Repository: stalwartlabs/stalwart Branch: main Commit: 2266634f36e2 Files: 1534 Total size: 11.2 MB Directory structure: gitextract_l0u7gilb/ ├── .dockerignore ├── .editorconfig ├── .github/ │ ├── DISCUSSION_TEMPLATE/ │ │ └── issue-triage.yml │ ├── ISSUE_TEMPLATE/ │ │ ├── config.yml │ │ └── confirmed_issue.yml │ ├── dependabot.yml │ └── workflows/ │ ├── ci.yml │ ├── scorecard.yml │ ├── test.yml │ └── trivy.yml ├── .gitignore ├── CHANGELOG.md ├── CNAME ├── CONTRIBUTING.md ├── Cargo.toml ├── Dockerfile ├── Dockerfile.build ├── LICENSES/ │ ├── AGPL-3.0-only.txt │ └── LicenseRef-SEL.txt ├── README.md ├── SECURITY.md ├── SECURITY_PROCESS.md ├── SECURITY_TEMPLATE.md ├── UPGRADING/ │ ├── v0_04.md │ ├── v0_05.md │ ├── v0_06.md │ ├── v0_07.md │ ├── v0_08.md │ ├── v0_09.md │ ├── v0_10.md │ ├── v0_11.md │ ├── v0_12.md │ ├── v0_13.md │ ├── v0_14.md │ └── v0_15.md ├── api/ │ └── v1/ │ └── openapi.yml ├── crates/ │ ├── cli/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── main.rs │ │ └── modules/ │ │ ├── account.rs │ │ ├── cli.rs │ │ ├── database.rs │ │ ├── dkim.rs │ │ ├── domain.rs │ │ ├── export.rs │ │ ├── group.rs │ │ ├── import.rs │ │ ├── list.rs │ │ ├── mod.rs │ │ ├── queue.rs │ │ └── report.rs │ ├── common/ │ │ ├── Cargo.toml │ │ ├── build.rs │ │ └── src/ │ │ ├── addresses.rs │ │ ├── auth/ │ │ │ ├── access_token.rs │ │ │ ├── mod.rs │ │ │ ├── oauth/ │ │ │ │ ├── config.rs │ │ │ │ ├── crypto.rs │ │ │ │ ├── introspect.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── oidc.rs │ │ │ │ ├── registration.rs │ │ │ │ └── token.rs │ │ │ ├── rate_limit.rs │ │ │ ├── roles.rs │ │ │ └── sasl.rs │ │ ├── config/ │ │ │ ├── groupware.rs │ │ │ ├── imap.rs │ │ │ ├── inner.rs │ │ │ ├── jmap/ │ │ │ │ ├── capabilities.rs │ │ │ │ ├── mod.rs │ │ │ │ └── settings.rs │ │ │ ├── mod.rs │ │ │ ├── network.rs │ │ │ ├── scripts.rs │ │ │ ├── server/ │ │ │ │ ├── listener.rs │ │ │ │ ├── mod.rs │ │ │ │ └── tls.rs │ │ │ ├── smtp/ │ │ │ │ ├── auth.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── queue.rs │ │ │ │ ├── report.rs │ │ │ │ ├── resolver.rs │ │ │ │ ├── session.rs │ │ │ │ └── throttle.rs │ │ │ ├── spamfilter.rs │ │ │ ├── storage.rs │ │ │ └── telemetry.rs │ │ ├── core.rs │ │ ├── dns.rs │ │ ├── enterprise/ │ │ │ ├── alerts.rs │ │ │ ├── config.rs │ │ │ ├── license.rs │ │ │ ├── llm.rs │ │ │ ├── mod.rs │ │ │ └── undelete.rs │ │ ├── expr/ │ │ │ ├── eval.rs │ │ │ ├── functions/ │ │ │ │ ├── array.rs │ │ │ │ ├── asynch.rs │ │ │ │ ├── email.rs │ │ │ │ ├── misc.rs │ │ │ │ ├── mod.rs │ │ │ │ └── text.rs │ │ │ ├── if_block.rs │ │ │ ├── mod.rs │ │ │ ├── parser.rs │ │ │ └── tokenizer.rs │ │ ├── i18n.rs │ │ ├── ipc.rs │ │ ├── lib.rs │ │ ├── listener/ │ │ │ ├── acme/ │ │ │ │ ├── cache.rs │ │ │ │ ├── directory.rs │ │ │ │ ├── jose.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── order.rs │ │ │ │ └── resolver.rs │ │ │ ├── asn.rs │ │ │ ├── blocked.rs │ │ │ ├── limiter.rs │ │ │ ├── listen.rs │ │ │ ├── mod.rs │ │ │ ├── stream.rs │ │ │ └── tls.rs │ │ ├── manager/ │ │ │ ├── backup.rs │ │ │ ├── boot.rs │ │ │ ├── config.rs │ │ │ ├── console.rs │ │ │ ├── mod.rs │ │ │ ├── reload.rs │ │ │ ├── restore.rs │ │ │ └── webadmin.rs │ │ ├── scripts/ │ │ │ ├── functions/ │ │ │ │ ├── array.rs │ │ │ │ ├── email.rs │ │ │ │ ├── header.rs │ │ │ │ ├── image.rs │ │ │ │ ├── misc.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── text.rs │ │ │ │ ├── unicode.rs │ │ │ │ └── url.rs │ │ │ ├── mod.rs │ │ │ └── plugins/ │ │ │ ├── dns.rs │ │ │ ├── exec.rs │ │ │ ├── headers.rs │ │ │ ├── http.rs │ │ │ ├── llm_prompt.rs │ │ │ ├── lookup.rs │ │ │ ├── mod.rs │ │ │ ├── query.rs │ │ │ └── text.rs │ │ ├── sharing/ │ │ │ ├── acl.rs │ │ │ ├── mod.rs │ │ │ ├── notification.rs │ │ │ └── resources.rs │ │ ├── storage/ │ │ │ ├── blob.rs │ │ │ ├── index.rs │ │ │ ├── mod.rs │ │ │ └── state.rs │ │ └── telemetry/ │ │ ├── metrics/ │ │ │ ├── mod.rs │ │ │ ├── otel.rs │ │ │ ├── prometheus.rs │ │ │ └── store.rs │ │ ├── mod.rs │ │ ├── tracers/ │ │ │ ├── journald.rs │ │ │ ├── log.rs │ │ │ ├── mod.rs │ │ │ ├── otel.rs │ │ │ ├── stdout.rs │ │ │ └── store.rs │ │ └── webhooks/ │ │ └── mod.rs │ ├── dav/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── calendar/ │ │ │ ├── copy_move.rs │ │ │ ├── delete.rs │ │ │ ├── freebusy.rs │ │ │ ├── get.rs │ │ │ ├── mkcol.rs │ │ │ ├── mod.rs │ │ │ ├── proppatch.rs │ │ │ ├── query.rs │ │ │ ├── scheduling.rs │ │ │ └── update.rs │ │ ├── card/ │ │ │ ├── copy_move.rs │ │ │ ├── delete.rs │ │ │ ├── get.rs │ │ │ ├── mkcol.rs │ │ │ ├── mod.rs │ │ │ ├── proppatch.rs │ │ │ ├── query.rs │ │ │ └── update.rs │ │ ├── common/ │ │ │ ├── acl.rs │ │ │ ├── lock.rs │ │ │ ├── mod.rs │ │ │ ├── propfind.rs │ │ │ └── uri.rs │ │ ├── file/ │ │ │ ├── copy_move.rs │ │ │ ├── delete.rs │ │ │ ├── get.rs │ │ │ ├── mkcol.rs │ │ │ ├── mod.rs │ │ │ ├── proppatch.rs │ │ │ └── update.rs │ │ ├── lib.rs │ │ ├── principal/ │ │ │ ├── matching.rs │ │ │ ├── mod.rs │ │ │ ├── propfind.rs │ │ │ └── propsearch.rs │ │ └── request.rs │ ├── dav-proto/ │ │ ├── Cargo.toml │ │ ├── resources/ │ │ │ ├── requests/ │ │ │ │ ├── acl-001.json │ │ │ │ ├── acl-001.xml │ │ │ │ ├── acl-002.json │ │ │ │ ├── acl-002.xml │ │ │ │ ├── acl-003.json │ │ │ │ ├── acl-003.xml │ │ │ │ ├── acl-004.json │ │ │ │ ├── acl-004.xml │ │ │ │ ├── lockinfo-001.json │ │ │ │ ├── lockinfo-001.xml │ │ │ │ ├── lockinfo-002.json │ │ │ │ ├── lockinfo-002.xml │ │ │ │ ├── mkcol-001.json │ │ │ │ ├── mkcol-001.xml │ │ │ │ ├── mkcol-002.json │ │ │ │ ├── mkcol-002.xml │ │ │ │ ├── mkcol-003.json │ │ │ │ ├── mkcol-003.xml │ │ │ │ ├── mkcol-004.json │ │ │ │ ├── mkcol-004.xml │ │ │ │ ├── propertyupdate-001.json │ │ │ │ ├── propertyupdate-001.xml │ │ │ │ ├── propertyupdate-002.json │ │ │ │ ├── propertyupdate-002.xml │ │ │ │ ├── propfind-001.json │ │ │ │ ├── propfind-001.xml │ │ │ │ ├── propfind-002.json │ │ │ │ ├── propfind-002.xml │ │ │ │ ├── propfind-003.json │ │ │ │ ├── propfind-003.xml │ │ │ │ ├── propfind-004.json │ │ │ │ ├── propfind-004.xml │ │ │ │ ├── propfind-005.json │ │ │ │ ├── propfind-005.xml │ │ │ │ ├── propfind-006.json │ │ │ │ ├── propfind-006.xml │ │ │ │ ├── propfind-007.json │ │ │ │ ├── propfind-007.xml │ │ │ │ ├── propfind-008.json │ │ │ │ ├── propfind-008.xml │ │ │ │ ├── propfind-009.json │ │ │ │ ├── propfind-009.xml │ │ │ │ ├── propfind-010.json │ │ │ │ ├── propfind-010.xml │ │ │ │ ├── report-001.json │ │ │ │ ├── report-001.xml │ │ │ │ ├── report-002.json │ │ │ │ ├── report-002.xml │ │ │ │ ├── report-003.json │ │ │ │ ├── report-003.xml │ │ │ │ ├── report-004.json │ │ │ │ ├── report-004.xml │ │ │ │ ├── report-005.json │ │ │ │ ├── report-005.xml │ │ │ │ ├── report-006.json │ │ │ │ ├── report-006.xml │ │ │ │ ├── report-007.json │ │ │ │ ├── report-007.xml │ │ │ │ ├── report-008.json │ │ │ │ ├── report-008.xml │ │ │ │ ├── report-009.json │ │ │ │ ├── report-009.xml │ │ │ │ ├── report-010.json │ │ │ │ ├── report-010.xml │ │ │ │ ├── report-011.json │ │ │ │ ├── report-011.xml │ │ │ │ ├── report-012.json │ │ │ │ ├── report-012.xml │ │ │ │ ├── report-013.json │ │ │ │ ├── report-013.xml │ │ │ │ ├── report-014.json │ │ │ │ ├── report-014.xml │ │ │ │ ├── report-015.json │ │ │ │ ├── report-015.xml │ │ │ │ ├── report-016.json │ │ │ │ ├── report-016.xml │ │ │ │ ├── report-017.json │ │ │ │ ├── report-017.xml │ │ │ │ ├── report-018.json │ │ │ │ ├── report-018.xml │ │ │ │ ├── report-019.json │ │ │ │ ├── report-019.xml │ │ │ │ ├── report-020.json │ │ │ │ ├── report-020.xml │ │ │ │ ├── report-021.json │ │ │ │ ├── report-021.xml │ │ │ │ ├── report-022.json │ │ │ │ ├── report-022.xml │ │ │ │ ├── report-023.json │ │ │ │ ├── report-023.xml │ │ │ │ ├── report-024.json │ │ │ │ ├── report-024.xml │ │ │ │ ├── report-025.json │ │ │ │ └── report-025.xml │ │ │ └── responses/ │ │ │ ├── 001.xml │ │ │ ├── 002.xml │ │ │ ├── 003.xml │ │ │ ├── 004.xml │ │ │ ├── 005.xml │ │ │ ├── 006.xml │ │ │ ├── 007.xml │ │ │ ├── 008.xml │ │ │ ├── 009.xml │ │ │ ├── 010.xml │ │ │ ├── 011.xml │ │ │ ├── 012.xml │ │ │ ├── 013.xml │ │ │ ├── 014.xml │ │ │ ├── 015.xml │ │ │ ├── 016.xml │ │ │ ├── 017.xml │ │ │ ├── 018.xml │ │ │ ├── 019.xml │ │ │ └── 020.xml │ │ └── src/ │ │ ├── lib.rs │ │ ├── parser/ │ │ │ ├── header.rs │ │ │ ├── mod.rs │ │ │ ├── property.rs │ │ │ └── tokenizer.rs │ │ ├── requests/ │ │ │ ├── acl.rs │ │ │ ├── lockinfo.rs │ │ │ ├── mkcol.rs │ │ │ ├── mod.rs │ │ │ ├── propertyupdate.rs │ │ │ ├── propfind.rs │ │ │ └── report.rs │ │ ├── responses/ │ │ │ ├── acl.rs │ │ │ ├── error.rs │ │ │ ├── lock.rs │ │ │ ├── mkcol.rs │ │ │ ├── mod.rs │ │ │ ├── multistatus.rs │ │ │ ├── property.rs │ │ │ ├── propstat.rs │ │ │ └── schedule.rs │ │ └── schema/ │ │ ├── mod.rs │ │ ├── property.rs │ │ ├── request.rs │ │ └── response.rs │ ├── directory/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── backend/ │ │ │ ├── imap/ │ │ │ │ ├── client.rs │ │ │ │ ├── config.rs │ │ │ │ ├── lookup.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── pool.rs │ │ │ │ └── tls.rs │ │ │ ├── internal/ │ │ │ │ ├── lookup.rs │ │ │ │ ├── manage.rs │ │ │ │ └── mod.rs │ │ │ ├── ldap/ │ │ │ │ ├── config.rs │ │ │ │ ├── lookup.rs │ │ │ │ ├── mod.rs │ │ │ │ └── pool.rs │ │ │ ├── memory/ │ │ │ │ ├── config.rs │ │ │ │ ├── lookup.rs │ │ │ │ └── mod.rs │ │ │ ├── mod.rs │ │ │ ├── oidc/ │ │ │ │ ├── config.rs │ │ │ │ ├── lookup.rs │ │ │ │ └── mod.rs │ │ │ ├── smtp/ │ │ │ │ ├── config.rs │ │ │ │ ├── lookup.rs │ │ │ │ ├── mod.rs │ │ │ │ └── pool.rs │ │ │ └── sql/ │ │ │ ├── config.rs │ │ │ ├── lookup.rs │ │ │ └── mod.rs │ │ ├── core/ │ │ │ ├── cache.rs │ │ │ ├── config.rs │ │ │ ├── dispatch.rs │ │ │ ├── mod.rs │ │ │ ├── principal.rs │ │ │ └── secret.rs │ │ └── lib.rs │ ├── email/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── cache/ │ │ │ ├── email.rs │ │ │ ├── mailbox.rs │ │ │ └── mod.rs │ │ ├── identity/ │ │ │ ├── index.rs │ │ │ └── mod.rs │ │ ├── lib.rs │ │ ├── mailbox/ │ │ │ ├── destroy.rs │ │ │ ├── index.rs │ │ │ ├── manage.rs │ │ │ └── mod.rs │ │ ├── message/ │ │ │ ├── copy.rs │ │ │ ├── crypto.rs │ │ │ ├── delete.rs │ │ │ ├── delivery.rs │ │ │ ├── index/ │ │ │ │ ├── extractors.rs │ │ │ │ ├── metadata.rs │ │ │ │ ├── mod.rs │ │ │ │ └── search.rs │ │ │ ├── ingest.rs │ │ │ ├── metadata.rs │ │ │ └── mod.rs │ │ ├── push/ │ │ │ └── mod.rs │ │ ├── sieve/ │ │ │ ├── delete.rs │ │ │ ├── index.rs │ │ │ ├── ingest.rs │ │ │ └── mod.rs │ │ └── submission/ │ │ ├── index.rs │ │ └── mod.rs │ ├── groupware/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── cache/ │ │ │ ├── calcard.rs │ │ │ ├── file.rs │ │ │ └── mod.rs │ │ ├── calendar/ │ │ │ ├── alarm.rs │ │ │ ├── dates.rs │ │ │ ├── expand.rs │ │ │ ├── index.rs │ │ │ ├── itip.rs │ │ │ ├── mod.rs │ │ │ └── storage.rs │ │ ├── contact/ │ │ │ ├── index.rs │ │ │ ├── mod.rs │ │ │ └── storage.rs │ │ ├── file/ │ │ │ ├── index.rs │ │ │ ├── mod.rs │ │ │ └── storage.rs │ │ ├── lib.rs │ │ └── scheduling/ │ │ ├── attendee.rs │ │ ├── event_cancel.rs │ │ ├── event_create.rs │ │ ├── event_update.rs │ │ ├── inbound.rs │ │ ├── itip.rs │ │ ├── mod.rs │ │ ├── organizer.rs │ │ └── snapshot.rs │ ├── http/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── auth/ │ │ │ ├── authenticate.rs │ │ │ ├── mod.rs │ │ │ └── oauth/ │ │ │ ├── auth.rs │ │ │ ├── mod.rs │ │ │ ├── openid.rs │ │ │ ├── registration.rs │ │ │ └── token.rs │ │ ├── autoconfig/ │ │ │ └── mod.rs │ │ ├── form/ │ │ │ └── mod.rs │ │ ├── lib.rs │ │ ├── management/ │ │ │ ├── crypto.rs │ │ │ ├── dkim.rs │ │ │ ├── dns.rs │ │ │ ├── enterprise/ │ │ │ │ ├── mod.rs │ │ │ │ ├── telemetry.rs │ │ │ │ └── undelete.rs │ │ │ ├── log.rs │ │ │ ├── mod.rs │ │ │ ├── principal.rs │ │ │ ├── queue.rs │ │ │ ├── reload.rs │ │ │ ├── report.rs │ │ │ ├── settings.rs │ │ │ ├── spam.rs │ │ │ ├── stores.rs │ │ │ └── troubleshoot.rs │ │ └── request.rs │ ├── http-proto/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── context.rs │ │ ├── lib.rs │ │ ├── request.rs │ │ └── response.rs │ ├── imap/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── core/ │ │ │ ├── client.rs │ │ │ ├── mailbox.rs │ │ │ ├── message.rs │ │ │ ├── mod.rs │ │ │ └── session.rs │ │ ├── lib.rs │ │ └── op/ │ │ ├── acl.rs │ │ ├── append.rs │ │ ├── authenticate.rs │ │ ├── capability.rs │ │ ├── close.rs │ │ ├── copy_move.rs │ │ ├── create.rs │ │ ├── delete.rs │ │ ├── enable.rs │ │ ├── expunge.rs │ │ ├── fetch.rs │ │ ├── idle.rs │ │ ├── list.rs │ │ ├── login.rs │ │ ├── logout.rs │ │ ├── mod.rs │ │ ├── namespace.rs │ │ ├── noop.rs │ │ ├── quota.rs │ │ ├── rename.rs │ │ ├── search.rs │ │ ├── select.rs │ │ ├── status.rs │ │ ├── store.rs │ │ ├── subscribe.rs │ │ └── thread.rs │ ├── imap-proto/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── lib.rs │ │ ├── parser/ │ │ │ ├── acl.rs │ │ │ ├── append.rs │ │ │ ├── authenticate.rs │ │ │ ├── copy_move.rs │ │ │ ├── create.rs │ │ │ ├── delete.rs │ │ │ ├── enable.rs │ │ │ ├── fetch.rs │ │ │ ├── list.rs │ │ │ ├── login.rs │ │ │ ├── lsub.rs │ │ │ ├── mod.rs │ │ │ ├── quota.rs │ │ │ ├── rename.rs │ │ │ ├── search.rs │ │ │ ├── select.rs │ │ │ ├── sort.rs │ │ │ ├── status.rs │ │ │ ├── store.rs │ │ │ ├── subscribe.rs │ │ │ └── thread.rs │ │ ├── protocol/ │ │ │ ├── acl.rs │ │ │ ├── append.rs │ │ │ ├── authenticate.rs │ │ │ ├── capability.rs │ │ │ ├── copy_move.rs │ │ │ ├── create.rs │ │ │ ├── delete.rs │ │ │ ├── enable.rs │ │ │ ├── expunge.rs │ │ │ ├── fetch.rs │ │ │ ├── list.rs │ │ │ ├── login.rs │ │ │ ├── mod.rs │ │ │ ├── namespace.rs │ │ │ ├── quota.rs │ │ │ ├── rename.rs │ │ │ ├── search.rs │ │ │ ├── select.rs │ │ │ ├── status.rs │ │ │ ├── store.rs │ │ │ ├── subscribe.rs │ │ │ └── thread.rs │ │ ├── receiver.rs │ │ └── utf7.rs │ ├── jmap/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── addressbook/ │ │ │ ├── get.rs │ │ │ ├── mod.rs │ │ │ └── set.rs │ │ ├── api/ │ │ │ ├── acl.rs │ │ │ ├── auth.rs │ │ │ ├── event_source.rs │ │ │ ├── mod.rs │ │ │ ├── query.rs │ │ │ ├── request.rs │ │ │ └── session.rs │ │ ├── blob/ │ │ │ ├── copy.rs │ │ │ ├── download.rs │ │ │ ├── get.rs │ │ │ ├── mod.rs │ │ │ └── upload.rs │ │ ├── calendar/ │ │ │ ├── get.rs │ │ │ ├── mod.rs │ │ │ └── set.rs │ │ ├── calendar_event/ │ │ │ ├── copy.rs │ │ │ ├── get.rs │ │ │ ├── mod.rs │ │ │ ├── parse.rs │ │ │ ├── query.rs │ │ │ └── set.rs │ │ ├── calendar_event_notification/ │ │ │ ├── get.rs │ │ │ ├── mod.rs │ │ │ ├── query.rs │ │ │ └── set.rs │ │ ├── changes/ │ │ │ ├── get.rs │ │ │ ├── mod.rs │ │ │ ├── query.rs │ │ │ └── state.rs │ │ ├── contact/ │ │ │ ├── copy.rs │ │ │ ├── get.rs │ │ │ ├── mod.rs │ │ │ ├── parse.rs │ │ │ ├── query.rs │ │ │ └── set.rs │ │ ├── email/ │ │ │ ├── body.rs │ │ │ ├── copy.rs │ │ │ ├── get.rs │ │ │ ├── headers.rs │ │ │ ├── import.rs │ │ │ ├── mod.rs │ │ │ ├── parse.rs │ │ │ ├── query.rs │ │ │ ├── set.rs │ │ │ └── snippet.rs │ │ ├── file/ │ │ │ ├── get.rs │ │ │ ├── mod.rs │ │ │ ├── query.rs │ │ │ └── set.rs │ │ ├── identity/ │ │ │ ├── get.rs │ │ │ ├── mod.rs │ │ │ └── set.rs │ │ ├── lib.rs │ │ ├── mailbox/ │ │ │ ├── get.rs │ │ │ ├── mod.rs │ │ │ ├── query.rs │ │ │ └── set.rs │ │ ├── participant_identity/ │ │ │ ├── get.rs │ │ │ ├── mod.rs │ │ │ └── set.rs │ │ ├── principal/ │ │ │ ├── availability.rs │ │ │ ├── get.rs │ │ │ ├── mod.rs │ │ │ └── query.rs │ │ ├── push/ │ │ │ ├── get.rs │ │ │ ├── mod.rs │ │ │ └── set.rs │ │ ├── quota/ │ │ │ ├── get.rs │ │ │ ├── mod.rs │ │ │ └── query.rs │ │ ├── share_notification/ │ │ │ ├── get.rs │ │ │ ├── mod.rs │ │ │ ├── query.rs │ │ │ └── set.rs │ │ ├── sieve/ │ │ │ ├── get.rs │ │ │ ├── mod.rs │ │ │ ├── query.rs │ │ │ ├── set.rs │ │ │ └── validate.rs │ │ ├── submission/ │ │ │ ├── get.rs │ │ │ ├── mod.rs │ │ │ ├── query.rs │ │ │ └── set.rs │ │ ├── thread/ │ │ │ ├── get.rs │ │ │ └── mod.rs │ │ ├── vacation/ │ │ │ ├── get.rs │ │ │ ├── mod.rs │ │ │ └── set.rs │ │ └── websocket/ │ │ ├── mod.rs │ │ ├── stream.rs │ │ └── upgrade.rs │ ├── jmap-proto/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── error/ │ │ │ ├── method.rs │ │ │ ├── mod.rs │ │ │ ├── request.rs │ │ │ └── set.rs │ │ ├── lib.rs │ │ ├── method/ │ │ │ ├── availability.rs │ │ │ ├── changes.rs │ │ │ ├── copy.rs │ │ │ ├── get.rs │ │ │ ├── import.rs │ │ │ ├── lookup.rs │ │ │ ├── mod.rs │ │ │ ├── parse.rs │ │ │ ├── query.rs │ │ │ ├── query_changes.rs │ │ │ ├── search_snippet.rs │ │ │ ├── set.rs │ │ │ ├── upload.rs │ │ │ └── validate.rs │ │ ├── object/ │ │ │ ├── addressbook.rs │ │ │ ├── blob.rs │ │ │ ├── calendar.rs │ │ │ ├── calendar_event.rs │ │ │ ├── calendar_event_notification.rs │ │ │ ├── contact.rs │ │ │ ├── email.rs │ │ │ ├── email_submission.rs │ │ │ ├── file_node.rs │ │ │ ├── identity.rs │ │ │ ├── mailbox.rs │ │ │ ├── mod.rs │ │ │ ├── participant_identity.rs │ │ │ ├── principal.rs │ │ │ ├── push_subscription.rs │ │ │ ├── quota.rs │ │ │ ├── search_snippet.rs │ │ │ ├── share_notification.rs │ │ │ ├── sieve.rs │ │ │ ├── thread.rs │ │ │ └── vacation_response.rs │ │ ├── references/ │ │ │ ├── eval.rs │ │ │ ├── jsptr.rs │ │ │ ├── mod.rs │ │ │ └── resolve.rs │ │ ├── request/ │ │ │ ├── capability.rs │ │ │ ├── deserialize.rs │ │ │ ├── method.rs │ │ │ ├── mod.rs │ │ │ ├── parser.rs │ │ │ ├── reference.rs │ │ │ └── websocket.rs │ │ ├── response/ │ │ │ ├── mod.rs │ │ │ ├── serialize.rs │ │ │ └── status.rs │ │ └── types/ │ │ ├── date.rs │ │ ├── mod.rs │ │ └── state.rs │ ├── main/ │ │ ├── Cargo.toml │ │ └── src/ │ │ └── main.rs │ ├── managesieve/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── core/ │ │ │ ├── client.rs │ │ │ ├── mod.rs │ │ │ └── session.rs │ │ ├── lib.rs │ │ └── op/ │ │ ├── authenticate.rs │ │ ├── capability.rs │ │ ├── checkscript.rs │ │ ├── deletescript.rs │ │ ├── getscript.rs │ │ ├── havespace.rs │ │ ├── listscripts.rs │ │ ├── logout.rs │ │ ├── mod.rs │ │ ├── noop.rs │ │ ├── putscript.rs │ │ ├── renamescript.rs │ │ └── setactive.rs │ ├── migration/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── addressbook_v2.rs │ │ ├── blob.rs │ │ ├── calendar_v2.rs │ │ ├── changelog.rs │ │ ├── contact_v2.rs │ │ ├── email_v1.rs │ │ ├── email_v2.rs │ │ ├── encryption_v1.rs │ │ ├── encryption_v2.rs │ │ ├── event_v1.rs │ │ ├── event_v2.rs │ │ ├── identity_v1.rs │ │ ├── lib.rs │ │ ├── mailbox.rs │ │ ├── object.rs │ │ ├── principal_v1.rs │ │ ├── principal_v2.rs │ │ ├── push_v1.rs │ │ ├── push_v2.rs │ │ ├── queue_v1.rs │ │ ├── queue_v2.rs │ │ ├── report.rs │ │ ├── sieve_v1.rs │ │ ├── sieve_v2.rs │ │ ├── submission.rs │ │ ├── tasks_v1.rs │ │ ├── tasks_v2.rs │ │ ├── threads.rs │ │ ├── v011.rs │ │ ├── v012.rs │ │ ├── v013.rs │ │ └── v014.rs │ ├── nlp/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── classifier/ │ │ │ ├── adam.rs │ │ │ ├── feature.rs │ │ │ ├── ftrl.rs │ │ │ ├── mod.rs │ │ │ ├── model.rs │ │ │ ├── reservoir.rs │ │ │ ├── sgd.rs │ │ │ └── train.rs │ │ ├── language/ │ │ │ ├── detect.rs │ │ │ ├── mod.rs │ │ │ ├── search_snippet.rs │ │ │ ├── stemmer.rs │ │ │ └── stopwords.rs │ │ ├── lib.rs │ │ └── tokenizers/ │ │ ├── chinese.rs │ │ ├── japanese.rs │ │ ├── mod.rs │ │ ├── space.rs │ │ ├── stream.rs │ │ ├── types.rs │ │ └── word.rs │ ├── pop3/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── client.rs │ │ ├── lib.rs │ │ ├── mailbox.rs │ │ ├── op/ │ │ │ ├── authenticate.rs │ │ │ ├── delete.rs │ │ │ ├── fetch.rs │ │ │ ├── list.rs │ │ │ └── mod.rs │ │ ├── protocol/ │ │ │ ├── mod.rs │ │ │ ├── request.rs │ │ │ └── response.rs │ │ └── session.rs │ ├── services/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── broadcast/ │ │ │ ├── mod.rs │ │ │ ├── publisher.rs │ │ │ └── subscriber.rs │ │ ├── housekeeper/ │ │ │ └── mod.rs │ │ ├── lib.rs │ │ ├── state_manager/ │ │ │ ├── ece.rs │ │ │ ├── http.rs │ │ │ ├── manager.rs │ │ │ ├── mod.rs │ │ │ └── push.rs │ │ └── task_manager/ │ │ ├── alarm.rs │ │ ├── imip.rs │ │ ├── index.rs │ │ ├── lock.rs │ │ ├── merge_threads.rs │ │ └── mod.rs │ ├── smtp/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── core/ │ │ │ ├── mod.rs │ │ │ ├── params.rs │ │ │ └── throttle.rs │ │ ├── inbound/ │ │ │ ├── auth.rs │ │ │ ├── data.rs │ │ │ ├── ehlo.rs │ │ │ ├── hooks/ │ │ │ │ ├── client.rs │ │ │ │ ├── message.rs │ │ │ │ └── mod.rs │ │ │ ├── mail.rs │ │ │ ├── milter/ │ │ │ │ ├── client.rs │ │ │ │ ├── macros.rs │ │ │ │ ├── message.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── protocol.rs │ │ │ │ └── receiver.rs │ │ │ ├── mod.rs │ │ │ ├── rcpt.rs │ │ │ ├── session.rs │ │ │ ├── spam.rs │ │ │ ├── spawn.rs │ │ │ └── vrfy.rs │ │ ├── lib.rs │ │ ├── outbound/ │ │ │ ├── client.rs │ │ │ ├── dane/ │ │ │ │ ├── dnssec.rs │ │ │ │ ├── mod.rs │ │ │ │ └── verify.rs │ │ │ ├── delivery.rs │ │ │ ├── local.rs │ │ │ ├── lookup.rs │ │ │ ├── mod.rs │ │ │ ├── mta_sts/ │ │ │ │ ├── lookup.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── parse.rs │ │ │ │ └── verify.rs │ │ │ └── session.rs │ │ ├── queue/ │ │ │ ├── dsn.rs │ │ │ ├── manager.rs │ │ │ ├── mod.rs │ │ │ ├── quota.rs │ │ │ ├── spool.rs │ │ │ └── throttle.rs │ │ ├── reporting/ │ │ │ ├── analysis.rs │ │ │ ├── dkim.rs │ │ │ ├── dmarc.rs │ │ │ ├── mod.rs │ │ │ ├── scheduler.rs │ │ │ ├── spf.rs │ │ │ └── tls.rs │ │ └── scripts/ │ │ ├── envelope.rs │ │ ├── event_loop.rs │ │ ├── exec.rs │ │ └── mod.rs │ ├── spam-filter/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── analysis/ │ │ │ ├── classifier.rs │ │ │ ├── date.rs │ │ │ ├── dmarc.rs │ │ │ ├── domain.rs │ │ │ ├── ehlo.rs │ │ │ ├── from.rs │ │ │ ├── headers.rs │ │ │ ├── html.rs │ │ │ ├── init.rs │ │ │ ├── ip.rs │ │ │ ├── llm.rs │ │ │ ├── messageid.rs │ │ │ ├── mime.rs │ │ │ ├── mod.rs │ │ │ ├── pyzor.rs │ │ │ ├── received.rs │ │ │ ├── recipient.rs │ │ │ ├── replyto.rs │ │ │ ├── rules.rs │ │ │ ├── score.rs │ │ │ ├── subject.rs │ │ │ └── url.rs │ │ ├── lib.rs │ │ └── modules/ │ │ ├── classifier.rs │ │ ├── dnsbl.rs │ │ ├── expression.rs │ │ ├── html.rs │ │ ├── mod.rs │ │ ├── pyzor.rs │ │ └── sanitize.rs │ ├── store/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── backend/ │ │ │ ├── azure/ │ │ │ │ └── mod.rs │ │ │ ├── composite/ │ │ │ │ ├── mod.rs │ │ │ │ ├── read_replica.rs │ │ │ │ ├── sharded_blob.rs │ │ │ │ └── sharded_lookup.rs │ │ │ ├── elastic/ │ │ │ │ ├── main.rs │ │ │ │ ├── mod.rs │ │ │ │ └── search.rs │ │ │ ├── foundationdb/ │ │ │ │ ├── blob.rs │ │ │ │ ├── main.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── read.rs │ │ │ │ └── write.rs │ │ │ ├── fs/ │ │ │ │ └── mod.rs │ │ │ ├── http/ │ │ │ │ ├── config.rs │ │ │ │ ├── lookup.rs │ │ │ │ └── mod.rs │ │ │ ├── kafka/ │ │ │ │ ├── mod.rs │ │ │ │ └── pubsub.rs │ │ │ ├── meili/ │ │ │ │ ├── main.rs │ │ │ │ ├── mod.rs │ │ │ │ └── search.rs │ │ │ ├── memory/ │ │ │ │ └── mod.rs │ │ │ ├── mod.rs │ │ │ ├── mysql/ │ │ │ │ ├── blob.rs │ │ │ │ ├── lookup.rs │ │ │ │ ├── main.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── read.rs │ │ │ │ ├── search.rs │ │ │ │ └── write.rs │ │ │ ├── nats/ │ │ │ │ ├── mod.rs │ │ │ │ └── pubsub.rs │ │ │ ├── postgres/ │ │ │ │ ├── blob.rs │ │ │ │ ├── lookup.rs │ │ │ │ ├── main.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── read.rs │ │ │ │ ├── search.rs │ │ │ │ ├── tls.rs │ │ │ │ └── write.rs │ │ │ ├── redis/ │ │ │ │ ├── lookup.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── pool.rs │ │ │ │ └── pubsub.rs │ │ │ ├── rocksdb/ │ │ │ │ ├── blob.rs │ │ │ │ ├── main.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── read.rs │ │ │ │ └── write.rs │ │ │ ├── s3/ │ │ │ │ └── mod.rs │ │ │ ├── sqlite/ │ │ │ │ ├── blob.rs │ │ │ │ ├── lookup.rs │ │ │ │ ├── main.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── pool.rs │ │ │ │ ├── read.rs │ │ │ │ └── write.rs │ │ │ └── zenoh/ │ │ │ ├── mod.rs │ │ │ └── pubsub.rs │ │ ├── config.rs │ │ ├── dispatch/ │ │ │ ├── blob.rs │ │ │ ├── lookup.rs │ │ │ ├── mod.rs │ │ │ ├── pubsub.rs │ │ │ ├── search.rs │ │ │ └── store.rs │ │ ├── lib.rs │ │ ├── query/ │ │ │ ├── acl.rs │ │ │ ├── log.rs │ │ │ └── mod.rs │ │ ├── search/ │ │ │ ├── bm_u32.rs │ │ │ ├── bm_u64.rs │ │ │ ├── document.rs │ │ │ ├── fields.rs │ │ │ ├── index.rs │ │ │ ├── local.rs │ │ │ ├── mod.rs │ │ │ ├── query.rs │ │ │ ├── split.rs │ │ │ └── term.rs │ │ └── write/ │ │ ├── assert.rs │ │ ├── batch.rs │ │ ├── bitpack.rs │ │ ├── blob.rs │ │ ├── key.rs │ │ ├── log.rs │ │ ├── mod.rs │ │ └── serialize.rs │ ├── trc/ │ │ ├── Cargo.toml │ │ ├── event-macro/ │ │ │ ├── Cargo.toml │ │ │ └── src/ │ │ │ └── lib.rs │ │ └── src/ │ │ ├── atomics/ │ │ │ ├── array.rs │ │ │ ├── bitset.rs │ │ │ ├── counter.rs │ │ │ ├── gauge.rs │ │ │ ├── histogram.rs │ │ │ └── mod.rs │ │ ├── event/ │ │ │ ├── conv.rs │ │ │ ├── description.rs │ │ │ ├── level.rs │ │ │ ├── metrics.rs │ │ │ └── mod.rs │ │ ├── ipc/ │ │ │ ├── bitset.rs │ │ │ ├── channel.rs │ │ │ ├── collector.rs │ │ │ ├── metrics.rs │ │ │ ├── mod.rs │ │ │ └── subscriber.rs │ │ ├── lib.rs │ │ ├── macros.rs │ │ └── serializers/ │ │ ├── binary.rs │ │ ├── json.rs │ │ ├── mod.rs │ │ └── text.rs │ ├── types/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── acl.rs │ │ ├── blob.rs │ │ ├── blob_hash.rs │ │ ├── collection.rs │ │ ├── dead_property.rs │ │ ├── field.rs │ │ ├── id.rs │ │ ├── keyword.rs │ │ ├── lib.rs │ │ ├── semver.rs │ │ ├── special_use.rs │ │ └── type_state.rs │ └── utils/ │ ├── Cargo.toml │ ├── proc-macros/ │ │ ├── Cargo.toml │ │ └── src/ │ │ └── lib.rs │ └── src/ │ ├── bimap.rs │ ├── cache.rs │ ├── chained_bytes.rs │ ├── cheeky_hash.rs │ ├── codec/ │ │ ├── base32_custom.rs │ │ ├── leb128.rs │ │ └── mod.rs │ ├── config/ │ │ ├── cron.rs │ │ ├── http.rs │ │ ├── ipmask.rs │ │ ├── mod.rs │ │ ├── parser.rs │ │ └── utils.rs │ ├── glob.rs │ ├── lib.rs │ ├── map/ │ │ ├── bitmap.rs │ │ ├── mod.rs │ │ ├── mutex_map.rs │ │ └── vec_map.rs │ ├── snowflake.rs │ ├── suffixlist.rs │ ├── template.rs │ ├── topological.rs │ └── url_params.rs ├── docker-bake.hcl ├── install.sh ├── resources/ │ ├── apparmor.d/ │ │ └── stalwart-mail │ ├── config/ │ │ └── config.toml │ ├── docker/ │ │ ├── Dockerfile.fdb │ │ ├── download.sh │ │ └── entrypoint.sh │ ├── html-templates/ │ │ ├── calendar-alarm.html │ │ ├── calendar-alarm.html.min │ │ ├── calendar-alarm.mjml │ │ ├── calendar-invite.html │ │ ├── calendar-invite.html.min │ │ └── calendar-invite.mjml │ ├── locales/ │ │ └── i18n.yml │ ├── scripts/ │ │ └── ossify.py │ └── systemd/ │ ├── stalwart-mail.service │ └── stalwart.mail.plist └── tests/ ├── Cargo.toml ├── resources/ │ ├── acme/ │ │ ├── Docker.pebble │ │ ├── config.toml │ │ ├── docker-compose-pebble.yaml │ │ └── test_acme.sh │ ├── crypto/ │ │ ├── cert_mixed.pem │ │ ├── cert_pgp.der │ │ ├── cert_pgp.pem │ │ ├── cert_smime.der │ │ ├── cert_smime.pem │ │ └── is_encrypted.txt │ ├── imap/ │ │ ├── 000.imap │ │ ├── 000.txt │ │ ├── 001.imap │ │ ├── 001.txt │ │ ├── 002.imap │ │ ├── 002.txt │ │ ├── 003.imap │ │ ├── 003.txt │ │ ├── 004.imap │ │ ├── 004.txt │ │ ├── 005.imap │ │ ├── 005.txt │ │ ├── 006.imap │ │ ├── 006.txt │ │ ├── 007.imap │ │ ├── 007.txt │ │ ├── 008.imap │ │ ├── 008.txt │ │ ├── 009.imap │ │ ├── 009.txt │ │ ├── 010.imap │ │ ├── 010.txt │ │ ├── 011.imap │ │ ├── 011.txt │ │ ├── 012.imap │ │ ├── 012.txt │ │ ├── 013.imap │ │ ├── 013.txt │ │ ├── 014.imap │ │ └── 014.txt │ ├── imap-test/ │ │ ├── append │ │ ├── append-binary │ │ ├── atoms │ │ ├── broken/ │ │ │ ├── search-intdate │ │ │ └── search-intdate.mbox │ │ ├── catenate │ │ ├── catenate-multiappend │ │ ├── close │ │ ├── copy │ │ ├── default.mbox │ │ ├── esearch │ │ ├── esearch-condstore │ │ ├── esearch.mbox │ │ ├── esort │ │ ├── expunge │ │ ├── expunge2 │ │ ├── fetch │ │ ├── fetch-binary-mime │ │ ├── fetch-binary-mime-base64 │ │ ├── fetch-binary-mime-base64.mbox │ │ ├── fetch-binary-mime-qp │ │ ├── fetch-binary-mime-qp.mbox │ │ ├── fetch-binary-mime.mbox │ │ ├── fetch-body │ │ ├── fetch-body-message-rfc822 │ │ ├── fetch-body-message-rfc822-mime │ │ ├── fetch-body-message-rfc822-mime.mbox │ │ ├── fetch-body-message-rfc822-x2 │ │ ├── fetch-body-message-rfc822-x2.mbox │ │ ├── fetch-body-message-rfc822.mbox │ │ ├── fetch-body-mime │ │ ├── fetch-body-mime.mbox │ │ ├── fetch-body.mbox │ │ ├── fetch-bodystructure │ │ ├── fetch-bodystructure.mbox │ │ ├── fetch-envelope │ │ ├── fetch-envelope.mbox │ │ ├── id │ │ ├── list │ │ ├── listext │ │ ├── logout │ │ ├── move │ │ ├── multiappend │ │ ├── mutf7 │ │ ├── nil │ │ ├── nil.mbox │ │ ├── notify │ │ ├── pipeline │ │ ├── search-addresses │ │ ├── search-addresses.mbox │ │ ├── search-body │ │ ├── search-body.mbox │ │ ├── search-context-update │ │ ├── search-context-update2 │ │ ├── search-context-update3 │ │ ├── search-date │ │ ├── search-date.mbox │ │ ├── search-flags │ │ ├── search-header │ │ ├── search-header.mbox │ │ ├── search-partial │ │ ├── search-partial.mbox │ │ ├── search-sets │ │ ├── search-size │ │ ├── search-size.mbox │ │ ├── select │ │ ├── select.mbox │ │ ├── sort-addresses │ │ ├── sort-addresses.mbox │ │ ├── sort-arrival │ │ ├── sort-arrival.mbox │ │ ├── sort-date │ │ ├── sort-date.mbox │ │ ├── sort-display-from │ │ ├── sort-display-from.mbox │ │ ├── sort-display-to │ │ ├── sort-display-to.mbox │ │ ├── sort-partial │ │ ├── sort-partial.mbox │ │ ├── sort-size │ │ ├── sort-size.mbox │ │ ├── sort-subject │ │ ├── sort-subject.mbox │ │ ├── store │ │ ├── subscribe │ │ ├── thread │ │ ├── thread-orderedsubject │ │ ├── thread-orderedsubject.mbox │ │ ├── thread-orderedsubject2 │ │ ├── thread-orderedsubject2.mbox │ │ ├── thread.mbox │ │ ├── thread2 │ │ ├── thread2.mbox │ │ ├── thread3 │ │ ├── thread3.mbox │ │ ├── thread4 │ │ ├── thread4.mbox │ │ ├── thread5 │ │ ├── thread5.mbox │ │ ├── thread6 │ │ ├── thread6.mbox │ │ ├── thread7 │ │ ├── thread7.mbox │ │ ├── thread8 │ │ ├── thread8.mbox │ │ ├── uidplus │ │ ├── uidvalidity │ │ ├── uidvalidity-rename │ │ ├── urlauth │ │ ├── urlauth-binary │ │ ├── urlauth-binary.mbox │ │ └── urlauth2 │ ├── itip/ │ │ ├── google_calendar.txt │ │ ├── itip_incoming.txt │ │ ├── put_validation.txt │ │ ├── rfc5546_event_recurring.txt │ │ ├── rfc5546_event_single.txt │ │ ├── rfc5546_todo.txt │ │ ├── rfc6638_recurring.txt │ │ └── rfc6638_single.txt │ ├── jmap/ │ │ ├── email_get/ │ │ │ ├── headers.eml │ │ │ ├── headers.json │ │ │ ├── message_attachment.eml │ │ │ ├── message_attachment.json │ │ │ ├── multipart_alternative.eml │ │ │ ├── multipart_alternative.json │ │ │ ├── multipart_cid.eml │ │ │ ├── multipart_cid.json │ │ │ ├── multipart_mixed.eml │ │ │ ├── multipart_mixed.json │ │ │ ├── multipart_related.eml │ │ │ ├── multipart_related.json │ │ │ ├── rfc8621.eml │ │ │ ├── rfc8621.json │ │ │ ├── single_part.eml │ │ │ ├── single_part.json │ │ │ ├── text_body_missing.eml │ │ │ ├── text_body_missing.json │ │ │ ├── text_body_missing_multipart.eml │ │ │ └── text_body_missing_multipart.json │ │ ├── email_parse/ │ │ │ ├── attachment.eml │ │ │ ├── attachment.json │ │ │ ├── attachment.part1 │ │ │ ├── attachment.part2 │ │ │ ├── attachment.part3 │ │ │ ├── attachment_b64.eml │ │ │ ├── attachment_b64.json │ │ │ ├── attachment_b64.part1 │ │ │ ├── attachment_b64.part2 │ │ │ ├── headers.eml │ │ │ └── headers.json │ │ ├── email_set/ │ │ │ ├── headers.eml │ │ │ ├── headers.jmap │ │ │ ├── headers.json │ │ │ ├── minimal.eml │ │ │ ├── minimal.jmap │ │ │ ├── minimal.json │ │ │ ├── mixed.eml │ │ │ ├── mixed.jmap │ │ │ ├── mixed.json │ │ │ ├── nested_body.eml │ │ │ ├── nested_body.jmap │ │ │ ├── nested_body.json │ │ │ ├── rfc8621_1.eml │ │ │ ├── rfc8621_1.jmap │ │ │ ├── rfc8621_1.json │ │ │ ├── rfc8621_2.eml │ │ │ ├── rfc8621_2.jmap │ │ │ └── rfc8621_2.json │ │ ├── email_snippet/ │ │ │ ├── html.eml │ │ │ ├── mixed.eml │ │ │ ├── subpart.eml │ │ │ ├── text_plain.eml │ │ │ └── text_plain_chinese.eml │ │ └── sieve/ │ │ ├── test_discard_reject.sieve │ │ ├── test_include.sieve │ │ ├── test_include_global.sieve │ │ ├── test_include_this.sieve │ │ ├── test_mailbox.sieve │ │ ├── test_notify_fcc.sieve │ │ ├── test_redirect_enclose.sieve │ │ ├── validate_error.sieve │ │ └── validate_ok.sieve │ ├── ldap/ │ │ ├── ldap.cfg │ │ └── run_glauth.sh │ ├── otel/ │ │ ├── docker-compose.yaml │ │ ├── otel-collector-config.yaml │ │ └── stalwart-config.toml │ ├── proxy-protocol/ │ │ ├── Docker.haproxy │ │ └── haproxy.cfg │ ├── scripts/ │ │ ├── create_test_cluster.sh │ │ ├── create_test_env.sh │ │ ├── create_test_users.sh │ │ ├── imap-log-parser.py │ │ ├── imap_import.py │ │ ├── imap_import_single.py │ │ ├── stress_test.py │ │ └── stress_test_prepare.py │ ├── smtp/ │ │ ├── antispam/ │ │ │ ├── bounce.test │ │ │ ├── classifier.ham │ │ │ ├── classifier.spam │ │ │ ├── classifier.test │ │ │ ├── classifier_features.test │ │ │ ├── classifier_html.test │ │ │ ├── combined.test │ │ │ ├── date.test │ │ │ ├── dmarc.test │ │ │ ├── from.test │ │ │ ├── headers.test │ │ │ ├── helo.test │ │ │ ├── html.test │ │ │ ├── ip.test │ │ │ ├── llm.test │ │ │ ├── messageid.test │ │ │ ├── mime.test │ │ │ ├── pyzor.test │ │ │ ├── rbl.test │ │ │ ├── received.test │ │ │ ├── recipient.test │ │ │ ├── replyto.test │ │ │ ├── spamtrap.test │ │ │ ├── subject.test │ │ │ └── url.test │ │ ├── certs/ │ │ │ ├── tls_cert.pem │ │ │ └── tls_privatekey.pem │ │ ├── config/ │ │ │ ├── if-blocks.toml │ │ │ ├── lists.toml │ │ │ ├── rules-dynvalue.toml │ │ │ ├── rules-eval.toml │ │ │ ├── rules.toml │ │ │ ├── servers.toml │ │ │ ├── throttle.toml │ │ │ └── toml-parser.toml │ │ ├── dane/ │ │ │ ├── dns.txt │ │ │ ├── internet.nl.0.cert │ │ │ ├── internet.nl.1.cert │ │ │ ├── mail.ietf.org.0.cert │ │ │ ├── mail.ietf.org.1.cert │ │ │ ├── mail.ietf.org.2.cert │ │ │ └── mail.ietf.org.3.cert │ │ ├── dsn/ │ │ │ ├── delay.eml │ │ │ ├── failure.eml │ │ │ ├── mixed.eml │ │ │ ├── original.txt │ │ │ └── success.eml │ │ ├── lists/ │ │ │ ├── test-list1.txt │ │ │ └── test-list2.txt │ │ ├── messages/ │ │ │ ├── arc.eml │ │ │ ├── dkim.eml │ │ │ ├── invalid_arc.eml │ │ │ ├── invalid_dkim.eml │ │ │ ├── loop.eml │ │ │ ├── multipart.eml │ │ │ ├── no_dkim.eml │ │ │ └── no_msgid.eml │ │ ├── milter/ │ │ │ ├── message.eml │ │ │ └── message.json │ │ ├── reports/ │ │ │ ├── arf1.eml │ │ │ ├── arf2.eml │ │ │ ├── arf3.eml │ │ │ ├── arf4.eml │ │ │ ├── arf5.eml │ │ │ ├── dmarc1.eml │ │ │ ├── dmarc2.eml │ │ │ ├── dmarc3.eml │ │ │ ├── dmarc4.eml │ │ │ ├── dmarc5.eml │ │ │ ├── tls1.eml │ │ │ └── tls2.eml │ │ └── sieve/ │ │ ├── awl.sieve │ │ ├── awl_include.sieve │ │ ├── stage_connect.sieve │ │ ├── stage_data.sieve │ │ ├── stage_ehlo.sieve │ │ ├── stage_mail.sieve │ │ └── stage_rcpt.sieve │ ├── tls_cert.pem │ └── tls_privatekey.pem └── src/ ├── cluster/ │ ├── broadcast.rs │ ├── mod.rs │ └── stress.rs ├── directory/ │ ├── imap.rs │ ├── internal.rs │ ├── ldap.rs │ ├── mod.rs │ ├── oidc.rs │ ├── smtp.rs │ └── sql.rs ├── http_server.rs ├── imap/ │ ├── acl.rs │ ├── antispam.rs │ ├── append.rs │ ├── basic.rs │ ├── body_structure.rs │ ├── condstore.rs │ ├── copy_move.rs │ ├── fetch.rs │ ├── idle.rs │ ├── mailbox.rs │ ├── managesieve.rs │ ├── mod.rs │ ├── pop.rs │ ├── search.rs │ ├── store.rs │ └── thread.rs ├── jmap/ │ ├── auth/ │ │ ├── limits.rs │ │ ├── mod.rs │ │ ├── oauth.rs │ │ ├── permissions.rs │ │ └── quota.rs │ ├── calendar/ │ │ ├── acl.rs │ │ ├── alarm.rs │ │ ├── calendars.rs │ │ ├── event.rs │ │ ├── identity.rs │ │ ├── mod.rs │ │ └── notification.rs │ ├── contacts/ │ │ ├── acl.rs │ │ ├── addressbook.rs │ │ ├── contact.rs │ │ └── mod.rs │ ├── core/ │ │ ├── blob.rs │ │ ├── event_source.rs │ │ ├── mod.rs │ │ ├── push_subscription.rs │ │ └── websocket.rs │ ├── files/ │ │ ├── acl.rs │ │ ├── mod.rs │ │ └── node.rs │ ├── mail/ │ │ ├── acl.rs │ │ ├── antispam.rs │ │ ├── changes.rs │ │ ├── copy.rs │ │ ├── crypto.rs │ │ ├── delivery.rs │ │ ├── get.rs │ │ ├── mailbox.rs │ │ ├── mod.rs │ │ ├── parse.rs │ │ ├── query.rs │ │ ├── query_changes.rs │ │ ├── search_snippet.rs │ │ ├── set.rs │ │ ├── sieve_script.rs │ │ ├── submission.rs │ │ ├── thread_get.rs │ │ ├── thread_merge.rs │ │ └── vacation_response.rs │ ├── mod.rs │ ├── principal/ │ │ ├── availability.rs │ │ ├── get.rs │ │ └── mod.rs │ └── server/ │ ├── enterprise.rs │ ├── mod.rs │ ├── purge.rs │ └── webhooks.rs ├── lib.rs ├── smtp/ │ ├── config.rs │ ├── inbound/ │ │ ├── antispam.rs │ │ ├── asn.rs │ │ ├── auth.rs │ │ ├── basic.rs │ │ ├── data.rs │ │ ├── dmarc.rs │ │ ├── ehlo.rs │ │ ├── limits.rs │ │ ├── mail.rs │ │ ├── milter.rs │ │ ├── mod.rs │ │ ├── rcpt.rs │ │ ├── rewrite.rs │ │ ├── scripts.rs │ │ ├── sign.rs │ │ ├── throttle.rs │ │ └── vrfy.rs │ ├── lookup/ │ │ ├── mod.rs │ │ ├── sql.rs │ │ └── utils.rs │ ├── management/ │ │ ├── mod.rs │ │ ├── queue.rs │ │ └── report.rs │ ├── mod.rs │ ├── outbound/ │ │ ├── dane.rs │ │ ├── extensions.rs │ │ ├── fallback_relay.rs │ │ ├── ip_lookup.rs │ │ ├── lmtp.rs │ │ ├── mod.rs │ │ ├── mta_sts.rs │ │ ├── smtp.rs │ │ ├── throttle.rs │ │ └── tls.rs │ ├── queue/ │ │ ├── concurrent.rs │ │ ├── dsn.rs │ │ ├── manager.rs │ │ ├── mod.rs │ │ ├── retry.rs │ │ └── virtualq.rs │ ├── reporting/ │ │ ├── analyze.rs │ │ ├── dmarc.rs │ │ ├── mod.rs │ │ ├── scheduler.rs │ │ └── tls.rs │ └── session.rs ├── store/ │ ├── blob.rs │ ├── cleanup.rs │ ├── import_export.rs │ ├── lookup.rs │ ├── mod.rs │ ├── ops.rs │ └── query.rs └── webdav/ ├── acl.rs ├── basic.rs ├── cal_alarm.rs ├── cal_itip.rs ├── cal_query.rs ├── cal_scheduling.rs ├── card_query.rs ├── copy_move.rs ├── lock.rs ├── mkcol.rs ├── mod.rs ├── multiget.rs ├── principals.rs ├── prop.rs ├── put_get.rs └── sync.rs ================================================ FILE CONTENTS ================================================ ================================================ FILE: .dockerignore ================================================ // Ignore everything * // Allow what is needed !crates !tests !resources !Cargo.lock !Cargo.toml ================================================ FILE: .editorconfig ================================================ # https://EditorConfig.org root = true [*] charset = utf-8 indent_size = 4 indent_style = space end_of_line = lf insert_final_newline = true trim_trailing_whitespace = true max_line_length = 100 ================================================ FILE: .github/DISCUSSION_TEMPLATE/issue-triage.yml ================================================ labels: ["triage"] body: - type: markdown attributes: value: | > [!IMPORTANT] > Please review the [documentation](https://stalw.art/docs/), check the [FAQ](https://stalw.art/docs/faq), and search existing [Discussions](https://github.com/stalwartlabs/stalwart/discussions) and [Issues](https://github.com/stalwartlabs/stalwart/issues?q=sort%3Areactions-desc) before opening a new Discussion. > > Most reported issues turn out to be configuration problems rather than actual bugs. Starting here helps us triage effectively—if this is confirmed as a bug, we'll create an Issue for tracking. - type: markdown attributes: value: "# Issue Details" - type: textarea attributes: label: Issue Description description: | Provide a detailed description of the issue. Include relevant context such as your configuration, environment, and any recent changes that might have led to the issue. placeholder: | When trying to send an email via SMTP, the connection is accepted but the message is rejected with error 550. validations: required: true - type: textarea attributes: label: Expected Behavior description: | Describe how you expect Stalwart to behave in this situation. Include any relevant documentation links. placeholder: | The email should be accepted and delivered to the recipient's mailbox. validations: required: false - type: textarea attributes: label: Actual Behavior description: | Describe how Stalwart actually behaves in this situation. If it is not immediately obvious how the actual behavior differs from the expected behavior described above, please mention the deviation specifically. placeholder: | The email is rejected with error: "550 5.7.1 Relay access denied" validations: required: false - type: textarea attributes: label: Reproduction Steps description: | Provide a detailed set of step-by-step instructions for reproducing this issue. placeholder: | 1. Configure Stalwart with the attached configuration. 2. Attempt to send an email from user@example.com to external@domain.com. 3. Observe the 550 error in the SMTP session. validations: required: false - type: textarea attributes: label: Relevant Log Output description: | Please copy and paste any relevant log output. Set logging level to `trace` if you can't find any relevant errors in the log. render: shell - type: dropdown attributes: label: Stalwart Version description: What version of Stalwart are you running? options: - v0.15.x - v0.14.x - v0.13.x - v0.12.x or lower validations: required: true - type: dropdown attributes: label: Installation Method description: How did you install Stalwart? options: - Docker - Binary (Linux) - Binary (macOS) - Binary (FreeBSD) - Binary (Windows) - NixOS - Built from source validations: required: true - type: dropdown attributes: label: Database Backend description: What database are you using for the data store? options: - RocksDB - FoundationDB - PostgreSQL - MySQL - SQLite validations: required: true - type: dropdown attributes: label: Blob Storage description: What blob storage are you using? options: - RocksDB - FoundationDB - PostgreSQL - MySQL - SQLite - Filesystem - S3-compatible - Azure validations: required: true - type: dropdown attributes: label: Search Engine description: What search engine are you using? options: - Internal - Meilisearch - Elasticsearch - PostgreSQL - MySQL validations: required: true - type: dropdown attributes: label: Directory Backend description: Where is your directory/user database located? options: - Internal - SQL - LDAP - OIDC validations: required: true - type: textarea attributes: label: Additional Context description: | Add any other context about the problem here. This could include: - Client software and versions (Thunderbird, Apple Mail, K-9, etc.) - Proxy or reverse proxy configuration (nginx, Traefik, etc.) - Network setup (NAT, firewall rules, etc.) validations: required: false - type: markdown attributes: value: | # Acknowledgements > [!TIP] > Use these links to review existing [Discussions](https://github.com/stalwartlabs/stalwart/discussions) and [Issues](https://github.com/stalwartlabs/stalwart/issues?q=sort%3Areactions-desc). - type: checkboxes attributes: label: "I acknowledge that:" options: - label: I have reviewed the documentation and FAQ and confirm that my issue is NOT addressed there. required: true - label: I have searched the Stalwart repository (both open and closed Discussions and Issues) and confirm this is not a duplicate. required: true - label: I have set the logging level to `trace` and included relevant log output if applicable. required: false - label: I agree to follow the project's [Code of Conduct](https://github.com/stalwartlabs/.github/blob/main/CODE_OF_CONDUCT.md). required: true ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: false contact_links: - name: Report an Issue url: https://github.com/stalwartlabs/stalwart/discussions/new?category=issue-triage about: Report a potential bug. Confirmed bugs will be converted to Issues. - name: Questions & Support url: https://github.com/stalwartlabs/stalwart/discussions/new?category=q-a about: Get help with configuration, troubleshooting, or general questions. - name: Feature Requests url: https://github.com/stalwartlabs/stalwart/discussions/new?category=feature-requests-and-ideas about: Suggest new features or improvements. - name: Join Stalwart's Reddit url: https://www.reddit.com/r/stalwartlabs about: Join our subreddit for help, discussions and release announcements. - name: Join Stalwart's Discord url: https://discord.com/servers/stalwart-923615863037390889 about: Join our Discord server for help, discussions and release announcements. ================================================ FILE: .github/ISSUE_TEMPLATE/confirmed_issue.yml ================================================ name: Confirmed Bug Report description: Only for issues that have been discussed and confirmed as bugs in GitHub Discussions. labels: ["bug"] title: "🪲: " body: - type: markdown attributes: value: | > [!IMPORTANT] > This template is **only** for issues that have already been discussed and confirmed as bugs in the [Discussions](https://github.com/stalwartlabs/stalwart/discussions) section. > > If you haven't had your issue confirmed yet, please [start a Discussion](https://github.com/stalwartlabs/stalwart/discussions/new?category=issue-triage) first. - type: input attributes: label: Discussion Link description: Provide the link to the Discussion where this issue was confirmed as a bug. placeholder: https://github.com/stalwartlabs/stalwart/discussions/1234 validations: required: true - type: textarea attributes: label: Bug Summary description: Brief summary of the confirmed bug. validations: required: true - type: textarea attributes: label: Additional Information description: | Include any additional information not covered in the original Discussion, such as new findings, workarounds discovered, or updated reproduction steps. validations: required: false - type: checkboxes attributes: label: Confirmation options: - label: This issue was discussed and confirmed as a bug in the linked Discussion. required: true - label: I have included the link to the Discussion above. required: true ================================================ FILE: .github/dependabot.yml ================================================ # To get started with Dependabot version updates, you'll need to specify which # package ecosystems to update and where the package manifests are located. # Please see the documentation for all configuration options: # https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file version: 2 updates: - package-ecosystem: "cargo" # See documentation for possible values directory: "/" # Location of package manifests schedule: interval: "weekly" # Enable version updates for GitHub Actions - package-ecosystem: "github-actions" # Workflow files stored in the default location of `.github/workflows` # You don't need to specify `/.github/workflows` for `directory`. You can use `directory: "/"`. directory: "/" schedule: interval: "weekly" ================================================ FILE: .github/workflows/ci.yml ================================================ name: "CI" on: workflow_dispatch: inputs: Docker: required: false default: false type: boolean Release: required: false default: false type: boolean push: tags: ["v*.*.*"] env: SCCACHE_GHA_ENABLED: true RUSTC_WRAPPER: sccache CARGO_TERM_COLOR: always concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: multiarch: strategy: fail-fast: false matrix: include: - variant: gnu - variant: musl name: Merge image / ${{matrix.variant}} runs-on: ubuntu-latest permissions: id-token: write contents: read attestations: write packages: write needs: [linux] if: github.event_name == 'push' || inputs.Docker steps: - name: Install Cosign uses: sigstore/cosign-installer@v3 - name: Log In to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{github.repository_owner}} password: ${{github.token}} - name: Log In to DockerHub uses: docker/login-action@v3 with: username: ${{secrets.DOCKERHUB_USERNAME}} password: ${{secrets.DOCKERHUB_TOKEN}} - name: Download ${{matrix.variant}} meta bake definition uses: actions/download-artifact@v7 with: name: bake-meta-${{matrix.variant}} path: ${{ runner.temp }}/${{matrix.variant}} - name: Download ${{matrix.variant}} digests uses: actions/download-artifact@v7 with: path: ${{ runner.temp }}/${{matrix.variant}}/digests pattern: digests-${{matrix.variant}}-* merge-multiple: true - name: Create ${{matrix.variant}} manifest list and push working-directory: ${{ runner.temp }}/${{matrix.variant}}/digests run: | docker buildx imagetools create $(jq -cr '.target."docker-metadata-action".tags | map(select(startswith("ghcr.io/${{github.repository}}")) | "-t " + .) | join(" ")' ${{ runner.temp }}/${{matrix.variant}}/bake-meta.json) \ $(printf 'ghcr.io/${{github.repository}}@sha256:%s ' *) docker buildx imagetools create $(jq -cr '.target."docker-metadata-action".tags | map(select(startswith("index.docker.io/${{github.repository}}")) | "-t " + .) | join(" ")' ${{ runner.temp }}/${{matrix.variant}}/bake-meta.json) \ $(printf 'index.docker.io/${{github.repository}}@sha256:%s ' *) - name: Inspect ${{matrix.variant}} image id: manifest-digest run: | docker buildx imagetools inspect --format '{{json .Manifest}}' ghcr.io/${{github.repository}}:$(jq -r '.target."docker-metadata-action".args.DOCKER_META_VERSION' ${{ runner.temp }}/${{matrix.variant}}/bake-meta.json) | jq -r '.digest' > GHCR_DIGEST_SHA echo "GHCR_DIGEST_SHA=$(cat GHCR_DIGEST_SHA)" | tee -a "${GITHUB_ENV}" docker buildx imagetools inspect --format '{{json .Manifest}}' index.docker.io/${{github.repository}}:$(jq -r '.target."docker-metadata-action".args.DOCKER_META_VERSION' ${{ runner.temp }}/${{matrix.variant}}/bake-meta.json) | jq -r '.digest' > DOCKERHUB_DIGEST_SHA echo "DOCKERHUB_DIGEST_SHA=$(cat DOCKERHUB_DIGEST_SHA)" | tee -a "${GITHUB_ENV}" cosign sign --yes $(jq --arg GHCR_DIGEST_SHA "$(cat GHCR_DIGEST_SHA)" -cr '.target."docker-metadata-action".tags | map(select(startswith("ghcr.io/${{github.repository}}")) | . + "@" + $GHCR_DIGEST_SHA) | join(" ")' ${{ runner.temp }}/${{matrix.variant}}/bake-meta.json) cosign sign --yes $(jq --arg DOCKERHUB_DIGEST_SHA "$(cat DOCKERHUB_DIGEST_SHA)" -cr '.target."docker-metadata-action".tags | map(select(startswith("index.docker.io/${{github.repository}}")) | . + "@" + $DOCKERHUB_DIGEST_SHA) | join(" ")' ${{ runner.temp }}/${{matrix.variant}}/bake-meta.json) - name: Attest GHCR uses: actions/attest-build-provenance@v3 with: subject-name: ghcr.io/${{github.repository}} subject-digest: ${{ env.GHCR_DIGEST_SHA }} push-to-registry: true - name: Attest Dockerhub uses: actions/attest-build-provenance@v3 with: subject-name: index.docker.io/${{github.repository}} subject-digest: ${{ env.DOCKERHUB_DIGEST_SHA }} push-to-registry: true linux: permissions: id-token: write contents: write attestations: write packages: write strategy: fail-fast: false matrix: include: - target: x86_64-unknown-linux-gnu platform: linux/amd64 suffix: "" build_env: "" - target: x86_64-unknown-linux-musl platform: linux/amd64 suffix: "-alpine" build_env: "" - target: aarch64-unknown-linux-gnu platform: linux/arm64 suffix: "" build_env: "JEMALLOC_SYS_WITH_LG_PAGE=16 " - target: aarch64-unknown-linux-musl platform: linux/arm64 suffix: "-alpine" build_env: "JEMALLOC_SYS_WITH_LG_PAGE=16 " - target: armv7-unknown-linux-gnueabihf platform: linux/arm/v7 suffix: "" build_env: "JEMALLOC_SYS_WITH_LG_PAGE=16 " - target: armv7-unknown-linux-musleabihf platform: linux/arm/v7 suffix: "-alpine" build_env: "JEMALLOC_SYS_WITH_LG_PAGE=16 " - target: arm-unknown-linux-gnueabihf platform: linux/arm/v6 suffix: "" build_env: "" - target: arm-unknown-linux-musleabihf platform: linux/arm/v6 suffix: "-alpine" build_env: "" name: Build / ${{matrix.target}} runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v6.0.1 - name: Set up QEMU uses: docker/setup-qemu-action@v3 with: platforms: "arm64,arm" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 with: buildkitd-config-inline: | [registry."docker.io"] mirrors = ["https://mirror.gcr.io"] driver-opts: | network=host - name: Log In to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{github.repository_owner}} password: ${{github.token}} - name: Log In to DockerHub uses: docker/login-action@v3 with: username: ${{secrets.DOCKERHUB_USERNAME}} password: ${{secrets.DOCKERHUB_TOKEN}} - name: Calculate shasum of external deps id: cal-dep-shasum run: | echo "checksum=$(yq -p toml -oy '.package[] | select((.source | contains("")) or (.checksum | contains("")))' Cargo.lock | sha256sum | awk '{print $1}')" >> "$GITHUB_OUTPUT" - name: Cache apt uses: actions/cache@v5 id: apt-cache with: path: | var-cache-apt var-lib-apt key: apt-cache-${{ hashFiles('Dockerfile.build') }} - name: Cache Cargo uses: actions/cache@v5 id: cargo-cache with: path: | usr-local-cargo-registry usr-local-cargo-git key: cargo-cache-${{ steps.cal-dep-shasum.outputs.checksum }} - name: Inject cache into docker uses: reproducible-containers/buildkit-cache-dance@v3.3.0 with: cache-map: | { "var-cache-apt": "/var/cache/apt", "var-lib-apt": "/var/lib/apt", "usr-local-cargo-registry": "/usr/local/cargo/registry", "usr-local-cargo-git": "/usr/local/cargo/git" } skip-extraction: ${{ steps.cargo-cache.outputs.cache-hit }} && ${{ steps.apt-cache.outputs.cache-hit }} - name: Extract Metadata for Docker uses: docker/metadata-action@v5 id: meta with: images: | index.docker.io/${{github.repository}} ghcr.io/${{github.repository}} flavor: | suffix=${{matrix.suffix}},onlatest=true tags: | type=ref,event=tag type=ref,event=branch,prefix=branch- type=edge,branch=main type=semver,pattern=v{{major}}.{{minor}} - name: Build Artifact id: bake uses: docker/bake-action@v6 env: DOCKER_BUILD_RECORD_UPLOAD: false TARGET: ${{matrix.target}} GHCR_REPO: ghcr.io/${{github.repository}} BUILD_ENV: ${{matrix.build_env}} DOCKER_PLATFORM: ${{matrix.platform}} SUFFIX: ${{matrix.suffix}} with: source: . set: | *.tags= image.output=type=image,"name=ghcr.io/${{github.repository}},index.docker.io/${{github.repository}}",push-by-digest=true,name-canonical=true,push=true,compression=zstd,compression-level=9,force-compression=true,oci-mediatypes=true files: | docker-bake.hcl ${{ steps.meta.outputs.bake-file }} targets: ${{(github.event_name == 'push' || inputs.Docker) && 'build,image' || 'build'}} - name: Upload Artifacts uses: actions/upload-artifact@v6 with: name: artifact-${{matrix.target}} path: | artifact !artifact/*.json - name: Export digest & Rename meta bake definition file if: github.event_name == 'push' || inputs.Docker run: | mv "${{ steps.meta.outputs.bake-file }}" "${{ runner.temp }}/bake-meta.json" mkdir -p ${{ runner.temp }}/digests digest="${{ fromJSON(steps.bake.outputs.metadata).image['containerimage.digest'] }}" touch "${{ runner.temp }}/digests/${digest#sha256:}" - name: Upload digest if: github.event_name == 'push' || inputs.Docker uses: actions/upload-artifact@v6 with: name: digests-${{matrix.suffix == '' && 'gnu' || 'musl'}}-${{ matrix.target }} path: ${{ runner.temp }}/digests/* if-no-files-found: error retention-days: 1 - name: Upload GNU meta bake definition uses: actions/upload-artifact@v6 if: (github.event_name == 'push' || inputs.Docker) && endsWith(matrix.target,'gnu') && startsWith(matrix.target,'x86') with: name: bake-meta-gnu path: ${{ runner.temp }}/bake-meta.json if-no-files-found: error retention-days: 1 - name: Upload musl meta bake definition uses: actions/upload-artifact@v6 if: (github.event_name == 'push' || inputs.Docker) && endsWith(matrix.target,'musl') && startsWith(matrix.target,'x86') with: name: bake-meta-musl path: ${{ runner.temp }}/bake-meta.json if-no-files-found: error retention-days: 1 windows: name: Build / ${{matrix.target}} runs-on: windows-latest strategy: fail-fast: false matrix: include: # - target: aarch64-pc-windows-msvc - target: x86_64-pc-windows-msvc steps: - name: Checkout uses: actions/checkout@v6.0.1 - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.9 with: disable_annotations: true - name: Build run: | rustup target add ${{matrix.target}} cargo build --release --target ${{matrix.target}} -p stalwart --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats enterprise" cargo build --release --target ${{matrix.target}} -p stalwart-cli mkdir -p artifacts mv ./target/${{matrix.target}}/release/stalwart.exe ./artifacts/stalwart.exe mv ./target/${{matrix.target}}/release/stalwart-cli.exe ./artifacts/stalwart-cli.exe - name: Upload Artifacts uses: actions/upload-artifact@v6 with: name: artifact-${{matrix.target}} path: artifacts macos: name: Build / ${{matrix.target}} runs-on: macos-latest strategy: fail-fast: false matrix: include: - target: aarch64-apple-darwin - target: x86_64-apple-darwin steps: - name: Checkout uses: actions/checkout@v6.0.1 - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.9 with: disable_annotations: true #- name: Build FoundationDB Edition # env: # GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # run: | # rustup target add ${{matrix.target}} # # Get latest FoundationDB installer # curl --retry 5 -Lso foundationdb.pkg "$(gh api -X GET /repos/apple/foundationdb/releases --jq '.[] | select(.prerelease == false) | .assets[] | select(.name | test("${{startsWith(matrix.target, 'x86') && 'x86_64' || 'arm64'}}" + ".pkg$")) | .browser_download_url' | head -n1)" # echo "=== Package contents ===" # pkgutil --payload-files foundationdb.pkg || true # sudo installer -allowUntrusted -verbose -dumplog -pkg foundationdb.pkg -target / # cargo build --release --target ${{matrix.target}} -p stalwart --no-default-features --features "foundationdb s3 redis nats enterprise" # mkdir -p artifacts # mv ./target/${{matrix.target}}/release/stalwart ./artifacts/stalwart-foundationdb - name: Build run: | rustup target add ${{matrix.target}} cargo build --release --target ${{matrix.target}} -p stalwart --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats enterprise" cargo build --release --target ${{matrix.target}} -p stalwart-cli mkdir -p artifacts mv ./target/${{matrix.target}}/release/stalwart ./artifacts/stalwart mv ./target/${{matrix.target}}/release/stalwart-cli ./artifacts/stalwart-cli - name: Upload Artifacts uses: actions/upload-artifact@v6 with: name: artifact-${{matrix.target}} path: artifacts release: name: Release permissions: id-token: write contents: write attestations: write if: github.event_name == 'push' || inputs.Release needs: [linux, windows, macos] runs-on: ubuntu-latest steps: - name: Download Artifacts uses: actions/download-artifact@v7 with: path: archive pattern: artifact-* - name: Compress run: | set -eux BASE_DIR="$(pwd)/archive" compress_files() { local dir="$1" local archive_dir_name="${dir#artifact-}" cd "$dir" # Process each file in the directory for file in `ls`; do filename="${file%.*}" extension="${file##*.}" if [ "$extension" = "exe" ]; then 7z a -tzip "${filename}-${archive_dir_name}.zip" "$file" > /dev/null else tar -czf "${filename}-${archive_dir_name}.tar.gz" "$file" fi done cd $BASE_DIR } cd $BASE_DIR for arch_dir in `ls`; do dir_name=$(basename "$arch_dir") compress_files "$dir_name" done - name: Attest binary id: attest uses: actions/attest-build-provenance@v3 with: subject-path: | archive/**/*.tar.gz archive/**/*.zip - name: Use cosign to sign existing artifacts uses: sigstore/gh-action-sigstore-python@v3.2.0 with: inputs: | archive/**/*.tar.gz archive/**/*.zip - name: Release uses: softprops/action-gh-release@v2 with: files: | archive/**/*.tar.gz archive/**/*.zip archive/**/*.sigstore.json prerelease: ${{!startsWith(github.ref, 'refs/tags/') || null}} tag_name: ${{!startsWith(github.ref, 'refs/tags/') && 'nightly' || null}} # TODO add instructions about using cosign to verify binary artifact append_body: true body: |
### Check binary attestation at [here](${{ steps.attest.outputs.attestation-url }}) ================================================ FILE: .github/workflows/scorecard.yml ================================================ # This workflow uses actions that are not certified by GitHub. They are provided # by a third-party and are governed by separate terms of service, privacy # policy, and support documentation. name: Scorecard supply-chain security on: # For Branch-Protection check. Only the default branch is supported. See # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection branch_protection_rule: # To guarantee Maintained check is occasionally updated. See # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained schedule: - cron: '31 6 * * 0' push: branches: [ "main" ] # Declare default permissions as read only. permissions: read-all jobs: analysis: name: Scorecard analysis runs-on: ubuntu-latest # `publish_results: true` only works when run from the default branch. conditional can be removed if disabled. if: github.event.repository.default_branch == github.ref_name || github.event_name == 'pull_request' permissions: # Needed to upload the results to code-scanning dashboard. security-events: write # Needed to publish results and get a badge (see publish_results below). id-token: write # Uncomment the permissions below if installing in a private repository. # contents: read # actions: read steps: - name: "Checkout code" uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v4.2.2 with: persist-credentials: false - name: "Run analysis" uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3 with: results_file: results.sarif results_format: sarif # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: # - you want to enable the Branch-Protection check on a *public* repository, or # - you are installing Scorecard on a *private* repository # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional. # repo_token: ${{ secrets.SCORECARD_TOKEN }} # Public repositories: # - Publish results to OpenSSF REST API for easy access by consumers # - Allows the repository to include the Scorecard badge. # - See https://github.com/ossf/scorecard-action#publishing-results. # For private repositories: # - `publish_results` will always be set to `false`, regardless # of the value entered here. publish_results: true # (Optional) Uncomment file_mode if you have a .gitattributes with files marked export-ignore # file_mode: git # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: SARIF file path: results.sarif retention-days: 5 # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" uses: github/codeql-action/upload-sarif@v4 with: sarif_file: results.sarif ================================================ FILE: .github/workflows/test.yml ================================================ name: Test on: workflow_dispatch: jobs: style: name: Check Style runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v6.0.1 - name: Check Style run: cargo fmt --all --check test: name: Test needs: style runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v6.0.1 - name: Install dependencies run: | sudo apt-get update -y curl -LO https://github.com/glauth/glauth/releases/download/v2.2.0/glauth-linux-arm64 chmod a+rx glauth-linux-arm64 nohup ./glauth-linux-arm64 -c tests/resources/ldap.cfg & curl -Lo minio.deb https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20230629051228.0.0_amd64.deb sudo dpkg -i minio.deb mkdir ~/minio nohup minio server ~/minio --console-address :9090 & curl -LO https://dl.min.io/client/mc/release/linux-amd64/mc chmod a+rx mc ./mc alias set myminio http://localhost:9000 minioadmin minioadmin ./mc mb tmp - name: Rust Cache uses: Swatinem/rust-cache@v2 - name: JMAP Protocol Tests run: cargo test -p jmap_proto -- --nocapture - name: IMAP Protocol Tests run: cargo test -p imap_proto -- --nocapture - name: Full-text search Tests run: cargo test -p store -- --nocapture - name: Directory Tests run: cargo test -p tests directory -- --nocapture - name: SMTP Tests run: cargo test -p tests smtp -- --nocapture - name: IMAP Tests run: cargo test -p tests imap -- --nocapture - name: JMAP Tests run: cargo test -p tests jmap -- --nocapture ================================================ FILE: .github/workflows/trivy.yml ================================================ # trivy ci workflow name: trivy on: workflow_dispatch: push: branches: [ "main" ] pull_request: # The branches below must be a subset of the branches above branches: [ "main" ] schedule: - cron: '00 12 * * *' permissions: contents: read jobs: build: permissions: contents: read # for actions/checkout to fetch code security-events: write # for github/codeql-action/upload-sarif to upload SARIF results actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status name: Check runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v6.0.1 - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@master with: scan-type: 'fs' ignore-unfixed: true format: 'sarif' output: 'trivy-results.sarif' severity: 'CRITICAL,HIGH' - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@v4 with: sarif_file: 'trivy-results.sarif' ================================================ FILE: .gitignore ================================================ /target .vscode .idea *.failed *_failed run.sh _ignore .DS_Store ================================================ FILE: CHANGELOG.md ================================================ # Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). ## [0.15.5] - 2026-02-14 If you are upgrading from v0.14.x and below, this version includes **multiple breaking changes**. Please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_15.md) for more information on how to upgrade from previous versions. If you are upgrading from v0.15.x, replace the binary and update the webadmin. ## Added ## Changed ## Fixed - IMAP/JMAP: OOM when `mail-parser` returns cyclical MIME structures [CVE-2026-26312](https://github.com/stalwartlabs/stalwart/security/advisories/GHSA-jm95-876q-c9gw). - Tracing: Fix tracing indexing when using separate stores. - JMAP: Fix `upToId` computation in `*/queryChanges`. - JMAP: Include createdIds when the property is present. - JMAP: Respect query arguments in `Email/queryChanges`. - JMAP: Return the correct container/item change id when there are no changes. ## [0.15.4] - 2026-01-19 If you are upgrading from v0.14.x and below, this version includes **multiple breaking changes**. Please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_15.md) for more information on how to upgrade from previous versions. If you are upgrading from v0.15.x, replace the binary and update the webadmin. ## Added - IMAP: Map `HEADER SUBJECT/FROM/TO` searches to `SUBJECT/FROM/TO` queries. - Sieve: Update spam status on user scripts. ## Changed ## Fixed - Search: Return all document ids when no filters are provided. - Search: Filters not applied when a single message is in the account. - IMAP: Return `ALREADYEXISTS` code when creating existing mailboxes. - IMAP: Do not return quota resources if no quota is set. - JMAP/changes: Update `newState` with last changeId if an invalid fromChangeId is provided. - JMAP/CalendarIdentity: Do not update invalid calendar identities. - AI API: Include request error details if available. ## [0.15.3] - 2025-12-29 If you are upgrading from v0.14.x and below, this version includes **multiple breaking changes**. Please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_15.md) for more information on how to upgrade from previous versions. If you are upgrading from v0.15.x, replace the binary and update the webadmin. ## Added - Polish locale support (contributed by @mrxkp) (#2480) ## Changed ## Fixed - Meilisearch: Return correct error messages when failing to create indexes (#2574) - PostgreSQL search: Truncate emails to 650kb for full-text search indexing. - FoundationDB search: Batch large transactions (#2567). - Spam filter: Fix training sample size checks - IMAP: Fix UTF7 encoding with Emojis (contributed by @dojiong) (#2564). ## [0.15.2] - 2025-12-22 If you are upgrading from v0.14.x and below, this version includes **multiple breaking changes**. Please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_15.md) for more information on how to upgrade from previous versions. If you are upgrading from v0.15.x, replace the binary and update the webadmin. ## Added - OAuth: Add device authorization endpoint (#2225). ## Changed - Antispam: Only auto-learn spam from traps or multiple RBL hits. ## Fixed - mySQL search: Use `MEDIUMTEXT` field type for email body and attachments (#2544). - PostgreSQL search: Truncate large text fields. - ElasticSearch: Implement pagination (#2551). - Antispam: Fix `NO_SPACE_IN_FROM` spam tag detection logic (#2372). - IMAP: Fix shared folder double nesting (test suite credits to @ochnygosch) (#2358). - JMAP: Use latest `Received` header in JMAP `Email/import` (credits to @apexskier) (#2374). - JMAP: Return unsorted search results when the index is not ready (#2544). - LDAP: Lowercase attribute comparison (credits to @pdf) (#2363). - CLI: Fix same-host JMAP redirection on non-standard ports (#2271). ## [0.15.1] - 2025-12-17 This version includes **multiple breaking changes**. If you are upgrading from v0.14.x and below, please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_15.md) for more information on how to upgrade from previous versions. ## Added ## Changed ## Fixed - PostgreSQL: Sanitize search index values (#2533) - Elasticsearch: Ignore `resource_already_exists_exception` errors when creating indexes (#2535) - Migrate 0.13.x data (#2534) ## [0.15.0] - 2025-12-16 This version includes **multiple breaking changes**. Please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_15.md) for more information on how to upgrade from previous versions. ## Added - Linear spam classifier using FTRL-Proximal and feature/cuckoo hashing. - Meilisearch store backend implementation (#1482). - PostgreSQL and mySQL native full-text search support. - Multiple performance improvements and database access optimizations. - Encryption-at-rest: Spam training privacy setting. - Enterprise: Undelete e-mail feature now includes From/Subject/Received information. - IMAP: Implemented new keywords and mailbox attributes described in [draft-ietf-mailmaint-messageflag-mailboxattribute-13](https://datatracker.ietf.org/doc/html/draft-ietf-mailmaint-messageflag-mailboxattribute-13) ## Changed - IMAP: Always return special use flags in responses. ## Fixed - JMAP: `FileNode/set` fails to delete files (#2485). - JMAP: Return error when using `blobId` in JSContact and JSCalendar (#2431). - Directory: Deletion of list or domain issues (#2415). - MTA: Headers and body stripped from mail delivery subsystem failure notifications (#2344). - MTA: Hooks only run if sieve script, milter or rewrite is configured (#2317). - Autodiscover: Endpoint should be case insensitive (#2440). - Housekeeper: Panic during DST transition (#2366). - Import/Export: Fix import/export utility (#1882). - Enterprise: Remove tenant admin permissions when license is invalid. ## [0.14.1] - 2025-10-28 If you are upgrading from v0.13.4 and below, this version includes **breaking changes** to the internal directory, calendar and contacts. Please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_14.md) for more information on how to upgrade from previous versions. ## Added - Autoconfig for CalDAV, CardDAV and WebDAV (#1937) ## Changed - HTTP: Remove HTTP STS `preload` directive. ## Fixed - Directory: Keep OTP Auth and AppPasswords unless the remote directory provides new ones (#2319) - JMAP: Fix `ContactCard/set` and `CalendarEvent/set` destroy methods (#2308). ## [0.14.0] - 2025-10-22 If you are upgrading from v0.13.4 and below, this version includes **breaking changes** to the internal directory, calendar and contacts. Please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_14.md) for more information on how to upgrade from previous versions. ## Added - JMAP for Calendars ([draft-ietf-jmap-calendars](https://datatracker.ietf.org/doc/draft-ietf-jmap-calendars/)). - JMAP for Contacts ([RFC 9610](https://datatracker.ietf.org/doc/rfc9610/)). - JMAP for File Storage ([draft-ietf-jmap-filenode](https://datatracker.ietf.org/doc/draft-ietf-jmap-filenode/)). - JMAP Sharing ([RFC 9670](https://datatracker.ietf.org/doc/rfc9670/)) - CalDAV: support for `supported-calendar-component-set` (#1893) - i18n: Greek language support (contributed by @infl00p) - i18n: Swedish language support (contributed by @purung) ## Changed - **Breaking Database Changes** (migrated automatically on first start): - Internal directory schema changed. - Calendar and Contacts storage schema changed. - Sieve scripts storage schema changed. - Push Subscriptions storage schema changed. - Replaced `sieve.untrusted.limits.max-scripts` and `jmap.push.max-total` with `object-quota.*` settings. - Cluster node roles now allow sharding. ## Fixed - Push Subscription: Clean-up of expired subscriptions and cluster notification of changes (#1248) - CalDAV: Per-user CalDAV properties (#2058) ## [0.13.4] - 2025-09-30 If you are upgrading from v0.11.x or v0.12.x, this version includes **breaking changes** to the message queue and MTA configuration. Please read the [UPGRADING.md](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING.md) file for more information on how to upgrade from previous versions. ## Added ## Changed - JMAP: Protocol layer rewrite for zero-copy deserialization and architectural improvements. ## Fixed - IMAP: Unbounded memory allocation in request parser ([CVE-2025-61600 ](https://github.com/stalwartlabs/stalwart/security/advisories/GHSA-8jqj-qj5p-v5rr)). - IMAP: Wrong permission checked for GETACL. - JMAP: References to previous method fail when there are no results (#1507). - JMAP: Enforce quota checks on `Blob/copy`. - JMAP: `Mailbox/get` fails without `accountId` argument (#1936). - JMAP: Do not return `invalidProperties` when email update doesn't contain changes (#1139) - iTIP: Include date properties in `REPLY` (#2102). - OIDC: Do not set `username` field if it is the same as the `email` field. - Telemetry: Fix `calculateMetrics` housekeeper task (#2155). - Directory: Always use `rsplit` to extract the domain part from email addresses. ## [0.13.3] - 2025-09-10 If you are upgrading from v0.11.x or v0.12.x, this version includes **breaking changes** to the message queue and MTA configuration. Please read the [UPGRADING.md](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING.md) file for more information on how to upgrade from previous versions. ## Added - CLI: Health checks (contributed by @Codekloeppler) ## Changed - WebDAV: Assisted discovery v2 ## Fixed - iTIP: Do not send a REPLY when deleting an event that was not accepted. - iTIP: Include event details in REPLY messages (#2102). - iTIP: Add organizer to iMIP replies if missing to deal with MS Exchange 2010 bug. - OIDC: Do not overwrite locally defined aliases (#2065). - HTTP: Scan ban should only be triggered by HTTP parse errors. - HTTP: Skip scanner fail2ban checks when the proxy client IP can't be parsed (#2121). - JMAP: Do not allow roles to be removed from system mailboxes (#1977). - JMAP WS: Fix panic when using invalid server url. - SMTP: Do no send `EHLO` twice when `STARTTLS` is unavailable (#2050). - IMAP: Allow `ENABLE UTF8` in IMAPrev1. - IMAP: Include `administer` permission in ACL responses. - IMAP: Add owner rights to ACL get responses. - IMAP: Do not auto-train Bayes when moving messages from Junk to Trash. - IMAP/ManageSieve: Increase maximum quoted argument size (#2039). - CalDAV: Limit recurrence expansions in calendar reports ([CVE-2025-59045](https://github.com/stalwartlabs/stalwart/security/advisories/GHSA-xv4r-q6gr-6pfg)). - WebDAV: Do not fix percent encoding on WebDAV FS (#2036). ## [0.13.2] - 2025-07-28 If you are upgrading from v0.11.x or v0.12.x, this version includes **breaking changes** to the message queue and MTA configuration. Please read the [UPGRADING.md](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING.md) file for more information on how to upgrade from previous versions. ## Added - ACME: DeSEC cloud DNS provider support (contributed by @Tyr3al). - ACME: OVH cloud DNS provider support (contributed by @srachner). - CalDAV Scheduling: Catalan language support (contributed by @jolupa) (#1873). - MTA: Allow to send e-mails as group, while member of that group (#485). - OIDC: Allow local access tokens to be used with third-party OIDC backends (#1311 stalwartlabs/webadmin#52). ## Changed - IMAP: Return `OK` when moving/copying non-existent messages (#670). - IMAP: Copy flags when copying/moving messages between accounts. ## Fixed - MTA: Do not convert e-mail local parts to lowercase (#1916). - Sieve: `fileinto` should override spam filter (#1917). - JMAP: Incorrect `accountId` used in email set and import methods (#1777). - WebDAV: Always return `MULTISTATUS` when calendar-query yields no results. - LDAP: Only set account name if not returned in LDAP query (#1471). - Enterprise: Invalidate logo cache when changes are made (#1856). - Enterprise: Fix tenant quota update API. ## [0.13.1] - 2025-07-16 If you are upgrading from v0.11.x or v0.12.x, this version includes **breaking changes** to the message queue and MTA configuration. Please read the [UPGRADING.md](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING.md) file for more information on how to upgrade from previous versions. ## Added - ACME: DigitalOcean cloud DNS provider support (#1667). ## Changed ## Fixed - Migration: Old queue events not deleted causing high CPU usage in some deployments (#1833). - MTA: `mta-sts` setting parsing issue (#1830). - JMAP: `sortOrder` should not be null (#1831). - Allow invalid TOML when parsing database settings (#1822). ## [0.13.0] - 2025-07-15 If you are upgrading from v0.11.x or v0.12.x, this version includes **breaking changes** to the message queue and MTA configuration. Please read the [UPGRADING.md](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING.md) file for more information on how to upgrade from previous versions. ## Added - MTA queue enhancements (#1246 #1035 #457). - Danish locale support (contributed by @Fadil2k) (#1772). - DKIM support for `stalwart-cli` (contributed by @rmsc) (#1804). ## Changed - Invalidate access token caches in a cluster using pub/sub (#1741). - Allow updating secrets for all directory types. ## Fixed - WebDAV: Return all shared resources in `calendar-home-set` and `addressbook-home-set` (#1796). - WebDAV ACL: Fix write permission and `multiget` reports (#1768). - CalDAV Scheduling: Include `DTSTART`/`DTEND` properties in iMIP `CANCEL` messages (#1775). - HTTP: Do not include `WWW-Authenticate` headers in API responses (#1795). - API: Allow API keys to be used with external directories (#1815). - IMAP: Fix issue creating subfolders under INBOX for group shared folder (#1817). - IMAP: Custom Name for Shared Folders ignored (#1620). - LDAP: `local` placeholder should return username when its not an email address (#1784). ## [0.12.5] - 2025-06-25 If you are upgrading from v0.11.x, this version includes **breaking changes** to the database layout and requires a migration. Please read the [UPGRADING.md](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING.md) file for more information on how to upgrade from previous versions. ## Added - Calendar Scheduling Extensions to CalDAV - RFC6368 (#1514) - Calendar E-Mail Notifications (#1514) - Limited i18n support for calendaring events. - Assisted CalDAV/CardDAV shared resource discovery (#1691). ## Changed - JMAP: Allow unauthenticated access to JMAP session object. ## Fixed - WebDAV: Return NOTFOUND error instead of MULTISTATUS on empty PROPFIND responses (#1657). - WebDAV: Update account name when refreshing DAV caches (#1694). - JMAP: Do not include email address in identity names (#1688). - IMAP: Normalize `INBOX` name when creating/renaming folders (#1636). - LDAP: Request `secret-changed` attribute in LDAP queries (#1409). - Branding: Unable to change logos (#1652). - Antispam: Skip `card-is-ham` override when sender does not pass DMARC (#1648). - FoundationDB: Renew old/expired FDB read transactions after the `1007` error code is received rather than estimating expiration time. ## [0.12.4] - 2025-06-03 If you are upgrading from v0.11.x, this version includes **breaking changes** to the database layout and requires a migration. Please read the [UPGRADING.md](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING.md) file for more information on how to upgrade from previous versions. ## Added - LDAP authentication enhancements (#1269 #1471 #795 #1496). - MTA: Return Queue IDs during message acceptance (#927). ## Changed - LDAP: `bind.auth.enable` is now `bind.auth.method`, read the updated [LDAP documentation](https://stalw.art/docs/auth/backend/ldap) for more information. ## Fixed - DNS: `hickory-resolver` bug hitting 100% CPU usage when resolving DNSSEC records. - IMAP: Return the message UID in the destination mailbox if the message already exists (#1201). - MTA: TLS reports being issued for sent TLS reports (infinite loop) (#1301). - WebDAV: Return `CTag` on `/dav/cal/account` resources to force iOS synchronize. - CardDAV: Strict vCard parsing (#1607). - WebDAV: Dead property updates (#1611). - WebDAV: Use last change id in `CTag`. ## [0.12.3] - 2025-05-30 If you are upgrading from v0.11.x, this version includes **breaking changes** to the database layout and requires a migration. Please read the [UPGRADING.md](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING.md) file for more information on how to upgrade from previous versions. ## Added - Store vanished IMAP UIDs and WebDAV paths in the changelog. ## Changed ## Fixed - XML `CDATA` injection (credits to @andreymal for the report). - Macro references are replaced with their content when writing config file (#1595). - Double nested CalDAV and CardDAV property tags (#1591). - Allow empty properties in PROPPATCH requests (#1580). ## [0.12.2] - 2025-05-27 If you are upgrading from v0.11.x, this version includes **breaking changes** to the database layout and requires a migration. Please read the [UPGRADING.md](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING.md) file for more information on how to upgrade from previous versions. ## Added - CardDAV: Legacy vCard 2.1 and 3.0 serialization support. - WebDAV: Add SRV Records to help DAV autodiscovery (closes #1565). ## Changed ## Fixed - Report list attempts to deserialize empty values (#1562) - Refresh expired FoundationDB transactions while retrieving large blobs (#1555). ## [0.12.1] - 2025-05-26 If you are upgrading from v0.11.x, this version includes **breaking changes** to the database layout and requires a migration. Please read the [UPGRADING.md](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING.md) file for more information on how to upgrade from previous versions. ## Added ## Changed ## Fixed - Migration tool to generate the correct next id (#1561). - Failed to parse setting dav.lock.max-timeout (closes #1559). - Failed to build OpenTelemetry span exporter: no http client specified (#1571). ## [0.12.0] - 2025-05-26 This version includes **breaking changes** to the database layout and requires a migration. Please read the [UPGRADING.md](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING.md) file for more information on how to upgrade from previous versions. ### Added - [Collaboration](https://stalw.art/docs/collaboration/overview) features including [Calendars over CalDAV](https://stalw.art/docs/http/calendar/), [Contacts over CardDAV](https://stalw.art/docs/http/contact/) and [File Storage over WebDAV](https://stalw.art/docs/http/file-storage/). - Peer-to-peer [cluster coordination](https://stalw.art/docs/cluster/coordination/overview) or with Apache Kafka, Redpanda, NATS or Redis. - Incremental caching of emails, calendars, contacts and file metadata. - Zero-copy deserialization. - Train spam messages as ham when the sender is in the user's address book. - `XOAUTH2` SASL mechanism support (#1194 #1369). - Support for RFC9698, the `JMAPACCESS` Extension for IMAP. - Search index for accounts and other principals (#1368). - Add `description` property to OIDC ID token (#1234). ### Changed - Deprecated gossip protocol in favor of the new [coordinator](https://stalw.art/docs/cluster/coordination/overview) options. - Renamed Git repository from `stalwartlabs/mail-server` to `stalwartlabs/stalwart` and the Docker image from `stalwartlabs/mail-server` to `stalwartlabs/stalwart`. - Renamed multiple settings: - `server.http.*` to `http.*`. - `jmap.folders.*` to `email.folders.*`. - `jmap.account.purge.frequency` to `account.purge.frequency`. - `jmap.email.auto-expunge` to `email.auto-expunge`. - `jmap.protocol.changes.max-history` to `changes.max-history`. - `storage.encryption.*` to `email.encryption.*`. - Deprecated `lookup.default.*` settings in favor of `server.hostname` and `report.domain`. v0.11 and before supported both, v0.12 will only support the new settings. ### Fixed - Allow undiscovered UIDs to be used in IMAP `COPY`/`MOVE` operations (#1201). - Refuse loopback SMTP delivery (#1377). - Hide the current server version (#1435). - Use the newest `X-Spam-Status` Header (#1308). - MySQL Driver error: Transactions couldn't be nested (#1271). - Spawn a delivery thread for `EmailSubmission/set` requests (#1540). - ACME: Don't restrict challenge types (#1522). - Autoconfig: return `%EMAILADDRESS%` if no e-mail address is provided (#1537). ## [0.11.8] - 2025-04-30 To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin. ### Added ### Changed ### Fixed - Allow undiscovered UIDs to be used in `COPY`/`MOVE` operations (#1201). ## [0.11.7] - 2025-03-23 To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin. ### Added - LDAP attribute to indicate password change (#1156). ### Changed - Lazy DKIM key parsing (#1211). - Enable `edns0` for system resolver by default (#1282). - Bump FoundationDB to `7.3`. ### Fixed - Fix incorrect `UIDNEXT` when mailbox is empty (#1201). - Sender variable not set when evaluating `must-match-sender` (#1294). - Do not panic when mailboxId is not found (#1293). - Prioritize local over span keys when serializing webhook payloads (#1250). - Allow TLS name mismatch as per RFC7671 Section 5.1. - Try with implicit MX when no MX records are found. - SQL `secrets` directory query. ## [0.11.5] - 2025-02-01 To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin. ### Added ### Changed - Open source third party OIDC support. ### Fixed - Case insensitive flag parsing (#1138). - BCC not removed from JMAP EmailSubmissions (#618). - Group pipelined IMAP FETCH and STATUS operations (#1096). ## [0.11.4] - 2025-01-29 To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin. ### Added - RFC 9208 - IMAP QUOTA Extension (#484). ### Changed - `session.throttle.*` is now `queue.limiter.inbound.*`. - `queue.throttle.*` is now `queue.limiter.outbound.*`. - Changed DNSBL error level to debug (#1107). ### Fixed - Creating a mailbox in a shared folder results in wrong hierarchy (#1128). - IMAP LIST-STATUS (RFC 5819) returns items in wrong order (#1129). - Avoid non-RFC SMTP status codes (#1109). - Do not DNSBL check invalid domains (#1107). - Sieve message flag parser (#1059). - Sieve script import case insensitivity (#962). - `mailto:` parsing in HTMLs. ## [0.11.2] - 2025-01-17 To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin. ### Added - Automatic revoking of access tokens when secrets, permissions, ACLs or group memberships change (#649). - Increased concurrency for local message delivery (configurable via `queue.threads.local`). - Cluster node roles. - `config_get` expression function. ### Changed - `queue.outbound.concurrency` is now `queue.threads.remote`. - `lookup.default.hostname` is now `server.hostname`. - `lookup.default.domain` is now `report.domain`. ### Fixed - Distributed locking issues in non-Redis stores (#1066). - S3 incorrect backoff wait time after failures. - Panic parsing broken HTMLs. - Update CLI response serializer to v0.11.x (#1082). - Histogram bucket counts (#1079). - Do not rate limit trusted IPs (#1078). - Avoid double encrypting PGP parts encoded as plain text (#1083). - Return empty SASL challenge rather than "" (#1064). ## [0.11.0] - 2025-01-06 This version includes breaking changes to the configuration file, please read [UPGRADING.md](UPGRADING.md) for details. To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin. ### Added - Spam filter rewritten in Rust for a significant performance improvement. - Multiple spam filter improvements (#947) such as training spam/ham when moving between inbox and spam folders (#819). - Improved distributed locking and handling of large distributed SMTP queues. - ASN and GeoIP lookups. - Bulk operations REST endpoints (#925). - Faster S3-FIFO caching. - Support adding the `Delivered-To` header (#916). - Semver compatibility checks when upgrading (#844). - Sharded In-Memory Store. ### Changed - Removed authentication rate limit (no longer necessary since there is fail2ban). - Pipes have been deprecated in favor of MTA hooks. ### Fixed - OpenPGP EOF error (#1024). - Convert emails obtained from external directories to lowercase (#1004). - LDAP: Support both name and email fields to be mapped to the same attribute. - Admin role can't be assigned if an account with the same name exists. - Fix macro detection in DNS record generation (#978). - Use host FQDN in install script (#1003). ## [0.10.7] - 2024-12-04 To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin. ### Added - Delivery and DMARC Troubleshooting (#420). - Support for external email addresses on mailing lists (#152). - Azure blob storage support. ### Changed ### Fixed - Some mails can't be moved out of the junk folder (#670). - Out of bound index error on Sieve script (#941). - Missing `User-Agent` header for ACME (#937). - UTF8 support in IMAP4rev1 (#948). - Account alias owner leak on autodiscover. - Include all events in OTEL traces + Include spanId in webhooks. - Implement `todo!()` causing panic on concurrency and rate limits. - Mark SQL store as active if used as a telemetry store. - Discard empty form submissions. ## [0.10.6] - 2024-11-07 To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin. ### Added - Enterprise license automatic renewals before expiration (disabled by default). - Allow to LDAP search using bind dn instead of auth bind connection when bind auth is enabled (#873) ### Changed ### Fixed - Include `preferred_username` and `email` in OIDC `id_token`. - Verify roles and permissions when creating or modifying accounts (#874) ## [0.10.5] - 2024-10-15 To upgrade replace the `stalwart-mail` binary. ### Added - Data store CLI. ### Changed ### Fixed - Tokenizer performance issue (#863) - Incorrect AI model endpoint setting. ## [0.10.4] - 2024-10-08 To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin. ### Added - Detect and ban port scanners as well as other forms of abuse (#820). - ACME External Account Binding support (#379). ### Changed - The settings `server.fail2ban.*` have been moved to `server.auto-ban.*`. - The event `security.brute-force-ban` is now `security.abuse-ban`. ### Fixed - Do not send SPF failures reports to local domains. - Allow `nonce` in OAuth code requests. - Warn when there are errors migrating domains rather than aborting migration. ## [0.10.3] - 2024-10-07 To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin. Enterprise users wishing to use the new LLM-powered spam filter should also upgrade the spam filter rules. ### Added - AI-powered Spam filtering and Sieve scripting (Enterprise feature). ### Changed - The untrusted Sieve interpreter now has the `vnd.stalwart.expressions` extension enabled by default. This allows Sieve users to use the `eval` function to evaluate expressions in their scripts. If you would like to disable this extension, you can do so by adding `vnd.stalwart.expressions` to `sieve.untrusted.disabled-capabilities`. ### Fixed - S3-compatible backends: Retry on `5xx` errors. - OIDC: Include `nonce` parameter in `id_token` response. ## [0.10.2] - 2024-10-02 To upgrade first upgrade the webadmin and then replace the `stalwart-mail` binary. If you read these instructions too late, you can upgrade to the latest web-admin using `curl -k -u admin:yourpass https://yourserver/api/update/webadmin`. ### Added - OpenID Connect server (#298). - OpenID Connect backend support (Enterprise feature). - OpenID Connect Dynamic Client Registration (#4) - OAuth 2.0 Dynamic Client Registration Protocol ([RFC7591](https://datatracker.ietf.org/doc/html/rfc7591)) (#136) - OAuth 2.0 Token Introspection ([RFC7662](https://datatracker.ietf.org/doc/html/rfc7662)). - Contact form submission handling. - `webadmin.path` setting to override unpack directory (#792). ### Changed ### Fixed - Missing `LIST-STATUS` from RFC5819 in IMAP capability responses (#816). - Do not allow tenant domains to be deleted if they have members (#812). - Tenant principal limits (#810). ## [0.10.1] - 2024-09-26 To upgrade replace the `stalwart-mail` binary. ### Added - `OAUTHBEARER` SASL support in all services (#627). ### Changed ### Fixed - Fixed `migrate_directory` range scan (#784). ## [0.10.0] - 2024-09-21 This version includes breaking changes to how accounts are stored. Please read [UPGRADING.md](UPGRADING.md) for details. ### Added - Multi-tenancy (Enterprise feature). - Branding (Enterprise feature). - Roles and permissions. - Full-text search re-indexing. - Partial database backups (#497). ### Changed ### Fixed - IMAP `IDLE` support for command pipelining, aka the Apple Mail iOS 18 bug (#765). - Case insensitive INBOX `fileinto` (#763). - Properly decode undelete account name (#761). ## [0.9.4] - 2024-09-09 To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin. ### Added - Support for global Sieve scripts that can be used by users to filter their incoming mail. - Allow localhost to override HTTP access controls to prevent lockouts. ### Changed - Sieve runtime error default log level is now `debug`. ### Fixed - Ignore INBOX case on Sieve's `fileinto` (#725) - Local keys parsing and retrieval issues. - Lookup reload does not include database settings. - Account count is incorrect. ## [0.9.3] - 2024-08-29 To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin. ### Added - Dashboard (Enterprise feature) - Alerts (Enterprise feature) - SYN Flood (session "loitering") attack protection (#482) - Mailbox brute force protection (#688) - Mail from is allowed (`session.mail.is-allowed`) expression (#609) ### Changed - `authentication.fail2ban` setting renamed to `server.fail2ban.authentication`. - Added elapsed times to message filtering events. ### Fixed - Include queueId in MTA Hooks (#708) - Do not insert empty keywords in FTS index. ## [0.9.2] - 2024-08-21 To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin. ### Added - Message delivery history (Enterprise feature) - Live tracing and logging (Enterprise feature) - SQL Read Replicas (Enterprise feature) - Distributed S3 Blob Store (Enterprise feature) ### Changed ### Fixed - Autodiscover request parser issues. - Do not create tables when using SQL as an external directory (fixes #291) - Do not hardcode logger id (fixes #348) - Include `Forwarded-For IP` address in `http.request-url` event (fixes #682) ## [0.9.1] - 2024-08-08 To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin. ### Added - Metrics support (closes #478) - OpenTelemetry Push Exporter - Prometheus Pull Exporter (closes #275) - HTTP endpoint access controls (closes #266 #329 #542) - Add `options` setting to PostgreSQL driver (closes #662) - Add `isActive` property to defaults on Sieve/get JMAP method (closes #624) ### Changed - Perform `must-match-sender` checks after sender rewriting (closes #394) - Only perform email ingest duplicate check on the target mailbox (closes #632) ### Fixed - Properly parse `Forwarded` and `X-Forwarded-For` headers (fixes #669) - Resolve DKIM macros when generating DNS records (fixes #666) - Fixed `is_local_domain` Sieve function (fixes #622) ## [0.9.0] - 2024-08-01 To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin. This version includes breaking changes to the Webhooks configuration and produces a slightly different log output, read [UPGRADING.md](UPGRADING.md) for details. ### Added - Improved and faster tracing and logging. - Customizable event logging levels. ### Changed ### Fixed - ManageSieve: Return capabilities after successful `STARTTLS` - Do not provide `{auth_authen}` Milter macro unless the user is authenticated ## [0.8.5] - 2024-07-07 To upgrade replace the `stalwart-mail` binary. ### Added - Restore deleted e-mails (Enterprise Edition only) - Kubernetes (K8S) livenessProbe and readinessProbe endpoints. ### Changed - Avoid sending reports for DMARC/delivery reports (#173) ### Fixed - Refresh old FoundationDB read transactions (#520) - Subscribing shared mailboxes doesn't work (#251) ## [0.8.4] - 2024-07-03 To upgrade replace the `stalwart-mail` binary. ### Added ### Changed ### Fixed - Fix TOTP validation order. - Increase Jemalloc page size on armv7 builds. ## [0.8.3] - 2024-07-01 To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin. ### Added - Two-factor authentication with Time-based One-Time Passwords (#436) - Application passwords (#479). - Option to disable user accounts. ### Changed - DANE success on EndEntity match regardless of TrustAnchor validation. ### Fixed - Fix ManageSieve GETSCRIPT response: Add missing CRLF (#563) - Do not return CAPABILITIES after ManageSieve AUTH=PLAIN SASL exchange (#548) - POP3 QUIT must write a response (#568) ## [0.8.2] - 2024-06-22 To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin and spam filter versions. ### Added - Webhooks support (#480) - MTA Hooks (like milter but over HTTP) - Manually train and test spam classifier (#473 #264 #257 #471) - Allow configuring default mailbox names, roles and subscriptions (#125 #290 #458 #498) - Include `robots.txt` (#542) ### Changed - Milter support on all SMTP stages (#183) - Do not announce `STARTTLS` if the listener does not support it. ### Fixed - Incoming reports stored in the wrong subspace (#543) - Return `OK` after a successful ManageSieve SASL authentication flow (#187) - Case-insensitive search in settings API (#487) - Fix `session.rcpt.script` default variable name (#502) ## [0.8.1] - 2024-05-23 To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin and spam filter versions. ### Added - POP3 support. - DKIM signature length exploit protection. - Faster email deletion. - Junk/Trash folder auto-expunge and changelog auto-expiry (#403) - IP allowlists. - HTTP Strict Transport Security option. - Add TLS Reporting DNS entry (#464). ### Changed - Use separate account for master user. - Include server hostname in SMTP greetings (#448). ### Fixed - IP addresses trigger `R_SUSPICIOUS_URL` false positive (#461 #419). - JMAP identities should not return null signatures. - Include authentication headers and check queue quotas on Sieve message forwards. - ARC seal using just one signature. - Remove technical subdomains from MTA-STS policies and TLS records (#429). ## [0.8.0] - 2024-05-13 This version uses a different database layout which is incompatible with previous versions. Please read the [UPGRADING.md](UPGRADING.md) file for more information on how to upgrade from previous versions. ### Added - Clustering support with node auto-discovery and partition-tolerant failure detection. - Autoconfig and MS Autodiscover support (#336) - New variables `retry_num`, `notify_num`, `last_error` add `last_status` available in queue expressions. - Performance improvements, in particular for FoundationDB. - Improved full-text indexing with lower disk space usage. - MTA-STS policy management. - TLSA Records generation for DANE (#397) - Queued message visualization from the web-admin. - Master user support. ### Changed - Make `certificate.*` local keys by default. - Removed `server.run-as.*` settings. - Add Microsoft Office Macro types to bad mime types (#391) ### Fixed - mySQL TLS support (#415) - Resolve file macros after dropping root privileges. - Updated order of SPF Records (#395). - Avoid duplicate accountIds when using case insensitive external directories (#399) - `authenticated_as` variable not usable for must-match-sender (#372) - Remove `StandardOutput`, `StandardError` in service (#390) - SMTP `AUTH=LOGIN` compatibility issues with Microsoft Outlook (#400) ## [0.7.3] - 2024-05-01 To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin version. ### Added - Full database export and import functionality - Add --help and --version command line arguments (#365) - Allow catch-all addresses when validating must match sender ### Changed - Add `groupOfUniqueNames` to the list of LDAP object classes ### Fixed - Trim spaces in DNS-01 ACME secrets (#382) - Allow only one journald tracer (#375) - `authenticated_as` variable not usable for must-match-sender (#372) - Fixed `BOGUS_ENCRYPTED_AND_TEXT` spam filter rule - Fixed parsing of IPv6 DNS server addresses ## [0.7.2] - 2024-04-17 To upgrade replace the `stalwart-mail` binary and then upgrade to the latest web-admin version. ### Added - Support for `DNS-01` and `HTTP-01` ACME challenges (#226) - Configurable external resources (#355) ### Changed ### Fixed - Startup failure when Elasticsearch is down/starting up (#334) - URL decode path elements in REST API. ## [0.7.1] - 2024-04-12 To upgrade replace the `stalwart-mail` binary. ### Added - Make initial admin password configurable via env (#311) ### Changed - WebAdmin download URL. ### Fixed - Remove ASN.1 DER structure from DKIM ED25519 public keys. - Filter out invalid timestamps on log entries. ## [0.7.0] - 2024-04-09 This version uses a different database layout and introduces multiple breaking changes in the configuration files. Please read the [UPGRADING.md](UPGRADING.md) file for more information on how to upgrade from previous versions. ### Added - Web-based administration interface. - REST API for management and configuration. - Automatic RSA and ED25519 DKIM key generation. - Support for compressing binaries in the blob store (#227). - Improved performance accessing IMAP mailboxes with a large number of messages. - Support for custom DNS resolvers. - Support for multiple loggers with different levels and outputs. ### Changed ### Fixed - Store quotas as `u64` rather than `u32`. - Second IDLE connections disconnects the first one (#280). - Use relaxed DNS parsing, allowing underscores in DNS labels (#172). - Escape regexes within `matches()` expressions (#155). - ManageSieve LOGOUT should reply with `OK` instead of `BYE`. ## [0.6.0] - 2024-02-14 This version introduces breaking changes in the configuration file. Please read the [UPGRADING.md](UPGRADING.md) file for more information on how to upgrade from previous versions. ### Added - Distributed and fault-tolerant SMTP message queues. - Distributed rate-limiting and fail2ban. - Expressions in configuration files. ### Changed ### Fixed - Do not include `STATUS` in IMAP `NOOP` responses (#234). - Allow multiple SMTP `HELO` commands. - Redirect OAuth using a `301` instead of a `307` code. ## [0.5.3] - 2024-01-14 Please read the [UPGRADING.md](UPGRADING.md) file for more information on how to upgrade from previous versions. ### Added - Built-in [fail2ban](https://stalw.art/docs/server/fail2ban) and IP address/mask blocking (#164). - CLI: Read URL and credentials from environment variables (#88). - mySQL driver: Add `max-allowed-packet` setting (#201). ### Changed - Unified storage settings for all services (read the [UPGRADING.md](UPGRADING.md) for details) ### Fixed - IMAP retrieval of auto-encrypted emails (#203). - mySQL driver: Parse `timeout.wait` property as duration (#202). - `X-Forwarded-For` header on JMAP Rate-Limit does not work (#208). - Use timeouts in install script (#138). ## [0.5.2] - 2024-01-07 Please read the [UPGRADING.md](UPGRADING.md) file for more information on how to upgrade from previous versions. ### Added - [ACME](https://stalw.art/docs/server/tls/acme) support for automatic TLS certificate generation and renewal (#160). - TLS certificate [hot-reloading](https://stalw.art/docs/management/database/maintenance#tls-certificate-reloading). - [HAProxy protocol](https://stalw.art/docs/server/proxy) support (#36). ### Changed ### Fixed - IMAP command `SEARCH ` is using UIDs rather than sequence numbers. - IMAP responses to `APPEND` and `EXPUNGE` should include `HIGHESTMODSEQ` when `CONDSTORE` is enabled. ## [0.5.1] - 2024-01-02 ### Added - SMTP smuggling protection: Sanitization of outgoing messages that do not use `CRLF` as line endings. - SMTP sender validation for authenticated users: Added the `session.auth.must-match-sender` configuration option to enforce that the sender address used in the `MAIL FROM` command matches the authenticated user or any of their associated e-mail addresses. ### Changed ### Fixed - Invalid DKIM signatures for empty message bodies. - IMAP command `SEARCH BEFORE` is not properly parsed. - IMAP command `FETCH` fails to parse single arguments without parentheses. - IMAP command `ENABLE QRESYNC` should also enable `CONDSTORE` extension. - IMAP response to `ENABLE` command does not include enabled capabilities list. - IMAP response to `FETCH ENVELOPE` should not return `NIL` when the `From` header is missing. ## [0.5.0] - 2023-12-27 This version requires a database migration and introduces breaking changes in the configuration file. Please read the [UPGRADING.md](UPGRADING.md) file for more information. ### Added - Performance enhancements: - Messages are parsed only once and their offsets stored in the database, which avoids having to parse them on every `FETCH` request. - Background full-text indexing. - Optimization of database access functions. - Storage layer improvements: - In addition to `FoundationDB` and `SQLite`, now it is also possible to use `RocksDB`, `PostgreSQL` and `mySQL` as a storage backend. - Blobs can now be stored in any of the supported data stores, it is no longer limited to the file system or S3/MinIO. - Full-text searching con now be done internally or delegated to `ElasticSearch`. - Spam databases can now be stored in any of the supported data stores or `Redis`. It is no longer necessary to have an SQL server to use the spam filter. - Internal directory: - User account, groups and mailing lists can now be managed directly from Stalwart without the need of an external LDAP or SQL directory. - HTTP API to manage users, groups, domains and mailing lists. - IMAP4rev1 `Recent` flag support, which improves compatibility with old IMAP clients. - LDAP bind authentication, to support some LDAP servers such as `lldap` which do not expose the userPassword attribute. - Messages marked a spam by the spam filter can now be automatically moved to the account's `Junk Mail` folder. - Automatic creation of JMAP identities. ### Changed ### Fixed - Spamhaus DNSBL return codes. - CLI tool reports authentication errors rather than a parsing error. ## [0.4.2] - 2023-11-01 ### Added - JMAP for Quotas support ([RFC9425](https://www.rfc-editor.org/rfc/rfc9425.html)) - JMAP Blob Management Extension support ([RFC9404](https://www.rfc-editor.org/rfc/rfc9404.html)) - Spam Filter - Empty header rules. ### Changed ### Fixed - Daylight savings time support for crontabs. - JMAP `oldState` doesn’t reflect in `*/changes` (#56) ## [0.4.1] - 2023-10-26 ### Added ### Changed ### Fixed - Dockerfile entrypoint script. - `bayes_is_balanced` function. ## [0.4.0] - 2023-10-25 This version introduces some breaking changes in the configuration file. Please read the [UPGRADING.md](UPGRADING.md) file for more information. ### Added - Built-in Spam and Phishing filter. - Scheduled queries on some directory types. - In-memory maps and lists containing glob or regex patterns. - Remote retrieval of in-memory list/maps with fallback mechanisms. - Macros and support for including files from TOML config files. ### Changed - `config.toml` is now split in multiple TOML files for better organization. - **BREAKING:** Configuration key prefix `jmap.sieve` (JMAP Sieve Interpreter) has been renamed to `sieve.untrusted`. - **BREAKING:** Configuration key prefix `sieve` (SMTP Sieve Interpreter) has been renamed to `sieve.trusted`. ### Fixed ## [0.3.10] - 2023-10-17 ### Added - Option to allow invalid certificates on outbound SMTP connections. - Option to disable ansi colors on `stdout`. ### Changed - SMTP reject messages are now logged as `info` rather than `debug`. ### Fixed ## [0.3.9] - 2023-10-07 ### Added - Support for reading environment variables from the configuration file using the `!ENV_VAR_NAME` special keyword. - Option to disable ANSI color codes in logs. ### Changed - Querying directories from a Sieve script is now done using the `query()` method from `eval`. Your scripts will need to be updated, please refer to the [new syntax](https://stalw.art/docs/smtp/filter/sieve#directory-queries). ### Fixed - IPrev lookups of IPv4 mapped to IPv6 addresses. ## [0.3.8] - 2023-09-19 ### Added - Journal logging support - IMAP support for UTF8 APPEND ### Changed - Replaced `rpgp` with `sequoia-pgp` due to rpgp bug. ### Fixed - Fix: IMAP folders that contain a & can't be used (#90) - Fix: Ignore empty lines in IMAP requests ## [0.3.7] - 2023-09-05 ### Added - Option to disable IMAP All Messages folder (#68). - Option to allow unencrypted SMTP AUTH (#72) - Support for `rcpt-domain` key in `rcpt.relay` SMTP rule evaluation. ### Changed ### Fixed - SMTP strategy `Ipv6thenIpv4` returns only IPv6 addresses (#70) - Invalid IMAP `FETCH` responses for non-UTF-8 messages (#70) - Allow `STATUS` and `ACL` IMAP operations on virtual mailboxes. - IMAP `SELECT QRESYNC` without specifying a UID causes panic (#67) - Milter `DATA` command is sent after headers which causes ClamAV to hang. - Sieve `redirect` of unmodified messages does not work. ## [0.3.6] - 2023-08-29 ### Added - Arithmetic and logical expression evaluation in Sieve scripts. - Support for storing query results in Sieve variables. - Results of SPF, DKIM, ARC, DMARC and IPREV checks available as environment variables in Sieve scripts. - Configurable protocol flags for Milter filters. - Fall-back to plain text when `STARTTLS` fails and `starttls` is set to `optional`. ### Changed ### Fixed - Do not panic when `hash = 0` in reports. (#60) - JMAP Session resource returns `EmailSubmission` capabilities using arrays rather than objects. - ManageSieve `PUTSCRIPT` should replace existing scripts. ## [0.3.5] - 2023-08-18 ### Added - TCP listener option `nodelay`. ### Changed ### Fixed - SMTP: Allow disabling `STARTTLS`. - JMAP: Support for `OPTIONS` HTTP method. ## [0.3.4] - 2023-08-09 ### Added - JMAP: Support for setting custom HTTP response headers (#52) ### Changed ### Fixed - SMTP: Missing envelope keys in rewrite rules (#25) - SMTP: Remove CRLF from Milter headers - JMAP/IMAP: Successful authentication requests should not count when rate limiting - IMAP: Case insensitive Inbox selection - IMAP: Automatically create Inbox for group accounts ## [0.3.3] - 2023-08-02 ### Added - Encryption at rest with **S/MIME** or **OpenPGP**. - Support for referencing context variables from dynamic values. ### Changed ### Fixed - Support for PKCS8v1 ED25519 keys (#20). - Automatic retry for import/export blob downloads (#14) ## [0.3.2] - 2023-07-28 ### Added - Sender and recipient address rewriting using regular expressions and sieve scripts. - Subaddressing and catch-all addresses using regular expressions (#10). - Dynamic variables in SMTP rules. ### Changed - Added CLI to Docker container (#19). ### Fixed - Workaround for a bug in `sqlx` that caused SQL time-outs (#15). - Support for ED25519 certificates in PEM files (#20). - Better handling of concurrent IMAP UID map modifications (#17). - LDAP domain lookups from SMTP rules. ## [0.3.1] - 2023-07-22 ### Added - Milter filter support. - Match IP address type using /0 mask (#16). ### Changed ### Fixed - Support for OpenLDAP password hashing schemes between curly brackets (#8). - Add CA certificates to Docker runtime (#5). ## [0.3.0] - 2023-07-16 ### Added - **LDAP** and **SQL** authentication. - **subaddressing** and **catch-all** addresses. - **S3-compatible** storage. ### Changed - Merged the `stalwart-jmap`, `stalwart-imap` and `stalwart-smtp` repositories into `stalwart-mail`. - Removed clustering module and replaced it with a **FoundationDB** backend option. - Integrated Stalwart SMTP into Stalwart JMAP. - Rewritten JMAP protocol parser. - Rewritten store backend. - Rewritten IMAP server to have direct access to the message store (no more IMAP proxy). - Replaced `actix` with `hyper`. ### Fixed ================================================ FILE: CNAME ================================================ get.stalw.art ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing ## Contributions are Temporarily Limited Thank you for your interest in contributing to Stalwart. We appreciate the support and enthusiasm of the open-source community. However, at this stage of the project, we are **limiting the scope of external contributions**. Stalwart is currently **not accepting external contributions**, except for bug fixes and small, well-scoped changes. The project is approaching version 1.0, and as we move toward this milestone, development is progressing rapidly. The architecture of Stalwart is still evolving, and many internal components are subject to change. Due to these ongoing changes and the fast pace of development, we do not have the time or resources to thoroughly review and integrate most pull requests. Accepting broad contributions at this time could lead to confusion and unnecessary rework for both contributors and maintainers. While we are not accepting most code contributions, you can still support the project in meaningful ways. Reporting bugs, providing feedback, and helping test the software are all valuable forms of participation. If you encounter an issue, please open a detailed report that includes steps to reproduce the problem and any relevant logs or context. We also welcome thoughtful suggestions and questions through our issue tracker or discussion channels. We plan to open the project to broader contributions once we reach a stable 1.0 release. At that point, with a more mature architecture and clearer development roadmap, we will be better positioned to collaborate with the community. We will update this policy accordingly when the time comes. Thank you for your understanding and continued support. We’re excited about the future of Stalwart and look forward to working with the community in the near future. ## Code of Conduct Please note we have a code of conduct, please follow it in all your interactions with the project. ## Licensing This project is licensed under the Affero General Public License (AGPL) version 3.0. By contributing to this project, you agree that your contributions will be licensed under the AGPL-3.0 license. ## Fiduciary Contributor License Agreement Before making any contributions, all contributors are required to sign the Fiduciary Contributor License Agreement (FLA). The FLA is a legal agreement that assigns the copyright of contributions to a designated fiduciary, who manages these rights on behalf of the project. This arrangement ensures that the software remains free and open, even as contributors come and go. Key points of the FLA: - Ensures the software remains free and open source - Protects the project from potential copyright issues - Includes a reversion clause: if the fiduciary violates Free Software principles, rights revert to the original contributors For more details about FLA, please refer to the [FLA FAQ](https://fsfe.org/activities/fla/fla.en.html). ## Pull Request Process 1. Ensure any install or build dependencies are removed before the end of the layer when doing a build. 2. Update the README.md with details of changes to the interface, this includes new environment variables, exposed ports, useful file locations and container parameters. 3. Increase the version numbers in any examples files and the README.md to the new version that this Pull Request would represent. The versioning scheme we use is [SemVer](http://semver.org/). 4. You may merge the Pull Request in once you have the sign-off of two other developers, or if you do not have permission to do that, you may request the second reviewer to merge it for you. ## Code of Conduct We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. You can read the full Code of Conduct [here](https://github.com/stalwartlabs/.github/blob/main/CODE_OF_CONDUCT.md). ================================================ FILE: Cargo.toml ================================================ [workspace] resolver = "2" members = [ "crates/main", "crates/types", "crates/http", "crates/http-proto", "crates/jmap", "crates/jmap-proto", "crates/email", "crates/imap", "crates/imap-proto", "crates/smtp", "crates/managesieve", "crates/pop3", "crates/dav-proto", "crates/dav", "crates/groupware", "crates/spam-filter", "crates/nlp", "crates/store", "crates/directory", "crates/services", "crates/utils", "crates/common", "crates/trc", "crates/migration", "crates/cli", "tests", ] [profile.dev] opt-level = 0 debug = 1 #codegen-units = 4 lto = false incremental = true panic = 'unwind' debug-assertions = true overflow-checks = false rpath = false [profile.release] opt-level = 3 debug = false codegen-units = 1 lto = true incremental = false panic = 'unwind' debug-assertions = false overflow-checks = false rpath = false strip = true [profile.test] opt-level = 0 debug = 1 #codegen-units = 16 lto = false incremental = true debug-assertions = true overflow-checks = true rpath = false [profile.bench] opt-level = 3 debug = false codegen-units = 1 lto = true incremental = false debug-assertions = false overflow-checks = false rpath = false ================================================ FILE: Dockerfile ================================================ # Stalwart Dockerfile # Credits: https://github.com/33KK FROM --platform=$BUILDPLATFORM docker.io/lukemathwalker/cargo-chef:latest-rust-slim-trixie AS chef WORKDIR /build FROM --platform=$BUILDPLATFORM chef AS planner COPY . . RUN cargo chef prepare --recipe-path /recipe.json FROM --platform=$BUILDPLATFORM chef AS builder ARG TARGETPLATFORM RUN case "${TARGETPLATFORM}" in \ "linux/arm64") echo "aarch64-unknown-linux-gnu" > /target.txt && echo "-C linker=aarch64-linux-gnu-gcc" > /flags.txt ;; \ "linux/amd64") echo "x86_64-unknown-linux-gnu" > /target.txt && echo "-C linker=x86_64-linux-gnu-gcc" > /flags.txt ;; \ *) exit 1 ;; \ esac RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get install -yq --no-install-recommends build-essential libclang-19-dev \ g++-aarch64-linux-gnu binutils-aarch64-linux-gnu \ g++-x86-64-linux-gnu binutils-x86-64-linux-gnu RUN rustup target add "$(cat /target.txt)" COPY --from=planner /recipe.json /recipe.json RUN RUSTFLAGS="$(cat /flags.txt)" cargo chef cook --target "$(cat /target.txt)" --release --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats enterprise" --recipe-path /recipe.json COPY . . RUN RUSTFLAGS="$(cat /flags.txt)" cargo build --target "$(cat /target.txt)" --release -p stalwart --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats enterprise" RUN RUSTFLAGS="$(cat /flags.txt)" cargo build --target "$(cat /target.txt)" --release -p stalwart-cli RUN mv "/build/target/$(cat /target.txt)/release" "/output" FROM docker.io/debian:trixie-slim WORKDIR /opt/stalwart RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get install -yq --no-install-recommends ca-certificates COPY --from=builder /output/stalwart /usr/local/bin COPY --from=builder /output/stalwart-cli /usr/local/bin COPY ./resources/docker/entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod -R 755 /usr/local/bin CMD ["/usr/local/bin/stalwart"] VOLUME [ "/opt/stalwart" ] EXPOSE 443 25 110 587 465 143 993 995 4190 8080 ENTRYPOINT ["/bin/sh", "/usr/local/bin/entrypoint.sh"] ================================================ FILE: Dockerfile.build ================================================ # syntax=docker/dockerfile:1 # check=skip=FromPlatformFlagConstDisallowed,RedundantTargetPlatform # ***************** # Base image for planner & builder # ***************** FROM --platform=$BUILDPLATFORM rust:slim-trixie AS base ENV DEBIAN_FRONTEND="noninteractive" \ BINSTALL_DISABLE_TELEMETRY=true \ CARGO_TERM_COLOR=always \ LANG=C.UTF-8 \ TZ=UTC \ TERM=xterm-256color # With zig, we only need libclang and make RUN \ --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ rm -f /etc/apt/apt.conf.d/docker-clean && \ echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' >/etc/apt/apt.conf.d/keep-cache && \ apt-get update && \ apt-get install -yq --no-install-recommends curl jq xz-utils make libclang-19-dev # Install zig RUN \ ZIG_VERSION=0.13.0 && \ [ ! -z "$ZIG_VERSION" ] && \ curl --retry 5 -Ls "https://ziglang.org/download/${ZIG_VERSION}/zig-linux-$(uname -m)-${ZIG_VERSION}.tar.xz" | tar -J -x -C /usr/local && \ ln -s "/usr/local/zig-linux-$(uname -m)-${ZIG_VERSION}/zig" /usr/local/bin/zig # Install cargo-binstall RUN curl --retry 5 -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash # Install cargo-chef & sccache & cargo-zigbuild RUN cargo binstall --no-confirm cargo-chef sccache cargo-zigbuild # ***************** # Planner # ***************** FROM base AS planner WORKDIR /app COPY . . # Generate recipe file RUN cargo chef prepare --recipe-path recipe.json # ***************** # Builder # ***************** FROM base AS builder WORKDIR /app COPY --from=planner /app/recipe.json recipe.json ARG TARGET ARG BUILD_ENV SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Install toolchain and specify some env variables RUN \ rustup set profile minimal && \ rustup target add ${TARGET} && \ mkdir -p artifact && \ touch /env-cargo && \ if [ ! -z "${BUILD_ENV}" ]; then \ echo "export ${BUILD_ENV}" >> /env-cargo; \ echo "Setting up ${BUILD_ENV}"; \ fi && \ if [[ "${TARGET}" == *gnu ]]; then \ base_arch="${TARGET%%-*}"; \ case "$base_arch" in \ x86_64) \ echo "export FDB_ARCH=amd64" >> /env-cargo; \ ;; \ aarch64) \ echo "export FDB_ARCH=aarch64" >> /env-cargo; \ ;; \ *) \ exit 1; \ ;; \ esac; \ fi # Install FoundationDB RUN \ source /env-cargo && \ if [ ! -z "${FDB_ARCH}" ]; then \ curl --retry 5 -Lso fdb-client.deb "$(curl --retry 5 -Ls 'https://api.github.com/repos/apple/foundationdb/releases' | jq --arg FDB_ARCH "$FDB_ARCH" -r '.[] | select(.prerelease == false) | .assets[] | select(.name | test("foundationdb-clients.*" + $FDB_ARCH + ".deb$")) | .browser_download_url' | head -n1)" && \ mkdir -p /fdb && \ dpkg -x fdb-client.deb /fdb && \ mv /fdb/usr/include/foundationdb /usr/include && \ mv /fdb/usr/lib/libfdb_c.so /usr/lib && \ rm -rf fdb-client.deb /fdb; \ fi # Cargo-chef Cache layer RUN \ --mount=type=secret,id=ACTIONS_RESULTS_URL,env=ACTIONS_RESULTS_URL \ --mount=type=secret,id=ACTIONS_RUNTIME_TOKEN,env=ACTIONS_RUNTIME_TOKEN \ --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git \ source /env-cargo && \ if [ ! -z "${FDB_ARCH}" ]; then \ RUSTFLAGS="-L /usr/lib" cargo chef cook --recipe-path recipe.json --zigbuild --release --target ${TARGET} -p stalwart --no-default-features --features "foundationdb s3 redis nats enterprise"; \ fi RUN \ --mount=type=secret,id=ACTIONS_RESULTS_URL,env=ACTIONS_RESULTS_URL \ --mount=type=secret,id=ACTIONS_RUNTIME_TOKEN,env=ACTIONS_RUNTIME_TOKEN \ --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git \ source /env-cargo && \ cargo chef cook --recipe-path recipe.json --zigbuild --release --target ${TARGET} -p stalwart --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats enterprise" && \ cargo chef cook --recipe-path recipe.json --zigbuild --release --target ${TARGET} -p stalwart-cli # Copy the source code COPY . . ENV RUSTC_WRAPPER="sccache" \ SCCACHE_GHA_ENABLED=true # Build FoundationDB version RUN \ --mount=type=secret,id=ACTIONS_RESULTS_URL,env=ACTIONS_RESULTS_URL \ --mount=type=secret,id=ACTIONS_RUNTIME_TOKEN,env=ACTIONS_RUNTIME_TOKEN \ --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git \ source /env-cargo && \ if [ ! -z "${FDB_ARCH}" ]; then \ RUSTFLAGS="-L /usr/lib" cargo zigbuild --release --target ${TARGET} -p stalwart --no-default-features --features "foundationdb s3 redis nats enterprise" && \ mv /app/target/${TARGET}/release/stalwart /app/artifact/stalwart-foundationdb; \ fi # Build generic version RUN \ --mount=type=secret,id=ACTIONS_RESULTS_URL,env=ACTIONS_RESULTS_URL \ --mount=type=secret,id=ACTIONS_RUNTIME_TOKEN,env=ACTIONS_RUNTIME_TOKEN \ --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git \ source /env-cargo && \ cargo zigbuild --release --target ${TARGET} -p stalwart --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats enterprise" && \ cargo zigbuild --release --target ${TARGET} -p stalwart-cli && \ mv /app/target/${TARGET}/release/stalwart /app/artifact/stalwart && \ mv /app/target/${TARGET}/release/stalwart-cli /app/artifact/stalwart-cli # ***************** # Binary stage # ***************** FROM scratch AS binaries COPY --from=builder /app/artifact / # ***************** # Runtime image for GNU targets # ***************** FROM --platform=$TARGETPLATFORM docker.io/library/debian:trixie-slim AS gnu WORKDIR /opt/stalwart RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get install -yq --no-install-recommends ca-certificates tzdata COPY --from=builder /app/artifact/stalwart /usr/local/bin COPY --from=builder /app/artifact/stalwart-cli /usr/local/bin COPY ./resources/docker/entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod -R 755 /usr/local/bin CMD ["/usr/local/bin/stalwart"] VOLUME [ "/opt/stalwart" ] EXPOSE 443 25 110 587 465 143 993 995 4190 8080 ENTRYPOINT ["/bin/sh", "/usr/local/bin/entrypoint.sh"] # ***************** # Runtime image for musl targets # ***************** FROM --platform=$TARGETPLATFORM alpine AS musl WORKDIR /opt/stalwart RUN apk add --update --no-cache ca-certificates tzdata && rm -rf /var/cache/apk/* COPY --from=builder /app/artifact/stalwart /usr/local/bin COPY --from=builder /app/artifact/stalwart-cli /usr/local/bin COPY ./resources/docker/entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod -R 755 /usr/local/bin CMD ["/usr/local/bin/stalwart"] VOLUME [ "/opt/stalwart" ] EXPOSE 443 25 110 587 465 143 993 995 4190 8080 ENTRYPOINT ["/bin/sh", "/usr/local/bin/entrypoint.sh"] ================================================ FILE: LICENSES/AGPL-3.0-only.txt ================================================ GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU Affero General Public License is a free, copyleft license for software and other kinds of works, specifically designed to ensure cooperation with the community in the case of network server software. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, our General Public Licenses are intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. Developers that use our General Public Licenses protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License which gives you legal permission to copy, distribute and/or modify the software. A secondary benefit of defending all users' freedom is that improvements made in alternate versions of the program, if they receive widespread use, become available for other developers to incorporate. Many developers of free software are heartened and encouraged by the resulting cooperation. However, in the case of software used on network servers, this result may fail to come about. The GNU General Public License permits making a modified version and letting the public access it on a server without ever releasing its source code to the public. The GNU Affero General Public License is designed specifically to ensure that, in such cases, the modified source code becomes available to the community. It requires the operator of a network server to provide the source code of the modified version running there to the users of that server. Therefore, public use of a modified version, on a publicly accessible server, gives the public access to the source code of the modified version. An older license, called the Affero General Public License and published by Affero, was designed to accomplish similar goals. This is a different license, not a version of the Affero GPL, but Affero has released a new version of the Affero GPL which permits relicensing under this license. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU Affero General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Remote Network Interaction; Use with the GNU General Public License. Notwithstanding any other provision of this License, if you modify the Program, your modified version must prominently offer all users interacting with it remotely through a computer network (if your version supports such interaction) an opportunity to receive the Corresponding Source of your version by providing access to the Corresponding Source from a network server at no charge, through some standard or customary means of facilitating copying of software. This Corresponding Source shall include the Corresponding Source for any work covered by version 3 of the GNU General Public License that is incorporated pursuant to the following paragraph. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the work with which it is combined will remain governed by version 3 of the GNU General Public License. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU Affero General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU Affero General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU Affero General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU Affero General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. If your software can interact with users remotely through a computer network, you should also make sure that it provides a way for users to get its source. For example, if your program is a web application, its interface could display a "Source" link that leads users to an archive of the code. There are many ways you could offer source, and different solutions will be better for different programs; see section 13 for the specific requirements. You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see . ================================================ FILE: LICENSES/LicenseRef-SEL.txt ================================================ Stalwart Enterprise License 1.0 (SELv1) Agreement ================================================= Last Update: April 29, 2025 PLEASE CAREFULLY READ THIS STALWART ENTERPRISE LICENSE AGREEMENT ("AGREEMENT"). THIS AGREEMENT CONSTITUTES A LEGALLY BINDING AGREEMENT BETWEEN YOU AND Stalwart Labs LLC AND GOVERNS YOUR USE OF THE SOFTWARE (DEFINED BELOW). IF YOU DO NOT AGREE WITH THIS AGREEMENT, YOU MAY NOT USE THE SOFTWARE. IF YOU ARE USING THE SOFTWARE ON BEHALF OF A LEGAL ENTITY, YOU REPRESENT AND WARRANT THAT YOU HAVE AUTHORITY TO AGREE TO THIS AGREEMENT ON BEHALF OF SUCH ENTITY. IF YOU DO NOT HAVE SUCH AUTHORITY, DO NOT USE THE SOFTWARE IN ANY MANNER. This Agreement is entered into by and between Stalwart Labs LLC and you, or the legal entity on behalf of whom you are acting. 1. DEFINITIONS 1.1. "Software" refers to the Stalwart Server Enterprise Edition software, including all its versions, updates, modifications, accompanying documentation, and related materials. 1.2. "Subscription" refers to the paid access to the Software provided by Licensor to Licensee. 1.3. "Licensor" refers to Stalwart Labs LLC, the entity providing the Software. 1.4. "Licensee" refers to the individual or entity installing, accessing, or using the Software with a valid Subscription. 1.5. "License Key" refers to the unique code provided by Licensor upon purchasing a Subscription which activates the full features of the Software. 1.6. "Source Code" refers to the human-readable version of the Software's code, as opposed to the compiled machine-readable version. 2. GRANT OF LICENSE 2.1. Licensor grants Licensee a revocable, non-exclusive, non-transferable, non-sublicensable, limited license to download, install, and use the Software. 2.2. The use of the Software is conditioned upon Licensee maintaining an active and valid paid subscription with Licensor. The paid subscription covers all versions of the Software and all updates and modifications. 2.3. This license grants Licensee the right to use the Software for both personal and commercial purposes. However, Licensee is expressly prohibited from reselling, leasing, sublicensing, or otherwise redistributing the Software itself. 2.4. This license is further governed by the terms and conditions set forth in any licensing agreements separately executed between Licensor and Licensee. In the event of any conflict between the terms of this Agreement and the terms of a signed licensing agreement, the terms of the signed licensing agreement shall control. 2.5. You are not granted any other rights beyond what is expressly stated herein. 3. LICENSE KEYS 3.1. The Software shall not be used without a valid License Key issued by Licensor. 3.2. Licensee is required to use valid License Keys issued by Licensor to run the Software, including any modified versions. Any attempts to bypass the License Key requirement is a violation of this Agreement. 3.3. Distribution or sharing of License Keys to third parties, not associated with Licensee, is strictly prohibited. 3.4. License Keys are bound to the subscription period. Should your subscription expire, all License Keys will become invalid after 15 days from the subscription expiration date. 3.5. Any instance of the Software using such an expired key will revert to the Community Edition functionality after the aforementioned 15-day period. 4. SOURCE CODE USAGE 4.1. Licensee is permitted to view, copy, and modify the Software's Source Code, as made available by Licensor, solely for Licensee's internal business use and in compliance with this Agreement's terms. 4.2. Any modifications to the Source Code do not grant Licensee any ownership rights to the original Software or any modifications. All rights, title, and interest to the Software and its Source Code remain exclusively with Licensor. 4.3. Licensee is strictly prohibited from altering, removing, or in any way tampering with the License Key validation system within the Software. Any such unauthorized modifications will be considered a material breach of this Agreement and may result in legal action. 4.3. Notwithstanding the availability of the Software's Source Code for review and limited modification, the Software and its Source Code are not open source and remain proprietary to Licensor. The provision of access to the Source Code does not confer any rights typically associated with open source software, including but not limited to the right to freely sublicense, or create derivative works for public distribution. All rights not expressly granted herein are reserved by Licensor. 4.4. Notwithstanding the foregoing, you may copy the Source Code for development and testing purposes, without requiring a Subscription. 5. INTELLECTUAL PROPERTY RIGHTS 5.1. The Licensor retains all rights, title, and interest in and to the Software, including all intellectual property rights therein. This Agreement does not transfer any ownership rights to the Licensee. 5.2. The Licensee must not remove, alter, or obscure any proprietary notices (including copyright and trademark notices) on the Software. 6. TERMINATION 6.1. Licensor reserves the right to terminate this Agreement immediately if the Licensee fails to comply with any terms and conditions of this Agreement. 6.2. In the event of a termination, you will be provided with a written notice, sent to the email address used during your subscription to the Software, outlining the reasons for the termination. 6.3. Upon termination, all rights granted to you under this Agreement will cease, and you must promptly cease all use of the Software. 7. LIMITATION OF LIABILITY 7.1. In no event will the Licensor be liable for any indirect, incidental, special, consequential, or punitive damages, or any loss of profits or revenues, whether incurred directly or indirectly, or any loss of data, use, goodwill, or other intangible losses, resulting from (i) your use or inability to use the Software; (ii) any unauthorized access to or use of our servers and/or any personal information stored therein. 7.2. Except for liability arising from death or personal injury caused by negligence, fraud, or willful misconduct, Licensor's total aggregate liability for any and all claims under this Agreement shall be limited to the total Subscription fees paid by Licensee to Licensor in the twelve (12) months immediately preceding the event giving rise to the claim. 8. GOVERNING LAW & JURISDICTION This Agreement shall be governed by and construed under the laws of the United Kingdom. Any disputes arising from or related to this Agreement shall be resolved in the jurisdiction of London, UK. 9. DATA PROTECTION & PRIVACY By using the Software, you consent to the collection, processing, and use of any personal data as required for the functionality of the Software. The specifics of data handling and storage will be outlined in the company's Privacy Policy, which can be accessed on the company's website. 10. ACCEPTANCE By downloading, installing, or using the Software software, even without explicitly clicking on an "I Agree" button or a similar mechanism, you acknowledge that you have read, understood, and agreed to be bound by the terms and conditions of this Agreement. 11. ASSIGNMENT This Agreement and the rights granted hereunder may not be transferred or assigned by you but may be assigned by Licensor without restriction. 12. SEVERABILITY If any provision of this Agreement is held to be unenforceable or invalid for any reason, that provision shall be reformed to the extent necessary to make it enforceable and consistent with the intent of the parties, and the remaining provisions shall remain in full force and effect. 13. ENTIRE AGREEMENT This Agreement constitutes the entire agreement between the Licensor and the Licensee with respect to the subject matter hereof and supersedes all prior or contemporaneous understandings regarding such subject matter. No amendment to or modification of this Agreement will be binding unless in writing and signed by the Licensor. 14. DISCLAIMERS AND WARRANTIES The Software is provided "AS IS" and "AS AVAILABLE", without warranty of any kind, either express or implied, including, without limitation, warranties of merchantability, fitness for a particular purpose, and non-infringement. Licensor does not warrant that the Software will be error-free, that access thereto will be uninterrupted, or that defects will be corrected. 15. INDEMNIFICATION Licensee agrees to indemnify, defend, and hold harmless Licensor, its officers, directors, employees, agents, licensors, suppliers, and any third-party information providers from and against all claims, losses, expenses, damages, and costs, including reasonable attorneys' fees, resulting from any violation of this Agreement or any activity related to your use or misuse of the Software (including negligent or wrongful conduct). 16. FORCE MAJEURE Neither party shall be in default or otherwise liable for any delay in or failure of its performance under this Agreement if such delay or failure arises by any reason of any event beyond the reasonable control of a party, including acts of God, the elements, earthquakes, floods, fires, epidemics, riots, failures or delays in transportation or communications, or any act or failure to act by the other party or such other party’s officers, employees, agents, or contractors. The parties will promptly inform and consult with each other as to any of the above causes which, in their judgment, may or could be the cause of a delay in the performance of this Agreement. 17. CONTACT INFORMATION If you have any questions about this Agreement, please contact Stalwart Labs LLC at: Stalwart Labs LLC 1309 Coffeen Avenue STE 1200 Sheridan, Wyoming 82801 USA hello@stalw.art ================================================ FILE: README.md ================================================

Secure, scalable mail & collaboration server with comprehensive protocol support 🛡️
(IMAP, JMAP, SMTP, CalDAV, CardDAV, WebDAV)


continuous integration   License: AGPL v3   Documentation

Mastodon   Twitter

Discord   Reddit

## Features **Stalwart** is an open-source mail & collaboration server with JMAP, IMAP4, POP3, SMTP, CalDAV, CardDAV and WebDAV support and a wide range of modern features. It is written in Rust and designed to be secure, fast, robust and scalable. Key features: - **Email** server with complete protocol support: - JMAP: * [JMAP for Mail](https://datatracker.ietf.org/doc/html/rfc8621) server. * [JMAP for Sieve Scripts](https://www.ietf.org/archive/id/draft-ietf-jmap-sieve-22.html). * [WebSocket](https://datatracker.ietf.org/doc/html/rfc8887), [Blob Management](https://www.rfc-editor.org/rfc/rfc9404.html) and [Quotas](https://www.rfc-editor.org/rfc/rfc9425.html) extensions. - IMAP: * [IMAP4rev2](https://datatracker.ietf.org/doc/html/rfc9051) and [IMAP4rev1](https://datatracker.ietf.org/doc/html/rfc3501) server. * [ManageSieve](https://datatracker.ietf.org/doc/html/rfc5804) server. * Numerous [extensions](https://stalw.art/docs/development/rfcs#imap4-and-extensions) supported. - POP3: - [POP3](https://datatracker.ietf.org/doc/html/rfc1939) server. - [STLS](https://datatracker.ietf.org/doc/html/rfc2595) and [SASL](https://datatracker.ietf.org/doc/html/rfc5034) support as well as other [extensions](https://datatracker.ietf.org/doc/html/rfc2449). - SMTP: * SMTP server with built-in [DMARC](https://datatracker.ietf.org/doc/html/rfc7489), [DKIM](https://datatracker.ietf.org/doc/html/rfc6376), [SPF](https://datatracker.ietf.org/doc/html/rfc7208) and [ARC](https://datatracker.ietf.org/doc/html/rfc8617) support for message authentication. * Strong transport security through [DANE](https://datatracker.ietf.org/doc/html/rfc6698), [MTA-STS](https://datatracker.ietf.org/doc/html/rfc8461) and [SMTP TLS](https://datatracker.ietf.org/doc/html/rfc8460) reporting. * Inbound throttling and filtering with granular configuration rules, sieve scripting, MTA hooks and milter integration. * Distributed virtual queues with delayed delivery, priority delivery, quotas, routing rules and throttling support. * Envelope rewriting and message modification. - **Collaboration** server: - Calendaring and scheduling: - [CalDAV](https://datatracker.ietf.org/doc/html/rfc4791) and [CalDAV Scheduling](https://datatracker.ietf.org/doc/html/rfc6638) support. - [JMAP for Calendars](https://datatracker.ietf.org/doc/html/draft-ietf-jmap-calendars-24) support. - Contact management: - [CardDAV](https://datatracker.ietf.org/doc/html/rfc6352) support. - [JMAP for Contacts](https://datatracker.ietf.org/doc/html/rfc9610) support. - File storage: - [WebDAV](https://datatracker.ietf.org/doc/html/rfc4918) support. - [JMAP for File Storage](https://datatracker.ietf.org/doc/html/draft-ietf-jmap-filenode-03) support. - Sharing with fine-grained access controls: - [WebDAV ACL](https://datatracker.ietf.org/doc/html/rfc3744) support. - [JMAP Sharing](https://datatracker.ietf.org/doc/html/rfc9670) support. - **Spam** and **Phishing** built-in filter: - Comprehensive set of filtering **rules** on par with popular solutions. - LLM-driven spam filtering and message analysis. - Statistical **spam classifier** with collaborative filtering, automatic training capabilities and address book integration. - DNS Blocklists (**DNSBLs**) checking of IP addresses, domains, and hashes. - Collaborative digest-based spam filtering with **Pyzor**. - **Phishing** protection against homographic URL attacks, sender spoofing and other techniques. - Trusted **reply** tracking to recognize and prioritize genuine e-mail replies. - Sender **reputation** monitoring by IP address, ASN, domain and email address. - **Greylisting** to temporarily defer unknown senders. - **Spam traps** to set up decoy email addresses that catch and analyze spam. - **Flexible**: - Pluggable storage backends with **RocksDB**, **FoundationDB**, **PostgreSQL**, **mySQL**, **SQLite**, **S3-Compatible**, **Azure** and **Redis** support. - Full-text search available in 17 languages using the built-in search engine or via **Meilisearch**, **ElasticSearch**, **OpenSearch**, **PostgreSQL** or **mySQL** backends. - Sieve scripting language with support for all [registered extensions](https://www.iana.org/assignments/sieve-extensions/sieve-extensions.xhtml). - Email aliases, mailing lists, subaddressing and catch-all addresses support. - Automatic account configuration and discovery with [autoconfig](https://www.ietf.org/id/draft-bucksch-autoconfig-02.html) and [autodiscover](https://learn.microsoft.com/en-us/exchange/architecture/client-access/autodiscover?view=exchserver-2019). - Multi-tenancy support with domain and tenant isolation. - Disk quotas per user and tenant. - **Secure and robust**: - Encryption at rest with **S/MIME** or **OpenPGP**. - Automatic TLS certificate provisioning with [ACME](https://datatracker.ietf.org/doc/html/rfc8555) using `TLS-ALPN-01`, `DNS-01` or `HTTP-01` challenges. - Automated blocking of IP addresses that attack, abuse or scan the server for exploits. - Rate limiting. - Security audited (read the [report](https://stalw.art/blog/security-audit)). - Memory safe (thanks to Rust). - **Scalable and fault-tolerant**: - Designed to handle growth seamlessly, from small setups to large-scale deployments of thousands of nodes. - Built with **fault tolerance** and **high availability** in mind, recovers from hardware or software failures with minimal operational impact. - Peer-to-peer cluster coordination or with **Kafka**, **Redpanda**, **NATS** or **Redis**. - **Kubernetes**, **Apache Mesos** and **Docker Swarm** support for automated scaling and container orchestration. - Read replicas, sharded blob storage and in-memory data stores for high performance and low latency. - **Authentication and Authorization**: - **OpenID Connect** authentication. - OAuth 2.0 authorization with [authorization code](https://www.rfc-editor.org/rfc/rfc8628) and [device authorization](https://www.rfc-editor.org/rfc/rfc8628) flows. - **LDAP**, **OIDC**, **SQL** or built-in authentication backend support. - Two-factor authentication with Time-based One-Time Passwords (`2FA-TOTP`) - Application passwords (App Passwords). - Roles and permissions. - Access Control Lists (ACLs). - **Observability**: - Logging and tracing with **OpenTelemetry**, journald, log files and console support. - Metrics with **OpenTelemetry** and **Prometheus** integration. - Webhooks for event-driven automation. - Alerts with email and webhook notifications. - Live tracing and metrics. - **Web-based administration**: - Dashboard with real-time statistics and monitoring. - Account, domain, group and mailing list management. - SMTP queue management for messages and outbound DMARC and TLS reports. - Report visualization interface for received DMARC, TLS-RPT and Failure (ARF) reports. - Configuration of every aspect of the mail server. - Log viewer with search and filtering capabilities. - Self-service portal for password reset and encryption-at-rest key management. ## Screenshots ## Presentation **Want a deeper dive?** Need to explain to your boss why Stalwart is the perfect fit? Whether you're evaluating options, making a case to your team, or simply curious about how it all works under the hood, these slides walk you through the key features, architecture, and benefits of Stalwart. Browse the [slides](https://stalw.art/slides) to see what makes it stand out. ## Get Started Install Stalwart on your server by following the instructions for your platform: - [Linux / MacOS](https://stalw.art/docs/install/platform/linux) - [Windows](https://stalw.art/docs/install/platform/windows) - [Docker](https://stalw.art/docs/install/platform/docker) All documentation is available at [stalw.art/docs](https://stalw.art/docs/install/get-started). ## Support If you are having problems running Stalwart, you found a bug or just have a question, do not hesitate to reach us on [GitHub Discussions](https://github.com/stalwartlabs/stalwart/discussions), [Reddit](https://www.reddit.com/r/stalwartlabs) or [Discord](https://discord.com/servers/stalwart-923615863037390889). Additionally you may purchase an [Enterprise License](https://stalw.art/enterprise) to obtain priority support from Stalwart Labs LLC. ## Roadmap Stalwart has reached an exciting point in its journey, it’s now **feature complete**. All the core functionality and open standard email and collaboration protocols that we set out to support are in place. In other words, Stalwart already does everything you’d expect from a modern, standards-compliant mail and collaboration platform. The next major milestone is all about refinement: finalizing the database schema and focusing on performance optimizations to ensure everything runs as efficiently and reliably as possible. Once that’s done, we’ll be ready to roll out version **1.0**. Of course, development doesn’t stop there. The community has contributed hundreds of great ideas for improvements and new features, everything from subtle usability tweaks to entirely new integrations. You can see the full list of proposals over on our [GitHub issues](https://github.com/stalwartlabs/stalwart/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc+label%3Aenhancement). If there’s something you’d like to see prioritized, just give it a thumbs up as we plan to implement enhancements based on the community’s votes. ## Sponsorship Your support is crucial in helping us continue to improve the project, add new features, and maintain the highest level of quality. By [becoming a sponsor](https://opencollective.com/stalwart), you help fund the development and future of Stalwart. As a thank-you, sponsors who contribute $5 per month or more will automatically receive a [Enterprise edition](https://stalw.art/enterprise/) license. And, sponsors who contribute $30 per month or more, also have access to [Premium Support](https://stalw.art/support) from Stalwart Labs. ## Funding Part of the development of this project was funded through: - [NGI0 Entrust Fund](https://nlnet.nl/entrust), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 101069594. - [NGI Zero Core](https://nlnet.nl/NGI0/), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 101092990. If you find the project useful you can help by [becoming a sponsor](https://opencollective.com/stalwart). Thank you! ## License This project is dual-licensed under the **GNU Affero General Public License v3.0** (AGPL-3.0; as published by the Free Software Foundation) and the **Stalwart Enterprise License v1 (SELv1)**: - The [GNU Affero General Public License v3.0](./LICENSES/AGPL-3.0-only.txt) is a free software license that ensures your freedom to use, modify, and distribute the software, with the condition that any modified versions of the software must also be distributed under the same license. - The [Stalwart Enterprise License v1 (SELv1)](./LICENSES/LicenseRef-SEL.txt) is a proprietary license designed for commercial use. It offers additional features and greater flexibility for businesses that do not wish to comply with the AGPL-3.0 license requirements. Each file in this project contains a license notice at the top, indicating the applicable license(s). The license notice follows the [REUSE guidelines](https://reuse.software/) to ensure clarity and consistency. The full text of each license is available in the [LICENSES](./LICENSES/) directory. ## Copyright Copyright (C) 2020, Stalwart Labs LLC ================================================ FILE: SECURITY.md ================================================ # Security Policy for Stalwart ## Supported Versions We provide security updates for the following versions of Stalwart: | Version | Supported | End of Support | | ------- | ------------------ | -------------- | | 0.15.x | :white_check_mark: | TBD | | 0.14.x | :white_check_mark: | 2026-06-08 | | 0.13.x | :white_check_mark: | 2026-03-31 | | < 0.13 | :x: | Ended | **Note**: We typically support the current major version and one previous major version. Users are strongly encouraged to upgrade to the latest version for the best security posture. ## Reporting a Vulnerability We take the security of Stalwart very seriously. If you believe you've found a security vulnerability, we encourage you to inform us responsibly through coordinated disclosure. ### How to Report **Do not report security vulnerabilities through public GitHub issues, discussions, or social media.** Instead, please use one of these secure channels: 1. **Email** (preferred): Send details to `security@stalw.art` 2. **GitHub Security Advisories**: Use the "Report a vulnerability" button in the Security tab 3. **Backup contact**: If no response within 48 hours, email `hello@stalw.art` ### What to Include To help us understand and address the issue quickly, please include: **Required Information:** - Brief description of the vulnerability type - Affected version(s) and components - Steps to reproduce the issue - Impact assessment (what could an attacker achieve?) **Helpful Additional Details:** - Full paths of affected source files - Specific commit/branch where the issue exists - Required configuration to reproduce - Proof-of-concept code (if available) - Suggested mitigation or fix (if you have ideas) ### Our Response Process **Timeline Commitments:** - **Initial acknowledgment**: Within 24 hours - **Detailed response**: Within 72 hours - **Status updates**: Every 7 days until resolved - **Resolution target**: 90 days for most issues **What We'll Do:** 1. Acknowledge your report and assign a tracking ID 2. Assess the vulnerability and determine severity 3. Develop and test a fix 4. Coordinate disclosure timeline with you 5. Release security update and publish advisory 6. Credit you in our security advisory (if desired) ## Disclosure Policy We follow responsible disclosure principles: - **Coordinated disclosure**: We'll work with you to determine appropriate disclosure timing - **Typical timeline**: 90 days from report to public disclosure - **Early disclosure**: May occur if issue is being actively exploited - **Delayed disclosure**: May be necessary for complex issues requiring significant changes ## Scope This security policy applies to: **In Scope:** - Stalwart (all supported versions) - Official Docker images - Documentation that could lead to insecure configurations - Dependencies with security implications **Out of Scope:** - Third-party integrations or plugins - Issues requiring physical access to the server - Social engineering attacks - Attacks requiring compromised credentials (unless the vulnerability enables credential compromise) - Theoretical vulnerabilities without practical exploitation ## Security Measures **Our Commitments:** - Regular security audits of dependencies using `cargo audit` - Automated security scanning in CI/CD pipeline - Following Rust security best practices - Prompt security updates for critical dependencies - Security-focused code review process **User Responsibilities:** - Keep Stalwart updated to supported versions - Follow security configuration guidelines - Implement proper network security (firewalls, TLS, etc.) - Regular security monitoring and logging - Secure credential management ## Legal Safe Harbor We support security research conducted in good faith. If you follow these guidelines: **We will NOT:** - Initiate legal action against you - Contact law enforcement about your research - Suspend or terminate your access to Stalwart services **You must:** - Only test against your own Stalwart installations - Not access, modify, or delete user data - Not perform testing that could degrade service availability - Not publicly disclose the issue before coordinated disclosure - Act in good faith and not for malicious purposes ## Recognition We believe in recognizing security researchers who help keep Stalwart secure: - **Security Advisory Credits**: We'll credit you in our GitHub Security Advisories (unless you prefer to remain anonymous) - **Hall of Fame**: Significant contributors may be listed in our security acknowledgments - **Swag**: We may send Stalwart merchandise for notable contributions ## Security Updates **Stay Informed:** - Subscribe to our [GitHub releases](https://github.com/stalwartlabs/stalwart/releases) for security updates - Join our community channels for security announcements - Enable GitHub notifications for security advisories **Update Process:** - Security updates are published as patch releases (e.g., 0.12.1 → 0.12.2) - Critical vulnerabilities may receive out-of-band releases - Docker images are updated simultaneously with releases - Security advisories are published through GitHub Security Advisories ## Contact Information - **Security reports**: security@stalw.art - **General inquiries**: hello@stalw.art - **PGP Key**: Available upon request for sensitive communications ## Additional Resources - [Stalwart Security Incident Response Process](SECURITY_PROCESS.md) - [Security Configuration Guide](https://stalw.art/docs/install/security) - [Rust Security Advisory Database](https://rustsec.org/) *This security policy is effective as of June 20, 2025 and may be updated periodically. Check back regularly for updates.* ================================================ FILE: SECURITY_PROCESS.md ================================================ # Stalwart Security Incident Response Checklist ## Phase 1 : Initial Assessment & Validation ### Updates << Use this section to detail the report received, initial assessment, and validation results >> Example: I've reviewed the security report and confirmed this vulnerability exists in Stalwart version X.Y.Z. Assessment of exploitability: - Attack complexity: [High/Medium/Low] - Prerequisites: [Authentication required/Network access/Specific configuration/etc.] - User interaction required: [Yes/No] Potential impact: - Email data confidentiality: [At risk/Not affected] - Server integrity: [At risk/Not affected] - Service availability: [At risk/Not affected] - Estimated affected installations: [Number/Percentage] ### Resources - [Stalwart Security Policy](https://github.com/stalwartlabs/stalwart/blob/main/SECURITY.md) - [CVE Scoring Calculator](https://nvd.nist.gov/vuln-metrics/cvss/v3-calculator) - [Rust Security Advisory Database](https://rustsec.org/) ### Tasks - [ ] Reproduce the vulnerability in test environment - [ ] Assess CVSS score and severity level - [ ] Check if vulnerability affects current stable version - [ ] Check if vulnerability affects LTS versions (if applicable) - [ ] Determine if this requires immediate action or can wait for next release cycle - [ ] Document technical details and root cause ### Assessment Summary - **Severity Level**: `Critical|High|Medium|Low` - **CVSS Score**: `X.X` - **Affects versions**: `X.Y.Z to X.Y.Z` - **Root cause**: Brief technical explanation - **Introduced in commit/version**: `commit-hash` or `vX.Y.Z` - **Attack vector**: `Network|Local|Physical` - **Estimated timeline for fix**: `X days/weeks` ## Phase 2: Immediate Response & Mitigation ### Updates << Document immediate actions taken and mitigation strategies >> Example: Working on hotfix for version X.Y.Z. Temporary workaround available by disabling [feature] in configuration. ### Tasks - [ ] Implement immediate workaround if possible - [ ] Update security advisory draft - [ ] Prepare patch/hotfix - [ ] Test fix thoroughly in development environment - [ ] Prepare updated Docker images and binaries - [ ] Draft security advisory for GitHub Security Advisories - [ ] Consider if coordinated disclosure timeline needs adjustment ### Mitigation Details - **Workaround available**: `Yes|No` - If yes, describe briefly - **Fix implemented on**: `YYYY-MM-DD` - **Patch/hotfix version**: `vX.Y.Z` - **GitHub Security Advisory ID**: `GHSA-XXXX-XXXX-XXXX` ## Phase 3: Impact Assessment & User Analysis ### Updates << Analysis of potential impact on the Stalwart deployments >> Based on telemetry data and version statistics, approximately X installations may be affected. ### Tasks - [ ] Analyze version adoption from update checks (if available) - [ ] Estimate number of vulnerable installations - [ ] Assess if default configurations are vulnerable - [ ] Review if vulnerability has been exploited (check logs, reports) - [ ] Determine if any user data may have been compromised - [ ] Check for indicators of active exploitation in the wild ### Analysis Notes _Document your impact assessment process and findings_ ### Impact Summary - **Estimated vulnerable installations**: `~X out of Y` - **Default configuration vulnerable**: `Yes|No` - **Evidence of exploitation**: `Found|Not found|Unknown` - **User data potentially at risk**: `Email content|Credentials|Configuration|None` - **Confidence in assessment**: `High|Medium|Low` ## Phase 4: Communication & Release ### Updates << Communication strategy and release timeline >> Security release vX.Y.Z will be published on YYYY-MM-DD with coordinated disclosure. ### Tasks **Pre-release preparation:** - [ ] Finalize security patch - [ ] Prepare release notes with security details - [ ] Update documentation if needed - [ ] Test automated update mechanisms - [ ] Prepare GitHub Security Advisory **Communication channels:** - [ ] Draft announcement for Stalwart community forum/Discord - [ ] Prepare release announcement for GitHub - [ ] Draft security advisory content - [ ] Consider notification to major distributors/packagers **Release execution:** - [ ] Publish patched version to GitHub releases - [ ] Update Docker images on Docker Hub - [ ] Publish GitHub Security Advisory - [ ] Post to community channels (Discord/forum) - [ ] Update project website/documentation - [ ] Submit CVE request if warranted (CVSS ≥ 4.0) **Post-release:** - [ ] Monitor community channels for questions - [ ] Track adoption of security update - [ ] Follow up on any additional reports - [ ] Document lessons learned ### Communication Record - **Security release published**: `YYYY-MM-DD HH:MM UTC` - **GitHub Security Advisory**: `GHSA-XXXX-XXXX-XXXX` - **CVE ID** (if applicable): `CVE-YYYY-XXXXX` - **Community announcement**: [Link to forum/Discord post] - **Estimated time to 50% adoption**: `X days/weeks` ## Post-Incident Review ### What went well? - ### What could be improved? - ### Action items for future incidents: - [ ] - [ ] - [ ] ### Process improvements: - [ ] - [ ] ## Emergency Contacts - **Primary maintainer**: hello@stalw.art ================================================ FILE: SECURITY_TEMPLATE.md ================================================ # Stalwart Security Advisory **CVE ID:** CVE-YYYY-NNNNN **Publication Date:** YYYY-MM-DD **Last Updated:** YYYY-MM-DD ## Summary [Provide a brief, non-technical summary of the vulnerability in 1-2 sentences] ## Affected Products and Versions **Product:** Stalwart Mail and Collaboration Server **Affected Versions:** - Version X.X.X through Y.Y.Y - [List specific affected version ranges] **Fixed Versions:** - Version Z.Z.Z and later - [List all versions that include the fix] ## Vulnerability Details ### Description [Detailed technical description of the vulnerability, including how it can be exploited] ### Impact [Describe the potential impact if this vulnerability is exploited] ### CVSS Score **CVSS v3.1 Base Score:** X.X ([SEVERITY]) **Vector String:** CVSS:3.1/AV:X/AC:X/PR:X/UI:X/S:X/C:X/I:X/A:X **Severity Breakdown:** - **Attack Vector:** [Network/Adjacent/Local/Physical] - **Attack Complexity:** [Low/High] - **Privileges Required:** [None/Low/High] - **User Interaction:** [None/Required] - **Scope:** [Unchanged/Changed] - **Confidentiality Impact:** [None/Low/High] - **Integrity Impact:** [None/Low/High] - **Availability Impact:** [None/Low/High] ### CWE Classification **CWE-XXX:** [Weakness Name] ## Technical Details ### Root Cause [Explain the underlying cause of the vulnerability] ### Attack Scenario [Describe a realistic attack scenario or proof of concept, without providing exploit code] ### Prerequisites [List any conditions that must be met for successful exploitation] ## Remediation ### Recommended Actions 1. **Immediate:** Upgrade to version Z.Z.Z or later 2. **Short-term:** [Any temporary mitigation measures] 3. **Long-term:** [Any additional security hardening recommendations] ### Upgrade Instructions ```bash # Example upgrade commands [Provide specific upgrade instructions for Stalwart] ``` ### Workarounds [If applicable, describe any temporary workarounds for systems that cannot be immediately upgraded] **Note:** Workarounds are temporary measures and do not fully resolve the vulnerability. Upgrading is strongly recommended. ## Detection ### Indicators of Compromise [List any logs, patterns, or indicators that may suggest exploitation attempts] ### Log Entries ``` [Example log entries that administrators should look for] ``` ## Timeline - **YYYY-MM-DD:** Vulnerability discovered [by researcher/team name] - **YYYY-MM-DD:** Vendor notified - **YYYY-MM-DD:** Vendor acknowledged issue - **YYYY-MM-DD:** Fix developed and tested - **YYYY-MM-DD:** Fixed version released - **YYYY-MM-DD:** Public disclosure ## Credits This vulnerability was discovered by [Researcher Name / Organization]. ## References - Stalwart Mail Server: https://stalw.art/ - CVE Entry: https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-YYYY-NNNNN - GitHub Advisory: [Link to GitHub Security Advisory if applicable] - Release Notes: [Link to release notes with fix] ## Contact Information For questions or concerns regarding this advisory, please contact: **Security Team:** hello@stalw.art **Website:** https://stalw.art To report security vulnerabilities in Stalwart, please follow our [responsible disclosure policy](https://github.com/stalwartlabs/stalwart/security/policy). ## Disclaimer This advisory is provided "as is" without warranty of any kind. The information contained in this advisory is subject to change without notice. --- **Document Version:** 1.0 **Classification:** Public ================================================ FILE: UPGRADING/v0_04.md ================================================ # Upgrading from `v0.4.0` to `v0.4.x` - Replace the binary with the new version. - Restart the service. # Upgrading from `v0.3.x` to `v0.4.0` ## What's changed - **Configuration File Split:** While the `config.toml` configuration file format hasn't changed much, the new version has divided it into multiple sub-files. These sub-files are now included from the new `config.toml`. This division was implemented because the config file had grown significantly, and splitting it improves organization. - **Changes in the Sieve Interpreter Attribute Names:** - The configuration key prefix `jmap.sieve` (JMAP Sieve Interpreter) has been renamed to `sieve.untrusted`. - The configuration key prefix `sieve` (SMTP Sieve Interpreter) has been renamed to `sieve.trusted`. ## What's been added - **SPAM Filter Module:** The most notable addition in this version is the SPAM filter module. It comprises: - A TOML configuration file located at `etc/smtp/spamfilter.toml`. - A set of Sieve scripts in `etc/spamfilter/scripts`. - Lookup maps in `etc/spamfilter/maps`. - **New Configuration Key:** A new key `resolver.public-suffix` has been added. This specifies the URL of the list of public suffixes. ## Migration Steps 1. **Backup:** Ensure you have a backup of your current `config.toml` file. 2. **Download Configuration Bundle:** Fetch the new configuration bundle from [this link](https://get.stalw.art/resources/config.zip). Unpack it under `BASE_DIR/etc` (for example `/opt/stalwart-mail/etc`). 3. **Update Configuration Files:** Modify the following files with your domain name, host name, certificate paths, DKIM signatures, and so on: - `etc/config.toml` - `etc/jmap/store.toml` - `etc/jmap/oauth.toml` - `etc/smtp/signature.toml` - `etc/common/tls.toml` 4. **Adjust included files:** If you are using an LDAP directory for authentication, edit `etc/config.toml` and replace the `etc/directory/sql.toml` include with `etc/directory/ldap.toml`. 5. **Configure the SPAM Filter Database:** Set up and configure the SPAM filter database. More details can be found [here](https://stalw.art/docs/spamfilter/settings/database). 6. **Review All TOML Files:** Navigate to every TOML file under the `etc/` directory and make necessary changes. 7. **Update Binary:** Download and substitute the v0.4.0 binary suitable for your platform from [here](https://github.com/stalwartlabs/mail-server/releases/tag/v0.4.0). 8. **Restart Service:** Conclude by restarting the Stalwart service. ### Alternative Method: 1. **Separate Installation:** Install v0.4.0 in a distinct directory. This will auto-update all configuration files and establish the spam filter database in SQLite format. 2. **Move Configuration Files:** Transfer the configuration files from `etc/` and the SQLite spam filter database from `data/` to your current installation's directory. 3. **Replace Binary:** Move the binary from the `bin/` directory to your current installation's `data/` directory. 4. **Restart Service:** Finally, restart the Stalwart service. We apologize for the lack of an automated migration tool for this upgrade. However, we are planning on introducing an automated migration tool in the near future. Thank you for your understanding and patience. ================================================ FILE: UPGRADING/v0_05.md ================================================ # Upgrading from `v0.5.2` to `v0.5.3` - The following configuration attributes have been renamed, see [store.toml](https://github.com/stalwartlabs/mail-server/blob/main/resources/config/common/store.toml) for an example: - `jmap.store.data` -> `storage.data` - `jmap.store.fts` -> `storage.fts` - `jmap.store.blob` -> `storage.blob` - `jmap.encryption.*` -> `storage.encryption.*` - `jmap.spam.header` -> `storage.spam.header` - `jmap.fts.default-language` -> `storage.fts.default-language` - `jmap.cluster.node-id` -> `storage.cluster.node-id` - `management.directory` and `sieve.trusted.default.directory` -> `storage.directory` - `sieve.trusted.default.store` -> `storage.lookup` - Proxy networks are now configured under `server.proxy.trusted-networks` rather than `server.proxy-trusted-networks`. IP addresses/masks have to be defined within a set (`{}`) rather than a list (`[]`), see [server.toml](https://github.com/stalwartlabs/mail-server/blob/main/resources/config/common/server.toml) for an example. # Upgrading from `v0.5.1` to `v0.5.2` - Make sure that implicit TLS is enabled for the JMAP [listener](https://stalw.art/docs/server/listener) configured under `ets/jmap/listener.toml`: ```toml [server.listener."jmap".tls] implicit = true ``` - Optional: Enable automatic TLS with [ACME](https://stalw.art/docs/server/tls/acme). - Replace the binary with the new version. - Restart the service. # Upgrading from `v0.5.0` to `v0.5.1` - Replace the binary with the new version. - Restart the service. # Upgrading from `v0.4.x` to `v0.5.0` ## What's changed - **Database Layout**: Version 0.5.0 utilizes a different database layout which is more efficient and allows multiple backends to be supported. For this reason, the database must be migrated to the new layout. - **Configuration file changes**: The configuration file has been updated to support multiple stores, most configuration attributes starting with `store.*` and `directory.*` need to be reviewed. - **SPAM filter**: Sieve scripts that interact with databases need to be updated. The functions `lookup` and `lookup_map` has been renamed to `key_exists` and `key_get`. It is recommended to replace all scripts with the new versions rather than updating them manually. Additionally, the SPAM database no longer requires an SQL server, it can now be stored in Redis or any of the supported databases. - **Directory superusers**: Due to problems and confusion with the `superuser-group` attribute, the concept of a superuser group has been removed. Instead, a new attribute `type` has been added to external directories. The value of this attribute can be `individual`, `group` or `admin`. The `admin` type is equivalent to the old superuser group. The `type` attribute is required for all principals in the directory, it defaults to `individual` if not specified. - **Purge schedules**: The attributes `jmap.purge.schedule.db` and `jmap.purge.schedule.blobs` have been removed. Instead, the purge frequency is now specified per store in `store..purge.frequency`. The attribute `jmap.purge.schedule.sessions` has been renamed to `jmap.purge.sessions.frequency`. ## What's been added - **Multiple stores**: The server now supports multiple stores to be defined in the configuration file under `store.`. Which store to use is defined in the `jmap.store.data`, `jmap.store.fts` and `jmap.store.blob` settings. - **More backend options**: It is now possible to use `RocksDB`, `PostgreSQL` and `MySQL` as data stores. It is also now possible to store blobs in any of the supported databases instead of being limited to the filesystem or an S3-compatible storage. Full-text indexing can now be done using `Elasticsearch` and the Spam database stored in `Redis`. - **Internal Directory**: The server now has an internal directory that can be used to store user accounts, passwords and group membership. This directory can be used instead of an external directory such as LDAP or SQL. - **New settings**: When running Stalwart in a cluster, `jmap.cluster.node-id` allows to specify a unique identifier for each node. Messages containing the SPAM headers defined in `jmap.spam.header` are moved automatically to the user's Junk Mail folder. - **Default Sieve stores**: For Sieve scripts such as the Spam filter that require access to a directory and a lookup store, it is now possible to configure the default lookup store and directory using the `sieve.trusted.default.directory` and `sieve.trusted.default.store` settings. ## Migration Steps Rather than manually updating the configuration file, it is recommended to start with a fresh configuration file and update it with the necessary settings: - Install `v0.5.0` in a distinct directory. You now have the option to use an [internal directory](https://stalw.art/docs/directory/types/internal), which will allow you to manage users and groups directly from Stalwart server. Alternatively, you can continue to use an external directory such as LDAP or SQL. - Update the configuration files with your previous settings. All configuration attributes are backward compatible, except those starting with `store.*`, `directory.*` and `jmap.purge.*`. - Export each account following the procedure described in the [migration guide](https://stalw.art/docs/management/database/migrate). - Stop the old `v0.4.x` server. - If there are messages pending to be delivered in the SMTP queue, move the `queue` directory to the new installation. - Start the new `v0.5.0` server. - Import each account following the procedure described in the [migration guide](https://stalw.art/docs/management/database/migrate). Once again, we apologize for the lack of an automated migration tool for this upgrade. However, we are planning on introducing an automated migration tool once the web-admin is released in Q1 2024. Thank you for your understanding and patience. ================================================ FILE: UPGRADING/v0_06.md ================================================ # Upgrading from `v0.5.3` to `v0.6.0` - In order to support [expressions](https://stalw.art/docs/configuration/expressions/overview), version `0.6.0` introduces multiple breaking changes in the SMTP server configuration file. It is recommended to download the new SMTP configuration files from the [repository](https://github.com/stalwartlabs/mail-server/tree/main/resources/config/smtp), make any necessary changes and replace the old files under `INSTALL_DIR/etc/smtp` with the new ones. - If you are using custom subaddressing of catch-all rules, you'll need to replace these rules with expressions. Check out the updated [syntax](https://stalw.art/docs/directory/addresses). - Message queues are now distributed and stored in the backend specified by the `storage.data` and `storage.blob` settings. Make sure to flush your SMTP message queue before upgrading to `0.6.0` to avoid losing any outgoing messages pending delivery. - Replace the binary with the new version. - Restart the service. ================================================ FILE: UPGRADING/v0_07.md ================================================ # Upgrading from `v0.6.0` to `v0.7.0` Version `0.7.0` of Stalwart introduces significant improvements and features that enhance performance and functionality. However, it also comes with multiple breaking changes in the configuration files and a revamped database layout optimized for accessing large mailboxes. Additionally, Stalwart now supports compression for binaries stored in the blob store, further increasing efficiency. Due to these extensive changes, the recommended approach for upgrading is to perform a clean reinstallation of Stalwart and manually migrate your accounts to the new version. ## Pre-Upgrade Steps - Download the `v0.7.0` mail-server and CLI binaries for your platform from the [releases page](https://github.com/stalwartlabs/mail-server/releases/latest/). - Initialize the setup on a distinct directory using the command `sudo ./stalwart-mail --init /path/to/new-install`. This command will print the administrator password required to access the web-admin. - Create the `bin` directory using `mkdir /path/to/new-install/bin`. - Move the downloaded binaries to the `bin` directory using the command `mv stalwart-mail stalwart-cli /path/to/new-install/bin`. - Open `/path/to/new-install/etc/config.toml` in a text editor and comment out all listeners except the HTTP listener for port `8080`. - Start the new installation from the terminal using the command `sudo /path/to/new-install/bin/stalwart-mail --config /path/to/new-install/etc/config.toml`. - Point your browser to the web-admin at `http://yourserver.org:8080` and login using the auto-generated administrator password. - Configure the new installation with your domain, hostname, certificates, and other settings following the instructions at [stalw.art/docs/get-started](https://stalw.art/docs/get-started). Ignore the part about using the installation script, we are performing a manual installation. - Add your user accounts. - Configure Stalwart to run as the `stalwart-mail` user and `stalwart-mail` group from `Settings` > `Server` > `System`. This is not necessary if you are using Docker. - Stop the new installation by pressing `Ctrl+C` in the terminal. ## Upgrade Steps - On your `v0.6.0` installation, open in a text editor the `smtp/listener.toml`, `imap/listener.toml` files and comment out all listeners except the JMAP/HTTP listener (we are going to need it to export the user accounts) and then restart the service. - If you are using an external store, backup the database using the appropriate method for your database system. - Create the `~/exports` directory, here we will store the exported accounts. - Using the existing CLI tool (not the one you just downloaded as it is not compatible), export each user account using the command `./stalwart-cli -u https://your-old-server.org -c export account ~/exports`. - Stop the `v0.6.0` installation using the command `sudo systemctl stop stalwart-mail`. - Move the old `v0.6.0` installation to a backup directory, for example `mv /opt/stalwart-mail /opt/stalwart-mail-backup`. - Move the new `v0.7.0` installation to the old installation directory, for example `mv /path/to/new-install /opt/stalwart-mail`. - Set the right permissions for the new installation using the command `sudo chown -R stalwart-mail:stalwart-mail /opt/stalwart-mail`. - Start the new installation using the command `sudo systemctl start stalwart-mail`. - Import the accounts using the new CLI tool with the command `./stalwart-cli -u http://yourserver.org:8080 -c import account ~/exports/`. - Using the admin tool, reactivate all the necessary listener (SMTP, IMAP, etc.) - Restart the service using the command `sudo systemctl restart stalwart-mail`. We apologize for the complexity of the upgrade process associated with this version of Stalwart. We understand the challenges and inconveniences that the requirement for a clean reinstallation and manual account migration poses. Moving forward, an automated migration tool will be included in any future releases that necessitate changes to the database layout, aiming to streamline the upgrade process for you. Furthermore, as we approach the milestone of version 1.0.0, we anticipate that such foundational changes will become increasingly infrequent, leading to more straightforward updates. We appreciate your patience and commitment to Stalwart during this upgrade. ================================================ FILE: UPGRADING/v0_08.md ================================================ # Upgrading from `v0.7.3` to `v0.8.0` Version `0.8.0` includes both performance and security enhancements that require your data to be migrated to a new database layout. Luckily version `0.7.3` includes a migration tool which should make this process much easier than previous upgrades. In addition to the new layout, you will have to change the systemd service file to use the `CAP_NET_BIND_SERVICE` capability. ## Preparation - Upgrade to version `0.7.3` if you haven't already. If you are on a version previous to `0.7.0`, you will have to do a manual migration of your data using the Command-line Interface. - Create a directory where your data will be exported to, for example `/opt/stalwart-mail/export`. ## Systemd service upgrade (Linux only) - Stop the `v0.7.3` installation: ```bash $ sudo systemctl stop stalwart-mail ``` - Update your systemd file to include the `CAP_NET_BIND_SERVICE` capability. Open the file `/etc/systemd/system/stalwart-mail.service` in a text editor and add the following lines under the `[Service]` section: ``` User=stalwart-mail Group=stalwart-mail AmbientCapabilities=CAP_NET_BIND_SERVICE ``` - Reload the daemon: ```bash $ systemctl daemon-reload ``` - Do not start the service yet. ## Data migration - Stop Stalwart and export your data: ```bash $ sudo systemctl stop stalwart-mail $ sudo /opt/stalwart-mail/bin/stalwart-mail --config /opt/stalwart-mail/etc/config.toml --export /opt/stalwart-mail/export $ sudo chown -R stalwart-mail:stalwart-mail /opt/stalwart-mail/export ``` or, if you are using the Docker image: ```bash $ docker stop stalwart-mail $ docker run --rm -v :/opt/stalwart-mail -it stalwart-mail /opt/stalwart-mail/bin/stalwart-mail --config /opt/stalwart-mail/etc/config.toml --export /opt/stalwart-mail/export ``` - Backup your `v0.7.3` installation: - If you are using RocksDB or SQLite, simply rename the `data` directory to `data-backup`, for example: ```bash $ mv /opt/stalwart-mail/data /opt/stalwart-mail/data-backup $ mkdir /opt/stalwart-mail/data $ chown stalwart-mail:stalwart-mail /opt/stalwart-mail/data ``` - If you are using PostgreSQL, rename the database and create a blank database with the same name, for example: ```sql ALTER DATABASE stalwart RENAME TO stalwart_old; CREATE database stalwart; ``` - If you are using MySQL, rename the database and create a blank database with the same name, for example: ```sql CREATE DATABASE stalwart_old; RENAME TABLE stalwart.b TO stalwart_old.b; RENAME TABLE stalwart.v TO stalwart_old.v; RENAME TABLE stalwart.l TO stalwart_old.l; RENAME TABLE stalwart.i TO stalwart_old.i; RENAME TABLE stalwart.t TO stalwart_old.t; RENAME TABLE stalwart.c TO stalwart_old.c; DROP DATABASE stalwart; CREATE database stalwart; ``` - If you are using FoundationDB, backup your database and clean the entire key range. - Download the `v0.8.0` mail-server for your platform from the [releases page](https://github.com/stalwartlabs/mail-server/releases/latest/) and replace the binary in `/opt/stalwart-mail/bin`. If you are using the Docker image, pull the latest image. - Import your data: ```bash $ sudo -u stalwart-mail /opt/stalwart-mail/bin/stalwart-mail --config /opt/stalwart-mail/etc/config.toml --import /opt/stalwart-mail/export ``` or, if you are using the Docker image: ```bash $ docker run --rm -v :/opt/stalwart-mail -it stalwart-mail /opt/stalwart-mail/bin/stalwart-mail --config /opt/stalwart-mail/etc/config.toml --import /opt/stalwart-mail/export ``` - Start the service: ```bash $ sudo systemctl start stalwart-mail ``` Or, if you are using the Docker image: ```bash $ docker start stalwart-mail ``` ================================================ FILE: UPGRADING/v0_09.md ================================================ # Upgrading from `v0.8.x` to `v0.9.0` Version `0.9.0` introduces significant internal improvements while maintaining compatibility with existing database layouts and configuration file formats from version `0.8.0`. As a result, no data or configuration migration is necessary. This release focuses on enhancing performance and functionality, particularly in logging and tracing capabilities. To upgrade to Stalwart version `0.9.0` from `0.8.x`, begin by downloading the latest version of the `stalwart-mail` binary. Once downloaded, replace the existing binary with the new version. Additionally, it's important to update the WebAdmin interface to the latest version to ensure compatibility and to access new features introduced in this release. In terms of breaking changes, this release brings significant updates to webhooks. All webhook event names have been modified, requiring a thorough review and adjustment of existing webhook configurations. Furthermore, the update introduces hundreds of new event types, enhancing the granularity and specificity of event handling capabilities. Users should familiarize themselves with these changes to effectively integrate them into their systems. The reason for this release being classified as a major version, despite the absence of changes to the database or configuration formats, is the complete rewrite of the logging and tracing layer. This overhaul substantially improves the efficiency and speed of generating detailed tracing and logging events, making the system more robust and facilitating easier debugging and monitoring. ================================================ FILE: UPGRADING/v0_10.md ================================================ # Upgrading from `v0.9.x` to `v0.10.0` ## Important Notes - In version `0.10.0` accounts are associated with roles and permissions, which define what resources they can access. The concept of administrator or super user accounts no longer exists, now there is a single account type (the `individual` principal) which can be assigned the `admin` role or custom permissions to have administrator access. - Due to the changes in the database layout in order to support roles and permissions, the database must be migrated to the new layout. The migration is automatic and should not require any manual intervention. - While the database migration is automatic, it's recommended to **back up your data** before upgrading. - The webadmin must be upgraded **before** the mail server to maintain access post-upgrade. This is true even if you run Stalwart in Docker. ## Step-by-Step Upgrade Process - Upgrade the webadmin by clicking on `Manage` > `Maintenance` > `Update Webadmin`. - Stop Stalwart and backup your data: ```bash $ sudo systemctl stop stalwart-mail $ sudo /opt/stalwart-mail/bin/stalwart-mail --config /opt/stalwart-mail/etc/config.toml --export /opt/stalwart-mail/export $ sudo chown -R stalwart-mail:stalwart-mail /opt/stalwart-mail/export ``` or, if you are using the Docker image: ```bash $ docker stop stalwart-mail $ docker run --rm -v :/opt/stalwart-mail -it stalwart-mail /usr/local/bin/stalwart-mail --config /opt/stalwart-mail/etc/config.toml --export /opt/stalwart-mail/export ``` - Download the `v0.10.0` mail-server for your platform from the [releases page](https://github.com/stalwartlabs/mail-server/releases/latest/) and replace the binary in `/opt/stalwart-mail/bin`. If you are using the Docker image, pull the latest image. - Start the service: ```bash $ sudo systemctl start stalwart-mail ``` Or, if you are using the Docker image: ```bash $ docker start stalwart-mail ``` ================================================ FILE: UPGRADING/v0_11.md ================================================ # Upgrading from `v0.10.x` to `v0.11.0` Version `0.11.0` introduces breaking changes to the spam filter configuration. Although no data migration is required, if changes were made to the previous spam filter, the configuration of the new spam filter should be reviewed. In particular: - `lookup.spam-*` settings are no longer used, these have been replaced by `spam-filter.*` settings. Review the [updated documentation](http://stalw.art/docs/spamfilter/overview). - Previous `spam-filter` and `track-replies` Sieve scripts cannot be used with the new version. They have been replaced by a built-in spam filter written in Rust. - Cache settings have changed, see the [documentation](https://stalw.art/docs/server/cache) for details. - Support for Pipes was removed in favor of MTA hooks and Milter. - `config.resource.spam-filter` is now `spam-filter.resource`. - `config.resource.webadmin` is now `webadmin.resource`. - `authentication.rate-limit` was removed as security is handled by fail2ban. ================================================ FILE: UPGRADING/v0_12.md ================================================ # Upgrading from `v0.11.x` to `v0.12.x` ## Important Notes Version `0.12.x` introduces significant improvements such as zero-copy deserialization which make the new database layout incompatible with the previous version. As a result, the database must be migrated to the new layout. The migration is done automatically on startup and should not require any manual intervention. However, it is highly recommended to **back up your data** before upgrading since it is not possible to downgrade the database once it has been migrated. You may also want to run a mock migration before upgrading to ensure that everything works as expected. In addition to the database layout changes, multiple settings were renamed: - `server.http.*` to `http.*`. - `jmap.folders.*` to `email.folders.*`. - `jmap.account.purge.frequency` to `account.purge.frequency`. - `jmap.email.auto-expunge` to `email.auto-expunge`. - `jmap.protocol.changes.max-history` to `changes.max-history`. - `storage.encryption.*` to `email.encryption.*`. ## Step-by-Step Upgrade Process - Stop Stalwart in **every single node of your cluster**. If you are using the systemd service, you can do this with the following command: ```bash $ sudo systemctl stop stalwart-mail ``` - Backup your data following your database system's instructions. For example, if you are using RocksDB or SQLite, you can simply copy the `data` directory to a backup location. If you are using PostgreSQL or MySQL, you can use the `pg_dump` or `mysqldump` commands to create a backup of your database. If your database does not support backups, you can use the [built-in migration utility](https://stalw.art/docs/management/migration) to export your data to a file. For example: ```bash $ sudo /opt/stalwart-mail/bin/stalwart-mail --config /opt/stalwart-mail/etc/config.toml --export /opt/stalwart-mail/export $ sudo chown -R stalwart-mail:stalwart-mail /opt/stalwart-mail/export ``` - Download the `v0.12.x` binary for your platform (which is now called `stalwart` rather than `mail-server`) from the [releases page](https://github.com/stalwartlabs/stalwart/releases/latest/) and replace the binary in `/opt/stalwart-mail/bin`. If you rename the binary from `stalwart` to `stalwart-mail`, you can keep the same systemd service file, otherwise you will need to update the service file to point to the new binary name. - Start the service. In a cluster, you can speed up the migration process by starting all nodes at once. ```bash $ sudo systemctl start stalwart-mail ``` - Upgrade the webadmin by clicking on `Manage` > `Maintenance` > `Update Webadmin`. ## Step-by-Step Upgrade Process (Docker) - Stop the Stalwart container in **every single node of your cluster**. If you are using Docker, you can do this with the following command: ```bash $ docker stop stalwart-mail ``` - Backup your data following your database system's instructions. For example, if you are using RocksDB or SQLite, you can simply copy the `data` directory to a backup location. If you are using PostgreSQL or MySQL, you can use the `pg_dump` or `mysqldump` commands to create a backup of your database. If your database does not support backups, you can use the `--export` command to export your data to a file. For example: ```bash $ docker run --rm -v :/opt/stalwart-mail -it stalwart-mail /usr/local/bin/stalwart-mail --config /opt/stalwart-mail/etc/config.toml --export /opt/stalwart-mail/export ``` - The Docker image location has now changed to `stalwartlabs/stalwart` instead of `stalwartlabs/mail-server`. Pull the latest image and configure it to use your existing data directory: ```bash $ docker run -d -ti -p 443:443 -p 8080:8080 \ -p 25:25 -p 587:587 -p 465:465 \ -p 143:143 -p 993:993 -p 4190:4190 \ -p 110:110 -p 995:995 \ -v :/opt/stalwart \ --name stalwart stalwartlabs/stalwart:latest ``` - Since the mount point has changed from `/opt/stalwart-mail` to `/opt/stalwart`, you will need to update your Stalwart's configuration file to reflect this change. Open the file `/opt/stalwart/etc/config.toml` and update the paths accordingly. - Upgrade the webadmin by clicking on `Manage` > `Maintenance` > `Update Webadmin`. ================================================ FILE: UPGRADING/v0_13.md ================================================ # Upgrading from `v0.12.x` (and `v0.11.x`) to `v0.13.x` ## Important Notes Version `0.13.x` introduces a significant redesign of the MTA’s delivery and queueing subsystem. This includes a transition to a new message queue serialization format and a move to a strategy-based configuration model for routing, scheduling, and delivery control. Upon first launch of version `0.13.0`, any messages currently in the outbound queue will be automatically migrated to the new format. This migration is handled internally and does not require manual intervention. However, if your deployment includes custom routing rules or queueing logic, it is important to manually reconfigure those settings using the new strategy framework. The previous configuration format for routing is no longer compatible and will need to be updated. For systems that rely solely on the default configuration, no changes are required and the upgrade should proceed without issue. Even if your system uses the default settings, it is strongly recommended to read the accompanying [blog announcement](https://stalw.art/blog/virtual-queues) and consult the [updated documentation](https://stalw.art/docs/mta/outbound/overview). These resources provide a full overview of the new delivery architecture and can help you determine whether any adjustments are needed for your environment. Before applying the upgrade to a production system, take time to familiarize yourself with the new configuration structure and validate that your delivery behavior aligns with the new model. ## Step-by-Step Upgrade Process - Stop Stalwart in **every single node of your cluster**. If you are using the systemd service, you can do this with the following command: ```bash $ sudo systemctl stop stalwart ``` - Backup your data following your database system's instructions. For example, if you are using RocksDB or SQLite, you can simply copy the `data` directory to a backup location. If you are using PostgreSQL or MySQL, you can use the `pg_dump` or `mysqldump` commands to create a backup of your database. If your database does not support backups, you can use the [built-in migration utility](https://stalw.art/docs/management/migration) to export your data to a file. For example: ```bash $ sudo /opt/stalwart/bin/stalwart --config /opt/stalwart/etc/config.toml --export /opt/stalwart/export $ sudo chown -R stalwart:stalwart /opt/stalwart/export ``` - Download the `v0.13.x` binary for your platform from the [releases page](https://github.com/stalwartlabs/stalwart/releases/latest/) and replace the binary in `/opt/stalwart/bin`. - Start the service. In a cluster, you can speed up the migration process by starting all nodes at once. ```bash $ sudo systemctl start stalwart ``` - Upgrade the webadmin by clicking on `Manage` > `Maintenance` > `Update Webadmin`. ## Step-by-Step Upgrade Process (Docker) - Stop the Stalwart container in **every single node of your cluster**. If you are using Docker, you can do this with the following command: ```bash $ docker stop stalwart ``` - Backup your data following your database system's instructions. For example, if you are using RocksDB or SQLite, you can simply copy the `data` directory to a backup location. If you are using PostgreSQL or MySQL, you can use the `pg_dump` or `mysqldump` commands to create a backup of your database. If your database does not support backups, you can use the `--export` command to export your data to a file. For example: ```bash $ docker run --rm -v :/opt/stalwart -it stalwart /usr/local/bin/stalwart --config /opt/stalwart/etc/config.toml --export /opt/stalwart/export ``` - Pull the latest image and restart the container: ```bash $ docker pull stalwartlabs/stalwart:latest $ docker start stalwart ``` - Upgrade the webadmin by clicking on `Manage` > `Maintenance` > `Update Webadmin`. ================================================ FILE: UPGRADING/v0_14.md ================================================ # Upgrading from `v0.13.x` to `v0.14.x` ## Binary installation - Stop Stalwart in **every single node of your cluster**. If you are using the systemd service, you can do this with the following command: ```bash $ sudo systemctl stop stalwart ``` - Backup your data following your database system's instructions. For example, if you are using RocksDB or SQLite, you can simply copy the `data` directory to a backup location. If you are using PostgreSQL or MySQL, you can use the `pg_dump` or `mysqldump` commands to create a backup of your database. If your database does not support backups, you can use the [built-in migration utility](https://stalw.art/docs/management/migration) to export your data to a file. For example: ```bash $ sudo /opt/stalwart/bin/stalwart --config /opt/stalwart/etc/config.toml --export /opt/stalwart/export $ sudo chown -R stalwart:stalwart /opt/stalwart/export ``` - Download the latest binary for your platform from the [releases page](https://github.com/stalwartlabs/stalwart/releases/latest/) and replace the binary in `/opt/stalwart/bin`. - Start the service. In a cluster, you can speed up the migration process by starting all nodes at once. ```bash $ sudo systemctl start stalwart ``` - Upgrade the webadmin by clicking on `Manage` > `Maintenance` > `Update Webadmin`. ## Containerized - Stop the Stalwart container in **every single node of your cluster**. If you are using Docker, you can do this with the following command: ```bash $ docker stop stalwart ``` - Backup your data following your database system's instructions. For example, if you are using RocksDB or SQLite, you can simply copy the `data` directory to a backup location. If you are using PostgreSQL or MySQL, you can use the `pg_dump` or `mysqldump` commands to create a backup of your database. If your database does not support backups, you can use the `--export` command to export your data to a file. For example: ```bash $ docker run --rm -v :/opt/stalwart -it stalwart /usr/local/bin/stalwart --config /opt/stalwart/etc/config.toml --export /opt/stalwart/export ``` - Pull the latest image and restart the container: ```bash $ docker pull stalwartlabs/stalwart:latest $ docker start stalwart ``` - Upgrade the webadmin by clicking on `Manage` > `Maintenance` > `Update Webadmin`. ================================================ FILE: UPGRADING/v0_15.md ================================================ # Upgrading from `v0.14.x` to `v0.15.x` Stalwart `v0.15.x` introduces **breaking changes** to both the **database schema** and some **configuration options**. Upgrading to this version **requires a schema migration**, which is performed **automatically when Stalwart starts** for the first time on `v0.15.x`. Because this migration modifies how data is stored and indexed, it is important to understand what will change, what will be migrated, and how the upgrade may impact your deployment—especially for larger installations. ## What's changed Version `0.15.x` introduces significant internal improvements focused on performance, storage efficiency, and accuracy: - **Optimized database schema**: The database schema has been redesigned to use less storage space and significantly reduce the number of read and write operations required for common tasks. - **Rewritten search layer**: The search subsystem has been completely rewritten to use a more efficient and scalable indexing strategy. - **Native full-text search for SQL backends**: When using **PostgreSQL** or **MySQL** as the backend, Stalwart now leverages the database’s **native full-text search capabilities**, replacing the previous custom full-text search implementation. - **New spam classifier engine** : The spam classifier has been rewritten to use the **FTRL-Proximal** algorithm instead of the previous **Naive Bayes** implementation. This change improves classification accuracy, reduces memory usage, and reduces storage requirements for training data. ## What will be migrated The migration process runs automatically at startup and will migrate the following data: - **E-mail metadata**, including flags, folders, and parsed message representations. *(The raw e-mail content stored in the blob store is not migrated.)* - **Encryption-at-rest settings**, which now also include a **spam training privacy option** - **MTA message queue metadata** *(The actual message contents are not migrated.)* - **Maintenance tasks** - **Blob links** *(The underlying blobs themselves are not migrated.)* - **Search indexes**, which will be **rebuilt** using the new indexing strategy ## Important considerations - For deployments with **1,000 or more mailboxes**, the migration may take a **considerable amount of time**, depending on the volume of stored data. - During migration, **Stalwart runs in read-only mode**: - No new e-mail can be received - No outbound e-mail can be sent - It is **strongly recommended** to perform this upgrade during a **maintenance window**. - By default, the migration process is **multithreaded** and uses two threads for each available CPUs. You can control the number of threads by setting the following environment variable ``NUM_THREADS=`` > **Note:** If you do **not** require any of the features introduced in `v0.15.x`, consider **waiting for the next major release**, which will introduce a proxy-based architecture allowing **zero-downtime upgrades**. ## Upgrading steps ### Binary installation - Stop Stalwart in **every single node of your cluster**. If you are using the systemd service, you can do this with the following command: ```bash $ sudo systemctl stop stalwart ``` - Backup your data following your database system's instructions. For example, if you are using RocksDB or SQLite, you can simply copy the `data` directory to a backup location. If you are using PostgreSQL or MySQL, you can use the `pg_dump` or `mysqldump` commands to create a backup of your database. - Download the latest binary for your platform from the [releases page](https://github.com/stalwartlabs/stalwart/releases/latest/) and replace the binary in `/opt/stalwart/bin`. - Start the service. In a cluster, you can speed up the migration process by starting all nodes at once. ```bash $ sudo systemctl start stalwart ``` ### Containerized - Stop the Stalwart container in **every single node of your cluster**. If you are using Docker, you can do this with the following command: ```bash $ docker stop stalwart ``` - Backup your data following your database system's instructions. For example, if you are using RocksDB or SQLite, you can simply copy the `data` directory to a backup location. If you are using PostgreSQL or MySQL, you can use the `pg_dump` or `mysqldump` commands to create a backup of your database. - Pull the latest image and restart the container: ```bash $ docker pull stalwartlabs/stalwart:latest $ docker start stalwart ``` ## Post-upgrade steps After the upgrade and migration complete, several follow-up steps are required or recommended: - **Upgrade the webadmin**: Upgrade the webadmin interface by navigating to ``Manage → Maintenance → Update Webadmin`` - **Update the spam rules**: Download and apply the latest spam rules from the webadmin ``Manage → Maintenance → Update Spam rules`` - **Update search settings**: Review the updated documentation for search settings, as some configuration options have changed. In particular, the Elasticsearch backend now uses **different authentication settings** than previous versions. - **Rebuild search indexes**: All search indexes must be rebuilt to take advantage of the new indexing strategy. This can be done from the webadmin interface ``Manage → Maintenance``. - **Recalculate disk quotas for all accounts**: This step is **not required immediately**, but it is recommended to perform it at some point after the upgrade. The new version includes additional metadata in quota calculations, so recalculating ensures accurate disk usage reporting. ```bash $ curl -X DELETE https://myserver.org/api/store/quota/ -u : -k ``` - **Delete deprecated spam classifier keys**: Remove deprecated spam classifier keys from the memory store. These are the keys starting with the integer prefixes `12` to `16` and `17` to `18`: - If you are using Redis: ```bash $ for code in {12..18}; do char=$(printf "\\x$(printf '%02x' $code)") redis-cli --scan --pattern "${char}*" | xargs -r redis-cli DEL done ``` - If you are using your database as the in-memory store: ```bash $ /opt/stalwart/bin/stalwart --config /opt/stalwart/etc/config.toml --console Stalwart Server v0.15.2 Data Store CLI > delete y\x0c\x00 y\x12\xff > delete m\x0c\x00 m\x12\xff > exit ``` - If you are using your database as the in-memory store with Docker: ```bash $ docker stop stalwart $ docker run -it --rm \ -v :/opt/stalwart \ --entrypoint /usr/local/bin/stalwart \ stalwartlabs/stalwart:latest \ --config /opt/stalwart/etc/config.toml --console Stalwart Server v0.15.2 Data Store CLI > delete y\x0c\x00 y\x12\xff > delete m\x0c\x00 m\x12\xff > exit $ docker start stalwart ``` ## Troubleshooting ### Interrupted or stopped migration If the migration process is interrupted or stopped, it can be **resumed automatically** by simply restarting Stalwart. ### `Data corruption detected` error If you see an error message similar to: ``Data corruption detected``. This indicates that **another node wrote data using the old format while the migration was in progress**. This usually happens when the cluster was **not fully stopped** before starting the upgrade. In order to resolve this issue, follow these steps: 1. Stop **all** Stalwart nodes. 2. Ensure **all nodes are upgraded** to `v0.15.x`. 3. Start the nodes again. ### Forcing a migration If the migration does not resume because the node responsible for it already marked it as completed, you can force migration using environment variables: - **Force re-migration of MTA queue metadata**: ``FORCE_MIGRATE_QUEUE=4`` - **Force re-migration of blob links**: ``FORCE_MIGRATE_BLOBS=4`` - **Force re-migration of a specific account**: ``FORCE_MIGRATE_ACCOUNT=`` - **Force re-migration of all data**: ``FORCE_MIGRATE=4`` Use these options with care and only when necessary. ================================================ FILE: api/v1/openapi.yml ================================================ # SPDX-FileCopyrightText: 2025 Stalwart Labs LLC # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL openapi: 3.0.0 info: title: Stalwart API version: 1.0.0 servers: - url: https://mail.example.org/api description: Sample server paths: /oauth: post: summary: Obtain OAuth token responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object properties: code: type: string permissions: type: array items: type: string version: type: string isEnterprise: type: boolean example: data: code: 4YmRFLu9Df1t4JO7Iffnuney4B8tVLAxjimdRxEg permissions: - webadmin-update - spam-filter-update - dkim-signature-get - dkim-signature-create - undelete - fts-reindex - purge-account - purge-in-memory-store - purge-data-store - purge-blob-store version: 0.11.0 isEnterprise: true "401": description: Unauthorized content: application/json: schema: type: object properties: type: type: string status: type: number title: type: string detail: type: string example: type: about:blank status: 401 title: Unauthorized detail: You have to authenticate first. requestBody: content: application/json: schema: type: object properties: type: type: string client_id: type: string redirect_uri: type: string nonce: type: string example: type: code client_id: webadmin redirect_uri: stalwart://auth nonce: ttsaXca3qx /telemetry/metrics: get: summary: Fetch Telemetry Metrics responses: "200": description: OK content: application/json: schema: type: object properties: error: type: string details: type: string reason: type: string example: error: other details: No metrics store has been defined reason: You need to configure a metrics store in order to use this feature. parameters: - name: after in: query required: false schema: type: string /telemetry/live/metrics-token: get: summary: Obtain Metrics Telemetry token responses: "200": description: OK content: application/json: schema: type: object properties: data: type: string example: data: 2GO4RahIkSAms6S00R9BRsroo97ZdYTz4QVxFCOwGrGkr7zguP0AVyTMA/iha3Vz/////w8DhZi1+ALBmLX4AndlYg== /telemetry/metrics/live: get: summary: Live Metrics responses: "200": description: OK content: {} parameters: - name: metrics in: query required: false schema: type: string - name: interval in: query required: false schema: type: number - name: token in: query required: false schema: type: string /principal: get: summary: List Principals responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object properties: items: type: array items: {} total: type: number example: data: items: [] total: 0 parameters: - name: page in: query required: false schema: type: number - name: limit in: query required: false schema: type: number - name: types in: query required: false schema: type: string post: summary: Create Principal responses: "200": description: OK content: application/json: schema: type: object properties: data: type: number example: data: 50 requestBody: content: application/json: schema: type: object properties: type: type: string quota: type: number name: type: string description: type: string secrets: type: array items: {} emails: type: array items: {} urls: type: array items: {} memberOf: type: array items: {} roles: type: array items: {} lists: type: array items: {} members: type: array items: {} enabledPermissions: type: array items: {} disabledPermissions: type: array items: {} externalMembers: type: array items: {} example: type: domain quota: 0 name: example.org description: Example domain secrets: [] emails: [] urls: [] memberOf: [] roles: [] lists: [] members: [] enabledPermissions: [] disabledPermissions: [] externalMembers: [] /dkim: post: summary: Create DKIM Signature responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object nullable: true example: data: requestBody: content: application/json: schema: type: object properties: id: type: object nullable: true algorithm: type: string domain: type: string selector: type: object nullable: true example: id: algorithm: Ed25519 domain: example.org selector: /principal/{principal_id}: get: summary: Fetch Principal responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object properties: id: type: number type: type: string secrets: type: string name: type: string quota: type: number description: type: string emails: type: string roles: type: array items: type: string lists: type: array items: type: string example: data: id: 90 type: individual secrets: $6$ONjGT6nQtmPNaxw0$NNF5DXtPfOay2mfVnPJ0uQ77C.L3LNxXO/QMyphP/DzpODqbDBBGd4/gCnckYPQj3st6pqwY8/KeBsCJ.oe1Y1 name: jane quota: 0 description: Jane Doe emails: jane@example.org roles: - user lists: - all parameters: - name: principal_id in: path required: true schema: type: string patch: summary: Update Principal responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object nullable: true example: data: parameters: - name: principal_id in: path required: true schema: type: string requestBody: content: application/json: schema: type: array items: type: object properties: action: type: string field: type: string value: type: string example: - action: set field: name value: jane.doe - action: set field: description value: Jane Mary Doe - action: addItem field: emails value: jane-doe@example.org - action: removeItem field: emails value: jane@example.org delete: summary: Delete Principal responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object nullable: true example: data: parameters: - name: principal_id in: path required: true schema: type: string /queue/messages: get: summary: List Queued Messages responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object properties: items: type: array items: {} total: type: number status: type: boolean example: data: items: [] total: 0 status: true parameters: - name: page in: query required: false schema: type: number - name: max-total in: query required: false schema: type: number - name: limit in: query required: false schema: type: number - name: values in: query required: false schema: type: number patch: summary: Reschedule Queued Messages responses: "200": description: OK content: application/json: schema: type: object properties: data: type: boolean example: data: true parameters: - name: filter in: query required: false schema: type: string delete: summary: Delete Queued Messages responses: "200": description: OK content: application/json: schema: type: object properties: data: type: boolean example: data: true parameters: - name: text in: query required: false schema: type: string /queue/reports: get: summary: List Queued Reports responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object properties: items: type: array items: {} total: type: number example: data: items: [] total: 0 parameters: - name: max-total in: query required: false schema: type: number - name: limit in: query required: false schema: type: number - name: page in: query required: false schema: type: number /reports/dmarc: get: summary: List Incoming DMARC Reports responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object properties: items: type: array items: {} total: type: number example: data: items: [] total: 0 parameters: - name: max-total in: query required: false schema: type: number - name: limit in: query required: false schema: type: number - name: page in: query required: false schema: type: number /reports/tls: get: summary: List Incoming TLS Reports responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object properties: items: type: array items: {} total: type: number example: data: items: [] total: 0 parameters: - name: limit in: query required: false schema: type: number - name: max-total in: query required: false schema: type: number - name: page in: query required: false schema: type: number /reports/arf: get: summary: List Incoming ARF Reports responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object properties: items: type: array items: {} total: type: number example: data: items: [] total: 0 parameters: - name: page in: query required: false schema: type: number - name: limit in: query required: false schema: type: number - name: max-total in: query required: false schema: type: number /telemetry/traces: get: summary: List Stored Traces responses: "200": description: OK content: application/json: schema: type: object properties: error: type: string details: type: string example: error: unsupported details: No tracing store has been configured parameters: - name: type in: query required: false schema: type: string - name: page in: query required: false schema: type: number - name: limit in: query required: false schema: type: number - name: values in: query required: false schema: type: number /logs: get: summary: Quere Log Files responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object properties: items: type: array items: type: object properties: timestamp: type: string level: type: string event: type: string event_id: type: string details: type: string total: type: number example: data: items: - timestamp: "2025-01-05T14:06:29Z" level: TRACE event: HTTP request body event_id: http.request-body details: listenerId = "http", localPort = 1443, remoteIp = ::1, remotePort = 57223, contents = "", size = 0 - timestamp: "2025-01-05T14:06:29Z" level: TRACE event: Write batch operation event_id: store.data-write details: elapsed = 0ms, total = 2 - timestamp: "2025-01-05T14:06:29Z" level: TRACE event: Expression evaluation result event_id: eval.result details: listenerId = "http", localPort = 1443, remoteIp = ::1, remotePort = 57223, id = "server.http.allowed-endpoint", result = "Integer(200)" - timestamp: "2025-01-05T14:06:29Z" level: DEBUG event: HTTP request URL event_id: http.request-url details: listenerId = "http", localPort = 1443, remoteIp = ::1, remotePort = 57223, url = "/api/logs?page=1&limit=50&" - timestamp: "2025-01-05T14:06:23Z" level: TRACE event: HTTP response body event_id: http.response-body details: listenerId = "http", localPort = 1443, remoteIp = ::1, remotePort = 57223, contents = "{"error":"unsupported","details":"No tracing store has been configured"}", code = 200, size = 72 - timestamp: "2025-01-05T14:06:23Z" level: DEBUG event: Management operation not supported event_id: manage.not-supported details: listenerId = "http", localPort = 1443, remoteIp = ::1, remotePort = 57223, details = No tracing store has been configured - timestamp: "2025-01-05T14:06:23Z" level: TRACE event: HTTP request body event_id: http.request-body details: listenerId = "http", localPort = 1443, remoteIp = ::1, remotePort = 57223, contents = "", size = 0 - timestamp: "2025-01-05T14:06:23Z" level: TRACE event: Write batch operation event_id: store.data-write details: elapsed = 0ms, total = 2 - timestamp: "2025-01-05T14:06:23Z" level: TRACE event: Expression evaluation result event_id: eval.result details: listenerId = "http", localPort = 1443, remoteIp = ::1, remotePort = 57223, id = "server.http.allowed-endpoint", result = "Integer(200)" - timestamp: "2025-01-05T14:06:23Z" level: DEBUG event: HTTP request URL event_id: http.request-url details: listenerId = "http", localPort = 1443, remoteIp = ::1, remotePort = 57223, url = "/api/telemetry/traces?page=1&type=delivery.attempt-start&limit=10&values=1&" total: 100 parameters: - name: page in: query required: false schema: type: number - name: limit in: query required: false schema: type: number /spam-filter/train/spam: post: summary: Train Spam Filter as Spam responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object nullable: true example: data: requestBody: content: application/x-www-form-urlencoded: schema: type: object properties: ? "From: john@example.org\nTo: list@example.org\nSubject: Testing, please ignore\nContent-Type: text/plain; charset" : type: string example: ? "From: john@example.org\nTo: list@example.org\nSubject: Testing, please ignore\nContent-Type: text/plain; charset" : "\"utf-8\"\nContent-Transfer-Encoding: 8bit\n\nTesting 1, 2, 3\n" /spam-filter/train/ham: post: summary: Train Spam Filter as Ham responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object nullable: true example: data: requestBody: content: application/x-www-form-urlencoded: schema: type: object properties: ? "From: john@example.org\nTo: list@example.org\nSubject: Testing, please ignore\nContent-Type: text/plain; charset" : type: string example: ? "From: john@example.org\nTo: list@example.org\nSubject: Testing, please ignore\nContent-Type: text/plain; charset" : "\"utf-8\"\nContent-Transfer-Encoding: 8bit\n\nTesting 1, 2, 3\n" /spam-filter/train/spam/{account_id}: post: summary: Train Account's Spam Filter as Spam responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object nullable: true example: data: parameters: - name: account_id in: path required: true schema: type: string requestBody: content: application/x-www-form-urlencoded: schema: type: object properties: ? "From: john@example.org\nTo: list@example.org\nSubject: Testing, please ignore\nContent-Type: text/plain; charset" : type: string example: ? "From: john@example.org\nTo: list@example.org\nSubject: Testing, please ignore\nContent-Type: text/plain; charset" : "\"utf-8\"\nContent-Transfer-Encoding: 8bit\n\nTesting 1, 2, 3\n" /spam-filter/train/ham/{account_id}: post: summary: Train Account's Spam Filter as Ham responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object nullable: true example: data: parameters: - name: account_id in: path required: true schema: type: string requestBody: content: application/x-www-form-urlencoded: schema: type: object properties: ? "From: john@example.org\nTo: list@example.org\nSubject: Testing, please ignore\nContent-Type: text/plain; charset" : type: string example: ? "From: john@example.org\nTo: list@example.org\nSubject: Testing, please ignore\nContent-Type: text/plain; charset" : "\"utf-8\"\nContent-Transfer-Encoding: 8bit\n\nTesting 1, 2, 3\n" /spam-filter/classify: post: summary: Test Spam Filter Classification responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object properties: score: type: number tags: type: object properties: FROM_NO_DN: type: object properties: action: type: string value: type: number SOURCE_ASN_15169: type: object properties: action: type: string value: type: number SOURCE_COUNTRY_US: type: object properties: action: type: string value: type: number MISSING_DATE: type: object properties: action: type: string value: type: number FROMHOST_NORES_A_OR_MX: type: object properties: action: type: string value: type: number MISSING_MIME_VERSION: type: object properties: action: type: string value: type: number FORGED_SENDER: type: object properties: action: type: string value: type: number SPF_NA: type: object properties: action: type: string value: type: number X_HDR_TO: type: object properties: action: type: string value: type: number HELO_IPREV_MISMATCH: type: object properties: action: type: string value: type: number X_HDR_CONTENT_TYPE: type: object properties: action: type: string value: type: number AUTH_NA: type: object properties: action: type: string value: type: number FORGED_RECIPIENTS: type: object properties: action: type: string value: type: number RBL_SENDERSCORE_REPUT_BLOCKED: type: object properties: action: type: string value: type: number RCVD_COUNT_ZERO: type: object properties: action: type: string value: type: number X_HDR_SUBJECT: type: object properties: action: type: string value: type: number X_HDR_FROM: type: object properties: action: type: string value: type: number RCPT_COUNT_ONE: type: object properties: action: type: string value: type: number MISSING_MID: type: object properties: action: type: string value: type: number TO_DOM_EQ_FROM_DOM: type: object properties: action: type: string value: type: number ARC_NA: type: object properties: action: type: string value: type: number RCVD_TLS_LAST: type: object properties: action: type: string value: type: number X_HDR_CONTENT_TRANSFER_ENCODING: type: object properties: action: type: string value: type: number HELO_NORES_A_OR_MX: type: object properties: action: type: string value: type: number TO_DN_NONE: type: object properties: action: type: string value: type: number FROM_NEQ_ENV_FROM: type: object properties: action: type: string value: type: number DMARC_NA: type: object properties: action: type: string value: type: number SINGLE_SHORT_PART: type: object properties: action: type: string value: type: number DKIM_NA: type: object properties: action: type: string value: type: number disposition: type: object properties: action: type: string value: type: string example: data: score: 12.7 tags: FROM_NO_DN: action: allow value: 0.0 SOURCE_ASN_15169: action: allow value: 0.0 SOURCE_COUNTRY_US: action: allow value: 0.0 MISSING_DATE: action: allow value: 1.0 FROMHOST_NORES_A_OR_MX: action: allow value: 1.5 MISSING_MIME_VERSION: action: allow value: 2.0 FORGED_SENDER: action: allow value: 0.3 SPF_NA: action: allow value: 0.0 X_HDR_TO: action: allow value: 0.0 HELO_IPREV_MISMATCH: action: allow value: 1.0 X_HDR_CONTENT_TYPE: action: allow value: 0.0 AUTH_NA: action: allow value: 1.0 FORGED_RECIPIENTS: action: allow value: 2.0 RBL_SENDERSCORE_REPUT_BLOCKED: action: allow value: 0.0 RCVD_COUNT_ZERO: action: allow value: 0.1 X_HDR_SUBJECT: action: allow value: 0.0 X_HDR_FROM: action: allow value: 0.0 RCPT_COUNT_ONE: action: allow value: 0.0 MISSING_MID: action: allow value: 2.5 TO_DOM_EQ_FROM_DOM: action: allow value: 0.0 ARC_NA: action: allow value: 0.0 RCVD_TLS_LAST: action: allow value: 0.0 X_HDR_CONTENT_TRANSFER_ENCODING: action: allow value: 0.0 HELO_NORES_A_OR_MX: action: allow value: 0.3 TO_DN_NONE: action: allow value: 0.0 FROM_NEQ_ENV_FROM: action: allow value: 0.0 DMARC_NA: action: allow value: 1.0 SINGLE_SHORT_PART: action: allow value: 0.0 DKIM_NA: action: allow value: 0.0 disposition: action: allow value: "X-Spam-Result: ARC_NA (0.00),\r\n\tDKIM_NA (0.00),\r\n \tFROM_NEQ_ENV_FROM (0.00),\r\n\tFROM_NO_DN (0.00),\r\n\tRBL_SENDERSCORE_REPUT_BLOCKED (0.00),\r\n\tRCPT_COUNT_ONE (0.00),\r\n\tRCVD_TLS_LAST (0.00),\r \n\tSINGLE_SHORT_PART (0.00),\r\n\tSPF_NA (0.00),\r\n\tTO_DN_NONE (0.00),\r\n\tTO_DOM_EQ_FROM_DOM (0.00),\r\n\tRCVD_COUNT_ZERO (0.10),\r\n\tFORGED_SENDER (0.30),\r\n\tHELO_NORES_A_OR_MX (0.30),\r \n\tAUTH_NA (1.00),\r\n\tDMARC_NA (1.00),\r\n\tHELO_IPREV_MISMATCH (1.00),\r\n\tMISSING_DATE (1.00),\r\n\tFROMHOST_NORES_A_OR_MX (1.50),\r\n\tFORGED_RECIPIENTS (2.00),\r\n\tMISSING_MIME_VERSION (2.00),\r\n\tMISSING_MID (2.50)\r\nX-Spam-Status: Yes, score=12.70\r\ \n" requestBody: content: application/json: schema: type: object properties: message: type: string remoteIp: type: string ehloDomain: type: string authenticatedAs: type: object nullable: true isTls: type: boolean envFrom: type: string envFromFlags: type: number envRcptTo: type: array items: type: string example: message: "From: john@example.org\nTo: list@example.org\nSubject: Testing, please ignore\nContent-Type: text/plain; charset=\"utf-8\"\nContent-Transfer-Encoding: 8bit\n\nTesting 1, 2, 3\n" remoteIp: 8.8.8.8 ehloDomain: foo.org authenticatedAs: isTls: true envFrom: bill@foo.org envFromFlags: 0 envRcptTo: - john@example.org /troubleshoot/token: get: summary: Obtain a Troubleshooting Token responses: "200": description: OK content: application/json: schema: type: object properties: data: type: string example: data: +bS1rCUcrjoEtl9f7Vz1P6daqVs4nywxa56bHltPIASijRFrj1JrwvHxJCWphPKs/////w8E8p21+AKunrX4AndlYg== /troubleshoot/delivery/{recipient}: get: summary: Run Delivery Troubleshooting responses: "200": description: OK content: {} parameters: - name: recipient in: path required: true schema: type: string - name: token in: query required: false schema: type: string /troubleshoot/dmarc: post: summary: Run DMARC Troubleshooting responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object properties: spfEhloDomain: type: string spfEhloResult: type: object properties: type: type: string spfMailFromDomain: type: string spfMailFromResult: type: object properties: type: type: string details: type: object nullable: true ipRevResult: type: object properties: type: type: string ipRevPtr: type: array items: type: string dkimResults: type: array items: {} dkimPass: type: boolean arcResult: type: object properties: type: type: string dmarcResult: type: object properties: type: type: string dmarcPass: type: boolean dmarcPolicy: type: string elapsed: type: number example: data: spfEhloDomain: mx.google.com spfEhloResult: type: none spfMailFromDomain: google.com spfMailFromResult: type: softFail details: ipRevResult: type: pass ipRevPtr: - dns.google. dkimResults: [] dkimPass: false arcResult: type: none dmarcResult: type: none dmarcPass: false dmarcPolicy: reject elapsed: 200 requestBody: content: application/json: schema: type: object properties: remoteIp: type: string ehloDomain: type: string mailFrom: type: string body: type: string example: remoteIp: 8.8.8.8 ehloDomain: mx.google.com mailFrom: john@google.com body: "From: john@example.org\nTo: list@example.org\nSubject: Testing, please ignore\nContent-Type: text/plain; charset=\"utf-8\"\nContent-Transfer-Encoding: 8bit\n\nTesting 1, 2, 3\n" /reload: get: summary: Reload Settings responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object properties: warnings: type: object properties: {} errors: type: object properties: {} example: data: warnings: {} errors: {} parameters: - name: dry-run in: query required: false schema: type: string /update/spam-filter: get: summary: Update Spam Filter responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object nullable: true example: data: /update/webadmin: get: summary: Update WebAdmin responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object nullable: true example: data: /store/reindex: get: summary: Request FTS Reindex responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object nullable: true example: data: /store/purge/in-memory/default/bayes-global: get: summary: Delete Global Bayes Model responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object nullable: true example: data: /settings/keys: get: summary: List Settings by Key responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object properties: lookup.default.hostname: type: string example: data: lookup.default.hostname: mx.fr.email parameters: - name: prefixes in: query required: false schema: type: string - name: keys in: query required: false schema: type: string /settings/group: get: summary: List Settings by Group responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object properties: total: type: number items: type: array items: type: object properties: _id: type: string bind: type: string protocol: type: string example: data: total: 11 items: - _id: http bind: "[::]:1443" protocol: http - bind: "[::]:443" _id: https protocol: http tls.implicit: "true" - protocol: imap bind: "[::]:143" _id: imap - bind: "[::]:1143" tls.implicit: "false" _id: imapnotls protocol: imap proxy.override: "false" tls.override: "false" tls.enable: "false" socket.override: "false" - bind: "[::]:993" tls.implicit: "true" protocol: imap _id: imaptls - bind: "[::]:110" protocol: pop3 _id: pop3 - tls.implicit: "true" _id: pop3s protocol: pop3 bind: "[::]:995" - protocol: managesieve _id: sieve bind: "[::]:4190" - bind: "[::]:25" _id: smtp protocol: smtp - _id: submission bind: "[::]:587" protocol: smtp parameters: - name: limit in: query required: false schema: type: number - name: page in: query required: false schema: type: number - name: suffix in: query required: false schema: type: string - name: prefix in: query required: false schema: type: string /settings/list: get: summary: List Settings responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object properties: total: type: number items: type: object properties: enable: type: string format: type: string limits.entries: type: string limits.entry-size: type: string limits.size: type: string refresh: type: string retry: type: string timeout: type: string url: type: string example: data: total: 9 items: enable: "true" format: list limits.entries: "100000" limits.entry-size: "512" limits.size: "104857600" refresh: 12h retry: 1h timeout: 30s url: https://openphish.com/feed.txt parameters: - name: prefix in: query required: false schema: type: string /settings: post: summary: Update Settings responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object nullable: true example: data: requestBody: content: application/json: schema: type: array items: type: object properties: type: type: string prefix: type: string example: - type: clear prefix: spam-filter.rule.stwt_arc_signed. /account/crypto: get: summary: Obtain Encryption-at-Rest Settings responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object properties: type: type: string example: data: type: disabled post: summary: Update Encryption-at-Rest Settings responses: "200": description: OK content: application/json: schema: type: object properties: data: type: number example: data: 1 requestBody: content: application/json: schema: type: object properties: type: type: string algo: type: string certs: type: string example: type: pGP algo: Aes256 certs: "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nxsFNBGTGHwkBEADRB5EEtfsnUwgF2ZRg6h1fp2E8LNhv4lb9AWersI8KNFoWM6qx\n Bk/MfEpgILSPdW3g7PWHOxPV/hxjtStFHfbU/Ye5VvfbkU49faIPiw1V3MQJJ171\n cN6kgMnABfdixNiutDkHP4f34ABrEqexX2myOP+btxL24gI/N9UpOD5PiKTyKR7i\n GwNpi+O022rs/KvjlWR7iSJ4vk7bGFfTNHvWI6dZworey1tZoTIZ0CgvgMeB/F1q\n OOa0FvrJdNYR227RpHmICqFqTptNZ2EfdkJ6QUXW7bZ9dWgL36ds9QPJOGcG3c5i\n JebeX5YdJnniBefiWjfZElcqh/N6SqVuEwoTLyMCnMZ6gjNMn6tddwPH24kavZhT\n p6+vhTHmyq8XBqK/XEt9r+clSfg2hi5s7GO7hQV+W26xRjX7sQJY41PfzkgYJ0BM\n 6+w09X1ZO/iMjEp44t2rd3xSudwGYhlbazXbdB+OJaa3RtyjOAeFgY8OyNlODx3V\n xXLtF+104HGSL7nkpBsu6LLighSgEEF2Vok43grr0omyb1NPhWoAZhM8sT5iv5gW\n fKvB1O13c+hDc/iGTAvcrtdLLnF2Cs+6HD7r7zPPM4L6DrD1+oQt510H/oOEE5NZ\n wIS9CmBf0txqwk7n1U5V95lonaCK9nfoKeQ1fKl/tu01dCeERRbMXG2nCQARAQAB\n zRtKb2huIERvZSA8am9obkBleGFtcGxlLm9yZz7CwYcEEwEIADEWIQQWwx1eM+Aa\n o8okGzL45grMTSggxQUCZMYfCQIbAwQLCQgHBRUICQoLBRYCAwEAAAoJEPjmCsxN\n KCDFWP4QAI3eS5nPxmU0AC9/h8jeKNgjgpENroNQZKeWZQ8x4PfncDRkcbsJfT7Y\n IVZl4zw6gFKY5EoB1s1KkYJxPgYsqicmKNiR7Tnzabb3mzomU48FKaIyVCBzFUnJ\n YMroL/rm7QhoW2WWLvT+CPCPway/tA3By8Be/YOjhavJ8mf1W3rPzt87/4Vo6erf\n yzL0lN+FQmmhKfT4j42jF4SMSyyC2yzvfC7PT49u+KUKQm/LpQsfKHpwXZ/VI6+X\n GtZjTqsc+uglJYRo69oosImLzieA/ST1ltjmUutZQOSvlQFpDUEFrMej8XZ0qsrf\n 0gP2iwxyl0vkhV8c6wO6CacDHPivvQEHed9H1PNGn3DBfKb7Mq/jado2DapRtJg3\n 2OH0F0HTvQ0uNKl30xMUcwGQB0cKOlaFtksZT1LsosQPhtPLpFy1TuWaXOInpQLq\n JmNVcTbydOsCKq0mb6bgGcvhElC1q39tclKP3rOEDOnJ8hE6wYNaMGrt6WSKr3Tt\n h52M6KwTXOuMAecMvpDBSS3UFEVQ+T5puzInDTkjINxmj23ip+swA1x3HH2IgNrO\n VJ7O20oEf0+qC47R5rTRUxrvh/U0U3DRE5xt2J2T3xetFDT2mnQv0jcyMg/UlXXv\n GpGVfwNkvN0Cxmb1tFiBNLKCcPVizxq4MLrwx+MVfQBaRCwjJrUszsFNBGTGHwoB\n EACr5lA+j5pH0Er6Q76btbS4q9JgNjDNrjKJwX9brdBY1oXIUeBqCW9ekoqDTFpn\n xA5EFGJvPO++/0ZCa+zXE4IAcXS9+I9HVBouenPYBLETnXK0Phws+OCLoe0cAIvG\n e9Xo9VrHcGXCs9tJruVSAW3NF04YejHmnHNfEuD8mbaUdxVn5zc23w/2gLaY/ABL\n ZfNV8XZw0jBVBm3YXS3Ob3uIO+RvsNqBgnhGYN/C51QI9hdxXWUDlD1vdRacXmcI\n LDCYC3w6u8caxL0ktXTS4zwN+hEu7jHxBNiKcovCeIF5VZ5NcPpp6+6Y+vNdmmXw\n +lWNwAzj3ah6iu+y25LKSsz+7IkCh5liOwwYohO+YI7SjtTD+gL9HiHYAIO+PtBh\n 7GudmUwFoARu/q54hE4ThpzkeOzJzPqGkM/CzmwdKKM3u81ze+72ptJOqVKbFEsQ\n 3+RURrIAfyYyeJj4VVCfHNzrRRVpARZc9hJm1AXefxPnDN9dxbikjQgbg5UxrKaJ\n cjVU+go5CH5lg2D1LRGfKqTJtfiWFPjtztNgMp/SeslkhhFXsyJ0RJDcU8VfRBrO\n DBnZvPnZi4nLaWCL1LdHA8Y9EJgSwVOsfdRqL/Xk9qxqgl5R8m8lsNKZN2EYkfMN\n 4Vd+/8UBbmibHYoGIQi7UlNSPthc0XQcRzFen+3H4sg5kQARAQABwsF2BBgBCAAg\n FiEEFsMdXjPgGqPKJBsy+OYKzE0oIMUFAmTGHwsCGwwACgkQ+OYKzE0oIMXn4hAA\n lUWeF7tDdyENsOYyhsbtLIuLipYe6orHFY5m68NNOoLWwqEeTvutJgFeDT4WxYi0\n PJaNQYFPyGVyg7N0hCx5cGwajdnwGpb5zpSNyvG2Yes9I1O/u7+FFrbSwOuo61t1\n scGa8YlgTKoyGc9cwxl5U8krrlEwXTWQ/qF1Gq2wHG23wm1D2d2PXFDRvw3gPxJn\n yWkrx5k26ru1kguM7XFVyRi7B+uG4vdvMlxMBXM3jpH1CJRr82VvzYPv7f05Z5To\n C7XDqHpWKx3+AQvh/ZsSBpBhzK8qaixysMwnawe05rOPydWvsLlnMCGManKVnq9Y\n Wek1P2dwYT9zuroBR5nmrECY+xVWk7vhsDasKsYlQ/LdDyzSL7qh0Vq3DjcoHxLI\n uL7qQ3O0YRcKGfmQibpKdDzvIqA+48Nfh2nDnTxvfuwOxb41zdLTZQftaSXc0Xwd\n HgquBAFbRDr5TyWlUUc8iACowKkk01pEPc8coxPCp6F/hz6kgmebRevzs7sxwrS7\n aUWycSls783JC7WO267DRD30FNx+9S7SY4ECzhDGjLdne6wIoib1L9SFkk1AAKb3\n m2+6BB/HxCXtMqi95pFeCjV99bp+PBqoifx9SlFYZq9qcGDr/jyrdG8V2Wf/HF4n\n K8RIPxB+daAPMLTpj4WBhNquSE6mRQvABEf0GPi2eLA=\n=0TDv\n-----END PGP PUBLIC KEY BLOCK-----\n\n\n" /account/auth: get: summary: Obtain Account Authentication Settings responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object properties: otpEnabled: type: boolean appPasswords: type: array items: {} example: data: otpEnabled: false appPasswords: [] post: summary: Update Account Authentication Settings responses: "200": description: OK content: application/json: schema: type: object properties: error: type: string details: type: string reason: type: object nullable: true example: error: other details: Fallback administrator accounts do not support 2FA or AppPasswords reason: "401": description: Unauthorized content: application/json: schema: type: object properties: type: type: string status: type: number title: type: string detail: type: string example: type: about:blank status: 401 title: Unauthorized detail: You have to authenticate first. requestBody: content: application/json: schema: type: array items: type: object properties: type: type: string name: type: string password: type: string example: - type: addAppPassword name: dGVzdCQyMDI1LTAxLTA1VDE0OjEyOjUxLjg0NyswMDowMA== password: $6$4M/5LmG7b13r0cdE$6zb.i6wJ3pAQHA2MRHkKg0t8bgSYb2IeqiIU115t.NugwW6VXifE0VKI5n2BQUNwdeDMUzaX82TmhuVVgC0Gx1 /reload/: get: summary: Reload Settings responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object properties: warnings: type: object properties: {} errors: type: object properties: {} example: data: warnings: {} errors: {} /queue/status/stop: patch: summary: Stop Queue Processing responses: "200": description: OK content: application/json: schema: type: object properties: data: type: boolean example: data: true /queue/status/start: patch: summary: Resume Queue Processing responses: "200": description: OK content: application/json: schema: type: object properties: data: type: boolean example: data: false /queue/messages/{message_id}: get: summary: Obtain Queued Message Details responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object properties: id: type: number return_path: type: string domains: type: array items: type: object properties: name: type: string status: type: string recipients: type: array items: type: object properties: address: type: string status: type: string retry_num: type: number next_retry: type: string next_notify: type: string expires: type: string created: type: string size: type: number blob_hash: type: string example: data: id: 217700302698266624 return_path: pepe@pepe.com domains: - name: example.org status: scheduled recipients: - address: john@example.org status: scheduled retry_num: 0 next_retry: "2025-01-05T14:33:15Z" next_notify: "2025-01-06T14:33:15Z" expires: "2025-01-10T14:33:15Z" created: "2025-01-05T14:33:15Z" size: 1451 blob_hash: ykrZ_KghvdG2AdjH4AZajkSvZvcsxP_oI2HEZvw-tS0 "404": description: Not Found content: application/json: schema: type: object properties: type: type: string status: type: number title: type: string detail: type: string example: type: about:blank status: 404 title: Not Found detail: The requested resource does not exist on this server. parameters: - name: message_id in: path required: true schema: type: string patch: summary: Reschedule Delivery of Queued Message responses: "200": description: OK content: application/json: schema: type: object properties: data: type: boolean example: data: true parameters: - name: message_id in: path required: true schema: type: string delete: summary: Cancel Delivery of Queued Message responses: "200": description: OK content: application/json: schema: type: object properties: data: type: boolean example: data: true parameters: - name: message_id in: path required: true schema: type: string /store/blobs/{blob_id}: get: summary: Fetch Blob by ID responses: "200": description: OK content: {} parameters: - name: blob_id in: path required: true schema: type: string - name: limit in: query required: false schema: type: number /telemetry/trace/{trace_id}: get: summary: Obtain Trace Details responses: "200": description: OK content: application/json: schema: type: object properties: data: type: array items: type: object properties: text: type: string details: type: string createdAt: type: string type: type: string data: type: object properties: listenerId: type: string localPort: type: number remoteIp: type: string remotePort: type: number example: data: - text: SMTP connection started details: A new SMTP connection was started createdAt: "2025-01-05T14:34:50Z" type: smtp.connection-start data: listenerId: smtp localPort: 25 remoteIp: ::1 remotePort: 57513 - text: SMTP EHLO command details: The remote server sent an EHLO command createdAt: "2025-01-05T14:34:50Z" type: smtp.ehlo data: domain: test.eml - text: SPF EHLO check failed details: EHLO identity failed SPF check createdAt: "2025-01-05T14:34:50Z" type: smtp.spf-ehlo-fail data: domain: test.eml result: type: spf.none text: No SPF record details: No SPF record was found data: {} elapsed: 24 - text: IPREV check passed details: Reverse IP check passed createdAt: "2025-01-05T14:34:50Z" type: smtp.iprev-pass data: domain: test.eml result: type: iprev.pass text: IPREV check passed details: The IPREV check has passed data: details: - localhost. elapsed: 0 - text: SPF From check failed details: MAIL FROM identity failed SPF check createdAt: "2025-01-05T14:34:50Z" type: smtp.spf-from-fail data: domain: test.eml from: pepe@pepe.com result: type: spf.none text: No SPF record details: No SPF record was found data: {} elapsed: 18 - text: SMTP MAIL FROM command details: The remote client sent a MAIL FROM command createdAt: "2025-01-05T14:34:50Z" type: smtp.mail-from data: from: pepe@pepe.com - text: SMTP RCPT TO command details: The remote client sent an RCPT TO command createdAt: "2025-01-05T14:34:50Z" type: smtp.rcpt-to data: to: john@example.org - text: DKIM verification failed details: Failed to verify DKIM signature createdAt: "2025-01-05T14:34:50Z" type: smtp.dkim-fail data: strict: false result: [] elapsed: 0 - text: ARC verification passed details: Successful ARC verification createdAt: "2025-01-05T14:34:50Z" type: smtp.arc-pass data: strict: false result: type: dkim.none text: No DKIM signature details: No DKIM signature was found data: {} elapsed: 0 - text: DMARC check failed details: Failed to verify DMARC policy createdAt: "2025-01-05T14:34:50Z" type: smtp.dmarc-fail data: strict: false domain: example.org policy: reject result: type: dmarc.none text: No DMARC record details: No DMARC record was found data: {} elapsed: 0 parameters: - name: trace_id in: path required: true schema: type: string /telemetry/live/tracing-token: get: summary: Request a Tracing Token responses: "200": description: OK content: application/json: schema: type: object properties: data: type: string example: data: VLxkixOwgDF8Frj0wi8kPhx3SpzKqtsDvbo25wgKw2tBIz/O8La0dwioQw9pN11c/////w8Ctau1+ALxq7X4AndlYg== /telemetry/traces/live: get: summary: Start Live Tracing responses: "200": description: OK content: {} parameters: - name: filter in: query required: false schema: type: string - name: token in: query required: false schema: type: string /dns/records/{domain}: get: summary: Obtain DNS Records for Domain responses: "200": description: OK content: application/json: schema: type: object properties: data: type: array items: type: object properties: type: type: string name: type: string content: type: string example: data: - type: MX name: example.org. content: 10 mx.fr.email. - type: CNAME name: mail.example.org. content: mx.fr.email. - type: TXT name: 202501e._domainkey.example.org. content: v=DKIM1; k=ed25519; h=sha256; p=82LqzMGRHEBI2HGDogjojWGz+Crrv0TAi8pcaOBd1vw= - type: TXT name: 202501r._domainkey.example.org. content: v=DKIM1; k=rsa; h=sha256; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1qtCbIlrZffIqm7gHqpihPUlxOq1zD6K3j1RO/enhkZRp5dEdCqcLbyFk5d+rqRsVIWwUZiU4HXHWqMTN1hlKojUlzmU1JYtlHRMwtM5vN4mzG4x1KA0i8ZHxkahE8ITsP+kPByDF9x0vAySHXpyErNXq3BeFyu/VW+6X+fmUW6x39PfWq7kQQTcwU0Ogo447oJfmAX9H4Z+/cD5WJVNiLgvLY6faVgoXm0mJJjRU5xoEStXoUcKwrwbl7G3K7JfxtmWsgEn97auV6v4he2LRRfTxbY9smkqUtcJs61E9iyyYroJv0iRda2pv71qg8e4wTb2sqBloZv/F2FZQhM+wIDAQAB - type: TXT name: example.org. content: v=spf1 mx ra=postmaster -all - type: SRV name: _jmap._tcp.example.org. content: 0 1 443 mx.fr.email. - type: SRV name: _imaps._tcp.example.org. content: 0 1 993 mx.fr.email. - type: SRV name: _imap._tcp.example.org. content: 0 1 143 mx.fr.email. - type: SRV name: _imap._tcp.example.org. content: 0 1 1143 mx.fr.email. - type: SRV name: _pop3s._tcp.example.org. content: 0 1 995 mx.fr.email. parameters: - name: domain in: path required: true schema: type: string /store/purge/account/{account_id}: get: summary: Purge Account responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object nullable: true example: data: parameters: - name: account_id in: path required: true schema: type: string /store/purge/in-memory/default/bayes-account/{account_id}: get: summary: Delete Bayes Model for Account responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object nullable: true example: data: parameters: - name: account_id in: path required: true schema: type: string /store/undelete/{account_id}: get: summary: List Deleted Messages responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object properties: items: type: array items: {} total: type: number example: data: items: [] total: 0 parameters: - name: account_id in: path required: true schema: type: string - name: limit in: query required: false schema: type: number - name: page in: query required: false schema: type: number post: summary: Undelete Messages responses: "200": description: OK content: application/json: schema: type: object properties: data: type: array items: type: object properties: type: type: string example: data: - type: success parameters: - name: account_id in: path required: true schema: type: string requestBody: content: application/json: schema: type: array items: type: object properties: hash: type: string collection: type: string restoreTime: type: string cancelDeletion: type: string example: - hash: 9pDYGrkDlLYuBNl062qhi0wStnDYyq4ZWalnj2vXbLY collection: email restoreTime: "2025-01-05T14:50:13Z" cancelDeletion: "2025-02-04T14:50:13Z" /queue/status: get: summary: Obtain Queue Status responses: "200": description: OK content: application/json: schema: type: object properties: data: type: boolean example: data: true /store/purge/blob: get: summary: Purge Blob Store responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object nullable: true example: data: /store/purge/data: get: summary: Purge Data Store responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object nullable: true example: data: /store/purge/in-memory: get: summary: Purge In-Memory Store responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object nullable: true example: data: /store/purge/account: get: summary: Purge All Accounts responses: "200": description: OK content: application/json: schema: type: object properties: data: type: object nullable: true example: data: /store/uids/{account_id}: delete: summary: Reset IMAP UIDs for Account responses: "200": description: OK content: application/json: schema: type: object properties: data: type: array items: type: number example: data: - 0 - 0 parameters: - name: account_id in: path required: true schema: type: string ================================================ FILE: crates/cli/Cargo.toml ================================================ [package] name = "stalwart-cli" description = "Stalwart Server CLI" authors = ["Stalwart Labs LLC "] license = "AGPL-3.0-only OR LicenseRef-SEL" repository = "https://github.com/stalwartlabs/cli" homepage = "https://github.com/stalwartlabs/cli" version = "0.15.5" edition = "2024" readme = "README.md" [dependencies] jmap-client = { version = "0.3", features = ["async"] } mail-parser = { version = "0.11", features = ["full_encoding", "serde"] } reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-webpki-roots", "http2"]} tokio = { version = "1.47", features = ["full"] } num_cpus = "1.13.1" clap = { version = "4.1.6", features = ["derive"] } prettytable-rs = "0.10.0" rpassword = "7.0" indicatif = "0.17.0" console = { version = "0.15", default-features = false, features = ["ansi-parsing"] } serde = { version = "1.0", features = ["derive"]} serde_json = "1.0" csv = "1.1" form_urlencoded = "1.1.0" human-size = "0.4.2" futures = "0.3.28" pwhash = "1.0.0" rand = "0.9.0" mail-auth = { version = "0.7.1" } ================================================ FILE: crates/cli/src/main.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ collections::HashMap, fmt::Display, io::{BufRead, Write}, time::Duration, }; use clap::Parser; use console::style; use jmap_client::client::Credentials; use modules::{ UnwrapResult, cli::{Cli, Client, Commands}, host, is_localhost, }; use reqwest::{Method, StatusCode, header::AUTHORIZATION}; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use crate::modules::OAuthResponse; pub mod modules; #[tokio::main] async fn main() -> std::io::Result<()> { let args = Cli::parse(); let url = args .url .or_else(|| std::env::var("URL").ok()) .map(|url| url.trim_end_matches('/').to_string()) .unwrap_or_else(|| { eprintln!("No URL specified. Use --url or set the URL environment variable."); std::process::exit(1); }); let client = Client { credentials: if let Some(credentials) = args.credentials { parse_credentials(&credentials) } else if let Ok(credentials) = std::env::var("CREDENTIALS") { parse_credentials(&credentials) } else if args.anonymous { let credentials = "anonymous:".to_string(); parse_credentials(&credentials) } else { let credentials = rpassword::prompt_password( "\nEnter administrator credentials or press [ENTER] to use OAuth: ", ) .unwrap(); if !credentials.is_empty() { parse_credentials(&credentials) } else { oauth(&url).await } }, timeout: args.timeout, url, }; match args.command { Commands::Import(command) => { command.exec(client).await; } Commands::Export(command) => { command.exec(client).await; } Commands::Server(command) => command.exec(client).await, /*Commands::Account(command) => command.exec(client).await, Commands::Domain(command) => command.exec(client).await, Commands::List(command) => command.exec(client).await, Commands::Group(command) => command.exec(client).await,*/ Commands::Dkim(command) => command.exec(client).await, Commands::Queue(command) => command.exec(client).await, Commands::Report(command) => command.exec(client).await, } Ok(()) } fn parse_credentials(credentials: &str) -> Credentials { if let Some((account, secret)) = credentials.split_once(':') { Credentials::basic(account, secret) } else { Credentials::basic("admin", credentials) } } async fn oauth(url: &str) -> Credentials { let metadata: HashMap = serde_json::from_slice( &reqwest::Client::builder() .danger_accept_invalid_certs(is_localhost(url)) .build() .unwrap_or_default() .get(format!("{}/.well-known/oauth-authorization-server", url)) .send() .await .unwrap_result("send OAuth GET request") .bytes() .await .unwrap_result("fetch bytes"), ) .unwrap_result("deserialize OAuth GET response"); let token_endpoint = metadata.property("token_endpoint"); let mut params: HashMap = HashMap::from_iter([("client_id".to_string(), "Stalwart_CLI".to_string())]); let response: HashMap = serde_json::from_slice( &reqwest::Client::builder() .danger_accept_invalid_certs(is_localhost(url)) .build() .unwrap_or_default() .post(metadata.property("device_authorization_endpoint")) .form(¶ms) .send() .await .unwrap_result("send OAuth POST request") .bytes() .await .unwrap_result("fetch bytes"), ) .unwrap_result("deserialize OAuth POST response"); params.insert( "grant_type".to_string(), "urn:ietf:params:oauth:grant-type:device_code".to_string(), ); params.insert( "device_code".to_string(), response.property("device_code").to_string(), ); print!( "\nAuthenticate this request using code {} at {}. Please ENTER when done.", style(response.property("user_code")).bold(), style(response.property("verification_uri")).bold().dim() ); std::io::stdout().flush().unwrap(); std::io::stdin().lock().lines().next(); let mut response: HashMap = serde_json::from_slice( &reqwest::Client::builder() .danger_accept_invalid_certs(is_localhost(url)) .build() .unwrap_or_default() .post(token_endpoint) .form(¶ms) .send() .await .unwrap_result("send OAuth POST request") .bytes() .await .unwrap_result("fetch bytes"), ) .unwrap_result("deserialize OAuth POST response"); if let Some(serde_json::Value::String(access_token)) = response.remove("access_token") { Credentials::Bearer(access_token) } else { eprintln!( "OAuth failed with code {}.", response .get("error") .and_then(|s| s.as_str()) .unwrap_or("") ); std::process::exit(1); } } #[derive(Deserialize)] #[serde(untagged)] pub enum Response { Error(ManagementApiError), Data { data: T }, } #[derive(Deserialize)] #[serde(tag = "error")] #[serde(rename_all = "camelCase")] pub enum ManagementApiError { FieldAlreadyExists { field: String, value: String }, FieldMissing { field: String }, NotFound { item: String }, Unsupported { details: String }, AssertFailed, Other { details: String }, } impl Client { pub async fn into_jmap_client(self) -> jmap_client::client::Client { jmap_client::client::Client::new() .credentials(self.credentials) .accept_invalid_certs(is_localhost(&self.url)) .follow_redirects([host(&self.url).expect("Invalid host").to_owned()]) .timeout(Duration::from_secs(self.timeout.unwrap_or(60))) .connect(&self.url) .await .unwrap_or_else(|err| { eprintln!("Failed to connect to JMAP server {}: {}.", &self.url, err); std::process::exit(1); }) } pub async fn http_request( &self, method: Method, url: &str, body: Option, ) -> R { self.try_http_request(method, url, body) .await .unwrap_or_else(|| { eprintln!("Request failed: No data returned."); std::process::exit(1); }) } pub async fn try_http_request( &self, method: Method, url: &str, body: Option, ) -> Option { let url = format!( "{}{}{}", self.url, if !self.url.ends_with('/') && !url.starts_with('/') { "/" } else { "" }, url ); let mut request = reqwest::Client::builder() .danger_accept_invalid_certs(is_localhost(&url)) .timeout(Duration::from_secs(self.timeout.unwrap_or(60))) .build() .unwrap_or_default() .request(method, url) .header( AUTHORIZATION, match &self.credentials { Credentials::Basic(s) => format!("Basic {s}"), Credentials::Bearer(s) => format!("Bearer {s}"), }, ); if let Some(body) = body { request = request.body(serde_json::to_string(&body).unwrap_result("serialize body")); } let response = request.send().await.unwrap_result("send HTTP request"); match response.status() { StatusCode::OK => (), StatusCode::NOT_FOUND => { return None; } StatusCode::UNAUTHORIZED => { eprintln!( "Authentication failed. Make sure the credentials are correct and that the account has administrator rights." ); std::process::exit(1); } _ => { eprintln!( "Request failed: {}", response.text().await.unwrap_result("fetch text") ); std::process::exit(1); } } let bytes = response.bytes().await.unwrap_result("fetch bytes"); match serde_json::from_slice::>(&bytes).unwrap_result(&format!( "deserialize response {}", String::from_utf8_lossy(bytes.as_ref()) )) { Response::Data { data } => Some(data), Response::Error(error) => { eprintln!("Request failed: {error})"); std::process::exit(1); } } } } impl Display for ManagementApiError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { ManagementApiError::FieldAlreadyExists { field, value } => { write!(f, "Field {} already exists with value {}.", field, value) } ManagementApiError::FieldMissing { field } => { write!(f, "Field {} is missing.", field) } ManagementApiError::NotFound { item } => { write!(f, "{} not found.", item) } ManagementApiError::Unsupported { details } => { write!(f, "Unsupported: {}", details) } ManagementApiError::AssertFailed => { write!(f, "Assertion failed.") } ManagementApiError::Other { details } => { write!(f, "{}", details) } } } } ================================================ FILE: crates/cli/src/modules/account.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::fmt::Display; use prettytable::{Attr, Cell, Row, Table}; use pwhash::sha512_crypt; use reqwest::Method; use serde_json::Value; use super::{ Principal, PrincipalField, PrincipalUpdate, PrincipalValue, Type, cli::{AccountCommands, Client}, }; impl AccountCommands { pub async fn exec(self, client: Client) { match self { AccountCommands::Create { name, password, description, quota, is_admin, addresses, member_of, } => { let principal = Principal { typ: if is_admin.unwrap_or_default() { Type::Superuser } else { Type::Individual } .into(), quota, name: name.clone().into(), secrets: vec![sha512_crypt::hash(password).unwrap()], emails: addresses.unwrap_or_default(), member_of: member_of.unwrap_or_default(), description, ..Default::default() }; let account_id = client .http_request::(Method::POST, "/api/principal", Some(principal)) .await; eprintln!("Successfully created account {name:?} with id {account_id}."); } AccountCommands::Update { name, new_name, password, description, quota, is_admin, addresses, member_of, } => { let mut changes = Vec::new(); if let Some(new_name) = new_name { changes.push(PrincipalUpdate::set( PrincipalField::Name, PrincipalValue::String(new_name), )); } if let Some(password) = password { changes.push(PrincipalUpdate::add_item( PrincipalField::Secrets, PrincipalValue::String(sha512_crypt::hash(password).unwrap()), )); } if let Some(description) = description { changes.push(PrincipalUpdate::set( PrincipalField::Description, PrincipalValue::String(description), )); } if let Some(quota) = quota { changes.push(PrincipalUpdate::set( PrincipalField::Quota, PrincipalValue::Integer(quota), )); } if let Some(is_admin) = is_admin { changes.push(PrincipalUpdate::set( PrincipalField::Type, PrincipalValue::String( if is_admin { Type::Superuser } else { Type::Individual } .to_string() .to_ascii_lowercase(), ), )); } if let Some(addresses) = addresses { changes.push(PrincipalUpdate::set( PrincipalField::Emails, PrincipalValue::StringList(addresses), )); } if let Some(member_of) = member_of { changes.push(PrincipalUpdate::set( PrincipalField::MemberOf, PrincipalValue::StringList(member_of), )); } if !changes.is_empty() { client .http_request::( Method::PATCH, &format!("/api/principal/{name}"), Some(changes), ) .await; eprintln!("Successfully updated account {name:?}."); } else { eprintln!("No changes to apply."); } } AccountCommands::AddEmail { name, addresses } => { client .http_request::( Method::PATCH, &format!("/api/principal/{name}"), Some( addresses .into_iter() .map(|address| { PrincipalUpdate::add_item( PrincipalField::Emails, PrincipalValue::String(address), ) }) .collect::>(), ), ) .await; eprintln!("Successfully updated account {name:?}."); } AccountCommands::RemoveEmail { name, addresses } => { client .http_request::( Method::PATCH, &format!("/api/principal/{name}"), Some( addresses .into_iter() .map(|address| { PrincipalUpdate::remove_item( PrincipalField::Emails, PrincipalValue::String(address), ) }) .collect::>(), ), ) .await; eprintln!("Successfully updated account {name:?}."); } AccountCommands::AddToGroup { name, member_of } => { client .http_request::( Method::PATCH, &format!("/api/principal/{name}"), Some( member_of .into_iter() .map(|group| { PrincipalUpdate::add_item( PrincipalField::MemberOf, PrincipalValue::String(group), ) }) .collect::>(), ), ) .await; eprintln!("Successfully updated account {name:?}."); } AccountCommands::RemoveFromGroup { name, member_of } => { client .http_request::( Method::PATCH, &format!("/api/principal/{name}"), Some( member_of .into_iter() .map(|group| { PrincipalUpdate::remove_item( PrincipalField::MemberOf, PrincipalValue::String(group), ) }) .collect::>(), ), ) .await; eprintln!("Successfully updated account {name:?}."); } AccountCommands::Delete { name } => { client .http_request::( Method::DELETE, &format!("/api/principal/{name}"), None, ) .await; eprintln!("Successfully deleted account {name:?}."); } AccountCommands::Display { name } => { client.display_principal(&name).await; } AccountCommands::List { filter, limit, page, } => { client .list_principals("individual", "Account", filter, page, limit) .await; } } } } impl Client { pub async fn display_principal(&self, name: &str) { let principal = self .http_request::(Method::GET, &format!("/api/principal/{name}"), None) .await; let mut table = Table::new(); if let Some(name) = principal.name { table.add_row(Row::new(vec![ Cell::new("Name").with_style(Attr::Bold), Cell::new(&name), ])); } if let Some(typ) = principal.typ { table.add_row(Row::new(vec![ Cell::new("Type").with_style(Attr::Bold), Cell::new(&typ.to_string()), ])); } if let Some(description) = principal.description { table.add_row(Row::new(vec![ Cell::new("Description").with_style(Attr::Bold), Cell::new(&description), ])); } if matches!( principal.typ, Some(Type::Individual | Type::Superuser | Type::Group) ) { if let Some(quota) = principal.quota { table.add_row(Row::new(vec![ Cell::new("Quota").with_style(Attr::Bold), if quota != 0 { Cell::new("a.to_string()) } else { Cell::new("Unlimited") }, ])); } if let Some(used_quota) = principal.used_quota { table.add_row(Row::new(vec![ Cell::new("Used Quota").with_style(Attr::Bold), Cell::new(&used_quota.to_string()), ])); } } if !principal.members.is_empty() { table.add_row(Row::new(vec![ Cell::new("Members").with_style(Attr::Bold), Cell::new(&principal.members.join(", ")), ])); } if !principal.member_of.is_empty() { table.add_row(Row::new(vec![ Cell::new("Member of").with_style(Attr::Bold), Cell::new(&principal.member_of.join(", ")), ])); } if !principal.emails.is_empty() { table.add_row(Row::new(vec![ Cell::new("E-mail address(es)").with_style(Attr::Bold), Cell::new(&principal.emails.join(", ")), ])); } eprintln!(); table.printstd(); eprintln!(); } pub async fn list_principals( &self, record_type: &str, record_name: &str, filter: Option, page: Option, limit: Option, ) { let mut query = form_urlencoded::Serializer::new("/api/principal?".to_string()); query.append_pair("type", record_type); if let Some(filter) = &filter { query.append_pair("filter", filter); } if let Some(limit) = limit { query.append_pair("limit", &limit.to_string()); } if let Some(page) = page { query.append_pair("page", &page.to_string()); } let results = self .http_request::(Method::GET, &query.finish(), None) .await; if !results.items.is_empty() { let mut table = Table::new(); table.add_row(Row::new(vec![ Cell::new(&format!("{record_name} Name")).with_style(Attr::Bold), ])); for item in &results.items { table.add_row(Row::new(vec![Cell::new(item)])); } eprintln!(); table.printstd(); eprintln!(); } eprintln!( "\n\n{} {}{} found.\n", results.total, record_name.to_ascii_lowercase(), if results.total == 1 { "" } else { "s" } ); } } #[derive(Debug, serde::Deserialize)] struct ListResponse { pub total: usize, pub items: Vec, } impl Display for Type { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Type::Superuser => write!(f, "Superuser"), Type::Individual => write!(f, "Individual"), Type::Group => write!(f, "Group"), Type::List => write!(f, "List"), Type::Resource => write!(f, "Resource"), Type::Location => write!(f, "Location"), Type::Other => write!(f, "Other"), } } } ================================================ FILE: crates/cli/src/modules/cli.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::dkim::Algorithm; use clap::{Parser, Subcommand, ValueEnum}; use jmap_client::client::Credentials; use mail_parser::DateTime; use serde::Deserialize; #[derive(Parser)] #[clap(version, about, long_about = None)] #[clap(name = "stalwart-cli")] pub struct Cli { #[clap(subcommand)] pub command: Commands, /// Server base URL #[clap(short, long)] pub url: Option, /// Authentication credentials #[clap(short, long)] pub credentials: Option, /// Connection timeout in seconds #[clap(short, long)] pub timeout: Option, /// Do not ask for credentials #[clap(short, long)] pub anonymous: bool, } #[derive(Subcommand)] pub enum Commands { /// Manage user accounts /* #[clap(subcommand)] Account(AccountCommands), /// Manage domains #[clap(subcommand)] Domain(DomainCommands), /// Manage mailing lists #[clap(subcommand)] List(ListCommands), /// Manage groups #[clap(subcommand)] Group(GroupCommands), */ /// Manage DKIM signatures #[clap(subcommand)] Dkim(DkimCommands), /// Import JMAP accounts and Maildir/mbox mailboxes #[clap(subcommand)] Import(ImportCommands), /// Export JMAP accounts #[clap(subcommand)] Export(ExportCommands), /// Manage JMAP database #[clap(subcommand)] Server(ServerCommands), /// Manage SMTP message queue #[clap(subcommand)] Queue(QueueCommands), /// Manage SMTP DMARC/TLS report queue #[clap(subcommand)] Report(ReportCommands), } pub struct Client { pub url: String, pub credentials: Credentials, pub timeout: Option, } #[derive(Subcommand)] pub enum AccountCommands { /// Create a new user account Create { /// Login Name name: String, /// Password password: String, /// Account description #[clap(short, long)] description: Option, /// Quota in bytes #[clap(short, long)] quota: Option, /// Whether the account is an administrator #[clap(short, long)] is_admin: Option, /// E-mail addresses #[clap(short, long)] addresses: Option>, /// Groups this account is a member of #[clap(short, long)] member_of: Option>, }, /// Update an existing user account Update { /// Account login name: String, /// Rename account login #[clap(short, long)] new_name: Option, /// Update password #[clap(short, long)] password: Option, /// Update account description #[clap(short, long)] description: Option, /// Update quota in bytes #[clap(short, long)] quota: Option, /// Whether the account is an administrator #[clap(short, long)] is_admin: Option, /// Update e-mail addresses #[clap(short, long)] addresses: Option>, /// Update groups this account is a member of #[clap(short, long)] member_of: Option>, }, /// Add e-mail aliases to a user account AddEmail { /// Account login name: String, /// E-mail aliases to add #[clap(required = true)] addresses: Vec, }, /// Remove e-mail aliases to a user account RemoveEmail { /// Account login name: String, /// E-mail aliases to remove #[clap(required = true)] addresses: Vec, }, /// Add a user account to groups AddToGroup { /// Account login name: String, /// Groups to add #[clap(required = true)] member_of: Vec, }, /// Remove a user account from groups RemoveFromGroup { /// Account login name: String, /// Groups to remove #[clap(required = true)] member_of: Vec, }, /// Delete an existing user account Delete { /// Account name to delete name: String, }, /// Display an existing user account Display { /// Account name to display name: String, }, /// List all user accounts List { /// Filter accounts by keywords filter: Option, /// Maximum number of accounts to list limit: Option, /// Page number page: Option, }, } #[derive(Subcommand)] pub enum ListCommands { /// Create a new mailing list Create { /// List Name name: String, /// List email address email: String, /// Description #[clap(short, long)] description: Option, /// Mailing list members #[clap(short, long)] members: Option>, }, /// Update an existing mailing list Update { /// List Name name: String, /// Rename list new_name: Option, /// List email address email: Option, /// Description #[clap(short, long)] description: Option, /// Mailing list members #[clap(short, long)] members: Option>, }, /// Add members to a mailing list AddMembers { /// List Name name: String, /// Members to add #[clap(required = true)] members: Vec, }, /// Remove members from a mailing list RemoveMembers { /// List Name name: String, /// Members to remove #[clap(required = true)] members: Vec, }, /// Display an existing mailing list Display { /// Mailing list to display name: String, }, /// List all mailing lists List { /// Filter mailing lists by keywords filter: Option, /// Maximum number of mailing lists to list limit: Option, /// Page number page: Option, }, } #[derive(Subcommand)] pub enum GroupCommands { /// Create a group Create { /// Group Name name: String, /// Group email address email: Option, /// Description #[clap(short, long)] description: Option, /// Group members #[clap(short, long)] members: Option>, }, /// Update an existing group Update { /// Group Name name: String, /// Rename group new_name: Option, /// Group email address email: Option, /// Description #[clap(short, long)] description: Option, /// Update groups that this group is a member of #[clap(short, long)] members: Option>, }, /// Add members to a group AddMembers { /// Group name name: String, /// Groups to add #[clap(required = true)] members: Vec, }, /// Remove members from a group RemoveMembers { /// Group name name: String, /// Groups to remove #[clap(required = true)] members: Vec, }, /// Display an existing group Display { /// Group name to display name: String, }, /// List all groups List { /// Filter groups by keywords filter: Option, /// Maximum number of groups to list limit: Option, /// Page number page: Option, }, } #[derive(Subcommand)] pub enum DomainCommands { /// Create a new domain Create { /// Domain name to create name: String, }, /// Delete an existing domain Delete { /// Domain name to delete name: String, }, /// List DNS records for domain DNSRecords { /// Domain name to list DNS records for name: String, }, /// List all domains List { /// Starting point for listing domains from: Option, /// Maximum number of domains to list limit: Option, }, } #[derive(Subcommand)] pub enum DkimCommands { /// Create DKIM signature Create { /// Algorithm to use algorithm: Algorithm, /// Domain name for which to create domain: String, /// Id signature_id: Option, /// Selector selector: Option, }, /// Get DKIM public key GetPublicKey { /// Signature id signature_id: String, }, } #[derive(Subcommand)] pub enum ImportCommands { /// Import messages and folders Messages { #[clap(value_enum)] #[clap(short, long)] format: MailboxFormat, /// Number of messages to import concurrently, defaults to the number of CPUs. #[clap(short, long)] num_concurrent: Option, /// Account name or email to import messages into account: String, /// Path to the mailbox to import, or '-' for stdin (stdin only supported for mbox) path: String, }, /// Import a JMAP account Account { /// Number of concurrent requests, defaults to the number of CPUs. #[clap(short, long)] num_concurrent: Option, /// Account name or email to import messages into account: String, /// Path to the exported account directory path: String, }, } #[derive(Subcommand)] pub enum ExportCommands { /// Export a JMAP account Account { /// Number of concurrent blob downloads to perform, defaults to the number of CPUs. #[clap(short, long)] num_concurrent: Option, /// Account name or email to import messages into account: String, /// Path to export the account to path: String, }, } #[derive(Subcommand)] pub enum ServerCommands { /// Perform database maintenance DatabaseMaintenance {}, /// Reload TLS certificates ReloadCertificates {}, /// Reload configuration ReloadConfig {}, /// Create a new configuration key AddConfig { /// Key to add key: String, /// Value to set value: Option, }, /// Delete a configuration key or prefix DeleteConfig { /// Configuration key or prefix to delete key: String, }, /// List all configuration entries ListConfig { /// Prefix to filter configuration entries by prefix: Option, }, /// Perform Healthcheck Healthcheck { /// Status `ready` (default) or `live` to check for check: Option }, } #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] pub enum MailboxFormat { /// Mbox format Mbox, /// Maildir and Maildir++ formats Maildir, /// Maildir with hierarchical folders (i.e. Dovecot) MaildirNested, } #[derive(Subcommand)] pub enum QueueCommands { /// Shows messages queued for delivery List { /// Filter by sender address #[clap(short, long)] sender: Option, /// Filter by recipient #[clap(short, long)] rcpt: Option, /// Filter messages due for delivery before a certain datetime #[clap(short, long)] #[arg(value_parser = parse_datetime)] before: Option, /// Filter messages due for delivery after a certain datetime #[clap(short, long)] #[arg(value_parser = parse_datetime)] after: Option, /// Number of items to show per page #[clap(short, long)] page_size: Option, }, /// Displays details about a queued message Status { #[clap(required = true)] ids: Vec, }, /// Reschedule delivery Retry { /// Apply to messages matching a sender address #[clap(short, long)] sender: Option, /// Apply to a specific domain #[clap(short, long)] domain: Option, /// Apply to messages due before a certain datetime #[clap(short, long)] #[arg(value_parser = parse_datetime)] before: Option, /// Apply to messages due after a certain datetime #[clap(short, long)] #[arg(value_parser = parse_datetime)] after: Option, /// Schedule delivery at a specific time #[clap(short, long)] #[arg(value_parser = parse_datetime)] time: Option, // Reschedule one or multiple message ids ids: Vec, }, /// Cancel delivery Cancel { /// Apply to messages matching a sender address #[clap(short, long)] sender: Option, /// Apply to specific recipients or domains #[clap(short, long)] rcpt: Option, /// Apply to messages due before a certain datetime #[clap(short, long)] #[arg(value_parser = parse_datetime)] before: Option, /// Apply to messages due after a certain datetime #[clap(short, long)] #[arg(value_parser = parse_datetime)] after: Option, // Cancel one or multiple message ids ids: Vec, }, } #[derive(Subcommand)] pub enum ReportCommands { /// Shows reports queued for delivery List { /// Filter by report domain #[clap(short, long)] domain: Option, /// Filter by report type #[clap(short, long)] #[clap(value_enum)] format: Option, /// Number of items to show per page #[clap(short, long)] page_size: Option, }, /// Displays details about a queued report Status { #[clap(required = true)] ids: Vec, }, /// Cancel report delivery Cancel { #[clap(required = true)] ids: Vec, }, } #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum, Deserialize)] pub enum ReportFormat { /// DMARC report #[serde(rename = "dmarc")] Dmarc, /// TLS report #[serde(rename = "tls")] Tls, } fn parse_datetime(arg: &str) -> Result { if arg.contains('T') { DateTime::parse_rfc3339(arg).ok_or("Failed to parse RFC3339 datetime") } else { DateTime::parse_rfc3339(&format!("{arg}T00:00:00Z")) .ok_or("Failed to parse RFC3339 datetime") } } ================================================ FILE: crates/cli/src/modules/database.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::collections::HashMap; use prettytable::{Attr, Cell, Row, Table}; use reqwest::{Method, StatusCode}; use serde_json::Value; use crate::modules::{Response, UnwrapResult}; use super::cli::{Client, ServerCommands}; #[derive(Debug, serde::Serialize, serde::Deserialize)] #[serde(tag = "type")] #[serde(rename_all = "camelCase")] pub enum UpdateSettings { Delete { keys: Vec, }, Clear { prefix: String, #[serde(default)] filter: Option, }, Insert { prefix: Option, values: Vec<(String, String)>, assert_empty: bool, }, } impl ServerCommands { pub async fn exec(self, client: Client) { match self { ServerCommands::DatabaseMaintenance {} => { client .http_request::(Method::GET, "/api/store/maintenance", None) .await; eprintln!("Success."); } ServerCommands::ReloadCertificates {} => { client .http_request::(Method::GET, "/api/reload/certificate", None) .await; eprintln!("Success."); } ServerCommands::ReloadConfig {} => { client .http_request::(Method::GET, "/api/reload", None) .await; eprintln!("Success."); } ServerCommands::AddConfig { key, value } => { client .http_request::( Method::POST, "/api/settings", Some(vec![UpdateSettings::Insert { prefix: None, values: vec![(key.clone(), value.unwrap_or_default())], assert_empty: false, }]), ) .await; eprintln!("Successfully added key {key}."); } ServerCommands::DeleteConfig { key } => { client .http_request::( Method::POST, "/api/settings", Some(vec![UpdateSettings::Delete { keys: vec![key.clone()], }]), ) .await; eprintln!("Successfully deleted key {key}."); } ServerCommands::ListConfig { prefix } => { let results = client .http_request::>, String>( Method::GET, &format!("/api/settings/list?prefix={}", prefix.unwrap_or_default()), None, ) .await .items; if !results.is_empty() { let mut table = Table::new(); table.add_row(Row::new(vec![ Cell::new("Key").with_style(Attr::Bold), Cell::new("Value").with_style(Attr::Bold), ])); for (key, value) in &results { table.add_row(Row::new(vec![Cell::new(key), Cell::new(value)])); } eprintln!(); table.printstd(); eprintln!(); } eprintln!( "\n\n{} key{} found.\n", results.len(), if results.len() == 1 { "" } else { "s" } ); } ServerCommands::Healthcheck { check } => { let response = reqwest::get( format!("{}/healthz/{}", client.url, check.unwrap_or("ready".to_string())) ).await; match response { Ok(resp) => { match resp.status() { StatusCode::OK => { eprintln!("Success") }, _ => { eprintln!( "Request failed: {}", resp.text().await.unwrap_result("fetch text") ); std::process::exit(1); } } } Err(err) => { eprintln!("Request failed: {}", err); std::process::exit(1); } } } } } } ================================================ FILE: crates/cli/src/modules/dkim.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::cli::{Client, DkimCommands}; use clap::ValueEnum; use reqwest::Method; use serde::{Deserialize, Serialize}; use serde_json::Value; #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ValueEnum)] pub enum Algorithm { /// RSA #[default] Rsa, /// ED25519 Ed25519, } #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize)] struct DkimSignature { #[serde(skip_serializing_if = "Option::is_none")] pub id: Option, pub algorithm: Algorithm, pub domain: String, #[serde(skip_serializing_if = "Option::is_none")] pub selector: Option, } impl DkimCommands { pub async fn exec(self, client: Client) { match self { DkimCommands::Create { signature_id, algorithm, domain, selector, } => { let signature_req = DkimSignature { id: signature_id, algorithm, domain: domain.clone(), selector, }; client .http_request::(Method::POST, "/api/dkim", Some(signature_req)) .await; eprintln!("Successfully created {algorithm:?} signature for domain {domain:?}"); } DkimCommands::GetPublicKey { signature_id } => { let response = client .http_request::( Method::GET, &format!("/api/dkim/{signature_id}"), None, ) .await; eprintln!(); eprintln!("Public DKIM key for signature {signature_id}: {response}"); eprintln!(); } } } } ================================================ FILE: crates/cli/src/modules/domain.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::borrow::Cow; use prettytable::{Attr, Cell, Row, Table, format}; use reqwest::Method; use serde_json::Value; use crate::modules::List; use super::cli::{Client, DomainCommands}; use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize, Clone)] struct DnsRecord { #[serde(rename = "type")] typ: String, name: String, content: String, } impl DomainCommands { pub async fn exec(self, client: Client) { match self { DomainCommands::Create { name } => { client .http_request::( Method::POST, &format!("/api/domain/{name}"), None, ) .await; eprintln!("Successfully created domain {name:?}"); } DomainCommands::Delete { name } => { client .http_request::( Method::DELETE, &format!("/api/domain/{name}"), None, ) .await; eprintln!("Successfully deleted domain {name:?}"); } DomainCommands::DNSRecords { name } => { let records = client .http_request::, String>( Method::GET, &format!("/api/domain/{name}"), None, ) .await; if !records.is_empty() { let mut table = Table::new(); // no borderline separator separator, as long values will mess it up table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR); table.add_row(Row::new(vec![ Cell::new("Type").with_style(Attr::Bold), Cell::new("Name").with_style(Attr::Bold), Cell::new("Contents").with_style(Attr::Bold), ])); for record in &records { table.add_row(Row::new(vec![ Cell::new(&record.typ), Cell::new(&record.name), Cell::new(&record.content), ])); } eprintln!(); table.printstd(); eprintln!(); } } DomainCommands::List { from, limit } => { let query = if from.is_none() && limit.is_none() { Cow::Borrowed("/api/domain") } else { let mut query = "/api/domain?".to_string(); if let Some(from) = &from { query.push_str(&format!("from={from}")); } if let Some(limit) = limit { query.push_str(&format!( "{}limit={limit}", if from.is_some() { "&" } else { "" } )); } Cow::Owned(query) }; let domains = client .http_request::, String>(Method::GET, query.as_ref(), None) .await; if !domains.items.is_empty() { let mut table = Table::new(); table.add_row(Row::new(vec![ Cell::new("Domain Name").with_style(Attr::Bold), ])); for domain in &domains.items { table.add_row(Row::new(vec![Cell::new(domain)])); } eprintln!(); table.printstd(); eprintln!(); } eprintln!( "\n\n{} domain{} found.\n", domains.total, if domains.total == 1 { "" } else { "s" } ); } } } } ================================================ FILE: crates/cli/src/modules/export.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ path::{Path, PathBuf}, sync::Arc, }; use futures::{StreamExt, stream::FuturesUnordered}; use jmap_client::{ email::{self, Email}, identity::{self, Identity}, mailbox::{self, Mailbox}, sieve::{self, SieveScript}, vacation_response::{self, VacationResponse}, }; use serde::Serialize; use tokio::io::AsyncWriteExt; use crate::modules::RETRY_ATTEMPTS; use super::{ UnwrapResult, cli::{Client, ExportCommands}, name_to_id, }; impl ExportCommands { pub async fn exec(self, client: Client) { let mut client = client.into_jmap_client().await; match self { ExportCommands::Account { num_concurrent, account, path, } => { client.set_default_account_id(name_to_id(&client, &account).await); let max_objects_in_get = client .session() .core_capabilities() .map(|c| c.max_objects_in_get()) .unwrap_or(500); // Create directory let mut path = PathBuf::from(path); if !path.is_dir() { eprintln!("Directory {} does not exist.", path.display()); std::process::exit(1); } path.push(&account); if !path.is_dir() { std::fs::create_dir(&path).unwrap_or_else(|_| { eprintln!("Failed to create directory: {}", path.display()); std::process::exit(1); }); } // Export metadata let mut blobs = Vec::new(); export_mailboxes(&client, max_objects_in_get, &path).await; export_emails(&client, max_objects_in_get, &mut blobs, &path).await; export_sieve_scripts(&client, max_objects_in_get, &mut blobs, &path).await; export_identities(&client, &path).await; export_vacation_responses(&client, &path).await; // Export blobs path.push("blobs"); if !path.exists() { std::fs::create_dir(&path).unwrap_or_else(|_| { eprintln!("Failed to create directory: {}", path.display()); std::process::exit(1); }); } let client = Arc::new(client); let num_concurrent = num_concurrent.unwrap_or_else(num_cpus::get); let mut futures = FuturesUnordered::new(); eprintln!("Exporting {} blobs...", blobs.len()); for blob_id in blobs { let client = client.clone(); let mut blob_path = path.clone(); blob_path.push(&blob_id); if tokio::fs::metadata(&blob_path).await.is_err() { futures.push(async move { let mut retry_count = 0; let bytes = loop { match client.download(&blob_id).await { Ok(bytes) => break bytes, Err(_) if retry_count < RETRY_ATTEMPTS => { tokio::time::sleep(std::time::Duration::from_secs(1)).await; retry_count += 1; } result => { result.unwrap_result("download blob"); return; } } }; tokio::fs::OpenOptions::new() .create(true) .write(true) .truncate(true) .open(&blob_path) .await .unwrap_result(&format!("open {}", blob_path.display())) .write_all(&bytes) .await .unwrap_result(&format!("write {}", blob_path.display())); }); } if futures.len() == num_concurrent { futures.next().await.unwrap(); } } // Wait for remaining futures while futures.next().await.is_some() {} } } } } pub async fn fetch_mailboxes( client: &jmap_client::client::Client, max_objects_in_get: usize, ) -> Vec { let mut position = 0; let mut results = Vec::new(); loop { let mut request = client.build(); let query_result = request .query_mailbox() .calculate_total(true) .position(position) .limit(max_objects_in_get) .result_reference(); request.get_mailbox().ids_ref(query_result).properties([ mailbox::Property::Id, mailbox::Property::Name, mailbox::Property::IsSubscribed, mailbox::Property::ParentId, mailbox::Property::Role, mailbox::Property::SortOrder, mailbox::Property::ShareWith, ]); let mut response = request .send() .await .unwrap_result("send JMAP request") .unwrap_method_responses(); if response.len() != 2 { eprintln!("Invalid response while fetching mailboxes"); std::process::exit(1); } let mut get_response = response .pop() .unwrap() .unwrap_get_mailbox() .unwrap_result("fetch mailboxes"); let mailboxes_part = get_response.take_list(); let total_mailboxes = response .pop() .unwrap() .unwrap_query_mailbox() .unwrap_result("query mailboxes") .total() .unwrap_or(0); let mailboxes_part_len = mailboxes_part.len(); if mailboxes_part_len > 0 { results.extend(mailboxes_part); if results.len() < total_mailboxes { position += mailboxes_part_len as i32; continue; } } break; } results } async fn export_mailboxes( client: &jmap_client::client::Client, max_objects_in_get: usize, path: &Path, ) { eprintln!( "Exported {} mailboxes.", write_file( path, "mailboxes.json", fetch_mailboxes(client, max_objects_in_get).await, ) .await ); } pub async fn fetch_emails( client: &jmap_client::client::Client, max_objects_in_get: usize, ) -> Vec { let mut position = 0; let mut results = Vec::new(); loop { let mut request = client.build(); let query_result = request .query_email() .calculate_total(true) .position(position) .limit(max_objects_in_get) .result_reference(); request.get_email().ids_ref(query_result).properties([ email::Property::Id, email::Property::MailboxIds, email::Property::Keywords, email::Property::ReceivedAt, email::Property::BlobId, email::Property::MessageId, ]); let mut response = request .send() .await .unwrap_result("send JMAP request") .unwrap_method_responses(); if response.len() != 2 { eprintln!("Invalid response while fetching emails"); std::process::exit(1); } let mut get_response = response .pop() .unwrap() .unwrap_get_email() .unwrap_result("fetch emails"); let emails_part = get_response.take_list(); let total_emails = response .pop() .unwrap() .unwrap_query_email() .unwrap_result("query emails") .total() .unwrap_or(0); let emails_part_len = emails_part.len(); if emails_part_len > 0 { results.extend(emails_part); if results.len() < total_emails { position += emails_part_len as i32; continue; } } break; } results } async fn export_emails( client: &jmap_client::client::Client, max_objects_in_get: usize, blobs: &mut Vec, path: &Path, ) { let emails = fetch_emails(client, max_objects_in_get).await; for email in &emails { if let Some(blob_id) = email.blob_id() { blobs.push(blob_id.to_string()); } else { eprintln!( "Warning: email {:?} has no blobId", email.id().unwrap_or_default() ); } } eprintln!( "Exported {} emails.", write_file(path, "emails.json", emails,).await ); } pub async fn fetch_sieve_scripts( client: &jmap_client::client::Client, max_objects_in_get: usize, ) -> Vec { let mut position = 0; let mut results = Vec::new(); loop { let mut request = client.build(); let query_result = request .query_sieve_script() .calculate_total(true) .position(position) .limit(max_objects_in_get) .result_reference(); request .get_sieve_script() .ids_ref(query_result) .properties([ sieve::Property::Id, sieve::Property::Name, sieve::Property::BlobId, sieve::Property::IsActive, ]); let mut response = request .send() .await .unwrap_result("send JMAP request") .unwrap_method_responses(); if response.len() != 2 { eprintln!("Invalid response while fetching sieve_scripts"); std::process::exit(1); } let mut get_response = response .pop() .unwrap() .unwrap_get_sieve_script() .unwrap_result("fetch sieve_scripts"); let sieve_scripts_part = get_response.take_list(); let total_sieve_scripts = response .pop() .unwrap() .unwrap_query_sieve_script() .unwrap_result("query sieve_scripts") .total() .unwrap_or(0); let sieve_scripts_part_len = sieve_scripts_part.len(); if sieve_scripts_part_len > 0 { results.extend(sieve_scripts_part); if results.len() < total_sieve_scripts { position += sieve_scripts_part_len as i32; continue; } } break; } results } async fn export_sieve_scripts( client: &jmap_client::client::Client, max_objects_in_get: usize, blobs: &mut Vec, path: &Path, ) { let sieves = fetch_sieve_scripts(client, max_objects_in_get).await; for sieve in &sieves { if let Some(blob_id) = sieve.blob_id() { blobs.push(blob_id.to_string()); } else { eprintln!( "Warning: sieve script {:?} has no blobId", sieve.id().unwrap_or_default() ); } } eprintln!( "Exported {} sieve scripts.", write_file(path, "sieve.json", sieves,).await ); } pub async fn fetch_identities(client: &jmap_client::client::Client) -> Vec { let mut request = client.build(); request.get_identity().properties([ identity::Property::Id, identity::Property::Name, identity::Property::Email, identity::Property::ReplyTo, identity::Property::Bcc, identity::Property::TextSignature, identity::Property::HtmlSignature, ]); request .send_get_identity() .await .unwrap_result("send JMAP request") .take_list() } async fn export_identities(client: &jmap_client::client::Client, path: &Path) { eprintln!( "Exported {} identities.", write_file(path, "identities.json", fetch_identities(client).await).await ); } pub async fn fetch_vacation_responses( client: &jmap_client::client::Client, ) -> Vec { let mut request = client.build(); request.get_vacation_response().properties([ vacation_response::Property::Id, vacation_response::Property::FromDate, vacation_response::Property::ToDate, vacation_response::Property::Subject, vacation_response::Property::TextBody, vacation_response::Property::HtmlBody, vacation_response::Property::IsEnabled, ]); request .send_get_vacation_response() .await .unwrap_result("send JMAP request") .take_list() } async fn export_vacation_responses(client: &jmap_client::client::Client, path: &Path) { eprintln!( "Exported {} vacation responses.", write_file( path, "vacation.json", fetch_vacation_responses(client).await ) .await ); } async fn write_file(path: &Path, name: &str, contents: Vec) -> usize { let mut path = PathBuf::from(path); path.push(name); let len = contents.len(); tokio::fs::OpenOptions::new() .create(true) .write(true) .truncate(true) .open(&path) .await .unwrap_result(&format!("open {}", path.display())) .write_all(serde_json::to_string(&contents).unwrap().as_bytes()) .await .unwrap_result(&format!("write to {}", path.display())); len } ================================================ FILE: crates/cli/src/modules/group.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::vec; use reqwest::Method; use serde_json::Value; use crate::modules::{Principal, Type}; use super::{ PrincipalField, PrincipalUpdate, PrincipalValue, cli::{Client, GroupCommands}, }; impl GroupCommands { pub async fn exec(self, client: Client) { match self { GroupCommands::Create { name, email, description, members, } => { let principal = Principal { typ: Some(Type::Group), name: name.clone().into(), emails: email.map(|e| vec![e]).unwrap_or_default(), description, ..Default::default() }; let account_id = client .http_request::(Method::POST, "/api/principal", Some(principal)) .await; if let Some(members) = members { client .http_request::( Method::PATCH, &format!("/api/principal/{name}"), Some(vec![PrincipalUpdate::set( PrincipalField::Members, PrincipalValue::StringList(members), )]), ) .await; } eprintln!("Successfully created group {name:?} with id {account_id}."); } GroupCommands::Update { name, new_name, email, description, members, } => { let mut changes = Vec::new(); if let Some(new_name) = new_name { changes.push(PrincipalUpdate::set( PrincipalField::Name, PrincipalValue::String(new_name), )); } if let Some(email) = email { changes.push(PrincipalUpdate::set( PrincipalField::Emails, PrincipalValue::StringList(vec![email]), )); } if let Some(members) = members { changes.push(PrincipalUpdate::set( PrincipalField::Members, PrincipalValue::StringList(members), )); } if let Some(description) = description { changes.push(PrincipalUpdate::set( PrincipalField::Description, PrincipalValue::String(description), )); } if !changes.is_empty() { client .http_request::( Method::PATCH, &format!("/api/principal/{name}"), Some(changes), ) .await; eprintln!("Successfully updated group {name:?}."); } else { eprintln!("No changes to apply."); } } GroupCommands::AddMembers { name, members } => { client .http_request::( Method::PATCH, &format!("/api/principal/{name}"), Some( members .into_iter() .map(|group| { PrincipalUpdate::add_item( PrincipalField::Members, PrincipalValue::String(group), ) }) .collect::>(), ), ) .await; eprintln!("Successfully updated group {name:?}."); } GroupCommands::RemoveMembers { name, members } => { client .http_request::( Method::PATCH, &format!("/api/principal/{name}"), Some( members .into_iter() .map(|group| { PrincipalUpdate::remove_item( PrincipalField::Members, PrincipalValue::String(group), ) }) .collect::>(), ), ) .await; eprintln!("Successfully updated group {name:?}."); } GroupCommands::Display { name } => { client.display_principal(&name).await; } GroupCommands::List { filter, limit, page, } => { client .list_principals("group", "Group", filter, page, limit) .await; } } } } ================================================ FILE: crates/cli/src/modules/import.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ collections::{HashMap, HashSet}, io::{self, Cursor}, path::{Path, PathBuf}, sync::{ Arc, Mutex, atomic::{AtomicUsize, Ordering}, }, time::Duration, }; use console::style; use futures::{StreamExt, stream::FuturesUnordered}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use jmap_client::{ core::set::SetObject, mailbox::{self, Role}, }; use mail_parser::mailbox::{ maildir, mbox::{self, MessageIterator}, }; use rand::Rng; use serde::de::DeserializeOwned; use tokio::{fs::File, io::AsyncReadExt}; use crate::modules::{RETRY_ATTEMPTS, UnwrapResult, name_to_id}; use super::{ cli::{Client, ImportCommands, MailboxFormat}, export::{ fetch_emails, fetch_identities, fetch_mailboxes, fetch_sieve_scripts, fetch_vacation_responses, }, read_file, }; enum Mailbox { Mbox(mbox::MessageIterator>>), Maildir(maildir::MessageIterator), None, } #[derive(Debug)] enum MailboxId<'x> { ExistingId(&'x str), CreateId(String), None, } #[derive(Debug)] struct Message { identifier: String, flags: Vec, internal_date: u64, contents: Vec, } impl ImportCommands { pub async fn exec(self, client: Client) { let mut client = client.into_jmap_client().await; match self { ImportCommands::Messages { num_concurrent, format, account, path, } => { client.set_default_account_id(name_to_id(&client, &account).await); let mut create_mailboxes = Vec::new(); let mut create_mailbox_names = Vec::new(); let mut create_mailbox_ids = Vec::new(); eprintln!("{} Parsing mailbox...", style("[1/4]").bold().dim(),); match format { MailboxFormat::Mbox => { create_mailbox_names.push(Vec::new()); create_mailboxes.push(Mailbox::Mbox(MessageIterator::new(Cursor::new( read_file(&path), )))); } MailboxFormat::Maildir | MailboxFormat::MaildirNested => { let (folder_sep, folder_split) = if format == MailboxFormat::Maildir { (Some("."), ".") } else { (None, "/") }; for folder in maildir::FolderIterator::new(path, folder_sep) .unwrap_result("read Maildir folder") { let folder = folder.unwrap_result("read Maildir folder"); if let Some(folder_name) = folder.name() { let mut folder_parts = Vec::new(); for folder_name in folder_name.split(folder_split) { let mut folder_name = folder_name.trim(); if folder_name.is_empty() { folder_name = "."; } folder_parts.push(folder_name.to_string()); if !create_mailbox_names.contains(&folder_parts) { create_mailboxes.push(Mailbox::None); create_mailbox_names.push(folder_parts.clone()); } } *create_mailboxes.last_mut().unwrap() = Mailbox::Maildir(folder); } else { create_mailboxes.push(Mailbox::Maildir(folder)); create_mailbox_names.push(Vec::new()); }; } } } // Fetch all mailboxes for the account eprintln!( "{} Fetching existing mailboxes for account...", style("[2/4]").bold().dim(), ); let mut inbox_id = None; let mut mailbox_ids = HashMap::new(); let mut children: HashMap, Vec<&str>> = HashMap::from_iter([(None, Vec::new())]); let mut request = client.build(); request.get_mailbox().properties([ mailbox::Property::Name, mailbox::Property::ParentId, mailbox::Property::Role, mailbox::Property::Id, ]); let response = request .send_get_mailbox() .await .unwrap_result("fetch mailboxes"); for mailbox in response.list() { let mailbox_id = mailbox.id().unwrap(); if mailbox.role() == Role::Inbox { inbox_id = mailbox_id.into(); } children .entry(mailbox.parent_id()) .or_default() .push(mailbox_id); mailbox_ids.insert(mailbox_id, mailbox.name().unwrap_or("Untitled")); } let inbox_id = inbox_id .unwrap_result("locate Inbox on account, please check the server logs."); let mut it = children.get(&None).unwrap().iter(); let mut it_stack = Vec::new(); let mut name_stack = Vec::new(); let mut mailbox_names = HashMap::with_capacity(mailbox_ids.len()); // Build mailbox hierarchy on the server eprintln!( "{} Creating missing mailboxes...", style("[3/4]").bold().dim(), ); loop { while let Some(mailbox_id) = it.next() { let name = mailbox_ids[mailbox_id]; let mut mailbox_name = name_stack.clone(); mailbox_name.push(name.to_string()); mailbox_names.insert(mailbox_name, mailbox_id); if let Some(next_it) = children.get(&Some(mailbox_id)).map(|c| c.iter()) { name_stack.push(name.to_string()); it_stack.push(it); it = next_it; } } if let Some(prev_it) = it_stack.pop() { name_stack.pop(); it = prev_it; } else { break; } } // Check whether the mailboxes to be created already exist let mut has_missing_mailboxes = false; for mailbox_name in &create_mailbox_names { create_mailbox_ids.push(if !mailbox_name.is_empty() { if let Some(mailbox_id) = mailbox_names.get(mailbox_name) { MailboxId::ExistingId(mailbox_id) } else { has_missing_mailboxes = true; MailboxId::None } } else { MailboxId::ExistingId(inbox_id) }); } // Create any missing mailboxes if has_missing_mailboxes { let mut request = client.build(); let set_request = request.set_mailbox(); for pos in 0..create_mailbox_ids.len() { if let MailboxId::None = create_mailbox_ids[pos] { let mailbox_name = &create_mailbox_names[pos]; let create_request = set_request.create().name(mailbox_name.last().unwrap()); if mailbox_name.len() > 1 { let parent_mailbox_name = &mailbox_name[..mailbox_name.len() - 1]; let parent_mailbox_pos = create_mailbox_names .iter() .position(|n| n == parent_mailbox_name) .unwrap(); match &create_mailbox_ids[parent_mailbox_pos] { MailboxId::ExistingId(id) => { create_request.parent_id((*id).into()); } MailboxId::CreateId(id_ref) => { create_request.parent_id_ref(id_ref); } MailboxId::None => unreachable!(), } } else { create_request.parent_id(None::); } create_mailbox_ids[pos] = MailboxId::CreateId(create_request.create_id().unwrap()); } } // Create mailboxes let mut response = request .send_set_mailbox() .await .unwrap_result("create mailboxes"); for create_mailbox_id in create_mailbox_ids.iter_mut() { if let MailboxId::CreateId(id) = create_mailbox_id { *id = response .created(id) .unwrap_result("create mailbox") .take_id(); } } } // Import messages eprintln!("{} Importing messages...", style("[4/4]").bold().dim(),); let client = Arc::new(client); let total_imported = Arc::new(AtomicUsize::from(0)); let m = MultiProgress::new(); let num_concurrent = num_concurrent.unwrap_or_else(num_cpus::get); let spinner_style = ProgressStyle::with_template("{prefix:.bold.dim} {spinner} {wide_msg}") .unwrap() .tick_chars("⠁⠂⠄⡀⢀⠠⠐⠈ "); let pbs = Arc::new(Mutex::new(( (0..num_concurrent) .map(|n| { let pb = m.add(ProgressBar::new(40)); pb.set_style(spinner_style.clone()); pb.set_prefix(format!("[{}/?]", n + 1)); pb }) .collect::>(), 0usize, ))); let failures = Arc::new(Mutex::new(Vec::new())); let mut message_num = 0; for ((mut mailbox, mailbox_id), mailbox_name) in create_mailboxes .into_iter() .zip(create_mailbox_ids) .zip(create_mailbox_names) { let mut futures = FuturesUnordered::new(); let mailbox_id = Arc::new(match mailbox_id { MailboxId::ExistingId(id) => id.to_string(), MailboxId::CreateId(id) => id, MailboxId::None => unreachable!(), }); let mailbox_name = Arc::new(if !mailbox_name.is_empty() { mailbox_name.join("/") } else { "Inbox".to_string() }); for result in mailbox.by_ref() { match result { Ok(message) => { message_num += 1; let client = client.clone(); let mailbox_id = mailbox_id.clone(); let mailbox_name = mailbox_name.clone(); let total_imported = total_imported.clone(); let pbs = pbs.clone(); let failures = failures.clone(); futures.push(async move { // Update progress bar { let mut pbs = pbs.lock().unwrap(); let pb = &pbs.0[pbs.1 % pbs.0.len()]; pb.set_message(format!( "Importing {}: {}/{}", message_num, mailbox_name, message.identifier )); pb.inc(1); pbs.1 += 1; } let mut retry_count = 0; loop { // Sanitize message let mut contents = Vec::with_capacity(message.contents.len()); let mut last_ch = 0; for &ch in message.contents.iter() { if ch == b'\n' && last_ch != b'\r' { contents.push(b'\r'); } contents.push(ch); last_ch = ch; } match client .email_import( contents, [mailbox_id.as_ref()], if !message.flags.is_empty() { message .flags .iter() .map(|f| match f { maildir::Flag::Passed => "$passed", maildir::Flag::Replied => "$answered", maildir::Flag::Seen => "$seen", maildir::Flag::Trashed => "$deleted", maildir::Flag::Draft => "$draft", maildir::Flag::Flagged => "$flagged", }) .into() } else { None }, if message.internal_date > 0 { (message.internal_date as i64).into() } else { None }, ) .await { Ok(_) => { total_imported.fetch_add(1, Ordering::Relaxed); } Err(_) if retry_count < RETRY_ATTEMPTS => { let backoff = rand::rng().random_range(50..=300); tokio::time::sleep(Duration::from_millis(backoff)) .await; retry_count += 1; continue; } Err(err) => { failures.lock().unwrap().push(format!( concat!( "Failed to import message {} ", "with identifier '{}': {}" ), message_num, message.identifier, err )); } } break; } }); if futures.len() == num_concurrent { futures.next().await.unwrap(); } } Err(e) => { failures .lock() .unwrap() .push(format!("I/O error reading message: {}", e)); } } } // Wait for remaining futures while futures.next().await.is_some() {} } // Done for pb in pbs.lock().unwrap().0.iter() { pb.finish_with_message("Done"); } let failures = failures.lock().unwrap(); eprintln!( "\n\nSuccessfully imported {} messages.\n", total_imported.load(Ordering::Relaxed) ); if !failures.is_empty() { eprintln!("There were {} failures:\n", failures.len()); for failure in failures.iter() { eprintln!("{}", failure); } } } ImportCommands::Account { num_concurrent, account, path, } => { client.set_default_account_id(name_to_id(&client, &account).await); let path = PathBuf::from(path); if !path.exists() { eprintln!("Path '{}' does not exist.", path.display()); return; } let num_concurrent = num_concurrent.unwrap_or_else(num_cpus::get); // Import objects import_emails( &client, &path, import_mailboxes(&client, &path).await.into(), num_concurrent, ) .await; import_sieve_scripts(&client, &path, num_concurrent).await; import_identities(&client, &path).await; import_vacation_responses(&client, &path).await; } } } } async fn import_mailboxes( client: &jmap_client::client::Client, path: &Path, ) -> HashMap { // Deserialize mailboxes let mailboxes = read_json::(path, "mailboxes.json").await; if mailboxes.is_empty() { return HashMap::new(); } // Obtain current mailboxes let existing_mailboxes = fetch_mailboxes( client, client .session() .core_capabilities() .map(|c| c.max_objects_in_get()) .unwrap_or(500), ) .await; let nested_existing_mailboxes = build_mailbox_tree(&existing_mailboxes); let mut id_mappings: HashMap = HashMap::new(); let mut id_missing = Vec::new(); for (path, mailbox) in build_mailbox_tree(&mailboxes) { let id = mailbox.id().unwrap_result("obtain mailbox id"); // Find existing mailbox based on role if !matches!(mailbox.role(), Role::None) && let Some(existing_mailbox) = existing_mailboxes .iter() .find(|m| m.role() == mailbox.role()) { id_mappings.insert( id.to_string(), existing_mailbox .id() .unwrap_result("obtain mailbox id") .to_string(), ); continue; } // Find existing mailbox by name if let Some(mailbox) = nested_existing_mailboxes.get(&path) { id_mappings.insert( id.to_string(), mailbox.id().unwrap_result("obtain mailbox id").to_string(), ); } else { id_missing.push(id); } } let mut total_imported = 0; let mut total_existing = 0; if !id_missing.is_empty() { let mut request = client.build(); let set_request = request.set_mailbox(); for mailbox in &mailboxes { // Skip if mailbox already exists let id = mailbox.id().unwrap_result("obtain mailbox id").to_string(); if id_mappings.contains_key(&id) { total_existing += 1; continue; } let create_request = set_request .create_with_id(&id) .name(mailbox.name().unwrap()) .role(mailbox.role()); if let Some(parent_id) = mailbox.parent_id() { if let Some(existing_id) = id_mappings.get(parent_id) { create_request.parent_id(Some(existing_id.to_string())); } else { create_request.parent_id_ref(parent_id); } } else { create_request.parent_id(None::); } if mailbox.sort_order() > 0 { create_request.sort_order(mailbox.sort_order()); } /*if let Some(acls) = mailbox.acl() { create_request.acls(acls.clone().into_iter()); }*/ if mailbox.is_subscribed() { create_request.is_subscribed(true); } } // Create mailboxes let mut response = request .send_set_mailbox() .await .unwrap_result("create mailboxes"); for missing_id in id_missing { id_mappings.insert( missing_id.to_string(), response .created(missing_id) .unwrap_result("create mailbox") .take_id(), ); total_imported += 1; } } else { total_existing = mailboxes.len(); } eprintln!( "Successfully processed {} mailboxes ({} imported, {} already exist).", total_existing + total_imported, total_imported, total_existing ); id_mappings } async fn import_emails( client: &jmap_client::client::Client, path: &Path, mailbox_ids: Arc>, num_concurrent: usize, ) { // Deserialize emails let emails = read_json::(path, "emails.json").await; if emails.is_empty() { return; } // Obtain existing emails let existing_emails = fetch_emails( client, client .session() .core_capabilities() .map(|c| c.max_objects_in_get()) .unwrap_or(500), ) .await; let existing_ids = existing_emails .iter() .map(|email| (email.message_id(), email.received_at())) .collect::>(); let mut futures = FuturesUnordered::new(); let total_imported = Arc::new(AtomicUsize::from(0)); let mut total_existing = 0; let mut path = PathBuf::from(path); path.push("blobs"); for email in emails { // Skip messages that already exist in the server if existing_ids.contains(&(email.message_id(), email.received_at())) { total_existing += 1; continue; } // Spawn import tasks let mailbox_ids = mailbox_ids.clone(); let mut path = path.clone(); let total_imported = total_imported.clone(); futures.push(async move { // Obtain mailbox ids let id = if let Some(id) = email.id() { id } else { eprintln!("Skipping email with no id"); return; }; if email.mailbox_ids().is_empty() { eprintln!("Skipping emailId {id} with no mailboxIds"); return; } let mut mailboxes = Vec::with_capacity(email.mailbox_ids().len()); for mailbox_id in email.mailbox_ids() { if let Some(mailbox_id) = mailbox_ids.get(mailbox_id) { mailboxes.push(mailbox_id.to_string()); } else { eprintln!("Skipping emailId {id} with unknown mailboxId {mailbox_id}"); return; } } let keywords = email.keywords(); // Read blob if let Some(blob_id) = email.blob_id() { path.push(blob_id); } else { eprintln!("Skipping emailId {id} with no blobId"); return; } let mut contents = vec![]; match File::open(&path).await { Ok(mut file) => match file.read_to_end(&mut contents).await { Ok(_) => {} Err(err) => { eprintln!( "Failed to read blob file for emailId {id} at {path:?}: {err}", id = id, path = path, err = err ); return; } }, Err(err) => { eprintln!( "Failed to open blob file for emailId {id} at {path:?}: {err}", id = id, path = path, err = err ); return; } } let mut retry_count = 0; loop { match client .email_import( contents.clone(), mailboxes.clone(), if !keywords.is_empty() { Some(keywords.clone()) } else { None }, email.received_at(), ) .await { Ok(_) => { total_imported.fetch_add(1, Ordering::Relaxed); } Err(_) if retry_count < RETRY_ATTEMPTS => { retry_count += 1; continue; } Err(err) => { eprintln!("Failed to import emailId {id}: {err}"); } } break; } }); if futures.len() == num_concurrent { futures.next().await.unwrap(); } } // Wait for remaining futures while futures.next().await.is_some() {} // Done eprintln!( "Successfully processed {} emails ({} imported, {} already exist).", total_imported.load(Ordering::Relaxed) + total_existing, total_imported.load(Ordering::Relaxed), total_existing ); } async fn import_sieve_scripts( client: &jmap_client::client::Client, path: &Path, num_concurrent: usize, ) { // Deserialize scripts let scripts = read_json::(path, "sieve.json").await; if scripts.is_empty() { return; } let existing_scripts = fetch_sieve_scripts( client, client .session() .core_capabilities() .map(|c| c.max_objects_in_get()) .unwrap_or(500), ) .await; let mut path = PathBuf::from(path); path.push("blobs"); // Spawn tasks let mut futures = FuturesUnordered::new(); let total_imported = Arc::new(AtomicUsize::from(0)); let mut total_existing = 0; 'outer: for script in scripts { // Skip scripts that already exist for existing_script in &existing_scripts { if existing_script.name() == script.name() { total_existing += 1; continue 'outer; } } let mut path = path.clone(); let total_imported = total_imported.clone(); futures.push(async move { let id = if let Some(id) = script.id() { id } else { eprintln!("Skipping script with no id."); return; }; // Read blob let name = if let (Some(blob_id), Some(name)) = (script.blob_id(), script.name()) { path.push(blob_id); name } else { eprintln!("Skipping script {id} with no blobId and/or name"); return; }; let mut contents = vec![]; match File::open(&path).await { Ok(mut file) => match file.read_to_end(&mut contents).await { Ok(_) => {} Err(err) => { eprintln!( "Failed to read blob file for script {id} at {path:?}: {err}", id = id, path = path, err = err ); return; } }, Err(err) => { eprintln!( "Failed to open blob file for script {id} at {path:?}: {err}", id = id, path = path, err = err ); return; } } // Upload blob match client .sieve_script_create(name, contents, script.is_active()) .await { Ok(_) => { total_imported.fetch_add(1, Ordering::Relaxed); } Err(err) => { eprintln!("Failed to import script {id}: {err}"); } } }); if futures.len() == num_concurrent { futures.next().await.unwrap(); } } // Wait for remaining futures while futures.next().await.is_some() {} // Done eprintln!( "Successfully processed {} sieve scripts ({} imported, {} already exist).", total_imported.load(Ordering::Relaxed) + total_existing, total_imported.load(Ordering::Relaxed), total_existing ); } async fn import_identities(client: &jmap_client::client::Client, path: &Path) { // Deserialize mailboxes let identities = read_json::(path, "identities.json").await; if identities.is_empty() { return; } let existing_identities = fetch_identities(client).await; let mut request = client.build(); let set_request = request.set_identity(); let mut create_ids = Vec::new(); let mut total_existing = 0; 'outer: for identity in &identities { for existing_identity in &existing_identities { if identity.name() == existing_identity.name() && identity.email() == existing_identity.email() { total_existing += 1; continue 'outer; } } if let (Some(id), Some(name), Some(email)) = (identity.id(), identity.name(), identity.email()) { if name != "vacation" { create_ids.push(id); let create_request = set_request.create_with_id(id).name(name).email(email); if let Some(reply_to) = identity.reply_to() { create_request.reply_to(reply_to.iter().cloned().into()); } if let Some(bcc) = identity.bcc() { create_request.bcc(bcc.iter().cloned().into()); } if let Some(html_signature) = identity.html_signature() { create_request.html_signature(html_signature); } if let Some(text_signature) = identity.text_signature() { create_request.text_signature(text_signature); } } } else { eprintln!("Skipping identity with no id, name, and/or email."); continue; } } let mut total_imported = 0; if !create_ids.is_empty() { match request.send_set_identity().await { Ok(mut response) => { for id in create_ids { if let Err(err) = response.created(id) { eprintln!("Failed to import identity {id}: {err}"); } else { total_imported += 1; } } } Err(err) => { eprintln!("Failed to import identities: {err}"); return; } } } eprintln!( "Successfully processed {} identities ({} imported, {} already exist).", total_imported + total_existing, total_imported, total_existing ); } async fn import_vacation_responses(client: &jmap_client::client::Client, path: &Path) { // Deserialize mailboxes let vacation_responses = read_json::(path, "vacation.json").await; if vacation_responses.is_empty() { return; } let existing_vacation_responses = fetch_vacation_responses(client).await; if !existing_vacation_responses.is_empty() { eprintln!("Successfully processed 1 vacation response (0 imported, 1 already exist).",); return; } let vacation_response = vacation_responses.into_iter().next().unwrap(); let mut request = client.build(); let set_request = request.set_vacation_response().create(); if vacation_response.is_enabled() { set_request.is_enabled(true); } if let Some(from_date) = vacation_response.from_date() { set_request.from_date(from_date.into()); } if let Some(to_date) = vacation_response.to_date() { set_request.to_date(to_date.into()); } if let Some(subject) = vacation_response.subject() { set_request.subject(subject.into()); } if let Some(text_body) = vacation_response.text_body() { set_request.text_body(text_body.into()); } if let Some(html_body) = vacation_response.html_body() { set_request.html_body(html_body.into()); } let create_id = set_request.create_id().unwrap(); match request.send_set_vacation_response().await { Ok(mut response) => { if let Err(err) = response.created(&create_id) { eprintln!("Failed to import vacation response: {err}"); } else { eprintln!( "Successfully processed 1 vacation response (1 imported, 0 already exist).", ); } } Err(err) => { eprintln!("Failed to import vacation response: {err}"); } } } fn build_mailbox_tree( mailboxes: &[jmap_client::mailbox::Mailbox], ) -> HashMap, &jmap_client::mailbox::Mailbox> { let mut path = Vec::new(); let mut parent_id = None; let mut mailboxes_iter = mailboxes.iter(); let mut stack = Vec::new(); let mut results = HashMap::with_capacity(mailboxes.len()); let parents = mailboxes .iter() .map(|m| m.parent_id()) .collect::>(); 'outer: loop { while let Some(mailbox) = mailboxes_iter.next() { if parent_id == mailbox.parent_id() { let name = mailbox.name().unwrap_result("obtain mailbox name"); if parents.contains(&mailbox.id()) { stack.push((path.clone(), parent_id, mailboxes_iter)); parent_id = mailbox.id(); path.push(name); results.insert(path.clone(), mailbox); mailboxes_iter = mailboxes.iter(); continue 'outer; } else { let mut path = path.clone(); path.push(name); results.insert(path, mailbox); } } } if let Some((prev_path, prev_parent_id, prev_iter)) = stack.pop() { parent_id = prev_parent_id; path = prev_path; mailboxes_iter = prev_iter; } else { break; } } debug_assert_eq!(results.len(), mailboxes.len()); results } async fn read_json(path: &Path, filename: &str) -> Vec { let mut path = PathBuf::from(path); path.push(filename); if path.exists() { let mut file = File::open(path).await.unwrap_result("open file"); let mut contents = String::new(); file.read_to_string(&mut contents) .await .unwrap_result("read file"); serde_json::from_str(&contents).unwrap_result("parse JSON") } else { Vec::new() } } impl Iterator for Mailbox { type Item = io::Result; fn next(&mut self) -> Option { match self { Mailbox::Mbox(it) => it.next().map(|r| { r.map(|m| Message { identifier: m.from().to_string(), flags: Vec::new(), internal_date: m.internal_date(), contents: m.unwrap_contents(), }) .map_err(|_| std::io::Error::other("Failed to parse from mbox file.")) }), Mailbox::Maildir(it) => it.next().map(|r| { r.map(|m| Message { identifier: m .path() .file_name() .and_then(|f| f.to_str()) .unwrap_or("unknown") .to_string(), flags: m.flags().to_vec(), internal_date: m.internal_date(), contents: m.unwrap_contents(), }) }), Mailbox::None => None, } } } ================================================ FILE: crates/cli/src/modules/list.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::vec; use reqwest::Method; use serde_json::Value; use crate::modules::{Principal, Type}; use super::{ PrincipalField, PrincipalUpdate, PrincipalValue, cli::{Client, ListCommands}, }; impl ListCommands { pub async fn exec(self, client: Client) { match self { ListCommands::Create { name, email, description, members, } => { let principal = Principal { typ: Some(Type::List), name: name.clone().into(), emails: vec![email], description, ..Default::default() }; let account_id = client .http_request::(Method::POST, "/api/principal", Some(principal)) .await; if let Some(members) = members { client .http_request::( Method::PATCH, &format!("/api/principal/{name}"), Some(vec![PrincipalUpdate::set( PrincipalField::Members, PrincipalValue::StringList(members), )]), ) .await; } eprintln!("Successfully created mailing list {name:?} with id {account_id}."); } ListCommands::Update { name, new_name, email, description, members, } => { let mut changes = Vec::new(); if let Some(new_name) = new_name { changes.push(PrincipalUpdate::set( PrincipalField::Name, PrincipalValue::String(new_name), )); } if let Some(email) = email { changes.push(PrincipalUpdate::set( PrincipalField::Emails, PrincipalValue::StringList(vec![email]), )); } if let Some(members) = members { changes.push(PrincipalUpdate::set( PrincipalField::Members, PrincipalValue::StringList(members), )); } if let Some(description) = description { changes.push(PrincipalUpdate::set( PrincipalField::Description, PrincipalValue::String(description), )); } if !changes.is_empty() { client .http_request::( Method::PATCH, &format!("/api/principal/{name}"), Some(changes), ) .await; eprintln!("Successfully updated mailing list {name:?}."); } else { eprintln!("No changes to apply."); } } ListCommands::AddMembers { name, members } => { client .http_request::( Method::PATCH, &format!("/api/principal/{name}"), Some( members .into_iter() .map(|group| { PrincipalUpdate::add_item( PrincipalField::Members, PrincipalValue::String(group), ) }) .collect::>(), ), ) .await; eprintln!("Successfully updated mailing list {name:?}."); } ListCommands::RemoveMembers { name, members } => { client .http_request::( Method::PATCH, &format!("/api/principal/{name}"), Some( members .into_iter() .map(|group| { PrincipalUpdate::remove_item( PrincipalField::Members, PrincipalValue::String(group), ) }) .collect::>(), ), ) .await; eprintln!("Successfully updated mailing list {name:?}."); } ListCommands::Display { name } => { client.display_principal(&name).await; } ListCommands::List { filter, limit, page, } => { client .list_principals("list", "Mailing List", filter, page, limit) .await; } } } } ================================================ FILE: crates/cli/src/modules/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{collections::HashMap, fmt::Display, io::Read}; use jmap_client::{ client::Client, principal::query::{self}, }; use serde::{Deserialize, Serialize}; pub mod account; pub mod cli; pub mod database; pub mod dkim; pub mod domain; pub mod export; pub mod group; pub mod import; pub mod list; pub mod queue; pub mod report; const RETRY_ATTEMPTS: usize = 5; #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Principal { #[serde(default, skip_serializing_if = "Option::is_none")] pub id: Option, #[serde(rename = "type")] pub typ: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub quota: Option, #[serde(rename = "usedQuota")] #[serde(default, skip_serializing_if = "Option::is_none")] pub used_quota: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub name: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub secrets: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub emails: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] #[serde(rename = "memberOf")] pub member_of: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] #[serde(rename = "members")] pub members: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub description: Option, } #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum Type { #[serde(rename = "individual")] #[default] Individual = 0, #[serde(rename = "group")] Group = 1, #[serde(rename = "resource")] Resource = 2, #[serde(rename = "location")] Location = 3, #[serde(rename = "superuser")] Superuser = 4, #[serde(rename = "list")] List = 5, #[serde(rename = "other")] Other = 6, } #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub enum PrincipalField { #[serde(rename = "name")] Name, #[serde(rename = "type")] Type, #[serde(rename = "quota")] Quota, #[serde(rename = "description")] Description, #[serde(rename = "secrets")] Secrets, #[serde(rename = "emails")] Emails, #[serde(rename = "memberOf")] MemberOf, #[serde(rename = "members")] Members, } #[derive(Clone, serde::Serialize, serde::Deserialize, Default)] pub struct List { pub items: Vec, pub total: u64, } #[derive(Clone, serde::Serialize, serde::Deserialize, Default)] pub struct Response { pub items: T, pub total: u64, } #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct PrincipalUpdate { action: PrincipalAction, field: PrincipalField, value: PrincipalValue, } #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub enum PrincipalAction { #[serde(rename = "set")] Set, #[serde(rename = "addItem")] AddItem, #[serde(rename = "removeItem")] RemoveItem, } #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(untagged)] pub enum PrincipalValue { String(String), StringList(Vec), Integer(u64), } impl PrincipalUpdate { pub fn set(field: PrincipalField, value: PrincipalValue) -> PrincipalUpdate { PrincipalUpdate { action: PrincipalAction::Set, field, value, } } pub fn add_item(field: PrincipalField, value: PrincipalValue) -> PrincipalUpdate { PrincipalUpdate { action: PrincipalAction::AddItem, field, value, } } pub fn remove_item(field: PrincipalField, value: PrincipalValue) -> PrincipalUpdate { PrincipalUpdate { action: PrincipalAction::RemoveItem, field, value, } } } pub trait UnwrapResult { fn unwrap_result(self, action: &str) -> T; } impl UnwrapResult for Option { fn unwrap_result(self, message: &str) -> T { match self { Some(result) => result, None => { eprintln!("Failed to {}", message); std::process::exit(1); } } } } impl UnwrapResult for Result { fn unwrap_result(self, message: &str) -> T { match self { Ok(result) => result, Err(err) => { eprintln!("Failed to {}: {}", message, err); std::process::exit(1); } } } } pub fn read_file(path: &str) -> Vec { if path == "-" { let mut stdin = std::io::stdin().lock(); let mut raw_message = Vec::with_capacity(1024); let mut buf = [0; 1024]; loop { let n = stdin.read(&mut buf).unwrap(); if n == 0 { break; } raw_message.extend_from_slice(&buf[..n]); } raw_message } else { std::fs::read(path).unwrap_or_else(|_| { eprintln!("Failed to read file: {}", path); std::process::exit(1); }) } } pub async fn name_to_id(client: &Client, name: &str) -> String { let filter = if name.contains('@') { query::Filter::email(name) } else { query::Filter::name(name) }; let mut response = client .principal_query(filter.into(), None::>) .await .unwrap_result("query principals"); match response.ids().len() { 1 => response.take_ids().pop().unwrap(), 0 => { eprintln!("Error: No principal found with name '{}'.", name); std::process::exit(1); } _ => { eprintln!("Error: Multiple principals found with name '{}'.", name); std::process::exit(1); } } } pub fn host(url: &str) -> Option<&str> { url.split_once("://") .map(|(_, url)| url.split_once('/').map_or(url, |(host, _)| host)) .map(|host| host.rsplit_once(':').map_or(host, |(host, _)| host)) } pub fn is_localhost(url: &str) -> bool { host(url).is_some_and(|host| host == "localhost" || host == "127.0.0.1" || host == "[::1]") } pub trait OAuthResponse { fn property(&self, name: &str) -> &str; } impl OAuthResponse for HashMap { fn property(&self, name: &str) -> &str { self.get(name) .unwrap_result(&format!("find '{}' in OAuth response", name)) .as_str() .unwrap_result(&format!("invalid '{}' value", name)) } } ================================================ FILE: crates/cli/src/modules/queue.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ List, cli::{Client, QueueCommands}, }; use console::Term; use human_size::{Byte, SpecificSize}; use mail_parser::DateTime; use prettytable::{Attr, Cell, Row, Table, format::Alignment}; use reqwest::Method; use serde::{Deserialize, Deserializer}; #[derive(Debug, Deserialize, PartialEq, Eq)] pub struct Message { pub id: u64, pub return_path: String, pub recipients: Vec, #[serde(deserialize_with = "deserialize_datetime")] pub created: DateTime, pub size: usize, #[serde(default)] pub priority: i16, #[serde(default)] pub env_id: Option, pub blob_hash: String, } #[derive(Debug, Deserialize, PartialEq, Eq)] pub struct Recipient { pub address: String, pub status: Status, pub queue: String, pub retry_num: u32, #[serde(deserialize_with = "deserialize_maybe_datetime", default)] pub next_retry: Option, #[serde(deserialize_with = "deserialize_maybe_datetime", default)] pub next_notify: Option, #[serde(deserialize_with = "deserialize_maybe_datetime", default)] pub expires: Option, #[serde(default)] pub orcpt: Option, } #[derive(Debug, PartialEq, Eq, Deserialize)] pub enum Status { #[serde(rename = "scheduled")] Scheduled, #[serde(rename = "completed")] Completed(String), #[serde(rename = "temp_fail")] TemporaryFailure(String), #[serde(rename = "perm_fail")] PermanentFailure(String), } impl QueueCommands { pub async fn exec(self, client: Client) { match self { QueueCommands::List { sender, rcpt, before, after, page_size, } => { let stdout = Term::buffered_stdout(); let ids = client.query_messages(&sender, &rcpt, &before, &after).await; let ids_len = ids.len(); let page_size = page_size.map(|p| std::cmp::max(p, 1)).unwrap_or(20); let pages_total = (ids_len as f64 / page_size as f64).ceil() as usize; for (page_num, chunk) in ids.chunks(page_size).enumerate() { // Build table let mut table = Table::new(); table.add_row(Row::new( ["ID", "Delivery Due", "Sender", "Recipients", "Size"] .iter() .map(|p| Cell::new(p).with_style(Attr::Bold)) .collect(), )); for id in chunk { let message = client .http_request::( Method::GET, &format!("/api/queue/messages/{id}"), None, ) .await; let mut rcpts = String::new(); let mut deliver_at = i64::MAX; let mut deliver_pos = 0; for (pos, rcpt) in message.recipients.iter().enumerate() { if let Some(next_retry) = &rcpt.next_retry { let ts = next_retry.to_timestamp(); if ts < deliver_at { deliver_at = ts; deliver_pos = pos; } } if !rcpts.is_empty() { rcpts.push('\n'); } rcpts.push_str(&rcpt.address); rcpts.push_str(" ("); rcpts.push_str(rcpt.status.status_short()); rcpts.push(')'); } let mut cells = Vec::new(); cells.push(Cell::new(&format!("{id:X}"))); cells.push(if deliver_at != i64::MAX { Cell::new( &message.recipients[deliver_pos] .next_retry .as_ref() .unwrap() .to_rfc822(), ) } else { Cell::new("None") }); cells.push(Cell::new(if !message.return_path.is_empty() { &message.return_path } else { "<>" })); cells.push(Cell::new(&rcpts)); cells.push(Cell::new( &SpecificSize::new(message.size as u32, Byte) .unwrap() .to_string(), )); table.add_row(Row::new(cells)); } eprintln!(); table.printstd(); eprintln!(); if page_num + 1 != pages_total { eprintln!("\n--- Press any key to continue or 'q' to exit ---"); if let Ok('q' | 'Q') = stdout.read_char() { break; } } } eprintln!("\n{ids_len} queued message(s) found.") } QueueCommands::Status { ids } => { for (uid, id) in parse_ids(&ids).into_iter().zip(ids) { let message = client .try_http_request::( Method::GET, &format!("/api/queue/messages/{uid}"), None, ) .await; let mut table = Table::new(); table.add_row(Row::new(vec![ Cell::new("ID").with_style(Attr::Bold), Cell::new(&id), ])); if let Some(message) = message { table.add_row(Row::new(vec![ Cell::new("Sender").with_style(Attr::Bold), Cell::new(if !message.return_path.is_empty() { &message.return_path } else { "<>" }), ])); table.add_row(Row::new(vec![ Cell::new("Created").with_style(Attr::Bold), Cell::new(&message.created.to_rfc822()), ])); table.add_row(Row::new(vec![ Cell::new("Size").with_style(Attr::Bold), Cell::new( &SpecificSize::new(message.size as u32, Byte) .unwrap() .to_string(), ), ])); if let Some(env_id) = &message.env_id { table.add_row(Row::new(vec![ Cell::new("Env-Id").with_style(Attr::Bold), Cell::new(env_id), ])); } if message.priority != 0 { table.add_row(Row::new(vec![ Cell::new("Priority").with_style(Attr::Bold), Cell::new(&message.priority.to_string()), ])); } for rcpt in &message.recipients { table.add_row(Row::new(vec![ Cell::new_align(&rcpt.address, Alignment::RIGHT) .with_style(Attr::Bold) .with_style(Attr::Italic(true)) .with_hspan(2), ])); table.add_row(Row::new(vec![ Cell::new("Status").with_style(Attr::Bold), Cell::new(rcpt.status.status()), ])); table.add_row(Row::new(vec![ Cell::new("Details").with_style(Attr::Bold), Cell::new(rcpt.status.details()), ])); table.add_row(Row::new(vec![ Cell::new("Retry #").with_style(Attr::Bold), Cell::new(&rcpt.retry_num.to_string()), ])); if let Some(dt) = &rcpt.next_retry { table.add_row(Row::new(vec![ Cell::new("Delivery Due").with_style(Attr::Bold), Cell::new(&dt.to_rfc822()), ])); } if let Some(dt) = &rcpt.next_notify { table.add_row(Row::new(vec![ Cell::new("Notify at").with_style(Attr::Bold), Cell::new(&dt.to_rfc822()), ])); } table.add_row(Row::new(vec![ Cell::new("Expires").with_style(Attr::Bold), if let Some(dt) = &rcpt.expires { Cell::new(&dt.to_rfc822()) } else { Cell::new("N/A") }, ])); } } else { table.add_row(Row::new(vec![ Cell::new_align("-- Not found --", Alignment::CENTER).with_hspan(2), ])); } eprintln!(); table.printstd(); eprintln!(); } } QueueCommands::Retry { sender, domain, before, after, time, ids, } => { let (parsed_ids, ids) = if ids.is_empty() { if sender.is_some() || domain.is_some() || before.is_some() || after.is_some() { let parsed_ids = client .query_messages(&sender, &domain, &before, &after) .await; let ids = parsed_ids.iter().map(|id| format!("{id:X}")).collect(); (parsed_ids, ids) } else { (vec![], vec![]) } } else { (parse_ids(&ids), ids) }; if ids.is_empty() { eprintln!("No messages were found."); std::process::exit(1); } let mut success_count = 0; let mut failed_list = vec![]; for id in parsed_ids { let mut query = form_urlencoded::Serializer::new(format!("/api/queue/messages/{id}")); if let Some(filter) = &domain { query.append_pair("filter", filter); } if let Some(at) = time { query.append_pair("at", &at.to_rfc3339()); } if client .try_http_request::(Method::PATCH, &query.finish(), None) .await .unwrap_or(false) { success_count += 1; } else { failed_list.push(id.to_string()); } } eprint!("\nSuccessfully rescheduled {success_count} message(s)."); if !failed_list.is_empty() { eprint!(" Unable to reschedule id(s): {}.", failed_list.join(", ")); } eprintln!(); } QueueCommands::Cancel { sender, rcpt, before, after, ids, } => { let (parsed_ids, ids) = if ids.is_empty() { if sender.is_some() || rcpt.is_some() || before.is_some() || after.is_some() { let parsed_ids = client.query_messages(&sender, &rcpt, &before, &after).await; let ids = parsed_ids.iter().map(|id| format!("{id:X}")).collect(); (parsed_ids, ids) } else { (vec![], vec![]) } } else { (parse_ids(&ids), ids) }; if ids.is_empty() { eprintln!("No messages were found."); std::process::exit(1); } let mut success_count = 0; let mut failed_list = vec![]; for id in parsed_ids { let mut query = form_urlencoded::Serializer::new(format!("/api/queue/messages/{id}")); if let Some(filter) = &rcpt { query.append_pair("filter", filter); } if client .try_http_request::(Method::DELETE, &query.finish(), None) .await .unwrap_or(false) { success_count += 1; } else { failed_list.push(id.to_string()); } } eprint!("\nCancelled delivery of {success_count} message(s)."); if !failed_list.is_empty() { eprint!( " Unable to cancel delivery for id(s): {}.", failed_list.join(", ") ); } eprintln!(); } } } } impl Client { async fn query_messages( &self, from: &Option, rcpt: &Option, before: &Option, after: &Option, ) -> Vec { let mut query = form_urlencoded::Serializer::new("/api/queue/messages".to_string()); if let Some(sender) = from { query.append_pair("from", sender); } if let Some(rcpt) = rcpt { query.append_pair("to", rcpt); } if let Some(before) = before { query.append_pair("before", &before.to_rfc3339()); } if let Some(after) = after { query.append_pair("after", &after.to_rfc3339()); } self.http_request::, String>(Method::GET, &query.finish(), None) .await .items } } fn deserialize_maybe_datetime<'de, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, { if let Some(value) = Option::<&str>::deserialize(deserializer)? { if let Some(value) = DateTime::parse_rfc3339(value) { Ok(Some(value)) } else { Err(serde::de::Error::custom( "Failed to parse RFC3339 timestamp", )) } } else { Ok(None) } } pub fn deserialize_datetime<'de, D>(deserializer: D) -> Result where D: Deserializer<'de>, { if let Some(value) = DateTime::parse_rfc3339(<&str>::deserialize(deserializer)?) { Ok(value) } else { Err(serde::de::Error::custom( "Failed to parse RFC3339 timestamp", )) } } fn parse_ids(ids: &[String]) -> Vec { let mut result = Vec::with_capacity(ids.len()); for id in ids { match u64::from_str_radix(id, 16) { Ok(id) => { result.push(id); } Err(_) => { eprintln!("Failed to parse id {id:?}."); std::process::exit(1); } } } result } impl Status { fn status_short(&self) -> &str { match self { Status::Scheduled => "scheduled", Status::Completed(_) => "delivered", Status::TemporaryFailure(_) => "tempfail", Status::PermanentFailure(_) => "permfail", } } fn status(&self) -> &str { match self { Status::Scheduled => "Scheduled", Status::Completed(_) => "Delivered", Status::TemporaryFailure(_) => "Temporary Failure", Status::PermanentFailure(_) => "Permanent Failure", } } fn details(&self) -> &str { match self { Status::Scheduled => "", Status::Completed(status) => status, Status::TemporaryFailure(status) => status, Status::PermanentFailure(status) => status, } } } ================================================ FILE: crates/cli/src/modules/report.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::cli::{Client, ReportCommands, ReportFormat}; use crate::modules::{List, queue::deserialize_datetime}; use console::Term; use human_size::{Byte, SpecificSize}; use mail_auth::{ dmarc::URI, mta_sts::ReportUri, report::{self, tlsrpt::TlsReport}, }; use mail_parser::DateTime; use prettytable::{Attr, Cell, Row, Table, format}; use reqwest::Method; use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize)] #[serde(tag = "type")] #[serde(rename_all = "camelCase")] pub enum Report { Tls { id: String, domain: String, #[serde(deserialize_with = "deserialize_datetime")] range_from: DateTime, #[serde(deserialize_with = "deserialize_datetime")] range_to: DateTime, report: TlsReport, rua: Vec, }, Dmarc { id: String, domain: String, #[serde(deserialize_with = "deserialize_datetime")] range_from: DateTime, #[serde(deserialize_with = "deserialize_datetime")] range_to: DateTime, report: report::Report, rua: Vec, }, } impl Report { pub fn domain(&self) -> &str { match self { Report::Tls { domain, .. } => domain, Report::Dmarc { domain, .. } => domain, } } pub fn type_(&self) -> &str { match self { Report::Tls { .. } => "TLS", Report::Dmarc { .. } => "DMARC", } } pub fn range_from(&self) -> &DateTime { match self { Report::Tls { range_from, .. } => range_from, Report::Dmarc { range_from, .. } => range_from, } } pub fn range_to(&self) -> &DateTime { match self { Report::Tls { range_to, .. } => range_to, Report::Dmarc { range_to, .. } => range_to, } } pub fn num_records(&self) -> usize { match self { Report::Tls { report, .. } => report .policies .iter() .map(|p| p.failure_details.len()) .sum(), Report::Dmarc { report, .. } => report.records().len(), } } } impl ReportCommands { pub async fn exec(self, client: Client) { match self { ReportCommands::List { domain, format, page_size, } => { let stdout = Term::buffered_stdout(); let mut query = form_urlencoded::Serializer::new("/api/queue/reports".to_string()); if let Some(domain) = &domain { query.append_pair("domain", domain); } if let Some(format) = &format { query.append_pair("type", format.id()); } let ids = client .http_request::, String>(Method::GET, &query.finish(), None) .await .items; let ids_len = ids.len(); let page_size = page_size.map(|p| std::cmp::max(p, 1)).unwrap_or(20); let pages_total = (ids_len as f64 / page_size as f64).ceil() as usize; for (page_num, chunk) in ids.chunks(page_size).enumerate() { // Build table let mut table = Table::new(); table.add_row(Row::new( ["ID", "Domain", "Type", "From Date", "To Date", "Records"] .iter() .map(|p| Cell::new(p).with_style(Attr::Bold)) .collect(), )); for id in chunk { let report = client .try_http_request::( Method::GET, &format!("/api/queue/reports/{id}"), None, ) .await; if let Some(report) = report { table.add_row(Row::new(vec![ Cell::new(id), Cell::new(report.domain()), Cell::new(report.type_()), Cell::new(&report.range_from().to_rfc822()), Cell::new(&report.range_to().to_rfc822()), Cell::new( &SpecificSize::new(report.num_records() as u32, Byte) .unwrap() .to_string(), ), ])); } } eprintln!(); table.printstd(); eprintln!(); if page_num + 1 != pages_total { eprintln!("\n--- Press any key to continue or 'q' to exit ---"); if let Ok('q' | 'Q') = stdout.read_char() { break; } } } eprintln!("\n{ids_len} queued message(s) found.") } ReportCommands::Status { ids } => { for id in ids { let report = client .try_http_request::( Method::GET, &format!("/api/queue/reports/{id}"), None, ) .await; let mut table = Table::new(); table.add_row(Row::new(vec![ Cell::new("ID").with_style(Attr::Bold), Cell::new(&id), ])); if let Some(report) = report { table.add_row(Row::new(vec![ Cell::new("Domain Name").with_style(Attr::Bold), Cell::new(report.domain()), ])); table.add_row(Row::new(vec![ Cell::new("Type").with_style(Attr::Bold), Cell::new(report.type_()), ])); table.add_row(Row::new(vec![ Cell::new("From Date").with_style(Attr::Bold), Cell::new(&report.range_from().to_rfc822()), ])); table.add_row(Row::new(vec![ Cell::new("To Date").with_style(Attr::Bold), Cell::new(&report.range_to().to_rfc822()), ])); table.add_row(Row::new(vec![ Cell::new("Records").with_style(Attr::Bold), Cell::new( &SpecificSize::new(report.num_records() as u32, Byte) .unwrap() .to_string(), ), ])); } else { table.add_row(Row::new(vec![ Cell::new_align("-- Not found --", format::Alignment::CENTER) .with_hspan(2), ])); } eprintln!(); table.printstd(); eprintln!(); } } ReportCommands::Cancel { ids } => { let mut success_count = 0; let mut failed_list = vec![]; for id in ids { let success = client .try_http_request::( Method::DELETE, &format!("/api/queue/reports/{id}"), None, ) .await; if success.unwrap_or_default() { success_count += 1; } else { failed_list.push(id); } } eprint!("\nRemoved {success_count} report(s)."); if !failed_list.is_empty() { eprint!( " Unable to remove report id(s): {}.", failed_list.join(", ") ); } eprintln!(); } } } } impl ReportFormat { fn id(&self) -> &'static str { match self { ReportFormat::Dmarc => "dmarc", ReportFormat::Tls => "tls", } } } ================================================ FILE: crates/common/Cargo.toml ================================================ [package] name = "common" version = "0.15.5" edition = "2024" build = "build.rs" [dependencies] utils = { path = "../utils" } nlp = { path = "../nlp" } store = { path = "../store" } trc = { path = "../trc" } directory = { path = "../directory" } types = { path = "../types" } jmap_proto = { path = "../jmap-proto" } imap_proto = { path = "../imap-proto" } sieve-rs = { version = "0.7", features = ["rkyv", "serde"] } mail-parser = { version = "0.11", features = ["full_encoding"] } mail-builder = { version = "0.4" } mail-auth = { version = "0.7.1" } mail-send = { version = "0.5", default-features = false, features = ["cram-md5", "ring", "tls12"] } smtp-proto = { version = "0.2", features = ["rkyv"] } dns-update = { version = "0.1.5" } calcard = { version = "0.3", features = ["rkyv"] } ahash = { version = "0.8.2", features = ["serde"] } parking_lot = "0.12.1" regex = "1.7.0" proxy-header = { version = "0.1.0", features = ["tokio"] } arc-swap = "1.6.0" rustls = { version = "0.23.5", default-features = false, features = ["std", "ring", "tls12"] } rustls-pemfile = "2.0" rustls-pki-types = { version = "1" } ring = { version = "0.17" } tokio = { version = "1.47", features = ["net", "macros"] } tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } futures = "0.3" rcgen = "0.12" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-webpki-roots", "http2", "stream"]} serde = { version = "1.0", features = ["derive"]} serde_json = "1.0" base64 = "0.22" x509-parser = "0.18" pem = "3.0" chrono = { version = "0.4", features = ["serde"] } hyper = { version = "1.0.1", features = ["server", "http1", "http2"] } opentelemetry = { version = "0.29" } opentelemetry_sdk = { version = "0.29" } opentelemetry-otlp = { version = "0.29", default-features = false, features = ["reqwest-client", "http-proto", "trace", "metrics", "logs", "internal-logs", "grpc-tonic", "tls-webpki-roots", "reqwest-rustls-webpki-roots"] } opentelemetry-semantic-conventions = { version = "0.29.0" } prometheus = { version = "0.14", default-features = false } imagesize = "0.14" sha1 = "0.10" sha2 = "0.10.6" md5 = "0.8.0" whatlang = "0.18" idna = "1.0" decancer = "3.0.1" unicode-security = "0.1.0" infer = "0.19" bincode = { version = "2.0", features = ["serde"] } hostname = "0.4.0" zip = "6.0" pwhash = "1.0.0" xxhash-rust = { version = "0.8.5", features = ["xxh3"] } psl = "2" aes-gcm-siv = "0.11.1" biscuit = "0.7.0" rsa = "0.9.2" p256 = { version = "0.13", features = ["ecdh"] } p384 = { version = "0.13", features = ["ecdh"] } num_cpus = "1.13.1" hashify = "0.2" rkyv = { version = "0.8.10", features = ["little_endian"] } indexmap = "2.7.1" tinyvec = "1.9.0" compact_str = { version = "0.9.0", features = ["rkyv", "serde"] } lz4_flex = { version = "0.12", features = ["frame"], default-features = false } [target.'cfg(unix)'.dependencies] privdrop = "0.5.3" libc = "0.2.126" [features] test_mode = [] enterprise = [] foundation = [] [dev-dependencies] tokio = { version = "1.47", features = ["full"] } ================================================ FILE: crates/common/build.rs ================================================ use std::collections::HashMap; use std::env; use std::fs; use std::path::Path; fn main() { let out_dir = env::var("OUT_DIR").unwrap(); let dest_path = Path::new(&out_dir).join("locales.rs"); // Read the YAML file let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); let repo_root = Path::new(&manifest_dir).parent().unwrap().parent().unwrap(); let yaml_path = repo_root.join("resources/locales/i18n.yml"); let yaml_content = fs::read_to_string(&yaml_path).unwrap_or_else(|_| panic!("Failed to read {yaml_path:?}")); let locales = parse_yaml(&yaml_content); let generated_code = generate_locale_code(&locales); fs::write(&dest_path, generated_code).expect("Failed to write generated locales."); println!("cargo:rerun-if-changed={}", yaml_path.display()); } fn parse_yaml(content: &str) -> HashMap> { let mut result: HashMap> = HashMap::new(); let mut current_key = None; for line in content.lines() { if let Some((key, value)) = line.split_once(':') { let is_translation = key .as_bytes() .first() .is_some_and(|&b| b.is_ascii_whitespace()); let key = key.trim(); if !key.starts_with('#') && !key.is_empty() { if !is_translation { current_key = result.entry(key.replace('.', "_")).or_default().into(); } else { current_key .as_mut() .unwrap() .insert(key.to_string(), value.trim().trim_matches('"').to_string()); } } } } result } fn generate_locale_code(locales: &HashMap>) -> String { let mut code = String::new(); code.push_str("#[derive(Debug, Clone)]\n"); code.push_str("pub struct Locale {\n"); for key in locales.keys() { code.push_str(&format!(" pub {}: &'static str,\n", key)); } code.push_str("}\n\n"); let mut languages = std::collections::HashSet::new(); for translations in locales.values() { for lang in translations.keys() { languages.insert(lang.clone()); } } for lang in &languages { code.push_str(&format!( "pub static {}_LOCALES: Locale = Locale {{\n", lang.to_uppercase() )); for (key, translations) in locales { let value = translations .get(lang) .unwrap_or_else(|| panic!("Missing: {}", key)); code.push_str(&format!(" {key}: {value:?},\n")); } code.push_str("};\n\n"); } code.push_str("pub fn locale(name: &str) -> Option<&'static Locale> {\n"); code.push_str(" hashify::tiny_map!(name.as_bytes(),\n"); for lang in &languages { code.push_str(&format!( " \"{}\" => &{}_LOCALES,\n", lang, lang.to_uppercase() )); } code.push_str(" )\n"); code.push_str("}\n"); code } ================================================ FILE: crates/common/src/addresses.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use directory::{Directory, backend::RcptType}; use std::borrow::Cow; use utils::config::{Config, utils::AsKey}; use crate::{ Server, config::smtp::session::AddressMapping, expr::{ V_RECIPIENT, Variable, functions::ResolveVariable, if_block::IfBlock, tokenizer::TokenMap, }, }; impl Server { pub async fn email_to_id( &self, directory: &Directory, email: &str, session_id: u64, ) -> trc::Result> { let mut address = self .core .smtp .session .rcpt .subaddressing .to_subaddress(self, email, session_id) .await; for _ in 0..2 { let result = directory.email_to_id(address.as_ref()).await?; if result.is_some() { return Ok(result); } else if let Some(catch_all) = self .core .smtp .session .rcpt .catch_all .to_catch_all(self, email, session_id) .await { address = catch_all; } else { break; } } Ok(None) } pub async fn rcpt( &self, directory: &Directory, email: &str, session_id: u64, ) -> trc::Result { // Expand subaddress let mut address = self .core .smtp .session .rcpt .subaddressing .to_subaddress(self, email, session_id) .await; for _ in 0..2 { let rcpt_type = directory.rcpt(address.as_ref()).await?; if rcpt_type != RcptType::Invalid { return Ok(rcpt_type); } else if let Some(catch_all) = self .core .smtp .session .rcpt .catch_all .to_catch_all(self, email, session_id) .await { address = catch_all; } else { break; } } Ok(RcptType::Invalid) } pub async fn vrfy( &self, directory: &Directory, address: &str, session_id: u64, ) -> trc::Result> { directory .vrfy( self.core .smtp .session .rcpt .subaddressing .to_subaddress(self, address, session_id) .await .as_ref(), ) .await } pub async fn expn( &self, directory: &Directory, address: &str, session_id: u64, ) -> trc::Result> { directory .expn( self.core .smtp .session .rcpt .subaddressing .to_subaddress(self, address, session_id) .await .as_ref(), ) .await } } impl AddressMapping { pub fn parse(config: &mut Config, key: impl AsKey) -> Self { let key = key.as_key(); if let Some(value) = config.value(key.as_str()) { match value { "true" => AddressMapping::Enable, "false" => AddressMapping::Disable, _ => { config.new_parse_error( key, format!("Invalid value for address mapping {value:?}",), ); AddressMapping::Disable } } } else if let Some(if_block) = IfBlock::try_parse( config, key, &TokenMap::default().with_variables_map([ ("address", V_RECIPIENT), ("email", V_RECIPIENT), ("rcpt", V_RECIPIENT), ]), ) { AddressMapping::Custom(if_block) } else { AddressMapping::Enable } } } struct Address<'x>(&'x str); impl ResolveVariable for Address<'_> { fn resolve_variable(&'_ self, _: u32) -> crate::expr::Variable<'_> { Variable::from(self.0) } fn resolve_global(&self, _: &str) -> Variable<'_> { Variable::Integer(0) } } impl AddressMapping { pub async fn to_subaddress<'x, 'y: 'x>( &'x self, core: &Server, address: &'y str, session_id: u64, ) -> Cow<'x, str> { match self { AddressMapping::Enable => { if let Some((local_part, domain_part)) = address.rsplit_once('@') && let Some((local_part, _)) = local_part.split_once('+') { return format!("{}@{}", local_part, domain_part).into(); } } AddressMapping::Custom(if_block) => { if let Some(result) = core .eval_if::(if_block, &Address(address), session_id) .await { return result.into(); } } AddressMapping::Disable => (), } address.into() } pub async fn to_catch_all<'x, 'y: 'x>( &'x self, core: &Server, address: &'y str, session_id: u64, ) -> Option> { match self { AddressMapping::Enable => address .rsplit_once('@') .map(|(_, domain_part)| format!("@{}", domain_part)) .map(Cow::Owned), AddressMapping::Custom(if_block) => core .eval_if::(if_block, &Address(address), session_id) .await .map(Cow::Owned), AddressMapping::Disable => None, } } } ================================================ FILE: crates/common/src/auth/access_token.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{AccessToken, ResourceToken, TenantInfo, roles::RolePermissions}; use crate::{ Server, ipc::BroadcastEvent, listener::limiter::{ConcurrencyLimiter, LimiterResult}, }; use ahash::AHashSet; use directory::{ Permission, Principal, PrincipalData, QueryParams, Type, backend::internal::{ lookup::DirectoryStore, manage::{ChangedPrincipals, ManageDirectory}, }, }; use std::{ hash::{DefaultHasher, Hash, Hasher}, sync::Arc, }; use store::{query::acl::AclQuery, rand}; use trc::AddContext; use types::{acl::Acl, collection::Collection}; use utils::map::{ bitmap::{Bitmap, BitmapItem}, vec_map::VecMap, }; pub enum PrincipalOrId { Principal(Principal), Id(u32), } impl Server { async fn build_access_token_from_principal( &self, principal: Principal, revision: u64, ) -> trc::Result { let mut role_permissions = RolePermissions::default(); // Extract data let mut object_quota = self.core.jmap.max_objects; let mut description = None; let mut tenant_id = None; let mut quota = None; let mut locale = None; let mut member_of = Vec::new(); let mut emails = Vec::new(); for data in principal.data { match data { PrincipalData::Tenant(v) => tenant_id = Some(v), PrincipalData::MemberOf(v) => member_of.push(v), PrincipalData::Role(v) => { role_permissions.union(self.get_role_permissions(v).await?.as_ref()); } PrincipalData::Permission { permission_id, grant, } => { if grant { role_permissions.enabled.set(permission_id as usize); } else { role_permissions.disabled.set(permission_id as usize); } } PrincipalData::DiskQuota(v) => quota = Some(v), PrincipalData::ObjectQuota { quota, typ } => { object_quota[typ as usize] = quota; } PrincipalData::Description(v) => description = Some(v), PrincipalData::PrimaryEmail(v) => { if emails.is_empty() { emails.push(v); } else { emails.insert(0, v); } } PrincipalData::EmailAlias(v) => { emails.push(v); } PrincipalData::Locale(v) => locale = Some(v), _ => (), } } // Apply principal permissions let mut permissions = role_permissions.finalize(); let mut tenant = None; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] { use directory::{QueryParams, ROLE_USER}; if let Some(tenant_id) = tenant_id { if self.is_enterprise_edition() { // Limit tenant permissions permissions.intersection(&self.get_role_permissions(tenant_id).await?.enabled); // Obtain tenant quota tenant = Some(TenantInfo { id: tenant_id, quota: self .store() .query(QueryParams::id(tenant_id).with_return_member_of(false)) .await .caused_by(trc::location!())? .ok_or_else(|| { trc::SecurityEvent::Unauthorized .into_err() .details("Tenant not found") .id(tenant_id) .caused_by(trc::location!()) })? .quota() .unwrap_or_default(), }); } else { // Enterprise edition downgrade, remove any tenant administrator permissions permissions.intersection(&self.get_role_permissions(ROLE_USER).await?.enabled); } } } // SPDX-SnippetEnd // Build member of and e-mail addresses for &group_id in &member_of { if let Some(group) = self .store() .query(QueryParams::id(group_id).with_return_member_of(false)) .await .caused_by(trc::location!())? && group.typ == Type::Group { emails.extend(group.into_email_addresses()); } } // Build access token let mut access_token = AccessToken { primary_id: principal.id, member_of, access_to: VecMap::new(), tenant, name: principal.name, description, emails, quota: quota.unwrap_or_default(), locale, permissions, object_quota, concurrent_imap_requests: self.core.imap.rate_concurrent.map(ConcurrencyLimiter::new), concurrent_http_requests: self .core .jmap .request_max_concurrent .map(ConcurrencyLimiter::new), concurrent_uploads: self .core .jmap .upload_max_concurrent .map(ConcurrencyLimiter::new), obj_size: 0, revision, }; for grant_account_id in [access_token.primary_id] .into_iter() .chain(access_token.member_of.iter().copied()) { for acl_item in self .store() .acl_query(AclQuery::HasAccess { grant_account_id }) .await .caused_by(trc::location!())? { if !access_token.is_member(acl_item.to_account_id) { let acl = Bitmap::::from(acl_item.permissions); let collection = acl_item.to_collection; if !collection.is_valid() { return Err(trc::StoreEvent::DataCorruption .ctx(trc::Key::Reason, "Corrupted collection found in ACL key.") .details(format!("{acl_item:?}")) .account_id(grant_account_id) .caused_by(trc::location!())); } let mut collections: Bitmap = Bitmap::new(); if acl.contains(Acl::Read) { collections.insert(collection); } if acl.contains(Acl::ReadItems) && let Some(child_col) = collection.child_collection() { collections.insert(child_col); } if !collections.is_empty() { access_token .access_to .get_mut_or_insert_with(acl_item.to_account_id, Bitmap::new) .union(&collections); } } } } Ok(access_token.update_size()) } async fn build_access_token(&self, account_id: u32, revision: u64) -> trc::Result { let err = match self .directory() .query(QueryParams::id(account_id).with_return_member_of(true)) .await { Ok(Some(principal)) => { return self .build_access_token_from_principal(principal, revision) .await; } Ok(None) => Err(trc::AuthEvent::Error .into_err() .details("Account not found.") .caused_by(trc::location!())), Err(err) => Err(err), }; match &self.core.jmap.fallback_admin { Some((_, secret)) if account_id == u32::MAX => { self.build_access_token_from_principal(Principal::fallback_admin(secret), revision) .await } _ => err, } } pub async fn get_access_token( &self, principal: impl Into, ) -> trc::Result> { let principal = principal.into(); // Obtain current revision let principal_id = principal.id(); match self .inner .cache .access_tokens .get_value_or_guard_async(&principal_id) .await { Ok(token) => Ok(token), Err(guard) => { let revision = rand::random::(); let token: Arc = match principal { PrincipalOrId::Principal(principal) => { self.build_access_token_from_principal(principal, revision) .await? } PrincipalOrId::Id(account_id) => { self.build_access_token(account_id, revision).await? } } .into(); let _ = guard.insert(token.clone()); Ok(token) } } } pub async fn invalidate_principal_caches(&self, changed_principals: ChangedPrincipals) { let mut nested_principals = Vec::new(); let mut changed_ids = AHashSet::new(); let mut changed_names = Vec::new(); for (id, changed_principal) in changed_principals.iter() { changed_ids.insert(*id); if changed_principal.name_change { self.inner.cache.files.remove(id); self.inner.cache.contacts.remove(id); self.inner.cache.events.remove(id); self.inner.cache.scheduling.remove(id); changed_names.push(*id); } if changed_principal.member_change { if changed_principal.typ == Type::Tenant { match self .store() .list_principals( None, (*id).into(), &[Type::Individual, Type::Group, Type::Role, Type::ApiKey], false, 0, 0, ) .await { Ok(principals) => { for principal in principals.items { changed_ids.insert(principal.id()); } } Err(err) => { trc::error!( err.details("Failed to list principals") .caused_by(trc::location!()) .account_id(*id) ); } } } else { nested_principals.push(*id); } } } if !nested_principals.is_empty() { let mut ids = nested_principals.into_iter(); let mut ids_stack = vec![]; loop { if let Some(id) = ids.next() { // Skip if already fetched if !changed_ids.insert(id) { continue; } // Obtain principal match self.store().get_members(id).await { Ok(members) => { ids_stack.push(ids); ids = members.into_iter(); } Err(err) => { trc::error!( err.details("Failed to obtain principal") .caused_by(trc::location!()) .account_id(id) ); } } } else if let Some(prev_ids) = ids_stack.pop() { ids = prev_ids; } else { break; } } } // Invalidate access tokens in cluster if !changed_ids.is_empty() { let mut ids = Vec::with_capacity(changed_ids.len()); for id in changed_ids { self.inner.cache.permissions.remove(&id); self.inner.cache.access_tokens.remove(&id); ids.push(id); } self.cluster_broadcast(BroadcastEvent::InvalidateAccessTokens(ids)) .await; } // Invalidate DAV caches if !changed_names.is_empty() { self.cluster_broadcast(BroadcastEvent::InvalidateGroupwareCache(changed_names)) .await; } } } impl From for PrincipalOrId { fn from(id: u32) -> Self { Self::Id(id) } } impl From for PrincipalOrId { fn from(principal: Principal) -> Self { Self::Principal(principal) } } impl PrincipalOrId { pub fn id(&self) -> u32 { match self { Self::Principal(principal) => principal.id(), Self::Id(id) => *id, } } } impl AccessToken { pub fn from_id(primary_id: u32) -> Self { Self { primary_id, ..Default::default() } } pub fn with_access_to(self, access_to: VecMap>) -> Self { Self { access_to, ..self } } pub fn with_permission(mut self, permission: Permission) -> Self { self.permissions.set(permission.id() as usize); self } pub fn with_tenant_id(mut self, tenant_id: Option) -> Self { self.tenant = tenant_id.map(|id| TenantInfo { id, quota: 0 }); self } pub fn state(&self) -> u32 { // Hash state let mut s = DefaultHasher::new(); self.member_of.hash(&mut s); self.access_to.hash(&mut s); s.finish() as u32 } #[inline(always)] pub fn primary_id(&self) -> u32 { self.primary_id } #[inline(always)] pub fn tenant_id(&self) -> Option { self.tenant.as_ref().map(|t| t.id) } pub fn secondary_ids(&self) -> impl Iterator { self.member_of .iter() .chain(self.access_to.iter().map(|(id, _)| id)) } pub fn member_ids(&self) -> impl Iterator { [self.primary_id] .into_iter() .chain(self.member_of.iter().copied()) } pub fn all_ids(&self) -> impl Iterator { [self.primary_id] .into_iter() .chain(self.member_of.iter().copied()) .chain(self.access_to.iter().map(|(id, _)| *id)) } pub fn all_ids_by_collection(&self, collection: Collection) -> impl Iterator { [self.primary_id] .into_iter() .chain(self.member_of.iter().copied()) .chain(self.access_to.iter().filter_map(move |(id, cols)| { if cols.contains(collection) { Some(*id) } else { None } })) } pub fn is_member(&self, account_id: u32) -> bool { self.primary_id == account_id || self.member_of.contains(&account_id) || self.has_permission(Permission::Impersonate) } pub fn is_primary_id(&self, account_id: u32) -> bool { self.primary_id == account_id } #[inline(always)] pub fn has_permission(&self, permission: Permission) -> bool { self.permissions.get(permission.id() as usize) } pub fn assert_has_permission(&self, permission: Permission) -> trc::Result { if self.has_permission(permission) { Ok(true) } else { Err(trc::SecurityEvent::Unauthorized .into_err() .details(permission.name())) } } pub fn permissions(&self) -> Vec { const USIZE_BITS: usize = std::mem::size_of::() * 8; const USIZE_MASK: u32 = USIZE_BITS as u32 - 1; let mut permissions = Vec::new(); for (block_num, bytes) in self.permissions.inner().iter().enumerate() { let mut bytes = *bytes; while bytes != 0 { let item = USIZE_MASK - bytes.leading_zeros(); bytes ^= 1 << item; if let Some(permission) = Permission::from_id(((block_num * USIZE_BITS) + item as usize) as u32) { permissions.push(permission); } } } permissions } #[inline(always)] pub fn object_quota(&self, collection: Collection) -> u32 { self.object_quota[collection as usize] } pub fn is_shared(&self, account_id: u32) -> bool { !self.is_member(account_id) && self.access_to.iter().any(|(id, _)| *id == account_id) } pub fn shared_accounts(&self, collection: Collection) -> impl Iterator { self.member_of .iter() .chain(self.access_to.iter().filter_map(move |(id, cols)| { if cols.contains(collection) { id.into() } else { None } })) } pub fn has_access(&self, to_account_id: u32, to_collection: impl Into) -> bool { let to_collection = to_collection.into(); self.is_member(to_account_id) || self.access_to.iter().any(|(id, collections)| { *id == to_account_id && collections.contains(to_collection) }) } pub fn has_account_access(&self, to_account_id: u32) -> bool { self.is_member(to_account_id) || self.access_to.iter().any(|(id, _)| *id == to_account_id) } pub fn as_resource_token(&self) -> ResourceToken { ResourceToken { account_id: self.primary_id, quota: self.quota, tenant: self.tenant, } } pub fn is_http_request_allowed(&self) -> LimiterResult { self.concurrent_http_requests .as_ref() .map_or(LimiterResult::Disabled, |limiter| limiter.is_allowed()) } pub fn is_imap_request_allowed(&self) -> LimiterResult { self.concurrent_imap_requests .as_ref() .map_or(LimiterResult::Disabled, |limiter| limiter.is_allowed()) } pub fn is_upload_allowed(&self) -> LimiterResult { self.concurrent_uploads .as_ref() .map_or(LimiterResult::Disabled, |limiter| limiter.is_allowed()) } pub fn update_size(mut self) -> Self { self.obj_size = (std::mem::size_of::() + (self.member_of.len() * std::mem::size_of::()) + (self.access_to.len() * (std::mem::size_of::() + std::mem::size_of::())) + self.name.len() + self.description.as_ref().map_or(0, |v| v.len()) + self.locale.as_ref().map_or(0, |v| v.len()) + self.emails.iter().map(|v| v.len()).sum::()) as u64; self } } ================================================ FILE: crates/common/src/auth/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{Server, listener::limiter::ConcurrencyLimiter}; use directory::{ Directory, FALLBACK_ADMIN_ID, Permission, Permissions, Principal, QueryParams, Type, backend::internal::lookup::DirectoryStore, core::secret::verify_secret_hash, }; use mail_send::Credentials; use oauth::GrantType; use std::{net::IpAddr, sync::Arc}; use types::collection::Collection; use utils::{ cache::CacheItemWeight, map::{bitmap::Bitmap, vec_map::VecMap}, }; pub mod access_token; pub mod oauth; pub mod rate_limit; pub mod roles; pub mod sasl; #[derive(Debug, Default)] pub struct AccessToken { pub primary_id: u32, pub member_of: Vec, pub access_to: VecMap>, pub name: String, pub description: Option, pub locale: Option, pub emails: Vec, pub quota: u64, pub object_quota: [u32; Collection::MAX], pub permissions: Permissions, pub tenant: Option, pub concurrent_http_requests: Option, pub concurrent_imap_requests: Option, pub concurrent_uploads: Option, pub revision: u64, pub obj_size: u64, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct TenantInfo { pub id: u32, pub quota: u64, } #[derive(Debug, Clone, Default)] pub struct ResourceToken { pub account_id: u32, pub quota: u64, pub tenant: Option, } pub struct AuthRequest<'x> { credentials: Credentials, session_id: u64, remote_ip: IpAddr, return_member_of: bool, allow_api_access: bool, directory: Option<&'x Directory>, } impl Server { pub async fn authenticate(&self, req: &AuthRequest<'_>) -> trc::Result> { // Resolve directory let directory = req.directory.unwrap_or(&self.core.storage.directory); // Validate credentials match &req.credentials { Credentials::OAuthBearer { token } if !directory.has_bearer_token_support() => { match self .validate_access_token(GrantType::AccessToken.into(), token) .await { Ok(token_into) => self.get_access_token(token_into.account_id).await, Err(err) => Err(err), } } _ => match self.authenticate_credentials(req, directory).await { Ok(principal) => self.get_access_token(principal).await, Err(err) => Err(err), }, } .and_then(|token| { token .assert_has_permission(Permission::Authenticate) .map(|_| token) }) } async fn authenticate_credentials( &self, req: &AuthRequest<'_>, directory: &Directory, ) -> trc::Result { // First try to authenticate the user against the default directory let result = match directory .query( QueryParams::credentials(&req.credentials) .with_return_member_of(req.return_member_of), ) .await { Ok(Some(principal)) => { trc::event!( Auth(trc::AuthEvent::Success), AccountName = principal.name().to_string(), AccountId = principal.id(), SpanId = req.session_id, ); return Ok(principal); } Ok(None) => Ok(()), Err(err) => { if err.matches(trc::EventType::Auth(trc::AuthEvent::MissingTotp)) { return Err(err); } else { Err(err) } } }; match &req.credentials { Credentials::Plain { username, secret } => { // Then check if the credentials match the fallback admin or master user match (&self.core.jmap.fallback_admin, &self.core.jmap.master_user) { (Some((fallback_admin, fallback_pass)), _) if username == fallback_admin => { if verify_secret_hash(fallback_pass, secret).await? { trc::event!( Auth(trc::AuthEvent::Success), AccountName = username.clone(), SpanId = req.session_id, ); return Ok(Principal::fallback_admin(fallback_pass)); } } (_, Some((master_user, master_pass))) if username.ends_with(master_user) => { if verify_secret_hash(master_pass, secret).await? { let username = username.strip_suffix(master_user).unwrap(); let username = username.strip_suffix('%').unwrap_or(username); if let Some(principal) = directory .query( QueryParams::name(username) .with_return_member_of(req.return_member_of), ) .await? { trc::event!( Auth(trc::AuthEvent::Success), AccountName = username.to_string(), SpanId = req.session_id, AccountId = principal.id(), Type = principal.typ().description(), ); return Ok(principal); } } } _ => { // Validate API credentials if req.allow_api_access && let Ok(Some(principal)) = self .store() .query( QueryParams::credentials(&req.credentials) .with_return_member_of(req.return_member_of), ) .await && principal.typ == Type::ApiKey { trc::event!( Auth(trc::AuthEvent::Success), AccountName = principal.name().to_string(), AccountId = principal.id(), SpanId = req.session_id, ); return Ok(principal); } } } } Credentials::OAuthBearer { token } if directory.has_bearer_token_support() => { // Check for bearer tokens issued locally if let Ok(token_info) = self .validate_access_token(GrantType::AccessToken.into(), token) .await { let principal = if token_info.account_id != FALLBACK_ADMIN_ID { directory .query( QueryParams::id(token_info.account_id) .with_return_member_of(req.return_member_of), ) .await .unwrap_or_default() } else if let Some((_, fallback_pass)) = &self.core.jmap.fallback_admin { Principal::fallback_admin(fallback_pass).into() } else { None }; if let Some(principal) = principal { trc::event!( Auth(trc::AuthEvent::Success), AccountName = principal.name().to_string(), AccountId = principal.id(), SpanId = req.session_id, ); return Ok(principal); } } } _ => (), }; if let Err(err) = result { Err(err) } else if self.has_auth_fail2ban() { let login = req.credentials.login(); if self.is_auth_fail2banned(req.remote_ip, login).await? { Err(trc::SecurityEvent::AuthenticationBan .into_err() .ctx(trc::Key::RemoteIp, req.remote_ip) .ctx_opt(trc::Key::AccountName, login.map(|s| s.to_string()))) } else { Err(trc::AuthEvent::Failed .ctx(trc::Key::RemoteIp, req.remote_ip) .ctx_opt(trc::Key::AccountName, login.map(|s| s.to_string()))) } } else { Err(trc::AuthEvent::Failed .ctx(trc::Key::RemoteIp, req.remote_ip) .ctx_opt( trc::Key::AccountName, req.credentials.login().map(|s| s.to_string()), )) } } } impl<'x> AuthRequest<'x> { pub fn from_credentials( credentials: Credentials, session_id: u64, remote_ip: IpAddr, ) -> Self { Self { credentials, session_id, remote_ip, return_member_of: true, directory: None, allow_api_access: false, } } pub fn from_plain( user: impl Into, pass: impl Into, session_id: u64, remote_ip: IpAddr, ) -> Self { Self::from_credentials( Credentials::Plain { username: user.into(), secret: pass.into(), }, session_id, remote_ip, ) } pub fn without_members(mut self) -> Self { self.return_member_of = false; self } pub fn with_directory(mut self, directory: &'x Directory) -> Self { self.directory = Some(directory); self } pub fn with_api_access(mut self, allow_api_access: bool) -> Self { self.allow_api_access = allow_api_access; self } } impl CacheItemWeight for AccessToken { fn weight(&self) -> u64 { self.obj_size } } pub(crate) trait CredentialsUsername { fn login(&self) -> Option<&str>; } impl CredentialsUsername for Credentials { fn login(&self) -> Option<&str> { match self { Credentials::Plain { username, .. } | Credentials::XOauth2 { username, .. } => { username.as_str().into() } Credentials::OAuthBearer { .. } => None, } } } ================================================ FILE: crates/common/src/auth/oauth/config.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use biscuit::{ jwa::{Algorithm, SignatureAlgorithm}, jwk::{ AlgorithmParameters, CommonParameters, EllipticCurve, EllipticCurveKeyParameters, EllipticCurveKeyType, JWK, JWKSet, OctetKeyParameters, OctetKeyType, PublicKeyUse, RSAKeyParameters, RSAKeyType, }, jws::Secret, }; use ring::signature::{self, KeyPair}; use rsa::{RsaPublicKey, pkcs1::DecodeRsaPublicKey, traits::PublicKeyParts}; use store::rand::{Rng, distr::Alphanumeric, rng}; use utils::config::Config; use x509_parser::num_bigint::BigUint; use crate::{ config::{build_ecdsa_pem, build_rsa_keypair}, manager::webadmin::Resource, }; #[derive(Clone)] pub struct OAuthConfig { pub oauth_key: String, pub oauth_expiry_user_code: u64, pub oauth_expiry_auth_code: u64, pub oauth_expiry_token: u64, pub oauth_expiry_refresh_token: u64, pub oauth_expiry_refresh_token_renew: u64, pub oauth_max_auth_attempts: u32, pub allow_anonymous_client_registration: bool, pub require_client_authentication: bool, pub oidc_expiry_id_token: u64, pub oidc_signing_secret: Secret, pub oidc_signature_algorithm: SignatureAlgorithm, pub oidc_jwks: Resource>, } impl OAuthConfig { pub fn parse(config: &mut Config) -> Self { let oidc_signature_algorithm = match config.value("oauth.oidc.signature-algorithm") { Some(alg) => match alg.to_uppercase().as_str() { "HS256" => SignatureAlgorithm::HS256, "HS384" => SignatureAlgorithm::HS384, "HS512" => SignatureAlgorithm::HS512, "RS256" => SignatureAlgorithm::RS256, "RS384" => SignatureAlgorithm::RS384, "RS512" => SignatureAlgorithm::RS512, "ES256" => SignatureAlgorithm::ES256, "ES384" => SignatureAlgorithm::ES384, "PS256" => SignatureAlgorithm::PS256, "PS384" => SignatureAlgorithm::PS384, "PS512" => SignatureAlgorithm::PS512, _ => { config.new_parse_error( "oauth.oidc.signature-algorithm", format!("Invalid OIDC signature algorithm: {}", alg), ); SignatureAlgorithm::HS256 } }, None => SignatureAlgorithm::HS256, }; let rand_key = rng() .sample_iter(Alphanumeric) .take(64) .map(char::from) .collect::() .into_bytes(); let (oidc_signing_secret, algorithm) = match oidc_signature_algorithm { SignatureAlgorithm::None | SignatureAlgorithm::HS256 | SignatureAlgorithm::HS384 | SignatureAlgorithm::HS512 => { let key = config .value("oauth.oidc.signature-key") .map(|s| s.to_string().into_bytes()) .unwrap_or(rand_key); ( Secret::Bytes(key.clone()), AlgorithmParameters::OctetKey(OctetKeyParameters { key_type: OctetKeyType::Octet, value: key, }), ) } SignatureAlgorithm::RS256 | SignatureAlgorithm::RS384 | SignatureAlgorithm::RS512 | SignatureAlgorithm::PS256 | SignatureAlgorithm::PS384 | SignatureAlgorithm::PS512 => parse_rsa_key(config).unwrap_or_else(|| { ( Secret::Bytes(rand_key.clone()), AlgorithmParameters::OctetKey(OctetKeyParameters { key_type: OctetKeyType::Octet, value: rand_key, }), ) }), SignatureAlgorithm::ES256 | SignatureAlgorithm::ES384 | SignatureAlgorithm::ES512 => { parse_ecdsa_key(config, oidc_signature_algorithm).unwrap_or_else(|| { ( Secret::Bytes(rand_key.clone()), AlgorithmParameters::OctetKey(OctetKeyParameters { key_type: OctetKeyType::Octet, value: rand_key, }), ) }) } }; let oidc_jwks = Resource { content_type: "application/json".into(), contents: serde_json::to_string(&JWKSet { keys: vec![JWK { common: CommonParameters { public_key_use: PublicKeyUse::Signature.into(), algorithm: Algorithm::Signature(oidc_signature_algorithm).into(), key_id: "default".to_string().into(), ..Default::default() }, algorithm, additional: (), }], }) .unwrap_or_default() .into_bytes(), }; OAuthConfig { oauth_key: config .value("oauth.key") .map(|s| s.to_string()) .unwrap_or_else(|| { rng() .sample_iter(Alphanumeric) .take(64) .map(char::from) .collect::() }), oauth_expiry_user_code: config .property_or_default::("oauth.expiry.user-code", "30m") .unwrap_or_else(|| Duration::from_secs(30 * 60)) .as_secs(), oauth_expiry_auth_code: config .property_or_default::("oauth.expiry.auth-code", "10m") .unwrap_or_else(|| Duration::from_secs(10 * 60)) .as_secs(), oauth_expiry_token: config .property_or_default::("oauth.expiry.token", "1h") .unwrap_or_else(|| Duration::from_secs(60 * 60)) .as_secs(), oauth_expiry_refresh_token: config .property_or_default::("oauth.expiry.refresh-token", "30d") .unwrap_or_else(|| Duration::from_secs(30 * 24 * 60 * 60)) .as_secs(), oauth_expiry_refresh_token_renew: config .property_or_default::("oauth.expiry.refresh-token-renew", "4d") .unwrap_or_else(|| Duration::from_secs(4 * 24 * 60 * 60)) .as_secs(), oauth_max_auth_attempts: config .property_or_default("oauth.auth.max-attempts", "3") .unwrap_or(10), oidc_expiry_id_token: config .property_or_default::("oauth.oidc.expiry.id-token", "15m") .unwrap_or_else(|| Duration::from_secs(15 * 60)) .as_secs(), allow_anonymous_client_registration: config .property_or_default("oauth.client-registration.anonymous", "false") .unwrap_or(false), require_client_authentication: config .property_or_default("oauth.client-registration.require", "false") .unwrap_or(true), oidc_signing_secret, oidc_signature_algorithm, oidc_jwks, } } } impl Default for OAuthConfig { fn default() -> Self { Self { oauth_key: Default::default(), oauth_expiry_user_code: Default::default(), oauth_expiry_auth_code: Default::default(), oauth_expiry_token: Default::default(), oauth_expiry_refresh_token: Default::default(), oauth_expiry_refresh_token_renew: Default::default(), oauth_max_auth_attempts: Default::default(), oidc_expiry_id_token: Default::default(), allow_anonymous_client_registration: Default::default(), require_client_authentication: Default::default(), oidc_signing_secret: Secret::Bytes("secret".to_string().into_bytes()), oidc_signature_algorithm: SignatureAlgorithm::HS256, oidc_jwks: Resource { content_type: "application/json".into(), contents: serde_json::to_string(&JWKSet::<()> { keys: vec![] }) .unwrap_or_default() .into_bytes(), }, } } } fn parse_rsa_key(config: &mut Config) -> Option<(Secret, AlgorithmParameters)> { let rsa_key_pair = match build_rsa_keypair(config.value_require("oauth.oidc.signature-key")?) { Ok(key) => key, Err(err) => { config.new_build_error( "oauth.oidc.signature-key", format!("Failed to build RSA key: {}", err), ); return None; } }; let rsa_public_key = match RsaPublicKey::from_pkcs1_der(rsa_key_pair.public_key().as_ref()) { Ok(key) => key, Err(err) => { config.new_build_error( "oauth.oidc.signature-key", format!("Failed to obtain RSA public key: {}", err), ); return None; } }; let rsa_key_params = RSAKeyParameters { key_type: RSAKeyType::RSA, n: BigUint::from_bytes_be(&rsa_public_key.n().to_bytes_be()), e: BigUint::from_bytes_be(&rsa_public_key.e().to_bytes_be()), ..Default::default() }; ( Secret::RsaKeyPair(rsa_key_pair.into()), AlgorithmParameters::RSA(rsa_key_params), ) .into() } fn parse_ecdsa_key( config: &mut Config, oidc_signature_algorithm: SignatureAlgorithm, ) -> Option<(Secret, AlgorithmParameters)> { let (alg, curve) = match oidc_signature_algorithm { SignatureAlgorithm::ES256 => ( &signature::ECDSA_P256_SHA256_FIXED_SIGNING, EllipticCurve::P256, ), SignatureAlgorithm::ES384 => ( &signature::ECDSA_P384_SHA384_FIXED_SIGNING, EllipticCurve::P384, ), _ => unreachable!(), }; let ecdsa_key_pair = match build_ecdsa_pem(alg, config.value_require("oauth.oidc.signature-key")?) { Ok(key) => key, Err(err) => { config.new_build_error( "oauth.oidc.signature-key", format!("Failed to build ECDSA key: {}", err), ); return None; } }; let ecdsa_public_key = ecdsa_key_pair.public_key().as_ref(); let (x, y) = match oidc_signature_algorithm { SignatureAlgorithm::ES256 => { let points = match p256::EncodedPoint::from_bytes(ecdsa_public_key) { Ok(points) => points, Err(err) => { config.new_build_error( "oauth.oidc.signature-key", format!("Failed to parse ECDSA key: {}", err), ); return None; } }; ( points.x().map(|x| x.to_vec()).unwrap_or_default(), points.y().map(|y| y.to_vec()).unwrap_or_default(), ) } SignatureAlgorithm::ES384 => { let points = match p384::EncodedPoint::from_bytes(ecdsa_public_key) { Ok(points) => points, Err(err) => { config.new_build_error( "oauth.oidc.signature-key", format!("Failed to parse ECDSA key: {}", err), ); return None; } }; ( points.x().map(|x| x.to_vec()).unwrap_or_default(), points.y().map(|y| y.to_vec()).unwrap_or_default(), ) } _ => unreachable!(), }; let ecdsa_key_params = EllipticCurveKeyParameters { key_type: EllipticCurveKeyType::EC, curve, x, y, d: None, }; ( Secret::EcdsaKeyPair(ecdsa_key_pair.into()), AlgorithmParameters::EllipticCurve(ecdsa_key_params), ) .into() } ================================================ FILE: crates/common/src/auth/oauth/crypto.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use aes_gcm_siv::{AeadInPlace, Aes256GcmSiv, KeyInit, Nonce, aead::Aead}; use store::blake3; pub struct SymmetricEncrypt { aes: Aes256GcmSiv, } //TODO: Remove allow deprecated when aes-gcm is updated #[allow(deprecated)] impl SymmetricEncrypt { pub const ENCRYPT_TAG_LEN: usize = 16; pub const NONCE_LEN: usize = 12; pub fn new(key: &[u8], context: &str) -> Self { SymmetricEncrypt { aes: Aes256GcmSiv::new( &sha1::digest::generic_array::GenericArray::clone_from_slice( &blake3::derive_key(context, key)[..], ), ), } } #[allow(clippy::ptr_arg)] pub fn encrypt_in_place(&self, bytes: &mut Vec, nonce: &[u8]) -> Result<(), String> { self.aes .encrypt_in_place(Nonce::from_slice(nonce), b"", bytes) .map_err(|e| e.to_string()) } pub fn encrypt(&self, bytes: &[u8], nonce: &[u8]) -> Result, String> { self.aes .encrypt(Nonce::from_slice(nonce), bytes) .map_err(|e| e.to_string()) } pub fn decrypt(&self, bytes: &[u8], nonce: &[u8]) -> Result, String> { self.aes .decrypt(Nonce::from_slice(nonce), bytes) .map_err(|e| e.to_string()) } } ================================================ FILE: crates/common/src/auth/oauth/introspect.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use serde::{Deserialize, Serialize}; use trc::{AddContext, AuthEvent, EventType}; use crate::{Server, auth::AccessToken}; #[derive(Debug, Default, Clone, Eq, PartialEq, Deserialize, Serialize)] pub struct OAuthIntrospect { #[serde(default)] pub active: bool, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub scope: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub client_id: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub username: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub token_type: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub exp: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub iat: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub nbf: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub sub: Option, } impl Server { pub async fn introspect_access_token( &self, token: &str, access_token: &AccessToken, ) -> trc::Result { match self.validate_access_token(None, token).await { Ok(token_info) => Ok(OAuthIntrospect { active: true, client_id: Some(token_info.client_id), username: if access_token.primary_id() == token_info.account_id { access_token.name.clone() } else { self.get_access_token(token_info.account_id) .await .caused_by(trc::location!())? .name .clone() } .into(), token_type: Some("bearer".into()), exp: Some(token_info.expiry as i64), iat: Some(token_info.issued_at as i64), ..Default::default() }), Err(err) if matches!( err.event_type(), EventType::Auth(AuthEvent::Error) | EventType::Auth(AuthEvent::TokenExpired) ) => { Ok(OAuthIntrospect::default()) } Err(err) => Err(err), } } } ================================================ FILE: crates/common/src/auth/oauth/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod config; pub mod crypto; pub mod introspect; pub mod oidc; pub mod registration; pub mod token; pub const DEVICE_CODE_LEN: usize = 40; pub const USER_CODE_LEN: usize = 8; pub const RANDOM_CODE_LEN: usize = 32; pub const CLIENT_ID_MAX_LEN: usize = 20; pub const USER_CODE_ALPHABET: &[u8] = b"ABCDEFGHJKLMNPQRSTUVWXYZ23456789"; // No 0, O, I, 1 #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum GrantType { AccessToken, RefreshToken, LiveTracing, LiveMetrics, Troubleshoot, Rsvp, } impl GrantType { pub fn as_str(&self) -> &'static str { match self { GrantType::AccessToken => "access_token", GrantType::RefreshToken => "refresh_token", GrantType::LiveTracing => "live_tracing", GrantType::LiveMetrics => "live_metrics", GrantType::Troubleshoot => "troubleshoot", GrantType::Rsvp => "rsvp", } } pub fn id(&self) -> u8 { match self { GrantType::AccessToken => 0, GrantType::RefreshToken => 1, GrantType::LiveTracing => 2, GrantType::LiveMetrics => 3, GrantType::Troubleshoot => 4, GrantType::Rsvp => 5, } } pub fn from_id(id: u8) -> Option { match id { 0 => Some(GrantType::AccessToken), 1 => Some(GrantType::RefreshToken), 2 => Some(GrantType::LiveTracing), 3 => Some(GrantType::LiveMetrics), 4 => Some(GrantType::Troubleshoot), 5 => Some(GrantType::Rsvp), _ => None, } } } ================================================ FILE: crates/common/src/auth/oauth/oidc.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::fmt; use biscuit::{ClaimsSet, JWT, RegisteredClaims, SingleOrMultiple, jws::RegisteredHeader}; use serde::{ Deserialize, Deserializer, Serialize, de::{self, Visitor}, }; use store::write::now; use crate::Server; #[derive(Debug, Default, Clone, Eq, PartialEq, Deserialize, Serialize)] pub struct Userinfo { #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub sub: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub name: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub given_name: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub family_name: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub middle_name: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub nickname: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub preferred_username: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub profile: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub picture: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub website: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub email: Option, #[serde(default, deserialize_with = "any_bool")] #[serde(skip_serializing_if = "std::ops::Not::not")] pub email_verified: bool, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub zoneinfo: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub locale: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub updated_at: Option, } #[derive(Debug, Default, Eq, PartialEq, Deserialize, Serialize)] pub struct StandardClaims { #[serde(skip_serializing_if = "Option::is_none")] #[serde(default)] pub nonce: Option, #[serde(skip_serializing_if = "Option::is_none")] #[serde(default)] pub preferred_username: Option, #[serde(skip_serializing_if = "Option::is_none")] #[serde(default)] pub email: Option, #[serde(skip_serializing_if = "Option::is_none")] #[serde(default)] pub description: Option, } impl Server { pub fn issue_id_token( &self, subject: impl Into, issuer: impl Into, audience: impl Into, claims: StandardClaims, ) -> trc::Result { let now = now() as i64; JWT::new_decoded( From::from(RegisteredHeader { algorithm: self.core.oauth.oidc_signature_algorithm, key_id: Some("default".into()), ..Default::default() }), ClaimsSet:: { registered: RegisteredClaims { issuer: Some(issuer.into()), subject: Some(subject.into()), audience: Some(SingleOrMultiple::Single(audience.into())), not_before: Some(now.into()), issued_at: Some(now.into()), expiry: Some((now + self.core.oauth.oidc_expiry_id_token as i64).into()), ..Default::default() }, private: claims, }, ) .into_encoded(&self.core.oauth.oidc_signing_secret) .map(|token| token.unwrap_encoded().to_string()) .map_err(|err| { trc::AuthEvent::Error .into_err() .reason(err) .details("Failed to encode ID token") }) } } fn any_bool<'de, D>(deserializer: D) -> Result where D: Deserializer<'de>, { struct AnyBoolVisitor; impl Visitor<'_> for AnyBoolVisitor { type Value = bool; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("a boolean value") } fn visit_str(self, value: &str) -> Result where E: de::Error, { match value { "true" => Ok(true), "false" => Ok(false), _ => Err(E::custom(format!("Unknown boolean: {value}"))), } } fn visit_bool(self, value: bool) -> Result where E: de::Error, { Ok(value) } } deserializer.deserialize_any(AnyBoolVisitor) } ================================================ FILE: crates/common/src/auth/oauth/registration.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use serde::{Deserialize, Serialize}; use std::collections::HashMap; #[derive(Serialize, Deserialize, Debug, Default)] #[serde(rename_all = "snake_case")] pub struct ClientRegistrationRequest { pub redirect_uris: Vec, #[serde(default)] #[serde(skip_serializing_if = "Vec::is_empty")] pub response_types: Vec, #[serde(default)] #[serde(skip_serializing_if = "Vec::is_empty")] pub grant_types: Vec, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub application_type: Option, #[serde(default)] #[serde(skip_serializing_if = "Vec::is_empty")] pub contacts: Vec, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub client_name: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub logo_uri: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub client_uri: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub policy_uri: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub tos_uri: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub jwks_uri: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub jwks: Option, // Using serde_json::Value for flexibility #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub sector_identifier_uri: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub subject_type: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub id_token_signed_response_alg: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub id_token_encrypted_response_alg: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub id_token_encrypted_response_enc: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub userinfo_signed_response_alg: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub userinfo_encrypted_response_alg: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub userinfo_encrypted_response_enc: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub request_object_signing_alg: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub request_object_encryption_alg: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub request_object_encryption_enc: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub token_endpoint_auth_method: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub token_endpoint_auth_signing_alg: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub default_max_age: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub require_auth_time: Option, #[serde(default)] #[serde(skip_serializing_if = "Vec::is_empty")] pub default_acr_values: Vec, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub initiate_login_uri: Option, #[serde(default)] #[serde(skip_serializing_if = "Vec::is_empty")] pub request_uris: Vec, #[serde(flatten)] #[serde(skip_serializing_if = "HashMap::is_empty")] pub additional_fields: HashMap, } #[derive(Serialize, Deserialize, Debug, Default)] #[serde(rename_all = "snake_case")] pub struct ClientRegistrationResponse { // Required fields pub client_id: String, // Optional fields specific to the response #[serde(skip_serializing_if = "Option::is_none")] pub client_secret: Option, #[serde(skip_serializing_if = "Option::is_none")] pub registration_access_token: Option, #[serde(skip_serializing_if = "Option::is_none")] pub registration_client_uri: Option, #[serde(skip_serializing_if = "Option::is_none")] pub client_id_issued_at: Option, #[serde(skip_serializing_if = "Option::is_none")] pub client_secret_expires_at: Option, // Echo back the request #[serde(flatten)] pub request: ClientRegistrationRequest, } #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "lowercase")] pub enum ApplicationType { Web, Native, } #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "lowercase")] pub enum SubjectType { Pairwise, Public, } #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "snake_case")] pub enum TokenEndpointAuthMethod { ClientSecretPost, ClientSecretBasic, ClientSecretJwt, PrivateKeyJwt, None, } ================================================ FILE: crates/common/src/auth/oauth/token.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{CLIENT_ID_MAX_LEN, GrantType, RANDOM_CODE_LEN, crypto::SymmetricEncrypt}; use crate::Server; use directory::{PrincipalData, QueryParams}; use mail_builder::encoders::base64::base64_encode; use mail_parser::decoders::base64::base64_decode; use std::time::SystemTime; use store::{ blake3, rand::{Rng, rng}, }; use trc::AddContext; use utils::codec::leb128::{Leb128Iterator, Leb128Vec}; pub struct TokenInfo { pub grant_type: GrantType, pub account_id: u32, pub client_id: String, pub expiry: u64, pub issued_at: u64, pub expires_in: u64, } const OAUTH_EPOCH: u64 = 946684800; // Jan 1, 2000 impl Server { pub async fn encode_access_token( &self, grant_type: GrantType, account_id: u32, client_id: &str, expiry_in: u64, ) -> trc::Result { // Build context let mut password_hash = String::new(); if !matches!(grant_type, GrantType::Rsvp) { if client_id.len() > CLIENT_ID_MAX_LEN { return Err(trc::AuthEvent::Error .into_err() .details("Client id too long")); } // Include password hash if expiration is over 1 hour if expiry_in > 3600 { password_hash = self .password_hash(account_id) .await .caused_by(trc::location!())? } } let key = &self.core.oauth.oauth_key; let context = format!( "{} {} {} {}", grant_type.as_str(), client_id, account_id, password_hash ); // Set expiration time let issued_at = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()) .saturating_sub(OAUTH_EPOCH); // Jan 1, 2000 let expiry = issued_at + expiry_in; // Calculate nonce let mut hasher = blake3::Hasher::new(); if !password_hash.is_empty() { hasher.update(password_hash.as_bytes()); } hasher.update(grant_type.as_str().as_bytes()); hasher.update(issued_at.to_be_bytes().as_slice()); hasher.update(expiry.to_be_bytes().as_slice()); let nonce = hasher .finalize() .as_bytes() .iter() .take(SymmetricEncrypt::NONCE_LEN) .copied() .collect::>(); // Encrypt random bytes let mut token = SymmetricEncrypt::new(key.as_bytes(), &context) .encrypt(&rng().random::<[u8; RANDOM_CODE_LEN]>(), &nonce) .map_err(|_| { trc::AuthEvent::Error .into_err() .ctx(trc::Key::Reason, "Failed to encrypt token") .caused_by(trc::location!()) })?; token.push_leb128(account_id); token.push(grant_type.id()); token.push_leb128(issued_at); token.push_leb128(expiry); token.extend_from_slice(client_id.as_bytes()); Ok(String::from_utf8(base64_encode(&token).unwrap_or_default()).unwrap()) } pub async fn validate_access_token( &self, expected_grant_type: Option, token_: &str, ) -> trc::Result { // Base64 decode token let token = base64_decode(token_.as_bytes()).ok_or_else(|| { trc::AuthEvent::Error .into_err() .ctx(trc::Key::Reason, "Failed to decode token") .caused_by(trc::location!()) .details(token_.to_string()) })?; let (account_id, grant_type, issued_at, expiry, client_id) = token .get((RANDOM_CODE_LEN + SymmetricEncrypt::ENCRYPT_TAG_LEN)..) .and_then(|bytes| { let mut bytes = bytes.iter(); ( bytes.next_leb128()?, GrantType::from_id(bytes.next().copied()?)?, bytes.next_leb128::()?, bytes.next_leb128::()?, bytes.copied().map(char::from).collect::(), ) .into() }) .ok_or_else(|| { trc::AuthEvent::Error .into_err() .ctx(trc::Key::Reason, "Failed to decode token") .caused_by(trc::location!()) .details(token_.to_string()) })?; // Validate expiration let now = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()) .saturating_sub(OAUTH_EPOCH); // Jan 1, 2000 if expiry <= now || issued_at > now { return Err(trc::AuthEvent::TokenExpired.into_err()); } // Validate grant type if expected_grant_type.is_some_and(|g| g != grant_type) { return Err(trc::AuthEvent::Error .into_err() .details("Invalid grant type")); } // Obtain password hash let password_hash = if !matches!(grant_type, GrantType::Rsvp) && expiry - issued_at > 3600 { self.password_hash(account_id) .await .map_err(|err| trc::AuthEvent::Error.into_err().ctx(trc::Key::Details, err))? } else { "".into() }; // Build context let key = self.core.oauth.oauth_key.clone(); let context = format!( "{} {} {} {}", grant_type.as_str(), client_id, account_id, password_hash ); // Calculate nonce let mut hasher = blake3::Hasher::new(); if !password_hash.is_empty() { hasher.update(password_hash.as_bytes()); } hasher.update(grant_type.as_str().as_bytes()); hasher.update(issued_at.to_be_bytes().as_slice()); hasher.update(expiry.to_be_bytes().as_slice()); let nonce = hasher .finalize() .as_bytes() .iter() .take(SymmetricEncrypt::NONCE_LEN) .copied() .collect::>(); // Decrypt SymmetricEncrypt::new(key.as_bytes(), &context) .decrypt( &token[..RANDOM_CODE_LEN + SymmetricEncrypt::ENCRYPT_TAG_LEN], &nonce, ) .map_err(|err| { trc::AuthEvent::Error .into_err() .ctx(trc::Key::Details, "Failed to decode token") .caused_by(trc::location!()) .reason(err) })?; // Success Ok(TokenInfo { grant_type, account_id, client_id, expiry: expiry + OAUTH_EPOCH, issued_at: issued_at + OAUTH_EPOCH, expires_in: expiry - now, }) } pub async fn password_hash(&self, account_id: u32) -> trc::Result { if account_id != u32::MAX { self.core .storage .directory .query(QueryParams::id(account_id).with_return_member_of(false)) .await .caused_by(trc::location!())? .ok_or_else(|| { trc::AuthEvent::Error .into_err() .details("Account no longer exists") })? .data .into_iter() .filter_map(|v| { if let PrincipalData::Password(secret) = v { Some(secret) } else { None } }) .next() .ok_or( trc::AuthEvent::Error .into_err() .details("Account does not contain secrets") .caused_by(trc::location!()), ) } else if let Some((_, secret)) = &self.core.jmap.fallback_admin { Ok(secret.into()) } else { Err(trc::AuthEvent::Error .into_err() .details("Invalid account ID") .caused_by(trc::location!())) } } } ================================================ FILE: crates/common/src/auth/rate_limit.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::net::IpAddr; use crate::{ KV_RATE_LIMIT_HTTP_ANONYMOUS, KV_RATE_LIMIT_HTTP_AUTHENTICATED, Server, ip_to_bytes, listener::limiter::{InFlight, LimiterResult}, }; use directory::Permission; use trc::AddContext; use crate::auth::AccessToken; impl Server { pub async fn is_http_authenticated_request_allowed( &self, access_token: &AccessToken, ) -> trc::Result> { let is_rate_allowed = if let Some(rate) = &self.core.jmap.rate_authenticated { self.core .storage .lookup .is_rate_allowed( KV_RATE_LIMIT_HTTP_AUTHENTICATED, &access_token.primary_id.to_be_bytes(), rate, false, ) .await .caused_by(trc::location!())? .is_none() } else { true }; if is_rate_allowed { match access_token.is_http_request_allowed() { LimiterResult::Allowed(in_flight) => Ok(Some(in_flight)), LimiterResult::Forbidden => { if access_token.has_permission(Permission::UnlimitedRequests) { Ok(None) } else { Err(trc::LimitEvent::ConcurrentRequest.into_err()) } } LimiterResult::Disabled => Ok(None), } } else if access_token.has_permission(Permission::UnlimitedRequests) { Ok(None) } else { Err(trc::LimitEvent::TooManyRequests.into_err()) } } pub async fn is_http_anonymous_request_allowed(&self, addr: &IpAddr) -> trc::Result<()> { if let Some(rate) = &self.core.jmap.rate_anonymous && !self.is_ip_allowed(addr) && self .core .storage .lookup .is_rate_allowed( KV_RATE_LIMIT_HTTP_ANONYMOUS, &ip_to_bytes(addr), rate, false, ) .await .caused_by(trc::location!())? .is_some() { return Err(trc::LimitEvent::TooManyRequests.into_err()); } Ok(()) } pub fn is_upload_allowed(&self, access_token: &AccessToken) -> trc::Result> { match access_token.is_upload_allowed() { LimiterResult::Allowed(in_flight) => Ok(Some(in_flight)), LimiterResult::Forbidden => { if access_token.has_permission(Permission::UnlimitedRequests) { Ok(None) } else { Err(trc::LimitEvent::ConcurrentUpload.into_err()) } } LimiterResult::Disabled => Ok(None), } } } ================================================ FILE: crates/common/src/auth/roles.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::Server; use ahash::AHashSet; use directory::{ Permission, Permissions, QueryParams, ROLE_ADMIN, ROLE_TENANT_ADMIN, ROLE_USER, backend::internal::lookup::DirectoryStore, }; use std::sync::{Arc, LazyLock}; use trc::AddContext; use utils::cache::CacheItemWeight; #[derive(Debug, Clone, Default)] pub struct RolePermissions { pub enabled: Permissions, pub disabled: Permissions, } static USER_PERMISSIONS: LazyLock> = LazyLock::new(user_permissions); static ADMIN_PERMISSIONS: LazyLock> = LazyLock::new(admin_permissions); static TENANT_ADMIN_PERMISSIONS: LazyLock> = LazyLock::new(tenant_admin_permissions); impl Server { pub async fn get_role_permissions(&self, role_id: u32) -> trc::Result> { match role_id { ROLE_USER => Ok(USER_PERMISSIONS.clone()), ROLE_ADMIN => Ok(ADMIN_PERMISSIONS.clone()), ROLE_TENANT_ADMIN => Ok(TENANT_ADMIN_PERMISSIONS.clone()), role_id => { match self .inner .cache .permissions .get_value_or_guard_async(&role_id) .await { Ok(permissions) => Ok(permissions), Err(guard) => { let permissions = self.build_role_permissions(role_id).await?; let _ = guard.insert(permissions.clone()); Ok(permissions) } } } } } async fn build_role_permissions(&self, role_id: u32) -> trc::Result> { let mut role_ids = vec![role_id].into_iter(); let mut role_ids_stack = vec![]; let mut fetched_role_ids = AHashSet::new(); let mut return_permissions = RolePermissions::default(); 'outer: loop { if let Some(role_id) = role_ids.next() { // Skip if already fetched if !fetched_role_ids.insert(role_id) { continue; } match role_id { ROLE_USER => { return_permissions.enabled.union(&USER_PERMISSIONS.enabled); return_permissions .disabled .union(&USER_PERMISSIONS.disabled); } ROLE_ADMIN => { return_permissions.enabled.union(&ADMIN_PERMISSIONS.enabled); return_permissions .disabled .union(&ADMIN_PERMISSIONS.disabled); break 'outer; } ROLE_TENANT_ADMIN => { return_permissions .enabled .union(&TENANT_ADMIN_PERMISSIONS.enabled); return_permissions .disabled .union(&TENANT_ADMIN_PERMISSIONS.disabled); } role_id => { // Try with the cache if let Some(role_permissions) = self.inner.cache.permissions.get(&role_id) { return_permissions.union(role_permissions.as_ref()); } else { let mut role_permissions = RolePermissions::default(); // Obtain principal let principal = self .store() .query(QueryParams::id(role_id).with_return_member_of(true)) .await .caused_by(trc::location!())? .ok_or_else(|| { trc::SecurityEvent::Unauthorized .into_err() .details( "Principal not found while building role permissions", ) .ctx(trc::Key::Id, role_id) })?; // Add permissions for permission in principal.permissions() { if permission.grant { role_permissions.enabled.set(permission.permission as usize); } else { role_permissions .disabled .set(permission.permission as usize); } } // Add permissions return_permissions.union(&role_permissions); // Add parent roles let mut principal_roles = principal.roles().peekable(); if principal_roles.peek().is_some() { role_ids_stack.push(role_ids); role_ids = principal_roles.collect::>().into_iter(); } else { // Cache role self.inner .cache .permissions .insert(role_id, Arc::new(role_permissions)); } } } } } else if let Some(prev_role_ids) = role_ids_stack.pop() { role_ids = prev_role_ids; } else { break; } } Ok(Arc::new(return_permissions)) } } impl RolePermissions { pub fn union(&mut self, other: &RolePermissions) { self.enabled.union(&other.enabled); self.disabled.union(&other.disabled); } pub fn finalize(mut self) -> Permissions { self.enabled.difference(&self.disabled); self.enabled } pub fn finalize_as_ref(&self) -> Permissions { let mut enabled = self.enabled.clone(); enabled.difference(&self.disabled); enabled } } fn tenant_admin_permissions() -> Arc { let mut permissions = RolePermissions::default(); for permission_id in 0..Permission::COUNT { let permission = Permission::from_id(permission_id as u32).unwrap(); if permission.is_tenant_admin_permission() { permissions.enabled.set(permission_id); } } Arc::new(permissions) } fn user_permissions() -> Arc { let mut permissions = RolePermissions::default(); for permission_id in 0..Permission::COUNT { let permission = Permission::from_id(permission_id as u32).unwrap(); if permission.is_user_permission() { permissions.enabled.set(permission_id); } } Arc::new(permissions) } fn admin_permissions() -> Arc { Arc::new(RolePermissions { enabled: Permissions::all(), disabled: Permissions::new(), }) } impl CacheItemWeight for RolePermissions { fn weight(&self) -> u64 { std::mem::size_of::() as u64 } } ================================================ FILE: crates/common/src/auth/sasl.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use mail_send::Credentials; pub fn sasl_decode_challenge_plain(challenge: &[u8]) -> Option> { let mut username = Vec::new(); let mut secret = Vec::new(); let mut arg_num = 0; for &ch in challenge { if ch != 0 { if arg_num == 1 { username.push(ch); } else if arg_num == 2 { secret.push(ch); } } else { arg_num += 1; } } match (String::from_utf8(username), String::from_utf8(secret)) { (Ok(username), Ok(secret)) if !username.is_empty() && !secret.is_empty() => { Some((username, secret).into()) } _ => None, } } pub fn sasl_decode_challenge_oauth(challenge: &[u8]) -> Option> { extract_oauth_bearer(challenge).map(|s| Credentials::OAuthBearer { token: s.into() }) } fn extract_oauth_bearer(bytes: &[u8]) -> Option<&str> { let mut start_pos = 0; let eof = bytes.len().saturating_sub(1); for (pos, ch) in bytes.iter().enumerate() { let is_separator = *ch == 1; if is_separator || pos == eof { if bytes .get(start_pos..start_pos + 12) .is_some_and(|s| s.eq_ignore_ascii_case(b"auth=Bearer ")) { return bytes .get(start_pos + 12..if is_separator { pos } else { bytes.len() }) .and_then(|s| std::str::from_utf8(s).ok()); } start_pos = pos + 1; } } None } #[cfg(test)] mod tests { use super::*; #[test] fn test_extract_oauth_bearer() { let input = b"auth=Bearer validtoken"; let result = extract_oauth_bearer(input); assert_eq!(result, Some("validtoken")); let input = b"auth=Invalid validtoken"; let result = extract_oauth_bearer(input); assert_eq!(result, None); let input = b"auth=Bearer"; let result = extract_oauth_bearer(input); assert_eq!(result, None); let input = b""; let result = extract_oauth_bearer(input); assert_eq!(result, None); let input = b"auth=Bearer token1\x01auth=Bearer token2"; let result = extract_oauth_bearer(input); assert_eq!(result, Some("token1")); let input = b"auth=Bearer VALIDTOKEN"; let result = extract_oauth_bearer(input); assert_eq!(result, Some("VALIDTOKEN")); let input = b"auth=Bearer token with spaces"; let result = extract_oauth_bearer(input); assert_eq!(result, Some("token with spaces")); let input = b"auth=Bearer token_with_special_chars!@#"; let result = extract_oauth_bearer(input); assert_eq!(result, Some("token_with_special_chars!@#")); let input = "n,a=user@example.com,\x01host=server.example.com\x01port=143\x01auth=Bearer vF9dft4qmTc2Nvb3RlckBhbHRhdmlzdGEuY29tCg==\x01\x01"; let result = extract_oauth_bearer(input.as_bytes()); assert_eq!(result, Some("vF9dft4qmTc2Nvb3RlckBhbHRhdmlzdGEuY29tCg==")); } } ================================================ FILE: crates/common/src/config/groupware.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{str::FromStr, time::Duration}; use utils::{config::Config, template::Template}; #[derive(Debug, Clone, Default)] pub struct GroupwareConfig { // DAV settings pub max_request_size: usize, pub dead_property_size: Option, pub live_property_size: usize, pub max_lock_timeout: u64, pub max_locks_per_user: usize, pub max_results: usize, pub assisted_discovery: bool, // Calendar settings pub max_ical_size: usize, pub max_ical_instances: usize, pub max_ical_attendees_per_instance: usize, pub default_calendar_name: Option, pub default_calendar_display_name: Option, pub alarms_enabled: bool, pub alarms_minimum_interval: i64, pub alarms_allow_external_recipients: bool, pub alarms_from_name: String, pub alarms_from_email: Option, pub alarms_template: Template, pub itip_enabled: bool, pub itip_auto_add: bool, pub itip_inbound_max_ical_size: usize, pub itip_outbound_max_recipients: usize, pub itip_http_rsvp_url: Option, pub itip_http_rsvp_expiration: u64, pub itip_inbox_auto_expunge: Option, pub itip_template: Template, // Addressbook settings pub max_vcard_size: usize, pub default_addressbook_name: Option, pub default_addressbook_display_name: Option, // File storage settings pub max_file_size: usize, // Sharing settings pub max_shares_per_item: usize, pub allow_directory_query: bool, } #[derive(Debug, Clone, PartialEq, Eq, Default, Hash)] pub enum CalendarTemplateVariable { #[default] PageTitle, Header, Footer, EventTitle, EventDescription, EventDetails, Actions, ActionUrl, ActionName, AttendeesTitle, Attendees, Key, Color, Changed, Value, LogoCid, OldValue, Rsvp, } impl GroupwareConfig { pub fn parse(config: &mut Config) -> Self { GroupwareConfig { max_request_size: config .property("dav.request.max-size") .unwrap_or(25 * 1024 * 1024), dead_property_size: config .property_or_default::>("dav.property.max-size.dead", "1024") .unwrap_or(Some(1024)), live_property_size: config.property("dav.property.max-size.live").unwrap_or(250), assisted_discovery: config .property("dav.collection.assisted-discovery") .unwrap_or(true), max_lock_timeout: config .property::("dav.lock.max-timeout") .map(|d| d.as_secs()) .unwrap_or(3600), max_locks_per_user: config.property("dav.locks.max-per-user").unwrap_or(10), max_results: config.property("dav.response.max-results").unwrap_or(2000), default_calendar_name: config .property_or_default::>("calendar.default.href-name", "default") .unwrap_or_default(), default_calendar_display_name: config .property_or_default::>( "calendar.default.display-name", "Stalwart Calendar", ) .unwrap_or_default(), default_addressbook_name: config .property_or_default::>("contacts.default.href-name", "default") .unwrap_or_default(), default_addressbook_display_name: config .property_or_default::>( "contacts.default.display-name", "Stalwart Address Book", ) .unwrap_or_default(), max_ical_size: config.property("calendar.max-size").unwrap_or(512 * 1024), max_ical_instances: config .property("calendar.max-recurrence-expansions") .unwrap_or(3000), max_ical_attendees_per_instance: config .property("calendar.max-attendees-per-instance") .unwrap_or(20), max_vcard_size: config.property("contacts.max-size").unwrap_or(512 * 1024), max_file_size: config .property("file-storage.max-size") .unwrap_or(25 * 1024 * 1024), alarms_enabled: config.property("calendar.alarms.enabled").unwrap_or(true), alarms_minimum_interval: config .property_or_default::("calendar.alarms.minimum-interval", "1h") .unwrap_or(Duration::from_secs(60 * 60)) .as_secs() as i64, alarms_allow_external_recipients: config .property("calendar.alarms.allow-external-recipients") .unwrap_or(false), alarms_from_name: config .value("calendar.alarms.from.name") .unwrap_or("Stalwart Calendar") .to_string(), alarms_from_email: config .value("calendar.alarms.from.email") .map(|s| s.to_string()), alarms_template: Template::parse(include_str!(concat!( env!("CARGO_MANIFEST_DIR"), "/../../resources/html-templates/calendar-alarm.html.min" ))) .expect("Failed to parse calendar template"), itip_enabled: config .property("calendar.scheduling.enable") .unwrap_or(true), itip_auto_add: config .property("calendar.scheduling.inbound.auto-add") .unwrap_or(false), itip_inbound_max_ical_size: config .property("calendar.scheduling.inbound.max-size") .unwrap_or(512 * 1024), itip_outbound_max_recipients: config .property("calendar.scheduling.outbound.max-recipients") .unwrap_or(100), itip_inbox_auto_expunge: config .property_or_default::>( "calendar.scheduling.inbox.auto-expunge", "30d", ) .map(|d| d.map(|d| d.as_secs())) .unwrap_or(Some(30 * 24 * 60 * 60)), itip_http_rsvp_url: if config .property("calendar.scheduling.http-rsvp.enable") .unwrap_or(true) { if let Some(url) = config .value("calendar.scheduling.http-rsvp.url") .map(|v| v.trim().trim_end_matches('/')) .filter(|v| !v.is_empty()) { Some(url.to_string()) } else { Some(format!( "https://{}/calendar/rsvp", config.value("server.hostname").unwrap_or("localhost") )) } } else { None }, max_shares_per_item: config.property("sharing.max-shares-per-item").unwrap_or(10), allow_directory_query: config .property("sharing.allow-directory-query") .unwrap_or(false), itip_http_rsvp_expiration: config .property_or_default::("calendar.scheduling.http-rsvp.expiration", "90d") .map(|d| d.as_secs()) .unwrap_or(90 * 24 * 60 * 60), itip_template: Template::parse(include_str!(concat!( env!("CARGO_MANIFEST_DIR"), "/../../resources/html-templates/calendar-invite.html.min" ))) .expect("Failed to parse calendar template"), } } } impl FromStr for CalendarTemplateVariable { type Err = String; fn from_str(s: &str) -> Result { match s { "page_title" => Ok(CalendarTemplateVariable::PageTitle), "header" => Ok(CalendarTemplateVariable::Header), "footer" => Ok(CalendarTemplateVariable::Footer), "event_title" => Ok(CalendarTemplateVariable::EventTitle), "event_description" => Ok(CalendarTemplateVariable::EventDescription), "event_details" => Ok(CalendarTemplateVariable::EventDetails), "action_url" => Ok(CalendarTemplateVariable::ActionUrl), "action_name" => Ok(CalendarTemplateVariable::ActionName), "attendees" => Ok(CalendarTemplateVariable::Attendees), "attendees_title" => Ok(CalendarTemplateVariable::AttendeesTitle), "key" => Ok(CalendarTemplateVariable::Key), "value" => Ok(CalendarTemplateVariable::Value), "logo_cid" => Ok(CalendarTemplateVariable::LogoCid), "actions" => Ok(CalendarTemplateVariable::Actions), "changed" => Ok(CalendarTemplateVariable::Changed), "old_value" => Ok(CalendarTemplateVariable::OldValue), "rsvp" => Ok(CalendarTemplateVariable::Rsvp), "color" => Ok(CalendarTemplateVariable::Color), _ => Err(format!("Unknown calendar template variable: {}", s)), } } } ================================================ FILE: crates/common/src/config/imap.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use utils::config::{Config, Rate}; #[derive(Default, Clone)] pub struct ImapConfig { pub max_request_size: usize, pub max_auth_failures: u32, pub allow_plain_auth: bool, pub timeout_auth: Duration, pub timeout_unauth: Duration, pub timeout_idle: Duration, pub rate_requests: Option, pub rate_concurrent: Option, } impl ImapConfig { pub fn parse(config: &mut Config) -> Self { ImapConfig { max_request_size: config .property_or_default("imap.request.max-size", "52428800") .unwrap_or(52428800), max_auth_failures: config .property_or_default("imap.auth.max-failures", "3") .unwrap_or(3), timeout_auth: config .property_or_default("imap.timeout.authenticated", "30m") .unwrap_or_else(|| Duration::from_secs(1800)), timeout_unauth: config .property_or_default("imap.timeout.anonymous", "1m") .unwrap_or_else(|| Duration::from_secs(60)), timeout_idle: config .property_or_default("imap.timeout.idle", "30m") .unwrap_or_else(|| Duration::from_secs(1800)), rate_requests: config .property_or_default::>("imap.rate-limit.requests", "2000/1m") .unwrap_or_default(), rate_concurrent: config .property::>("imap.rate-limit.concurrent") .unwrap_or_default(), allow_plain_auth: config .property_or_default("imap.auth.allow-plain-text", "false") .unwrap_or(false), } } } ================================================ FILE: crates/common/src/config/inner.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::server::tls::{build_self_signed_cert, parse_certificates}; use crate::{ CacheSwap, Caches, Data, DavResource, DavResources, MailboxCache, MessageStoreCache, MessageUidCache, TlsConnectors, auth::{AccessToken, roles::RolePermissions}, config::{ smtp::resolver::{Policy, Tlsa}, spamfilter::SpamClassifier, }, listener::blocked::BlockedIps, manager::webadmin::WebAdminManager, }; use ahash::{AHashMap, AHashSet}; use arc_swap::ArcSwap; use mail_auth::{MX, Parameters, Txt}; use mail_send::smtp::tls::build_tls_connector; use parking_lot::RwLock; use std::{ net::{IpAddr, Ipv4Addr, Ipv6Addr}, sync::Arc, }; use utils::{ cache::{Cache, CacheWithTtl}, config::Config, snowflake::SnowflakeIdGenerator, }; impl Data { pub fn parse(config: &mut Config) -> Self { // Parse certificates let mut certificates = AHashMap::new(); let mut subject_names = AHashSet::new(); parse_certificates(config, &mut certificates, &mut subject_names); if subject_names.is_empty() { subject_names.insert("localhost".to_string()); } // Build and test snowflake id generator let node_id = config .property::("cluster.node-id") .unwrap_or_else(store::rand::random); let id_generator = SnowflakeIdGenerator::with_node_id(node_id); if !id_generator.is_valid() { panic!("Invalid system time, panicking to avoid data corruption"); } Data { spam_classifier: ArcSwap::from_pointee(SpamClassifier::default()), tls_certificates: ArcSwap::from_pointee(certificates), tls_self_signed_cert: build_self_signed_cert( subject_names.into_iter().collect::>(), ) .or_else(|err| { config.new_build_error("certificate.self-signed", err); build_self_signed_cert(vec!["localhost".to_string()]) }) .ok() .map(Arc::new), blocked_ips: RwLock::new(BlockedIps::parse(config).blocked_ip_addresses), jmap_id_gen: id_generator.clone(), queue_id_gen: id_generator.clone(), span_id_gen: id_generator, queue_status: true.into(), webadmin: config .value("webadmin.path") .map(|path| WebAdminManager::new(path.into())) .unwrap_or_default(), logos: Default::default(), smtp_connectors: TlsConnectors::default(), asn_geo_data: Default::default(), } } } impl Caches { pub fn parse(config: &mut Config) -> Self { const MB_50: u64 = 50 * 1024 * 1024; const MB_10: u64 = 10 * 1024 * 1024; const MB_5: u64 = 5 * 1024 * 1024; const MB_1: u64 = 1024 * 1024; Caches { access_tokens: Cache::from_config( config, "access-token", MB_10, (std::mem::size_of::() + 255) as u64, ), http_auth: Cache::from_config( config, "http-auth", MB_1, (50 + std::mem::size_of::()) as u64, ), permissions: Cache::from_config( config, "permission", MB_5, std::mem::size_of::() as u64, ), messages: Cache::from_config( config, "message", MB_50, (std::mem::size_of::() + std::mem::size_of::>() + (1024 * std::mem::size_of::()) + (15 * (std::mem::size_of::() + 60))) as u64, ), files: Cache::from_config( config, "files", MB_10, (std::mem::size_of::() + (500 * std::mem::size_of::())) as u64, ), events: Cache::from_config( config, "events", MB_10, (std::mem::size_of::() + (500 * std::mem::size_of::())) as u64, ), contacts: Cache::from_config( config, "contacts", MB_10, (std::mem::size_of::() + (500 * std::mem::size_of::())) as u64, ), scheduling: Cache::from_config( config, "scheduling", MB_1, (std::mem::size_of::() + (500 * std::mem::size_of::())) as u64, ), dns_txt: CacheWithTtl::from_config( config, "dns.txt", MB_5, (std::mem::size_of::() + 255) as u64, ), dns_mx: CacheWithTtl::from_config( config, "dns.mx", MB_5, ((std::mem::size_of::() + 255) * 2) as u64, ), dns_ptr: CacheWithTtl::from_config( config, "dns.ptr", MB_1, (std::mem::size_of::() + 255) as u64, ), dns_ipv4: CacheWithTtl::from_config( config, "dns.ipv4", MB_5, ((std::mem::size_of::() + 255) * 2) as u64, ), dns_ipv6: CacheWithTtl::from_config( config, "dns.ipv6", MB_5, ((std::mem::size_of::() + 255) * 2) as u64, ), dns_tlsa: CacheWithTtl::from_config( config, "dns.tlsa", MB_1, (std::mem::size_of::() + 255) as u64, ), dbs_mta_sts: CacheWithTtl::from_config( config, "dns.mta-sts", MB_1, (std::mem::size_of::() + 255) as u64, ), dns_rbl: CacheWithTtl::from_config( config, "dns.rbl", MB_5, ((std::mem::size_of::() + 255) * 2) as u64, ), } } #[allow(clippy::type_complexity)] #[inline(always)] pub fn build_auth_parameters( &self, params: T, ) -> Parameters< '_, T, CacheWithTtl, CacheWithTtl>>, CacheWithTtl>>, CacheWithTtl>>, CacheWithTtl>>, > { Parameters { params, cache_txt: Some(&self.dns_txt), cache_mx: Some(&self.dns_mx), cache_ptr: Some(&self.dns_ptr), cache_ipv4: Some(&self.dns_ipv4), cache_ipv6: Some(&self.dns_ipv6), } } } impl Default for Data { fn default() -> Self { Self { spam_classifier: Default::default(), tls_certificates: Default::default(), tls_self_signed_cert: Default::default(), blocked_ips: Default::default(), jmap_id_gen: Default::default(), queue_id_gen: Default::default(), span_id_gen: Default::default(), queue_status: true.into(), webadmin: Default::default(), logos: Default::default(), smtp_connectors: Default::default(), asn_geo_data: Default::default(), } } } impl Default for TlsConnectors { fn default() -> Self { TlsConnectors { pki_verify: build_tls_connector(false), dummy_verify: build_tls_connector(true), } } } ================================================ FILE: crates/common/src/config/jmap/capabilities.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::settings::JmapConfig; use crate::config::groupware::GroupwareConfig; use ahash::AHashSet; use calcard::icalendar::ICalendarDuration; use chrono::{DateTime, Utc}; use jmap_proto::{ object::email::EmailComparator, request::capability::{ BlobCapabilities, CalendarCapabilities, Capabilities, Capability, ContactsCapabilities, CoreCapabilities, EmptyCapabilities, FileNodeCapabilities, MailCapabilities, PrincipalAvailabilityCapabilities, PrincipalCapabilities, SieveAccountCapabilities, SieveSessionCapabilities, SubmissionCapabilities, }, types::date::UTCDate, }; use types::{collection::Collection, type_state::DataType}; use utils::{config::Config, map::vec_map::VecMap}; impl JmapConfig { pub fn add_capabilities(&mut self, config: &mut Config, groupware_config: &GroupwareConfig) { // Add core capabilities self.capabilities.session.append( Capability::Core, Capabilities::Core(CoreCapabilities { max_size_upload: self.upload_max_size, max_concurrent_upload: self.upload_max_concurrent.unwrap_or(u32::MAX as u64) as usize, max_size_request: self.request_max_size, max_concurrent_requests: self.request_max_concurrent.unwrap_or(u32::MAX as u64) as usize, max_calls_in_request: self.request_max_calls, max_objects_in_get: self.get_max_objects, max_objects_in_set: self.set_max_objects, collation_algorithms: vec![ "i;ascii-numeric".to_string(), "i;ascii-casemap".to_string(), "i;unicode-casemap".to_string(), ], }), ); // Add email capabilities self.capabilities.session.append( Capability::Mail, Capabilities::Empty(EmptyCapabilities::default()), ); self.capabilities.account.insert( Capability::Mail, Capabilities::Mail(MailCapabilities { max_mailboxes_per_email: None, max_mailbox_depth: self.mailbox_max_depth, max_size_mailbox_name: self.mailbox_name_max_len, max_size_attachments_per_email: self.mail_attachments_max_size, email_query_sort_options: vec![ EmailComparator::ReceivedAt, EmailComparator::Size, EmailComparator::From, EmailComparator::To, EmailComparator::Subject, EmailComparator::SentAt, EmailComparator::HasKeyword(Default::default()), EmailComparator::AllInThreadHaveKeyword(Default::default()), EmailComparator::SomeInThreadHaveKeyword(Default::default()), ], may_create_top_level_mailbox: true, }), ); // Add calendar capabilities self.capabilities.session.append( Capability::Calendars, Capabilities::Empty(EmptyCapabilities::default()), ); self.capabilities.account.insert( Capability::Calendars, Capabilities::Calendar(CalendarCapabilities { max_calendars_per_event: None, min_date_time: UTCDate::from_timestamp(DateTime::::MIN_UTC.timestamp()), max_date_time: UTCDate::from_timestamp(DateTime::::MAX_UTC.timestamp()), max_expanded_query_duration: ICalendarDuration::from_seconds(86400 * 365) .to_string(), max_participants_per_event: groupware_config.max_ical_attendees_per_instance.into(), may_create_calendar: true, }), ); self.capabilities.session.append( Capability::CalendarsParse, Capabilities::Empty(EmptyCapabilities::default()), ); self.capabilities.account.insert( Capability::CalendarsParse, Capabilities::Empty(EmptyCapabilities::default()), ); // Add contacts capabilities self.capabilities.session.append( Capability::Contacts, Capabilities::Empty(EmptyCapabilities::default()), ); self.capabilities.account.insert( Capability::Contacts, Capabilities::Contacts(ContactsCapabilities { max_address_books_per_card: None, may_create_address_book: true, }), ); self.capabilities.session.append( Capability::ContactsParse, Capabilities::Empty(EmptyCapabilities::default()), ); self.capabilities.account.insert( Capability::ContactsParse, Capabilities::Empty(EmptyCapabilities::default()), ); // Add file node capabilities self.capabilities.session.append( Capability::FileNode, Capabilities::Empty(EmptyCapabilities::default()), ); self.capabilities.account.insert( Capability::FileNode, Capabilities::FileNode(FileNodeCapabilities { max_file_node_depth: None, max_size_file_node_name: 255, file_node_query_sort_options: vec![], may_create_top_level_file_node: true, }), ); // Add principal capabilities self.capabilities.session.append( Capability::Principals, Capabilities::Empty(EmptyCapabilities::default()), ); self.capabilities.account.insert( Capability::Principals, Capabilities::Principals(PrincipalCapabilities { current_user_principal_id: None, }), ); self.capabilities.session.append( Capability::PrincipalsAvailability, Capabilities::Empty(EmptyCapabilities::default()), ); self.capabilities.account.insert( Capability::PrincipalsAvailability, Capabilities::PrincipalsAvailability(PrincipalAvailabilityCapabilities { max_availability_duration: ICalendarDuration::from_seconds(86400 * 365).to_string(), }), ); // Add submission capabilities self.capabilities.session.append( Capability::Submission, Capabilities::Empty(EmptyCapabilities::default()), ); self.capabilities.account.insert( Capability::Submission, Capabilities::Submission(SubmissionCapabilities { max_delayed_send: 86400 * 30, submission_extensions: VecMap::from_iter([ ("FUTURERELEASE".to_string(), Vec::new()), ("SIZE".to_string(), Vec::new()), ("DSN".to_string(), Vec::new()), ("DELIVERYBY".to_string(), Vec::new()), ("MT-PRIORITY".to_string(), vec!["MIXER".to_string()]), ("REQUIRETLS".to_string(), vec![]), ]), }), ); // Add vacation response capabilities self.capabilities.session.append( Capability::VacationResponse, Capabilities::Empty(EmptyCapabilities::default()), ); self.capabilities.account.insert( Capability::VacationResponse, Capabilities::Empty(EmptyCapabilities::default()), ); // Add Sieve capabilities let mut notification_methods = Vec::new(); for (_, uri) in config.values("sieve.untrusted.notification-uris") { notification_methods.push(uri.to_string()); } if notification_methods.is_empty() { notification_methods.push("mailto".to_string()); } let mut capabilities: AHashSet = AHashSet::from_iter(sieve::compiler::grammar::Capability::all().iter().cloned()); for (_, capability) in config.values("sieve.untrusted.disabled-capabilities") { capabilities.remove(&sieve::compiler::grammar::Capability::parse(capability)); } let mut extensions = capabilities .into_iter() .map(|c| c.to_string()) .collect::>(); extensions.sort_unstable(); self.capabilities.session.append( Capability::Sieve, Capabilities::SieveSession(SieveSessionCapabilities::default()), ); self.capabilities.account.insert( Capability::Sieve, Capabilities::SieveAccount(SieveAccountCapabilities { max_script_name: self.sieve_max_script_name, max_script_size: config .property("sieve.untrusted.max-script-size") .unwrap_or(1024 * 1024), max_scripts: self.max_objects[Collection::SieveScript as usize] as usize, max_redirects: config .property("sieve.untrusted.max-redirects") .unwrap_or(1), extensions, notification_methods: if !notification_methods.is_empty() { notification_methods.into() } else { None }, ext_lists: None, }), ); // Add Blob capabilities self.capabilities.session.append( Capability::Blob, Capabilities::Empty(EmptyCapabilities::default()), ); self.capabilities.account.insert( Capability::Blob, Capabilities::Blob(BlobCapabilities { max_size_blob_set: (self.request_max_size * 3 / 4) - 512, max_data_sources: self.request_max_calls, supported_type_names: vec![ DataType::Email, DataType::Thread, DataType::SieveScript, ], supported_digest_algorithms: vec!["sha", "sha-256", "sha-512"], }), ); // Add Quota capabilities self.capabilities.session.append( Capability::Quota, Capabilities::Empty(EmptyCapabilities::default()), ); self.capabilities.account.insert( Capability::Quota, Capabilities::Empty(EmptyCapabilities::default()), ); } } ================================================ FILE: crates/common/src/config/jmap/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod capabilities; pub mod settings; ================================================ FILE: crates/common/src/config/jmap/settings.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::config::groupware::GroupwareConfig; use ahash::{AHashMap, AHashSet}; use jmap_proto::request::capability::BaseCapabilities; use nlp::language::Language; use std::{str::FromStr, time::Duration}; use store::{search::SearchField, write::SearchIndex}; use types::{collection::Collection, special_use::SpecialUse}; use utils::{ config::{Config, Rate, cron::SimpleCron, utils::ParseValue}, map::bitmap::Bitmap, }; #[derive(Default, Clone)] pub struct JmapConfig { pub default_language: Language, pub query_max_results: usize, pub snippet_max_results: usize, pub changes_max_results: Option, pub changes_max_history: Option, pub share_notification_max_history: Option, pub request_max_size: usize, pub request_max_calls: usize, pub request_max_concurrent: Option, pub get_max_objects: usize, pub set_max_objects: usize, pub upload_max_size: usize, pub upload_max_concurrent: Option, pub upload_tmp_quota_size: usize, pub upload_tmp_quota_amount: usize, pub upload_tmp_ttl: u64, pub mailbox_max_depth: usize, pub mailbox_name_max_len: usize, pub mail_attachments_max_size: usize, pub mail_parse_max_items: usize, pub mail_max_size: usize, pub mail_autoexpunge_after: Option, pub email_submission_autoexpunge_after: Option, pub contact_parse_max_items: usize, pub calendar_parse_max_items: usize, pub sieve_max_script_name: usize, pub max_objects: [u32; Collection::MAX], pub rate_authenticated: Option, pub rate_anonymous: Option, pub event_source_throttle: Duration, pub push_attempt_interval: Duration, pub push_attempts_max: u32, pub push_retry_interval: Duration, pub push_timeout: Duration, pub push_verify_timeout: Duration, pub push_throttle: Duration, pub web_socket_throttle: Duration, pub web_socket_timeout: Duration, pub web_socket_heartbeat: Duration, pub fallback_admin: Option<(String, String)>, pub master_user: Option<(String, String)>, pub default_folders: Vec, pub shared_folder: String, pub http_headers: Vec<(hyper::header::HeaderName, hyper::header::HeaderValue)>, pub http_use_forwarded: bool, pub encrypt: bool, pub encrypt_append: bool, pub index_batch_size: usize, pub index_fields: AHashMap>, pub capabilities: BaseCapabilities, pub account_purge_frequency: SimpleCron, } #[derive(Clone, Debug)] pub struct DefaultFolder { pub name: String, pub aliases: Vec, pub special_use: SpecialUse, pub subscribe: bool, pub create: bool, } impl JmapConfig { pub fn parse(config: &mut Config, groupware_config: &GroupwareConfig) -> Self { // Parse HTTP headers let mut http_headers = config .values("http.headers") .map(|(_, v)| { if let Some((k, v)) = v.split_once(':') { Ok(( hyper::header::HeaderName::from_str(k.trim()).map_err(|err| { format!("Invalid header found in property \"http.headers\": {}", err) })?, hyper::header::HeaderValue::from_str(v.trim()).map_err(|err| { format!("Invalid header found in property \"http.headers\": {}", err) })?, )) } else { Err(format!( "Invalid header found in property \"http.headers\": {}", v )) } }) .collect::, String>>() .map_err(|e| config.new_parse_error("http.headers", e)) .unwrap_or_default(); // Parse default folders let mut default_folders = Vec::new(); let mut shared_folder = "Shared Folders".to_string(); for key in config.sub_keys("email.folders", ".name") { match SpecialUse::parse_value(&key) { Ok(SpecialUse::Shared) => { if let Some(value) = config.value(("email.folders", key.as_str(), "name")) { shared_folder = value.to_string(); } } Ok(special_use) => { let subscribe = config .property_or_default(("email.folders", key.as_str(), "subscribe"), "true") .unwrap_or(true); let create = config .property_or_default(("email.folders", key.as_str(), "create"), "true") .unwrap_or(true) | [SpecialUse::Inbox, SpecialUse::Trash, SpecialUse::Junk] .contains(&special_use); if let Some(name) = config .value(("email.folders", key.as_str(), "name")) .map(|name| name.trim()) .filter(|name| !name.is_empty()) { default_folders.push(DefaultFolder { name: name.to_string(), aliases: config .value(("email.folders", key.as_str(), "aliases")) .unwrap_or_default() .split(',') .map(|s| s.trim().to_string()) .filter(|s| !s.is_empty()) .collect(), special_use, subscribe, create, }); } } Err(err) => { config.new_parse_error(key, err); } } } for (special_use, name) in [ (SpecialUse::Inbox, "Inbox"), (SpecialUse::Trash, "Deleted Items"), (SpecialUse::Junk, "Junk Mail"), (SpecialUse::Drafts, "Drafts"), (SpecialUse::Sent, "Sent Items"), ] { if !default_folders.iter().any(|f| f.special_use == special_use) { default_folders.push(DefaultFolder { name: name.to_string(), aliases: Vec::new(), special_use, subscribe: true, create: true, }); } } // Add permissive CORS headers if config .property::("http.permissive-cors") .unwrap_or(false) { http_headers.push(( hyper::header::ACCESS_CONTROL_ALLOW_ORIGIN, hyper::header::HeaderValue::from_static("*"), )); http_headers.push(( hyper::header::ACCESS_CONTROL_ALLOW_HEADERS, hyper::header::HeaderValue::from_static( "Authorization, Content-Type, Accept, X-Requested-With", ), )); http_headers.push(( hyper::header::ACCESS_CONTROL_ALLOW_METHODS, hyper::header::HeaderValue::from_static( "POST, GET, PATCH, PUT, DELETE, HEAD, OPTIONS", ), )); } // Add HTTP Strict Transport Security if config.property::("http.hsts").unwrap_or(false) { http_headers.push(( hyper::header::STRICT_TRANSPORT_SECURITY, hyper::header::HeaderValue::from_static("max-age=31536000; includeSubDomains"), )); } let mut jmap = JmapConfig { default_language: Language::from_iso_639( config .value("storage.search-index.default-language") .unwrap_or("en"), ) .unwrap_or(Language::English), query_max_results: config .property("jmap.protocol.query.max-results") .unwrap_or(5000), changes_max_results: config .property_or_default::>("jmap.protocol.changes.max-results", "5000") .unwrap_or_default(), changes_max_history: config .property_or_default::>("changes.max-history", "10000") .unwrap_or_default(), share_notification_max_history: config .property_or_default::>("sharing.max-history", "30d") .unwrap_or_default(), snippet_max_results: config .property("jmap.protocol.search-snippet.max-results") .unwrap_or(100), request_max_size: config .property("jmap.protocol.request.max-size") .unwrap_or(10000000), request_max_calls: config .property("jmap.protocol.request.max-calls") .unwrap_or(16), request_max_concurrent: config .property_or_default::>("jmap.protocol.request.max-concurrent", "4") .unwrap_or(Some(4)), get_max_objects: config .property("jmap.protocol.get.max-objects") .unwrap_or(500), set_max_objects: config .property("jmap.protocol.set.max-objects") .unwrap_or(500), upload_max_size: config .property("jmap.protocol.upload.max-size") .unwrap_or(50000000), upload_max_concurrent: config .property_or_default::>("jmap.protocol.upload.max-concurrent", "4") .unwrap_or(Some(4)), upload_tmp_quota_size: config .property("jmap.protocol.upload.quota.size") .unwrap_or(50000000), upload_tmp_quota_amount: config .property("jmap.protocol.upload.quota.files") .unwrap_or(1000), upload_tmp_ttl: config .property_or_default::("jmap.protocol.upload.ttl", "1h") .unwrap_or_else(|| Duration::from_secs(3600)) .as_secs(), mailbox_max_depth: config.property("jmap.mailbox.max-depth").unwrap_or(10), mailbox_name_max_len: config .property("jmap.mailbox.max-name-length") .unwrap_or(255), mail_attachments_max_size: config .property("jmap.email.max-attachment-size") .unwrap_or(50000000), mail_max_size: config.property("jmap.email.max-size").unwrap_or(75000000), mail_parse_max_items: config.property("jmap.email.parse.max-items").unwrap_or(10), mail_autoexpunge_after: config .property_or_default::>("email.auto-expunge", "30d") .map(|d| d.map(|d| d.as_secs())) .unwrap_or_default(), email_submission_autoexpunge_after: config .property_or_default::>("email-submission.auto-expunge", "3d") .map(|d| d.map(|d| d.as_secs())) .unwrap_or_default(), sieve_max_script_name: config .property("sieve.untrusted.limits.name-length") .unwrap_or(512), max_objects: [u32::MAX; Collection::MAX], capabilities: BaseCapabilities::default(), rate_authenticated: config .property_or_default::>("http.rate-limit.account", "1000/1m") .unwrap_or_default(), rate_anonymous: config .property_or_default::>("http.rate-limit.anonymous", "100/1m") .unwrap_or_default(), event_source_throttle: config .property_or_default("jmap.event-source.throttle", "1s") .unwrap_or_else(|| Duration::from_secs(1)), web_socket_throttle: config .property_or_default("jmap.web-socket.throttle", "1s") .unwrap_or_else(|| Duration::from_secs(1)), web_socket_timeout: config .property_or_default("jmap.web-socket.timeout", "10m") .unwrap_or_else(|| Duration::from_secs(10 * 60)), web_socket_heartbeat: config .property_or_default("jmap.web-socket.heartbeat", "1m") .unwrap_or_else(|| Duration::from_secs(60)), encrypt: config .property_or_default("email.encryption.enable", "true") .unwrap_or(true), encrypt_append: config .property_or_default("email.encryption.append", "false") .unwrap_or(false), http_use_forwarded: config.property("http.use-x-forwarded").unwrap_or(false), http_headers, push_attempt_interval: config .property_or_default("jmap.push.attempts.interval", "1m") .unwrap_or_else(|| Duration::from_secs(60)), push_attempts_max: config .property_or_default("jmap.push.attempts.max", "3") .unwrap_or(3), push_retry_interval: config .property_or_default("jmap.push.retry.interval", "1s") .unwrap_or_else(|| Duration::from_secs(1)), push_timeout: config .property_or_default("jmap.push.timeout.request", "10s") .unwrap_or_else(|| Duration::from_secs(10)), push_verify_timeout: config .property_or_default("jmap.push.timeout.verify", "1m") .unwrap_or_else(|| Duration::from_secs(60)), push_throttle: config .property_or_default("jmap.push.throttle", "1s") .unwrap_or_else(|| Duration::from_secs(1)), account_purge_frequency: config .property_or_default::("account.purge.frequency", "0 0 *") .unwrap_or_else(|| SimpleCron::parse_value("0 0 *").unwrap()), fallback_admin: config .value("authentication.fallback-admin.user") .and_then(|u| { config .value("authentication.fallback-admin.secret") .map(|p| (u.to_string(), p.to_string())) }), master_user: config.value("authentication.master.user").and_then(|u| { config .value("authentication.master.secret") .map(|p| (u.to_string(), p.to_string())) }), contact_parse_max_items: config .property("jmap.contact.parse.max-items") .unwrap_or(10), calendar_parse_max_items: config .property("jmap.calendar.parse.max-items") .unwrap_or(10), index_batch_size: config .property("storage.search-index.batch-size") .unwrap_or(100), index_fields: AHashMap::new(), default_folders, shared_folder, }; // Parse index fields for index in [ SearchIndex::Email, SearchIndex::Contacts, SearchIndex::Calendar, SearchIndex::Tracing, ] { let mut fields = AHashSet::new(); let index_name = match index { SearchIndex::Email => "email", SearchIndex::Contacts => "contacts", SearchIndex::Calendar => "calendar", SearchIndex::Tracing => "tracing", _ => unreachable!(), }; if !config .property_or_default::( &format!("storage.search-index.{index_name}.enabled"), "true", ) .unwrap_or(true) { continue; } for (_, field) in config .properties::(&format!("storage.search-index.{index_name}.fields")) { fields.insert(field); } jmap.index_fields.insert(index, fields); } for collection in Bitmap::::all() { let key = format!("object-quota.{}", collection.as_config_case()); jmap.max_objects[collection as usize] = if let Some(value) = config.property::(&key) { value } else { match collection { Collection::Mailbox => 250, Collection::SieveScript => 100, Collection::Identity => 20, Collection::EmailSubmission => 500, Collection::PushSubscription => 15, Collection::Calendar => 250, Collection::AddressBook => 250, Collection::Principal | Collection::None | Collection::CalendarEventNotification | Collection::CalendarEvent | Collection::ContactCard | Collection::FileNode | Collection::Email | Collection::Thread => u32::MAX, } }; } // Add capabilities jmap.add_capabilities(config, groupware_config); jmap } } ================================================ FILE: crates/common/src/config/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use self::{ imap::ImapConfig, jmap::settings::JmapConfig, scripts::Scripting, smtp::SmtpConfig, storage::Storage, }; use crate::{ Core, Network, Security, auth::oauth::config::OAuthConfig, expr::*, listener::tls::AcmeProviders, manager::config::ConfigManager, }; use arc_swap::ArcSwap; use directory::{Directories, Directory}; use groupware::GroupwareConfig; use hyper::HeaderMap; use ring::signature::{EcdsaKeyPair, RsaKeyPair}; use spamfilter::SpamFilterConfig; use std::sync::Arc; use store::{BlobBackend, BlobStore, InMemoryStore, SearchStore, Store, Stores}; use telemetry::Metrics; use utils::config::{Config, utils::AsKey}; pub mod groupware; pub mod imap; pub mod inner; pub mod jmap; pub mod network; pub mod scripts; pub mod server; pub mod smtp; pub mod spamfilter; pub mod storage; pub mod telemetry; pub(crate) const CONNECTION_VARS: &[u32; 9] = &[ V_LISTENER, V_REMOTE_IP, V_REMOTE_PORT, V_LOCAL_IP, V_LOCAL_PORT, V_PROTOCOL, V_TLS, V_ASN, V_COUNTRY, ]; impl Core { pub async fn parse( config: &mut Config, mut stores: Stores, config_manager: ConfigManager, ) -> Self { let mut data = config .value_require("storage.data") .map(|id| id.to_string()) .and_then(|id| { if let Some(store) = stores.stores.get(&id) { store.clone().into() } else { config.new_parse_error("storage.data", format!("Data store {id:?} not found")); None } }) .unwrap_or_default(); #[cfg(not(feature = "enterprise"))] let is_enterprise = false; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] let enterprise = crate::enterprise::Enterprise::parse(config, &config_manager, &stores, &data).await; #[cfg(feature = "enterprise")] let is_enterprise = enterprise.is_some(); #[cfg(feature = "enterprise")] if !is_enterprise { if data.is_enterprise_store() { config .new_build_error("storage.data", "SQL read replicas is an Enterprise feature"); data = Store::None; } stores.disable_enterprise_only(); } // SPDX-SnippetEnd let mut blob = config .value_require("storage.blob") .map(|id| id.to_string()) .and_then(|id| { if let Some(store) = stores.blob_stores.get(&id) { store.clone().into() } else { config.new_parse_error("storage.blob", format!("Blob store {id:?} not found")); None } }) .unwrap_or_default(); let mut lookup = config .value_require("storage.lookup") .map(|id| id.to_string()) .and_then(|id| { if let Some(store) = stores.in_memory_stores.get(&id) { store.clone().into() } else { config.new_parse_error( "storage.lookup", format!("In-memory store {id:?} not found"), ); None } }) .unwrap_or_default(); let mut fts = config .value_require("storage.fts") .map(|id| id.to_string()) .and_then(|id| { if let Some(store) = stores.search_stores.get(&id) { store.clone().into() } else { config.new_parse_error( "storage.fts", format!("Full-text store {id:?} not found"), ); None } }) .unwrap_or_default(); let pubsub = config .value("cluster.coordinator") .map(|id| id.to_string()) .and_then(|id| { if let Some(store) = stores.pubsub_stores.get(&id) { store.clone().into() } else { config.new_parse_error( "cluster.coordinator", format!("Coordinator backend {id:?} not found"), ); None } }) .unwrap_or_default(); let mut directories = Directories::parse(config, &stores, data.clone(), is_enterprise).await; let directory = config .value_require("storage.directory") .map(|id| id.to_string()) .and_then(|id| { if let Some(directory) = directories.directories.get(&id) { directory.clone().into() } else { config.new_parse_error( "storage.directory", format!("Directory {id:?} not found"), ); None } }) .unwrap_or_else(|| Arc::new(Directory::default())); directories .directories .insert("*".to_string(), directory.clone()); // If any of the stores are missing, disable all stores to avoid data loss if matches!(data, Store::None) || matches!(&blob.backend, BlobBackend::Store(Store::None)) || matches!(lookup, InMemoryStore::Store(Store::None)) || matches!(fts, SearchStore::Store(Store::None)) { data = Store::default(); blob = BlobStore::default(); lookup = InMemoryStore::default(); fts = SearchStore::default(); config.new_build_error( "storage.*", "One or more stores are missing, disabling all stores", ) } let groupware = GroupwareConfig::parse(config); Self { // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] enterprise, // SPDX-SnippetEnd sieve: Scripting::parse(config, &stores).await, network: Network::parse(config), smtp: SmtpConfig::parse(config).await, jmap: JmapConfig::parse(config, &groupware), imap: ImapConfig::parse(config), oauth: OAuthConfig::parse(config), acme: AcmeProviders::parse(config), metrics: Metrics::parse(config), spam: SpamFilterConfig::parse(config).await, groupware, storage: Storage { data, blob, fts, lookup, pubsub, directory, directories: directories.directories, purge_schedules: stores.purge_schedules, config: config_manager, stores: stores.stores, lookups: stores.in_memory_stores, blobs: stores.blob_stores, ftss: stores.search_stores, }, } } pub fn into_shared(self) -> ArcSwap { ArcSwap::from_pointee(self) } } pub fn build_rsa_keypair(pem: &str) -> Result { match rustls_pemfile::read_one(&mut pem.as_bytes()) { Ok(Some(rustls_pemfile::Item::Pkcs1Key(key))) => { RsaKeyPair::from_der(key.secret_pkcs1_der()) .map_err(|err| format!("Failed to parse PKCS1 RSA key: {err}")) } Ok(Some(rustls_pemfile::Item::Pkcs8Key(key))) => { RsaKeyPair::from_pkcs8(key.secret_pkcs8_der()) .map_err(|err| format!("Failed to parse PKCS8 RSA key: {err}")) } Err(err) => Err(format!("Failed to read PEM: {err}")), Ok(Some(key)) => Err(format!("Unsupported key type: {key:?}")), Ok(None) => Err("No RSA key found in PEM".to_string()), } } pub fn build_ecdsa_pem( alg: &'static ring::signature::EcdsaSigningAlgorithm, pem: &str, ) -> Result { match rustls_pemfile::read_one(&mut pem.as_bytes()) { Ok(Some(rustls_pemfile::Item::Pkcs8Key(key))) => EcdsaKeyPair::from_pkcs8( alg, key.secret_pkcs8_der(), &ring::rand::SystemRandom::new(), ) .map_err(|err| format!("Failed to parse PKCS8 ECDSA key: {err}")), Err(err) => Err(format!("Failed to read PEM: {err}")), Ok(Some(key)) => Err(format!("Unsupported key type: {key:?}")), Ok(None) => Err("No ECDSA key found in PEM".to_string()), } } ================================================ FILE: crates/common/src/config/network.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::*; use crate::expr::{if_block::IfBlock, tokenizer::TokenMap}; use ahash::AHashSet; use std::{hash::Hasher, time::Duration}; use utils::config::{Config, Rate, http::parse_http_headers, utils::ParseValue}; use xxhash_rust::xxh3::Xxh3Builder; #[derive(Clone)] pub struct Network { pub node_id: u64, pub roles: ClusterRoles, pub server_name: String, pub report_domain: String, pub security: Security, pub contact_form: Option, pub http_response_url: IfBlock, pub http_allowed_endpoint: IfBlock, pub asn_geo_lookup: AsnGeoLookupConfig, } #[derive(Clone)] pub struct ContactForm { pub rcpt_to: Vec, pub max_size: usize, pub rate: Option, pub validate_domain: bool, pub from_email: FieldOrDefault, pub from_subject: FieldOrDefault, pub from_name: FieldOrDefault, pub field_honey_pot: Option, } #[derive(Clone, Default)] pub struct ClusterRoles { pub purge_stores: ClusterRole, pub purge_accounts: ClusterRole, pub push_notifications: ClusterRole, pub fts_indexing: ClusterRole, pub spam_training: ClusterRole, pub imip_processing: ClusterRole, pub merge_threads: ClusterRole, pub calendar_alerts: ClusterRole, pub renew_acme: ClusterRole, pub calculate_metrics: ClusterRole, pub push_metrics: ClusterRole, } #[derive(Clone, Copy, Default)] pub enum ClusterRole { #[default] Enabled, Disabled, Sharded { shard_id: u32, total_shards: u32, }, } #[derive(Clone, Default)] pub enum AsnGeoLookupConfig { Resource { expires: Duration, timeout: Duration, max_size: usize, headers: HeaderMap, asn_resources: Vec, geo_resources: Vec, }, Dns { zone_ipv4: String, zone_ipv6: String, separator: String, index_asn: usize, index_asn_name: Option, index_country: Option, }, #[default] Disabled, } #[derive(Clone)] pub struct FieldOrDefault { pub field: Option, pub default: String, } pub(crate) const HTTP_VARS: &[u32; 11] = &[ V_LISTENER, V_REMOTE_IP, V_REMOTE_PORT, V_LOCAL_IP, V_LOCAL_PORT, V_PROTOCOL, V_TLS, V_URL, V_URL_PATH, V_HEADERS, V_METHOD, ]; impl Default for Network { fn default() -> Self { Self { security: Default::default(), contact_form: None, node_id: 1, http_response_url: IfBlock::new::<()>( "http.url", [], "protocol + '://' + config_get('server.hostname') + ':' + local_port", ), http_allowed_endpoint: IfBlock::new::<()>("http.allowed-endpoint", [], "200"), asn_geo_lookup: AsnGeoLookupConfig::Disabled, server_name: Default::default(), report_domain: Default::default(), roles: ClusterRoles::default(), } } } impl ContactForm { pub fn parse(config: &mut Config) -> Option { if !config .property_or_default::("form.enable", "false") .unwrap_or_default() { return None; } let form = ContactForm { rcpt_to: config .values("form.deliver-to") .filter_map(|(_, addr)| { if addr.contains('@') && addr.contains('.') { Some(addr.trim().to_lowercase()) } else { None } }) .collect(), max_size: config.property("form.max-size").unwrap_or(100 * 1024), validate_domain: config .property_or_default::("form.validate-domain", "true") .unwrap_or(true), from_email: FieldOrDefault::parse(config, "form.email", "postmaster@localhost"), from_subject: FieldOrDefault::parse(config, "form.subject", "Contact form submission"), from_name: FieldOrDefault::parse(config, "form.name", "Anonymous"), field_honey_pot: config.value("form.honey-pot.field").map(|v| v.into()), rate: config .property_or_default::>("form.rate-limit", "5/1h") .unwrap_or_default(), }; if !form.rcpt_to.is_empty() { Some(form) } else { config.new_build_error("form.deliver-to", "No valid email addresses found"); None } } } impl FieldOrDefault { pub fn parse(config: &mut Config, key: &str, default: &str) -> Self { FieldOrDefault { field: config.value((key, "field")).map(|s| s.to_string()), default: config .value((key, "default")) .unwrap_or(default) .to_string(), } } } impl Network { pub fn parse(config: &mut Config) -> Self { let server_name = config .value("server.hostname") .map(|v| v.to_string()) .or_else(|| { config .value("lookup.default.hostname") .map(|v| v.to_lowercase()) }) .unwrap_or_else(|| { hostname::get() .map(|v| v.to_string_lossy().to_lowercase()) .unwrap_or_else(|_| "localhost".to_string()) }); let report_domain = config .value("report.domain") .map(|v| v.to_lowercase()) .or_else(|| { config .value("lookup.default.domain") .map(|v| v.to_lowercase()) }) .unwrap_or_else(|| { psl::domain_str(&server_name) .unwrap_or(server_name.as_str()) .to_string() }); let mut network = Network { node_id: config.property("cluster.node-id").unwrap_or(1), report_domain, server_name, security: Security::parse(config), contact_form: ContactForm::parse(config), asn_geo_lookup: AsnGeoLookupConfig::parse(config).unwrap_or_default(), ..Default::default() }; let token_map = &TokenMap::default().with_variables(HTTP_VARS); // Node roles for (value, key) in [ ( &mut network.roles.purge_stores, "cluster.roles.purge.stores", ), ( &mut network.roles.purge_accounts, "cluster.roles.purge.accounts", ), (&mut network.roles.renew_acme, "cluster.roles.acme.renew"), ( &mut network.roles.calculate_metrics, "cluster.roles.metrics.calculate", ), ( &mut network.roles.push_metrics, "cluster.roles.metrics.push", ), ( &mut network.roles.push_notifications, "cluster.roles.push-notifications", ), ( &mut network.roles.fts_indexing, "cluster.roles.fts-indexing", ), ( &mut network.roles.spam_training, "cluster.roles.spam-training", ), ( &mut network.roles.imip_processing, "cluster.roles.imip-processing", ), ( &mut network.roles.calendar_alerts, "cluster.roles.calendar-alerts", ), ( &mut network.roles.merge_threads, "cluster.roles.merge-threads", ), ] { let shards = config .properties::(key) .into_iter() .map(|(_, v)| v) .collect::>(); let shard_size = shards.len() as u32; let mut found_node = false; for (shard_id, shard) in shards.iter().enumerate() { if shard.0.contains(&network.node_id) { if shard_size > 1 { *value = ClusterRole::Sharded { shard_id: shard_id as u32, total_shards: shard_size, }; } found_node = true; break; } } if !shards.is_empty() && !found_node { *value = ClusterRole::Disabled; } } for (value, key) in [ (&mut network.http_response_url, "http.url"), (&mut network.http_allowed_endpoint, "http.allowed-endpoint"), ] { if let Some(if_block) = IfBlock::try_parse(config, key, token_map) { *value = if_block; } } network } } struct NodeList(AHashSet); impl ParseValue for NodeList { fn parse_value(value: &str) -> utils::config::Result { value .split(',') .map(|s| s.trim().parse::().map_err(|e| e.to_string())) .collect::, String>>() .map(NodeList) } } impl AsnGeoLookupConfig { pub fn parse(config: &mut Config) -> Option { match config.value("asn.type")? { "dns" => AsnGeoLookupConfig::Dns { zone_ipv4: config.value_require_non_empty("asn.zone.ipv4")?.to_string(), zone_ipv6: config.value_require_non_empty("asn.zone.ipv6")?.to_string(), separator: config.value_require_non_empty("asn.separator")?.to_string(), index_asn: config.property_require("asn.index.asn")?, index_asn_name: config.property("asn.index.asn-name"), index_country: config.property("asn.index.country"), } .into(), "resource" => { let asn_resources = config .values("asn.urls.asn") .map(|(_, v)| v.to_string()) .collect::>(); let geo_resources = config .values("asn.urls.geo") .map(|(_, v)| v.to_string()) .collect::>(); if asn_resources.is_empty() && geo_resources.is_empty() { config.new_build_error("asn.urls", "No resources found"); return None; } AsnGeoLookupConfig::Resource { headers: parse_http_headers(config, "asn"), expires: config.property_or_default::("asn.expires", "1d")?, timeout: config.property_or_default::("asn.timeout", "5m")?, max_size: config.property("asn.max-size").unwrap_or(100 * 1024 * 1024), asn_resources, geo_resources, } .into() } "disable" | "disabled" | "none" | "false" => AsnGeoLookupConfig::Disabled.into(), _ => { config.new_build_error("asn.type", "Invalid value"); None } } } } impl ClusterRole { pub fn is_enabled_or_sharded(&self) -> bool { matches!(self, ClusterRole::Enabled | ClusterRole::Sharded { .. }) } pub fn is_enabled_for_integer(&self, value: u32) -> bool { match self { ClusterRole::Enabled => true, ClusterRole::Disabled => false, ClusterRole::Sharded { shard_id, total_shards, } => (value % total_shards) == *shard_id, } } pub fn is_enabled_for_hash(&self, item: &impl std::hash::Hash) -> bool { match self { ClusterRole::Enabled => true, ClusterRole::Disabled => false, ClusterRole::Sharded { shard_id, total_shards, } => { let mut hasher = Xxh3Builder::new().with_seed(191179).build(); item.hash(&mut hasher); hasher.finish() % (*total_shards as u64) == *shard_id as u64 } } } } ================================================ FILE: crates/common/src/config/scripts.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{sync::Arc, time::Duration}; use ahash::AHashMap; use sieve::{Compiler, Runtime, Sieve, compiler::grammar::Capability}; use store::Stores; use utils::config::Config; use crate::{ VERSION_PUBLIC, scripts::{ functions::{register_functions_trusted, register_functions_untrusted}, plugins::RegisterSievePlugins, }, }; use super::{if_block::IfBlock, smtp::SMTP_RCPT_TO_VARS, tokenizer::TokenMap}; pub struct Scripting { pub untrusted_compiler: Compiler, pub untrusted_runtime: Runtime, pub trusted_runtime: Runtime, pub from_addr: IfBlock, pub from_name: IfBlock, pub return_path: IfBlock, pub sign: IfBlock, pub trusted_scripts: AHashMap>, pub untrusted_scripts: AHashMap>, } impl Scripting { pub async fn parse(config: &mut Config, stores: &Stores) -> Self { // Parse untrusted compiler let mut fnc_map_untrusted = register_functions_untrusted().register_plugins_untrusted(); let untrusted_compiler = Compiler::new() .with_max_script_size( config .property("sieve.untrusted.limits.script-size") .unwrap_or(1024 * 1024), ) .with_max_string_size( config .property("sieve.untrusted.limits.string-length") .unwrap_or(4096), ) .with_max_variable_name_size( config .property("sieve.untrusted.limits.variable-name-length") .unwrap_or(32), ) .with_max_nested_blocks( config .property("sieve.untrusted.limits.nested-blocks") .unwrap_or(15), ) .with_max_nested_tests( config .property("sieve.untrusted.limits.nested-tests") .unwrap_or(15), ) .with_max_nested_foreverypart( config .property("sieve.untrusted.limits.nested-foreverypart") .unwrap_or(3), ) .with_max_match_variables( config .property("sieve.untrusted.limits.match-variables") .unwrap_or(30), ) .with_max_local_variables( config .property("sieve.untrusted.limits.local-variables") .unwrap_or(128), ) .with_max_header_size( config .property("sieve.untrusted.limits.header-size") .unwrap_or(1024), ) .with_max_includes( config .property("sieve.untrusted.limits.includes") .unwrap_or(3), ) .register_functions(&mut fnc_map_untrusted); // Parse untrusted runtime let untrusted_runtime = Runtime::new() .with_functions(&mut fnc_map_untrusted) .with_max_nested_includes( config .property("sieve.untrusted.limits.nested-includes") .unwrap_or(3), ) .with_cpu_limit( config .property("sieve.untrusted.limits.cpu") .unwrap_or(5000), ) .with_max_variable_size( config .property("sieve.untrusted.limits.variable-size") .unwrap_or(4096), ) .with_max_redirects( config .property("sieve.untrusted.limits.redirects") .unwrap_or(1), ) .with_max_received_headers( config .property("sieve.untrusted.limits.received-headers") .unwrap_or(10), ) .with_max_header_size( config .property("sieve.untrusted.limits.header-size") .unwrap_or(1024), ) .with_max_out_messages( config .property("sieve.untrusted.limits.outgoing-messages") .unwrap_or(3), ) .with_default_vacation_expiry( config .property::("sieve.untrusted.default-expiry.vacation") .unwrap_or(Duration::from_secs(30 * 86400)) .as_secs(), ) .with_default_duplicate_expiry( config .property::("sieve.untrusted.default-expiry.duplicate") .unwrap_or(Duration::from_secs(7 * 86400)) .as_secs(), ) .with_capability(Capability::Expressions) .without_capabilities( config .values("sieve.untrusted.disable-capabilities") .map(|(_, v)| v), ) .with_valid_notification_uris({ let values = config .values("sieve.untrusted.notification-uris") .map(|(_, v)| v.to_string()) .collect::>(); if !values.is_empty() { values } else { vec!["mailto".to_string()] } }) .with_protected_headers({ let values = config .values("sieve.untrusted.protected-headers") .map(|(_, v)| v.to_string()) .collect::>(); if !values.is_empty() { values } else { vec![ "Original-Subject".to_string(), "Original-From".to_string(), "Received".to_string(), "Auto-Submitted".to_string(), ] } }) .with_vacation_default_subject( config .value("sieve.untrusted.vacation.default-subject") .unwrap_or("Automated reply") .to_string(), ) .with_vacation_subject_prefix( config .value("sieve.untrusted.vacation.subject-prefix") .unwrap_or("Auto: ") .to_string(), ) .with_env_variable("name", "Stalwart Server") .with_env_variable("version", VERSION_PUBLIC) .with_env_variable("location", "MS") .with_env_variable("phase", "during"); // Parse trusted compiler and runtime let mut fnc_map_trusted = register_functions_trusted().register_plugins_trusted(); // Allocate compiler and runtime let trusted_compiler = Compiler::new() .with_max_string_size(52428800) .with_max_variable_name_size(100) .with_max_nested_blocks(50) .with_max_nested_tests(50) .with_max_nested_foreverypart(10) .with_max_local_variables(8192) .with_max_header_size(10240) .with_max_includes(10) .with_no_capability_check( config .property_or_default("sieve.trusted.no-capability-check", "true") .unwrap_or(true), ) .register_functions(&mut fnc_map_trusted); let mut trusted_runtime = Runtime::new() .without_capabilities([ Capability::FileInto, Capability::Vacation, Capability::VacationSeconds, Capability::Fcc, Capability::Mailbox, Capability::MailboxId, Capability::MboxMetadata, Capability::ServerMetadata, Capability::ImapSieve, Capability::Duplicate, ]) .with_capability(Capability::Expressions) .with_capability(Capability::While) .with_max_variable_size( config .property_or_default("sieve.trusted.limits.variable-size", "52428800") .unwrap_or(52428800), ) .with_max_header_size(10240) .with_valid_notification_uri("mailto") .with_valid_ext_lists(stores.in_memory_stores.keys().map(|k| k.to_string())) .with_functions(&mut fnc_map_trusted) .with_max_redirects( config .property_or_default("sieve.trusted.limits.redirects", "3") .unwrap_or(3), ) .with_max_out_messages( config .property_or_default("sieve.trusted.limits.out-messages", "5") .unwrap_or(5), ) .with_cpu_limit( config .property_or_default("sieve.trusted.limits.cpu", "1048576") .unwrap_or(1048576), ) .with_max_nested_includes( config .property_or_default("sieve.trusted.limits.nested-includes", "5") .unwrap_or(5), ) .with_max_received_headers( config .property_or_default("sieve.trusted.limits.received-headers", "50") .unwrap_or(50), ) .with_default_duplicate_expiry( config .property_or_default::("sieve.trusted.limits.duplicate-expiry", "7d") .unwrap_or_else(|| Duration::from_secs(604800)) .as_secs(), ); let hostname = config .value("sieve.trusted.hostname") .or_else(|| config.value("server.hostname")) .unwrap_or("localhost") .to_string(); trusted_runtime.set_local_hostname(hostname.clone()); // Parse trusted scripts let mut trusted_scripts = AHashMap::new(); for id in config.sub_keys("sieve.trusted.scripts", ".contents") { match trusted_compiler.compile( config .value(("sieve.trusted.scripts", id.as_str(), "contents")) .unwrap() .as_bytes(), ) { Ok(compiled) => { trusted_scripts.insert(id, compiled.into()); } Err(err) => config.new_build_error( ("sieve.trusted.scripts", id.as_str(), "contents"), format!("Failed to compile trusted Sieve script: {err}"), ), } } // Parse untrusted scripts let mut untrusted_scripts = AHashMap::new(); for id in config.sub_keys("sieve.untrusted.scripts", ".contents") { match untrusted_compiler.compile( config .value(("sieve.untrusted.scripts", id.as_str(), "contents")) .unwrap() .as_bytes(), ) { Ok(compiled) => { untrusted_scripts.insert(id, compiled.into()); } Err(err) => config.new_build_error( ("sieve.untrusted.scripts", id.as_str(), "contents"), format!("Failed to compile untrusted Sieve script: {err}"), ), } } let token_map = TokenMap::default().with_variables(SMTP_RCPT_TO_VARS); Scripting { untrusted_compiler, untrusted_runtime, trusted_runtime, from_addr: IfBlock::try_parse(config, "sieve.trusted.from-addr", &token_map) .unwrap_or_else(|| { IfBlock::new::<()>( "sieve.trusted.from-addr", [], "'MAILER-DAEMON@' + config_get('report.domain')", ) }), from_name: IfBlock::try_parse(config, "sieve.trusted.from-name", &token_map) .unwrap_or_else(|| { IfBlock::new::<()>("sieve.trusted.from-name", [], "'Automated Message'") }), return_path: IfBlock::try_parse(config, "sieve.trusted.return-path", &token_map) .unwrap_or_else(|| IfBlock::empty("sieve.trusted.return-path")), sign: IfBlock::try_parse(config, "sieve.trusted.sign", &token_map).unwrap_or_else( || { IfBlock::new::<()>( "sieve.trusted.sign", [], concat!( "['rsa-' + config_get('report.domain'), ", "'ed25519-' + config_get('report.domain')]" ), ) }, ), untrusted_scripts, trusted_scripts, } } } impl Default for Scripting { fn default() -> Self { Scripting { untrusted_compiler: Compiler::new(), untrusted_runtime: Runtime::new(), trusted_runtime: Runtime::new(), from_addr: IfBlock::new::<()>( "sieve.trusted.from-addr", [], "'MAILER-DAEMON@' + config_get('report.domain')", ), from_name: IfBlock::new::<()>("sieve.trusted.from-name", [], "'Mailer Daemon'"), return_path: IfBlock::empty("sieve.trusted.return-path"), sign: IfBlock::new::<()>( "sieve.trusted.sign", [], concat!( "['rsa-' + config_get('report.domain'), ", "'ed25519-' + config_get('report.domain')]" ), ), untrusted_scripts: AHashMap::new(), trusted_scripts: AHashMap::new(), } } } impl Clone for Scripting { fn clone(&self) -> Self { Self { untrusted_compiler: self.untrusted_compiler.clone(), untrusted_runtime: self.untrusted_runtime.clone(), trusted_runtime: self.trusted_runtime.clone(), from_addr: self.from_addr.clone(), from_name: self.from_name.clone(), return_path: self.return_path.clone(), sign: self.sign.clone(), trusted_scripts: self.trusted_scripts.clone(), untrusted_scripts: self.untrusted_scripts.clone(), } } } ================================================ FILE: crates/common/src/config/server/listener.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{net::SocketAddr, sync::Arc, time::Duration}; use rustls::{ ALL_VERSIONS, ServerConfig, SupportedCipherSuite, crypto::ring::{ALL_CIPHER_SUITES, default_provider}, }; use tokio::net::TcpSocket; use tokio_rustls::TlsAcceptor; use utils::{ config::{ Config, utils::{AsKey, ParseValue}, }, snowflake::SnowflakeIdGenerator, }; use crate::{ Inner, listener::{TcpAcceptor, tls::CertificateResolver}, }; use super::{ Listener, Listeners, ServerProtocol, TcpListener, tls::{TLS12_VERSION, TLS13_VERSION}, }; impl Listeners { pub fn parse(config: &mut Config) -> Self { // Parse ACME managers let mut servers = Listeners { span_id_gen: Arc::new( config .property::("cluster.node-id") .map(SnowflakeIdGenerator::with_node_id) .unwrap_or_default(), ), ..Default::default() }; // Parse servers for id in config.sub_keys("server.listener", ".protocol") { servers.parse_server(config, id); } servers } fn parse_server(&mut self, config: &mut Config, id_: String) { // Parse protocol let id = id_.as_str(); let protocol = if let Some(protocol) = config.property_require(("server.listener", id, "protocol")) { protocol } else { return; }; // Build listeners let mut listeners = Vec::new(); for (_, addr) in config.properties::(("server.listener", id, "bind")) { // Parse bind address and build socket let socket = match if addr.is_ipv4() { TcpSocket::new_v4() } else { TcpSocket::new_v6() } { Ok(socket) => socket, Err(err) => { config.new_build_error( ("server.listener", id, "bind"), format!("Failed to create socket: {err}"), ); return; } }; // Set socket options for option in [ "reuse-addr", "reuse-port", "send-buffer-size", "recv-buffer-size", "tos", ] { if let Some(value) = config.value_or_else( ("server.listener", id, "socket", option), ("server.socket", option), ) { let value = value.to_string(); let key = ("server.listener", id, "socket", option); let result = match option { "reuse-addr" => socket .set_reuseaddr(config.try_parse_value(key, &value).unwrap_or(true)), #[cfg(not(target_env = "msvc"))] "reuse-port" => socket .set_reuseport(config.try_parse_value(key, &value).unwrap_or(false)), "send-buffer-size" => { if let Some(value) = config.try_parse_value(key, &value) { socket.set_send_buffer_size(value) } else { continue; } } "recv-buffer-size" => { if let Some(value) = config.try_parse_value(key, &value) { socket.set_recv_buffer_size(value) } else { continue; } } "tos" => { if let Some(value) = config.try_parse_value(key, &value) { socket.set_tos(value) } else { continue; } } _ => continue, }; if let Err(err) = result { config.new_build_error(key, format!("Failed to set socket option: {err}")); } } } // Set default options if !config.contains_key(("server.listener", id, "socket.reuse-addr")) { let _ = socket.set_reuseaddr(true); } listeners.push(TcpListener { socket, addr, ttl: config .property_or_else::>( ("server.listener", id, "socket.ttl"), "server.socket.ttl", "false", ) .unwrap_or_default(), backlog: config .property_or_else::>( ("server.listener", id, "socket.backlog"), "server.socket.backlog", "1024", ) .unwrap_or_default(), linger: config .property_or_else::>( ("server.listener", id, "socket.linger"), "server.socket.linger", "false", ) .unwrap_or_default(), nodelay: config .property_or_else( ("server.listener", id, "socket.nodelay"), "server.socket.nodelay", "true", ) .unwrap_or(true), }); } if listeners.is_empty() { config.new_build_error( ("server.listener", id), "No 'bind' directive found for listener", ); return; } // Parse proxy networks let mut proxy_networks = Vec::new(); let proxy_keys = if config .value(("server.listener", id, "proxy.trusted-networks")) .is_some() || config.has_prefix(("server.listener", id, "proxy.trusted-networks")) { ("server.listener", id, "proxy.trusted-networks").as_key() } else { "server.proxy.trusted-networks".as_key() }; for (_, network) in config.properties(proxy_keys) { proxy_networks.push(network); } let span_id_gen = self.span_id_gen.clone(); self.servers.push(Listener { max_connections: config .property_or_else( ("server.listener", id, "max-connections"), "server.max-connections", "8192", ) .unwrap_or(8192), id: id_, protocol, listeners, proxy_networks, span_id_gen, }); } pub fn parse_tcp_acceptors(&mut self, config: &mut Config, inner: Arc) { let resolver = Arc::new(CertificateResolver::new(inner.clone())); for id_ in config.sub_keys("server.listener", ".protocol") { let id = id_.as_str(); // Build TLS config let acceptor = if config .property_or_default(("server.listener", id, "tls.enable"), "true") .unwrap_or(true) { // Parse protocol versions let mut tls_v2 = true; let mut tls_v3 = true; let mut proto_err = None; for (_, protocol) in config.values_or_else( ("server.listener", id, "tls.disable-protocols"), "server.tls.disable-protocols", ) { match protocol { "TLSv1.2" | "0x0303" => tls_v2 = false, "TLSv1.3" | "0x0304" => tls_v3 = false, protocol => { proto_err = format!("Unsupported TLS protocol {protocol:?}").into(); } } } if let Some(proto_err) = proto_err { config.new_parse_error( ("server.listener", id, "tls.disable-protocols"), proto_err, ); } // Parse cipher suites let mut disabled_ciphers: Vec = Vec::new(); let cipher_keys = if config.has_prefix(("server.listener", id, "tls.disable-ciphers")) { ("server.listener", id, "tls.disable-ciphers").as_key() } else { "server.tls.disable-ciphers".as_key() }; for (_, protocol) in config.properties::(cipher_keys) { disabled_ciphers.push(protocol); } // Build cert provider let mut provider = default_provider(); if !disabled_ciphers.is_empty() { provider.cipher_suites = ALL_CIPHER_SUITES .iter() .filter(|suite| !disabled_ciphers.contains(suite)) .copied() .collect(); } // Build server config let mut server_config = match ServerConfig::builder_with_provider(provider.into()) .with_protocol_versions(if tls_v3 == tls_v2 { ALL_VERSIONS } else if tls_v3 { TLS13_VERSION } else { TLS12_VERSION }) { Ok(server_config) => server_config .with_no_client_auth() .with_cert_resolver(resolver.clone()), Err(err) => { config.new_build_error( ("server.listener", id, "tls"), format!("Failed to build TLS server config: {err}"), ); return; } }; server_config.ignore_client_order = config .property_or_else( ("server.listener", id, "tls.ignore-client-order"), "server.tls.ignore-client-order", "true", ) .unwrap_or(true); // Build acceptor let default_config = Arc::new(server_config); TcpAcceptor::Tls { acceptor: TlsAcceptor::from(default_config.clone()), config: default_config, implicit: config .property_or_default(("server.listener", id, "tls.implicit"), "false") .unwrap_or(false), } } else { TcpAcceptor::Plain }; self.tcp_acceptors.insert(id_, acceptor); } } } impl ParseValue for ServerProtocol { fn parse_value(value: &str) -> Result { if value.eq_ignore_ascii_case("smtp") { Ok(Self::Smtp) } else if value.eq_ignore_ascii_case("lmtp") { Ok(Self::Lmtp) } else if value.eq_ignore_ascii_case("imap") { Ok(Self::Imap) } else if value.eq_ignore_ascii_case("http") | value.eq_ignore_ascii_case("https") { Ok(Self::Http) } else if value.eq_ignore_ascii_case("managesieve") { Ok(Self::ManageSieve) } else if value.eq_ignore_ascii_case("pop3") { Ok(Self::Pop3) } else { Err(format!("Invalid server protocol type {:?}.", value,)) } } } ================================================ FILE: crates/common/src/config/server/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{fmt::Display, net::SocketAddr, sync::Arc, time::Duration}; use ahash::AHashMap; use serde::{Deserialize, Serialize}; use tokio::net::TcpSocket; use utils::{config::ipmask::IpAddrMask, snowflake::SnowflakeIdGenerator}; use crate::listener::TcpAcceptor; pub mod listener; pub mod tls; #[derive(Default)] pub struct Listeners { pub servers: Vec, pub tcp_acceptors: AHashMap, pub span_id_gen: Arc, } #[derive(Debug, Default)] pub struct Listener { pub id: String, pub protocol: ServerProtocol, pub listeners: Vec, pub proxy_networks: Vec, pub max_connections: u64, pub span_id_gen: Arc, } #[derive(Debug)] pub struct TcpListener { pub socket: TcpSocket, pub addr: SocketAddr, pub backlog: Option, // TCP options pub ttl: Option, pub linger: Option, pub nodelay: bool, } #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy, Default, Serialize, Deserialize)] pub enum ServerProtocol { #[default] Smtp, Lmtp, Imap, Pop3, Http, ManageSieve, } impl ServerProtocol { pub fn as_str(&self) -> &'static str { match self { ServerProtocol::Smtp => "smtp", ServerProtocol::Lmtp => "lmtp", ServerProtocol::Imap => "imap", ServerProtocol::Http => "http", ServerProtocol::Pop3 => "pop3", ServerProtocol::ManageSieve => "managesieve", } } } impl Display for ServerProtocol { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(self.as_str()) } } ================================================ FILE: crates/common/src/config/server/tls.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ io::Cursor, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, sync::Arc, time::Duration, }; use ahash::{AHashMap, AHashSet}; use base64::{ Engine, engine::general_purpose::{self, STANDARD}, }; use dns_update::{DnsUpdater, TsigAlgorithm, providers::rfc2136::DnsAddress}; use rcgen::generate_simple_self_signed; use rustls::{ SupportedProtocolVersion, crypto::ring::sign::any_supported_type, sign::CertifiedKey, version::{TLS12, TLS13}, }; use rustls_pemfile::{Item, certs, read_one}; use rustls_pki_types::PrivateKeyDer; use utils::config::Config; use x509_parser::{ certificate::X509Certificate, der_parser::asn1_rs::FromDer, extensions::{GeneralName, ParsedExtension}, }; use crate::listener::{ acme::{ AcmeProvider, ChallengeSettings, EabSettings, directory::LETS_ENCRYPT_PRODUCTION_DIRECTORY, }, tls::AcmeProviders, }; pub static TLS13_VERSION: &[&SupportedProtocolVersion] = &[&TLS13]; pub static TLS12_VERSION: &[&SupportedProtocolVersion] = &[&TLS12]; impl AcmeProviders { pub fn parse(config: &mut Config) -> Self { let mut providers = AHashMap::new(); // Parse ACME providers 'outer: for acme_id in config.sub_keys("acme", ".directory") { let acme_id = acme_id.as_str(); let directory = config .value(("acme", acme_id, "directory")) .unwrap_or(LETS_ENCRYPT_PRODUCTION_DIRECTORY) .trim() .to_string(); let contact = config .values(("acme", acme_id, "contact")) .filter_map(|(_, v)| { let v = v.trim().to_string(); if !v.is_empty() { Some(v) } else { None } }) .collect::>(); let renew_before: Duration = config .property_or_default(("acme", acme_id, "renew-before"), "30d") .unwrap_or_else(|| Duration::from_secs(30 * 24 * 60 * 60)); if directory.is_empty() { config.new_parse_error(format!("acme.{acme_id}.directory"), "Missing property"); continue; } if contact.is_empty() { config.new_parse_error(format!("acme.{acme_id}.contact"), "Missing property"); continue; } // Parse challenge type let challenge = match config .value(("acme", acme_id, "challenge")) .unwrap_or("tls-alpn-01") { "tls-alpn-01" => ChallengeSettings::TlsAlpn01, "http-01" => ChallengeSettings::Http01, "dns-01" => match build_dns_updater(config, acme_id) { Some(updater) => ChallengeSettings::Dns01 { updater, origin: config .value(("acme", acme_id, "origin")) .map(|s| s.to_string()), polling_interval: config .property_or_default(("acme", acme_id, "polling-interval"), "15s") .unwrap_or_else(|| Duration::from_secs(15)), propagation_timeout: config .property_or_default(("acme", acme_id, "propagation-timeout"), "1m") .unwrap_or_else(|| Duration::from_secs(60)), ttl: config .property_or_default(("acme", acme_id, "ttl"), "5m") .unwrap_or_else(|| Duration::from_secs(5 * 60)) .as_secs() as u32, }, None => { continue; } }, _ => { config .new_parse_error(("acme", acme_id, "challenge"), "Invalid challenge type"); continue; } }; // Domains covered by this ACME manager let domains = config .values(("acme", acme_id, "domains")) .map(|(_, s)| s.trim().to_string()) .collect::>(); if !matches!(challenge, ChallengeSettings::Dns01 { .. }) && domains.iter().any(|d| d.starts_with("*.")) { config.new_parse_error( ("acme", acme_id, "domains"), "Wildcard domains are only supported with DNS-01 challenge", ); continue 'outer; } // Obtain EAB settings let eab = if let (Some(eab_kid), Some(eab_hmac_key)) = ( config .value(("acme", acme_id, "eab.kid")) .filter(|s| !s.is_empty()), config .value(("acme", acme_id, "eab.hmac-key")) .filter(|s| !s.is_empty()), ) { if let Ok(hmac_key) = general_purpose::URL_SAFE_NO_PAD.decode(eab_hmac_key.trim().as_bytes()) { EabSettings { kid: eab_kid.to_string(), hmac_key, } .into() } else { config.new_build_error( format!("acme.{acme_id}.eab.hmac-key"), "Failed to base64 decode HMAC key", ); None } } else { None }; // This ACME manager is the default when SNI is not available let default = config .property::(("acme", acme_id, "default")) .unwrap_or_default(); if !domains.is_empty() { match AcmeProvider::new( acme_id.to_string(), directory, domains, contact, challenge, eab, renew_before, default, ) { Ok(acme_provider) => { providers.insert(acme_id.to_string(), acme_provider); } Err(err) => { config.new_build_error(format!("acme.{acme_id}"), err.to_string()); } } } } AcmeProviders { providers } } } #[allow(clippy::unnecessary_to_owned)] fn build_dns_updater(config: &mut Config, acme_id: &str) -> Option { let timeout = config .property_or_default(("acme", acme_id, "timeout"), "30s") .unwrap_or_else(|| Duration::from_secs(30)); match config.value_require(("acme", acme_id, "provider"))? { "rfc2136-tsig" => { let algorithm: TsigAlgorithm = config .value_require(("acme", acme_id, "tsig-algorithm"))? .parse() .map_err(|_| { config.new_parse_error(("acme", acme_id, "tsig-algorithm"), "Invalid algorithm") }) .ok()?; let key = STANDARD .decode(config.value_require(("acme", acme_id, "secret"))?.trim()) .map_err(|_| { config.new_parse_error( ("acme", acme_id, "secret"), "Failed to base64 decode secret", ) }) .ok()?; let host = config.property_require::(("acme", acme_id, "host"))?; let port = config .property_or_default::(("acme", acme_id, "port"), "53") .unwrap_or(53); let addr = if config.value(("acme", acme_id, "protocol")) == Some("tcp") { DnsAddress::Tcp(SocketAddr::new(host, port)) } else { DnsAddress::Udp(SocketAddr::new(host, port)) }; DnsUpdater::new_rfc2136_tsig( addr, config .value_require(("acme", acme_id, "key"))? .trim() .to_string(), key, algorithm, ) .map_err(|err| { config.new_build_error( ("acme", acme_id, "provider"), format!("Failed to create RFC2136-TSIG DNS updater: {err}"), ) }) .ok() } "cloudflare" => DnsUpdater::new_cloudflare( config .value_require(("acme", acme_id, "secret"))? .trim() .to_string(), config.value(("acme", acme_id, "user")).map(|s| s.trim()), timeout.into(), ) .map_err(|err| { config.new_build_error( ("acme", acme_id, "provider"), format!("Failed to create Cloudflare DNS updater: {err}"), ) }) .ok(), "digitalocean" => DnsUpdater::new_digitalocean( config .value_require(("acme", acme_id, "secret"))? .trim() .to_string(), timeout.into(), ) .map_err(|err| { config.new_build_error( ("acme", acme_id, "provider"), format!("Failed to create DigitalOcean DNS updater: {err}"), ) }) .ok(), "desec" => DnsUpdater::new_desec( config .value_require(("acme", acme_id, "secret"))? .trim() .to_string(), timeout.into(), ) .map_err(|err| { config.new_build_error( ("acme", acme_id, "provider"), format!("Failed to create Desec DNS updater: {err}"), ) }) .ok(), "ovh" => DnsUpdater::new_ovh( config .value_require(("acme", acme_id, "key")) .map(|s| s.trim())? .to_string(), config .value_require(("acme", acme_id, "secret"))? .trim() .to_string(), config .value_require(("acme", acme_id, "consumer-key"))? .trim() .to_string(), config .value_require(("acme", acme_id, "ovh-endpoint"))? .parse() .map_err(|_| { config .new_parse_error(("acme", acme_id, "ovh-endpoint"), "Invalid OVH endpoint") }) .ok()?, timeout.into(), ) .map_err(|err| { config.new_build_error( ("acme", acme_id, "provider"), format!("Failed to create OVH DNS updater: {err}"), ) }) .ok(), _ => { config.new_parse_error(("acme", acme_id, "provider"), "Unsupported provider"); None } } } pub(crate) fn parse_certificates( config: &mut Config, certificates: &mut AHashMap>, subject_names: &mut AHashSet, ) { // Parse certificates for cert_id in config.sub_keys("certificate", ".cert") { let cert_id = cert_id.as_str(); let key_cert = ("certificate", cert_id, "cert"); let key_pk = ("certificate", cert_id, "private-key"); let cert = config .value_require(key_cert) .map(|s| s.as_bytes().to_vec()); let pk = config.value_require(key_pk).map(|s| s.as_bytes().to_vec()); if let (Some(cert), Some(pk)) = (cert, pk) { match build_certified_key(cert, pk) { Ok(cert) => { match cert .end_entity_cert() .map_err(|err| format!("Failed to obtain end entity cert: {err}")) .and_then(|cert| { X509Certificate::from_der(cert.as_ref()) .map_err(|err| format!("Failed to parse end entity cert: {err}")) }) { Ok((_, parsed)) => { // Add CNs and SANs to the list of names let mut names = AHashSet::new(); for name in parsed.subject().iter_common_name() { if let Ok(name) = name.as_str() { names.insert(name.to_string()); } } for ext in parsed.extensions() { if let ParsedExtension::SubjectAlternativeName(san) = ext.parsed_extension() { for name in &san.general_names { let name = match name { GeneralName::DNSName(name) => name.to_string(), GeneralName::IPAddress(ip) => match ip.len() { 4 => Ipv4Addr::from( <[u8; 4]>::try_from(*ip).unwrap(), ) .to_string(), 16 => Ipv6Addr::from( <[u8; 16]>::try_from(*ip).unwrap(), ) .to_string(), _ => continue, }, _ => { continue; } }; names.insert(name); } } } // Add custom SNIs names.extend( config .values(("certificate", cert_id, "subjects")) .map(|(_, v)| v.trim().to_string()), ); // Add domain names subject_names.extend(names.iter().cloned()); // Add certificates let cert = Arc::new(cert); for name in names { certificates.insert( name.strip_prefix("*.") .map(|name| name.to_string()) .unwrap_or(name), cert.clone(), ); } // Add default certificate if config .property::(("certificate", cert_id, "default")) .unwrap_or_default() { certificates.insert("*".to_string(), cert.clone()); } } Err(err) => config.new_build_error(format!("certificate.{cert_id}"), err), } } Err(err) => config.new_build_error(format!("certificate.{cert_id}"), err), } } } } pub(crate) fn build_certified_key(cert: Vec, pk: Vec) -> Result { let cert = certs(&mut Cursor::new(cert)) .collect::, _>>() .map_err(|err| format!("Failed to read certificates: {err}"))?; if cert.is_empty() { return Err("No certificates found.".to_string()); } let pk = match read_one(&mut Cursor::new(pk)) .map_err(|err| format!("Failed to read private keys.: {err}",))? .into_iter() .next() { Some(Item::Pkcs8Key(key)) => PrivateKeyDer::Pkcs8(key), Some(Item::Pkcs1Key(key)) => PrivateKeyDer::Pkcs1(key), Some(Item::Sec1Key(key)) => PrivateKeyDer::Sec1(key), Some(_) => return Err("Unsupported private keys found.".to_string()), None => return Err("No private keys found.".to_string()), }; Ok(CertifiedKey { cert, key: any_supported_type(&pk) .map_err(|err| format!("Failed to sign certificate: {err}",))?, ocsp: None, }) } pub(crate) fn build_self_signed_cert( domains: impl Into>, ) -> Result { let cert = generate_simple_self_signed(domains) .map_err(|err| format!("Failed to generate self-signed certificate: {err}",))?; build_certified_key( cert.serialize_pem().unwrap().into_bytes(), cert.serialize_private_key_pem().into_bytes(), ) } ================================================ FILE: crates/common/src/config/smtp/auth.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{sync::Arc, time::Duration}; use ahash::AHashMap; use mail_auth::{ common::crypto::{Algorithm, Ed25519Key, HashAlgorithm, RsaKey, Sha256, SigningKey}, dkim::{Canonicalization, Done}, }; use mail_parser::decoders::base64::base64_decode; use utils::config::{ Config, utils::{AsKey, ParseValue}, }; use crate::{ config::CONNECTION_VARS, expr::{self, Constant, ConstantValue, if_block::IfBlock, tokenizer::TokenMap}, }; use super::*; #[derive(Clone)] pub struct MailAuthConfig { pub dkim: DkimAuthConfig, pub arc: ArcAuthConfig, pub spf: SpfAuthConfig, pub dmarc: DmarcAuthConfig, pub iprev: IpRevAuthConfig, pub signatures: AHashMap>>, } #[allow(clippy::large_enum_variant)] pub enum LazySignature { Resolved(ResolvedSignature), Pending(Config), Failed, } #[derive(Clone)] pub struct ResolvedSignature { pub signer: Arc, pub sealer: Arc, } #[derive(Clone)] pub struct DkimAuthConfig { pub verify: IfBlock, pub sign: IfBlock, pub strict: bool, } #[derive(Clone)] pub struct ArcAuthConfig { pub verify: IfBlock, pub seal: IfBlock, } #[derive(Clone)] pub struct SpfAuthConfig { pub verify_ehlo: IfBlock, pub verify_mail_from: IfBlock, } #[derive(Clone)] pub struct DmarcAuthConfig { pub verify: IfBlock, } #[derive(Clone)] pub struct IpRevAuthConfig { pub verify: IfBlock, } #[derive(Debug, Clone, Copy, Default)] pub enum VerifyStrategy { #[default] Relaxed, Strict, Disable, } #[derive(Debug, Clone)] pub struct DkimCanonicalization { pub headers: Canonicalization, pub body: Canonicalization, } pub enum DkimSigner { RsaSha256(mail_auth::dkim::DkimSigner, Done>), Ed25519Sha256(mail_auth::dkim::DkimSigner), } pub enum ArcSealer { RsaSha256(mail_auth::arc::ArcSealer, Done>), Ed25519Sha256(mail_auth::arc::ArcSealer), } impl Default for MailAuthConfig { fn default() -> Self { Self { dkim: DkimAuthConfig { verify: IfBlock::new::("auth.dkim.verify", [], "relaxed"), sign: IfBlock::new::<()>( "auth.dkim.sign", [( "is_local_domain('*', sender_domain)", "['rsa-' + sender_domain, 'ed25519-' + sender_domain]", )], "false", ), strict: true, }, arc: ArcAuthConfig { verify: IfBlock::new::("auth.arc.verify", [], "relaxed"), seal: IfBlock::new::<()>( "auth.arc.seal", [], "'rsa-' + config_get('report.domain')", ), }, spf: SpfAuthConfig { verify_ehlo: IfBlock::new::( "auth.spf.verify.ehlo", [("local_port == 25", "relaxed")], #[cfg(not(feature = "test_mode"))] "disable", #[cfg(feature = "test_mode")] "relaxed", ), verify_mail_from: IfBlock::new::( "auth.spf.verify.mail-from", [("local_port == 25", "relaxed")], #[cfg(not(feature = "test_mode"))] "disable", #[cfg(feature = "test_mode")] "relaxed", ), }, dmarc: DmarcAuthConfig { verify: IfBlock::new::( "auth.dmarc.verify", [("local_port == 25", "relaxed")], #[cfg(not(feature = "test_mode"))] "disable", #[cfg(feature = "test_mode")] "relaxed", ), }, iprev: IpRevAuthConfig { verify: IfBlock::new::( "auth.iprev.verify", [("local_port == 25", "relaxed")], #[cfg(not(feature = "test_mode"))] "disable", #[cfg(feature = "test_mode")] "relaxed", ), }, signatures: Default::default(), } } } impl MailAuthConfig { pub fn parse(config: &mut Config) -> Self { let rcpt_vars = TokenMap::default() .with_variables(SMTP_RCPT_TO_VARS) .with_constants::(); let conn_vars = TokenMap::default() .with_variables(CONNECTION_VARS) .with_constants::(); let mut mail_auth = Self::default(); for (value, key, token_map) in [ (&mut mail_auth.dkim.verify, "auth.dkim.verify", &rcpt_vars), (&mut mail_auth.dkim.sign, "auth.dkim.sign", &rcpt_vars), (&mut mail_auth.arc.verify, "auth.arc.verify", &rcpt_vars), (&mut mail_auth.arc.seal, "auth.arc.seal", &rcpt_vars), ( &mut mail_auth.spf.verify_ehlo, "auth.spf.verify.ehlo", &conn_vars, ), ( &mut mail_auth.spf.verify_mail_from, "auth.spf.verify.mail-from", &conn_vars, ), (&mut mail_auth.dmarc.verify, "auth.dmarc.verify", &rcpt_vars), (&mut mail_auth.iprev.verify, "auth.iprev.verify", &conn_vars), ] { if let Some(if_block) = IfBlock::try_parse(config, key, token_map) { *value = if_block; } } mail_auth.dkim.strict = config .property_or_default("auth.dkim.strict", "true") .unwrap_or(true); // Parse signatures let mut signatures: AHashMap<&str, Config> = AHashMap::new(); let mut current_id = None; for (k, v) in config.keys.iter() { if let Some(prefix) = k.strip_prefix("signature.") { if let Some(id) = prefix.strip_suffix(".algorithm") { current_id = Some(id); } #[allow(clippy::unwrap_or_default)] if let Some(current_id) = current_id { signatures .entry(current_id) .or_insert_with(Config::default) .keys .insert(k.to_string(), v.to_string()); } } else if !signatures.is_empty() { break; } } mail_auth.signatures = signatures .into_iter() .map(|(id, config)| { ( id.to_string(), Arc::new(ArcSwap::from_pointee(LazySignature::Pending(config))), ) }) .collect(); mail_auth } } pub fn build_signature(config: &mut Config, id: &str) -> Option<(DkimSigner, ArcSealer)> { match config.property_require::(("signature", id, "algorithm"))? { Algorithm::RsaSha256 => { let pk = config .value_require(("signature", id, "private-key"))? .trim() .to_string(); let key = RsaKey::::from_rsa_pem(&pk) .or_else(|_| RsaKey::::from_pkcs8_pem(&pk)) .map_err(|err| { config.new_build_error( ("signature", id, "private-key"), format!("Failed to build RSA key: {err}",), ) }) .ok()?; let key_clone = RsaKey::::from_rsa_pem(&pk) .or_else(|_| RsaKey::::from_pkcs8_pem(&pk)) .map_err(|err| { config.new_build_error( ("signature", id, "private-key"), format!("Failed to build RSA key: {err}",), ) }) .ok()?; let (signer, sealer) = parse_signature(config, id, key_clone, key)?; (DkimSigner::RsaSha256(signer), ArcSealer::RsaSha256(sealer)).into() } Algorithm::Ed25519Sha256 => { let private_key = parse_pem(config, ("signature", id, "private-key"))?; let key = Ed25519Key::from_pkcs8_maybe_unchecked_der(&private_key) .map_err(|err| { config.new_build_error( ("signature", id), format!("Failed to build ED25519 key for signature {id:?}: {err}"), ) }) .ok()?; let key_clone = Ed25519Key::from_pkcs8_maybe_unchecked_der(&private_key) .map_err(|err| { config.new_build_error( ("signature", id), format!("Failed to build ED25519 key for signature {id:?}: {err}"), ) }) .ok()?; let (signer, sealer) = parse_signature(config, id, key_clone, key)?; ( DkimSigner::Ed25519Sha256(signer), ArcSealer::Ed25519Sha256(sealer), ) .into() } Algorithm::RsaSha1 => { config.new_build_error( ("signature", id), format!("Could not build signature {id:?}: SHA1 signatures are deprecated.",), ); None } } } fn parse_pem(config: &mut Config, key: impl AsKey) -> Option> { if let Some(der) = simple_pem_parse(config.value_require(key.clone())?) { Some(der) } else { config.new_build_error(key, "Failed to base64 decode key."); None } } pub fn simple_pem_parse(contents: &str) -> Option> { let mut contents = contents.as_bytes().iter().copied(); let mut base64 = vec![]; 'outer: while let Some(ch) = contents.next() { if !ch.is_ascii_whitespace() { if ch == b'-' { for ch in contents.by_ref() { if ch == b'\n' { break; } } } else { base64.push(ch); } for ch in contents.by_ref() { if ch == b'-' { break 'outer; } else if !ch.is_ascii_whitespace() { base64.push(ch); } } } } base64_decode(&base64) } fn parse_signature>( config: &mut Config, id: &str, key_dkim: T, key_arc: U, ) -> Option<( mail_auth::dkim::DkimSigner, mail_auth::arc::ArcSealer, )> { let domain = config .value_require(("signature", id, "domain"))? .to_string(); let selector = config .value_require(("signature", id, "selector"))? .to_string(); let mut headers = config .values(("signature", id, "headers")) .filter_map(|(_, v)| { if !v.is_empty() { v.to_string().into() } else { None } }) .collect::>(); if headers.is_empty() { headers = vec![ "From".to_string(), "To".to_string(), "Date".to_string(), "Subject".to_string(), "Message-ID".to_string(), ]; } let mut signer = mail_auth::dkim::DkimSigner::from_key(key_dkim) .domain(&domain) .selector(&selector) .headers(headers.clone()); if !headers .iter() .any(|h| h.eq_ignore_ascii_case("DKIM-Signature")) { headers.push("DKIM-Signature".to_string()); } let mut sealer = mail_auth::arc::ArcSealer::from_key(key_arc) .domain(domain) .selector(selector) .headers(headers); if let Some(c) = config.property::(("signature", id, "canonicalization")) { signer = signer .body_canonicalization(c.body) .header_canonicalization(c.headers); sealer = sealer .body_canonicalization(c.body) .header_canonicalization(c.headers); } if let Some(c) = config.property::(("signature", id, "expire")) { signer = signer.expiration(c.as_secs()); sealer = sealer.expiration(c.as_secs()); } if let Some(true) = config.property::(("signature", id, "report")) { signer = signer.reporting(true); } if let Some(auid) = config.property::(("signature", id, "auid")) { signer = signer.agent_user_identifier(auid); } if let Some(atps) = config.property::(("signature", id, "third-party")) { signer = signer.atps(atps); } if let Some(atpsh) = config.property::(("signature", id, "third-party-algo")) { signer = signer.atpsh(atpsh); } Some((signer, sealer)) } impl<'x> TryFrom> for VerifyStrategy { type Error = (); fn try_from(value: expr::Variable<'x>) -> Result { match value { expr::Variable::Integer(c) => match c { 2 => Ok(VerifyStrategy::Relaxed), 3 => Ok(VerifyStrategy::Strict), 4 => Ok(VerifyStrategy::Disable), _ => Err(()), }, _ => Err(()), } } } impl From for Constant { fn from(value: VerifyStrategy) -> Self { Constant::Integer(match value { VerifyStrategy::Relaxed => 2, VerifyStrategy::Strict => 3, VerifyStrategy::Disable => 4, }) } } impl VerifyStrategy { #[inline(always)] pub fn verify(&self) -> bool { matches!(self, VerifyStrategy::Strict | VerifyStrategy::Relaxed) } #[inline(always)] pub fn is_strict(&self) -> bool { matches!(self, VerifyStrategy::Strict) } } impl ParseValue for VerifyStrategy { fn parse_value(value: &str) -> Result { match value { "relaxed" => Ok(VerifyStrategy::Relaxed), "strict" => Ok(VerifyStrategy::Strict), "disable" | "disabled" | "never" | "none" => Ok(VerifyStrategy::Disable), _ => Err(format!("Invalid value {:?}.", value)), } } } impl ConstantValue for VerifyStrategy { fn add_constants(token_map: &mut TokenMap) { token_map .add_constant("relaxed", VerifyStrategy::Relaxed) .add_constant("strict", VerifyStrategy::Strict) .add_constant("disable", VerifyStrategy::Disable) .add_constant("disabled", VerifyStrategy::Disable) .add_constant("never", VerifyStrategy::Disable) .add_constant("none", VerifyStrategy::Disable); } } impl ParseValue for DkimCanonicalization { fn parse_value(value: &str) -> Result { if let Some((headers, body)) = value.split_once('/') { Ok(DkimCanonicalization { headers: Canonicalization::parse_value(headers.trim())?, body: Canonicalization::parse_value(body.trim())?, }) } else { let c = Canonicalization::parse_value(value)?; Ok(DkimCanonicalization { headers: c, body: c, }) } } } impl Default for DkimCanonicalization { fn default() -> Self { Self { headers: Canonicalization::Relaxed, body: Canonicalization::Relaxed, } } } ================================================ FILE: crates/common/src/config/smtp/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use utils::config::{Config, Rate}; pub mod auth; pub mod queue; pub mod report; pub mod resolver; pub mod session; pub mod throttle; use crate::expr::{Expression, tokenizer::TokenMap}; use self::{ auth::MailAuthConfig, queue::QueueConfig, report::ReportConfig, resolver::Resolvers, session::SessionConfig, }; use super::*; #[derive(Default, Clone)] pub struct SmtpConfig { pub session: SessionConfig, pub queue: QueueConfig, pub resolvers: Resolvers, pub mail_auth: MailAuthConfig, pub report: ReportConfig, } #[derive(Debug, Default, Clone)] #[cfg_attr(feature = "test_mode", derive(PartialEq, Eq))] pub struct QueueRateLimiter { pub id: String, pub expr: Expression, pub keys: u16, pub rate: Rate, } pub const THROTTLE_RCPT: u16 = 1 << 0; pub const THROTTLE_RCPT_DOMAIN: u16 = 1 << 1; pub const THROTTLE_SENDER: u16 = 1 << 2; pub const THROTTLE_SENDER_DOMAIN: u16 = 1 << 3; pub const THROTTLE_AUTH_AS: u16 = 1 << 4; pub const THROTTLE_LISTENER: u16 = 1 << 5; pub const THROTTLE_MX: u16 = 1 << 6; pub const THROTTLE_REMOTE_IP: u16 = 1 << 7; pub const THROTTLE_LOCAL_IP: u16 = 1 << 8; pub const THROTTLE_HELO_DOMAIN: u16 = 1 << 9; pub(crate) const RCPT_DOMAIN_VARS: &[u32; 1] = &[V_RECIPIENT_DOMAIN]; pub(crate) const SMTP_EHLO_VARS: &[u32; 10] = &[ V_LISTENER, V_REMOTE_IP, V_REMOTE_PORT, V_LOCAL_IP, V_LOCAL_PORT, V_PROTOCOL, V_TLS, V_HELO_DOMAIN, V_ASN, V_COUNTRY, ]; pub(crate) const SMTP_MAIL_FROM_VARS: &[u32; 12] = &[ V_LISTENER, V_REMOTE_IP, V_REMOTE_PORT, V_LOCAL_IP, V_LOCAL_PORT, V_PROTOCOL, V_TLS, V_SENDER, V_SENDER_DOMAIN, V_AUTHENTICATED_AS, V_ASN, V_COUNTRY, ]; pub(crate) const SMTP_RCPT_TO_VARS: &[u32; 17] = &[ V_SENDER, V_SENDER_DOMAIN, V_RECIPIENTS, V_RECIPIENT, V_RECIPIENT_DOMAIN, V_AUTHENTICATED_AS, V_LISTENER, V_REMOTE_IP, V_REMOTE_PORT, V_LOCAL_IP, V_LOCAL_PORT, V_PROTOCOL, V_TLS, V_PRIORITY, V_HELO_DOMAIN, V_ASN, V_COUNTRY, ]; pub(crate) const SMTP_QUEUE_HOST_VARS: &[u32; 20] = &[ V_SENDER, V_SENDER_DOMAIN, V_RECIPIENT_DOMAIN, V_RECIPIENT, V_RECIPIENTS, V_MX, V_PRIORITY, V_REMOTE_IP, V_LOCAL_IP, V_QUEUE_RETRY_NUM, V_QUEUE_NOTIFY_NUM, V_QUEUE_EXPIRES_IN, V_QUEUE_LAST_STATUS, V_QUEUE_LAST_ERROR, V_QUEUE_NAME, V_QUEUE_AGE, V_RECEIVED_FROM_IP, V_RECEIVED_VIA_PORT, V_SOURCE, V_SIZE, ]; pub(crate) const SMTP_QUEUE_RCPT_VARS: &[u32; 17] = &[ V_RECIPIENT, V_RECIPIENT_DOMAIN, V_RECIPIENTS, V_SENDER, V_SENDER_DOMAIN, V_PRIORITY, V_QUEUE_RETRY_NUM, V_QUEUE_NOTIFY_NUM, V_QUEUE_EXPIRES_IN, V_QUEUE_LAST_STATUS, V_QUEUE_LAST_ERROR, V_QUEUE_NAME, V_QUEUE_AGE, V_RECEIVED_FROM_IP, V_RECEIVED_VIA_PORT, V_SOURCE, V_SIZE, ]; pub(crate) const SMTP_QUEUE_SENDER_VARS: &[u32; 8] = &[ V_SENDER, V_SENDER_DOMAIN, V_PRIORITY, V_QUEUE_RETRY_NUM, V_QUEUE_NOTIFY_NUM, V_QUEUE_EXPIRES_IN, V_QUEUE_LAST_STATUS, V_QUEUE_LAST_ERROR, ]; impl SmtpConfig { pub async fn parse(config: &mut Config) -> Self { Self { session: SessionConfig::parse(config), queue: QueueConfig::parse(config), resolvers: Resolvers::parse(config).await, mail_auth: MailAuthConfig::parse(config), report: ReportConfig::parse(config), } } } ================================================ FILE: crates/common/src/config/smtp/queue.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use self::throttle::parse_queue_rate_limiter; use super::*; use crate::{ config::server::ServerProtocol, expr::{if_block::IfBlock, *}, }; use ahash::AHashMap; use mail_auth::IpLookupStrategy; use mail_send::Credentials; use std::{ fmt::Display, hash::{Hash, Hasher}, net::IpAddr, time::Duration, }; use throttle::parse_queue_rate_limiter_key; use utils::config::{Config, utils::ParseValue}; #[derive( Debug, Clone, Copy, PartialEq, Eq, Hash, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, serde::Deserialize, )] #[rkyv(derive(Debug, Clone, Copy, PartialEq), compare(PartialEq))] #[repr(transparent)] pub struct QueueName([u8; 8]); pub const DEFAULT_QUEUE_NAME: QueueName = QueueName([b'd', b'e', b'f', b'a', b'u', b'l', b't', 0]); #[derive(Clone)] pub struct QueueConfig { // Strategy resolver pub route: IfBlock, pub queue: IfBlock, pub connection: IfBlock, pub tls: IfBlock, // DSN pub dsn: Dsn, // Rate limits pub inbound_limiters: QueueRateLimiters, pub outbound_limiters: QueueRateLimiters, pub quota: QueueQuotas, // Strategies pub queue_strategy: AHashMap, pub connection_strategy: AHashMap, pub routing_strategy: AHashMap, pub tls_strategy: AHashMap, pub virtual_queues: AHashMap, } #[derive(Clone, Hash, PartialEq, Eq, Debug)] pub enum RoutingStrategy { Local, Mx(MxConfig), Relay(RelayConfig), } #[derive(Clone, Debug)] pub struct MxConfig { pub max_mx: usize, pub max_multi_homed: usize, pub ip_lookup_strategy: IpLookupStrategy, } #[derive(Clone)] pub struct Dsn { pub name: IfBlock, pub address: IfBlock, pub sign: IfBlock, } #[derive(Clone, Debug)] pub struct VirtualQueue { pub threads: usize, } #[derive(Clone, Debug)] pub struct QueueStrategy { pub retry: Vec, pub notify: Vec, pub expiry: QueueExpiry, pub virtual_queue: QueueName, } #[derive( rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Clone, Copy, PartialEq, Eq, serde::Deserialize, )] pub enum QueueExpiry { Ttl(u64), Attempts(u32), } #[derive(Clone, Debug)] pub struct TlsStrategy { pub dane: RequireOptional, pub mta_sts: RequireOptional, pub tls: RequireOptional, pub allow_invalid_certs: bool, pub timeout_tls: Duration, pub timeout_mta_sts: Duration, } #[derive(Clone, Debug)] pub struct ConnectionStrategy { pub source_ipv4: Vec, pub source_ipv6: Vec, pub ehlo_hostname: Option, pub timeout_connect: Duration, pub timeout_greeting: Duration, pub timeout_ehlo: Duration, pub timeout_mail: Duration, pub timeout_rcpt: Duration, pub timeout_data: Duration, } #[derive(Clone, Debug)] pub struct IpAndHost { pub ip: IpAddr, pub host: Option, } #[derive(Debug, Clone, Default)] pub struct QueueRateLimiters { pub sender: Vec, pub rcpt: Vec, pub remote: Vec, } #[derive(Clone, Default)] pub struct QueueQuotas { pub sender: Vec, pub rcpt: Vec, pub rcpt_domain: Vec, } #[derive(Clone)] pub struct QueueQuota { pub id: String, pub expr: Expression, pub keys: u16, pub size: Option, pub messages: Option, } #[derive(Clone, Hash, PartialEq, Eq)] pub struct RelayConfig { pub address: String, pub port: u16, pub protocol: ServerProtocol, pub auth: Option>, pub tls_implicit: bool, pub tls_allow_invalid_certs: bool, } #[derive(Debug, Clone, Copy, Default)] pub enum RequireOptional { #[default] Optional, Require, Disable, } impl Default for QueueConfig { fn default() -> Self { Self { route: IfBlock::new::<()>( "queue.strategy.route", #[cfg(not(feature = "test_mode"))] [("is_local_domain('*', rcpt_domain)", "'local'")], #[cfg(feature = "test_mode")] [], "'mx'", ), queue: IfBlock::new::<()>( "queue.strategy.schedule", #[cfg(not(feature = "test_mode"))] [ ("is_local_domain('*', rcpt_domain)", "'local'"), ("source == 'dsn'", "'dsn'"), ("source == 'report'", "'report'"), ], #[cfg(feature = "test_mode")] [], #[cfg(not(feature = "test_mode"))] "'remote'", #[cfg(feature = "test_mode")] "'default'", ), connection: IfBlock::new::<()>("queue.strategy.connection", [], "'default'"), tls: IfBlock::new::<()>( "queue.strategy.tls", #[cfg(not(feature = "test_mode"))] [("retry_num > 0 && last_error == 'tls'", "'invalid-tls'")], #[cfg(feature = "test_mode")] [], "'default'", ), dsn: Dsn { name: IfBlock::new::<()>("report.dsn.from-name", [], "'Mail Delivery Subsystem'"), address: IfBlock::new::<()>( "report.dsn.from-address", [], "'MAILER-DAEMON@' + config_get('report.domain')", ), sign: IfBlock::new::<()>( "report.dsn.sign", [], "['rsa-' + config_get('report.domain'), 'ed25519-' + config_get('report.domain')]", ), }, inbound_limiters: QueueRateLimiters::default(), outbound_limiters: QueueRateLimiters::default(), quota: QueueQuotas::default(), queue_strategy: Default::default(), virtual_queues: Default::default(), connection_strategy: Default::default(), routing_strategy: Default::default(), tls_strategy: Default::default(), } } } impl QueueConfig { pub fn parse(config: &mut Config) -> Self { let mut queue = QueueConfig::default(); let rcpt_vars = TokenMap::default().with_variables(SMTP_QUEUE_RCPT_VARS); let sender_vars = TokenMap::default().with_variables(SMTP_QUEUE_SENDER_VARS); let host_vars = TokenMap::default().with_variables(SMTP_QUEUE_HOST_VARS); for (value, key, token_map) in [ (&mut queue.route, "queue.strategy.route", &rcpt_vars), (&mut queue.queue, "queue.strategy.schedule", &rcpt_vars), ( &mut queue.connection, "queue.strategy.connection", &host_vars, ), (&mut queue.tls, "queue.strategy.tls", &host_vars), (&mut queue.dsn.name, "report.dsn.from-name", &sender_vars), ( &mut queue.dsn.address, "report.dsn.from-address", &sender_vars, ), (&mut queue.dsn.sign, "report.dsn.sign", &sender_vars), ] { if let Some(if_block) = IfBlock::try_parse(config, key, token_map) { *value = if_block; } } // Parse strategies queue.virtual_queues = parse_virtual_queues(config); queue.queue_strategy = parse_queue_strategies(config, &queue.virtual_queues); queue.connection_strategy = parse_connection_strategies(config); queue.routing_strategy = parse_routing_strategies(config); queue.tls_strategy = parse_tls_strategies(config); // Parse rate limiters queue.inbound_limiters = parse_inbound_rate_limiters(config); queue.outbound_limiters = parse_outbound_rate_limiters(config); queue.quota = parse_queue_quota(config); queue } } fn parse_queue_strategies( config: &mut Config, queues: &AHashMap, ) -> AHashMap { let mut entries = AHashMap::new(); for key in config.sub_keys_with_suffixes( "queue.schedule", &[ ".queue-name", ".retry", ".notify", ".expire", ".max-attempts", ], ) { if let Some(strategy) = parse_queue_strategy(config, &key, queues) { entries.insert(key, strategy); } } entries } fn parse_queue_strategy( config: &mut Config, id: &str, queues: &AHashMap, ) -> Option { let virtual_queue = config .property_require::(("queue.schedule", id, "queue-name")) .unwrap_or_default(); if virtual_queue != DEFAULT_QUEUE_NAME && !queues.contains_key(&virtual_queue) { config.new_parse_error( ("queue.schedule", id, "queue-name"), format!("Virtual queue '{virtual_queue}' does not exist."), ); return None; } let mut retry: Vec = config .properties::(("queue.schedule", id, "retry")) .into_iter() .map(|(_, d)| d.as_secs()) .collect(); let mut notify: Vec = config .properties::(("queue.schedule", id, "notify")) .into_iter() .map(|(_, d)| d.as_secs()) .collect(); if retry.is_empty() { config.new_parse_error( ("queue.schedule", id, "retry"), "At least one 'retry' duration must be specified.".to_string(), ); retry.push(60 * 60); // Default to 1 minute } if notify.is_empty() { notify.push(10000 * 86400); // Disable notifications by default } Some(QueueStrategy { retry, notify, expiry: match ( config.property::(("queue.schedule", id, "expire")), config.property::(("queue.schedule", id, "max-attempts")), ) { (Some(duration), None) => QueueExpiry::Ttl(duration.as_secs()), (None, Some(count)) => QueueExpiry::Attempts(count), (Some(_), Some(_)) => { config.new_parse_error( ("queue.schedule", id, "expire"), "Cannot specify both 'expire' and 'max-attempts'.".to_string(), ); return None; } (None, None) => QueueExpiry::Ttl(60 * 60 * 24 * 3), // Default to 3 days }, virtual_queue, }) } fn parse_virtual_queues(config: &mut Config) -> AHashMap { let mut entries = AHashMap::new(); for key in config.sub_keys("queue.virtual", ".threads-per-node") { if let Some(queue_name) = QueueName::new(&key) { if let Some(queue) = parse_virtual_queue(config, &key) { entries.insert(queue_name, queue); } } else { config.new_parse_error( ("queue.virtual", &key, "threads-per-node"), format!("Invalid virtual queue name: {key:?}. Must be 1-8 bytes long."), ); } } entries } fn parse_virtual_queue(config: &mut Config, id: &str) -> Option { Some(VirtualQueue { threads: config .property_require::(("queue.virtual", id, "threads-per-node")) .unwrap_or(1), }) } fn parse_routing_strategies(config: &mut Config) -> AHashMap { let mut entries = AHashMap::new(); for key in config.sub_keys("queue.route", ".type") { if let Some(strategy) = parse_route(config, &key) { entries.insert(key, strategy); } } entries } fn parse_route(config: &mut Config, id: &str) -> Option { match config.value_require_non_empty(("queue.route", id, "type"))? { "relay" => RoutingStrategy::Relay(RelayConfig { address: config.property_require(("queue.route", id, "address"))?, port: config .property_require(("queue.route", id, "port")) .unwrap_or(25), protocol: config .property_require(("queue.route", id, "protocol")) .unwrap_or(ServerProtocol::Smtp), auth: if let (Some(username), Some(secret)) = ( config.value(("queue.route", id, "auth.username")), config.value(("queue.route", id, "auth.secret")), ) { Credentials::new(username.to_string(), secret.to_string()).into() } else { None }, tls_implicit: config .property(("queue.route", id, "tls.implicit")) .unwrap_or(true), tls_allow_invalid_certs: config .property(("queue.route", id, "tls.allow-invalid-certs")) .unwrap_or(false), }) .into(), "local" => RoutingStrategy::Local.into(), "mx" => RoutingStrategy::Mx(MxConfig { max_mx: config .property(("queue.route", id, "limits.mx")) .unwrap_or(5), max_multi_homed: config .property(("queue.route", id, "limits.multihomed")) .unwrap_or(2), ip_lookup_strategy: config .property(("queue.route", id, "ip-lookup")) .unwrap_or(IpLookupStrategy::Ipv4thenIpv6), }) .into(), invalid => { let details = format!("Invalid route type: {invalid:?}. Expected 'relay', 'local', or 'mx'."); config.new_parse_error(("queue.route", id, "type"), details); None } } } fn parse_tls_strategies(config: &mut Config) -> AHashMap { let mut entries = AHashMap::new(); for key in config.sub_keys_with_suffixes( "queue.tls", &[ ".allow-invalid-certs", ".dane", ".starttls", ".timeout.tls", ".timeout.mta-sts", ], ) { if let Some(strategy) = parse_tls(config, &key) { entries.insert(key, strategy); } } entries } fn parse_tls(config: &mut Config, id: &str) -> Option { Some(TlsStrategy { dane: config .property::(("queue.tls", id, "dane")) .unwrap_or(RequireOptional::Optional), mta_sts: config .property::(("queue.tls", id, "mta-sts")) .unwrap_or(RequireOptional::Optional), tls: config .property::(("queue.tls", id, "starttls")) .unwrap_or(RequireOptional::Optional), allow_invalid_certs: config .property::(("queue.tls", id, "allow-invalid-certs")) .unwrap_or(false), timeout_tls: config .property::(("queue.tls", id, "timeout.tls")) .unwrap_or(Duration::from_secs(3 * 60)), timeout_mta_sts: config .property::(("queue.tls", id, "timeout.mta-sts")) .unwrap_or(Duration::from_secs(5 * 60)), }) } fn parse_connection_strategies(config: &mut Config) -> AHashMap { let mut entries = AHashMap::new(); for key in config.sub_keys_with_suffixes( "queue.connection", &[ ".timeout.connect", ".timeout.greeting", ".timeout.ehlo", ".timeout.mail-from", ".timeout.rcpt-to", ".timeout.data", ".ehlo-hostname", ], ) { if let Some(strategy) = parse_connection(config, &key) { entries.insert(key, strategy); } } entries } fn parse_connection(config: &mut Config, id: &str) -> Option { let mut source_ipv4 = Vec::new(); let mut source_ipv6 = Vec::new(); for (_, ip) in config.properties::(("queue.connection", id, "source-ips")) { let ip_and_host = IpAndHost { ip, host: config.property::(("queue.source-ip", ip.to_string(), "ehlo-hostname")), }; if ip.is_ipv4() { source_ipv4.push(ip_and_host); } else { source_ipv6.push(ip_and_host); } } Some(ConnectionStrategy { source_ipv4, source_ipv6, ehlo_hostname: config.property::(("queue.connection", id, "ehlo-hostname")), timeout_connect: config .property::(("queue.connection", id, "timeout.connect")) .unwrap_or(Duration::from_secs(5 * 60)), timeout_greeting: config .property::(("queue.connection", id, "timeout.greeting")) .unwrap_or(Duration::from_secs(5 * 60)), timeout_ehlo: config .property::(("queue.connection", id, "timeout.ehlo")) .unwrap_or(Duration::from_secs(5 * 60)), timeout_mail: config .property::(("queue.connection", id, "timeout.mail-from")) .unwrap_or(Duration::from_secs(5 * 60)), timeout_rcpt: config .property::(("queue.connection", id, "timeout.rcpt-to")) .unwrap_or(Duration::from_secs(5 * 60)), timeout_data: config .property::(("queue.connection", id, "timeout.data")) .unwrap_or(Duration::from_secs(10 * 60)), }) } fn parse_inbound_rate_limiters(config: &mut Config) -> QueueRateLimiters { let mut throttle = QueueRateLimiters::default(); let all_throttles = parse_queue_rate_limiter( config, "queue.limiter.inbound", &TokenMap::default().with_variables(SMTP_RCPT_TO_VARS), THROTTLE_LISTENER | THROTTLE_REMOTE_IP | THROTTLE_LOCAL_IP | THROTTLE_AUTH_AS | THROTTLE_HELO_DOMAIN | THROTTLE_RCPT | THROTTLE_RCPT_DOMAIN | THROTTLE_SENDER | THROTTLE_SENDER_DOMAIN, ); for t in all_throttles { if (t.keys & (THROTTLE_RCPT | THROTTLE_RCPT_DOMAIN)) != 0 || t.expr.items().iter().any(|c| { matches!( c, ExpressionItem::Variable(V_RECIPIENT | V_RECIPIENT_DOMAIN) ) }) { throttle.rcpt.push(t); } else if (t.keys & (THROTTLE_SENDER | THROTTLE_SENDER_DOMAIN | THROTTLE_HELO_DOMAIN | THROTTLE_AUTH_AS)) != 0 || t.expr.items().iter().any(|c| { matches!( c, ExpressionItem::Variable( V_SENDER | V_SENDER_DOMAIN | V_HELO_DOMAIN | V_AUTHENTICATED_AS ) ) }) { throttle.sender.push(t); } else { throttle.remote.push(t); } } throttle } fn parse_outbound_rate_limiters(config: &mut Config) -> QueueRateLimiters { // Parse throttle let mut throttle = QueueRateLimiters::default(); let all_throttles = parse_queue_rate_limiter( config, "queue.limiter.outbound", &TokenMap::default().with_variables(SMTP_QUEUE_HOST_VARS), THROTTLE_RCPT_DOMAIN | THROTTLE_SENDER | THROTTLE_SENDER_DOMAIN | THROTTLE_MX | THROTTLE_REMOTE_IP | THROTTLE_LOCAL_IP, ); for t in all_throttles { if (t.keys & (THROTTLE_MX | THROTTLE_REMOTE_IP | THROTTLE_LOCAL_IP)) != 0 || t.expr .items() .iter() .any(|c| matches!(c, ExpressionItem::Variable(V_MX | V_REMOTE_IP | V_LOCAL_IP))) { throttle.remote.push(t); } else if (t.keys & (THROTTLE_RCPT_DOMAIN)) != 0 || t.expr .items() .iter() .any(|c| matches!(c, ExpressionItem::Variable(V_RECIPIENT_DOMAIN))) { throttle.rcpt.push(t); } else { throttle.sender.push(t); } } throttle } fn parse_queue_quota(config: &mut Config) -> QueueQuotas { let mut capacities = QueueQuotas { sender: Vec::new(), rcpt: Vec::new(), rcpt_domain: Vec::new(), }; for quota_id in config.sub_keys("queue.quota", "") { if let Some(quota) = parse_queue_quota_item(config, ("queue.quota", "a_id), "a_id) { if (quota.keys & THROTTLE_RCPT) != 0 || quota .expr .items() .iter() .any(|c| matches!(c, ExpressionItem::Variable(V_RECIPIENT))) { capacities.rcpt.push(quota); } else if (quota.keys & THROTTLE_RCPT_DOMAIN) != 0 || quota .expr .items() .iter() .any(|c| matches!(c, ExpressionItem::Variable(V_RECIPIENT_DOMAIN))) { capacities.rcpt_domain.push(quota); } else { capacities.sender.push(quota); } } } capacities } fn parse_queue_quota_item(config: &mut Config, prefix: impl AsKey, id: &str) -> Option { let prefix = prefix.as_key(); // Skip disabled throttles if !config .property::((prefix.as_str(), "enable")) .unwrap_or(true) { return None; } let mut keys = 0; for (key_, value) in config .values((&prefix, "key")) .map(|(k, v)| (k.to_string(), v.to_string())) .collect::>() { match parse_queue_rate_limiter_key(&value) { Ok(key) => { if (key & (THROTTLE_RCPT_DOMAIN | THROTTLE_RCPT | THROTTLE_SENDER | THROTTLE_SENDER_DOMAIN)) != 0 { keys |= key; } else { let err = format!("Quota key {value:?} is not available in this context"); config.new_build_error(key_, err); } } Err(err) => { config.new_parse_error(key_, err); } } } let quota = QueueQuota { id: id.to_string(), expr: Expression::try_parse( config, (prefix.as_str(), "match"), &TokenMap::default().with_variables(SMTP_QUEUE_HOST_VARS), ) .unwrap_or_default(), keys, size: config .property::>((prefix.as_str(), "size")) .filter(|&v| v.as_ref().is_some_and(|v| *v > 0)) .unwrap_or_default(), messages: config .property::>((prefix.as_str(), "messages")) .filter(|&v| v.as_ref().is_some_and(|v| *v > 0)) .unwrap_or_default(), }; // Validate if quota.size.is_none() && quota.messages.is_none() { config.new_parse_error( prefix.as_str(), concat!( "Queue quota needs to define a ", "valid 'size' and/or 'messages' property." ) .to_string(), ); None } else { Some(quota) } } impl ParseValue for RequireOptional { fn parse_value(value: &str) -> Result { match value { "optional" => Ok(RequireOptional::Optional), "require" | "required" => Ok(RequireOptional::Require), "disable" | "disabled" | "none" | "false" => Ok(RequireOptional::Disable), _ => Err(format!("Invalid TLS option value {:?}.", value,)), } } } impl<'x> TryFrom> for RequireOptional { type Error = (); fn try_from(value: Variable<'x>) -> Result { match value { Variable::Integer(2) => Ok(RequireOptional::Optional), Variable::Integer(1) => Ok(RequireOptional::Require), Variable::Integer(0) => Ok(RequireOptional::Disable), _ => Err(()), } } } impl From for Constant { fn from(value: RequireOptional) -> Self { Constant::Integer(match value { RequireOptional::Optional => 2, RequireOptional::Require => 1, RequireOptional::Disable => 0, }) } } impl ConstantValue for RequireOptional { fn add_constants(token_map: &mut crate::expr::tokenizer::TokenMap) { token_map .add_constant("optional", RequireOptional::Optional) .add_constant("require", RequireOptional::Require) .add_constant("required", RequireOptional::Require) .add_constant("disable", RequireOptional::Disable) .add_constant("disabled", RequireOptional::Disable) .add_constant("none", RequireOptional::Disable) .add_constant("false", RequireOptional::Disable); } } impl<'x> TryFrom> for IpLookupStrategy { type Error = (); fn try_from(value: Variable<'x>) -> Result { match value { Variable::Integer(value) => match value { 2 => Ok(IpLookupStrategy::Ipv4Only), 3 => Ok(IpLookupStrategy::Ipv6Only), 4 => Ok(IpLookupStrategy::Ipv6thenIpv4), 5 => Ok(IpLookupStrategy::Ipv4thenIpv6), _ => Err(()), }, Variable::String(value) => { IpLookupStrategy::parse_value(value.as_str()).map_err(|_| ()) } _ => Err(()), } } } impl From for Constant { fn from(value: IpLookupStrategy) -> Self { Constant::Integer(match value { IpLookupStrategy::Ipv4Only => 2, IpLookupStrategy::Ipv6Only => 3, IpLookupStrategy::Ipv6thenIpv4 => 4, IpLookupStrategy::Ipv4thenIpv6 => 5, }) } } impl ConstantValue for IpLookupStrategy { fn add_constants(token_map: &mut crate::expr::tokenizer::TokenMap) { token_map .add_constant("ipv4_only", IpLookupStrategy::Ipv4Only) .add_constant("ipv6_only", IpLookupStrategy::Ipv6Only) .add_constant("ipv6_then_ipv4", IpLookupStrategy::Ipv6thenIpv4) .add_constant("ipv4_then_ipv6", IpLookupStrategy::Ipv4thenIpv6); } } impl std::fmt::Debug for RelayConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("RelayConfig") .field("address", &self.address) .field("port", &self.port) .field("protocol", &self.protocol) .field("tls_implicit", &self.tls_implicit) .field("tls_allow_invalid_certs", &self.tls_allow_invalid_certs) .finish() } } impl TlsStrategy { #[inline(always)] pub fn try_dane(&self) -> bool { matches!( self.dane, RequireOptional::Require | RequireOptional::Optional ) } #[inline(always)] pub fn try_start_tls(&self) -> bool { matches!( self.tls, RequireOptional::Require | RequireOptional::Optional ) } #[inline(always)] pub fn is_dane_required(&self) -> bool { matches!(self.dane, RequireOptional::Require) } #[inline(always)] pub fn try_mta_sts(&self) -> bool { matches!( self.mta_sts, RequireOptional::Require | RequireOptional::Optional ) } #[inline(always)] pub fn is_mta_sts_required(&self) -> bool { matches!(self.mta_sts, RequireOptional::Require) } #[inline(always)] pub fn is_tls_required(&self) -> bool { matches!(self.tls, RequireOptional::Require) || self.is_dane_required() || self.is_mta_sts_required() } } impl Hash for MxConfig { fn hash(&self, state: &mut H) { self.max_mx.hash(state); self.max_multi_homed.hash(state); } } impl PartialEq for MxConfig { fn eq(&self, other: &Self) -> bool { self.max_mx == other.max_mx && self.max_multi_homed == other.max_multi_homed } } impl Eq for MxConfig {} impl QueueName { pub fn new(name: impl AsRef<[u8]>) -> Option { let name_bytes = name.as_ref(); if (1..=8).contains(&name_bytes.len()) { let mut bytes = [0; 8]; bytes[..name_bytes.len()].copy_from_slice(name_bytes); QueueName(bytes).into() } else { None } } pub fn from_bytes(name: &[u8]) -> Option { name.try_into().ok().map(|bytes: [u8; 8]| QueueName(bytes)) } pub fn as_str(&self) -> &str { std::str::from_utf8(&self.0) .unwrap_or_default() .trim_end_matches('\0') } pub fn into_inner(self) -> [u8; 8] { self.0 } pub fn as_slice(&self) -> &[u8] { &self.0 } } impl ArchivedQueueName { pub fn as_str(&self) -> &str { std::str::from_utf8(self.0.as_ref()) .unwrap_or_default() .trim_end_matches('\0') } pub fn as_slice(&self) -> &[u8] { self.0.as_ref() } } impl Default for QueueName { fn default() -> Self { DEFAULT_QUEUE_NAME } } impl ParseValue for QueueName { fn parse_value(value: &str) -> Result { if let Some(name) = QueueName::new(value.trim().as_bytes()) { Ok(name) } else { Err(format!( "Queue name '{value}' is too long. Maximum length is 8 bytes." )) } } } impl Display for QueueName { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { self.as_str().fmt(f) } } impl Display for ArchivedQueueName { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { self.as_str().fmt(f) } } impl AsRef<[u8]> for QueueName { fn as_ref(&self) -> &[u8] { &self.0 } } ================================================ FILE: crates/common/src/config/smtp/report.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use utils::config::{Config, utils::ParseValue}; use crate::expr::{Constant, ConstantValue, Variable, if_block::IfBlock, tokenizer::TokenMap}; use super::*; #[derive(Clone)] pub struct ReportConfig { pub submitter: IfBlock, pub analysis: ReportAnalysis, pub dkim: Report, pub spf: Report, pub dmarc: Report, pub dmarc_aggregate: AggregateReport, pub tls: AggregateReport, } #[derive(Clone)] pub struct ReportAnalysis { pub addresses: Vec, pub forward: bool, pub store: Option, } #[derive(Clone)] pub enum AddressMatch { StartsWith(String), EndsWith(String), Equals(String), } #[derive(Clone)] pub struct AggregateReport { pub name: IfBlock, pub address: IfBlock, pub org_name: IfBlock, pub contact_info: IfBlock, pub send: IfBlock, pub sign: IfBlock, pub max_size: IfBlock, } #[derive(Clone)] pub struct Report { pub name: IfBlock, pub address: IfBlock, pub subject: IfBlock, pub sign: IfBlock, pub send: IfBlock, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub enum AggregateFrequency { Hourly, Daily, Weekly, #[default] Never, } impl ReportConfig { pub fn parse(config: &mut Config) -> Self { let sender_vars = TokenMap::default().with_variables(SMTP_MAIL_FROM_VARS); let rcpt_vars = TokenMap::default().with_variables(SMTP_RCPT_TO_VARS); Self { submitter: IfBlock::try_parse( config, "report.submitter", &TokenMap::default().with_variables(RCPT_DOMAIN_VARS), ) .unwrap_or_else(|| { IfBlock::new::<()>("report.submitter", [], "config_get('server.hostname')") }), analysis: ReportAnalysis { addresses: config .properties::("report.analysis.addresses") .into_iter() .map(|(_, m)| m) .collect(), forward: config.property("report.analysis.forward").unwrap_or(true), store: config .property_or_default::>("report.analysis.store", "30d") .unwrap_or_default(), }, dkim: Report::parse(config, "dkim", &rcpt_vars), spf: Report::parse(config, "spf", &sender_vars), dmarc: Report::parse(config, "dmarc", &rcpt_vars), dmarc_aggregate: AggregateReport::parse( config, "dmarc", &rcpt_vars.with_constants::(), ), tls: AggregateReport::parse( config, "tls", &TokenMap::default() .with_variables(SMTP_QUEUE_HOST_VARS) .with_constants::(), ), } } } impl Report { pub fn parse(config: &mut Config, id: &str, token_map: &TokenMap) -> Self { let mut report = Self { name: IfBlock::new::<()>(format!("report.{id}.from-name"), [], "'Report Subsystem'"), address: IfBlock::new::<()>( format!("report.{id}.from-address"), [], format!("'noreply-{id}@' + config_get('report.domain')"), ), subject: IfBlock::new::<()>( format!("report.{id}.subject"), [], format!( "'{} Authentication Failure Report'", id.to_ascii_uppercase() ), ), sign: IfBlock::new::<()>( format!("report.{id}.sign"), [], "['rsa-' + config_get('report.domain'), 'ed25519-' + config_get('report.domain')]", ), send: IfBlock::new::<()>(format!("report.{id}.send"), [], "[1, 1d]"), }; for (value, key) in [ (&mut report.name, "from-name"), (&mut report.address, "from-address"), (&mut report.subject, "subject"), (&mut report.sign, "sign"), (&mut report.send, "send"), ] { if let Some(if_block) = IfBlock::try_parse(config, ("report", id, key), token_map) { *value = if_block; } } report } } impl AggregateReport { pub fn parse(config: &mut Config, id: &str, token_map: &TokenMap) -> Self { let rcpt_vars = TokenMap::default().with_variables(RCPT_DOMAIN_VARS); let mut report = Self { name: IfBlock::new::<()>( format!("report.{id}.aggregate.from-name"), [], format!("'{} Aggregate Report'", id.to_ascii_uppercase()), ), address: IfBlock::new::<()>( format!("report.{id}.aggregate.from-address"), [], format!("'noreply-{id}@' + config_get('report.domain')"), ), org_name: IfBlock::new::<()>( format!("report.{id}.aggregate.org-name"), [], "config_get('report.domain')", ), contact_info: IfBlock::empty(format!("report.{id}.aggregate.contact-info")), send: IfBlock::new::( format!("report.{id}.aggregate.send"), [], "daily", ), sign: IfBlock::new::<()>( format!("report.{id}.aggregate.sign"), [], "['rsa-' + config_get('report.domain'), 'ed25519-' + config_get('report.domain')]", ), max_size: IfBlock::new::<()>(format!("report.{id}.aggregate.max-size"), [], "26214400"), }; for (value, key, token_map) in [ (&mut report.name, "aggregate.from-name", &rcpt_vars), (&mut report.address, "aggregate.from-address", &rcpt_vars), (&mut report.org_name, "aggregate.org-name", &rcpt_vars), ( &mut report.contact_info, "aggregate.contact-info", &rcpt_vars, ), (&mut report.send, "aggregate.send", token_map), (&mut report.sign, "aggregate.sign", &rcpt_vars), (&mut report.max_size, "aggregate.max-size", &rcpt_vars), ] { if let Some(if_block) = IfBlock::try_parse(config, ("report", id, key), token_map) { *value = if_block; } } report } } impl Default for ReportConfig { fn default() -> Self { Self::parse(&mut Config::default()) } } impl ParseValue for AggregateFrequency { fn parse_value(value: &str) -> Result { match value { "daily" | "day" => Ok(AggregateFrequency::Daily), "hourly" | "hour" => Ok(AggregateFrequency::Hourly), "weekly" | "week" => Ok(AggregateFrequency::Weekly), "never" | "disable" | "false" => Ok(AggregateFrequency::Never), _ => Err(format!("Invalid aggregate frequency value {:?}.", value,)), } } } impl From for Constant { fn from(value: AggregateFrequency) -> Self { match value { AggregateFrequency::Never => 0.into(), AggregateFrequency::Hourly => 2.into(), AggregateFrequency::Daily => 3.into(), AggregateFrequency::Weekly => 4.into(), } } } impl<'x> TryFrom> for AggregateFrequency { type Error = (); fn try_from(value: Variable<'x>) -> Result { match value { Variable::Integer(0) => Ok(AggregateFrequency::Never), Variable::Integer(2) => Ok(AggregateFrequency::Hourly), Variable::Integer(3) => Ok(AggregateFrequency::Daily), Variable::Integer(4) => Ok(AggregateFrequency::Weekly), _ => Err(()), } } } impl ConstantValue for AggregateFrequency { fn add_constants(token_map: &mut crate::expr::tokenizer::TokenMap) { token_map .add_constant("never", AggregateFrequency::Never) .add_constant("hourly", AggregateFrequency::Hourly) .add_constant("hour", AggregateFrequency::Hourly) .add_constant("daily", AggregateFrequency::Daily) .add_constant("day", AggregateFrequency::Daily) .add_constant("weekly", AggregateFrequency::Weekly) .add_constant("week", AggregateFrequency::Weekly) .add_constant("never", AggregateFrequency::Never) .add_constant("disable", AggregateFrequency::Never) .add_constant("false", AggregateFrequency::Never); } } impl ParseValue for AddressMatch { fn parse_value(value: &str) -> Result { if let Some(value) = value.strip_prefix('*').map(|v| v.trim()) { if !value.is_empty() { return Ok(AddressMatch::EndsWith(value.to_lowercase())); } } else if let Some(value) = value.strip_suffix('*').map(|v| v.trim()) { if !value.is_empty() { return Ok(AddressMatch::StartsWith(value.to_lowercase())); } } else if value.contains('@') { return Ok(AddressMatch::Equals(value.trim().to_lowercase())); } Err(format!("Invalid address match value {:?}.", value,)) } } ================================================ FILE: crates/common/src/config/smtp/resolver.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ fmt::Display, hash::{DefaultHasher, Hash, Hasher}, net::{IpAddr, Ipv4Addr, SocketAddr}, time::Duration, }; use mail_auth::{ MessageAuthenticator, hickory_resolver::{ TokioResolver, config::{NameServerConfig, ProtocolConfig, ResolverConfig, ResolverOpts}, name_server::TokioConnectionProvider, system_conf::read_system_conf, }, }; use serde::{Deserialize, Serialize}; use utils::{ cache::CacheItemWeight, config::{Config, utils::ParseValue}, }; use crate::Server; pub struct Resolvers { pub dns: MessageAuthenticator, pub dnssec: DnssecResolver, } #[derive(Clone)] pub struct DnssecResolver { pub resolver: TokioResolver, } #[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] pub struct TlsaEntry { pub is_end_entity: bool, pub is_sha256: bool, pub is_spki: bool, pub data: Vec, } #[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] pub struct Tlsa { pub entries: Vec, pub has_end_entities: bool, pub has_intermediates: bool, } #[derive(Debug, PartialEq, Eq, Hash, Default, Clone, Copy, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum Mode { Enforce, Testing, #[default] None, } #[derive(Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum MxPattern { Equals(String), StartsWith(String), } #[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize)] pub struct Policy { pub id: String, pub mode: Mode, pub mx: Vec, pub max_age: u64, } impl CacheItemWeight for Tlsa { fn weight(&self) -> u64 { self.entries .iter() .map(|entry| (entry.data.len() + std::mem::size_of::()) as u64) .sum::() + std::mem::size_of::() as u64 } } impl CacheItemWeight for Policy { fn weight(&self) -> u64 { (std::mem::size_of::() + self .mx .iter() .map(|mx| match mx { MxPattern::Equals(t) => t.len(), MxPattern::StartsWith(t) => t.len(), }) .sum::()) as u64 } } impl Resolvers { pub async fn parse(config: &mut Config) -> Self { let (resolver_config, mut opts) = match config.value("resolver.type").unwrap_or("system") { "cloudflare" => (ResolverConfig::cloudflare(), ResolverOpts::default()), "cloudflare-tls" => (ResolverConfig::cloudflare_tls(), ResolverOpts::default()), "quad9" => (ResolverConfig::quad9(), ResolverOpts::default()), "quad9-tls" => (ResolverConfig::quad9_tls(), ResolverOpts::default()), "google" => (ResolverConfig::google(), ResolverOpts::default()), "system" => read_system_conf() .map_err(|err| { config.new_build_error( "resolver.type", format!("Failed to read system DNS config: {err}"), ) }) .unwrap_or_else(|_| (ResolverConfig::cloudflare(), ResolverOpts::default())), "custom" => { let mut resolver_config = ResolverConfig::default(); for url in config .values("resolver.custom") .map(|(_, v)| v.to_string()) .collect::>() { let (proto, host) = if let Some((proto, host)) = url .split_once("://") .map(|(a, b)| (a.to_string(), b.to_string())) { ( match proto.as_str() { "udp" => ProtocolConfig::Udp, "tcp" => ProtocolConfig::Tcp, "tls" => ProtocolConfig::Tls { server_name: host.clone().into(), }, _ => { config.new_parse_error( "resolver.custom", format!("Invalid custom resolver protocol {url:?}"), ); ProtocolConfig::Udp } }, host.to_string(), ) } else { (ProtocolConfig::Udp, url) }; let (host, port) = if let Some(host) = host.strip_prefix('[') { let (host, maybe_port) = host.rsplit_once(']').unwrap_or_default(); ( host, maybe_port .rsplit_once(':') .map(|(_, port)| port) .unwrap_or("53"), ) } else if let Some((host, port)) = host.split_once(':') { (host, port) } else { (host.as_str(), "53") }; let port = port .parse::() .map_err(|err| { config.new_parse_error( "resolver.custom", format!("Invalid custom resolver port {port:?}: {err}"), ); }) .unwrap_or(53); let host = host .parse::() .map_err(|err| { config.new_parse_error( "resolver.custom", format!("Invalid custom resolver IP {host:?}: {err}"), ) }) .unwrap_or(IpAddr::V4(Ipv4Addr::new(8, 8, 8, 8))); resolver_config .add_name_server(NameServerConfig::new(SocketAddr::new(host, port), proto)); } if !resolver_config.name_servers().is_empty() { (resolver_config, ResolverOpts::default()) } else { config.new_parse_error( "resolver.custom", "At least one custom resolver must be specified.", ); (ResolverConfig::cloudflare(), ResolverOpts::default()) } } other => { let err = format!("Unknown resolver type {other:?}."); config.new_parse_error("resolver.custom", err); (ResolverConfig::cloudflare(), ResolverOpts::default()) } }; if let Some(concurrency) = config.property("resolver.concurrency") { opts.num_concurrent_reqs = concurrency; } if let Some(timeout) = config.property("resolver.timeout") { opts.timeout = timeout; } if let Some(preserve) = config.property("resolver.preserve-intermediates") { opts.preserve_intermediates = preserve; } if let Some(try_tcp_on_error) = config.property("resolver.try-tcp-on-error") { opts.try_tcp_on_error = try_tcp_on_error; } if let Some(attempts) = config.property("resolver.attempts") { opts.attempts = attempts; } opts.edns0 = config .property_or_default("resolver.edns", "true") .unwrap_or(true); // We already have a cache, so disable the built-in cache opts.cache_size = 0; // Prepare DNSSEC resolver options let config_dnssec = resolver_config.clone(); let mut opts_dnssec = opts.clone(); opts_dnssec.validate = true; Resolvers { dns: MessageAuthenticator::new(resolver_config, opts).unwrap(), dnssec: DnssecResolver { resolver: TokioResolver::builder_with_config( config_dnssec, TokioConnectionProvider::default(), ) .with_options(opts_dnssec) .build(), }, } } } impl Policy { pub fn try_parse(config: &mut Config) -> Option { let mode = config .property_or_default::>("session.mta-sts.mode", "testing") .unwrap_or_default()?; let max_age = config .property_or_default::("session.mta-sts.max-age", "7d") .unwrap_or_else(|| Duration::from_secs(604800)) .as_secs(); let mut mx = Vec::new(); for (_, item) in config.values("session.mta-sts.mx") { if let Some(item) = item.strip_prefix("*.") { mx.push(MxPattern::StartsWith(item.to_string())); } else { mx.push(MxPattern::Equals(item.to_string())); } } let mut policy = Self { id: Default::default(), mode, mx, max_age, }; if !policy.mx.is_empty() { policy.mx.sort_unstable(); policy.id = policy.hash().to_string(); } policy.into() } pub fn try_build(mut self, names: I) -> Option where I: IntoIterator, T: AsRef, { if self.mx.is_empty() { for name in names { let name = name.as_ref(); if let Some(domain) = name.strip_prefix('.') { self.mx.push(MxPattern::StartsWith(domain.to_string())); } else if name != "*" && !name.is_empty() { self.mx.push(MxPattern::Equals(name.to_string())); } } if !self.mx.is_empty() { self.mx.sort_unstable(); self.id = self.hash().to_string(); Some(self) } else { None } } else { Some(self) } } fn hash(&self) -> u64 { let mut s = DefaultHasher::new(); self.mode.hash(&mut s); self.max_age.hash(&mut s); self.mx.hash(&mut s); s.finish() } } impl Server { pub fn build_mta_sts_policy(&self) -> Option { self.core .smtp .session .mta_sts_policy .clone() .and_then(|policy| { policy.try_build( self.inner .data .tls_certificates .load() .keys() .filter(|key| { !key.starts_with("mta-sts.") && !key.starts_with("autoconfig.") && !key.starts_with("autodiscover.") }), ) }) } } impl ParseValue for Mode { fn parse_value(value: &str) -> Result { match value { "enforce" => Ok(Self::Enforce), "testing" | "test" => Ok(Self::Testing), "none" => Ok(Self::None), _ => Err(format!("Invalid mode value {value:?}")), } } } impl Default for Resolvers { fn default() -> Self { let (config, opts) = match read_system_conf() { Ok(conf) => conf, Err(_) => (ResolverConfig::cloudflare(), ResolverOpts::default()), }; let config_dnssec = config.clone(); let mut opts_dnssec = opts.clone(); opts_dnssec.validate = true; Self { dns: MessageAuthenticator::new(config, opts).expect("Failed to build DNS resolver"), dnssec: DnssecResolver { resolver: TokioResolver::builder_with_config( config_dnssec, TokioConnectionProvider::default(), ) .with_options(opts_dnssec) .build(), }, } } } impl Display for Policy { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("version: STSv1\r\n")?; f.write_str("mode: ")?; match self.mode { Mode::Enforce => f.write_str("enforce")?, Mode::Testing => f.write_str("testing")?, Mode::None => unreachable!(), } f.write_str("\r\nmax_age: ")?; self.max_age.fmt(f)?; f.write_str("\r\n")?; for mx in &self.mx { f.write_str("mx: ")?; mx.fmt(f)?; f.write_str("\r\n")?; } Ok(()) } } impl Display for MxPattern { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { MxPattern::Equals(mx) => f.write_str(mx), MxPattern::StartsWith(mx) => { f.write_str("*.")?; f.write_str(mx) } } } } impl Clone for Resolvers { fn clone(&self) -> Self { Self { dns: self.dns.clone(), dnssec: self.dnssec.clone(), } } } ================================================ FILE: crates/common/src/config/smtp/session.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ net::{SocketAddr, ToSocketAddrs}, str::FromStr, time::Duration, }; use ahash::AHashSet; use base64::{Engine, engine::general_purpose::STANDARD}; use hyper::{ HeaderMap, header::{AUTHORIZATION, CONTENT_TYPE, HeaderName, HeaderValue}, }; use smtp_proto::*; use utils::config::{Config, utils::ParseValue}; use crate::{ config::CONNECTION_VARS, expr::{if_block::IfBlock, tokenizer::TokenMap, *}, }; use self::resolver::Policy; use super::*; #[derive(Clone)] pub struct SessionConfig { pub timeout: IfBlock, pub duration: IfBlock, pub transfer_limit: IfBlock, pub connect: Connect, pub ehlo: Ehlo, pub auth: Auth, pub mail: Mail, pub rcpt: Rcpt, pub data: Data, pub extensions: Extensions, pub mta_sts_policy: Option, pub milters: Vec, pub hooks: Vec, } #[derive(Clone)] pub struct Connect { pub hostname: IfBlock, pub script: IfBlock, pub greeting: IfBlock, } #[derive(Clone)] pub struct Ehlo { pub script: IfBlock, pub require: IfBlock, pub reject_non_fqdn: IfBlock, } #[derive(Clone)] pub struct Extensions { pub pipelining: IfBlock, pub chunking: IfBlock, pub requiretls: IfBlock, pub dsn: IfBlock, pub vrfy: IfBlock, pub expn: IfBlock, pub no_soliciting: IfBlock, pub future_release: IfBlock, pub deliver_by: IfBlock, pub mt_priority: IfBlock, } #[derive(Clone)] pub struct Auth { pub directory: IfBlock, pub mechanisms: IfBlock, pub require: IfBlock, pub must_match_sender: IfBlock, pub errors_max: IfBlock, pub errors_wait: IfBlock, } #[derive(Clone)] pub struct Mail { pub script: IfBlock, pub rewrite: IfBlock, pub is_allowed: IfBlock, } #[derive(Clone)] pub struct Rcpt { pub script: IfBlock, pub relay: IfBlock, pub directory: IfBlock, pub rewrite: IfBlock, // Errors pub errors_max: IfBlock, pub errors_wait: IfBlock, // Limits pub max_recipients: IfBlock, // Catch-all and sub-addressing pub catch_all: AddressMapping, pub subaddressing: AddressMapping, } #[derive(Debug, Default, Clone)] pub enum AddressMapping { Enable, Custom(IfBlock), #[default] Disable, } #[derive(Clone)] pub struct Data { pub script: IfBlock, pub spam_filter: IfBlock, // Limits pub max_messages: IfBlock, pub max_message_size: IfBlock, pub max_received_headers: IfBlock, // Headers pub add_received: IfBlock, pub add_received_spf: IfBlock, pub add_return_path: IfBlock, pub add_auth_results: IfBlock, pub add_message_id: IfBlock, pub add_date: IfBlock, pub add_delivered_to: bool, } #[derive(Clone)] pub struct Milter { pub enable: IfBlock, pub id: Arc, pub addrs: Vec, pub hostname: String, pub port: u16, pub timeout_connect: Duration, pub timeout_command: Duration, pub timeout_data: Duration, pub tls: bool, pub tls_allow_invalid_certs: bool, pub tempfail_on_error: bool, pub max_frame_len: usize, pub protocol_version: MilterVersion, pub flags_actions: Option, pub flags_protocol: Option, pub run_on_stage: AHashSet, } #[derive(Clone, Copy)] pub enum MilterVersion { V2, V6, } #[derive(Clone)] pub struct MTAHook { pub enable: IfBlock, pub id: String, pub url: String, pub timeout: Duration, pub headers: HeaderMap, pub tls_allow_invalid_certs: bool, pub tempfail_on_error: bool, pub run_on_stage: AHashSet, pub max_response_size: usize, } #[derive(Clone, Copy, PartialEq, Eq, Hash)] pub enum Stage { Connect, Ehlo, Auth, Mail, Rcpt, Data, } impl SessionConfig { pub fn parse(config: &mut Config) -> Self { let has_conn_vars = TokenMap::default().with_variables(CONNECTION_VARS); let has_ehlo_hars = TokenMap::default().with_variables(SMTP_EHLO_VARS); let has_sender_vars = TokenMap::default().with_variables(SMTP_MAIL_FROM_VARS); let has_rcpt_vars = TokenMap::default().with_variables(SMTP_RCPT_TO_VARS); let mt_priority_vars = has_sender_vars.clone().with_constants::(); let mechanisms_vars = has_ehlo_hars.clone().with_constants::(); let mut session = SessionConfig::default(); session.rcpt.catch_all = AddressMapping::parse(config, "session.rcpt.catch-all"); session.rcpt.subaddressing = AddressMapping::parse(config, "session.rcpt.sub-addressing"); session.milters = config .sub_keys("session.milter", ".hostname") .into_iter() .filter_map(|id| parse_milter(config, &id, &has_rcpt_vars)) .collect(); session.hooks = config .sub_keys("session.hook", ".url") .into_iter() .filter_map(|id| parse_hooks(config, &id, &has_rcpt_vars)) .collect(); session.mta_sts_policy = Policy::try_parse(config); for (value, key, token_map) in [ (&mut session.duration, "session.duration", &has_conn_vars), ( &mut session.transfer_limit, "session.transfer-limit", &has_conn_vars, ), (&mut session.timeout, "session.timeout", &has_conn_vars), ( &mut session.connect.script, "session.connect.script", &has_conn_vars, ), ( &mut session.connect.hostname, "session.connect.hostname", &has_conn_vars, ), ( &mut session.connect.greeting, "session.connect.greeting", &has_conn_vars, ), ( &mut session.extensions.pipelining, "session.extensions.pipelining", &has_sender_vars, ), ( &mut session.extensions.dsn, "session.extensions.dsn", &has_sender_vars, ), ( &mut session.extensions.vrfy, "session.extensions.vrfy", &has_sender_vars, ), ( &mut session.extensions.expn, "session.extensions.expn", &has_sender_vars, ), ( &mut session.extensions.chunking, "session.extensions.chunking", &has_sender_vars, ), ( &mut session.extensions.requiretls, "session.extensions.requiretls", &has_sender_vars, ), ( &mut session.extensions.no_soliciting, "session.extensions.no-soliciting", &has_sender_vars, ), ( &mut session.extensions.future_release, "session.extensions.future-release", &has_sender_vars, ), ( &mut session.extensions.deliver_by, "session.extensions.deliver-by", &has_sender_vars, ), ( &mut session.extensions.mt_priority, "session.extensions.mt-priority", &mt_priority_vars, ), ( &mut session.ehlo.script, "session.ehlo.script", &has_conn_vars, ), ( &mut session.ehlo.require, "session.ehlo.require", &has_conn_vars, ), ( &mut session.ehlo.reject_non_fqdn, "session.ehlo.reject-non-fqdn", &has_conn_vars, ), ( &mut session.auth.directory, "session.auth.directory", &has_ehlo_hars, ), ( &mut session.auth.mechanisms, "session.auth.mechanisms", &mechanisms_vars, ), ( &mut session.auth.require, "session.auth.require", &has_ehlo_hars, ), ( &mut session.auth.errors_max, "session.auth.errors.total", &has_ehlo_hars, ), ( &mut session.auth.errors_wait, "session.auth.errors.wait", &has_ehlo_hars, ), ( &mut session.auth.must_match_sender, "session.auth.must-match-sender", &has_sender_vars, ), ( &mut session.mail.script, "session.mail.script", &has_sender_vars, ), ( &mut session.mail.rewrite, "session.mail.rewrite", &has_sender_vars, ), ( &mut session.mail.is_allowed, "session.mail.is-allowed", &has_sender_vars, ), ( &mut session.rcpt.script, "session.rcpt.script", &has_rcpt_vars, ), ( &mut session.rcpt.relay, "session.rcpt.relay", &has_rcpt_vars, ), ( &mut session.rcpt.directory, "session.rcpt.directory", &has_rcpt_vars, ), ( &mut session.rcpt.errors_max, "session.rcpt.errors.total", &has_sender_vars, ), ( &mut session.rcpt.errors_wait, "session.rcpt.errors.wait", &has_sender_vars, ), ( &mut session.rcpt.max_recipients, "session.rcpt.max-recipients", &has_sender_vars, ), ( &mut session.rcpt.rewrite, "session.rcpt.rewrite", &has_rcpt_vars, ), ( &mut session.data.script, "session.data.script", &has_rcpt_vars, ), ( &mut session.data.max_messages, "session.data.limits.messages", &has_rcpt_vars, ), ( &mut session.data.max_message_size, "session.data.limits.size", &has_rcpt_vars, ), ( &mut session.data.max_received_headers, "session.data.limits.received-headers", &has_rcpt_vars, ), ( &mut session.data.spam_filter, "session.data.spam-filter", &has_rcpt_vars, ), ( &mut session.data.add_received, "session.data.add-headers.received", &has_rcpt_vars, ), ( &mut session.data.add_received_spf, "session.data.add-headers.received-spf", &has_rcpt_vars, ), ( &mut session.data.add_return_path, "session.data.add-headers.return-path", &has_rcpt_vars, ), ( &mut session.data.add_auth_results, "session.data.add-headers.auth-results", &has_rcpt_vars, ), ( &mut session.data.add_message_id, "session.data.add-headers.message-id", &has_rcpt_vars, ), ( &mut session.data.add_date, "session.data.add-headers.date", &has_rcpt_vars, ), ] { if let Some(if_block) = IfBlock::try_parse(config, key, token_map) { *value = if_block; } } session.data.add_delivered_to = config .property_or_default("session.data.add-headers.delivered-to", "true") .unwrap_or(true); session } } fn parse_milter(config: &mut Config, id: &str, token_map: &TokenMap) -> Option { let hostname = config .value_require(("session.milter", id, "hostname"))? .to_string(); let port = config.property_require(("session.milter", id, "port"))?; Some(Milter { enable: IfBlock::try_parse(config, ("session.milter", id, "enable"), token_map) .unwrap_or_else(|| { IfBlock::new::<()>(format!("session.milter.{id}.enable"), [], "false") }), id: Arc::new(id.into()), addrs: format!("{}:{}", hostname, port) .to_socket_addrs() .map_err(|err| { config.new_build_error( ("session.milter", id, "hostname"), format!("Unable to resolve milter hostname {hostname}: {err}"), ) }) .ok()? .collect(), hostname, port, timeout_connect: config .property_or_default(("session.milter", id, "timeout.connect"), "30s") .unwrap_or_else(|| Duration::from_secs(30)), timeout_command: config .property_or_default(("session.milter", id, "timeout.command"), "30s") .unwrap_or_else(|| Duration::from_secs(30)), timeout_data: config .property_or_default(("session.milter", id, "timeout.data"), "60s") .unwrap_or_else(|| Duration::from_secs(60)), tls: config .property_or_default(("session.milter", id, "tls"), "false") .unwrap_or_default(), tls_allow_invalid_certs: config .property_or_default(("session.milter", id, "allow-invalid-certs"), "false") .unwrap_or_default(), tempfail_on_error: config .property_or_default(("session.milter", id, "options.tempfail-on-error"), "true") .unwrap_or(true), max_frame_len: config .property_or_default( ("session.milter", id, "options.max-response-size"), "52428800", ) .unwrap_or(52428800), protocol_version: match config .property_or_default::(("session.milter", id, "options.version"), "6") .unwrap_or(6) { 6 => MilterVersion::V6, 2 => MilterVersion::V2, v => { config.new_parse_error( ("session.milter", id, "options.version"), format!("Unsupported milter protocol version {v}"), ); MilterVersion::V6 } }, flags_actions: config.property(("session.milter", id, "options.flags.actions")), flags_protocol: config.property(("session.milter", id, "options.flags.protocol")), run_on_stage: parse_stages(config, "session.milter", id), }) } fn parse_hooks(config: &mut Config, id: &str, token_map: &TokenMap) -> Option { let mut headers = HeaderMap::new(); for (header, value) in config .values(("session.hook", id, "headers")) .map(|(_, v)| { if let Some((k, v)) = v.split_once(':') { Ok(( HeaderName::from_str(k.trim()).map_err(|err| { format!( "Invalid header found in property \"session.hook.{id}.headers\": {err}", ) })?, HeaderValue::from_str(v.trim()).map_err(|err| { format!( "Invalid header found in property \"session.hook.{id}.headers\": {err}", ) })?, )) } else { Err(format!( "Invalid header found in property \"session.hook.{id}.headers\": {v}", )) } }) .collect::, String>>() .map_err(|e| config.new_parse_error(("session.hook", id, "headers"), e)) .unwrap_or_default() { headers.insert(header, value); } headers.insert(CONTENT_TYPE, "application/json".parse().unwrap()); if let (Some(name), Some(secret)) = ( config.value(("session.hook", id, "auth.username")), config.value(("session.hook", id, "auth.secret")), ) { headers.insert( AUTHORIZATION, format!("Basic {}", STANDARD.encode(format!("{}:{}", name, secret))) .parse() .unwrap(), ); } Some(MTAHook { enable: IfBlock::try_parse(config, ("session.hook", id, "enable"), token_map) .unwrap_or_else(|| { IfBlock::new::<()>(format!("session.hook.{id}.enable"), [], "false") }), id: id.to_string(), url: config .value_require(("session.hook", id, "url"))? .to_string(), timeout: config .property_or_default(("session.hook", id, "timeout"), "30s") .unwrap_or_else(|| Duration::from_secs(30)), tls_allow_invalid_certs: config .property_or_default(("session.hook", id, "allow-invalid-certs"), "false") .unwrap_or_default(), tempfail_on_error: config .property_or_default(("session.hook", id, "options.tempfail-on-error"), "true") .unwrap_or(true), run_on_stage: parse_stages(config, "session.hook", id), max_response_size: config .property_or_default( ("session.hook", id, "options.max-response-size"), "52428800", ) .unwrap_or(52428800), headers, }) } fn parse_stages(config: &mut Config, prefix: &str, id: &str) -> AHashSet { let mut stages = AHashSet::default(); let mut invalid = Vec::new(); for (_, value) in config.values((prefix, id, "stages")) { let value = value.to_ascii_lowercase(); let state = match value.as_str() { "connect" => Stage::Connect, "ehlo" => Stage::Ehlo, "auth" => Stage::Auth, "mail" => Stage::Mail, "rcpt" => Stage::Rcpt, "data" => Stage::Data, _ => { invalid.push(value); continue; } }; stages.insert(state); } if !invalid.is_empty() { config.new_parse_error( (prefix, id, "stages"), format!("Invalid stages: {}", invalid.join(", ")), ); } if stages.is_empty() { stages.insert(Stage::Data); } stages } impl Default for SessionConfig { fn default() -> Self { Self { timeout: IfBlock::new::<()>("session.timeout", [], "5m"), duration: IfBlock::new::<()>("session.duration", [], "10m"), transfer_limit: IfBlock::new::<()>("session.transfer-limit", [], "262144000"), connect: Connect { hostname: IfBlock::new::<()>( "server.connect.hostname", [], "config_get('server.hostname')", ), script: IfBlock::empty("session.connect.script"), greeting: IfBlock::new::<()>( "session.connect.greeting", [], "config_get('server.hostname') + ' Stalwart ESMTP at your service'", ), }, ehlo: Ehlo { script: IfBlock::empty("session.ehlo.script"), require: IfBlock::new::<()>("session.ehlo.require", [], "true"), reject_non_fqdn: IfBlock::new::<()>( "session.ehlo.reject-non-fqdn", [("local_port == 25", "true")], "false", ), }, auth: Auth { directory: IfBlock::new::<()>( "session.auth.directory", #[cfg(feature = "test_mode")] [], #[cfg(not(feature = "test_mode"))] [("local_port != 25", "'*'")], "false", ), mechanisms: IfBlock::new::( "session.auth.mechanisms", [ ( "local_port != 25 && is_tls", "[plain, login, oauthbearer, xoauth2]", ), ("local_port != 25", "[oauthbearer, xoauth2]"), ], "false", ), require: IfBlock::new::<()>( "session.auth.require", #[cfg(feature = "test_mode")] [], #[cfg(not(feature = "test_mode"))] [("local_port != 25", "true")], "false", ), must_match_sender: IfBlock::new::<()>("session.auth.must-match-sender", [], "true"), errors_max: IfBlock::new::<()>("session.auth.errors.total", [], "3"), errors_wait: IfBlock::new::<()>("session.auth.errors.wait", [], "5s"), }, mail: Mail { script: IfBlock::empty("session.mail.script"), rewrite: IfBlock::empty("session.mail.rewrite"), is_allowed: IfBlock::new::<()>( "session.mail.is-allowed", [], "!is_empty(authenticated_as) || !key_exists('blocked-domains', sender_domain)", ), }, rcpt: Rcpt { script: IfBlock::empty("session.rcpt.script"), relay: IfBlock::new::<()>( "session.rcpt.relay", [("!is_empty(authenticated_as)", "true")], "false", ), directory: IfBlock::new::<()>( "session.rcpt.directory", [], #[cfg(feature = "test_mode")] "false", #[cfg(not(feature = "test_mode"))] "'*'", ), rewrite: IfBlock::empty("session.rcpt.rewrite"), errors_max: IfBlock::new::<()>("session.rcpt.errors.total", [], "5"), errors_wait: IfBlock::new::<()>("session.rcpt.errors.wait", [], "5s"), max_recipients: IfBlock::new::<()>("session.rcpt.max-recipients", [], "100"), catch_all: AddressMapping::Enable, subaddressing: AddressMapping::Enable, }, data: Data { script: IfBlock::empty("session.data.script"), spam_filter: IfBlock::new::<()>("session.data.spam-filter", [], "true"), max_messages: IfBlock::new::<()>("session.data.limits.messages", [], "10"), max_message_size: IfBlock::new::<()>("session.data.limits.size", [], "104857600"), max_received_headers: IfBlock::new::<()>( "session.data.limits.received-headers", [], "50", ), add_received: IfBlock::new::<()>( "session.data.add-headers.received", [("local_port == 25", "true")], "false", ), add_received_spf: IfBlock::new::<()>( "session.data.add-headers.received-spf", [("local_port == 25", "true")], "false", ), add_return_path: IfBlock::new::<()>( "session.data.add-headers.return-path", [("local_port == 25", "true")], "false", ), add_auth_results: IfBlock::new::<()>( "session.data.add-headers.auth-results", [("local_port == 25", "true")], "false", ), add_message_id: IfBlock::new::<()>( "session.data.add-headers.message-id", [("local_port == 25", "true")], "false", ), add_date: IfBlock::new::<()>( "session.data.add-headers.date", [("local_port == 25", "true")], "false", ), add_delivered_to: false, }, extensions: Extensions { pipelining: IfBlock::new::<()>("session.extensions.pipelining", [], "true"), chunking: IfBlock::new::<()>("session.extensions.chunking", [], "true"), requiretls: IfBlock::new::<()>("session.extensions.requiretls", [], "true"), dsn: IfBlock::new::<()>( "session.extensions.dsn", [("!is_empty(authenticated_as)", "true")], "false", ), vrfy: IfBlock::new::<()>( "session.extensions.vrfy", [("!is_empty(authenticated_as)", "true")], "false", ), expn: IfBlock::new::<()>( "session.extensions.expn", [("!is_empty(authenticated_as)", "true")], "false", ), no_soliciting: IfBlock::new::<()>("session.extensions.no-soliciting", [], "''"), future_release: IfBlock::new::<()>( "session.extensions.future-release", [("!is_empty(authenticated_as)", "7d")], "false", ), deliver_by: IfBlock::new::<()>( "session.extensions.deliver-by", [("!is_empty(authenticated_as)", "15d")], "false", ), mt_priority: IfBlock::new::( "session.extensions.mt-priority", [("!is_empty(authenticated_as)", "mixer")], "false", ), }, mta_sts_policy: None, milters: Default::default(), hooks: Default::default(), } } } #[derive(Default)] pub struct Mechanism(u64); impl ParseValue for Mechanism { fn parse_value(value: &str) -> Result { Ok(Mechanism(match value.to_ascii_uppercase().as_str() { "LOGIN" => AUTH_LOGIN, "PLAIN" => AUTH_PLAIN, "XOAUTH2" => AUTH_XOAUTH2, "OAUTHBEARER" => AUTH_OAUTHBEARER, /*"SCRAM-SHA-256-PLUS" => AUTH_SCRAM_SHA_256_PLUS, "SCRAM-SHA-256" => AUTH_SCRAM_SHA_256, "SCRAM-SHA-1-PLUS" => AUTH_SCRAM_SHA_1_PLUS, "SCRAM-SHA-1" => AUTH_SCRAM_SHA_1, "XOAUTH" => AUTH_XOAUTH, "9798-M-DSA-SHA1" => AUTH_9798_M_DSA_SHA1, "9798-M-ECDSA-SHA1" => AUTH_9798_M_ECDSA_SHA1, "9798-M-RSA-SHA1-ENC" => AUTH_9798_M_RSA_SHA1_ENC, "9798-U-DSA-SHA1" => AUTH_9798_U_DSA_SHA1, "9798-U-ECDSA-SHA1" => AUTH_9798_U_ECDSA_SHA1, "9798-U-RSA-SHA1-ENC" => AUTH_9798_U_RSA_SHA1_ENC, "EAP-AES128" => AUTH_EAP_AES128, "EAP-AES128-PLUS" => AUTH_EAP_AES128_PLUS, "ECDH-X25519-CHALLENGE" => AUTH_ECDH_X25519_CHALLENGE, "ECDSA-NIST256P-CHALLENGE" => AUTH_ECDSA_NIST256P_CHALLENGE, "EXTERNAL" => AUTH_EXTERNAL, "GS2-KRB5" => AUTH_GS2_KRB5, "GS2-KRB5-PLUS" => AUTH_GS2_KRB5_PLUS, "GSS-SPNEGO" => AUTH_GSS_SPNEGO, "GSSAPI" => AUTH_GSSAPI, "KERBEROS_V4" => AUTH_KERBEROS_V4, "KERBEROS_V5" => AUTH_KERBEROS_V5, "NMAS-SAMBA-AUTH" => AUTH_NMAS_SAMBA_AUTH, "NMAS_AUTHEN" => AUTH_NMAS_AUTHEN, "NMAS_LOGIN" => AUTH_NMAS_LOGIN, "NTLM" => AUTH_NTLM, "OAUTH10A" => AUTH_OAUTH10A, "OPENID20" => AUTH_OPENID20, "OTP" => AUTH_OTP, "SAML20" => AUTH_SAML20, "SECURID" => AUTH_SECURID, "SKEY" => AUTH_SKEY, "SPNEGO" => AUTH_SPNEGO, "SPNEGO-PLUS" => AUTH_SPNEGO_PLUS, "SXOVER-PLUS" => AUTH_SXOVER_PLUS, "CRAM-MD5" => AUTH_CRAM_MD5, "DIGEST-MD5" => AUTH_DIGEST_MD5, "ANONYMOUS" => AUTH_ANONYMOUS,*/ _ => return Err(format!("Unsupported mechanism {:?}.", value)), })) } } impl<'x> TryFrom> for Mechanism { type Error = (); fn try_from(value: Variable<'x>) -> Result { match value { Variable::Integer(value) => Ok(Mechanism(value as u64)), Variable::Array(items) => { let mut mechanism = 0; for item in items { match item { Variable::Integer(value) => mechanism |= value as u64, _ => return Err(()), } } Ok(Mechanism(mechanism)) } _ => Err(()), } } } impl From for Constant { fn from(value: Mechanism) -> Self { Constant::Integer(value.0 as i64) } } impl ConstantValue for Mechanism { fn add_constants(token_map: &mut crate::expr::tokenizer::TokenMap) { token_map .add_constant("login", Mechanism(AUTH_LOGIN)) .add_constant("plain", Mechanism(AUTH_PLAIN)) .add_constant("xoauth2", Mechanism(AUTH_XOAUTH2)) .add_constant("oauthbearer", Mechanism(AUTH_OAUTHBEARER)); } } impl From for u64 { fn from(value: Mechanism) -> Self { value.0 } } impl From for Mechanism { fn from(value: u64) -> Self { Mechanism(value) } } impl<'x> TryFrom> for MtPriority { type Error = (); fn try_from(value: Variable<'x>) -> Result { match value { Variable::Integer(value) => match value { 2 => Ok(MtPriority::Mixer), 3 => Ok(MtPriority::Stanag4406), 4 => Ok(MtPriority::Nsep), _ => Err(()), }, Variable::String(value) => MtPriority::parse_value(value.as_str()).map_err(|_| ()), _ => Err(()), } } } impl From for Constant { fn from(value: MtPriority) -> Self { Constant::Integer(match value { MtPriority::Mixer => 2, MtPriority::Stanag4406 => 3, MtPriority::Nsep => 4, }) } } impl ConstantValue for MtPriority { fn add_constants(token_map: &mut TokenMap) { token_map .add_constant("mixer", MtPriority::Mixer) .add_constant("stanag4406", MtPriority::Stanag4406) .add_constant("nsep", MtPriority::Nsep); } } ================================================ FILE: crates/common/src/config/smtp/throttle.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use utils::config::{Config, Rate, utils::AsKey}; use crate::expr::{Expression, tokenizer::TokenMap}; use super::*; pub fn parse_queue_rate_limiter( config: &mut Config, prefix: impl AsKey, token_map: &TokenMap, available_rate_limiter_keys: u16, ) -> Vec { let prefix_ = prefix.as_key(); let mut rate_limiters = Vec::new(); for rate_limiter_id in config.sub_keys(prefix, "") { let rate_limiter_id = rate_limiter_id.as_str(); if let Some(rate_limiter) = parse_queue_rate_limiter_item( config, (&prefix_, rate_limiter_id), rate_limiter_id, token_map, available_rate_limiter_keys, ) { rate_limiters.push(rate_limiter); } } rate_limiters } fn parse_queue_rate_limiter_item( config: &mut Config, prefix: impl AsKey, rate_limiter_id: &str, token_map: &TokenMap, available_rate_limiter_keys: u16, ) -> Option { let prefix = prefix.as_key(); // Skip disabled rate_limiters if !config .property::((prefix.as_str(), "enable")) .unwrap_or(true) { return None; } let mut keys = 0; for (key_, value) in config .values((&prefix, "key")) .map(|(k, v)| (k.to_string(), v.to_string())) .collect::>() { match parse_queue_rate_limiter_key(&value) { Ok(key) => { if (key & available_rate_limiter_keys) != 0 { keys |= key; } else { let err = format!("Rate limiter key {value:?} is not available in this context"); config.new_build_error(key_, err); } } Err(err) => { config.new_parse_error(key_, err); } } } Some(QueueRateLimiter { id: rate_limiter_id.to_string(), expr: Expression::try_parse(config, (prefix.as_str(), "match"), token_map) .unwrap_or_default(), keys, rate: config .property_require::((prefix.as_str(), "rate")) .filter(|r| r.requests > 0)?, }) } pub(crate) fn parse_queue_rate_limiter_key(value: &str) -> Result { match value { "rcpt" => Ok(THROTTLE_RCPT), "rcpt_domain" => Ok(THROTTLE_RCPT_DOMAIN), "sender" => Ok(THROTTLE_SENDER), "sender_domain" => Ok(THROTTLE_SENDER_DOMAIN), "authenticated_as" => Ok(THROTTLE_AUTH_AS), "listener" => Ok(THROTTLE_LISTENER), "mx" => Ok(THROTTLE_MX), "remote_ip" => Ok(THROTTLE_REMOTE_IP), "local_ip" => Ok(THROTTLE_LOCAL_IP), "helo_domain" => Ok(THROTTLE_HELO_DOMAIN), _ => Err(format!("Invalid THROTTLE key {value:?}")), } } ================================================ FILE: crates/common/src/config/spamfilter.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{Variable, functions::ResolveVariable, if_block::IfBlock, tokenizer::TokenMap}; use ahash::AHashSet; use mail_auth::common::resolver::ToReverseName; use nlp::classifier::model::{CcfhClassifier, FhClassifier}; use std::{ net::{IpAddr, SocketAddr}, time::Duration, }; use tokio::net::lookup_host; use utils::{ cache::CacheItemWeight, config::{Config, utils::ParseValue}, glob::GlobMap, }; #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default)] pub enum SpamClassifier { FhClassifier { classifier: FhClassifier, last_trained_at: u64, }, CcfhClassifier { classifier: CcfhClassifier, last_trained_at: u64, }, #[default] Disabled, } #[derive(Debug, Clone, Default)] pub struct SpamFilterConfig { pub enabled: bool, pub card_is_ham: bool, pub trusted_reply: bool, pub grey_list_expiry: Option, pub dnsbl: DnsBlConfig, pub rules: SpamFilterRules, pub lists: SpamFilterLists, pub pyzor: Option, pub classifier: Option, pub scores: SpamFilterScoreConfig, } #[derive(Debug, Clone, Default)] pub struct SpamFilterScoreConfig { pub reject_threshold: f32, pub discard_threshold: f32, pub spam_threshold: f32, } #[derive(Debug, Clone, Default)] pub struct DnsBlConfig { pub max_ip_checks: usize, pub max_domain_checks: usize, pub max_email_checks: usize, pub max_url_checks: usize, pub servers: Vec, } #[derive(Debug, Clone, Default)] pub struct SpamFilterLists { pub file_extensions: GlobMap, pub scores: GlobMap>, } #[derive(Debug, Clone)] pub enum SpamFilterAction { Allow(T), Discard, Reject, Disabled, } #[derive(Debug, Clone, Default)] pub struct ClassifierConfig { pub w_params: FtrlParameters, pub i_params: Option, pub reservoir_capacity: usize, pub min_ham_samples: u64, pub min_spam_samples: u64, pub auto_learn_reply_ham: bool, pub auto_learn_card_is_ham: bool, pub auto_learn_spam_trap: bool, pub auto_learn_spam_rbl_count: u32, pub hold_samples_for: u64, pub train_frequency: Option, pub log_scale: bool, pub l2_normalize: bool, } #[derive(Debug, Clone, Default)] pub struct FtrlParameters { pub feature_hash_size: usize, pub alpha: f64, pub beta: f64, pub l1_ratio: f64, pub l2_ratio: f64, } #[derive(Debug, Clone)] pub struct PyzorConfig { pub address: SocketAddr, pub timeout: Duration, pub min_count: u64, pub min_wl_count: u64, pub ratio: f64, } #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct SpamFilterRules { pub url: Vec, pub domain: Vec, pub email: Vec, pub ip: Vec, pub header: Vec, pub body: Vec, pub any: Vec, } #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct FileExtension { pub known_types: AHashSet, pub is_bad: bool, pub is_archive: bool, pub is_nz: bool, } #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] pub enum Element { Url, Domain, Email, Ip, Header, Body, #[default] Any, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Location { EnvelopeFrom, EnvelopeTo, HeaderDkimPass, HeaderReceived, HeaderFrom, HeaderReplyTo, HeaderSubject, HeaderTo, HeaderCc, HeaderBcc, HeaderMid, HeaderDnt, Ehlo, BodyText, BodyHtml, Attachment, Tcp, } #[derive(Debug, Clone)] pub struct DnsBlServer { pub id: String, pub zone: IfBlock, pub scope: Element, pub tags: IfBlock, } impl SpamFilterConfig { pub async fn parse(config: &mut Config) -> Self { SpamFilterConfig { enabled: config .property_or_default("spam-filter.enable", "true") .unwrap_or(true), card_is_ham: config .property_or_default("spam-filter.card-is-ham.enable", "true") .unwrap_or(true), trusted_reply: config .property_or_default("spam-filter.trusted-reply.enable", "true") .unwrap_or(true), dnsbl: DnsBlConfig::parse(config), rules: SpamFilterRules::parse(config), lists: SpamFilterLists::parse(config), pyzor: PyzorConfig::parse(config).await, classifier: ClassifierConfig::parse(config), scores: SpamFilterScoreConfig::parse(config), grey_list_expiry: config .property::>("spam-filter.grey-list.duration") .unwrap_or_default() .map(|d| d.as_secs()), } } } impl SpamFilterRules { pub fn parse(config: &mut Config) -> SpamFilterRules { let mut rules = vec![]; for id in config.sub_keys("spam-filter.rule", ".scope") { if let Some(rule) = SpamFilterRule::parse(config, id) { rules.push(rule); } } rules.sort_by(|a, b| a.priority.cmp(&b.priority)); let mut result = SpamFilterRules::default(); for rule in rules { match rule.scope { Element::Url => result.url.push(rule.rule), Element::Domain => result.domain.push(rule.rule), Element::Email => result.email.push(rule.rule), Element::Ip => result.ip.push(rule.rule), Element::Header => result.header.push(rule.rule), Element::Body => result.body.push(rule.rule), Element::Any => result.any.push(rule.rule), } } result } } struct SpamFilterRule { rule: IfBlock, priority: i32, scope: Element, } impl SpamFilterRule { pub fn parse(config: &mut Config, id: String) -> Option { let id = id.as_str(); if !config .property_or_default(("spam-filter.rule", id, "enable"), "true") .unwrap_or(true) { return None; } let priority = config .property_or_default(("spam-filter.rule", id, "priority"), "0") .unwrap_or(0); let scope = config .property_or_default::(("spam-filter.rule", id, "scope"), "any") .unwrap_or_default(); SpamFilterRule { rule: IfBlock::try_parse( config, ("spam-filter.rule", id, "condition"), &scope.token_map(), )?, scope, priority, } .into() } } impl DnsBlConfig { pub fn parse(config: &mut Config) -> Self { let mut servers = vec![]; for id in config.sub_keys("spam-filter.dnsbl.server", ".scope") { if let Some(server) = DnsBlServer::parse(config, id) { servers.push(server); } } DnsBlConfig { max_ip_checks: config .property_or_default("spam-filter.dnsbl.max-check.ip", "50") .unwrap_or(20), max_domain_checks: config .property_or_default("spam-filter.dnsbl.max-check.domain", "50") .unwrap_or(20), max_email_checks: config .property_or_default("spam-filter.dnsbl.max-check.email", "50") .unwrap_or(20), max_url_checks: config .property_or_default("spam-filter.dnsbl.max-check.url", "50") .unwrap_or(20), servers, } } } impl DnsBlServer { pub fn parse(config: &mut Config, id: String) -> Option { let id_ = id.as_str(); if !config .property_or_default(("spam-filter.dnsbl.server", id_, "enable"), "true") .unwrap_or(true) { return None; } let scope = config.property_require::(("spam-filter.dnsbl.server", id_, "scope"))?; DnsBlServer { zone: IfBlock::try_parse( config, ("spam-filter.dnsbl.server", id_, "zone"), &scope.token_map(), )?, scope, tags: IfBlock::try_parse( config, ("spam-filter.dnsbl.server", id_, "tag"), &Element::Ip.token_map(), )?, id, } .into() } } impl SpamFilterLists { pub fn parse(config: &mut Config) -> Self { let mut lists = SpamFilterLists { file_extensions: GlobMap::default(), scores: GlobMap::default(), }; // Parse local lists let mut errors = vec![]; for (key, value) in config.iterate_prefix("spam-filter.list") { if let Some((id, key)) = key .split_once('.') .filter(|(id, key)| !id.is_empty() && !key.is_empty()) { match id { "scores" => { let action = match value.to_lowercase().as_str() { "reject" => SpamFilterAction::Reject, "discard" => SpamFilterAction::Discard, score => match score.parse() { Ok(score) => SpamFilterAction::Allow(score), Err(err) => { errors.push(( format!("spam-filter.list.{id}.{key}"), format!("Invalid score: {}", err), )); continue; } }, }; lists.scores.insert(key, action); } "file-extensions" => { let mut ext = FileExtension::default(); for part in value.split('|') { let part = part.trim(); match part { "AR" => { ext.is_archive = true; } "NZ" => { ext.is_nz = true; } "BAD" => { ext.is_bad = true; } other => { if other.contains('/') { ext.known_types.insert(other.to_string()); } else if !other.is_empty() { errors.push(( format!("spam-filter.list.{id}.{key}"), format!("Invalid file extension: {}", other), )); } } } } lists.file_extensions.insert(key, ext); } _ => (), } } } for (key, error) in errors { config.new_parse_error(key, error); } lists } } impl PyzorConfig { pub async fn parse(config: &mut Config) -> Option { if !config .property_or_default("spam-filter.pyzor.enable", "true") .unwrap_or(true) { return None; } let port = config .property_or_default::("spam-filter.pyzor.port", "24441") .unwrap_or(24441); let host = config .value("spam-filter.pyzor.host") .unwrap_or("public.pyzor.org"); let address = match lookup_host(format!("{host}:{port}")) .await .map(|mut a| a.next()) { Ok(Some(address)) => address, Ok(None) => { config.new_build_error( "spam-filter.pyzor.host", "Invalid address: No addresses found.", ); return None; } Err(err) => { config.new_build_error( "spam-filter.pyzor.host", format!("Invalid address: {}", err), ); return None; } }; PyzorConfig { address, timeout: config .property_or_default::("spam-filter.pyzor.timeout", "5s") .unwrap_or(Duration::from_secs(5)), min_count: config .property_or_default("spam-filter.pyzor.count", "5") .unwrap_or(5), min_wl_count: config .property_or_default("spam-filter.pyzor.wl-count", "10") .unwrap_or(10), ratio: config .property_or_default("spam-filter.pyzor.ratio", "0.2") .unwrap_or(0.2), } .into() } } impl ClassifierConfig { pub fn parse(config: &mut Config) -> Option { let ccfh = match config.value("spam-filter.classifier.model") { Some("ftrl-fh") | None => false, Some("ftrl-ccfh") => true, Some("disabled" | "disable") => return None, Some(other) => { config.new_build_error( "spam-filter.classifier.model", format!("Invalid model type: {}", other), ); return None; } }; let w_params = FtrlParameters::parse(config, "spam-filter.classifier.parameters", 20); let i_params = if ccfh { Some(FtrlParameters::parse( config, "spam-filter.classifier.parameters.ccfh", w_params.feature_hash_size - 2, )) } else { None }; ClassifierConfig { w_params, i_params, reservoir_capacity: config .property_or_default("spam-filter.classifier.samples.reservoir-capacity", "1024") .unwrap_or(1024), auto_learn_card_is_ham: config .property_or_default("spam-filter.card-is-ham.learn", "true") .unwrap_or(true), auto_learn_reply_ham: config .property_or_default("spam-filter.trusted-reply.learn", "true") .unwrap_or(true), auto_learn_spam_trap: config .property_or_default("spam-filter.classifier.auto-learn.spam-trap", "true") .unwrap_or(true), auto_learn_spam_rbl_count: config .property_or_default("spam-filter.classifier.auto-learn.spam-rbl-count", "2") .unwrap_or(2), hold_samples_for: config .property_or_default::("spam-filter.classifier.samples.hold-for", "180d") .unwrap_or(Duration::from_secs(180 * 24 * 60 * 60)) .as_secs(), min_ham_samples: config .property_or_default("spam-filter.classifier.samples.min-ham", "100") .unwrap_or(100), min_spam_samples: config .property_or_default("spam-filter.classifier.samples.min-spam", "100") .unwrap_or(100), train_frequency: config .property_or_default::>( "spam-filter.classifier.training.frequency", "12h", ) .unwrap_or(Some(Duration::from_secs(12 * 60 * 60))) .map(|d| d.as_secs()), log_scale: config .property_or_default("spam-filter.classifier.features.log-scale", "true") .unwrap_or(true), l2_normalize: config .property_or_default("spam-filter.classifier.features.l2-normalize", "true") .unwrap_or(true), } .into() } } impl FtrlParameters { pub fn parse(config: &mut Config, prefix: &str, default_features: usize) -> Self { let feature_hash_size: usize = config .property((prefix, "features")) .unwrap_or(default_features); if !(16..=28).contains(&feature_hash_size) { config.new_build_error( (prefix, "features"), "Feature size must be between 2^16 and 2^28.", ); } FtrlParameters { feature_hash_size: 1 << feature_hash_size, alpha: config .property_or_default((prefix, "alpha"), "2.0") .unwrap_or(2.0), beta: config .property_or_default((prefix, "beta"), "1.0") .unwrap_or(1.0), l1_ratio: config .property_or_default((prefix, "l1"), "0.001") .unwrap_or(0.001), l2_ratio: config .property_or_default((prefix, "l2"), "0.0001") .unwrap_or(0.0001), } } } impl SpamClassifier { pub fn is_active(&self) -> bool { !matches!(self, SpamClassifier::Disabled) } } impl SpamFilterScoreConfig { pub fn parse(config: &mut Config) -> Self { SpamFilterScoreConfig { reject_threshold: config .property("spam-filter.score.reject") .unwrap_or_default(), discard_threshold: config .property("spam-filter.score.discard") .unwrap_or_default(), spam_threshold: config .property_or_default("spam-filter.score.spam", "5.0") .unwrap_or(5.0), } } } impl ParseValue for Element { fn parse_value(value: &str) -> utils::config::Result { match value { "url" => Ok(Element::Url), "domain" => Ok(Element::Domain), "email" => Ok(Element::Email), "ip" => Ok(Element::Ip), "header" => Ok(Element::Header), "body" => Ok(Element::Body), "any" | "message" => Ok(Element::Any), other => Err(format!("Invalid type {other:?}.",)), } } } impl Location { pub fn as_str(&self) -> &'static str { match self { Location::EnvelopeFrom => "env_from", Location::EnvelopeTo => "env_to", Location::HeaderDkimPass => "dkim_pass", Location::HeaderReceived => "received", Location::HeaderFrom => "from", Location::HeaderReplyTo => "reply_to", Location::HeaderSubject => "subject", Location::HeaderTo => "to", Location::HeaderCc => "cc", Location::HeaderBcc => "bcc", Location::HeaderMid => "message_id", Location::HeaderDnt => "dnt", Location::Ehlo => "ehlo", Location::BodyText => "body_text", Location::BodyHtml => "body_html", Location::Attachment => "attachment", Location::Tcp => "tcp", } } } pub const V_SPAM_REMOTE_IP: u32 = 100; pub const V_SPAM_REMOTE_IP_PTR: u32 = 101; pub const V_SPAM_EHLO_DOMAIN: u32 = 102; pub const V_SPAM_AUTH_AS: u32 = 103; pub const V_SPAM_ASN: u32 = 104; pub const V_SPAM_COUNTRY: u32 = 105; pub const V_SPAM_IS_TLS: u32 = 106; pub const V_SPAM_ENV_FROM: u32 = 108; pub const V_SPAM_ENV_FROM_LOCAL: u32 = 109; pub const V_SPAM_ENV_FROM_DOMAIN: u32 = 110; pub const V_SPAM_ENV_TO: u32 = 111; pub const V_SPAM_FROM: u32 = 112; pub const V_SPAM_FROM_NAME: u32 = 113; pub const V_SPAM_FROM_LOCAL: u32 = 114; pub const V_SPAM_FROM_DOMAIN: u32 = 115; pub const V_SPAM_REPLY_TO: u32 = 116; pub const V_SPAM_REPLY_TO_NAME: u32 = 117; pub const V_SPAM_REPLY_TO_LOCAL: u32 = 118; pub const V_SPAM_REPLY_TO_DOMAIN: u32 = 119; pub const V_SPAM_TO: u32 = 120; pub const V_SPAM_TO_NAME: u32 = 121; pub const V_SPAM_TO_LOCAL: u32 = 122; pub const V_SPAM_TO_DOMAIN: u32 = 123; pub const V_SPAM_CC: u32 = 124; pub const V_SPAM_CC_NAME: u32 = 125; pub const V_SPAM_CC_LOCAL: u32 = 126; pub const V_SPAM_CC_DOMAIN: u32 = 127; pub const V_SPAM_BCC: u32 = 128; pub const V_SPAM_BCC_NAME: u32 = 129; pub const V_SPAM_BCC_LOCAL: u32 = 130; pub const V_SPAM_BCC_DOMAIN: u32 = 131; pub const V_SPAM_BODY_TEXT: u32 = 132; pub const V_SPAM_BODY_HTML: u32 = 133; pub const V_SPAM_BODY_RAW: u32 = 134; pub const V_SPAM_SUBJECT: u32 = 135; pub const V_SPAM_SUBJECT_THREAD: u32 = 136; pub const V_SPAM_LOCATION: u32 = 137; pub const V_WORDS_SUBJECT: u32 = 138; pub const V_WORDS_BODY: u32 = 139; pub const V_RCPT_EMAIL: u32 = 0; pub const V_RCPT_NAME: u32 = 1; pub const V_RCPT_LOCAL: u32 = 2; pub const V_RCPT_DOMAIN: u32 = 3; pub const V_RCPT_DOMAIN_SLD: u32 = 4; pub const V_URL_FULL: u32 = 0; pub const V_URL_PATH_QUERY: u32 = 1; pub const V_URL_PATH: u32 = 2; pub const V_URL_QUERY: u32 = 3; pub const V_URL_SCHEME: u32 = 4; pub const V_URL_AUTHORITY: u32 = 5; pub const V_URL_HOST: u32 = 6; pub const V_URL_HOST_SLD: u32 = 7; pub const V_URL_PORT: u32 = 8; pub const V_HEADER_NAME: u32 = 0; pub const V_HEADER_NAME_LOWER: u32 = 1; pub const V_HEADER_VALUE: u32 = 2; pub const V_HEADER_VALUE_LOWER: u32 = 3; pub const V_HEADER_PROPERTY: u32 = 4; pub const V_HEADER_RAW: u32 = 5; pub const V_HEADER_RAW_LOWER: u32 = 6; pub const V_IP: u32 = 0; pub const V_IP_REVERSE: u32 = 1; pub const V_IP_OCTETS: u32 = 2; pub const V_IP_IS_V4: u32 = 3; pub const V_IP_IS_V6: u32 = 4; impl Element { pub fn token_map(&self) -> TokenMap { let map = TokenMap::default().with_variables_map([ ("remote_ip", V_SPAM_REMOTE_IP), ("remote_ip.ptr", V_SPAM_REMOTE_IP_PTR), ("ehlo_domain", V_SPAM_EHLO_DOMAIN), ("auth_as", V_SPAM_AUTH_AS), ("asn", V_SPAM_ASN), ("country", V_SPAM_COUNTRY), ("is_tls", V_SPAM_IS_TLS), ("env_from", V_SPAM_ENV_FROM), ("env_from.local", V_SPAM_ENV_FROM_LOCAL), ("env_from.domain", V_SPAM_ENV_FROM_DOMAIN), ("env_to", V_SPAM_ENV_TO), ("from", V_SPAM_FROM), ("from.name", V_SPAM_FROM_NAME), ("from.local", V_SPAM_FROM_LOCAL), ("from.domain", V_SPAM_FROM_DOMAIN), ("reply_to", V_SPAM_REPLY_TO), ("reply_to.name", V_SPAM_REPLY_TO_NAME), ("reply_to.local", V_SPAM_REPLY_TO_LOCAL), ("reply_to.domain", V_SPAM_REPLY_TO_DOMAIN), ("to", V_SPAM_TO), ("to.name", V_SPAM_TO_NAME), ("to.local", V_SPAM_TO_LOCAL), ("to.domain", V_SPAM_TO_DOMAIN), ("cc", V_SPAM_CC), ("cc.name", V_SPAM_CC_NAME), ("cc.local", V_SPAM_CC_LOCAL), ("cc.domain", V_SPAM_CC_DOMAIN), ("bcc", V_SPAM_BCC), ("bcc.name", V_SPAM_BCC_NAME), ("bcc.local", V_SPAM_BCC_LOCAL), ("bcc.domain", V_SPAM_BCC_DOMAIN), ("body", V_SPAM_BODY_TEXT), ("body.text", V_SPAM_BODY_TEXT), ("body.html", V_SPAM_BODY_HTML), ("body.words", V_WORDS_BODY), ("body.raw", V_SPAM_BODY_RAW), ("subject", V_SPAM_SUBJECT), ("subject.thread", V_SPAM_SUBJECT_THREAD), ("subject.words", V_WORDS_SUBJECT), ("location", V_SPAM_LOCATION), ]); match self { Element::Url => map.with_variables_map([ ("url", V_URL_FULL), ("value", V_URL_FULL), ("path_query", V_URL_PATH_QUERY), ("path", V_URL_PATH), ("query", V_URL_QUERY), ("scheme", V_URL_SCHEME), ("authority", V_URL_AUTHORITY), ("host", V_URL_HOST), ("sld", V_URL_HOST_SLD), ("port", V_URL_PORT), ]), Element::Email => map.with_variables_map([ ("email", V_RCPT_EMAIL), ("value", V_RCPT_EMAIL), ("name", V_RCPT_NAME), ("local", V_RCPT_LOCAL), ("domain", V_RCPT_DOMAIN), ("sld", V_RCPT_DOMAIN_SLD), ]), Element::Ip => map.with_variables_map([ ("ip", V_IP), ("value", V_IP), ("input", V_IP), ("reverse_ip", V_IP_REVERSE), ("ip_reverse", V_IP_REVERSE), ("octets", V_IP_OCTETS), ("is_v4", V_IP_IS_V4), ("is_v6", V_IP_IS_V6), ]), Element::Header => map.with_variables_map([ ("name", V_HEADER_NAME), ("name_lower", V_HEADER_NAME_LOWER), ("value", V_HEADER_VALUE), ("value_lower", V_HEADER_VALUE_LOWER), ("email", V_HEADER_VALUE), ("email_lower", V_HEADER_VALUE_LOWER), ("attributes", V_HEADER_PROPERTY), ("raw", V_HEADER_RAW), ("raw_lower", V_HEADER_RAW_LOWER), ]), Element::Body | Element::Domain => { map.with_variables_map([("input", 0), ("value", 0), ("result", 0)]) } Element::Any => map, } } pub fn as_str(&self) -> &'static str { match self { Element::Url => "url", Element::Domain => "domain", Element::Email => "email", Element::Ip => "ip", Element::Header => "header", Element::Body => "body", Element::Any => "any", } } } pub struct IpResolver { ip: IpAddr, ip_string: String, reverse: String, octets: Variable<'static>, } impl ResolveVariable for IpResolver { fn resolve_variable(&self, variable: u32) -> Variable<'_> { match variable { V_IP => self.ip_string.as_str().into(), V_IP_REVERSE => self.reverse.as_str().into(), V_IP_OCTETS => self.octets.clone(), V_IP_IS_V4 => Variable::Integer(self.ip.is_ipv4() as _), V_IP_IS_V6 => Variable::Integer(self.ip.is_ipv6() as _), _ => Variable::Integer(0), } } fn resolve_global(&self, _: &str) -> Variable<'_> { Variable::Integer(0) } } impl IpResolver { pub fn new(ip: IpAddr) -> Self { Self { ip_string: ip.to_string(), reverse: ip.to_reverse_name(), octets: Variable::Array(match ip { IpAddr::V4(ipv4_addr) => ipv4_addr .octets() .iter() .map(|o| Variable::Integer(*o as _)) .collect(), IpAddr::V6(ipv6_addr) => ipv6_addr .octets() .iter() .map(|o| Variable::Integer(*o as _)) .collect(), }), ip, } } } impl CacheItemWeight for IpResolver { fn weight(&self) -> u64 { (std::mem::size_of::() + self.ip_string.len() + self.reverse.len()) as u64 } } impl SpamFilterAction { pub fn as_score(&self) -> Option<&T> { match self { SpamFilterAction::Allow(value) => Some(value), _ => None, } } } ================================================ FILE: crates/common/src/config/storage.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::Arc; use ahash::AHashMap; use directory::Directory; use store::{BlobStore, SearchStore, InMemoryStore, PubSubStore, PurgeSchedule, Store}; use crate::manager::config::ConfigManager; #[derive(Default, Clone)] pub struct Storage { pub data: Store, pub blob: BlobStore, pub fts: SearchStore, pub lookup: InMemoryStore, pub pubsub: PubSubStore, pub directory: Arc, pub directories: AHashMap>, pub purge_schedules: Vec, pub config: ConfigManager, pub stores: AHashMap, pub blobs: AHashMap, pub lookups: AHashMap, pub ftss: AHashMap, } ================================================ FILE: crates/common/src/config/telemetry.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use ahash::{AHashMap, AHashSet}; use base64::{Engine, engine::general_purpose::STANDARD}; use hyper::{HeaderMap, header::CONTENT_TYPE}; use opentelemetry::{InstrumentationScope, KeyValue, logs::LoggerProvider}; use opentelemetry_otlp::{ LogExporter, MetricExporter, SpanExporter, WithExportConfig, WithHttpConfig, }; use opentelemetry_sdk::{ Resource, logs::{SdkLogger, SdkLoggerProvider}, metrics::Temporality, }; use opentelemetry_semantic_conventions::resource::SERVICE_VERSION; use std::{collections::HashMap, str::FromStr, sync::Arc, time::Duration}; use store::Stores; use trc::{EventType, Level, TelemetryEvent, ipc::subscriber::Interests}; use utils::config::{Config, http::parse_http_headers, utils::ParseValue}; #[derive(Debug)] pub struct TelemetrySubscriber { pub id: String, pub interests: Interests, pub typ: TelemetrySubscriberType, pub lossy: bool, } #[allow(clippy::large_enum_variant)] #[derive(Debug)] pub enum TelemetrySubscriberType { ConsoleTracer(ConsoleTracer), LogTracer(LogTracer), OtelTracer(OtelTracer), Webhook(WebhookTracer), #[cfg(unix)] JournalTracer(crate::telemetry::tracers::journald::Subscriber), // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] StoreTracer(StoreTracer), // SPDX-SnippetEnd } #[derive(Debug)] pub struct OtelTracer { pub span_exporter: SpanExporter, pub span_exporter_enable: bool, pub log_exporter: LogExporter, pub log_provider: SdkLogger, pub log_exporter_enable: bool, pub throttle: Duration, } pub struct OtelMetrics { pub resource: Resource, pub instrumentation: InstrumentationScope, pub exporter: MetricExporter, pub interval: Duration, } #[derive(Debug)] pub struct ConsoleTracer { pub ansi: bool, pub multiline: bool, pub buffered: bool, } #[derive(Debug)] pub struct LogTracer { pub path: String, pub prefix: String, pub rotate: RotationStrategy, pub ansi: bool, pub multiline: bool, } #[derive(Debug)] pub struct WebhookTracer { pub url: String, pub key: String, pub timeout: Duration, pub throttle: Duration, pub discard_after: Duration, pub tls_allow_invalid_certs: bool, pub headers: HeaderMap, } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[derive(Debug)] #[cfg(feature = "enterprise")] pub struct StoreTracer { pub store: store::Store, } // SPDX-SnippetEnd #[derive(Debug)] pub enum RotationStrategy { Daily, Hourly, Minutely, Never, } #[derive(Debug)] pub struct Telemetry { pub tracers: Tracers, pub metrics: Interests, } #[derive(Debug)] pub struct Tracers { pub interests: Interests, pub levels: AHashMap, pub subscribers: Vec, } #[derive(Debug, Clone, Default)] pub struct Metrics { pub prometheus: Option, pub otel: Option>, pub log_path: Option, } #[derive(Debug, Clone, Default)] pub struct PrometheusMetrics { pub auth: Option, } impl Telemetry { pub fn parse(config: &mut Config, stores: &Stores) -> Self { let mut telemetry = Telemetry { tracers: Tracers::parse(config, stores), metrics: Interests::default(), }; // Parse metrics apply_events( config .properties::("metrics.disabled-events") .into_iter() .map(|(_, e)| e), false, |event_type| { if event_type.is_metric() { telemetry.metrics.set(event_type); } }, ); telemetry } } impl Tracers { pub fn parse(config: &mut Config, stores: &Stores) -> Self { // Parse custom logging levels let mut custom_levels = AHashMap::new(); for event_name in config .prefix("tracing.level") .map(|s| s.to_string()) .collect::>() { if let Some(event_type) = config.try_parse_value::(("tracing.level", &event_name), &event_name) && let Some(level) = config.property_require::(("tracing.level", &event_name)) { custom_levels.insert(event_type, level); } } // Parse tracers let mut tracers: Vec = Vec::new(); let mut global_interests = Interests::default(); for tracer_id in config.sub_keys("tracer", ".type") { let id = tracer_id.as_str(); // Skip disabled tracers if !config .property::(("tracer", id, "enable")) .unwrap_or(true) { continue; } // Parse tracer let typ = match config .value(("tracer", id, "type")) .unwrap_or_default() .to_string() .as_str() { "log" => { if let Some(path) = config .value_require(("tracer", id, "path")) .map(|s| s.to_string()) { TelemetrySubscriberType::LogTracer(LogTracer { path, prefix: config .value(("tracer", id, "prefix")) .unwrap_or("stalwart") .to_string(), rotate: match config.value(("tracer", id, "rotate")).unwrap_or("daily") { "daily" => RotationStrategy::Daily, "hourly" => RotationStrategy::Hourly, "minutely" => RotationStrategy::Minutely, "never" => RotationStrategy::Never, rotate => { let err = format!("Invalid rotation strategy: {rotate}"); config.new_parse_error(("tracer", id, "rotate"), err); RotationStrategy::Daily } }, ansi: config .property_or_default(("tracer", id, "ansi"), "false") .unwrap_or(false), multiline: config .property_or_default(("tracer", id, "multiline"), "false") .unwrap_or(false), }) } else { continue; } } "console" | "stdout" | "stderr" => { if !tracers .iter() .any(|t| matches!(t.typ, TelemetrySubscriberType::ConsoleTracer(_))) { TelemetrySubscriberType::ConsoleTracer(ConsoleTracer { ansi: config .property_or_default(("tracer", id, "ansi"), "true") .unwrap_or(true), multiline: config .property_or_default(("tracer", id, "multiline"), "false") .unwrap_or(false), buffered: config .property_or_default(("tracer", id, "buffered"), "true") .unwrap_or(true), }) } else { config.new_build_error( ("tracer", id, "type"), "Only one console tracer is allowed".to_string(), ); continue; } } "otel" | "open-telemetry" => { let timeout = config .property::(("tracer", id, "timeout")) .unwrap_or(opentelemetry_otlp::OTEL_EXPORTER_OTLP_TIMEOUT_DEFAULT); let throttle = config .property_or_default(("tracer", id, "throttle"), "1s") .unwrap_or_else(|| Duration::from_secs(1)); let log_exporter_enable = config .property_or_default(("tracer", id, "enable.log-exporter"), "true") .unwrap_or(true); let span_exporter_enable = config .property_or_default(("tracer", id, "enable.span-exporter"), "true") .unwrap_or(true); match config .value_require(("tracer", id, "transport")) .unwrap_or_default() { "grpc" => { let mut span_exporter = SpanExporter::builder() .with_tonic() .with_protocol(opentelemetry_otlp::Protocol::Grpc) .with_timeout(timeout); let mut log_exporter = LogExporter::builder() .with_tonic() .with_protocol(opentelemetry_otlp::Protocol::Grpc) .with_timeout(timeout); if let Some(endpoint) = config.value(("tracer", id, "endpoint")) { span_exporter = span_exporter.with_endpoint(endpoint); log_exporter = log_exporter.with_endpoint(endpoint); } match (span_exporter.build(), log_exporter.build()) { (Ok(span_exporter), Ok(log_exporter)) => { TelemetrySubscriberType::OtelTracer(OtelTracer { span_exporter, log_exporter, throttle, span_exporter_enable, log_exporter_enable, log_provider: SdkLoggerProvider::builder() .build() .logger("stalwart"), }) } (Err(err), _) => { config.new_build_error( ("tracer", id), format!( "Failed to build OpenTelemetry span exporter: {err}" ), ); continue; } (_, Err(err)) => { config.new_build_error( ("tracer", id), format!( "Failed to build OpenTelemetry log exporter: {err}" ), ); continue; } } } "http" => { if let Some(endpoint) = config .value_require(("tracer", id, "endpoint")) .map(|s| s.to_string()) { let mut headers = HashMap::new(); let mut err = None; for (_, value) in config.values(("tracer", id, "headers")) { if let Some((key, value)) = value.split_once(':') { headers.insert( key.trim().to_string(), value.trim().to_string(), ); } else { err = format!("Invalid open-telemetry header {value:?}") .into(); break; } } if let Some(err) = err { config.new_parse_error(("tracer", id, "headers"), err); } let mut span_exporter = SpanExporter::builder() .with_http() .with_endpoint(&endpoint) .with_timeout(timeout); let mut log_exporter = LogExporter::builder() .with_http() .with_endpoint(&endpoint) .with_timeout(timeout); if !headers.is_empty() { span_exporter = span_exporter.with_headers(headers.clone()); log_exporter = log_exporter.with_headers(headers); } match (span_exporter.build(), log_exporter.build()) { (Ok(span_exporter), Ok(log_exporter)) => { TelemetrySubscriberType::OtelTracer(OtelTracer { span_exporter, log_exporter, throttle, span_exporter_enable, log_exporter_enable, log_provider: SdkLoggerProvider::builder() .build() .logger("stalwart"), }) } (Err(err), _) => { config.new_build_error( ("tracer", id), format!( "Failed to build OpenTelemetry span exporter: {err}" ), ); continue; } (_, Err(err)) => { config.new_build_error( ("tracer", id), format!( "Failed to build OpenTelemetry log exporter: {err}" ), ); continue; } } } else { continue; } } transport => { let err = format!("Invalid transport: {transport}"); config.new_parse_error(("tracer", id, "transport"), err); continue; } } } "journal" => { #[cfg(unix)] { if !tracers .iter() .any(|t| matches!(t.typ, TelemetrySubscriberType::JournalTracer(_))) { match crate::telemetry::tracers::journald::Subscriber::new() { Ok(subscriber) => { TelemetrySubscriberType::JournalTracer(subscriber) } Err(e) => { config.new_build_error( ("tracer", id, "type"), format!("Failed to create journald subscriber: {e}"), ); continue; } } } else { config.new_build_error( ("tracer", id, "type"), "Only one journal tracer is allowed".to_string(), ); continue; } } #[cfg(not(unix))] { config.new_build_error( ("tracer", id, "type"), "Journald is only available on Unix systems.", ); continue; } } unknown => { config.new_parse_error( ("tracer", id, "type"), format!("Unknown tracer type: {unknown}"), ); continue; } }; // Create tracer let mut tracer = TelemetrySubscriber { id: format!("t_{id}"), interests: Default::default(), lossy: config .property_or_default(("tracer", id, "lossy"), "false") .unwrap_or(false), typ, }; // Parse level let level = Level::from_str(config.value(("tracer", id, "level")).unwrap_or("info")) .map_err(|err| { config.new_parse_error( ("tracer", id, "level"), format!("Invalid log level: {err}"), ) }) .unwrap_or(Level::Info); // Parse disabled events let exclude_event = match &tracer.typ { TelemetrySubscriberType::ConsoleTracer(_) => None, TelemetrySubscriberType::LogTracer(_) => { EventType::Telemetry(TelemetryEvent::LogError).into() } TelemetrySubscriberType::OtelTracer(_) => { EventType::Telemetry(TelemetryEvent::OtelExporterError).into() } TelemetrySubscriberType::Webhook(_) => { EventType::Telemetry(TelemetryEvent::WebhookError).into() } #[cfg(unix)] TelemetrySubscriberType::JournalTracer(_) => { EventType::Telemetry(TelemetryEvent::JournalError).into() } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] TelemetrySubscriberType::StoreTracer(_) => None, // SPDX-SnippetEnd }; // Parse disabled events apply_events( config .properties::(("tracer", id, "disabled-events")) .into_iter() .map(|(_, e)| e), false, |event_type| { if exclude_event != Some(event_type) { let event_level = custom_levels .get(&event_type) .copied() .unwrap_or(event_type.level()); if level.is_contained(event_level) { tracer.interests.set(event_type); global_interests.set(event_type); } } }, ); if !tracer.interests.is_empty() { tracers.push(tracer); } else { config.new_build_warning(("tracer", "id"), "No events enabled for tracer"); } } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL // Parse tracing history #[cfg(feature = "enterprise")] { if config .property_or_default("tracing.history.enable", "false") .unwrap_or(false) && let Some(store_id) = config.value_require("tracing.history.store") { if let Some(store) = stores.stores.get(store_id) { let mut tracer = TelemetrySubscriber { id: "history".to_string(), interests: Default::default(), lossy: false, typ: TelemetrySubscriberType::StoreTracer(StoreTracer { store: store.clone(), }), }; for event_type in StoreTracer::default_events() { tracer.interests.set(event_type); global_interests.set(event_type); } tracers.push(tracer); } else { let err = format!("Store {store_id} not found"); config.new_build_error("tracing.history.store", err); } } } // SPDX-SnippetEnd // Parse webhooks for id in config.sub_keys("webhook", ".url") { if let Some(webhook) = parse_webhook(config, &id, &mut global_interests) { tracers.push(webhook); } } // Add default tracer if none were found #[cfg(not(feature = "test_mode"))] if tracers.is_empty() { for event_type in EventType::variants() { let event_level = custom_levels .get(&event_type) .copied() .unwrap_or(event_type.level()); if Level::Info.is_contained(event_level) { global_interests.set(event_type); } } tracers.push(TelemetrySubscriber { id: "default".to_string(), interests: global_interests.clone(), typ: TelemetrySubscriberType::ConsoleTracer(ConsoleTracer { ansi: true, multiline: false, buffered: true, }), lossy: false, }); } Tracers { subscribers: tracers, interests: global_interests, levels: custom_levels, } } } impl Metrics { pub fn parse(config: &mut Config) -> Self { let mut metrics = Metrics { prometheus: None, otel: None, log_path: None, }; // Obtain log path for tracer_id in config.sub_keys("tracer", ".type") { let tracer_id = tracer_id.as_str(); if config .value(("tracer", tracer_id, "enable")) .unwrap_or("true") == "true" && config .value(("tracer", tracer_id, "type")) .unwrap_or_default() == "log" && let Some(path) = config .value(("tracer", tracer_id, "path")) .map(|s| s.to_string()) { metrics.log_path = Some(path); break; } } if config .property_or_default("metrics.prometheus.enable", "false") .unwrap_or(false) { metrics.prometheus = Some(PrometheusMetrics { auth: config .value("metrics.prometheus.auth.username") .and_then(|user| { config .value("metrics.prometheus.auth.secret") .map(|secret| STANDARD.encode(format!("{user}:{secret}"))) }), }); } let otel_enabled = match config .value("metrics.open-telemetry.transport") .unwrap_or("disable") { "grpc" => true.into(), "http" | "https" => false.into(), "disable" | "disabled" => None, transport => { let err = format!("Invalid transport: {transport}"); config.new_parse_error("metrics.open-telemetry.transport", err); None } }; if let Some(is_grpc) = otel_enabled { let timeout = config .property::("metrics.open-telemetry.timeout") .unwrap_or(opentelemetry_otlp::OTEL_EXPORTER_OTLP_TIMEOUT_DEFAULT); let interval = config .property_or_default("metrics.open-telemetry.interval", "1m") .unwrap_or_else(|| Duration::from_secs(60)); let resource = Resource::builder() .with_service_name("stalwart") .with_attribute(KeyValue::new(SERVICE_VERSION, env!("CARGO_PKG_VERSION"))) .build(); let instrumentation = InstrumentationScope::builder("stalwart") .with_version(env!("CARGO_PKG_VERSION")) .build(); if is_grpc { let mut exporter = MetricExporter::builder() .with_temporality(Temporality::Delta) .with_tonic() .with_protocol(opentelemetry_otlp::Protocol::Grpc) .with_timeout(timeout); if let Some(endpoint) = config.value("metrics.open-telemetry.endpoint") { exporter = exporter.with_endpoint(endpoint); } match exporter.build() { Ok(exporter) => { metrics.otel = Some(Arc::new(OtelMetrics { exporter, interval, resource, instrumentation, })); } Err(err) => { config.new_build_error( "metrics.open-telemetry", format!("Failed to build OpenTelemetry metrics exporter: {err}"), ); } } } else if let Some(endpoint) = config .value_require("metrics.open-telemetry.endpoint") .map(|s| s.to_string()) { let mut headers = HashMap::new(); let mut err = None; for (_, value) in config.values("metrics.open-telemetry.headers") { if let Some((key, value)) = value.split_once(':') { headers.insert(key.trim().to_string(), value.trim().to_string()); } else { err = format!("Invalid open-telemetry header {value:?}").into(); break; } } if let Some(err) = err { config.new_parse_error("metrics.open-telemetry.headers", err); } let mut exporter = MetricExporter::builder() .with_temporality(Temporality::Delta) .with_http() .with_endpoint(&endpoint) .with_timeout(timeout); if !headers.is_empty() { exporter = exporter.with_headers(headers); } match exporter.build() { Ok(exporter) => { metrics.otel = Some(Arc::new(OtelMetrics { exporter, interval, resource, instrumentation, })); } Err(err) => { config.new_build_error( "metrics.open-telemetry", format!("Failed to build OpenTelemetry metrics exporter: {err}"), ); } } } } metrics } } fn parse_webhook( config: &mut Config, id: &str, global_interests: &mut Interests, ) -> Option { let mut headers = parse_http_headers(config, ("webhook", id)); headers.insert(CONTENT_TYPE, "application/json".parse().unwrap()); // Build tracer let mut tracer = TelemetrySubscriber { id: format!("w_{id}"), interests: Default::default(), lossy: config .property_or_default(("webhook", id, "lossy"), "false") .unwrap_or(false), typ: TelemetrySubscriberType::Webhook(WebhookTracer { url: config.value_require(("webhook", id, "url"))?.to_string(), timeout: config .property_or_default(("webhook", id, "timeout"), "30s") .unwrap_or_else(|| Duration::from_secs(30)), tls_allow_invalid_certs: config .property_or_default(("webhook", id, "allow-invalid-certs"), "false") .unwrap_or_default(), headers, key: config .value(("webhook", id, "signature-key")) .unwrap_or_default() .to_string(), throttle: config .property_or_default(("webhook", id, "throttle"), "1s") .unwrap_or_else(|| Duration::from_secs(1)), discard_after: config .property_or_default(("webhook", id, "discard-after"), "5m") .unwrap_or_else(|| Duration::from_secs(300)), }), }; // Parse webhook events apply_events( config .properties::(("webhook", id, "events")) .into_iter() .map(|(_, e)| e), true, |event_type| { if event_type != EventType::Telemetry(TelemetryEvent::WebhookError) { tracer.interests.set(event_type); global_interests.set(event_type); } }, ); if !tracer.interests.is_empty() { Some(tracer) } else { config.new_build_warning(("webhook", id), "No events enabled for webhook"); None } } enum EventOrMany { Event(EventType), StartsWith(String), EndsWith(String), All, } fn apply_events( event_types: impl IntoIterator, inclusive: bool, mut apply_fn: impl FnMut(EventType), ) { let event_names = EventType::variants() .into_iter() .map(|e| (e, e.name())) .collect::>(); let mut exclude_events = AHashSet::new(); for event_or_many in event_types { match event_or_many { EventOrMany::Event(event_type) => { if inclusive { apply_fn(event_type); } else { exclude_events.insert(event_type); } } EventOrMany::StartsWith(value) => { for (event_type, name) in event_names.iter() { if name.starts_with(&value) { if inclusive { apply_fn(*event_type); } else { exclude_events.insert(*event_type); } } } } EventOrMany::EndsWith(value) => { for (event_type, name) in event_names.iter() { if name.ends_with(&value) { if inclusive { apply_fn(*event_type); } else { exclude_events.insert(*event_type); } } } } EventOrMany::All => { for (event_type, _) in event_names.iter() { if inclusive { apply_fn(*event_type); } else { exclude_events.insert(*event_type); } } break; } } } if !inclusive { for (event_type, _) in event_names.iter() { if !exclude_events.contains(event_type) { apply_fn(*event_type); } } } } impl ParseValue for EventOrMany { fn parse_value(value: &str) -> Result { let value = value.trim(); if value == "*" { Ok(EventOrMany::All) } else if let Some(suffix) = value.strip_prefix("*") { Ok(EventOrMany::EndsWith(suffix.to_string())) } else if let Some(prefix) = value.strip_suffix("*") { Ok(EventOrMany::StartsWith(prefix.to_string())) } else { EventType::parse_value(value).map(EventOrMany::Event) } } } impl std::fmt::Debug for OtelMetrics { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("OtelMetrics") .field("interval", &self.interval) .finish() } } ================================================ FILE: crates/common/src/core.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ Inner, Server, auth::{AccessToken, ResourceToken, TenantInfo}, config::{ smtp::{ auth::{ArcSealer, DkimSigner, LazySignature, ResolvedSignature, build_signature}, queue::{ ConnectionStrategy, DEFAULT_QUEUE_NAME, MxConfig, QueueExpiry, QueueName, QueueStrategy, RequireOptional, RoutingStrategy, TlsStrategy, VirtualQueue, }, }, spamfilter::SpamClassifier, }, ipc::{BroadcastEvent, PushEvent, PushNotification}, manager::SPAM_CLASSIFIER_KEY, }; use directory::{Directory, QueryParams, Type, backend::internal::manage::ManageDirectory}; use mail_auth::IpLookupStrategy; use sieve::Sieve; use std::{ sync::{Arc, LazyLock}, time::Duration, }; use store::{ BlobStore, Deserialize, InMemoryStore, IndexKey, IndexKeyPrefix, IterateParams, Key, LogKey, SUBSPACE_LOGS, SearchStore, SerializeInfallible, Store, U32_LEN, U64_LEN, ValueKey, dispatch::DocumentSet, roaring::RoaringBitmap, write::{ AlignedBytes, AnyClass, Archive, AssignedIds, BatchBuilder, BlobLink, BlobOp, DirectoryClass, QueueClass, ValueClass, key::DeserializeBigEndian, now, }, }; use trc::{AddContext, SpamEvent}; use types::{ blob::{BlobClass, BlobId}, blob_hash::BlobHash, collection::{Collection, SyncCollection}, field::Field, type_state::{DataType, StateChange}, }; use utils::{map::bitmap::Bitmap, snowflake::SnowflakeIdGenerator}; impl Server { #[inline(always)] pub fn store(&self) -> &Store { &self.core.storage.data } #[inline(always)] pub fn blob_store(&self) -> &BlobStore { &self.core.storage.blob } #[inline(always)] pub fn search_store(&self) -> &SearchStore { &self.core.storage.fts } #[inline(always)] pub fn in_memory_store(&self) -> &InMemoryStore { &self.core.storage.lookup } #[inline(always)] pub fn directory(&self) -> &Directory { &self.core.storage.directory } pub fn get_directory(&self, name: &str) -> Option<&Arc> { self.core.storage.directories.get(name) } pub fn get_directory_or_default(&self, name: &str, session_id: u64) -> &Arc { self.core.storage.directories.get(name).unwrap_or_else(|| { if !name.is_empty() { trc::event!( Eval(trc::EvalEvent::DirectoryNotFound), Id = name.to_string(), SpanId = session_id, ); } &self.core.storage.directory }) } pub fn get_in_memory_store(&self, name: &str) -> Option<&InMemoryStore> { self.core.storage.lookups.get(name) } pub fn get_in_memory_store_or_default(&self, name: &str, session_id: u64) -> &InMemoryStore { self.core.storage.lookups.get(name).unwrap_or_else(|| { if !name.is_empty() { trc::event!( Eval(trc::EvalEvent::StoreNotFound), Id = name.to_string(), SpanId = session_id, ); } &self.core.storage.lookup }) } pub fn get_data_store(&self, name: &str, session_id: u64) -> &Store { self.core.storage.stores.get(name).unwrap_or_else(|| { if !name.is_empty() { trc::event!( Eval(trc::EvalEvent::StoreNotFound), Id = name.to_string(), SpanId = session_id, ); } &self.core.storage.data }) } pub fn get_arc_sealer(&self, name: &str, session_id: u64) -> Option> { self.resolve_signature(name).map(|s| s.sealer).or_else(|| { trc::event!( Arc(trc::ArcEvent::SealerNotFound), Id = name.to_string(), SpanId = session_id, ); None }) } pub fn get_dkim_signer(&self, name: &str, session_id: u64) -> Option> { self.resolve_signature(name).map(|s| s.signer).or_else(|| { trc::event!( Dkim(trc::DkimEvent::SignerNotFound), Id = name.to_string(), SpanId = session_id, ); None }) } fn resolve_signature(&self, name: &str) -> Option { let lazy_resolver_ = self.core.smtp.mail_auth.signatures.get(name)?; match lazy_resolver_.load().as_ref() { LazySignature::Resolved(resolved_signature) => Some(resolved_signature.clone()), LazySignature::Pending(config) => { let mut config = config.clone(); if let Some((signer, sealer)) = build_signature(&mut config, name) { let resolved = ResolvedSignature { signer: Arc::new(signer), sealer: Arc::new(sealer), }; lazy_resolver_.store(Arc::new(LazySignature::Resolved(resolved.clone()))); Some(resolved) } else { config.log_errors(); lazy_resolver_.store(Arc::new(LazySignature::Failed)); None } } LazySignature::Failed => None, } } pub fn get_trusted_sieve_script(&self, name: &str, session_id: u64) -> Option<&Arc> { self.core.sieve.trusted_scripts.get(name).or_else(|| { trc::event!( Sieve(trc::SieveEvent::ScriptNotFound), Id = name.to_string(), SpanId = session_id, ); None }) } pub fn get_untrusted_sieve_script(&self, name: &str, session_id: u64) -> Option<&Arc> { self.core.sieve.untrusted_scripts.get(name).or_else(|| { trc::event!( Sieve(trc::SieveEvent::ScriptNotFound), Id = name.to_string(), SpanId = session_id, ); None }) } pub fn get_route_or_default(&self, name: &str, session_id: u64) -> &RoutingStrategy { static LOCAL_GATEWAY: RoutingStrategy = RoutingStrategy::Local; static MX_GATEWAY: RoutingStrategy = RoutingStrategy::Mx(MxConfig { max_mx: 5, max_multi_homed: 2, ip_lookup_strategy: IpLookupStrategy::Ipv4thenIpv6, }); self.core .smtp .queue .routing_strategy .get(name) .unwrap_or_else(|| match name { "local" => &LOCAL_GATEWAY, "mx" => &MX_GATEWAY, _ => { trc::event!( Smtp(trc::SmtpEvent::IdNotFound), Id = name.to_string(), Details = "Gateway not found", SpanId = session_id, ); &MX_GATEWAY } }) } pub fn get_virtual_queue_or_default(&self, name: &QueueName) -> &VirtualQueue { static DEFAULT_QUEUE: VirtualQueue = VirtualQueue { threads: 25 }; self.core .smtp .queue .virtual_queues .get(name) .unwrap_or_else(|| { if name != &DEFAULT_QUEUE_NAME { trc::event!( Smtp(trc::SmtpEvent::IdNotFound), Id = name.to_string(), Details = "Virtual queue not found", ); } &DEFAULT_QUEUE }) } pub fn get_queue_or_default(&self, name: &str, session_id: u64) -> &QueueStrategy { static DEFAULT_SCHEDULE: LazyLock = LazyLock::new(|| QueueStrategy { retry: vec![ 120, // 2 minutes 300, // 5 minutes 600, // 10 minutes 900, // 15 minutes 1800, // 30 minutes 3600, // 1 hour 7200, // 2 hours ], notify: vec![ 86400, // 1 day 259200, // 3 days ], expiry: QueueExpiry::Ttl(432000), // 5 days virtual_queue: QueueName::default(), }); self.core .smtp .queue .queue_strategy .get(name) .unwrap_or_else(|| { if name != "default" { trc::event!( Smtp(trc::SmtpEvent::IdNotFound), Id = name.to_string(), Details = "Queue strategy not found", SpanId = session_id, ); } &DEFAULT_SCHEDULE }) } pub fn get_tls_or_default(&self, name: &str, session_id: u64) -> &TlsStrategy { static DEFAULT_TLS: TlsStrategy = TlsStrategy { dane: RequireOptional::Optional, mta_sts: RequireOptional::Optional, tls: RequireOptional::Optional, allow_invalid_certs: false, timeout_tls: Duration::from_secs(3 * 60), timeout_mta_sts: Duration::from_secs(5 * 60), }; self.core .smtp .queue .tls_strategy .get(name) .unwrap_or_else(|| { if name != "default" { trc::event!( Smtp(trc::SmtpEvent::IdNotFound), Id = name.to_string(), Details = "TLS strategy not found", SpanId = session_id, ); } &DEFAULT_TLS }) } pub fn get_connection_or_default(&self, name: &str, session_id: u64) -> &ConnectionStrategy { static DEFAULT_CONNECTION: ConnectionStrategy = ConnectionStrategy { source_ipv4: Vec::new(), source_ipv6: Vec::new(), ehlo_hostname: None, timeout_connect: Duration::from_secs(5 * 60), timeout_greeting: Duration::from_secs(5 * 60), timeout_ehlo: Duration::from_secs(5 * 60), timeout_mail: Duration::from_secs(5 * 60), timeout_rcpt: Duration::from_secs(5 * 60), timeout_data: Duration::from_secs(10 * 60), }; self.core .smtp .queue .connection_strategy .get(name) .unwrap_or_else(|| { if name != "default" { trc::event!( Smtp(trc::SmtpEvent::IdNotFound), Id = name.to_string(), Details = "Connection strategy not found", SpanId = session_id, ); } &DEFAULT_CONNECTION }) } pub async fn get_used_quota(&self, account_id: u32) -> trc::Result { self.core .storage .data .get_counter(DirectoryClass::UsedQuota(account_id)) .await .add_context(|err| err.caused_by(trc::location!()).account_id(account_id)) } pub async fn has_available_quota( &self, quotas: &ResourceToken, item_size: u64, ) -> trc::Result<()> { if quotas.quota != 0 { let used_quota = self.get_used_quota(quotas.account_id).await? as u64; if used_quota + item_size > quotas.quota { return Err(trc::LimitEvent::Quota .into_err() .ctx(trc::Key::Limit, quotas.quota) .ctx(trc::Key::Size, used_quota)); } } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] if self.core.is_enterprise_edition() && let Some(tenant) = quotas.tenant.filter(|tenant| tenant.quota != 0) { let used_quota = self.get_used_quota(tenant.id).await? as u64; if used_quota + item_size > tenant.quota { return Err(trc::LimitEvent::TenantQuota .into_err() .ctx(trc::Key::Limit, tenant.quota) .ctx(trc::Key::Size, used_quota)); } } // SPDX-SnippetEnd Ok(()) } pub async fn get_resource_token( &self, access_token: &AccessToken, account_id: u32, ) -> trc::Result { Ok(if access_token.primary_id == account_id { ResourceToken { account_id, quota: access_token.quota, tenant: access_token.tenant, } } else { let mut quotas = ResourceToken { account_id, ..Default::default() }; if let Some(principal) = self .core .storage .directory .query(QueryParams::id(account_id).with_return_member_of(false)) .await .add_context(|err| err.caused_by(trc::location!()).account_id(account_id))? { quotas.quota = principal.quota().unwrap_or_default(); // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] if self.core.is_enterprise_edition() && let Some(tenant_id) = principal.tenant() { quotas.tenant = TenantInfo { id: tenant_id, quota: self .core .storage .directory .query(QueryParams::id(tenant_id).with_return_member_of(false)) .await .add_context(|err| { err.caused_by(trc::location!()).account_id(tenant_id) })? .and_then(|tenant| tenant.quota()) .unwrap_or_default(), } .into(); } // SPDX-SnippetEnd } quotas }) } pub async fn archives( &self, account_id: u32, collection: Collection, documents: &I, mut cb: CB, ) -> trc::Result<()> where I: DocumentSet + Send + Sync, CB: FnMut(u32, Archive) -> trc::Result + Send + Sync, { let collection: u8 = collection.into(); self.core .storage .data .iterate( IterateParams::new( ValueKey { account_id, collection, document_id: documents.min(), class: ValueClass::Property(Field::ARCHIVE.into()), }, ValueKey { account_id, collection, document_id: documents.max(), class: ValueClass::Property(Field::ARCHIVE.into()), }, ), |key, value| { let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; if documents.contains(document_id) { as Deserialize>::deserialize(value) .and_then(|archive| cb(document_id, archive)) } else { Ok(true) } }, ) .await .add_context(|err| { err.caused_by(trc::location!()) .account_id(account_id) .collection(collection) }) } pub async fn all_archives( &self, account_id: u32, collection: Collection, field: u8, mut cb: CB, ) -> trc::Result<()> where CB: FnMut(u32, Archive) -> trc::Result<()> + Send + Sync, { let collection: u8 = collection.into(); self.core .storage .data .iterate( IterateParams::new( ValueKey { account_id, collection, document_id: 0, class: ValueClass::Property(field), }, ValueKey { account_id, collection, document_id: u32::MAX, class: ValueClass::Property(field), }, ), |key, value| { let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; let archive = as Deserialize>::deserialize(value)?; cb(document_id, archive)?; Ok(true) }, ) .await .add_context(|err| { err.caused_by(trc::location!()) .account_id(account_id) .collection(collection) }) } pub async fn document_ids( &self, account_id: u32, collection: Collection, field: impl Into, ) -> trc::Result { let field = field.into(); let mut results = RoaringBitmap::new(); self.store() .iterate( IterateParams::new( IndexKeyPrefix { account_id, collection: collection.into(), field, }, IndexKeyPrefix { account_id, collection: collection.into(), field: field + 1, }, ) .no_values(), |key, _| { results.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); Ok(true) }, ) .await .caused_by(trc::location!()) .map(|_| results) } pub async fn document_exists( &self, account_id: u32, collection: Collection, field: impl Into, filter: impl AsRef<[u8]>, ) -> trc::Result { let field = field.into(); let mut exists = false; let filter = filter.as_ref(); let key_len = IndexKeyPrefix::len() + filter.len() + U32_LEN; self.store() .iterate( IterateParams::new( IndexKey { account_id, collection: collection.into(), document_id: 0, field, key: filter, }, IndexKey { account_id, collection: collection.into(), document_id: u32::MAX, field, key: filter, }, ) .no_values(), |key, _| { exists = key.len() == key_len; Ok(!exists) }, ) .await .caused_by(trc::location!()) .map(|_| exists) } pub async fn document_ids_matching( &self, account_id: u32, collection: Collection, field: impl Into, filter: impl AsRef<[u8]>, ) -> trc::Result { let field = field.into(); let filter = filter.as_ref(); let key_len = IndexKeyPrefix::len() + filter.len() + U32_LEN; let mut results = RoaringBitmap::new(); self.store() .iterate( IterateParams::new( IndexKey { account_id, collection: collection.into(), document_id: 0, field, key: filter, }, IndexKey { account_id, collection: collection.into(), document_id: u32::MAX, field, key: filter, }, ) .no_values(), |key, _| { if key.len() == key_len { results.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); } Ok(true) }, ) .await .caused_by(trc::location!()) .map(|_| results) } #[inline(always)] pub fn notify_task_queue(&self) { self.inner.ipc.task_tx.notify_one(); } pub async fn total_queued_messages(&self) -> trc::Result { let mut total = 0; self.store() .iterate( IterateParams::new( ValueKey::from(ValueClass::Queue(QueueClass::Message(0))), ValueKey::from(ValueClass::Queue(QueueClass::Message(u64::MAX))), ) .no_values(), |_, _| { total += 1; Ok(true) }, ) .await .caused_by(trc::location!()) .map(|_| total) } #[inline(always)] pub fn generate_snowflake_id(&self) -> u64 { self.inner.data.jmap_id_gen.generate() } pub async fn commit_batch(&self, mut builder: BatchBuilder) -> trc::Result { let mut assigned_ids = AssignedIds::default(); let mut commit_points = builder.commit_points(); for commit_point in commit_points.iter() { let batch = builder.build_one(commit_point); assigned_ids .ids .extend(self.store().write(batch).await?.ids); } if let Some(changes) = builder.changes() { for (account_id, changed_collections) in changes { let mut state_change = StateChange::new(account_id); for changed_collection in changed_collections.changed_containers { if let Some(data_type) = DataType::try_from_sync(changed_collection, true) { state_change.set_change(data_type); } } for changed_collection in changed_collections.changed_items { if let Some(data_type) = DataType::try_from_sync(changed_collection, false) { state_change.set_change(data_type); } } if state_change.has_changes() { self.broadcast_push_notification(PushNotification::StateChange( state_change.with_change_id(assigned_ids.last_change_id(account_id)?), )) .await; } if let Some(change_id) = changed_collections.share_notification_id { self.broadcast_push_notification(PushNotification::StateChange(StateChange { account_id, change_id, types: Bitmap::from_iter([DataType::ShareNotification]), })) .await; } } } Ok(assigned_ids) } pub async fn delete_changes( &self, account_id: u32, max_entries: Option, max_duration: Option, ) -> trc::Result<()> { if let Some(max_entries) = max_entries { for sync_collection in [ SyncCollection::Email, SyncCollection::Thread, SyncCollection::Identity, SyncCollection::EmailSubmission, SyncCollection::SieveScript, SyncCollection::FileNode, SyncCollection::AddressBook, SyncCollection::Calendar, SyncCollection::CalendarEventNotification, ] { let collection = sync_collection.into(); let from_key = LogKey { account_id, collection, change_id: 0, }; let to_key = LogKey { account_id, collection, change_id: u64::MAX, }; let mut first_change_id = 0; let mut num_changes = 0; self.store() .iterate( IterateParams::new(from_key, to_key) .descending() .no_values(), |key, _| { first_change_id = key.deserialize_be_u64(key.len() - U64_LEN)?; num_changes += 1; Ok(num_changes <= max_entries) }, ) .await .caused_by(trc::location!())?; if num_changes > max_entries { self.store() .delete_range( LogKey { account_id, collection, change_id: 0, }, LogKey { account_id, collection, change_id: first_change_id, }, ) .await .caused_by(trc::location!())?; // Delete vanished items if let Some(vanished_collection) = sync_collection.vanished_collection().map(u8::from) { self.store() .delete_range( LogKey { account_id, collection: vanished_collection, change_id: 0, }, LogKey { account_id, collection: vanished_collection, change_id: first_change_id, }, ) .await .caused_by(trc::location!())?; } // Write truncation entry for cache let mut batch = BatchBuilder::new(); batch.with_account_id(account_id).set( ValueClass::Any(AnyClass { subspace: SUBSPACE_LOGS, key: LogKey { account_id, collection, change_id: first_change_id, } .serialize(0), }), Vec::new(), ); self.store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } } } if let Some(max_duration) = max_duration { self.store() .delete_range( LogKey { account_id, collection: SyncCollection::ShareNotification.into(), change_id: 0, }, LogKey { account_id, collection: SyncCollection::ShareNotification.into(), change_id: SnowflakeIdGenerator::from_duration(max_duration) .unwrap_or_default(), }, ) .await .caused_by(trc::location!())?; } Ok(()) } pub async fn broadcast_push_notification(&self, notification: PushNotification) -> bool { match self .inner .ipc .push_tx .clone() .send(PushEvent::Publish { notification, broadcast: true, }) .await { Ok(_) => true, Err(_) => { trc::event!( Server(trc::ServerEvent::ThreadError), Details = "Error sending state change.", CausedBy = trc::location!() ); false } } } pub async fn cluster_broadcast(&self, event: BroadcastEvent) { if let Some(broadcast_tx) = &self.inner.ipc.broadcast_tx.clone() && broadcast_tx.send(event).await.is_err() { trc::event!( Server(trc::ServerEvent::ThreadError), Details = "Error sending broadcast event.", CausedBy = trc::location!() ); } } #[allow(clippy::blocks_in_conditions)] pub async fn put_jmap_blob(&self, account_id: u32, data: &[u8]) -> trc::Result { // First reserve the hash let hash = BlobHash::generate(data); let mut batch = BatchBuilder::new(); let until = now() + self.core.jmap.upload_tmp_ttl; batch .with_account_id(account_id) .set( BlobOp::Link { hash: hash.clone(), to: BlobLink::Temporary { until }, }, vec![BlobLink::QUOTA_LINK], ) .set( BlobOp::Quota { hash: hash.clone(), until, }, (data.len() as u32).serialize(), ); self.core .storage .data .write(batch.build_all()) .await .caused_by(trc::location!())?; if !self .core .storage .data .blob_exists(&hash) .await .caused_by(trc::location!())? { // Upload blob to store self.core .storage .blob .put_blob(hash.as_ref(), data) .await .caused_by(trc::location!())?; // Commit blob let mut batch = BatchBuilder::new(); batch.set(BlobOp::Commit { hash: hash.clone() }, Vec::new()); self.core .storage .data .write(batch.build_all()) .await .caused_by(trc::location!())?; } Ok(BlobId { hash, class: BlobClass::Reserved { account_id, expires: until, }, section: None, }) } pub async fn put_temporary_blob( &self, account_id: u32, data: &[u8], hold_for: u64, ) -> trc::Result<(BlobHash, BlobOp)> { // First reserve the hash let hash = BlobHash::generate(data); let mut batch = BatchBuilder::new(); let until = now() + hold_for; batch.with_account_id(account_id).set( BlobOp::Link { hash: hash.clone(), to: BlobLink::Temporary { until }, }, vec![], ); self.core .storage .data .write(batch.build_all()) .await .caused_by(trc::location!())?; if !self .core .storage .data .blob_exists(&hash) .await .caused_by(trc::location!())? { // Upload blob to store self.core .storage .blob .put_blob(hash.as_ref(), data) .await .caused_by(trc::location!())?; // Commit blob let mut batch = BatchBuilder::new(); batch.set(BlobOp::Commit { hash: hash.clone() }, Vec::new()); self.core .storage .data .write(batch.build_all()) .await .caused_by(trc::location!())?; } Ok(( hash.clone(), BlobOp::Link { hash, to: BlobLink::Temporary { until }, }, )) } pub async fn total_accounts(&self) -> trc::Result { self.store() .count_principals(None, Type::Individual.into(), None) .await .caused_by(trc::location!()) } pub async fn total_domains(&self) -> trc::Result { self.store() .count_principals(None, Type::Domain.into(), None) .await .caused_by(trc::location!()) } pub async fn spam_model_reload(&self) -> trc::Result<()> { if self.core.spam.classifier.is_some() { if let Some(model) = self .blob_store() .get_blob(SPAM_CLASSIFIER_KEY, 0..usize::MAX) .await .and_then(|archive| match archive { Some(archive) => as Deserialize>::deserialize(&archive) .and_then(|archive| archive.deserialize_untrusted::()) .map(Some), None => Ok(None), }) .caused_by(trc::location!())? { self.inner.data.spam_classifier.store(Arc::new(model)); } else { trc::event!(Spam(SpamEvent::ModelNotFound)); } } Ok(()) } #[cfg(not(feature = "enterprise"))] pub async fn logo_resource( &self, _: &str, ) -> trc::Result>>> { Ok(None) } } pub trait BuildServer { fn build_server(&self) -> Server; } impl BuildServer for Arc { fn build_server(&self) -> Server { Server { inner: self.clone(), core: self.shared_core.load_full(), } } } ================================================ FILE: crates/common/src/dns.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::net::IpAddr; use mail_auth::{Error, IpLookupStrategy}; use crate::Server; impl Server { pub async fn dns_exists_mx(&self, entry: &str) -> trc::Result { match self .core .smtp .resolvers .dns .mx_lookup(entry, Some(&self.inner.cache.dns_mx)) .await { Ok(result) => Ok(result.iter().any(|mx| !mx.exchanges.is_empty())), Err(Error::DnsRecordNotFound(_)) => Ok(false), Err(err) => Err(err.into()), } } pub async fn dns_exists_ip(&self, entry: &str) -> trc::Result { match self .core .smtp .resolvers .dns .ip_lookup( entry, IpLookupStrategy::Ipv4thenIpv6, 10, Some(&self.inner.cache.dns_ipv4), Some(&self.inner.cache.dns_ipv6), ) .await { Ok(result) => Ok(!result.is_empty()), Err(Error::DnsRecordNotFound(_)) => Ok(false), Err(err) => Err(err.into()), } } pub async fn dns_exists_ptr(&self, entry: &str) -> trc::Result { if let Ok(addr) = entry.parse::() { match self .core .smtp .resolvers .dns .ptr_lookup(addr, Some(&self.inner.cache.dns_ptr)) .await { Ok(result) => Ok(!result.is_empty()), Err(Error::DnsRecordNotFound(_)) => Ok(false), Err(err) => Err(err.into()), } } else { Err(trc::EventType::Resource(trc::ResourceEvent::BadParameters).into_err()) } } pub async fn dns_exists_ipv4(&self, entry: &str) -> trc::Result { match self .core .smtp .resolvers .dns .ipv4_lookup(entry, Some(&self.inner.cache.dns_ipv4)) .await { Ok(result) => Ok(!result.is_empty()), Err(Error::DnsRecordNotFound(_)) => Ok(false), Err(err) => Err(err.into()), } } pub async fn dns_exists_ipv6(&self, entry: &str) -> trc::Result { match self .core .smtp .resolvers .dns .ipv6_lookup(entry, Some(&self.inner.cache.dns_ipv6)) .await { Ok(result) => Ok(!result.is_empty()), Err(Error::DnsRecordNotFound(_)) => Ok(false), Err(err) => Err(err.into()), } } } ================================================ FILE: crates/common/src/enterprise/alerts.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: LicenseRef-SEL * * This file is subject to the Stalwart Enterprise License Agreement (SEL) and * is NOT open source software. * */ use mail_builder::{ MessageBuilder, headers::{ HeaderType, address::{Address, EmailAddress}, }, }; use trc::{Collector, MetricType, TOTAL_EVENT_COUNT, TelemetryEvent}; use super::{AlertContent, AlertContentToken, AlertMethod}; use crate::{ Server, expr::{Variable, functions::ResolveVariable}, }; use std::fmt::Write; #[derive(Debug, PartialEq, Eq)] pub struct AlertMessage { pub from: String, pub to: Vec, pub body: Vec, } struct CollectorResolver; impl Server { pub async fn process_alerts(&self) -> Option> { let alerts = &self.core.enterprise.as_ref()?.metrics_alerts; if alerts.is_empty() { return None; } let mut messages = Vec::new(); for alert in alerts { if !self .eval_expr(&alert.condition, &CollectorResolver, &alert.id, 0) .await .unwrap_or(false) { continue; } for method in &alert.method { match method { AlertMethod::Email { from_name, from_addr, to, subject, body, } => { messages.push(AlertMessage { from: from_addr.clone(), to: to.clone(), body: MessageBuilder::new() .from(Address::Address(EmailAddress { name: from_name.as_ref().map(|s| s.into()), email: from_addr.as_str().into(), })) .header( "To", HeaderType::Address(Address::List( to.iter() .map(|to| { Address::Address(EmailAddress { name: None, email: to.as_str().into(), }) }) .collect(), )), ) .header("Auto-Submitted", HeaderType::Text("auto-generated".into())) .subject(subject.build()) .text_body(body.build()) .write_to_vec() .unwrap_or_default(), }); } AlertMethod::Event { message } => { trc::event!( Telemetry(TelemetryEvent::Alert), Id = alert.id.to_string(), Details = message.as_ref().map(|m| m.build()) ); #[cfg(feature = "test_mode")] Collector::update_event_counter( trc::EventType::Telemetry(TelemetryEvent::Alert), 1, ); } } } } (!messages.is_empty()).then_some(messages) } } impl ResolveVariable for CollectorResolver { fn resolve_variable(&self, variable: u32) -> Variable<'_> { if (variable as usize) < TOTAL_EVENT_COUNT { Variable::Integer(Collector::read_event_metric(variable as usize) as i64) } else if let Some(metric_type) = MetricType::from_code(variable as u64 - TOTAL_EVENT_COUNT as u64) { Variable::Float(Collector::read_metric(metric_type)) } else { Variable::Integer(0) } } fn resolve_global(&self, _: &str) -> Variable<'_> { Variable::Integer(0) } } impl AlertContent { pub fn build(&self) -> String { let mut buf = String::with_capacity(self.len()); for token in &self.0 { token.write(&mut buf); } buf } #[allow(clippy::len_without_is_empty)] pub fn len(&self) -> usize { self.0.iter().map(|t| t.len()).sum() } } impl AlertContentToken { fn write(&self, buf: &mut String) { match self { AlertContentToken::Text(text) => buf.push_str(text), AlertContentToken::Metric(metric_type) => { let _ = write!(buf, "{}", Collector::read_metric(*metric_type)); } AlertContentToken::Event(event_type) => { let _ = write!(buf, "{}", Collector::read_event_metric(event_type.id())); } } } fn len(&self) -> usize { match self { AlertContentToken::Text(s) => s.len(), AlertContentToken::Metric(_) | AlertContentToken::Event(_) => 10, } } } ================================================ FILE: crates/common/src/enterprise/config.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: LicenseRef-SEL * * This file is subject to the Stalwart Enterprise License Agreement (SEL) and * is NOT open source software. * */ use super::{ AlertContent, AlertContentToken, AlertMethod, Enterprise, MetricAlert, MetricStore, SpamFilterLlmConfig, TraceStore, Undelete, license::LicenseKey, llm::AiApiConfig, }; use crate::{ expr::{Expression, tokenizer::TokenMap}, manager::config::ConfigManager, }; use ahash::AHashMap; use directory::{Type, backend::internal::manage::ManageDirectory}; use std::{sync::Arc, time::Duration}; use store::{Store, Stores}; use trc::{EventType, MetricType, TOTAL_EVENT_COUNT}; use utils::{ config::{ Config, ConfigKey, cron::SimpleCron, utils::{AsKey, ParseValue}, }, template::Template, }; impl Enterprise { pub async fn parse( config: &mut Config, config_manager: &ConfigManager, stores: &Stores, data: &Store, ) -> Option { let server_hostname = config .value("server.hostname") .or_else(|| config.value("lookup.default.hostname"))?; let mut update_license = None; let license_result = match ( config.value("enterprise.license-key"), config.value("enterprise.api-key"), ) { (Some(license_key), maybe_api_key) => { match (LicenseKey::new(license_key, server_hostname), maybe_api_key) { (Ok(license), Some(api_key)) if license.is_near_expiration() => Ok(license .try_renew(api_key) .await .map(|result| { update_license = Some(result.encoded_key); result.key }) .unwrap_or(license)), (Ok(license), None) => Ok(license), (Err(_), Some(api_key)) => LicenseKey::invalid(server_hostname) .try_renew(api_key) .await .map(|result| { update_license = Some(result.encoded_key); result.key }), (maybe_license, _) => maybe_license, } } (None, Some(api_key)) => LicenseKey::invalid(server_hostname) .try_renew(api_key) .await .map(|result| { update_license = Some(result.encoded_key); result.key }), (None, None) => { return None; } }; // Report error let license = match license_result { Ok(license) => license, Err(err) => { config.new_build_warning("enterprise.license-key", err.to_string()); return None; } }; // Update the license if a new one was obtained if let Some(license) = update_license { config .keys .insert("enterprise.license-key".to_string(), license.clone()); if let Err(err) = config_manager .set( [ConfigKey { key: "enterprise.license-key".to_string(), value: license.to_string(), }], true, ) .await { trc::error!( err.caused_by(trc::location!()) .details("Failed to update license key") ); } } match data .count_principals(None, Type::Individual.into(), None) .await { Ok(total) if total > license.accounts as u64 => { config.new_build_warning( "enterprise.license-key", format!( "License key is valid but only allows {} accounts, found {}.", license.accounts, total ), ); return None; } Err(e) => { if !matches!(data, Store::None) { config.new_build_error("enterprise.license-key", e.to_string()); } return None; } _ => (), } let trace_store = if config .property_or_default("tracing.history.enable", "false") .unwrap_or(false) { if let Some(store) = config .value("tracing.history.store") .and_then(|name| stores.stores.get(name)) .cloned() { TraceStore { retention: config .property_or_default::>("tracing.history.retention", "30d") .unwrap_or(Some(Duration::from_secs(30 * 24 * 60 * 60))), store, } .into() } else { None } } else { None }; let metrics_store = if config .property_or_default("metrics.history.enable", "false") .unwrap_or(false) { if let Some(store) = config .value("metrics.history.store") .and_then(|name| stores.stores.get(name)) .cloned() { MetricStore { retention: config .property_or_default::>("metrics.history.retention", "90d") .unwrap_or(Some(Duration::from_secs(90 * 24 * 60 * 60))), store, interval: config .property_or_default::("metrics.history.interval", "0 * *") .unwrap_or_else(|| SimpleCron::parse_value("0 * *").unwrap()), } .into() } else { None } } else { None }; // Parse AI APIs let mut ai_apis = AHashMap::new(); for id in config.sub_keys("enterprise.ai", ".url") { if let Some(api) = AiApiConfig::parse(config, &id) { ai_apis.insert(id, api.into()); } } // Build the enterprise configuration let mut enterprise = Enterprise { license, undelete: config .property_or_default::>("storage.undelete.retention", "false") .unwrap_or_default() .map(|retention| Undelete { retention }), logo_url: config.value("enterprise.logo-url").map(|s| s.to_string()), trace_store, metrics_store, metrics_alerts: parse_metric_alerts(config), spam_filter_llm: SpamFilterLlmConfig::parse(config, &ai_apis), ai_apis, template_calendar_alarm: None, template_scheduling_email: None, template_scheduling_web: None, }; // Parse templates for (key, value) in [ ( "calendar.alarms.template", &mut enterprise.template_calendar_alarm, ), ( "calendar.scheduling.template.email", &mut enterprise.template_scheduling_email, ), ( "calendar.scheduling.template.web", &mut enterprise.template_scheduling_web, ), ] { if let Some(template) = config.value(key) { match Template::parse(template) { Ok(template) => *value = Some(template), Err(err) => { config.new_build_error(key, format!("Invalid template: {err}")); } } } } Some(enterprise) } } impl SpamFilterLlmConfig { pub fn parse(config: &mut Config, models: &AHashMap>) -> Option { if !config .property_or_default::("spam-filter.llm.enable", "false") .unwrap_or_default() { return None; } let model = config.value_require_non_empty("spam-filter.llm.model")?; let model = if let Some(model) = models.get(model) { model.clone() } else { let message = format!("Model {model:?} not found in AI API configuration"); config.new_build_error("spam-filter.llm.model", message); return None; }; let llm = SpamFilterLlmConfig { model, temperature: config .property_or_default("spam-filter.llm.temperature", "0.5") .unwrap_or(0.5), prompt: config .value_require_non_empty("spam-filter.llm.prompt")? .to_string(), separator: config .value_require_non_empty("spam-filter.llm.separator") .unwrap_or_default() .chars() .next() .unwrap_or(','), index_category: config .property("spam-filter.llm.index.category") .unwrap_or_default(), index_confidence: config.property("spam-filter.llm.index.confidence"), index_explanation: config.property("spam-filter.llm.index.explanation"), categories: config .values("spam-filter.llm.categories") .map(|(_, v)| v.trim().to_uppercase()) .collect(), confidence: config .values("spam-filter.llm.confidence") .map(|(_, v)| v.trim().to_uppercase()) .collect(), }; if llm.categories.is_empty() { config.new_build_error("spam-filter.llm.categories", "No categories defined"); return None; } if llm.index_confidence.is_some() && llm.confidence.is_empty() { config.new_build_error( "spam-filter.llm.confidence", "Confidence index is defined but no confidence values are provided", ); return None; } llm.into() } } pub fn parse_metric_alerts(config: &mut Config) -> Vec { let mut alerts = Vec::new(); for metric_id in config.sub_keys("metrics.alerts", ".enable") { if let Some(alert) = parse_metric_alert(config, metric_id) { alerts.push(alert); } } alerts } fn parse_metric_alert(config: &mut Config, id: String) -> Option { if !config.property_or_default::(("metrics.alerts", id.as_str(), "enable"), "false")? { return None; } let mut alert = MetricAlert { condition: Expression::try_parse( config, ("metrics.alerts", id.as_str(), "condition"), &TokenMap::default().with_variables_map( EventType::variants() .into_iter() .map(|e| (sanitize_metric_name(e.name()), e.id() as u32)) .chain(MetricType::variants().iter().map(|m| { ( sanitize_metric_name(m.name()), m.code() as u32 + TOTAL_EVENT_COUNT as u32, ) })), ), )?, method: Vec::new(), id, }; let id_str = alert.id.as_str(); if config .property_or_default::(("metrics.alerts", id_str, "notify.event.enable"), "false") .unwrap_or_default() { alert.method.push(AlertMethod::Event { message: parse_alert_content( ("metrics.alerts", id_str, "notify.event.message"), config, ), }); } if config .property_or_default::(("metrics.alerts", id_str, "notify.email.enable"), "false") .unwrap_or_default() { let from_addr = config .value_require(("metrics.alerts", id_str, "notify.email.from-addr"))? .trim() .to_string(); let from_name = config .value(("metrics.alerts", id_str, "notify.email.from-name")) .map(|s| s.to_string()); let to = config .values(("metrics.alerts", id_str, "notify.email.to")) .filter_map(|(_, s)| { if s.contains('@') { s.trim().to_string().into() } else { None } }) .collect::>(); let subject = parse_alert_content(("metrics.alerts", id_str, "notify.email.subject"), config)?; let body = parse_alert_content(("metrics.alerts", id_str, "notify.email.body"), config)?; if !from_addr.contains('@') { config.new_build_error( ("metrics.alerts", id_str, "notify.email.from-addr"), "Invalid from email address", ); } if to.is_empty() { config.new_build_error( ("metrics.alerts", id_str, "notify.email.to"), "Missing recipient address(es)", ); } if subject.0.is_empty() { config.new_build_error( ("metrics.alerts", id_str, "notify.email.subject"), "Missing email subject", ); } if body.0.is_empty() { config.new_build_error( ("metrics.alerts", id_str, "notify.email.body"), "Missing email body", ); } alert.method.push(AlertMethod::Email { from_name, from_addr, to, subject, body, }); } if alert.method.is_empty() { config.new_build_error( ("metrics.alerts", id_str), "No notification method enabled for alert", ); } alert.into() } fn parse_alert_content(key: impl AsKey, config: &mut Config) -> Option { let mut tokens = Vec::new(); let mut value = config.value(key)?.chars().peekable(); let mut buf = String::new(); while let Some(ch) = value.next() { if ch == '%' && value.peek() == Some(&'{') { value.next(); let mut var_name = String::new(); let mut found_curly = false; for ch in value.by_ref() { if ch == '}' { found_curly = true; break; } var_name.push(ch); } if found_curly && value.peek() == Some(&'%') { value.next(); if let Some(event_type) = EventType::try_parse(&var_name) .map(AlertContentToken::Event) .or_else(|| MetricType::try_parse(&var_name).map(AlertContentToken::Metric)) { if !buf.is_empty() { tokens.push(AlertContentToken::Text(std::mem::take(&mut buf))); } tokens.push(event_type); } else { buf.push('%'); buf.push('{'); buf.push_str(&var_name); buf.push('}'); buf.push('%'); } } else { buf.push('%'); buf.push('{'); buf.push_str(&var_name); } } else { buf.push(ch); } } if !buf.is_empty() { tokens.push(AlertContentToken::Text(buf)); } AlertContent(tokens).into() } fn sanitize_metric_name(name: &str) -> String { let mut result = String::with_capacity(name.len()); for ch in name.chars() { if ch.is_ascii_alphanumeric() { result.push(ch); } else { result.push('_'); } } result } ================================================ FILE: crates/common/src/enterprise/license.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: LicenseRef-SEL * * This file is subject to the Stalwart Enterprise License Agreement (SEL) and * is NOT open source software. * */ /* * WARNING: TAMPERING WITH THIS CODE IS STRICTLY PROHIBITED * Any attempt to modify, bypass, or disable the license validation mechanism * constitutes a severe violation of the Stalwart Enterprise License Agreement. * Such actions may result in immediate termination of your license, legal action, * and substantial financial penalties. Stalwart Labs LLC actively monitors for * unauthorized modifications and will pursue all available legal remedies against * violators to the fullest extent of the law, including but not limited to claims * for copyright infringement, breach of contract, and fraud. */ use crate::manager::fetch_resource; use base64::{Engine, engine::general_purpose::STANDARD}; use hyper::{HeaderMap, header::AUTHORIZATION}; use ring::signature::{ED25519, UnparsedPublicKey}; use std::{ fmt::{Display, Formatter}, time::Duration, }; use store::write::now; use trc::ServerEvent; //const LICENSING_API: &str = "https://localhost:444/api/license/"; const LICENSING_API: &str = "https://license.stalw.art/api/license/"; const RENEW_THRESHOLD: u64 = 60 * 60 * 24 * 4; // 4 days pub struct LicenseValidator { public_key: UnparsedPublicKey>, } #[derive(Debug, Clone)] pub struct LicenseKey { pub valid_to: u64, pub valid_from: u64, pub domain: String, pub accounts: u32, } #[derive(Debug)] pub enum LicenseError { Expired, InvalidDomain { domain: String }, DomainMismatch { issued_to: String, current: String }, Parse, Validation, Decode, InvalidParameters, RenewalFailed { reason: String }, } pub struct RenewedLicense { pub key: LicenseKey, pub encoded_key: String, } const U64_LEN: usize = std::mem::size_of::(); const U32_LEN: usize = std::mem::size_of::(); impl LicenseValidator { #[allow(clippy::new_without_default)] pub fn new() -> Self { LicenseValidator { public_key: UnparsedPublicKey::new( &ED25519, vec![ 118, 10, 182, 35, 89, 111, 11, 60, 154, 47, 205, 127, 107, 229, 55, 104, 72, 54, 141, 14, 97, 219, 2, 4, 119, 143, 156, 10, 152, 216, 32, 194, ], ), } } pub fn try_parse(&self, key: impl AsRef) -> Result { let key = STANDARD .decode(key.as_ref()) .map_err(|_| LicenseError::Decode)?; let valid_from = u64::from_le_bytes( key.get(..U64_LEN) .ok_or(LicenseError::Parse)? .try_into() .unwrap(), ); let valid_to = u64::from_le_bytes( key.get(U64_LEN..(U64_LEN * 2)) .ok_or(LicenseError::Parse)? .try_into() .unwrap(), ); let accounts = u32::from_le_bytes( key.get((U64_LEN * 2)..(U64_LEN * 2) + U32_LEN) .ok_or(LicenseError::Parse)? .try_into() .unwrap(), ); let domain_len = u32::from_le_bytes( key.get((U64_LEN * 2) + U32_LEN..(U64_LEN * 2) + (U32_LEN * 2)) .ok_or(LicenseError::Parse)? .try_into() .unwrap(), ) as usize; let domain = String::from_utf8( key.get((U64_LEN * 2) + (U32_LEN * 2)..(U64_LEN * 2) + (U32_LEN * 2) + domain_len) .ok_or(LicenseError::Parse)? .to_vec(), ) .map_err(|_| LicenseError::Parse)?; let signature = key .get((U64_LEN * 2) + (U32_LEN * 2) + domain_len..) .ok_or(LicenseError::Parse)?; if valid_from == 0 || valid_to == 0 || valid_from >= valid_to || accounts == 0 || domain.is_empty() { return Err(LicenseError::InvalidParameters); } // Validate signature self.public_key .verify( &key[..(U64_LEN * 2) + (U32_LEN * 2) + domain_len], signature, ) .map_err(|_| LicenseError::Validation)?; let key = LicenseKey { valid_from, valid_to, domain, accounts, }; if !key.is_expired() { Ok(key) } else { Err(LicenseError::Expired) } } } impl LicenseKey { pub fn new( license_key: impl AsRef, hostname: impl AsRef, ) -> Result { LicenseValidator::new() .try_parse(license_key) .and_then(|key| { let local_domain = Self::base_domain(hostname)?; let license_domain = Self::base_domain(&key.domain)?; if local_domain == license_domain { Ok(key) } else { Err(LicenseError::DomainMismatch { issued_to: license_domain, current: local_domain, }) } }) } pub fn invalid(domain: impl AsRef) -> Self { LicenseKey { valid_from: 0, valid_to: 0, domain: Self::base_domain(domain).unwrap_or_default(), accounts: 0, } } pub async fn try_renew(&self, api_key: &str) -> Result { let mut headers = HeaderMap::new(); headers.insert( AUTHORIZATION, format!("Bearer {api_key}") .parse() .map_err(|_| LicenseError::Validation)?, ); trc::event!( Server(ServerEvent::Licensing), Details = "Attempting to renew Enterprise license from license.stalw.art", ); match fetch_resource( &format!("{}{}", LICENSING_API, self.domain), headers.into(), Duration::from_secs(60), 1024, ) .await .and_then(|bytes| { String::from_utf8(bytes) .map_err(|_| String::from("Failed to UTF-8 decode server response")) }) { Ok(encoded_key) => match LicenseKey::new(&encoded_key, &self.domain) { Ok(key) => Ok(RenewedLicense { key, encoded_key }), Err(err) => { trc::event!( Server(ServerEvent::Licensing), Details = "Failed to decode license renewal", Reason = err.to_string(), ); Err(err) } }, Err(err) => { trc::event!( Server(ServerEvent::Licensing), Details = "Failed to renew Enterprise license", Reason = err.clone(), ); Err(LicenseError::RenewalFailed { reason: err }) } } } pub fn is_near_expiration(&self) -> bool { let now = now(); self.valid_to.saturating_sub(now) <= RENEW_THRESHOLD } pub fn expires_in(&self) -> Duration { Duration::from_secs(self.valid_to.saturating_sub(now())) } pub fn renew_in(&self) -> Duration { Duration::from_secs(self.valid_to.saturating_sub(now() + RENEW_THRESHOLD)) } pub fn is_expired(&self) -> bool { let now = now(); now >= self.valid_to || now < self.valid_from } pub fn base_domain(domain: impl AsRef) -> Result { let domain = domain.as_ref(); psl::domain_str(domain) .map(|d| d.to_string()) .ok_or(LicenseError::InvalidDomain { domain: domain.to_string(), }) } } impl Display for LicenseError { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { LicenseError::Expired => write!(f, "License is expired"), LicenseError::Parse => write!(f, "Failed to parse license key"), LicenseError::Validation => write!(f, "Failed to validate license key"), LicenseError::Decode => write!(f, "Failed to decode license key"), LicenseError::InvalidParameters => write!(f, "Invalid license key parameters"), LicenseError::DomainMismatch { issued_to, current } => { write!( f, "License issued to domain {issued_to:?} does not match {current:?}", ) } LicenseError::InvalidDomain { domain } => { write!(f, "Invalid domain {domain:?}") } LicenseError::RenewalFailed { reason } => { write!(f, "Failed to renew license: {reason}") } } } } ================================================ FILE: crates/common/src/enterprise/llm.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: LicenseRef-SEL * * This file is subject to the Stalwart Enterprise License Agreement (SEL) and * is NOT open source software. * */ use hyper::{HeaderMap, header::CONTENT_TYPE}; use serde::{Deserialize, Serialize}; use std::time::Duration; use utils::config::{Config, http::parse_http_headers}; #[derive(Clone, Debug)] pub struct AiApiConfig { pub id: String, pub api_type: ApiType, pub url: String, pub model: String, pub timeout: Duration, pub headers: HeaderMap, pub tls_allow_invalid_certs: bool, pub default_temperature: f64, } #[derive(Clone, Copy, Debug)] pub enum ApiType { ChatCompletion, TextCompletion, } #[derive(Serialize, Deserialize, Debug)] pub struct ChatCompletionRequest { pub model: String, pub messages: Vec, pub temperature: f64, } #[derive(Serialize, Deserialize, Debug)] pub struct Message { pub role: String, pub content: String, } #[derive(Serialize, Deserialize, Debug)] pub struct ChatCompletionResponse { pub created: i64, pub object: String, pub id: String, pub model: String, pub choices: Vec, } #[derive(Serialize, Deserialize, Debug)] pub struct ChatCompletionChoice { pub index: i32, pub finish_reason: String, pub message: Message, } #[derive(Serialize, Deserialize, Debug)] pub struct TextCompletionRequest { pub model: String, pub prompt: String, pub temperature: f64, } #[derive(Deserialize, Debug)] pub struct TextCompletionResponse { pub created: i64, pub object: String, pub id: String, pub model: String, pub choices: Vec, } #[derive(Deserialize, Debug)] pub struct TextCompletionChoice { pub index: i32, pub finish_reason: String, pub text: String, } impl AiApiConfig { pub async fn send_request( &self, prompt: impl Into, temperature: Option, ) -> trc::Result { self.post_api(prompt, temperature).await.map_err(|err| { trc::Error::new(trc::EventType::Ai(trc::AiEvent::ApiError)) .id(self.id.clone()) .details("OpenAPI request failed") .reason(err) }) } async fn post_api( &self, prompt: impl Into, temperature: Option, ) -> Result { // Serialize body let body = match self.api_type { ApiType::ChatCompletion => serde_json::to_string(&ChatCompletionRequest { model: self.model.to_string(), messages: vec![Message { role: "user".to_string(), content: prompt.into(), }], temperature: temperature.unwrap_or(self.default_temperature), }) .map_err(|err| format!("Failed to serialize request: {}", err))?, ApiType::TextCompletion => serde_json::to_string(&TextCompletionRequest { model: self.model.to_string(), prompt: prompt.into(), temperature: temperature.unwrap_or(self.default_temperature), }) .map_err(|err| format!("Failed to serialize request: {}", err))?, }; // Send request let response = reqwest::Client::builder() .timeout(self.timeout) .danger_accept_invalid_certs(self.tls_allow_invalid_certs) .build() .map_err(|err| format!("Failed to create HTTP client: {}", err))? .post(&self.url) .headers(self.headers.clone()) .body(body) .send() .await .map_err(|err| format!("API request to {} failed: {err}", self.url))?; if response.status().is_success() { let bytes = response.bytes().await.map_err(|err| { format!("Failed to read response body from {}: {}", self.url, err) })?; match self.api_type { ApiType::ChatCompletion => { let response = serde_json::from_slice::(&bytes) .map_err(|err| { format!( "Failed to chat completion parse response from {}: {}", self.url, err ) })?; response .choices .into_iter() .next() .map(|choice| choice.message.content) .filter(|text| !text.is_empty()) .ok_or_else(|| { format!( "Chat completion response from {} did not contain any choices: {}", self.url, std::str::from_utf8(&bytes).unwrap_or_default() ) }) } ApiType::TextCompletion => { let response = serde_json::from_slice::(&bytes) .map_err(|err| { format!( "Failed to parse text completion response from {}: {}", self.url, err ) })?; response .choices .into_iter() .next() .map(|choice| choice.text) .filter(|text| !text.is_empty()) .ok_or_else(|| { format!( "Text completion response from {} did not contain any choices: {}", self.url, std::str::from_utf8(&bytes).unwrap_or_default() ) }) } } } else { let status = response.status(); let bytes = response.bytes().await.unwrap_or_default(); Err(format!( "OpenAPI request to {} failed with code {} ({}): {}", self.url, status.as_u16(), status.canonical_reason().unwrap_or("Unknown"), std::str::from_utf8(&bytes).unwrap_or_default() )) } } pub fn parse(config: &mut Config, id: &str) -> Option { let url = config.value(("enterprise.ai", id, "url"))?.to_string(); let api_type = match config.value(("enterprise.ai", id, "type"))? { "chat" => ApiType::ChatCompletion, "text" => ApiType::TextCompletion, _ => { config.new_build_error(("enterprise.ai", id, "type"), "Invalid API type"); return None; } }; let mut headers = parse_http_headers(config, ("enterprise.ai", id)); headers.insert(CONTENT_TYPE, "application/json".parse().unwrap()); Some(AiApiConfig { id: id.to_string(), api_type, url, headers, model: config .value_require(("enterprise.ai", id, "model"))? .to_string(), timeout: config .property_or_default(("enterprise.ai", id, "timeout"), "2m") .unwrap_or_else(|| Duration::from_secs(120)), tls_allow_invalid_certs: config .property_or_default(("enterprise.ai", id, "allow-invalid-certs"), "false") .unwrap_or_default(), default_temperature: config .property_or_default(("enterprise.ai", id, "default-temperature"), "0.7") .unwrap_or(0.7), }) } } ================================================ FILE: crates/common/src/enterprise/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: LicenseRef-SEL * * This file is subject to the Stalwart Enterprise License Agreement (SEL) and * is NOT open source software. * */ pub mod alerts; pub mod config; pub mod license; pub mod llm; pub mod undelete; use ahash::{AHashMap, AHashSet}; use directory::{ QueryParams, Type, backend::internal::{lookup::DirectoryStore, manage::ManageDirectory}, }; use license::LicenseKey; use llm::AiApiConfig; use mail_parser::DateTime; use std::{sync::Arc, time::Duration}; use store::Store; use trc::{AddContext, EventType, MetricType}; use utils::{HttpLimitResponse, config::cron::SimpleCron, template::Template}; use crate::{ Core, Server, config::groupware::CalendarTemplateVariable, expr::Expression, manager::webadmin::Resource, }; #[derive(Clone)] pub struct Enterprise { pub license: LicenseKey, pub logo_url: Option, pub undelete: Option, pub trace_store: Option, pub metrics_store: Option, pub metrics_alerts: Vec, pub ai_apis: AHashMap>, pub spam_filter_llm: Option, pub template_calendar_alarm: Option>, pub template_scheduling_email: Option>, pub template_scheduling_web: Option>, } #[derive(Debug, Clone)] pub struct SpamFilterLlmConfig { pub model: Arc, pub temperature: f64, pub prompt: String, pub separator: char, pub index_category: usize, pub index_confidence: Option, pub index_explanation: Option, pub categories: AHashSet, pub confidence: AHashSet, } #[derive(Clone)] pub struct Undelete { pub retention: Duration, } #[derive(Clone)] pub struct TraceStore { pub retention: Option, pub store: Store, } #[derive(Clone)] pub struct MetricStore { pub retention: Option, pub store: Store, pub interval: SimpleCron, } #[derive(Clone, Debug)] pub struct MetricAlert { pub id: String, pub condition: Expression, pub method: Vec, } #[derive(Clone, Debug)] pub enum AlertMethod { Email { from_name: Option, from_addr: String, to: Vec, subject: AlertContent, body: AlertContent, }, Event { message: Option, }, } #[derive(Clone, Debug)] pub struct AlertContent(pub Vec); #[derive(Clone, Debug)] pub enum AlertContentToken { Text(String), Metric(MetricType), Event(EventType), } impl Core { pub fn is_enterprise_edition(&self) -> bool { self.enterprise .as_ref() .is_some_and(|e| !e.license.is_expired()) } } impl Server { // WARNING: TAMPERING WITH THIS FUNCTION IS STRICTLY PROHIBITED // Any attempt to modify, bypass, or disable this license validation mechanism // constitutes a severe violation of the Stalwart Enterprise License Agreement. // Such actions may result in immediate termination of your license, legal action, // and substantial financial penalties. Stalwart Labs LLC actively monitors for // unauthorized modifications and will pursue all available legal remedies against // violators to the fullest extent of the law, including but not limited to claims // for copyright infringement, breach of contract, and fraud. #[inline] pub fn is_enterprise_edition(&self) -> bool { self.core.is_enterprise_edition() } pub fn licensed_accounts(&self) -> u32 { self.core .enterprise .as_ref() .map_or(0, |e| e.license.accounts) } pub fn log_license_details(&self) { if let Some(enterprise) = &self.core.enterprise { trc::event!( Server(trc::ServerEvent::Licensing), Details = "Stalwart Enterprise Edition license key is valid", Domain = enterprise.license.domain.clone(), Total = enterprise.license.accounts, ValidFrom = DateTime::from_timestamp(enterprise.license.valid_from as i64).to_rfc3339(), ValidTo = DateTime::from_timestamp(enterprise.license.valid_to as i64).to_rfc3339(), ); } } pub async fn can_create_account(&self) -> trc::Result { if let Some(enterprise) = &self.core.enterprise { let total_accounts = self .store() .count_principals(None, Type::Individual.into(), None) .await .caused_by(trc::location!())?; if total_accounts + 1 > enterprise.license.accounts as u64 { trc::event!( Server(trc::ServerEvent::Licensing), Details = "Account creation not possible: license key account limit reached", Domain = enterprise.license.domain.clone(), Total = total_accounts, Limit = enterprise.license.accounts, ); return Ok(false); } } Ok(true) } pub async fn logo_resource(&self, domain: &str) -> trc::Result>>> { const MAX_IMAGE_SIZE: usize = 1024 * 1024; if self.is_enterprise_edition() { let domain = psl::domain_str(domain).unwrap_or(domain); let logo = { self.inner.data.logos.lock().get(domain).cloned() }; if let Some(logo) = logo { Ok(logo) } else { // Try fetching the logo for the domain let logo_url = if let Some(mut principal) = self .store() .query(QueryParams::name(domain).with_return_member_of(false)) .await .caused_by(trc::location!())? .filter(|p| p.typ() == Type::Domain) { if let Some(logo) = principal.picture_mut().filter(|l| l.starts_with("http")) { std::mem::take(logo).into() } else if let Some(tenant_id) = principal.tenant() { if let Some(logo) = self .store() .query(QueryParams::id(tenant_id).with_return_member_of(false)) .await .caused_by(trc::location!())? .and_then(|mut p| p.picture_mut().map(std::mem::take)) .filter(|l| l.starts_with("http")) { logo.clone().into() } else { self.default_logo_url() } } else { self.default_logo_url() } } else { self.default_logo_url() }; let mut logo = None; if let Some(logo_url) = logo_url { let response = reqwest::get(logo_url.as_str()).await.map_err(|err| { trc::ResourceEvent::DownloadExternal .into_err() .details("Failed to download logo") .reason(err) })?; let content_type = response .headers() .get(reqwest::header::CONTENT_TYPE) .and_then(|ct| ct.to_str().ok()) .unwrap_or("image/svg+xml") .to_string(); let contents = response .bytes_with_limit(MAX_IMAGE_SIZE) .await .map_err(|err| { trc::ResourceEvent::DownloadExternal .into_err() .details("Failed to download logo") .reason(err) })? .ok_or_else(|| { trc::ResourceEvent::DownloadExternal .into_err() .details("Download exceeded maximum size") })?; logo = Resource::new(content_type, contents).into(); } self.inner .data .logos .lock() .insert(domain.to_string(), logo.clone()); Ok(logo) } } else { Ok(None) } } fn default_logo_url(&self) -> Option { self.core .enterprise .as_ref() .and_then(|e| e.logo_url.as_ref().map(|l| l.into())) } } ================================================ FILE: crates/common/src/enterprise/undelete.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: LicenseRef-SEL * * This file is subject to the Stalwart Enterprise License Agreement (SEL) and * is NOT open source software. * */ use crate::Core; use store::{ Deserialize, IterateParams, U32_LEN, U64_LEN, ValueKey, write::{AlignedBytes, Archive, BlobOp, ValueClass, key::DeserializeBigEndian, now}, }; use trc::AddContext; use types::blob_hash::{BLOB_HASH_LEN, BlobHash}; pub struct DeletedBlob { pub hash: BlobHash, pub expires_at: u64, pub item: DeletedItem, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] pub struct DeletedItem { pub typ: DeletedItemType, pub size: u32, pub deleted_at: u64, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] pub enum DeletedItemType { Email { from: Box, subject: Box, received_at: u64, }, FileNode { name: Box, }, CalendarEvent { title: Box, start_time: u64, }, ContactCard { name: Box, }, SieveScript { name: Box, }, } impl Core { pub async fn list_deleted(&self, account_id: u32) -> trc::Result> { let from_key = ValueKey { account_id, collection: 0, document_id: 0, class: ValueClass::Blob(BlobOp::Undelete { hash: BlobHash::default(), until: 0, }), }; let to_key = ValueKey { account_id, collection: 0, document_id: u32::MAX, class: ValueClass::Blob(BlobOp::Undelete { hash: BlobHash::new_max(), until: u64::MAX, }), }; let now = now(); let mut results = Vec::new(); self.storage .data .iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { let expires_at = key.deserialize_be_u64(key.len() - U64_LEN)?; if expires_at > now { let item = as Deserialize>::deserialize(value) .and_then(|bytes| bytes.deserialize::()) .add_context(|ctx| ctx.ctx(trc::Key::Key, key))?; results.push(DeletedBlob { hash: BlobHash::try_from_hash_slice( key.get(U32_LEN + 1..U32_LEN + 1 + BLOB_HASH_LEN) .ok_or_else(|| { trc::Error::corrupted_key( key, value.into(), trc::location!(), ) })?, ) .unwrap(), expires_at, item, }); } Ok(true) }, ) .await .caused_by(trc::location!())?; Ok(results) } } ================================================ FILE: crates/common/src/expr/eval.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{cmp::Ordering, fmt::Display}; use compact_str::{CompactString, ToCompactString, format_compact}; use hyper::StatusCode; use trc::EvalEvent; use crate::Server; use super::{ BinaryOperator, Constant, Expression, ExpressionItem, Setting, StringCow, UnaryOperator, Variable, functions::{FUNCTIONS, ResolveVariable}, if_block::IfBlock, }; impl Server { pub async fn eval_if<'x, R: TryFrom>, V: ResolveVariable>( &'x self, if_block: &'x IfBlock, resolver: &'x V, session_id: u64, ) -> Option { if if_block.is_empty() { trc::event!( Eval(EvalEvent::Result), SpanId = session_id, Id = if_block.key.clone(), Result = "" ); return None; } match (EvalContext { resolver, core: self, expr: if_block, captures: Vec::new(), session_id, }) .eval() .await { Ok(result) => { trc::event!( Eval(EvalEvent::Result), SpanId = session_id, Id = if_block.key.clone(), Result = format!("{result:?}"), ); match result.try_into() { Ok(value) => Some(value), Err(_) => { trc::event!( Eval(EvalEvent::Result), SpanId = session_id, Id = if_block.key.clone(), Result = "", ); None } } } Err(err) => { trc::event!( Eval(EvalEvent::Error), SpanId = session_id, Id = if_block.key.clone(), CausedBy = err, ); None } } } pub async fn eval_expr<'x, R: TryFrom>, V: ResolveVariable>( &'x self, expr: &'x Expression, resolver: &'x V, expr_id: &str, session_id: u64, ) -> Option { if expr.is_empty() { return None; } match (EvalContext { resolver, core: self, expr, captures: &mut Vec::new(), session_id, }) .eval() .await { Ok(result) => { trc::event!( Eval(EvalEvent::Result), SpanId = session_id, Id = expr_id.to_compact_string(), Result = format!("{result:?}"), ); match result.try_into() { Ok(value) => Some(value), Err(_) => { trc::event!( Eval(EvalEvent::Error), SpanId = session_id, Id = expr_id.to_compact_string(), Details = "Failed to convert result", ); None } } } Err(err) => { trc::event!( Eval(EvalEvent::Error), SpanId = session_id, Id = expr_id.to_compact_string(), CausedBy = err, ); None } } } } struct EvalContext<'x, V: ResolveVariable, T, C> { resolver: &'x V, core: &'x Server, expr: &'x T, captures: C, session_id: u64, } impl<'x, V: ResolveVariable> EvalContext<'x, V, IfBlock, Vec> { async fn eval(&mut self) -> trc::Result> { for if_then in &self.expr.if_then { if (EvalContext { resolver: self.resolver, core: self.core, expr: &if_then.expr, captures: &mut self.captures, session_id: self.session_id, }) .eval() .await? .to_bool() { return (EvalContext { resolver: self.resolver, core: self.core, expr: &if_then.then, captures: &mut self.captures, session_id: self.session_id, }) .eval() .await; } } (EvalContext { resolver: self.resolver, core: self.core, expr: &self.expr.default, captures: &mut self.captures, session_id: self.session_id, }) .eval() .await } } impl<'x, V: ResolveVariable> EvalContext<'x, V, Expression, &mut Vec> { async fn eval(&mut self) -> trc::Result> { let mut stack = Vec::new(); let mut exprs = self.expr.items.iter(); while let Some(expr) = exprs.next() { match expr { ExpressionItem::Variable(v) => { stack.push(self.resolver.resolve_variable(*v)); } ExpressionItem::Global(v) => { stack.push(self.resolver.resolve_global(v)); } ExpressionItem::Constant(val) => { stack.push(Variable::from(val)); } ExpressionItem::Capture(v) => { stack.push(Variable::String(StringCow::Owned( self.captures .get(*v as usize) .map(|v| v.as_str()) .unwrap_or_default() .to_compact_string(), ))); } ExpressionItem::Setting(setting) => match setting { Setting::Hostname => { stack.push(self.core.core.network.server_name.as_str().into()) } Setting::ReportDomain => { stack.push(self.core.core.network.report_domain.as_str().into()) } Setting::NodeId => stack.push(self.core.core.network.node_id.into()), Setting::Other(key) => stack.push( self.core .core .storage .config .get(key) .await? .unwrap_or_default() .to_compact_string() .into(), ), }, ExpressionItem::UnaryOperator(op) => { let value = stack.pop().unwrap_or_default(); stack.push(match op { UnaryOperator::Not => value.op_not(), UnaryOperator::Minus => value.op_minus(), }); } ExpressionItem::BinaryOperator(op) => { let right = stack.pop().unwrap_or_default(); let left = stack.pop().unwrap_or_default(); stack.push(match op { BinaryOperator::Add => left.op_add(right), BinaryOperator::Subtract => left.op_subtract(right), BinaryOperator::Multiply => left.op_multiply(right), BinaryOperator::Divide => left.op_divide(right), BinaryOperator::And => left.op_and(right), BinaryOperator::Or => left.op_or(right), BinaryOperator::Xor => left.op_xor(right), BinaryOperator::Eq => left.op_eq(right), BinaryOperator::Ne => left.op_ne(right), BinaryOperator::Lt => left.op_lt(right), BinaryOperator::Le => left.op_le(right), BinaryOperator::Gt => left.op_gt(right), BinaryOperator::Ge => left.op_ge(right), }); } ExpressionItem::Function { id, num_args } => { let num_args = *num_args as usize; let mut arguments = Variable::array(num_args); for arg_num in 0..num_args { arguments[num_args - arg_num - 1] = stack.pop().unwrap_or_default(); } let result = if let Some((_, fnc, _)) = FUNCTIONS.get(*id as usize) { (fnc)(arguments) } else { Box::pin(self.core.eval_fnc( *id - FUNCTIONS.len() as u32, arguments, self.session_id, )) .await? }; stack.push(result); } ExpressionItem::JmpIf { val, pos } => { if stack.last().is_some_and(|v| v.to_bool()) == *val { for _ in 0..*pos { exprs.next(); } } } ExpressionItem::ArrayAccess => { let index = stack .pop() .unwrap_or_default() .to_usize() .unwrap_or_default(); let array = stack.pop().unwrap_or_default().into_array(); stack.push(array.into_iter().nth(index).unwrap_or_default()); } ExpressionItem::ArrayBuild(num_items) => { let num_items = *num_items as usize; let mut items = Variable::array(num_items); for arg_num in 0..num_items { items[num_items - arg_num - 1] = stack.pop().unwrap_or_default(); } stack.push(Variable::Array(items)); } ExpressionItem::Regex(regex) => { self.captures.clear(); let value = stack.pop().unwrap_or_default().into_string(); if let Some(captures_) = regex.captures(value.as_ref()) { for capture in captures_.iter() { self.captures .push(capture.map_or("", |m| m.as_str()).to_compact_string()); } } stack.push(Variable::Integer(!self.captures.is_empty() as i64)); } } } Ok(stack.pop().unwrap_or_default()) } } impl Expression { pub fn is_empty(&self) -> bool { self.items.is_empty() } pub fn items(&self) -> &[ExpressionItem] { &self.items } } impl<'x> Variable<'x> { pub fn op_add(self, other: Variable<'x>) -> Variable<'x> { match (self, other) { (Variable::Integer(a), Variable::Integer(b)) => Variable::Integer(a.saturating_add(b)), (Variable::Float(a), Variable::Float(b)) => Variable::Float(a + b), (Variable::Integer(i), Variable::Float(f)) | (Variable::Float(f), Variable::Integer(i)) => Variable::Float(i as f64 + f), (Variable::Array(a), Variable::Array(b)) => { Variable::Array(a.into_iter().chain(b).collect::>()) } (Variable::Array(a), b) => { Variable::Array(a.into_iter().chain([b]).collect::>()) } (a, Variable::Array(b)) => { Variable::Array([a].into_iter().chain(b).collect::>()) } (Variable::String(a), b) => { if !a.is_empty() { Variable::String(StringCow::Owned(format_compact!("{}{}", a, b))) } else { b } } (a, Variable::String(b)) => { if !b.is_empty() { Variable::String(StringCow::Owned(format_compact!("{}{}", a, b))) } else { a } } } } pub fn op_subtract(self, other: Variable<'x>) -> Variable<'x> { match (self, other) { (Variable::Integer(a), Variable::Integer(b)) => Variable::Integer(a.saturating_sub(b)), (Variable::Float(a), Variable::Float(b)) => Variable::Float(a - b), (Variable::Integer(a), Variable::Float(b)) => Variable::Float(a as f64 - b), (Variable::Float(a), Variable::Integer(b)) => Variable::Float(a - b as f64), (Variable::Array(a), b) | (b, Variable::Array(a)) => { Variable::Array(a.into_iter().filter(|v| v != &b).collect::>()) } (a, b) => a.parse_number().op_subtract(b.parse_number()), } } pub fn op_multiply(self, other: Variable<'x>) -> Variable<'x> { match (self, other) { (Variable::Integer(a), Variable::Integer(b)) => Variable::Integer(a.saturating_mul(b)), (Variable::Float(a), Variable::Float(b)) => Variable::Float(a * b), (Variable::Integer(i), Variable::Float(f)) | (Variable::Float(f), Variable::Integer(i)) => Variable::Float(i as f64 * f), (a, b) => a.parse_number().op_multiply(b.parse_number()), } } pub fn op_divide(self, other: Variable<'x>) -> Variable<'x> { match (self, other) { (Variable::Integer(a), Variable::Integer(b)) => { Variable::Float(if b != 0 { a as f64 / b as f64 } else { 0.0 }) } (Variable::Float(a), Variable::Float(b)) => { Variable::Float(if b != 0.0 { a / b } else { 0.0 }) } (Variable::Integer(a), Variable::Float(b)) => { Variable::Float(if b != 0.0 { a as f64 / b } else { 0.0 }) } (Variable::Float(a), Variable::Integer(b)) => { Variable::Float(if b != 0 { a / b as f64 } else { 0.0 }) } (a, b) => a.parse_number().op_divide(b.parse_number()), } } pub fn op_and(self, other: Variable) -> Variable { Variable::Integer(i64::from(self.to_bool() & other.to_bool())) } pub fn op_or(self, other: Variable) -> Variable { Variable::Integer(i64::from(self.to_bool() | other.to_bool())) } pub fn op_xor(self, other: Variable) -> Variable { Variable::Integer(i64::from(self.to_bool() ^ other.to_bool())) } pub fn op_eq(self, other: Variable) -> Variable { Variable::Integer(i64::from(self == other)) } pub fn op_ne(self, other: Variable) -> Variable { Variable::Integer(i64::from(self != other)) } pub fn op_lt(self, other: Variable) -> Variable { Variable::Integer(i64::from(self < other)) } pub fn op_le(self, other: Variable) -> Variable { Variable::Integer(i64::from(self <= other)) } pub fn op_gt(self, other: Variable) -> Variable { Variable::Integer(i64::from(self > other)) } pub fn op_ge(self, other: Variable) -> Variable { Variable::Integer(i64::from(self >= other)) } pub fn op_not(self) -> Variable<'static> { Variable::Integer(i64::from(!self.to_bool())) } pub fn op_minus(self) -> Variable<'static> { match self { Variable::Integer(n) => Variable::Integer(-n), Variable::Float(n) => Variable::Float(-n), _ => self.parse_number().op_minus(), } } pub fn parse_number(&self) -> Variable<'static> { match self { Variable::String(s) if !s.is_empty() => { if let Ok(n) = s.as_str().parse::() { Variable::Integer(n) } else if let Ok(n) = s.as_str().parse::() { Variable::Float(n) } else { Variable::Integer(0) } } Variable::Integer(n) => Variable::Integer(*n), Variable::Float(n) => Variable::Float(*n), Variable::Array(l) => Variable::Integer(l.is_empty() as i64), _ => Variable::Integer(0), } } #[inline(always)] fn array(num_items: usize) -> Vec> { let mut items = Vec::with_capacity(num_items); for _ in 0..num_items { items.push(Variable::Integer(0)); } items } pub fn to_ref<'y: 'x>(&'y self) -> Variable<'x> { match self { Variable::String(s) => Variable::String(StringCow::Borrowed(s.as_str())), Variable::Integer(n) => Variable::Integer(*n), Variable::Float(n) => Variable::Float(*n), Variable::Array(l) => Variable::Array(l.iter().map(|v| v.to_ref()).collect::>()), } } pub fn to_bool(&self) -> bool { match self { Variable::Float(f) => *f != 0.0, Variable::Integer(n) => *n != 0, Variable::String(s) => !s.is_empty(), Variable::Array(a) => !a.is_empty(), } } pub fn to_string(&'_ self) -> StringCow<'_> { match self { Variable::String(s) => StringCow::Borrowed(s.as_str()), Variable::Integer(n) => StringCow::Owned(n.to_compact_string()), Variable::Float(n) => StringCow::Owned(n.to_compact_string()), Variable::Array(l) => { let mut result = CompactString::with_capacity(self.len() * 10); for item in l { if !result.is_empty() { result.push_str("\r\n"); } match item { Variable::String(v) => result.push_str(v.as_str()), Variable::Integer(v) => result.push_str(&v.to_compact_string()), Variable::Float(v) => result.push_str(&v.to_compact_string()), Variable::Array(_) => {} } } StringCow::Owned(result) } } } pub fn into_string(self) -> StringCow<'x> { match self { Variable::String(s) => s, Variable::Integer(n) => StringCow::Owned(n.to_compact_string()), Variable::Float(n) => StringCow::Owned(n.to_compact_string()), Variable::Array(l) => { let mut result = CompactString::with_capacity(l.len() * 10); for item in l { if !result.is_empty() { result.push_str("\r\n"); } match item { Variable::String(v) => result.push_str(v.as_ref()), Variable::Integer(v) => result.push_str(&v.to_compact_string()), Variable::Float(v) => result.push_str(&v.to_compact_string()), Variable::Array(_) => {} } } StringCow::Owned(result) } } } pub fn to_integer(&self) -> Option { match self { Variable::Integer(n) => Some(*n), Variable::Float(n) => Some(*n as i64), Variable::String(s) if !s.is_empty() => s.as_str().parse::().ok(), _ => None, } } pub fn to_usize(&self) -> Option { match self { Variable::Integer(n) => Some(*n as usize), Variable::Float(n) => Some(*n as usize), Variable::String(s) if !s.is_empty() => s.as_str().parse::().ok(), _ => None, } } pub fn len(&self) -> usize { match self { Variable::String(s) => s.len(), Variable::Integer(_) | Variable::Float(_) => 2, Variable::Array(l) => l.iter().map(|v| v.len() + 2).sum(), } } pub fn is_empty(&self) -> bool { match self { Variable::String(s) => s.is_empty(), _ => false, } } pub fn as_array(&'_ self) -> Option<&'_ [Variable<'_>]> { match self { Variable::Array(l) => Some(l), _ => None, } } pub fn into_array(self) -> Vec> { match self { Variable::Array(l) => l, v if !v.is_empty() => vec![v], _ => vec![], } } pub fn to_array(&self) -> Vec> { match self { Variable::Array(l) => l.iter().map(|v| v.to_ref()).collect::>(), v if !v.is_empty() => vec![v.to_ref()], _ => vec![], } } pub fn into_owned(self) -> Variable<'static> { match self { Variable::String(s) => Variable::String(StringCow::Owned(s.into_owned())), Variable::Integer(n) => Variable::Integer(n), Variable::Float(n) => Variable::Float(n), Variable::Array(l) => Variable::Array(l.into_iter().map(|v| v.into_owned()).collect()), } } } impl PartialEq for Variable<'_> { fn eq(&self, other: &Self) -> bool { match (self, other) { (Self::Integer(a), Self::Integer(b)) => a == b, (Self::Float(a), Self::Float(b)) => a == b, (Self::Integer(a), Self::Float(b)) | (Self::Float(b), Self::Integer(a)) => { *a as f64 == *b } (Self::String(a), Self::String(b)) => a.as_str() == b.as_str(), (Self::String(_), Self::Integer(_) | Self::Float(_)) => &self.parse_number() == other, (Self::Integer(_) | Self::Float(_), Self::String(_)) => self == &other.parse_number(), (Self::Array(a), Self::Array(b)) => a == b, _ => false, } } } impl Eq for Variable<'_> {} #[allow(clippy::non_canonical_partial_ord_impl)] impl PartialOrd for Variable<'_> { fn partial_cmp(&self, other: &Self) -> Option { match (self, other) { (Self::Integer(a), Self::Integer(b)) => a.partial_cmp(b), (Self::Float(a), Self::Float(b)) => a.partial_cmp(b), (Self::Integer(a), Self::Float(b)) => (*a as f64).partial_cmp(b), (Self::Float(a), Self::Integer(b)) => a.partial_cmp(&(*b as f64)), (Self::String(a), Self::String(b)) => a.as_str().partial_cmp(b.as_str()), (Self::String(_), Self::Integer(_) | Self::Float(_)) => { self.parse_number().partial_cmp(other) } (Self::Integer(_) | Self::Float(_), Self::String(_)) => { self.partial_cmp(&other.parse_number()) } (Self::Array(a), Self::Array(b)) => a.partial_cmp(b), (Self::Array(_) | Self::String(_), _) => Ordering::Greater.into(), (_, Self::Array(_)) => Ordering::Less.into(), } } } impl Ord for Variable<'_> { fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.partial_cmp(other).unwrap_or(Ordering::Greater) } } impl Display for Variable<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Variable::String(v) => v.fmt(f), Variable::Integer(v) => v.fmt(f), Variable::Float(v) => v.fmt(f), Variable::Array(v) => { for (i, v) in v.iter().enumerate() { if i > 0 { f.write_str("\n")?; } v.fmt(f)?; } Ok(()) } } } } impl<'x> From<&'x Constant> for Variable<'x> { fn from(value: &'x Constant) -> Self { match value { Constant::Integer(i) => Variable::Integer(*i), Constant::Float(f) => Variable::Float(*f), Constant::String(s) => Variable::String(StringCow::Borrowed(s.as_str())), } } } impl<'x> TryFrom> for CompactString { type Error = (); fn try_from(value: Variable<'x>) -> Result { if let Variable::String(s) = value { Ok(match s { StringCow::Borrowed(v) => v.into(), StringCow::Owned(v) => v, }) } else { Err(()) } } } impl<'x> TryFrom> for String { type Error = (); fn try_from(value: Variable<'x>) -> Result { if let Variable::String(s) = value { Ok(match s { StringCow::Borrowed(v) => v.to_string(), StringCow::Owned(v) => v.into_string(), }) } else { Err(()) } } } impl<'x> From> for bool { fn from(val: Variable<'x>) -> Self { val.to_bool() } } impl<'x> TryFrom> for i64 { type Error = (); fn try_from(value: Variable<'x>) -> Result { value.to_integer().ok_or(()) } } impl<'x> TryFrom> for u64 { type Error = (); fn try_from(value: Variable<'x>) -> Result { value.to_integer().map(|v| v as u64).ok_or(()) } } impl<'x> TryFrom> for usize { type Error = (); fn try_from(value: Variable<'x>) -> Result { value.to_usize().ok_or(()) } } impl<'x> TryFrom> for StatusCode { type Error = (); fn try_from(value: Variable<'x>) -> Result { match value.to_integer() { Some(v) => match StatusCode::from_u16(v as u16) { Ok(status) => Ok(status), Err(_) => Err(()), }, None => Err(()), } } } ================================================ FILE: crates/common/src/expr/functions/array.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::expr::Variable; pub(crate) fn fn_count(v: Vec) -> Variable { match &v[0] { Variable::Array(a) => a.len(), v => { if !v.is_empty() { 1 } else { 0 } } } .into() } pub(crate) fn fn_sort(mut v: Vec) -> Variable { let is_asc = v[1].to_bool(); let mut arr = v.remove(0).into_array(); if is_asc { arr.sort_unstable_by(|a, b| b.cmp(a)); } else { arr.sort_unstable(); } arr.into() } pub(crate) fn fn_dedup(mut v: Vec) -> Variable { let arr = v.remove(0).into_array(); let mut result = Vec::with_capacity(arr.len()); for item in arr { if !result.contains(&item) { result.push(item); } } result.into() } pub(crate) fn fn_is_intersect(v: Vec) -> Variable { match (&v[0], &v[1]) { (Variable::Array(a), Variable::Array(b)) => a.iter().any(|x| b.contains(x)), (Variable::Array(a), item) | (item, Variable::Array(a)) => a.contains(item), _ => false, } .into() } pub(crate) fn fn_winnow(mut v: Vec) -> Variable { match v.remove(0) { Variable::Array(a) => a .into_iter() .filter(|i| !i.is_empty()) .collect::>() .into(), v => v, } } ================================================ FILE: crates/common/src/expr/functions/asynch.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{cmp::Ordering, net::IpAddr, vec::IntoIter}; use compact_str::{CompactString, ToCompactString}; use directory::backend::RcptType; use mail_auth::IpLookupStrategy; use store::{Deserialize, Rows, Value, dispatch::lookup::KeyValue}; use trc::AddContext; use crate::{Server, expr::StringCow}; use super::*; impl Server { pub(crate) async fn eval_fnc<'x>( &self, fnc_id: u32, params: Vec>, session_id: u64, ) -> trc::Result> { let mut params = FncParams::new(params); match fnc_id { F_IS_LOCAL_DOMAIN => { let directory = params.next_as_string(); let domain = params.next_as_string(); self.get_directory_or_default(directory.as_ref(), session_id) .is_local_domain(domain.as_ref()) .await .caused_by(trc::location!()) .map(|v| v.into()) } F_IS_LOCAL_ADDRESS => { let directory = params.next_as_string(); let address = params.next_as_string(); self.get_directory_or_default(directory.as_ref(), session_id) .rcpt(address.as_ref()) .await .caused_by(trc::location!()) .map(|v| (v != RcptType::Invalid).into()) } F_KEY_GET => { let store = params.next_as_string(); let key = params.next_as_string(); self.get_in_memory_store_or_default(store.as_str(), session_id) .key_get::(key.as_str()) .await .map(|value| value.map(|v| v.into_inner()).unwrap_or_default()) .caused_by(trc::location!()) } F_KEY_EXISTS => { let store = params.next_as_string(); let key = params.next_as_string(); self.get_in_memory_store_or_default(store.as_str(), session_id) .key_exists(key.as_str()) .await .caused_by(trc::location!()) .map(|v| v.into()) } F_KEY_SET => { let store = params.next_as_string(); let key = params.next_as_string(); let value = params.next_as_string(); self.get_in_memory_store_or_default(store.as_ref(), session_id) .key_set(KeyValue::new( key.as_bytes().to_vec(), value.as_bytes().to_vec(), )) .await .map(|_| true) .caused_by(trc::location!()) .map(|v| v.into()) } F_COUNTER_INCR => { let store = params.next_as_string(); let key = params.next_as_string(); let value = params.next_as_integer(); self.get_in_memory_store_or_default(store.as_ref(), session_id) .counter_incr(KeyValue::new(key.into_owned(), value), true) .await .map(Variable::Integer) .caused_by(trc::location!()) } F_COUNTER_GET => { let store = params.next_as_string(); let key = params.next_as_string(); self.get_in_memory_store_or_default(store.as_ref(), session_id) .counter_get(key.as_bytes().to_vec()) .await .map(Variable::Integer) .caused_by(trc::location!()) } F_DNS_QUERY => self.dns_query(params).await, F_SQL_QUERY => self.sql_query(params, session_id).await, _ => Ok(Variable::default()), } } async fn sql_query<'x>( &self, mut arguments: FncParams<'x>, session_id: u64, ) -> trc::Result> { let store = self.get_data_store(arguments.next_as_string().as_ref(), session_id); let query = arguments.next_as_string(); if query.is_empty() { return Err(trc::EventType::Eval(trc::EvalEvent::Error) .into_err() .details("Empty query string")); } // Obtain arguments let arguments = match arguments.next() { Variable::Array(l) => l.into_iter().map(to_store_value).collect(), v => vec![to_store_value(v)], }; // Run query if query .as_bytes() .get(..6) .is_some_and(|q| q.eq_ignore_ascii_case(b"SELECT")) { let mut rows = store .sql_query::(query.as_str(), arguments) .await .caused_by(trc::location!())?; Ok(match rows.rows.len().cmp(&1) { Ordering::Equal => { let mut row = rows.rows.pop().unwrap().values; match row.len().cmp(&1) { Ordering::Equal if !matches!(row.first(), Some(Value::Null)) => { row.pop().map(into_variable).unwrap() } Ordering::Less => Variable::default(), _ => { Variable::Array(row.into_iter().map(into_variable).collect::>()) } } } Ordering::Less => Variable::default(), Ordering::Greater => rows .rows .into_iter() .map(|r| { Variable::Array(r.values.into_iter().map(into_variable).collect::>()) }) .collect::>() .into(), }) } else { store .sql_query::(query.as_str(), arguments) .await .caused_by(trc::location!()) .map(|v| v.into()) } } async fn dns_query<'x>(&self, mut arguments: FncParams<'x>) -> trc::Result> { let entry = arguments.next_as_string(); let record_type = arguments.next_as_string(); if record_type.as_str().eq_ignore_ascii_case("ip") { self.core .smtp .resolvers .dns .ip_lookup( entry.as_ref(), IpLookupStrategy::Ipv4thenIpv6, 10, Some(&self.inner.cache.dns_ipv4), Some(&self.inner.cache.dns_ipv6), ) .await .map_err(|err| trc::Error::from(err).caused_by(trc::location!())) .map(|result| { result .iter() .map(|ip| Variable::from(ip.to_compact_string())) .collect::>() .into() }) } else if record_type.as_str().eq_ignore_ascii_case("mx") { self.core .smtp .resolvers .dns .mx_lookup(entry.as_str(), Some(&self.inner.cache.dns_mx)) .await .map_err(|err| trc::Error::from(err).caused_by(trc::location!())) .map(|result| { result .iter() .flat_map(|mx| { mx.exchanges.iter().map(|host| { Variable::String(StringCow::Owned( host.strip_suffix('.') .unwrap_or(host.as_str()) .to_compact_string(), )) }) }) .collect::>() .into() }) } else if record_type.as_str().eq_ignore_ascii_case("txt") { self.core .smtp .resolvers .dns .txt_raw_lookup(entry.as_str()) .await .map_err(|err| trc::Error::from(err).caused_by(trc::location!())) .map(|result| Variable::from(CompactString::from_utf8(result).unwrap_or_default())) } else if record_type.as_str().eq_ignore_ascii_case("ptr") { self.core .smtp .resolvers .dns .ptr_lookup( entry.as_str().parse::().map_err(|err| { trc::EventType::Eval(trc::EvalEvent::Error) .into_err() .details("Failed to parse IP address") .reason(err) })?, Some(&self.inner.cache.dns_ptr), ) .await .map_err(|err| trc::Error::from(err).caused_by(trc::location!())) .map(|result| { result .iter() .map(|host| Variable::from(host.to_compact_string())) .collect::>() .into() }) } else if record_type.as_str().eq_ignore_ascii_case("ipv4") { self.core .smtp .resolvers .dns .ipv4_lookup(entry.as_str(), Some(&self.inner.cache.dns_ipv4)) .await .map_err(|err| trc::Error::from(err).caused_by(trc::location!())) .map(|result| { result .iter() .map(|ip| Variable::from(ip.to_compact_string())) .collect::>() .into() }) } else if record_type.as_str().eq_ignore_ascii_case("ipv6") { self.core .smtp .resolvers .dns .ipv6_lookup(entry.as_str(), Some(&self.inner.cache.dns_ipv6)) .await .map_err(|err| trc::Error::from(err).caused_by(trc::location!())) .map(|result| { result .iter() .map(|ip| Variable::from(ip.to_compact_string())) .collect::>() .into() }) } else { Ok(Variable::default()) } } } struct FncParams<'x> { params: IntoIter>, } impl<'x> FncParams<'x> { pub fn new(params: Vec>) -> Self { Self { params: params.into_iter(), } } pub fn next_as_string(&mut self) -> StringCow<'x> { self.params.next().unwrap().into_string() } pub fn next_as_integer(&mut self) -> i64 { self.params.next().unwrap().to_integer().unwrap_or_default() } pub fn next(&mut self) -> Variable<'x> { self.params.next().unwrap() } } #[derive(Debug)] struct VariableWrapper(Variable<'static>); impl From for VariableWrapper { fn from(value: i64) -> Self { VariableWrapper(Variable::Integer(value)) } } impl Deserialize for VariableWrapper { fn deserialize(bytes: &[u8]) -> trc::Result { Ok(VariableWrapper(Variable::String(StringCow::Owned( CompactString::from_utf8_lossy(bytes), )))) } } impl From> for VariableWrapper { fn from(value: store::Value<'static>) -> Self { VariableWrapper(match value { Value::Integer(v) => Variable::Integer(v), Value::Bool(v) => Variable::Integer(v as i64), Value::Float(v) => Variable::Float(v), Value::Text(v) => Variable::String(StringCow::Owned(v.into())), Value::Blob(v) => Variable::String(StringCow::Owned(match v { std::borrow::Cow::Borrowed(v) => CompactString::from_utf8_lossy(v), std::borrow::Cow::Owned(v) => CompactString::from_utf8_lossy(&v), })), Value::Null => Variable::String(StringCow::Borrowed("")), }) } } impl VariableWrapper { pub fn into_inner(self) -> Variable<'static> { self.0 } } fn to_store_value(value: Variable) -> Value { match value { Variable::String(v) => Value::Text(v.to_string().into()), Variable::Integer(v) => Value::Integer(v), Variable::Float(v) => Value::Float(v), v => Value::Text(v.to_string().into_owned().into()), } } fn into_variable(value: Value) -> Variable { match value { Value::Integer(v) => Variable::Integer(v), Value::Bool(v) => Variable::Integer(i64::from(v)), Value::Float(v) => Variable::Float(v), Value::Text(v) => Variable::String(v.into()), Value::Blob(v) => Variable::String(StringCow::Owned(CompactString::from_utf8_lossy(&v))), Value::Null => Variable::default(), } } ================================================ FILE: crates/common/src/expr/functions/email.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::CompactString; use crate::expr::{StringCow, Variable}; pub(crate) fn fn_is_email(v: Vec) -> Variable { let mut last_ch = 0; let mut in_quote = false; let mut at_count = 0; let mut dot_count = 0; let mut lp_len = 0; let mut value = 0; for &ch in v[0].to_string().as_bytes() { match ch { b'0'..=b'9' | b'a'..=b'z' | b'A'..=b'Z' | b'!' | b'#' | b'$' | b'%' | b'&' | b'\'' | b'*' | b'+' | b'-' | b'/' | b'=' | b'?' | b'^' | b'_' | b'`' | b'{' | b'|' | b'}' | b'~' | 0x7f..=u8::MAX => { value += 1; } b'.' if !in_quote => { if last_ch != b'.' && last_ch != b'@' && value != 0 { value += 1; if at_count == 1 { dot_count += 1; } } else { return false.into(); } } b'@' if !in_quote => { at_count += 1; lp_len = value; value = 0; } b'>' | b':' | b',' | b' ' if in_quote => { value += 1; } b'\"' if !in_quote || last_ch != b'\\' => { in_quote = !in_quote; } b'\\' if in_quote && last_ch != b'\\' => (), _ => { if !in_quote { return false.into(); } } } last_ch = ch; } (at_count == 1 && dot_count > 0 && lp_len > 0 && value > 0).into() } pub(crate) fn fn_email_part(v: Vec) -> Variable { let mut v = v.into_iter(); let value = v.next().unwrap(); let part = v.next().unwrap().into_string(); value.transform(|s| match s { StringCow::Borrowed(s) => s .rsplit_once('@') .map(|(u, d)| match part.as_str() { "local" => Variable::from(u.trim()), "domain" => Variable::from(d.trim()), _ => Variable::default(), }) .unwrap_or_default(), StringCow::Owned(s) => s .rsplit_once('@') .map(|(u, d)| match part.as_str() { "local" => Variable::from(CompactString::new(u.trim())), "domain" => Variable::from(CompactString::new(d.trim())), _ => Variable::default(), }) .unwrap_or_default(), }) } ================================================ FILE: crates/common/src/expr/functions/misc.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::net::IpAddr; use compact_str::CompactString; use mail_auth::common::resolver::ToReverseName; use crate::expr::Variable; pub(crate) fn fn_is_empty(v: Vec) -> Variable { match &v[0] { Variable::String(s) => s.is_empty(), Variable::Integer(_) | Variable::Float(_) => false, Variable::Array(a) => a.is_empty(), } .into() } pub(crate) fn fn_is_number(v: Vec) -> Variable { matches!(&v[0], Variable::Integer(_) | Variable::Float(_)).into() } pub(crate) fn fn_is_ip_addr(v: Vec) -> Variable { v[0].to_string() .as_str() .parse::() .is_ok() .into() } pub(crate) fn fn_is_ipv4_addr(v: Vec) -> Variable { v[0].to_string() .as_str() .parse::() .is_ok_and(|ip| matches!(ip, IpAddr::V4(_))) .into() } pub(crate) fn fn_is_ipv6_addr(v: Vec) -> Variable { v[0].to_string() .as_str() .parse::() .is_ok_and(|ip| matches!(ip, IpAddr::V6(_))) .into() } pub(crate) fn fn_ip_reverse_name(v: Vec) -> Variable { CompactString::new( v[0].to_string() .as_str() .parse::() .map(|ip| ip.to_reverse_name()) .unwrap_or_default(), ) .into() } pub(crate) fn fn_if_then(v: Vec) -> Variable { let mut v = v.into_iter(); let condition = v.next().unwrap(); let iff = v.next().unwrap(); let then = v.next().unwrap(); if condition.to_bool() { iff } else { then } } ================================================ FILE: crates/common/src/expr/functions/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{StringCow, Variable}; pub mod array; pub mod asynch; pub mod email; pub mod misc; pub mod text; pub trait ResolveVariable: Sync + Send { fn resolve_variable(&self, variable: u32) -> Variable<'_>; fn resolve_global(&self, variable: &str) -> Variable<'_>; } impl<'x> Variable<'x> { fn transform(self, f: impl Fn(StringCow<'x>) -> Variable<'x>) -> Variable<'x> { match self { Variable::String(s) => f(s), Variable::Array(list) => Variable::Array( list.into_iter() .map(|v| match v { Variable::String(s) => f(s), v => f(v.into_string()), }) .collect::>(), ), v => f(v.into_string()), } } } #[allow(clippy::type_complexity)] pub(crate) const FUNCTIONS: &[(&str, fn(Vec) -> Variable, u32)] = &[ ("count", array::fn_count, 1), ("sort", array::fn_sort, 2), ("dedup", array::fn_dedup, 1), ("winnow", array::fn_winnow, 1), ("is_intersect", array::fn_is_intersect, 2), ("is_email", email::fn_is_email, 1), ("email_part", email::fn_email_part, 2), ("is_empty", misc::fn_is_empty, 1), ("is_number", misc::fn_is_number, 1), ("is_ip_addr", misc::fn_is_ip_addr, 1), ("is_ipv4_addr", misc::fn_is_ipv4_addr, 1), ("is_ipv6_addr", misc::fn_is_ipv6_addr, 1), ("ip_reverse_name", misc::fn_ip_reverse_name, 1), ("trim", text::fn_trim, 1), ("trim_end", text::fn_trim_end, 1), ("trim_start", text::fn_trim_start, 1), ("len", text::fn_len, 1), ("to_lowercase", text::fn_to_lowercase, 1), ("to_uppercase", text::fn_to_uppercase, 1), ("is_uppercase", text::fn_is_uppercase, 1), ("is_lowercase", text::fn_is_lowercase, 1), ("has_digits", text::fn_has_digits, 1), ("count_spaces", text::fn_count_spaces, 1), ("count_uppercase", text::fn_count_uppercase, 1), ("count_lowercase", text::fn_count_lowercase, 1), ("count_chars", text::fn_count_chars, 1), ("contains", text::fn_contains, 2), ("contains_ignore_case", text::fn_contains_ignore_case, 2), ("eq_ignore_case", text::fn_eq_ignore_case, 2), ("starts_with", text::fn_starts_with, 2), ("ends_with", text::fn_ends_with, 2), ("lines", text::fn_lines, 1), ("substring", text::fn_substring, 3), ("strip_prefix", text::fn_strip_prefix, 2), ("strip_suffix", text::fn_strip_suffix, 2), ("split", text::fn_split, 2), ("rsplit", text::fn_rsplit, 2), ("split_once", text::fn_split_once, 2), ("rsplit_once", text::fn_rsplit_once, 2), ("split_n", text::fn_split_n, 3), ("split_words", text::fn_split_words, 1), ("hash", text::fn_hash, 2), ("if_then", misc::fn_if_then, 3), ]; pub const F_IS_LOCAL_DOMAIN: u32 = 0; pub const F_IS_LOCAL_ADDRESS: u32 = 1; pub const F_KEY_GET: u32 = 2; pub const F_KEY_EXISTS: u32 = 3; pub const F_KEY_SET: u32 = 4; pub const F_COUNTER_INCR: u32 = 5; pub const F_COUNTER_GET: u32 = 6; pub const F_SQL_QUERY: u32 = 7; pub const F_DNS_QUERY: u32 = 8; pub const ASYNC_FUNCTIONS: &[(&str, u32, u32)] = &[ ("is_local_domain", F_IS_LOCAL_DOMAIN, 2), ("is_local_address", F_IS_LOCAL_ADDRESS, 2), ("key_get", F_KEY_GET, 2), ("key_exists", F_KEY_EXISTS, 2), ("key_set", F_KEY_SET, 3), ("counter_incr", F_COUNTER_INCR, 3), ("counter_get", F_COUNTER_GET, 2), ("dns_query", F_DNS_QUERY, 2), ("sql_query", F_SQL_QUERY, 3), ]; ================================================ FILE: crates/common/src/expr/functions/text.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::{CompactString, ToCompactString, format_compact}; use sha1::Sha1; use sha2::{Sha256, Sha512}; use crate::expr::{StringCow, Variable}; pub(crate) fn fn_trim(mut v: Vec) -> Variable { v.remove(0).transform(|s| match s { StringCow::Borrowed(s) => Variable::from(s.trim()), StringCow::Owned(s) => Variable::from(s.trim().to_compact_string()), }) } pub(crate) fn fn_trim_end(mut v: Vec) -> Variable { v.remove(0).transform(|s| match s { StringCow::Borrowed(s) => Variable::from(s.trim_end()), StringCow::Owned(s) => Variable::from(s.trim_end().to_compact_string()), }) } pub(crate) fn fn_trim_start(mut v: Vec) -> Variable { v.remove(0).transform(|s| match s { StringCow::Borrowed(s) => Variable::from(s.trim_start()), StringCow::Owned(s) => Variable::from(s.trim_start().to_compact_string()), }) } pub(crate) fn fn_len(v: Vec) -> Variable { match &v[0] { Variable::String(s) => s.len(), Variable::Array(a) => a.len(), v => v.to_string().len(), } .into() } pub(crate) fn fn_to_lowercase(mut v: Vec) -> Variable { v.remove(0) .transform(|s| Variable::from(CompactString::from_str_to_lowercase(s.as_str()))) } pub(crate) fn fn_to_uppercase(mut v: Vec) -> Variable { v.remove(0) .transform(|s| Variable::from(CompactString::from_str_to_uppercase(s.as_str()))) } pub(crate) fn fn_is_uppercase(mut v: Vec) -> Variable { v.remove(0).transform(|s| { s.as_str() .chars() .filter(|c| c.is_alphabetic()) .all(|c| c.is_uppercase()) .into() }) } pub(crate) fn fn_is_lowercase(mut v: Vec) -> Variable { v.remove(0).transform(|s| { s.as_str() .chars() .filter(|c| c.is_alphabetic()) .all(|c| c.is_lowercase()) .into() }) } pub(crate) fn fn_has_digits(mut v: Vec) -> Variable { v.remove(0) .transform(|s| s.as_str().chars().any(|c| c.is_ascii_digit()).into()) } pub(crate) fn fn_split_words(v: Vec) -> Variable { v[0].to_string() .as_str() .split_whitespace() .filter(|word| word.chars().all(|c| c.is_alphanumeric())) .map(|word| Variable::from(CompactString::new(word))) .collect::>() .into() } pub(crate) fn fn_count_spaces(v: Vec) -> Variable { v[0].to_string() .as_str() .chars() .filter(|c| c.is_whitespace()) .count() .into() } pub(crate) fn fn_count_uppercase(v: Vec) -> Variable { v[0].to_string() .as_str() .chars() .filter(|c| c.is_alphabetic() && c.is_uppercase()) .count() .into() } pub(crate) fn fn_count_lowercase(v: Vec) -> Variable { v[0].to_string() .as_str() .chars() .filter(|c| c.is_alphabetic() && c.is_lowercase()) .count() .into() } pub(crate) fn fn_count_chars(v: Vec) -> Variable { v[0].to_string().as_str().chars().count().into() } pub(crate) fn fn_eq_ignore_case(v: Vec) -> Variable { v[0].to_string() .as_str() .eq_ignore_ascii_case(v[1].to_string().as_str()) .into() } pub(crate) fn fn_contains(v: Vec) -> Variable { match &v[0] { Variable::String(s) => s.as_str().contains(v[1].to_string().as_str()), Variable::Array(arr) => arr.contains(&v[1]), val => val.to_string().as_str().contains(v[1].to_string().as_str()), } .into() } pub(crate) fn fn_contains_ignore_case(v: Vec) -> Variable { let needle = v[1].to_string(); match &v[0] { Variable::String(s) => s .as_str() .to_lowercase() .contains(&needle.as_str().to_lowercase()), Variable::Array(arr) => arr.iter().any(|v| match v { Variable::String(s) => s.as_str().eq_ignore_ascii_case(needle.as_str()), _ => false, }), val => val.to_string().as_str().contains(needle.as_str()), } .into() } pub(crate) fn fn_starts_with(v: Vec) -> Variable { v[0].to_string() .as_str() .starts_with(v[1].to_string().as_str()) .into() } pub(crate) fn fn_ends_with(v: Vec) -> Variable { v[0].to_string() .as_str() .ends_with(v[1].to_string().as_str()) .into() } pub(crate) fn fn_lines(mut v: Vec) -> Variable { match v.remove(0) { Variable::String(s) => s .as_str() .lines() .map(|s| Variable::from(CompactString::new(s))) .collect::>() .into(), val => val, } } pub(crate) fn fn_substring(v: Vec) -> Variable { v[0].to_string() .as_str() .chars() .skip(v[1].to_usize().unwrap_or_default()) .take(v[2].to_usize().unwrap_or_default()) .collect::() .into() } pub(crate) fn fn_strip_prefix(v: Vec) -> Variable { let mut v = v.into_iter(); let value = v.next().unwrap(); let prefix = v.next().unwrap().into_string(); value.transform(|s| match s { StringCow::Borrowed(s) => s .strip_prefix(prefix.as_str()) .map(Variable::from) .unwrap_or_default(), StringCow::Owned(s) => s .strip_prefix(prefix.as_str()) .map(|s| Variable::from(CompactString::new(s))) .unwrap_or_default(), }) } pub(crate) fn fn_strip_suffix(v: Vec) -> Variable { let mut v = v.into_iter(); let value = v.next().unwrap(); let suffix = v.next().unwrap().into_string(); value.transform(|s| match s { StringCow::Borrowed(s) => s .strip_suffix(suffix.as_str()) .map(Variable::from) .unwrap_or_default(), StringCow::Owned(s) => s .strip_suffix(suffix.as_str()) .map(|s| Variable::from(CompactString::new(s))) .unwrap_or_default(), }) } pub(crate) fn fn_split(v: Vec) -> Variable { let mut v = v.into_iter(); let value = v.next().unwrap().into_string(); let arg = v.next().unwrap().into_string(); match value { StringCow::Borrowed(s) => s .split(arg.as_str()) .map(Variable::from) .collect::>() .into(), StringCow::Owned(s) => s .split(arg.as_str()) .map(|s| Variable::from(CompactString::new(s))) .collect::>() .into(), } } pub(crate) fn fn_rsplit(v: Vec) -> Variable { let mut v = v.into_iter(); let value = v.next().unwrap().into_string(); let arg = v.next().unwrap().into_string(); match value { StringCow::Borrowed(s) => s .rsplit(arg.as_str()) .map(Variable::from) .collect::>() .into(), StringCow::Owned(s) => s .rsplit(arg.as_str()) .map(|s| Variable::from(CompactString::new(s))) .collect::>() .into(), } } pub(crate) fn fn_split_n(v: Vec) -> Variable { let mut v = v.into_iter(); let value = v.next().unwrap().into_string(); let arg = v.next().unwrap().into_string(); let num = v.next().unwrap().to_integer().unwrap_or_default() as usize; fn split_n<'x, 'y>(s: &'x str, arg: &'y str, num: usize, mut f: impl FnMut(&'x str)) { let mut s = s; for _ in 0..num { if let Some((a, b)) = s.split_once(arg) { f(a); s = b; } else { break; } } f(s); } let mut result = Vec::new(); match value { StringCow::Borrowed(s) => split_n(s, arg.as_str(), num, |s| result.push(Variable::from(s))), StringCow::Owned(s) => split_n(&s, arg.as_str(), num, |s| { result.push(Variable::from(CompactString::new(s))) }), } result.into() } pub(crate) fn fn_split_once(v: Vec) -> Variable { let mut v = v.into_iter(); let value = v.next().unwrap().into_string(); let arg = v.next().unwrap().into_string(); match value { StringCow::Borrowed(s) => s .split_once(arg.as_str()) .map(|(a, b)| Variable::Array(vec![Variable::from(a), Variable::from(b)])) .unwrap_or_default(), StringCow::Owned(s) => s .split_once(arg.as_str()) .map(|(a, b)| { Variable::Array(vec![ Variable::from(CompactString::new(a)), Variable::from(CompactString::new(b)), ]) }) .unwrap_or_default(), } } pub(crate) fn fn_rsplit_once(v: Vec) -> Variable { let mut v = v.into_iter(); let value = v.next().unwrap().into_string(); let arg = v.next().unwrap().into_string(); match value { StringCow::Borrowed(s) => s .rsplit_once(arg.as_str()) .map(|(a, b)| Variable::Array(vec![Variable::from(a), Variable::from(b)])) .unwrap_or_default(), StringCow::Owned(s) => s .rsplit_once(arg.as_str()) .map(|(a, b)| { Variable::Array(vec![ Variable::from(CompactString::new(a)), Variable::from(CompactString::new(b)), ]) }) .unwrap_or_default(), } } pub(crate) fn fn_hash(v: Vec) -> Variable { use sha1::Digest; let mut v = v.into_iter(); let value = v.next().unwrap().into_string(); let algo = v.next().unwrap().into_string(); match algo.as_str() { "md5" => format_compact!("{:x}", md5::compute(value.as_bytes())).into(), "sha1" => { let mut hasher = Sha1::new(); hasher.update(value.as_bytes()); format_compact!("{:x}", hasher.finalize()).into() } "sha256" => { let mut hasher = Sha256::new(); hasher.update(value.as_bytes()); format_compact!("{:x}", hasher.finalize()).into() } "sha512" => { let mut hasher = Sha512::new(); hasher.update(value.as_bytes()); format_compact!("{:x}", hasher.finalize()).into() } _ => Variable::default(), } } ================================================ FILE: crates/common/src/expr/if_block.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::CompactString; use utils::config::{Config, utils::AsKey}; use crate::expr::{Constant, Expression}; use super::{ ConstantValue, ExpressionItem, parser::ExpressionParser, tokenizer::{TokenMap, Tokenizer}, }; #[derive(Debug, Clone, PartialEq, Eq)] pub struct IfThen { pub expr: Expression, pub then: Expression, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct IfBlock { pub key: String, pub if_then: Vec, pub default: Expression, } impl IfBlock { pub fn new( key: impl Into, if_thens: impl IntoIterator, default: impl AsRef, ) -> Self { let token_map = TokenMap::default() .with_all_variables() .with_constants::(); Self { key: key.into(), if_then: if_thens .into_iter() .map(|(if_, then)| IfThen { expr: Expression::parse(&token_map, if_), then: Expression::parse(&token_map, then), }) .collect(), default: Expression::parse(&token_map, default.as_ref()), } } pub fn empty(key: impl Into) -> Self { Self { key: key.into(), if_then: Default::default(), default: Expression { items: Default::default(), }, } } pub fn is_empty(&self) -> bool { self.default.is_empty() && self.if_then.is_empty() } } impl Expression { pub fn try_parse( config: &mut Config, key: impl AsKey, token_map: &TokenMap, ) -> Option { if let Some(expr) = config.value(key.as_key()) { match ExpressionParser::new(Tokenizer::new(expr, token_map)).parse() { Ok(expr) => Some(expr), Err(err) => { config.new_parse_error(key, err); None } } } else { None } } fn parse(token_map: &TokenMap, expr: &str) -> Self { ExpressionParser::new(Tokenizer::new(expr, token_map)) .parse() .unwrap() } } impl IfBlock { pub fn try_parse( config: &mut Config, prefix: impl AsKey, token_map: &TokenMap, ) -> Option { let key = prefix.as_key(); // Parse conditions let mut if_block = IfBlock { key, if_then: Default::default(), default: Expression { items: Default::default(), }, }; // Try first with a single value if config.contains_key(if_block.key.as_str()) { if_block.default = Expression::try_parse(config, &if_block.key, token_map)?; return Some(if_block); } // Collect prefixes let prefix = prefix.as_prefix(); let keys = config .keys .keys() .filter(|k| k.starts_with(&prefix)) .cloned() .collect::>(); let mut found_if = false; let mut found_else = ""; let mut found_then = false; let mut last_array_pos = ""; for item in &keys { let suffix_ = item.strip_prefix(&prefix).unwrap(); if let Some((array_pos, suffix)) = suffix_.split_once('.') { let if_key = suffix.split_once('.').map(|(v, _)| v).unwrap_or(suffix); if if_key == "if" { if array_pos != last_array_pos { if !last_array_pos.is_empty() && !found_then { config.new_parse_error( if_block.key, format!( "Missing 'then' in 'if' condition {}.", last_array_pos.parse().unwrap_or(0) + 1, ), ); return None; } if_block.if_then.push(IfThen { expr: Expression::try_parse(config, item, token_map)?, then: Expression::default(), }); found_then = false; last_array_pos = array_pos; } found_if = true; } else if if_key == "else" { if found_else.is_empty() { if found_if { if_block.default = Expression::try_parse(config, item, token_map)?; found_else = array_pos; } else { config.new_parse_error(if_block.key, "Found 'else' before 'if'"); return None; } } else if array_pos != found_else { config.new_parse_error(if_block.key, "Multiple 'else' found"); return None; } } else if if_key == "then" { if found_else.is_empty() { if array_pos == last_array_pos { if !found_then { if_block.if_then.last_mut().unwrap().then = Expression::try_parse(config, item, token_map)?; found_then = true; } } else { config.new_parse_error(if_block.key, "Found 'then' without 'if'"); return None; } } else { config.new_parse_error(if_block.key, "Found 'then' in 'else' block"); return None; } } } else { config.new_parse_error( if_block.key, format!("Invalid property {item:?} found in 'if' block."), ); return None; } } if !found_if { None } else if !found_then { config.new_parse_error( if_block.key, format!( "Missing 'then' in 'if' condition {}", last_array_pos.parse().unwrap_or(0) + 1, ), ); None } else if found_else.is_empty() { config.new_parse_error(if_block.key, "Missing 'else'"); None } else { Some(if_block) } } pub fn into_default(self, key: impl Into) -> IfBlock { IfBlock { key: key.into(), if_then: Default::default(), default: self.default, } } pub fn default_string(&self) -> Option<&str> { for expr_item in &self.default.items { if let ExpressionItem::Constant(Constant::String(value)) = expr_item { return Some(value.as_str()); } } None } pub fn into_default_string(self) -> Option { for expr_item in self.default.items { if let ExpressionItem::Constant(Constant::String(value)) = expr_item { return Some(value); } } None } } ================================================ FILE: crates/common/src/expr/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use self::tokenizer::TokenMap; use compact_str::CompactString; use regex::Regex; use std::{ borrow::Cow, fmt::{Display, Formatter}, net::{IpAddr, Ipv4Addr, Ipv6Addr}, time::Duration, }; use utils::config::{Rate, utils::ParseValue}; pub const V_RECIPIENT: u32 = 0; pub const V_RECIPIENT_DOMAIN: u32 = 1; pub const V_SENDER: u32 = 2; pub const V_SENDER_DOMAIN: u32 = 3; pub const V_MX: u32 = 4; pub const V_HELO_DOMAIN: u32 = 5; pub const V_AUTHENTICATED_AS: u32 = 6; pub const V_LISTENER: u32 = 7; pub const V_REMOTE_IP: u32 = 8; pub const V_REMOTE_PORT: u32 = 9; pub const V_LOCAL_IP: u32 = 10; pub const V_LOCAL_PORT: u32 = 11; pub const V_PRIORITY: u32 = 12; pub const V_PROTOCOL: u32 = 13; pub const V_TLS: u32 = 14; pub const V_RECIPIENTS: u32 = 15; pub const V_QUEUE_RETRY_NUM: u32 = 16; pub const V_QUEUE_NOTIFY_NUM: u32 = 17; pub const V_QUEUE_EXPIRES_IN: u32 = 18; pub const V_QUEUE_LAST_STATUS: u32 = 19; pub const V_QUEUE_LAST_ERROR: u32 = 20; pub const V_URL: u32 = 21; pub const V_URL_PATH: u32 = 22; pub const V_HEADERS: u32 = 23; pub const V_METHOD: u32 = 24; pub const V_ASN: u32 = 25; pub const V_COUNTRY: u32 = 26; pub const V_RECEIVED_VIA_PORT: u32 = 27; pub const V_RECEIVED_FROM_IP: u32 = 28; pub const V_QUEUE_NAME: u32 = 29; pub const V_SOURCE: u32 = 30; pub const V_SIZE: u32 = 31; pub const V_QUEUE_AGE: u32 = 32; pub const VARIABLES_MAP: &[(&str, u32)] = &[ ("rcpt", V_RECIPIENT), ("rcpt_domain", V_RECIPIENT_DOMAIN), ("sender", V_SENDER), ("sender_domain", V_SENDER_DOMAIN), ("mx", V_MX), ("helo_domain", V_HELO_DOMAIN), ("authenticated_as", V_AUTHENTICATED_AS), ("listener", V_LISTENER), ("remote_ip", V_REMOTE_IP), ("local_ip", V_LOCAL_IP), ("priority", V_PRIORITY), ("local_port", V_LOCAL_PORT), ("remote_port", V_REMOTE_PORT), ("protocol", V_PROTOCOL), ("is_tls", V_TLS), ("recipients", V_RECIPIENTS), ("retry_num", V_QUEUE_RETRY_NUM), ("notify_num", V_QUEUE_NOTIFY_NUM), ("expires_in", V_QUEUE_EXPIRES_IN), ("last_status", V_QUEUE_LAST_STATUS), ("last_error", V_QUEUE_LAST_ERROR), ("url", V_URL), ("url_path", V_URL_PATH), ("headers", V_HEADERS), ("method", V_METHOD), ("asn", V_ASN), ("country", V_COUNTRY), ("received_via_port", V_RECEIVED_VIA_PORT), ("received_from_ip", V_RECEIVED_FROM_IP), ("queue_name", V_QUEUE_NAME), ("source", V_SOURCE), ("size", V_SIZE), ("queue_age", V_QUEUE_AGE), ]; pub mod eval; pub mod functions; pub mod if_block; pub mod parser; pub mod tokenizer; #[derive(Debug, PartialEq, Eq, Clone, Default)] pub struct Expression { pub items: Vec, } #[derive(Debug, Clone)] pub enum ExpressionItem { Variable(u32), Global(CompactString), Setting(Setting), Capture(u32), Constant(Constant), BinaryOperator(BinaryOperator), UnaryOperator(UnaryOperator), Regex(Regex), JmpIf { val: bool, pos: u32 }, Function { id: u32, num_args: u32 }, ArrayAccess, ArrayBuild(u32), } #[derive(Debug, Clone)] pub enum Variable<'x> { String(StringCow<'x>), Integer(i64), Float(f64), Array(Vec>), } #[derive(Debug, Clone)] pub enum StringCow<'x> { Owned(CompactString), Borrowed(&'x str), } impl Default for Variable<'_> { fn default() -> Self { Variable::String(StringCow::Borrowed("")) } } #[derive(Debug, PartialEq, Clone)] pub enum Constant { Integer(i64), Float(f64), String(CompactString), } impl Eq for Constant {} impl From for Constant { fn from(value: CompactString) -> Self { Constant::String(value) } } impl From for Constant { fn from(value: bool) -> Self { Constant::Integer(value as i64) } } impl From for Constant { fn from(value: i64) -> Self { Constant::Integer(value) } } impl From for Constant { fn from(value: i32) -> Self { Constant::Integer(value as i64) } } impl From for Constant { fn from(value: i16) -> Self { Constant::Integer(value as i64) } } impl From for Constant { fn from(value: f64) -> Self { Constant::Float(value) } } impl From for Constant { fn from(value: usize) -> Self { Constant::Integer(value as i64) } } #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum BinaryOperator { Add, Subtract, Multiply, Divide, And, Or, Xor, Eq, Ne, Lt, Le, Gt, Ge, } #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum UnaryOperator { Not, Minus, } #[derive(Debug, Clone)] pub enum Token { Variable(u32), Global(CompactString), Capture(u32), Function { name: Cow<'static, str>, id: u32, num_args: u32, }, Constant(Constant), Setting(Setting), Regex(Regex), BinaryOperator(BinaryOperator), UnaryOperator(UnaryOperator), OpenParen, CloseParen, OpenBracket, CloseBracket, Comma, } #[derive(Debug, Clone)] pub enum Setting { Hostname, ReportDomain, NodeId, Other(CompactString), } impl From for Setting { fn from(value: CompactString) -> Self { match value.as_str() { "server.hostname" => Setting::Hostname, "report.domain" => Setting::ReportDomain, "cluster.node-id" => Setting::NodeId, _ => Setting::Other(value), } } } impl From for Variable<'_> { fn from(value: usize) -> Self { Variable::Integer(value as i64) } } impl From for Variable<'_> { fn from(value: i64) -> Self { Variable::Integer(value) } } impl From for Variable<'_> { fn from(value: u64) -> Self { Variable::Integer(value as i64) } } impl From for Variable<'_> { fn from(value: i32) -> Self { Variable::Integer(value as i64) } } impl From for Variable<'_> { fn from(value: u32) -> Self { Variable::Integer(value as i64) } } impl From for Variable<'_> { fn from(value: u16) -> Self { Variable::Integer(value as i64) } } impl From for Variable<'_> { fn from(value: i16) -> Self { Variable::Integer(value as i64) } } impl From for Variable<'_> { fn from(value: f64) -> Self { Variable::Float(value) } } impl<'x> From<&'x str> for Variable<'x> { fn from(value: &'x str) -> Self { Variable::String(StringCow::Borrowed(value)) } } impl From for Variable<'_> { fn from(value: CompactString) -> Self { Variable::String(StringCow::Owned(value)) } } impl<'x> From>> for Variable<'x> { fn from(value: Vec>) -> Self { Variable::Array(value) } } impl From for Variable<'_> { fn from(value: bool) -> Self { Variable::Integer(value as i64) } } impl> From for Expression { fn from(value: T) -> Self { Expression { items: vec![ExpressionItem::Constant(value.into())], } } } impl PartialEq for ExpressionItem { fn eq(&self, other: &Self) -> bool { match (self, other) { (Self::Variable(l0), Self::Variable(r0)) => l0 == r0, (Self::Constant(l0), Self::Constant(r0)) => l0 == r0, (Self::BinaryOperator(l0), Self::BinaryOperator(r0)) => l0 == r0, (Self::UnaryOperator(l0), Self::UnaryOperator(r0)) => l0 == r0, (Self::Regex(_), Self::Regex(_)) => true, ( Self::JmpIf { val: l_val, pos: l_pos, }, Self::JmpIf { val: r_val, pos: r_pos, }, ) => l_val == r_val && l_pos == r_pos, ( Self::Function { id: l_id, num_args: l_num_args, }, Self::Function { id: r_id, num_args: r_num_args, }, ) => l_id == r_id && l_num_args == r_num_args, (Self::ArrayBuild(l0), Self::ArrayBuild(r0)) => l0 == r0, _ => core::mem::discriminant(self) == core::mem::discriminant(other), } } } impl Eq for ExpressionItem {} impl PartialEq for Token { fn eq(&self, other: &Self) -> bool { match (self, other) { (Self::Variable(l0), Self::Variable(r0)) => l0 == r0, ( Self::Function { name: l_name, id: l_id, num_args: l_num_args, }, Self::Function { name: r_name, id: r_id, num_args: r_num_args, }, ) => l_name == r_name && l_id == r_id && l_num_args == r_num_args, (Self::Constant(l0), Self::Constant(r0)) => l0 == r0, (Self::Regex(_), Self::Regex(_)) => true, (Self::BinaryOperator(l0), Self::BinaryOperator(r0)) => l0 == r0, (Self::UnaryOperator(l0), Self::UnaryOperator(r0)) => l0 == r0, _ => core::mem::discriminant(self) == core::mem::discriminant(other), } } } impl Eq for Token {} pub struct NoConstants; pub trait ConstantValue: ParseValue + for<'x> TryFrom> + Into + Sized { fn add_constants(token_map: &mut TokenMap); } impl ConstantValue for () { fn add_constants(_: &mut TokenMap) {} } impl From<()> for Constant { fn from(_: ()) -> Self { Constant::Integer(0) } } impl<'x> TryFrom> for () { type Error = (); fn try_from(_: Variable<'x>) -> Result { Ok(()) } } impl ConstantValue for Duration { fn add_constants(_: &mut TokenMap) {} } impl<'x> TryFrom> for Duration { type Error = (); fn try_from(value: Variable<'x>) -> Result { match value { Variable::Integer(value) if value > 0 => Ok(Duration::from_millis(value as u64)), Variable::Float(value) if value > 0.0 => Ok(Duration::from_millis(value as u64)), Variable::String(value) if !value.is_empty() => { Duration::parse_value(value.as_str()).map_err(|_| ()) } _ => Err(()), } } } impl StringCow<'_> { pub fn as_str(&self) -> &str { match self { StringCow::Owned(s) => s.as_str(), StringCow::Borrowed(s) => s, } } pub fn as_bytes(&self) -> &[u8] { match self { StringCow::Owned(s) => s.as_bytes(), StringCow::Borrowed(s) => s.as_bytes(), } } pub fn is_empty(&self) -> bool { match self { StringCow::Owned(s) => s.is_empty(), StringCow::Borrowed(s) => s.is_empty(), } } pub fn len(&self) -> usize { match self { StringCow::Owned(s) => s.len(), StringCow::Borrowed(s) => s.len(), } } pub fn into_owned(self) -> CompactString { match self { StringCow::Owned(s) => s, StringCow::Borrowed(s) => s.into(), } } } impl<'x> From> for StringCow<'x> { fn from(value: Cow<'x, str>) -> Self { match value { Cow::Borrowed(s) => StringCow::Borrowed(s), Cow::Owned(s) => StringCow::Owned(s.into()), } } } impl From for StringCow<'_> { fn from(value: CompactString) -> Self { StringCow::Owned(value) } } impl AsRef for StringCow<'_> { fn as_ref(&self) -> &str { self.as_str() } } impl AsRef<[u8]> for StringCow<'_> { fn as_ref(&self) -> &[u8] { self.as_str().as_bytes() } } impl Display for StringCow<'_> { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { StringCow::Owned(s) => write!(f, "{}", s), StringCow::Borrowed(s) => write!(f, "{}", s), } } } impl From for Constant { fn from(value: Duration) -> Self { Constant::Integer(value.as_millis() as i64) } } impl<'x> TryFrom> for Rate { type Error = (); fn try_from(value: Variable<'x>) -> Result { match value { Variable::Array(items) if items.len() == 2 => { let requests = items[0].to_integer().ok_or(())?; let period = items[1].to_integer().ok_or(())?; if requests > 0 && period > 0 { Ok(Rate { requests: requests as u64, period: Duration::from_millis(period as u64), }) } else { Err(()) } } _ => Err(()), } } } impl<'x> TryFrom> for Ipv4Addr { type Error = (); fn try_from(value: Variable<'x>) -> Result { match value { Variable::String(value) => value.as_str().parse().map_err(|_| ()), _ => Err(()), } } } impl<'x> TryFrom> for Ipv6Addr { type Error = (); fn try_from(value: Variable<'x>) -> Result { match value { Variable::String(value) => value.as_str().parse().map_err(|_| ()), _ => Err(()), } } } impl<'x> TryFrom> for IpAddr { type Error = (); fn try_from(value: Variable<'x>) -> Result { match value { Variable::String(value) => value.as_str().parse().map_err(|_| ()), _ => Err(()), } } } impl<'x, T: TryFrom>> TryFrom> for Vec where Result, ()>: FromIterator>>::Error>>, { type Error = (); fn try_from(value: Variable<'x>) -> Result { value .into_array() .into_iter() .map(|v| T::try_from(v)) .collect() } } ================================================ FILE: crates/common/src/expr/parser.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{BinaryOperator, Expression, ExpressionItem, Token, tokenizer::Tokenizer}; pub struct ExpressionParser<'x> { pub(crate) tokenizer: Tokenizer<'x>, pub(crate) output: Vec, operator_stack: Vec<(Token, Option)>, arg_count: Vec, } pub(crate) const ID_ARRAY_ACCESS: u32 = u32::MAX; pub(crate) const ID_ARRAY_BUILD: u32 = u32::MAX - 1; impl<'x> ExpressionParser<'x> { pub fn new(tokenizer: Tokenizer<'x>) -> Self { Self { tokenizer, output: Vec::new(), operator_stack: Vec::new(), arg_count: Vec::new(), } } pub fn parse(mut self) -> Result { let mut last_is_var_or_fnc = false; while let Some(token) = self.tokenizer.next()? { let mut is_var_or_fnc = false; match token { Token::Variable(v) => { self.inc_arg_count(); is_var_or_fnc = true; self.output.push(ExpressionItem::Variable(v)) } Token::Constant(c) => { self.inc_arg_count(); self.output.push(ExpressionItem::Constant(c)) } Token::Global(g) => { self.inc_arg_count(); self.output.push(ExpressionItem::Global(g)) } Token::Capture(c) => { self.inc_arg_count(); self.output.push(ExpressionItem::Capture(c)) } Token::UnaryOperator(uop) => { self.operator_stack.push((Token::UnaryOperator(uop), None)) } Token::OpenParen => self.operator_stack.push((token, None)), Token::CloseParen | Token::CloseBracket => { let expect_token = if matches!(token, Token::CloseParen) { Token::OpenParen } else { Token::OpenBracket }; loop { match self.operator_stack.pop() { Some((t, _)) if t == expect_token => { break; } Some((Token::BinaryOperator(bop), jmp_pos)) => { self.update_jmp_pos(jmp_pos); self.output.push(ExpressionItem::BinaryOperator(bop)) } Some((Token::UnaryOperator(uop), _)) => { self.output.push(ExpressionItem::UnaryOperator(uop)) } _ => return Err("Mismatched parentheses".to_string()), } } match self.operator_stack.last() { Some((Token::Function { id, num_args, name }, _)) => { let got_args = self.arg_count.pop().unwrap(); if got_args != *num_args as i32 { return Err(if *id != u32::MAX { format!( "Expression function {:?} expected {} arguments, got {}", name, num_args, got_args ) } else { "Missing array index".to_string() }); } let expr = match *id { ID_ARRAY_ACCESS => ExpressionItem::ArrayAccess, ID_ARRAY_BUILD => ExpressionItem::ArrayBuild(*num_args), id => ExpressionItem::Function { id, num_args: *num_args, }, }; self.operator_stack.pop(); self.output.push(expr); } Some((Token::Regex(regex), _)) => { if self.arg_count.pop().unwrap() != 1 { return Err("Expression function \"matches\" expected 2 arguments" .to_string()); } self.output.push(ExpressionItem::Regex(regex.clone())); self.operator_stack.pop(); } Some((Token::Setting(setting), _)) => { if self.arg_count.pop().unwrap() != 0 { return Err( "Expression function \"config_get\" expected 1 argument" .to_string(), ); } self.output.push(ExpressionItem::Setting(setting.clone())); self.operator_stack.pop(); } _ => {} } is_var_or_fnc = true; } Token::BinaryOperator(bop) => { self.dec_arg_count(); while let Some((top_token, prev_jmp_pos)) = self.operator_stack.last() { match top_token { Token::BinaryOperator(top_bop) => { if bop.precedence() <= top_bop.precedence() { let top_bop = *top_bop; let jmp_pos = *prev_jmp_pos; self.update_jmp_pos(jmp_pos); self.operator_stack.pop(); self.output.push(ExpressionItem::BinaryOperator(top_bop)); } else { break; } } Token::UnaryOperator(top_uop) => { let top_uop = *top_uop; self.operator_stack.pop(); self.output.push(ExpressionItem::UnaryOperator(top_uop)); } _ => break, } } // Add jump instruction for short-circuiting let jmp_pos = match bop { BinaryOperator::And => { self.output .push(ExpressionItem::JmpIf { val: false, pos: 0 }); Some(self.output.len() - 1) } BinaryOperator::Or => { self.output .push(ExpressionItem::JmpIf { val: true, pos: 0 }); Some(self.output.len() - 1) } _ => None, }; self.operator_stack .push((Token::BinaryOperator(bop), jmp_pos)); } token @ (Token::Function { .. } | Token::Regex(_) | Token::Setting(_)) => { self.inc_arg_count(); self.arg_count.push(0); self.operator_stack.push((token, None)) } Token::OpenBracket => { // Array functions let (id, num_args, arg_count) = if last_is_var_or_fnc { (ID_ARRAY_ACCESS, 2, 1) } else { self.inc_arg_count(); (ID_ARRAY_BUILD, 0, 0) }; self.arg_count.push(arg_count); self.operator_stack.push(( Token::Function { id, name: "array".into(), num_args, }, None, )); self.operator_stack.push((token, None)); } Token::Comma => { while let Some((token, jmp_pos)) = self.operator_stack.last() { match token { Token::OpenParen => break, Token::BinaryOperator(bop) => { let bop = *bop; let jmp_pos = *jmp_pos; self.update_jmp_pos(jmp_pos); self.output.push(ExpressionItem::BinaryOperator(bop)); self.operator_stack.pop(); } Token::UnaryOperator(uop) => { self.output.push(ExpressionItem::UnaryOperator(*uop)); self.operator_stack.pop(); } _ => break, } } } } last_is_var_or_fnc = is_var_or_fnc; } while let Some((token, jmp_pos)) = self.operator_stack.pop() { match token { Token::BinaryOperator(bop) => { self.update_jmp_pos(jmp_pos); self.output.push(ExpressionItem::BinaryOperator(bop)) } Token::UnaryOperator(uop) => self.output.push(ExpressionItem::UnaryOperator(uop)), _ => return Err("Invalid token on the operator stack".to_string()), } } if self.operator_stack.is_empty() { Ok(Expression { items: self.output }) } else { Err("Invalid expression".to_string()) } } fn inc_arg_count(&mut self) { if let Some(x) = self.arg_count.last_mut() { *x = x.saturating_add(1); let op_pos = self.operator_stack.len().saturating_sub(2); match self.operator_stack.get_mut(op_pos) { Some((Token::Function { num_args, id, .. }, _)) if *id == ID_ARRAY_BUILD => { *num_args += 1; } _ => {} } } } fn dec_arg_count(&mut self) { if let Some(x) = self.arg_count.last_mut() { *x = x.saturating_sub(1); } } fn update_jmp_pos(&mut self, jmp_pos: Option) { if let Some(jmp_pos) = jmp_pos { let cur_pos = self.output.len(); if let ExpressionItem::JmpIf { pos, .. } = &mut self.output[jmp_pos] { *pos = (cur_pos - jmp_pos) as u32; } else { #[cfg(test)] panic!("Invalid jump position"); } } } } impl BinaryOperator { fn precedence(&self) -> i32 { match self { BinaryOperator::Multiply | BinaryOperator::Divide => 7, BinaryOperator::Add | BinaryOperator::Subtract => 6, BinaryOperator::Gt | BinaryOperator::Ge | BinaryOperator::Lt | BinaryOperator::Le => 5, BinaryOperator::Eq | BinaryOperator::Ne => 4, BinaryOperator::Xor => 3, BinaryOperator::And => 2, BinaryOperator::Or => 1, } } } ================================================ FILE: crates/common/src/expr/tokenizer.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{borrow::Cow, iter::Peekable, slice::Iter, time::Duration}; use ahash::AHashMap; use regex::Regex; use utils::config::utils::ParseValue; use super::{ functions::{ASYNC_FUNCTIONS, FUNCTIONS}, *, }; pub struct Tokenizer<'x> { pub(crate) iter: Peekable>, token_map: &'x TokenMap, buf: Vec, depth: u32, next_token: Vec, has_number: bool, has_dot: bool, has_alpha: bool, is_start: bool, is_eof: bool, } #[derive(Debug, Default, Clone)] pub struct TokenMap { pub tokens: AHashMap, Token>, } impl<'x> Tokenizer<'x> { #[allow(clippy::should_implement_trait)] pub fn new(expr: &'x str, token_map: &'x TokenMap) -> Self { Self { iter: expr.as_bytes().iter().peekable(), buf: Vec::new(), depth: 0, next_token: Vec::with_capacity(2), has_number: false, has_dot: false, has_alpha: false, is_start: true, is_eof: false, token_map, } } #[allow(clippy::should_implement_trait)] pub fn next(&mut self) -> Result, String> { if let Some(token) = self.next_token.pop() { return Ok(Some(token)); } else if self.is_eof { return Ok(None); } while let Some(&ch) = self.iter.next() { match ch { b'A'..=b'Z' | b'a'..=b'z' | b'_' | b'$' => { self.buf.push(ch); self.has_alpha = true; } b'0'..=b'9' => { self.buf.push(ch); self.has_number = true; } b'.' => { self.buf.push(ch); self.has_dot = true; } b'}' => { self.is_eof = true; break; } b'-' if self.buf.last().is_some_and(|c| *c == b'[') => { self.buf.push(ch); } b':' if self.buf.contains(&b'.') => { self.buf.push(ch); } b']' if self.buf.contains(&b'[') => { self.buf.push(b']'); } b'*' if self.buf.last().is_some_and(|&c| c == b'[' || c == b'.') => { self.buf.push(ch); } _ => { let (prev_token, ch) = if ch == b'(' && self.buf.eq(b"matches") { // Parse regular expressions let stop_ch = self.find_char(b"\"'")?; let regex_str = self.parse_string(stop_ch)?; let regex = Regex::new(®ex_str).map_err(|e| { format!("Invalid regular expression {:?}: {}", regex_str, e) })?; self.has_alpha = false; self.buf.clear(); self.find_char(b",")?; (Token::Regex(regex).into(), b'(') } else if ch == b'(' && self.buf.eq(b"config_get") { // Parse setting let stop_ch = self.find_char(b"\"'")?; let setting_str = self.parse_string(stop_ch)?; self.has_alpha = false; self.buf.clear(); (Token::Setting(Setting::from(setting_str)).into(), b'(') } else if !self.buf.is_empty() { self.is_start = false; (self.parse_buf()?.into(), ch) } else { (None, ch) }; let token = match ch { b'&' => { if matches!(self.iter.peek(), Some(b'&')) { self.iter.next(); } Token::BinaryOperator(BinaryOperator::And) } b'|' => { if matches!(self.iter.peek(), Some(b'|')) { self.iter.next(); } Token::BinaryOperator(BinaryOperator::Or) } b'!' => { if matches!(self.iter.peek(), Some(b'=')) { self.iter.next(); Token::BinaryOperator(BinaryOperator::Ne) } else { Token::UnaryOperator(UnaryOperator::Not) } } b'^' => Token::BinaryOperator(BinaryOperator::Xor), b'(' => { self.depth += 1; Token::OpenParen } b')' => { if self.depth == 0 { return Err("Unmatched close parenthesis".to_string()); } self.depth -= 1; Token::CloseParen } b'+' => Token::BinaryOperator(BinaryOperator::Add), b'*' => Token::BinaryOperator(BinaryOperator::Multiply), b'/' => Token::BinaryOperator(BinaryOperator::Divide), b'-' => { if self.is_start { Token::UnaryOperator(UnaryOperator::Minus) } else { Token::BinaryOperator(BinaryOperator::Subtract) } } b'=' => match self.iter.next() { Some(b'=') => Token::BinaryOperator(BinaryOperator::Eq), Some(b'>') => Token::BinaryOperator(BinaryOperator::Ge), Some(b'<') => Token::BinaryOperator(BinaryOperator::Le), _ => Token::BinaryOperator(BinaryOperator::Eq), }, b'>' => match self.iter.peek() { Some(b'=') => { self.iter.next(); Token::BinaryOperator(BinaryOperator::Ge) } _ => Token::BinaryOperator(BinaryOperator::Gt), }, b'<' => match self.iter.peek() { Some(b'=') => { self.iter.next(); Token::BinaryOperator(BinaryOperator::Le) } _ => Token::BinaryOperator(BinaryOperator::Lt), }, b',' => Token::Comma, b'[' => Token::OpenBracket, b']' => Token::CloseBracket, b' ' | b'\r' | b'\n' => { if prev_token.is_some() { return Ok(prev_token); } else { continue; } } b'\"' | b'\'' => Token::Constant(Constant::String(self.parse_string(ch)?)), _ => { return Err(format!("Invalid character {:?}", char::from(ch),)); } }; self.is_start = matches!( token, Token::OpenParen | Token::Comma | Token::BinaryOperator(_) ); return if prev_token.is_some() { self.next_token.push(token); Ok(prev_token) } else { Ok(Some(token)) }; } } } if self.depth > 0 { Err("Unmatched open parenthesis".to_string()) } else if !self.buf.is_empty() { self.parse_buf().map(Some) } else { Ok(None) } } fn find_char(&mut self, chars: &[u8]) -> Result { for &ch in self.iter.by_ref() { if !ch.is_ascii_whitespace() { return if chars.contains(&ch) { Ok(ch) } else { Err(format!( "Expected {:?}, found invalid character {:?}", char::from(chars[0]), char::from(ch), )) }; } } Err("Unexpected end of expression".to_string()) } fn parse_string(&mut self, stop_ch: u8) -> Result { let mut buf = Vec::with_capacity(16); let mut last_ch = 0; let mut found_end = false; for &ch in self.iter.by_ref() { if last_ch != b'\\' { if ch != stop_ch { buf.push(ch); } else { found_end = true; break; } } else { match ch { b'n' => { buf.push(b'\n'); } b'r' => { buf.push(b'\r'); } b't' => { buf.push(b'\t'); } _ => { buf.push(ch); } } } last_ch = ch; } if found_end { CompactString::from_utf8(buf).map_err(|_| "Invalid UTF-8".into()) } else { Err("Unterminated string".to_string()) } } fn parse_buf(&mut self) -> Result { let buf = String::from_utf8(std::mem::take(&mut self.buf)).unwrap_or_default(); if self.has_number && !self.has_alpha { self.has_number = false; if self.has_dot { self.has_dot = false; buf.parse::() .map(|f| Token::Constant(Constant::Float(f))) .map_err(|_| format!("Invalid float value {}", buf,)) } else { buf.parse::() .map(|i| Token::Constant(Constant::Integer(i))) .map_err(|_| format!("Invalid integer value {}", buf,)) } } else { let has_dot = self.has_dot; let has_number = self.has_number; self.has_alpha = false; self.has_number = false; self.has_dot = false; if !has_number && !has_dot && [4, 5].contains(&buf.len()) { if buf == "true" { return Ok(Token::Constant(Constant::Integer(1))); } else if buf == "false" { return Ok(Token::Constant(Constant::Integer(0))); } } if let Some(variable) = buf.strip_prefix('$').filter(|s| !s.is_empty()) { if variable.chars().all(|c| c.is_ascii_digit()) { Ok(variable .parse::() .map(Token::Capture) .unwrap_or_else(|_| Token::Global(variable.into()))) } else { Ok(Token::Global(variable.into())) } } else if let Some((idx, (name, _, num_args))) = FUNCTIONS .iter() .enumerate() .find(|(_, (name, _, _))| name == &buf) { Ok(Token::Function { name: Cow::Borrowed(*name), id: idx as u32, num_args: *num_args, }) } else if let Some((name, idx, num_args)) = ASYNC_FUNCTIONS.iter().find(|(name, _, _)| name == &buf) { Ok(Token::Function { name: Cow::Borrowed(*name), id: *idx + FUNCTIONS.len() as u32, num_args: *num_args, }) } else if let Some(token) = self.token_map.tokens.get(buf.as_str()) { Ok(token.clone()) } else if let Ok(duration) = Duration::parse_value(&buf) { Ok(Token::Constant(Constant::Integer( duration.as_millis() as i64 ))) } else { Err(format!("Invalid variable or constant {buf:?}")) } } } } impl TokenMap { pub fn with_all_variables(self) -> Self { self.with_variables(&[ V_RECIPIENT, V_RECIPIENT_DOMAIN, V_SENDER, V_SENDER_DOMAIN, V_MX, V_HELO_DOMAIN, V_AUTHENTICATED_AS, V_LISTENER, V_REMOTE_IP, V_REMOTE_PORT, V_LOCAL_IP, V_LOCAL_PORT, V_PRIORITY, V_PROTOCOL, V_TLS, V_QUEUE_RETRY_NUM, V_QUEUE_NOTIFY_NUM, V_QUEUE_EXPIRES_IN, V_QUEUE_LAST_STATUS, V_QUEUE_LAST_ERROR, V_QUEUE_NAME, V_QUEUE_AGE, V_ASN, V_COUNTRY, V_RECEIVED_FROM_IP, V_RECEIVED_VIA_PORT, V_SOURCE, V_SIZE, ]) } pub fn with_variables(mut self, variables: &[u32]) -> Self { for (name, idx) in VARIABLES_MAP { if variables.contains(idx) { self.tokens .insert(Cow::Borrowed(name), Token::Variable(*idx)); } } self } pub fn with_variables_map(mut self, vars: I) -> Self where I: IntoIterator, V: Into>, { for (name, idx) in vars { self.tokens.insert(name.into(), Token::Variable(idx)); } self } pub fn set_constants(mut self, consts: I) -> Self where I: IntoIterator, T: Into, { for (name, constant) in consts { self.tokens .insert(Cow::Borrowed(name), Token::Constant(constant.into())); } self } pub fn with_constants(mut self) -> Self { T::add_constants(&mut self); self } pub fn add_constant(&mut self, name: &'static str, constant: impl Into) -> &mut Self { self.tokens .insert(Cow::Borrowed(name), Token::Constant(constant.into())); self } } ================================================ FILE: crates/common/src/i18n.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ include!(concat!(env!("OUT_DIR"), "/locales.rs")); pub fn locale_or_default(name: &str) -> &'static Locale { locale(name) .or_else(|| name.split_once('_').and_then(|(lang, _)| locale(lang))) .unwrap_or(&EN_LOCALES) } #[cfg(test)] mod tests { use super::locale; #[test] fn calendar_templates_include_minutes() { for lang in ["en", "es", "fr", "de", "it", "pt", "nl", "da", "ca", "el", "sv", "pl"] { let locale = locale(lang).expect("locale must exist"); assert!( locale.calendar_date_template.contains("%M"), "{lang} calendar.date_template must include minutes" ); assert!( locale.calendar_date_template_long.contains("%M"), "{lang} calendar.date_template_long must include minutes" ); } } } ================================================ FILE: crates/common/src/ipc.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::config::smtp::{ queue::QueueName, report::AggregateFrequency, resolver::{Policy, Tlsa}, }; use ahash::RandomState; use mail_auth::{ dmarc::Dmarc, mta_sts::TlsRpt, report::{Record, tlsrpt::FailureDetails}, }; use std::{ sync::{ Arc, atomic::{AtomicBool, Ordering}, }, time::Instant, }; use store::{BlobStore, InMemoryStore, Store}; use tokio::sync::{Semaphore, SemaphorePermit, mpsc}; use types::type_state::{DataType, StateChange}; use utils::map::bitmap::Bitmap; pub enum HousekeeperEvent { AcmeReschedule { provider_id: String, renew_at: Instant, }, Purge(PurgeType), ReloadSettings, Exit, } pub enum PurgeType { Data(Store), Blobs { store: Store, blob_store: BlobStore, }, Lookup { store: InMemoryStore, prefix: Option>, }, Account { account_id: Option, use_roles: bool, }, } #[derive(Debug)] pub enum PushEvent { Subscribe { account_ids: Vec, types: Bitmap, tx: mpsc::Sender, }, Publish { notification: PushNotification, broadcast: bool, }, PushServerRegister { activate: Vec, expired: Vec, }, PushServerUpdate { account_id: u32, broadcast: bool, }, Stop, } #[derive(Debug, Clone)] pub enum PushNotification { StateChange(StateChange), CalendarAlert(CalendarAlert), EmailPush(EmailPush), } #[derive(Debug, Clone)] pub struct EmailPush { pub account_id: u32, pub email_id: u32, pub change_id: u64, } #[derive(Debug, Clone)] pub struct CalendarAlert { pub account_id: u32, pub event_id: u32, pub recurrence_id: Option, pub uid: String, pub alert_id: String, } #[derive(Debug)] pub enum BroadcastEvent { PushNotification(PushNotification), InvalidateAccessTokens(Vec), InvalidateGroupwareCache(Vec), ReloadPushServers(u32), ReloadSettings, ReloadBlockedIps, ReloadSpamFilter, } #[derive(Debug)] pub enum QueueEvent { Refresh, WorkerDone { queue_id: u64, queue_name: QueueName, status: QueueEventStatus, }, Paused(bool), ReloadSettings, Stop, } #[derive(Debug)] pub enum QueueEventStatus { Completed, Locked, Deferred, } #[derive(Debug)] pub enum ReportingEvent { Dmarc(Box), Tls(Box), Stop, } #[derive(Debug)] pub struct DmarcEvent { pub domain: String, pub report_record: Record, pub dmarc_record: Arc, pub interval: AggregateFrequency, } #[derive(Debug)] pub struct TlsEvent { pub domain: String, pub policy: PolicyType, pub failure: Option, pub tls_record: Arc, pub interval: AggregateFrequency, } #[derive(Debug, Hash, PartialEq, Eq)] pub enum PolicyType { Tlsa(Option>), Sts(Option>), None, } pub struct TrainTaskController { semaphore: Semaphore, stop_flag: AtomicBool, } impl Default for TrainTaskController { fn default() -> Self { Self { semaphore: Semaphore::new(1), stop_flag: AtomicBool::new(false), } } } impl TrainTaskController { pub fn try_run(&self) -> Option> { let permit = self.semaphore.try_acquire().ok()?; self.stop_flag.store(false, Ordering::SeqCst); Some(permit) } pub fn is_running(&self) -> bool { self.semaphore.available_permits() == 0 } pub fn stop(&self) { self.stop_flag.store(true, Ordering::SeqCst); } pub fn should_stop(&self) -> bool { self.stop_flag.load(Ordering::SeqCst) } } pub trait ToHash { fn to_hash(&self) -> u64; } impl ToHash for Dmarc { fn to_hash(&self) -> u64 { RandomState::with_seeds(1, 9, 7, 9).hash_one(self) } } impl ToHash for PolicyType { fn to_hash(&self) -> u64 { RandomState::with_seeds(1, 9, 7, 9).hash_one(self) } } impl From for ReportingEvent { fn from(value: DmarcEvent) -> Self { ReportingEvent::Dmarc(Box::new(value)) } } impl From for ReportingEvent { fn from(value: TlsEvent) -> Self { ReportingEvent::Tls(Box::new(value)) } } impl From> for PolicyType { fn from(value: Arc) -> Self { PolicyType::Tlsa(Some(value)) } } impl From> for PolicyType { fn from(value: Arc) -> Self { PolicyType::Sts(Some(value)) } } impl From<&Arc> for PolicyType { fn from(value: &Arc) -> Self { PolicyType::Tlsa(Some(value.clone())) } } impl From<&Arc> for PolicyType { fn from(value: &Arc) -> Self { PolicyType::Sts(Some(value.clone())) } } impl From<(&Option>, &Option>)> for PolicyType { fn from(value: (&Option>, &Option>)) -> Self { match value { (Some(value), _) => PolicyType::Sts(Some(value.clone())), (_, Some(value)) => PolicyType::Tlsa(Some(value.clone())), _ => PolicyType::None, } } } impl PushNotification { pub fn account_id(&self) -> u32 { match self { PushNotification::StateChange(state_change) => state_change.account_id, PushNotification::CalendarAlert(calendar_alert) => calendar_alert.account_id, PushNotification::EmailPush(email_push) => email_push.account_id, } } pub fn filter_types(&self, types: &Bitmap) -> Option { match self { PushNotification::StateChange(state_change) => { let mut filtered_types = state_change.types; filtered_types.intersection(types); if !filtered_types.is_empty() { Some(PushNotification::StateChange(StateChange { account_id: state_change.account_id, change_id: state_change.change_id, types: filtered_types, })) } else { None } } PushNotification::CalendarAlert(_) => { if types.contains(DataType::CalendarAlert) { Some(self.clone()) } else { None } } PushNotification::EmailPush(_) => { if types.contains_any( [ DataType::EmailDelivery, DataType::Email, DataType::Mailbox, DataType::Thread, ] .into_iter(), ) { Some(self.clone()) } else { None } } } } } impl EmailPush { pub fn to_state_change(&self) -> StateChange { StateChange { account_id: self.account_id, change_id: self.change_id, types: Bitmap::from_iter([ DataType::EmailDelivery, DataType::Email, DataType::Mailbox, DataType::Thread, ]), } } } ================================================ FILE: crates/common/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ #![warn(clippy::large_futures)] use ahash::{AHashMap, AHashSet}; use arc_swap::ArcSwap; use auth::{AccessToken, oauth::config::OAuthConfig, roles::RolePermissions}; use calcard::common::timezone::Tz; use config::{ groupware::GroupwareConfig, imap::ImapConfig, jmap::settings::JmapConfig, network::Network, scripts::Scripting, smtp::{ SmtpConfig, resolver::{Policy, Tlsa}, }, spamfilter::{IpResolver, SpamFilterConfig}, storage::Storage, telemetry::Metrics, }; use ipc::{BroadcastEvent, HousekeeperEvent, PushEvent, QueueEvent, ReportingEvent}; use listener::{asn::AsnGeoLookupData, blocked::Security, tls::AcmeProviders}; use mail_auth::{MX, Txt}; use manager::webadmin::{Resource, WebAdminManager}; use parking_lot::{Mutex, RwLock}; use rustls::sign::CertifiedKey; use std::{ hash::{BuildHasher, Hash, Hasher}, net::{IpAddr, Ipv4Addr, Ipv6Addr}, sync::{Arc, atomic::AtomicBool}, time::{Duration, Instant}, }; use store::rand::{Rng, distr::Alphanumeric}; use tinyvec::TinyVec; use tokio::sync::{Notify, Semaphore, mpsc}; use tokio_rustls::TlsConnector; use types::{acl::AclGrant, special_use::SpecialUse}; use utils::{ cache::{Cache, CacheItemWeight, CacheWithTtl}, snowflake::SnowflakeIdGenerator, }; pub mod addresses; pub mod auth; pub mod config; pub mod core; pub mod dns; pub mod expr; pub mod i18n; pub mod ipc; pub mod listener; pub mod manager; pub mod scripts; pub mod sharing; pub mod storage; pub mod telemetry; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] pub mod enterprise; // SPDX-SnippetEnd pub use psl; use crate::{config::spamfilter::SpamClassifier, ipc::TrainTaskController}; pub static VERSION_PRIVATE: &str = env!("CARGO_PKG_VERSION"); pub static VERSION_PUBLIC: &str = "1.0.0"; pub static USER_AGENT: &str = "Stalwart/1.0.0"; pub static DAEMON_NAME: &str = concat!("Stalwart v", env!("CARGO_PKG_VERSION"),); pub static PROD_ID: &str = "-//Stalwart Labs LLC//Stalwart Server//EN"; /* Schema history: 1 - v0.12.0 2 - v0.12.4 3 - v0.13.0 4 - v0.14.0 5 - v0.15.0 */ pub const DATABASE_SCHEMA_VERSION: u32 = 5; pub const LONG_1D_SLUMBER: Duration = Duration::from_secs(60 * 60 * 24); pub const LONG_1Y_SLUMBER: Duration = Duration::from_secs(60 * 60 * 24 * 365); pub const IPC_CHANNEL_BUFFER: usize = 1024; pub const KV_ACME: u8 = 0; pub const KV_OAUTH: u8 = 1; pub const KV_RATE_LIMIT_RCPT: u8 = 2; pub const KV_RATE_LIMIT_SCAN: u8 = 3; pub const KV_RATE_LIMIT_LOITER: u8 = 4; pub const KV_RATE_LIMIT_AUTH: u8 = 5; pub const KV_RATE_LIMIT_SMTP: u8 = 6; pub const KV_RATE_LIMIT_CONTACT: u8 = 7; pub const KV_RATE_LIMIT_HTTP_AUTHENTICATED: u8 = 8; pub const KV_RATE_LIMIT_HTTP_ANONYMOUS: u8 = 9; pub const KV_RATE_LIMIT_IMAP: u8 = 10; pub const KV_GREYLIST: u8 = 16; pub const KV_LOCK_PURGE_ACCOUNT: u8 = 20; pub const KV_LOCK_QUEUE_MESSAGE: u8 = 21; pub const KV_LOCK_QUEUE_REPORT: u8 = 22; pub const KV_LOCK_TASK: u8 = 23; pub const KV_LOCK_HOUSEKEEPER: u8 = 24; pub const KV_LOCK_DAV: u8 = 25; pub const KV_SIEVE_ID: u8 = 26; #[derive(Clone)] pub struct Server { pub inner: Arc, pub core: Arc, } pub struct Inner { pub shared_core: ArcSwap, pub data: Data, pub cache: Caches, pub ipc: Ipc, } pub struct Data { pub spam_classifier: ArcSwap, pub tls_certificates: ArcSwap>>, pub tls_self_signed_cert: Option>, pub blocked_ips: RwLock>, pub asn_geo_data: AsnGeoLookupData, pub jmap_id_gen: SnowflakeIdGenerator, pub queue_id_gen: SnowflakeIdGenerator, pub span_id_gen: SnowflakeIdGenerator, pub queue_status: AtomicBool, pub webadmin: WebAdminManager, pub logos: Mutex>>>>, pub smtp_connectors: TlsConnectors, } pub struct Caches { pub access_tokens: Cache>, pub http_auth: Cache, pub permissions: Cache>, pub messages: Cache>, pub files: Cache>, pub contacts: Cache>, pub events: Cache>, pub scheduling: Cache>, pub dns_txt: CacheWithTtl, pub dns_mx: CacheWithTtl>>, pub dns_ptr: CacheWithTtl>>, pub dns_ipv4: CacheWithTtl>>, pub dns_ipv6: CacheWithTtl>>, pub dns_tlsa: CacheWithTtl>, pub dbs_mta_sts: CacheWithTtl>, pub dns_rbl: CacheWithTtl>>, } #[derive(Debug, Clone)] pub struct CacheSwap(pub Arc>); #[derive(Debug, Clone)] pub struct MessageStoreCache { pub emails: Arc, pub mailboxes: Arc, pub update_lock: Arc, pub last_change_id: u64, pub size: u64, } #[derive(Debug, Clone)] pub struct MailboxesCache { pub change_id: u64, pub index: AHashMap, pub items: Box<[MailboxCache]>, pub size: u64, } #[derive(Debug, Clone)] pub struct MessagesCache { pub change_id: u64, pub items: Box<[MessageCache]>, pub index: AHashMap, pub keywords: Box<[Box]>, pub size: u64, } #[derive(Debug, Clone)] pub struct MessageCache { pub document_id: u32, pub mailboxes: TinyVec<[MessageUidCache; 2]>, pub keywords: u128, pub thread_id: u32, pub change_id: u64, pub size: u32, } #[derive(Debug, Default, Clone, Copy)] pub struct MessageUidCache { pub mailbox_id: u32, pub uid: u32, } #[derive(Debug, Clone)] pub struct MailboxCache { pub document_id: u32, pub name: String, pub path: String, pub role: SpecialUse, pub parent_id: u32, pub sort_order: u32, pub subscribers: TinyVec<[u32; 4]>, pub uid_validity: u32, pub acls: TinyVec<[AclGrant; 2]>, } #[derive(Debug, Clone)] pub struct HttpAuthCache { pub account_id: u32, pub revision: u64, pub expires: Instant, } pub struct Ipc { pub push_tx: mpsc::Sender, pub housekeeper_tx: mpsc::Sender, pub task_tx: Arc, pub queue_tx: mpsc::Sender, pub report_tx: mpsc::Sender, pub broadcast_tx: Option>, pub train_task_controller: Arc, } pub struct TlsConnectors { pub pki_verify: TlsConnector, pub dummy_verify: TlsConnector, } pub struct NameWrapper(pub String); #[derive(Debug, Clone)] pub struct DavResources { pub base_path: String, pub paths: AHashSet, pub resources: Vec, pub item_change_id: u64, pub container_change_id: u64, pub highest_change_id: u64, pub size: u64, pub update_lock: Arc, } #[derive(Debug, Clone)] pub struct DavPath { pub path: String, pub parent_id: Option, pub hierarchy_seq: u32, pub resource_idx: usize, } #[derive(Debug, Clone)] pub struct DavResource { pub document_id: u32, pub data: DavResourceMetadata, } #[derive(Debug, Clone, Copy)] pub struct DavResourcePath<'x> { pub path: &'x DavPath, pub resource: &'x DavResource, } #[derive(Debug, Clone)] pub enum DavResourceMetadata { File { name: String, size: Option, parent_id: Option, acls: TinyVec<[AclGrant; 2]>, }, Calendar { name: String, acls: TinyVec<[AclGrant; 2]>, preferences: TinyVec<[TinyCalendarPreferences; 2]>, }, CalendarEvent { names: TinyVec<[DavName; 2]>, start: i64, duration: u32, }, CalendarEventNotification { names: TinyVec<[DavName; 2]>, }, AddressBook { name: String, acls: TinyVec<[AclGrant; 2]>, }, ContactCard { names: TinyVec<[DavName; 2]>, }, } #[derive(Debug, Clone, Default)] pub struct TinyCalendarPreferences { pub account_id: u32, pub tz: Tz, pub flags: u16, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] #[rkyv(derive(Debug))] pub struct DavName { pub name: String, pub parent_id: u32, } #[derive(Clone, Default)] pub struct Core { pub storage: Storage, pub sieve: Scripting, pub network: Network, pub acme: AcmeProviders, pub oauth: OAuthConfig, pub smtp: SmtpConfig, pub jmap: JmapConfig, pub groupware: GroupwareConfig, pub spam: SpamFilterConfig, pub imap: ImapConfig, pub metrics: Metrics, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] pub enterprise: Option, // SPDX-SnippetEnd } impl CacheItemWeight for CacheSwap { fn weight(&self) -> u64 { std::mem::size_of::>() as u64 + self.0.load().weight() } } impl CacheItemWeight for MessageStoreCache { fn weight(&self) -> u64 { self.size } } impl CacheItemWeight for HttpAuthCache { fn weight(&self) -> u64 { std::mem::size_of::() as u64 } } impl CacheItemWeight for DavResources { fn weight(&self) -> u64 { self.size } } pub trait IntoString: Sized { fn into_string(self) -> String; } impl IntoString for Vec { fn into_string(self) -> String { String::from_utf8(self) .unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned()) } } #[derive(Debug, Clone, Eq)] pub struct ThrottleKey { pub hash: [u8; 32], } impl PartialEq for ThrottleKey { fn eq(&self, other: &Self) -> bool { self.hash == other.hash } } impl std::hash::Hash for ThrottleKey { fn hash(&self, state: &mut H) { self.hash.hash(state); } } impl AsRef<[u8]> for ThrottleKey { fn as_ref(&self) -> &[u8] { &self.hash } } #[derive(Default)] pub struct ThrottleKeyHasher { hash: u64, } impl Hasher for ThrottleKeyHasher { fn finish(&self) -> u64 { self.hash } fn write(&mut self, bytes: &[u8]) { debug_assert!( bytes.len() >= std::mem::size_of::(), "ThrottleKeyHasher: input too short {bytes:?}" ); self.hash = bytes .get(0..std::mem::size_of::()) .map_or(0, |b| u64::from_ne_bytes(b.try_into().unwrap())); } } #[derive(Clone, Default)] pub struct ThrottleKeyHasherBuilder {} impl BuildHasher for ThrottleKeyHasherBuilder { type Hasher = ThrottleKeyHasher; fn build_hasher(&self) -> Self::Hasher { ThrottleKeyHasher::default() } } #[cfg(feature = "test_mode")] #[allow(clippy::derivable_impls)] impl Default for Server { fn default() -> Self { Self { inner: Default::default(), core: Default::default(), } } } #[cfg(feature = "test_mode")] #[allow(clippy::derivable_impls)] impl Default for Inner { fn default() -> Self { Self { shared_core: Default::default(), data: Default::default(), ipc: Default::default(), cache: Default::default(), } } } #[cfg(feature = "test_mode")] #[allow(clippy::derivable_impls)] impl Default for Caches { fn default() -> Self { Self { access_tokens: Cache::new(1024, 10 * 1024 * 1024), http_auth: Cache::new(1024, 10 * 1024 * 1024), permissions: Cache::new(1024, 10 * 1024 * 1024), messages: Cache::new(1024, 25 * 1024 * 1024), files: Cache::new(1024, 10 * 1024 * 1024), contacts: Cache::new(1024, 10 * 1024 * 1024), events: Cache::new(1024, 10 * 1024 * 1024), scheduling: Cache::new(1024, 10 * 1024 * 1024), dns_rbl: CacheWithTtl::new(1024, 10 * 1024 * 1024), dns_txt: CacheWithTtl::new(1024, 10 * 1024 * 1024), dns_mx: CacheWithTtl::new(1024, 10 * 1024 * 1024), dns_ptr: CacheWithTtl::new(1024, 10 * 1024 * 1024), dns_ipv4: CacheWithTtl::new(1024, 10 * 1024 * 1024), dns_ipv6: CacheWithTtl::new(1024, 10 * 1024 * 1024), dns_tlsa: CacheWithTtl::new(1024, 10 * 1024 * 1024), dbs_mta_sts: CacheWithTtl::new(1024, 10 * 1024 * 1024), } } } #[cfg(feature = "test_mode")] impl Default for Ipc { fn default() -> Self { Self { push_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, housekeeper_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, task_tx: Default::default(), queue_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, report_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, broadcast_tx: None, train_task_controller: Arc::new(TrainTaskController::default()), } } } pub fn ip_to_bytes(ip: &IpAddr) -> Vec { match ip { IpAddr::V4(ip) => ip.octets().to_vec(), IpAddr::V6(ip) => ip.octets().to_vec(), } } pub fn ip_to_bytes_prefix(prefix: u8, ip: &IpAddr) -> Vec { match ip { IpAddr::V4(ip) => { let mut buf = Vec::with_capacity(5); buf.push(prefix); buf.extend_from_slice(&ip.octets()); buf } IpAddr::V6(ip) => { let mut buf = Vec::with_capacity(17); buf.push(prefix); buf.extend_from_slice(&ip.octets()); buf } } } impl DavResourcePath<'_> { #[inline(always)] pub fn document_id(&self) -> u32 { self.resource.document_id } #[inline(always)] pub fn parent_id(&self) -> Option { self.path.parent_id } #[inline(always)] pub fn path(&self) -> &str { self.path.path.as_str() } #[inline(always)] pub fn is_container(&self) -> bool { self.resource.is_container() } #[inline(always)] pub fn hierarchy_seq(&self) -> u32 { self.path.hierarchy_seq } #[inline(always)] pub fn size(&self) -> u32 { self.resource.size().unwrap_or_default() } } impl DavResources { pub fn by_path(&self, name: &str) -> Option> { self.paths.get(name).map(|path| DavResourcePath { path, resource: &self.resources[path.resource_idx], }) } pub fn container_resource_by_id(&self, id: u32) -> Option<&DavResource> { self.resources .iter() .find(|res| res.document_id == id && res.is_container()) } pub fn container_resource_path_by_id(&self, id: u32) -> Option> { self.resources .iter() .enumerate() .find(|(_, resource)| resource.document_id == id && resource.is_container()) .and_then(|(idx, resource)| { self.paths .iter() .find(|path| path.resource_idx == idx) .map(|path| DavResourcePath { path, resource }) }) } pub fn any_resource_path_by_id(&self, id: u32) -> Option> { self.resources .iter() .enumerate() .find(|(_, resource)| resource.document_id == id) .and_then(|(idx, resource)| { self.paths .iter() .find(|path| path.resource_idx == idx) .map(|path| DavResourcePath { path, resource }) }) } pub fn subtree(&self, search_path: &str) -> impl Iterator> { let prefix = format!("{search_path}/"); self.paths.iter().filter_map(move |path| { if path.path.starts_with(&prefix) || path.path == search_path { Some(DavResourcePath { path, resource: &self.resources[path.resource_idx], }) } else { None } }) } pub fn subtree_with_depth( &self, search_path: &str, depth: usize, ) -> impl Iterator> { let prefix = format!("{search_path}/"); self.paths.iter().filter_map(move |path| { if path .path .strip_prefix(&prefix) .is_some_and(|name| name.as_bytes().iter().filter(|&&c| c == b'/').count() < depth) || path.path.as_str() == search_path { Some(DavResourcePath { path, resource: &self.resources[path.resource_idx], }) } else { None } }) } pub fn tree_with_depth(&self, depth: usize) -> impl Iterator> { self.paths.iter().filter_map(move |path| { if path.path.as_bytes().iter().filter(|&&c| c == b'/').count() <= depth { Some(DavResourcePath { path, resource: &self.resources[path.resource_idx], }) } else { None } }) } pub fn children(&self, parent_id: u32) -> impl Iterator> { self.paths .iter() .filter(move |item| item.parent_id.is_some_and(|id| id == parent_id)) .map(|path| DavResourcePath { path, resource: &self.resources[path.resource_idx], }) } pub fn children_ids(&self, parent_id: u32) -> impl Iterator { self.paths .iter() .filter(move |item| item.parent_id.is_some_and(|id| id == parent_id)) .map(|path| self.resources[path.resource_idx].document_id) } pub fn format_resource(&self, resource: DavResourcePath<'_>) -> String { if resource.resource.is_container() { format!("{}{}/", self.base_path, resource.path.path) } else { format!("{}{}", self.base_path, resource.path.path) } } pub fn format_collection(&self, name: &str) -> String { format!("{}{name}/", self.base_path) } pub fn format_item(&self, name: &str) -> String { format!("{}{}", self.base_path, name) } } const SCHEDULE_INBOX_ID: u32 = u32::MAX - 1; impl DavResource { pub fn is_child_of(&self, parent_id: u32) -> bool { match &self.data { DavResourceMetadata::File { parent_id: id, .. } => id.is_some_and(|id| id == parent_id), DavResourceMetadata::CalendarEvent { names, .. } => { names.iter().any(|name| name.parent_id == parent_id) } DavResourceMetadata::ContactCard { names } => { names.iter().any(|name| name.parent_id == parent_id) } DavResourceMetadata::CalendarEventNotification { names } => { names.is_empty() && parent_id == SCHEDULE_INBOX_ID } _ => false, } } pub fn parent_id(&self) -> Option { match &self.data { DavResourceMetadata::File { parent_id, .. } => *parent_id, DavResourceMetadata::CalendarEvent { names, .. } => { names.first().map(|name| name.parent_id) } DavResourceMetadata::ContactCard { names } => names.first().map(|name| name.parent_id), DavResourceMetadata::CalendarEventNotification { names } if names.is_empty() => { Some(SCHEDULE_INBOX_ID) } _ => None, } } pub fn child_names(&self) -> Option<&[DavName]> { match &self.data { DavResourceMetadata::CalendarEvent { names, .. } => Some(names.as_slice()), DavResourceMetadata::ContactCard { names } => Some(names.as_slice()), DavResourceMetadata::CalendarEventNotification { names } if !names.is_empty() => { Some(names.as_slice()) } _ => None, } } pub fn container_name(&self) -> Option<&str> { match &self.data { DavResourceMetadata::File { name, .. } => Some(name.as_str()), DavResourceMetadata::Calendar { name, .. } => Some(name.as_str()), DavResourceMetadata::AddressBook { name, .. } => Some(name.as_str()), DavResourceMetadata::CalendarEventNotification { names } if names.is_empty() => { Some(if self.document_id == SCHEDULE_INBOX_ID { "inbox" } else { "outbox" }) } _ => None, } } pub fn has_hierarchy_changes(&self, other: &DavResource) -> bool { match (&self.data, &other.data) { ( DavResourceMetadata::File { name: a, parent_id: c, .. }, DavResourceMetadata::File { name: b, parent_id: d, .. }, ) => a != b || c != d, ( DavResourceMetadata::Calendar { name: a, .. }, DavResourceMetadata::Calendar { name: b, .. }, ) => a != b, ( DavResourceMetadata::AddressBook { name: a, .. }, DavResourceMetadata::AddressBook { name: b, .. }, ) => a != b, ( DavResourceMetadata::CalendarEvent { names: a, .. }, DavResourceMetadata::CalendarEvent { names: b, .. }, ) => a != b, ( DavResourceMetadata::ContactCard { names: a, .. }, DavResourceMetadata::ContactCard { names: b, .. }, ) => a != b, ( DavResourceMetadata::CalendarEventNotification { names: a, .. }, DavResourceMetadata::CalendarEventNotification { names: b, .. }, ) => a != b, _ => unreachable!(), } } pub fn event_time_range(&self) -> Option<(i64, i64)> { match &self.data { DavResourceMetadata::CalendarEvent { start, duration, .. } => Some((*start, *start + *duration as i64)), _ => None, } } pub fn calendar_preferences(&self, account_id: u32) -> Option<&TinyCalendarPreferences> { match &self.data { DavResourceMetadata::Calendar { preferences, .. } => preferences .iter() .find(|pref| pref.account_id == account_id) .or_else(|| preferences.first()), _ => None, } } pub fn is_container(&self) -> bool { match &self.data { DavResourceMetadata::File { size, .. } => size.is_none(), DavResourceMetadata::Calendar { .. } | DavResourceMetadata::AddressBook { .. } => true, DavResourceMetadata::CalendarEventNotification { names } => names.is_empty(), _ => false, } } pub fn size(&self) -> Option { match &self.data { DavResourceMetadata::File { size, .. } => *size, _ => None, } } pub fn acls(&self) -> Option<&[AclGrant]> { match &self.data { DavResourceMetadata::File { acls, .. } => Some(acls.as_slice()), DavResourceMetadata::Calendar { acls, .. } => Some(acls.as_slice()), DavResourceMetadata::AddressBook { acls, .. } => Some(acls.as_slice()), _ => None, } } } impl Hash for DavPath { fn hash(&self, state: &mut H) { self.path.hash(state); } } impl PartialEq for DavPath { fn eq(&self, other: &Self) -> bool { self.path == other.path } } impl Eq for DavPath {} impl std::borrow::Borrow for DavPath { fn borrow(&self) -> &str { &self.path } } impl std::hash::Hash for DavResource { fn hash(&self, state: &mut H) { self.document_id.hash(state); } } impl PartialEq for DavResource { fn eq(&self, other: &Self) -> bool { self.document_id == other.document_id } } impl Eq for DavResource {} impl std::borrow::Borrow for DavResource { fn borrow(&self) -> &u32 { &self.document_id } } impl DavName { pub fn new(name: String, parent_id: u32) -> Self { Self { name, parent_id } } pub fn new_with_rand_name(parent_id: u32) -> Self { Self { name: store::rand::rng() .sample_iter(Alphanumeric) .take(10) .map(char::from) .collect::(), parent_id, } } } impl CacheSwap { pub fn new(value: Arc) -> Self { Self(Arc::new(ArcSwap::new(value))) } pub fn load_full(&self) -> Arc { self.0.load_full() } pub fn update(&self, value: Arc) { self.0.store(value); } } impl MailboxCache { pub fn parent_id(&self) -> Option { if self.parent_id != u32::MAX { Some(self.parent_id) } else { None } } pub fn sort_order(&self) -> Option { if self.sort_order != u32::MAX { Some(self.sort_order) } else { None } } pub fn is_root(&self) -> bool { self.parent_id == u32::MAX } } pub const DEFAULT_LOGO_RAW: &str = r#" "#; pub const DEFAULT_LOGO_BASE64: &str = "iVBORw0KGgoAAAANSUhEUgAAAMgAAAAnCAMAAAB9lPf7AAABOFBMVEUAAAAAADoAAEkPDkIPDkIQ DkIQDkLcLVTYMVTbLVTbLVQPDkLWM2YQDkIPD0IODEHaJlPbLVQWDT8MC0IQDkIQDkIPDkIODkEQ D0ITDEAQDkIPDkIODkIPDkIPDkIRDUIQDkIQDUIPDkLcLVQQDkIQDkIQDkIQDULbLVQQDUIPDkIQ EEIPDkIPD0EPDkHcLlUPDkLbLFQPDULbLFPbLVQQDkLbLFPcK1TbLVTbLVQQDUIPDkIPDkIPDULc LVTcLVQQDkLbLFTcLFTbLFQPDkIQDkIQDULbLVUSEkPcLVTbLVQQDULbLFTcLVTbLVTbLVPbLFQP DUHbLVMPDkPbLVTdLFXbLFQNDULcLVTcLlMRD0cQDkIQDkXpMFnrMFrtMFvlL1jjLlfdLVThLlYS EErnL1gRD0n0Ml2YjG1wAAAAWnRSTlMABAb59/379wr7/lMF9HgoBvQJF/BtZSQwGu7JNulAO95x WD3TsF1M3b6kH5MRiRe5saujyH9oHezk4tjGmn9fwkc1vrVqYA8N19DPqnFQkYh1V0a4LSITmSiJ LN30AAAKZUlEQVRYw91ZCVvbRhCVrcuWD7AxYLANtrEx5opDIBCOQBLO5qTQ0uyubyD//x/0za4O RIG0/dLvazu0tbTaWc3bOd6sqv2PJen9/LcF9o++fnf58r8OBcb/vNBttzvrc0Ck/VcFMN6+a/fs uG23+x9+fcopkUiE/ks/35ew2j8ucMCz3wbteNwZtIcZp2O/jj3mlWjo+t8nHzNdx3ac/sLrTDee GXRf7cMpD+OYnF9dO6zvnc/gOuLv93dBRScTkz/KWrwu+mBUfTrrwhHtod1b0J696N/E7T5S5UEo xbrBlIidyp90CvCO1cu56you/jFBAP1y2evZdq/z4suwv3CiaXPr7X7Gbg9fbP4xvtKccSNlGEYq ZQpWG3GRTM2ki49biUkjgpks8aOAZCcT6ZX7ME4+Izkyw85Pn7T9m+FCjMb2X3UHSJXM62TIKxFt lXHdEPAG/jUNnelFQjJVM8T1BK4e98hIiqd+DBCskcgZjE/dW+zlFiVHd+sjXfcICEnsS7wTtwfd hZ8DJFBsMNPirLxabSQqTSFMQ5RWyErBDTH/NBCdGz8MyPm1wUtjocXe/NQZZuLtwednMN4FkpSO +vqhh1Tp3bx7qwXyXuhcLLsLNHJAwvZg/6TBLRYGEoE8AOTplAqrPx6nR8wQ01N3Fht90evF4+2b y1+k7QEQLTkqQfbt+M3wi29NUZgWO8RSJLg1ucVTY6hj5j0geCh//pZHAvX7s+nNtOr4fSCbw0Hc cdbd8ElKIIESMf3AsftbSW8nJphusqqWdTMOGQNgVRdINUQvkZWrLP24Ix6Q74qn46kHEtwqIKda IJtbDgQ4YpoLxMk89+Xy+W/rAyc+OPOBpBki6wIX7v2s4CZblUBMXjtstgoNLYK/6PxarVQqt8Zh zEThEKMKSCPaah4WKporK/Vm6yCP+SqDC61ma0paW10k9SYVpirUq3Jqcw23F8tr0K8wg+sHrSbN l/LMadvxwfDyKyKL/vn4DQSipNdG19XtO3G7+8rf6Ap5pKJlPZ9H8/l8ccoFQrXsukIoG2Vcco6R UkNrXfPrI83zSA135YhXOm6hseHt8yEe1Wh4puapT89rixjdQF3kGFvKHtA7FskOeow/RbGKCG2n 7Xw5kdG1+bzXaUvp9uIkjiRGH0iCWSafzitmDUhdATFNpP4RZm0wYRm6EFTKxHxBpMSEC6QKE1I6 B+NItWWWMsSupmRsmudok7QjqoaGVOdsoiVS7Fw+tfhx4dpI5cSyAoL3cWPyTmvScWx0JC9din+h 5POH+MBx4rJVCWTKgLrQV4tZt7QADWxSQCAWgCCATcsSorxbKDGMUUELgOQt/KSBltR3hMW5vuRG Fowz84glMJUl2PbuLtRNqOssDSA5WJ7CdeAREssDEqNmcQg+7Ldp42PJoCwPHLgq/iWG0WDrl29T WJzx7cONxEjUTz9KdpFeQqCNaXm80WKFGWBdmS8JHdABxE/2XWGwA68GcuBgx4CFu0WWYvDOEqw1 We0C6qeJMtR9ILRVopmeqMxQsvPczCTeB9UACrXvGftGNu8xd6SD0PJHAokc3ILPLcoHxsuL1RUM uUC8klRnusXqXryUsekhIGkGnyxBi0JQJycuyiWy20JavMcMizW9TgS8FQJSfaRqPfuEFB+VB6qB HVfNu+sjGz3LG8DA0093XBJdRIaZOgmhKW1g0OeRLHEjDBZlTJSGSP+EgOTJdtXN7FJkmWI7S+l3 IWBoXpvKcZ26haxSh39MH4iOZEFqRh7ikcE7UGGMjrh2x3Gb949bXVx2tl5iGI/mzohHgkI+u1gS sqpYhoFYLqxo0YAQI0Q1iJagQi8y/S6QCJoDg61pEQlZ7BSELmaI5pZhW0GT5QSGZz31VWZ5QHRA VlXCAxIJPJLpqOYEmF4M2zZS5d16Fz1Lx/l8Iod//TDsnmlhOb3YqO+kBMBYpkFxfRfIHkuJ3NSd 7iwMJIuAMtSWTyApNiZuU2yV7N4RBhm8qlI+YKogRwwZhA8D2cygXfS3fl12JNTQ954rR8Veo3V0 FkbvdkBu2V1qrL5HncTy46EW5RDm1QLQ8lEABFPyVPgasL3ODFacFJbYpcTnkBGNHCi2Tz0TI1R2 fY/IaveYR7po4LuUDKNu827b6j6WdJv5+Dd4JCwemEZJwKhCCMgBgBz4M5UFodBCbKXYsqZdlbjY jmo1IclgA2oEqA4gO8GLqAYEQMYfBYIjleuBrzBdNu/tNjX0SSpdOF5RMcPxypVssUgFNjhrFok/ SlfaiOUDacKiwl3qSYWBRMhmUctqDYHOmWLJYMdIfKhRaqwBSPlOr3la+nNAvENuB9wuoXzd3/dy BgfeTMDrpL9kCo5AjfqOgdk6zy3dBbIGM8ungQ5VqRAQFUVFmd3owWYo92kB1CwA2aMFx4LQWjL4 nwIympSfHRS3K0dIGMnX7uA+BZ3vEbCCKAdNaJQ2kE+PBUAitMMwM+h6j8LJTuM12XXsCF46pTW5 KMnEL+BGcrZoBMleFeb3gYS5XW1+LBaTvNIhN4V5XUWwicYp6582doCshhxxgeCv4R22ICojrDCQ qIymwxGhUxWKkg9EHrROqeyeeOq4UrUFLrceBxINkv1jUkv63N6TTC7v+rbP9JgUC5pGHXVqlq6j tEqF9mtZJrvOqpIQT7cFtfowEUJJbPH7QGaJBfdAIChiWFMYYrEMXlJFt0bq85qm1I+Y+Vho5a5k BCjZ7C/c5/ZRfKgL8fp+JqP5sssMU5jpK1WQVgU3qWhSJFOkS0lLsG6/soGyFgaimkWTQ6ZX5KFk mq5l8dMUoeLhsVI/4vxBIJjExXHoYNXuXXqUobh94VWY19fbnbNk0K8bwtA5m24try63cozrqrhH y7SP7+v11h7CDWC5aI03EukaQ3m4D4RYXPaCa3QXkXkWkIR2INUPKonEeUFA/SEgM7Q/VrNeb07g joDEbfrwcBJwu90P83q/ZzuvRoMzbDHHcFjgjISncIHAIj6/zYFRmLgFM0xOM90wGROMERuDskNA lB2cYlG1vccEhCIroihwG+oW1AWDZr0mjHtAVIeZMjm19Mtq6GSBPgXJ7deSHrcrXh9VTsInoW/v 7n7AWGoJOjfRNzpYnhpXWzRVuuVwj8Fr5Lb3DCwH0QXbI4aUQAw/tKQDLaqyEBmXpi4K/hvGcAz0 1NdQX1J+izLu15AqY1DSU2LVHXr22ekPB+1vZ59wI7m815M8j7uXW996g2Evg5Y4EKhdrJWE8sjO xpRf79emucAmluWU8RqXE94nNDqr3tJRlwt+W/WOhtecX9e9NZu3uEsH3KEdF6S62DnWaOo1AdGh XgnqVKNg4H3c8wjk69zc3Nu3b+ZG3c+Oc3SVJG+9efP2LR5u/vFDxkqxOn5+lMhTwPujK/nZ2dmZ vDslX61U5vMS4szsDPY+S0+vvL7lAoNeZ4kZeHQaesNIYvz8uCg7AzUzWpwNNJRWkVZceur/vSUf GAtDCZrI0KAvoeG/Lt9Xx5MfIhFiicj9QSmhGd649zg098nPik+rB+/T/k/yO9A7bEvKcQkCAAAA AElFTkSuQmCC"; ================================================ FILE: crates/common/src/listener/acme/cache.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; use trc::AddContext; use utils::config::ConfigKey; use crate::Server; use super::AcmeProvider; impl Server { pub(crate) async fn load_cert(&self, provider: &AcmeProvider) -> trc::Result>> { self.read_if_exists(provider, "cert", provider.domains.as_slice()) .await .add_context(|err| { err.caused_by(trc::location!()) .details("Failed to load certificates") }) } pub(crate) async fn store_cert(&self, provider: &AcmeProvider, cert: &[u8]) -> trc::Result<()> { self.write(provider, "cert", provider.domains.as_slice(), cert) .await .add_context(|err| { err.caused_by(trc::location!()) .details("Failed to store certificate") }) } pub(crate) async fn load_account( &self, provider: &AcmeProvider, ) -> trc::Result>> { self.read_if_exists(provider, "account-key", provider.contact.as_slice()) .await .add_context(|err| { err.caused_by(trc::location!()) .details("Failed to load account") }) } pub(crate) async fn store_account( &self, provider: &AcmeProvider, account: &[u8], ) -> trc::Result<()> { self.write( provider, "account-key", provider.contact.as_slice(), account, ) .await .add_context(|err| { err.caused_by(trc::location!()) .details("Failed to store account") }) } async fn read_if_exists( &self, provider: &AcmeProvider, class: &str, items: &[String], ) -> trc::Result>> { if let Some(content) = self .core .storage .config .get(self.build_key(provider, class, items)) .await? { URL_SAFE_NO_PAD .decode(content.as_bytes()) .map_err(|err| { trc::EventType::Acme(trc::AcmeEvent::Error) .caused_by(trc::location!()) .reason(err) .details("failed to decode certificate") }) .map(Some) } else { Ok(None) } } async fn write( &self, provider: &AcmeProvider, class: &str, items: &[String], contents: impl AsRef<[u8]>, ) -> trc::Result<()> { self.core .storage .config .set( [ConfigKey { key: self.build_key(provider, class, items), value: URL_SAFE_NO_PAD.encode(contents.as_ref()), }], true, ) .await } fn build_key(&self, provider: &AcmeProvider, class: &str, _: &[String]) -> String { /*let mut ctx = Context::new(&SHA512); for el in items { ctx.update(el.as_ref()); ctx.update(&[0]) } ctx.update(provider.directory_url.as_bytes()); format!( "certificate.acme-{}-{}.{}", provider.id, URL_SAFE_NO_PAD.encode(ctx.finish()), class )*/ format!("acme.{}.{}", provider.id, class) } } ================================================ FILE: crates/common/src/listener/acme/directory.rs ================================================ // Adapted from rustls-acme (https://github.com/FlorianUekermann/rustls-acme), licensed under MIT/Apache-2.0. use super::AcmeProvider; use super::jose::{ Body, eab_sign, key_authorization, key_authorization_sha256, key_authorization_sha256_base64, sign, }; use base64::Engine; use base64::engine::general_purpose::URL_SAFE_NO_PAD; use hyper::header::USER_AGENT; use rcgen::{Certificate, CustomExtension, PKCS_ECDSA_P256_SHA256}; use reqwest::header::CONTENT_TYPE; use reqwest::{Method, Response}; use ring::rand::SystemRandom; use ring::signature::{ECDSA_P256_SHA256_FIXED_SIGNING, EcdsaKeyPair, EcdsaSigningAlgorithm}; use serde::Deserialize; use std::time::Duration; use store::Serialize; use store::write::Archiver; use trc::AddContext; use trc::event::conv::AssertSuccess; pub const LETS_ENCRYPT_STAGING_DIRECTORY: &str = "https://acme-staging-v02.api.letsencrypt.org/directory"; pub const LETS_ENCRYPT_PRODUCTION_DIRECTORY: &str = "https://acme-v02.api.letsencrypt.org/directory"; pub const ACME_TLS_ALPN_NAME: &[u8] = b"acme-tls/1"; #[derive(Debug)] pub struct Account { pub key_pair: EcdsaKeyPair, pub directory: Directory, pub kid: String, } #[derive(Debug, serde::Serialize)] pub struct NewAccountPayload<'x> { #[serde(rename = "termsOfServiceAgreed")] tos_agreed: bool, contact: &'x [String], #[serde(rename = "externalAccountBinding")] #[serde(skip_serializing_if = "Option::is_none")] eab: Option, } static ALG: &EcdsaSigningAlgorithm = &ECDSA_P256_SHA256_FIXED_SIGNING; impl Account { pub fn generate_key_pair() -> Vec { EcdsaKeyPair::generate_pkcs8(ALG, &SystemRandom::new()) .unwrap() .as_ref() .to_vec() } pub async fn create(directory: Directory, provider: &AcmeProvider) -> trc::Result { Self::create_with_keypair(directory, provider).await } pub async fn create_with_keypair( directory: Directory, provider: &AcmeProvider, ) -> trc::Result { let key_pair = EcdsaKeyPair::from_pkcs8( ALG, provider.account_key.load().as_slice(), &SystemRandom::new(), ) .map_err(|err| { trc::EventType::Acme(trc::AcmeEvent::Error) .reason(err) .caused_by(trc::location!()) })?; let eab = if let Some(eab) = &provider.eab { eab_sign(&key_pair, &eab.kid, &eab.hmac_key, &directory.new_account) .caused_by(trc::location!())? .into() } else { None }; let payload = serde_json::to_string(&NewAccountPayload { tos_agreed: true, contact: &provider.contact, eab, }) .unwrap_or_default(); let body = sign( &key_pair, None, directory.nonce().await?, &directory.new_account, &payload, )?; let response = https(&directory.new_account, Method::POST, Some(body)).await?; let kid = get_header(&response, "Location")?; Ok(Account { key_pair, kid, directory, }) } async fn request( &self, url: impl AsRef, payload: &str, ) -> trc::Result<(Option, String)> { let body = sign( &self.key_pair, Some(&self.kid), self.directory.nonce().await?, url.as_ref(), payload, )?; let response = https(url.as_ref(), Method::POST, Some(body)).await?; let location = get_header(&response, "Location").ok(); let body = response .text() .await .map_err(|err| trc::EventType::Acme(trc::AcmeEvent::Error).from_http_error(err))?; Ok((location, body)) } pub async fn new_order(&self, domains: Vec) -> trc::Result<(String, Order)> { let domains: Vec = domains.into_iter().map(Identifier::Dns).collect(); let payload = format!( "{{\"identifiers\":{}}}", serde_json::to_string(&domains) .map_err(|err| trc::EventType::Acme(trc::AcmeEvent::Error).from_json_error(err))? ); let response = self.request(&self.directory.new_order, &payload).await?; let url = response.0.ok_or( trc::EventType::Acme(trc::AcmeEvent::Error) .caused_by(trc::location!()) .details("Missing header") .ctx(trc::Key::Id, "Location"), )?; let order = serde_json::from_str(&response.1) .map_err(|err| trc::EventType::Acme(trc::AcmeEvent::Error).from_json_error(err))?; Ok((url, order)) } pub async fn auth(&self, url: impl AsRef) -> trc::Result { let response = self.request(url, "").await?; serde_json::from_str(&response.1) .map_err(|err| trc::EventType::Acme(trc::AcmeEvent::Error).from_json_error(err)) } pub async fn challenge(&self, url: impl AsRef) -> trc::Result<()> { self.request(&url, "{}").await.map(|_| ()) } pub async fn order(&self, url: impl AsRef) -> trc::Result { let response = self.request(&url, "").await?; serde_json::from_str(&response.1) .map_err(|err| trc::EventType::Acme(trc::AcmeEvent::Error).from_json_error(err)) } pub async fn finalize(&self, url: impl AsRef, csr: Vec) -> trc::Result { let payload = format!("{{\"csr\":\"{}\"}}", URL_SAFE_NO_PAD.encode(csr)); let response = self.request(&url, &payload).await?; serde_json::from_str(&response.1) .map_err(|err| trc::EventType::Acme(trc::AcmeEvent::Error).from_json_error(err)) } pub async fn certificate(&self, url: impl AsRef) -> trc::Result { Ok(self.request(&url, "").await?.1) } pub fn http_proof(&self, challenge: &Challenge) -> trc::Result> { key_authorization(&self.key_pair, &challenge.token).map(|key| key.into_bytes()) } pub fn dns_proof(&self, challenge: &Challenge) -> trc::Result { key_authorization_sha256_base64(&self.key_pair, &challenge.token) } pub fn tls_alpn_key(&self, challenge: &Challenge, domain: String) -> trc::Result> { let mut params = rcgen::CertificateParams::new(vec![domain]); let key_auth = key_authorization_sha256(&self.key_pair, &challenge.token)?; params.alg = &PKCS_ECDSA_P256_SHA256; params.custom_extensions = vec![CustomExtension::new_acme_identifier(key_auth.as_ref())]; let cert = Certificate::from_params(params).map_err(|err| { trc::EventType::Acme(trc::AcmeEvent::Error) .caused_by(trc::location!()) .reason(err) })?; Archiver::new(SerializedCert { certificate: cert.serialize_der().map_err(|err| { trc::EventType::Acme(trc::AcmeEvent::Error) .caused_by(trc::location!()) .reason(err) })?, private_key: cert.serialize_private_key_der(), }) .untrusted() .serialize() } } #[derive( rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Clone, serde::Serialize, Deserialize, )] pub struct SerializedCert { pub certificate: Vec, pub private_key: Vec, } #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Directory { pub new_nonce: String, pub new_account: String, pub new_order: String, } impl Directory { pub async fn discover(url: impl AsRef) -> trc::Result { serde_json::from_str( &https(url, Method::GET, None) .await? .text() .await .map_err(|err| trc::EventType::Acme(trc::AcmeEvent::Error).from_http_error(err))?, ) .map_err(|err| trc::EventType::Acme(trc::AcmeEvent::Error).from_json_error(err)) } pub async fn nonce(&self) -> trc::Result { get_header( &https(&self.new_nonce.as_str(), Method::HEAD, None).await?, "replay-nonce", ) } } #[derive(Debug, Deserialize, Eq, PartialEq, Clone, Copy)] pub enum ChallengeType { #[serde(rename = "http-01")] Http01, #[serde(rename = "dns-01")] Dns01, #[serde(rename = "tls-alpn-01")] TlsAlpn01, #[serde(other)] Unknown, } #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Order { #[serde(flatten)] pub status: OrderStatus, pub authorizations: Vec, pub finalize: String, pub error: Option, } #[derive(Debug, Deserialize, Clone, PartialEq, Eq)] #[serde(tag = "status", rename_all = "camelCase")] pub enum OrderStatus { Pending, Ready, Valid { certificate: String }, Invalid, Processing, } #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Auth { pub status: AuthStatus, pub identifier: Identifier, pub challenges: Vec, pub wildcard: Option, } #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub enum AuthStatus { Pending, Valid, Invalid, Revoked, Expired, Deactivated, } #[derive(Clone, Debug, serde::Serialize, Deserialize)] #[serde(tag = "type", content = "value", rename_all = "camelCase")] pub enum Identifier { Dns(String), } #[derive(Debug, Deserialize)] pub struct Challenge { #[serde(rename = "type")] pub typ: ChallengeType, pub url: String, pub token: String, pub error: Option, } #[derive(Clone, Debug, serde::Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Problem { #[serde(rename = "type")] pub typ: Option, pub detail: Option, } #[allow(unused_mut)] async fn https( url: impl AsRef, method: Method, body: Option, ) -> trc::Result { let url = url.as_ref(); let mut builder = reqwest::Client::builder() .timeout(Duration::from_secs(30)) .http1_only(); #[cfg(debug_assertions)] { builder = builder.danger_accept_invalid_certs( url.starts_with("https://localhost") || url.starts_with("https://127.0.0.1"), ); } let mut request = builder .build() .map_err(|err| trc::EventType::Acme(trc::AcmeEvent::Error).from_http_error(err))? .request(method, url) .header(USER_AGENT, crate::USER_AGENT); if let Some(body) = body { request = request .header(CONTENT_TYPE, "application/jose+json") .body(body); } request .send() .await .map_err(|err| trc::EventType::Acme(trc::AcmeEvent::Error).from_http_error(err))? .assert_success(trc::EventType::Acme(trc::AcmeEvent::Error)) .await } fn get_header(response: &Response, header: &'static str) -> trc::Result { match response.headers().get_all(header).iter().next_back() { Some(value) => Ok(value .to_str() .map_err(|err| trc::EventType::Acme(trc::AcmeEvent::Error).from_http_str_error(err))? .to_string()), None => Err(trc::EventType::Acme(trc::AcmeEvent::Error) .caused_by(trc::location!()) .details("Missing header") .ctx(trc::Key::Id, header)), } } impl ChallengeType { pub fn as_str(&self) -> &'static str { match self { Self::Http01 => "http-01", Self::Dns01 => "dns-01", Self::TlsAlpn01 => "tls-alpn-01", Self::Unknown => "unknown", } } } impl AuthStatus { pub fn as_str(&self) -> &'static str { match self { Self::Pending => "pending", Self::Valid => "valid", Self::Invalid => "invalid", Self::Revoked => "revoked", Self::Expired => "expired", Self::Deactivated => "deactivated", } } } ================================================ FILE: crates/common/src/listener/acme/jose.rs ================================================ // Adapted from rustls-acme (https://github.com/FlorianUekermann/rustls-acme), licensed under MIT/Apache-2.0. use base64::Engine; use base64::engine::general_purpose::URL_SAFE_NO_PAD; use ring::digest::{Digest, SHA256, digest}; use ring::hmac; use ring::rand::SystemRandom; use ring::signature::{EcdsaKeyPair, KeyPair}; use serde::Serialize; pub(crate) fn sign( key: &EcdsaKeyPair, kid: Option<&str>, nonce: String, url: &str, payload: &str, ) -> trc::Result { let jwk = match kid { None => Some(Jwk::new(key)), Some(_) => None, }; let protected = Protected::encode("ES256", jwk, kid, nonce.into(), url)?; let payload = URL_SAFE_NO_PAD.encode(payload); let combined = format!("{}.{}", &protected, &payload); let signature = key .sign(&SystemRandom::new(), combined.as_bytes()) .map_err(|err| { trc::EventType::Acme(trc::AcmeEvent::Error) .caused_by(trc::location!()) .reason(err) })?; serde_json::to_string(&Body { protected, payload, signature: URL_SAFE_NO_PAD.encode(signature.as_ref()), }) .map_err(|err| trc::EventType::Acme(trc::AcmeEvent::Error).from_json_error(err)) } pub(crate) fn eab_sign( key: &EcdsaKeyPair, kid: &str, hmac_key: &[u8], url: &str, ) -> trc::Result { let protected = Protected::encode("HS256", None, kid.into(), None, url)?; let payload = Jwk::new(key).base64()?; let combined = format!("{}.{}", &protected, &payload); let key = hmac::Key::new(hmac::HMAC_SHA256, hmac_key); let tag = hmac::sign(&key, combined.as_bytes()); let signature = URL_SAFE_NO_PAD.encode(tag.as_ref()); Ok(Body { protected, payload, signature, }) } pub(crate) fn key_authorization(key: &EcdsaKeyPair, token: &str) -> trc::Result { Ok(format!( "{}.{}", token, Jwk::new(key).thumb_sha256_base64()? )) } pub(crate) fn key_authorization_sha256(key: &EcdsaKeyPair, token: &str) -> trc::Result { key_authorization(key, token).map(|s| digest(&SHA256, s.as_bytes())) } pub(crate) fn key_authorization_sha256_base64( key: &EcdsaKeyPair, token: &str, ) -> trc::Result { key_authorization_sha256(key, token).map(|s| URL_SAFE_NO_PAD.encode(s.as_ref())) } #[derive(Debug, Serialize)] pub(crate) struct Body { protected: String, payload: String, signature: String, } #[derive(Serialize)] struct Protected<'a> { alg: &'static str, #[serde(skip_serializing_if = "Option::is_none")] jwk: Option, #[serde(skip_serializing_if = "Option::is_none")] kid: Option<&'a str>, #[serde(skip_serializing_if = "Option::is_none")] nonce: Option, url: &'a str, } impl<'a> Protected<'a> { fn encode( alg: &'static str, jwk: Option, kid: Option<&'a str>, nonce: Option, url: &'a str, ) -> trc::Result { serde_json::to_vec(&Protected { alg, jwk, kid, nonce, url, }) .map_err(|err| trc::EventType::Acme(trc::AcmeEvent::Error).from_json_error(err)) .map(|v| URL_SAFE_NO_PAD.encode(v.as_slice())) } } #[derive(Serialize)] struct Jwk { alg: &'static str, crv: &'static str, kty: &'static str, #[serde(rename = "use")] u: &'static str, x: String, y: String, } impl Jwk { pub(crate) fn new(key: &EcdsaKeyPair) -> Self { let (x, y) = key.public_key().as_ref()[1..].split_at(32); Self { alg: "ES256", crv: "P-256", kty: "EC", u: "sig", x: URL_SAFE_NO_PAD.encode(x), y: URL_SAFE_NO_PAD.encode(y), } } pub(crate) fn base64(&self) -> trc::Result { serde_json::to_vec(self) .map_err(|err| trc::EventType::Acme(trc::AcmeEvent::Error).from_json_error(err)) .map(|v| URL_SAFE_NO_PAD.encode(v.as_slice())) } pub(crate) fn thumb_sha256_base64(&self) -> trc::Result { Ok(URL_SAFE_NO_PAD.encode(digest( &SHA256, &serde_json::to_vec(&JwkThumb { crv: self.crv, kty: self.kty, x: &self.x, y: &self.y, }) .map_err(|err| trc::EventType::Acme(trc::AcmeEvent::Error).from_json_error(err))?, ))) } } #[derive(Serialize)] struct JwkThumb<'a> { crv: &'a str, kty: &'a str, x: &'a str, y: &'a str, } ================================================ FILE: crates/common/src/listener/acme/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod cache; pub mod directory; pub mod jose; pub mod order; pub mod resolver; use std::{fmt::Debug, sync::Arc, time::Duration}; use arc_swap::ArcSwap; use dns_update::DnsUpdater; use rustls::sign::CertifiedKey; use crate::Server; use self::directory::{Account, ChallengeType}; pub struct AcmeProvider { pub id: String, pub directory_url: String, pub domains: Vec, pub contact: Vec, pub challenge: ChallengeSettings, pub eab: Option, renew_before: chrono::Duration, account_key: ArcSwap>, default: bool, } #[derive(Clone)] pub struct EabSettings { pub kid: String, pub hmac_key: Vec, } #[derive(Clone)] pub enum ChallengeSettings { Http01, TlsAlpn01, Dns01 { updater: DnsUpdater, origin: Option, polling_interval: Duration, propagation_timeout: Duration, ttl: u32, }, } pub struct StaticResolver { pub key: Option>, } impl AcmeProvider { #[allow(clippy::too_many_arguments)] pub fn new( id: String, directory_url: String, domains: Vec, contact: Vec, challenge: ChallengeSettings, eab: Option, renew_before: Duration, default: bool, ) -> trc::Result { Ok(AcmeProvider { id, directory_url, contact: contact .into_iter() .map(|c| { if !c.starts_with("mailto:") { format!("mailto:{}", c) } else { c } }) .collect(), renew_before: chrono::Duration::from_std(renew_before).unwrap(), domains, account_key: Default::default(), challenge, eab, default, }) } } impl Server { pub async fn init_acme(&self, provider: &AcmeProvider) -> trc::Result { // Load account key from cache or generate a new one if let Some(account_key) = self.load_account(provider).await? { provider.account_key.store(Arc::new(account_key)); } else { let account_key = Account::generate_key_pair(); self.store_account(provider, &account_key).await?; provider.account_key.store(Arc::new(account_key)); } // Load certificate from cache or request a new one Ok(if let Some(pem) = self.load_cert(provider).await? { self.process_cert(provider, pem, true).await? } else { Duration::from_millis(1000) }) } pub fn has_acme_tls_providers(&self) -> bool { self.core .acme .providers .values() .any(|p| matches!(p.challenge, ChallengeSettings::TlsAlpn01)) } pub fn has_acme_http_providers(&self) -> bool { self.core .acme .providers .values() .any(|p| matches!(p.challenge, ChallengeSettings::Http01)) } } impl ChallengeSettings { pub fn challenge_type(&self) -> ChallengeType { match self { ChallengeSettings::Http01 => ChallengeType::Http01, ChallengeSettings::TlsAlpn01 => ChallengeType::TlsAlpn01, ChallengeSettings::Dns01 { .. } => ChallengeType::Dns01, } } } impl Debug for StaticResolver { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("StaticResolver").finish() } } impl Clone for AcmeProvider { fn clone(&self) -> Self { Self { id: self.id.clone(), directory_url: self.directory_url.clone(), domains: self.domains.clone(), contact: self.contact.clone(), challenge: self.challenge.clone(), renew_before: self.renew_before, account_key: ArcSwap::from_pointee(self.account_key.load().as_ref().clone()), eab: self.eab.clone(), default: self.default, } } } ================================================ FILE: crates/common/src/listener/acme/order.rs ================================================ // Adapted from rustls-acme (https://github.com/FlorianUekermann/rustls-acme), licensed under MIT/Apache-2.0. use chrono::{DateTime, TimeZone, Utc}; use compact_str::CompactString; use dns_update::{DnsRecord, DnsRecordType}; use futures::future::try_join_all; use rcgen::{CertificateParams, DistinguishedName, PKCS_ECDSA_P256_SHA256}; use rustls::crypto::ring::sign::any_ecdsa_type; use rustls::sign::CertifiedKey; use rustls_pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer}; use std::sync::Arc; use std::time::{Duration, Instant}; use store::dispatch::lookup::KeyValue; use trc::{AcmeEvent, EventType}; use x509_parser::parse_x509_certificate; use crate::listener::acme::ChallengeSettings; use crate::listener::acme::directory::Identifier; use crate::{KV_ACME, Server}; use super::AcmeProvider; use super::directory::{Account, AuthStatus, Directory, OrderStatus}; impl Server { pub(crate) async fn process_cert( &self, provider: &AcmeProvider, pem: Vec, cached: bool, ) -> trc::Result { let (cert, validity) = parse_cert(&pem)?; self.set_cert(provider, Arc::new(cert)); let renew_at = (validity[1] - provider.renew_before - Utc::now()) .max(chrono::Duration::zero()) .to_std() .unwrap_or_default(); let renewal_date = validity[1] - provider.renew_before; trc::event!( Acme(AcmeEvent::ProcessCert), Id = provider.id.to_string(), Hostname = provider.domains.as_slice(), ValidFrom = trc::Value::Timestamp(validity[0].timestamp() as u64), ValidTo = trc::Value::Timestamp(validity[1].timestamp() as u64), Due = trc::Value::Timestamp(renewal_date.timestamp() as u64), ); if !cached { self.store_cert(provider, &pem).await?; } Ok(renew_at) } pub async fn renew(&self, provider: &AcmeProvider) -> trc::Result { let mut backoff = 0; loop { match self.order(provider).await { Ok(pem) => return self.process_cert(provider, pem, false).await, Err(err) if !err.matches(EventType::Acme(AcmeEvent::OrderInvalid)) && backoff < 9 => { trc::event!( Acme(AcmeEvent::RenewBackoff), Id = provider.id.to_string(), Hostname = provider.domains.as_slice(), Total = backoff, NextRetry = 1 << backoff, CausedBy = err, ); backoff += 1; tokio::time::sleep(Duration::from_secs(1 << backoff)).await; } Err(err) => { return Err(err .details("Failed to renew certificate") .ctx_unique(trc::Key::Id, provider.id.to_string()) .ctx_unique(trc::Key::Hostname, provider.domains.as_slice())); } } } } async fn order(&self, provider: &AcmeProvider) -> trc::Result> { let directory = Directory::discover(&provider.directory_url).await?; let account = Account::create_with_keypair(directory, provider).await?; let mut params = CertificateParams::new(provider.domains.clone()); params.distinguished_name = DistinguishedName::new(); params.alg = &PKCS_ECDSA_P256_SHA256; let cert = rcgen::Certificate::from_params(params).map_err(|err| { EventType::Acme(AcmeEvent::Error) .caused_by(trc::location!()) .reason(err) })?; let (order_url, mut order) = account.new_order(provider.domains.clone()).await?; loop { match order.status { OrderStatus::Pending => { let auth_futures = order .authorizations .iter() .map(|url| self.authorize(provider, &account, url)); try_join_all(auth_futures).await?; trc::event!( Acme(AcmeEvent::AuthCompleted), Id = provider.id.to_string(), Hostname = provider.domains.as_slice(), ); order = account.order(&order_url).await?; } OrderStatus::Processing => { for i in 0u64..10 { trc::event!( Acme(AcmeEvent::OrderProcessing), Id = provider.id.to_string(), Hostname = provider.domains.as_slice(), Total = i, ); tokio::time::sleep(Duration::from_secs(1u64 << i)).await; order = account.order(&order_url).await?; if order.status != OrderStatus::Processing { break; } } if order.status == OrderStatus::Processing { return Err(EventType::Acme(AcmeEvent::Error) .caused_by(trc::location!()) .details("Order processing timed out")); } } OrderStatus::Ready => { trc::event!( Acme(AcmeEvent::OrderReady), Id = provider.id.to_string(), Hostname = provider.domains.as_slice(), ); let csr = cert.serialize_request_der().map_err(|err| { EventType::Acme(AcmeEvent::Error) .caused_by(trc::location!()) .reason(err) })?; order = account.finalize(order.finalize, csr).await? } OrderStatus::Valid { certificate } => { trc::event!( Acme(AcmeEvent::OrderValid), Id = provider.id.to_string(), Hostname = provider.domains.as_slice(), ); let pem = [ &cert.serialize_private_key_pem(), "\n", &account.certificate(certificate).await?, ] .concat(); return Ok(pem.into_bytes()); } OrderStatus::Invalid => { return Err(EventType::Acme(AcmeEvent::OrderInvalid).into_err()); } } } } async fn authorize( &self, provider: &AcmeProvider, account: &Account, url: &String, ) -> trc::Result<()> { let auth = account.auth(url).await?; let (domain, challenge_url) = match auth.status { AuthStatus::Pending => { let Identifier::Dns(domain) = auth.identifier; let challenge_type = provider.challenge.challenge_type(); trc::event!( Acme(AcmeEvent::AuthStart), Hostname = domain.to_string(), Type = challenge_type.as_str(), Id = provider.id.to_string(), ); let challenge = auth .challenges .iter() .find(|c| c.typ == challenge_type) .ok_or( EventType::Acme(AcmeEvent::OrderInvalid) .into_err() .details("Challenge not supported by ACME provider") .ctx(trc::Key::Id, provider.id.to_string()) .ctx(trc::Key::Type, challenge_type.as_str()) .ctx( trc::Key::Contents, auth.challenges .iter() .map(|c| { trc::Value::String(CompactString::const_new(c.typ.as_str())) }) .collect::>(), ), )?; match &provider.challenge { ChallengeSettings::TlsAlpn01 => { self.in_memory_store() .key_set( KeyValue::with_prefix( KV_ACME, &domain, account.tls_alpn_key(challenge, domain.clone())?, ) .expires(3600), ) .await?; } ChallengeSettings::Http01 => { self.in_memory_store() .key_set( KeyValue::with_prefix( KV_ACME, &challenge.token, account.http_proof(challenge)?, ) .expires(3600), ) .await?; } ChallengeSettings::Dns01 { updater, origin, polling_interval, propagation_timeout, ttl, } => { let dns_proof = account.dns_proof(challenge)?; let domain = domain.strip_prefix("*.").unwrap_or(&domain); let name = format!("_acme-challenge.{}", domain); let origin = origin .as_deref() .or_else(|| psl::domain_str(domain)) .unwrap_or(domain) .to_string(); // First try deleting the record if let Err(err) = updater.delete(&name, &origin, DnsRecordType::TXT).await { // Errors are expected if the record does not exist trc::event!( Acme(AcmeEvent::DnsRecordDeletionFailed), Hostname = name.to_string(), Reason = err.to_string(), Details = origin.to_string(), Id = provider.id.to_string(), ); } // Create the record if let Err(err) = updater .create( &name, DnsRecord::TXT { content: dns_proof.clone(), }, *ttl, &origin, ) .await { return Err(EventType::Acme(AcmeEvent::DnsRecordCreationFailed) .ctx(trc::Key::Id, provider.id.to_string()) .ctx(trc::Key::Hostname, name) .ctx(trc::Key::Details, origin) .reason(err)); } trc::event!( Acme(AcmeEvent::DnsRecordCreated), Hostname = name.to_string(), Details = origin.to_string(), Id = provider.id.to_string(), ); // Wait for changes to propagate let wait_until = Instant::now() + *propagation_timeout; let mut did_propagate = false; while Instant::now() < wait_until { match self.core.smtp.resolvers.dns.txt_raw_lookup(&name).await { Ok(result) => { let result = std::str::from_utf8(&result).unwrap_or_default(); if result.contains(&dns_proof) { did_propagate = true; break; } else { trc::event!( Acme(AcmeEvent::DnsRecordNotPropagated), Id = provider.id.to_string(), Hostname = name.to_string(), Details = origin.to_string(), Result = result.to_string(), Value = dns_proof.to_string(), ); } } Err(err) => { trc::event!( Acme(AcmeEvent::DnsRecordLookupFailed), Id = provider.id.to_string(), Hostname = name.to_string(), Details = origin.to_string(), Reason = err.to_string(), ); } } tokio::time::sleep(*polling_interval).await; } if did_propagate { trc::event!( Acme(AcmeEvent::DnsRecordPropagated), Id = provider.id.to_string(), Hostname = name.to_string(), Details = origin.to_string(), ); } else { trc::event!( Acme(AcmeEvent::DnsRecordPropagationTimeout), Id = provider.id.to_string(), Hostname = name.to_string(), Details = origin.to_string(), ); } } } account.challenge(&challenge.url).await?; (domain, challenge.url.clone()) } AuthStatus::Valid => return Ok(()), _ => { return Err(EventType::Acme(AcmeEvent::AuthError) .into_err() .ctx(trc::Key::Id, provider.id.to_string()) .ctx(trc::Key::Details, auth.status.as_str())); } }; for i in 0u64..5 { tokio::time::sleep(Duration::from_secs(1u64 << i)).await; let auth = account.auth(url).await?; match auth.status { AuthStatus::Pending => { trc::event!( Acme(AcmeEvent::AuthPending), Hostname = domain.to_string(), Id = provider.id.to_string(), Total = i, ); account.challenge(&challenge_url).await? } AuthStatus::Valid => { trc::event!( Acme(AcmeEvent::AuthValid), Hostname = domain.to_string(), Id = provider.id.to_string(), ); return Ok(()); } _ => { return Err(EventType::Acme(AcmeEvent::AuthError) .into_err() .ctx(trc::Key::Id, provider.id.to_string()) .ctx(trc::Key::Details, auth.status.as_str())); } } } Err(EventType::Acme(AcmeEvent::AuthTooManyAttempts) .into_err() .ctx(trc::Key::Id, provider.id.to_string()) .ctx(trc::Key::Hostname, domain)) } } fn parse_cert(pem: &[u8]) -> trc::Result<(CertifiedKey, [DateTime; 2])> { let mut pems = pem::parse_many(pem).map_err(|err| { EventType::Acme(AcmeEvent::Error) .reason(err) .caused_by(trc::location!()) })?; if pems.len() < 2 { return Err(EventType::Acme(AcmeEvent::Error) .caused_by(trc::location!()) .ctx(trc::Key::Size, pems.len()) .details("Too few PEMs")); } let pk = match any_ecdsa_type(&PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from( pems.remove(0).contents(), ))) { Ok(pk) => pk, Err(err) => { return Err(EventType::Acme(AcmeEvent::Error) .reason(err) .caused_by(trc::location!())); } }; let cert_chain: Vec = pems .into_iter() .map(|p| CertificateDer::from(p.into_contents())) .collect(); let validity = match parse_x509_certificate(&cert_chain[0]) { Ok((_, cert)) => { let validity = cert.validity(); [validity.not_before, validity.not_after].map(|t| { Utc.timestamp_opt(t.timestamp(), 0) .earliest() .unwrap_or_default() }) } Err(err) => { return Err(EventType::Acme(AcmeEvent::Error) .reason(err) .caused_by(trc::location!())); } }; let cert = CertifiedKey::new(cert_chain, pk); Ok((cert, validity)) } ================================================ FILE: crates/common/src/listener/acme/resolver.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ AcmeProvider, StaticResolver, directory::{ACME_TLS_ALPN_NAME, SerializedCert}, }; use crate::{KV_ACME, Server}; use rustls::{ ServerConfig, crypto::ring::sign::any_ecdsa_type, server::{ClientHello, ResolvesServerCert}, sign::CertifiedKey, }; use rustls_pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer}; use std::sync::Arc; use store::{ dispatch::lookup::KeyValue, write::{AlignedBytes, Archive}, }; use trc::AcmeEvent; impl Server { pub(crate) fn set_cert(&self, provider: &AcmeProvider, cert: Arc) { // Add certificates let mut certificates = self.inner.data.tls_certificates.load().as_ref().clone(); for domain in provider.domains.iter() { certificates.insert( domain .strip_prefix("*.") .unwrap_or(domain.as_str()) .to_string(), cert.clone(), ); } // Add default certificate if provider.default { certificates.insert("*".to_string(), cert); } self.inner.data.tls_certificates.store(certificates.into()); } pub(crate) async fn build_acme_certificate(&self, domain: &str) -> Option> { match self .in_memory_store() .key_get::>(KeyValue::<()>::build_key(KV_ACME, domain)) .await { Ok(Some(cert_)) => match cert_.unarchive::() { Ok(cert) => { match any_ecdsa_type(&PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from( cert.private_key.as_ref(), ))) { Ok(key) => Some(Arc::new(CertifiedKey::new( vec![CertificateDer::from(cert.certificate.to_vec())], key, ))), Err(err) => { trc::event!( Acme(AcmeEvent::Error), Domain = domain.to_string(), Reason = err.to_string(), Details = "Failed to parse private key" ); None } } } Err(err) => { trc::event!( Acme(AcmeEvent::Error), Domain = domain.to_string(), CausedBy = err, Details = "Failed to unarchive certificate" ); None } }, Err(err) => { trc::event!( Acme(AcmeEvent::Error), Domain = domain.to_string(), CausedBy = err ); None } Ok(None) => { trc::event!(Acme(AcmeEvent::TokenNotFound), Domain = domain.to_string()); None } } } } impl ResolvesServerCert for StaticResolver { fn resolve(&self, _: ClientHello) -> Option> { self.key.clone() } } pub(crate) fn build_acme_static_resolver(key: Option>) -> Arc { let mut challenge = ServerConfig::builder() .with_no_client_auth() .with_cert_resolver(Arc::new(StaticResolver { key })); challenge.alpn_protocols.push(ACME_TLS_ALPN_NAME.to_vec()); Arc::new(challenge) } pub trait IsTlsAlpnChallenge { fn is_tls_alpn_challenge(&self) -> bool; } impl IsTlsAlpnChallenge for ClientHello<'_> { fn is_tls_alpn_challenge(&self) -> bool { self.alpn().into_iter().flatten().eq([ACME_TLS_ALPN_NAME]) } } ================================================ FILE: crates/common/src/listener/asn.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ net::IpAddr, sync::{Arc, atomic::AtomicU64}, time::{Duration, Instant}, }; use ahash::AHashMap; use arc_swap::ArcSwap; use mail_auth::common::resolver::ToReverseName; use store::write::now; use tokio::sync::Semaphore; use crate::{Server, config::network::AsnGeoLookupConfig, manager::fetch_resource}; pub struct AsnGeoLookupData { pub lock: Semaphore, expires: AtomicU64, asn: ArcSwap>>, country: ArcSwap>>, } #[derive(Clone, Default, Debug)] pub struct AsnData { pub id: u32, pub name: Option, } #[derive(Clone, Default, Debug)] pub struct AsnGeoLookupResult { pub asn: Option>, pub country: Option>, } struct Data { ip4_ranges: Vec>, ip6_ranges: Vec>, } pub struct IpRange { pub start: I, pub end: I, pub data: T, } impl Server { pub async fn lookup_asn_country(&self, ip: IpAddr) -> AsnGeoLookupResult { let mut result = AsnGeoLookupResult::default(); match &self.core.network.asn_geo_lookup { AsnGeoLookupConfig::Resource { .. } if !ip.is_loopback() => { let asn_geo = &self.inner.data.asn_geo_data; if asn_geo.expires.load(std::sync::atomic::Ordering::Relaxed) <= now() && asn_geo.lock.available_permits() > 0 { self.refresh_asn_geo_tables(); } result.asn = asn_geo.asn.load().lookup(ip).cloned(); result.country = asn_geo.country.load().lookup(ip).cloned(); } AsnGeoLookupConfig::Dns { zone_ipv4, zone_ipv6, separator, index_asn, index_asn_name, index_country, } if !ip.is_loopback() => { let zone = if ip.is_ipv4() { zone_ipv4 } else { zone_ipv6 }; match self .core .smtp .resolvers .dns .txt_raw_lookup(format!("{}.{}.", ip.to_reverse_name(), zone)) .await .map(String::from_utf8) { Ok(Ok(entry)) => { let mut asn = None; let mut asn_name = None; let mut country = None; for (idx, part) in entry.split(separator).enumerate() { let part = part.trim(); if !part.is_empty() { if idx == *index_asn { asn = part.parse::().ok(); } else if index_asn_name.is_some_and(|i| i == idx) { asn_name = Some(part.to_string()); } else if index_country.is_some_and(|i| i == idx) { country = Some(part.to_string()); } } } if let Some(asn) = asn { result.asn = Some(Arc::new(AsnData { id: asn, name: asn_name, })); } if let Some(country) = country { result.country = Some(Arc::new(country)); } } Ok(Err(_)) => { trc::event!( Resource(trc::ResourceEvent::Error), Details = "Failed to UTF-8 decode ASN/Geo data", Hostname = format!("{}.{}.", ip.to_reverse_name(), zone), ); } Err(err) => { trc::event!( Resource(trc::ResourceEvent::Error), Details = "Failed to lookup ASN/Geo data", Hostname = format!("{}.{}.", ip.to_reverse_name(), zone), CausedBy = err.to_string() ); } } } _ => (), } result } fn refresh_asn_geo_tables(&self) { let server = self.clone(); tokio::spawn(async move { let asn_geo = &server.inner.data.asn_geo_data; let _permit = asn_geo.lock.acquire().await; if asn_geo.expires.load(std::sync::atomic::Ordering::Relaxed) > now() { return; } if let AsnGeoLookupConfig::Resource { expires, timeout, max_size, asn_resources, geo_resources, headers, } = &server.core.network.asn_geo_lookup { let mut asn_data = Data::new(); let mut country_data = Data::new(); for (is_asn, url) in asn_resources .iter() .map(|url| (true, url)) .chain(geo_resources.iter().map(|url| (false, url))) { let time = Instant::now(); match fetch_resource(url, headers.clone().into(), *timeout, *max_size) .await .map(String::from_utf8) { Ok(Ok(data)) => { let mut has_errors = false; let mut asn_mappings = AHashMap::new(); let mut geo_mappings = AHashMap::new(); let mut from_ip = None; let mut to_ip = None; let mut asn = None; let mut details = None; let mut in_quote = false; let mut col_num = 0; let mut col_start = 0; let mut line_start = 0; for (idx, ch) in data.char_indices() { match ch { '"' => in_quote = !in_quote, ',' | '\n' if !in_quote => { let column = data.get(col_start..idx).unwrap_or_default().trim(); match col_num { 0 => from_ip = column.parse::().ok(), 1 => to_ip = column.parse::().ok(), 2 if is_asn => asn = column.parse::().ok(), 2 | 3 => { let column = column .strip_prefix('"') .and_then(|s| s.strip_suffix('"')) .unwrap_or(column); if !column.is_empty() || details.is_none() { details = Some(column); } } _ => break, } if ch == '\n' { let is_success = match (from_ip, to_ip, asn, details) { ( Some(from_ip), Some(to_ip), Some(asn), asn_name, ) if is_asn => { let data = asn_mappings .entry(asn) .or_insert_with(|| { Arc::new(AsnData { id: asn, name: asn_name.map(String::from), }) }) .clone(); asn_data.insert(from_ip, to_ip, data) } (Some(from_ip), Some(to_ip), _, Some(code)) if !is_asn && [2, 3].contains(&code.len()) => { let code = code.to_uppercase(); let data = geo_mappings .entry(code.clone()) .or_insert_with(|| Arc::new(code)) .clone(); country_data.insert(from_ip, to_ip, data) } (None, None, _, _) => true, // Ignore empty rows _ => false, }; if !is_success && !has_errors { trc::event!( Resource(trc::ResourceEvent::Error), Details = "Invalid ASN/Geo data", Url = url.clone(), Details = data .get(line_start..idx) .unwrap_or_default() .to_string(), ); has_errors = true; } col_num = 0; from_ip = None; to_ip = None; asn = None; details = None; line_start = idx + 1; } else { col_num += 1; } col_start = idx + 1; } _ => {} } } trc::event!( Resource(trc::ResourceEvent::DownloadExternal), Details = "Downloaded ASN/Geo data", Url = url.clone(), Elapsed = time.elapsed() ); } Ok(Err(_)) => { trc::event!( Resource(trc::ResourceEvent::Error), Details = "Failed to UTF-8 decode ASN/Geo data", Url = url.clone(), ); } Err(err) => { trc::event!( Resource(trc::ResourceEvent::Error), Details = "Failed to download ASN/Geo data", Url = url.clone(), CausedBy = err ); } } } let expires = if !asn_data.is_empty() || !country_data.is_empty() { *expires } else { Duration::from_secs(60) }; if !asn_data.is_empty() { asn_geo.asn.store(Arc::new(asn_data.sorted())); } if !country_data.is_empty() { asn_geo.country.store(Arc::new(country_data.sorted())); } asn_geo.expires.store( now() + expires.as_secs(), std::sync::atomic::Ordering::Relaxed, ); } }); } } impl Data { fn new() -> Self { Self { ip4_ranges: Vec::new(), ip6_ranges: Vec::new(), } } pub fn lookup(&self, ip: IpAddr) -> Option<&T> { match ip { IpAddr::V4(ip) => { let ip = u32::from(ip); match self.ip4_ranges.binary_search_by(|range| { if ip < range.start { std::cmp::Ordering::Greater } else if ip > range.end { std::cmp::Ordering::Less } else { std::cmp::Ordering::Equal } }) { Ok(idx) => Some(&self.ip4_ranges[idx].data), Err(_) => None, } } IpAddr::V6(ip) => { let ip = u128::from(ip); match self.ip6_ranges.binary_search_by(|range| { if ip < range.start { std::cmp::Ordering::Greater } else if ip > range.end { std::cmp::Ordering::Less } else { std::cmp::Ordering::Equal } }) { Ok(idx) => Some(&self.ip6_ranges[idx].data), Err(_) => None, } } } } pub fn insert(&mut self, from_ip: IpAddr, to_ip: IpAddr, data: T) -> bool { match (from_ip, to_ip) { (IpAddr::V4(from), IpAddr::V4(to)) => { self.ip4_ranges.push(IpRange { start: u32::from(from), end: u32::from(to), data, }); true } (IpAddr::V6(from), IpAddr::V6(to)) => { self.ip6_ranges.push(IpRange { start: u128::from(from), end: u128::from(to), data, }); true } _ => false, } } pub fn sorted(mut self) -> Self { self.ip4_ranges.sort_unstable_by_key(|range| range.start); self.ip6_ranges.sort_unstable_by_key(|range| range.start); self } pub fn is_empty(&self) -> bool { self.ip4_ranges.is_empty() && self.ip6_ranges.is_empty() } } impl Default for AsnGeoLookupData { fn default() -> Self { Self { lock: Semaphore::new(1), expires: AtomicU64::new(0), asn: ArcSwap::new(Arc::new(Data::new())), country: ArcSwap::new(Arc::new(Data::new())), } } } ================================================ FILE: crates/common/src/listener/blocked.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{fmt::Debug, net::IpAddr}; use ahash::AHashSet; use utils::{ config::{ Config, ConfigKey, Rate, ipmask::{IpAddrMask, IpAddrOrMask}, utils::ParseValue, }, glob::GlobPattern, }; use crate::{ KV_RATE_LIMIT_AUTH, KV_RATE_LIMIT_LOITER, KV_RATE_LIMIT_RCPT, KV_RATE_LIMIT_SCAN, Server, ip_to_bytes, ipc::BroadcastEvent, manager::config::MatchType, }; #[derive(Debug, Clone)] pub struct Security { blocked_ip_networks: Vec, has_blocked_networks: bool, allowed_ip_addresses: AHashSet, allowed_ip_networks: Vec, has_allowed_networks: bool, http_banned_paths: Vec, scanner_fail_rate: Option, auth_fail_rate: Option, rcpt_fail_rate: Option, loiter_fail_rate: Option, } pub const BLOCKED_IP_KEY: &str = "server.blocked-ip"; pub const BLOCKED_IP_PREFIX: &str = "server.blocked-ip."; pub const ALLOWED_IP_KEY: &str = "server.allowed-ip"; pub const ALLOWED_IP_PREFIX: &str = "server.allowed-ip."; pub struct BlockedIps { pub blocked_ip_addresses: AHashSet, pub blocked_ip_networks: Vec, } impl Security { pub fn parse(config: &mut Config) -> Self { let mut allowed_ip_addresses = AHashSet::new(); let mut allowed_ip_networks = Vec::new(); for ip in config .set_values(ALLOWED_IP_KEY) .map(IpAddrOrMask::parse_value) .collect::>() { match ip { Ok(IpAddrOrMask::Ip(ip)) => { allowed_ip_addresses.insert(ip); } Ok(IpAddrOrMask::Mask(ip)) => { allowed_ip_networks.push(ip); } Err(err) => { config.new_parse_error(ALLOWED_IP_KEY, err); } } } #[cfg(not(feature = "test_mode"))] { // Add loopback addresses allowed_ip_addresses.insert(IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)); allowed_ip_addresses.insert(IpAddr::V6(std::net::Ipv6Addr::LOCALHOST)); } let blocked = BlockedIps::parse(config); // Parse blocked HTTP paths let mut http_banned_paths = config .values("server.auto-ban.scan.paths") .filter_map(|(_, v)| { let v = v.trim(); if !v.is_empty() { MatchType::parse(v).into() } else { None } }) .collect::>(); if http_banned_paths.is_empty() { for pattern in [ "*.php*", "*.cgi*", "*.asp*", "*/wp-*", "*/php*", "*/cgi-bin*", "*xmlrpc*", "*../*", "*/..*", "*joomla*", "*wordpress*", "*drupal*", ] .iter() { http_banned_paths.push(MatchType::Matches(GlobPattern::compile(pattern, true))); } } Security { has_blocked_networks: !blocked.blocked_ip_networks.is_empty(), blocked_ip_networks: blocked.blocked_ip_networks, has_allowed_networks: !allowed_ip_networks.is_empty(), allowed_ip_addresses, allowed_ip_networks, auth_fail_rate: config .property_or_default::>("server.auto-ban.auth.rate", "100/1d") .unwrap_or_default(), rcpt_fail_rate: config .property_or_default::>("server.auto-ban.abuse.rate", "35/1d") .unwrap_or_default(), loiter_fail_rate: config .property_or_default::>("server.auto-ban.loiter.rate", "150/1d") .unwrap_or_default(), http_banned_paths, scanner_fail_rate: config .property_or_default::>("server.auto-ban.scan.rate", "30/1d") .unwrap_or_default(), } } } impl Server { pub async fn is_rcpt_fail2banned(&self, ip: IpAddr, rcpt: &str) -> trc::Result { if let Some(rate) = &self.core.network.security.rcpt_fail_rate { let is_allowed = self.is_ip_allowed(&ip) || (self .in_memory_store() .is_rate_allowed(KV_RATE_LIMIT_RCPT, &ip_to_bytes(&ip), rate, false) .await? .is_none() && self .in_memory_store() .is_rate_allowed(KV_RATE_LIMIT_RCPT, rcpt.as_bytes(), rate, false) .await? .is_none()); if !is_allowed { return self.block_ip(ip).await.map(|_| true); } } Ok(false) } pub async fn is_scanner_fail2banned(&self, ip: IpAddr) -> trc::Result { if let Some(rate) = &self.core.network.security.scanner_fail_rate { let is_allowed = self.is_ip_allowed(&ip) || self .in_memory_store() .is_rate_allowed(KV_RATE_LIMIT_SCAN, &ip_to_bytes(&ip), rate, false) .await? .is_none(); if !is_allowed { return self.block_ip(ip).await.map(|_| true); } } Ok(false) } pub async fn is_http_banned_path(&self, path: &str, ip: IpAddr) -> trc::Result { let paths = &self.core.network.security.http_banned_paths; if !paths.is_empty() && paths.iter().any(|p| p.matches(path)) && !self.is_ip_allowed(&ip) { self.block_ip(ip).await.map(|_| true) } else { Ok(false) } } pub async fn is_loiter_fail2banned(&self, ip: IpAddr) -> trc::Result { if let Some(rate) = &self.core.network.security.loiter_fail_rate { let is_allowed = self.is_ip_allowed(&ip) || self .in_memory_store() .is_rate_allowed(KV_RATE_LIMIT_LOITER, &ip_to_bytes(&ip), rate, false) .await? .is_none(); if !is_allowed { return self.block_ip(ip).await.map(|_| true); } } Ok(false) } pub async fn is_auth_fail2banned(&self, ip: IpAddr, login: Option<&str>) -> trc::Result { if let Some(rate) = &self.core.network.security.auth_fail_rate { let login = login.unwrap_or_default(); let is_allowed = self.is_ip_allowed(&ip) || (self .in_memory_store() .is_rate_allowed(KV_RATE_LIMIT_AUTH, &ip_to_bytes(&ip), rate, false) .await? .is_none() && (login.is_empty() || self .in_memory_store() .is_rate_allowed(KV_RATE_LIMIT_AUTH, login.as_bytes(), rate, false) .await? .is_none())); if !is_allowed { return self.block_ip(ip).await.map(|_| true); } } Ok(false) } pub async fn block_ip(&self, ip: IpAddr) -> trc::Result<()> { // Add IP to blocked list self.inner.data.blocked_ips.write().insert(ip); // Write blocked IP to config self.core .storage .config .set( [ConfigKey { key: format!("{}.{}", BLOCKED_IP_KEY, ip), value: String::new(), }], true, ) .await?; // Increment version self.cluster_broadcast(BroadcastEvent::ReloadBlockedIps) .await; Ok(()) } pub fn has_auth_fail2ban(&self) -> bool { self.core.network.security.auth_fail_rate.is_some() } pub fn is_ip_blocked(&self, ip: &IpAddr) -> bool { self.inner.data.blocked_ips.read().contains(ip) || (self.core.network.security.has_blocked_networks && self .core .network .security .blocked_ip_networks .iter() .any(|network| network.matches(ip))) } pub fn is_ip_allowed(&self, ip: &IpAddr) -> bool { self.core.network.security.allowed_ip_addresses.contains(ip) || (self.core.network.security.has_allowed_networks && self .core .network .security .allowed_ip_networks .iter() .any(|network| network.matches(ip))) } } impl BlockedIps { pub fn parse(config: &mut Config) -> Self { let mut blocked_ip_addresses = AHashSet::new(); let mut blocked_ip_networks = Vec::new(); for ip in config .set_values(BLOCKED_IP_KEY) .map(IpAddrOrMask::parse_value) .collect::>() { match ip { Ok(IpAddrOrMask::Ip(ip)) => { blocked_ip_addresses.insert(ip); } Ok(IpAddrOrMask::Mask(ip)) => { blocked_ip_networks.push(ip); } Err(err) => { config.new_parse_error(BLOCKED_IP_KEY, err); } } } Self { blocked_ip_addresses, blocked_ip_networks, } } } #[allow(clippy::derivable_impls)] impl Default for Security { fn default() -> Self { // Add IPv4 and IPv6 loopback addresses Self { #[cfg(not(feature = "test_mode"))] allowed_ip_addresses: AHashSet::from_iter([ IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), IpAddr::V6(std::net::Ipv6Addr::LOCALHOST), ]), #[cfg(feature = "test_mode")] allowed_ip_addresses: Default::default(), allowed_ip_networks: Default::default(), has_allowed_networks: Default::default(), blocked_ip_networks: Default::default(), has_blocked_networks: Default::default(), auth_fail_rate: Default::default(), rcpt_fail_rate: Default::default(), loiter_fail_rate: Default::default(), scanner_fail_rate: Default::default(), http_banned_paths: Default::default(), } } } ================================================ FILE: crates/common/src/listener/limiter.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::{ Arc, atomic::{AtomicU64, Ordering}, }; #[derive(Debug, Clone)] pub struct ConcurrencyLimiter { pub max_concurrent: u64, pub concurrent: Arc, } #[derive(Default)] pub struct InFlight { concurrent: Arc, } pub enum LimiterResult { Allowed(InFlight), Forbidden, Disabled, } impl Drop for InFlight { fn drop(&mut self) { self.concurrent.fetch_sub(1, Ordering::Relaxed); } } impl ConcurrencyLimiter { pub fn new(max_concurrent: u64) -> Self { ConcurrencyLimiter { max_concurrent, concurrent: Arc::new(0.into()), } } pub fn is_allowed(&self) -> LimiterResult { if self.concurrent.load(Ordering::Relaxed) < self.max_concurrent { // Return in-flight request self.concurrent.fetch_add(1, Ordering::Relaxed); LimiterResult::Allowed(InFlight { concurrent: self.concurrent.clone(), }) } else { LimiterResult::Forbidden } } pub fn check_is_allowed(&self) -> bool { self.concurrent.load(Ordering::Relaxed) < self.max_concurrent } pub fn is_active(&self) -> bool { self.concurrent.load(Ordering::Relaxed) > 0 } } impl InFlight { pub fn num_concurrent(&self) -> u64 { self.concurrent.load(Ordering::Relaxed) } } impl From for Option { fn from(result: LimiterResult) -> Self { match result { LimiterResult::Allowed(in_flight) => Some(in_flight), LimiterResult::Forbidden => None, LimiterResult::Disabled => Some(InFlight::default()), } } } ================================================ FILE: crates/common/src/listener/listen.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ net::{IpAddr, SocketAddr}, sync::Arc, time::Duration, }; use proxy_header::io::ProxiedStream; use rustls::crypto::ring::cipher_suite::TLS13_AES_128_GCM_SHA256; use tokio::{net::TcpStream, sync::watch}; use tokio_rustls::server::TlsStream; use trc::{EventType, HttpEvent, ImapEvent, ManageSieveEvent, Pop3Event, SmtpEvent}; use utils::{UnwrapFailure, config::Config}; use crate::{ Inner, Server, config::server::{Listener, Listeners, ServerProtocol, TcpListener}, core::BuildServer, }; use super::{ ServerInstance, SessionData, SessionManager, SessionStream, TcpAcceptor, limiter::{ConcurrencyLimiter, LimiterResult}, }; impl Listener { pub fn spawn( self, manager: impl SessionManager, inner: Arc, acceptor: TcpAcceptor, shutdown_rx: watch::Receiver, ) { // Prepare instance let instance = Arc::new(ServerInstance { id: self.id, protocol: self.protocol, proxy_networks: self.proxy_networks, limiter: ConcurrencyLimiter::new(self.max_connections), acceptor, shutdown_rx, span_id_gen: self.span_id_gen, }); let is_tls = matches!(instance.acceptor, TcpAcceptor::Tls { implicit, .. } if implicit); let is_https = is_tls && self.protocol == ServerProtocol::Http; let has_proxies = !instance.proxy_networks.is_empty(); // Spawn listeners for listener in self.listeners { let local_addr = listener.addr; // Obtain TCP options let opts = SocketOpts { nodelay: listener.nodelay, ttl: listener.ttl, linger: listener.linger, }; // Bind socket let listener = match listener.listen() { Ok(listener) => { trc::event!( Network(trc::NetworkEvent::ListenStart), ListenerId = instance.id.clone(), LocalIp = local_addr.ip(), LocalPort = local_addr.port(), Tls = is_tls, ); listener } Err(err) => { trc::event!( Network(trc::NetworkEvent::ListenError), ListenerId = instance.id.clone(), LocalIp = local_addr.ip(), LocalPort = local_addr.port(), Tls = is_tls, Reason = err, ); continue; } }; // Spawn listener let mut shutdown_rx = instance.shutdown_rx.clone(); let manager = manager.clone(); let instance = instance.clone(); let inner = inner.clone(); tokio::spawn(async move { let (span_start, span_end) = match self.protocol { ServerProtocol::Smtp | ServerProtocol::Lmtp => ( EventType::Smtp(SmtpEvent::ConnectionStart), EventType::Smtp(SmtpEvent::ConnectionEnd), ), ServerProtocol::Imap => ( EventType::Imap(ImapEvent::ConnectionStart), EventType::Imap(ImapEvent::ConnectionEnd), ), ServerProtocol::Pop3 => ( EventType::Pop3(Pop3Event::ConnectionStart), EventType::Pop3(Pop3Event::ConnectionEnd), ), ServerProtocol::Http => ( EventType::Http(HttpEvent::ConnectionStart), EventType::Http(HttpEvent::ConnectionEnd), ), ServerProtocol::ManageSieve => ( EventType::ManageSieve(ManageSieveEvent::ConnectionStart), EventType::ManageSieve(ManageSieveEvent::ConnectionEnd), ), }; loop { tokio::select! { stream = listener.accept() => { match stream { Ok((stream, remote_addr)) => { let server = inner.build_server(); let enable_acme = (is_https && server.has_acme_tls_providers()).then(|| server.clone()); if has_proxies && instance.proxy_networks.iter().any(|network| network.matches(&remote_addr.ip())) { let instance = instance.clone(); let manager = manager.clone(); // Set socket options opts.apply(&stream); tokio::spawn(async move { match ProxiedStream::create_from_tokio(stream, Default::default()).await { Ok(stream) =>{ let remote_addr = stream.proxy_header() .proxied_address() .map(|addr| addr.source) .unwrap_or(remote_addr); if let Some(session) = instance.build_session(stream, local_addr, remote_addr, &server) { // Spawn session manager.spawn(session, is_tls, enable_acme, span_start, span_end); } } Err(err) => { trc::event!( Network(trc::NetworkEvent::ProxyError), ListenerId = instance.id.clone(), LocalIp = local_addr.ip(), LocalPort = local_addr.port(), Tls = is_tls, Reason = err.to_string(), ); } } }); } else if let Some(session) = instance.build_session(stream, local_addr, remote_addr, &server) { // Set socket options opts.apply(&session.stream); // Spawn session manager.spawn(session, is_tls, enable_acme, span_start, span_end); } } Err(err) => { trc::event!( Network(trc::NetworkEvent::AcceptError), ListenerId = instance.id.clone(), LocalIp = local_addr.ip(), LocalPort = local_addr.port(), Tls = is_tls, Reason = err.to_string(), ); } } }, _ = shutdown_rx.changed() => { trc::event!( Network(trc::NetworkEvent::ListenStop), ListenerId = instance.id.clone(), LocalIp = local_addr.ip(), Tls = is_tls, LocalPort = local_addr.port(), ); manager.shutdown().await; break; } }; } }); } } } trait BuildSession { fn build_session( &self, stream: T, local_addr: SocketAddr, remote_addr: SocketAddr, server: &Server, ) -> Option>; } impl BuildSession for Arc { fn build_session( &self, stream: T, local_addr: SocketAddr, remote_addr: SocketAddr, server: &Server, ) -> Option> { // Convert mapped IPv6 addresses to IPv4 let remote_ip = match remote_addr.ip() { IpAddr::V6(ip) => ip .to_ipv4_mapped() .map(IpAddr::V4) .unwrap_or(IpAddr::V6(ip)), remote_ip => remote_ip, }; let remote_port = remote_addr.port(); // Check if blocked if server.is_ip_blocked(&remote_ip) { trc::event!( Security(trc::SecurityEvent::IpBlocked), ListenerId = self.id.clone(), LocalPort = local_addr.port(), RemoteIp = remote_ip, RemotePort = remote_port, ); None } else if let LimiterResult::Allowed(in_flight) = self.limiter.is_allowed() { // Enforce concurrency SessionData { stream, in_flight, local_ip: local_addr.ip(), local_port: local_addr.port(), session_id: 0, remote_ip, remote_port, protocol: self.protocol, instance: self.clone(), } .into() } else { trc::event!( Limit(trc::LimitEvent::ConcurrentConnection), ListenerId = self.id.clone(), LocalPort = local_addr.port(), RemoteIp = remote_ip, RemotePort = remote_port, Limit = self.limiter.max_concurrent, ); None } } } pub struct SocketOpts { pub nodelay: bool, pub ttl: Option, pub linger: Option, } impl SocketOpts { pub fn apply(&self, stream: &TcpStream) { // Set TCP options if let Err(err) = stream.set_nodelay(self.nodelay) { trc::event!( Network(trc::NetworkEvent::SetOptError), Reason = err.to_string(), Details = "Failed to set TCP_NODELAY", ); } if let Some(ttl) = self.ttl && let Err(err) = stream.set_ttl(ttl) { trc::event!( Network(trc::NetworkEvent::SetOptError), Reason = err.to_string(), Details = "Failed to set TTL", ); } if self.linger.is_some() && let Err(err) = stream.set_linger(self.linger) { trc::event!( Network(trc::NetworkEvent::SetOptError), Reason = err.to_string(), Details = "Failed to set LINGER", ); } } } impl Listeners { pub fn bind_and_drop_priv(&self, config: &mut Config) { // Bind as root for server in &self.servers { for listener in &server.listeners { if let Err(err) = listener.socket.bind(listener.addr) { config.new_build_error( format!("server.listener.{}", server.id), format!("Failed to bind to {}: {}", listener.addr, err), ); } } } // Drop privileges #[cfg(not(target_env = "msvc"))] { if let Ok(run_as_user) = std::env::var("RUN_AS_USER") { let mut pd = privdrop::PrivDrop::default() .user(run_as_user) .fallback_to_ids_if_names_are_numeric(); if let Ok(run_as_group) = std::env::var("RUN_AS_GROUP") { pd = pd .group(run_as_group) .fallback_to_ids_if_names_are_numeric(); } pd.apply().failed("Failed to drop privileges"); } } } pub fn spawn( mut self, spawn: impl Fn(Listener, TcpAcceptor, watch::Receiver), ) -> (watch::Sender, watch::Receiver) { // Spawn listeners let (shutdown_tx, shutdown_rx) = watch::channel(false); for server in self.servers { let acceptor = self .tcp_acceptors .remove(&server.id) .unwrap_or(TcpAcceptor::Plain); spawn(server, acceptor, shutdown_rx.clone()); } (shutdown_tx, shutdown_rx) } } impl TcpListener { pub fn listen(self) -> Result { self.socket .listen(self.backlog.unwrap_or(1024)) .map_err(|err| format!("Failed to listen on {}: {}", self.addr, err)) } } impl ServerInstance { pub async fn tls_accept( &self, stream: T, session_id: u64, ) -> Result, ()> { match &self.acceptor { TcpAcceptor::Tls { acceptor, .. } => match acceptor.accept(stream).await { Ok(stream) => { trc::event!( Tls(trc::TlsEvent::Handshake), ListenerId = self.id.clone(), SpanId = session_id, Version = format!( "{:?}", stream .get_ref() .1 .protocol_version() .unwrap_or(rustls::ProtocolVersion::TLSv1_3) ), Details = format!( "{:?}", stream .get_ref() .1 .negotiated_cipher_suite() .unwrap_or(TLS13_AES_128_GCM_SHA256) ) ); Ok(stream) } Err(err) => { trc::event!( Tls(trc::TlsEvent::HandshakeError), ListenerId = self.id.clone(), SpanId = session_id, Reason = err.to_string(), ); Err(()) } }, TcpAcceptor::Plain => { trc::event!( Tls(trc::TlsEvent::NotConfigured), ListenerId = self.id.clone(), SpanId = session_id, ); Err(()) } } } } ================================================ FILE: crates/common/src/listener/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{borrow::Cow, net::IpAddr, sync::Arc, time::Instant}; use compact_str::ToCompactString; use rustls::ServerConfig; use std::fmt::Debug; use tokio::{ io::{AsyncRead, AsyncWrite}, sync::watch, }; use tokio_rustls::{Accept, TlsAcceptor}; use trc::{Event, EventType, Key}; use utils::{config::ipmask::IpAddrMask, snowflake::SnowflakeIdGenerator}; use crate::{ Server, config::server::ServerProtocol, expr::{functions::ResolveVariable, *}, }; use self::limiter::{ConcurrencyLimiter, InFlight}; pub mod acme; pub mod asn; pub mod blocked; pub mod limiter; pub mod listen; pub mod stream; pub mod tls; pub struct ServerInstance { pub id: String, pub protocol: ServerProtocol, pub acceptor: TcpAcceptor, pub limiter: ConcurrencyLimiter, pub proxy_networks: Vec, pub shutdown_rx: watch::Receiver, pub span_id_gen: Arc, } #[derive(Default)] pub enum TcpAcceptor { Tls { config: Arc, acceptor: TlsAcceptor, implicit: bool, }, #[default] Plain, } #[allow(clippy::large_enum_variant)] pub enum TcpAcceptorResult where IO: AsyncRead + AsyncWrite + Unpin, { Tls(Accept), Plain(IO), Close, } pub struct SessionData { pub stream: T, pub local_ip: IpAddr, pub local_port: u16, pub remote_ip: IpAddr, pub remote_port: u16, pub protocol: ServerProtocol, pub session_id: u64, pub in_flight: InFlight, pub instance: Arc, } pub trait SessionStream: AsyncRead + AsyncWrite + Unpin + 'static + Sync + Send { fn is_tls(&self) -> bool; fn tls_version_and_cipher(&self) -> (Cow<'static, str>, Cow<'static, str>); } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SessionResult { Continue, Close, UpgradeTls, } pub trait SessionManager: Sync + Send + 'static + Clone { fn spawn( &self, mut session: SessionData, is_tls: bool, acme_core: Option, span_start: EventType, span_end: EventType, ) { let manager = self.clone(); tokio::spawn(async move { let start_time = Instant::now(); let local_port = session.local_port; let session_id; if is_tls { match session .instance .acceptor .accept(session.stream, acme_core, &session.instance) .await { TcpAcceptorResult::Tls(accept) => match accept.await { Ok(stream) => { // Generate sessionId session.session_id = session.instance.span_id_gen.generate(); session_id = session.session_id; // Send span Event::with_keys( span_start, vec![ (Key::ListenerId, session.instance.id.clone().into()), (Key::LocalPort, session.local_port.into()), (Key::RemoteIp, session.remote_ip.into()), (Key::RemotePort, session.remote_port.into()), (Key::SpanId, session.session_id.into()), ], ) .send_with_metrics(); manager .handle(SessionData { stream, local_ip: session.local_ip, local_port: session.local_port, remote_ip: session.remote_ip, remote_port: session.remote_port, protocol: session.protocol, session_id: session.session_id, in_flight: session.in_flight, instance: session.instance, }) .await; } Err(err) => { trc::event!( Tls(trc::TlsEvent::HandshakeError), ListenerId = session.instance.id.clone(), LocalPort = local_port, RemoteIp = session.remote_ip, RemotePort = session.remote_port, Reason = err.to_string(), ); return; } }, TcpAcceptorResult::Plain(stream) => { // Generate sessionId session.session_id = session.instance.span_id_gen.generate(); session_id = session.session_id; // Send span Event::with_keys( span_start, vec![ (Key::ListenerId, session.instance.id.clone().into()), (Key::LocalPort, session.local_port.into()), (Key::RemoteIp, session.remote_ip.into()), (Key::RemotePort, session.remote_port.into()), (Key::SpanId, session.session_id.into()), ], ) .send_with_metrics(); session.stream = stream; manager.handle(session).await; } TcpAcceptorResult::Close => return, } } else { // Generate sessionId session.session_id = session.instance.span_id_gen.generate(); session_id = session.session_id; // Send span Event::with_keys( span_start, vec![ (Key::ListenerId, session.instance.id.clone().into()), (Key::LocalPort, session.local_port.into()), (Key::RemoteIp, session.remote_ip.into()), (Key::RemotePort, session.remote_port.into()), (Key::SpanId, session.session_id.into()), ], ) .send_with_metrics(); manager.handle(session).await; } // End span Event::with_keys( span_end, vec![ (Key::SpanId, session_id.into()), (Key::Elapsed, start_time.elapsed().into()), ], ) .send_with_metrics(); }); } fn handle( self, session: SessionData, ) -> impl std::future::Future + Send; fn shutdown(&self) -> impl std::future::Future + Send; } impl ResolveVariable for SessionData { fn resolve_variable(&self, variable: u32) -> crate::expr::Variable<'_> { match variable { V_REMOTE_IP => self.remote_ip.to_compact_string().into(), V_REMOTE_PORT => self.remote_port.into(), V_LOCAL_IP => self.local_ip.to_compact_string().into(), V_LOCAL_PORT => self.local_port.into(), V_LISTENER => self.instance.id.as_str().into(), V_PROTOCOL => self.protocol.as_str().into(), V_TLS => self.stream.is_tls().into(), _ => crate::expr::Variable::default(), } } fn resolve_global(&self, _: &str) -> Variable<'_> { Variable::Integer(0) } } impl Debug for TcpAcceptor { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Tls { config, implicit, .. } => f .debug_struct("Tls") .field("config", config) .field("implicit", implicit) .finish(), Self::Plain => write!(f, "Plain"), } } } ================================================ FILE: crates/common/src/listener/stream.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::borrow::Cow; use proxy_header::io::ProxiedStream; use tokio::{ io::{AsyncRead, AsyncWrite}, net::TcpStream, }; use tokio_rustls::server::TlsStream; use super::SessionStream; impl SessionStream for TcpStream { fn is_tls(&self) -> bool { false } fn tls_version_and_cipher(&self) -> (Cow<'static, str>, Cow<'static, str>) { (Cow::Borrowed(""), Cow::Borrowed("")) } } impl SessionStream for TlsStream { fn is_tls(&self) -> bool { true } fn tls_version_and_cipher(&self) -> (Cow<'static, str>, Cow<'static, str>) { let (_, conn) = self.get_ref(); ( match conn .protocol_version() .unwrap_or(rustls::ProtocolVersion::Unknown(0)) { rustls::ProtocolVersion::SSLv2 => "SSLv2", rustls::ProtocolVersion::SSLv3 => "SSLv3", rustls::ProtocolVersion::TLSv1_0 => "TLSv1.0", rustls::ProtocolVersion::TLSv1_1 => "TLSv1.1", rustls::ProtocolVersion::TLSv1_2 => "TLSv1.2", rustls::ProtocolVersion::TLSv1_3 => "TLSv1.3", rustls::ProtocolVersion::DTLSv1_0 => "DTLSv1.0", rustls::ProtocolVersion::DTLSv1_2 => "DTLSv1.2", rustls::ProtocolVersion::DTLSv1_3 => "DTLSv1.3", _ => "unknown", } .into(), match conn.negotiated_cipher_suite() { Some(rustls::SupportedCipherSuite::Tls13(cs)) => { cs.common.suite.as_str().unwrap_or("unknown") } Some(rustls::SupportedCipherSuite::Tls12(cs)) => { cs.common.suite.as_str().unwrap_or("unknown") } None => "unknown", } .into(), ) } } impl SessionStream for ProxiedStream { fn is_tls(&self) -> bool { self.proxy_header() .ssl() .is_some_and(|ssl| ssl.client_ssl()) } fn tls_version_and_cipher(&self) -> (Cow<'static, str>, Cow<'static, str>) { self.proxy_header() .ssl() .map(|ssl| { ( ssl.version().unwrap_or("unknown").to_string().into(), ssl.cipher().unwrap_or("unknown").to_string().into(), ) }) .unwrap_or((Cow::Borrowed("unknown"), Cow::Borrowed("unknown"))) } } #[derive(Default)] pub struct NullIo { pub tx_buf: Vec, } impl AsyncWrite for NullIo { fn poll_write( mut self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>, buf: &[u8], ) -> std::task::Poll> { self.tx_buf.extend_from_slice(buf); std::task::Poll::Ready(Ok(buf.len())) } fn poll_flush( self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>, ) -> std::task::Poll> { std::task::Poll::Ready(Ok(())) } fn poll_shutdown( self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>, ) -> std::task::Poll> { std::task::Poll::Ready(Ok(())) } } impl AsyncRead for NullIo { fn poll_read( self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>, _buf: &mut tokio::io::ReadBuf<'_>, ) -> std::task::Poll> { unreachable!() } } impl SessionStream for NullIo { fn is_tls(&self) -> bool { true } fn tls_version_and_cipher( &self, ) -> ( std::borrow::Cow<'static, str>, std::borrow::Cow<'static, str>, ) { ( std::borrow::Cow::Borrowed(""), std::borrow::Cow::Borrowed(""), ) } } ================================================ FILE: crates/common/src/listener/tls.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ cmp::Ordering, fmt::{self, Formatter}, sync::Arc, }; use ahash::AHashMap; use rustls::{ SupportedProtocolVersion, server::{ClientHello, ResolvesServerCert}, sign::CertifiedKey, version::{TLS12, TLS13}, }; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use tokio_rustls::{Accept, LazyConfigAcceptor}; use crate::{Inner, Server}; use super::{ ServerInstance, SessionStream, TcpAcceptor, TcpAcceptorResult, acme::{ AcmeProvider, resolver::{IsTlsAlpnChallenge, build_acme_static_resolver}, }, }; pub static TLS13_VERSION: &[&SupportedProtocolVersion] = &[&TLS13]; pub static TLS12_VERSION: &[&SupportedProtocolVersion] = &[&TLS12]; #[derive(Default, Clone)] pub struct AcmeProviders { pub providers: AHashMap, } #[derive(Clone)] pub struct CertificateResolver { pub inner: Arc, } impl CertificateResolver { pub fn new(inner: Arc) -> Self { Self { inner } } } impl ResolvesServerCert for CertificateResolver { fn resolve(&self, hello: ClientHello<'_>) -> Option> { self.resolve_certificate(hello.server_name()) } } impl CertificateResolver { pub(crate) fn resolve_certificate(&self, name: Option<&str>) -> Option> { let certs = self.inner.data.tls_certificates.load(); name.map_or_else( || certs.get("*"), |name| { certs .get(name) .or_else(|| { // Try with a wildcard certificate name.split_once('.') .and_then(|(_, domain)| certs.get(domain)) }) .or_else(|| { trc::event!( Tls(trc::TlsEvent::CertificateNotFound), Hostname = name.to_string(), ); certs.get("*") }) }, ) .or_else(|| match certs.len().cmp(&1) { Ordering::Equal => certs.values().next(), Ordering::Greater => { trc::event!( Tls(trc::TlsEvent::MultipleCertificatesAvailable), Total = certs.len(), ); certs.values().next() } Ordering::Less => { trc::event!( Tls(trc::TlsEvent::NoCertificatesAvailable), Total = certs.len(), ); self.inner.data.tls_self_signed_cert.as_ref() } }) .cloned() } } impl TcpAcceptor { pub async fn accept( &self, stream: IO, enable_acme: Option, instance: &ServerInstance, ) -> TcpAcceptorResult where IO: SessionStream, { match self { TcpAcceptor::Tls { config, acceptor, implicit, } if *implicit => match enable_acme { None => TcpAcceptorResult::Tls(acceptor.accept(stream)), Some(core) => { match LazyConfigAcceptor::new(Default::default(), stream).await { Ok(start_handshake) => { if core.has_acme_tls_providers() && start_handshake.client_hello().is_tls_alpn_challenge() { let key = match start_handshake.client_hello().server_name() { Some(domain) => { let key = core.build_acme_certificate(domain).await; trc::event!( Acme(trc::AcmeEvent::ClientSuppliedSni), ListenerId = instance.id.clone(), Domain = domain.to_string(), Result = key.is_some(), ); key } None => { trc::event!( Acme(trc::AcmeEvent::ClientMissingSni), ListenerId = instance.id.clone(), ); None } }; match start_handshake .into_stream(build_acme_static_resolver(key)) .await { Ok(mut tls) => { trc::event!( Acme(trc::AcmeEvent::TlsAlpnReceived), ListenerId = instance.id.clone(), ); let _ = tls.shutdown().await; } Err(err) => { trc::event!( Acme(trc::AcmeEvent::TlsAlpnError), ListenerId = instance.id.clone(), Reason = err.to_string(), ); } } } else { return TcpAcceptorResult::Tls( start_handshake.into_stream(config.clone()), ); } } Err(err) => { trc::event!( Tls(trc::TlsEvent::HandshakeError), ListenerId = instance.id.clone(), Reason = err.to_string(), ); } } TcpAcceptorResult::Close } }, _ => TcpAcceptorResult::Plain(stream), } } pub fn is_tls(&self) -> bool { matches!(self, TcpAcceptor::Tls { .. }) } } impl TcpAcceptorResult where IO: AsyncRead + AsyncWrite + Unpin, { pub fn unwrap_tls(self) -> Accept { match self { TcpAcceptorResult::Tls(accept) => accept, _ => panic!("unwrap_tls called on non-TLS acceptor"), } } } impl std::fmt::Debug for CertificateResolver { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { f.debug_struct("CertificateResolver").finish() } } ================================================ FILE: crates/common/src/manager/backup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::Core; use ahash::AHashSet; use lz4_flex::frame::FrameEncoder; use std::{ io::{BufWriter, Write}, path::{Path, PathBuf}, sync::mpsc::{self, SyncSender}, }; use store::{ write::{AnyClass, AnyKey, ValueClass}, *, }; use types::blob_hash::{BLOB_HASH_LEN, BlobHash}; use utils::{UnwrapFailure, codec::leb128::Leb128_}; pub(super) const MAGIC_MARKER: u8 = 123; #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] pub(super) enum Family { Data = 0, Directory = 1, Blob = 2, Config = 3, Changelog = 4, Queue = 5, Report = 6, Telemetry = 7, Tasks = 8, } type TaskHandle = (tokio::task::JoinHandle<()>, std::thread::JoinHandle<()>); #[derive(Debug, Default, PartialEq, Eq)] pub struct BackupParams { dest: PathBuf, families: AHashSet, } impl Core { pub async fn backup(&self, mut params: BackupParams) { if !params.dest.exists() { std::fs::create_dir_all(¶ms.dest).failed("Failed to create backup directory"); } else if !params.dest.is_dir() { eprintln!("Backup destination {:?} is not a directory.", params.dest); std::process::exit(1); } let mut sync_handles = Vec::new(); let schema_version = self .storage .data .get_value::(AnyKey { subspace: SUBSPACE_PROPERTY, key: vec![0u8], }) .await .failed("Could not retrieve database schema version.") .failed("Could not retrieve database schema version."); if params.families.is_empty() { params.families = [ Family::Data, Family::Directory, Family::Blob, Family::Config, Family::Changelog, Family::Queue, Family::Report, Family::Telemetry, Family::Tasks, ] .into_iter() .collect(); } for subspace in params .families .into_iter() .flat_map(|f| f.subspaces()) .copied() { let (async_handle, sync_handle) = if subspace == SUBSPACE_BLOBS { self.backup_blobs(¶ms.dest, subspace, schema_version) } else { self.backup_subspace(¶ms.dest, subspace, schema_version) }; async_handle.await.failed("Task failed"); sync_handles.push(sync_handle); } for handle in sync_handles { handle.join().expect("Failed to join thread"); } } fn backup_blobs(&self, dest: &Path, subspace: u8, schema_version: u32) -> TaskHandle { let store = self.storage.data.clone(); let blob_store = self.storage.blob.clone(); let (handle, writer) = spawn_writer( dest.join(format!("subspace_{}", char::from(subspace))), subspace, schema_version, ); ( tokio::spawn(async move { let mut blobs = Vec::new(); let mut last_hash = BlobHash::default(); store .iterate( IterateParams::new( AnyKey { subspace: SUBSPACE_BLOB_LINK, key: vec![0u8], }, AnyKey { subspace: SUBSPACE_BLOB_LINK, key: vec![u8::MAX; 32], }, ) .no_values(), |key, _| { let hash = BlobHash::try_from_hash_slice( key.get(0..BLOB_HASH_LEN).ok_or_else(|| { trc::Error::corrupted_key(key, None, trc::location!()) })?, ) .unwrap(); if last_hash != hash { blobs.push(hash.clone()); last_hash = hash; } Ok(true) }, ) .await .failed("Failed to iterate over data store"); for hash in blobs { if let Some(blob) = blob_store .get_blob(hash.as_slice(), 0..usize::MAX) .await .failed("Failed to get blob") { writer .send((hash.as_slice().to_vec(), blob)) .failed("Failed to send key"); } } }), handle, ) } fn backup_subspace(&self, dest: &Path, subspace: u8, schema_version: u32) -> TaskHandle { let store = self.storage.data.clone(); let (handle, writer) = spawn_writer( dest.join(format!("subspace_{}", char::from(subspace))), subspace, schema_version, ); ( tokio::spawn(async move { if !store.is_sql() || (subspace != SUBSPACE_COUNTER && subspace != SUBSPACE_QUOTA) { store .iterate( IterateParams::new( AnyKey { subspace, key: vec![0u8], }, AnyKey { subspace, key: vec![u8::MAX; 32], }, ) .set_values(subspace != SUBSPACE_INDEXES), |key, value| { writer .send((key.to_vec(), value.to_vec())) .failed("Failed to send key"); Ok(true) }, ) .await .failed("Failed to iterate over data store"); } else { let mut keys = Vec::with_capacity(128); store .iterate( IterateParams::new( AnyKey { subspace, key: vec![0u8], }, AnyKey { subspace, key: vec![u8::MAX; 32], }, ) .no_values(), |key, _| { keys.push(key.to_vec()); Ok(true) }, ) .await .failed("Failed to iterate over data store"); for key in keys { let counter = store .get_counter(ValueClass::Any(AnyClass { subspace, key: key.clone(), })) .await .failed("Failed to get counter"); writer .send((key.to_vec(), (counter as u64).to_le_bytes().to_vec())) .failed("Failed to send key"); } } }), handle, ) } } #[allow(clippy::type_complexity)] fn spawn_writer( path: PathBuf, subspace: u8, version: u32, ) -> (std::thread::JoinHandle<()>, SyncSender<(Vec, Vec)>) { let (tx, rx) = mpsc::sync_channel::<(Vec, Vec)>(10); let handle = std::thread::spawn(move || { println!("Exporting database to {}.", path.to_str().unwrap()); let mut file = FrameEncoder::new(BufWriter::new( std::fs::File::create(path).failed("Failed to create backup file"), )); file.write_all(&[MAGIC_MARKER, subspace]) .failed("Failed to write version"); file.write_all(&version.to_le_bytes()) .failed("Failed to write version"); while let Ok((key, value)) = rx.recv() { key.len() .to_leb128_writer(&mut file) .failed("Failed to write key value"); file.write_all(&key).failed("Failed to write key"); value .len() .to_leb128_writer(&mut file) .failed("Failed to write key value"); if !value.is_empty() { file.write_all(&value).failed("Failed to write key value"); } } file.flush().failed("Failed to flush backup file"); }); (handle, tx) } impl BackupParams { pub fn new(dest: PathBuf) -> Self { let mut params = Self { dest, families: AHashSet::new(), }; if let Ok(families) = std::env::var("EXPORT_TYPES") { params.parse_families(&families); } params } fn parse_families(&mut self, families: &str) { for family in families.split(',') { let family = family.trim(); match Family::parse(family) { Ok(family) => { self.families.insert(family); } Err(err) => { eprintln!("Backup failed: {err}."); std::process::exit(1); } } } } } impl Family { pub fn subspaces(&self) -> &'static [u8] { match self { Family::Data => &[ SUBSPACE_ACL, SUBSPACE_INDEXES, SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_PROPERTY, ], Family::Directory => &[SUBSPACE_DIRECTORY], Family::Blob => &[SUBSPACE_BLOBS, SUBSPACE_BLOB_EXTRA, SUBSPACE_BLOB_LINK], Family::Config => &[SUBSPACE_SETTINGS], Family::Changelog => &[SUBSPACE_LOGS], Family::Queue => &[SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUEUE_EVENT], Family::Report => &[SUBSPACE_REPORT_OUT, SUBSPACE_REPORT_IN], Family::Telemetry => &[SUBSPACE_TELEMETRY_SPAN, SUBSPACE_TELEMETRY_METRIC], Family::Tasks => &[SUBSPACE_TASK_QUEUE], } } pub fn parse(family: &str) -> Result { match family { "data" => Ok(Family::Data), "directory" => Ok(Family::Directory), "blob" => Ok(Family::Blob), "config" => Ok(Family::Config), "changelog" => Ok(Family::Changelog), "queue" => Ok(Family::Queue), "report" => Ok(Family::Report), "telemetry" => Ok(Family::Telemetry), "tasks" => Ok(Family::Tasks), _ => Err(format!("Unknown family {}", family)), } } } ================================================ FILE: crates/common/src/manager/boot.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ WEBADMIN_KEY, backup::BackupParams, config::{ConfigManager, Patterns}, console::store_console, }; use crate::{ Caches, Core, Data, IPC_CHANNEL_BUFFER, Inner, Ipc, config::{network::AsnGeoLookupConfig, server::Listeners, telemetry::Telemetry}, core::BuildServer, ipc::{ BroadcastEvent, HousekeeperEvent, PushEvent, QueueEvent, ReportingEvent, TrainTaskController, }, }; use arc_swap::ArcSwap; use pwhash::sha512_crypt; use std::{ net::{IpAddr, Ipv4Addr}, path::PathBuf, sync::Arc, }; use store::{ Stores, rand::{Rng, distr::Alphanumeric, rng}, }; use tokio::sync::{Notify, mpsc}; use utils::{ UnwrapFailure, config::{Config, ConfigKey}, failed, }; pub struct BootManager { pub config: Config, pub inner: Arc, pub servers: Listeners, pub ipc_rxs: IpcReceivers, } pub struct IpcReceivers { pub push_rx: Option>, pub housekeeper_rx: Option>, pub queue_rx: Option>, pub report_rx: Option>, pub broadcast_rx: Option>, } const HELP: &str = concat!( "Stalwart Server v", env!("CARGO_PKG_VERSION"), r#" Usage: stalwart [OPTIONS] Options: -c, --config Start server with the specified configuration file -e, --export Export all store data to a specific path -i, --import Import store data from a specific path -o, --console Open the store console -I, --init Initialize a new server at a specific path -h, --help Print help -V, --version Print version "# ); #[derive(PartialEq, Eq)] enum StoreOp { Export(BackupParams), Import(PathBuf), Console, None, } pub const DEFAULT_SETTINGS: &[(&str, &str)] = &[ ("queue.quota.size.messages", "100000"), ("queue.quota.size.size", "10737418240"), ("queue.quota.size.enable", "true"), ("queue.limiter.inbound.ip.key", "remote_ip"), ("queue.limiter.inbound.ip.rate", "5/1s"), ("queue.limiter.inbound.ip.enable", "true"), ("queue.limiter.inbound.sender.key.0", "sender_domain"), ("queue.limiter.inbound.sender.key.1", "rcpt"), ("queue.limiter.inbound.sender.rate", "25/1h"), ("queue.limiter.inbound.sender.enable", "true"), ("report.analysis.addresses", "postmaster@*"), ("queue.virtual.local.threads-per-node", "25"), ("queue.virtual.local.description", "Local delivery queue"), ("queue.virtual.remote.threads-per-node", "50"), ("queue.virtual.remote.description", "Remote delivery queue"), ("queue.virtual.dsn.threads-per-node", "5"), ( "queue.virtual.dsn.description", "Delivery Status Notification delivery queue", ), ("queue.virtual.report.threads-per-node", "5"), ( "queue.virtual.report.description", "DMARC and TLS report delivery queue", ), ("queue.schedule.local.queue-name", "local"), ("queue.schedule.local.retry.0", "2m"), ("queue.schedule.local.retry.1", "5m"), ("queue.schedule.local.retry.2", "10m"), ("queue.schedule.local.retry.3", "15m"), ("queue.schedule.local.retry.4", "30m"), ("queue.schedule.local.retry.5", "1h"), ("queue.schedule.local.retry.6", "2h"), ("queue.schedule.local.notify.0", "1d"), ("queue.schedule.local.notify.1", "3d"), ("queue.schedule.local.expire-type", "ttl"), ("queue.schedule.local.expire", "3d"), ( "queue.schedule.local.description", "Local delivery schedule", ), ("queue.schedule.remote.queue-name", "remote"), ("queue.schedule.remote.retry.0", "2m"), ("queue.schedule.remote.retry.1", "5m"), ("queue.schedule.remote.retry.2", "10m"), ("queue.schedule.remote.retry.3", "15m"), ("queue.schedule.remote.retry.4", "30m"), ("queue.schedule.remote.retry.5", "1h"), ("queue.schedule.remote.retry.6", "2h"), ("queue.schedule.remote.notify.0", "1d"), ("queue.schedule.remote.notify.1", "3d"), ("queue.schedule.remote.expire-type", "ttl"), ("queue.schedule.remote.expire", "3d"), ( "queue.schedule.remote.description", "Remote delivery schedule", ), ("queue.schedule.dsn.queue-name", "dsn"), ("queue.schedule.dsn.retry.0", "15m"), ("queue.schedule.dsn.retry.1", "30m"), ("queue.schedule.dsn.retry.2", "1h"), ("queue.schedule.dsn.retry.3", "2h"), ("queue.schedule.dsn.expire-type", "attempts"), ("queue.schedule.dsn.max-attempts", "10"), ( "queue.schedule.dsn.description", "Delivery Status Notification delivery schedule", ), ("queue.schedule.report.queue-name", "report"), ("queue.schedule.report.retry.0", "30m"), ("queue.schedule.report.retry.1", "1h"), ("queue.schedule.report.retry.2", "2h"), ("queue.schedule.report.expire-type", "attempts"), ("queue.schedule.report.max-attempts", "8"), ( "queue.schedule.report.description", "DMARC and TLS report delivery schedule", ), ("queue.tls.invalid-tls.allow-invalid-certs", "true"), ( "queue.tls.invalid-tls.description", "Allow invalid TLS certificates", ), ("queue.tls.default.allow-invalid-certs", "false"), ("queue.tls.default.description", "Default TLS settings"), ("queue.route.local.type", "local"), ("queue.route.local.description", "Local delivery route"), ("queue.route.mx.type", "mx"), ("queue.route.mx.limits.multihomed", "2"), ("queue.route.mx.limits.mx", "5"), ("queue.route.mx.ip-lookup", "ipv4_then_ipv6"), ("queue.route.mx.description", "MX delivery route"), ("queue.connection.default.timeout.connect", "5m"), ( "queue.connection.default.description", "Default connection settings", ), ]; impl BootManager { pub async fn init() -> Self { let mut config_path = std::env::var("CONFIG_PATH").ok(); let mut import_export = StoreOp::None; if config_path.is_none() { let mut args = std::env::args().skip(1); while let Some(arg) = args.next().and_then(|arg| { arg.strip_prefix("--") .or_else(|| arg.strip_prefix('-')) .map(|arg| arg.to_string()) }) { let (key, value) = if let Some((key, value)) = arg.split_once('=') { (key.to_string(), Some(value.trim().to_string())) } else { (arg, args.next()) }; match (key.as_str(), value) { ("help" | "h", _) => { eprintln!("{HELP}"); std::process::exit(0); } ("version" | "V", _) => { println!("{}", env!("CARGO_PKG_VERSION")); std::process::exit(0); } ("config" | "c", Some(value)) => { config_path = Some(value); } ("init" | "I", Some(value)) => { quickstart(value); std::process::exit(0); } ("export" | "e", Some(value)) => { import_export = StoreOp::Export(BackupParams::new(value.into())); } ("import" | "i", Some(value)) => { import_export = StoreOp::Import(value.into()); } ("console" | "o", None) => { import_export = StoreOp::Console; } (_, None) => { failed(&format!("Unrecognized command '{key}', try '--help'.")); } (_, Some(_)) => failed(&format!( "Missing value for argument '{key}', try '--help'." )), } } if config_path.is_none() { if import_export == StoreOp::None { eprintln!("{HELP}"); } else { eprintln!("Missing '--config' argument for import/export.") } std::process::exit(0); } } // Read main configuration file let cfg_local_path = PathBuf::from(config_path.unwrap()); let mut config = Config::default(); match std::fs::read_to_string(&cfg_local_path) { Ok(value) => { config.parse(&value).failed("Invalid configuration file"); } Err(err) => { config.new_build_error("*", format!("Could not read configuration file: {err}")); } } let cfg_local = config.keys.clone(); // Resolve environment macros config.resolve_macros(&["env"]).await; // Parser servers let mut servers = Listeners::parse(&mut config); // Bind ports and drop privileges servers.bind_and_drop_priv(&mut config); // Resolve file and configuration macros config.resolve_macros(&["file", "cfg"]).await; // Load stores let mut stores = Stores::parse(&mut config).await; let local_patterns = Patterns::parse(&mut config); // Build local keys and warn about database keys defined in the local configuration let mut warn_keys = Vec::new(); for key in config.keys.keys() { if !local_patterns.is_local_key(key) { warn_keys.push(key.clone()); } } for warn_key in warn_keys { config.new_build_warning( warn_key, concat!( "Database key defined in local configuration, this might cause issues. ", "See https://stalw.art/docs/configuration/overview/#loc", "al-and-database-settings" ), ); } // Build manager let manager = ConfigManager { cfg_local: ArcSwap::from_pointee(cfg_local), cfg_local_path, cfg_local_patterns: local_patterns.into(), cfg_store: config .value("storage.data") .and_then(|id| stores.stores.get(id)) .cloned() .unwrap_or_default(), }; // Extend configuration with settings stored in the db if !manager.cfg_store.is_none() { for (key, value) in manager .db_list("", false) .await .failed("Failed to read database configuration") { if manager.cfg_local_patterns.is_local_key(&key) { config.new_build_warning( &key, concat!( "Local key defined in database, this might cause issues. ", "See https://stalw.art/docs/configuration/overview/#loc", "al-and-database-settings" ), ); } config.keys.entry(key).or_insert(value); } } // Parse telemetry let telemetry = Telemetry::parse(&mut config, &stores); match import_export { StoreOp::None => { // Add hostname lookup if missing let mut insert_keys = Vec::new(); // Generate an OAuth key if missing if config .value("oauth.key") .filter(|v| !v.is_empty()) .is_none() { insert_keys.push(ConfigKey::from(( "oauth.key", rng() .sample_iter(Alphanumeric) .take(64) .map(char::from) .collect::(), ))); } // Download Spam filter rules if missing if config.value("version.spam-filter").is_none() { match manager.fetch_spam_rules().await { Ok(external_config) => { trc::event!( Config(trc::ConfigEvent::ImportExternal), Version = external_config.version.to_string(), Id = "spam-filter" ); insert_keys.extend(external_config.keys); } Err(err) => { config.new_build_error( "*", format!("Failed to fetch spam filter: {err}"), ); } } // Add default settings for key in DEFAULT_SETTINGS { insert_keys.push(ConfigKey::from(*key)); } } // Download webadmin if missing if let Some(blob_store) = config .value("storage.blob") .and_then(|id| stores.blob_stores.get(id)) { match blob_store.get_blob(WEBADMIN_KEY, 0..usize::MAX).await { Ok(Some(_)) => (), Ok(None) => match manager.fetch_resource("webadmin").await { Ok(bytes) => match blob_store.put_blob(WEBADMIN_KEY, &bytes).await { Ok(_) => { trc::event!( Resource(trc::ResourceEvent::DownloadExternal), Id = "webadmin" ); } Err(err) => { config.new_build_error( "*", format!("Failed to store webadmin blob: {err}"), ); } }, Err(err) => { config.new_build_error( "*", format!("Failed to download webadmin: {err}"), ); } }, Err(err) => config .new_build_error("*", format!("Failed to access webadmin blob: {err}")), } } // Add missing settings if !insert_keys.is_empty() { for item in &insert_keys { config.keys.insert(item.key.clone(), item.value.clone()); } if let Err(err) = manager.set(insert_keys, true).await { config .new_build_error("*", format!("Failed to update configuration: {err}")); } } // Parse in-memory stores stores.parse_in_memory(&mut config, false).await; // Parse settings let core = Box::pin(Core::parse(&mut config, stores, manager)).await; // Parse data let data = Data::parse(&mut config); // Parse caches let cache = Caches::parse(&mut config); // Enable telemetry // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] telemetry.enable(core.is_enterprise_edition()); // SPDX-SnippetEnd #[cfg(not(feature = "enterprise"))] telemetry.enable(false); trc::event!( Server(trc::ServerEvent::Startup), Version = env!("CARGO_PKG_VERSION"), ); // Webadmin auto-update // Disabled temporarily until selective updates are implemented /*if config .property_or_default::("webadmin.auto-update", "false") .unwrap_or_default() { if let Err(err) = data.webadmin.update(&core).await { trc::event!( Resource(trc::ResourceEvent::Error), Details = "Failed to update webadmin", CausedBy = err ); } }*/ // Spam filter auto-update if config .property_or_default::("spam-filter.auto-update", "false") .unwrap_or_default() && let Err(err) = core.storage.config.update_spam_rules(false, false).await { trc::event!( Resource(trc::ResourceEvent::Error), Details = "Failed to update spam-filter", CausedBy = err ); } // Build shared inner let has_remote_asn = matches!( core.network.asn_geo_lookup, AsnGeoLookupConfig::Resource { .. } ); let (ipc, ipc_rxs) = build_ipc(!core.storage.pubsub.is_none()); let inner = Arc::new(Inner { shared_core: ArcSwap::from_pointee(core), data, ipc, cache, }); // Load spam model if let Err(err) = inner.build_server().spam_model_reload().await { trc::error!( err.details("Failed to load spam filter model") .caused_by(trc::location!()) ); } // Fetch ASN database if has_remote_asn { inner .build_server() .lookup_asn_country(IpAddr::V4(Ipv4Addr::new(8, 8, 8, 8))) .await; } // Parse TCP acceptors servers.parse_tcp_acceptors(&mut config, inner.clone()); BootManager { inner, config, servers, ipc_rxs, } } StoreOp::Export(path) => { // Enable telemetry telemetry.enable(false); // Parse settings and backup Box::pin(Core::parse(&mut config, stores, manager)) .await .backup(path) .await; std::process::exit(0); } StoreOp::Import(path) => { // Enable telemetry telemetry.enable(false); // Parse settings and restore Box::pin(Core::parse(&mut config, stores, manager)) .await .restore(path) .await; std::process::exit(0); } StoreOp::Console => { // Store console store_console( Box::pin(Core::parse(&mut config, stores, manager)) .await .storage .data, ) .await; std::process::exit(0); } } } } pub fn build_ipc(has_pubsub: bool) -> (Ipc, IpcReceivers) { // Build ipc receivers let (push_tx, push_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); let (housekeeper_tx, housekeeper_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); let (queue_tx, queue_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); let (report_tx, report_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); let (broadcast_tx, broadcast_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); ( Ipc { push_tx, housekeeper_tx, queue_tx, report_tx, broadcast_tx: has_pubsub.then_some(broadcast_tx), task_tx: Arc::new(Notify::new()), train_task_controller: Arc::new(TrainTaskController::default()), }, IpcReceivers { push_rx: Some(push_rx), housekeeper_rx: Some(housekeeper_rx), queue_rx: Some(queue_rx), report_rx: Some(report_rx), broadcast_rx: has_pubsub.then_some(broadcast_rx), }, ) } fn quickstart(path: impl Into) { let path = path.into(); if !path.exists() { std::fs::create_dir_all(&path).failed("Failed to create directory"); } for dir in &["etc", "data", "logs"] { let sub_path = path.join(dir); if !sub_path.exists() { std::fs::create_dir(sub_path).failed(&format!("Failed to create {dir} directory")); } } let admin_pass = std::env::var("STALWART_ADMIN_PASSWORD").unwrap_or_else(|_| { rng() .sample_iter(Alphanumeric) .take(10) .map(char::from) .collect::() }); std::fs::write( path.join("etc").join("config.toml"), QUICKSTART_CONFIG .replace("_P_", &path.to_string_lossy()) .replace("_S_", &sha512_crypt::hash(&admin_pass).unwrap()), ) .failed("Failed to write configuration file"); eprintln!( "✅ Configuration file written to {}/etc/config.toml", path.to_string_lossy() ); eprintln!("🔑 Your administrator account is 'admin' with password '{admin_pass}'."); } #[cfg(not(feature = "foundation"))] const QUICKSTART_CONFIG: &str = r#"[server.listener.smtp] bind = "[::]:25" protocol = "smtp" [server.listener.submission] bind = "[::]:587" protocol = "smtp" [server.listener.submissions] bind = "[::]:465" protocol = "smtp" tls.implicit = true [server.listener.imap] bind = "[::]:143" protocol = "imap" [server.listener.imaptls] bind = "[::]:993" protocol = "imap" tls.implicit = true [server.listener.pop3] bind = "[::]:110" protocol = "pop3" [server.listener.pop3s] bind = "[::]:995" protocol = "pop3" tls.implicit = true [server.listener.sieve] bind = "[::]:4190" protocol = "managesieve" [server.listener.https] protocol = "http" bind = "[::]:443" tls.implicit = true [server.listener.http] protocol = "http" bind = "[::]:8080" [storage] data = "rocksdb" fts = "rocksdb" blob = "rocksdb" lookup = "rocksdb" directory = "internal" [store.rocksdb] type = "rocksdb" path = "_P_/data" compression = "lz4" [directory.internal] type = "internal" store = "rocksdb" [tracer.log] type = "log" level = "info" path = "_P_/logs" prefix = "stalwart.log" rotate = "daily" ansi = false enable = true [authentication.fallback-admin] user = "admin" secret = "_S_" "#; #[cfg(feature = "foundation")] const QUICKSTART_CONFIG: &str = r#"[server.listener.smtp] bind = "[::]:25" protocol = "smtp" [server.listener.submission] bind = "[::]:587" protocol = "smtp" [server.listener.submissions] bind = "[::]:465" protocol = "smtp" tls.implicit = true [server.listener.imap] bind = "[::]:143" protocol = "imap" [server.listener.imaptls] bind = "[::]:993" protocol = "imap" tls.implicit = true [server.listener.pop3] bind = "[::]:110" protocol = "pop3" [server.listener.pop3s] bind = "[::]:995" protocol = "pop3" tls.implicit = true [server.listener.sieve] bind = "[::]:4190" protocol = "managesieve" [server.listener.https] protocol = "http" bind = "[::]:443" tls.implicit = true [server.listener.http] protocol = "http" bind = "[::]:8080" [storage] data = "foundation-db" fts = "foundation-db" blob = "foundation-db" lookup = "foundation-db" directory = "internal" [store.foundation-db] type = "foundationdb" compression = "lz4" [directory.internal] type = "internal" store = "foundation-db" [tracer.log] type = "log" level = "info" path = "_P_/logs" prefix = "stalwart.log" rotate = "daily" ansi = false enable = true [authentication.fallback-admin] user = "admin" secret = "_S_" "#; ================================================ FILE: crates/common/src/manager/config.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ collections::{BTreeMap, btree_map::Entry}, path::PathBuf, sync::Arc, }; use ahash::AHashMap; use arc_swap::ArcSwap; use store::{ Deserialize, IterateParams, Store, ValueKey, write::{BatchBuilder, ValueClass}, }; use trc::AddContext; use types::semver::Semver; use utils::{ config::{Config, ConfigKey}, glob::GlobPattern, }; #[derive(Default)] pub struct ConfigManager { pub cfg_local: ArcSwap>, pub cfg_local_path: PathBuf, pub cfg_local_patterns: Arc, pub cfg_store: Store, } #[derive(Default)] pub struct Patterns { patterns: Vec, } #[derive(Debug, PartialEq, Eq)] enum Pattern { Include(MatchType), Exclude(MatchType), } #[derive(Debug, Clone, PartialEq, Eq)] pub enum MatchType { Equal(String), StartsWith(String), EndsWith(String), Matches(GlobPattern), All, } pub(crate) struct ExternalSpamRules { pub version: Semver, pub keys: Vec, } impl ConfigManager { pub async fn build_config(&self, prefix: &str) -> trc::Result { let mut config = Config { keys: self.cfg_local.load().as_ref().clone(), ..Default::default() }; config.resolve_all_macros().await; self.extend_config(&mut config, prefix) .await .map(|_| config) } pub(crate) async fn extend_config(&self, config: &mut Config, prefix: &str) -> trc::Result<()> { for (key, value) in self.db_list(prefix, false).await? { config.keys.entry(key).or_insert(value); } Ok(()) } pub async fn get(&self, key: impl AsRef) -> trc::Result> { let key = key.as_ref(); match self.cfg_local.load().get(key) { Some(value) => Ok(Some(value.to_string())), None => { self.cfg_store .get_value(ValueKey::from(ValueClass::Config( key.to_string().into_bytes(), ))) .await } } } pub async fn list( &self, prefix: &str, strip_prefix: bool, ) -> trc::Result> { let mut results = self.db_list(prefix, strip_prefix).await?; for (key, value) in self.cfg_local.load().iter() { if prefix.is_empty() || (!strip_prefix && key.starts_with(prefix)) { results.insert(key.clone(), value.clone()); } else if let Some(key) = key.strip_prefix(prefix) { results.insert(key.to_string(), value.clone()); } } Ok(results) } pub async fn group( &self, prefix: &str, suffix: &str, ) -> trc::Result>> { let mut grouped = AHashMap::new(); let mut list = self.list(prefix, true).await?; for key in list.keys() { if let Some(key) = key.strip_suffix(suffix) { grouped.insert(key.to_string(), AHashMap::new()); } } for (name, entries) in &mut grouped { let prefix = format!("{name}."); for (key, value) in &mut list { if let Some(key) = key.strip_prefix(&prefix) { entries.insert(key.to_string(), std::mem::take(value)); } } } Ok(grouped) } pub async fn db_list( &self, prefix: &str, strip_prefix: bool, ) -> trc::Result> { let key = prefix.as_bytes(); let from_key = ValueKey::from(ValueClass::Config(key.to_vec())); let to_key = ValueKey::from(ValueClass::Config( key.iter() .copied() .chain([u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX]) .collect::>(), )); let mut results = BTreeMap::new(); let patterns = self.cfg_local_patterns.clone(); self.cfg_store .iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { let mut key = std::str::from_utf8(key).map_err(|_| { trc::Error::corrupted_key(key, value.into(), trc::location!()) })?; if !patterns.is_local_key(key) { if strip_prefix && !prefix.is_empty() { key = key.strip_prefix(prefix).unwrap_or(key); } results.insert(key.to_string(), String::deserialize(value)?); } Ok(true) }, ) .await .caused_by(trc::location!())?; Ok(results) } pub async fn set(&self, keys: I, overwrite: bool) -> trc::Result<()> where I: IntoIterator, T: Into, { let mut batch = BatchBuilder::new(); let mut local_batch = Vec::new(); for key in keys { let key = key.into(); if overwrite || self.get(&key.key).await?.is_none() || key.key.starts_with("version.") { if self.cfg_local_patterns.is_local_key(&key.key) { local_batch.push(key); } else { batch.set(ValueClass::Config(key.key.into_bytes()), key.value); } } } if !batch.is_empty() { self.cfg_store.write(batch.build_all()).await?; } if !local_batch.is_empty() { let mut local = self.cfg_local.load().as_ref().clone(); let mut has_changes = false; for key in local_batch { match local.entry(key.key) { Entry::Vacant(v) => { v.insert(key.value); has_changes = true; } Entry::Occupied(mut v) => { if v.get() != &key.value { v.insert(key.value); has_changes = true; } } } } if has_changes { self.update_local(local).await?; } } Ok(()) } pub async fn clear(&self, key: impl AsRef) -> trc::Result<()> { let key = key.as_ref(); if self.cfg_local_patterns.is_local_key(key) { let mut local = self.cfg_local.load().as_ref().clone(); if local.remove(key).is_some() { self.update_local(local).await } else { Ok(()) } } else { let mut batch = BatchBuilder::new(); batch.clear(ValueClass::Config(key.to_string().into_bytes())); self.cfg_store.write(batch.build_all()).await.map(|_| ()) } } pub async fn clear_prefix(&self, key: impl AsRef) -> trc::Result<()> { let key = key.as_ref(); // Delete local keys let local = self.cfg_local.load(); if local.keys().any(|k| k.starts_with(key)) { let mut local = local.as_ref().clone(); local.retain(|k, _| !k.starts_with(key)); self.update_local(local).await?; } // Delete db keys self.cfg_store .delete_range( ValueKey::from(ValueClass::Config(key.as_bytes().to_vec())), ValueKey::from(ValueClass::Config( key.as_bytes() .iter() .copied() .chain([u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX]) .collect::>(), )), ) .await } async fn update_local(&self, map: BTreeMap) -> trc::Result<()> { let mut cfg_text = String::with_capacity(1024); for (key, value) in &map { cfg_text.push_str(key); cfg_text.push_str(" = "); if value == "true" || value == "false" || value.parse::().is_ok() { cfg_text.push_str(value); } else { let mut needs_escape = false; let mut has_lf = false; for ch in value.chars() { match ch { '"' | '\\' => { needs_escape = true; if has_lf { break; } } '\n' => { has_lf = true; if needs_escape { break; } } _ => {} } } if has_lf || (value.len() > 50 && needs_escape) { cfg_text.push_str("'''"); cfg_text.push_str(value); cfg_text.push_str("'''"); } else { cfg_text.push('"'); if needs_escape { for ch in value.chars() { if ch == '\\' || ch == '"' { cfg_text.push('\\'); } cfg_text.push(ch); } } else { cfg_text.push_str(value); } cfg_text.push('"'); } } cfg_text.push('\n'); } self.cfg_local.store(map.into()); tokio::fs::write(&self.cfg_local_path, cfg_text) .await .map_err(|err| { trc::EventType::Config(trc::ConfigEvent::WriteError) .reason(err) .details("Failed to write local configuration") .ctx(trc::Key::Path, self.cfg_local_path.display().to_string()) }) } pub async fn update_spam_rules( &self, force_update: bool, overwrite: bool, ) -> trc::Result> { let current_version = self .get("version.spam-filter") .await? .and_then(|v| Semver::try_from(v.as_str()).ok()); let is_update = current_version.is_some(); let mut external = self.fetch_spam_rules().await.map_err(|reason| { trc::EventType::Config(trc::ConfigEvent::FetchError) .caused_by(trc::location!()) .details("Failed to update spam filter rules") .ctx(trc::Key::Reason, reason) })?; if current_version.is_none_or(|v| external.version > v || force_update) { if is_update { // Delete previous STWT_* rules let mut rule_settings = AHashMap::new(); for prefix in [ "spam-filter.rule.stwt_", "spam-filter.dnsbl.server.stwt_", "http-lookup.stwt_", ] { for (key, value) in self.list(prefix, false).await? { if key.ends_with(".enable") { rule_settings.insert(key, value); } } self.clear_prefix(prefix).await?; } // Update keys if !rule_settings.is_empty() { for key in &mut external.keys { if let Some(value) = rule_settings.remove(&key.key) { key.value = value; } } } if !overwrite { // Do not overwrite ASN or LLM settings external.keys.retain(|key| { !key.key.starts_with("spam-filter.llm.") && !key.key.starts_with("asn.") }); } } self.set(external.keys, overwrite).await?; trc::event!( Config(trc::ConfigEvent::ImportExternal), Version = external.version.to_string(), Id = "spam-filter", ); Ok(Some(external.version)) } else { trc::event!( Config(trc::ConfigEvent::AlreadyUpToDate), Version = external.version.to_string(), Id = "spam-filter", ); Ok(None) } } pub(crate) async fn fetch_spam_rules(&self) -> Result { let config = String::from_utf8(self.fetch_resource("spam-filter").await?) .map_err(|err| format!("Configuration file has invalid UTF-8: {err}"))?; let config = Config::new(config) .map_err(|err| format!("Failed to parse external configuration: {err}"))?; // Import configuration let mut external = ExternalSpamRules { version: Semver::default(), keys: Vec::new(), }; let mut required_semver = Semver::default(); let server_semver: Semver = env!("CARGO_PKG_VERSION").try_into().unwrap(); for (key, value) in config.keys { if key == "version.spam-filter" { external.version = value.as_str().try_into().unwrap_or_default(); external.keys.push(ConfigKey::from((key, value))); } else if key == "version.server" { required_semver = value.as_str().try_into().unwrap_or_default(); } else if key.starts_with("spam-filter.") || key.starts_with("http-lookup.") || key.starts_with("lookup.") || key.starts_with("asn.") { external.keys.push(ConfigKey::from((key, value))); } } if !required_semver.is_valid() { Err("External spam filter rules do not contain a valid server version".to_string()) } else if required_semver > server_semver { Err(format!( "External spam filter rules require server version {required_semver}, but this is version {server_semver}", )) } else if external.version.is_valid() { Ok(external) } else { Err("External spam filter rules do not contain a version key".to_string()) } } pub async fn get_services(&self) -> trc::Result> { let mut result = Vec::new(); for listener in self .group("server.listener.", ".protocol") .await .unwrap_or_default() .into_values() { let is_tls = listener .get("tls.implicit") .is_some_and(|tls| tls == "true"); let protocol = listener .get("protocol") .map(|s| s.as_str()) .unwrap_or_default(); let port = listener .get("bind") .or_else(|| { listener.iter().find_map(|(key, value)| { if key.starts_with("bind.") { Some(value) } else { None } }) }) .and_then(|s| s.rsplit_once(':').and_then(|(_, p)| p.parse::().ok())) .unwrap_or_default(); if port > 0 { result.push((protocol.to_string(), port, is_tls)); } } // Sort by name, then tls and finally port result.sort_unstable_by(|a, b| { a.0.cmp(&b.0) .then_with(|| b.2.cmp(&a.2)) .then_with(|| a.1.cmp(&b.1)) }); Ok(result) } } impl Patterns { pub fn parse(config: &mut Config) -> Self { let mut cfg_local_patterns = Vec::new(); for (key, value) in &config.keys { if !key.starts_with("config.local-keys") { if cfg_local_patterns.is_empty() { continue; } else { break; } }; let value = value.trim(); let (value, is_include) = value .strip_prefix('!') .map_or((value, true), |value| (value, false)); let value = value.trim().to_ascii_lowercase(); if value.is_empty() { continue; } let match_type = MatchType::parse(&value); cfg_local_patterns.push(if is_include { Pattern::Include(match_type) } else { Pattern::Exclude(match_type) }); } if cfg_local_patterns.is_empty() { cfg_local_patterns = vec![ Pattern::Include(MatchType::StartsWith("store.".to_string())), Pattern::Include(MatchType::StartsWith("directory.".to_string())), Pattern::Include(MatchType::StartsWith("tracer.".to_string())), Pattern::Exclude(MatchType::StartsWith("server.blocked-ip.".to_string())), Pattern::Exclude(MatchType::StartsWith("server.allowed-ip.".to_string())), Pattern::Include(MatchType::StartsWith("server.".to_string())), Pattern::Include(MatchType::StartsWith("certificate.".to_string())), Pattern::Include(MatchType::StartsWith("config.local-keys.".to_string())), Pattern::Include(MatchType::StartsWith( "authentication.fallback-admin.".to_string(), )), Pattern::Include(MatchType::StartsWith("cluster.".to_string())), Pattern::Include(MatchType::Equal("storage.data".to_string())), Pattern::Include(MatchType::Equal("storage.blob".to_string())), Pattern::Include(MatchType::Equal("storage.lookup".to_string())), Pattern::Include(MatchType::Equal("storage.fts".to_string())), Pattern::Include(MatchType::Equal("storage.directory".to_string())), Pattern::Include(MatchType::Equal("enterprise.license-key".to_string())), ]; } else if !cfg_local_patterns.contains(&Pattern::Include(MatchType::StartsWith( "config.local-keys.".to_string(), ))) { cfg_local_patterns.push(Pattern::Include(MatchType::StartsWith( "config.local-keys.".to_string(), ))); } Patterns { patterns: cfg_local_patterns, } } pub fn is_local_key(&self, key: &str) -> bool { let mut is_local = false; for pattern in &self.patterns { match pattern { Pattern::Include(pattern) => { if !is_local && pattern.matches(key) { is_local = true; } } Pattern::Exclude(pattern) => { if pattern.matches(key) { return false; } } } } is_local } } impl MatchType { pub fn parse(value: &str) -> Self { if value == "*" { MatchType::All } else if let Some(value) = value.strip_suffix('*') { MatchType::StartsWith(value.to_string()) } else if let Some(value) = value.strip_prefix('*') { MatchType::EndsWith(value.to_string()) } else if value.contains('*') { MatchType::Matches(GlobPattern::compile(value, false)) } else { MatchType::Equal(value.to_string()) } } pub fn matches(&self, value: &str) -> bool { match self { MatchType::Equal(pattern) => value == pattern, MatchType::StartsWith(pattern) => value.starts_with(pattern), MatchType::EndsWith(pattern) => value.ends_with(pattern), MatchType::Matches(pattern) => pattern.matches(value), MatchType::All => true, } } } impl Clone for ConfigManager { fn clone(&self) -> Self { Self { cfg_local: ArcSwap::from_pointee(self.cfg_local.load().as_ref().clone()), cfg_local_path: self.cfg_local_path.clone(), cfg_local_patterns: self.cfg_local_patterns.clone(), cfg_store: self.cfg_store.clone(), } } } ================================================ FILE: crates/common/src/manager/console.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use base64::Engine; use base64::engine::general_purpose; use std::env; use std::io::{self, Write}; use store::write::{AnyClass, AnyKey, BatchBuilder, ValueClass}; use store::{Deserialize, IterateParams, SUBSPACE_INDEXES, Store}; const HELP: &str = concat!( "Stalwart Server v", env!("CARGO_PKG_VERSION"), r#" Data Store CLI Enter commands (type 'help' for available commands). "# ); pub async fn store_console(store: Store) { print!("{HELP}"); if matches!(store, Store::None) { println!("No store available. Verify your configuration."); return; } loop { print!("> "); io::stdout().flush().unwrap(); let mut input = String::new(); io::stdin().read_line(&mut input).unwrap(); let input = input.trim(); let parts: Vec<&str> = input.split_whitespace().collect(); if parts.is_empty() { continue; } match parts[0] { "scan" => { if parts.len() != 3 { println!("Usage: scan "); } else if let (Some(from_key), Some(to_key)) = (parse_key(parts[1]), parse_key(parts[2])) { println!("Scanning from {:?} to {:?}", from_key, to_key); let mut from_key = from_key.into_iter(); let mut to_key = to_key.into_iter(); let from_subspace = from_key.next().unwrap(); let to_subspace = to_key.next().unwrap(); if from_subspace != to_subspace { println!("Keys must be in the same subspace."); return; } store .iterate( IterateParams::new( AnyKey { subspace: from_subspace, key: from_key.collect::>(), }, AnyKey { subspace: to_subspace, key: to_key.collect::>(), }, ) .set_values(![SUBSPACE_INDEXES].contains(&from_subspace)), |key, value| { print!("{}", char::from(from_subspace)); print_escaped(key); print!(" : "); print_escaped(value); println!(); Ok(true) }, ) .await .expect("Failed to scan keys"); } } "delete" => match (parts.get(1), parts.get(2)) { (Some(from_key), Some(to_key)) => { if let (Some(from_key), Some(to_key)) = (parse_key(from_key), parse_key(to_key)) { let mut from_key = from_key.into_iter(); let mut to_key = to_key.into_iter(); let from_key = AnyKey { subspace: from_key.next().unwrap(), key: from_key.collect::>(), }; let to_key = AnyKey { subspace: to_key.next().unwrap(), key: to_key.collect::>(), }; if from_key.subspace != to_key.subspace { println!("Keys must be in the same subspace."); return; } let mut total = 0; store .iterate( IterateParams::new(from_key.clone(), to_key.clone()).no_values(), |_, _| { total += 1; Ok(true) }, ) .await .expect("Failed to scan keys"); if total > 0 { print!("Are you sure you want to delete {total} keys? (y/N): "); io::stdout().flush().unwrap(); let mut response = String::new(); io::stdin().read_line(&mut response).unwrap(); if !response.trim().eq_ignore_ascii_case("y") { println!("Aborted."); return; } store .delete_range(from_key, to_key) .await .expect("Failed to delete keys"); println!("Deleted {total} keys."); } else { println!("No keys found."); } } } (Some(key), None) => { if let Some(key) = parse_key(key) { println!("Deleting key: {:?}", key); let mut key = key.into_iter(); let mut batch = BatchBuilder::new(); batch.clear(ValueClass::Any(AnyClass { subspace: key.next().unwrap(), key: key.collect(), })); if let Err(err) = store.write(batch.build_all()).await { println!("Failed to delete key: {}", err); } } } _ => { println!("Usage: delete []"); } }, "get" => { if parts.len() != 2 { println!("Usage: get "); } else if let Some(key) = parse_key(parts[1]) { let mut key = key.into_iter(); match store .get_value::(AnyKey { subspace: key.next().unwrap(), key: key.collect::>(), }) .await { Ok(Some(data)) => { print_escaped(&data.0); println!(); } Ok(None) => { println!("Key not found."); } Err(err) => { println!("Failed to retrieve key: {}", err); } } } } "put" => { if parts.len() < 2 { println!("Usage: put []"); } else if let Some(key) = parse_key(parts[1]) { let value = parts.get(2).map(|v| parse_value(v)).unwrap_or_default(); println!("Putting key: {key:?}"); let mut key = key.into_iter(); let mut batch = BatchBuilder::new(); batch.set( ValueClass::Any(AnyClass { subspace: key.next().unwrap(), key: key.collect(), }), value, ); if let Err(err) = store.write(batch.build_all()).await { println!("Failed to insert key: {}", err); } } } "help" => { print_help(); } "exit" | "quit" => { println!("Exiting..."); break; } _ => { println!("Unknown command. Type 'help' for available commands."); } } } } fn parse_key(input: &str) -> Option> { let result = if let Some(key) = input.strip_prefix("base64:") { base64_decode(key) } else { parse_binary(input) }; if matches!(result.first(), Some(ch) if ch.is_ascii_alphabetic() && ch.is_ascii_lowercase()) { Some(result) } else { println!("Invalid key: {result:?}"); None } } fn parse_value(input: &str) -> Vec { if let Some(key) = input.strip_prefix("base64:") { base64_decode(key) } else { parse_binary(input) } } fn base64_decode(input: &str) -> Vec { general_purpose::STANDARD .decode(input) .expect("Failed to decode base64") } fn parse_binary(input: &str) -> Vec { let mut result = Vec::new(); let mut chars = input.chars().peekable(); while let Some(c) = chars.next() { if c == '\\' { match chars.next() { Some('x') => { let hex: String = chars.by_ref().take(2).collect(); if hex.len() == 2 { if let Ok(byte) = u8::from_str_radix(&hex, 16) { result.push(byte); } else { result.extend_from_slice(b"\\x"); result.extend_from_slice(hex.as_bytes()); } } else { result.push(b'\\'); result.push(b'x'); result.extend_from_slice(hex.as_bytes()); } } Some(other) => { result.push(b'\\'); result.push(other as u8); } None => { result.push(b'\\'); } } } else { result.push(c as u8); } } result } fn print_escaped(bytes: &[u8]) { for ch in bytes { if ch.is_ascii() && !ch.is_ascii_control() && *ch != b'\\' { print!("{}", *ch as char); } else { print!("\\x{:02x}", ch); } } } fn print_help() { println!("Available commands:"); println!(" scan "); println!(" delete []"); println!(" get "); println!(" put []"); println!(" help"); println!(" exit/quit"); println!("Note: Keys and values can be prefixed with 'base64:' for base64 encoding"); println!(" or use escaped hex values (e.g., \\x41 for 'A')"); } struct RawValue(Vec); impl Deserialize for RawValue { fn deserialize(bytes: &[u8]) -> trc::Result { Ok(RawValue(bytes.to_vec())) } } ================================================ FILE: crates/common/src/manager/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use self::config::ConfigManager; use crate::USER_AGENT; use hyper::HeaderMap; use std::time::Duration; use utils::HttpLimitResponse; pub mod backup; pub mod boot; pub mod config; pub mod console; pub mod reload; pub mod restore; pub mod webadmin; const DEFAULT_SPAMFILTER_URL: &str = "https://github.com/stalwartlabs/spam-filter/releases/latest/download/spam-filter.toml"; pub const WEBADMIN_KEY: &[u8] = "STALWART_WEBADMIN".as_bytes(); pub const SPAM_TRAINER_KEY: &[u8] = "STALWART_SPAM_TRAIN_DATA.lz4".as_bytes(); pub const SPAM_CLASSIFIER_KEY: &[u8] = "STALWART_SPAM_CLASSIFIER_MODEL.lz4".as_bytes(); // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] const DEFAULT_WEBADMIN_URL: &str = "https://github.com/stalwartlabs/webadmin/releases/latest/download/webadmin.zip"; // SPDX-SnippetEnd #[cfg(not(feature = "enterprise"))] const DEFAULT_WEBADMIN_URL: &str = "https://github.com/stalwartlabs/webadmin/releases/latest/download/webadmin-oss.zip"; impl ConfigManager { pub async fn fetch_resource(&self, resource_id: &str) -> Result, String> { if let Some(url) = self .get(&format!("{resource_id}.resource")) .await .map_err(|err| { format!("Failed to fetch configuration key '{resource_id}.resource': {err}",) })? { fetch_resource(&url, None, Duration::from_secs(60), MAX_SIZE).await } else { match resource_id { "spam-filter" => { fetch_resource( DEFAULT_SPAMFILTER_URL, None, Duration::from_secs(60), MAX_SIZE, ) .await } "webadmin" => { fetch_resource( DEFAULT_WEBADMIN_URL, None, Duration::from_secs(60), MAX_SIZE, ) .await } _ => Err(format!("Unknown resource: {resource_id}")), } } } } const MAX_SIZE: usize = 100 * 1024 * 1024; pub async fn fetch_resource( url: &str, headers: Option, timeout: Duration, max_size: usize, ) -> Result, String> { if let Some(path) = url.strip_prefix("file://") { tokio::fs::read(path) .await .map_err(|err| format!("Failed to read {path}: {err}")) } else { let response = reqwest::Client::builder() .timeout(timeout) .danger_accept_invalid_certs(is_localhost_url(url)) .user_agent(USER_AGENT) .build() .unwrap_or_default() .get(url) .headers(headers.unwrap_or_default()) .send() .await .map_err(|err| format!("Failed to fetch {url}: {err}"))?; if response.status().is_success() { response .bytes_with_limit(max_size) .await .map_err(|err| format!("Failed to fetch {url}: {err}")) .and_then(|bytes| bytes.ok_or_else(|| format!("Resource too large: {url}"))) } else { let code = response.status().canonical_reason().unwrap_or_default(); let reason = response.text().await.unwrap_or_default(); Err(format!( "Failed to fetch {url}: Code: {code}, Details: {reason}", )) } } } pub fn is_localhost_url(url: &str) -> bool { url.split_once("://") .map(|(_, url)| url.split_once('/').map_or(url, |(host, _)| host)) .is_some_and(|host| { let host = host.rsplit_once(':').map_or(host, |(host, _)| host); host == "localhost" || host == "127.0.0.1" || host == "[::1]" }) } ================================================ FILE: crates/common/src/manager/reload.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use ahash::AHashMap; use arc_swap::ArcSwap; use store::Stores; use utils::config::Config; use crate::{ Core, Server, config::{ server::{Listeners, tls::parse_certificates}, telemetry::Telemetry, }, listener::blocked::{BLOCKED_IP_KEY, BlockedIps}, }; use super::config::{ConfigManager, Patterns}; pub struct ReloadResult { pub config: Config, pub new_core: Option, pub tracers: Option, } impl Server { pub async fn reload_blocked_ips(&self) -> trc::Result { let mut config = self .core .storage .config .build_config(BLOCKED_IP_KEY) .await?; *self.inner.data.blocked_ips.write() = BlockedIps::parse(&mut config).blocked_ip_addresses; Ok(config.into()) } pub async fn reload_certificates(&self) -> trc::Result { let mut config = self.core.storage.config.build_config("certificate").await?; let mut certificates = self.inner.data.tls_certificates.load().as_ref().clone(); parse_certificates(&mut config, &mut certificates, &mut Default::default()); self.inner.data.tls_certificates.store(certificates.into()); Ok(config.into()) } pub async fn reload_lookups(&self) -> trc::Result { let mut config = self.core.storage.config.build_config("lookup").await?; let mut stores = Stores::default(); stores.parse_static_stores(&mut config, true); let mut core = self.core.as_ref().clone(); for (id, store) in stores.in_memory_stores { core.storage.lookups.insert(id, store); } Ok(ReloadResult { config, new_core: core.into(), tracers: None, }) } pub async fn reload(&self) -> trc::Result { let mut config = self.core.storage.config.build_config("").await?; // Load stores let mut stores = Stores { stores: self.core.storage.stores.clone(), blob_stores: self.core.storage.blobs.clone(), search_stores: self.core.storage.ftss.clone(), in_memory_stores: self.core.storage.lookups.clone(), pubsub_stores: Default::default(), purge_schedules: Default::default(), }; stores.parse_stores(&mut config).await; stores.parse_in_memory(&mut config, true).await; // Parse tracers let tracers = Telemetry::parse(&mut config, &stores); if !config.errors.is_empty() { return Ok(config.into()); } // Build manager let manager = ConfigManager { cfg_local: ArcSwap::from_pointee( self.core.storage.config.cfg_local.load().as_ref().clone(), ), cfg_local_path: self.core.storage.config.cfg_local_path.clone(), cfg_local_patterns: Patterns::parse(&mut config).into(), cfg_store: config .value("storage.data") .and_then(|id| stores.stores.get(id)) .cloned() .unwrap_or_default(), }; // Parse settings and build shared core let core = Box::pin(Core::parse(&mut config, stores, manager)).await; if !config.errors.is_empty() { return Ok(config.into()); } // Update TLS certificates let mut new_certificates = AHashMap::new(); parse_certificates(&mut config, &mut new_certificates, &mut Default::default()); let mut current_certificates = self.inner.data.tls_certificates.load().as_ref().clone(); for (cert_id, cert) in new_certificates { current_certificates.insert(cert_id, cert); } self.inner .data .tls_certificates .store(current_certificates.into()); // Update blocked IPs *self.inner.data.blocked_ips.write() = BlockedIps::parse(&mut config).blocked_ip_addresses; // Parser servers let mut servers = Listeners::parse(&mut config); servers.parse_tcp_acceptors(&mut config, self.inner.clone()); Ok(if config.errors.is_empty() { ReloadResult { config, new_core: core.into(), tracers: tracers.into(), } } else { config.into() }) } } impl From for ReloadResult { fn from(config: Config) -> Self { Self { config, new_core: None, tracers: None, } } } ================================================ FILE: crates/common/src/manager/restore.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::backup::MAGIC_MARKER; use crate::{Core, DATABASE_SCHEMA_VERSION}; use lz4_flex::frame::FrameDecoder; use std::{ fs::File, io::{BufReader, ErrorKind, Read}, path::{Path, PathBuf}, }; use store::{ BlobStore, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INDEXES, SUBSPACE_QUOTA, Store, U32_LEN, write::{AnyClass, BatchBuilder, ValueClass, key::DeserializeBigEndian}, }; use types::{collection::Collection, field::Field}; use utils::{UnwrapFailure, failed}; impl Core { pub async fn restore(&self, src: PathBuf) { // Backup the core if src.is_dir() { // Iterate directory and spawn a task for each file let mut tasks = Vec::new(); for entry in std::fs::read_dir(&src).failed("Failed to read directory") { let entry = entry.failed("Failed to read entry"); let path = entry.path(); if path.is_file() { let storage = self.storage.clone(); let blob_store = self.storage.blob.clone(); tasks.push(tokio::spawn(async move { restore_file(storage.data, blob_store, &path).await; })); } } for task in tasks { task.await.failed("Failed to wait for task"); } } else { restore_file(self.storage.data.clone(), self.storage.blob.clone(), &src).await; } } } async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { println!("Importing database dump from {}.", path.to_str().unwrap()); let mut reader = KeyValueReader::new(path); let mut batch = BatchBuilder::new(); match reader.subspace { SUBSPACE_BLOBS => { while let Some((key, value)) = reader.next() { blob_store .put_blob(&key, &value) .await .failed("Failed to write blob"); } } SUBSPACE_COUNTER | SUBSPACE_QUOTA => { while let Some((key, value)) = reader.next() { batch.add( ValueClass::Any(AnyClass { subspace: reader.subspace, key, }), u64::from_le_bytes( value .try_into() .expect("Failed to deserialize counter/quota"), ) as i64, ); if batch.is_large_batch() { store .write(batch.build_all()) .await .failed("Failed to write batch"); batch = BatchBuilder::new(); } } } SUBSPACE_INDEXES => { while let Some((key, _)) = reader.next() { let account_id = key .as_slice() .deserialize_be_u32(0) .failed("Failed to deserialize account ID"); let collection = *key.get(U32_LEN).failed("Missing collection byte"); let field = *key.get(U32_LEN + 1).failed("Missing field byte"); let value = key .get(U32_LEN + 2..key.len() - U32_LEN) .failed("Missing index key") .to_vec(); let document_id = key .as_slice() .deserialize_be_u32(key.len() - U32_LEN) .failed("Failed to deserialize document ID"); batch .with_account_id(account_id) .with_collection(Collection::from(collection)) .with_document(document_id) .index(Field::new(field), value); if batch.is_large_batch() { store .write(batch.build_all()) .await .failed("Failed to write batch"); batch = BatchBuilder::new(); } } } _ => { while let Some((key, value)) = reader.next() { batch.set( ValueClass::Any(AnyClass { subspace: reader.subspace, key, }), value, ); if batch.is_large_batch() { store .write(batch.build_all()) .await .failed("Failed to write batch"); batch = BatchBuilder::new(); } } } } if !batch.is_empty() { store .write(batch.build_all()) .await .failed("Failed to write batch"); } } struct KeyValueReader { subspace: u8, file: FrameDecoder>, } impl KeyValueReader { fn new(path: &Path) -> Self { let mut file = FrameDecoder::new(BufReader::new( File::open(path).failed("Failed to open file"), )); let mut buf = [0u8; 1]; file.read_exact(&mut buf) .failed(&format!("Failed to read magic marker from {path:?}")); if buf[0] != MAGIC_MARKER { failed(&format!("Invalid magic marker in {path:?}")); } file.read_exact(&mut buf) .failed(&format!("Failed to read subspace from {path:?}")); let subspace = buf[0]; let mut buf = [0u8; 4]; file.read_exact(&mut buf) .failed(&format!("Failed to read version from {path:?}")); let version = u32::from_le_bytes(buf); if version != DATABASE_SCHEMA_VERSION { failed(&format!( "Invalid database schema version in {path:?}: Expected {DATABASE_SCHEMA_VERSION}, found {version}" )); } Self { file, subspace } } fn next(&mut self) -> Option<(Vec, Vec)> { let size = self.read_size()?; let mut key = vec![0; size as usize]; self.file .read_exact(&mut key) .failed("Failed to read bytes"); let value = self.expect_sized_bytes(); Some((key, value)) } fn read_size(&mut self) -> Option { let mut result = 0; let mut buf = [0u8; 1]; for shift in [0, 7, 14, 21, 28] { if let Err(err) = self.file.read_exact(&mut buf) { if err.kind() == ErrorKind::UnexpectedEof { return None; } else { failed(&format!("Failed to read file: {err:?}")); } } let byte = buf[0]; if (byte & 0x80) == 0 { result |= (byte as u32) << shift; return Some(result); } else { result |= ((byte & 0x7F) as u32) << shift; } } failed("Invalid leb128 sequence") } fn expect_sized_bytes(&mut self) -> Vec { let len = self.read_size().failed("Missing leb128 value sequence") as usize; let mut bytes = vec![0; len]; self.file .read_exact(&mut bytes) .failed("Failed to read bytes"); bytes } } ================================================ FILE: crates/common/src/manager/webadmin.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ borrow::Cow, io::{self, Cursor, Read}, path::PathBuf, }; use ahash::AHashMap; use arc_swap::ArcSwap; use store::BlobStore; use crate::Core; use super::WEBADMIN_KEY; pub struct WebAdminManager { bundle_path: TempDir, routes: ArcSwap>>, } #[derive(Default, Clone)] pub struct Resource { pub content_type: Cow<'static, str>, pub contents: T, } impl Resource { pub fn new(content_type: impl Into>, contents: T) -> Self { Self { content_type: content_type.into(), contents, } } } impl WebAdminManager { pub fn new(base_path: PathBuf) -> Self { Self { bundle_path: TempDir::new(base_path), routes: ArcSwap::from_pointee(Default::default()), } } pub async fn get(&self, path: &str) -> trc::Result>> { let routes = self.routes.load(); if let Some(resource) = routes.get(path).or_else(|| routes.get("index.html")) { tokio::fs::read(&resource.contents) .await .map(|contents| Resource { content_type: resource.content_type.clone(), contents, }) .map_err(|err| { trc::ResourceEvent::Error .reason(err) .ctx(trc::Key::Path, path.to_string()) .caused_by(trc::location!()) }) } else { Ok(Resource::default()) } } pub async fn unpack(&self, blob_store: &BlobStore) -> trc::Result<()> { // Delete any existing bundles self.bundle_path.clean().await.map_err(unpack_error)?; // Obtain webadmin bundle let bundle = blob_store .get_blob(WEBADMIN_KEY, 0..usize::MAX) .await? .ok_or_else(|| { trc::ResourceEvent::NotFound .caused_by(trc::location!()) .details("Webadmin bundle not found") })?; // Uncompress let mut bundle = zip::ZipArchive::new(Cursor::new(bundle)).map_err(|err| { trc::ResourceEvent::Error .caused_by(trc::location!()) .reason(err) .details("Failed to decompress webadmin bundle") })?; let mut routes = AHashMap::new(); for i in 0..bundle.len() { let (file_name, contents) = { let mut file = bundle.by_index(i).map_err(|err| { trc::ResourceEvent::Error .caused_by(trc::location!()) .reason(err) .details("Failed to read file from webadmin bundle") })?; if file.is_dir() { continue; } let mut contents = Vec::new(); file.read_to_end(&mut contents).map_err(unpack_error)?; (file.name().to_string(), contents) }; let path = self.bundle_path.path.join(format!("{i:02}")); tokio::fs::write(&path, contents) .await .map_err(unpack_error)?; let resource = Resource { content_type: match file_name .rsplit_once('.') .map(|(_, ext)| ext) .unwrap_or_default() { "html" => "text/html", "css" => "text/css", "wasm" => "application/wasm", "js" => "application/javascript", "json" => "application/json", "png" => "image/png", "svg" => "image/svg+xml", "ico" => "image/x-icon", _ => "application/octet-stream", } .into(), contents: path, }; routes.insert(file_name, resource); } // Update routes self.routes.store(routes.into()); trc::event!( Resource(trc::ResourceEvent::WebadminUnpacked), Path = self.bundle_path.path.to_string_lossy().into_owned(), ); Ok(()) } pub async fn update(&self, core: &Core) -> trc::Result<()> { let bytes = core .storage .config .fetch_resource("webadmin") .await .map_err(|err| { trc::ResourceEvent::Error .caused_by(trc::location!()) .reason(err) .details("Failed to download webadmin") })?; core.storage.blob.put_blob(WEBADMIN_KEY, &bytes).await } pub async fn update_and_unpack(&self, core: &Core) -> trc::Result<()> { self.update(core).await?; self.unpack(&core.storage.blob).await } } impl Resource> { pub fn is_empty(&self) -> bool { self.content_type.is_empty() && self.contents.is_empty() } } pub struct TempDir { pub path: PathBuf, } impl TempDir { pub fn new(path: PathBuf) -> TempDir { TempDir { path: path.join(std::str::from_utf8(WEBADMIN_KEY).unwrap()), } } pub async fn clean(&self) -> io::Result<()> { if tokio::fs::metadata(&self.path).await.is_ok() { let _ = tokio::fs::remove_dir_all(&self.path).await; } tokio::fs::create_dir(&self.path).await } } fn unpack_error(err: std::io::Error) -> trc::Error { trc::ResourceEvent::Error .reason(err) .details("Failed to unpack webadmin bundle") } impl Default for WebAdminManager { fn default() -> Self { Self::new(std::env::temp_dir()) } } impl Default for TempDir { fn default() -> Self { Self::new(std::env::temp_dir()) } } impl Drop for TempDir { fn drop(&mut self) { let _ = std::fs::remove_dir_all(&self.path); } } ================================================ FILE: crates/common/src/scripts/functions/array.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::collections::{HashMap, HashSet}; use sieve::{Context, runtime::Variable}; pub fn fn_count<'x>(_: &'x Context<'x>, v: Vec) -> Variable { match &v[0] { Variable::Array(a) => a.len(), v => { if !v.is_empty() { 1 } else { 0 } } } .into() } pub fn fn_sort<'x>(_: &'x Context<'x>, v: Vec) -> Variable { let is_asc = v[1].to_bool(); let mut arr = (*v[0].to_array()).clone(); if is_asc { arr.sort_unstable_by(|a, b| b.cmp(a)); } else { arr.sort_unstable(); } arr.into() } pub fn fn_dedup<'x>(_: &'x Context<'x>, v: Vec) -> Variable { let arr = v[0].to_array(); let mut result = Vec::with_capacity(arr.len()); for item in arr.iter() { if !result.contains(item) { result.push(item.clone()); } } result.into() } pub fn fn_cosine_similarity<'x>(_: &'x Context<'x>, v: Vec) -> Variable { let mut word_freq: HashMap = HashMap::new(); for (idx, var) in v.into_iter().enumerate() { match var { Variable::Array(l) => { for item in l.iter() { word_freq.entry(item.clone()).or_insert([0, 0])[idx] += 1; } } _ => { for char in var.to_string().chars() { word_freq.entry(char.to_string().into()).or_insert([0, 0])[idx] += 1; } } } } let mut dot_product = 0; let mut magnitude_a = 0; let mut magnitude_b = 0; for (_word, count) in word_freq.iter() { dot_product += count[0] * count[1]; magnitude_a += count[0] * count[0]; magnitude_b += count[1] * count[1]; } if magnitude_a != 0 && magnitude_b != 0 { dot_product as f64 / (magnitude_a as f64).sqrt() / (magnitude_b as f64).sqrt() } else { 0.0 } .into() } pub fn cosine_similarity(a: &[&str], b: &[&str]) -> f64 { let mut word_freq: HashMap<&str, [u32; 2]> = HashMap::new(); for (idx, items) in [a, b].into_iter().enumerate() { for item in items { word_freq.entry(item).or_insert([0, 0])[idx] += 1; } } let mut dot_product = 0; let mut magnitude_a = 0; let mut magnitude_b = 0; for (_word, count) in word_freq.iter() { dot_product += count[0] * count[1]; magnitude_a += count[0] * count[0]; magnitude_b += count[1] * count[1]; } if magnitude_a != 0 && magnitude_b != 0 { dot_product as f64 / (magnitude_a as f64).sqrt() / (magnitude_b as f64).sqrt() } else { 0.0 } } pub fn fn_jaccard_similarity<'x>(_: &'x Context<'x>, v: Vec) -> Variable { let mut word_freq = [HashSet::new(), HashSet::new()]; for (idx, var) in v.into_iter().enumerate() { match var { Variable::Array(l) => { for item in l.iter() { word_freq[idx].insert(item.clone()); } } _ => { for char in var.to_string().chars() { word_freq[idx].insert(char.to_string().into()); } } } } let intersection_size = word_freq[0].intersection(&word_freq[1]).count(); let union_size = word_freq[0].union(&word_freq[1]).count(); if union_size != 0 { intersection_size as f64 / union_size as f64 } else { 0.0 } .into() } pub fn fn_is_intersect<'x>(_: &'x Context<'x>, v: Vec) -> Variable { match (&v[0], &v[1]) { (Variable::Array(a), Variable::Array(b)) => a.iter().any(|x| b.contains(x)), (Variable::Array(a), item) | (item, Variable::Array(a)) => a.contains(item), _ => false, } .into() } pub fn fn_winnow<'x>(_: &'x Context<'x>, mut v: Vec) -> Variable { match v.remove(0) { Variable::Array(a) => a .iter() .filter(|i| !i.is_empty()) .cloned() .collect::>() .into(), v => v, } } ================================================ FILE: crates/common/src/scripts/functions/email.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use sieve::{Context, runtime::Variable}; use super::ApplyString; pub fn fn_is_email<'x>(_: &'x Context<'x>, v: Vec) -> Variable { let mut last_ch = 0; let mut in_quote = false; let mut at_count = 0; let mut dot_count = 0; let mut lp_len = 0; let mut value = 0; for ch in v[0].to_string().bytes() { match ch { b'0'..=b'9' | b'a'..=b'z' | b'A'..=b'Z' | b'!' | b'#' | b'$' | b'%' | b'&' | b'\'' | b'*' | b'+' | b'-' | b'/' | b'=' | b'?' | b'^' | b'_' | b'`' | b'{' | b'|' | b'}' | b'~' | 0x7f..=u8::MAX => { value += 1; } b'.' if !in_quote => { if last_ch != b'.' && last_ch != b'@' && value != 0 { value += 1; if at_count == 1 { dot_count += 1; } } else { return false.into(); } } b'@' if !in_quote => { at_count += 1; lp_len = value; value = 0; } b'>' | b':' | b',' | b' ' if in_quote => { value += 1; } b'\"' if !in_quote || last_ch != b'\\' => { in_quote = !in_quote; } b'\\' if in_quote && last_ch != b'\\' => (), _ => { if !in_quote { return false.into(); } } } last_ch = ch; } (at_count == 1 && dot_count > 0 && lp_len > 0 && value > 0).into() } pub fn fn_email_part<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].transform(|s| { s.rsplit_once('@') .map(|(u, d)| match v[1].to_string().as_ref() { "local" => Variable::from(u.trim()), "domain" => Variable::from(d.trim()), _ => Variable::default(), }) .unwrap_or_default() }) } ================================================ FILE: crates/common/src/scripts/functions/header.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use mail_parser::{HeaderName, HeaderValue, MimeHeaders, parsers::fields::thread::thread_name}; use sieve::{Context, compiler::ReceivedPart, runtime::Variable}; use super::ApplyString; pub fn fn_received_part<'x>(ctx: &'x Context<'x>, v: Vec) -> Variable { if let (Ok(part), Some(HeaderValue::Received(rcvd))) = ( ReceivedPart::try_from(v[1].to_string().as_ref()), ctx.message() .part(ctx.part()) .and_then(|p| { p.headers .iter() .filter(|h| h.name == HeaderName::Received) .nth((v[0].to_integer() as usize).saturating_sub(1)) }) .map(|h| &h.value), ) { part.eval(rcvd).unwrap_or_default() } else { Variable::default() } } pub fn fn_is_encoding_problem<'x>(ctx: &'x Context<'x>, _: Vec) -> Variable { ctx.message() .part(ctx.part()) .map(|p| p.is_encoding_problem) .unwrap_or_default() .into() } pub fn fn_is_attachment<'x>(ctx: &'x Context<'x>, _: Vec) -> Variable { ctx.message().attachments.contains(&ctx.part()).into() } pub fn fn_is_body<'x>(ctx: &'x Context<'x>, _: Vec) -> Variable { (ctx.message().text_body.contains(&ctx.part()) || ctx.message().html_body.contains(&ctx.part())) .into() } pub fn fn_attachment_name<'x>(ctx: &'x Context<'x>, _: Vec) -> Variable { ctx.message() .part(ctx.part()) .and_then(|p| p.attachment_name()) .unwrap_or_default() .into() } pub fn fn_mime_part_len<'x>(ctx: &'x Context<'x>, _: Vec) -> Variable { ctx.message() .part(ctx.part()) .map(|p| p.len()) .unwrap_or_default() .into() } pub fn fn_thread_name<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].transform(|s| thread_name(s).into()) } pub fn fn_is_header_utf8_valid<'x>(ctx: &'x Context<'x>, v: Vec) -> Variable { ctx.message() .part(ctx.part()) .map(|p| { let raw = ctx.message().raw_message(); let mut is_valid = true; if let Some(header_name) = HeaderName::parse(v[0].to_string().as_ref()) { for header in &p.headers { if header.name == header_name && raw .get(header.offset_start() as usize..header.offset_end() as usize) .and_then(|raw| std::str::from_utf8(raw).ok()) .is_none() { is_valid = false; break; } } } else { is_valid = raw .get(p.raw_header_offset() as usize..p.raw_body_offset() as usize) .and_then(|raw| std::str::from_utf8(raw).ok()) .is_some(); } Variable::from(is_valid) }) .unwrap_or(Variable::Integer(1)) } ================================================ FILE: crates/common/src/scripts/functions/image.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use sieve::{Context, runtime::Variable}; pub fn fn_img_metadata<'x>(ctx: &'x Context<'x>, v: Vec) -> Variable { ctx.message() .part(ctx.part()) .map(|p| p.contents()) .and_then(|bytes| { let arg = v[1].to_string(); match arg.as_ref() { "type" => imagesize::image_type(bytes).ok().map(|t| { Variable::from(match t { imagesize::ImageType::Aseprite => "aseprite", imagesize::ImageType::Bmp => "bmp", imagesize::ImageType::Dds => "dds", imagesize::ImageType::Exr => "exr", imagesize::ImageType::Farbfeld => "farbfeld", imagesize::ImageType::Gif => "gif", imagesize::ImageType::Hdr => "hdr", imagesize::ImageType::Heif(_) => "heif", imagesize::ImageType::Ico => "ico", imagesize::ImageType::Jpeg => "jpeg", imagesize::ImageType::Jxl => "jxl", imagesize::ImageType::Ktx2 => "ktx2", imagesize::ImageType::Png => "png", imagesize::ImageType::Pnm => "pnm", imagesize::ImageType::Psd => "psd", imagesize::ImageType::Qoi => "qoi", imagesize::ImageType::Tga => "tga", imagesize::ImageType::Tiff => "tiff", imagesize::ImageType::Vtf => "vtf", imagesize::ImageType::Webp => "webp", imagesize::ImageType::Ilbm => "ilbm", _ => "unknown", }) }), "width" => imagesize::blob_size(bytes) .ok() .map(|s| Variable::Integer(s.width as i64)), "height" => imagesize::blob_size(bytes) .ok() .map(|s| Variable::Integer(s.height as i64)), "area" => imagesize::blob_size(bytes) .ok() .map(|s| Variable::Integer(s.width.saturating_mul(s.height) as i64)), "dimension" => imagesize::blob_size(bytes) .ok() .map(|s| Variable::Integer(s.width.saturating_add(s.height) as i64)), _ => None, } }) .unwrap_or_default() } ================================================ FILE: crates/common/src/scripts/functions/misc.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::net::IpAddr; use mail_auth::common::resolver::ToReverseName; use sha1::Sha1; use sha2::{Sha256, Sha512}; use sieve::{Context, runtime::Variable}; use super::ApplyString; pub fn fn_is_empty<'x>(_: &'x Context<'x>, v: Vec) -> Variable { match &v[0] { Variable::String(s) => s.is_empty(), Variable::Integer(_) | Variable::Float(_) => false, Variable::Array(a) => a.is_empty(), } .into() } pub fn fn_is_number<'x>(_: &'x Context<'x>, v: Vec) -> Variable { matches!(&v[0], Variable::Integer(_) | Variable::Float(_)).into() } pub fn fn_is_ip_addr<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].to_string().parse::().is_ok().into() } pub fn fn_is_ipv4_addr<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].to_string() .parse::() .is_ok_and(|ip| matches!(ip, IpAddr::V4(_))) .into() } pub fn fn_is_ipv6_addr<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].to_string() .parse::() .is_ok_and(|ip| matches!(ip, IpAddr::V6(_))) .into() } pub fn fn_ip_reverse_name<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].to_string() .parse::() .map(|ip| ip.to_reverse_name()) .unwrap_or_default() .into() } pub fn fn_detect_file_type<'x>(ctx: &'x Context<'x>, v: Vec) -> Variable { ctx.message() .part(ctx.part()) .and_then(|p| infer::get(p.contents())) .map(|t| { Variable::from( if v[0].to_string() != "ext" { t.mime_type() } else { t.extension() } .to_string(), ) }) .unwrap_or_default() } pub fn fn_hash<'x>(_: &'x Context<'x>, v: Vec) -> Variable { use sha1::Digest; let hash = v[1].to_string(); v[0].transform(|value| match hash.as_ref() { "md5" => format!("{:x}", md5::compute(value.as_bytes())).into(), "sha1" => { let mut hasher = Sha1::new(); hasher.update(value.as_bytes()); format!("{:x}", hasher.finalize()).into() } "sha256" => { let mut hasher = Sha256::new(); hasher.update(value.as_bytes()); format!("{:x}", hasher.finalize()).into() } "sha512" => { let mut hasher = Sha512::new(); hasher.update(value.as_bytes()); format!("{:x}", hasher.finalize()).into() } _ => Variable::default(), }) } pub fn fn_get_var_names<'x>(ctx: &'x Context<'x>, _: Vec) -> Variable { Variable::Array( ctx.global_variable_names() .map(|v| Variable::from(v.to_uppercase())) .collect::>() .into(), ) } ================================================ FILE: crates/common/src/scripts/functions/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod array; mod email; mod header; pub mod image; pub mod misc; pub mod text; pub mod unicode; pub mod url; use sieve::{FunctionMap, runtime::Variable}; use self::{array::*, email::*, header::*, image::*, misc::*, text::*, unicode::*, url::*}; pub fn register_functions_trusted() -> FunctionMap { FunctionMap::new() .with_function("trim", fn_trim) .with_function("trim_start", fn_trim_start) .with_function("trim_end", fn_trim_end) .with_function("len", fn_len) .with_function("count", fn_count) .with_function("is_empty", fn_is_empty) .with_function("is_number", fn_is_number) .with_function("is_ascii", fn_is_ascii) .with_function("to_lowercase", fn_to_lowercase) .with_function("to_uppercase", fn_to_uppercase) .with_function("detect_language", fn_detect_language) .with_function("is_email", fn_is_email) .with_function("thread_name", fn_thread_name) .with_function("html_to_text", fn_html_to_text) .with_function("is_uppercase", fn_is_uppercase) .with_function("is_lowercase", fn_is_lowercase) .with_function("has_digits", fn_has_digits) .with_function("count_spaces", fn_count_spaces) .with_function("count_uppercase", fn_count_uppercase) .with_function("count_lowercase", fn_count_lowercase) .with_function("count_chars", fn_count_chars) .with_function("dedup", fn_dedup) .with_function("lines", fn_lines) .with_function("is_header_utf8_valid", fn_is_header_utf8_valid) .with_function("img_metadata", fn_img_metadata) .with_function("is_ip_addr", fn_is_ip_addr) .with_function("is_ipv4_addr", fn_is_ipv4_addr) .with_function("is_ipv6_addr", fn_is_ipv6_addr) .with_function("ip_reverse_name", fn_ip_reverse_name) .with_function("winnow", fn_winnow) .with_function("has_zwsp", fn_has_zwsp) .with_function("has_obscured", fn_has_obscured) .with_function("is_mixed_charset", fn_is_mixed_charset) .with_function("puny_decode", fn_puny_decode) .with_function("unicode_skeleton", fn_unicode_skeleton) .with_function("cure_text", fn_cure_text) .with_function("detect_file_type", fn_detect_file_type) .with_function_args("sort", fn_sort, 2) .with_function_args("email_part", fn_email_part, 2) .with_function_args("eq_ignore_case", fn_eq_ignore_case, 2) .with_function_args("contains", fn_contains, 2) .with_function_args("contains_ignore_case", fn_contains_ignore_case, 2) .with_function_args("starts_with", fn_starts_with, 2) .with_function_args("ends_with", fn_ends_with, 2) .with_function_args("received_part", fn_received_part, 2) .with_function_args("cosine_similarity", fn_cosine_similarity, 2) .with_function_args("jaccard_similarity", fn_jaccard_similarity, 2) .with_function_args("levenshtein_distance", fn_levenshtein_distance, 2) .with_function_args("uri_part", fn_uri_part, 2) .with_function_args("substring", fn_substring, 3) .with_function_args("split", fn_split, 2) .with_function_args("rsplit", fn_rsplit, 2) .with_function_args("split_once", fn_split_once, 2) .with_function_args("rsplit_once", fn_rsplit_once, 2) .with_function_args("split_n", fn_split_n, 3) .with_function_args("strip_prefix", fn_strip_prefix, 2) .with_function_args("strip_suffix", fn_strip_suffix, 2) .with_function_args("is_intersect", fn_is_intersect, 2) .with_function_args("hash", fn_hash, 2) .with_function_no_args("is_encoding_problem", fn_is_encoding_problem) .with_function_no_args("is_attachment", fn_is_attachment) .with_function_no_args("is_body", fn_is_body) .with_function_no_args("var_names", fn_get_var_names) .with_function_no_args("attachment_name", fn_attachment_name) .with_function_no_args("mime_part_len", fn_mime_part_len) } pub fn register_functions_untrusted() -> FunctionMap { FunctionMap::new() .with_function("trim", fn_trim) .with_function("trim_start", fn_trim_start) .with_function("trim_end", fn_trim_end) .with_function("len", fn_len) .with_function("count", fn_count) .with_function("is_empty", fn_is_empty) .with_function("is_number", fn_is_number) .with_function("is_ascii", fn_is_ascii) .with_function("to_lowercase", fn_to_lowercase) .with_function("to_uppercase", fn_to_uppercase) .with_function("is_email", fn_is_email) .with_function("thread_name", fn_thread_name) .with_function("html_to_text", fn_html_to_text) .with_function("is_uppercase", fn_is_uppercase) .with_function("is_lowercase", fn_is_lowercase) .with_function("has_digits", fn_has_digits) .with_function("count_spaces", fn_count_spaces) .with_function("count_uppercase", fn_count_uppercase) .with_function("count_lowercase", fn_count_lowercase) .with_function("count_chars", fn_count_chars) .with_function("dedup", fn_dedup) .with_function("lines", fn_lines) .with_function("is_ip_addr", fn_is_ip_addr) .with_function("is_ipv4_addr", fn_is_ipv4_addr) .with_function("is_ipv6_addr", fn_is_ipv6_addr) .with_function("winnow", fn_winnow) .with_function_args("sort", fn_sort, 2) .with_function_args("email_part", fn_email_part, 2) .with_function_args("eq_ignore_case", fn_eq_ignore_case, 2) .with_function_args("contains", fn_contains, 2) .with_function_args("contains_ignore_case", fn_contains_ignore_case, 2) .with_function_args("starts_with", fn_starts_with, 2) .with_function_args("ends_with", fn_ends_with, 2) .with_function_args("uri_part", fn_uri_part, 2) .with_function_args("substring", fn_substring, 3) .with_function_args("split", fn_split, 2) .with_function_args("rsplit", fn_rsplit, 2) .with_function_args("split_once", fn_split_once, 2) .with_function_args("rsplit_once", fn_rsplit_once, 2) .with_function_args("split_n", fn_split_n, 3) .with_function_args("strip_prefix", fn_strip_prefix, 2) .with_function_args("strip_suffix", fn_strip_suffix, 2) .with_function_args("is_intersect", fn_is_intersect, 2) } pub trait ApplyString<'x> { fn transform(&self, f: impl Fn(&'_ str) -> Variable) -> Variable; } impl ApplyString<'_> for Variable { fn transform(&self, f: impl Fn(&'_ str) -> Variable) -> Variable { match self { Variable::String(s) => f(s), Variable::Array(list) => list .iter() .map(|v| match v { Variable::String(s) => f(s), v => f(v.to_string().as_ref()), }) .collect::>() .into(), v => f(v.to_string().as_ref()), } } } ================================================ FILE: crates/common/src/scripts/functions/text.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use mail_parser::decoders::html::html_to_text; use sieve::{Context, runtime::Variable}; use super::ApplyString; pub fn fn_trim<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].transform(|s| Variable::from(s.trim())) } pub fn fn_trim_end<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].transform(|s| Variable::from(s.trim_end())) } pub fn fn_trim_start<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].transform(|s| Variable::from(s.trim_start())) } pub fn fn_len<'x>(_: &'x Context<'x>, v: Vec) -> Variable { match &v[0] { Variable::String(s) => s.len(), Variable::Array(a) => a.len(), v => v.to_string().len(), } .into() } pub fn fn_to_lowercase<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].transform(|s| Variable::from(s.to_lowercase())) } pub fn fn_to_uppercase<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].transform(|s| Variable::from(s.to_uppercase())) } pub fn fn_is_uppercase<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].transform(|s| { s.chars() .filter(|c| c.is_alphabetic()) .all(|c| c.is_uppercase()) .into() }) } pub fn fn_is_lowercase<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].transform(|s| { s.chars() .filter(|c| c.is_alphabetic()) .all(|c| c.is_lowercase()) .into() }) } pub fn fn_has_digits<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].transform(|s| s.chars().any(|c| c.is_ascii_digit()).into()) } pub fn tokenize_words(v: &Variable) -> Variable { v.to_string() .split_whitespace() .filter(|word| word.chars().all(|c| c.is_alphanumeric())) .map(|word| Variable::from(word.to_string())) .collect::>() .into() } pub fn fn_count_spaces<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].to_string() .as_ref() .chars() .filter(|c| c.is_whitespace()) .count() .into() } pub fn fn_count_uppercase<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].to_string() .as_ref() .chars() .filter(|c| c.is_alphabetic() && c.is_uppercase()) .count() .into() } pub fn fn_count_lowercase<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].to_string() .as_ref() .chars() .filter(|c| c.is_alphabetic() && c.is_lowercase()) .count() .into() } pub fn fn_count_chars<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].to_string().as_ref().chars().count().into() } pub fn fn_eq_ignore_case<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].to_string() .eq_ignore_ascii_case(v[1].to_string().as_ref()) .into() } pub fn fn_contains<'x>(_: &'x Context<'x>, v: Vec) -> Variable { match &v[0] { Variable::String(s) => s.contains(v[1].to_string().as_ref()), Variable::Array(arr) => arr.contains(&v[1]), val => val.to_string().contains(v[1].to_string().as_ref()), } .into() } pub fn fn_contains_ignore_case<'x>(_: &'x Context<'x>, v: Vec) -> Variable { let needle = v[1].to_string(); match &v[0] { Variable::String(s) => s.to_lowercase().contains(&needle.to_lowercase()), Variable::Array(arr) => arr.iter().any(|v| match v { Variable::String(s) => s.eq_ignore_ascii_case(needle.as_ref()), _ => false, }), val => val.to_string().contains(needle.as_ref()), } .into() } pub fn fn_starts_with<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].to_string() .starts_with(v[1].to_string().as_ref()) .into() } pub fn fn_ends_with<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].to_string().ends_with(v[1].to_string().as_ref()).into() } pub fn fn_lines<'x>(_: &'x Context<'x>, mut v: Vec) -> Variable { match v.remove(0) { Variable::String(s) => s .lines() .map(|s| Variable::from(s.to_string())) .collect::>() .into(), val => val, } } pub fn fn_substring<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].to_string() .chars() .skip(v[1].to_usize()) .take(v[2].to_usize()) .collect::() .into() } pub fn fn_strip_prefix<'x>(_: &'x Context<'x>, v: Vec) -> Variable { let prefix = v[1].to_string(); v[0].transform(|s| { s.strip_prefix(prefix.as_ref()) .map(Variable::from) .unwrap_or_default() }) } pub fn fn_strip_suffix<'x>(_: &'x Context<'x>, v: Vec) -> Variable { let suffix = v[1].to_string(); v[0].transform(|s| { s.strip_suffix(suffix.as_ref()) .map(Variable::from) .unwrap_or_default() }) } pub fn fn_split<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].to_string() .split(v[1].to_string().as_ref()) .map(|s| Variable::from(s.to_string())) .collect::>() .into() } pub fn fn_rsplit<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].to_string() .rsplit(v[1].to_string().as_ref()) .map(|s| Variable::from(s.to_string())) .collect::>() .into() } pub fn fn_split_n<'x>(_: &'x Context<'x>, v: Vec) -> Variable { let value = v[0].to_string(); let arg = v[1].to_string(); let num = v[2].to_integer() as usize; let mut result = Vec::new(); let mut s = value.as_ref(); for _ in 0..num { if let Some((a, b)) = s.split_once(arg.as_ref()) { result.push(Variable::from(a.to_string())); s = b; } else { break; } } result.push(Variable::from(s.to_string())); result.into() } pub fn fn_split_once<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].to_string() .split_once(v[1].to_string().as_ref()) .map(|(a, b)| { Variable::Array( vec![Variable::from(a.to_string()), Variable::from(b.to_string())].into(), ) }) .unwrap_or_default() } pub fn fn_rsplit_once<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].to_string() .rsplit_once(v[1].to_string().as_ref()) .map(|(a, b)| { Variable::Array( vec![Variable::from(a.to_string()), Variable::from(b.to_string())].into(), ) }) .unwrap_or_default() } /** * `levenshtein-rs` - levenshtein * * MIT licensed. * * Copyright (c) 2016 Titus Wormer */ pub fn fn_levenshtein_distance<'x>(_: &'x Context<'x>, v: Vec) -> Variable { let a = v[0].to_string(); let b = v[1].to_string(); levenshtein_distance(a.as_ref(), b.as_ref()).into() } pub fn levenshtein_distance(a: &str, b: &str) -> usize { let mut result = 0; /* Shortcut optimizations / degenerate cases. */ if a == b { return result; } let length_a = a.chars().count(); let length_b = b.chars().count(); if length_a == 0 { return length_b; } else if length_b == 0 { return length_a; } /* Initialize the vector. * * This is why it’s fast, normally a matrix is used, * here we use a single vector. */ let mut cache: Vec = (1..).take(length_a).collect(); let mut distance_a; let mut distance_b; /* Loop. */ for (index_b, code_b) in b.chars().enumerate() { result = index_b; distance_a = index_b; for (index_a, code_a) in a.chars().enumerate() { distance_b = if code_a == code_b { distance_a } else { distance_a + 1 }; distance_a = cache[index_a]; result = if distance_a > result { if distance_b > result { result + 1 } else { distance_b } } else if distance_b > distance_a { distance_a + 1 } else { distance_b }; cache[index_a] = result; } } result } pub fn fn_detect_language<'x>(_: &'x Context<'x>, v: Vec) -> Variable { whatlang::detect_lang(v[0].to_string().as_ref()) .map(|l| l.code()) .unwrap_or("unknown") .into() } pub fn fn_html_to_text<'x>(_: &'x Context<'x>, v: Vec) -> Variable { html_to_text(v[0].to_string().as_ref()).into() } ================================================ FILE: crates/common/src/scripts/functions/unicode.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use sieve::{Context, runtime::Variable}; use crate::scripts::IsMixedCharset; pub fn fn_is_ascii<'x>(_: &'x Context<'x>, v: Vec) -> Variable { match &v[0] { Variable::String(s) => s.is_ascii(), Variable::Integer(_) | Variable::Float(_) => true, Variable::Array(a) => a.iter().all(|v| match v { Variable::String(s) => s.is_ascii(), _ => true, }), } .into() } pub fn fn_has_zwsp<'x>(_: &'x Context<'x>, v: Vec) -> Variable { match &v[0] { Variable::String(s) => s.chars().any(|c| c.is_zwsp()), Variable::Array(a) => a.iter().any(|v| match v { Variable::String(s) => s.chars().any(|c| c.is_zwsp()), _ => true, }), Variable::Integer(_) | Variable::Float(_) => false, } .into() } pub fn fn_has_obscured<'x>(_: &'x Context<'x>, v: Vec) -> Variable { match &v[0] { Variable::String(s) => s.chars().any(|c| c.is_obscured()), Variable::Array(a) => a.iter().any(|v| match v { Variable::String(s) => s.chars().any(|c| c.is_obscured()), _ => true, }), Variable::Integer(_) | Variable::Float(_) => false, } .into() } pub trait CharUtils { fn is_zwsp(&self) -> bool; fn is_obscured(&self) -> bool; } impl CharUtils for char { fn is_zwsp(&self) -> bool { matches!( self, '\u{200B}' | '\u{200C}' | '\u{200D}' | '\u{FEFF}' | '\u{00AD}' ) } fn is_obscured(&self) -> bool { matches!( self, '\u{200B}'..='\u{200F}' | '\u{2028}'..='\u{202F}' | '\u{205F}'..='\u{206F}' | '\u{FEFF}' ) } } pub fn fn_cure_text<'x>(_: &'x Context<'x>, v: Vec) -> Variable { decancer::cure(v[0].to_string().as_ref(), decancer::Options::default()) .map(String::from) .unwrap_or_default() .into() } pub fn fn_unicode_skeleton<'x>(_: &'x Context<'x>, v: Vec) -> Variable { unicode_security::skeleton(v[0].to_string().as_ref()) .collect::() .into() } pub fn fn_is_mixed_charset<'x>(_: &'x Context<'x>, v: Vec) -> Variable { let text = v[0].to_string(); if !text.is_empty() { text.as_ref().is_mixed_charset() } else { false } .into() } ================================================ FILE: crates/common/src/scripts/functions/url.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use hyper::Uri; use sieve::{Context, runtime::Variable}; use super::ApplyString; pub fn fn_uri_part<'x>(_: &'x Context<'x>, v: Vec) -> Variable { let part = v[1].to_string(); v[0].transform(|uri| { uri.parse::() .ok() .and_then(|uri| match part.as_ref() { "scheme" => uri.scheme_str().map(|s| Variable::from(s.to_string())), "host" => uri.host().map(|s| Variable::from(s.to_string())), "scheme_host" => uri .scheme_str() .and_then(|s| (s, uri.host()?).into()) .map(|(s, h)| Variable::from(format!("{}://{}", s, h))), "path" => Variable::from(uri.path().to_string()).into(), "port" => uri.port_u16().map(|port| Variable::Integer(port as i64)), "query" => uri.query().map(|s| Variable::from(s.to_string())), "path_query" => uri.path_and_query().map(|s| Variable::from(s.to_string())), "authority" => uri.authority().map(|s| Variable::from(s.to_string())), _ => None, }) .unwrap_or_default() }) } pub fn fn_puny_decode<'x>(_: &'x Context<'x>, v: Vec) -> Variable { v[0].transform(|domain| { if domain.contains("xn--") { let mut decoded = String::with_capacity(domain.len()); for part in domain.split('.') { if !decoded.is_empty() { decoded.push('.'); } if let Some(puny) = part .strip_prefix("xn--") .and_then(idna::punycode::decode_to_string) { decoded.push_str(&puny); } else { decoded.push_str(part); } } decoded.into() } else { domain.into() } }) } ================================================ FILE: crates/common/src/scripts/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::Arc; use sieve::{Envelope, runtime::Variable}; use store::Value; use unicode_security::mixed_script::AugmentedScriptSet; use crate::IntoString; pub mod functions; pub mod plugins; #[derive(Debug, serde::Serialize)] #[serde(tag = "action")] #[serde(rename_all = "camelCase")] pub enum ScriptModification { SetEnvelope { name: Envelope, value: String, }, AddHeader { name: Arc, value: Arc, }, } pub fn into_sieve_value(value: Value) -> Variable { match value { Value::Integer(v) => Variable::Integer(v), Value::Bool(v) => Variable::Integer(i64::from(v)), Value::Float(v) => Variable::Float(v), Value::Text(v) => Variable::String(v.into_owned().into()), Value::Blob(v) => Variable::String(v.into_owned().into_string().into()), Value::Null => Variable::default(), } } pub fn into_store_value(value: Variable) -> Value<'static> { match value { Variable::String(v) => Value::Text(v.to_string().into()), Variable::Integer(v) => Value::Integer(v), Variable::Float(v) => Value::Float(v), v => Value::Text(v.to_string().into_owned().into()), } } pub fn to_store_value(value: &Variable) -> Value<'static> { match value { Variable::String(v) => Value::Text(v.to_string().into()), Variable::Integer(v) => Value::Integer(*v), Variable::Float(v) => Value::Float(*v), v => Value::Text(v.to_string().into_owned().into()), } } pub trait IsMixedCharset { fn is_mixed_charset(&self) -> bool; } impl> IsMixedCharset for T { fn is_mixed_charset(&self) -> bool { let mut set: Option = None; for ch in self.as_ref().chars() { if !ch.is_ascii() { set.get_or_insert_default().intersect_with(ch.into()); } } set.is_some_and(|set| set.is_empty()) } } ================================================ FILE: crates/common/src/scripts/plugins/dns.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::net::IpAddr; use mail_auth::IpLookupStrategy; use sieve::{FunctionMap, runtime::Variable}; use super::PluginContext; pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("dns_query", plugin_id, 2); } pub fn register_exists(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("dns_exists", plugin_id, 2); } pub async fn exec(ctx: PluginContext<'_>) -> trc::Result { let entry = ctx.arguments[0].to_string(); let record_type = ctx.arguments[1].to_string(); Ok(if record_type.eq_ignore_ascii_case("ip") { match ctx .server .core .smtp .resolvers .dns .ip_lookup( entry.as_ref(), IpLookupStrategy::Ipv4thenIpv6, 10, Some(&ctx.server.inner.cache.dns_ipv4), Some(&ctx.server.inner.cache.dns_ipv6), ) .await { Ok(result) => result .iter() .map(|ip| Variable::from(ip.to_string())) .collect::>() .into(), Err(err) => err.short_error().into(), } } else if record_type.eq_ignore_ascii_case("mx") { match ctx .server .core .smtp .resolvers .dns .mx_lookup(entry.as_ref(), Some(&ctx.server.inner.cache.dns_mx)) .await { Ok(result) => result .iter() .flat_map(|mx| { mx.exchanges .iter() .map(|host| Variable::from(format!("{} {}", mx.preference, host))) }) .collect::>() .into(), Err(err) => err.short_error().into(), } } else if record_type.eq_ignore_ascii_case("txt") { #[cfg(feature = "test_mode")] { if entry.contains("origin") { return Ok(Variable::from("23028|US|arin|2002-01-04".to_string())); } } match ctx .server .core .smtp .resolvers .dns .txt_raw_lookup(entry.as_ref()) .await { Ok(result) => Variable::from(String::from_utf8(result).unwrap_or_default()), Err(err) => err.short_error().into(), } } else if record_type.eq_ignore_ascii_case("ptr") { if let Ok(addr) = entry.parse::() { match ctx .server .core .smtp .resolvers .dns .ptr_lookup(addr, Some(&ctx.server.inner.cache.dns_ptr)) .await { Ok(result) => result .iter() .map(|host| Variable::from(host.to_string())) .collect::>() .into(), Err(err) => err.short_error().into(), } } else { Variable::default() } } else if record_type.eq_ignore_ascii_case("ipv4") { #[cfg(feature = "test_mode")] { if entry.contains(".168.192.") { let parts = entry.split('.').collect::>(); return Ok(vec![Variable::from(format!("127.0.{}.{}", parts[1], parts[0]))].into()); } } match ctx .server .core .smtp .resolvers .dns .ipv4_lookup(entry.as_ref(), Some(&ctx.server.inner.cache.dns_ipv4)) .await { Ok(result) => result .iter() .map(|ip| Variable::from(ip.to_string())) .collect::>() .into(), Err(err) => err.short_error().into(), } } else if record_type.eq_ignore_ascii_case("ipv6") { match ctx .server .core .smtp .resolvers .dns .ipv6_lookup(entry.as_ref(), Some(&ctx.server.inner.cache.dns_ipv6)) .await { Ok(result) => result .iter() .map(|ip| Variable::from(ip.to_string())) .collect::>() .into(), Err(err) => err.short_error().into(), } } else { Variable::default() }) } pub async fn exec_exists(ctx: PluginContext<'_>) -> trc::Result { let entry = ctx.arguments[0].to_string(); let record_type = ctx.arguments[1].to_string(); let result = if record_type.eq_ignore_ascii_case("ip") { ctx.server.dns_exists_ip(entry.as_ref()).await } else if record_type.eq_ignore_ascii_case("mx") { ctx.server.dns_exists_mx(entry.as_ref()).await } else if record_type.eq_ignore_ascii_case("ptr") { ctx.server.dns_exists_ptr(entry.as_ref()).await } else if record_type.eq_ignore_ascii_case("ipv4") { #[cfg(feature = "test_mode")] { if entry.starts_with("2.0.168.192.") { return Ok(1.into()); } } ctx.server.dns_exists_ipv4(entry.as_ref()).await } else if record_type.eq_ignore_ascii_case("ipv6") { ctx.server.dns_exists_ipv6(entry.as_ref()).await } else { return Ok((-1).into()); }; Ok(result.map(i64::from).unwrap_or(-1).into()) } trait ShortError { fn short_error(&self) -> &'static str; } impl ShortError for mail_auth::Error { fn short_error(&self) -> &'static str { match self { mail_auth::Error::DnsError(_) => "temp_fail", mail_auth::Error::DnsRecordNotFound(_) => "not_found", mail_auth::Error::Io(_) => "io_error", mail_auth::Error::InvalidRecordType => "invalid_record", _ => "unknown_error", } } } ================================================ FILE: crates/common/src/scripts/plugins/exec.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::process::Command; use sieve::{FunctionMap, runtime::Variable}; use super::PluginContext; pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("exec", plugin_id, 2); } pub async fn exec(ctx: PluginContext<'_>) -> trc::Result { let mut arguments = ctx.arguments.into_iter(); tokio::task::spawn_blocking(move || { let command = arguments .next() .map(|a| a.to_string().into_owned()) .unwrap_or_default(); match Command::new(&command) .args( arguments .next() .map(|a| a.into_string_array()) .unwrap_or_default(), ) .output() { Ok(result) => Ok(result.status.success()), Err(err) => Err(trc::SieveEvent::RuntimeError .ctx(trc::Key::Path, command) .reason(err) .details("Failed to execute command")), } }) .await .map_err(|err| { trc::EventType::Server(trc::ServerEvent::ThreadError) .reason(err) .caused_by(trc::location!()) .details("Join Error") })? .map(Into::into) } ================================================ FILE: crates/common/src/scripts/plugins/headers.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use sieve::{FunctionMap, runtime::Variable}; use crate::scripts::ScriptModification; use super::PluginContext; pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("add_header", plugin_id, 2); } pub fn exec(ctx: PluginContext<'_>) -> trc::Result { Ok(if let (Variable::String(name), Variable::String(value)) = (&ctx.arguments[0], &ctx.arguments[1]) { ctx.modifications.push(ScriptModification::AddHeader { name: name.clone(), value: value.clone(), }); true } else { false } .into()) } ================================================ FILE: crates/common/src/scripts/plugins/http.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use reqwest::redirect::Policy; use sieve::{FunctionMap, runtime::Variable}; use super::PluginContext; pub fn register_header(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("http_header", plugin_id, 4); } pub async fn exec_header(ctx: PluginContext<'_>) -> trc::Result { let url = ctx.arguments[0].to_string(); let header = ctx.arguments[1].to_string(); let agent = ctx.arguments[2].to_string(); let timeout = ctx.arguments[3].to_string().parse::().unwrap_or(5000); #[cfg(feature = "test_mode")] if url.contains("redirect.") { return Ok(Variable::from(url.split_once("/?").unwrap().1.to_string())); } reqwest::Client::builder() .user_agent(agent.as_ref()) .timeout(Duration::from_millis(timeout)) .redirect(Policy::none()) .danger_accept_invalid_certs(true) .build() .map_err(|err| { trc::SieveEvent::RuntimeError .into_err() .reason(err) .details("Failed to build request") })? .get(url.as_ref()) .send() .await .map_err(|err| { trc::SieveEvent::RuntimeError .into_err() .reason(err) .details("Failed to send request") }) .map(|response| { response .headers() .get(header.as_ref()) .and_then(|h| h.to_str().ok()) .map(|h| Variable::from(h.to_string())) .unwrap_or_default() }) } ================================================ FILE: crates/common/src/scripts/plugins/llm_prompt.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Instant; use directory::Permission; use sieve::{FunctionMap, compiler::Number, runtime::Variable}; use trc::{AiEvent, SecurityEvent}; use super::PluginContext; pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("llm_prompt", plugin_id, 3); } pub async fn exec(ctx: PluginContext<'_>) -> trc::Result { // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] if let (Variable::String(name), Variable::String(prompt)) = (&ctx.arguments[0], &ctx.arguments[1]) { #[cfg(feature = "test_mode")] if name.as_ref() == "echo-test" { return Ok(prompt.to_string().into()); } let temperature = ctx.arguments[2].to_number_checked().map(|n| match n { Number::Integer(n) => (n as f64).clamp(0.0, 1.0), Number::Float(n) => n.clamp(0.0, 1.0), }); if let Some(ai_api) = ctx.server.core.enterprise.as_ref().and_then(|e| { if ctx.access_token.is_none_or(|token| { if token.has_permission(Permission::AiModelInteract) { true } else { trc::event!( Security(SecurityEvent::Unauthorized), AccountId = token.primary_id(), Details = Permission::AiModelInteract.name(), SpanId = ctx.session_id, ); false } }) { if e.ai_apis.len() == 1 && name.is_empty() { e.ai_apis.values().next() } else { e.ai_apis.get(name.as_ref()) } } else { None } }) { let time = Instant::now(); match ai_api.send_request(prompt.as_ref(), temperature).await { Ok(response) => { trc::event!( Ai(AiEvent::LlmResponse), Id = ai_api.id.clone(), Value = prompt.to_string(), Details = response.clone(), Elapsed = time.elapsed(), SpanId = ctx.session_id, ); return Ok(response.into()); } Err(err) => { trc::error!(err.span_id(ctx.session_id)); } } } } // SPDX-SnippetEnd Ok(false.into()) } ================================================ FILE: crates/common/src/scripts/plugins/lookup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::PluginContext; use crate::scripts::into_sieve_value; use sieve::{FunctionMap, runtime::Variable}; use store::{Deserialize, Value, dispatch::lookup::KeyValue}; pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("key_exists", plugin_id, 2); } pub fn register_get(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("key_get", plugin_id, 2); } pub fn register_set(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("key_set", plugin_id, 4); } pub fn register_local_domain(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("is_local_domain", plugin_id, 2); } pub async fn exec(ctx: PluginContext<'_>) -> trc::Result { let store = match &ctx.arguments[0] { Variable::String(v) if !v.is_empty() => ctx.server.core.storage.lookups.get(v.as_ref()), _ => Some(&ctx.server.core.storage.lookup), } .ok_or_else(|| { trc::SieveEvent::RuntimeError .ctx(trc::Key::Id, ctx.arguments[0].to_string().into_owned()) .details("Unknown store") })?; Ok(match &ctx.arguments[1] { Variable::Array(items) => { for item in items.iter() { if !item.is_empty() && store.key_exists(item.to_string()).await? { return Ok(true.into()); } } false } v if !v.is_empty() => store.key_exists(v.to_string()).await?, _ => false, } .into()) } pub async fn exec_get(ctx: PluginContext<'_>) -> trc::Result { match &ctx.arguments[0] { Variable::String(v) if !v.is_empty() => ctx.server.core.storage.lookups.get(v.as_ref()), _ => Some(&ctx.server.core.storage.lookup), } .ok_or_else(|| { trc::SieveEvent::RuntimeError .ctx(trc::Key::Id, ctx.arguments[0].to_string().into_owned()) .details("Unknown store") })? .key_get::(ctx.arguments[1].to_string()) .await .map(|v| v.map(|v| v.into_inner()).unwrap_or_default()) } pub async fn exec_set(ctx: PluginContext<'_>) -> trc::Result { let expires = match &ctx.arguments[3] { Variable::Integer(v) => Some(*v as u64), Variable::Float(v) => Some(*v as u64), _ => None, }; match &ctx.arguments[0] { Variable::String(v) if !v.is_empty() => ctx.server.core.storage.lookups.get(v.as_ref()), _ => Some(&ctx.server.core.storage.lookup), } .ok_or_else(|| { trc::SieveEvent::RuntimeError .ctx(trc::Key::Id, ctx.arguments[0].to_string().into_owned()) .details("Unknown store") })? .key_set( KeyValue::new( ctx.arguments[1].to_string().into_owned().into_bytes(), if !ctx.arguments[2].is_empty() { bincode::serde::encode_to_vec(&ctx.arguments[2], bincode::config::standard()) .unwrap_or_default() } else { vec![] }, ) .expires_opt(expires), ) .await .map(|_| true.into()) } pub async fn exec_local_domain(ctx: PluginContext<'_>) -> trc::Result { let domain = ctx.arguments[1].to_string(); if !domain.is_empty() { return match &ctx.arguments[0] { Variable::String(v) if !v.is_empty() => { ctx.server.core.storage.directories.get(v.as_ref()) } _ => Some(&ctx.server.core.storage.directory), } .ok_or_else(|| { trc::SieveEvent::RuntimeError .ctx(trc::Key::Id, ctx.arguments[0].to_string().into_owned()) .details("Unknown directory") })? .is_local_domain(domain.as_ref()) .await .map(Into::into); } Ok(Variable::default()) } #[derive(Debug, PartialEq, Eq)] pub struct VariableWrapper(Variable); impl Deserialize for VariableWrapper { fn deserialize(bytes: &[u8]) -> trc::Result { Ok(VariableWrapper( bincode::serde::decode_from_slice::(bytes, bincode::config::standard()) .map(|v| v.0) .unwrap_or_else(|_| { Variable::String(String::from_utf8_lossy(bytes).into_owned().into()) }), )) } } impl From for VariableWrapper { fn from(value: i64) -> Self { VariableWrapper(value.into()) } } impl VariableWrapper { pub fn into_inner(self) -> Variable { self.0 } } impl From> for VariableWrapper { fn from(value: Value<'static>) -> Self { VariableWrapper(into_sieve_value(value)) } } ================================================ FILE: crates/common/src/scripts/plugins/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod dns; pub mod exec; pub mod headers; pub mod http; pub mod llm_prompt; pub mod lookup; pub mod query; pub mod text; use mail_parser::Message; use sieve::{FunctionMap, Input, runtime::Variable}; use crate::{Core, Server, auth::AccessToken}; use super::ScriptModification; type RegisterPluginFnc = fn(u32, &mut FunctionMap) -> (); pub struct PluginContext<'x> { pub session_id: u64, pub access_token: Option<&'x AccessToken>, pub server: &'x Server, pub message: &'x Message<'x>, pub modifications: &'x mut Vec, pub arguments: Vec, } const PLUGINS_REGISTER: [RegisterPluginFnc; 13] = [ query::register, exec::register, lookup::register, lookup::register_get, lookup::register_set, lookup::register_local_domain, dns::register, dns::register_exists, http::register_header, headers::register, text::register_tokenize, text::register_domain_part, llm_prompt::register, ]; pub trait RegisterSievePlugins { fn register_plugins_trusted(self) -> Self; fn register_plugins_untrusted(self) -> Self; } impl RegisterSievePlugins for FunctionMap { fn register_plugins_trusted(mut self) -> Self { #[cfg(feature = "test_mode")] { self.set_external_function("print", PLUGINS_REGISTER.len() as u32, 1) } for (i, fnc) in PLUGINS_REGISTER.iter().enumerate() { fnc(i as u32, &mut self); } self } fn register_plugins_untrusted(mut self) -> Self { llm_prompt::register(12, &mut self); self } } impl Core { pub async fn run_plugin(&self, id: u32, ctx: PluginContext<'_>) -> Input { #[cfg(feature = "test_mode")] if id == PLUGINS_REGISTER.len() as u32 { return test_print(ctx); } let session_id = ctx.session_id; let result = match id { 0 => query::exec(ctx).await, 1 => exec::exec(ctx).await, 2 => lookup::exec(ctx).await, 3 => lookup::exec_get(ctx).await, 4 => lookup::exec_set(ctx).await, 5 => lookup::exec_local_domain(ctx).await, 6 => dns::exec(ctx).await, 7 => dns::exec_exists(ctx).await, 8 => http::exec_header(ctx).await, 9 => headers::exec(ctx), 10 => text::exec_tokenize(ctx), 11 => text::exec_domain_part(ctx), 12 => llm_prompt::exec(ctx).await, _ => unreachable!(), }; match result { Ok(result) => result.into(), Err(err) => { trc::error!(err.span_id(session_id).details("Sieve runtime error")); Input::FncResult(Variable::default()) } } } } #[cfg(feature = "test_mode")] pub fn test_print(ctx: PluginContext<'_>) -> Input { println!("{}", ctx.arguments[0].to_string()); Input::True } ================================================ FILE: crates/common/src/scripts/plugins/query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::cmp::Ordering; use crate::scripts::{into_sieve_value, to_store_value}; use sieve::{FunctionMap, runtime::Variable}; use store::{Rows, Value}; use super::PluginContext; pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("query", plugin_id, 3); } pub async fn exec(ctx: PluginContext<'_>) -> trc::Result { // Obtain store name let store = match &ctx.arguments[0] { Variable::String(v) if !v.is_empty() => ctx.server.core.storage.stores.get(v.as_ref()), _ => Some(&ctx.server.core.storage.data), } .ok_or_else(|| { trc::SieveEvent::RuntimeError .ctx(trc::Key::Id, ctx.arguments[0].to_string().into_owned()) .details("Unknown store") })?; // Obtain query string let query = ctx.arguments[1].to_string(); if query.is_empty() { trc::bail!( trc::SieveEvent::RuntimeError .ctx(trc::Key::Id, ctx.arguments[0].to_string().into_owned()) .details("Empty query string") ); } // Obtain arguments let arguments = match &ctx.arguments[2] { Variable::Array(l) => l.iter().map(to_store_value).collect(), v => vec![to_store_value(v)], }; // Run query if query .as_bytes() .get(..6) .is_some_and(|q| q.eq_ignore_ascii_case(b"SELECT")) { let mut rows = store.sql_query::(&query, arguments).await?; Ok(match rows.rows.len().cmp(&1) { Ordering::Equal => { let mut row = rows.rows.pop().unwrap().values; match row.len().cmp(&1) { Ordering::Equal if !matches!(row.first(), Some(Value::Null)) => { row.pop().map(into_sieve_value).unwrap() } Ordering::Less => Variable::default(), _ => Variable::Array( row.into_iter() .map(into_sieve_value) .collect::>() .into(), ), } } Ordering::Less => Variable::default(), Ordering::Greater => rows .rows .into_iter() .map(|r| { Variable::Array( r.values .into_iter() .map(into_sieve_value) .collect::>() .into(), ) }) .collect::>() .into(), }) } else { Ok(store .sql_query::(&query, arguments) .await .is_ok() .into()) } } ================================================ FILE: crates/common/src/scripts/plugins/text.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use nlp::tokenizers::types::{TokenType, TypesTokenizer}; use sieve::{FunctionMap, runtime::Variable}; use crate::scripts::functions::{ApplyString, text::tokenize_words}; use super::PluginContext; pub fn register_tokenize(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("tokenize", plugin_id, 2); } pub fn register_domain_part(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("domain_part", plugin_id, 2); } pub fn exec_tokenize(ctx: PluginContext<'_>) -> trc::Result { let mut v = ctx.arguments; let (urls, urls_without_scheme, emails) = match v[1].to_string().as_ref() { "words" => return Ok(tokenize_words(&v[0])), "uri" | "url" => (true, true, true), "uri_strict" | "url_strict" => (true, false, false), "email" => (false, false, true), _ => return Ok(Variable::default()), }; Ok(match v.remove(0) { v @ (Variable::String(_) | Variable::Array(_)) => { TypesTokenizer::new(v.to_string().as_ref()) .tokenize_numbers(false) .tokenize_urls(urls) .tokenize_urls_without_scheme(urls_without_scheme) .tokenize_emails(emails) .filter_map(|t| match t.word { TokenType::Url(text) if urls => Variable::from(text.to_string()).into(), TokenType::UrlNoScheme(text) if urls_without_scheme => { Variable::from(format!("https://{text}")).into() } TokenType::Email(text) if emails => Variable::from(text.to_string()).into(), _ => None, }) .collect::>() .into() } v => v, }) } enum DomainPart { Sld, Tld, Host, } pub fn exec_domain_part(ctx: PluginContext<'_>) -> trc::Result { let v = ctx.arguments; let part = match v[1].to_string().as_ref() { "sld" => DomainPart::Sld, "tld" => DomainPart::Tld, "host" => DomainPart::Host, _ => return Ok(Variable::default()), }; Ok(v[0].transform(|domain| { match part { DomainPart::Sld => psl::domain_str(domain), DomainPart::Tld => domain.rsplit_once('.').map(|(_, tld)| tld), DomainPart::Host => domain.split_once('.').map(|(host, _)| host), } .map(Variable::from) .unwrap_or_default() })) } ================================================ FILE: crates/common/src/sharing/acl.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::Server; use directory::{ Type, backend::internal::{PrincipalField, manage::ChangedPrincipals}, }; use types::acl::{AclGrant, ArchivedAclGrant}; impl Server { pub async fn refresh_acls(&self, acl_changes: &[AclGrant], current: Option<&[AclGrant]>) { let mut changed_principals = ChangedPrincipals::new(); if let Some(acl_current) = current { for current_item in acl_current { let mut invalidate = true; for change_item in acl_changes { if change_item.account_id == current_item.account_id { invalidate = change_item.grants != current_item.grants; break; } } if invalidate { changed_principals.add_change( current_item.account_id, Type::Individual, PrincipalField::EnabledPermissions, ); } } for change_item in acl_changes { let mut invalidate = true; for current_item in acl_current { if change_item.account_id == current_item.account_id { invalidate = change_item.grants != current_item.grants; break; } } if invalidate { changed_principals.add_change( change_item.account_id, Type::Individual, PrincipalField::EnabledPermissions, ); } } } else { for value in acl_changes { changed_principals.add_change( value.account_id, Type::Individual, PrincipalField::EnabledPermissions, ); } } self.invalidate_principal_caches(changed_principals).await; } pub async fn refresh_archived_acls( &self, acl_changes: &[AclGrant], acl_current: &[ArchivedAclGrant], ) { let mut changed_principals = ChangedPrincipals::new(); for current_item in acl_current.iter() { let mut invalidate = true; for change_item in acl_changes { if change_item.account_id == current_item.account_id { invalidate = change_item.grants != current_item.grants; break; } } if invalidate { changed_principals.add_change( current_item.account_id.to_native(), Type::Individual, PrincipalField::EnabledPermissions, ); } } for change_item in acl_changes { let mut invalidate = true; for current_item in acl_current.iter() { if change_item.account_id == current_item.account_id { invalidate = change_item.grants != current_item.grants; break; } } if invalidate { changed_principals.add_change( change_item.account_id, Type::Individual, PrincipalField::EnabledPermissions, ); } } self.invalidate_principal_caches(changed_principals).await; } } ================================================ FILE: crates/common/src/sharing/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::auth::AccessToken; use rkyv::vec::ArchivedVec; use types::acl::{Acl, AclGrant, ArchivedAclGrant}; use utils::map::bitmap::Bitmap; pub mod acl; pub mod notification; pub mod resources; pub trait EffectiveAcl { fn effective_acl(&self, access_token: &AccessToken) -> Bitmap; } impl EffectiveAcl for Vec { fn effective_acl(&self, access_token: &AccessToken) -> Bitmap { self.as_slice().effective_acl(access_token) } } impl EffectiveAcl for &[AclGrant] { fn effective_acl(&self, access_token: &AccessToken) -> Bitmap { let mut acl = Bitmap::::new(); for item in self.iter() { if access_token.is_member(item.account_id) { acl.union(&item.grants); } } acl } } impl EffectiveAcl for ArchivedVec { fn effective_acl(&self, access_token: &AccessToken) -> Bitmap { let mut acl = Bitmap::::new(); for item in self.iter() { if access_token.is_member(item.account_id.into()) { acl.union_raw(item.grants.bitmap); } } acl } } ================================================ FILE: crates/common/src/sharing/notification.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use store::{Deserialize, SerializeInfallible, U32_LEN, U64_LEN, write::key::KeySerializer}; use types::{acl::Acl, collection::Collection}; use utils::map::bitmap::Bitmap; #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct ShareNotification { pub object_account_id: u32, pub object_id: u32, pub object_type: Collection, pub changed_by: u32, pub old_rights: Bitmap, pub new_rights: Bitmap, pub name: String, } impl SerializeInfallible for ShareNotification { fn serialize(&self) -> Vec { KeySerializer::new(U64_LEN * 2 + U32_LEN * 3 + 1 + self.name.len()) .write(self.object_account_id) .write(self.object_id) .write(self.object_type as u8) .write(self.changed_by) .write(self.old_rights.bitmap) .write(self.new_rights.bitmap) .write(self.name.as_bytes()) .finalize() } } impl Deserialize for ShareNotification { fn deserialize(bytes: &[u8]) -> trc::Result { Self::deserialize_from_slice(bytes) .ok_or(trc::StoreEvent::DataCorruption.caused_by(trc::location!())) } } impl ShareNotification { fn deserialize_from_slice(bytes: &[u8]) -> Option { Some(Self { object_account_id: bytes .get(..U32_LEN) .and_then(|b| b.try_into().ok()) .map(u32::from_be_bytes)?, object_id: bytes .get(U32_LEN..U32_LEN * 2) .and_then(|b| b.try_into().ok()) .map(u32::from_be_bytes)?, object_type: bytes.get(U32_LEN * 2).copied().map(Collection::from)?, changed_by: bytes .get(U32_LEN * 2 + 1..U32_LEN * 3 + 1) .and_then(|b| b.try_into().ok()) .map(u32::from_be_bytes)?, old_rights: bytes .get(U32_LEN * 3 + 1..U32_LEN * 3 + U64_LEN + 1) .and_then(|b| b.try_into().ok()) .map(u64::from_be_bytes) .map(Bitmap::from)?, new_rights: bytes .get(U32_LEN * 3 + U64_LEN + 1..U32_LEN * 3 + U64_LEN * 2 + 1) .and_then(|b| b.try_into().ok()) .map(u64::from_be_bytes) .map(Bitmap::from)?, name: bytes .get(U32_LEN * 3 + U64_LEN * 2 + 1..) .and_then(|b| String::from_utf8(b.to_vec()).ok()) .unwrap_or_default(), }) } } ================================================ FILE: crates/common/src/sharing/resources.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{DavResources, auth::AccessToken}; use store::roaring::RoaringBitmap; use types::acl::Acl; use utils::map::bitmap::Bitmap; impl DavResources { pub fn shared_containers( &self, access_token: &AccessToken, check_acls: impl IntoIterator, match_any: bool, ) -> RoaringBitmap { let check_acls = Bitmap::::from_iter(check_acls); let mut document_ids = RoaringBitmap::new(); for resource in &self.resources { if let Some(acls) = resource.acls() { for acl in acls { if access_token.is_member(acl.account_id) { let mut grants = acl.grants; grants.intersection(&check_acls); if grants == check_acls || (match_any && !grants.is_empty()) { document_ids.insert(resource.document_id); } } } } } document_ids } pub fn shared_items( &self, access_token: &AccessToken, check_acls: impl IntoIterator, match_any: bool, ) -> RoaringBitmap { let shared_containers = self.shared_containers(access_token, check_acls, match_any); if !shared_containers.is_empty() { let mut document_ids = RoaringBitmap::new(); for path in &self.paths { if let Some(parent_id) = path.parent_id && shared_containers.contains(parent_id) { document_ids.insert(self.resources[path.resource_idx].document_id); } } document_ids } else { shared_containers } } pub fn has_access_to_container( &self, access_token: &AccessToken, document_id: u32, check_acls: impl Into>, ) -> bool { let check_acls = check_acls.into(); for resource in &self.resources { if resource.document_id == document_id && let Some(acls) = resource.acls() { for acl in acls { if access_token.is_member(acl.account_id) { let mut grants = acl.grants; grants.intersection(&check_acls); return !grants.is_empty(); } } break; } } false } pub fn container_acl(&self, access_token: &AccessToken, document_id: u32) -> Bitmap { let mut account_acls = Bitmap::::new(); for resource in &self.resources { if resource.document_id == document_id && let Some(acls) = resource.acls() { for acl in acls { if access_token.is_member(acl.account_id) { account_acls.union(&acl.grants); } } break; } } account_acls } pub fn document_ids(&self, is_container: bool) -> impl Iterator { self.resources.iter().filter_map(move |resource| { if resource.is_container() == is_container { Some(resource.document_id) } else { None } }) } pub fn has_container_id(&self, id: &u32) -> bool { self.resources .iter() .any(|r| r.document_id == *id && r.is_container()) } pub fn has_item_id(&self, id: &u32) -> bool { self.resources .iter() .any(|r| r.document_id == *id && !r.is_container()) } } ================================================ FILE: crates/common/src/storage/blob.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::Server; use mail_parser::{ Encoding, decoders::{base64::base64_decode, quoted_printable::quoted_printable_decode}, }; use types::{blob::BlobSection, blob_hash::BlobHash}; impl Server { pub async fn get_blob_section( &self, hash: &BlobHash, section: &BlobSection, ) -> trc::Result>> { Ok(self .blob_store() .get_blob( hash.as_slice(), (section.offset_start)..(section.offset_start.saturating_add(section.size)), ) .await? .and_then(|bytes| match Encoding::from(section.encoding) { Encoding::None => Some(bytes), Encoding::Base64 => base64_decode(&bytes), Encoding::QuotedPrintable => quoted_printable_decode(&bytes), })) } } ================================================ FILE: crates/common/src/storage/index.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{auth::AccessToken, sharing::notification::ShareNotification}; use rkyv::{ option::ArchivedOption, primitive::{ArchivedU32, ArchivedU64}, string::ArchivedString, }; use std::{borrow::Cow, fmt::Debug}; use store::{ Serialize, SerializeInfallible, write::{ Archive, Archiver, BatchBuilder, BlobLink, BlobOp, DirectoryClass, IntoOperations, Params, SearchIndex, TaskEpoch, TaskQueueClass, ValueClass, }, }; use types::{ acl::AclGrant, blob_hash::BlobHash, collection::{Collection, SyncCollection}, field::Field, }; use utils::{cheeky_hash::CheekyHash, map::bitmap::Bitmap, snowflake::SnowflakeIdGenerator}; #[derive(Debug, Clone, PartialEq, Eq)] pub enum IndexValue<'x> { Index { field: Field, value: IndexItem<'x>, }, Property { field: ValueClass, value: IndexItem<'x>, }, SearchIndex { index: SearchIndex, hash: u64, }, Blob { value: BlobHash, }, Quota { used: u32, }, LogContainer { sync_collection: SyncCollection, }, LogContainerProperty { sync_collection: SyncCollection, ids: Vec, }, LogItem { sync_collection: SyncCollection, prefix: Option, }, Acl { value: Cow<'x, [AclGrant]>, }, } #[derive(Debug, Clone)] pub enum IndexItem<'x> { Vec(Vec), Slice(&'x [u8]), ShortInt([u8; std::mem::size_of::()]), LongInt([u8; std::mem::size_of::()]), Hash(CheekyHash), None, } impl IndexItem<'_> { pub fn as_slice(&self) -> &[u8] { match self { IndexItem::Vec(v) => v, IndexItem::Slice(s) => s, IndexItem::ShortInt(s) => s, IndexItem::LongInt(s) => s, IndexItem::Hash(h) => h.as_bytes(), IndexItem::None => &[], } } pub fn into_owned(self) -> Vec { match self { IndexItem::Vec(v) => v, IndexItem::Slice(s) => s.to_vec(), IndexItem::ShortInt(s) => s.to_vec(), IndexItem::LongInt(s) => s.to_vec(), IndexItem::Hash(h) => h.as_bytes().to_vec(), IndexItem::None => vec![], } } pub fn is_empty(&self) -> bool { match self { IndexItem::Vec(v) => v.is_empty(), IndexItem::Slice(s) => s.is_empty(), IndexItem::None => true, _ => false, } } pub fn is_none(&self) -> bool { matches!(self, IndexItem::None) } pub fn is_some(&self) -> bool { !self.is_none() } } impl PartialEq for IndexItem<'_> { fn eq(&self, other: &Self) -> bool { self.as_slice() == other.as_slice() } } impl Eq for IndexItem<'_> {} impl std::hash::Hash for IndexItem<'_> { fn hash(&self, state: &mut H) { match self { IndexItem::Vec(v) => v.as_slice().hash(state), IndexItem::Slice(s) => s.hash(state), IndexItem::ShortInt(s) => s.as_slice().hash(state), IndexItem::LongInt(s) => s.as_slice().hash(state), IndexItem::Hash(h) => h.hash(state), IndexItem::None => 0.hash(state), } } } impl From for IndexItem<'_> { fn from(value: u32) -> Self { IndexItem::ShortInt(value.to_be_bytes()) } } impl From<&u32> for IndexItem<'_> { fn from(value: &u32) -> Self { IndexItem::ShortInt(value.to_be_bytes()) } } impl From for IndexItem<'_> { fn from(value: u64) -> Self { IndexItem::LongInt(value.to_be_bytes()) } } impl From for IndexItem<'_> { fn from(value: i64) -> Self { IndexItem::LongInt(value.to_be_bytes()) } } impl<'x> From<&'x [u8]> for IndexItem<'x> { fn from(value: &'x [u8]) -> Self { IndexItem::Slice(value) } } impl From> for IndexItem<'_> { fn from(value: Vec) -> Self { IndexItem::Vec(value) } } impl<'x> From<&'x str> for IndexItem<'x> { fn from(value: &'x str) -> Self { IndexItem::Slice(value.as_bytes()) } } impl<'x> From<&'x String> for IndexItem<'x> { fn from(value: &'x String) -> Self { IndexItem::Slice(value.as_bytes()) } } impl From for IndexItem<'_> { fn from(value: String) -> Self { IndexItem::Vec(value.into_bytes()) } } impl<'x> From<&'x ArchivedString> for IndexItem<'x> { fn from(value: &'x ArchivedString) -> Self { IndexItem::Slice(value.as_bytes()) } } impl From for IndexItem<'_> { fn from(value: ArchivedU32) -> Self { IndexItem::ShortInt(value.to_native().to_be_bytes()) } } impl From<&ArchivedU32> for IndexItem<'_> { fn from(value: &ArchivedU32) -> Self { IndexItem::ShortInt(value.to_native().to_be_bytes()) } } impl From for IndexItem<'_> { fn from(value: ArchivedU64) -> Self { IndexItem::LongInt(value.to_native().to_be_bytes()) } } impl<'x, T: Into>> From> for IndexItem<'x> { fn from(value: Option) -> Self { match value { Some(v) => v.into(), None => IndexItem::None, } } } impl<'x, T: Into>> From> for IndexItem<'x> { fn from(value: ArchivedOption) -> Self { match value { ArchivedOption::Some(v) => v.into(), ArchivedOption::None => IndexItem::None, } } } pub trait IndexableObject: Sync + Send { fn index_values(&self) -> impl Iterator>; } pub trait IndexableAndSerializableObject: IndexableObject + rkyv::Archive + for<'a> rkyv::Serialize< rkyv::api::high::HighSerializer< rkyv::util::AlignedVec, rkyv::ser::allocator::ArenaHandle<'a>, rkyv::rancor::Error, >, > { fn is_versioned() -> bool; } #[derive(Debug)] pub struct ObjectIndexBuilder { changed_by: u32, tenant_id: Option, current: Option>, changes: Option, } impl Default for ObjectIndexBuilder { fn default() -> Self { Self::new() } } impl ObjectIndexBuilder { pub fn new() -> Self { Self { current: None, changes: None, tenant_id: None, changed_by: u32::MAX, } } pub fn with_current(mut self, current: Archive) -> Self { self.current = Some(current); self } pub fn with_changes(mut self, changes: N) -> Self { self.changes = Some(changes); self } pub fn with_current_opt(mut self, current: Option>) -> Self { self.current = current; self } pub fn changes(&self) -> Option<&N> { self.changes.as_ref() } pub fn changes_mut(&mut self) -> Option<&mut N> { self.changes.as_mut() } pub fn current(&self) -> Option<&Archive> { self.current.as_ref() } pub fn with_access_token(mut self, access_token: &AccessToken) -> Self { self.tenant_id = access_token.tenant.as_ref().map(|t| t.id); self.changed_by = access_token.primary_id(); self } pub fn with_tenant_id(mut self, tenant_id: Option) -> Self { self.tenant_id = tenant_id; self } } impl IntoOperations for ObjectIndexBuilder { fn build(self, batch: &mut BatchBuilder) -> trc::Result<()> { match (self.current, self.changes) { (None, Some(changes)) => { // Insertion for item in changes.index_values() { build_index(batch, item, self.changed_by, self.tenant_id, true); } if N::is_versioned() { let (offset, bytes) = Archiver::new(changes).serialize_versioned()?; batch.set_fnc( Field::ARCHIVE, Params::with_capacity(2).with_bytes(bytes).with_u64(offset), |params, ids| { let change_id = ids.current_change_id()?; let archive = params.bytes(0); let offset = params.u64(1); let mut bytes = Vec::with_capacity(archive.len()); bytes.extend_from_slice(&archive[..offset as usize]); bytes.extend_from_slice(&change_id.to_be_bytes()[..]); bytes.push(archive.last().copied().unwrap()); // Marker Ok(bytes) }, ); } else { batch.set(Field::ARCHIVE, Archiver::new(changes).serialize()?); } } (Some(current), Some(changes)) => { // Update batch.assert_value(Field::ARCHIVE, ¤t); for (current, change) in current.inner.index_values().zip(changes.index_values()) { if current != change { merge_index(batch, current, change, self.changed_by, self.tenant_id)?; } else { match current { IndexValue::LogContainer { sync_collection } => { batch.log_container_update(sync_collection); } IndexValue::LogItem { sync_collection, prefix, } => { batch.log_item_update(sync_collection, prefix); } _ => (), } } } if N::is_versioned() { let (offset, bytes) = Archiver::new(changes).serialize_versioned()?; batch.set_fnc( Field::ARCHIVE, Params::with_capacity(2).with_bytes(bytes).with_u64(offset), |params, ids| { let change_id = ids.current_change_id()?; let archive = params.bytes(0); let offset = params.u64(1); let mut bytes = Vec::with_capacity(archive.len()); bytes.extend_from_slice(&archive[..offset as usize]); bytes.extend_from_slice(&change_id.to_be_bytes()[..]); bytes.push(archive.last().copied().unwrap()); // Marker Ok(bytes) }, ); } else { batch.set(Field::ARCHIVE, Archiver::new(changes).serialize()?); } } (Some(current), None) => { // Deletion batch.assert_value(Field::ARCHIVE, ¤t); for item in current.inner.index_values() { build_index(batch, item, self.changed_by, self.tenant_id, false); } batch.clear(Field::ARCHIVE); } (None, None) => unreachable!(), } Ok(()) } } fn build_index( batch: &mut BatchBuilder, item: IndexValue<'_>, changed_by: u32, tenant_id: Option, set: bool, ) { match item { IndexValue::Index { field, value } => { if !value.is_empty() { if set { batch.index(field, value.into_owned()); } else { batch.unindex(field, value.into_owned()); } } } IndexValue::SearchIndex { index, .. } => { batch.set( ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { due: TaskEpoch::now().with_random_sequence_id(), index, is_insert: set, }), vec![], ); } IndexValue::Property { field, value } => { if !value.is_none() { if set { batch.set(field, value.into_owned()); } else { batch.clear(field); } } } IndexValue::Blob { value } => { if set { batch.set( BlobOp::Link { hash: value, to: BlobLink::Document, }, vec![], ); } else { batch.clear(BlobOp::Link { hash: value, to: BlobLink::Document, }); } } IndexValue::Acl { value } => { let object_account_id = batch.last_account_id().unwrap_or_default(); let object_type = batch.last_collection().unwrap_or(Collection::None); let object_id = batch.last_document_id().unwrap_or_default(); let notification_id = SnowflakeIdGenerator::from_sequence_and_node_id( object_type as u64 ^ object_account_id as u64, None, ) .unwrap_or_default(); for item in value.as_ref() { if set { batch.acl_grant(item.account_id, item.grants.bitmap.serialize()); batch.log_share_notification( notification_id, item.account_id, ShareNotification { object_account_id, object_id, object_type, changed_by, old_rights: Default::default(), new_rights: item.grants, name: Default::default(), }, ); } else { batch.acl_revoke(item.account_id); batch.log_share_notification( notification_id, item.account_id, ShareNotification { object_account_id, object_id, object_type, changed_by, old_rights: item.grants, new_rights: Default::default(), name: Default::default(), }, ); } } } IndexValue::Quota { used } => { let value = if set { used as i64 } else { -(used as i64) }; if let Some(account_id) = batch.last_account_id() { batch.add(DirectoryClass::UsedQuota(account_id), value); } if let Some(tenant_id) = tenant_id { batch.add(DirectoryClass::UsedQuota(tenant_id), value); } } IndexValue::LogItem { sync_collection, prefix, } => { if set { batch.log_item_insert(sync_collection, prefix); } else { batch.log_item_delete(sync_collection, prefix); } } IndexValue::LogContainer { sync_collection } => { if set { batch.log_container_insert(sync_collection); } else { batch.log_container_delete(sync_collection); } } IndexValue::LogContainerProperty { sync_collection, ids, } => { for parent_id in ids { batch.log_container_property_change(sync_collection, parent_id); } } } } fn merge_index( batch: &mut BatchBuilder, current: IndexValue<'_>, change: IndexValue<'_>, changed_by: u32, tenant_id: Option, ) -> trc::Result<()> { match (current, change) { ( IndexValue::Index { field, value: old_value, }, IndexValue::Index { value: new_value, .. }, ) => { if !old_value.is_empty() { batch.unindex(field, old_value.into_owned()); } if !new_value.is_empty() { batch.index(field, new_value.into_owned()); } } (IndexValue::SearchIndex { index, .. }, IndexValue::SearchIndex { .. }) => { batch.set( ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { due: TaskEpoch::now().with_random_sequence_id(), index, is_insert: true, }), vec![], ); } ( IndexValue::Property { field: old_field, value: old_value, }, IndexValue::Property { field: new_field, value: new_value, .. }, ) => { if old_field != new_field { batch.clear(old_field); batch.set(new_field, new_value.into_owned()); } else if new_value != old_value { if new_value.is_some() { batch.set(old_field, new_value.into_owned()); } else { batch.clear(old_field); } } } (IndexValue::Blob { value: old_hash }, IndexValue::Blob { value: new_hash }) => { batch.clear(BlobOp::Link { hash: old_hash, to: BlobLink::Document, }); batch.set( BlobOp::Link { hash: new_hash, to: BlobLink::Document, }, vec![], ); } (IndexValue::Acl { value: old_acl }, IndexValue::Acl { value: new_acl }) => { let has_old_acl = !old_acl.is_empty(); let has_new_acl = !new_acl.is_empty(); if !has_old_acl && !has_new_acl { return Ok(()); } let object_account_id = batch.last_account_id().unwrap_or_default(); let object_type = batch.last_collection().unwrap_or(Collection::None); let object_id = batch.last_document_id().unwrap_or_default(); let notification_id = SnowflakeIdGenerator::from_sequence_and_node_id( object_type as u64 ^ object_account_id as u64, None, ) .unwrap_or_default(); match (has_old_acl, has_new_acl) { (true, true) => { // Remove deleted ACLs for current_item in old_acl.as_ref() { if !new_acl .iter() .any(|item| item.account_id == current_item.account_id) { batch.acl_revoke(current_item.account_id); batch.log_share_notification( notification_id, current_item.account_id, ShareNotification { object_account_id, object_id, object_type, changed_by, old_rights: current_item.grants, new_rights: Default::default(), name: Default::default(), }, ); } } // Update ACLs for item in new_acl.as_ref() { let mut add_item = true; let mut old_rights = Bitmap::default(); for current_item in old_acl.as_ref() { if item.account_id == current_item.account_id { if item.grants == current_item.grants { add_item = false; } else { old_rights = current_item.grants; } break; } } if add_item { batch.acl_grant(item.account_id, item.grants.bitmap.serialize()); batch.log_share_notification( notification_id, item.account_id, ShareNotification { object_account_id, object_id, object_type, changed_by, old_rights, new_rights: item.grants, name: Default::default(), }, ); } } } (false, true) => { // Add all ACLs for item in new_acl.as_ref() { batch.acl_grant(item.account_id, item.grants.bitmap.serialize()); batch.log_share_notification( notification_id, item.account_id, ShareNotification { object_account_id, object_id, object_type, changed_by, old_rights: Default::default(), new_rights: item.grants, name: Default::default(), }, ); } } (true, false) => { // Remove all ACLs for item in old_acl.as_ref() { batch.acl_revoke(item.account_id); batch.log_share_notification( notification_id, item.account_id, ShareNotification { object_account_id, object_id, object_type, changed_by, old_rights: item.grants, new_rights: Default::default(), name: Default::default(), }, ); } } _ => {} } } (IndexValue::Quota { used: old_used }, IndexValue::Quota { used: new_used }) => { let value = new_used as i64 - old_used as i64; if let Some(account_id) = batch.last_account_id() { batch.add(DirectoryClass::UsedQuota(account_id), value); } if let Some(tenant_id) = tenant_id { batch.add(DirectoryClass::UsedQuota(tenant_id), value); } } ( IndexValue::LogItem { sync_collection, prefix: old_prefix, }, IndexValue::LogItem { prefix: new_prefix, .. }, ) => { batch.log_item_delete(sync_collection, old_prefix); batch.log_item_insert(sync_collection, new_prefix); } ( IndexValue::LogContainerProperty { sync_collection, ids: old_ids, }, IndexValue::LogContainerProperty { ids: new_ids, .. }, ) => { for parent_id in &old_ids { if !new_ids.contains(parent_id) { batch.log_container_property_change(sync_collection, *parent_id); } } for parent_id in new_ids { if !old_ids.contains(&parent_id) { batch.log_container_property_change(sync_collection, parent_id); } } } _ => unreachable!(), } Ok(()) } impl IndexableObject for () { fn index_values(&self) -> impl Iterator> { std::iter::empty() } } impl IndexableAndSerializableObject for () { fn is_versioned() -> bool { false } } ================================================ FILE: crates/common/src/storage/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod blob; pub mod index; pub mod state; ================================================ FILE: crates/common/src/storage/state.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ IPC_CHANNEL_BUFFER, Server, auth::AccessToken, ipc::{PushEvent, PushNotification}, }; use tokio::sync::mpsc; use types::type_state::DataType; use utils::map::bitmap::Bitmap; impl Server { pub async fn subscribe_push_manager( &self, access_token: &AccessToken, types: Bitmap, ) -> trc::Result> { let (tx, rx) = mpsc::channel::(IPC_CHANNEL_BUFFER); let push_tx = self.inner.ipc.push_tx.clone(); push_tx .send(PushEvent::Subscribe { account_ids: access_token.member_ids().collect(), types, tx, }) .await .map_err(|err| { trc::EventType::Server(trc::ServerEvent::ThreadError) .reason(err) .caused_by(trc::location!()) })?; Ok(rx) } } ================================================ FILE: crates/common/src/telemetry/metrics/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod otel; pub mod prometheus; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] pub mod store; // SPDX-SnippetEnd ================================================ FILE: crates/common/src/telemetry/metrics/otel.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::config::telemetry::OtelMetrics; use opentelemetry_sdk::metrics::{ Temporality, data::{ Gauge, GaugeDataPoint, Histogram, HistogramDataPoint, Metric, ResourceMetrics, ScopeMetrics, Sum, SumDataPoint, }, exporter::PushMetricExporter, }; use std::time::SystemTime; use trc::{Collector, TelemetryEvent}; impl OtelMetrics { pub async fn push_metrics(&self, is_enterprise: bool, start_time: SystemTime) { let mut metrics = Vec::with_capacity(256); let time = SystemTime::now(); // Add counters for counter in Collector::collect_counters(is_enterprise) { metrics.push(Metric { name: counter.id().name().into(), description: counter.id().description().into(), unit: "events".into(), data: Box::new(Sum { data_points: vec![SumDataPoint { attributes: vec![], value: counter.value(), exemplars: vec![], }], temporality: Temporality::Cumulative, is_monotonic: true, start_time, time, }), }); } // Add gauges for gauge in Collector::collect_gauges(is_enterprise) { metrics.push(Metric { name: gauge.id().name().into(), description: gauge.id().description().into(), unit: gauge.id().unit().into(), data: Box::new(Gauge { data_points: vec![GaugeDataPoint { attributes: vec![], value: gauge.get(), exemplars: vec![], }], start_time: start_time.into(), time, }), }); } // Add histograms for histogram in Collector::collect_histograms(is_enterprise) { metrics.push(Metric { name: histogram.id().name().into(), description: histogram.id().description().into(), unit: histogram.id().unit().into(), data: Box::new(Histogram { data_points: vec![HistogramDataPoint { attributes: vec![], count: histogram.count(), bounds: histogram.upper_bounds_vec(), bucket_counts: histogram.buckets_vec(), min: histogram.min(), max: histogram.max(), sum: histogram.sum(), exemplars: vec![], }], temporality: Temporality::Cumulative, start_time, time, }), }); } // Export metrics if let Err(err) = self .exporter .export(&mut ResourceMetrics { resource: self.resource.clone(), scope_metrics: vec![ScopeMetrics { scope: self.instrumentation.clone(), metrics, }], }) .await { trc::event!( Telemetry(TelemetryEvent::OtelMetricsExporterError), Reason = err.to_string(), ); } } pub fn enable_errors() { // TODO: Remove this when the OpenTelemetry SDK supports error handling /*let _ = set_error_handler(|error| { trc::event!( Telemetry(TelemetryEvent::OtelMetricsExporterError), Reason = error.to_string(), ); });*/ } } ================================================ FILE: crates/common/src/telemetry/metrics/prometheus.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use prometheus::{ TextEncoder, proto::{Bucket, Counter, Gauge, Histogram, Metric, MetricFamily, MetricType}, }; use trc::{Collector, atomics::histogram::AtomicHistogram}; use crate::Server; impl Server { pub async fn export_prometheus_metrics(&self) -> trc::Result { let mut metrics = Vec::new(); // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] let is_enterprise = self.is_enterprise_edition(); // SPDX-SnippetEnd #[cfg(not(feature = "enterprise"))] let is_enterprise = false; // Add counters for counter in Collector::collect_counters(is_enterprise) { let mut metric = MetricFamily::default(); metric.set_name(metric_name(counter.id().name())); metric.set_help(counter.id().description().into()); metric.set_field_type(MetricType::COUNTER); metric.set_metric(vec![new_counter(counter.value())]); metrics.push(metric); } // Add gauges for gauge in Collector::collect_gauges(is_enterprise) { let mut metric = MetricFamily::default(); metric.set_name(metric_name(gauge.id().name())); metric.set_help(gauge.id().description().into()); metric.set_field_type(MetricType::GAUGE); metric.set_metric(vec![new_gauge(gauge.get())]); metrics.push(metric); } // Add histograms for histogram in Collector::collect_histograms(is_enterprise) { let mut metric = MetricFamily::default(); metric.set_name(metric_name(histogram.id().name())); metric.set_help(histogram.id().description().into()); metric.set_field_type(MetricType::HISTOGRAM); metric.set_metric(vec![new_histogram(histogram)]); metrics.push(metric); } TextEncoder::new().encode_to_string(&metrics).map_err(|e| { trc::EventType::Telemetry(trc::TelemetryEvent::OtelExporterError).reason(e) }) } } fn metric_name(id: impl AsRef) -> String { let id = id.as_ref(); let mut name = String::with_capacity(id.len()); for c in id.chars() { if c.is_ascii_alphanumeric() { name.push(c); } else { name.push('_'); } } name } fn new_counter(value: u64) -> Metric { let mut m = Metric::default(); let mut counter = Counter::default(); counter.set_value(value as f64); m.set_counter(counter); m } fn new_gauge(value: u64) -> Metric { let mut m = Metric::default(); let mut gauge = Gauge::default(); gauge.set_value(value as f64); m.set_gauge(gauge); m } fn new_histogram(histogram: &AtomicHistogram<12>) -> Metric { let mut m = Metric::default(); let mut h = Histogram::default(); h.set_sample_count(histogram.count()); h.set_sample_sum(histogram.sum() as f64); h.set_bucket( histogram .buckets_iter() .into_iter() .zip(histogram.upper_bounds_iter()) .map(|(count, upper_bound)| { let mut b = Bucket::default(); b.set_cumulative_count(count); b.set_upper_bound(if upper_bound != u64::MAX { upper_bound as f64 } else { f64::INFINITY }); b }) .collect(), ); m.set_histogram(h); m } ================================================ FILE: crates/common/src/telemetry/metrics/store.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: LicenseRef-SEL * * This file is subject to the Stalwart Enterprise License Agreement (SEL) and * is NOT open source software. * */ use std::{future::Future, sync::Arc, time::Duration}; use ahash::AHashMap; use parking_lot::Mutex; use serde::{Deserialize, Serialize}; use store::{ IterateParams, Store, U32_LEN, U64_LEN, ValueKey, write::{ BatchBuilder, TelemetryClass, ValueClass, key::{DeserializeBigEndian, KeySerializer}, now, }, }; use trc::*; use utils::codec::leb128::Leb128Reader; use crate::Core; pub trait MetricsStore: Sync + Send { fn write_metrics( &self, core: Arc, timestamp: u64, history: SharedMetricHistory, ) -> impl Future> + Send; fn query_metrics( &self, from_timestamp: u64, to_timestamp: u64, ) -> impl Future>>> + Send; fn purge_metrics(&self, period: Duration) -> impl Future> + Send; } #[derive(Default)] pub struct MetricsHistory { events: AHashMap, histograms: AHashMap, } #[derive(Default)] struct HistogramHistory { sum: u64, count: u64, } #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(tag = "type")] #[serde(rename_all = "camelCase")] pub enum Metric { Counter { id: CI, timestamp: T, value: u64, }, Gauge { id: MI, timestamp: T, value: u64, }, Histogram { id: MI, timestamp: T, count: u64, sum: u64, }, } pub type SharedMetricHistory = Arc>; const TYPE_COUNTER: u64 = 0x00; const TYPE_HISTOGRAM: u64 = 0x01; const TYPE_GAUGE: u64 = 0x02; impl MetricsStore for Store { async fn write_metrics( &self, core: Arc, timestamp: u64, history_: SharedMetricHistory, ) -> trc::Result<()> { let mut batch = BatchBuilder::new(); { let node_id = core.network.node_id; let mut history = history_.lock(); for event in [ EventType::Smtp(SmtpEvent::ConnectionStart), EventType::Imap(ImapEvent::ConnectionStart), EventType::Pop3(Pop3Event::ConnectionStart), EventType::ManageSieve(ManageSieveEvent::ConnectionStart), EventType::Http(HttpEvent::ConnectionStart), EventType::Delivery(DeliveryEvent::AttemptStart), EventType::Queue(QueueEvent::QueueMessage), EventType::Queue(QueueEvent::QueueMessageAuthenticated), EventType::Queue(QueueEvent::QueueDsn), EventType::Queue(QueueEvent::QueueReport), EventType::MessageIngest(MessageIngestEvent::Ham), EventType::MessageIngest(MessageIngestEvent::Spam), EventType::Auth(AuthEvent::Failed), EventType::Security(SecurityEvent::AuthenticationBan), EventType::Security(SecurityEvent::ScanBan), EventType::Security(SecurityEvent::AbuseBan), EventType::Security(SecurityEvent::LoiterBan), EventType::Security(SecurityEvent::IpBlocked), EventType::IncomingReport(IncomingReportEvent::DmarcReport), EventType::IncomingReport(IncomingReportEvent::DmarcReportWithWarnings), EventType::IncomingReport(IncomingReportEvent::TlsReport), EventType::IncomingReport(IncomingReportEvent::TlsReportWithWarnings), ] { let reading = Collector::read_event_metric(event.id()); if reading > 0 { let history = history.events.entry(event).or_insert(0); let diff = reading - *history; if diff > 0 { batch.set( ValueClass::Telemetry(TelemetryClass::Metric { timestamp, metric_id: (event.code() << 2) | TYPE_COUNTER, node_id, }), KeySerializer::new(U32_LEN).write_leb128(diff).finalize(), ); } *history = reading; } } for gauge in Collector::collect_gauges(true) { let gauge_id = gauge.id(); if matches!(gauge_id, MetricType::QueueCount | MetricType::ServerMemory) { let value = gauge.get(); if value > 0 { batch.set( ValueClass::Telemetry(TelemetryClass::Metric { timestamp, metric_id: (gauge_id.code() << 2) | TYPE_GAUGE, node_id, }), KeySerializer::new(U32_LEN).write_leb128(value).finalize(), ); } } } for histogram in Collector::collect_histograms(true) { let histogram_id = histogram.id(); if matches!( histogram_id, MetricType::MessageIngestionTime | MetricType::MessageFtsIndexTime | MetricType::DeliveryTotalTime | MetricType::DeliveryTime | MetricType::DnsLookupTime ) { let history = history.histograms.entry(histogram_id).or_default(); let sum = histogram.sum(); let count = histogram.count(); let diff_sum = sum - history.sum; let diff_count = count - history.count; if diff_sum > 0 || diff_count > 0 { batch.set( ValueClass::Telemetry(TelemetryClass::Metric { timestamp, metric_id: (histogram_id.code() << 2) | TYPE_HISTOGRAM, node_id, }), KeySerializer::new(U32_LEN) .write_leb128(diff_count) .write_leb128(diff_sum) .finalize(), ); } history.sum = sum; history.count = count; } } } if !batch.is_empty() { self.write(batch.build_all()) .await .caused_by(trc::location!())?; } Ok(()) } async fn query_metrics( &self, from_timestamp: u64, to_timestamp: u64, ) -> trc::Result>> { let mut metrics = Vec::new(); self.iterate( IterateParams::new( ValueKey::from(ValueClass::Telemetry(TelemetryClass::Metric { timestamp: from_timestamp, metric_id: 0, node_id: 0, })), ValueKey::from(ValueClass::Telemetry(TelemetryClass::Metric { timestamp: to_timestamp, metric_id: 0, node_id: 0, })), ), |key, value| { let timestamp = key.deserialize_be_u64(0).caused_by(trc::location!())?; let (metric_type, _) = key .get(U64_LEN..) .and_then(|bytes| bytes.read_leb128::()) .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?; match metric_type & 0x03 { TYPE_COUNTER => { let id = EventType::from_code(metric_type >> 2).ok_or_else(|| { trc::Error::corrupted_key(key, None, trc::location!()) })?; let (value, _) = value.read_leb128::().ok_or_else(|| { trc::Error::corrupted_key(key, value.into(), trc::location!()) })?; metrics.push(Metric::Counter { id, timestamp, value, }); } TYPE_HISTOGRAM => { let id = MetricType::from_code(metric_type >> 2).ok_or_else(|| { trc::Error::corrupted_key(key, None, trc::location!()) })?; let (count, bytes_read) = value.read_leb128::().ok_or_else(|| { trc::Error::corrupted_key(key, value.into(), trc::location!()) })?; let (sum, _) = value .get(bytes_read..) .and_then(|bytes| bytes.read_leb128::()) .ok_or_else(|| { trc::Error::corrupted_key(key, value.into(), trc::location!()) })?; metrics.push(Metric::Histogram { id, timestamp, count, sum, }); } TYPE_GAUGE => { let id = MetricType::from_code(metric_type >> 2).ok_or_else(|| { trc::Error::corrupted_key(key, None, trc::location!()) })?; let (value, _) = value.read_leb128::().ok_or_else(|| { trc::Error::corrupted_key(key, value.into(), trc::location!()) })?; metrics.push(Metric::Gauge { id, timestamp, value, }); } _ => return Err(trc::Error::corrupted_key(key, None, trc::location!())), } Ok(true) }, ) .await .caused_by(trc::location!())?; Ok(metrics) } async fn purge_metrics(&self, period: Duration) -> trc::Result<()> { self.delete_range( ValueKey::from(ValueClass::Telemetry(TelemetryClass::Metric { timestamp: 0, metric_id: 0, node_id: 0, })), ValueKey::from(ValueClass::Telemetry(TelemetryClass::Metric { timestamp: now() - period.as_secs(), metric_id: 0, node_id: 0, })), ) .await .caused_by(trc::location!()) } } impl MetricsHistory { pub fn init() -> SharedMetricHistory { Arc::new(Mutex::new(Self::default())) } } ================================================ FILE: crates/common/src/telemetry/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod metrics; pub mod tracers; pub mod webhooks; use tracers::log::spawn_log_tracer; use tracers::otel::spawn_otel_tracer; use tracers::stdout::spawn_console_tracer; use trc::{Collector, ipc::subscriber::SubscriberBuilder}; use webhooks::spawn_webhook_tracer; use crate::config::telemetry::{Telemetry, TelemetrySubscriberType}; impl Telemetry { pub fn enable(self, is_enterprise: bool) { // Spawn tracers for tracer in self.tracers.subscribers { tracer.typ.spawn( SubscriberBuilder::new(tracer.id) .with_interests(tracer.interests) .with_lossy(tracer.lossy), is_enterprise, ); } // Update global collector Collector::set_interests(self.tracers.interests); Collector::update_custom_levels(self.tracers.levels); Collector::set_metrics(self.metrics); Collector::reload(); } pub fn update(self, is_enterprise: bool) { // Remove tracers that are no longer active let active_subscribers = Collector::get_subscribers(); for subscribed_id in &active_subscribers { if !self .tracers .subscribers .iter() .any(|tracer| tracer.id == *subscribed_id) { Collector::remove_subscriber(subscribed_id.clone()); } } // Activate new tracers or update existing ones for tracer in self.tracers.subscribers { if active_subscribers.contains(&tracer.id) { Collector::update_subscriber(tracer.id, tracer.interests, tracer.lossy); } else { tracer.typ.spawn( SubscriberBuilder::new(tracer.id) .with_interests(tracer.interests) .with_lossy(tracer.lossy), is_enterprise, ); } } // Update global collector Collector::set_interests(self.tracers.interests); Collector::update_custom_levels(self.tracers.levels); Collector::set_metrics(self.metrics); Collector::reload(); } #[cfg(feature = "test_mode")] pub fn test_tracer(level: trc::Level) { let mut interests = trc::ipc::subscriber::Interests::default(); for event in trc::EventType::variants() { if level.is_contained(event.level()) { interests.set(event); } } spawn_console_tracer( SubscriberBuilder::new("stderr".to_string()) .with_interests(interests.clone()) .with_lossy(false), crate::config::telemetry::ConsoleTracer { ansi: true, multiline: false, buffered: false, }, ); Collector::union_interests(interests); Collector::reload(); } } impl TelemetrySubscriberType { pub fn spawn(self, builder: SubscriberBuilder, is_enterprise: bool) { match self { TelemetrySubscriberType::ConsoleTracer(settings) => { spawn_console_tracer(builder, settings) } TelemetrySubscriberType::LogTracer(settings) => spawn_log_tracer(builder, settings), TelemetrySubscriberType::Webhook(settings) => spawn_webhook_tracer(builder, settings), TelemetrySubscriberType::OtelTracer(settings) => spawn_otel_tracer(builder, settings), #[cfg(unix)] TelemetrySubscriberType::JournalTracer(subscriber) => { tracers::journald::spawn_journald_tracer(builder, subscriber) } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] TelemetrySubscriberType::StoreTracer(subscriber) => { if is_enterprise { tracers::store::spawn_store_tracer(builder, subscriber) } } // SPDX-SnippetEnd } } } ================================================ FILE: crates/common/src/telemetry/tracers/journald.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use ahash::AHashSet; use std::io::Write; use trc::ipc::subscriber::SubscriberBuilder; use trc::{Event, EventDetails, Level, TelemetryEvent}; pub(crate) fn spawn_journald_tracer(builder: SubscriberBuilder, subscriber: Subscriber) { let (_, mut rx) = builder.register(); tokio::spawn(async move { while let Some(events) = rx.recv().await { for event in events { subscriber.send_event(&event); } } }); } impl Subscriber { fn send_event(&self, event: &Event) { let mut buf = Vec::with_capacity(256); put_field_wellformed( &mut buf, "PRIORITY", &[match event.inner.level { Level::Error => self.priority_mappings.error as u8, Level::Warn => self.priority_mappings.warn as u8, Level::Info => self.priority_mappings.info as u8, Level::Debug => self.priority_mappings.debug as u8, Level::Trace | Level::Disable => self.priority_mappings.trace as u8, }], ); put_field_length_encoded(&mut buf, "SYSLOG_IDENTIFIER", |buf| { write!(buf, "{}", self.syslog_identifier).unwrap() }); put_field_length_encoded(&mut buf, "MESSAGE", |buf| { write!(buf, "{}", event.inner.typ.description()).unwrap() }); let mut seen_keys = AHashSet::new(); for (key, value) in &event.keys { if seen_keys.insert(*key) { put_field_length_encoded(&mut buf, key.name(), |buf| { write!(buf, "{value}").unwrap() }); } } if let Err(err) = self.send_payload(&buf) { trc::event!( Telemetry(TelemetryEvent::JournalError), Details = "Failed to send event to journald", Reason = err.to_string() ); } } } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2018 Benjamin Saunders // SPDX-License-Identifier: MIT #[cfg(target_os = "linux")] use std::fs::File; use std::io::{self, Error, Result}; use std::mem::{size_of, zeroed}; #[cfg(target_os = "linux")] use std::os::raw::c_uint; use std::os::unix::ffi::OsStrExt; use std::os::unix::net::UnixDatagram; #[cfg(target_os = "linux")] use std::os::unix::prelude::FromRawFd; use std::os::unix::prelude::{AsRawFd, RawFd}; use std::path::Path; use std::ptr; use libc::*; #[cfg(unix)] const JOURNALD_PATH: &str = "/run/systemd/journal/socket"; const CMSG_BUFSIZE: usize = 64; pub struct Subscriber { #[cfg(unix)] socket: UnixDatagram, syslog_identifier: String, priority_mappings: PriorityMappings, } #[derive(Debug, Clone)] pub struct PriorityMappings { /// Priority mapped to the `ERROR` level pub error: Priority, /// Priority mapped to the `WARN` level pub warn: Priority, /// Priority mapped to the `INFO` level pub info: Priority, /// Priority mapped to the `DEBUG` level pub debug: Priority, /// Priority mapped to the `TRACE` level pub trace: Priority, } #[repr(C)] union AlignedBuffer { buffer: T, align: cmsghdr, } #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] #[repr(u8)] pub enum Priority { /// System is unusable. /// /// Examples: /// /// - severe Kernel BUG /// - systemd dumped core /// /// This level should not be used by applications. Emergency = b'0', /// Should be corrected immediately. /// /// Examples: /// /// - Vital subsystem goes out of work, data loss: /// - `kernel: BUG: unable to handle kernel paging request at ffffc90403238ffc` Alert = b'1', /// Critical conditions /// /// Examples: /// /// - Crashe, coredumps /// - `systemd-coredump[25319]: Process 25310 (plugin-container) of user 1000 dumped core` Critical = b'2', /// Error conditions /// /// Examples: /// /// - Not severe error reported /// - `kernel: usb 1-3: 3:1: cannot get freq at ep 0x84, systemd[1]: Failed unmounting /var` /// - `libvirtd[1720]: internal error: Failed to initialize a valid firewall backend` Error = b'3', /// May indicate that an error will occur if action is not taken. /// /// Examples: /// /// - a non-root file system has only 1GB free /// - `org.freedesktop. Notifications[1860]: (process:5999): Gtk-WARNING **: Locale not supported by C library. Using the fallback 'C' locale` Warning = b'4', /// Events that are unusual, but not error conditions. /// /// Examples: /// /// - `systemd[1]: var.mount: Directory /var to mount over is not empty, mounting anyway` /// - `gcr-prompter[4997]: Gtk: GtkDialog mapped without a transient parent. This is discouraged` Notice = b'5', /// Normal operational messages that require no action. /// /// Example: `lvm[585]: 7 logical volume(s) in volume group "archvg" now active` Informational = b'6', /// Information useful to developers for debugging the /// application. /// /// Example: `kdeinit5[1900]: powerdevil: Scheduling inhibition from ":1.14" "firefox" with cookie 13 and reason "screen"` Debug = b'7', } impl Subscriber { /// Construct a journald subscriber /// /// Fails if the journald socket couldn't be opened. Returns a `NotFound` error unconditionally /// in non-Unix environments. pub fn new() -> io::Result { #[cfg(unix)] { let socket = UnixDatagram::unbound()?; let sub = Self { socket, syslog_identifier: std::env::current_exe() .ok() .as_ref() .and_then(|p| p.file_name()) .map(|n| n.to_string_lossy().into_owned()) // If we fail to get the name of the current executable fall back to an empty string. .unwrap_or_default(), priority_mappings: PriorityMappings::new(), }; // Check that we can talk to journald, by sending empty payload which journald discards. // However if the socket didn't exist or if none listened we'd get an error here. sub.send_payload(&[])?; Ok(sub) } #[cfg(not(unix))] Err(io::Error::new( io::ErrorKind::NotFound, "journald does not exist in this environment", )) } /// Sets how [`tracing_core::Level`]s are mapped to [journald priorities](Priority). /// pub fn with_priority_mappings(mut self, mappings: PriorityMappings) -> Self { self.priority_mappings = mappings; self } /// Sets the syslog identifier for this logger. /// /// The syslog identifier comes from the classic syslog interface (`openlog()` /// and `syslog()`) and tags log entries with a given identifier. /// Systemd exposes it in the `SYSLOG_IDENTIFIER` journal field, and allows /// filtering log messages by syslog identifier with `journalctl -t`. /// Unlike the unit (`journalctl -u`) this field is not trusted, i.e. applications /// can set it freely, and use it e.g. to further categorize log entries emitted under /// the same systemd unit or in the same process. It also allows to filter for log /// entries of processes not started in their own unit. /// /// See [Journal Fields](https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html) /// and [journalctl](https://www.freedesktop.org/software/systemd/man/journalctl.html) /// for more information. /// /// Defaults to the file name of the executable of the current process, if any. pub fn with_syslog_identifier(mut self, identifier: String) -> Self { self.syslog_identifier = identifier; self } /// Returns the syslog identifier in use. pub fn syslog_identifier(&self) -> &str { &self.syslog_identifier } #[cfg(not(unix))] fn send_payload(&self, _opayload: &[u8]) -> io::Result<()> { Err(io::Error::new( io::ErrorKind::Other, "journald not supported on non-Unix", )) } #[cfg(unix)] fn send_payload(&self, payload: &[u8]) -> io::Result { self.socket .send_to(payload, JOURNALD_PATH) .or_else(|error| { if Some(libc::EMSGSIZE) == error.raw_os_error() { self.send_large_payload(payload) } else { Err(error) } }) } #[cfg(all(unix, not(target_os = "linux")))] fn send_large_payload(&self, _payload: &[u8]) -> io::Result { Err(std::io::Error::other( "Large payloads not supported on non-Linux OS", )) } /// Send large payloads to journald via a memfd. #[cfg(target_os = "linux")] fn send_large_payload(&self, payload: &[u8]) -> io::Result { // If the payload's too large for a single datagram, send it through a memfd, see // https://systemd.io/JOURNAL_NATIVE_PROTOCOL/ use std::os::unix::prelude::AsRawFd; // Write the whole payload to a memfd let mut mem = create_sealable()?; mem.write_all(payload)?; // Fully seal the memfd to signal journald that its backing data won't resize anymore // and so is safe to mmap. seal_fully(mem.as_raw_fd())?; send_one_fd_to(&self.socket, mem.as_raw_fd(), JOURNALD_PATH) } } impl PriorityMappings { /// Returns the default priority mappings: /// pub fn new() -> PriorityMappings { Self { error: Priority::Error, warn: Priority::Warning, info: Priority::Notice, debug: Priority::Informational, trace: Priority::Debug, } } } impl Default for PriorityMappings { fn default() -> Self { Self::new() } } impl std::fmt::Debug for Subscriber { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Subscriber") .field("socket", &self.socket) .field("syslog_identifier", &self.syslog_identifier) .field("priority_mappings", &self.priority_mappings) .finish() } } /// Append a sanitized and length-encoded field into `buf`. /// /// Unlike `put_field_wellformed` this function handles arbitrary field names and values. /// /// `name` denotes the field name. It gets sanitized before being appended to `buf`. /// /// `write_value` is invoked with `buf` as argument to append the value data to `buf`. It must /// not delete from `buf`, but may append arbitrary data. This function then determines the length /// of the data written and adds it in the appropriate place in `buf`. fn put_field_length_encoded(buf: &mut Vec, name: &str, write_value: impl FnOnce(&mut Vec)) { for ch in name.as_bytes() { buf.push(ch.to_ascii_uppercase()); } buf.push(b'\n'); buf.extend_from_slice(&[0; 8]); // Length tag, to be populated let start = buf.len(); write_value(buf); let end = buf.len(); buf[start - 8..start].copy_from_slice(&((end - start) as u64).to_le_bytes()); buf.push(b'\n'); } /// Append arbitrary data with a well-formed name and value. /// /// `value` must not contain an internal newline, because this function writes /// `value` in the new-line separated format. /// /// For a "newline-safe" variant, see `put_field_length_encoded`. fn put_field_wellformed(buf: &mut Vec, name: &str, value: &[u8]) { buf.extend_from_slice(name.as_bytes()); buf.push(b'\n'); put_value(buf, value); } /// Write the value portion of a key-value pair, in newline separated format. /// /// `value` must not contain an internal newline. /// /// For a "newline-safe" variant, see `put_field_length_encoded`. fn put_value(buf: &mut Vec, value: &[u8]) { buf.extend_from_slice(&(value.len() as u64).to_le_bytes()); buf.extend_from_slice(value); buf.push(b'\n'); } fn assert_cmsg_bufsize() { let space_one_fd = unsafe { CMSG_SPACE(size_of::() as u32) }; assert!( space_one_fd <= CMSG_BUFSIZE as u32, "cmsghdr buffer too small (< {}) to hold a single fd", space_one_fd ); } pub fn send_one_fd_to>(socket: &UnixDatagram, fd: RawFd, path: P) -> Result { assert_cmsg_bufsize(); let mut addr: sockaddr_un = unsafe { zeroed() }; let path_bytes = path.as_ref().as_os_str().as_bytes(); // path_bytes may have at most sun_path + 1 bytes, to account for the trailing NUL byte. if addr.sun_path.len() <= path_bytes.len() { return Err(Error::from_raw_os_error(ENAMETOOLONG)); } addr.sun_family = AF_UNIX as _; unsafe { std::ptr::copy_nonoverlapping( path_bytes.as_ptr(), addr.sun_path.as_mut_ptr() as *mut u8, path_bytes.len(), ) }; let mut msg: msghdr = unsafe { zeroed() }; // Set the target address. msg.msg_name = &mut addr as *mut _ as *mut c_void; msg.msg_namelen = size_of::() as socklen_t; // We send no data body with this message. msg.msg_iov = ptr::null_mut(); msg.msg_iovlen = 0; // Create and fill the control message buffer with our file descriptor let mut cmsg_buffer = AlignedBuffer { buffer: ([0u8; CMSG_BUFSIZE]), }; msg.msg_control = unsafe { cmsg_buffer.buffer.as_mut_ptr() as _ }; msg.msg_controllen = unsafe { CMSG_SPACE(size_of::() as _) as _ }; let cmsg: &mut cmsghdr = unsafe { CMSG_FIRSTHDR(&msg).as_mut() }.expect("Control message buffer exhausted"); cmsg.cmsg_level = SOL_SOCKET; cmsg.cmsg_type = SCM_RIGHTS; cmsg.cmsg_len = unsafe { CMSG_LEN(size_of::() as _) as _ }; unsafe { ptr::write(CMSG_DATA(cmsg) as *mut RawFd, fd) }; let result = unsafe { sendmsg(socket.as_raw_fd(), &msg, libc::MSG_NOSIGNAL) }; if result < 0 { Err(Error::last_os_error()) } else { // sendmsg returns the number of bytes written Ok(result as usize) } } #[cfg(target_os = "linux")] fn create(flags: c_uint) -> Result { let fd = memfd_create_syscall(flags); if fd < 0 { Err(Error::last_os_error()) } else { Ok(unsafe { File::from_raw_fd(fd as RawFd) }) } } /// Make the `memfd_create` syscall ourself instead of going through `libc`; /// `memfd_create` isn't supported on `glibc<2.27` so this allows us to /// support old-but-still-used distros like Ubuntu Xenial, Debian Stretch, /// RHEL 7, etc. /// /// See: https://github.com/tokio-rs/tracing/issues/1879 #[cfg(target_os = "linux")] fn memfd_create_syscall(flags: c_uint) -> c_int { unsafe { syscall( SYS_memfd_create, "tracing-journald\0".as_ptr() as *const c_char, flags, ) as c_int } } #[cfg(target_os = "linux")] pub fn create_sealable() -> Result { create(MFD_ALLOW_SEALING | MFD_CLOEXEC) } #[cfg(target_os = "linux")] pub fn seal_fully(fd: RawFd) -> Result<()> { let all_seals = F_SEAL_SHRINK | F_SEAL_GROW | F_SEAL_WRITE | F_SEAL_SEAL; let result = unsafe { fcntl(fd, F_ADD_SEALS, all_seals) }; if result < 0 { Err(Error::last_os_error()) } else { Ok(()) } } // SPDX-SnippetEnd ================================================ FILE: crates/common/src/telemetry/tracers/log.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{path::PathBuf, time::SystemTime}; use crate::config::telemetry::{LogTracer, RotationStrategy}; use mail_parser::DateTime; use tokio::{ fs::{File, OpenOptions}, io::BufWriter, }; use trc::{TelemetryEvent, ipc::subscriber::SubscriberBuilder, serializers::text::FmtWriter}; pub(crate) fn spawn_log_tracer(builder: SubscriberBuilder, settings: LogTracer) { let (_, mut rx) = builder.register(); tokio::spawn(async move { if let Some(writer) = settings.build_writer().await { let mut buf = FmtWriter::new(writer) .with_ansi(settings.ansi) .with_multiline(settings.multiline); let mut roatation_timestamp = settings.next_rotation(); while let Some(events) = rx.recv().await { for event in events { // Check if we need to rotate the log file if roatation_timestamp != 0 && event.inner.timestamp > roatation_timestamp { if let Err(err) = buf.flush().await { trc::event!( Telemetry(TelemetryEvent::LogError), Reason = err.to_string(), Details = "Failed to flush log buffer" ); } if let Some(writer) = settings.build_writer().await { buf.update_writer(writer); roatation_timestamp = settings.next_rotation(); } else { return; }; } if let Err(err) = buf.write(&event).await { trc::event!( Telemetry(TelemetryEvent::LogError), Reason = err.to_string(), Details = "Failed to write event to log" ); return; } } if let Err(err) = buf.flush().await { trc::event!( Telemetry(TelemetryEvent::LogError), Reason = err.to_string(), Details = "Failed to flush log buffer" ); } } } }); } impl LogTracer { pub async fn build_writer(&self) -> Option> { let now = DateTime::from_timestamp( SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()) as i64, ); let file_name = match self.rotate { RotationStrategy::Daily => { format!( "{}.{:04}-{:02}-{:02}", self.prefix, now.year, now.month, now.day ) } RotationStrategy::Hourly => { format!( "{}.{:04}-{:02}-{:02}T{:02}", self.prefix, now.year, now.month, now.day, now.hour ) } RotationStrategy::Minutely => { format!( "{}.{:04}-{:02}-{:02}T{:02}:{:02}", self.prefix, now.year, now.month, now.day, now.hour, now.minute ) } RotationStrategy::Never => self.prefix.clone(), }; let path = PathBuf::from(&self.path).join(file_name); match OpenOptions::new() .create(true) .append(true) .open(&path) .await { Ok(writer) => Some(BufWriter::new(writer)), Err(err) => { trc::event!( Telemetry(TelemetryEvent::LogError), Details = "Failed to create log file", Path = path.to_string_lossy().into_owned(), Reason = err.to_string(), ); None } } } pub fn next_rotation(&self) -> u64 { let mut now = DateTime::from_timestamp( SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()) as i64, ); now.second = 0; match self.rotate { RotationStrategy::Daily => { now.hour = 0; now.minute = 0; now.to_timestamp() as u64 + 86400 } RotationStrategy::Hourly => { now.minute = 0; now.to_timestamp() as u64 + 3600 } RotationStrategy::Minutely => now.to_timestamp() as u64 + 60, RotationStrategy::Never => 0, } } } ================================================ FILE: crates/common/src/telemetry/tracers/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ #[cfg(unix)] pub mod journald; pub mod log; pub mod otel; pub mod stdout; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] pub mod store; // SPDX-SnippetEnd ================================================ FILE: crates/common/src/telemetry/tracers/otel.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{LONG_1Y_SLUMBER, config::telemetry::OtelTracer}; use ahash::AHashMap; use mail_parser::DateTime; use opentelemetry::{ InstrumentationScope, Key, KeyValue, Value, logs::{AnyValue, Severity}, trace::{SpanContext, SpanKind, Status, TraceFlags, TraceState}, }; use opentelemetry_sdk::{ Resource, logs::{LogBatch, LogExporter, SdkLogRecord}, trace::{SpanData, SpanEvents, SpanExporter, SpanLinks}, }; use opentelemetry_semantic_conventions::resource::SERVICE_VERSION; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use trc::{Event, EventDetails, Level, TelemetryEvent, ipc::subscriber::SubscriberBuilder}; const MAX_EVENTS: usize = 2048; pub(crate) fn spawn_otel_tracer(builder: SubscriberBuilder, mut otel: OtelTracer) { let (_, mut rx) = builder.register(); tokio::spawn(async move { let resource = Resource::builder() .with_service_name("stalwart") .with_attribute(KeyValue::new(SERVICE_VERSION, env!("CARGO_PKG_VERSION"))) .build(); let instrumentation = InstrumentationScope::builder("stalwart") .with_version(env!("CARGO_PKG_VERSION")) .build(); otel.log_exporter.set_resource(&resource); otel.span_exporter.set_resource(&resource); let mut wakeup_time = LONG_1Y_SLUMBER; let mut next_delivery = Instant::now(); let mut pending_logs = Vec::new(); let mut pending_spans = Vec::new(); let mut active_spans = AHashMap::new(); loop { // Wait for the next event or timeout let event_or_timeout = tokio::time::timeout(wakeup_time, rx.recv()).await; match event_or_timeout { Ok(Some(events)) => { for event in events { if otel.log_exporter_enable { pending_logs.push(otel.build_log_record(&event)); } if otel.span_exporter_enable && let Some(span) = event.inner.span.as_ref() { let span_id = span.span_id().unwrap(); if !event.inner.typ.is_span_end() { let events = active_spans.entry(span_id).or_insert_with(Vec::new); if events.len() < MAX_EVENTS { events.push(event); } } else if let Some(events) = active_spans.remove(&span_id) { pending_spans.push(build_span_data( span, &event, events.iter().chain(std::iter::once(&event)), &instrumentation, )); } } } } Ok(None) => { break; } Err(_) => (), } // Process events let mut next_retry = None; let now = Instant::now(); if next_delivery <= now { if !pending_spans.is_empty() || !pending_logs.is_empty() { next_delivery = now + otel.throttle; if !pending_spans.is_empty() && let Err(err) = otel .span_exporter .export(std::mem::take(&mut pending_spans)) .await { trc::event!( Telemetry(TelemetryEvent::OtelExporterError), Details = "Failed to export spans", Reason = err.to_string() ); } if !pending_logs.is_empty() { let logs = pending_logs .iter() .map(|log| (log, &instrumentation)) .collect::>(); if let Err(err) = otel.log_exporter.export(LogBatch::new(&logs)).await { trc::event!( Telemetry(TelemetryEvent::OtelExporterError), Details = "Failed to export logs", Reason = err.to_string() ); } pending_logs.clear(); } } } else if !pending_logs.is_empty() || !pending_spans.is_empty() { // Retry later let this_retry = next_delivery - now; match next_retry { Some(next_retry) if this_retry >= next_retry => {} _ => { next_retry = Some(this_retry); } } } wakeup_time = next_retry.unwrap_or(LONG_1Y_SLUMBER); } }); } fn build_span_data( start_span: &Event, end_span: &Event, span_events: I, instrumentation: &InstrumentationScope, ) -> SpanData where I: IntoIterator, T: AsRef>, { let span_id = start_span.span_id().unwrap(); let mut events = SpanEvents::default(); events.events = span_events .into_iter() .map(|event| { let event = event.as_ref(); opentelemetry::trace::Event::new( event.inner.typ.name(), UNIX_EPOCH + Duration::from_secs(event.inner.timestamp), event.keys.iter().filter_map(build_key_value).collect(), 0, ) }) .collect(); SpanData { span_context: SpanContext::new( (span_id as u128).into(), span_id.into(), TraceFlags::default(), false, TraceState::default(), ), dropped_attributes_count: 0, parent_span_id: 0.into(), name: start_span.inner.typ.name().into(), start_time: UNIX_EPOCH + Duration::from_secs(start_span.inner.timestamp), end_time: UNIX_EPOCH + Duration::from_secs(end_span.inner.timestamp), attributes: start_span.keys.iter().filter_map(build_key_value).collect(), events, links: SpanLinks::default(), status: Status::default(), span_kind: SpanKind::Server, instrumentation_scope: instrumentation.clone(), } } impl OtelTracer { fn build_log_record(&self, event: &Event) -> SdkLogRecord { use opentelemetry::logs::LogRecord; use opentelemetry::logs::Logger; let mut record = self.log_provider.create_log_record(); record.set_event_name(event.inner.typ.name()); record.set_severity_number(match event.inner.level { Level::Trace => Severity::Trace, Level::Debug => Severity::Debug, Level::Info => Severity::Info, Level::Warn => Severity::Warn, Level::Error => Severity::Error, Level::Disable => Severity::Error, }); record.set_severity_text(event.inner.level.as_str()); record.set_body(AnyValue::String(event.inner.typ.description().into())); record.set_timestamp(UNIX_EPOCH + Duration::from_secs(event.inner.timestamp)); record.set_observed_timestamp(SystemTime::now()); for (k, v) in &event.keys { record.add_attribute(k.name(), build_any_value(v)); } record } } fn build_key_value(key_value: &(trc::Key, trc::Value)) -> Option { (key_value.0 != trc::Key::SpanId).then(|| { KeyValue::new( build_key(&key_value.0), match &key_value.1 { trc::Value::String(v) => Value::String(v.to_string().into()), trc::Value::UInt(v) => Value::I64(*v as i64), trc::Value::Int(v) => Value::I64(*v), trc::Value::Float(v) => Value::F64(*v), trc::Value::Timestamp(v) => { Value::String(DateTime::from_timestamp(*v as i64).to_rfc3339().into()) } trc::Value::Duration(v) => Value::I64(*v as i64), trc::Value::Bytes(_) => Value::String("[binary data]".into()), trc::Value::Bool(v) => Value::Bool(*v), trc::Value::Ipv4(v) => Value::String(v.to_string().into()), trc::Value::Ipv6(v) => Value::String(v.to_string().into()), trc::Value::Event(_) => Value::String("[event data]".into()), trc::Value::Array(_) => Value::String("[array]".into()), trc::Value::None => Value::Bool(false), }, ) }) } fn build_key(key: &trc::Key) -> Key { Key::from_static_str(key.name()) } fn build_any_value(value: &trc::Value) -> AnyValue { match value { trc::Value::String(v) => AnyValue::String(v.to_string().into()), trc::Value::UInt(v) => AnyValue::Int(*v as i64), trc::Value::Int(v) => AnyValue::Int(*v), trc::Value::Float(v) => AnyValue::Double(*v), trc::Value::Timestamp(v) => { AnyValue::String(DateTime::from_timestamp(*v as i64).to_rfc3339().into()) } trc::Value::Duration(v) => AnyValue::Int(*v as i64), trc::Value::Bytes(v) => AnyValue::Bytes(Box::new(v.clone())), trc::Value::Bool(v) => AnyValue::Boolean(*v), trc::Value::Ipv4(v) => AnyValue::String(v.to_string().into()), trc::Value::Ipv6(v) => AnyValue::String(v.to_string().into()), trc::Value::Event(v) => AnyValue::Map(Box::new( [( Key::from_static_str("eventName"), AnyValue::String(v.event_type().name().into()), )] .into_iter() .chain( v.keys() .iter() .map(|(k, v)| (build_key(k), build_any_value(v))), ) .collect(), )), trc::Value::Array(v) => { AnyValue::ListAny(Box::new(v.iter().map(build_any_value).collect())) } trc::Value::None => AnyValue::Boolean(false), } } ================================================ FILE: crates/common/src/telemetry/tracers/stdout.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ io::{Error, stderr}, pin::Pin, task::{Context, Poll}, }; use crate::config::telemetry::ConsoleTracer; use std::io::Write; use tokio::io::AsyncWrite; use trc::{ipc::subscriber::SubscriberBuilder, serializers::text::FmtWriter}; pub(crate) fn spawn_console_tracer(builder: SubscriberBuilder, settings: ConsoleTracer) { let (_, mut rx) = builder.register(); tokio::spawn(async move { let mut buf = FmtWriter::new(StdErrWriter::default()) .with_ansi(settings.ansi) .with_multiline(settings.multiline); while let Some(events) = rx.recv().await { for event in events { let _ = buf.write(&event).await; if !settings.buffered { let _ = buf.flush().await; } } if settings.buffered { let _ = buf.flush().await; } } }); } const BUFFER_CAPACITY: usize = 4096; pub struct StdErrWriter { buffer: Vec, } impl AsyncWrite for StdErrWriter { fn poll_write( mut self: Pin<&mut Self>, _: &mut Context<'_>, bytes: &[u8], ) -> Poll> { let bytes_len = bytes.len(); let buffer_len = self.buffer.len(); if buffer_len + bytes_len < BUFFER_CAPACITY { self.buffer.extend_from_slice(bytes); Poll::Ready(Ok(bytes_len)) } else if bytes_len > BUFFER_CAPACITY { let result = stderr() .write_all(&self.buffer) .and_then(|_| stderr().write_all(bytes)); self.buffer.clear(); Poll::Ready(result.map(|_| bytes_len)) } else { let result = stderr().write_all(&self.buffer); self.buffer.clear(); self.buffer.extend_from_slice(bytes); Poll::Ready(result.map(|_| bytes_len)) } } fn poll_flush(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { Poll::Ready(if !self.buffer.is_empty() { let result = stderr().write_all(&self.buffer); self.buffer.clear(); result } else { Ok(()) }) } fn poll_shutdown(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { Poll::Ready(Ok(())) } } impl Default for StdErrWriter { fn default() -> Self { Self { buffer: Vec::with_capacity(BUFFER_CAPACITY), } } } ================================================ FILE: crates/common/src/telemetry/tracers/store.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: LicenseRef-SEL * * This file is subject to the Stalwart Enterprise License Agreement (SEL) and * is NOT open source software. * */ use crate::config::telemetry::StoreTracer; use ahash::{AHashMap, AHashSet}; use std::{collections::HashSet, future::Future, time::Duration}; use store::{ Deserialize, SearchStore, Store, ValueKey, search::{IndexDocument, SearchField, SearchFilter, SearchQuery, TracingSearchField}, write::{BatchBuilder, SearchIndex, TaskEpoch, TaskQueueClass, TelemetryClass, ValueClass}, }; use trc::{ AddContext, AuthEvent, Event, EventDetails, EventType, Key, MessageIngestEvent, OutgoingReportEvent, QueueEvent, Value, ipc::subscriber::SubscriberBuilder, serializers::binary::{deserialize_events, serialize_events}, }; use utils::snowflake::SnowflakeIdGenerator; const MAX_EVENTS: usize = 2048; pub(crate) fn spawn_store_tracer(builder: SubscriberBuilder, settings: StoreTracer) { let (_, mut rx) = builder.register(); tokio::spawn(async move { let mut active_spans = AHashMap::new(); let store = settings.store; let mut batch = BatchBuilder::new(); while let Some(events) = rx.recv().await { for event in events { if let Some(span) = &event.inner.span { let span_id = span.span_id().unwrap(); if !event.inner.typ.is_span_end() { let events = active_spans.entry(span_id).or_insert_with(Vec::new); if events.len() < MAX_EVENTS { events.push(event); } } else if let Some(events) = active_spans.remove(&span_id) && events .iter() .chain([span, &event]) .flat_map(|event| event.keys.iter()) .any(|(k, v)| matches!((k, v), (Key::QueueId, Value::UInt(_)))) { // Serialize events batch .set( ValueClass::Telemetry(TelemetryClass::Span { span_id }), serialize_events( [span.as_ref()] .into_iter() .chain(events.iter().map(|event| event.as_ref())) .chain([event.as_ref()].into_iter()), events.len() + 2, ), ) .with_account_id((span_id >> 32) as u32) // TODO: This is hacky, improve .with_document(span_id as u32) .set( ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { due: TaskEpoch::now(), index: SearchIndex::Tracing, is_insert: true, }), vec![], ); } } } if !batch.is_empty() { if let Err(err) = store.write(batch.build_all()).await { trc::error!(err.caused_by(trc::location!())); } batch = BatchBuilder::new(); } } }); } pub trait TracingStore: Sync + Send { fn get_span( &self, span_id: u64, ) -> impl Future>>> + Send; fn get_raw_span( &self, span_id: u64, ) -> impl Future>>> + Send; fn purge_spans( &self, period: Duration, search_store: Option<&SearchStore>, ) -> impl Future> + Send; } impl TracingStore for Store { async fn get_span(&self, span_id: u64) -> trc::Result>> { self.get_value::(ValueKey::from(ValueClass::Telemetry( TelemetryClass::Span { span_id }, ))) .await .caused_by(trc::location!()) .map(|span| span.map(|span| span.0).unwrap_or_default()) } async fn get_raw_span(&self, span_id: u64) -> trc::Result>> { self.get_value::(ValueKey::from(ValueClass::Telemetry( TelemetryClass::Span { span_id }, ))) .await .caused_by(trc::location!()) .map(|span| span.map(|span| span.0)) } async fn purge_spans( &self, period: Duration, search_store: Option<&SearchStore>, ) -> trc::Result<()> { let until_span_id = SnowflakeIdGenerator::from_duration(period).ok_or_else(|| { trc::StoreEvent::UnexpectedError .caused_by(trc::location!()) .ctx(trc::Key::Reason, "Failed to generate reference span id.") })?; self.delete_range( ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span { span_id: 0 })), ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span { span_id: until_span_id, })), ) .await .caused_by(trc::location!())?; if let Some(search_store) = search_store { search_store .unindex( SearchQuery::new(SearchIndex::Tracing) .with_filter(SearchFilter::lt(SearchField::Id, until_span_id)), ) .await .caused_by(trc::location!())?; } Ok(()) } } impl StoreTracer { pub fn default_events() -> impl IntoIterator { EventType::variants().into_iter().filter(|event| { !event.is_raw_io() && matches!( event, EventType::MessageIngest( MessageIngestEvent::Ham | MessageIngestEvent::Spam | MessageIngestEvent::Duplicate | MessageIngestEvent::Error ) | EventType::Smtp(_) | EventType::Delivery(_) | EventType::MtaSts(_) | EventType::TlsRpt(_) | EventType::Dane(_) | EventType::Iprev(_) | EventType::Spf(_) | EventType::Dmarc(_) | EventType::Dkim(_) | EventType::MailAuth(_) | EventType::Queue( QueueEvent::QueueMessage | QueueEvent::QueueMessageAuthenticated | QueueEvent::QueueReport | QueueEvent::QueueDsn | QueueEvent::QueueAutogenerated | QueueEvent::Rescheduled | QueueEvent::RateLimitExceeded | QueueEvent::ConcurrencyLimitExceeded | QueueEvent::QuotaExceeded ) | EventType::Limit(_) | EventType::Tls(_) | EventType::IncomingReport(_) | EventType::OutgoingReport( OutgoingReportEvent::SpfReport | OutgoingReportEvent::SpfRateLimited | OutgoingReportEvent::DkimReport | OutgoingReportEvent::DkimRateLimited | OutgoingReportEvent::DmarcReport | OutgoingReportEvent::DmarcRateLimited | OutgoingReportEvent::DmarcAggregateReport | OutgoingReportEvent::TlsAggregate | OutgoingReportEvent::HttpSubmission | OutgoingReportEvent::UnauthorizedReportingAddress | OutgoingReportEvent::ReportingAddressValidationError | OutgoingReportEvent::NotFound | OutgoingReportEvent::SubmissionError | OutgoingReportEvent::NoRecipientsFound ) | EventType::Auth( AuthEvent::Success | AuthEvent::Failed | AuthEvent::TooManyAttempts | AuthEvent::Error ) | EventType::Sieve(_) | EventType::Milter(_) | EventType::MtaHook(_) | EventType::Security(_) ) }) } } struct RawSpan(Vec); struct Span(Vec>); impl Deserialize for Span { fn deserialize(bytes: &[u8]) -> trc::Result { deserialize_events(bytes).map(Self) } } impl Deserialize for RawSpan { fn deserialize(bytes: &[u8]) -> trc::Result { Ok(Self(bytes.to_vec())) } } pub fn build_span_document( span_id: u64, events: Vec>, index_fields: &AHashSet, ) -> IndexDocument { let mut document = IndexDocument::new(SearchIndex::Tracing).with_id(span_id); let mut keywords = HashSet::new(); for (idx, event) in events.into_iter().enumerate() { if idx == 0 && (index_fields.is_empty() || index_fields.contains(&TracingSearchField::EventType.into())) { document.index_unsigned(TracingSearchField::EventType, event.inner.typ.code()); } for (key, value) in event.keys { match (key, value) { (Key::QueueId, Value::UInt(queue_id)) => { if index_fields.is_empty() || index_fields.contains(&TracingSearchField::QueueId.into()) { document.index_unsigned(TracingSearchField::QueueId, queue_id); } } (Key::From | Key::To | Key::Domain | Key::Hostname, Value::String(address)) => { if index_fields.is_empty() || index_fields.contains(&TracingSearchField::Keywords.into()) { keywords.insert(address.to_string()); } } (Key::To, Value::Array(value)) => { if index_fields.is_empty() || index_fields.contains(&TracingSearchField::Keywords.into()) { for value in value { if let Value::String(address) = value { keywords.insert(address.to_string()); } } } } (Key::RemoteIp, Value::Ipv4(ip)) => { if index_fields.is_empty() || index_fields.contains(&TracingSearchField::Keywords.into()) { keywords.insert(ip.to_string()); } } (Key::RemoteIp, Value::Ipv6(ip)) => { if index_fields.is_empty() || index_fields.contains(&TracingSearchField::Keywords.into()) { keywords.insert(ip.to_string()); } } _ => {} } } } if !keywords.is_empty() { let mut keyword_str = String::new(); for keyword in keywords { if !keyword_str.is_empty() { keyword_str.push(' '); } keyword_str.push_str(&keyword); } document.index_keyword(TracingSearchField::Keywords, keyword_str); } document } ================================================ FILE: crates/common/src/telemetry/webhooks/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ sync::{ Arc, atomic::{AtomicBool, Ordering}, }, time::Instant, }; use crate::{LONG_1Y_SLUMBER, config::telemetry::WebhookTracer}; use base64::{Engine, engine::general_purpose::STANDARD}; use ring::hmac; use serde::Serialize; use store::write::now; use tokio::sync::mpsc; use trc::{ Event, EventDetails, ServerEvent, TelemetryEvent, ipc::subscriber::{EventBatch, SubscriberBuilder}, serializers::json::JsonEventSerializer, }; pub(crate) fn spawn_webhook_tracer(builder: SubscriberBuilder, settings: WebhookTracer) { let (tx, mut rx) = builder.register(); tokio::spawn(async move { let settings = Arc::new(settings); let mut wakeup_time = LONG_1Y_SLUMBER; let discard_after = settings.discard_after.as_secs(); let mut pending_events = Vec::new(); let mut next_delivery = Instant::now(); let in_flight = Arc::new(AtomicBool::new(false)); loop { // Wait for the next event or timeout let event_or_timeout = tokio::time::timeout(wakeup_time, rx.recv()).await; let now = now(); match event_or_timeout { Ok(Some(events)) => { let mut discard_count = 0; for event in events { if now.saturating_sub(event.inner.timestamp) < discard_after { pending_events.push(event) } else { discard_count += 1; } } if discard_count > 0 { trc::event!( Telemetry(TelemetryEvent::WebhookError), Details = "Discarded stale events", Total = discard_count ); } } Ok(None) => { break; } Err(_) => (), } // Process events let mut next_retry = None; let now = Instant::now(); if next_delivery <= now { if !pending_events.is_empty() { next_delivery = now + settings.throttle; if !in_flight.load(Ordering::Relaxed) { spawn_webhook_handler( settings.clone(), in_flight.clone(), std::mem::take(&mut pending_events), tx.clone(), ); } } } else if !pending_events.is_empty() { // Retry later let this_retry = next_delivery - now; match next_retry { Some(next_retry) if this_retry >= next_retry => {} _ => { next_retry = Some(this_retry); } } } wakeup_time = next_retry.unwrap_or(LONG_1Y_SLUMBER); } }); } #[derive(Serialize)] struct EventWrapper { events: JsonEventSerializer>>>, } fn spawn_webhook_handler( settings: Arc, in_flight: Arc, events: EventBatch, webhook_tx: mpsc::Sender, ) { tokio::spawn(async move { in_flight.store(true, Ordering::Relaxed); let wrapper = EventWrapper { events: JsonEventSerializer::new(events).with_id().with_spans(), }; if let Err(err) = post_webhook_events(&settings, &wrapper).await { trc::event!(Telemetry(TelemetryEvent::WebhookError), Details = err); if webhook_tx.send(wrapper.events.into_inner()).await.is_err() { trc::event!( Server(ServerEvent::ThreadError), Details = "Failed to send failed webhook events back to main thread", CausedBy = trc::location!() ); } } in_flight.store(false, Ordering::Relaxed); }); } async fn post_webhook_events( settings: &WebhookTracer, events: &EventWrapper, ) -> Result<(), String> { // Serialize body let body = serde_json::to_string(events) .map_err(|err| format!("Failed to serialize events: {}", err))?; // Add HMAC-SHA256 signature let mut headers = settings.headers.clone(); if !settings.key.is_empty() { let key = hmac::Key::new(hmac::HMAC_SHA256, settings.key.as_bytes()); let tag = hmac::sign(&key, body.as_bytes()); headers.insert( "X-Signature", STANDARD.encode(tag.as_ref()).parse().unwrap(), ); } // Send request let response = reqwest::Client::builder() .timeout(settings.timeout) .danger_accept_invalid_certs(settings.tls_allow_invalid_certs) .build() .map_err(|err| format!("Failed to create HTTP client: {}", err))? .post(&settings.url) .headers(headers) .body(body) .send() .await .map_err(|err| format!("Webhook request to {} failed: {err}", settings.url))?; if response.status().is_success() { Ok(()) } else { Err(format!( "Webhook request to {} failed with code {}: {}", settings.url, response.status().as_u16(), response.status().canonical_reason().unwrap_or("Unknown") )) } } ================================================ FILE: crates/dav/Cargo.toml ================================================ [package] name = "dav" version = "0.15.5" edition = "2024" [dependencies] dav-proto = { path = "../dav-proto" } common = { path = "../common" } store = { path = "../store" } utils = { path = "../utils" } groupware = { path = "../groupware" } directory = { path = "../directory" } http_proto = { path = "../http-proto" } types = { path = "../types" } trc = { path = "../trc" } calcard = { version = "0.3", features = ["rkyv"] } hashify = { version = "0.2" } hyper = { version = "1.0.1", features = ["server", "http1", "http2"] } percent-encoding = "2.3.1" rkyv = { version = "0.8.10", features = ["little_endian"] } compact_str = "0.9.0" chrono = "0.4.40" [dev-dependencies] [features] test_mode = [] enterprise = [] ================================================ FILE: crates/dav/src/calendar/copy_move.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::assert_is_unique_uid; use crate::{ DavError, DavMethod, common::{ lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, file::DavFileResource, }; use calcard::common::timezone::Tz; use common::{DavName, Server, auth::AccessToken}; use dav_proto::{Depth, RequestHeaders}; use groupware::{ DestroyArchive, cache::GroupwareCache, calendar::{Calendar, CalendarEvent, CalendarPreferences, Timezone}, }; use http_proto::HttpResponse; use hyper::StatusCode; use store::write::{BatchBuilder, now}; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection, VanishedCollection}, }; pub(crate) trait CalendarCopyMoveRequestHandler: Sync + Send { fn handle_calendar_copy_move_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, is_move: bool, ) -> impl Future> + Send; } impl CalendarCopyMoveRequestHandler for Server { async fn handle_calendar_copy_move_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, is_move: bool, ) -> crate::Result { // Validate source let from_resource_ = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let from_account_id = from_resource_.account_id; let from_resources = self .fetch_dav_resources(access_token, from_account_id, SyncCollection::Calendar) .await .caused_by(trc::location!())?; let from_resource_name = from_resource_ .resource .ok_or(DavError::Code(StatusCode::FORBIDDEN))?; let from_resource = from_resources .by_path(from_resource_name) .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; #[cfg(not(debug_assertions))] if is_move && from_resource.is_container() && self .core .groupware .default_calendar_name .as_ref() .is_some_and(|name| name == from_resource_name) { return Err(DavError::Condition(crate::DavErrorCondition::new( StatusCode::FORBIDDEN, dav_proto::schema::response::CalCondition::DefaultCalendarNeeded, ))); } // Validate ACL if !access_token.is_member(from_account_id) && !from_resources.has_access_to_container( access_token, if from_resource.is_container() { from_resource.document_id() } else { from_resource.parent_id().unwrap() }, Acl::ReadItems, ) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Validate destination let destination = self .validate_uri_with_status( access_token, headers .destination .ok_or(DavError::Code(StatusCode::BAD_GATEWAY))?, StatusCode::BAD_GATEWAY, ) .await?; if destination.collection != Collection::Calendar { return Err(DavError::Code(StatusCode::BAD_GATEWAY)); } let to_account_id = destination .account_id .ok_or(DavError::Code(StatusCode::BAD_GATEWAY))?; let to_resources = if to_account_id == from_account_id { from_resources.clone() } else { self.fetch_dav_resources(access_token, to_account_id, SyncCollection::Calendar) .await .caused_by(trc::location!())? }; // Validate headers let destination_resource_name = destination .resource .ok_or(DavError::Code(StatusCode::BAD_GATEWAY))?; let to_resource = to_resources.by_path(destination_resource_name); self.validate_headers( access_token, headers, vec![ ResourceState { account_id: from_account_id, collection: if from_resource.is_container() { Collection::Calendar } else { Collection::CalendarEvent }, document_id: Some(from_resource.document_id()), path: from_resource_name, ..Default::default() }, ResourceState { account_id: to_account_id, collection: to_resource .map(|r| { if r.is_container() { Collection::Calendar } else { Collection::CalendarEvent } }) .unwrap_or(Collection::Calendar), document_id: Some(to_resource.map(|r| r.document_id()).unwrap_or(u32::MAX)), path: destination_resource_name, ..Default::default() }, ], Default::default(), if is_move { DavMethod::MOVE } else { DavMethod::COPY }, ) .await?; // Map destination if let Some(to_resource) = to_resource { if from_resource.path() == to_resource.path() { // Same resource return Err(DavError::Code(StatusCode::BAD_GATEWAY)); } let new_name = destination_resource_name .rsplit_once('/') .map(|(_, name)| name) .unwrap_or(destination_resource_name); match (from_resource.is_container(), to_resource.is_container()) { (true, true) => { let from_children_ids = from_resources .subtree(from_resource_name) .filter(|r| !r.is_container()) .map(|r| r.document_id()) .collect::>(); let to_document_ids = to_resources .subtree(destination_resource_name) .filter(|r| !r.is_container()) .map(|r| r.document_id()) .collect::>(); // Validate ACLs if !access_token.is_member(to_account_id) || (!access_token.is_member(from_account_id) && !from_resources.has_access_to_container( access_token, from_resource.document_id(), if is_move { Acl::RemoveItems } else { Acl::ReadItems }, )) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Overwrite container copy_container( self, access_token, from_account_id, from_resource.document_id(), from_children_ids, from_resources.format_collection(from_resource_name), to_account_id, to_resource.document_id().into(), to_document_ids, new_name, is_move, ) .await } (false, false) => { // Overwrite event let from_calendar_id = from_resource.parent_id().unwrap(); let to_calendar_id = to_resource.parent_id().unwrap(); // Validate ACL if (!access_token.is_member(from_account_id) && !from_resources.has_access_to_container( access_token, from_calendar_id, if is_move { Acl::RemoveItems } else { Acl::ReadItems }, )) || (!access_token.is_member(to_account_id) && !to_resources.has_access_to_container( access_token, to_calendar_id, Acl::RemoveItems, )) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } if is_move { move_event( self, access_token, from_account_id, from_resource.document_id(), from_calendar_id, from_resources.format_item(from_resource_name), to_account_id, to_resource.document_id().into(), to_calendar_id, new_name, headers.if_schedule_tag, ) .await } else { copy_event( self, access_token, from_account_id, from_resource.document_id(), to_account_id, to_resource.document_id().into(), to_calendar_id, new_name, ) .await } } _ => Err(DavError::Code(StatusCode::BAD_GATEWAY)), } } else if let Some((parent_resource, new_name)) = to_resources.map_parent(destination_resource_name) { if let Some(parent_resource) = parent_resource { // Creating items under an event is not allowed // Copying/moving containers under a container is not allowed if !parent_resource.is_container() || from_resource.is_container() { return Err(DavError::Code(StatusCode::BAD_GATEWAY)); } // Validate ACL let from_calendar_id = from_resource.parent_id().unwrap(); let to_calendar_id = parent_resource.document_id(); if (!access_token.is_member(from_account_id) && !from_resources.has_access_to_container( access_token, from_calendar_id, if is_move { Acl::RemoveItems } else { Acl::ReadItems }, )) || (!access_token.is_member(to_account_id) && !to_resources.has_access_to_container( access_token, to_calendar_id, Acl::AddItems, )) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Copy/move event if is_move { if from_account_id != to_account_id || parent_resource.document_id() != from_calendar_id { move_event( self, access_token, from_account_id, from_resource.document_id(), from_calendar_id, from_resources.format_item(from_resource_name), to_account_id, None, to_calendar_id, new_name, headers.if_schedule_tag, ) .await } else { rename_event( self, access_token, from_account_id, from_resource.document_id(), from_calendar_id, new_name, from_resources.format_item(from_resource_name), ) .await } } else { copy_event( self, access_token, from_account_id, from_resource.document_id(), to_account_id, None, to_calendar_id, new_name, ) .await } } else { // Copying/moving events to the root is not allowed if !from_resource.is_container() { return Err(DavError::Code(StatusCode::BAD_GATEWAY)); } // Shared users cannot create containers if !access_token.is_member(to_account_id) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Validate ACLs if !access_token.is_member(from_account_id) && !from_resources.has_access_to_container( access_token, from_resource.document_id(), if is_move { Acl::RemoveItems } else { Acl::ReadItems }, ) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Copy/move container let from_children_ids = from_resources .subtree(from_resource_name) .filter(|r| !r.is_container()) .map(|r| r.document_id()) .collect::>(); if is_move { if from_account_id != to_account_id { copy_container( self, access_token, from_account_id, from_resource.document_id(), if headers.depth != Depth::Zero { from_children_ids } else { return Err(DavError::Code(StatusCode::BAD_GATEWAY)); }, from_resources.format_collection(from_resource_name), to_account_id, None, vec![], new_name, true, ) .await } else { rename_container( self, access_token, from_account_id, from_resource.document_id(), new_name, from_resources.format_collection(from_resource_name), ) .await } } else { copy_container( self, access_token, from_account_id, from_resource.document_id(), if headers.depth != Depth::Zero { from_children_ids } else { vec![] }, from_resources.format_collection(from_resource_name), to_account_id, None, vec![], new_name, false, ) .await } } } else { Err(DavError::Code(StatusCode::CONFLICT)) } } } #[allow(clippy::too_many_arguments)] async fn copy_event( server: &Server, access_token: &AccessToken, from_account_id: u32, from_document_id: u32, to_account_id: u32, to_document_id: Option, to_calendar_id: u32, new_name: &str, ) -> crate::Result { // Fetch event let event_ = server .store() .get_value::>(ValueKey::archive( from_account_id, Collection::CalendarEvent, from_document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let event = event_ .to_unarchived::() .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); // Validate UID assert_is_unique_uid( server, server .fetch_dav_resources(access_token, to_account_id, SyncCollection::Calendar) .await .caused_by(trc::location!())? .as_ref(), to_account_id, to_calendar_id, event.inner.data.event.uids().next(), ) .await?; if from_account_id == to_account_id { let mut new_event = event .deserialize::() .caused_by(trc::location!())?; new_event.names.push(DavName { name: new_name.to_string(), parent_id: to_calendar_id, }); new_event .update( access_token, event, from_account_id, from_document_id, &mut batch, ) .caused_by(trc::location!())?; } else { let next_email_alarm = event.inner.data.next_alarm(now() as i64, Tz::Floating); let mut new_event = event .deserialize::() .caused_by(trc::location!())?; new_event.names = vec![DavName { name: new_name.to_string(), parent_id: to_calendar_id, }]; let to_document_id = server .store() .assign_document_ids(to_account_id, Collection::CalendarEvent, 1) .await .caused_by(trc::location!())?; new_event .insert( access_token, to_account_id, to_document_id, next_email_alarm, &mut batch, ) .caused_by(trc::location!())?; } let response = if let Some(to_document_id) = to_document_id { // Overwrite event on destination let event_ = server .store() .get_value::>(ValueKey::archive( to_account_id, Collection::CalendarEvent, to_document_id, )) .await .caused_by(trc::location!())?; if let Some(event_) = event_ { let event = event_ .to_unarchived::() .caused_by(trc::location!())?; DestroyArchive(event) .delete( access_token, to_account_id, to_document_id, to_calendar_id, None, false, &mut batch, ) .caused_by(trc::location!())?; } Ok(HttpResponse::new(StatusCode::NO_CONTENT)) } else { Ok(HttpResponse::new(StatusCode::CREATED)) }; server .commit_batch(batch) .await .caused_by(trc::location!())?; server.notify_task_queue(); response } #[allow(clippy::too_many_arguments)] async fn move_event( server: &Server, access_token: &AccessToken, from_account_id: u32, from_document_id: u32, from_calendar_id: u32, from_resource_path: String, to_account_id: u32, to_document_id: Option, to_calendar_id: u32, new_name: &str, if_schedule_tag: Option, ) -> crate::Result { // Fetch event let event_ = server .store() .get_value::>(ValueKey::archive( from_account_id, Collection::CalendarEvent, from_document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let event = event_ .to_unarchived::() .caused_by(trc::location!())?; // Validate headers if if_schedule_tag.is_some() && event.inner.schedule_tag.as_ref().map(|t| t.to_native()) != if_schedule_tag { return Err(DavError::Code(StatusCode::PRECONDITION_FAILED)); } // Validate UID if from_account_id != to_account_id || from_calendar_id != to_calendar_id || to_document_id.is_none() { assert_is_unique_uid( server, server .fetch_dav_resources(access_token, to_account_id, SyncCollection::Calendar) .await .caused_by(trc::location!())? .as_ref(), to_account_id, to_calendar_id, event.inner.data.event.uids().next(), ) .await?; } let mut batch = BatchBuilder::new(); if from_account_id == to_account_id { let mut name_idx = None; for (idx, name) in event.inner.names.iter().enumerate() { if name.parent_id == from_calendar_id { name_idx = Some(idx); break; } } let name_idx = if let Some(name_idx) = name_idx { name_idx } else { return Err(DavError::Code(StatusCode::NOT_FOUND)); }; let mut new_event = event .deserialize::() .caused_by(trc::location!())?; new_event.names.swap_remove(name_idx); new_event.names.push(DavName { name: new_name.to_string(), parent_id: to_calendar_id, }); new_event .update( access_token, event.clone(), from_account_id, from_document_id, &mut batch, ) .caused_by(trc::location!())?; batch.log_vanished_item(VanishedCollection::Calendar, from_resource_path); } else { let next_email_alarm = event.inner.data.next_alarm(now() as i64, Tz::Floating); let mut new_event = event .deserialize::() .caused_by(trc::location!())?; new_event.names = vec![DavName { name: new_name.to_string(), parent_id: to_calendar_id, }]; DestroyArchive(event) .delete( access_token, from_account_id, from_document_id, from_calendar_id, from_resource_path.into(), false, &mut batch, ) .caused_by(trc::location!())?; let to_document_id = server .store() .assign_document_ids(to_account_id, Collection::CalendarEvent, 1) .await .caused_by(trc::location!())?; new_event .insert( access_token, to_account_id, to_document_id, next_email_alarm, &mut batch, ) .caused_by(trc::location!())?; } let response = if let Some(to_document_id) = to_document_id { // Overwrite event on destination let event_ = server .store() .get_value::>(ValueKey::archive( to_account_id, Collection::CalendarEvent, to_document_id, )) .await .caused_by(trc::location!())?; if let Some(event_) = event_ { let event = event_ .to_unarchived::() .caused_by(trc::location!())?; DestroyArchive(event) .delete( access_token, to_account_id, to_document_id, to_calendar_id, None, false, &mut batch, ) .caused_by(trc::location!())?; } Ok(HttpResponse::new(StatusCode::NO_CONTENT)) } else { Ok(HttpResponse::new(StatusCode::CREATED)) }; server .commit_batch(batch) .await .caused_by(trc::location!())?; server.notify_task_queue(); response } #[allow(clippy::too_many_arguments)] async fn rename_event( server: &Server, access_token: &AccessToken, account_id: u32, document_id: u32, calendar_id: u32, new_name: &str, from_resource_path: String, ) -> crate::Result { // Fetch event let event_ = server .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let event = event_ .to_unarchived::() .caused_by(trc::location!())?; let name_idx = event .inner .names .iter() .position(|n| n.parent_id == calendar_id) .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let mut new_event = event .deserialize::() .caused_by(trc::location!())?; new_event.names[name_idx].name = new_name.to_string(); let mut batch = BatchBuilder::new(); new_event .update(access_token, event, account_id, document_id, &mut batch) .caused_by(trc::location!())?; batch.log_vanished_item(VanishedCollection::Calendar, from_resource_path); server .commit_batch(batch) .await .caused_by(trc::location!())?; server.notify_task_queue(); Ok(HttpResponse::new(StatusCode::CREATED)) } #[allow(clippy::too_many_arguments)] async fn copy_container( server: &Server, access_token: &AccessToken, from_account_id: u32, from_document_id: u32, from_children_ids: Vec, from_resource_path: String, to_account_id: u32, to_document_id: Option, to_children_ids: Vec, new_name: &str, remove_source: bool, ) -> crate::Result { // Fetch calendar let calendar_ = server .store() .get_value::>(ValueKey::archive( from_account_id, Collection::Calendar, from_document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let old_calendar = calendar_ .to_unarchived::() .caused_by(trc::location!())?; let mut calendar = old_calendar .deserialize::() .caused_by(trc::location!())?; // Prepare write batch let mut batch = BatchBuilder::new(); if remove_source { DestroyArchive(old_calendar) .delete( access_token, from_account_id, from_document_id, from_resource_path.into(), &mut batch, ) .caused_by(trc::location!())?; } let preference = calendar.preferences.into_iter().next().unwrap(); calendar.name = new_name.to_string(); calendar.acls.clear(); calendar.preferences = vec![CalendarPreferences { account_id: to_account_id, name: preference.name, description: preference.description, default_alerts: preference.default_alerts, sort_order: 0, color: preference.color, flags: 0, time_zone: Timezone::Default, }]; let is_overwrite = to_document_id.is_some(); let to_document_id = if let Some(to_document_id) = to_document_id { // Overwrite destination let calendar_ = server .store() .get_value::>(ValueKey::archive( to_account_id, Collection::Calendar, to_document_id, )) .await .caused_by(trc::location!())?; if let Some(calendar_) = calendar_ { let calendar = calendar_ .to_unarchived::() .caused_by(trc::location!())?; DestroyArchive(calendar) .delete_with_events( server, access_token, to_account_id, to_document_id, to_children_ids, None, false, &mut batch, ) .await .caused_by(trc::location!())?; } to_document_id } else { server .store() .assign_document_ids(to_account_id, Collection::Calendar, 1) .await .caused_by(trc::location!())? }; calendar .insert(access_token, to_account_id, to_document_id, &mut batch) .caused_by(trc::location!())?; // Copy children let mut required_space = 0; for from_child_document_id in from_children_ids { if let Some(event_) = server .store() .get_value::>(ValueKey::archive( from_account_id, Collection::CalendarEvent, from_child_document_id, )) .await? { let event = event_ .to_unarchived::() .caused_by(trc::location!())?; let mut new_name = None; for name in event.inner.names.iter() { if name.parent_id == to_document_id { continue; } else if name.parent_id == from_document_id { new_name = Some(name.name.to_string()); } } let new_name = if let Some(new_name) = new_name { DavName { name: new_name, parent_id: to_document_id, } } else { continue; }; let event = event_ .to_unarchived::() .caused_by(trc::location!())?; let mut new_event = event .deserialize::() .caused_by(trc::location!())?; if from_account_id == to_account_id { if remove_source { new_event .names .retain(|name| name.parent_id != from_document_id); } new_event.names.push(new_name); new_event .update( access_token, event, from_account_id, from_child_document_id, &mut batch, ) .caused_by(trc::location!())?; } else { let next_email_alarm = event.inner.data.next_alarm(now() as i64, Tz::Floating); if remove_source { DestroyArchive(event) .delete( access_token, from_account_id, from_child_document_id, from_document_id, None, false, &mut batch, ) .caused_by(trc::location!())?; } let to_document_id = server .store() .assign_document_ids(to_account_id, Collection::CalendarEvent, 1) .await .caused_by(trc::location!())?; new_event.names = vec![new_name]; required_space += new_event.size as u64; new_event .insert( access_token, to_account_id, to_document_id, next_email_alarm, &mut batch, ) .caused_by(trc::location!())?; } } } if from_account_id != to_account_id && required_space > 0 { server .has_available_quota( &server .get_resource_token(access_token, to_account_id) .await?, required_space, ) .await?; } server .commit_batch(batch) .await .caused_by(trc::location!())?; server.notify_task_queue(); if !is_overwrite { Ok(HttpResponse::new(StatusCode::CREATED)) } else { Ok(HttpResponse::new(StatusCode::NO_CONTENT)) } } #[allow(clippy::too_many_arguments)] async fn rename_container( server: &Server, access_token: &AccessToken, account_id: u32, document_id: u32, new_name: &str, from_resource_path: String, ) -> crate::Result { // Fetch calendar let calendar_ = server .store() .get_value::>(ValueKey::archive( account_id, Collection::Calendar, document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let calendar = calendar_ .to_unarchived::() .caused_by(trc::location!())?; let mut new_calendar = calendar .deserialize::() .caused_by(trc::location!())?; new_calendar.name = new_name.to_string(); let mut batch = BatchBuilder::new(); new_calendar .update(access_token, calendar, account_id, document_id, &mut batch) .caused_by(trc::location!())?; batch.log_vanished_item(VanishedCollection::Calendar, from_resource_path); server .commit_batch(batch) .await .caused_by(trc::location!())?; server.notify_task_queue(); Ok(HttpResponse::new(StatusCode::CREATED)) } ================================================ FILE: crates/dav/src/calendar/delete.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ DavError, DavMethod, common::{ ETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, }; use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; use dav_proto::RequestHeaders; use directory::Permission; use groupware::{ DestroyArchive, cache::GroupwareCache, calendar::{Calendar, CalendarEvent}, }; use http_proto::HttpResponse; use hyper::StatusCode; use store::write::{BatchBuilder, ValueClass}; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, field::PrincipalField, }; pub(crate) trait CalendarDeleteRequestHandler: Sync + Send { fn handle_calendar_delete_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, ) -> impl Future> + Send; } impl CalendarDeleteRequestHandler for Server { async fn handle_calendar_delete_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, ) -> crate::Result { // Validate URI let resource = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let account_id = resource.account_id; let delete_path = resource .resource .filter(|r| !r.is_empty()) .ok_or(DavError::Code(StatusCode::FORBIDDEN))?; let resources = self .fetch_dav_resources(access_token, account_id, SyncCollection::Calendar) .await .caused_by(trc::location!())?; // Check resource type let delete_resource = resources .by_path(delete_path) .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let document_id = delete_resource.document_id(); let send_itip = self.core.groupware.itip_enabled && !headers.no_schedule_reply && !access_token.emails.is_empty() && access_token.has_permission(Permission::CalendarSchedulingSend); // Fetch entry let mut batch = BatchBuilder::new(); if delete_resource.is_container() { // Deleting the default calendar is not allowed #[cfg(not(debug_assertions))] if self .core .groupware .default_calendar_name .as_ref() .is_some_and(|name| name == delete_path) { return Err(DavError::Condition(crate::DavErrorCondition::new( StatusCode::FORBIDDEN, dav_proto::schema::response::CalCondition::DefaultCalendarNeeded, ))); } let calendar_ = self .store() .get_value::>(ValueKey::archive( account_id, Collection::Calendar, document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let calendar = calendar_ .to_unarchived::() .caused_by(trc::location!())?; // Validate ACL if !access_token.is_member(account_id) && !calendar .inner .acls .effective_acl(access_token) .contains_all([Acl::Delete, Acl::RemoveItems].into_iter()) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Validate headers self.validate_headers( access_token, headers, vec![ResourceState { account_id, collection: Collection::Calendar, document_id: document_id.into(), etag: calendar.etag().into(), path: delete_path, ..Default::default() }], Default::default(), DavMethod::DELETE, ) .await?; // Delete calendar and events DestroyArchive(calendar) .delete_with_events( self, access_token, account_id, document_id, resources .subtree(delete_path) .filter(|r| !r.is_container()) .map(|r| r.document_id()) .collect::>(), resources.format_resource(delete_resource).into(), send_itip, &mut batch, ) .await .caused_by(trc::location!())?; // Reset default calendar id let default_calendar_id = self .store() .get_value::(ValueKey { account_id, collection: Collection::Principal.into(), document_id: 0, class: ValueClass::Property(PrincipalField::DefaultCalendarId.into()), }) .await .caused_by(trc::location!())?; if default_calendar_id.is_some_and(|id| id == document_id) { batch .with_account_id(account_id) .with_collection(Collection::Principal) .with_document(0) .clear(PrincipalField::DefaultCalendarId); } } else { // Validate ACL let calendar_id = delete_resource.parent_id().unwrap(); if !access_token.is_member(account_id) && !resources.has_access_to_container(access_token, calendar_id, Acl::RemoveItems) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } let event_ = self .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; // Validate headers self.validate_headers( access_token, headers, vec![ResourceState { account_id, collection: Collection::CalendarEvent, document_id: document_id.into(), etag: event_.etag().into(), path: delete_path, ..Default::default() }], Default::default(), DavMethod::DELETE, ) .await?; // Validate schedule tag let event = event_ .to_unarchived::() .caused_by(trc::location!())?; if headers.if_schedule_tag.is_some() && event.inner.schedule_tag.as_ref().map(|t| t.to_native()) != headers.if_schedule_tag { return Err(DavError::Code(StatusCode::PRECONDITION_FAILED)); } // Delete event DestroyArchive(event) .delete( access_token, account_id, document_id, calendar_id, resources.format_resource(delete_resource).into(), send_itip, &mut batch, ) .caused_by(trc::location!())?; } self.commit_batch(batch).await.caused_by(trc::location!())?; self.notify_task_queue(); Ok(HttpResponse::new(StatusCode::NO_CONTENT)) } } ================================================ FILE: crates/dav/src/calendar/freebusy.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::query::CalendarQueryHandler; use crate::{DavError, calendar::query::is_resource_in_time_range, common::uri::DavUriResource}; use calcard::{ common::{PartialDateTime, timezone::Tz}, icalendar::{ ArchivedICalendarComponentType, ArchivedICalendarEntry, ArchivedICalendarParameterName, ArchivedICalendarParameterValue, ArchivedICalendarProperty, ArchivedICalendarStatus, ArchivedICalendarValue, ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarEntry, ICalendarFreeBusyType, ICalendarParameter, ICalendarPeriod, ICalendarProperty, ICalendarTransparency, ICalendarValue, }, }; use common::{DavResourcePath, DavResources, PROD_ID, Server, auth::AccessToken}; use dav_proto::{RequestHeaders, schema::request::FreeBusyQuery}; use groupware::{cache::GroupwareCache, calendar::CalendarEvent}; use http_proto::HttpResponse; use hyper::StatusCode; use std::str::FromStr; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use store::{ ahash::AHashMap, write::{now, serialize::rkyv_deserialize}, }; use trc::AddContext; use types::{ TimeRange, acl::Acl, collection::{Collection, SyncCollection}, }; pub(crate) trait CalendarFreebusyRequestHandler: Sync + Send { fn handle_calendar_freebusy_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: FreeBusyQuery, ) -> impl Future> + Send; fn build_freebusy_object( &self, access_token: &AccessToken, request: FreeBusyQuery, resources: &DavResources, account_id: u32, resource: DavResourcePath<'_>, ) -> impl Future> + Send; } impl CalendarFreebusyRequestHandler for Server { async fn handle_calendar_freebusy_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: FreeBusyQuery, ) -> crate::Result { // Validate URI let resource_ = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let account_id = resource_.account_id; let resources = self .fetch_dav_resources(access_token, account_id, SyncCollection::Calendar) .await .caused_by(trc::location!())?; let resource = resources .by_path( resource_ .resource .ok_or(DavError::Code(StatusCode::METHOD_NOT_ALLOWED))?, ) .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; if !resource.is_container() { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } self.build_freebusy_object(access_token, request, &resources, account_id, resource) .await .map(|ical| { HttpResponse::new(StatusCode::OK) .with_content_type("text/calendar; charset=utf-8") .with_text_body(ical.to_string()) }) } async fn build_freebusy_object( &self, access_token: &AccessToken, request: FreeBusyQuery, resources: &DavResources, account_id: u32, resource: DavResourcePath<'_>, ) -> crate::Result { // Obtain shared ids let shared_ids = if !access_token.is_member(account_id) { resources .shared_containers( access_token, [Acl::ReadItems, Acl::SchedulingReadFreeBusy], false, ) .into() } else { None }; // Build FreeBusy component let default_tz = resource .resource .calendar_preferences(account_id) .map(|p| p.tz) .unwrap_or(Tz::UTC); let mut entries = Vec::with_capacity(6); if let Some(range) = request.range { entries.push(ICalendarEntry { name: ICalendarProperty::Dtstart, params: vec![], values: vec![ICalendarValue::PartialDateTime(Box::new( PartialDateTime::from_utc_timestamp(range.start), ))], }); entries.push(ICalendarEntry { name: ICalendarProperty::Dtend, params: vec![], values: vec![ICalendarValue::PartialDateTime(Box::new( PartialDateTime::from_utc_timestamp(range.end), ))], }); entries.push(ICalendarEntry { name: ICalendarProperty::Dtstamp, params: vec![], values: vec![ICalendarValue::PartialDateTime(Box::new( PartialDateTime::from_utc_timestamp(now() as i64), ))], }); let document_ids = resources .children(resource.document_id()) .filter(|resource| { shared_ids .as_ref() .is_none_or(|ids| ids.contains(resource.document_id())) && is_resource_in_time_range(resource.resource, &range) }) .map(|resource| resource.document_id()) .collect::>(); let mut fb_entries: AHashMap> = AHashMap::with_capacity(document_ids.len()); for document_id in document_ids { let Some(archive) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, )) .await .caused_by(trc::location!())? else { continue; }; let event = archive .unarchive::() .caused_by(trc::location!())?; /* Only VEVENT components without a TRANSP property or with the TRANSP property set to OPAQUE, and VFREEBUSY components SHOULD be considered in generating the free busy time information. */ let mut components = event .data .event .components .iter() .enumerate() .filter(|(_, comp)| { (matches!(comp.component_type, ArchivedICalendarComponentType::VEvent) && comp .transparency() .is_none_or(|t| t == &ICalendarTransparency::Opaque)) || matches!( comp.component_type, ArchivedICalendarComponentType::VFreebusy ) }) .peekable(); if components.peek().is_none() { continue; } let events = CalendarQueryHandler::new(event, Some(range), default_tz).into_expanded_times(); if events.is_empty() { continue; } for (component_id, component) in components { let component_id = component_id as u32; match component.component_type { ArchivedICalendarComponentType::VEvent => { let fbtype = match component.status() { Some(ArchivedICalendarStatus::Cancelled) => continue, Some(ArchivedICalendarStatus::Tentative) => { ICalendarFreeBusyType::BusyTentative } _ => ICalendarFreeBusyType::Busy, }; let mut events_in_range = Vec::new(); for event in &events { if event.comp_id == component_id && range.is_in_range(false, event.start, event.end) { events_in_range.push((event.start, event.end)); } } if !events_in_range.is_empty() { fb_entries .entry(fbtype) .or_default() .extend(events_in_range); } } ArchivedICalendarComponentType::VFreebusy => { for entry in component.entries.iter() { if matches!(entry.name, ArchivedICalendarProperty::Freebusy) { let mut fb_in_range = freebusy_in_range_utc(entry, &range, default_tz).peekable(); if fb_in_range.peek().is_some() { let fb_type = entry .params .iter() .find_map(|param| { if let ( ArchivedICalendarParameterName::Fbtype, ArchivedICalendarParameterValue::Fbtype(param), ) = (¶m.name, ¶m.value) { rkyv_deserialize(param).ok() } else { None } }) .unwrap_or(ICalendarFreeBusyType::Busy); fb_entries.entry(fb_type).or_default().extend(fb_in_range); } } } } _ => {} } } } for (fbtype, events_in_range) in fb_entries { entries.push(ICalendarEntry { name: ICalendarProperty::Freebusy, params: vec![ICalendarParameter::fbtype(fbtype)], values: merge_intervals(events_in_range), }); } } // Build ICalendar Ok(ICalendar { components: vec![ ICalendarComponent { component_type: ICalendarComponentType::VCalendar, entries: vec![ ICalendarEntry { name: ICalendarProperty::Version, params: vec![], values: vec![ICalendarValue::Text("2.0".to_string())], }, ICalendarEntry { name: ICalendarProperty::Prodid, params: vec![], values: vec![ICalendarValue::Text(PROD_ID.to_string())], }, ], component_ids: vec![1], }, ICalendarComponent { component_type: ICalendarComponentType::VFreebusy, entries, component_ids: vec![], }, ], }) } } fn merge_intervals(mut intervals: Vec<(i64, i64)>) -> Vec { if intervals.len() > 1 { intervals.sort_unstable_by(|a, b| a.0.cmp(&b.0)); let mut unique_intervals = Vec::new(); let mut start_time = intervals[0].0; let mut end_time = intervals[0].1; for &(curr_start, curr_end) in intervals.iter().skip(1) { if curr_start <= end_time { end_time = end_time.max(curr_end); } else { unique_intervals.push(build_ical_value(start_time, end_time)); start_time = curr_start; end_time = curr_end; } } unique_intervals.push(build_ical_value(start_time, end_time)); unique_intervals } else { intervals .into_iter() .map(|(start, end)| build_ical_value(start, end)) .collect() } } fn build_ical_value(from: i64, to: i64) -> ICalendarValue { ICalendarValue::Period(ICalendarPeriod::Range { start: PartialDateTime::from_utc_timestamp(from), end: PartialDateTime::from_utc_timestamp(to), }) } pub(crate) fn freebusy_in_range( entry: &ArchivedICalendarEntry, range: &TimeRange, default_tz: Tz, ) -> impl Iterator { let tz = entry .tz_id() .and_then(|tz_id| Tz::from_str(tz_id).ok()) .unwrap_or(default_tz); entry.values.iter().filter_map(move |value| { if let ArchivedICalendarValue::Period(period) = &value { period.time_range(tz).and_then(|(start, end)| { let start = start.timestamp(); let end = end.timestamp(); if range.is_in_range(false, start, end) { rkyv_deserialize(value).ok() } else { None } }) } else { None } }) } fn freebusy_in_range_utc( entry: &ArchivedICalendarEntry, range: &TimeRange, default_tz: Tz, ) -> impl Iterator { let tz = entry .tz_id() .and_then(|tz_id| Tz::from_str(tz_id).ok()) .unwrap_or(default_tz); entry.values.iter().filter_map(move |value| { if let ArchivedICalendarValue::Period(period) = &value { period.time_range(tz).and_then(|(start, end)| { let start = start.timestamp(); let end = end.timestamp(); if range.is_in_range(false, start, end) { Some((start, end)) } else { None } }) } else { None } }) } ================================================ FILE: crates/dav/src/calendar/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ DavError, DavMethod, common::{ ETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, }; use common::{Server, auth::AccessToken}; use dav_proto::{RequestHeaders, schema::property::Rfc1123DateTime}; use groupware::{cache::GroupwareCache, calendar::CalendarEvent}; use http_proto::HttpResponse; use hyper::StatusCode; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, }; pub(crate) trait CalendarGetRequestHandler: Sync + Send { fn handle_calendar_get_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, is_head: bool, ) -> impl Future> + Send; } impl CalendarGetRequestHandler for Server { async fn handle_calendar_get_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, is_head: bool, ) -> crate::Result { // Validate URI let resource_ = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let account_id = resource_.account_id; let resources = self .fetch_dav_resources(access_token, account_id, SyncCollection::Calendar) .await .caused_by(trc::location!())?; let resource = resources .by_path( resource_ .resource .ok_or(DavError::Code(StatusCode::METHOD_NOT_ALLOWED))?, ) .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; if resource.is_container() { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } // Validate ACL if !access_token.is_member(account_id) && !resources.has_access_to_container( access_token, resource.parent_id().unwrap(), Acl::ReadItems, ) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Fetch event let event_ = self .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, resource.document_id(), )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let event = event_ .unarchive::() .caused_by(trc::location!())?; // Validate headers let etag = event_.etag(); let schedule_tag = event.schedule_tag.as_ref().map(|tag| tag.to_native()); self.validate_headers( access_token, headers, vec![ResourceState { account_id, collection: Collection::CalendarEvent, document_id: resource.document_id().into(), etag: etag.clone().into(), path: resource_.resource.unwrap(), ..Default::default() }], Default::default(), DavMethod::GET, ) .await?; let response = HttpResponse::new(StatusCode::OK) .with_content_type("text/calendar; charset=utf-8") .with_etag(etag) .with_schedule_tag_opt(schedule_tag) .with_last_modified(Rfc1123DateTime::new(i64::from(event.modified)).to_string()); let ical = event.data.event.to_string(); if !is_head { Ok(response.with_binary_body(ical)) } else { Ok(response.with_content_length(ical.len())) } } } ================================================ FILE: crates/dav/src/calendar/mkcol.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::proppatch::CalendarPropPatchRequestHandler; use crate::{ DavError, DavMethod, PropStatBuilder, common::{ ExtractETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, }; use common::{Server, auth::AccessToken}; use dav_proto::{ RequestHeaders, Return, schema::{Namespace, request::MkCol, response::MkColResponse}, }; use groupware::{ cache::GroupwareCache, calendar::{Calendar, CalendarPreferences}, }; use http_proto::HttpResponse; use hyper::StatusCode; use store::write::BatchBuilder; use trc::AddContext; use types::collection::{Collection, SyncCollection}; pub(crate) trait CalendarMkColRequestHandler: Sync + Send { fn handle_calendar_mkcol_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: Option, ) -> impl Future> + Send; } impl CalendarMkColRequestHandler for Server { async fn handle_calendar_mkcol_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: Option, ) -> crate::Result { // Validate URI let resource = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let account_id = resource.account_id; let name = resource .resource .ok_or(DavError::Code(StatusCode::FORBIDDEN))?; if !access_token.is_member(account_id) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } else if name.contains('/') || self .fetch_dav_resources(access_token, account_id, SyncCollection::Calendar) .await .caused_by(trc::location!())? .by_path(name) .is_some() { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } // Validate headers self.validate_headers( access_token, headers, vec![ResourceState { account_id, collection: resource.collection, document_id: Some(u32::MAX), path: name, ..Default::default() }], Default::default(), DavMethod::MKCOL, ) .await?; // Build file container let mut calendar = Calendar { name: name.to_string(), preferences: vec![CalendarPreferences { account_id, name: name.to_string(), ..Default::default() }], ..Default::default() }; // Apply MKCOL properties let mut return_prop_stat = None; let mut is_mkcalendar = false; if let Some(mkcol) = request { let mut prop_stat = PropStatBuilder::default(); is_mkcalendar = mkcol.is_mkcalendar; if !self.apply_calendar_properties( access_token, &mut calendar, false, mkcol.props, &mut prop_stat, ) { return Ok(HttpResponse::new(StatusCode::FORBIDDEN).with_xml_body( MkColResponse::new(prop_stat.build()) .with_namespace(Namespace::CalDav) .with_mkcalendar(is_mkcalendar) .to_string(), )); } if headers.ret != Return::Minimal { return_prop_stat = Some(prop_stat); } } // Prepare write batch let mut batch = BatchBuilder::new(); let document_id = self .store() .assign_document_ids(account_id, Collection::Calendar, 1) .await .caused_by(trc::location!())?; calendar .insert(access_token, account_id, document_id, &mut batch) .caused_by(trc::location!())?; let etag = batch.etag(); self.commit_batch(batch).await.caused_by(trc::location!())?; if let Some(prop_stat) = return_prop_stat { Ok(HttpResponse::new(StatusCode::CREATED) .with_xml_body( MkColResponse::new(prop_stat.build()) .with_namespace(Namespace::CalDav) .with_mkcalendar(is_mkcalendar) .to_string(), ) .with_etag_opt(etag)) } else { Ok(HttpResponse::new(StatusCode::CREATED).with_etag_opt(etag)) } } } ================================================ FILE: crates/dav/src/calendar/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod copy_move; pub mod delete; pub mod freebusy; pub mod get; pub mod mkcol; pub mod proppatch; pub mod query; pub mod scheduling; pub mod update; use crate::{DavError, DavErrorCondition}; use common::{DavResources, Server}; use dav_proto::schema::{ property::{CalDavProperty, CalendarData, DavProperty, WebDavProperty}, response::CalCondition, }; use groupware::scheduling::ItipError; use hyper::StatusCode; use trc::AddContext; use types::{collection::Collection, field::CalendarEventField}; pub(crate) static CALENDAR_CONTAINER_PROPS: [DavProperty; 31] = [ DavProperty::WebDav(WebDavProperty::CreationDate), DavProperty::WebDav(WebDavProperty::DisplayName), DavProperty::WebDav(WebDavProperty::GetETag), DavProperty::WebDav(WebDavProperty::GetLastModified), DavProperty::WebDav(WebDavProperty::ResourceType), DavProperty::WebDav(WebDavProperty::LockDiscovery), DavProperty::WebDav(WebDavProperty::SupportedLock), DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal), DavProperty::WebDav(WebDavProperty::SyncToken), DavProperty::WebDav(WebDavProperty::Owner), DavProperty::WebDav(WebDavProperty::SupportedPrivilegeSet), DavProperty::WebDav(WebDavProperty::CurrentUserPrivilegeSet), DavProperty::WebDav(WebDavProperty::Acl), DavProperty::WebDav(WebDavProperty::AclRestrictions), DavProperty::WebDav(WebDavProperty::InheritedAclSet), DavProperty::WebDav(WebDavProperty::PrincipalCollectionSet), DavProperty::WebDav(WebDavProperty::SupportedReportSet), DavProperty::WebDav(WebDavProperty::QuotaAvailableBytes), DavProperty::WebDav(WebDavProperty::QuotaUsedBytes), DavProperty::CalDav(CalDavProperty::CalendarDescription), DavProperty::CalDav(CalDavProperty::SupportedCalendarData), DavProperty::CalDav(CalDavProperty::SupportedCollationSet), DavProperty::CalDav(CalDavProperty::SupportedCalendarComponentSet), DavProperty::CalDav(CalDavProperty::CalendarTimezone), DavProperty::CalDav(CalDavProperty::MaxResourceSize), DavProperty::CalDav(CalDavProperty::MinDateTime), DavProperty::CalDav(CalDavProperty::MaxDateTime), DavProperty::CalDav(CalDavProperty::MaxInstances), DavProperty::CalDav(CalDavProperty::MaxAttendeesPerInstance), DavProperty::CalDav(CalDavProperty::TimezoneServiceSet), DavProperty::CalDav(CalDavProperty::TimezoneId), ]; pub(crate) static CALENDAR_ITEM_PROPS: [DavProperty; 20] = [ DavProperty::WebDav(WebDavProperty::CreationDate), DavProperty::WebDav(WebDavProperty::DisplayName), DavProperty::WebDav(WebDavProperty::GetETag), DavProperty::WebDav(WebDavProperty::GetLastModified), DavProperty::WebDav(WebDavProperty::ResourceType), DavProperty::WebDav(WebDavProperty::LockDiscovery), DavProperty::WebDav(WebDavProperty::SupportedLock), DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal), DavProperty::WebDav(WebDavProperty::SyncToken), DavProperty::WebDav(WebDavProperty::Owner), DavProperty::WebDav(WebDavProperty::SupportedPrivilegeSet), DavProperty::WebDav(WebDavProperty::CurrentUserPrivilegeSet), DavProperty::WebDav(WebDavProperty::Acl), DavProperty::WebDav(WebDavProperty::AclRestrictions), DavProperty::WebDav(WebDavProperty::InheritedAclSet), DavProperty::WebDav(WebDavProperty::PrincipalCollectionSet), DavProperty::WebDav(WebDavProperty::GetContentLanguage), DavProperty::WebDav(WebDavProperty::GetContentLength), DavProperty::WebDav(WebDavProperty::GetContentType), DavProperty::CalDav(CalDavProperty::CalendarData(CalendarData { properties: vec![], expand: None, limit_recurrence: None, limit_freebusy: None, })), ]; pub(crate) async fn assert_is_unique_uid( server: &Server, resources: &DavResources, account_id: u32, calendar_id: u32, uid: Option<&str>, ) -> crate::Result<()> { if let Some(uid) = uid { let hits = server .document_ids_matching( account_id, Collection::CalendarEvent, CalendarEventField::Uid, uid.as_bytes(), ) .await .caused_by(trc::location!())?; if !hits.is_empty() { for path in resources.children(calendar_id) { if hits.contains(path.document_id()) { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, CalCondition::NoUidConflict(resources.format_resource(path).into()), ))); } } } } Ok(()) } pub(crate) trait ItipPrecondition { fn failed_precondition(&self) -> Option; } impl ItipPrecondition for ItipError { fn failed_precondition(&self) -> Option { match self { ItipError::MultipleOrganizer => Some(CalCondition::SameOrganizerInAllComponents), ItipError::OrganizerIsLocalAddress | ItipError::SenderIsNotParticipant(_) | ItipError::OrganizerMismatch => Some(CalCondition::ValidOrganizer), ItipError::CannotModifyProperty(_) | ItipError::CannotModifyInstance | ItipError::CannotModifyAddress => Some(CalCondition::AllowedAttendeeObjectChange), ItipError::MissingUid | ItipError::MultipleUid | ItipError::MultipleObjectTypes | ItipError::MultipleObjectInstances | ItipError::MissingMethod | ItipError::InvalidComponentType | ItipError::OutOfSequence | ItipError::UnknownParticipant(_) | ItipError::UnsupportedMethod(_) => Some(CalCondition::ValidSchedulingMessage), _ => None, } } } ================================================ FILE: crates/dav/src/calendar/proppatch.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ DavError, DavMethod, PropStatBuilder, common::{ ETag, ExtractETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, }; use calcard::common::timezone::Tz; use common::{Server, auth::AccessToken}; use dav_proto::{ RequestHeaders, Return, schema::{ Namespace, property::{CalDavProperty, DavProperty, DavValue, ResourceType, WebDavProperty}, request::{DavPropertyValue, PropertyUpdate}, response::{BaseCondition, CalCondition, MultiStatus, Response}, }, }; use groupware::{ cache::GroupwareCache, calendar::{Calendar, CalendarEvent, SupportedComponent, Timezone}, }; use http_proto::HttpResponse; use hyper::StatusCode; use std::str::FromStr; use store::write::BatchBuilder; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, }; use utils::map::bitmap::Bitmap; pub(crate) trait CalendarPropPatchRequestHandler: Sync + Send { fn handle_calendar_proppatch_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: PropertyUpdate, ) -> impl Future> + Send; fn apply_calendar_properties( &self, access_token: &AccessToken, calendar: &mut Calendar, is_update: bool, properties: Vec, items: &mut PropStatBuilder, ) -> bool; fn apply_event_properties( &self, event: &mut CalendarEvent, is_update: bool, properties: Vec, items: &mut PropStatBuilder, ) -> bool; } impl CalendarPropPatchRequestHandler for Server { async fn handle_calendar_proppatch_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, mut request: PropertyUpdate, ) -> crate::Result { // Validate URI let resource_ = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let uri = headers.uri; let account_id = resource_.account_id; let resources = self .fetch_dav_resources(access_token, account_id, SyncCollection::Calendar) .await .caused_by(trc::location!())?; let resource = resource_ .resource .and_then(|r| resources.by_path(r)) .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let document_id = resource.document_id(); let collection = if resource.is_container() { Collection::Calendar } else { Collection::CalendarEvent }; if !request.has_changes() { return Ok(HttpResponse::new(StatusCode::NO_CONTENT)); } // Verify ACL if !access_token.is_member(account_id) { let (acl, document_id) = if resource.is_container() { (Acl::Modify, resource.document_id()) } else { (Acl::ModifyItems, resource.parent_id().unwrap()) }; if !resources.has_access_to_container(access_token, document_id, acl) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } } // Fetch archive let archive = self .store() .get_value::>(ValueKey::archive( account_id, collection, document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; // Validate headers self.validate_headers( access_token, headers, vec![ResourceState { account_id, collection, document_id: document_id.into(), etag: archive.etag().into(), path: resource_.resource.unwrap(), ..Default::default() }], Default::default(), DavMethod::PROPPATCH, ) .await?; let is_success; let mut batch = BatchBuilder::new(); let mut items = PropStatBuilder::default(); let etag = if resource.is_container() { // Deserialize let calendar = archive .to_unarchived::() .caused_by(trc::location!())?; let mut new_calendar = archive .deserialize::() .caused_by(trc::location!())?; // Remove properties if !request.set_first && !request.remove.is_empty() { remove_calendar_properties( access_token, &mut new_calendar, std::mem::take(&mut request.remove), &mut items, ); } // Set properties is_success = self.apply_calendar_properties( access_token, &mut new_calendar, true, request.set, &mut items, ); // Remove properties if is_success && !request.remove.is_empty() { remove_calendar_properties( access_token, &mut new_calendar, request.remove, &mut items, ); } if is_success { new_calendar .update(access_token, calendar, account_id, document_id, &mut batch) .caused_by(trc::location!())? .etag() } else { calendar.etag().into() } } else { // Deserialize let event = archive .to_unarchived::() .caused_by(trc::location!())?; let mut new_event = archive .deserialize::() .caused_by(trc::location!())?; // Remove properties if !request.set_first && !request.remove.is_empty() { remove_event_properties( &mut new_event, std::mem::take(&mut request.remove), &mut items, ); } // Set properties is_success = self.apply_event_properties(&mut new_event, true, request.set, &mut items); // Remove properties if is_success && !request.remove.is_empty() { remove_event_properties(&mut new_event, request.remove, &mut items); } if is_success { new_event .update(access_token, event, account_id, document_id, &mut batch) .caused_by(trc::location!())? .etag() } else { event.etag().into() } }; if is_success { self.commit_batch(batch).await.caused_by(trc::location!())?; } if headers.ret != Return::Minimal || !is_success { Ok(HttpResponse::new(StatusCode::MULTI_STATUS) .with_xml_body( MultiStatus::new(vec![Response::new_propstat(uri, items.build())]) .with_namespace(Namespace::CalDav) .to_string(), ) .with_etag_opt(etag)) } else { Ok(HttpResponse::new(StatusCode::NO_CONTENT).with_etag_opt(etag)) } } fn apply_calendar_properties( &self, access_token: &AccessToken, calendar: &mut Calendar, is_update: bool, properties: Vec, items: &mut PropStatBuilder, ) -> bool { let mut has_errors = false; for property in properties { match (&property.property, property.value) { (DavProperty::WebDav(WebDavProperty::DisplayName), DavValue::String(name)) => { if name.len() <= self.core.groupware.live_property_size { calendar.preferences_mut(access_token).name = name; items.insert_ok(property.property); } else { items.insert_error_with_description( property.property, StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); has_errors = true; } } ( DavProperty::CalDav(CalDavProperty::CalendarDescription), DavValue::String(name), ) => { if name.len() <= self.core.groupware.live_property_size { calendar.preferences_mut(access_token).description = Some(name); items.insert_ok(property.property); } else { items.insert_error_with_description( property.property, StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); has_errors = true; } } ( DavProperty::CalDav(CalDavProperty::CalendarTimezone), DavValue::ICalendar(ical), ) => { if ical.size() > self.core.groupware.max_ical_size { items.insert_error_with_description( property.property, StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); has_errors = true; } else if !ical.is_timezone() { items.insert_precondition_failed_with_description( property.property, StatusCode::PRECONDITION_FAILED, CalCondition::ValidCalendarData, "Invalid calendar timezone", ); has_errors = true; } else { calendar.preferences_mut(access_token).time_zone = Timezone::Custom(ical); items.insert_ok(property.property); } } (DavProperty::CalDav(CalDavProperty::TimezoneId), DavValue::String(tz_id)) => { if let Ok(tz) = Tz::from_str(&tz_id) { calendar.preferences_mut(access_token).time_zone = Timezone::IANA(tz.as_id()); items.insert_ok(property.property); } else { items.insert_precondition_failed_with_description( property.property, StatusCode::PRECONDITION_FAILED, CalCondition::ValidTimezone, "Invalid timezone ID", ); has_errors = true; } } (DavProperty::WebDav(WebDavProperty::CreationDate), DavValue::Timestamp(dt)) => { calendar.created = dt; items.insert_ok(property.property); } ( DavProperty::WebDav(WebDavProperty::ResourceType), DavValue::ResourceTypes(types), ) => { if !types .0 .iter() .all(|rt| matches!(rt, ResourceType::Collection | ResourceType::Calendar)) { items.insert_precondition_failed( property.property, StatusCode::FORBIDDEN, BaseCondition::ValidResourceType, ); has_errors = true; } else { items.insert_ok(property.property); } } ( DavProperty::CalDav(CalDavProperty::SupportedCalendarComponentSet), DavValue::Components(components), ) => { if !is_update { calendar.supported_components = Bitmap::::from_iter( components .0 .into_iter() .map(|v| SupportedComponent::from(v.0)), ) .into_inner(); if calendar.supported_components != 0 { items.insert_ok(property.property); } else { items.insert_precondition_failed_with_description( property.property, StatusCode::PRECONDITION_FAILED, CalCondition::SupportedCalendarComponent, "At least one supported component must be specified", ); has_errors = true; } } else { items.insert_precondition_failed_with_description( property.property, StatusCode::PRECONDITION_FAILED, CalCondition::SupportedCalendarComponent, "Property cannot be modified", ); has_errors = true; } } (DavProperty::DeadProperty(dead), DavValue::DeadProperty(values)) if self.core.groupware.dead_property_size.is_some() => { if is_update { calendar.dead_properties.remove_element(dead); } if calendar.dead_properties.size() + values.size() + dead.size() < self.core.groupware.dead_property_size.unwrap() { calendar.dead_properties.add_element(dead.clone(), values.0); items.insert_ok(property.property); } else { items.insert_error_with_description( property.property, StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); has_errors = true; } } (_, DavValue::Null) => { items.insert_ok(property.property); } _ => { items.insert_error_with_description( property.property, StatusCode::CONFLICT, "Property cannot be modified", ); has_errors = true; } } } !has_errors } fn apply_event_properties( &self, event: &mut CalendarEvent, is_update: bool, properties: Vec, items: &mut PropStatBuilder, ) -> bool { let mut has_errors = false; for property in properties { match (&property.property, property.value) { (DavProperty::WebDav(WebDavProperty::DisplayName), DavValue::String(name)) => { if name.len() <= self.core.groupware.live_property_size { event.display_name = Some(name); items.insert_ok(property.property); } else { items.insert_error_with_description( property.property, StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); has_errors = true; } } (DavProperty::WebDav(WebDavProperty::CreationDate), DavValue::Timestamp(dt)) => { event.created = dt; items.insert_ok(property.property); } (DavProperty::DeadProperty(dead), DavValue::DeadProperty(values)) if self.core.groupware.dead_property_size.is_some() => { if is_update { event.dead_properties.remove_element(dead); } if event.dead_properties.size() + values.size() + dead.size() < self.core.groupware.dead_property_size.unwrap() { event.dead_properties.add_element(dead.clone(), values.0); items.insert_ok(property.property); } else { items.insert_error_with_description( property.property, StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); has_errors = true; } } (_, DavValue::Null) => { items.insert_ok(property.property); } _ => { items.insert_error_with_description( property.property, StatusCode::CONFLICT, "Property cannot be modified", ); has_errors = true; } } } !has_errors } } fn remove_event_properties( event: &mut CalendarEvent, properties: Vec, items: &mut PropStatBuilder, ) { for property in properties { match &property { DavProperty::WebDav(WebDavProperty::DisplayName) => { event.display_name = None; items.insert_with_status(property, StatusCode::NO_CONTENT); } DavProperty::DeadProperty(dead) => { event.dead_properties.remove_element(dead); items.insert_with_status(property, StatusCode::NO_CONTENT); } _ => { items.insert_error_with_description( property, StatusCode::CONFLICT, "Property cannot be deleted", ); } } } } fn remove_calendar_properties( access_token: &AccessToken, calendar: &mut Calendar, properties: Vec, items: &mut PropStatBuilder, ) { for property in properties { match &property { DavProperty::CalDav(CalDavProperty::CalendarDescription) => { calendar.preferences_mut(access_token).description = None; items.insert_with_status(property, StatusCode::NO_CONTENT); } DavProperty::CalDav(CalDavProperty::CalendarTimezone) | DavProperty::CalDav(CalDavProperty::TimezoneId) => { calendar.preferences_mut(access_token).time_zone = Timezone::Default; items.insert_with_status(property, StatusCode::NO_CONTENT); } DavProperty::DeadProperty(dead) => { calendar.dead_properties.remove_element(dead); items.insert_with_status(property, StatusCode::NO_CONTENT); } _ => { items.insert_error_with_description( property, StatusCode::CONFLICT, "Property cannot be deleted", ); } } } } ================================================ FILE: crates/dav/src/calendar/query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::freebusy::freebusy_in_range; use crate::{ DavError, common::{ CalendarFilter, DavQuery, propfind::{PropFindItem, PropFindRequestHandler}, uri::DavUriResource, }, }; use calcard::{ common::{PartialDateTime, timezone::Tz}, icalendar::{ ArchivedICalendar, ArchivedICalendarComponent, ArchivedICalendarEntry, ArchivedICalendarParameter, ArchivedICalendarProperty, ArchivedICalendarValue, ICalendarComponentType, ICalendarEntry, ICalendarParameterName, ICalendarProperty, ICalendarValue, }, }; use common::{DavResource, Server, auth::AccessToken}; use dav_proto::{ RequestHeaders, schema::{ property::{CalDavProperty, CalendarData, DavProperty}, request::{CalendarQuery, Filter, FilterOp, PropFind, Timezone}, response::MultiStatus, }, }; use groupware::{ cache::GroupwareCache, calendar::{ArchivedCalendarEvent, expand::CalendarEventExpansion}, }; use http_proto::HttpResponse; use hyper::StatusCode; use std::{fmt::Write, slice::Iter, str::FromStr}; use store::{ ahash::{AHashMap, AHashSet}, write::serialize::rkyv_deserialize, }; use trc::AddContext; use types::{TimeRange, acl::Acl, collection::SyncCollection}; pub(crate) trait CalendarQueryRequestHandler: Sync + Send { fn handle_calendar_query_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: CalendarQuery, ) -> impl Future> + Send; } impl CalendarQueryRequestHandler for Server { async fn handle_calendar_query_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: CalendarQuery, ) -> crate::Result { // Validate URI let resource_ = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let account_id = resource_.account_id; let resources = self .fetch_dav_resources(access_token, account_id, SyncCollection::Calendar) .await .caused_by(trc::location!())?; let Some(resource) = resources.by_path( resource_ .resource .ok_or(DavError::Code(StatusCode::METHOD_NOT_ALLOWED))?, ) else { return Ok(HttpResponse::new(StatusCode::MULTI_STATUS) .with_xml_body(MultiStatus::not_found(headers.uri).to_string())); }; if !resource.is_container() { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } // Obtain shared ids let shared_ids = if !access_token.is_member(account_id) { resources .shared_containers(access_token, [Acl::ReadItems], false) .into() } else { None }; // Pre-filter by date range let filter_range = extract_filter_range(&request); // Obtain document ids in folder let mut items = Vec::with_capacity(16); for resource in resources.children(resource.document_id()) { if shared_ids .as_ref() .is_none_or(|ids| ids.contains(resource.document_id())) && filter_range .as_ref() .is_none_or(|range| is_resource_in_time_range(resource.resource, range)) { items.push(PropFindItem::new( resources.format_resource(resource), account_id, resource, )); } } // Extract the time range from the request let max_time_range = extract_data_range(&request.properties, filter_range); self.handle_dav_query( access_token, DavQuery::calendar_query(request, max_time_range, items, headers), ) .await } } pub(crate) fn is_resource_in_time_range(resource: &DavResource, filter: &TimeRange) -> bool { // Check whether the resource has a time range and if it overlaps with the filter if let Some((start, end)) = resource.event_time_range() { ((filter.start < end) || (filter.start <= start)) && (filter.end > start || filter.end >= end) } else { // If the resource does not have a time range, it is not in the range false } } fn extract_filter_range(query: &CalendarQuery) -> Option { let mut range = TimeRange { start: i64::MAX, end: i64::MIN, }; for filter in &query.filters { let op = match filter { Filter::Component { op, .. } => op, Filter::Property { op, .. } => op, Filter::Parameter { op, .. } => op, _ => continue, }; if let FilterOp::TimeRange(date_range) = op { if date_range.start < range.start { range.start = date_range.start; } if date_range.end > range.end { range.end = date_range.end; } } } if range.start != i64::MAX { Some(range) } else { None } } fn extract_data_range(propfind: &PropFind, filter_range: Option) -> Option { let props = match propfind { PropFind::AllProp(props) | PropFind::Prop(props) => props, PropFind::PropName => &[][..], }; for prop in props { if let DavProperty::CalDav(CalDavProperty::CalendarData(data)) = prop { let mut range = filter_range.unwrap_or(TimeRange { start: i64::MAX, end: i64::MIN, }); for data_range in [&data.expand, &data.limit_recurrence, &data.limit_freebusy] .into_iter() .flatten() { if data_range.start < range.start { range.start = data_range.start; } if data_range.end > range.end { range.end = data_range.end; } } return if range.start != i64::MAX { Some(range) } else { None }; } } filter_range } pub fn try_parse_tz(tz: &Timezone) -> Option { match tz { Timezone::Name(value) | Timezone::Id(value) => Tz::from_str(value).ok(), Timezone::None => None, } } pub(crate) struct CalendarQueryHandler { default_tz: Tz, expanded_times: Vec, } impl CalendarQueryHandler { pub fn new( event: &ArchivedCalendarEvent, max_time_range: Option, default_tz: Tz, ) -> Self { Self { default_tz, expanded_times: max_time_range .map(|max_time_range| { event .data .expand(default_tz, max_time_range) .unwrap_or_else(|| { trc::event!( Calendar(trc::CalendarEvent::RuleExpansionError), Reason = "chrono error", Details = event.data.event.to_string(), ); vec![] }) }) .unwrap_or_default(), } } pub fn filter(&mut self, event: &ArchivedCalendarEvent, filters: &CalendarFilter) -> bool { let ical = &event.data.event; let mut is_all = true; let mut matches_one = false; for filter in filters { match filter { Filter::AnyOf => { is_all = false; } Filter::AllOf => { is_all = true; } Filter::Property { prop, op, comp } => { let mut properties = find_components(ical, comp) .flat_map(|(_, comp)| find_properties(comp, prop)) .peekable(); let result = if properties.peek().is_some() { properties.any(|entry| { match op { FilterOp::Exists => true, FilterOp::Undefined => false, FilterOp::TextMatch(text_match) => { let mut matched_any = false; for value in entry.values.iter() { if let Some(text) = value.as_text() && text_match.matches(text) { matched_any = true; break; } } matched_any } FilterOp::TimeRange(range) => { if let Some(ArchivedICalendarValue::PartialDateTime(date)) = entry.values.first() { let tz = entry .tz_id() .and_then(|tz_id| Tz::from_str(tz_id).ok()) .unwrap_or(self.default_tz); if let Some(date) = date .to_date_time() .and_then(|date| date.to_date_time_with_tz(tz)) { let timestamp = date.timestamp(); // RFC4791#9.9: start <= DTSTART AND end > DTSTART range.start <= timestamp && range.end > timestamp } else { false } } else { false } } } }) } else { matches!(op, FilterOp::Undefined) }; if result { matches_one = true; } else if is_all { return false; } } Filter::Parameter { prop, param, op, comp, } => { let mut parameters = find_components(ical, comp) .flat_map(|(_, comp)| { find_properties(comp, prop) .filter_map(|entry| find_parameter(entry, param)) }) .peekable(); let result = if parameters.peek().is_some() { parameters.any(|entry| match op { FilterOp::Exists => true, FilterOp::Undefined => false, FilterOp::TextMatch(text_match) => { if let Some(text) = entry.value.as_text() { text_match.matches(text) } else { false } } FilterOp::TimeRange(_) => false, }) } else { matches!(op, FilterOp::Undefined) }; if result { matches_one = true; } else if is_all { return false; } } Filter::Component { comp, op } => { let result = match op { FilterOp::Exists => find_components(ical, comp).next().is_some(), FilterOp::Undefined => find_components(ical, comp).next().is_none(), FilterOp::TimeRange(range) => { if !matches!(comp.last(), Some(ICalendarComponentType::VAlarm)) { let matching_comp_ids = find_components(ical, comp) .map(|(id, comp)| (id as u32, &comp.component_type)) .collect::>(); !matching_comp_ids.is_empty() && self.expanded_times.iter().any(|event| { matching_comp_ids.get(&event.comp_id).is_some_and(|ct| { range.is_in_range( ct == &&ICalendarComponentType::VTodo, event.start, event.end, ) }) }) } else { let matching_comp_ids = event .data .alarms .iter() .map(|alarm| alarm.parent_id.to_native() as u32) .collect::>(); !matching_comp_ids.is_empty() && self.expanded_times.iter().any(|time| { matching_comp_ids.contains(&time.comp_id) && event.data.alarms.iter().any(|alarm| { alarm.parent_id.to_native() as u32 == time.comp_id && alarm .delta .to_timestamp( time.start, time.end, self.default_tz, ) .is_some_and(|timestamp| { range.is_in_range( false, timestamp, timestamp, ) }) }) }) } } FilterOp::TextMatch(_) => false, }; if result { matches_one = true; } else if is_all { return false; } } } } is_all || matches_one } pub fn serialize_ical( &mut self, event: &ArchivedCalendarEvent, data: &CalendarData, instances_limit: &mut usize, ) -> Option { let mut out = String::with_capacity(event.size.to_native() as usize); let _v = [0.into()]; let mut component_iter: Iter<'_, rkyv::rend::u32_le> = _v.iter(); let mut component_stack: Vec<(&ArchivedICalendarComponent, Iter<'_, rkyv::rend::u32_le>)> = Vec::with_capacity(4); if data.expand.is_some() { self.expanded_times .sort_unstable_by(|a, b| a.start.cmp(&b.start)); } loop { if let Some(component_id) = component_iter.next() { let component_id = component_id.to_native(); let component = event .data .event .components .get(component_id as usize) .unwrap(); // Limit recurrence override if let Some(limit_recurrence) = &data.limit_recurrence && component.is_recurrence_override() && !self.expanded_times.iter().any(|event| { event.comp_id == component_id && limit_recurrence.is_in_range( component.component_type == ICalendarComponentType::VTodo, event.start, event.end, ) }) { continue; } // Limit freebusy if let Some(limit_recurrence) = &data.limit_freebusy && component.component_type == ICalendarComponentType::VFreebusy && !self.expanded_times.iter().any(|event| { event.comp_id == component_id && limit_recurrence.is_in_range(false, event.start, event.end) }) { continue; } // Filter entries let mut entries = component .entries .iter() .filter_map(|entry| { if data.properties.is_empty() || component.component_type == ICalendarComponentType::VCalendar { Some((entry, true)) } else { data.properties .iter() .find(|prop| { prop.component.as_ref().is_none_or(|comp| { comp == &component.component_type || component_stack.iter().any(|(parent_comp, _)| { comp == &parent_comp.component_type }) }) && prop.name.as_ref().is_none_or(|name| name == &entry.name) }) .map(|prop| (entry, !prop.no_value)) } }) .peekable(); // Expand recurrences let component_name = component.component_type.as_str(); if let Some(expand) = &data .expand .filter(|_| component.component_type.has_time_ranges()) { let is_recurrent = component.is_recurrent(); let is_recurrent_or_override = is_recurrent || component.is_recurrence_override(); let is_todo = component.component_type == ICalendarComponentType::VTodo; let mut has_duration = false; let entries = entries .filter(|(entry, _)| match &entry.name { ArchivedICalendarProperty::Dtstart | ArchivedICalendarProperty::Dtend | ArchivedICalendarProperty::Exdate | ArchivedICalendarProperty::Exrule | ArchivedICalendarProperty::Rdate | ArchivedICalendarProperty::Rrule | ArchivedICalendarProperty::RecurrenceId => false, ArchivedICalendarProperty::Due | ArchivedICalendarProperty::Completed | ArchivedICalendarProperty::Created => is_recurrent, ArchivedICalendarProperty::Duration => { has_duration = true; true } _ => true, }) .collect::>(); for event in &self.expanded_times { if event.comp_id == component_id && (!is_recurrent_or_override || expand.is_in_range(is_todo, event.start, event.end)) { if *instances_limit > 0 { *instances_limit -= 1; } else { return None; } let _ = write!(&mut out, "BEGIN:{component_name}\r\n"); // Write DTSTART, DTEND and RECURRENCE-ID let mut entry = ICalendarEntry { name: ICalendarProperty::Dtstart, params: vec![], values: vec![ICalendarValue::PartialDateTime(Box::new( PartialDateTime::from_utc_timestamp(event.start), ))], }; let _ = entry.write_to(&mut out); if is_recurrent_or_override { entry.name = ICalendarProperty::RecurrenceId; let _ = entry.write_to(&mut out); } if !has_duration { entry.name = ICalendarProperty::Dtend; entry.values = vec![ICalendarValue::PartialDateTime(Box::new( PartialDateTime::from_utc_timestamp(event.end), ))]; let _ = entry.write_to(&mut out); } // Write other component entries for (entry, with_value) in &entries { let _ = entry.write_to(&mut out, *with_value); } let _ = write!(&mut out, "END:{component_name}\r\n"); } } } else if entries.peek().is_some() { let _ = write!(&mut out, "BEGIN:{component_name}\r\n"); if data.limit_freebusy.is_none() || component.component_type != ICalendarComponentType::VFreebusy { for (entry, with_value) in entries { let _ = entry.write_to(&mut out, with_value); } } else { // Filter freebusy let range = data.limit_freebusy.unwrap(); for (entry, with_value) in entries { if matches!(entry.name, ArchivedICalendarProperty::Freebusy) { let mut fb_in_range = freebusy_in_range(entry, &range, self.default_tz).peekable(); if fb_in_range.peek().is_none() { continue; } else { let _ = ICalendarEntry { name: ICalendarProperty::Freebusy, params: rkyv_deserialize(&entry.params) .ok() .unwrap_or_default(), values: fb_in_range.collect(), } .write_to(&mut out); } } else { let _ = entry.write_to(&mut out, with_value); } } } if !component.component_ids.is_empty() { component_stack.push((component, component_iter)); component_iter = component.component_ids.iter(); } else if component.component_ids.is_empty() { let _ = write!(&mut out, "END:{component_name}\r\n"); } } } else if let Some((component, iter)) = component_stack.pop() { let _ = write!(&mut out, "END:{}\r\n", component.component_type.as_str()); component_iter = iter; } else { break; } } Some(out) } pub fn into_expanded_times(self) -> Vec { self.expanded_times } } #[inline(always)] fn find_components<'x>( ical: &'x ArchivedICalendar, comp: &[ICalendarComponentType], ) -> impl Iterator { // TODO: Properly expand the component type path let comp = comp.last().unwrap_or(&ICalendarComponentType::VCalendar); ical.components .iter() .enumerate() .filter(move |(_, entry)| { comp == &ICalendarComponentType::VCalendar || &entry.component_type == comp }) } #[inline(always)] fn find_properties<'x>( comp: &'x ArchivedICalendarComponent, prop: &ICalendarProperty, ) -> impl Iterator { comp.entries.iter().filter(move |entry| &entry.name == prop) } #[inline(always)] fn find_parameter<'x>( entry: &'x ArchivedICalendarEntry, name: &ICalendarParameterName, ) -> Option<&'x ArchivedICalendarParameter> { entry.params.iter().find(|param| param.name == *name) } ================================================ FILE: crates/dav/src/calendar/scheduling.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ DavError, DavErrorCondition, DavMethod, calendar::freebusy::CalendarFreebusyRequestHandler, common::{ ETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, }; use calcard::{ Entry, Parser, icalendar::{ ICalendarComponentType, ICalendarEntry, ICalendarMethod, ICalendarProperty, ICalendarValue, Uri, }, }; use common::{Server, auth::AccessToken}; use dav_proto::{ RequestHeaders, schema::{ property::Rfc1123DateTime, request::FreeBusyQuery, response::{CalCondition, Href, ScheduleResponse, ScheduleResponseItem}, }, }; use groupware::{DestroyArchive, cache::GroupwareCache, calendar::CalendarEventNotification}; use http_proto::HttpResponse; use hyper::StatusCode; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use store::{ahash::AHashMap, write::BatchBuilder}; use trc::AddContext; use types::collection::{Collection, SyncCollection}; use utils::sanitize_email; pub(crate) trait CalendarEventNotificationHandler: Sync + Send { fn handle_scheduling_get_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, is_head: bool, ) -> impl Future> + Send; fn handle_scheduling_delete_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, ) -> impl Future> + Send; fn handle_scheduling_post_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, bytes: Vec, ) -> impl Future> + Send; } impl CalendarEventNotificationHandler for Server { async fn handle_scheduling_get_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, is_head: bool, ) -> crate::Result { // Validate URI let resource_ = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let account_id = resource_.account_id; let resources = self .fetch_dav_resources( access_token, account_id, SyncCollection::CalendarEventNotification, ) .await .caused_by(trc::location!())?; let resource = resources .by_path( resource_ .resource .ok_or(DavError::Code(StatusCode::METHOD_NOT_ALLOWED))?, ) .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; if resource.is_container() { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } // Validate ACL if !access_token.is_member(account_id) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Fetch event let event_ = self .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEventNotification, resource.document_id(), )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let event = event_ .unarchive::() .caused_by(trc::location!())?; // Validate headers let etag = event_.etag(); self.validate_headers( access_token, headers, vec![ResourceState { account_id, collection: Collection::CalendarEventNotification, document_id: resource.document_id().into(), etag: etag.clone().into(), path: resource_.resource.unwrap(), ..Default::default() }], Default::default(), DavMethod::GET, ) .await?; let response = HttpResponse::new(StatusCode::OK) .with_content_type("text/calendar; charset=utf-8") .with_etag(etag) .with_last_modified(Rfc1123DateTime::new(i64::from(event.modified)).to_string()); let ical = event.event.to_string(); if !is_head { Ok(response.with_binary_body(ical)) } else { Ok(response.with_content_length(ical.len())) } } async fn handle_scheduling_delete_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, ) -> crate::Result { // Validate URI let resource = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let account_id = resource.account_id; let delete_path = resource .resource .filter(|r| !r.is_empty()) .ok_or(DavError::Code(StatusCode::FORBIDDEN))?; let resources = self .fetch_dav_resources( access_token, account_id, SyncCollection::CalendarEventNotification, ) .await .caused_by(trc::location!())?; // Check resource type let resource = resources .by_path(delete_path) .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; if resource.is_container() { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } // Validate ACL if !access_token.is_member(account_id) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } let document_id = resource.document_id(); let event_ = self .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEventNotification, document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; // Validate headers self.validate_headers( access_token, headers, vec![ResourceState { account_id, collection: Collection::CalendarEventNotification, document_id: document_id.into(), etag: event_.etag().into(), path: delete_path, ..Default::default() }], Default::default(), DavMethod::DELETE, ) .await?; let event = event_ .to_unarchived::() .caused_by(trc::location!())?; // Delete event let mut batch = BatchBuilder::new(); DestroyArchive(event) .delete(access_token, account_id, document_id, &mut batch) .caused_by(trc::location!())?; self.commit_batch(batch).await.caused_by(trc::location!())?; Ok(HttpResponse::new(StatusCode::NO_CONTENT)) } async fn handle_scheduling_post_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, bytes: Vec, ) -> crate::Result { // Validate URI let resource = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; if resource.resource.is_none_or(|r| r != "outbox") { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Parse iTIP message if bytes.len() > self.core.groupware.max_ical_size { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, CalCondition::MaxResourceSize(self.core.groupware.max_ical_size as u32), ))); } let itip_raw = std::str::from_utf8(&bytes).map_err(|_| { DavError::Condition( DavErrorCondition::new( StatusCode::BAD_REQUEST, CalCondition::ValidSchedulingMessage, ) .with_details("Invalid UTF-8 in iCalendar data"), ) })?; let itip = match Parser::new(itip_raw).entry() { Entry::ICalendar(ical) if ical.components.len() > 1 => ical, _ => { return Err(DavError::Condition( DavErrorCondition::new( StatusCode::BAD_REQUEST, CalCondition::ValidSchedulingMessage, ) .with_details("Failed to parse iCalendar data"), )); } }; // Parse request let mut from_date = None; let mut to_date = None; let mut organizer = None; let mut attendees = AHashMap::new(); let mut uid = None; let tz_resolver = itip.build_tz_resolver(); let mut found_freebusy = false; for component in &itip.components { if component.component_type != ICalendarComponentType::VFreebusy { continue; } else if !found_freebusy { found_freebusy = true; } else { return Err(DavError::Condition( DavErrorCondition::new( StatusCode::BAD_REQUEST, CalCondition::ValidSchedulingMessage, ) .with_details("Multiple VFREEBUSY components found"), )); } for entry in &component.entries { let tz_id = entry.tz_id(); match (&entry.name, entry.values.first()) { (ICalendarProperty::Dtstart, Some(ICalendarValue::PartialDateTime(dt))) => { from_date = dt.to_date_time_with_tz(tz_resolver.resolve_or_default(tz_id)); } (ICalendarProperty::Dtend, Some(ICalendarValue::PartialDateTime(dt))) => { to_date = dt.to_date_time_with_tz(tz_resolver.resolve_or_default(tz_id)); } (ICalendarProperty::Uid, Some(ICalendarValue::Text(_))) => { uid = Some(entry); } ( ICalendarProperty::Organizer, Some(ICalendarValue::Text(_) | ICalendarValue::Uri(Uri::Location(_))), ) => { organizer = Some(entry); } ( ICalendarProperty::Attendee, Some( ICalendarValue::Text(value) | ICalendarValue::Uri(Uri::Location(value)), ), ) => { if let Some(email) = sanitize_email(value.strip_prefix("mailto:").unwrap_or(value.as_str())) { attendees.insert(email, entry); } } _ => {} } } } let (Some(from_date), Some(to_date)) = (from_date, to_date) else { return Err(DavError::Condition( DavErrorCondition::new( StatusCode::BAD_REQUEST, CalCondition::ValidSchedulingMessage, ) .with_details("Missing DTSTART or DTEND in VFREEBUSY component"), )); }; let Some(organizer) = organizer else { return Err(DavError::Condition( DavErrorCondition::new( StatusCode::BAD_REQUEST, CalCondition::ValidSchedulingMessage, ) .with_details("Missing ORGANIZER in VFREEBUSY component"), )); }; if attendees.is_empty() { return Err(DavError::Condition( DavErrorCondition::new( StatusCode::BAD_REQUEST, CalCondition::ValidSchedulingMessage, ) .with_details("Missing ATTENDEE in VFREEBUSY component"), )); } let mut response = ScheduleResponse::default(); for (email, attendee) in attendees { if let Some(account_id) = self .directory() .email_to_id(&email) .await .caused_by(trc::location!())? { let resources = self .fetch_dav_resources(access_token, account_id, SyncCollection::Calendar) .await .caused_by(trc::location!())?; if let Some(resource) = self .core .groupware .default_calendar_name .as_ref() .and_then(|name| resources.by_path(name)) { let mut free_busy = self .build_freebusy_object( access_token, FreeBusyQuery::new(from_date.timestamp(), to_date.timestamp()), &resources, account_id, resource, ) .await?; // Add iTIP method free_busy.components[0].entries.push(ICalendarEntry { name: ICalendarProperty::Method, params: vec![], values: vec![ICalendarValue::Method(ICalendarMethod::Reply)], }); // Add properties let component = &mut free_busy.components[1]; component.entries.push(organizer.clone()); component.entries.push(attendee.clone()); if let Some(uid) = uid { component.entries.push(uid.clone()); } response.items.0.push(ScheduleResponseItem { recipient: Href(format!("mailto:{email}")), request_status: "2.0;Success".into(), calendar_data: Some(free_busy.to_string()), }); } else { response.items.0.push(ScheduleResponseItem { recipient: Href(format!("mailto:{email}")), request_status: "3.7;Default calendar not found".into(), calendar_data: None, }); } } else { response.items.0.push(ScheduleResponseItem { recipient: Href(format!("mailto:{email}")), request_status: "3.7;Invalid calendar user or insufficient permissions".into(), calendar_data: None, }); } } Ok(HttpResponse::new(StatusCode::OK).with_xml_body(response.to_string())) } } ================================================ FILE: crates/dav/src/calendar/update.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::assert_is_unique_uid; use crate::{ DavError, DavErrorCondition, DavMethod, calendar::ItipPrecondition, common::{ ETag, ExtractETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, file::DavFileResource, fix_percent_encoding, }; use calcard::{ Entry, Parser, common::timezone::Tz, icalendar::{ICalendar, ICalendarComponentType}, }; use common::{DavName, Server, auth::AccessToken}; use dav_proto::{ RequestHeaders, Return, schema::{property::Rfc1123DateTime, response::CalCondition}, }; use directory::Permission; use groupware::{ cache::GroupwareCache, calendar::{CalendarEvent, CalendarEventData}, scheduling::{ItipMessages, event_create::itip_create, event_update::itip_update}, }; use http_proto::HttpResponse; use hyper::StatusCode; use std::collections::HashSet; use store::write::{BatchBuilder, now}; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, }; pub(crate) trait CalendarUpdateRequestHandler: Sync + Send { fn handle_calendar_update_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, bytes: Vec, is_patch: bool, ) -> impl Future> + Send; } impl CalendarUpdateRequestHandler for Server { async fn handle_calendar_update_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, bytes: Vec, _is_patch: bool, ) -> crate::Result { // Validate URI let resource = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let account_id = resource.account_id; let resources = self .fetch_dav_resources(access_token, account_id, SyncCollection::Calendar) .await .caused_by(trc::location!())?; let resource_name = fix_percent_encoding( resource .resource .ok_or(DavError::Code(StatusCode::CONFLICT))?, ); if bytes.len() > self.core.groupware.max_ical_size { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, CalCondition::MaxResourceSize(self.core.groupware.max_ical_size as u32), ))); } let ical_raw = std::str::from_utf8(&bytes).map_err(|_| { DavError::Condition( DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, CalCondition::SupportedCalendarData, ) .with_details("Invalid UTF-8 in iCalendar data"), ) })?; let ical = match Parser::new(ical_raw).entry() { Entry::ICalendar(ical) => ical, _ => { return Err(DavError::Condition( DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, CalCondition::SupportedCalendarData, ) .with_details("Failed to parse iCalendar data"), )); } }; if let Some(resource) = resources.by_path(resource_name.as_ref()) { if resource.is_container() { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } // Validate ACL let parent_id = resource.parent_id().unwrap(); let document_id = resource.document_id(); if !access_token.is_member(account_id) && !resources.has_access_to_container(access_token, parent_id, Acl::ModifyItems) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Update let event_ = self .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let event = event_ .to_unarchived::() .caused_by(trc::location!())?; // Validate headers match self .validate_headers( access_token, headers, vec![ResourceState { account_id, collection: Collection::CalendarEvent, document_id: Some(document_id), etag: event.etag().into(), path: resource_name.as_ref(), ..Default::default() }], Default::default(), DavMethod::PUT, ) .await { Ok(_) => {} Err(DavError::Code(StatusCode::PRECONDITION_FAILED)) if headers.ret == Return::Representation => { return Ok(HttpResponse::new(StatusCode::PRECONDITION_FAILED) .with_content_type("text/calendar; charset=utf-8") .with_etag(event.etag()) .with_last_modified( Rfc1123DateTime::new(i64::from(event.inner.modified)).to_string(), ) .with_header("Preference-Applied", "return=representation") .with_binary_body(event.inner.data.event.to_string())); } Err(e) => return Err(e), } if ical == event.inner.data.event { // No changes, return existing event return Ok(HttpResponse::new(StatusCode::NO_CONTENT)); } // Validate iCal if event.inner.data.event.uids().next().unwrap_or_default() != validate_ical(&ical)? { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, CalCondition::NoUidConflict(resources.format_resource(resource).into()), ))); } // Validate schedule tag if headers.if_schedule_tag.is_some() && event.inner.schedule_tag.as_ref().map(|t| t.to_native()) != headers.if_schedule_tag { return Err(DavError::Code(StatusCode::PRECONDITION_FAILED)); } // Obtain previous alarm let now = now() as i64; let prev_email_alarm = event.inner.data.next_alarm(now, Tz::Floating); // Build event let mut next_email_alarm = None; let mut new_event = event .deserialize::() .caused_by(trc::location!())?; let old_ical = new_event.data.event; new_event.size = bytes.len() as u32; new_event.data = CalendarEventData::new( ical, Tz::Floating, self.core.groupware.max_ical_instances, &mut next_email_alarm, ); // Scheduling let mut itip_messages = None; if self.core.groupware.itip_enabled && !access_token.emails.is_empty() && access_token.has_permission(Permission::CalendarSchedulingSend) && new_event.data.event_range_end() > now { let result = if new_event.schedule_tag.is_some() { itip_update( &mut new_event.data.event, &old_ical, access_token.emails.as_slice(), ) } else { itip_create(&mut new_event.data.event, access_token.emails.as_slice()) }; match result { Ok(messages) => { let mut is_organizer = false; if messages .iter() .map(|r| { is_organizer = r.from_organizer; r.to.len() }) .sum::() < self.core.groupware.itip_outbound_max_recipients { // Only update schedule tag if the user is the organizer if is_organizer { if let Some(schedule_tag) = &mut new_event.schedule_tag { *schedule_tag += 1; } else { new_event.schedule_tag = Some(1); } } itip_messages = Some(ItipMessages::new(messages)); } else { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, CalCondition::MaxAttendeesPerInstance, ))); } } Err(err) => { if let Some(failed_precondition) = err.failed_precondition() { return Err(DavError::Condition( DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, failed_precondition, ) .with_details(err.to_string()), )); } // Event changed, but there are no iTIP messages to send if let Some(schedule_tag) = &mut new_event.schedule_tag { *schedule_tag += 1; } } } } // Validate quota let extra_bytes = (bytes.len() as u64).saturating_sub(u32::from(event.inner.size) as u64); if extra_bytes > 0 { self.has_available_quota( &self.get_resource_token(access_token, account_id).await?, extra_bytes, ) .await?; } // Prepare write batch let mut batch = BatchBuilder::new(); let schedule_tag = new_event.schedule_tag; let etag = new_event .update(access_token, event, account_id, document_id, &mut batch) .caused_by(trc::location!())? .etag(); if prev_email_alarm != next_email_alarm { if let Some(prev_alarm) = prev_email_alarm { prev_alarm.delete_task(&mut batch); } if let Some(next_alarm) = next_email_alarm { next_alarm.write_task(&mut batch); } } if let Some(itip_messages) = itip_messages { itip_messages .queue(&mut batch) .caused_by(trc::location!())?; } self.commit_batch(batch).await.caused_by(trc::location!())?; self.notify_task_queue(); Ok(HttpResponse::new(StatusCode::NO_CONTENT) .with_etag_opt(etag) .with_schedule_tag_opt(schedule_tag)) } else if let Some((Some(parent), name)) = resources.map_parent(resource_name.as_ref()) { if !parent.is_container() { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } // Validate ACL if !access_token.is_member(account_id) && !resources.has_access_to_container( access_token, parent.document_id(), Acl::AddItems, ) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Validate headers self.validate_headers( access_token, headers, vec![ResourceState { account_id, collection: resource.collection, document_id: Some(u32::MAX), path: resource_name.as_ref(), ..Default::default() }], Default::default(), DavMethod::PUT, ) .await?; // Validate ical object assert_is_unique_uid( self, &resources, account_id, parent.document_id(), validate_ical(&ical)?.into(), ) .await?; // Build event let mut next_email_alarm = None; let mut event = CalendarEvent { names: vec![DavName { name: name.to_string(), parent_id: parent.document_id(), }], data: CalendarEventData::new( ical, Tz::Floating, self.core.groupware.max_ical_instances, &mut next_email_alarm, ), size: bytes.len() as u32, ..Default::default() }; // Scheduling let mut itip_messages = None; if self.core.groupware.itip_enabled && !access_token.emails.is_empty() && access_token.has_permission(Permission::CalendarSchedulingSend) && event.data.event_range_end() > now() as i64 { match itip_create(&mut event.data.event, access_token.emails.as_slice()) { Ok(messages) => { if messages.iter().map(|r| r.to.len()).sum::() < self.core.groupware.itip_outbound_max_recipients { event.schedule_tag = Some(1); itip_messages = Some(ItipMessages::new(messages)); } else { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, CalCondition::MaxAttendeesPerInstance, ))); } } Err(err) => { if let Some(failed_precondition) = err.failed_precondition() { return Err(DavError::Condition( DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, failed_precondition, ) .with_details(err.to_string()), )); } } } } // Validate quota if !bytes.is_empty() { self.has_available_quota( &self.get_resource_token(access_token, account_id).await?, bytes.len() as u64, ) .await?; } // Prepare write batch let mut batch = BatchBuilder::new(); let document_id = self .store() .assign_document_ids(account_id, Collection::CalendarEvent, 1) .await .caused_by(trc::location!())?; let schedule_tag = event.schedule_tag; let etag = event .insert( access_token, account_id, document_id, next_email_alarm, &mut batch, ) .caused_by(trc::location!())? .etag(); if let Some(itip_messages) = itip_messages { itip_messages .queue(&mut batch) .caused_by(trc::location!())?; } self.commit_batch(batch).await.caused_by(trc::location!())?; self.notify_task_queue(); Ok(HttpResponse::new(StatusCode::CREATED) .with_etag_opt(etag) .with_schedule_tag_opt(schedule_tag)) } else { Err(DavError::Code(StatusCode::CONFLICT))? } } } fn validate_ical(ical: &ICalendar) -> crate::Result<&str> { // Validate UIDs let mut uids = HashSet::with_capacity(1); // Validate component types let mut types: [u8; 5] = [0; 5]; for comp in &ical.components { *(match comp.component_type { ICalendarComponentType::VEvent => &mut types[0], ICalendarComponentType::VTodo => &mut types[1], ICalendarComponentType::VJournal => &mut types[2], ICalendarComponentType::VFreebusy => &mut types[3], ICalendarComponentType::VAvailability => &mut types[4], _ => { continue; } }) += 1; if let Some(uid) = comp.uid() { uids.insert(uid); } } if uids.len() == 1 && types.iter().filter(|&&v| v == 0).count() == 4 { Ok(uids.iter().next().unwrap()) } else { Err(DavError::Condition( DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, CalCondition::ValidCalendarObjectResource, ) .with_details("iCalendar must contain exactly one UID and same component types"), )) } } ================================================ FILE: crates/dav/src/card/copy_move.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::assert_is_unique_uid; use crate::{ DavError, DavMethod, common::{ lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, file::DavFileResource, }; use common::{DavName, Server, auth::AccessToken}; use dav_proto::{Depth, RequestHeaders}; use groupware::{ DestroyArchive, cache::GroupwareCache, contact::{AddressBook, AddressBookPreferences, ContactCard}, }; use http_proto::HttpResponse; use hyper::StatusCode; use store::write::BatchBuilder; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection, VanishedCollection}, }; pub(crate) trait CardCopyMoveRequestHandler: Sync + Send { fn handle_card_copy_move_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, is_move: bool, ) -> impl Future> + Send; } impl CardCopyMoveRequestHandler for Server { async fn handle_card_copy_move_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, is_move: bool, ) -> crate::Result { // Validate source let from_resource_ = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let from_account_id = from_resource_.account_id; let from_resources = self .fetch_dav_resources(access_token, from_account_id, SyncCollection::AddressBook) .await .caused_by(trc::location!())?; let from_resource_name = from_resource_ .resource .ok_or(DavError::Code(StatusCode::FORBIDDEN))?; let from_resource = from_resources .by_path(from_resource_name) .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; // Validate ACL if !access_token.is_member(from_account_id) && !from_resources.has_access_to_container( access_token, if from_resource.is_container() { from_resource.document_id() } else { from_resource.parent_id().unwrap() }, Acl::ReadItems, ) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Validate destination let destination = self .validate_uri_with_status( access_token, headers .destination .ok_or(DavError::Code(StatusCode::BAD_GATEWAY))?, StatusCode::BAD_GATEWAY, ) .await?; if destination.collection != Collection::AddressBook { return Err(DavError::Code(StatusCode::BAD_GATEWAY)); } let to_account_id = destination .account_id .ok_or(DavError::Code(StatusCode::BAD_GATEWAY))?; let to_resources = if to_account_id == from_account_id { from_resources.clone() } else { self.fetch_dav_resources(access_token, to_account_id, SyncCollection::AddressBook) .await .caused_by(trc::location!())? }; // Validate headers let destination_resource_name = destination .resource .ok_or(DavError::Code(StatusCode::BAD_GATEWAY))?; let to_resource = to_resources.by_path(destination_resource_name); self.validate_headers( access_token, headers, vec![ ResourceState { account_id: from_account_id, collection: if from_resource.is_container() { Collection::AddressBook } else { Collection::ContactCard }, document_id: Some(from_resource.document_id()), path: from_resource_name, ..Default::default() }, ResourceState { account_id: to_account_id, collection: to_resource .map(|r| { if r.is_container() { Collection::AddressBook } else { Collection::ContactCard } }) .unwrap_or(Collection::AddressBook), document_id: Some(to_resource.map(|r| r.document_id()).unwrap_or(u32::MAX)), path: destination_resource_name, ..Default::default() }, ], Default::default(), if is_move { DavMethod::MOVE } else { DavMethod::COPY }, ) .await?; // Map destination if let Some(to_resource) = to_resource { if from_resource.path() == to_resource.path() { // Same resource return Err(DavError::Code(StatusCode::BAD_GATEWAY)); } let new_name = destination_resource_name .rsplit_once('/') .map(|(_, name)| name) .unwrap_or(destination_resource_name); match (from_resource.is_container(), to_resource.is_container()) { (true, true) => { let from_children_ids = from_resources .subtree(from_resource_name) .filter(|r| !r.is_container()) .map(|r| r.document_id()) .collect::>(); let to_document_ids = to_resources .subtree(destination_resource_name) .filter(|r| !r.is_container()) .map(|r| r.document_id()) .collect::>(); // Validate ACLs if !access_token.is_member(to_account_id) || (!access_token.is_member(from_account_id) && !from_resources.has_access_to_container( access_token, from_resource.document_id(), if is_move { Acl::RemoveItems } else { Acl::ReadItems }, )) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Overwrite container copy_container( self, access_token, from_account_id, from_resource.document_id(), from_children_ids, from_resources.format_collection(from_resource_name), to_account_id, to_resource.document_id().into(), to_document_ids, new_name, is_move, ) .await } (false, false) => { // Overwrite card let from_addressbook_id = from_resource.parent_id().unwrap(); let to_addressbook_id = to_resource.parent_id().unwrap(); // Validate ACL if (!access_token.is_member(from_account_id) && !from_resources.has_access_to_container( access_token, from_addressbook_id, if is_move { Acl::RemoveItems } else { Acl::ReadItems }, )) || (!access_token.is_member(to_account_id) && !to_resources.has_access_to_container( access_token, to_addressbook_id, Acl::RemoveItems, )) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } if is_move { move_card( self, access_token, from_account_id, from_resource.document_id(), from_addressbook_id, from_resources.format_item(from_resource_name), to_account_id, to_resource.document_id().into(), to_addressbook_id, new_name, ) .await } else { copy_card( self, access_token, from_account_id, from_resource.document_id(), to_account_id, to_resource.document_id().into(), to_addressbook_id, new_name, ) .await } } _ => Err(DavError::Code(StatusCode::BAD_GATEWAY)), } } else if let Some((parent_resource, new_name)) = to_resources.map_parent(destination_resource_name) { if let Some(parent_resource) = parent_resource { // Creating items under a card is not allowed // Copying/moving containers under a container is not allowed if !parent_resource.is_container() || from_resource.is_container() { return Err(DavError::Code(StatusCode::BAD_GATEWAY)); } // Validate ACL let from_addressbook_id = from_resource.parent_id().unwrap(); let to_addressbook_id = parent_resource.document_id(); if (!access_token.is_member(from_account_id) && !from_resources.has_access_to_container( access_token, from_addressbook_id, if is_move { Acl::RemoveItems } else { Acl::ReadItems }, )) || (!access_token.is_member(to_account_id) && !to_resources.has_access_to_container( access_token, to_addressbook_id, Acl::AddItems, )) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Copy/move card if is_move { if from_account_id != to_account_id || parent_resource.document_id() != from_addressbook_id { move_card( self, access_token, from_account_id, from_resource.document_id(), from_addressbook_id, from_resources.format_item(from_resource_name), to_account_id, None, to_addressbook_id, new_name, ) .await } else { rename_card( self, access_token, from_account_id, from_resource.document_id(), from_addressbook_id, new_name, from_resources.format_item(from_resource_name), ) .await } } else { copy_card( self, access_token, from_account_id, from_resource.document_id(), to_account_id, None, to_addressbook_id, new_name, ) .await } } else { // Copying/moving cards to the root is not allowed if !from_resource.is_container() { return Err(DavError::Code(StatusCode::BAD_GATEWAY)); } // Shared users cannot create containers if !access_token.is_member(to_account_id) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Validate ACLs if !access_token.is_member(from_account_id) && !from_resources.has_access_to_container( access_token, from_resource.document_id(), if is_move { Acl::RemoveItems } else { Acl::ReadItems }, ) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Copy/move container let from_children_ids = from_resources .subtree(from_resource_name) .filter(|r| !r.is_container()) .map(|r| r.document_id()) .collect::>(); if is_move { if from_account_id != to_account_id { copy_container( self, access_token, from_account_id, from_resource.document_id(), if headers.depth != Depth::Zero { from_children_ids } else { return Err(DavError::Code(StatusCode::BAD_GATEWAY)); }, from_resources.format_collection(from_resource_name), to_account_id, None, vec![], new_name, true, ) .await } else { rename_container( self, access_token, from_account_id, from_resource.document_id(), new_name, from_resources.format_collection(from_resource_name), ) .await } } else { copy_container( self, access_token, from_account_id, from_resource.document_id(), if headers.depth != Depth::Zero { from_children_ids } else { vec![] }, from_resources.format_collection(from_resource_name), to_account_id, None, vec![], new_name, false, ) .await } } } else { Err(DavError::Code(StatusCode::CONFLICT)) } } } #[allow(clippy::too_many_arguments)] async fn copy_card( server: &Server, access_token: &AccessToken, from_account_id: u32, from_document_id: u32, to_account_id: u32, to_document_id: Option, to_addressbook_id: u32, new_name: &str, ) -> crate::Result { // Fetch card let card_ = server .store() .get_value::>(ValueKey::archive( from_account_id, Collection::ContactCard, from_document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let card = card_ .to_unarchived::() .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); // Validate UID assert_is_unique_uid( server, server .fetch_dav_resources(access_token, to_account_id, SyncCollection::AddressBook) .await .caused_by(trc::location!())? .as_ref(), to_account_id, to_addressbook_id, card.inner.card.uid(), ) .await?; if from_account_id == to_account_id { let mut new_card = card .deserialize::() .caused_by(trc::location!())?; new_card.names.push(DavName { name: new_name.to_string(), parent_id: to_addressbook_id, }); new_card .update( access_token, card, from_account_id, from_document_id, &mut batch, ) .caused_by(trc::location!())?; } else { let mut new_card = card .deserialize::() .caused_by(trc::location!())?; new_card.names = vec![DavName { name: new_name.to_string(), parent_id: to_addressbook_id, }]; let to_document_id = server .store() .assign_document_ids(to_account_id, Collection::ContactCard, 1) .await .caused_by(trc::location!())?; new_card .insert(access_token, to_account_id, to_document_id, &mut batch) .caused_by(trc::location!())?; } let response = if let Some(to_document_id) = to_document_id { // Overwrite card on destination let card_ = server .store() .get_value::>(ValueKey::archive( to_account_id, Collection::ContactCard, to_document_id, )) .await .caused_by(trc::location!())?; if let Some(card_) = card_ { let card = card_ .to_unarchived::() .caused_by(trc::location!())?; DestroyArchive(card) .delete( access_token, to_account_id, to_document_id, to_addressbook_id, None, &mut batch, ) .caused_by(trc::location!())?; } Ok(HttpResponse::new(StatusCode::NO_CONTENT)) } else { Ok(HttpResponse::new(StatusCode::CREATED)) }; server .commit_batch(batch) .await .caused_by(trc::location!())?; server.notify_task_queue(); response } #[allow(clippy::too_many_arguments)] async fn move_card( server: &Server, access_token: &AccessToken, from_account_id: u32, from_document_id: u32, from_addressbook_id: u32, from_resource_path: String, to_account_id: u32, to_document_id: Option, to_addressbook_id: u32, new_name: &str, ) -> crate::Result { // Fetch card let card_ = server .store() .get_value::>(ValueKey::archive( from_account_id, Collection::ContactCard, from_document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let card = card_ .to_unarchived::() .caused_by(trc::location!())?; // Validate UID if from_account_id != to_account_id || from_addressbook_id != to_addressbook_id || to_document_id.is_none() { assert_is_unique_uid( server, server .fetch_dav_resources(access_token, to_account_id, SyncCollection::AddressBook) .await .caused_by(trc::location!())? .as_ref(), to_account_id, to_addressbook_id, card.inner.card.uid(), ) .await?; } let mut batch = BatchBuilder::new(); if from_account_id == to_account_id { let mut name_idx = None; for (idx, name) in card.inner.names.iter().enumerate() { if name.parent_id == from_addressbook_id { name_idx = Some(idx); break; } } let name_idx = if let Some(name_idx) = name_idx { name_idx } else { return Err(DavError::Code(StatusCode::NOT_FOUND)); }; let mut new_card = card .deserialize::() .caused_by(trc::location!())?; new_card.names.swap_remove(name_idx); new_card.names.push(DavName { name: new_name.to_string(), parent_id: to_addressbook_id, }); new_card .update( access_token, card.clone(), from_account_id, from_document_id, &mut batch, ) .caused_by(trc::location!())?; batch.log_vanished_item(VanishedCollection::AddressBook, from_resource_path); } else { let mut new_card = card .deserialize::() .caused_by(trc::location!())?; new_card.names = vec![DavName { name: new_name.to_string(), parent_id: to_addressbook_id, }]; DestroyArchive(card) .delete( access_token, from_account_id, from_document_id, from_addressbook_id, from_resource_path.into(), &mut batch, ) .caused_by(trc::location!())?; let to_document_id = server .store() .assign_document_ids(to_account_id, Collection::ContactCard, 1) .await .caused_by(trc::location!())?; new_card .insert(access_token, to_account_id, to_document_id, &mut batch) .caused_by(trc::location!())?; } let response = if let Some(to_document_id) = to_document_id { // Overwrite card on destination let card_ = server .store() .get_value::>(ValueKey::archive( to_account_id, Collection::ContactCard, to_document_id, )) .await .caused_by(trc::location!())?; if let Some(card_) = card_ { let card = card_ .to_unarchived::() .caused_by(trc::location!())?; DestroyArchive(card) .delete( access_token, to_account_id, to_document_id, to_addressbook_id, None, &mut batch, ) .caused_by(trc::location!())?; } Ok(HttpResponse::new(StatusCode::NO_CONTENT)) } else { Ok(HttpResponse::new(StatusCode::CREATED)) }; server .commit_batch(batch) .await .caused_by(trc::location!())?; server.notify_task_queue(); response } #[allow(clippy::too_many_arguments)] async fn rename_card( server: &Server, access_token: &AccessToken, account_id: u32, document_id: u32, addressbook_id: u32, new_name: &str, from_resource_path: String, ) -> crate::Result { // Fetch card let card_ = server .store() .get_value::>(ValueKey::archive( account_id, Collection::ContactCard, document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let card = card_ .to_unarchived::() .caused_by(trc::location!())?; let name_idx = card .inner .names .iter() .position(|n| n.parent_id == addressbook_id) .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let mut new_card = card .deserialize::() .caused_by(trc::location!())?; new_card.names[name_idx].name = new_name.to_string(); let mut batch = BatchBuilder::new(); new_card .update(access_token, card, account_id, document_id, &mut batch) .caused_by(trc::location!())?; batch.log_vanished_item(VanishedCollection::AddressBook, from_resource_path); server .commit_batch(batch) .await .caused_by(trc::location!())?; server.notify_task_queue(); Ok(HttpResponse::new(StatusCode::CREATED)) } #[allow(clippy::too_many_arguments)] async fn copy_container( server: &Server, access_token: &AccessToken, from_account_id: u32, from_document_id: u32, from_children_ids: Vec, from_resource_path: String, to_account_id: u32, to_document_id: Option, to_children_ids: Vec, new_name: &str, remove_source: bool, ) -> crate::Result { // Fetch book let book_ = server .store() .get_value::>(ValueKey::archive( from_account_id, Collection::AddressBook, from_document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let old_book = book_ .to_unarchived::() .caused_by(trc::location!())?; let mut book = old_book .deserialize::() .caused_by(trc::location!())?; // Prepare write batch let mut batch = BatchBuilder::new(); if remove_source { DestroyArchive(old_book) .delete( access_token, from_account_id, from_document_id, from_resource_path.into(), &mut batch, ) .caused_by(trc::location!())?; } let preference = book.preferences.into_iter().next().unwrap(); book.name = new_name.to_string(); book.subscribers.clear(); book.acls.clear(); book.preferences = vec![AddressBookPreferences { account_id: to_account_id, name: preference.name, description: preference.description, sort_order: 0, }]; let is_overwrite = to_document_id.is_some(); let to_document_id = if let Some(to_document_id) = to_document_id { // Overwrite destination let book_ = server .store() .get_value::>(ValueKey::archive( to_account_id, Collection::AddressBook, to_document_id, )) .await .caused_by(trc::location!())?; if let Some(book_) = book_ { let book = book_ .to_unarchived::() .caused_by(trc::location!())?; DestroyArchive(book) .delete_with_cards( server, access_token, to_account_id, to_document_id, to_children_ids, None, &mut batch, ) .await .caused_by(trc::location!())?; } to_document_id } else { server .store() .assign_document_ids(to_account_id, Collection::AddressBook, 1) .await .caused_by(trc::location!())? }; book.insert(access_token, to_account_id, to_document_id, &mut batch) .caused_by(trc::location!())?; // Copy children let mut required_space = 0; for from_child_document_id in from_children_ids { if let Some(card_) = server .store() .get_value::>(ValueKey::archive( from_account_id, Collection::ContactCard, from_child_document_id, )) .await? { let card = card_ .to_unarchived::() .caused_by(trc::location!())?; let mut new_name = None; for name in card.inner.names.iter() { if name.parent_id == to_document_id { continue; } else if name.parent_id == from_document_id { new_name = Some(name.name.to_string()); } } let new_name = if let Some(new_name) = new_name { DavName { name: new_name, parent_id: to_document_id, } } else { continue; }; let card = card_ .to_unarchived::() .caused_by(trc::location!())?; let mut new_card = card .deserialize::() .caused_by(trc::location!())?; if from_account_id == to_account_id { if remove_source { new_card .names .retain(|name| name.parent_id != from_document_id); } new_card.names.push(new_name); new_card .update( access_token, card, from_account_id, from_child_document_id, &mut batch, ) .caused_by(trc::location!())?; } else { if remove_source { DestroyArchive(card) .delete( access_token, from_account_id, from_child_document_id, from_document_id, None, &mut batch, ) .caused_by(trc::location!())?; } let to_document_id = server .store() .assign_document_ids(to_account_id, Collection::ContactCard, 1) .await .caused_by(trc::location!())?; new_card.names = vec![new_name]; required_space += new_card.size as u64; new_card .insert(access_token, to_account_id, to_document_id, &mut batch) .caused_by(trc::location!())?; } } } if from_account_id != to_account_id && required_space > 0 { server .has_available_quota( &server .get_resource_token(access_token, to_account_id) .await?, required_space, ) .await?; } server .commit_batch(batch) .await .caused_by(trc::location!())?; server.notify_task_queue(); if !is_overwrite { Ok(HttpResponse::new(StatusCode::CREATED)) } else { Ok(HttpResponse::new(StatusCode::NO_CONTENT)) } } #[allow(clippy::too_many_arguments)] async fn rename_container( server: &Server, access_token: &AccessToken, account_id: u32, document_id: u32, new_name: &str, from_resource_path: String, ) -> crate::Result { // Fetch book let book_ = server .store() .get_value::>(ValueKey::archive( account_id, Collection::AddressBook, document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let book = book_ .to_unarchived::() .caused_by(trc::location!())?; let mut new_book = book .deserialize::() .caused_by(trc::location!())?; new_book.name = new_name.to_string(); let mut batch = BatchBuilder::new(); new_book .update(access_token, book, account_id, document_id, &mut batch) .caused_by(trc::location!())?; batch.log_vanished_item(VanishedCollection::AddressBook, from_resource_path); server .commit_batch(batch) .await .caused_by(trc::location!())?; server.notify_task_queue(); Ok(HttpResponse::new(StatusCode::CREATED)) } ================================================ FILE: crates/dav/src/card/delete.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ DavError, DavMethod, common::{ ETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, }; use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; use dav_proto::RequestHeaders; use groupware::{ DestroyArchive, cache::GroupwareCache, contact::{AddressBook, ContactCard}, }; use http_proto::HttpResponse; use hyper::StatusCode; use store::write::{BatchBuilder, ValueClass}; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, field::PrincipalField, }; pub(crate) trait CardDeleteRequestHandler: Sync + Send { fn handle_card_delete_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, ) -> impl Future> + Send; } impl CardDeleteRequestHandler for Server { async fn handle_card_delete_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, ) -> crate::Result { // Validate URI let resource = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let account_id = resource.account_id; let delete_path = resource .resource .filter(|r| !r.is_empty()) .ok_or(DavError::Code(StatusCode::FORBIDDEN))?; let resources = self .fetch_dav_resources(access_token, account_id, SyncCollection::AddressBook) .await .caused_by(trc::location!())?; // Check resource type let delete_resource = resources .by_path(delete_path) .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let document_id = delete_resource.document_id(); // Fetch entry let mut batch = BatchBuilder::new(); if delete_resource.is_container() { let book_ = self .store() .get_value::>(ValueKey::archive( account_id, Collection::AddressBook, document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let book = book_ .to_unarchived::() .caused_by(trc::location!())?; // Validate ACL if !access_token.is_member(account_id) && !book .inner .acls .effective_acl(access_token) .contains_all([Acl::Delete, Acl::RemoveItems].into_iter()) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Validate headers self.validate_headers( access_token, headers, vec![ResourceState { account_id, collection: Collection::AddressBook, document_id: document_id.into(), etag: book.etag().into(), path: delete_path, ..Default::default() }], Default::default(), DavMethod::DELETE, ) .await?; // Delete addressbook and cards DestroyArchive(book) .delete_with_cards( self, access_token, account_id, document_id, resources .subtree(delete_path) .filter(|r| !r.is_container()) .map(|r| r.document_id()) .collect::>(), resources.format_resource(delete_resource).into(), &mut batch, ) .await .caused_by(trc::location!())?; // Reset default address book id let default_book_id = self .store() .get_value::(ValueKey { account_id, collection: Collection::Principal.into(), document_id: 0, class: ValueClass::Property(PrincipalField::DefaultAddressBookId.into()), }) .await .caused_by(trc::location!())?; if default_book_id.is_some_and(|id| id == document_id) { batch .with_account_id(account_id) .with_collection(Collection::Principal) .with_document(0) .clear(PrincipalField::DefaultAddressBookId); } } else { // Validate ACL let addressbook_id = delete_resource.parent_id().unwrap(); if !access_token.is_member(account_id) && !resources.has_access_to_container( access_token, addressbook_id, Acl::RemoveItems, ) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } let card_ = self .store() .get_value::>(ValueKey::archive( account_id, Collection::ContactCard, document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; // Validate headers self.validate_headers( access_token, headers, vec![ResourceState { account_id, collection: Collection::ContactCard, document_id: document_id.into(), etag: card_.etag().into(), path: delete_path, ..Default::default() }], Default::default(), DavMethod::DELETE, ) .await?; // Delete card DestroyArchive( card_ .to_unarchived::() .caused_by(trc::location!())?, ) .delete( access_token, account_id, document_id, addressbook_id, resources.format_resource(delete_resource).into(), &mut batch, ) .caused_by(trc::location!())?; } self.commit_batch(batch).await.caused_by(trc::location!())?; self.notify_task_queue(); Ok(HttpResponse::new(StatusCode::NO_CONTENT)) } } ================================================ FILE: crates/dav/src/card/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ DavError, DavMethod, common::{ ETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, }; use common::{Server, auth::AccessToken}; use dav_proto::{RequestHeaders, schema::property::Rfc1123DateTime}; use groupware::{cache::GroupwareCache, contact::ContactCard}; use http_proto::HttpResponse; use hyper::StatusCode; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, }; pub(crate) trait CardGetRequestHandler: Sync + Send { fn handle_card_get_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, is_head: bool, ) -> impl Future> + Send; } impl CardGetRequestHandler for Server { async fn handle_card_get_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, is_head: bool, ) -> crate::Result { // Validate URI let resource_ = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let account_id = resource_.account_id; let resources = self .fetch_dav_resources(access_token, account_id, SyncCollection::AddressBook) .await .caused_by(trc::location!())?; let resource = resources .by_path( resource_ .resource .ok_or(DavError::Code(StatusCode::METHOD_NOT_ALLOWED))?, ) .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; if resource.is_container() { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } // Validate ACL if !access_token.is_member(account_id) && !resources.has_access_to_container( access_token, resource.parent_id().unwrap(), Acl::ReadItems, ) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Fetch card let card_ = self .store() .get_value::>(ValueKey::archive( account_id, Collection::ContactCard, resource.document_id(), )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let card = card_ .unarchive::() .caused_by(trc::location!())?; // Validate headers let etag = card_.etag(); self.validate_headers( access_token, headers, vec![ResourceState { account_id, collection: Collection::ContactCard, document_id: resource.document_id().into(), etag: etag.clone().into(), path: resource_.resource.unwrap(), ..Default::default() }], Default::default(), DavMethod::GET, ) .await?; let response = HttpResponse::new(StatusCode::OK) .with_content_type("text/vcard; charset=utf-8") .with_etag(etag) .with_last_modified(Rfc1123DateTime::new(i64::from(card.modified)).to_string()); let mut vcard = String::with_capacity(128); let _ = card.card.write_to( &mut vcard, headers .max_vcard_version .or_else(|| card.card.version()) .unwrap_or_default(), ); if !is_head { Ok(response.with_binary_body(vcard)) } else { Ok(response.with_content_length(vcard.len())) } } } ================================================ FILE: crates/dav/src/card/mkcol.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::proppatch::CardPropPatchRequestHandler; use crate::{ DavError, DavMethod, PropStatBuilder, common::{ ExtractETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, }; use common::{Server, auth::AccessToken}; use dav_proto::{ RequestHeaders, Return, schema::{Namespace, request::MkCol, response::MkColResponse}, }; use groupware::{ cache::GroupwareCache, contact::{AddressBook, AddressBookPreferences}, }; use http_proto::HttpResponse; use hyper::StatusCode; use store::write::BatchBuilder; use trc::AddContext; use types::collection::{Collection, SyncCollection}; pub(crate) trait CardMkColRequestHandler: Sync + Send { fn handle_card_mkcol_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: Option, ) -> impl Future> + Send; } impl CardMkColRequestHandler for Server { async fn handle_card_mkcol_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: Option, ) -> crate::Result { // Validate URI let resource = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let account_id = resource.account_id; let name = resource .resource .ok_or(DavError::Code(StatusCode::FORBIDDEN))?; if !access_token.is_member(account_id) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } else if name.contains('/') || self .fetch_dav_resources(access_token, account_id, SyncCollection::AddressBook) .await .caused_by(trc::location!())? .by_path(name) .is_some() { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } // Validate headers self.validate_headers( access_token, headers, vec![ResourceState { account_id, collection: resource.collection, document_id: Some(u32::MAX), path: name, ..Default::default() }], Default::default(), DavMethod::MKCOL, ) .await?; // Build file container let mut book = AddressBook { name: name.to_string(), preferences: vec![AddressBookPreferences { account_id, name: "Address Book".to_string(), ..Default::default() }], ..Default::default() }; // Apply MKCOL properties let mut return_prop_stat = None; if let Some(mkcol) = request { let mut prop_stat = PropStatBuilder::default(); if !self.apply_addressbook_properties( access_token, &mut book, false, mkcol.props, &mut prop_stat, ) { return Ok(HttpResponse::new(StatusCode::FORBIDDEN).with_xml_body( MkColResponse::new(prop_stat.build()) .with_namespace(Namespace::CardDav) .to_string(), )); } if headers.ret != Return::Minimal { return_prop_stat = Some(prop_stat); } } // Prepare write batch let mut batch = BatchBuilder::new(); let document_id = self .store() .assign_document_ids(account_id, Collection::AddressBook, 1) .await .caused_by(trc::location!())?; book.insert(access_token, account_id, document_id, &mut batch) .caused_by(trc::location!())?; let etag = batch.etag(); self.commit_batch(batch).await.caused_by(trc::location!())?; if let Some(prop_stat) = return_prop_stat { Ok(HttpResponse::new(StatusCode::CREATED) .with_xml_body( MkColResponse::new(prop_stat.build()) .with_namespace(Namespace::CardDav) .to_string(), ) .with_etag_opt(etag)) } else { Ok(HttpResponse::new(StatusCode::CREATED).with_etag_opt(etag)) } } } ================================================ FILE: crates/dav/src/card/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{DavError, DavErrorCondition}; use common::{DavResources, Server}; use dav_proto::schema::{ property::{CardDavProperty, DavProperty, WebDavProperty}, response::CardCondition, }; use hyper::StatusCode; use trc::AddContext; use types::{collection::Collection, field::ContactField}; pub mod copy_move; pub mod delete; pub mod get; pub mod mkcol; pub mod proppatch; pub mod query; pub mod update; pub(crate) static CARD_CONTAINER_PROPS: [DavProperty; 23] = [ DavProperty::WebDav(WebDavProperty::CreationDate), DavProperty::WebDav(WebDavProperty::DisplayName), DavProperty::WebDav(WebDavProperty::GetETag), DavProperty::WebDav(WebDavProperty::GetLastModified), DavProperty::WebDav(WebDavProperty::ResourceType), DavProperty::WebDav(WebDavProperty::LockDiscovery), DavProperty::WebDav(WebDavProperty::SupportedLock), DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal), DavProperty::WebDav(WebDavProperty::SyncToken), DavProperty::WebDav(WebDavProperty::Owner), DavProperty::WebDav(WebDavProperty::SupportedPrivilegeSet), DavProperty::WebDav(WebDavProperty::CurrentUserPrivilegeSet), DavProperty::WebDav(WebDavProperty::Acl), DavProperty::WebDav(WebDavProperty::AclRestrictions), DavProperty::WebDav(WebDavProperty::InheritedAclSet), DavProperty::WebDav(WebDavProperty::PrincipalCollectionSet), DavProperty::WebDav(WebDavProperty::SupportedReportSet), DavProperty::WebDav(WebDavProperty::QuotaAvailableBytes), DavProperty::WebDav(WebDavProperty::QuotaUsedBytes), DavProperty::CardDav(CardDavProperty::AddressbookDescription), DavProperty::CardDav(CardDavProperty::SupportedAddressData), DavProperty::CardDav(CardDavProperty::SupportedCollationSet), DavProperty::CardDav(CardDavProperty::MaxResourceSize), ]; pub(crate) static CARD_ITEM_PROPS: [DavProperty; 20] = [ DavProperty::WebDav(WebDavProperty::CreationDate), DavProperty::WebDav(WebDavProperty::DisplayName), DavProperty::WebDav(WebDavProperty::GetETag), DavProperty::WebDav(WebDavProperty::GetLastModified), DavProperty::WebDav(WebDavProperty::ResourceType), DavProperty::WebDav(WebDavProperty::LockDiscovery), DavProperty::WebDav(WebDavProperty::SupportedLock), DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal), DavProperty::WebDav(WebDavProperty::SyncToken), DavProperty::WebDav(WebDavProperty::Owner), DavProperty::WebDav(WebDavProperty::SupportedPrivilegeSet), DavProperty::WebDav(WebDavProperty::CurrentUserPrivilegeSet), DavProperty::WebDav(WebDavProperty::Acl), DavProperty::WebDav(WebDavProperty::AclRestrictions), DavProperty::WebDav(WebDavProperty::InheritedAclSet), DavProperty::WebDav(WebDavProperty::PrincipalCollectionSet), DavProperty::WebDav(WebDavProperty::GetContentLanguage), DavProperty::WebDav(WebDavProperty::GetContentLength), DavProperty::WebDav(WebDavProperty::GetContentType), DavProperty::CardDav(CardDavProperty::AddressData(vec![])), ]; pub(crate) async fn assert_is_unique_uid( server: &Server, resources: &DavResources, account_id: u32, addressbook_id: u32, uid: Option<&str>, ) -> crate::Result<()> { if let Some(uid) = uid { let hits = server .document_ids_matching( account_id, Collection::ContactCard, ContactField::Uid, uid.as_bytes(), ) .await .caused_by(trc::location!())?; if !hits.is_empty() { for path in resources.children(addressbook_id) { if hits.contains(path.document_id()) { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, CardCondition::NoUidConflict(resources.format_resource(path).into()), ))); } } } } Ok(()) } ================================================ FILE: crates/dav/src/card/proppatch.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ DavError, DavMethod, PropStatBuilder, common::{ ETag, ExtractETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, }; use common::{Server, auth::AccessToken}; use dav_proto::{ RequestHeaders, Return, schema::{ Namespace, property::{CardDavProperty, DavProperty, DavValue, ResourceType, WebDavProperty}, request::{DavPropertyValue, PropertyUpdate}, response::{BaseCondition, MultiStatus, Response}, }, }; use groupware::{ cache::GroupwareCache, contact::{AddressBook, ContactCard}, }; use http_proto::HttpResponse; use hyper::StatusCode; use store::write::BatchBuilder; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, }; pub(crate) trait CardPropPatchRequestHandler: Sync + Send { fn handle_card_proppatch_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: PropertyUpdate, ) -> impl Future> + Send; fn apply_addressbook_properties( &self, access_token: &AccessToken, address_book: &mut AddressBook, is_update: bool, properties: Vec, items: &mut PropStatBuilder, ) -> bool; fn apply_card_properties( &self, card: &mut ContactCard, is_update: bool, properties: Vec, items: &mut PropStatBuilder, ) -> bool; } impl CardPropPatchRequestHandler for Server { async fn handle_card_proppatch_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, mut request: PropertyUpdate, ) -> crate::Result { // Validate URI let resource_ = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let uri = headers.uri; let account_id = resource_.account_id; let resources = self .fetch_dav_resources(access_token, account_id, SyncCollection::AddressBook) .await .caused_by(trc::location!())?; let resource = resource_ .resource .and_then(|r| resources.by_path(r)) .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let document_id = resource.document_id(); let collection = if resource.is_container() { Collection::AddressBook } else { Collection::ContactCard }; if !request.has_changes() { return Ok(HttpResponse::new(StatusCode::NO_CONTENT)); } // Verify ACL if !access_token.is_member(account_id) { let (acl, document_id) = if resource.is_container() { (Acl::Modify, resource.document_id()) } else { (Acl::ModifyItems, resource.parent_id().unwrap()) }; if !resources.has_access_to_container(access_token, document_id, acl) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } } // Fetch archive let archive = self .store() .get_value::>(ValueKey::archive( account_id, collection, document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; // Validate headers self.validate_headers( access_token, headers, vec![ResourceState { account_id, collection, document_id: document_id.into(), etag: archive.etag().into(), path: resource_.resource.unwrap(), ..Default::default() }], Default::default(), DavMethod::PROPPATCH, ) .await?; let is_success; let mut batch = BatchBuilder::new(); let mut items = PropStatBuilder::default(); let etag = if resource.is_container() { // Deserialize let book = archive .to_unarchived::() .caused_by(trc::location!())?; let mut new_book = archive .deserialize::() .caused_by(trc::location!())?; // Remove properties if !request.set_first && !request.remove.is_empty() { remove_addressbook_properties( access_token, &mut new_book, std::mem::take(&mut request.remove), &mut items, ); } // Set properties is_success = self.apply_addressbook_properties( access_token, &mut new_book, true, request.set, &mut items, ); // Remove properties if is_success && !request.remove.is_empty() { remove_addressbook_properties( access_token, &mut new_book, request.remove, &mut items, ); } if is_success { new_book .update(access_token, book, account_id, document_id, &mut batch) .caused_by(trc::location!())? .etag() } else { book.etag().into() } } else { // Deserialize let card = archive .to_unarchived::() .caused_by(trc::location!())?; let mut new_card = archive .deserialize::() .caused_by(trc::location!())?; // Remove properties if !request.set_first && !request.remove.is_empty() { remove_card_properties( &mut new_card, std::mem::take(&mut request.remove), &mut items, ); } // Set properties is_success = self.apply_card_properties(&mut new_card, true, request.set, &mut items); // Remove properties if is_success && !request.remove.is_empty() { remove_card_properties(&mut new_card, request.remove, &mut items); } if is_success { new_card .update(access_token, card, account_id, document_id, &mut batch) .caused_by(trc::location!())? .etag() } else { card.etag().into() } }; if is_success { self.commit_batch(batch).await.caused_by(trc::location!())?; } if headers.ret != Return::Minimal || !is_success { Ok(HttpResponse::new(StatusCode::MULTI_STATUS) .with_xml_body( MultiStatus::new(vec![Response::new_propstat(uri, items.build())]) .with_namespace(Namespace::CardDav) .to_string(), ) .with_etag_opt(etag)) } else { Ok(HttpResponse::new(StatusCode::NO_CONTENT).with_etag_opt(etag)) } } fn apply_addressbook_properties( &self, access_token: &AccessToken, address_book: &mut AddressBook, is_update: bool, properties: Vec, items: &mut PropStatBuilder, ) -> bool { let mut has_errors = false; for property in properties { match (&property.property, property.value) { (DavProperty::WebDav(WebDavProperty::DisplayName), DavValue::String(name)) => { if name.len() <= self.core.groupware.live_property_size { address_book.preferences_mut(access_token).name = name; items.insert_ok(property.property); } else { items.insert_error_with_description( property.property, StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); has_errors = true; } } ( DavProperty::CardDav(CardDavProperty::AddressbookDescription), DavValue::String(name), ) => { if name.len() <= self.core.groupware.live_property_size { address_book.preferences_mut(access_token).description = Some(name); items.insert_ok(property.property); } else { items.insert_error_with_description( property.property, StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); has_errors = true; } } (DavProperty::WebDav(WebDavProperty::CreationDate), DavValue::Timestamp(dt)) => { address_book.created = dt; items.insert_ok(property.property); } ( DavProperty::WebDav(WebDavProperty::ResourceType), DavValue::ResourceTypes(types), ) => { if !types.0.iter().all(|rt| { matches!(rt, ResourceType::Collection | ResourceType::AddressBook) }) { items.insert_precondition_failed( property.property, StatusCode::FORBIDDEN, BaseCondition::ValidResourceType, ); has_errors = true; } else { items.insert_ok(property.property); } } (DavProperty::DeadProperty(dead), DavValue::DeadProperty(values)) if self.core.groupware.dead_property_size.is_some() => { if is_update { address_book.dead_properties.remove_element(dead); } if address_book.dead_properties.size() + values.size() + dead.size() < self.core.groupware.dead_property_size.unwrap() { address_book .dead_properties .add_element(dead.clone(), values.0); items.insert_ok(property.property); } else { items.insert_error_with_description( property.property, StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); has_errors = true; } } (_, DavValue::Null) => { items.insert_ok(property.property); } _ => { items.insert_error_with_description( property.property, StatusCode::CONFLICT, "Property cannot be modified", ); has_errors = true; } } } !has_errors } fn apply_card_properties( &self, card: &mut ContactCard, is_update: bool, properties: Vec, items: &mut PropStatBuilder, ) -> bool { let mut has_errors = false; for property in properties { match (&property.property, property.value) { (DavProperty::WebDav(WebDavProperty::DisplayName), DavValue::String(name)) => { if name.len() <= self.core.groupware.live_property_size { card.display_name = Some(name); items.insert_ok(property.property); } else { items.insert_error_with_description( property.property, StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); has_errors = true; } } (DavProperty::WebDav(WebDavProperty::CreationDate), DavValue::Timestamp(dt)) => { card.created = dt; items.insert_ok(property.property); } (DavProperty::DeadProperty(dead), DavValue::DeadProperty(values)) if self.core.groupware.dead_property_size.is_some() => { if is_update { card.dead_properties.remove_element(dead); } if card.dead_properties.size() + values.size() + dead.size() < self.core.groupware.dead_property_size.unwrap() { card.dead_properties.add_element(dead.clone(), values.0); items.insert_ok(property.property); } else { items.insert_error_with_description( property.property, StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); has_errors = true; } } (_, DavValue::Null) => { items.insert_ok(property.property); } _ => { items.insert_error_with_description( property.property, StatusCode::CONFLICT, "Property cannot be modified", ); has_errors = true; } } } !has_errors } } fn remove_card_properties( card: &mut ContactCard, properties: Vec, items: &mut PropStatBuilder, ) { for property in properties { match &property { DavProperty::WebDav(WebDavProperty::DisplayName) => { card.display_name = None; items.insert_with_status(property, StatusCode::NO_CONTENT); } DavProperty::DeadProperty(dead) => { card.dead_properties.remove_element(dead); items.insert_with_status(property, StatusCode::NO_CONTENT); } _ => { items.insert_error_with_description( property, StatusCode::CONFLICT, "Property cannot be deleted", ); } } } } fn remove_addressbook_properties( access_token: &AccessToken, book: &mut AddressBook, properties: Vec, items: &mut PropStatBuilder, ) { for property in properties { match &property { DavProperty::CardDav(CardDavProperty::AddressbookDescription) => { book.preferences_mut(access_token).description = None; items.insert_with_status(property, StatusCode::NO_CONTENT); } DavProperty::WebDav(WebDavProperty::DisplayName) => { book.preferences_mut(access_token).name.clear(); items.insert_with_status(property, StatusCode::NO_CONTENT); } DavProperty::DeadProperty(dead) => { book.dead_properties.remove_element(dead); items.insert_with_status(property, StatusCode::NO_CONTENT); } _ => { items.insert_error_with_description( property, StatusCode::CONFLICT, "Property cannot be deleted", ); } } } } ================================================ FILE: crates/dav/src/card/query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ DavError, common::{ AddressbookFilter, DavQuery, propfind::{PropFindItem, PropFindRequestHandler}, uri::DavUriResource, }, }; use calcard::vcard::{ ArchivedVCard, ArchivedVCardEntry, ArchivedVCardParameter, VCardParameterName, VCardProperty, VCardVersion, }; use common::{Server, auth::AccessToken}; use dav_proto::{ RequestHeaders, schema::{ property::CardDavPropertyName, request::{AddressbookQuery, Filter, FilterOp, VCardPropertyWithGroup}, response::MultiStatus, }, }; use groupware::cache::GroupwareCache; use http_proto::HttpResponse; use hyper::StatusCode; use std::fmt::Write; use trc::AddContext; use types::{acl::Acl, collection::SyncCollection}; pub(crate) trait CardQueryRequestHandler: Sync + Send { fn handle_card_query_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: AddressbookQuery, ) -> impl Future> + Send; } impl CardQueryRequestHandler for Server { async fn handle_card_query_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: AddressbookQuery, ) -> crate::Result { // Validate URI let resource_ = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let account_id = resource_.account_id; let resources = self .fetch_dav_resources(access_token, account_id, SyncCollection::AddressBook) .await .caused_by(trc::location!())?; let Some(resource) = resources.by_path( resource_ .resource .ok_or(DavError::Code(StatusCode::METHOD_NOT_ALLOWED))?, ) else { return Ok(HttpResponse::new(StatusCode::MULTI_STATUS) .with_xml_body(MultiStatus::not_found(headers.uri).to_string())); }; if !resource.is_container() { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } // Obtain shared ids let shared_ids = if !access_token.is_member(account_id) { resources .shared_containers(access_token, [Acl::ReadItems], false) .into() } else { None }; // Obtain document ids in folder let mut items = Vec::with_capacity(16); for resource in resources.children(resource.document_id()) { if shared_ids .as_ref() .is_none_or(|ids| ids.contains(resource.document_id())) { items.push(PropFindItem::new( resources.format_resource(resource), account_id, resource, )); } } self.handle_dav_query( access_token, DavQuery::addressbook_query(request, items, headers), ) .await } } pub(crate) fn vcard_query(card: &ArchivedVCard, filters: &AddressbookFilter) -> bool { let mut is_all = true; let mut matches_one = false; for filter in filters { match filter { Filter::AnyOf => { is_all = false; } Filter::AllOf => { is_all = true; } Filter::Property { prop, op, .. } => { let mut properties = find_properties(card, prop).peekable(); let result = if properties.peek().is_some() { properties.any(|entry| match op { FilterOp::Exists => true, FilterOp::Undefined => false, FilterOp::TextMatch(text_match) => { let mut matched_any = false; for value in entry.values.iter() { if let Some(text) = value.as_text() && text_match.matches(text) { matched_any = true; break; } } matched_any } FilterOp::TimeRange(_) => false, }) } else { matches!(op, FilterOp::Undefined) }; if result { matches_one = true; } else if is_all { return false; } } Filter::Parameter { prop, param, op, .. } => { let mut properties = find_properties(card, prop) .filter_map(|entry| find_parameter(entry, param)) .peekable(); let result = if properties.peek().is_some() { properties.any(|entry| match op { FilterOp::Exists => true, FilterOp::Undefined => false, FilterOp::TextMatch(text_match) => { if let Some(text) = entry.value.as_text() { text_match.matches(text) } else { false } } FilterOp::TimeRange(_) => false, }) } else { matches!(op, FilterOp::Undefined) }; if result { matches_one = true; } else if is_all { return false; } } Filter::Component { .. } => {} } } is_all || matches_one } #[inline(always)] fn find_properties<'x>( card: &'x ArchivedVCard, prop: &VCardPropertyWithGroup, ) -> impl Iterator { card.entries .iter() .filter(move |entry| entry.name == prop.name && entry.group == prop.group) } #[inline(always)] fn find_parameter<'x>( entry: &'x ArchivedVCardEntry, name: &VCardParameterName, ) -> Option<&'x ArchivedVCardParameter> { entry.params.iter().find(|param| param.name == *name) } pub(crate) fn serialize_vcard_with_props( card: &ArchivedVCard, props: &[CardDavPropertyName], version: Option, ) -> String { let mut vcard = String::with_capacity(128); let version = version.or_else(|| card.version()).unwrap_or_default(); if !props.is_empty() { let _ = write!(&mut vcard, "BEGIN:VCARD\r\n"); let is_v4 = matches!(version, VCardVersion::V4_0); for entry in card.entries.iter() { for item in props { if entry.name == item.name && entry.group == item.group { if item.name != VCardProperty::Version { let _ = entry.write_to(&mut vcard, !item.no_value, is_v4); } else { let _ = write!(&mut vcard, "VERSION:{version}\r\n"); } break; } } } let _ = write!(&mut vcard, "END:VCARD\r\n"); } else { let _ = card.write_to(&mut vcard, version); } vcard } ================================================ FILE: crates/dav/src/card/update.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::assert_is_unique_uid; use crate::{ DavError, DavErrorCondition, DavMethod, common::{ ETag, ExtractETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, file::DavFileResource, fix_percent_encoding, }; use calcard::{Entry, Parser}; use common::{DavName, Server, auth::AccessToken}; use dav_proto::{ RequestHeaders, Return, schema::{property::Rfc1123DateTime, response::CardCondition}, }; use groupware::{cache::GroupwareCache, contact::ContactCard}; use http_proto::HttpResponse; use hyper::StatusCode; use store::write::BatchBuilder; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, }; pub(crate) trait CardUpdateRequestHandler: Sync + Send { fn handle_card_update_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, bytes: Vec, is_patch: bool, ) -> impl Future> + Send; } impl CardUpdateRequestHandler for Server { async fn handle_card_update_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, bytes: Vec, _is_patch: bool, ) -> crate::Result { // Validate URI let resource = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let account_id = resource.account_id; let resources = self .fetch_dav_resources(access_token, account_id, SyncCollection::AddressBook) .await .caused_by(trc::location!())?; let resource_name = fix_percent_encoding( resource .resource .ok_or(DavError::Code(StatusCode::CONFLICT))?, ); if bytes.len() > self.core.groupware.max_vcard_size { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, CardCondition::MaxResourceSize(self.core.groupware.max_vcard_size as u32), ))); } let vcard_raw = std::str::from_utf8(&bytes).map_err(|_| { DavError::Condition( DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, CardCondition::SupportedAddressData, ) .with_details("The request body is not valid UTF-8."), ) })?; let vcard = match Parser::new(vcard_raw).strict().entry() { Entry::VCard(vcard) => vcard, _ => { return Err(DavError::Condition( DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, CardCondition::SupportedAddressData, ) .with_details("Failed to parse vCard data."), )); } }; if let Some(resource) = resources.by_path(resource_name.as_ref()) { if resource.is_container() { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } // Validate ACL let parent_id = resource.parent_id().unwrap(); let document_id = resource.document_id(); if !access_token.is_member(account_id) && !resources.has_access_to_container(access_token, parent_id, Acl::ModifyItems) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Update let card_ = self .store() .get_value::>(ValueKey::archive( account_id, Collection::ContactCard, document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let card = card_ .to_unarchived::() .caused_by(trc::location!())?; // Validate headers match self .validate_headers( access_token, headers, vec![ResourceState { account_id, collection: Collection::ContactCard, document_id: Some(document_id), etag: card.etag().into(), path: resource_name.as_ref(), ..Default::default() }], Default::default(), DavMethod::PUT, ) .await { Ok(_) => {} Err(DavError::Code(StatusCode::PRECONDITION_FAILED)) if headers.ret == Return::Representation => { return Ok(HttpResponse::new(StatusCode::PRECONDITION_FAILED) .with_content_type("text/vcard; charset=utf-8") .with_etag(card.etag()) .with_last_modified( Rfc1123DateTime::new(i64::from(card.inner.modified)).to_string(), ) .with_header("Preference-Applied", "return=representation") .with_binary_body(card.inner.card.to_string())); } Err(e) => return Err(e), } // Validate UID match (card.inner.card.uid(), vcard.uid()) { (Some(old_uid), Some(new_uid)) if old_uid == new_uid => {} (None, None) | (None, Some(_)) => {} _ => { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, CardCondition::NoUidConflict(resources.format_resource(resource).into()), ))); } } // Validate quota let extra_bytes = (bytes.len() as u64).saturating_sub(u32::from(card.inner.size) as u64); if extra_bytes > 0 { self.has_available_quota( &self.get_resource_token(access_token, account_id).await?, extra_bytes, ) .await?; } // Build node let mut new_card = card .deserialize::() .caused_by(trc::location!())?; new_card.size = bytes.len() as u32; new_card.card = vcard; // Prepare write batch let mut batch = BatchBuilder::new(); let etag = new_card .update(access_token, card, account_id, document_id, &mut batch) .caused_by(trc::location!())? .etag(); self.commit_batch(batch).await.caused_by(trc::location!())?; self.notify_task_queue(); Ok(HttpResponse::new(StatusCode::NO_CONTENT).with_etag_opt(etag)) } else if let Some((Some(parent), name)) = resources.map_parent(resource_name.as_ref()) { if !parent.is_container() { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } // Validate ACL if !access_token.is_member(account_id) && !resources.has_access_to_container( access_token, parent.document_id(), Acl::AddItems, ) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Validate headers self.validate_headers( access_token, headers, vec![ResourceState { account_id, collection: resource.collection, document_id: Some(u32::MAX), path: resource_name.as_ref(), ..Default::default() }], Default::default(), DavMethod::PUT, ) .await?; // Validate UID assert_is_unique_uid( self, &resources, account_id, parent.document_id(), vcard.uid(), ) .await?; // Validate quota if !bytes.is_empty() { self.has_available_quota( &self.get_resource_token(access_token, account_id).await?, bytes.len() as u64, ) .await?; } // Build node let card = ContactCard { names: vec![DavName { name: name.to_string(), parent_id: parent.document_id(), }], card: vcard, size: bytes.len() as u32, ..Default::default() }; // Prepare write batch let mut batch = BatchBuilder::new(); let document_id = self .store() .assign_document_ids(account_id, Collection::ContactCard, 1) .await .caused_by(trc::location!())?; let etag = card .insert(access_token, account_id, document_id, &mut batch) .caused_by(trc::location!())? .etag(); self.commit_batch(batch).await.caused_by(trc::location!())?; self.notify_task_queue(); Ok(HttpResponse::new(StatusCode::CREATED).with_etag_opt(etag)) } else { Err(DavError::Code(StatusCode::CONFLICT))? } } } ================================================ FILE: crates/dav/src/common/acl.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::ArchivedResource; use crate::{ DavError, DavErrorCondition, DavResourceName, common::uri::DavUriResource, principal::propfind::PrincipalPropFind, }; use common::{DavResources, Server, auth::AccessToken, sharing::EffectiveAcl}; use dav_proto::{ RequestHeaders, schema::{ property::{DavProperty, Privilege, WebDavProperty}, request::{AclPrincipalPropSet, PropFind}, response::{Ace, BaseCondition, GrantDeny, Href, MultiStatus, Principal}, }, }; use directory::{QueryParams, Type, backend::internal::manage::ManageDirectory}; use groupware::RFC_3986; use groupware::{cache::GroupwareCache, calendar::Calendar, contact::AddressBook, file::FileNode}; use http_proto::HttpResponse; use hyper::StatusCode; use rkyv::vec::ArchivedVec; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use store::{ahash::AHashSet, roaring::RoaringBitmap, write::BatchBuilder}; use trc::AddContext; use types::{ acl::{Acl, AclGrant, ArchivedAclGrant}, collection::Collection, }; use utils::map::bitmap::Bitmap; pub(crate) trait DavAclHandler: Sync + Send { fn handle_acl_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: dav_proto::schema::request::Acl, ) -> impl Future> + Send; fn handle_acl_prop_set( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: AclPrincipalPropSet, ) -> impl Future> + Send; fn validate_and_map_aces( &self, access_token: &AccessToken, acl: dav_proto::schema::request::Acl, collection: Collection, ) -> impl Future>> + Send; fn resolve_ace( &self, access_token: &AccessToken, account_id: u32, grants: &ArchivedVec, expand: Option<&PropFind>, ) -> impl Future>> + Send; } pub(crate) trait ResourceAcl { fn validate_and_map_parent_acl( &self, access_token: &AccessToken, is_member: bool, parent_id: Option, check_acls: impl Into> + Send, ) -> crate::Result; } impl DavAclHandler for Server { async fn handle_acl_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: dav_proto::schema::request::Acl, ) -> crate::Result { // Validate URI let resource_ = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let account_id = resource_.account_id; let collection = resource_.collection; if !matches!( collection, Collection::AddressBook | Collection::Calendar | Collection::FileNode ) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } let resources = self .fetch_dav_resources(access_token, account_id, collection.into()) .await .caused_by(trc::location!())?; let resource = resource_ .resource .and_then(|r| resources.by_path(r)) .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; if !resource.resource.is_container() && !matches!(collection, Collection::FileNode) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Fetch node let archive = self .store() .get_value::>(ValueKey::archive( account_id, collection, resource.document_id(), )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let container = ArchivedResource::from_archive(&archive, collection).caused_by(trc::location!())?; // Validate ACL let acls = container.acls().unwrap(); if !access_token.is_member(account_id) && !acls.effective_acl(access_token).contains(Acl::Share) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Validate ACEs let grants = self .validate_and_map_aces(access_token, request, collection) .await?; if grants.len() != acls.len() || acls.iter().zip(grants.iter()).any(|(a, b)| a != b) { // Refresh ACLs self.refresh_archived_acls(&grants, acls).await; let mut batch = BatchBuilder::new(); match container { ArchivedResource::Calendar(calendar) => { let mut new_calendar = calendar .deserialize::() .caused_by(trc::location!())?; new_calendar.acls = grants; new_calendar .update( access_token, calendar, account_id, resource.document_id(), &mut batch, ) .caused_by(trc::location!())?; } ArchivedResource::AddressBook(book) => { let mut new_book = book .deserialize::() .caused_by(trc::location!())?; new_book.acls = grants; new_book .update( access_token, book, account_id, resource.document_id(), &mut batch, ) .caused_by(trc::location!())?; } ArchivedResource::FileNode(node) => { let mut new_node = node.deserialize::().caused_by(trc::location!())?; new_node.acls = grants; new_node .update( access_token, node, account_id, resource.document_id(), &mut batch, ) .caused_by(trc::location!())?; } _ => unreachable!(), } self.commit_batch(batch).await.caused_by(trc::location!())?; } Ok(HttpResponse::new(StatusCode::OK)) } async fn handle_acl_prop_set( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, mut request: AclPrincipalPropSet, ) -> crate::Result { let uri = self .validate_uri(access_token, headers.uri) .await .and_then(|uri| uri.into_owned_uri())?; let uri = self .map_uri_resource(access_token, uri) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; if !matches!( uri.collection, Collection::Calendar | Collection::AddressBook | Collection::FileNode ) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } let archive = self .store() .get_value::>(ValueKey::archive( uri.account_id, uri.collection, uri.resource, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let acls = match uri.collection { Collection::FileNode => { &archive .unarchive::() .caused_by(trc::location!())? .acls } Collection::AddressBook => { &archive .unarchive::() .caused_by(trc::location!())? .acls } Collection::Calendar => { &archive .unarchive::() .caused_by(trc::location!())? .acls } _ => unreachable!(), }; // Validate ACLs if !access_token.is_member(uri.account_id) && !acls.effective_acl(access_token).contains(Acl::Read) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Validate let account_ids = RoaringBitmap::from_iter(acls.iter().map(|a| u32::from(a.account_id))); let mut response = MultiStatus::new(Vec::with_capacity(16)); if !account_ids.is_empty() { if request.properties.is_empty() { request .properties .push(DavProperty::WebDav(WebDavProperty::DisplayName)); } let request = PropFind::Prop(request.properties); self.prepare_principal_propfind_response( access_token, Collection::Principal, account_ids.into_iter(), &request, &mut response, ) .await?; } Ok(HttpResponse::new(StatusCode::MULTI_STATUS).with_xml_body(response.to_string())) } async fn validate_and_map_aces( &self, access_token: &AccessToken, acl: dav_proto::schema::request::Acl, collection: Collection, ) -> crate::Result> { let mut grants = Vec::with_capacity(acl.aces.len()); for ace in acl.aces { if ace.invert { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::FORBIDDEN, BaseCondition::NoInvert, ))); } let privileges = match ace.grant_deny { GrantDeny::Grant(list) => list.0, GrantDeny::Deny(_) => { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::FORBIDDEN, BaseCondition::GrantOnly, ))); } }; let principal_uri = match ace.principal { Principal::Href(href) => href.0, _ => { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::FORBIDDEN, BaseCondition::AllowedPrincipal, ))); } }; let mut acls = Bitmap::::default(); for privilege in privileges { match privilege { Privilege::Read => { acls.insert(Acl::Read); acls.insert(Acl::ReadItems); } Privilege::Write => { acls.insert(Acl::Modify); acls.insert(Acl::Delete); acls.insert(Acl::AddItems); acls.insert(Acl::ModifyItems); acls.insert(Acl::RemoveItems); } Privilege::WriteContent => { acls.insert(Acl::AddItems); acls.insert(Acl::Modify); acls.insert(Acl::ModifyItems); } Privilege::WriteProperties => { acls.insert(Acl::Modify); } Privilege::ReadCurrentUserPrivilegeSet | Privilege::Unlock | Privilege::Bind | Privilege::Unbind => {} Privilege::All => { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::FORBIDDEN, BaseCondition::NoAbstract, ))); } Privilege::ReadAcl => {} Privilege::WriteAcl => { acls.insert(Acl::Share); } Privilege::ReadFreeBusy | Privilege::ScheduleQueryFreeBusy | Privilege::ScheduleSendFreeBusy => { if collection == Collection::Calendar { acls.insert(Acl::SchedulingReadFreeBusy); } else { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::FORBIDDEN, BaseCondition::NotSupportedPrivilege, ))); } } Privilege::ScheduleDeliver | Privilege::ScheduleSend => { if collection == Collection::Calendar { acls.insert(Acl::SchedulingReadFreeBusy); acls.insert(Acl::SchedulingInvite); acls.insert(Acl::SchedulingReply); } else { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::FORBIDDEN, BaseCondition::NotSupportedPrivilege, ))); } } Privilege::ScheduleDeliverInvite | Privilege::ScheduleSendInvite => { if collection == Collection::Calendar { acls.insert(Acl::SchedulingInvite); } else { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::FORBIDDEN, BaseCondition::NotSupportedPrivilege, ))); } } Privilege::ScheduleDeliverReply | Privilege::ScheduleSendReply => { if collection == Collection::Calendar { acls.insert(Acl::SchedulingReply); } else { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::FORBIDDEN, BaseCondition::NotSupportedPrivilege, ))); } } } } if acls.is_empty() { continue; } let principal_id = self .validate_uri(access_token, &principal_uri) .await .map_err(|_| { DavError::Condition(DavErrorCondition::new( StatusCode::FORBIDDEN, BaseCondition::AllowedPrincipal, )) })? .account_id .ok_or_else(|| { DavError::Condition(DavErrorCondition::new( StatusCode::FORBIDDEN, BaseCondition::AllowedPrincipal, )) })?; // Verify that the principal is a valid principal let principal = self .directory() .query(QueryParams::id(principal_id).with_return_member_of(false)) .await .caused_by(trc::location!())? .ok_or_else(|| { DavError::Condition(DavErrorCondition::new( StatusCode::FORBIDDEN, BaseCondition::AllowedPrincipal, )) })?; if !matches!(principal.typ(), Type::Individual | Type::Group) { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::FORBIDDEN, BaseCondition::AllowedPrincipal, ))); } grants.push(AclGrant { account_id: principal_id, grants: acls, }); } Ok(grants) } async fn resolve_ace( &self, access_token: &AccessToken, account_id: u32, grants: &ArchivedVec, expand: Option<&PropFind>, ) -> crate::Result> { let mut aces = Vec::with_capacity(grants.len()); if access_token.is_member(account_id) || grants.effective_acl(access_token).contains(Acl::Share) { for grant in grants.iter() { let grant_account_id = u32::from(grant.account_id); let principal = if let Some(expand) = expand { self.expand_principal(access_token, grant_account_id, expand) .await? .map(Principal::Response) .unwrap_or_else(|| { Principal::Href(Href(format!( "{}/_{grant_account_id}/", DavResourceName::Principal.base_path(), ))) }) } else { let grant_account_name = self .store() .get_principal_name(grant_account_id) .await .caused_by(trc::location!())? .unwrap_or_else(|| format!("_{grant_account_id}")); Principal::Href(Href(format!( "{}/{}/", DavResourceName::Principal.base_path(), percent_encoding::utf8_percent_encode(&grant_account_name, RFC_3986), ))) }; aces.push(Ace::new( principal, GrantDeny::grant(current_user_privilege_set(Bitmap::::from( &grant.grants, ))), )); } } Ok(aces) } } impl ResourceAcl for DavResources { fn validate_and_map_parent_acl( &self, access_token: &AccessToken, is_member: bool, parent_id: Option, check_acls: impl Into> + Send, ) -> crate::Result { match parent_id { Some(parent_id) => { if is_member || self.has_access_to_container(access_token, parent_id, check_acls) { Ok(parent_id + 1) } else { Err(DavError::Code(StatusCode::FORBIDDEN)) } } None => { if is_member { Ok(0) } else { Err(DavError::Code(StatusCode::FORBIDDEN)) } } } } } pub(crate) trait Privileges { fn current_privilege_set( &self, account_id: u32, grants: &ArchivedVec, is_calendar: bool, ) -> Vec; } impl Privileges for AccessToken { fn current_privilege_set( &self, account_id: u32, grants: &ArchivedVec, is_calendar: bool, ) -> Vec { if self.is_member(account_id) { Privilege::all(is_calendar) } else { current_user_privilege_set(grants.effective_acl(self)) } } } pub(crate) fn current_user_privilege_set(acl_bitmap: Bitmap) -> Vec { let mut acls = AHashSet::with_capacity(16); for grant in acl_bitmap { match grant { Acl::Read | Acl::ReadItems => { acls.insert(Privilege::Read); acls.insert(Privilege::ReadCurrentUserPrivilegeSet); } Acl::Modify => { acls.insert(Privilege::WriteProperties); } Acl::ModifyItems => { acls.insert(Privilege::WriteContent); } Acl::Delete | Acl::RemoveItems => { acls.insert(Privilege::Write); } Acl::Share => { acls.insert(Privilege::ReadAcl); acls.insert(Privilege::WriteAcl); } Acl::SchedulingReadFreeBusy => { acls.insert(Privilege::ReadFreeBusy); } _ => {} } } acls.into_iter().collect() } ================================================ FILE: crates/dav/src/common/lock.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::ETag; use super::uri::{DavUriResource, OwnedUri, UriResource, Urn}; use crate::{DavError, DavErrorCondition, DavMethod}; use common::KV_LOCK_DAV; use common::{Server, auth::AccessToken}; use dav_proto::schema::property::{ActiveLock, LockScope, WebDavProperty}; use dav_proto::schema::request::DavPropertyValue; use dav_proto::schema::response::{BaseCondition, List, PropResponse}; use dav_proto::{Condition, Depth, Timeout}; use dav_proto::{RequestHeaders, schema::request::LockInfo}; use groupware::cache::GroupwareCache; use http_proto::HttpResponse; use hyper::StatusCode; use std::collections::HashMap; use store::ValueKey; use store::dispatch::lookup::KeyValue; use store::write::serialize::rkyv_deserialize; use store::write::{AlignedBytes, Archive, Archiver, now}; use store::{Serialize, U32_LEN}; use trc::AddContext; use types::collection::Collection; use types::dead_property::DeadProperty; #[derive(Debug, Default, Clone)] pub struct ResourceState<'x> { pub account_id: u32, pub collection: Collection, pub document_id: Option, pub etag: Option, pub lock_tokens: Vec, pub sync_token: Option, pub path: &'x str, } #[derive(Debug, Default, Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] pub(crate) struct LockData { locks: HashMap, } #[derive(Debug, Default, Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] #[repr(transparent)] pub(crate) struct LockItems(Vec); #[derive(Debug, Default, Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] pub(crate) struct LockItem { lock_id: u64, owner: u32, expires: u64, depth_infinity: bool, exclusive: bool, owner_dav: Option, } struct LockCache<'x> { account_id: u32, collection: Collection, lock_archive: LockArchive<'x>, } enum LockArchive<'x> { Unarchived(&'x ArchivedLockData), Archived(Archive), } #[derive(Default)] pub(crate) struct LockCaches<'x> { caches: Vec>, } pub(crate) trait LockRequestHandler: Sync + Send { fn handle_lock_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, lock_info: LockRequest, ) -> impl Future> + Send; fn validate_headers( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, resources: Vec>, locks: LockCaches<'_>, method: DavMethod, ) -> impl Future> + Send; } pub(crate) enum LockRequest { Lock(LockInfo), Unlock, Refresh, } impl LockRequestHandler for Server { async fn handle_lock_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, lock_info: LockRequest, ) -> crate::Result { let resource = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let resource_hash = resource.lock_key(); let resource_path = resource .resource .ok_or(DavError::Code(StatusCode::CONFLICT))?; let account_id = resource.account_id; if !access_token.is_member(account_id) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } let resources = vec![ResourceState { account_id, collection: resource.collection, path: resource_path, ..Default::default() }]; let mut base_path = None; let is_lock_request = !matches!(lock_info, LockRequest::Unlock); let if_lock_token = headers .if_ .iter() .flat_map(|if_| if_.list.iter()) .find_map(|cond| { if let Condition::StateToken { token, .. } = cond { Urn::parse(token).and_then(|u| u.try_unwrap_lock()) } else { None } }) .unwrap_or_default(); let mut lock_data = if let Some(lock_data) = self .in_memory_store() .key_get::>(resource_hash.as_slice()) .await .caused_by(trc::location!())? { let lock_data = lock_data .unarchive::() .caused_by(trc::location!())?; self.validate_headers( access_token, headers, resources, LockCaches::new_shared(account_id, resource.collection, lock_data), if is_lock_request { DavMethod::LOCK } else { DavMethod::UNLOCK }, ) .await?; if let LockRequest::Lock(lock_info) = &lock_info { let mut failed_locks = Vec::new(); let is_exclusive = matches!(lock_info.lock_scope, LockScope::Exclusive); let is_infinity = matches!(headers.depth, Depth::Infinity); for (lock_path, lock_item) in lock_data.find_locks(resource_path, true) { if if_lock_token != lock_item.lock_id && (lock_item.exclusive || is_exclusive) && (lock_path.len() == resource_path.len() || lock_item.depth_infinity && resource_path.len() > lock_path.len() || is_infinity && lock_path.len() > resource_path.len()) { let base_path = base_path.get_or_insert_with(|| headers.base_uri().unwrap_or_default()); failed_locks.push(format!("{base_path}/{lock_path}").into()); } } if !failed_locks.is_empty() { return Err(DavErrorCondition::new( StatusCode::LOCKED, BaseCondition::LockTokenSubmitted(List(failed_locks)), ) .into()); } // Validate lock_info if lock_info.owner.as_ref().is_some_and(|o| { o.size() > self.core.groupware.dead_property_size.unwrap_or(512) }) { return Err(DavError::Code(StatusCode::PAYLOAD_TOO_LARGE)); } if self.core.groupware.max_locks_per_user > 0 && lock_data .locks .values() .flat_map(|locks| { locks .0 .iter() .filter(|lock| lock.owner == access_token.primary_id) }) .count() >= self.core.groupware.max_locks_per_user { return Err(DavError::Code(StatusCode::TOO_MANY_REQUESTS)); } } rkyv_deserialize(lock_data).caused_by(trc::location!())? } else if is_lock_request { self.validate_headers( access_token, headers, resources, Default::default(), DavMethod::LOCK, ) .await?; LockData::default() } else { return Err(DavErrorCondition::new( StatusCode::CONFLICT, BaseCondition::LockTokenMatchesRequestUri, ) .into()); }; let now = now(); let response = if is_lock_request { let timeout = if let Timeout::Second(seconds) = headers.timeout { std::cmp::min(seconds, self.core.groupware.max_lock_timeout) } else { self.core.groupware.max_lock_timeout }; let expires = now + timeout; let lock_item = if if_lock_token > 0 { if let Some(lock_item) = lock_data .locks .values_mut() .flat_map(|locks| locks.0.iter_mut()) .find(|lock| lock.lock_id == if_lock_token) { lock_item } else { return Err(DavError::Code(StatusCode::PRECONDITION_FAILED)); } } else { let locks = lock_data .locks .entry(resource_path.to_string()) .or_insert_with(Default::default); locks.0.push(LockItem::default()); locks.0.last_mut().unwrap() }; lock_item.expires = expires; if let LockRequest::Lock(lock_info) = lock_info { // Validate lock_info if lock_info.owner.as_ref().is_some_and(|o| { o.size() > self.core.groupware.dead_property_size.unwrap_or(512) }) { return Err(DavError::Code(StatusCode::PAYLOAD_TOO_LARGE)); } lock_item.lock_id = store::rand::random::() ^ expires; lock_item.owner = access_token.primary_id; lock_item.depth_infinity = matches!(headers.depth, Depth::Infinity); lock_item.owner_dav = lock_info.owner; lock_item.exclusive = matches!(lock_info.lock_scope, LockScope::Exclusive); } let base_path = base_path.get_or_insert_with(|| headers.base_uri().unwrap_or_default()); let active_lock = lock_item.to_active_lock(format!("{base_path}/{resource_path}")); HttpResponse::new(if if_lock_token == 0 { StatusCode::CREATED } else { StatusCode::OK }) .with_lock_token(&active_lock.lock_token.as_ref().unwrap().0) .with_xml_body( PropResponse::new(vec![DavPropertyValue::new( WebDavProperty::LockDiscovery, vec![active_lock], )]) .to_string(), ) } else { let lock_id = headers .lock_token .and_then(Urn::parse) .and_then(|urn| urn.try_unwrap_lock()) .ok_or(DavError::Code(StatusCode::BAD_REQUEST))?; if lock_data.remove_lock(lock_id) { HttpResponse::new(StatusCode::NO_CONTENT) } else { return Err(DavErrorCondition::new( StatusCode::CONFLICT, BaseCondition::LockTokenMatchesRequestUri, ) .into()); } }; // Remove expired locks let max_expire = lock_data.remove_expired(); if max_expire > 0 { self.in_memory_store() .key_set( KeyValue::new( resource_hash, Archiver::new(lock_data) .untrusted() .serialize() .caused_by(trc::location!())?, ) .expires(max_expire), ) .await .caused_by(trc::location!())?; } else { self.in_memory_store() .key_delete(resource_hash) .await .caused_by(trc::location!())?; } Ok(response) } async fn validate_headers( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, mut resources: Vec>, mut locks_: LockCaches<'_>, method: DavMethod, ) -> crate::Result<()> { let no_if_headers = headers.if_.is_empty(); match method { DavMethod::GET | DavMethod::HEAD => { // Return early for GET/HEAD requests without If headers if no_if_headers { return Ok(()); } } DavMethod::COPY | DavMethod::MOVE | DavMethod::POST | DavMethod::PUT | DavMethod::PATCH => { if headers.overwrite_fail && resources.last().is_some_and(|r| { r.etag.is_some() || r.document_id.is_some_and(|id| id != u32::MAX) }) { return Err(DavError::Code(StatusCode::PRECONDITION_FAILED)); } } _ => {} } // Add lock data to the cache for resource in &resources { if locks_.is_cached(resource).is_none() { locks_.insert_lock_data(self, resource).await?; } } // Unarchive lock data let mut locks = locks_.to_unarchived().caused_by(trc::location!())?; // Validate locks for write operations let mut lock_response = Ok(()); if !matches!( method, DavMethod::GET | DavMethod::HEAD | DavMethod::LOCK | DavMethod::UNLOCK ) { let mut base_path = None; 'outer: for (pos, resource) in resources.iter().enumerate() { if pos == 0 && matches!(method, DavMethod::COPY) { continue; } if let Some(idx) = locks.find_cache_pos(self, resource).await? { let mut failed_locks = Vec::new(); for (lock_path, lock_item) in locks.find_locks_by_pos(idx, resource, true)? { let lock_token = lock_item.urn().to_string(); if headers.if_.iter().any(|if_| { if_.resource .is_none_or(|r| { r.trim_end_matches('/').ends_with(lock_path)}) && if_.list.iter().any(|cond| matches!(cond, Condition::StateToken { token, .. } if token == &lock_token)) }) { break 'outer; } else { let base_path = base_path.get_or_insert_with(|| { headers.base_uri() .unwrap_or_default() }); failed_locks.push(format!("{base_path}/{lock_path}").into()); } } if !failed_locks.is_empty() { lock_response = Err(DavErrorCondition::new( StatusCode::LOCKED, BaseCondition::LockTokenSubmitted(List(failed_locks)), ) .into()); break; } } } } // There are no If headers, so we can return early if no_if_headers { return lock_response; } let mut resource_not_found = ResourceState { account_id: u32::MAX, collection: Collection::None, path: "", ..Default::default() }; 'outer: for if_ in &headers.if_ { if if_.list.is_empty() { continue; } let mut resource_state = &mut resource_not_found; if let Some(resource) = if_.resource { if let Some(resource) = self .validate_uri(access_token, resource) .await .ok() .and_then(|r| { let path = r.resource?; Some(ResourceState { account_id: r.account_id?, collection: if !matches!(r.collection, Collection::FileNode) && path.contains('/') { r.collection.child_collection().unwrap_or(r.collection) } else { r.collection }, path, ..Default::default() }) }) { if let Some(known_resource) = resources.iter_mut().find(|r| { r.account_id == resource.account_id && r.collection == resource.collection && r.path == resource.path }) { resource_state = known_resource; } else if access_token.has_access(resource.account_id, resource.collection) { resources.push(resource); resource_state = resources.last_mut().unwrap(); } } } else if let Some(resource) = resources.first_mut() { resource_state = resource; }; // Fill missing data for resource if resource_state.collection != Collection::None && (resource_state.etag.is_none() || resource_state.lock_tokens.is_empty() || resource_state.sync_token.is_none()) { let mut needs_lock_token = false; let mut needs_sync_token = false; let mut needs_etag = false; for cond in &if_.list { match cond { Condition::StateToken { token, .. } => { if token.starts_with("urn:stalwart:davsync:") { needs_sync_token = true; } else { needs_lock_token = true; } } Condition::ETag { .. } | Condition::Exists { .. } => { needs_etag = true; } } } // Fetch eTag if needs_etag && resource_state.etag.is_none() { if resource_state.document_id.is_none() { resource_state.document_id = self .map_uri_resource( access_token, UriResource { collection: resource_state.collection, account_id: resource_state.account_id, resource: resource_state.path.into(), }, ) .await .caused_by(trc::location!())? .map(|uri| uri.resource) .unwrap_or(u32::MAX) .into(); } if let Some(document_id) = resource_state.document_id.filter(|&id| id != u32::MAX) && let Some(archive) = self .store() .get_value::>(ValueKey::archive( resource_state.account_id, resource_state.collection, document_id, )) .await .caused_by(trc::location!())? { resource_state.etag = archive.etag().into(); } } // Fetch lock token if needs_lock_token && resource_state.lock_tokens.is_empty() && let Some(idx) = locks.find_cache_pos(self, resource_state).await? { let found_locks = locks .find_locks_by_pos(idx, resource_state, false)? .iter() .map(|(_, lock)| lock.urn().to_string()) .collect::>(); resource_state.lock_tokens = found_locks; } // Fetch sync token if needs_sync_token && resource_state.sync_token.is_none() { let id = self .fetch_dav_resources( access_token, resource_state.account_id, resource_state.collection.into(), ) .await .caused_by(trc::location!())? .highest_change_id; resource_state.sync_token = Some(Urn::Sync { id, seq: 0 }.to_string()); } } for cond in &if_.list { match cond { Condition::StateToken { is_not, token } => { if let Some(token) = Urn::try_extract_sync_id(token) { if !((resource_state .sync_token .as_deref() .and_then(Urn::try_extract_sync_id) .is_some_and(|sync_token| sync_token == token)) ^ is_not) { continue 'outer; } } else if !((resource_state.lock_tokens.iter().any(|t| t == token)) ^ is_not) { continue 'outer; } } Condition::ETag { is_not, tag } => { if !((resource_state.etag.as_ref().is_some_and(|etag| etag == tag)) ^ is_not) { continue 'outer; } } Condition::Exists { is_not } => { if !((resource_state.etag.is_some()) ^ is_not) { continue 'outer; } } } } return lock_response; } Err(DavError::Code( if matches!(method, DavMethod::GET | DavMethod::HEAD) && headers .if_ .iter() .any(|if_| if_.list.iter().any(|cond| cond.is_none_match())) { StatusCode::NOT_MODIFIED } else { StatusCode::PRECONDITION_FAILED }, )) } } impl LockData { pub fn remove_lock(&mut self, lock_id: u64) -> bool { for (lock_path, lock_items) in self.locks.iter_mut() { for (idx, lock_item) in lock_items.0.iter().enumerate() { if lock_item.lock_id == lock_id { lock_items.0.swap_remove(idx); if lock_items.0.is_empty() { let lock_path = lock_path.clone(); self.locks.remove(&lock_path); } return true; } } } false } pub fn remove_expired(&mut self) -> u64 { let mut max_expire = 0; let now = now(); self.locks.retain(|_, locks| { locks.0.retain(|lock| { if lock.expires > now { max_expire = std::cmp::max(max_expire, lock.expires); true } else { false } }); !locks.0.is_empty() }); max_expire } } impl<'x> LockArchive<'x> { fn unarchive(&'x self) -> trc::Result<&'x ArchivedLockData> { match self { LockArchive::Unarchived(archived_lock_data) => Ok(archived_lock_data), LockArchive::Archived(archive) => { archive.unarchive::().caused_by(trc::location!()) } } } } impl<'x> LockCaches<'x> { pub(self) fn new_shared( account_id: u32, collection: Collection, lock_data: &'x ArchivedLockData, ) -> Self { Self { caches: vec![LockCache { account_id, collection, lock_archive: LockArchive::Unarchived(lock_data), }], } } pub fn to_unarchived(&'x self) -> trc::Result> { let caches = self .caches .iter() .map(|cache| { Ok(LockCache { account_id: cache.account_id, collection: cache.collection, lock_archive: LockArchive::Unarchived( cache.lock_archive.unarchive().caused_by(trc::location!())?, ), }) }) .collect::>>()?; Ok(LockCaches { caches }) } #[inline] pub fn is_cached(&self, resource_state: &ResourceState<'_>) -> Option { self.caches.iter().position(|cache| { resource_state.account_id == cache.account_id && resource_state.collection.main_collection() == cache.collection.main_collection() }) } pub async fn find_cache_pos( &mut self, server: &Server, resource_state: &ResourceState<'_>, ) -> trc::Result> { if let Some(idx) = self.is_cached(resource_state) { Ok(Some(idx)) } else if resource_state.collection != Collection::None { if self.insert_lock_data(server, resource_state).await? { Ok(Some(self.caches.len() - 1)) } else { Ok(None) } } else { Ok(None) } } fn find_locks_by_pos( &'x self, pos: usize, resource_state: &'x ResourceState<'_>, include_children: bool, ) -> trc::Result> { self.caches[pos] .lock_archive .unarchive() .map(|l| l.find_locks(resource_state.path, include_children)) } async fn insert_lock_data( &mut self, server: &Server, resource_state: &ResourceState<'_>, ) -> trc::Result { if let Some(lock_archive) = server .in_memory_store() .key_get::>(resource_state.lock_key().as_slice()) .await .caused_by(trc::location!())? { self.caches.push(LockCache { account_id: resource_state.account_id, collection: resource_state.collection, lock_archive: LockArchive::Archived(lock_archive), }); Ok(true) } else { Ok(false) } } } impl LockItem { pub fn to_active_lock(&self, href: String) -> ActiveLock { ActiveLock::new( href, if self.exclusive { LockScope::Exclusive } else { LockScope::Shared }, ) .with_depth(if self.depth_infinity { Depth::Infinity } else { Depth::Zero }) .with_owner_opt(self.owner_dav.clone()) .with_timeout(self.expires.saturating_sub(now())) .with_lock_token(self.urn().to_string()) } pub fn urn(&self) -> Urn { Urn::Lock(self.lock_id) } } impl ArchivedLockData { pub fn find_locks<'x: 'y, 'y>( &'x self, resource: &'y str, include_children: bool, ) -> Vec<(&'y str, &'x ArchivedLockItem)> { let now = now(); let mut resource_part = resource; let mut found_locks = Vec::new(); loop { if let Some(locks) = self.locks.get(resource_part) { found_locks.extend( locks .0 .iter() .filter(|lock| { lock.expires > now && (resource == resource_part || lock.depth_infinity) }) .map(|lock| (resource_part, lock)), ); } if let Some((resource_part_, _)) = resource_part.rsplit_once('/') { resource_part = resource_part_; } else { break; } } if include_children { let prefix = format!("{}/", resource); for (resource_part, locks) in self.locks.iter() { if resource_part.starts_with(&prefix) { found_locks.extend( locks .0 .iter() .filter(|lock| lock.expires > now) .map(|lock| (resource_part.as_str(), lock)), ); } } } found_locks } } impl ArchivedLockItem { pub fn to_active_lock(&self, href: String) -> ActiveLock { ActiveLock::new( href, if self.exclusive { LockScope::Exclusive } else { LockScope::Shared }, ) .with_depth(if self.depth_infinity { Depth::Infinity } else { Depth::Zero }) .with_owner_opt(self.owner_dav.as_ref().map(Into::into)) .with_timeout(u64::from(self.expires).saturating_sub(now())) .with_lock_token(self.urn().to_string()) } pub fn urn(&self) -> Urn { Urn::Lock(self.lock_id.into()) } } impl OwnedUri<'_> { pub fn lock_key(&self) -> Vec { build_lock_key(self.account_id, self.collection.main_collection()) } } impl ResourceState<'_> { pub fn lock_key(&self) -> Vec { build_lock_key(self.account_id, self.collection.main_collection()) } } pub(crate) fn build_lock_key(account_id: u32, collection: Collection) -> Vec { let mut result = Vec::with_capacity(U32_LEN + 2); result.push(KV_LOCK_DAV); result.extend_from_slice(account_id.to_be_bytes().as_slice()); result.push(u8::from(collection)); result } impl PartialEq for ResourceState<'_> { fn eq(&self, other: &Self) -> bool { self.account_id == other.account_id && self.collection == other.collection && self.document_id == other.document_id } } impl Eq for ResourceState<'_> {} ================================================ FILE: crates/dav/src/common/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use calcard::{ icalendar::{ICalendarComponentType, ICalendarParameterName, ICalendarProperty}, vcard::{VCardParameterName, VCardVersion}, }; use common::auth::AccessToken; use dav_proto::{ Depth, RequestHeaders, Return, schema::{ Namespace, property::{DavProperty, ReportSet, ResourceType}, request::{ AddressbookQuery, CalendarQuery, ExpandProperty, Filter, MultiGet, PropFind, SyncCollection, Timezone, VCardPropertyWithGroup, }, }, }; use groupware::{ calendar::{ ArchivedCalendar, ArchivedCalendarEvent, ArchivedCalendarEventNotification, Calendar, CalendarEvent, CalendarEventNotification, }, contact::{AddressBook, ArchivedAddressBook, ArchivedContactCard, ContactCard}, file::{ArchivedFileNode, FileNode}, }; use propfind::PropFindItem; use rkyv::vec::ArchivedVec; use store::write::{AlignedBytes, Archive, BatchBuilder, Operation, ValueClass, ValueOp}; use types::{ TimeRange, acl::ArchivedAclGrant, collection::Collection, dead_property::ArchivedDeadProperty, field::Field, }; use uri::{OwnedUri, Urn}; pub mod acl; pub mod lock; pub mod propfind; pub mod uri; #[derive(Debug)] pub(crate) struct DavQuery<'x> { pub uri: &'x str, pub resource: DavQueryResource<'x>, pub propfind: PropFind, pub sync_type: SyncType, pub depth: usize, pub limit: Option, pub max_vcard_version: Option, pub ret: Return, pub depth_no_root: bool, pub expand: bool, } #[derive(Default, Debug)] pub(crate) enum SyncType { #[default] None, Initial, From { id: u64, seq: u32, }, } #[derive(Default, Debug)] pub(crate) enum DavQueryResource<'x> { Uri(OwnedUri<'x>), Multiget { parent_collection: Collection, hrefs: Vec, }, Query { filter: DavQueryFilter, parent_collection: Collection, items: Vec, }, #[default] None, } pub(crate) type AddressbookFilter = Vec>; pub(crate) type CalendarFilter = Vec, ICalendarProperty, ICalendarParameterName>>; #[derive(Debug)] pub(crate) enum DavQueryFilter { Addressbook(AddressbookFilter), Calendar { filter: CalendarFilter, max_time_range: Option, timezone: Timezone, }, } pub(crate) trait ETag { fn etag(&self) -> String; } pub(crate) trait ExtractETag { fn etag(&self) -> Option; } impl ETag for Archive { fn etag(&self) -> String { format!("\"{}\"", self.version.hash().unwrap_or_default()) } } impl ExtractETag for BatchBuilder { fn etag(&self) -> Option { let p_value = u8::from(Field::ARCHIVE); for op in self.ops().iter().rev() { match op { Operation::Value { class: ValueClass::Property(p_id), op: ValueOp::Set(value), } if *p_id == p_value => { return Archive::::extract_hash(value) .map(|hash| format!("\"{}\"", hash)); } Operation::Value { class: ValueClass::Property(p_id), op: ValueOp::SetFnc(set_fnc), } if *p_id == p_value => { return Archive::::extract_hash(set_fnc.params().bytes(0)) .map(|hash| format!("\"{}\"", hash)); } _ => {} } } None } } pub(crate) trait DavCollection { fn namespace(&self) -> Namespace; } impl DavCollection for Collection { fn namespace(&self) -> Namespace { match self { Collection::Calendar | Collection::CalendarEvent | Collection::CalendarEventNotification => Namespace::CalDav, Collection::AddressBook | Collection::ContactCard => Namespace::CardDav, _ => Namespace::Dav, } } } impl<'x> DavQuery<'x> { pub fn propfind( resource: OwnedUri<'x>, propfind: PropFind, headers: &RequestHeaders<'x>, ) -> Self { Self { resource: DavQueryResource::Uri(resource), propfind, depth: match headers.depth { Depth::Zero => 0, _ => 1, }, ret: headers.ret, depth_no_root: headers.depth_no_root, uri: headers.uri, max_vcard_version: headers.max_vcard_version, sync_type: Default::default(), limit: Default::default(), expand: Default::default(), } } pub fn multiget( multiget: MultiGet, collection: Collection, headers: &RequestHeaders<'x>, ) -> Self { Self { resource: DavQueryResource::Multiget { hrefs: multiget.hrefs, parent_collection: collection, }, propfind: multiget.properties, ret: headers.ret, depth_no_root: headers.depth_no_root, uri: headers.uri, max_vcard_version: headers.max_vcard_version, sync_type: Default::default(), depth: Default::default(), limit: Default::default(), expand: Default::default(), } } pub fn addressbook_query( query: AddressbookQuery, items: Vec, headers: &RequestHeaders<'x>, ) -> Self { Self { resource: DavQueryResource::Query { filter: DavQueryFilter::Addressbook(query.filters), parent_collection: Collection::AddressBook, items, }, propfind: query.properties, limit: query.limit, ret: headers.ret, depth_no_root: headers.depth_no_root, uri: headers.uri, max_vcard_version: headers.max_vcard_version, sync_type: Default::default(), depth: Default::default(), expand: Default::default(), } } pub fn calendar_query( query: CalendarQuery, max_time_range: Option, items: Vec, headers: &RequestHeaders<'x>, ) -> Self { Self { resource: DavQueryResource::Query { filter: DavQueryFilter::Calendar { filter: query.filters, timezone: query.timezone, max_time_range, }, parent_collection: Collection::Calendar, items, }, propfind: query.properties, ret: headers.ret, depth_no_root: headers.depth_no_root, uri: headers.uri, sync_type: Default::default(), depth: Default::default(), limit: Default::default(), max_vcard_version: Default::default(), expand: Default::default(), } } pub fn changes( resource: OwnedUri<'x>, changes: SyncCollection, headers: &RequestHeaders<'x>, ) -> Self { Self { resource: DavQueryResource::Uri(resource), propfind: changes.properties, sync_type: changes .sync_token .as_deref() .and_then(Urn::parse) .and_then(|urn| urn.try_unwrap_sync()) .map(|(id, seq)| SyncType::From { id, seq }) .unwrap_or(SyncType::Initial), depth: match changes.depth { Depth::One => 1, Depth::Infinity => usize::MAX, _ => 0, }, limit: changes.limit, ret: headers.ret, depth_no_root: headers.depth_no_root, expand: false, uri: headers.uri, max_vcard_version: headers.max_vcard_version, } } pub fn expand( resource: OwnedUri<'x>, expand: ExpandProperty, headers: &RequestHeaders<'x>, ) -> Self { let mut props = Vec::with_capacity(expand.properties.len()); for item in expand.properties { if !matches!(item.property, DavProperty::DeadProperty(_)) && !props.contains(&item.property) { props.push(item.property); } } Self { resource: DavQueryResource::Uri(resource), propfind: PropFind::Prop(props), depth: match headers.depth { Depth::Zero => 0, _ => 1, }, ret: headers.ret, depth_no_root: headers.depth_no_root, expand: true, uri: headers.uri, sync_type: Default::default(), limit: Default::default(), max_vcard_version: headers.max_vcard_version, } } pub fn is_minimal(&self) -> bool { self.ret == Return::Minimal } } pub(crate) enum ArchivedResource<'x> { Calendar(Archive<&'x ArchivedCalendar>), CalendarEvent(Archive<&'x ArchivedCalendarEvent>), CalendarEventNotification(Archive<&'x ArchivedCalendarEventNotification>), CalendarEventNotificationCollection(bool), AddressBook(Archive<&'x ArchivedAddressBook>), ContactCard(Archive<&'x ArchivedContactCard>), FileNode(Archive<&'x ArchivedFileNode>), } impl<'x> ArchivedResource<'x> { pub fn from_archive( archive: &'x Archive, collection: Collection, ) -> trc::Result { match collection { Collection::Calendar => archive .to_unarchived::() .map(ArchivedResource::Calendar), Collection::CalendarEvent => archive .to_unarchived::() .map(ArchivedResource::CalendarEvent), Collection::CalendarEventNotification => archive .to_unarchived::() .map(ArchivedResource::CalendarEventNotification), Collection::AddressBook => archive .to_unarchived::() .map(ArchivedResource::AddressBook), Collection::FileNode => archive .to_unarchived::() .map(ArchivedResource::FileNode), Collection::ContactCard => archive .to_unarchived::() .map(ArchivedResource::ContactCard), _ => unreachable!(), } } pub fn acls(&self) -> Option<&ArchivedVec> { match self { Self::Calendar(archive) => Some(&archive.inner.acls), Self::AddressBook(archive) => Some(&archive.inner.acls), Self::FileNode(archive) => Some(&archive.inner.acls), _ => None, } } pub fn created(&self) -> i64 { match self { ArchivedResource::Calendar(archive) => archive.inner.created.to_native(), ArchivedResource::CalendarEvent(archive) => archive.inner.created.to_native(), ArchivedResource::AddressBook(archive) => archive.inner.created.to_native(), ArchivedResource::ContactCard(archive) => archive.inner.created.to_native(), ArchivedResource::FileNode(archive) => archive.inner.created.to_native(), ArchivedResource::CalendarEventNotification(archive) => { archive.inner.created.to_native() } ArchivedResource::CalendarEventNotificationCollection(_) => 1634515200, } } pub fn modified(&self) -> i64 { match self { ArchivedResource::Calendar(archive) => archive.inner.modified.to_native(), ArchivedResource::CalendarEvent(archive) => archive.inner.modified.to_native(), ArchivedResource::AddressBook(archive) => archive.inner.modified.to_native(), ArchivedResource::ContactCard(archive) => archive.inner.modified.to_native(), ArchivedResource::FileNode(archive) => archive.inner.modified.to_native(), ArchivedResource::CalendarEventNotification(archive) => { archive.inner.modified.to_native() } ArchivedResource::CalendarEventNotificationCollection(_) => 1634515200, } } pub fn dead_properties(&self) -> Option<&ArchivedDeadProperty> { match self { ArchivedResource::Calendar(archive) => Some(&archive.inner.dead_properties), ArchivedResource::CalendarEvent(archive) => Some(&archive.inner.dead_properties), ArchivedResource::AddressBook(archive) => Some(&archive.inner.dead_properties), ArchivedResource::ContactCard(archive) => Some(&archive.inner.dead_properties), ArchivedResource::FileNode(archive) => Some(&archive.inner.dead_properties), ArchivedResource::CalendarEventNotification(_) | ArchivedResource::CalendarEventNotificationCollection(_) => None, } } pub fn content_length(&self) -> Option { match self { ArchivedResource::FileNode(archive) => { archive.inner.file.as_ref().map(|f| f.size.to_native()) } ArchivedResource::CalendarEvent(archive) => archive.inner.size.to_native().into(), ArchivedResource::CalendarEventNotification(archive) => { archive.inner.size.to_native().into() } ArchivedResource::ContactCard(archive) => archive.inner.size.to_native().into(), ArchivedResource::AddressBook(_) | ArchivedResource::Calendar(_) | ArchivedResource::CalendarEventNotificationCollection(_) => None, } } pub fn content_type(&self) -> Option<&str> { match self { ArchivedResource::FileNode(archive) => archive .inner .file .as_ref() .and_then(|f| f.media_type.as_deref()), ArchivedResource::CalendarEvent(_) | ArchivedResource::CalendarEventNotification(_) => { "text/calendar".into() } ArchivedResource::ContactCard(_) => "text/vcard".into(), ArchivedResource::AddressBook(_) | ArchivedResource::Calendar(_) | ArchivedResource::CalendarEventNotificationCollection(_) => None, } } pub fn display_name(&self, access_token: &AccessToken) -> Option<&str> { match self { ArchivedResource::Calendar(archive) => { Some(archive.inner.preferences(access_token).name.as_str()) } ArchivedResource::CalendarEvent(archive) => archive.inner.display_name.as_deref(), ArchivedResource::AddressBook(archive) => { Some(archive.inner.preferences(access_token).name.as_str()) } ArchivedResource::ContactCard(archive) => archive.inner.display_name.as_deref(), ArchivedResource::FileNode(archive) => archive.inner.display_name.as_deref(), ArchivedResource::CalendarEventNotification(_) | ArchivedResource::CalendarEventNotificationCollection(_) => None, } } pub fn supported_report_set(&self) -> Option> { match self { ArchivedResource::Calendar(_) => vec![ ReportSet::SyncCollection, ReportSet::AclPrincipalPropSet, ReportSet::PrincipalMatch, ReportSet::ExpandProperty, ReportSet::CalendarQuery, ReportSet::CalendarMultiGet, ReportSet::FreeBusyQuery, ] .into(), ArchivedResource::AddressBook(_) => vec![ ReportSet::SyncCollection, ReportSet::AclPrincipalPropSet, ReportSet::PrincipalMatch, ReportSet::ExpandProperty, ReportSet::AddressbookQuery, ReportSet::AddressbookMultiGet, ] .into(), ArchivedResource::FileNode(archive) if archive.inner.file.is_none() => vec![ ReportSet::SyncCollection, ReportSet::AclPrincipalPropSet, ReportSet::PrincipalMatch, ] .into(), ArchivedResource::CalendarEventNotificationCollection(_) => vec![ ReportSet::SyncCollection, ReportSet::CalendarQuery, ReportSet::CalendarMultiGet, ] .into(), _ => None, } } pub fn resource_type(&self) -> Option> { match self { ArchivedResource::Calendar(_) => { vec![ResourceType::Collection, ResourceType::Calendar].into() } ArchivedResource::AddressBook(_) => { vec![ResourceType::Collection, ResourceType::AddressBook].into() } ArchivedResource::FileNode(archive) if archive.inner.file.is_none() => { vec![ResourceType::Collection].into() } ArchivedResource::CalendarEventNotificationCollection(true) => { vec![ResourceType::Collection, ResourceType::ScheduleInbox].into() } ArchivedResource::CalendarEventNotificationCollection(false) => { vec![ResourceType::Collection, ResourceType::ScheduleOutbox].into() } _ => None, } } } impl SyncType { pub fn is_none(&self) -> bool { matches!(self, SyncType::None) } pub fn is_none_or_initial(&self) -> bool { matches!(self, SyncType::None | SyncType::Initial) } } ================================================ FILE: crates/dav/src/common/propfind.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ ArchivedResource, DavCollection, DavQuery, DavQueryFilter, ETag, SyncType, acl::{DavAclHandler, Privileges}, lock::{LockData, build_lock_key}, uri::{UriResource, Urn}, }; use crate::{ DavError, DavErrorCondition, calendar::{ CALENDAR_CONTAINER_PROPS, CALENDAR_ITEM_PROPS, query::{CalendarQueryHandler, try_parse_tz}, }, card::{ CARD_CONTAINER_PROPS, CARD_ITEM_PROPS, query::{serialize_vcard_with_props, vcard_query}, }, common::{DavQueryResource, acl::current_user_privilege_set, uri::DavUriResource}, file::{FILE_CONTAINER_PROPS, FILE_ITEM_PROPS}, principal::{ CurrentUserPrincipal, propfind::{PrincipalPropFind, build_home_set}, }, }; use calcard::{common::timezone::Tz, icalendar::ICalendarComponentType}; use common::{DavResourcePath, DavResources, Server, auth::AccessToken}; use dav_proto::{ Depth, RequestHeaders, parser::header::dav_base_uri, requests::NsDeadProperty, schema::{ Collation, Namespace, property::{ ActiveLock, CalDavProperty, CardDavProperty, Comp, DavProperty, DavValue, PrincipalProperty, Privilege, ReportSet, ResourceType, Rfc1123DateTime, SupportedCollation, SupportedLock, WebDavProperty, }, request::{DavDeadProperty, DavPropertyValue, PropFind}, response::{ AclRestrictions, BaseCondition, Href, List, MultiStatus, PropStat, Response, SupportedPrivilege, }, }, }; use directory::{Permission, Type, backend::internal::manage::ManageDirectory}; use groupware::calendar::{SCHEDULE_INBOX_ID, SupportedComponent}; use groupware::{ DavCalendarResource, DavResourceName, cache::GroupwareCache, calendar::ArchivedTimezone, }; use http_proto::HttpResponse; use hyper::StatusCode; use std::sync::Arc; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use store::{ ahash::AHashMap, query::log::{Change, Query}, roaring::RoaringBitmap, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, dead_property::DeadProperty, }; use utils::map::bitmap::Bitmap; pub(crate) trait PropFindRequestHandler: Sync + Send { fn handle_propfind_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: PropFind, ) -> impl Future> + Send; fn handle_dav_query( &self, access_token: &AccessToken, query: DavQuery<'_>, ) -> impl Future> + Send; fn dav_quota( &self, access_token: &AccessToken, account_id: u32, ) -> impl Future> + Send; } pub(crate) struct PropFindData { pub accounts: AHashMap, } #[derive(Default)] pub(crate) struct PropFindAccountData { pub resources: Option>, pub quota: Option, pub owner: Option, pub locks: Option>, pub locks_not_found: bool, } #[derive(Clone, Default)] pub(crate) struct PropFindAccountQuota { pub used: u64, pub available: u64, } #[derive(Debug)] pub(crate) struct PropFindItem { pub name: String, pub account_id: u32, pub document_id: u32, pub parent_id: Option, pub is_container: bool, } impl PropFindRequestHandler for Server { async fn handle_propfind_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: PropFind, ) -> crate::Result { // Validate URI let resource = self.validate_uri(access_token, headers.uri).await?; // Reject Infinity depth for certain queries let return_children = match headers.depth { Depth::One | Depth::None => true, Depth::Zero => false, Depth::Infinity => match resource.collection { Collection::Principal => true, Collection::Calendar | Collection::AddressBook if resource.account_id.is_some() && resource.resource.is_some() => { true } Collection::CalendarEventNotification if resource.account_id.is_some() => true, _ => { return Err(DavErrorCondition::new( StatusCode::FORBIDDEN, BaseCondition::PropFindFiniteDepth, ) .into()); } }, }; // List shared resources if let Some(account_id) = resource.account_id { match resource.collection { Collection::FileNode | Collection::Calendar | Collection::AddressBook | Collection::CalendarEventNotification => { // Validate permissions access_token.assert_has_permission(match resource.collection { Collection::FileNode => Permission::DavFilePropFind, Collection::Calendar | Collection::CalendarEvent | Collection::CalendarEventNotification => Permission::DavCalPropFind, Collection::AddressBook | Collection::ContactCard => { Permission::DavCardPropFind } _ => unreachable!(), })?; self.handle_dav_query( access_token, DavQuery::propfind( UriResource::new_owned( resource.collection, account_id, resource.resource, ), request, headers, ), ) .await } Collection::Principal => { let mut response = MultiStatus::new(Vec::with_capacity(16)); if resource.resource.is_some() { response.add_response(Response::new_status( [headers.uri.to_string()], StatusCode::NOT_FOUND, )); } else if access_token.has_account_access(account_id) || (self.core.groupware.allow_directory_query && access_token.has_permission(Permission::DavPrincipalList)) || access_token.has_permission(Permission::IndividualList) { self.prepare_principal_propfind_response( access_token, Collection::Principal, [account_id].into_iter(), &request, &mut response, ) .await?; } else { response.add_response(Response::new_status( [headers.uri.to_string()], StatusCode::FORBIDDEN, )); } Ok(HttpResponse::new(StatusCode::MULTI_STATUS) .with_xml_body(response.to_string())) } _ => unreachable!(), } } else { let mut response = MultiStatus::new(Vec::with_capacity(16)); // Add container info if !headers.depth_no_root { add_base_collection_response( self, &request, resource.collection, access_token, &mut response, ) .await?; } if return_children { let ids = if !matches!(resource.collection, Collection::Principal) { // Validate permissions access_token.assert_has_permission(match resource.collection { Collection::FileNode => Permission::DavFilePropFind, Collection::Calendar | Collection::CalendarEvent | Collection::CalendarEventNotification => Permission::DavCalPropFind, Collection::AddressBook | Collection::ContactCard => { Permission::DavCardPropFind } _ => unreachable!(), })?; RoaringBitmap::from_iter( access_token.all_ids_by_collection(resource.collection), ) } else if (self.core.groupware.allow_directory_query && access_token.has_permission(Permission::DavPrincipalList)) || access_token.has_permission(Permission::IndividualList) { // Return all principals let principals = self .store() .list_principals( None, access_token.tenant_id(), &[Type::Individual, Type::Group], false, 0, 0, ) .await .caused_by(trc::location!())?; RoaringBitmap::from_iter(principals.items.into_iter().map(|p| p.id())) } else { RoaringBitmap::from_iter(access_token.all_ids()) }; self.prepare_principal_propfind_response( access_token, resource.collection, ids.into_iter(), &request, &mut response, ) .await?; } Ok(HttpResponse::new(StatusCode::MULTI_STATUS).with_xml_body(response.to_string())) } } async fn handle_dav_query( &self, access_token: &AccessToken, mut query: DavQuery<'_>, ) -> crate::Result { let mut response = MultiStatus::new(Vec::with_capacity(16)); let mut data = PropFindData::new(); let collection_container; let collection_children; let sync_collection; let mut query_filter = None; let mut limit = std::cmp::min( query.limit.unwrap_or(u32::MAX) as usize, self.core.groupware.max_results, ); let mut is_sync_limited = false; let mut is_propfind = false; let mut ical_instances_limit = self.core.groupware.max_ical_instances; let paths = match std::mem::take(&mut query.resource) { DavQueryResource::Uri(resource) => { collection_container = resource.collection; collection_children = collection_container.child_collection().unwrap(); sync_collection = SyncCollection::from(collection_container); is_propfind = true; get( self, access_token, collection_container, collection_children, sync_collection, &query, &mut data, &mut response, resource, limit, &mut is_sync_limited, ) .await? } DavQueryResource::Multiget { hrefs, parent_collection, } => { collection_container = parent_collection; collection_children = collection_container.child_collection().unwrap(); sync_collection = SyncCollection::from(collection_container); multiget( self, access_token, collection_container, collection_children, sync_collection, &mut data, &mut response, hrefs, ) .await? } DavQueryResource::Query { filter, parent_collection, items, } => { query_filter = Some(filter); collection_container = parent_collection; collection_children = collection_container.child_collection().unwrap(); sync_collection = SyncCollection::from(collection_container); items } DavQueryResource::None => unreachable!(), }; response.set_namespace(collection_container.namespace()); let mut skip_not_found = query.expand; let properties = match &query.propfind { PropFind::PropName => { let (container_props, children_props) = match collection_container { Collection::FileNode => { (FILE_CONTAINER_PROPS.as_slice(), FILE_ITEM_PROPS.as_slice()) } Collection::Calendar | Collection::CalendarEventNotification => ( CALENDAR_CONTAINER_PROPS.as_slice(), CALENDAR_ITEM_PROPS.as_slice(), ), Collection::AddressBook => { (CARD_CONTAINER_PROPS.as_slice(), CARD_ITEM_PROPS.as_slice()) } _ => unreachable!(), }; for item in paths { let props = if item.is_container { container_props .iter() .cloned() .map(DavPropertyValue::empty) .collect::>() } else { children_props .iter() .cloned() .map(DavPropertyValue::empty) .collect::>() }; response.add_response(Response::new_propstat( item.name, vec![PropStat::new_list(props)], )); } return Ok( HttpResponse::new(StatusCode::MULTI_STATUS).with_xml_body(response.to_string()) ); } PropFind::AllProp(items) => { skip_not_found = true; let mut result = Vec::with_capacity(items.len() + DavProperty::ALL_PROPS.len()); result.extend(DavProperty::ALL_PROPS); result.extend(items.iter().filter(|field| !field.is_all_prop()).cloned()); result } PropFind::Prop(items) => items.clone(), }; let is_scheduling = collection_container == Collection::CalendarEventNotification; 'outer: for item in paths { let account_id = item.account_id; let document_id = item.document_id; let collection = if item.is_container { collection_container } else { collection_children }; // Unarchive resource let archive_; let archive = if is_scheduling && item.is_container { archive_ = Archive::default(); ArchivedResource::CalendarEventNotificationCollection( item.document_id == SCHEDULE_INBOX_ID, ) } else if let Some(archive) = self .store() .get_value::>(ValueKey::archive( account_id, collection, document_id, )) .await .caused_by(trc::location!())? { archive_ = archive; ArchivedResource::from_archive(&archive_, collection).caused_by(trc::location!())? } else { response.add_response(Response::new_status([item.name], StatusCode::NOT_FOUND)); continue; }; // Filter let mut calendar_filter = None; if let Some(query_filter) = &query_filter { match (query_filter, &archive) { (DavQueryFilter::Addressbook(filter), ArchivedResource::ContactCard(card)) => { if !vcard_query(&card.inner.card, filter) { continue; } } ( DavQueryFilter::Calendar { filter, timezone, max_time_range, }, ArchivedResource::CalendarEvent(event), ) => { let default_tz = if let Some(tz) = try_parse_tz(timezone) { tz } else if let Some(calendar_id) = item.parent_id { data.resources(self, access_token, account_id, SyncCollection::Calendar) .await .caused_by(trc::location!())? .calendar_default_tz(calendar_id, account_id) .unwrap_or(Tz::UTC) } else { Tz::UTC }; let mut query_handler = CalendarQueryHandler::new(event.inner, *max_time_range, default_tz); if !query_handler.filter(event.inner, filter) { continue; } calendar_filter = Some(query_handler); } _ => (), } } // Fill properties let dead_properties = archive.dead_properties(); let mut fields = Vec::with_capacity(properties.len()); let mut fields_not_found = Vec::new(); for property in &properties { match property { DavProperty::WebDav(dav_property) => match dav_property { WebDavProperty::CreationDate => { fields.push(DavPropertyValue::new( property.clone(), DavValue::Timestamp(archive.created()), )); } WebDavProperty::DisplayName => { if let Some(name) = archive.display_name(access_token) { fields.push(DavPropertyValue::new( property.clone(), DavValue::String(name.to_string()), )); } else if !skip_not_found { fields_not_found.push(DavPropertyValue::empty(property.clone())); } } WebDavProperty::GetContentLanguage => { if !skip_not_found { fields_not_found.push(DavPropertyValue::empty(property.clone())); } } WebDavProperty::GetContentLength => { if let Some(value) = archive.content_length() { fields.push(DavPropertyValue::new( property.clone(), DavValue::Uint64(value as u64), )); } else if !skip_not_found { fields_not_found.push(DavPropertyValue::empty(property.clone())); } } WebDavProperty::GetContentType => { if let Some(value) = archive.content_type() { fields.push(DavPropertyValue::new( property.clone(), DavValue::String(value.to_string()), )); } else if !skip_not_found { fields_not_found.push(DavPropertyValue::empty(property.clone())); } } WebDavProperty::GetETag => { fields.push(DavPropertyValue::new( property.clone(), DavValue::String(archive_.etag()), )); } WebDavProperty::GetCTag => { if item.is_container { let ctag = data .resources(self, access_token, account_id, sync_collection) .await .caused_by(trc::location!())? .highest_change_id; fields.push(DavPropertyValue::new( property.clone(), DavValue::String(format!("\"{ctag}\"")), )); } else { fields_not_found.push(DavPropertyValue::empty(property.clone())); } response.set_namespace(Namespace::CalendarServer); } WebDavProperty::GetLastModified => { fields.push(DavPropertyValue::new( property.clone(), DavValue::Rfc1123Date(Rfc1123DateTime::new(archive.modified())), )); } WebDavProperty::ResourceType => { if let Some(resource_type) = archive.resource_type() { fields.push(DavPropertyValue::new(property.clone(), resource_type)); } else { fields.push(DavPropertyValue::empty(property.clone())); } } WebDavProperty::LockDiscovery => { if let Some(locks) = data .locks(self, account_id, collection_container, &item) .await .caused_by(trc::location!())? { fields.push(DavPropertyValue::new(property.clone(), locks)); } else { fields.push(DavPropertyValue::empty(property.clone())); } } WebDavProperty::SupportedLock => { if !is_scheduling { fields.push(DavPropertyValue::new( property.clone(), SupportedLock::default(), )); } else { fields.push(DavPropertyValue::empty(property.clone())); } } WebDavProperty::SupportedReportSet => { if let Some(report_set) = archive.supported_report_set() { fields.push(DavPropertyValue::new(property.clone(), report_set)); } else if !skip_not_found { fields_not_found.push(DavPropertyValue::empty(property.clone())); } } WebDavProperty::SyncToken => { let sync_token = data .resources(self, access_token, account_id, sync_collection) .await .caused_by(trc::location!())? .sync_token(); fields.push(DavPropertyValue::new(property.clone(), sync_token)); } WebDavProperty::CurrentUserPrincipal => { if !query.expand { fields.push(DavPropertyValue::new( property.clone(), vec![access_token.current_user_principal()], )); } else { fields.push(DavPropertyValue::new( property.clone(), self.expand_principal( access_token, access_token.primary_id(), &query.propfind, ) .await? .map(DavValue::Response) .unwrap_or(DavValue::Null), )); } } WebDavProperty::QuotaAvailableBytes => { if item.is_container { fields.push(DavPropertyValue::new( property.clone(), data.quota(self, access_token, account_id) .await .caused_by(trc::location!())? .available, )); } else if !skip_not_found { fields_not_found.push(DavPropertyValue::empty(property.clone())); } } WebDavProperty::QuotaUsedBytes => { if item.is_container { fields.push(DavPropertyValue::new( property.clone(), data.quota(self, access_token, account_id) .await .caused_by(trc::location!())? .used, )); } else if !skip_not_found { fields_not_found.push(DavPropertyValue::empty(property.clone())); } } WebDavProperty::Owner => { if !query.expand { fields.push(DavPropertyValue::new( property.clone(), vec![ data.owner(self, access_token, account_id) .await .caused_by(trc::location!())?, ], )); } else { fields.push(DavPropertyValue::new( property.clone(), self.expand_principal( access_token, account_id, &query.propfind, ) .await? .map(DavValue::Response) .unwrap_or(DavValue::Null), )); } } WebDavProperty::Group => { fields.push(DavPropertyValue::empty(property.clone())); } WebDavProperty::SupportedPrivilegeSet => { if !is_scheduling { fields.push(DavPropertyValue::new( property.clone(), vec![SupportedPrivilege::all_privileges( collection_container == Collection::Calendar, )], )); } else { fields.push(DavPropertyValue::new( property.clone(), vec![SupportedPrivilege::all_scheduling_privileges(matches!( archive, ArchivedResource::CalendarEventNotification(_) | ArchivedResource::CalendarEventNotificationCollection( true ) ))], )); } } WebDavProperty::CurrentUserPrivilegeSet => { let privileges = if is_scheduling { Privilege::scheduling( matches!( archive, ArchivedResource::CalendarEventNotification(_) | ArchivedResource::CalendarEventNotificationCollection( true ) ), access_token.is_member(account_id), ) } else if access_token.is_member(account_id) { Privilege::all(matches!( collection, Collection::Calendar | Collection::CalendarEvent )) } else if let Some(acls) = archive.acls() { access_token.current_privilege_set( account_id, acls, collection_container == Collection::Calendar, ) } else if let Some(parent_id) = item.parent_id { current_user_privilege_set( data.resources(self, access_token, account_id, sync_collection) .await .caused_by(trc::location!())? .container_acl(access_token, parent_id), ) } else { vec![] }; if !privileges.is_empty() { fields.push(DavPropertyValue::new(property.clone(), privileges)); } else if !skip_not_found { fields_not_found.push(DavPropertyValue::empty(property.clone())); } } WebDavProperty::Acl => { if let Some(acls) = archive.acls() { let aces = self .resolve_ace( access_token, account_id, acls, query.expand.then_some(&query.propfind), ) .await?; fields.push(DavPropertyValue::new(property.clone(), aces)); } else if !skip_not_found { fields_not_found.push(DavPropertyValue::empty(property.clone())); } } WebDavProperty::AclRestrictions => { fields.push(DavPropertyValue::new( property.clone(), AclRestrictions::default() .with_no_invert() .with_grant_only(), )); } WebDavProperty::InheritedAclSet => { fields.push(DavPropertyValue::empty(property.clone())); } WebDavProperty::PrincipalCollectionSet => { fields.push(DavPropertyValue::new( property.clone(), vec![Href( DavResourceName::Principal.collection_path().to_string(), )], )); } }, DavProperty::DeadProperty(tag) => { if let Some(value) = dead_properties.and_then(|props| props.find_tag(&tag.name)) { fields.push(DavPropertyValue::new(property.clone(), value)); } else { fields_not_found.push(DavPropertyValue::empty(property.clone())); } } DavProperty::CardDav(card_property) => match (card_property, &archive) { ( CardDavProperty::AddressbookDescription, ArchivedResource::AddressBook(book), ) => { if let Some(desc) = book.inner.preferences(access_token).description.as_deref() { fields.push(DavPropertyValue::new( property.clone(), desc.to_string(), )); } else { fields_not_found.push(DavPropertyValue::empty(property.clone())); } } ( CardDavProperty::SupportedAddressData, ArchivedResource::AddressBook(_), ) => { fields.push(DavPropertyValue::new( property.clone(), DavValue::SupportedAddressData, )); } ( CardDavProperty::SupportedCollationSet, ArchivedResource::AddressBook(_), ) => { fields.push(DavPropertyValue::new( property.clone(), DavValue::Collations(List(vec![ SupportedCollation { collation: Collation::AsciiCasemap, namespace: Namespace::CardDav, }, SupportedCollation { collation: Collation::UnicodeCasemap, namespace: Namespace::CardDav, }, ])), )); } (CardDavProperty::MaxResourceSize, ArchivedResource::AddressBook(_)) => { fields.push(DavPropertyValue::new( property.clone(), self.core.groupware.max_vcard_size as u64, )); } ( CardDavProperty::AddressData(items), ArchivedResource::ContactCard(card), ) => { fields.push(DavPropertyValue::new( property.clone(), DavValue::CData(serialize_vcard_with_props( &card.inner.card, items, query .max_vcard_version .or_else(|| card.inner.card.version()), )), )); } _ => { if !skip_not_found { fields_not_found.push(DavPropertyValue::empty(property.clone())); } } }, DavProperty::CalDav(cal_property) => match (cal_property, &archive) { ( CalDavProperty::CalendarDescription, ArchivedResource::Calendar(calendar), ) => { if let Some(desc) = calendar .inner .preferences(access_token) .description .as_deref() { fields.push(DavPropertyValue::new( property.clone(), desc.to_string(), )); } else { fields_not_found.push(DavPropertyValue::empty(property.clone())); } } ( CalDavProperty::CalendarTimezone, ArchivedResource::Calendar(calendar), ) => { if let ArchivedTimezone::Custom(tz) = &calendar.inner.preferences(access_token).time_zone { fields.push(DavPropertyValue::new( property.clone(), DavValue::CData(tz.to_string()), )); } else { fields_not_found.push(DavPropertyValue::empty(property.clone())); } } (CalDavProperty::TimezoneId, ArchivedResource::Calendar(calendar)) => { if let ArchivedTimezone::IANA(tz) = &calendar.inner.preferences(access_token).time_zone { fields.push(DavPropertyValue::new( property.clone(), Tz::from_id(tz.to_native()).unwrap_or(Tz::UTC).to_string(), )); } else { fields_not_found.push(DavPropertyValue::empty(property.clone())); } } ( CalDavProperty::SupportedCalendarComponentSet, ArchivedResource::Calendar(calendar), ) => { let supported_components = calendar.inner.supported_components.to_native(); fields.push(DavPropertyValue::new( property.clone(), if supported_components != 0 { DavValue::Components(List( Bitmap::::from(supported_components) .into_iter() .map(ICalendarComponentType::from) .map(Comp) .collect(), )) } else { DavValue::all_calendar_components() }, )); } (CalDavProperty::SupportedCalendarData, ArchivedResource::Calendar(_)) => { fields.push(DavPropertyValue::new( property.clone(), DavValue::SupportedCalendarData, )); } (CalDavProperty::SupportedCollationSet, ArchivedResource::Calendar(_)) => { fields.push(DavPropertyValue::new( property.clone(), DavValue::Collations(List(vec![ SupportedCollation { collation: Collation::AsciiCasemap, namespace: Namespace::CalDav, }, SupportedCollation { collation: Collation::UnicodeCasemap, namespace: Namespace::CalDav, }, ])), )); } (CalDavProperty::MaxResourceSize, ArchivedResource::Calendar(_)) => { fields.push(DavPropertyValue::new( property.clone(), self.core.groupware.max_ical_size as u64, )); } (CalDavProperty::MinDateTime, ArchivedResource::Calendar(_)) => { fields.push(DavPropertyValue::new( property.clone(), DavValue::String("0001-01-01T00:00:00Z".to_string()), )); } (CalDavProperty::MaxDateTime, ArchivedResource::Calendar(_)) => { fields.push(DavPropertyValue::new( property.clone(), DavValue::String("9999-12-31T23:59:59Z".to_string()), )); } (CalDavProperty::MaxInstances, ArchivedResource::Calendar(_)) => { fields.push(DavPropertyValue::new( property.clone(), self.core.groupware.max_ical_instances as u64, )); } ( CalDavProperty::MaxAttendeesPerInstance, ArchivedResource::Calendar(_), ) => { fields.push(DavPropertyValue::new( property.clone(), self.core.groupware.max_ical_attendees_per_instance as u64, )); } ( CalDavProperty::CalendarData(data), ArchivedResource::CalendarEvent(event), ) => { if calendar_filter.is_some() || !data.properties.is_empty() { if let Some(ical) = calendar_filter .get_or_insert_with(|| { CalendarQueryHandler::new(event.inner, None, Tz::UTC) }) .serialize_ical(event.inner, data, &mut ical_instances_limit) { fields.push(DavPropertyValue::new( property.clone(), DavValue::CData(ical), )); } else { limit = 0; break 'outer; } } else { fields.push(DavPropertyValue::new( property.clone(), DavValue::CData(event.inner.data.event.to_string()), )); } } ( CalDavProperty::CalendarData(_), ArchivedResource::CalendarEventNotification(event), ) => { fields.push(DavPropertyValue::new( property.clone(), DavValue::CData(event.inner.event.to_string()), )); } (CalDavProperty::ScheduleTag, ArchivedResource::CalendarEvent(event)) if event.inner.schedule_tag.is_some() => { fields.push(DavPropertyValue::new( property.clone(), DavValue::String(format!( "\"{}\"", event.inner.schedule_tag.as_ref().unwrap() )), )); } (CalDavProperty::ScheduleCalendarTransp, ArchivedResource::Calendar(_)) => { fields.push(DavPropertyValue::new( property.clone(), DavValue::DeadProperty(DeadProperty::single_with_ns( Namespace::CalDav, "opaque", )), )); } ( CalDavProperty::ScheduleDefaultCalendarURL, ArchivedResource::CalendarEventNotificationCollection(true), ) => { if let Some(default_cal) = &self.core.groupware.default_calendar_name { fields.push(DavPropertyValue::new( property.clone(), vec![Href(format!( "{}/{}/{default_cal}/", DavResourceName::Cal.base_path(), item.name.split('/').nth(3).unwrap_or_default() ))], )); } else { fields_not_found.push(DavPropertyValue::empty(property.clone())); } } _ => { if !skip_not_found { response.set_namespace(property.namespace()); fields_not_found.push(DavPropertyValue::empty(property.clone())); } } }, property => { if !skip_not_found { response.set_namespace(property.namespace()); fields_not_found.push(DavPropertyValue::empty(property.clone())); } } } } // Add dead properties if skip_not_found && let Some(dead_properties) = dead_properties.filter(|dead_properties| !dead_properties.0.is_empty()) { dead_properties.to_dav_values(&mut fields); } // Add response let mut prop_stat = Vec::with_capacity(2); if !fields.is_empty() { prop_stat.push(PropStat::new_list(fields)); } if !fields_not_found.is_empty() && !query.is_minimal() { prop_stat .push(PropStat::new_list(fields_not_found).with_status(StatusCode::NOT_FOUND)); } if prop_stat.is_empty() { prop_stat.push(PropStat::new_list(vec![])); } response.add_response(Response::new_propstat(item.name, prop_stat)); limit -= 1; if limit == 0 { break; } } if limit == 0 || is_sync_limited { response.add_response( Response::new_status([query.uri], StatusCode::INSUFFICIENT_STORAGE) .with_error(BaseCondition::NumberOfMatchesWithinLimit) .with_response_description(if ical_instances_limit > 0 { format!( "The number of matches exceeds the limit of {}", query .limit .unwrap_or(self.core.groupware.max_results as u32) ) } else { format!( "The number of recurrence instances exceeds the limit of {}", query .limit .unwrap_or(self.core.groupware.max_ical_instances as u32) ) }), ); } if !response.response.0.is_empty() || !query.sync_type.is_none() { Ok(HttpResponse::new(StatusCode::MULTI_STATUS).with_xml_body(response.to_string())) } else if !is_propfind { Ok(HttpResponse::new(StatusCode::MULTI_STATUS) .with_xml_body(MultiStatus::not_found(query.uri).to_string())) } else { Ok(HttpResponse::new(StatusCode::NOT_FOUND)) } } async fn dav_quota( &self, access_token: &AccessToken, account_id: u32, ) -> trc::Result { let resource_token = self .get_resource_token(access_token, account_id) .await .caused_by(trc::location!())?; let quota = if resource_token.quota > 0 { resource_token.quota } else if let Some(tenant) = resource_token.tenant.filter(|t| t.quota > 0) { tenant.quota } else { u32::MAX as u64 }; let used = self .get_used_quota(account_id) .await .caused_by(trc::location!())? as u64; Ok(PropFindAccountQuota { used, available: quota.saturating_sub(used), }) } } #[allow(clippy::too_many_arguments)] async fn get( server: &Server, access_token: &AccessToken, collection_container: Collection, collection_children: Collection, sync_collection: SyncCollection, query: &DavQuery<'_>, data: &mut PropFindData, response: &mut MultiStatus, resource: UriResource>, limit: usize, is_sync_limited: &mut bool, ) -> crate::Result> { let container_has_children = collection_children != collection_container; response.set_namespace(collection_container.namespace()); let account_id = resource.account_id; let resources = data .resources(server, access_token, account_id, sync_collection) .await .caused_by(trc::location!())?; // Obtain document ids let mut display_containers = if !access_token.is_member(account_id) { resources .shared_containers( access_token, [if container_has_children { Acl::ReadItems } else { Acl::Read }], true, ) .into() } else { None }; let mut display_children = display_containers .as_ref() .filter(|_| container_has_children) .map(|containers| { RoaringBitmap::from_iter(resources.resources.iter().filter_map(|r| { if r.child_names() .is_some_and(|n| n.iter().any(|n| containers.contains(n.parent_id))) { Some(r.document_id) } else { None } })) }); // Filter by changelog let is_sync = match query.sync_type { SyncType::From { id, seq } => { let changes = server .store() .changes(account_id, sync_collection.into(), Query::Since(id)) .await .caused_by(trc::location!())?; let mut vanished: Vec = Vec::new(); // Merge changes let mut total_changes = 0; let mut maybe_has_vanished = false; if container_has_children { let mut container_changes = RoaringBitmap::new(); let mut item_changes = RoaringBitmap::new(); for change in changes.changes { match change { Change::InsertItem(id) => { item_changes.insert(id as u32); } Change::UpdateItem(id) => { maybe_has_vanished = true; item_changes.insert(id as u32); } Change::InsertContainer(id) => { container_changes.insert(id as u32); } Change::UpdateContainer(id) => { maybe_has_vanished = true; container_changes.insert(id as u32); } Change::DeleteContainer(_) | Change::DeleteItem(_) => { maybe_has_vanished = true; } Change::UpdateContainerProperty(_) => (), } } for (document_ids, changes) in [ (&mut display_containers, container_changes), (&mut display_children, item_changes), ] { if let Some(document_ids) = document_ids { *document_ids &= changes; total_changes += document_ids.len() as usize; } else { total_changes += changes.len() as usize; *document_ids = Some(changes); } } } else { let changes = RoaringBitmap::from_iter(changes.changes.iter().filter_map( |change| match change { Change::InsertItem(id) | Change::InsertContainer(id) => Some(*id as u32), Change::UpdateItem(id) | Change::UpdateContainer(id) => { maybe_has_vanished = true; Some(*id as u32) } Change::DeleteContainer(_) | Change::DeleteItem(_) => { maybe_has_vanished = true; None } _ => None, }, )); if let Some(document_ids) = &mut display_containers { *document_ids &= changes; total_changes += document_ids.len() as usize; } else { total_changes += changes.len() as usize; display_containers = Some(changes); } } if maybe_has_vanished && let Some(vanished_collection) = sync_collection.vanished_collection() { vanished = server .store() .vanished(account_id, vanished_collection.into(), Query::Since(id)) .await .caused_by(trc::location!())?; total_changes += vanished.len(); } // Truncate changes if total_changes > limit { let mut offset = limit * seq as usize; let mut total_changes = 0; // Add vanished items to response for item in vanished { if offset > 0 { offset -= 1; } else if total_changes < limit { response.add_response(Response::new_status([item], StatusCode::NOT_FOUND)); total_changes += 1; } else { *is_sync_limited = true; } } // Add items to document set for document_ids in [&mut display_containers, &mut display_children] .into_iter() .flatten() { let mut new_document_ids = RoaringBitmap::new(); for id in document_ids.iter() { if offset > 0 { offset -= 1; } else if total_changes < limit { new_document_ids.insert(id); total_changes += 1; } else { *is_sync_limited = true; } } *document_ids = new_document_ids; } if *is_sync_limited { response.set_sync_token(Urn::Sync { id, seq: seq + 1 }.to_string()); } } else { // Add vanished items to response for item in vanished { response.add_response(Response::new_status([item], StatusCode::NOT_FOUND)); } } if !*is_sync_limited { response.set_sync_token(resources.sync_token()); } true } SyncType::Initial => { response.set_sync_token(resources.sync_token()); false } SyncType::None => false, }; let mut results = Vec::new(); if let Some(resource) = resource.resource { results = resources .subtree_with_depth(resource, query.depth) .filter(|item| { display_containers.as_ref().is_none_or(|containers| { if container_has_children { if item.is_container() { containers.contains(item.document_id()) } else { display_children .as_ref() .is_some_and(|children| children.contains(item.document_id())) } } else { containers.contains(item.document_id()) } }) && (!query.depth_no_root || item.path() != resource) }) .map(|item| PropFindItem::new(resources.format_resource(item), account_id, item)) .collect::>(); } else { if !query.depth_no_root && query.sync_type.is_none_or_initial() { server .prepare_principal_propfind_response( access_token, collection_container, [account_id].into_iter(), &query.propfind, response, ) .await?; } if query.depth != 0 { results = resources .tree_with_depth(query.depth - 1) .filter(|item| { display_containers.as_ref().is_none_or(|containers| { if container_has_children { if item.is_container() { containers.contains(item.document_id()) } else { display_children .as_ref() .is_some_and(|children| children.contains(item.document_id())) } } else { containers.contains(item.document_id()) } }) }) .map(|item| PropFindItem::new(resources.format_resource(item), account_id, item)) .collect::>(); // Assisted discovery: // If 'bob' has access to 'jane' and `bill` calendars, a query to '/dav/cal/bob' will return: // - /dav/cal/bob/default // - /dav/cal/jane/default // - /dav/cal/bill/default // This is invalid but it's the only workaround for clients which do not support multiple home-sets if server.core.groupware.assisted_discovery && !is_sync && account_id == access_token.primary_id() && matches!( sync_collection, SyncCollection::Calendar | SyncCollection::AddressBook ) { for shared_account_id in access_token.all_ids_by_collection(collection_container) { if shared_account_id == access_token.primary_id() { continue; } let shared_resources = data .resources(server, access_token, shared_account_id, sync_collection) .await .caused_by(trc::location!())?; let shared_containers = (!access_token.is_member(shared_account_id)).then(|| { shared_resources.shared_containers( access_token, [if container_has_children { Acl::ReadItems } else { Acl::Read }], true, ) }); if shared_containers .as_ref() .is_none_or(|containers| !containers.is_empty()) { results.extend( shared_resources .tree_with_depth(query.depth - 1) .filter(|item| { item.is_container() && shared_containers.as_ref().is_none_or(|containers| { containers.contains(item.document_id()) }) }) .map(|item| { PropFindItem::new( shared_resources.format_resource(item), shared_account_id, item, ) }), ); } } } } } Ok(results) } #[allow(clippy::too_many_arguments)] async fn multiget( server: &Server, access_token: &AccessToken, collection_container: Collection, collection_children: Collection, sync_collection: SyncCollection, data: &mut PropFindData, response: &mut MultiStatus, hrefs: Vec, ) -> crate::Result> { let mut paths = Vec::with_capacity(hrefs.len() * 2); let mut shared_folders_by_account: AHashMap> = AHashMap::with_capacity(3); for item in hrefs { let resource = match server .validate_uri(access_token, &item) .await .and_then(|r| r.into_owned_uri()) { Ok(resource) => resource, Err(DavError::Code(code)) => { response.add_response(Response::new_status([item], code)); continue; } Err(err) => { return Err(err); } }; let account_id = resource.account_id; let resources = data .resources(server, access_token, account_id, sync_collection) .await .caused_by(trc::location!())?; let document_ids = if !access_token.is_member(account_id) { if let Some(document_ids) = shared_folders_by_account.get(&account_id) { document_ids.clone().into() } else { let document_ids = Arc::new(resources.shared_containers( access_token, [if collection_children == collection_container { Acl::ReadItems } else { Acl::Read }], true, )); shared_folders_by_account.insert(account_id, document_ids.clone()); document_ids.into() } } else { None }; if let Some(resource) = resource.resource.and_then(|name| resources.by_path(name)) { if !resource.is_container() { if document_ids .as_ref() .is_none_or(|docs| docs.contains(resource.parent_id().unwrap())) { paths.push(PropFindItem::new( resources.format_resource(resource), account_id, resource, )); } else { response.add_response( Response::new_status([item], StatusCode::FORBIDDEN) .with_response_description( "Not enough permissions to access this shared resource", ), ); } } else { response.add_response( Response::new_status([item], StatusCode::FORBIDDEN) .with_response_description("Multiget not allowed for collections"), ); } } else { response.add_response(Response::new_status([item], StatusCode::NOT_FOUND)); } } Ok(paths) } impl PropFindItem { pub fn new(name: String, account_id: u32, resource: DavResourcePath<'_>) -> Self { Self { name, account_id, document_id: resource.document_id(), parent_id: resource.parent_id(), is_container: resource.is_container(), } } } impl PropFindData { pub fn new() -> Self { Self { accounts: AHashMap::with_capacity(2), } } pub async fn quota( &mut self, server: &Server, access_token: &AccessToken, account_id: u32, ) -> trc::Result { let data = self.accounts.entry(account_id).or_default(); if data.quota.is_none() { data.quota = server.dav_quota(access_token, account_id).await?.into(); } Ok(data.quota.clone().unwrap()) } pub async fn owner( &mut self, server: &Server, access_token: &AccessToken, account_id: u32, ) -> trc::Result { let data = self.accounts.entry(account_id).or_default(); if data.owner.is_none() { data.owner = server .owner_href(access_token, account_id) .await .caused_by(trc::location!())? .into(); } Ok(data.owner.clone().unwrap()) } pub async fn resources( &mut self, server: &Server, access_token: &AccessToken, account_id: u32, sync_collection: SyncCollection, ) -> trc::Result> { let data = self.accounts.entry(account_id).or_default(); if data.resources.is_none() { let resources = server .fetch_dav_resources(access_token, account_id, sync_collection) .await .caused_by(trc::location!())?; data.resources = resources.into(); } Ok(data.resources.clone().unwrap()) } pub async fn locks( &mut self, server: &Server, account_id: u32, collection_container: Collection, item: &PropFindItem, ) -> trc::Result>> { let data = self.accounts.entry(account_id).or_default(); if data.locks.is_none() && !data.locks_not_found { data.locks = server .in_memory_store() .key_get::>( build_lock_key(account_id, collection_container).as_slice(), ) .await .caused_by(trc::location!())?; if data.locks.is_none() { data.locks_not_found = true; } } if let Some(lock_data) = &data.locks { let base_uri = dav_base_uri(&item.name).unwrap_or_default(); lock_data.unarchive::().map(|locks| { locks .find_locks(&item.name.strip_prefix(base_uri).unwrap()[1..], false) .iter() .map(|(path, lock)| lock.to_active_lock(format!("{base_uri}/{path}"))) .collect::>() .into() }) } else { Ok(None) } } } pub(crate) trait SyncTokenUrn { fn sync_token(&self) -> String; } impl SyncTokenUrn for DavResources { fn sync_token(&self) -> String { Urn::Sync { id: self.highest_change_id, seq: 0, } .to_string() } } async fn add_base_collection_response( server: &Server, request: &PropFind, collection: Collection, access_token: &AccessToken, response: &mut MultiStatus, ) -> trc::Result<()> { let properties = match request { PropFind::PropName => { response.add_response(Response::new_propstat( DavResourceName::from(collection).collection_path(), vec![PropStat::new_list(vec![ DavPropertyValue::empty(DavProperty::WebDav(WebDavProperty::ResourceType)), DavPropertyValue::empty(DavProperty::WebDav( WebDavProperty::CurrentUserPrincipal, )), DavPropertyValue::empty(DavProperty::WebDav( WebDavProperty::SupportedReportSet, )), ])], )); return Ok(()); } PropFind::AllProp(_) => [ DavProperty::WebDav(WebDavProperty::ResourceType), DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal), DavProperty::WebDav(WebDavProperty::SupportedReportSet), ] .as_slice(), PropFind::Prop(items) => items, }; let mut fields = Vec::with_capacity(properties.len()); let mut fields_not_found = Vec::new(); for prop in properties { match &prop { DavProperty::WebDav(WebDavProperty::ResourceType) => { fields.push(DavPropertyValue::new( prop.clone(), vec![ResourceType::Collection], )); } DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal) => { fields.push(DavPropertyValue::new( prop.clone(), vec![access_token.current_user_principal()], )); } DavProperty::Principal(PrincipalProperty::CalendarHomeSet) => { let hrefs = build_home_set( server, access_token, &access_token.name, access_token.primary_id, true, ) .await .caused_by(trc::location!())?; fields.push(DavPropertyValue::new(prop.clone(), hrefs)); response.set_namespace(Namespace::CalDav); } DavProperty::Principal(PrincipalProperty::AddressbookHomeSet) => { let hrefs = build_home_set( server, access_token, &access_token.name, access_token.primary_id, false, ) .await .caused_by(trc::location!())?; fields.push(DavPropertyValue::new(prop.clone(), hrefs)); response.set_namespace(Namespace::CardDav); } DavProperty::WebDav(WebDavProperty::SupportedReportSet) => { let reports = match collection { Collection::Principal => ReportSet::principal(), Collection::Calendar | Collection::CalendarEvent => ReportSet::calendar(), Collection::AddressBook | Collection::ContactCard => ReportSet::addressbook(), _ => ReportSet::file(), }; fields.push(DavPropertyValue::new(prop.clone(), reports)); } _ => { response.set_namespace(prop.namespace()); fields_not_found.push(DavPropertyValue::empty(prop.clone())); } } } let mut prop_stat = Vec::with_capacity(2); if !fields.is_empty() { prop_stat.push(PropStat::new_list(fields)); } if !fields_not_found.is_empty() { prop_stat.push(PropStat::new_list(fields_not_found).with_status(StatusCode::NOT_FOUND)); } response.add_response(Response::new_propstat( DavResourceName::from(collection).collection_path(), prop_stat, )); Ok(()) } ================================================ FILE: crates/dav/src/common/uri.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{DavError, DavResourceName}; use common::{Server, auth::AccessToken}; use directory::backend::internal::manage::ManageDirectory; use groupware::cache::GroupwareCache; use http_proto::request::decode_path_element; use hyper::StatusCode; use std::fmt::Display; use trc::AddContext; use types::collection::Collection; #[derive(Debug)] pub(crate) struct UriResource { pub collection: Collection, pub account_id: A, pub resource: R, } pub(crate) enum Urn { Lock(u64), Sync { id: u64, seq: u32 }, } pub(crate) type UnresolvedUri<'x> = UriResource, Option<&'x str>>; pub(crate) type OwnedUri<'x> = UriResource>; pub(crate) type DocumentUri = UriResource; pub(crate) trait DavUriResource: Sync + Send { fn validate_uri_with_status<'x>( &self, access_token: &AccessToken, uri: &'x str, error_status: StatusCode, ) -> impl Future>> + Send; fn validate_uri<'x>( &self, access_token: &AccessToken, uri: &'x str, ) -> impl Future>> + Send; fn map_uri_resource( &self, access_token: &AccessToken, uri: OwnedUri<'_>, ) -> impl Future>> + Send; } impl DavUriResource for Server { async fn validate_uri<'x>( &self, access_token: &AccessToken, uri: &'x str, ) -> crate::Result> { self.validate_uri_with_status(access_token, uri, StatusCode::NOT_FOUND) .await } async fn validate_uri_with_status<'x>( &self, access_token: &AccessToken, uri: &'x str, error_status: StatusCode, ) -> crate::Result> { let (_, uri_parts) = uri .split_once("/dav/") .ok_or(DavError::Code(error_status))?; let mut uri_parts = uri_parts .trim_end_matches('/') .splitn(3, '/') .filter(|x| !x.is_empty()); let mut resource = UriResource { collection: uri_parts .next() .and_then(DavResourceName::parse) .ok_or(DavError::Code(error_status))? .into(), account_id: None, resource: None, }; if let Some(account) = uri_parts.next() { // Parse account id let account_id = if let Some(account_id) = account.strip_prefix('_') { account_id .parse::() .map_err(|_| DavError::Code(error_status))? } else { let account = decode_path_element(account); if access_token.name == account { access_token.primary_id } else { self.store() .get_principal_id(&account) .await .caused_by(trc::location!())? .ok_or(DavError::Code(error_status))? } }; // Validate access if resource.collection != Collection::Principal && !access_token.has_access(account_id, resource.collection) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Obtain remaining path resource.account_id = Some(account_id); resource.resource = uri_parts.next(); } Ok(resource) } async fn map_uri_resource( &self, access_token: &AccessToken, uri: OwnedUri<'_>, ) -> trc::Result> { if let Some(resource) = uri.resource { if let Some(resource) = self .fetch_dav_resources(access_token, uri.account_id, uri.collection.into()) .await .caused_by(trc::location!())? .by_path(resource) { Ok(Some(DocumentUri { collection: if resource.is_container() { uri.collection } else { uri.collection.child_collection().unwrap_or(uri.collection) }, account_id: uri.account_id, resource: resource.document_id(), })) } else { Ok(None) } } else { Ok(None) } } } impl<'x> UnresolvedUri<'x> { pub fn into_owned_uri(self) -> crate::Result> { Ok(OwnedUri { collection: self.collection, account_id: self .account_id .ok_or(DavError::Code(StatusCode::FORBIDDEN))?, resource: self.resource, }) } } impl OwnedUri<'_> { pub fn new_owned( collection: Collection, account_id: u32, resource: Option<&str>, ) -> OwnedUri<'_> { OwnedUri { collection, account_id, resource, } } } /*impl UriResource { pub fn collection_path(&self) -> &'static str { DavResourceName::from(self.collection).collection_path() } }*/ impl Urn { pub fn try_extract_sync_id(token: &str) -> Option<&str> { token .strip_prefix("urn:stalwart:davsync:") .map(|x| x.split_once(':').map(|(x, _)| x).unwrap_or(x)) } pub fn parse(input: &str) -> Option { let inbox = input.strip_prefix("urn:stalwart:")?; let (kind, id) = inbox.split_once(':')?; match kind { "davlock" => u64::from_str_radix(id, 16).ok().map(Urn::Lock), "davsync" => { if let Some((id, seq)) = id.split_once(':') { let id = u64::from_str_radix(id, 16).ok()?; let seq = u32::from_str_radix(seq, 16).ok()?; Some(Urn::Sync { id, seq }) } else { u64::from_str_radix(id, 16) .ok() .map(|id| Urn::Sync { id, seq: 0 }) } } _ => None, } } pub fn try_unwrap_lock(&self) -> Option { match self { Urn::Lock(id) => Some(*id), _ => None, } } pub fn try_unwrap_sync(&self) -> Option<(u64, u32)> { match self { Urn::Sync { id, seq } => Some((*id, *seq)), _ => None, } } } impl Display for Urn { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Urn::Lock(id) => write!(f, "urn:stalwart:davlock:{id:x}",), Urn::Sync { id, seq } => { if *seq == 0 { write!(f, "urn:stalwart:davsync:{id:x}") } else { write!(f, "urn:stalwart:davsync:{id:x}:{seq:x}") } } } } } ================================================ FILE: crates/dav/src/file/copy_move.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::FromDavResource; use crate::{ DavError, DavMethod, common::{ ExtractETag, lock::{LockRequestHandler, ResourceState}, uri::{DavUriResource, UriResource}, }, file::{DavFileResource, FileItemId}, }; use common::{ DavResourcePath, DavResources, Server, auth::AccessToken, storage::index::ObjectIndexBuilder, }; use dav_proto::{Depth, RequestHeaders}; use groupware::{DestroyArchive, cache::GroupwareCache, file::FileNode}; use http_proto::HttpResponse; use hyper::StatusCode; use std::sync::Arc; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use store::{ ahash::AHashMap, write::{BatchBuilder, now}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection, VanishedCollection}, }; pub(crate) trait FileCopyMoveRequestHandler: Sync + Send { fn handle_file_copy_move_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, is_move: bool, ) -> impl Future> + Send; } impl FileCopyMoveRequestHandler for Server { async fn handle_file_copy_move_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, is_move: bool, ) -> crate::Result { // Validate source let from_resource_ = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let from_account_id = from_resource_.account_id; let from_resources = self .fetch_dav_resources(access_token, from_account_id, SyncCollection::FileNode) .await .caused_by(trc::location!())?; let from_resource = from_resources.map_resource::(&from_resource_)?; let from_resource_name = from_resource_.resource.unwrap(); // Validate source ACLs if !access_token.is_member(from_account_id) { let shared = from_resources.shared_containers( access_token, if is_move { [Acl::Read, Acl::Delete].as_slice().iter().copied() } else { [Acl::Read].as_slice().iter().copied() }, false, ); for resource in from_resources.subtree(from_resource_.resource.unwrap()) { if !shared.contains(resource.document_id()) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } } } // Validate destination let destination = self .validate_uri_with_status( access_token, headers .destination .ok_or(DavError::Code(StatusCode::BAD_GATEWAY))?, StatusCode::BAD_GATEWAY, ) .await?; if destination.collection != Collection::FileNode { return Err(DavError::Code(StatusCode::BAD_GATEWAY)); } let to_account_id = destination .account_id .ok_or(DavError::Code(StatusCode::BAD_GATEWAY))?; let to_resources = if to_account_id == from_account_id { from_resources.clone() } else { self.fetch_dav_resources(access_token, to_account_id, SyncCollection::FileNode) .await .caused_by(trc::location!())? }; // Map file item let destination_resource_name = destination .resource .ok_or(DavError::Code(StatusCode::BAD_GATEWAY))?; if from_account_id == to_account_id && (from_resource_name == destination_resource_name || from_resource_name .strip_prefix(destination_resource_name) .is_some_and(|v| v.is_empty() || v.starts_with('/'))) { return Ok(HttpResponse::new(StatusCode::BAD_GATEWAY)); } // Check if the resource exists let mut delete_destination = None; let mut destination = if let Some((destination, new_name)) = to_resources.map_parent(destination_resource_name) { if let Some(mut existing_destination) = to_resources .by_path(destination_resource_name) .map(Destination::from_dav_resource) { if !headers.overwrite_fail { existing_destination.account_id = to_account_id; delete_destination = Some(existing_destination); } else { return Ok(HttpResponse::new(StatusCode::PRECONDITION_FAILED)); } } let mut destination = destination .map(Destination::from_dav_resource) .unwrap_or_default(); destination.new_name = Some(new_name.to_string()); destination } else { return Err(DavError::Code(StatusCode::CONFLICT)); }; destination.account_id = to_account_id; // Validate destination ACLs if let Some(document_id) = destination.document_id { if let Some(delete_destination) = &delete_destination && !access_token.is_member(to_account_id) && !from_resources.has_access_to_container( access_token, delete_destination.document_id.unwrap(), Acl::Delete, ) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } if !access_token.is_member(to_account_id) && !from_resources.has_access_to_container(access_token, document_id, Acl::Modify) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } } else if !access_token.is_member(to_account_id) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Validate headers self.validate_headers( access_token, headers, vec![ ResourceState { account_id: from_account_id, collection: Collection::FileNode, document_id: Some(from_resource.resource.document_id), path: from_resource_name, ..Default::default() }, ResourceState { account_id: to_account_id, collection: Collection::FileNode, document_id: Some( delete_destination .as_ref() .and_then(|d| d.document_id) .unwrap_or(u32::MAX), ), path: destination_resource_name, ..Default::default() }, ], Default::default(), if is_move { DavMethod::MOVE } else { DavMethod::COPY }, ) .await?; if delete_destination.is_none() && from_account_id == destination.account_id && from_resource.resource.parent_id == destination.document_id && destination.new_name.is_some() && is_move { // Rename let from_resource_path = if from_resource.resource.is_container { from_resources.format_collection(from_resource_name) } else { from_resources.format_item(from_resource_name) }; return rename_item( self, access_token, from_resource, from_resource_path, destination, ) .await; } // Validate quota if !is_move || from_account_id != to_account_id { let space_needed = from_resources .subtree(from_resource_name) .map(|a| a.size() as u64) .sum::(); self.has_available_quota( &self.get_resource_token(access_token, to_account_id).await?, space_needed, ) .await?; } // Delete collection let is_overwrite = delete_destination .as_ref() .is_some_and(|d| d.is_container || from_resource.resource.is_container); if is_overwrite { delete_destination = None; // Find ids to delete let mut ids = to_resources .subtree(destination_resource_name) .collect::>(); if !ids.is_empty() { ids.sort_unstable_by_key(|b| std::cmp::Reverse(b.hierarchy_seq())); let mut sorted_ids = Vec::with_capacity(ids.len()); sorted_ids.extend(ids.into_iter().map(|a| a.document_id())); DestroyArchive(sorted_ids) .delete(self, access_token, destination.account_id, None) .await .caused_by(trc::location!())?; } } match (from_resource.resource.is_container, is_move) { (true, true) => { move_container( self, access_token, from_resources, from_resource, from_resource_name, destination, headers.depth, ) .await } (true, false) => { copy_container( self, access_token, from_resources, from_resource, from_resource_name, destination, headers.depth, false, ) .await } (false, true) => { if let Some(delete_destination) = delete_destination { overwrite_and_delete_item( self, access_token, from_resource, from_resources.format_item(from_resource_name), delete_destination, ) .await } else { move_item( self, access_token, from_resource, from_resources.format_item(from_resource_name), destination, ) .await } } (false, false) => { if let Some(delete_destination) = delete_destination { overwrite_item(self, access_token, from_resource, delete_destination).await } else { copy_item(self, access_token, from_resource, destination).await } } } .map(|r| { if is_overwrite && r.status() == StatusCode::CREATED { r.with_status_code(StatusCode::NO_CONTENT) } else { r } }) } } #[derive(Debug)] pub(crate) struct Destination { pub account_id: u32, pub new_name: Option, pub document_id: Option, pub is_container: bool, } impl Default for Destination { fn default() -> Self { Self { account_id: Default::default(), document_id: Default::default(), new_name: Default::default(), is_container: true, } } } // Moves a container under an existing container async fn move_container( server: &Server, access_token: &AccessToken, from_resources: Arc, from_resource: UriResource, from_resource_name: &str, destination: Destination, depth: Depth, ) -> crate::Result { let from_account_id = from_resource.account_id; let to_account_id = destination.account_id; let from_document_id = from_resource.resource.document_id; let parent_id = destination.document_id.map(|id| id + 1).unwrap_or(0); if from_account_id == to_account_id { let node_ = server .store() .get_value::>(ValueKey::archive( from_account_id, Collection::FileNode, from_document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let node = node_ .to_unarchived::() .caused_by(trc::location!())?; let mut new_node = node.deserialize::().caused_by(trc::location!())?; new_node.parent_id = parent_id; if let Some(new_name) = destination.new_name { new_node.name = new_name; } let mut batch = BatchBuilder::new(); let etag = new_node .update( access_token, node, from_account_id, from_document_id, &mut batch, ) .caused_by(trc::location!())? .etag(); batch.with_account_id(from_account_id).log_vanished_item( VanishedCollection::FileNode, from_resources.format_collection(from_resource_name), ); server .commit_batch(batch) .await .caused_by(trc::location!())?; Ok(HttpResponse::new(StatusCode::CREATED).with_etag_opt(etag)) } else { copy_container( server, access_token, from_resources, from_resource, from_resource_name, destination, depth, true, ) .await } } #[allow(clippy::too_many_arguments)] async fn copy_container( server: &Server, access_token: &AccessToken, from_resources: Arc, from_resource: UriResource, from_resource_name: &str, mut destination: Destination, depth: Depth, delete_source: bool, ) -> crate::Result { let infinity_copy = match depth { Depth::Zero => { return copy_item(server, access_token, from_resource, destination).await; } Depth::One => false, _ => true, }; let from_account_id = from_resource.account_id; let to_account_id = destination.account_id; let parent_id = destination.document_id.map(|id| id + 1).unwrap_or(0); // Obtain files to copy let mut copy_files = if infinity_copy { from_resources .subtree(from_resource_name) .map(|r| (r.document_id(), r.hierarchy_seq())) .collect::>() } else { from_resources .subtree_with_depth(from_resource_name, 1) .map(|r| (r.document_id(), r.hierarchy_seq())) .collect::>() }; // Top-down copy let mut batch = BatchBuilder::new(); let mut id_map = AHashMap::with_capacity(copy_files.len()); let mut delete_files = if delete_source { Vec::with_capacity(copy_files.len()) } else { Vec::new() }; copy_files.sort_unstable_by(|a, b| a.1.cmp(&b.1)); let now = now() as i64; let mut next_document_id = server .store() .assign_document_ids(to_account_id, Collection::FileNode, copy_files.len() as u64) .await .caused_by(trc::location!())?; for (document_id, _) in copy_files.into_iter() { let node_ = server .store() .get_value::>(ValueKey::archive( from_account_id, Collection::FileNode, document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))? .into_deserialized::() .caused_by(trc::location!())?; // Build node let mut node = if !delete_source { node_.inner } else { let node = node_.inner.clone(); delete_files.push((document_id, node_)); node }; node.modified = now; node.created = now; if let Some(new_name) = destination.new_name.take() { node.name = new_name; } node.parent_id = if let Some(&prev_document_id) = id_map.get(&node.parent_id) { prev_document_id } else { parent_id }; // Prepare write batch let new_document_id = next_document_id; next_document_id -= 1; batch .with_account_id(to_account_id) .with_collection(Collection::FileNode) .with_document(new_document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(node) .with_access_token(access_token), ) .caused_by(trc::location!())? .commit_point(); id_map.insert(document_id + 1, new_document_id + 1); } // Delete nodes if !delete_files.is_empty() { for (document_id, node) in delete_files.into_iter().rev() { // Delete record batch .with_account_id(from_account_id) .with_collection(Collection::FileNode) .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() .with_access_token(access_token) .with_current(node), ) .caused_by(trc::location!())? .commit_point(); } batch.with_account_id(from_account_id).log_vanished_item( VanishedCollection::FileNode, from_resources.format_collection(from_resource_name), ); } // Write changes if !batch.is_empty() { server .commit_batch(batch) .await .caused_by(trc::location!())?; } Ok(HttpResponse::new(StatusCode::CREATED)) } // Overwrites the contents of one file with another, then deletes the original async fn overwrite_and_delete_item( server: &Server, access_token: &AccessToken, from_resource: UriResource, from_resource_path: String, destination: Destination, ) -> crate::Result { let from_account_id = from_resource.account_id; let to_account_id = destination.account_id; let from_document_id = from_resource.resource.document_id; let to_document_id = destination.document_id.unwrap(); // dest_node is the current file at the destination let dest_node_ = server .store() .get_value::>(ValueKey::archive( to_account_id, Collection::FileNode, to_document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let dest_node = dest_node_ .to_unarchived::() .caused_by(trc::location!())?; // source_node is the file to be copied let source_node__ = server .store() .get_value::>(ValueKey::archive( from_account_id, Collection::FileNode, from_document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let source_node_ = source_node__ .to_unarchived::() .caused_by(trc::location!())?; let mut source_node = source_node_ .deserialize::() .caused_by(trc::location!())?; source_node.name = if let Some(new_name) = destination.new_name { new_name } else { dest_node.inner.name.to_string() }; source_node.parent_id = dest_node.inner.parent_id.into(); let mut batch = BatchBuilder::new(); let etag = source_node .update( access_token, dest_node, to_account_id, to_document_id, &mut batch, ) .caused_by(trc::location!())? .etag(); DestroyArchive(source_node_) .delete( access_token, from_account_id, from_document_id, &mut batch, from_resource_path, ) .caused_by(trc::location!())?; server .commit_batch(batch) .await .caused_by(trc::location!())?; Ok(HttpResponse::new(StatusCode::NO_CONTENT).with_etag_opt(etag)) } // Overwrites the contents of one file with another async fn overwrite_item( server: &Server, access_token: &AccessToken, from_resource: UriResource, destination: Destination, ) -> crate::Result { let from_account_id = from_resource.account_id; let to_account_id = destination.account_id; let from_document_id = from_resource.resource.document_id; let to_document_id = destination.document_id.unwrap(); // dest_node is the current file at the destination let dest_node_ = server .store() .get_value::>(ValueKey::archive( to_account_id, Collection::FileNode, to_document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let dest_node = dest_node_ .to_unarchived::() .caused_by(trc::location!())?; // source_node is the file to be copied let mut source_node = server .store() .get_value::>(ValueKey::archive( from_account_id, Collection::FileNode, from_document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))? .deserialize::() .caused_by(trc::location!())?; source_node.name = if let Some(new_name) = destination.new_name { new_name } else { dest_node.inner.name.to_string() }; source_node.parent_id = dest_node.inner.parent_id.into(); let mut batch = BatchBuilder::new(); let etag = source_node .update( access_token, dest_node, to_account_id, to_document_id, &mut batch, ) .caused_by(trc::location!())? .etag(); server .commit_batch(batch) .await .caused_by(trc::location!())?; Ok(HttpResponse::new(StatusCode::NO_CONTENT).with_etag_opt(etag)) } // Moves an item under an existing container async fn move_item( server: &Server, access_token: &AccessToken, from_resource: UriResource, from_resource_path: String, destination: Destination, ) -> crate::Result { let from_account_id = from_resource.account_id; let to_account_id = destination.account_id; let from_document_id = from_resource.resource.document_id; let parent_id = destination.document_id.map(|id| id + 1).unwrap_or(0); let node_ = server .store() .get_value::>(ValueKey::archive( from_account_id, Collection::FileNode, from_document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let node = node_ .to_unarchived::() .caused_by(trc::location!())?; let mut new_node = node.deserialize::().caused_by(trc::location!())?; new_node.parent_id = parent_id; if let Some(new_name) = destination.new_name { new_node.name = new_name; } let mut batch = BatchBuilder::new(); let etag = if from_account_id == to_account_id { // Destination is in the same account: just update the parent id batch.log_vanished_item(VanishedCollection::FileNode, from_resource_path); new_node .update( access_token, node, from_account_id, from_document_id, &mut batch, ) .caused_by(trc::location!())? .etag() } else { // Destination is in a different account: insert a new node, then delete the old one let to_document_id = server .store() .assign_document_ids(to_account_id, Collection::FileNode, 1) .await .caused_by(trc::location!())?; let etag = new_node .insert(access_token, to_account_id, to_document_id, &mut batch) .caused_by(trc::location!())? .etag(); DestroyArchive(node) .delete( access_token, from_account_id, from_document_id, &mut batch, from_resource_path, ) .caused_by(trc::location!())?; etag }; server .commit_batch(batch) .await .caused_by(trc::location!())?; Ok(HttpResponse::new(StatusCode::CREATED).with_etag_opt(etag)) } // Copies an item under an existing container async fn copy_item( server: &Server, access_token: &AccessToken, from_resource: UriResource, destination: Destination, ) -> crate::Result { let from_account_id = from_resource.account_id; let to_account_id = destination.account_id; let from_document_id = from_resource.resource.document_id; let parent_id = destination.document_id.map(|id| id + 1).unwrap_or(0); let mut node = server .store() .get_value::>(ValueKey::archive( from_account_id, Collection::FileNode, from_document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))? .deserialize::() .caused_by(trc::location!())?; node.parent_id = parent_id; if let Some(new_name) = destination.new_name { node.name = new_name; } let mut batch = BatchBuilder::new(); let to_document_id = server .store() .assign_document_ids(to_account_id, Collection::FileNode, 1) .await .caused_by(trc::location!())?; let etag = node .insert(access_token, to_account_id, to_document_id, &mut batch) .caused_by(trc::location!())? .etag(); server .commit_batch(batch) .await .caused_by(trc::location!())?; Ok(HttpResponse::new(StatusCode::CREATED).with_etag_opt(etag)) } // Renames an item async fn rename_item( server: &Server, access_token: &AccessToken, from_resource: UriResource, from_resource_path: String, destination: Destination, ) -> crate::Result { let from_account_id = from_resource.account_id; let from_document_id = from_resource.resource.document_id; let node_ = server .store() .get_value::>(ValueKey::archive( from_account_id, Collection::FileNode, from_document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let node = node_ .to_unarchived::() .caused_by(trc::location!())?; let mut new_node = node.deserialize::().caused_by(trc::location!())?; if let Some(new_name) = destination.new_name { new_node.name = new_name; } let mut batch = BatchBuilder::new(); let etag = new_node .update( access_token, node, from_account_id, from_document_id, &mut batch, ) .caused_by(trc::location!())? .etag(); batch.log_vanished_item(VanishedCollection::FileNode, from_resource_path); server .commit_batch(batch) .await .caused_by(trc::location!())?; Ok(HttpResponse::new(StatusCode::CREATED).with_etag_opt(etag)) } impl FromDavResource for Destination { fn from_dav_resource(item: DavResourcePath<'_>) -> Self { Destination { account_id: u32::MAX, document_id: Some(item.document_id()), is_container: item.is_container(), new_name: None, } } } ================================================ FILE: crates/dav/src/file/delete.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ DavError, DavMethod, common::{ lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, }; use common::{Server, auth::AccessToken}; use dav_proto::RequestHeaders; use groupware::{DestroyArchive, cache::GroupwareCache}; use http_proto::HttpResponse; use hyper::StatusCode; use trc::AddContext; use types::{acl::Acl, collection::SyncCollection}; pub(crate) trait FileDeleteRequestHandler: Sync + Send { fn handle_file_delete_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, ) -> impl Future> + Send; } impl FileDeleteRequestHandler for Server { async fn handle_file_delete_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, ) -> crate::Result { // Validate URI let resource = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let account_id = resource.account_id; let delete_path = resource .resource .filter(|r| !r.is_empty()) .ok_or(DavError::Code(StatusCode::FORBIDDEN))?; let resources = self .fetch_dav_resources(access_token, account_id, SyncCollection::FileNode) .await .caused_by(trc::location!())?; // Find ids to delete let mut ids = resources.subtree(delete_path).collect::>(); if ids.is_empty() { return Err(DavError::Code(StatusCode::NOT_FOUND)); } // Sort ids descending from the deepest to the root ids.sort_unstable_by_key(|b| std::cmp::Reverse(b.hierarchy_seq())); let (document_id, full_delete_path) = ids .last() .map(|a| (a.document_id(), resources.format_resource(*a))) .unwrap(); let mut sorted_ids = Vec::with_capacity(ids.len()); sorted_ids.extend(ids.into_iter().map(|a| a.document_id())); // Validate ACLs if !access_token.is_member(account_id) { let permissions = resources.shared_containers(access_token, [Acl::Delete], false); if permissions.len() < sorted_ids.len() as u64 || !sorted_ids.iter().all(|id| permissions.contains(*id)) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } } // Validate headers self.validate_headers( access_token, headers, vec![ResourceState { account_id, collection: resource.collection, document_id: document_id.into(), path: delete_path, ..Default::default() }], Default::default(), DavMethod::DELETE, ) .await?; DestroyArchive(sorted_ids) .delete(self, access_token, account_id, full_delete_path.into()) .await?; Ok(HttpResponse::new(StatusCode::NO_CONTENT)) } } ================================================ FILE: crates/dav/src/file/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ DavError, DavMethod, common::{ ETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, file::DavFileResource, }; use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; use dav_proto::{RequestHeaders, schema::property::Rfc1123DateTime}; use groupware::{cache::GroupwareCache, file::FileNode}; use http_proto::HttpResponse; use hyper::StatusCode; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, }; pub(crate) trait FileGetRequestHandler: Sync + Send { fn handle_file_get_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, is_head: bool, ) -> impl Future> + Send; } impl FileGetRequestHandler for Server { async fn handle_file_get_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, is_head: bool, ) -> crate::Result { // Validate URI let resource_ = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let account_id = resource_.account_id; let files = self .fetch_dav_resources(access_token, account_id, SyncCollection::FileNode) .await .caused_by(trc::location!())?; let resource = files.map_resource(&resource_)?; // Fetch node let node_ = self .store() .get_value::>(ValueKey::archive( account_id, Collection::FileNode, resource.resource, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let node = node_.unarchive::().caused_by(trc::location!())?; // Validate ACL if !access_token.is_member(account_id) && !node.acls.effective_acl(access_token).contains(Acl::Read) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } let (hash, size, content_type) = if let Some(file) = node.file.as_ref() { ( file.blob_hash.0.as_ref(), u32::from(file.size) as usize, file.media_type.as_ref().map(|s| s.as_str()), ) } else { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); }; // Validate headers let etag = node_.etag(); self.validate_headers( access_token, headers, vec![ResourceState { account_id, collection: resource.collection, document_id: resource.resource.into(), etag: etag.clone().into(), path: resource_.resource.unwrap(), ..Default::default() }], Default::default(), DavMethod::GET, ) .await?; let response = HttpResponse::new(StatusCode::OK) .with_content_type(content_type.unwrap_or("application/octet-stream")) .with_etag(etag) .with_last_modified(Rfc1123DateTime::new(i64::from(node.modified)).to_string()); if !is_head { Ok(response.with_binary_body( self.blob_store() .get_blob(hash, 0..usize::MAX) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?, )) } else { Ok(response.with_content_length(size)) } } } ================================================ FILE: crates/dav/src/file/mkcol.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::proppatch::FilePropPatchRequestHandler; use crate::{ DavMethod, PropStatBuilder, common::{ ExtractETag, acl::ResourceAcl, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, file::DavFileResource, }; use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; use dav_proto::{ RequestHeaders, Return, schema::{Namespace, request::MkCol, response::MkColResponse}, }; use groupware::{cache::GroupwareCache, file::FileNode}; use http_proto::HttpResponse; use hyper::StatusCode; use store::write::{BatchBuilder, now}; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, }; pub(crate) trait FileMkColRequestHandler: Sync + Send { fn handle_file_mkcol_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: Option, ) -> impl Future> + Send; } impl FileMkColRequestHandler for Server { async fn handle_file_mkcol_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: Option, ) -> crate::Result { // Validate URI let resource_ = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let account_id = resource_.account_id; let resources = self .fetch_dav_resources(access_token, account_id, SyncCollection::FileNode) .await .caused_by(trc::location!())?; let resource = resources.map_parent_resource(&resource_)?; // Validate and map parent ACL let parent_id = resources.validate_and_map_parent_acl( access_token, access_token.is_member(account_id), resource.resource.0, Acl::AddItems, )?; // Validate headers self.validate_headers( access_token, headers, vec![ResourceState { account_id, collection: resource.collection, document_id: Some(u32::MAX), path: resource_.resource.unwrap(), ..Default::default() }], Default::default(), DavMethod::MKCOL, ) .await?; // Build file container let now = now(); let mut node = FileNode { parent_id, name: resource.resource.1.to_string(), display_name: None, file: None, created: now as i64, modified: now as i64, dead_properties: Default::default(), acls: Default::default(), }; // Apply MKCOL properties let mut return_prop_stat = None; if let Some(mkcol) = request { let mut prop_stat = PropStatBuilder::default(); if !self.apply_file_properties(&mut node, false, mkcol.props, &mut prop_stat) { return Ok(HttpResponse::new(StatusCode::FORBIDDEN).with_xml_body( MkColResponse::new(prop_stat.build()) .with_namespace(Namespace::Dav) .to_string(), )); } if headers.ret != Return::Minimal { return_prop_stat = Some(prop_stat); } } // Prepare write batch let document_id = self .store() .assign_document_ids(account_id, Collection::FileNode, 1) .await .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::FileNode) .with_document(document_id) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(node)) .caused_by(trc::location!())?; let etag = batch.etag(); self.commit_batch(batch).await.caused_by(trc::location!())?; if let Some(prop_stat) = return_prop_stat { Ok(HttpResponse::new(StatusCode::CREATED) .with_xml_body( MkColResponse::new(prop_stat.build()) .with_namespace(Namespace::Dav) .to_string(), ) .with_etag_opt(etag)) } else { Ok(HttpResponse::new(StatusCode::CREATED).with_etag_opt(etag)) } } } ================================================ FILE: crates/dav/src/file/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ DavError, common::uri::{OwnedUri, UriResource}, }; use common::{DavResourcePath, DavResources}; use dav_proto::schema::property::{DavProperty, WebDavProperty}; use hyper::StatusCode; pub mod copy_move; pub mod delete; pub mod get; pub mod mkcol; pub mod proppatch; pub mod update; pub(crate) static FILE_CONTAINER_PROPS: [DavProperty; 19] = [ DavProperty::WebDav(WebDavProperty::CreationDate), DavProperty::WebDav(WebDavProperty::DisplayName), DavProperty::WebDav(WebDavProperty::GetETag), DavProperty::WebDav(WebDavProperty::GetLastModified), DavProperty::WebDav(WebDavProperty::ResourceType), DavProperty::WebDav(WebDavProperty::LockDiscovery), DavProperty::WebDav(WebDavProperty::SupportedLock), DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal), DavProperty::WebDav(WebDavProperty::SyncToken), DavProperty::WebDav(WebDavProperty::Owner), DavProperty::WebDav(WebDavProperty::SupportedPrivilegeSet), DavProperty::WebDav(WebDavProperty::CurrentUserPrivilegeSet), DavProperty::WebDav(WebDavProperty::Acl), DavProperty::WebDav(WebDavProperty::AclRestrictions), DavProperty::WebDav(WebDavProperty::InheritedAclSet), DavProperty::WebDav(WebDavProperty::PrincipalCollectionSet), DavProperty::WebDav(WebDavProperty::SupportedReportSet), DavProperty::WebDav(WebDavProperty::QuotaAvailableBytes), DavProperty::WebDav(WebDavProperty::QuotaUsedBytes), ]; pub(crate) static FILE_ITEM_PROPS: [DavProperty; 19] = [ DavProperty::WebDav(WebDavProperty::CreationDate), DavProperty::WebDav(WebDavProperty::DisplayName), DavProperty::WebDav(WebDavProperty::GetETag), DavProperty::WebDav(WebDavProperty::GetLastModified), DavProperty::WebDav(WebDavProperty::ResourceType), DavProperty::WebDav(WebDavProperty::LockDiscovery), DavProperty::WebDav(WebDavProperty::SupportedLock), DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal), DavProperty::WebDav(WebDavProperty::SyncToken), DavProperty::WebDav(WebDavProperty::Owner), DavProperty::WebDav(WebDavProperty::SupportedPrivilegeSet), DavProperty::WebDav(WebDavProperty::CurrentUserPrivilegeSet), DavProperty::WebDav(WebDavProperty::Acl), DavProperty::WebDav(WebDavProperty::AclRestrictions), DavProperty::WebDav(WebDavProperty::InheritedAclSet), DavProperty::WebDav(WebDavProperty::PrincipalCollectionSet), DavProperty::WebDav(WebDavProperty::GetContentLanguage), DavProperty::WebDav(WebDavProperty::GetContentLength), DavProperty::WebDav(WebDavProperty::GetContentType), ]; pub(crate) trait FromDavResource { fn from_dav_resource(item: DavResourcePath<'_>) -> Self; } pub(crate) struct FileItemId { pub document_id: u32, pub parent_id: Option, pub is_container: bool, } pub(crate) trait DavFileResource { fn map_resource( &self, resource: &OwnedUri<'_>, ) -> crate::Result>; fn map_parent<'x>(&self, resource: &'x str) -> Option<(Option>, &'x str)>; #[allow(clippy::type_complexity)] fn map_parent_resource<'x, T: FromDavResource>( &self, resource: &OwnedUri<'x>, ) -> crate::Result, &'x str)>>; } impl DavFileResource for DavResources { fn map_resource( &self, resource: &OwnedUri<'_>, ) -> crate::Result> { resource .resource .and_then(|r| self.by_path(r)) .map(|r| UriResource { collection: resource.collection, account_id: resource.account_id, resource: T::from_dav_resource(r), }) .ok_or(DavError::Code(StatusCode::NOT_FOUND)) } fn map_parent<'x>(&self, resource: &'x str) -> Option<(Option>, &'x str)> { let (parent, child) = if let Some((parent, child)) = resource.rsplit_once('/') { (Some(self.by_path(parent)?), child) } else { (None, resource) }; Some((parent, child)) } fn map_parent_resource<'x, T: FromDavResource>( &self, resource: &OwnedUri<'x>, ) -> crate::Result, &'x str)>> { if let Some(r) = resource.resource { if self.by_path(r).is_none() { self.map_parent(r) .map(|(parent, child)| UriResource { collection: resource.collection, account_id: resource.account_id, resource: (parent.map(T::from_dav_resource), child), }) .ok_or(DavError::Code(StatusCode::CONFLICT)) } else { Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)) } } else { Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)) } } } impl FromDavResource for u32 { fn from_dav_resource(item: DavResourcePath) -> Self { item.document_id() } } impl FromDavResource for FileItemId { fn from_dav_resource(item: DavResourcePath) -> Self { FileItemId { document_id: item.document_id(), parent_id: item.parent_id(), is_container: item.is_container(), } } } ================================================ FILE: crates/dav/src/file/proppatch.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ DavError, DavMethod, PropStatBuilder, common::{ ETag, ExtractETag, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, file::DavFileResource, }; use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; use dav_proto::{ RequestHeaders, Return, schema::{ property::{DavProperty, DavValue, ResourceType, WebDavProperty}, request::{DavPropertyValue, PropertyUpdate}, response::{BaseCondition, MultiStatus, Response}, }, }; use groupware::{cache::GroupwareCache, file::FileNode}; use http_proto::HttpResponse; use hyper::StatusCode; use store::write::BatchBuilder; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, }; pub(crate) trait FilePropPatchRequestHandler: Sync + Send { fn handle_file_proppatch_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: PropertyUpdate, ) -> impl Future> + Send; fn apply_file_properties( &self, file: &mut FileNode, is_update: bool, properties: Vec, items: &mut PropStatBuilder, ) -> bool; } impl FilePropPatchRequestHandler for Server { async fn handle_file_proppatch_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, mut request: PropertyUpdate, ) -> crate::Result { // Validate URI let resource_ = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let uri = headers.uri; let account_id = resource_.account_id; let files = self .fetch_dav_resources(access_token, account_id, SyncCollection::FileNode) .await .caused_by(trc::location!())?; let resource = files.map_resource(&resource_)?; if !request.has_changes() { return Ok(HttpResponse::new(StatusCode::NO_CONTENT)); } // Fetch node let node_ = self .store() .get_value::>(ValueKey::archive( account_id, Collection::FileNode, resource.resource, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let node = node_ .to_unarchived::() .caused_by(trc::location!())?; // Validate ACL if !access_token.is_member(account_id) && !node .inner .acls .effective_acl(access_token) .contains(Acl::Modify) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Validate headers self.validate_headers( access_token, headers, vec![ResourceState { account_id, collection: resource.collection, document_id: resource.resource.into(), etag: node_.etag().into(), path: resource_.resource.unwrap(), ..Default::default() }], Default::default(), DavMethod::PROPPATCH, ) .await?; // Deserialize let mut new_node = node.deserialize::().caused_by(trc::location!())?; // Remove properties let mut items = PropStatBuilder::default(); if !request.set_first && !request.remove.is_empty() { remove_file_properties( &mut new_node, std::mem::take(&mut request.remove), &mut items, ); } // Set properties let is_success = self.apply_file_properties(&mut new_node, true, request.set, &mut items); // Remove properties if is_success && !request.remove.is_empty() { remove_file_properties(&mut new_node, request.remove, &mut items); } let etag = if is_success { let mut batch = BatchBuilder::new(); let etag = new_node .update( access_token, node, account_id, resource.resource, &mut batch, ) .caused_by(trc::location!())? .etag(); self.commit_batch(batch).await.caused_by(trc::location!())?; etag } else { node_.etag().into() }; if headers.ret != Return::Minimal || !is_success { Ok(HttpResponse::new(StatusCode::MULTI_STATUS) .with_xml_body( MultiStatus::new(vec![Response::new_propstat(uri, items.build())]).to_string(), ) .with_etag_opt(etag)) } else { Ok(HttpResponse::new(StatusCode::NO_CONTENT).with_etag_opt(etag)) } } fn apply_file_properties( &self, file: &mut FileNode, is_update: bool, properties: Vec, items: &mut PropStatBuilder, ) -> bool { let mut has_errors = false; for property in properties { match (&property.property, property.value) { (DavProperty::WebDav(WebDavProperty::DisplayName), DavValue::String(name)) => { if name.len() <= self.core.groupware.live_property_size { file.display_name = Some(name); items.insert_ok(property.property); } else { items.insert_error_with_description( property.property, StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); has_errors = true; } } (DavProperty::WebDav(WebDavProperty::CreationDate), DavValue::Timestamp(dt)) => { file.created = dt; items.insert_ok(property.property); } (DavProperty::WebDav(WebDavProperty::GetContentType), DavValue::String(name)) if file.file.is_some() => { if name.len() <= self.core.groupware.live_property_size { file.file.as_mut().unwrap().media_type = Some(name); items.insert_ok(property.property); } else { items.insert_error_with_description( property.property, StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); has_errors = true; } } ( DavProperty::WebDav(WebDavProperty::ResourceType), DavValue::ResourceTypes(types), ) if file.file.is_none() => { if types.0.len() != 1 || types.0.first() != Some(&ResourceType::Collection) { items.insert_precondition_failed( property.property, StatusCode::FORBIDDEN, BaseCondition::ValidResourceType, ); has_errors = true; } else { items.insert_ok(property.property); } } (DavProperty::DeadProperty(dead), DavValue::DeadProperty(values)) if self.core.groupware.dead_property_size.is_some() => { if is_update { file.dead_properties.remove_element(dead); } if file.dead_properties.size() + values.size() + dead.size() < self.core.groupware.dead_property_size.unwrap() { file.dead_properties.add_element(dead.clone(), values.0); items.insert_ok(property.property); } else { items.insert_error_with_description( property.property, StatusCode::INSUFFICIENT_STORAGE, "Property value is too long", ); has_errors = true; } } (_, DavValue::Null) => { items.insert_ok(property.property); } _ => { items.insert_error_with_description( property.property, StatusCode::CONFLICT, "Property cannot be modified", ); has_errors = true; } } } !has_errors } } fn remove_file_properties( node: &mut FileNode, properties: Vec, items: &mut PropStatBuilder, ) { for property in properties { match &property { DavProperty::WebDav(WebDavProperty::DisplayName) => { node.display_name = None; items.insert_with_status(property, StatusCode::NO_CONTENT); } DavProperty::WebDav(WebDavProperty::GetContentType) if node.file.is_some() => { node.file.as_mut().unwrap().media_type = None; items.insert_with_status(property, StatusCode::NO_CONTENT); } DavProperty::DeadProperty(dead) => { node.dead_properties.remove_element(dead); items.insert_with_status(property, StatusCode::NO_CONTENT); } _ => { items.insert_error_with_description( property, StatusCode::CONFLICT, "Property cannot be deleted", ); } } } } ================================================ FILE: crates/dav/src/file/update.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ DavError, DavMethod, common::{ ETag, ExtractETag, acl::ResourceAcl, lock::{LockRequestHandler, ResourceState}, uri::DavUriResource, }, file::DavFileResource, }; use common::{ Server, auth::AccessToken, sharing::EffectiveAcl, storage::index::ObjectIndexBuilder, }; use dav_proto::{RequestHeaders, Return, schema::property::Rfc1123DateTime}; use groupware::{ cache::GroupwareCache, file::{FileNode, FileProperties}, }; use http_proto::HttpResponse; use hyper::StatusCode; use store::write::{BatchBuilder, now}; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{ acl::Acl, blob_hash::BlobHash, collection::{Collection, SyncCollection}, }; pub(crate) trait FileUpdateRequestHandler: Sync + Send { fn handle_file_update_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, bytes: Vec, is_patch: bool, ) -> impl Future> + Send; } impl FileUpdateRequestHandler for Server { async fn handle_file_update_request( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, bytes: Vec, _is_patch: bool, ) -> crate::Result { // Validate URI let resource = self .validate_uri(access_token, headers.uri) .await? .into_owned_uri()?; let account_id = resource.account_id; let resources = self .fetch_dav_resources(access_token, account_id, SyncCollection::FileNode) .await .caused_by(trc::location!())?; let resource_name = resource .resource .ok_or(DavError::Code(StatusCode::CONFLICT))?; if bytes.len() > self.core.groupware.max_file_size { return Err(DavError::Code(StatusCode::PAYLOAD_TOO_LARGE)); } if let Some(document_id) = resources .by_path(resource_name.as_ref()) .map(|r| r.document_id()) { // Update let node_ = self .store() .get_value::>(ValueKey::archive( account_id, Collection::FileNode, document_id, )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; let node = node_ .to_unarchived::() .caused_by(trc::location!())?; // Validate ACL if !access_token.is_member(account_id) && !node .inner .acls .effective_acl(access_token) .contains(Acl::Modify) { return Err(DavError::Code(StatusCode::FORBIDDEN)); } // Validate headers match self .validate_headers( access_token, headers, vec![ResourceState { account_id, collection: resource.collection, document_id: Some(document_id), etag: node.etag().into(), path: resource_name, ..Default::default() }], Default::default(), DavMethod::PUT, ) .await { Ok(_) => {} Err(DavError::Code(StatusCode::PRECONDITION_FAILED)) if headers.ret == Return::Representation => { let file = node.inner.file.as_ref().unwrap(); let contents = self .blob_store() .get_blob(file.blob_hash.0.as_slice(), 0..usize::MAX) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::PRECONDITION_FAILED))?; return Ok(HttpResponse::new(StatusCode::PRECONDITION_FAILED) .with_content_type( file.media_type .as_ref() .map(|v| v.as_str()) .unwrap_or("application/octet-stream"), ) .with_etag(node.etag()) .with_last_modified( Rfc1123DateTime::new(i64::from(node.inner.modified)).to_string(), ) .with_header("Preference-Applied", "return=representation") .with_binary_body(contents)); } Err(e) => return Err(e), } // Verify that the node is a file if let Some(file) = node.inner.file.as_ref() { if BlobHash::generate(&bytes).as_slice() == file.blob_hash.0.as_slice() { return Ok(HttpResponse::new(StatusCode::NO_CONTENT)); } } else { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } // Validate quota let extra_bytes = (bytes.len() as u64) .saturating_sub(u32::from(node.inner.file.as_ref().unwrap().size) as u64); if extra_bytes > 0 { self.has_available_quota( &self.get_resource_token(access_token, account_id).await?, extra_bytes, ) .await?; } // Write blob let (blob_hash, blob_hold) = self .put_temporary_blob(account_id, &bytes, 60) .await .caused_by(trc::location!())?; // Build node let mut new_node = node.deserialize::().caused_by(trc::location!())?; let new_file = new_node.file.as_mut().unwrap(); new_file.blob_hash = blob_hash; new_file.media_type = headers .content_type .filter(|ct| !ct.is_empty() && *ct != "application/octet-stream") .map(|v| v.to_string()); new_file.size = bytes.len() as u32; new_node.modified = now() as i64; // Prepare write batch let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::FileNode) .with_document(document_id) .clear(blob_hold) .custom( ObjectIndexBuilder::new() .with_current(node) .with_changes(new_node) .with_access_token(access_token), ) .caused_by(trc::location!())?; let etag = batch.etag(); self.commit_batch(batch).await.caused_by(trc::location!())?; Ok(HttpResponse::new(StatusCode::NO_CONTENT).with_etag_opt(etag)) } else { // Insert let orig_resource_name = resource_name; let (parent, resource_name) = resources .map_parent(orig_resource_name.as_ref()) .ok_or(DavError::Code(StatusCode::CONFLICT))?; // Validate ACL let parent_id = resources.validate_and_map_parent_acl( access_token, access_token.is_member(account_id), parent.map(|r| r.document_id()), Acl::AddItems, )?; // Verify that parent is a collection if parent.as_ref().is_some_and(|r| !r.is_container()) { return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)); } // Validate headers self.validate_headers( access_token, headers, vec![ResourceState { account_id, collection: resource.collection, document_id: Some(u32::MAX), path: orig_resource_name, ..Default::default() }], Default::default(), DavMethod::PUT, ) .await?; // Validate quota if !bytes.is_empty() { self.has_available_quota( &self.get_resource_token(access_token, account_id).await?, bytes.len() as u64, ) .await?; } // Write blob let (blob_hash, blob_hold) = self .put_temporary_blob(account_id, &bytes, 60) .await .caused_by(trc::location!())?; // Build node let now = now(); let node = FileNode { parent_id, name: resource_name.to_string(), display_name: None, file: Some(FileProperties { blob_hash, size: bytes.len() as u32, media_type: headers.content_type.map(|v| v.to_string()), executable: false, }), created: now as i64, modified: now as i64, dead_properties: Default::default(), acls: parent .as_ref() .and_then(|p| p.resource.acls()) .map(|acls| acls.to_vec()) .unwrap_or_default(), }; // Prepare write batch let mut batch = BatchBuilder::new(); let document_id = self .store() .assign_document_ids(account_id, Collection::FileNode, 1) .await .caused_by(trc::location!())?; batch .with_account_id(account_id) .with_collection(Collection::FileNode) .with_document(document_id) .clear(blob_hold) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(node) .with_access_token(access_token), ) .caused_by(trc::location!())?; let etag = batch.etag(); self.commit_batch(batch).await.caused_by(trc::location!())?; Ok(HttpResponse::new(StatusCode::CREATED).with_etag_opt(etag)) } } } ================================================ FILE: crates/dav/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ #![warn(clippy::large_futures)] pub mod calendar; pub mod card; pub mod common; pub mod file; pub mod principal; pub mod request; use dav_proto::schema::{ request::DavPropertyValue, response::{Condition, List, Prop, PropStat, ResponseDescription, Status}, }; use groupware::{DavResourceName, RFC_3986}; use hyper::{Method, StatusCode}; use std::borrow::Cow; use store::ahash::AHashMap; pub(crate) type Result = std::result::Result; #[derive(Debug, Clone, Copy)] pub enum DavMethod { GET, PUT, POST, DELETE, HEAD, PATCH, PROPFIND, PROPPATCH, REPORT, MKCOL, MKCALENDAR, COPY, MOVE, LOCK, UNLOCK, OPTIONS, ACL, } impl From for trc::WebDavEvent { fn from(value: DavMethod) -> Self { match value { DavMethod::GET => trc::WebDavEvent::Get, DavMethod::PUT => trc::WebDavEvent::Put, DavMethod::POST => trc::WebDavEvent::Post, DavMethod::DELETE => trc::WebDavEvent::Delete, DavMethod::HEAD => trc::WebDavEvent::Head, DavMethod::PATCH => trc::WebDavEvent::Patch, DavMethod::PROPFIND => trc::WebDavEvent::Propfind, DavMethod::PROPPATCH => trc::WebDavEvent::Proppatch, DavMethod::REPORT => trc::WebDavEvent::Report, DavMethod::MKCOL => trc::WebDavEvent::Mkcol, DavMethod::MKCALENDAR => trc::WebDavEvent::Mkcalendar, DavMethod::COPY => trc::WebDavEvent::Copy, DavMethod::MOVE => trc::WebDavEvent::Move, DavMethod::LOCK => trc::WebDavEvent::Lock, DavMethod::UNLOCK => trc::WebDavEvent::Unlock, DavMethod::OPTIONS => trc::WebDavEvent::Options, DavMethod::ACL => trc::WebDavEvent::Acl, } } } pub(crate) enum DavError { Parse(dav_proto::parser::Error), Internal(trc::Error), Condition(DavErrorCondition), Code(StatusCode), } struct DavErrorCondition { pub code: StatusCode, pub condition: Condition, pub details: Option, } impl From for DavError { fn from(value: DavErrorCondition) -> Self { DavError::Condition(value) } } impl From for DavErrorCondition { fn from(value: Condition) -> Self { DavErrorCondition { code: StatusCode::CONFLICT, condition: value, details: None, } } } impl DavErrorCondition { pub fn new(code: StatusCode, condition: impl Into) -> Self { DavErrorCondition { code, condition: condition.into(), details: None, } } pub fn with_details(mut self, details: impl Into) -> Self { self.details = Some(details.into()); self } } impl DavMethod { pub fn parse(method: &Method) -> Option { match *method { Method::GET => Some(DavMethod::GET), Method::PUT => Some(DavMethod::PUT), Method::DELETE => Some(DavMethod::DELETE), Method::OPTIONS => Some(DavMethod::OPTIONS), Method::POST => Some(DavMethod::POST), Method::PATCH => Some(DavMethod::PATCH), Method::HEAD => Some(DavMethod::HEAD), _ => { hashify::tiny_map!(method.as_str().as_bytes(), "PROPFIND" => DavMethod::PROPFIND, "PROPPATCH" => DavMethod::PROPPATCH, "REPORT" => DavMethod::REPORT, "MKCOL" => DavMethod::MKCOL, "MKCALENDAR" => DavMethod::MKCALENDAR, "COPY" => DavMethod::COPY, "MOVE" => DavMethod::MOVE, "LOCK" => DavMethod::LOCK, "UNLOCK" => DavMethod::UNLOCK, "ACL" => DavMethod::ACL ) } } } #[inline] pub fn has_body(self) -> bool { matches!( self, DavMethod::PUT | DavMethod::POST | DavMethod::PATCH | DavMethod::PROPPATCH | DavMethod::PROPFIND | DavMethod::REPORT | DavMethod::LOCK | DavMethod::ACL | DavMethod::MKCALENDAR ) } } #[derive(Debug, Default)] pub struct PropStatBuilder { propstats: AHashMap<(StatusCode, Option, Option), Vec>, } impl PropStatBuilder { pub fn insert_ok(&mut self, prop: impl Into) -> &mut Self { self.propstats .entry((StatusCode::OK, None, None)) .or_default() .push(prop.into()); self } pub fn insert_with_status( &mut self, prop: impl Into, status: StatusCode, ) -> &mut Self { self.propstats .entry((status, None, None)) .or_default() .push(prop.into()); self } pub fn insert_error_with_description( &mut self, prop: impl Into, status: StatusCode, description: impl Into, ) -> &mut Self { self.propstats .entry((status, None, Some(description.into()))) .or_default() .push(prop.into()); self } pub fn insert_precondition_failed( &mut self, prop: impl Into, status: StatusCode, condition: impl Into, ) -> &mut Self { self.propstats .entry((status, Some(condition.into()), None)) .or_default() .push(prop.into()); self } pub fn insert_precondition_failed_with_description( &mut self, prop: impl Into, status: StatusCode, condition: impl Into, description: impl Into, ) -> &mut Self { self.propstats .entry((status, Some(condition.into()), Some(description.into()))) .or_default() .push(prop.into()); self } pub fn build(self) -> Vec { self.propstats .into_iter() .map(|((status, condition, description), props)| PropStat { prop: Prop(List(props)), status: Status(status), error: condition, response_description: description.map(ResponseDescription), }) .collect() } } // Workaround for Apple bug with missing percent encoding in paths pub(crate) fn fix_percent_encoding(path: &'_ str) -> Cow<'_, str> { let (parent, name) = if let Some((parent, name)) = path.rsplit_once('/') { (Some(parent), name) } else { (None, path) }; for &ch in name.as_bytes() { if !matches!(ch, b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z' | b'-' | b'.' | b'_' | b'~' | b'%') { let name = percent_encoding::percent_encode(name.as_bytes(), RFC_3986); return if let Some(parent) = parent { Cow::Owned(format!("{parent}/{name}")) } else { Cow::Owned(name.to_string()) }; } } path.into() } ================================================ FILE: crates/dav/src/principal/matching.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::propfind::PrincipalPropFind; use crate::{ DavError, common::{ DavQuery, DavQueryResource, propfind::PropFindRequestHandler, uri::{DavUriResource, UriResource}, }, }; use common::{Server, auth::AccessToken}; use dav_proto::{ RequestHeaders, schema::{ property::{DavProperty, WebDavProperty}, request::{PrincipalMatch, PropFind}, response::MultiStatus, }, }; use http_proto::HttpResponse; use hyper::StatusCode; use store::roaring::RoaringBitmap; use types::collection::Collection; pub(crate) trait PrincipalMatching: Sync + Send { fn handle_principal_match( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, request: PrincipalMatch, ) -> impl Future> + Send; } impl PrincipalMatching for Server { async fn handle_principal_match( &self, access_token: &AccessToken, headers: &RequestHeaders<'_>, mut request: PrincipalMatch, ) -> crate::Result { let resource = self.validate_uri(access_token, headers.uri).await?; match resource.collection { Collection::AddressBook | Collection::Calendar | Collection::FileNode => { if request.properties.is_empty() { request .properties .push(DavProperty::WebDav(WebDavProperty::Owner)); } if let Some(account_id) = resource.account_id { return self .handle_dav_query( access_token, DavQuery { resource: DavQueryResource::Uri(UriResource { collection: resource.collection, account_id, resource: resource.resource, }), propfind: PropFind::Prop(request.properties), depth: usize::MAX, ret: headers.ret, depth_no_root: headers.depth_no_root, uri: headers.uri, sync_type: Default::default(), limit: Default::default(), max_vcard_version: Default::default(), expand: Default::default(), }, ) .await; } } Collection::Principal => {} _ => return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)), } let mut response = MultiStatus::new(Vec::with_capacity(16)); if request.properties.is_empty() { request .properties .push(DavProperty::WebDav(WebDavProperty::DisplayName)); } let request = PropFind::Prop(request.properties); self.prepare_principal_propfind_response( access_token, resource.collection, RoaringBitmap::from_iter(access_token.all_ids()).into_iter(), &request, &mut response, ) .await?; Ok(HttpResponse::new(StatusCode::MULTI_STATUS).with_xml_body(response.to_string())) } } ================================================ FILE: crates/dav/src/principal/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::auth::AccessToken; use dav_proto::schema::response::Href; use groupware::RFC_3986; use crate::DavResourceName; pub mod matching; pub mod propfind; pub mod propsearch; pub trait CurrentUserPrincipal { fn current_user_principal(&self) -> Href; } impl CurrentUserPrincipal for AccessToken { fn current_user_principal(&self) -> Href { Href(format!( "{}/{}/", DavResourceName::Principal.base_path(), percent_encoding::utf8_percent_encode(&self.name, RFC_3986) )) } } ================================================ FILE: crates/dav/src/principal/propfind.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::CurrentUserPrincipal; use crate::{ DavResourceName, common::propfind::{PropFindRequestHandler, SyncTokenUrn}, }; use common::{Server, auth::AccessToken}; use dav_proto::schema::{ Namespace, property::{ DavProperty, DavValue, PrincipalProperty, Privilege, ReportSet, ResourceType, WebDavProperty, }, request::{DavPropertyValue, PropFind}, response::{Href, MultiStatus, PropStat, Response}, }; use directory::{PrincipalData, QueryParams, Type, backend::internal::manage::ManageDirectory}; use groupware::RFC_3986; use groupware::cache::GroupwareCache; use hyper::StatusCode; use std::borrow::Cow; use trc::AddContext; use types::collection::Collection; pub(crate) trait PrincipalPropFind: Sync + Send { fn prepare_principal_propfind_response( &self, access_token: &AccessToken, collection: Collection, documents: impl Iterator + Sync + Send, request: &PropFind, response: &mut MultiStatus, ) -> impl Future> + Send; fn expand_principal( &self, access_token: &AccessToken, account_id: u32, propfind: &PropFind, ) -> impl Future>> + Send; fn owner_href( &self, access_token: &AccessToken, account_id: u32, ) -> impl Future> + Send; } impl PrincipalPropFind for Server { async fn prepare_principal_propfind_response( &self, access_token: &AccessToken, collection: Collection, account_ids: impl Iterator + Sync + Send, request: &PropFind, response: &mut MultiStatus, ) -> crate::Result<()> { let properties = match request { PropFind::PropName => { let props = all_props(collection, None); for account_id in account_ids { response.add_response(Response::new_propstat( self.owner_href(access_token, account_id) .await .caused_by(trc::location!())?, vec![PropStat::new_list( props.iter().cloned().map(DavPropertyValue::empty).collect(), )], )); } return Ok(()); } PropFind::AllProp(items) => Cow::Owned(all_props(collection, items.as_slice().into())), PropFind::Prop(items) => Cow::Borrowed(items), }; let is_principal = match collection { Collection::AddressBook | Collection::ContactCard => { response.set_namespace(Namespace::CardDav); false } Collection::Calendar | Collection::CalendarEvent | Collection::CalendarEventNotification => { response.set_namespace(Namespace::CalDav); false } Collection::Principal => true, _ => false, }; let base_path = DavResourceName::from(collection).base_path(); let needs_quota = properties.iter().any(|property| { matches!( property, DavProperty::WebDav( WebDavProperty::QuotaAvailableBytes | WebDavProperty::QuotaUsedBytes ) ) }); for account_id in account_ids { let mut fields = Vec::with_capacity(properties.len()); let mut fields_not_found = Vec::new(); let (name, description, emails, typ) = if access_token.primary_id() == account_id { ( Cow::Borrowed(access_token.name.as_str()), access_token .description .as_deref() .unwrap_or(&access_token.name) .to_string(), Cow::Borrowed(access_token.emails.as_slice()), Type::Individual, ) } else { self.directory() .query(QueryParams::id(account_id).with_return_member_of(false)) .await .caused_by(trc::location!())? .map(|p| { let name = p.name; let mut description = None; let mut emails = Vec::new(); for data in p.data { match data { PrincipalData::Description(desc) => { description = Some(desc); } PrincipalData::PrimaryEmail(email) => { if emails.is_empty() { emails.push(email); } else { emails.insert(0, email); } } PrincipalData::EmailAlias(email) => { emails.push(email); } _ => {} } } let description = description.unwrap_or_else(|| name.clone()); ( Cow::Owned(name.to_string()), description, Cow::Owned(emails), p.typ, ) }) .unwrap_or_else(|| { ( Cow::Owned(format!("_{}", account_id)), format!("_{}", account_id), Cow::Owned(vec![]), Type::Individual, ) }) }; // Fetch quota let quota = if needs_quota { self.dav_quota(access_token, account_id) .await .caused_by(trc::location!())? } else { Default::default() }; for property in properties.as_slice() { match property { DavProperty::WebDav(dav_property) => match dav_property { WebDavProperty::DisplayName => { fields .push(DavPropertyValue::new(property.clone(), description.clone())); } WebDavProperty::ResourceType => { let resource_type = if !is_principal { vec![ResourceType::Collection] } else { vec![ResourceType::Principal, ResourceType::Collection] }; fields.push(DavPropertyValue::new(property.clone(), resource_type)); } WebDavProperty::SupportedReportSet => { let reports = match collection { Collection::Principal => ReportSet::principal(), Collection::Calendar | Collection::CalendarEvent => { ReportSet::calendar() } Collection::AddressBook | Collection::ContactCard => { ReportSet::addressbook() } _ => ReportSet::file(), }; fields.push(DavPropertyValue::new(property.clone(), reports)); } WebDavProperty::CurrentUserPrincipal => { fields.push(DavPropertyValue::new( property.clone(), vec![access_token.current_user_principal()], )); } WebDavProperty::QuotaAvailableBytes if !is_principal => { fields.push(DavPropertyValue::new(property.clone(), quota.available)); } WebDavProperty::QuotaUsedBytes if !is_principal => { fields.push(DavPropertyValue::new(property.clone(), quota.used)); } WebDavProperty::SyncToken if !is_principal => { let sync_token = self .fetch_dav_resources(access_token, account_id, collection.into()) .await .caused_by(trc::location!())? .sync_token(); fields.push(DavPropertyValue::new(property.clone(), sync_token)); } WebDavProperty::GetCTag if !is_principal => { let ctag = self .fetch_dav_resources(access_token, account_id, collection.into()) .await .caused_by(trc::location!())? .highest_change_id; fields.push(DavPropertyValue::new( property.clone(), DavValue::String(format!("\"{ctag}\"")), )); } WebDavProperty::Owner => { fields.push(DavPropertyValue::new( property.clone(), vec![Href(format!( "{}/{}/", DavResourceName::Principal.base_path(), percent_encoding::utf8_percent_encode(&name, RFC_3986), ))], )); } WebDavProperty::Group if !is_principal => { fields.push(DavPropertyValue::empty(property.clone())); } WebDavProperty::CurrentUserPrivilegeSet if !is_principal => { fields.push(DavPropertyValue::new( property.clone(), if access_token.is_member(account_id) { Privilege::all(matches!( collection, Collection::Calendar | Collection::CalendarEvent )) } else { vec![Privilege::Read] }, )); } WebDavProperty::PrincipalCollectionSet => { fields.push(DavPropertyValue::new( property.clone(), vec![Href( DavResourceName::Principal.collection_path().to_string(), )], )); } _ => { response.set_namespace(property.namespace()); fields_not_found.push(DavPropertyValue::empty(property.clone())); } }, DavProperty::Principal(principal_property) => match principal_property { PrincipalProperty::AlternateURISet | PrincipalProperty::GroupMemberSet | PrincipalProperty::GroupMembership => { fields.push(DavPropertyValue::empty(property.clone())); } PrincipalProperty::PrincipalURL => { fields.push(DavPropertyValue::new( property.clone(), vec![Href(format!( "{}/{}/", DavResourceName::Principal.base_path(), percent_encoding::utf8_percent_encode(&name, RFC_3986), ))], )); } PrincipalProperty::CalendarHomeSet => { let hrefs = build_home_set(self, access_token, name.as_ref(), account_id, true) .await .caused_by(trc::location!())?; fields.push(DavPropertyValue::new(property.clone(), hrefs)); response.set_namespace(Namespace::CalDav); } PrincipalProperty::AddressbookHomeSet => { let hrefs = build_home_set( self, access_token, name.as_ref(), account_id, false, ) .await .caused_by(trc::location!())?; fields.push(DavPropertyValue::new(property.clone(), hrefs)); response.set_namespace(Namespace::CardDav); } PrincipalProperty::PrincipalAddress => { fields_not_found.push(DavPropertyValue::empty(property.clone())); response.set_namespace(Namespace::CardDav); } PrincipalProperty::CalendarUserAddressSet => { fields.push(DavPropertyValue::new( property.clone(), emails .iter() .filter(|email| !email.starts_with("@")) .take(1) .map(|email| Href(format!("mailto:{email}",))) .collect::>(), )); response.set_namespace(Namespace::CalDav); } PrincipalProperty::CalendarUserType => { fields.push(DavPropertyValue::new( property.clone(), typ.as_str().to_uppercase(), )); response.set_namespace(Namespace::CalDav); } PrincipalProperty::ScheduleInboxURL => { fields.push(DavPropertyValue::new( property.clone(), vec![Href(format!( "{}/{}/inbox/", DavResourceName::Scheduling.base_path(), percent_encoding::utf8_percent_encode(&name, RFC_3986), ))], )); response.set_namespace(Namespace::CalDav); } PrincipalProperty::ScheduleOutboxURL => { fields.push(DavPropertyValue::new( property.clone(), vec![Href(format!( "{}/{}/outbox/", DavResourceName::Scheduling.base_path(), percent_encoding::utf8_percent_encode(&name, RFC_3986), ))], )); response.set_namespace(Namespace::CalDav); } }, _ => { response.set_namespace(property.namespace()); fields_not_found.push(DavPropertyValue::empty(property.clone())); } } } let mut prop_stats = Vec::with_capacity(2); if !fields_not_found.is_empty() { prop_stats .push(PropStat::new_list(fields_not_found).with_status(StatusCode::NOT_FOUND)); } if !fields.is_empty() || prop_stats.is_empty() { prop_stats.push(PropStat::new_list(fields)); } response.add_response(Response::new_propstat( Href(format!( "{}/{}/", base_path, percent_encoding::utf8_percent_encode(&name, RFC_3986), )), prop_stats, )); } Ok(()) } async fn expand_principal( &self, access_token: &AccessToken, account_id: u32, propfind: &PropFind, ) -> crate::Result> { let mut status = MultiStatus::new(vec![]); self.prepare_principal_propfind_response( access_token, Collection::Principal, [account_id].into_iter(), propfind, &mut status, ) .await?; Ok(status.response.0.into_iter().next()) } async fn owner_href(&self, access_token: &AccessToken, account_id: u32) -> trc::Result { if access_token.primary_id() == account_id { Ok(access_token.current_user_principal()) } else { let name = self .store() .get_principal_name(account_id) .await .caused_by(trc::location!())? .unwrap_or_else(|| format!("_{account_id}")); Ok(Href(format!( "{}/{}/", DavResourceName::Principal.base_path(), percent_encoding::utf8_percent_encode(&name, RFC_3986), ))) } } } pub(crate) async fn build_home_set( server: &Server, access_token: &AccessToken, name: &str, account_id: u32, is_calendar: bool, ) -> trc::Result> { let (collection, resource_name) = if is_calendar { (Collection::Calendar, DavResourceName::Cal) } else { (Collection::AddressBook, DavResourceName::Card) }; let mut hrefs = Vec::new(); hrefs.push(Href(format!( "{}/{}/", resource_name.base_path(), percent_encoding::utf8_percent_encode(name, RFC_3986), ))); if !server.core.groupware.assisted_discovery && account_id == access_token.primary_id() { for account_id in access_token.all_ids_by_collection(collection) { if account_id != access_token.primary_id() { let other_name = server .store() .get_principal_name(account_id) .await .caused_by(trc::location!())? .unwrap_or_else(|| format!("_{account_id}")); hrefs.push(Href(format!( "{}/{}/", resource_name.base_path(), percent_encoding::utf8_percent_encode(&other_name, RFC_3986), ))); } } } Ok(hrefs) } fn all_props(collection: Collection, all_props: Option<&[DavProperty]>) -> Vec { if collection == Collection::Principal { vec![ DavProperty::WebDav(WebDavProperty::DisplayName), DavProperty::WebDav(WebDavProperty::ResourceType), DavProperty::WebDav(WebDavProperty::SupportedReportSet), DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal), DavProperty::WebDav(WebDavProperty::PrincipalCollectionSet), DavProperty::Principal(PrincipalProperty::AlternateURISet), DavProperty::Principal(PrincipalProperty::PrincipalURL), DavProperty::Principal(PrincipalProperty::GroupMemberSet), DavProperty::Principal(PrincipalProperty::GroupMembership), ] } else { let mut props = vec![ DavProperty::WebDav(WebDavProperty::DisplayName), DavProperty::WebDav(WebDavProperty::ResourceType), DavProperty::WebDav(WebDavProperty::SupportedReportSet), DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal), DavProperty::WebDav(WebDavProperty::SyncToken), DavProperty::WebDav(WebDavProperty::Owner), DavProperty::WebDav(WebDavProperty::PrincipalCollectionSet), ]; if let Some(all_props) = all_props { props.extend(all_props.iter().filter(|p| !p.is_all_prop()).cloned()); props } else { props } } } ================================================ FILE: crates/dav/src/principal/propsearch.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::propfind::PrincipalPropFind; use common::{Server, auth::AccessToken}; use dav_proto::schema::{ property::{DavProperty, WebDavProperty}, request::{PrincipalPropertySearch, PropFind}, response::MultiStatus, }; use directory::{Type, backend::internal::manage::ManageDirectory}; use http_proto::HttpResponse; use hyper::StatusCode; use store::roaring::RoaringBitmap; use trc::AddContext; use types::collection::Collection; pub(crate) trait PrincipalPropSearch: Sync + Send { fn handle_principal_property_search( &self, access_token: &AccessToken, request: PrincipalPropertySearch, ) -> impl Future> + Send; } impl PrincipalPropSearch for Server { async fn handle_principal_property_search( &self, access_token: &AccessToken, mut request: PrincipalPropertySearch, ) -> crate::Result { let mut search_for = None; for prop_search in request.property_search { if matches!( prop_search.property, DavProperty::WebDav(WebDavProperty::DisplayName) ) && !prop_search.match_.is_empty() { search_for = Some(prop_search.match_); } } let mut response = MultiStatus::new(Vec::with_capacity(16)); if let Some(search_for) = search_for { // Return all principals let principals = self .store() .list_principals( search_for.as_str().into(), access_token.tenant_id(), &[Type::Individual, Type::Group], false, 0, 0, ) .await .caused_by(trc::location!())?; let ids = RoaringBitmap::from_iter(principals.items.into_iter().map(|p| p.id())); if !ids.is_empty() { if request.properties.is_empty() { request .properties .push(DavProperty::WebDav(WebDavProperty::DisplayName)); } let request = PropFind::Prop(request.properties); self.prepare_principal_propfind_response( access_token, Collection::Principal, ids.into_iter(), &request, &mut response, ) .await?; } } Ok(HttpResponse::new(StatusCode::MULTI_STATUS).with_xml_body(response.to_string())) } } ================================================ FILE: crates/dav/src/request.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ DavError, DavErrorCondition, DavMethod, DavResourceName, calendar::{ copy_move::CalendarCopyMoveRequestHandler, delete::CalendarDeleteRequestHandler, freebusy::CalendarFreebusyRequestHandler, get::CalendarGetRequestHandler, mkcol::CalendarMkColRequestHandler, proppatch::CalendarPropPatchRequestHandler, query::CalendarQueryRequestHandler, scheduling::CalendarEventNotificationHandler, update::CalendarUpdateRequestHandler, }, card::{ copy_move::CardCopyMoveRequestHandler, delete::CardDeleteRequestHandler, get::CardGetRequestHandler, mkcol::CardMkColRequestHandler, proppatch::CardPropPatchRequestHandler, query::CardQueryRequestHandler, update::CardUpdateRequestHandler, }, common::{ DavQuery, acl::DavAclHandler, lock::{LockRequest, LockRequestHandler}, propfind::PropFindRequestHandler, uri::DavUriResource, }, file::{ copy_move::FileCopyMoveRequestHandler, delete::FileDeleteRequestHandler, get::FileGetRequestHandler, mkcol::FileMkColRequestHandler, proppatch::FilePropPatchRequestHandler, update::FileUpdateRequestHandler, }, principal::{matching::PrincipalMatching, propsearch::PrincipalPropSearch}, }; use common::{Server, auth::AccessToken}; use compact_str::{CompactString, ToCompactString}; use dav_proto::{ RequestHeaders, parser::{DavParser, tokenizer::Tokenizer}, schema::{ Namespace, property::WebDavProperty, request::{Acl, LockInfo, MkCol, PropFind, PropertyUpdate, Report}, response::{ BaseCondition, ErrorResponse, List, PrincipalSearchProperty, PrincipalSearchPropertySet, }, }, }; use directory::Permission; use http_proto::{HttpRequest, HttpResponse, HttpSessionData, request::fetch_body}; use hyper::{StatusCode, header}; use std::{sync::Arc, time::Instant}; use trc::{EventType, LimitEvent, StoreEvent, WebDavEvent}; use types::collection::Collection; pub trait DavRequestHandler: Sync + Send { fn handle_dav_request( &self, request: HttpRequest, access_token: Arc, session: &HttpSessionData, resource: DavResourceName, method: DavMethod, ) -> impl Future + Send; } pub(crate) trait DavRequestDispatcher: Sync + Send { fn dispatch_dav_request( &self, headers: &RequestHeaders<'_>, access_token: Arc, resource: DavResourceName, method: DavMethod, body: Vec, ) -> impl Future> + Send; } impl DavRequestDispatcher for Server { async fn dispatch_dav_request( &self, headers: &RequestHeaders<'_>, access_token: Arc, resource: DavResourceName, method: DavMethod, body: Vec, ) -> crate::Result { // Dispatch match method { DavMethod::PROPFIND => { let request = PropFind::parse(&mut Tokenizer::new(&body))?; self.handle_propfind_request(&access_token, headers, request) .await } DavMethod::GET | DavMethod::HEAD => match resource { DavResourceName::Card => { // Validate permissions access_token.assert_has_permission(Permission::DavCardGet)?; self.handle_card_get_request( &access_token, headers, matches!(method, DavMethod::HEAD), ) .await } DavResourceName::Cal => { // Validate permissions access_token.assert_has_permission(Permission::DavCalGet)?; self.handle_calendar_get_request( &access_token, headers, matches!(method, DavMethod::HEAD), ) .await } DavResourceName::File => { // Validate permissions access_token.assert_has_permission(Permission::DavFileGet)?; // Deal with Litmus bug /*self.handle_file_get_request( &access_token, headers, matches!(method, DavMethod::HEAD) && !request.headers().contains_key("x-litmus"), ) .await*/ self.handle_file_get_request( &access_token, headers, matches!(method, DavMethod::HEAD), ) .await } DavResourceName::Scheduling => { // Validate permissions access_token.assert_has_permission(Permission::DavCalGet)?; self.handle_scheduling_get_request( &access_token, headers, matches!(method, DavMethod::HEAD), ) .await } DavResourceName::Principal => Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)), }, DavMethod::REPORT => match Report::parse(&mut Tokenizer::new(&body))? { Report::SyncCollection(sync_collection) => { // Validate permissions access_token.assert_has_permission(Permission::DavSyncCollection)?; let uri = self .validate_uri(&access_token, headers.uri) .await .and_then(|d| d.into_owned_uri())?; match resource { DavResourceName::Card | DavResourceName::Cal | DavResourceName::File | DavResourceName::Scheduling => { self.handle_dav_query( &access_token, DavQuery::changes(uri, sync_collection, headers), ) .await } DavResourceName::Principal => { Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)) } } } Report::AclPrincipalPropSet(report) => { // Validate permissions if !self.core.groupware.allow_directory_query && !access_token.has_permission(Permission::IndividualList) { return Err(DavError::Condition( DavErrorCondition::new( StatusCode::FORBIDDEN, BaseCondition::NeedPrivileges(List(Default::default())), ) .with_details("The administrator has disabled directory queries."), )); } access_token.assert_has_permission(Permission::DavPrincipalAcl)?; self.handle_acl_prop_set(&access_token, headers, report) .await } Report::PrincipalMatch(report) => { // Validate permissions if !self.core.groupware.allow_directory_query && !access_token.has_permission(Permission::IndividualList) { return Err(DavError::Condition( DavErrorCondition::new( StatusCode::FORBIDDEN, BaseCondition::NeedPrivileges(List(Default::default())), ) .with_details("The administrator has disabled directory queries."), )); } access_token.assert_has_permission(Permission::DavPrincipalMatch)?; self.handle_principal_match(&access_token, headers, report) .await } Report::PrincipalPropertySearch(report) => { if resource == DavResourceName::Principal { // Validate permissions if !self.core.groupware.allow_directory_query && !access_token.has_permission(Permission::IndividualList) { return Err(DavError::Condition( DavErrorCondition::new( StatusCode::FORBIDDEN, BaseCondition::NeedPrivileges(List(Default::default())), ) .with_details("The administrator has disabled directory queries."), )); } access_token.assert_has_permission(Permission::DavPrincipalSearch)?; self.handle_principal_property_search(&access_token, report) .await } else { Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)) } } Report::PrincipalSearchPropertySet => { if resource == DavResourceName::Principal { // Validate permissions access_token .assert_has_permission(Permission::DavPrincipalSearchPropSet)?; Ok(HttpResponse::new(StatusCode::OK).with_xml_body( PrincipalSearchPropertySet::new(vec![PrincipalSearchProperty::new( WebDavProperty::DisplayName, "Account or Group name", )]) .to_string(), )) } else { Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)) } } Report::AddressbookQuery(report) => { // Validate permissions access_token.assert_has_permission(Permission::DavCardQuery)?; self.handle_card_query_request(&access_token, headers, report) .await } Report::AddressbookMultiGet(report) => { // Validate permissions access_token.assert_has_permission(Permission::DavCardMultiGet)?; self.handle_dav_query( &access_token, DavQuery::multiget(report, Collection::AddressBook, headers), ) .await } Report::CalendarQuery(report) => { // Validate permissions access_token.assert_has_permission(Permission::DavCalQuery)?; self.handle_calendar_query_request(&access_token, headers, report) .await } Report::CalendarMultiGet(report) => { // Validate permissions access_token.assert_has_permission(Permission::DavCalMultiGet)?; self.handle_dav_query( &access_token, DavQuery::multiget(report, Collection::Calendar, headers), ) .await } Report::FreeBusyQuery(report) => { // Validate permissions access_token.assert_has_permission(Permission::DavCalFreeBusyQuery)?; self.handle_calendar_freebusy_request(&access_token, headers, report) .await } Report::ExpandProperty(report) => { let uri = self .validate_uri(&access_token, headers.uri) .await .and_then(|d| d.into_owned_uri())?; // Validate permissions access_token.assert_has_permission(Permission::DavExpandProperty)?; match resource { DavResourceName::Card | DavResourceName::Cal | DavResourceName::File => { self.handle_dav_query( &access_token, DavQuery::expand(uri, report, headers), ) .await } DavResourceName::Principal | DavResourceName::Scheduling => { Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)) } } } }, DavMethod::PROPPATCH => { let request = PropertyUpdate::parse(&mut Tokenizer::new(&body))?; match resource { DavResourceName::Card => { // Validate permissions access_token.assert_has_permission(Permission::DavCardPropPatch)?; self.handle_card_proppatch_request(&access_token, headers, request) .await } DavResourceName::Cal => { // Validate permissions access_token.assert_has_permission(Permission::DavCalPropPatch)?; self.handle_calendar_proppatch_request(&access_token, headers, request) .await } DavResourceName::File => { // Validate permissions access_token.assert_has_permission(Permission::DavFilePropPatch)?; self.handle_file_proppatch_request(&access_token, headers, request) .await } DavResourceName::Principal | DavResourceName::Scheduling => { Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)) } } } DavMethod::MKCOL => { let request = if !body.is_empty() { Some(MkCol::parse(&mut Tokenizer::new(&body))?) } else { None }; match resource { DavResourceName::Card => { // Validate permissions access_token.assert_has_permission(Permission::DavCardMkCol)?; self.handle_card_mkcol_request(&access_token, headers, request) .await } DavResourceName::Cal => { // Validate permissions access_token.assert_has_permission(Permission::DavCalMkCol)?; self.handle_calendar_mkcol_request(&access_token, headers, request) .await } DavResourceName::File => { // Validate permissions access_token.assert_has_permission(Permission::DavFileMkCol)?; self.handle_file_mkcol_request(&access_token, headers, request) .await } DavResourceName::Principal | DavResourceName::Scheduling => { Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)) } } } DavMethod::DELETE => match resource { DavResourceName::Card => { // Validate permissions access_token.assert_has_permission(Permission::DavCardDelete)?; self.handle_card_delete_request(&access_token, headers) .await } DavResourceName::Cal => { // Validate permissions access_token.assert_has_permission(Permission::DavCalDelete)?; self.handle_calendar_delete_request(&access_token, headers) .await } DavResourceName::File => { // Validate permissions access_token.assert_has_permission(Permission::DavFileDelete)?; self.handle_file_delete_request(&access_token, headers) .await } DavResourceName::Scheduling => { // Validate permissions access_token.assert_has_permission(Permission::DavCalDelete)?; self.handle_scheduling_delete_request(&access_token, headers) .await } DavResourceName::Principal => Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)), }, DavMethod::PUT | DavMethod::POST | DavMethod::PATCH => match resource { DavResourceName::Card => { // Validate permissions access_token.assert_has_permission(Permission::DavCardPut)?; self.handle_card_update_request( &access_token, headers, body, matches!(method, DavMethod::PATCH), ) .await } DavResourceName::Cal => { // Validate permissions access_token.assert_has_permission(Permission::DavCalPut)?; self.handle_calendar_update_request( &access_token, headers, body, matches!(method, DavMethod::PATCH), ) .await } DavResourceName::File => { // Validate permissions access_token.assert_has_permission(Permission::DavFilePut)?; self.handle_file_update_request( &access_token, headers, body, matches!(method, DavMethod::PATCH), ) .await } DavResourceName::Scheduling => { // Validate permissions access_token.assert_has_permission(Permission::DavCalFreeBusyQuery)?; self.handle_scheduling_post_request(&access_token, headers, body) .await } DavResourceName::Principal => Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)), }, DavMethod::COPY | DavMethod::MOVE => { let is_move = matches!(method, DavMethod::MOVE); match resource { DavResourceName::Card => { // Validate permissions access_token.assert_has_permission(if is_move { Permission::DavCardMove } else { Permission::DavCardCopy })?; self.handle_card_copy_move_request(&access_token, headers, is_move) .await } DavResourceName::Cal => { // Validate permissions access_token.assert_has_permission(if is_move { Permission::DavCalMove } else { Permission::DavCalCopy })?; self.handle_calendar_copy_move_request(&access_token, headers, is_move) .await } DavResourceName::File => { // Validate permissions access_token.assert_has_permission(if is_move { Permission::DavFileMove } else { Permission::DavFileCopy })?; self.handle_file_copy_move_request(&access_token, headers, is_move) .await } DavResourceName::Principal | DavResourceName::Scheduling => { Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)) } } } DavMethod::MKCALENDAR => match resource { DavResourceName::Cal => { // Validate permissions access_token.assert_has_permission(Permission::DavCalMkCol)?; self.handle_calendar_mkcol_request( &access_token, headers, Some(MkCol::parse(&mut Tokenizer::new(&body))?), ) .await } _ => Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)), }, DavMethod::LOCK => { // Validate permissions access_token.assert_has_permission(match resource { DavResourceName::File => Permission::DavFileLock, DavResourceName::Cal => Permission::DavCalLock, DavResourceName::Card => Permission::DavCardLock, _ => return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)), })?; self.handle_lock_request( &access_token, headers, if !body.is_empty() { LockRequest::Lock(LockInfo::parse(&mut Tokenizer::new(&body))?) } else { LockRequest::Refresh }, ) .await } DavMethod::UNLOCK => { // Validate permissions access_token.assert_has_permission(match resource { DavResourceName::File => Permission::DavFileLock, DavResourceName::Cal => Permission::DavCalLock, DavResourceName::Card => Permission::DavCardLock, _ => return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)), })?; self.handle_lock_request(&access_token, headers, LockRequest::Unlock) .await } DavMethod::ACL => { // Validate permissions access_token.assert_has_permission(match resource { DavResourceName::File => Permission::DavFileAcl, DavResourceName::Cal => Permission::DavCalAcl, DavResourceName::Card => Permission::DavCardAcl, _ => return Err(DavError::Code(StatusCode::METHOD_NOT_ALLOWED)), })?; self.handle_acl_request( &access_token, headers, Acl::parse(&mut Tokenizer::new(&body))?, ) .await } DavMethod::OPTIONS => unreachable!(), } } } impl DavRequestHandler for Server { async fn handle_dav_request( &self, mut request: HttpRequest, access_token: Arc, session: &HttpSessionData, resource: DavResourceName, method: DavMethod, ) -> HttpResponse { let body = if method.has_body() || request .headers() .get(header::CONTENT_LENGTH) .and_then(|v| v.to_str().ok()) .and_then(|v| v.parse::().ok()) .is_some_and(|len| len > 0) { if let Some(body) = fetch_body( &mut request, if !access_token.has_permission(Permission::UnlimitedUploads) { self.core.groupware.max_request_size } else { 0 }, session.session_id, ) .await { body } else { trc::event!( Limit(trc::LimitEvent::SizeRequest), SpanId = session.session_id, Contents = "Request body too large", ); return HttpResponse::new(StatusCode::PAYLOAD_TOO_LARGE); } } else { Vec::new() }; // Parse headers let mut headers = RequestHeaders::new(request.uri().path()); for (key, value) in request.headers() { headers.parse(key.as_str(), value.to_str().unwrap_or_default()); } let start_time = Instant::now(); match self .dispatch_dav_request(&headers, access_token, resource, method, body) .await { Ok(response) => { let event = WebDavEvent::from(method); trc::event!( WebDav(event), SpanId = session.session_id, Url = headers.uri.to_compact_string(), Type = resource.name(), Details = &headers, Result = response.status().as_u16(), Elapsed = start_time.elapsed(), ); response } Err(DavError::Internal(err)) => { let err_type = err.event_type(); trc::error!( err.span_id(session.session_id) .ctx(trc::Key::Url, headers.uri.to_compact_string()) .ctx(trc::Key::Type, resource.name()) .ctx(trc::Key::Elapsed, start_time.elapsed()) ); match err_type { EventType::Limit(LimitEvent::Quota | LimitEvent::TenantQuota) => { HttpResponse::new(StatusCode::PRECONDITION_FAILED) .with_xml_body( ErrorResponse::new(BaseCondition::QuotaNotExceeded) .with_namespace(match resource { DavResourceName::Card => Namespace::CardDav, DavResourceName::Cal | DavResourceName::Scheduling => { Namespace::CalDav } DavResourceName::File | DavResourceName::Principal => { Namespace::Dav } }) .to_string(), ) .with_no_cache() } EventType::Store(StoreEvent::AssertValueFailed) => { HttpResponse::new(StatusCode::CONFLICT) } EventType::Security(_) => HttpResponse::new(StatusCode::FORBIDDEN), _ => HttpResponse::new(StatusCode::INTERNAL_SERVER_ERROR), } } Err(DavError::Parse(err)) => { let result = if headers.content_type.is_some_and(|h| h.contains("/xml")) { StatusCode::BAD_REQUEST } else { StatusCode::UNSUPPORTED_MEDIA_TYPE }; trc::event!( WebDav(WebDavEvent::Error), SpanId = session.session_id, Url = headers.uri.to_compact_string(), Type = resource.name(), Details = &headers, Result = result.as_u16(), Reason = err.to_compact_string(), Elapsed = start_time.elapsed(), ); HttpResponse::new(result) } Err(DavError::Condition(condition)) => { let event = WebDavEvent::from(method); trc::event!( WebDav(event), SpanId = session.session_id, Url = headers.uri.to_compact_string(), Type = resource.name(), Details = &headers, Code = condition.code.as_u16(), Result = CompactString::const_new(condition.condition.display_name()), Reason = condition.details, Elapsed = start_time.elapsed(), ); HttpResponse::new(condition.code) .with_xml_body( ErrorResponse::new(condition.condition) .with_namespace(match resource { DavResourceName::Card => Namespace::CardDav, DavResourceName::Cal | DavResourceName::Scheduling => { Namespace::CalDav } DavResourceName::File | DavResourceName::Principal => { Namespace::Dav } }) .to_string(), ) .with_no_cache() } Err(DavError::Code(code)) => { let event = WebDavEvent::from(method); trc::event!( WebDav(event), SpanId = session.session_id, Url = headers.uri.to_compact_string(), Type = resource.name(), Details = &headers, Result = code.as_u16(), Elapsed = start_time.elapsed(), ); HttpResponse::new(code) } } } } impl From for DavError { fn from(err: dav_proto::parser::Error) -> Self { DavError::Parse(err) } } impl From for DavError { fn from(err: trc::Error) -> Self { DavError::Internal(err) } } ================================================ FILE: crates/dav-proto/Cargo.toml ================================================ [package] name = "dav-proto" version = "0.15.5" edition = "2024" [dependencies] trc = { path = "../trc" } types = { path = "../types" } hashify = "0.2.6" quick-xml = { version = "0.38" } calcard = { version = "0.3", features = ["rkyv"] } mail-parser = { version = "0.11", features = ["full_encoding", "rkyv"] } hyper = "1.6.0" rkyv = { version = "0.8.10", features = ["little_endian"] } chrono = { version = "0.4.40", features = ["serde"], optional = true } compact_str = "0.9.0" [dev-dependencies] calcard = { version = "0.3", features = ["serde", "rkyv"] } types = { path = "../types", features = ["test_mode"] } serde = { version = "1.0.217", features = ["derive"] } serde_json = "1.0.138" chrono = { version = "0.4.40", features = ["serde"] } [features] test_mode = ["chrono"] enterprise = [] ================================================ FILE: crates/dav-proto/resources/requests/acl-001.json ================================================ { "aces": [ { "principal": { "Href": "http://www.example.com/users/friends" }, "invert": false, "grant_deny": { "Grant": [ "Read" ] }, "protected": false, "inherited": null }, { "principal": { "Href": "http://www.example.com/users/ygoland-so" }, "invert": false, "grant_deny": { "Deny": [ "Read" ] }, "protected": false, "inherited": null } ] } ================================================ FILE: crates/dav-proto/resources/requests/acl-001.xml ================================================ http://www.example.com/users/friends http://www.example.com/users/ygoland-so ================================================ FILE: crates/dav-proto/resources/requests/acl-002.json ================================================ { "aces": [ { "principal": { "Href": "http://www.example.com/users/ejw" }, "invert": false, "grant_deny": { "Grant": [ "Write" ] }, "protected": false, "inherited": null } ] } ================================================ FILE: crates/dav-proto/resources/requests/acl-002.xml ================================================ http://www.example.com/users/ejw ================================================ FILE: crates/dav-proto/resources/requests/acl-003.json ================================================ { "aces": [ { "principal": { "Href": "http://www.example.com/users/esedlar" }, "invert": true, "grant_deny": { "Deny": [ "Write" ] }, "protected": true, "inherited": "http://www.example.com/container/" } ] } ================================================ FILE: crates/dav-proto/resources/requests/acl-003.xml ================================================ http://www.example.com/users/esedlar http://www.example.com/container/ ================================================ FILE: crates/dav-proto/resources/requests/acl-004.json ================================================ { "aces": [ { "principal": { "Href": "http://www.example.com/users/esedlar" }, "invert": false, "grant_deny": { "Grant": [ "Read", "Write" ] }, "protected": false, "inherited": null }, { "principal": { "Property": [ { "property": { "type": "WebDav", "data": { "type": "Owner" } }, "value": "Null" } ] }, "invert": false, "grant_deny": { "Grant": [ "ReadAcl", "WriteAcl" ] }, "protected": false, "inherited": null }, { "principal": "All", "invert": false, "grant_deny": { "Grant": [ "Read" ] }, "protected": false, "inherited": null } ] } ================================================ FILE: crates/dav-proto/resources/requests/acl-004.xml ================================================ http://www.example.com/users/esedlar ================================================ FILE: crates/dav-proto/resources/requests/lockinfo-001.json ================================================ { "lock_scope": "Shared", "lock_type": "Write", "owner": [ { "type": "ElementStart", "data": { "name": "href", "attrs": "xmlns=\"DAV:\"" } }, { "type": "Text", "data": "http://example.org/~ejw/contact.html" }, { "type": "ElementEnd" } ] } ================================================ FILE: crates/dav-proto/resources/requests/lockinfo-001.xml ================================================ http://example.org/~ejw/contact.html ================================================ FILE: crates/dav-proto/resources/requests/lockinfo-002.json ================================================ { "lock_scope": "Exclusive", "lock_type": "Write", "owner": [ { "type": "ElementStart", "data": { "name": "href", "attrs": "xmlns=\"DAV:\"" } }, { "type": "Text", "data": "http://example.org/~ejw/contact.html" }, { "type": "ElementEnd" } ] } ================================================ FILE: crates/dav-proto/resources/requests/lockinfo-002.xml ================================================ http://example.org/~ejw/contact.html ================================================ FILE: crates/dav-proto/resources/requests/mkcol-001.json ================================================ { "is_mkcalendar": false, "props": [ { "property": { "type": "WebDav", "data": { "type": "ResourceType" } }, "value": { "ResourceTypes": [ "Collection", "AddressBook" ] } }, { "property": { "type": "WebDav", "data": { "type": "DisplayName" } }, "value": { "String": "Lisa's Contacts" } }, { "property": { "type": "CardDav", "data": { "type": "AddressbookDescription" } }, "value": { "String": "My primary address book." } } ] } ================================================ FILE: crates/dav-proto/resources/requests/mkcol-001.xml ================================================ Lisa's Contacts My primary address book. ================================================ FILE: crates/dav-proto/resources/requests/mkcol-002.json ================================================ { "is_mkcalendar": true, "props": [ { "property": { "type": "WebDav", "data": { "type": "DisplayName" } }, "value": { "String": "Lisa's Events" } }, { "property": { "type": "CalDav", "data": { "type": "CalendarDescription" } }, "value": { "String": "Calendar restricted to events." } }, { "property": { "type": "CalDav", "data": { "type": "SupportedCalendarComponentSet" } }, "value": { "Components": [ "VEvent" ] } }, { "property": { "type": "CalDav", "data": { "type": "CalendarTimezone" } }, "value": { "ICalendar": { "components": [ { "component_type": "VCalendar", "entries": [ { "name": { "type": "Prodid" }, "params": [], "values": [ { "type": "Text", "data": "-//Example Corp.//CalDAV Client//EN" } ] }, { "name": { "type": "Version" }, "params": [], "values": [ { "type": "Text", "data": "2.0" } ] } ], "component_ids": [ 1 ] }, { "component_type": "VTimezone", "entries": [ { "name": { "type": "Tzid" }, "params": [], "values": [ { "type": "Text", "data": "US-Eastern" } ] }, { "name": { "type": "LastModified" }, "params": [], "values": [ { "type": "PartialDateTime", "data": { "year": 1987, "month": 1, "day": 1, "hour": 0, "minute": 0, "second": 0, "tz_hour": 0, "tz_minute": 0, "tz_minus": false } } ] } ], "component_ids": [ 2, 3 ] }, { "component_type": "Standard", "entries": [ { "name": { "type": "Dtstart" }, "params": [], "values": [ { "type": "PartialDateTime", "data": { "year": 1967, "month": 10, "day": 29, "hour": 2, "minute": 0, "second": 0, "tz_hour": null, "tz_minute": null, "tz_minus": false } } ] }, { "name": { "type": "Rrule" }, "params": [], "values": [ { "type": "RecurrenceRule", "data": { "freq": "Yearly", "until": null, "count": null, "interval": null, "bysecond": [], "byminute": [], "byhour": [], "byday": [ { "ordwk": -1, "weekday": "Sunday" } ], "bymonthday": [], "byyearday": [], "byweekno": [], "bymonth": [ 10 ], "bysetpos": [], "wkst": null } } ] }, { "name": { "type": "Tzoffsetfrom" }, "params": [], "values": [ { "type": "PartialDateTime", "data": { "year": null, "month": null, "day": null, "hour": null, "minute": null, "second": null, "tz_hour": 4, "tz_minute": 0, "tz_minus": true } } ] }, { "name": { "type": "Tzoffsetto" }, "params": [], "values": [ { "type": "PartialDateTime", "data": { "year": null, "month": null, "day": null, "hour": null, "minute": null, "second": null, "tz_hour": 5, "tz_minute": 0, "tz_minus": true } } ] }, { "name": { "type": "Tzname" }, "params": [], "values": [ { "type": "Text", "data": "Eastern Standard Time (US & Canada)" } ] } ], "component_ids": [] }, { "component_type": "Daylight", "entries": [ { "name": { "type": "Dtstart" }, "params": [], "values": [ { "type": "PartialDateTime", "data": { "year": 1987, "month": 4, "day": 5, "hour": 2, "minute": 0, "second": 0, "tz_hour": null, "tz_minute": null, "tz_minus": false } } ] }, { "name": { "type": "Rrule" }, "params": [], "values": [ { "type": "RecurrenceRule", "data": { "freq": "Yearly", "until": null, "count": null, "interval": null, "bysecond": [], "byminute": [], "byhour": [], "byday": [ { "ordwk": 1, "weekday": "Sunday" } ], "bymonthday": [], "byyearday": [], "byweekno": [], "bymonth": [ 4 ], "bysetpos": [], "wkst": null } } ] }, { "name": { "type": "Tzoffsetfrom" }, "params": [], "values": [ { "type": "PartialDateTime", "data": { "year": null, "month": null, "day": null, "hour": null, "minute": null, "second": null, "tz_hour": 5, "tz_minute": 0, "tz_minus": true } } ] }, { "name": { "type": "Tzoffsetto" }, "params": [], "values": [ { "type": "PartialDateTime", "data": { "year": null, "month": null, "day": null, "hour": null, "minute": null, "second": null, "tz_hour": 4, "tz_minute": 0, "tz_minus": true } } ] }, { "name": { "type": "Tzname" }, "params": [], "values": [ { "type": "Text", "data": "Eastern Daylight Time (US & Canada)" } ] } ], "component_ids": [] } ] } } } ] } ================================================ FILE: crates/dav-proto/resources/requests/mkcol-002.xml ================================================ Lisa's Events Calendar restricted to events. ================================================ FILE: crates/dav-proto/resources/requests/mkcol-003.json ================================================ { "is_mkcalendar": false, "props": [ { "property": { "type": "WebDav", "data": { "type": "ResourceType" } }, "value": { "ResourceTypes": [ "Collection" ] } }, { "property": { "type": "WebDav", "data": { "type": "DisplayName" } }, "value": { "String": "Special Resource" } } ] } ================================================ FILE: crates/dav-proto/resources/requests/mkcol-003.xml ================================================ Special Resource ================================================ FILE: crates/dav-proto/resources/requests/mkcol-004.json ================================================ { "is_mkcalendar": false, "props": [ { "property": { "type": "WebDav", "data": { "type": "ResourceType" } }, "value": { "ResourceTypes": [ "Collection", "Calendar" ] } }, { "property": { "type": "WebDav", "data": { "type": "DisplayName" } }, "value": { "String": "Lisa's Events" } } ] } ================================================ FILE: crates/dav-proto/resources/requests/mkcol-004.xml ================================================ Lisa's Events ================================================ FILE: crates/dav-proto/resources/requests/propertyupdate-001.json ================================================ { "set": [ { "property": { "type": "CardDav", "data": { "type": "AddressbookDescription" } }, "value": { "String": "Adresses de Oliver Daboo" } }, { "property": { "type": "CalDav", "data": { "type": "CalendarDescription" } }, "value": { "String": "Calendrier de Mathilde Desruisseaux" } }, { "property": { "type": "CalDav", "data": { "type": "SupportedCalendarComponentSet" } }, "value": { "Components": [ "VEvent", "VTodo" ] } }, { "property": { "type": "CalDav", "data": { "type": "CalendarTimezone" } }, "value": { "ICalendar": { "components": [ { "component_type": "VCalendar", "entries": [ { "name": { "type": "Prodid" }, "params": [], "values": [ { "type": "Text", "data": "-//Example Corp.//CalDAV Client//EN" } ] }, { "name": { "type": "Version" }, "params": [], "values": [ { "type": "Text", "data": "2.0" } ] } ], "component_ids": [ 1 ] }, { "component_type": "VTimezone", "entries": [ { "name": { "type": "Tzid" }, "params": [], "values": [ { "type": "Text", "data": "US-Eastern" } ] }, { "name": { "type": "LastModified" }, "params": [], "values": [ { "type": "PartialDateTime", "data": { "year": 1987, "month": 1, "day": 1, "hour": 0, "minute": 0, "second": 0, "tz_hour": 0, "tz_minute": 0, "tz_minus": false } } ] } ], "component_ids": [ 2, 3 ] }, { "component_type": "Standard", "entries": [ { "name": { "type": "Dtstart" }, "params": [], "values": [ { "type": "PartialDateTime", "data": { "year": 1967, "month": 10, "day": 29, "hour": 2, "minute": 0, "second": 0, "tz_hour": null, "tz_minute": null, "tz_minus": false } } ] }, { "name": { "type": "Rrule" }, "params": [], "values": [ { "type": "RecurrenceRule", "data": { "freq": "Yearly", "until": null, "count": null, "interval": null, "bysecond": [], "byminute": [], "byhour": [], "byday": [ { "ordwk": -1, "weekday": "Sunday" } ], "bymonthday": [], "byyearday": [], "byweekno": [], "bymonth": [ 10 ], "bysetpos": [], "wkst": null } } ] }, { "name": { "type": "Tzoffsetfrom" }, "params": [], "values": [ { "type": "PartialDateTime", "data": { "year": null, "month": null, "day": null, "hour": null, "minute": null, "second": null, "tz_hour": 4, "tz_minute": 0, "tz_minus": true } } ] }, { "name": { "type": "Tzoffsetto" }, "params": [], "values": [ { "type": "PartialDateTime", "data": { "year": null, "month": null, "day": null, "hour": null, "minute": null, "second": null, "tz_hour": 5, "tz_minute": 0, "tz_minus": true } } ] }, { "name": { "type": "Tzname" }, "params": [], "values": [ { "type": "Text", "data": "Eastern Standard Time (US & Canada)" } ] } ], "component_ids": [] }, { "component_type": "Daylight", "entries": [ { "name": { "type": "Dtstart" }, "params": [], "values": [ { "type": "PartialDateTime", "data": { "year": 1987, "month": 4, "day": 5, "hour": 2, "minute": 0, "second": 0, "tz_hour": null, "tz_minute": null, "tz_minus": false } } ] }, { "name": { "type": "Rrule" }, "params": [], "values": [ { "type": "RecurrenceRule", "data": { "freq": "Yearly", "until": null, "count": null, "interval": null, "bysecond": [], "byminute": [], "byhour": [], "byday": [ { "ordwk": 1, "weekday": "Sunday" } ], "bymonthday": [], "byyearday": [], "byweekno": [], "bymonth": [ 4 ], "bysetpos": [], "wkst": null } } ] }, { "name": { "type": "Tzoffsetfrom" }, "params": [], "values": [ { "type": "PartialDateTime", "data": { "year": null, "month": null, "day": null, "hour": null, "minute": null, "second": null, "tz_hour": 5, "tz_minute": 0, "tz_minus": true } } ] }, { "name": { "type": "Tzoffsetto" }, "params": [], "values": [ { "type": "PartialDateTime", "data": { "year": null, "month": null, "day": null, "hour": null, "minute": null, "second": null, "tz_hour": 4, "tz_minute": 0, "tz_minus": true } } ] }, { "name": { "type": "Tzname" }, "params": [], "values": [ { "type": "Text", "data": "Eastern Daylight Time (US & Canada)" } ] } ], "component_ids": [] } ] } } }, { "property": { "type": "WebDav", "data": { "type": "ResourceType" } }, "value": { "ResourceTypes": [ "Collection", "AddressBook" ] } } ], "remove": [ { "type": "CalDav", "data": { "type": "CalendarTimezone" } }, { "type": "WebDav", "data": { "type": "ResourceType" } } ], "set_first": true } ================================================ FILE: crates/dav-proto/resources/requests/propertyupdate-001.xml ================================================ Adresses de Oliver Daboo Calendrier de Mathilde Desruisseaux BEGIN:VCALENDAR PRODID:-//Example Corp.//CalDAV Client//EN VERSION:2.0 BEGIN:VTIMEZONE TZID:US-Eastern LAST-MODIFIED:19870101T000000Z BEGIN:STANDARD DTSTART:19671029T020000 RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 TZOFFSETFROM:-0400 TZOFFSETTO:-0500 TZNAME:Eastern Standard Time (US & Canada) END:STANDARD BEGIN:DAYLIGHT DTSTART:19870405T020000 RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4 TZOFFSETFROM:-0500 TZOFFSETTO:-0400 TZNAME:Eastern Daylight Time (US & Canada) END:DAYLIGHT END:VTIMEZONE END:VCALENDAR ================================================ FILE: crates/dav-proto/resources/requests/propertyupdate-002.json ================================================ { "set": [ { "property": { "type": "DeadProperty", "data": { "name": "prop0", "attrs": "xmlns=\"http://example.com/neon/litmus/\"" } }, "value": { "DeadProperty": [ { "type": "Text", "data": "value0" } ] } }, { "property": { "type": "DeadProperty", "data": { "name": "prop1", "attrs": "xmlns=\"http://example.com/neon/litmus/\"" } }, "value": { "DeadProperty": [ { "type": "Text", "data": "value1" } ] } }, { "property": { "type": "DeadProperty", "data": { "name": "prop2", "attrs": "xmlns=\"http://example.com/neon/litmus/\"" } }, "value": { "DeadProperty": [ { "type": "Text", "data": "value2" } ] } }, { "property": { "type": "DeadProperty", "data": { "name": "prop3", "attrs": "xmlns=\"http://example.com/neon/litmus/\"" } }, "value": { "DeadProperty": [ { "type": "Text", "data": "value3" } ] } }, { "property": { "type": "DeadProperty", "data": { "name": "prop4", "attrs": "xmlns=\"http://example.com/neon/litmus/\"" } }, "value": { "DeadProperty": [ { "type": "Text", "data": "value4" } ] } }, { "property": { "type": "DeadProperty", "data": { "name": "prop5", "attrs": "xmlns=\"http://example.com/neon/litmus/\"" } }, "value": { "DeadProperty": [ { "type": "Text", "data": "value5" } ] } }, { "property": { "type": "DeadProperty", "data": { "name": "prop6", "attrs": "xmlns=\"http://example.com/neon/litmus/\"" } }, "value": { "DeadProperty": [ { "type": "Text", "data": "value6" } ] } }, { "property": { "type": "DeadProperty", "data": { "name": "prop7", "attrs": "xmlns=\"http://example.com/neon/litmus/\"" } }, "value": { "DeadProperty": [ { "type": "Text", "data": "value7" } ] } }, { "property": { "type": "DeadProperty", "data": { "name": "prop8", "attrs": "xmlns=\"http://example.com/neon/litmus/\"" } }, "value": { "DeadProperty": [ { "type": "Text", "data": "value8" } ] } }, { "property": { "type": "DeadProperty", "data": { "name": "prop9", "attrs": "xmlns=\"http://example.com/neon/litmus/\"" } }, "value": { "DeadProperty": [ { "type": "Text", "data": "value9" } ] } } ], "remove": [], "set_first": true } ================================================ FILE: crates/dav-proto/resources/requests/propertyupdate-002.xml ================================================ value0 value1 value2 value3 value4 value5 value6 value7 value8 value9 ================================================ FILE: crates/dav-proto/resources/requests/propfind-001.json ================================================ { "type": "Prop", "data": [ { "type": "DeadProperty", "data": { "name": "bigbox", "attrs": "xmlns=\"http://ns.example.com/boxschema/\"" } }, { "type": "DeadProperty", "data": { "name": "author", "attrs": "xmlns=\"http://ns.example.com/boxschema/\"" } }, { "type": "DeadProperty", "data": { "name": "DingALing", "attrs": "xmlns=\"http://ns.example.com/boxschema/\"" } }, { "type": "DeadProperty", "data": { "name": "Random", "attrs": "xmlns=\"http://ns.example.com/boxschema/\"" } } ] } ================================================ FILE: crates/dav-proto/resources/requests/propfind-001.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/propfind-002.json ================================================ { "type": "PropName" } ================================================ FILE: crates/dav-proto/resources/requests/propfind-002.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/propfind-003.json ================================================ { "type": "AllProp", "data": [] } ================================================ FILE: crates/dav-proto/resources/requests/propfind-003.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/propfind-004.json ================================================ { "type": "AllProp", "data": [ { "type": "WebDav", "data": { "type": "SupportedReportSet" } } ] } ================================================ FILE: crates/dav-proto/resources/requests/propfind-004.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/propfind-005.json ================================================ { "type": "Prop", "data": [ { "type": "WebDav", "data": { "type": "CurrentUserPrincipal" } } ] } ================================================ FILE: crates/dav-proto/resources/requests/propfind-005.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/propfind-006.json ================================================ { "type": "Prop", "data": [ { "type": "Principal", "data": { "type": "CalendarHomeSet" } }, { "type": "Principal", "data": { "type": "GroupMembership" } } ] } ================================================ FILE: crates/dav-proto/resources/requests/propfind-006.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/propfind-007.json ================================================ { "type": "Prop", "data": [ { "type": "WebDav", "data": { "type": "CurrentUserPrivilegeSet" } }, { "type": "WebDav", "data": { "type": "ResourceType" } }, { "type": "WebDav", "data": { "type": "DisplayName" } }, { "type": "DeadProperty", "data": { "name": "calendar-color", "attrs": "xmlns=\"http://apple.com/ns/ical/\"" } }, { "type": "CalDav", "data": { "type": "SupportedCalendarComponentSet" } } ] } ================================================ FILE: crates/dav-proto/resources/requests/propfind-007.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/propfind-008.json ================================================ { "type": "Prop", "data": [ { "type": "WebDav", "data": { "type": "ResourceType" } }, { "type": "WebDav", "data": { "type": "DisplayName" } }, { "type": "WebDav", "data": { "type": "GetCTag" } }, { "type": "CalDav", "data": { "type": "SupportedCalendarComponentSet" } } ] } ================================================ FILE: crates/dav-proto/resources/requests/propfind-008.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/propfind-009.json ================================================ ================================================ FILE: crates/dav-proto/resources/requests/propfind-009.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/propfind-010.json ================================================ { "type": "Prop", "data": [ { "type": "WebDav", "data": { "type": "ResourceType" } }, { "type": "WebDav", "data": { "type": "DisplayName" } }, { "type": "WebDav", "data": { "type": "SyncToken" } }, { "type": "WebDav", "data": { "type": "GetCTag" } }, { "type": "DeadProperty", "data": { "name": "me-card", "attrs": "xmlns=\"http://calendarserver.org/ns/\" hello=\"world & test\"" } } ] } ================================================ FILE: crates/dav-proto/resources/requests/propfind-010.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/report-001.json ================================================ { "type": "CalendarQuery", "properties": { "type": "Prop", "data": [ { "type": "WebDav", "data": { "type": "GetETag" } }, { "type": "CalDav", "data": { "type": "CalendarData", "data": { "properties": [ { "component": "VCalendar", "name": { "type": "Version" }, "no_value": false }, { "component": "VEvent", "name": { "type": "Summary" }, "no_value": false }, { "component": "VEvent", "name": { "type": "Uid" }, "no_value": false }, { "component": "VEvent", "name": { "type": "Dtstart" }, "no_value": false }, { "component": "VEvent", "name": { "type": "Dtend" }, "no_value": false }, { "component": "VEvent", "name": { "type": "Duration" }, "no_value": false }, { "component": "VEvent", "name": { "type": "Rrule" }, "no_value": false }, { "component": "VEvent", "name": { "type": "Rdate" }, "no_value": false }, { "component": "VEvent", "name": { "type": "Exrule" }, "no_value": false }, { "component": "VEvent", "name": { "type": "Exdate" }, "no_value": false }, { "component": "VEvent", "name": { "type": "RecurrenceId" }, "no_value": false }, { "component": "VTimezone", "name": null, "no_value": false } ], "expand": null, "limit_recurrence": null, "limit_freebusy": null } } } ] }, "filters": [ { "type": "Component", "comp": [ "VCalendar", "VEvent" ], "op": { "type": "TimeRange", "data": { "start": 1136332800, "end": 1136419200 } } } ], "timezone": { "type": "None" } } ================================================ FILE: crates/dav-proto/resources/requests/report-001.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/report-002.json ================================================ { "type": "CalendarQuery", "properties": { "type": "Prop", "data": [ { "type": "CalDav", "data": { "type": "CalendarData", "data": { "properties": [], "expand": null, "limit_recurrence": { "start": 1136246400, "end": 1136419200 }, "limit_freebusy": null } } } ] }, "filters": [ { "type": "Component", "comp": [ "VCalendar", "VEvent" ], "op": { "type": "TimeRange", "data": { "start": 1136246400, "end": 1136419200 } } } ], "timezone": { "type": "None" } } ================================================ FILE: crates/dav-proto/resources/requests/report-002.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/report-003.json ================================================ { "type": "CalendarQuery", "properties": { "type": "Prop", "data": [ { "type": "CalDav", "data": { "type": "CalendarData", "data": { "properties": [], "expand": { "start": 1136246400, "end": 1136419200 }, "limit_recurrence": null, "limit_freebusy": null } } } ] }, "filters": [ { "type": "Component", "comp": [ "VCalendar", "VEvent" ], "op": { "type": "TimeRange", "data": { "start": 1136246400, "end": 1136419200 } } } ], "timezone": { "type": "None" } } ================================================ FILE: crates/dav-proto/resources/requests/report-003.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/report-004.json ================================================ { "type": "CalendarQuery", "properties": { "type": "Prop", "data": [ { "type": "CalDav", "data": { "type": "CalendarData", "data": { "properties": [], "expand": null, "limit_recurrence": null, "limit_freebusy": { "start": 1136160000, "end": 1136246400 } } } } ] }, "filters": [ { "type": "Component", "comp": [ "VCalendar", "VFreebusy" ], "op": { "type": "TimeRange", "data": { "start": 1136160000, "end": 1136246400 } } } ], "timezone": { "type": "None" } } ================================================ FILE: crates/dav-proto/resources/requests/report-004.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/report-005.json ================================================ { "type": "CalendarQuery", "properties": { "type": "Prop", "data": [ { "type": "WebDav", "data": { "type": "GetETag" } }, { "type": "CalDav", "data": { "type": "CalendarData", "data": { "properties": [], "expand": null, "limit_recurrence": null, "limit_freebusy": null } } } ] }, "filters": [ { "type": "Component", "comp": [ "VCalendar", "VTodo", "VAlarm" ], "op": { "type": "TimeRange", "data": { "start": 1136541600, "end": 1136628000 } } } ], "timezone": { "type": "None" } } ================================================ FILE: crates/dav-proto/resources/requests/report-005.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/report-006.json ================================================ { "type": "CalendarQuery", "properties": { "type": "Prop", "data": [ { "type": "WebDav", "data": { "type": "GetETag" } }, { "type": "CalDav", "data": { "type": "CalendarData", "data": { "properties": [], "expand": null, "limit_recurrence": null, "limit_freebusy": null } } } ] }, "filters": [ { "type": "Property", "comp": [ "VCalendar", "VEvent" ], "prop": { "type": "Uid" }, "op": { "type": "TextMatch", "data": { "type": "TextMatch", "match_type": "Contains", "value": "DC6C50A017428C5216A2F1CD@example.com", "collation": "Octet", "negate": false } } } ], "timezone": { "type": "None" } } ================================================ FILE: crates/dav-proto/resources/requests/report-006.xml ================================================ DC6C50A017428C5216A2F1CD@example.com ================================================ FILE: crates/dav-proto/resources/requests/report-007.json ================================================ { "type": "CalendarQuery", "properties": { "type": "Prop", "data": [ { "type": "WebDav", "data": { "type": "GetETag" } }, { "type": "CalDav", "data": { "type": "CalendarData", "data": { "properties": [], "expand": null, "limit_recurrence": null, "limit_freebusy": null } } } ] }, "filters": [ { "type": "Property", "comp": [ "VCalendar", "VEvent" ], "prop": { "type": "Attendee" }, "op": { "type": "TextMatch", "data": { "type": "TextMatch", "match_type": "Contains", "value": "mailto:lisa@example.com", "collation": "AsciiCasemap", "negate": false } } }, { "type": "Parameter", "comp": [ "VCalendar", "VEvent" ], "prop": { "type": "Attendee" }, "param": "Partstat", "op": { "type": "TextMatch", "data": { "type": "TextMatch", "match_type": "Contains", "value": "NEEDS-ACTION", "collation": "AsciiCasemap", "negate": false } } } ], "timezone": { "type": "None" } } ================================================ FILE: crates/dav-proto/resources/requests/report-007.xml ================================================ mailto:lisa@example.com NEEDS-ACTION ================================================ FILE: crates/dav-proto/resources/requests/report-008.json ================================================ { "type": "CalendarQuery", "properties": { "type": "Prop", "data": [ { "type": "WebDav", "data": { "type": "GetETag" } }, { "type": "CalDav", "data": { "type": "CalendarData", "data": { "properties": [], "expand": null, "limit_recurrence": null, "limit_freebusy": null } } } ] }, "filters": [ { "type": "Component", "comp": [ "VCalendar", "VEvent" ], "op": { "type": "Exists" } } ], "timezone": { "type": "None" } } ================================================ FILE: crates/dav-proto/resources/requests/report-008.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/report-009.json ================================================ { "type": "CalendarQuery", "properties": { "type": "Prop", "data": [ { "type": "WebDav", "data": { "type": "GetETag" } }, { "type": "CalDav", "data": { "type": "CalendarData", "data": { "properties": [], "expand": null, "limit_recurrence": null, "limit_freebusy": null } } } ] }, "filters": [ { "type": "Property", "comp": [ "VCalendar", "VTodo" ], "prop": { "type": "Completed" }, "op": { "type": "Undefined" } }, { "type": "Property", "comp": [ "VCalendar", "VTodo" ], "prop": { "type": "Status" }, "op": { "type": "TextMatch", "data": { "type": "TextMatch", "match_type": "Contains", "value": "CANCELLED", "collation": "AsciiCasemap", "negate": true } } } ], "timezone": { "type": "None" } } ================================================ FILE: crates/dav-proto/resources/requests/report-009.xml ================================================ CANCELLED ================================================ FILE: crates/dav-proto/resources/requests/report-010.json ================================================ { "type": "CalendarQuery", "properties": { "type": "Prop", "data": [ { "type": "WebDav", "data": { "type": "GetETag" } }, { "type": "CalDav", "data": { "type": "CalendarData", "data": { "properties": [], "expand": null, "limit_recurrence": null, "limit_freebusy": null } } } ] }, "filters": [ { "type": "Property", "comp": [ "VCalendar", "VEvent" ], "prop": { "type": "Other", "data": "X-ABC-GUID" }, "op": { "type": "TextMatch", "data": { "type": "TextMatch", "match_type": "Contains", "value": "ABC", "collation": "AsciiCasemap", "negate": false } } } ], "timezone": { "type": "None" } } ================================================ FILE: crates/dav-proto/resources/requests/report-010.xml ================================================ ABC ================================================ FILE: crates/dav-proto/resources/requests/report-011.json ================================================ { "type": "CalendarQuery", "properties": { "type": "Prop", "data": [ { "type": "WebDav", "data": { "type": "GetETag" } } ] }, "filters": [ { "type": "Component", "comp": [ "VCalendar", "VEvent" ], "op": { "type": "TimeRange", "data": { "start": 1094083200, "end": 1094169600 } } } ], "timezone": { "type": "None" } } ================================================ FILE: crates/dav-proto/resources/requests/report-011.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/report-012.json ================================================ { "type": "CalendarQuery", "properties": { "type": "Prop", "data": [ { "type": "WebDav", "data": { "type": "GetETag" } }, { "type": "CalDav", "data": { "type": "CalendarData", "data": { "properties": [], "expand": null, "limit_recurrence": null, "limit_freebusy": null } } } ] }, "filters": [], "timezone": { "type": "None" } } ================================================ FILE: crates/dav-proto/resources/requests/report-012.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/report-013.json ================================================ { "type": "CalendarMultiGet", "properties": { "type": "Prop", "data": [ { "type": "WebDav", "data": { "type": "GetETag" } }, { "type": "CalDav", "data": { "type": "CalendarData", "data": { "properties": [], "expand": null, "limit_recurrence": null, "limit_freebusy": null } } } ] }, "hrefs": [ "/bernard/work/abcd1.ics", "/bernard/work/mtg1.ics" ] } ================================================ FILE: crates/dav-proto/resources/requests/report-013.xml ================================================ /bernard/work/abcd1.ics /bernard/work/mtg1.ics ================================================ FILE: crates/dav-proto/resources/requests/report-014.json ================================================ { "type": "AddressbookQuery", "properties": { "type": "Prop", "data": [ { "type": "WebDav", "data": { "type": "GetETag" } }, { "type": "CardDav", "data": { "type": "AddressData", "data": [ { "group": null, "name": { "type": "Version" }, "no_value": false }, { "group": null, "name": { "type": "Uid" }, "no_value": false }, { "group": null, "name": { "type": "Nickname" }, "no_value": false }, { "group": null, "name": { "type": "Email" }, "no_value": false }, { "group": null, "name": { "type": "Fn" }, "no_value": false } ] } } ] }, "filters": [ { "type": "Property", "comp": null, "prop": { "name": { "type": "Nickname" }, "group": null }, "op": { "type": "TextMatch", "data": { "type": "TextMatch", "match_type": "Equals", "value": "me", "collation": "UnicodeCasemap", "negate": false } } } ], "limit": null } ================================================ FILE: crates/dav-proto/resources/requests/report-014.xml ================================================ me ================================================ FILE: crates/dav-proto/resources/requests/report-015.json ================================================ { "type": "AddressbookQuery", "properties": { "type": "Prop", "data": [ { "type": "WebDav", "data": { "type": "GetETag" } }, { "type": "CardDav", "data": { "type": "AddressData", "data": [ { "group": null, "name": { "type": "Version" }, "no_value": false }, { "group": null, "name": { "type": "Uid" }, "no_value": false }, { "group": null, "name": { "type": "Nickname" }, "no_value": false }, { "group": null, "name": { "type": "Email" }, "no_value": false }, { "group": null, "name": { "type": "Fn" }, "no_value": false } ] } } ] }, "filters": [ { "type": "AnyOf" }, { "type": "Property", "comp": null, "prop": { "name": { "type": "Fn" }, "group": null }, "op": { "type": "TextMatch", "data": { "type": "TextMatch", "match_type": "Contains", "value": "daboo", "collation": "UnicodeCasemap", "negate": false } } }, { "type": "Property", "comp": null, "prop": { "name": { "type": "Email" }, "group": null }, "op": { "type": "TextMatch", "data": { "type": "TextMatch", "match_type": "Contains", "value": "daboo", "collation": "UnicodeCasemap", "negate": false } } } ], "limit": null } ================================================ FILE: crates/dav-proto/resources/requests/report-015.xml ================================================ daboo daboo ================================================ FILE: crates/dav-proto/resources/requests/report-016.json ================================================ { "type": "AddressbookQuery", "properties": { "type": "Prop", "data": [ { "type": "WebDav", "data": { "type": "GetETag" } } ] }, "filters": [ { "type": "AnyOf" }, { "type": "Property", "comp": null, "prop": { "name": { "type": "Fn" }, "group": null }, "op": { "type": "TextMatch", "data": { "type": "TextMatch", "match_type": "Contains", "value": "daboo", "collation": "UnicodeCasemap", "negate": false } } } ], "limit": 2 } ================================================ FILE: crates/dav-proto/resources/requests/report-016.xml ================================================ daboo 2 ================================================ FILE: crates/dav-proto/resources/requests/report-017.json ================================================ { "type": "AddressbookMultiGet", "properties": { "type": "Prop", "data": [ { "type": "WebDav", "data": { "type": "GetETag" } }, { "type": "CardDav", "data": { "type": "AddressData", "data": [ { "group": null, "name": { "type": "Version" }, "no_value": false }, { "group": null, "name": { "type": "Uid" }, "no_value": false }, { "group": null, "name": { "type": "Nickname" }, "no_value": false }, { "group": null, "name": { "type": "Email" }, "no_value": false }, { "group": null, "name": { "type": "Fn" }, "no_value": false } ] } } ] }, "hrefs": [ "/home/bernard/addressbook/vcf102.vcf", "/home/bernard/addressbook/vcf1.vcf" ] } ================================================ FILE: crates/dav-proto/resources/requests/report-017.xml ================================================ /home/bernard/addressbook/vcf102.vcf /home/bernard/addressbook/vcf1.vcf ================================================ FILE: crates/dav-proto/resources/requests/report-018.json ================================================ { "type": "AddressbookMultiGet", "properties": { "type": "Prop", "data": [ { "type": "WebDav", "data": { "type": "GetETag" } }, { "type": "CardDav", "data": { "type": "AddressData", "data": [] } } ] }, "hrefs": [ "/home/bernard/addressbook/vcf3.vcf" ] } ================================================ FILE: crates/dav-proto/resources/requests/report-018.xml ================================================ /home/bernard/addressbook/vcf3.vcf ================================================ FILE: crates/dav-proto/resources/requests/report-019.json ================================================ { "type": "SyncCollection", "sync_token": "abc", "properties": { "type": "Prop", "data": [ { "type": "WebDav", "data": { "type": "GetETag" } } ] }, "depth": "Infinity", "limit": 9 } ================================================ FILE: crates/dav-proto/resources/requests/report-019.xml ================================================ abc infinite 9 ================================================ FILE: crates/dav-proto/resources/requests/report-020.json ================================================ { "type": "AclPrincipalPropSet", "properties": [ { "type": "WebDav", "data": { "type": "DisplayName" } } ] } ================================================ FILE: crates/dav-proto/resources/requests/report-020.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/report-021.json ================================================ { "type": "PrincipalMatch", "principal_properties": { "Properties": [ { "type": "WebDav", "data": { "type": "Owner" } } ] }, "properties": [] } ================================================ FILE: crates/dav-proto/resources/requests/report-021.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/report-022.json ================================================ { "type": "PrincipalPropertySearch", "property_search": [ { "property": { "type": "WebDav", "data": { "type": "DisplayName" } }, "match_": "doE" }, { "property": { "type": "DeadProperty", "data": { "name": "title", "attrs": "xmlns=\"http://www.example.com/ns/\"" } }, "match_": "Sales" } ], "properties": [ { "type": "WebDav", "data": { "type": "DisplayName" } }, { "type": "DeadProperty", "data": { "name": "department", "attrs": "xmlns=\"http://www.example.com/ns/\"" } }, { "type": "DeadProperty", "data": { "name": "phone", "attrs": "xmlns=\"http://www.example.com/ns/\"" } }, { "type": "DeadProperty", "data": { "name": "office", "attrs": "xmlns=\"http://www.example.com/ns/\"" } }, { "type": "DeadProperty", "data": { "name": "salary", "attrs": "xmlns=\"http://www.example.com/ns/\"" } } ], "apply_to_principal_collection_set": false } ================================================ FILE: crates/dav-proto/resources/requests/report-022.xml ================================================ doE Sales ================================================ FILE: crates/dav-proto/resources/requests/report-023.json ================================================ { "type": "PrincipalSearchPropertySet" } ================================================ FILE: crates/dav-proto/resources/requests/report-023.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/report-024.json ================================================ { "type": "ExpandProperty", "properties": [ { "property": { "type": "DeadProperty", "data": { "name": "version-history", "attrs": "name=\"version-history\"" } }, "depth": 0 }, { "property": { "type": "DeadProperty", "data": { "name": "version-set", "attrs": "name=\"version-set\"" } }, "depth": 1 }, { "property": { "type": "DeadProperty", "data": { "name": "creator-displayname", "attrs": "name=\"creator-displayname\"" } }, "depth": 2 }, { "property": { "type": "DeadProperty", "data": { "name": "activity-set", "attrs": "name=\"activity-set\"" } }, "depth": 2 } ] } ================================================ FILE: crates/dav-proto/resources/requests/report-024.xml ================================================ ================================================ FILE: crates/dav-proto/resources/requests/report-025.json ================================================ { "type": "ExpandProperty", "properties": [ { "property": { "type": "DeadProperty", "data": { "name": "calendar-proxy-read-for", "attrs": "name=\"calendar-proxy-read-for\" namespace=\"\nhttp://calendarserver.org/ns/\"" } }, "depth": 0 }, { "property": { "type": "DeadProperty", "data": { "name": "email-address-set", "attrs": "name=\"email-address-set\"\nnamespace=\"http://calendarserver.org/ns/\"" } }, "depth": 1 }, { "property": { "type": "WebDav", "data": { "type": "DisplayName" } }, "depth": 1 }, { "property": { "type": "DeadProperty", "data": { "name": "calendar-user-address-set", "attrs": "name=\"calendar-user-address-set\"\nnamespace=\"urn:ietf:params:xml:ns:caldav\"" } }, "depth": 1 }, { "property": { "type": "DeadProperty", "data": { "name": "calendar-proxy-write-for", "attrs": "name=\"calendar-proxy-write-for\"\nnamespace=\"http://calendarserver.org/ns/\"" } }, "depth": 0 }, { "property": { "type": "DeadProperty", "data": { "name": "email-address-set", "attrs": "name=\"email-address-set\" namespace=\"http://calendarserver.org/ns/\"" } }, "depth": 1 }, { "property": { "type": "WebDav", "data": { "type": "DisplayName" } }, "depth": 1 }, { "property": { "type": "DeadProperty", "data": { "name": "calendar-user-address-set", "attrs": "name=\"calendar-user-address-set\"\nnamespace=\"urn:ietf:params:xml:ns:caldav\"" } }, "depth": 1 } ] } ================================================ FILE: crates/dav-proto/resources/requests/report-025.xml ================================================ ================================================ FILE: crates/dav-proto/resources/responses/001.xml ================================================ /locked/ ================================================ FILE: crates/dav-proto/resources/responses/002.xml ================================================ http://www.example.com/file Box type A HTTP/1.1 200 OK Box type B HTTP/1.1 403 Forbidden The user does not have access to the DingALing property. There has been an access violation error. ================================================ FILE: crates/dav-proto/resources/responses/003.xml ================================================ /container/ 1997-12-02T01:42:21Z Example collection HTTP/1.1 200 OK /container/front.html 1997-12-02T02:27:21Z Example HTML resource 4525 text/html "zzyzx" Mon, 12 Jan 1998 09:25:56 GMT HTTP/1.1 200 OK ================================================ FILE: crates/dav-proto/resources/responses/004.xml ================================================ http://www.example.com/container/resource3 HTTP/1.1 423 Locked ================================================ FILE: crates/dav-proto/resources/responses/005.xml ================================================ infinity http://example.org/~ejw/contact.html Second-604800 urn:uuid:e71d4fae-5dec-22d6-fea5-00a0c91e6be4 http://example.com/workspace/webdav/proposal.doc ================================================ FILE: crates/dav-proto/resources/responses/006.xml ================================================ http://www.example.com/container/ 0 Jane Smith Infinite urn:uuid:f81de2ad-7f3d-a1b2-4f3c-00a0c91a9d76 http://www.example.com/container/ HTTP/1.1 200 OK ================================================ FILE: crates/dav-proto/resources/responses/007.xml ================================================ /workspace/webdav/ ================================================ FILE: crates/dav-proto/resources/responses/008.xml ================================================ http://cal.example.com/bernard/work/abcd2.ics "fffff-abcd2" HTTP/1.1 200 OK http://cal.example.com/bernard/work/abcd3.ics "fffff-abcd3" HTTP/1.1 200 OK ================================================ FILE: crates/dav-proto/resources/responses/009.xml ================================================ HTTP/1.1 200 OK ================================================ FILE: crates/dav-proto/resources/responses/010.xml ================================================ /home/bernard/addressbook/v102.vcf "23ba4d-ff11fb" HTTP/1.1 200 OK ================================================ FILE: crates/dav-proto/resources/responses/011.xml ================================================ /home/bernard/addressbook/ HTTP/1.1 507 Insufficient Storage Only two matching records were returned /home/bernard/addressbook/v102.vcf "23ba4d-ff11fb" HTTP/1.1 200 OK /home/bernard/addressbook/v104.vcf "23ba4d-ff11fc" HTTP/1.1 200 OK ================================================ FILE: crates/dav-proto/resources/responses/012.xml ================================================ /a /c ================================================ FILE: crates/dav-proto/resources/responses/013.xml ================================================ Full name Job title ================================================ FILE: crates/dav-proto/resources/responses/014.xml ================================================ http://www.example.com/papers/ Any operation Read any object Read ACL Read current user privilege set property Write any object Write ACL Write properties Write resource content Unlock resource HTTP/1.1 200 OK ================================================ FILE: crates/dav-proto/resources/responses/015.xml ================================================ http://www.example.com/papers/ HTTP/1.1 200 OK ================================================ FILE: crates/dav-proto/resources/responses/016.xml ================================================ http://www.example.com/papers/ http://www.example.com/acl/groups/maintainers HTTP/1.1 200 OK ================================================ FILE: crates/dav-proto/resources/responses/017.xml ================================================ http://www.example.com/papers/ HTTP/1.1 200 OK ================================================ FILE: crates/dav-proto/resources/responses/018.xml ================================================ http://www.example.com/papers/ http://www.example.com/acl/users/ http://www.example.com/acl/groups/ HTTP/1.1 200 OK ================================================ FILE: crates/dav-proto/resources/responses/019.xml ================================================ http://www.example.com/top/container/ http://www.example.com/users/gclemm Any operation Read any object Write any object Read the ACL Write the ACL http://www.example.com/users/esedlar http://www.example.com/groups/mrktng http://www.example.com/top HTTP/1.1 200 OK ================================================ FILE: crates/dav-proto/resources/responses/020.xml ================================================ mailto:wilfredo@example.com 2.0;Success mailto:bernard@example.net 2.0;Success mailto:mike@example.org 3.7;Invalid calendar user ================================================ FILE: crates/dav-proto/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use calcard::vcard::VCardVersion; use compact_str::{CompactString, ToCompactString}; use trc::Value; pub mod parser; pub mod requests; pub mod responses; pub mod schema; pub fn xml_pretty_print(xml_string: &str) -> String { // Create a reader let mut reader = quick_xml::Reader::from_str(xml_string); let mut writer = quick_xml::Writer::new_with_indent(std::io::Cursor::new(Vec::new()), b' ', 2); let mut buf = Vec::new(); loop { match reader.read_event_into(&mut buf) { Ok(quick_xml::events::Event::Eof) => break, Ok(event) => { writer.write_event(event).unwrap(); } Err(e) => panic!("Error at position {}: {:?}", reader.buffer_position(), e), } buf.clear(); } let result = writer.into_inner().into_inner(); String::from_utf8(result).unwrap() } #[derive(Debug, Default, PartialEq, Eq)] pub struct RequestHeaders<'x> { pub uri: &'x str, pub depth: Depth, pub timeout: Timeout, pub content_type: Option<&'x str>, pub destination: Option<&'x str>, pub lock_token: Option<&'x str>, pub max_vcard_version: Option, pub no_schedule_reply: bool, pub if_schedule_tag: Option, pub overwrite_fail: bool, pub no_timezones: bool, pub ret: Return, pub depth_no_root: bool, pub if_: Vec>, } pub struct ResourceState> { pub resource: Option, pub etag: T, pub state_token: T, } #[derive(Debug, Default, PartialEq, Eq, Clone, Copy)] pub enum Return { Minimal, Representation, #[default] Default, } #[derive(Debug, PartialEq, Eq, Clone)] pub struct If<'x> { pub resource: Option<&'x str>, pub list: Vec>, } #[derive(Debug, PartialEq, Eq, Clone)] pub enum Condition<'x> { StateToken { is_not: bool, token: &'x str }, ETag { is_not: bool, tag: &'x str }, Exists { is_not: bool }, } #[derive(Debug, Default, PartialEq, Eq, Clone, Copy)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(test, serde(tag = "type", content = "data"))] pub enum Timeout { Infinite, Second(u64), #[default] None, } #[derive(Debug, Default, PartialEq, Eq, Clone, Copy)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub enum Depth { Zero, One, Infinity, #[default] None, } impl From<&RequestHeaders<'_>> for Value { fn from(headers: &RequestHeaders<'_>) -> Self { let mut values = Vec::with_capacity(4); if headers.depth != Depth::None { values.push(Value::String(CompactString::const_new("Depth"))); values.push(match headers.depth { Depth::Zero => Value::Int(0), Depth::One => Value::Int(1), Depth::Infinity => Value::String(CompactString::const_new("infinity")), Depth::None => Value::None, }); } if headers.timeout != Timeout::None { values.push(Value::String(CompactString::const_new("Timeout"))); values.push(match headers.timeout { Timeout::Infinite => Value::String(CompactString::const_new("infinite")), Timeout::Second(n) => Value::Int(n as i64), Timeout::None => Value::None, }); } for (name, header_value) in [ ("Content-Type", headers.content_type), ("Destination", headers.destination), ("Lock-Token", headers.lock_token), ] { if let Some(value) = header_value { values.push(CompactString::const_new(name).into()); values.push(value.to_compact_string().into()); } } for (name, is_set) in [ ("Overwrite", headers.overwrite_fail), ("No-Timezones", headers.no_timezones), ("Depth-No-Root", headers.depth_no_root), ] { if is_set { values.push(CompactString::const_new(name).into()); } } for if_ in &headers.if_ { values.push(CompactString::const_new("If").into()); let mut if_values = Vec::with_capacity(if_.list.len() * 2 + 1); if let Some(resource) = if_.resource { if_values.push(Value::String(resource.to_compact_string())); } for condition in &if_.list { match condition { Condition::StateToken { is_not, token } => { if *is_not { if_values.push(Value::String(CompactString::const_new("!State-Token"))); } else { if_values.push(Value::String(CompactString::const_new("State-Token"))); } if_values.push(Value::String(token.to_compact_string())); } Condition::ETag { is_not, tag } => { if *is_not { if_values.push(Value::String(CompactString::const_new("!ETag"))); } else { if_values.push(Value::String(CompactString::const_new("ETag"))); } if_values.push(Value::String(tag.to_compact_string())); } Condition::Exists { is_not } => { if *is_not { if_values.push(Value::String(CompactString::const_new("!Exists"))); } else { if_values.push(Value::String(CompactString::const_new("Exists"))); } } } } values.push(Value::Array(if_values)); } Value::Array(values) } } /* Implemented: RFC4918 - HTTP Extensions for Web Distributed Authoring and Versioning (WebDAV) RFC5689 - Extended MKCOL for Web Distributed Authoring and Versioning (WebDAV) RFC6578 - Collection Synchronization for Web Distributed Authoring and Versioning (WebDAV) RFC3744 - Web Distributed Authoring and Versioning (WebDAV) Access Control Protocol RFC4331 - Quota and Size Properties for Distributed Authoring and Versioning (DAV) Collections RFC5397 - WebDAV Current Principal Extension RFC8144 - Use of the Prefer Header Field in Web Distributed Authoring and Versioning (WebDAV) RFC4791 - Calendaring Extensions to WebDAV (CalDAV) RFC7809 - Calendaring Extensions to WebDAV (CalDAV) Time Zones by Reference RFC6638 - Scheduling Extensions to CalDAV RFC6352 - CardDAV vCard Extensions to Web Distributed Authoring and Versioning (WebDAV) RFC6764 - Locating Services for Calendaring Extensions to WebDAV (CalDAV) and vCard Extensions to WebDAV (CardDAV) Out of scope: RFC5842 - Binding Extensions to Web Distributed Authoring and Versioning (WebDAV) RFC4316 - Datatypes for Web Distributed Authoring and Versioning (WebDAV) Properties RFC4709 - Mounting Web Distributed Authoring and Versioning (WebDAV) Servers RFC3648 - Web Distributed Authoring and Versioning (WebDAV) Ordered Collections Protocol RFC4437 - Web Distributed Authoring and Versioning (WebDAV) Redirect Reference Resources RFC8607 - Calendaring Extensions to WebDAV (CalDAV) Managed Attachments RFC5995 - Using POST to Add Members to Web Distributed Authoring and Versioning (WebDAV) Collections RFC3253 - Versioning Extensions to WebDAV (Web Distributed Authoring and Versioning) RFC5323 - Web Distributed Authoring and Versioning (WebDAV) SEARCH */ ================================================ FILE: crates/dav-proto/src/parser/header.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{Condition, Depth, If, RequestHeaders, ResourceState, Return, Timeout}; use calcard::vcard::VCardVersion; impl<'x> RequestHeaders<'x> { pub fn new(uri: &'x str) -> Self { RequestHeaders { uri, ..Default::default() } } pub fn parse(&mut self, key: &str, value: &'x str) -> bool { hashify::fnc_map_ignore_case!(key.as_bytes(), "Depth" => { if let Some(depth) = Depth::parse(value.as_bytes()) { self.depth = depth; return true; } }, "Destination" => { self.destination = Some(value); return true; }, "Lock-Token" => { self.lock_token = Some(try_unwrap_coded_url(value)); return true; }, "If" => { let num = self.if_.len(); self.parse_if(value); return self.if_.len() != num; }, "If-Match" => { let num = self.if_.len(); self.parse_if_match(value, false); return self.if_.len() != num; }, "If-None-Match" => { let num = self.if_.len(); self.parse_if_match(value, true); return self.if_.len() != num; }, "Timeout" => { let value = value.split_once(',').map(|(first, _)| first).unwrap_or(value).trim(); if let Some(seconds) = value.strip_prefix("Second-") { if let Ok(seconds) = seconds.parse() { self.timeout = Timeout::Second(seconds); return true; } } else if value == "Infinite" { self.timeout = Timeout::Infinite; return true; } }, "Overwrite" => { self.overwrite_fail = value == "F"; return true; }, "CalDAV-Timezones" => { self.no_timezones = value == "F"; return true; }, "Prefer" => { for value in value.split(&[',', ';']) { match value.trim() { "return=minimal" => self.ret = Return::Minimal, "return=representation" => self.ret = Return::Representation, "depth-noroot" => self.depth_no_root = true, _ => {} } } }, "Content-Type" => { let value = value.trim(); if (2..=127).contains(&value.len()) { self.content_type = Some(value); } return true; }, "Accept" => { for value in value.split(',') { if value.trim().starts_with("text/vcard") && let Some(version) = value.split_once("version=") .and_then(|(_, version)| VCardVersion::try_parse(version.trim())) { if let Some(max_vcard_version) = &mut self.max_vcard_version { if version > *max_vcard_version { *max_vcard_version = version; } } else { self.max_vcard_version = Some(version); } } } return true; }, "If-Schedule-Tag-Match" => { self.if_schedule_tag = value.trim().trim_matches('"').parse().ok(); return true; }, "Schedule-Reply" => { self.no_schedule_reply = value == "F"; return true; }, _ => {} ); false } pub fn has_if(&self) -> bool { !self.if_.is_empty() } pub fn eval_if_resources(&self) -> impl Iterator { self.if_.iter().filter_map(|if_| if_.resource) } pub fn eval_if(&self, resources: &[ResourceState]) -> bool where T: AsRef, { if self.if_.is_empty() { return true; } 'outer: for if_ in &self.if_ { if if_.list.is_empty() { continue; } let (current_token, current_etag) = resources .iter() .find_map(|r| { if if_.resource == r.resource.as_ref().map(|v| v.as_ref()) { Some((r.state_token.as_ref(), r.etag.as_ref())) } else { None } }) .unwrap_or_default(); for cond in if_.list.iter() { match cond { Condition::StateToken { is_not, token } => { if !((current_token == *token) ^ is_not) { continue 'outer; } } Condition::ETag { is_not, tag } => { if !((current_etag == *tag) ^ is_not) { continue 'outer; } } Condition::Exists { is_not } => { if !((current_etag.is_empty()) ^ is_not) { continue 'outer; } } } } return true; } false } fn parse_if(&mut self, value: &'x str) { let value = value.as_bytes(); let mut iter = value.iter().enumerate(); let mut resource = None; while let Some((idx, ch)) = iter.next() { match ch { b'<' if resource.is_none() => { for (to_idx, ch) in iter.by_ref() { if *ch == b'>' { resource = Some(std::str::from_utf8(&value[idx + 1..to_idx]).unwrap()); break; } } } b'(' => { let mut is_not = false; let mut conditions = Vec::new(); while let Some((idx, ch)) = iter.next() { match ch { b'N' => { if matches!(iter.next(), Some((_, b'o'))) && matches!(iter.next(), Some((_, b't'))) { is_not = true; } else { return; } } b'<' | b'[' => { let (stop_char, is_etag) = match ch { b'<' => (b'>', false), b'[' => (b']', true), _ => unreachable!(), }; for (to_idx, ch) in iter.by_ref() { if *ch == stop_char { let value = std::str::from_utf8(&value[idx + 1..to_idx]).unwrap(); let condition = if is_etag { Condition::ETag { is_not, tag: value } } else { Condition::StateToken { is_not, token: value, } }; conditions.push(condition); is_not = false; break; } } } b')' => { self.if_.push(If { resource: resource.take(), list: conditions, }); break; } _ => { if !ch.is_ascii_whitespace() { return; } } } } } _ => { if !ch.is_ascii_whitespace() { return; } } } } } pub fn parse_if_match(&mut self, value: &'x str, is_not: bool) { if value == "*" { self.if_.push(If { resource: None, list: vec![Condition::Exists { is_not }], }); } else if !is_not { for etag in value.split(',') { self.if_.push(If { resource: None, list: vec![Condition::ETag { is_not, tag: etag.trim(), }], }); } } else { let mut etags = Vec::new(); for etag in value.split(',') { etags.push(Condition::ETag { is_not, tag: etag.trim(), }); } self.if_.push(If { resource: None, list: etags, }); } } pub fn base_uri(&self) -> Option<&str> { dav_base_uri(self.uri) } } pub fn dav_base_uri(uri: &str) -> Option<&str> { // From a path ../dav/collection/account/.. // returns ../dav/collection/account without the trailing slash let uri = uri.as_bytes(); let mut found_dav = false; let mut last_idx = 0; let mut sep_count = 0; for (idx, ch) in uri.iter().enumerate() { if *ch == b'/' { if !found_dav { found_dav = uri.get(idx + 1..idx + 5).is_some_and(|s| s == b"dav/"); } else if found_dav { if sep_count == 2 { break; } sep_count += 1; } } last_idx = idx; } if sep_count == 2 { uri.get(..last_idx + 1) .map(|uri| std::str::from_utf8(uri).unwrap()) } else { None } } impl Depth { pub fn parse(value: &[u8]) -> Option { hashify::tiny_map!(value, "0" => Depth::Zero, "1" => Depth::One, "infinity" => Depth::Infinity, "infinite" => Depth::Infinity, ) } } fn try_unwrap_coded_url(url: &str) -> &str { url.strip_prefix("<") .and_then(|url| url.strip_suffix(">")) .unwrap_or(url) } #[cfg(test)] mod tests { use super::*; #[test] fn base_uri() { for (uri, expected_base) in [ ( "http://host/dav/collection/account/test/", Some("http://host/dav/collection/account"), ), ( "http://host/dav/collection/account/test", Some("http://host/dav/collection/account"), ), ( "http://host/dav/collection/account/", Some("http://host/dav/collection/account"), ), ( "http://host/dav/collection/account", Some("http://host/dav/collection/account"), ), ( "http://host/dev/dav/collection/account/test/", Some("http://host/dev/dav/collection/account"), ), ( "http://host/dev/dav/collection/account/test", Some("http://host/dev/dav/collection/account"), ), ( "http://host/dev/dav/collection/account/", Some("http://host/dev/dav/collection/account"), ), ( "http://host/dev/dav/collection/account", Some("http://host/dev/dav/collection/account"), ), ( "/dav/collection/account/test/", Some("/dav/collection/account"), ), ( "/dav/collection/account/test", Some("/dav/collection/account"), ), ("/dav/collection/account/", Some("/dav/collection/account")), ("/dav/collection/account", Some("/dav/collection/account")), ] { assert_eq!(RequestHeaders::new(uri).base_uri(), expected_base); } } #[test] fn eval_if_header() { let mut headers = RequestHeaders::default(); assert!(headers.parse( "If", r#"( ["I am an ETag"]) (["I am another ETag"])"#, )); assert!(headers.eval_if(&[ResourceState { resource: None, state_token: "urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2", etag: "\"I am an ETag\"" }])); assert!(headers.eval_if(&[ResourceState { resource: None, state_token: "", etag: "\"I am another ETag\"" }])); assert!(!headers.eval_if(&[ResourceState { resource: None, state_token: "", etag: "\"Unknown ETag\"" }])); assert!(!headers.eval_if(&[ResourceState { resource: None, state_token: "urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2", etag: "" }])); assert!(!headers.eval_if(&[ResourceState { resource: None, state_token: "urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2", etag: "\"Other ETag\"" }])); assert!(!headers.eval_if(&[ResourceState { resource: None, state_token: "", etag: "\"I am an ETag\"" }])); assert!(!headers.eval_if(&[ResourceState { resource: None, state_token: "urn:blah", etag: "\"I am an ETag\"" }])); assert!(headers.parse( "If", r#"(Not )"#, )); assert!(headers.eval_if(&[ResourceState { resource: None, state_token: "urn:uuid:58f202ac-22cf-11d1-b12d-002035b29092", etag: "" }])); assert!(!headers.eval_if(&[ResourceState { resource: None, state_token: "urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2", etag: "" }])); assert!(headers.parse( "If", r#"() (Not )"# )); assert!(headers.eval_if(&[ResourceState { resource: None, state_token: "urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2", etag: "" }])); assert!(headers.eval_if(&[ResourceState { resource: None, state_token: "urn:other-token", etag: "" }])); } #[test] fn parse_headers() { let mut headers = RequestHeaders::default(); assert!(headers.parse("Depth", "0")); assert_eq!(headers.depth, Depth::Zero); assert!(headers.parse("Destination", "/path/to/destination")); assert_eq!(headers.destination, Some("/path/to/destination")); assert!(headers.parse("Lock-Token", "")); assert_eq!(headers.lock_token, Some("urn:uuid:1234")); for (input, expected) in [ ( "()", vec![If { resource: "urn:uuid:1234".into(), list: vec![Condition::StateToken { is_not: false, token: "urn:uuid:1234", }], }], ), ( "<>(<>)", vec![If { resource: "".into(), list: vec![Condition::StateToken { is_not: false, token: "", }], }], ), ( r#"( ["I am an ETag"]) (["I am another ETag"])"#, vec![ If { resource: None, list: vec![ Condition::StateToken { is_not: false, token: "urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2", }, Condition::ETag { is_not: false, tag: "\"I am an ETag\"", }, ], }, If { resource: None, list: vec![Condition::ETag { is_not: false, tag: "\"I am another ETag\"", }], }, ], ), ( r#"(Not )"#, vec![If { resource: None, list: vec![ Condition::StateToken { is_not: true, token: "urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2", }, Condition::StateToken { is_not: false, token: "urn:uuid:58f202ac-22cf-11d1-b12d-002035b29092", }, ], }], ), ( r#"() (Not )"#, vec![ If { resource: None, list: vec![Condition::StateToken { is_not: false, token: "urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2", }], }, If { resource: None, list: vec![Condition::StateToken { is_not: true, token: "DAV:no-lock", }], }, ], ), ( r#" ( [W/"A weak ETag"]) (["strong ETag"])"#, vec![ If { resource: "/resource1".into(), list: vec![ Condition::StateToken { is_not: false, token: "urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2", }, Condition::ETag { is_not: false, tag: "W/\"A weak ETag\"", }, ], }, If { resource: None, list: vec![Condition::ETag { is_not: false, tag: "\"strong ETag\"", }], }, ], ), ( r#" ()"#, vec![If { resource: "http://www.example.com/specs/".into(), list: vec![Condition::StateToken { is_not: false, token: "urn:uuid:181d4fae-7d8c-11d0-a765-00a0c91e6bf2", }], }], ), ( r#" (["4217"])"#, vec![If { resource: "/specs/rfc2518.doc".into(), list: vec![Condition::ETag { is_not: false, tag: "\"4217\"", }], }], ), ( r#" (Not ["4217"])"#, vec![If { resource: "/specs/rfc2518.doc".into(), list: vec![Condition::ETag { is_not: true, tag: "\"4217\"", }], }], ), ( r#" (["1234"]) (Not ["4217"])"#, vec![ If { resource: "/test/file.txt".into(), list: vec![Condition::ETag { is_not: false, tag: "\"1234\"", }], }, If { resource: "/specs/rfc2518.doc".into(), list: vec![Condition::ETag { is_not: true, tag: "\"4217\"", }], }, ], ), ] { assert!(headers.parse("If", input)); assert_eq!(headers.if_, expected, "Failed for input: {}", input); headers.if_.clear(); } assert!(headers.parse("If-Match", "*")); assert_eq!( headers.if_, vec![If { resource: None, list: vec![Condition::Exists { is_not: false }], }] ); headers.if_.clear(); assert!(headers.parse("If-None-Match", "etag1, etag2")); assert_eq!( headers.if_, vec![If { resource: None, list: vec![ Condition::ETag { is_not: true, tag: "etag1", }, Condition::ETag { is_not: true, tag: "etag2", } ], },] ); assert!(headers.parse("Timeout", "Second-10")); assert_eq!(headers.timeout, Timeout::Second(10)); assert!(headers.parse("Timeout", "Infinite, Second-4100000000")); assert_eq!(headers.timeout, Timeout::Infinite); assert!(headers.parse("Overwrite", "F")); assert!(headers.overwrite_fail); } } ================================================ FILE: crates/dav-proto/src/parser/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ borrow::Cow, fmt::{Display, Formatter}, }; use quick_xml::events::BytesStart; use tokenizer::Tokenizer; use crate::schema::{Element, NamedElement, Namespace}; pub mod header; pub mod property; pub mod tokenizer; #[derive(Debug, Clone)] pub enum Error { Xml(Box), UnexpectedToken(Box), } #[derive(Debug, Clone)] pub struct UnexpectedToken { pub expected: Option>, pub found: Token<'static>, } pub type Result = std::result::Result; #[derive(Debug, Clone)] pub enum Token<'x> { ElementStart { name: NamedElement, raw: RawElement<'x>, }, ElementEnd, Bytes(Cow<'x, [u8]>), Text(Cow<'x, str>), UnknownElement(RawElement<'x>), Eof, } #[derive(Debug, Clone)] pub struct RawElement<'x> { pub element: BytesStart<'x>, pub namespace: Option>, } pub trait DavParser: Sized { fn parse(stream: &mut Tokenizer<'_>) -> Result; } pub trait XmlValueParser: Sized { fn parse_bytes(bytes: &[u8]) -> Option; fn parse_str(text: &str) -> Option; } impl NamedElement { pub fn dav(element: Element) -> NamedElement { NamedElement { ns: Namespace::Dav, element, } } pub fn caldav(element: Element) -> NamedElement { NamedElement { ns: Namespace::CalDav, element, } } pub fn carddav(element: Element) -> NamedElement { NamedElement { ns: Namespace::CardDav, element, } } pub fn calendarserver(element: Element) -> NamedElement { NamedElement { ns: Namespace::CalendarServer, element, } } } impl Token<'_> { pub fn into_owned(self) -> Token<'static> { match self { Token::ElementStart { name, raw } => Token::ElementStart { name, raw: raw.into_owned(), }, Token::ElementEnd => Token::ElementEnd, Token::Bytes(bytes) => Token::Bytes(bytes.into_owned().into()), Token::Text(text) => Token::Text(text.into_owned().into()), Token::UnknownElement(raw) => Token::UnknownElement(raw.into_owned()), Token::Eof => Token::Eof, } } pub fn into_unexpected(self) -> Error { Error::UnexpectedToken(Box::new(UnexpectedToken { expected: None, found: self.into_owned(), })) } } impl<'x> RawElement<'x> { pub fn new(element: BytesStart<'x>) -> Self { RawElement { element, namespace: None, } } pub fn with_namespace(self, namespace: quick_xml::name::Namespace<'_>) -> Self { RawElement { element: self.element, namespace: Some(Cow::Owned(namespace.into_inner().to_vec())), } } pub fn with_namespace_static(self, namespace: &'static [u8]) -> Self { RawElement { element: self.element, namespace: Some(Cow::Borrowed(namespace)), } } pub fn into_owned(self) -> RawElement<'static> { RawElement { element: self.element.into_owned(), namespace: self.namespace, } } } #[cfg(test)] impl PartialEq for Token<'_> { fn eq(&self, other: &Self) -> bool { match (self, other) { ( Self::ElementStart { name: l_name, raw: l_raw, }, Self::ElementStart { name: r_name, raw: r_raw, }, ) => { l_name == r_name && l_raw .element .attributes_raw() .trim_ascii() .eq_ignore_ascii_case(r_raw.element.attributes_raw().trim_ascii()) } (Self::Bytes(l0), Self::Bytes(r0)) => l0 == r0, (Self::Text(l0), Self::Text(r0)) => l0 == r0, (Self::UnknownElement(l0), Self::UnknownElement(r0)) => l0 .element .as_ref() .eq_ignore_ascii_case(r0.element.as_ref()), _ => core::mem::discriminant(self) == core::mem::discriminant(other), } } } impl NamedElement { pub fn into_unexpected(self) -> Error { Error::UnexpectedToken(Box::new(UnexpectedToken { expected: None, found: Token::ElementStart { name: self, raw: RawElement::new(BytesStart::new("")), }, })) } } impl Default for RawElement<'_> { fn default() -> Self { RawElement::new(BytesStart::new("")) } } impl Display for Error { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { Error::Xml(err) => write!(f, "XML error: {}", err), Error::UnexpectedToken(err) => { write!(f, "Unexpected token: {:?}", err.found)?; if let Some(expected) = &err.expected { write!(f, ", expected: {expected:?}")?; } Ok(()) } } } } ================================================ FILE: crates/dav-proto/src/parser/property.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{DavParser, RawElement, Token, XmlValueParser, tokenizer::Tokenizer}; use crate::schema::{ Attribute, AttributeValue, Element, NamedElement, Namespace, property::{ CalDavProperty, CalDavPropertyName, CalendarData, CardDavProperty, CardDavPropertyName, Comp, DavProperty, DavValue, PrincipalProperty, ResourceType, WebDavProperty, }, request::{DavPropertyValue, VCardPropertyWithGroup}, response::List, }; use calcard::{ Entry, Parser, common::{IanaParse, PartialDateTime}, icalendar::{ICalendar, ICalendarComponentType, ICalendarParameterName, ICalendarProperty}, vcard::{VCardParameterName, VCardProperty}, }; use mail_parser::DateTime; use types::{TimeRange, dead_property::DeadProperty}; impl Tokenizer<'_> { pub(crate) fn collect_properties( &mut self, mut elements: Vec, ) -> crate::parser::Result> { loop { match self.token()? { Token::ElementStart { name: NamedElement { ns: Namespace::CalDav, element: Element::CalendarData, }, .. } => { elements.push(DavProperty::CalDav(CalDavProperty::CalendarData( self.collect_calendar_data()?, ))); } Token::ElementStart { name: NamedElement { ns: Namespace::CardDav, element: Element::AddressData, }, .. } => { elements.push(DavProperty::CardDav(CardDavProperty::AddressData( self.collect_address_data()?, ))); } Token::ElementStart { name, .. } => { if let Some(property) = DavProperty::from_element(name) { elements.push(property); } self.expect_element_end()?; } Token::ElementEnd => { break; } Token::UnknownElement(name) => { elements.push(DavProperty::DeadProperty((&name).into())); self.expect_element_end()?; } token => return Err(token.into_unexpected()), } } Ok(elements) } pub(crate) fn collect_calendar_data(&mut self) -> crate::parser::Result { let mut depth = 1; let mut data = CalendarData { properties: Vec::with_capacity(4), expand: None, limit_recurrence: None, limit_freebusy: None, }; let mut components: Vec = Vec::new(); loop { match self.token()? { Token::ElementStart { name: NamedElement { ns: Namespace::CalDav, element: Element::Allcomp, }, .. } => { self.expect_element_end()?; } Token::ElementStart { name: NamedElement { ns: Namespace::CalDav, element: Element::Allprop, }, .. } => { if let Some(component) = components.last().cloned() { data.properties.push(CalDavPropertyName { component: Some(component), name: None, no_value: false, }); } self.expect_element_end()?; } Token::ElementStart { name: NamedElement { ns: Namespace::CalDav, element: Element::Comp, }, raw, } => { depth += 1; for attribute in raw.attributes::() { if let Attribute::Name(name) = attribute? { components.push(name); } } } Token::ElementStart { name: NamedElement { ns: Namespace::CalDav, element: Element::Prop, }, raw, } => { let mut name = None; let mut no_value = false; for attribute in raw.attributes::() { match attribute? { Attribute::Name(name_) => { name = Some(name_); } Attribute::NoValue(no_value_) => { no_value = no_value_; } _ => {} } } if let Some(name) = name { data.properties.push(CalDavPropertyName { component: components.last().cloned(), name: Some(name), no_value, }); } self.expect_element_end()?; } Token::ElementStart { name: NamedElement { ns: Namespace::CalDav, element: Element::Expand, }, raw, } => { data.expand = TimeRange::from_raw(&raw)?; self.expect_element_end()?; } Token::ElementStart { name: NamedElement { ns: Namespace::CalDav, element: Element::LimitRecurrenceSet, }, raw, } => { data.limit_recurrence = TimeRange::from_raw(&raw)?; self.expect_element_end()?; } Token::ElementStart { name: NamedElement { ns: Namespace::CalDav, element: Element::LimitFreebusySet, }, raw, } => { data.limit_freebusy = TimeRange::from_raw(&raw)?; self.expect_element_end()?; } Token::ElementEnd => { depth -= 1; if depth == 0 { break; } if let Some(last_component) = components.pop() && last_component != ICalendarComponentType::VCalendar && !matches!(data.properties.last(), Some(CalDavPropertyName { component: Some(component), .. }) if component == &last_component) { data.properties.push(CalDavPropertyName { component: Some(last_component), name: None, no_value: false, }); } } Token::Eof => { break; } token => return Err(token.into_unexpected()), } } Ok(data) } pub(crate) fn collect_address_data( &mut self, ) -> crate::parser::Result> { let mut items = Vec::with_capacity(4); loop { match self.token()? { Token::ElementStart { name: NamedElement { ns: Namespace::CardDav, element: Element::Allprop, }, .. } => { self.expect_element_end()?; } Token::ElementStart { name: NamedElement { ns: Namespace::CardDav, element: Element::Prop, }, raw, } => { let mut name = None; let mut group = None; let mut no_value = false; for attribute in raw.attributes::() { match attribute? { Attribute::Name(name_) => { name = Some(name_.name); group = name_.group; } Attribute::NoValue(no_value_) => { no_value = no_value_; } _ => {} } } if let Some(name) = name { items.push(CardDavPropertyName { name, group, no_value, }); } self.expect_element_end()?; } Token::ElementEnd | Token::Eof => { break; } token => return Err(token.into_unexpected()), } } Ok(items) } } impl Tokenizer<'_> { pub(crate) fn collect_property_values( &mut self, elements: &mut Vec, ) -> crate::parser::Result<()> { loop { match self.token()? { Token::ElementStart { name, .. } => { if let Some(property) = DavProperty::from_element(name) { let value = match property { DavProperty::WebDav(WebDavProperty::ResourceType) => { DavValue::ResourceTypes(List(self.collect_elements()?)) } DavProperty::WebDav(WebDavProperty::CreationDate) => { match self.parse_value::()? { Some(Ok(value)) => DavValue::Timestamp(value.to_timestamp()), Some(Err(value)) => DavValue::String(value), None => DavValue::Null, } } DavProperty::CalDav(CalDavProperty::CalendarTimezone) => { match self .collect_string_value()? .map(|v| ICalendar::parse(&v).map_err(|_| v)) { Some(Ok(value)) => DavValue::ICalendar(value), Some(Err(value)) => DavValue::String(value), None => DavValue::Null, } } DavProperty::CalDav(CalDavProperty::SupportedCalendarComponentSet) => { let mut components = Vec::new(); loop { match self.token()? { Token::ElementStart { name, raw } => { if name.ns == Namespace::CalDav && name.element == Element::Comp { for component in raw.attributes::() { if let Attribute::Name(name) = component? { components.push(Comp(name)); } } } self.seek_element_end()?; } Token::UnknownElement(_) => { // Ignore unknown elements self.seek_element_end()?; } Token::ElementEnd | Token::Eof => { break; } _ => {} } } DavValue::Components(List(components)) } DavProperty::CalDav( CalDavProperty::MaxInstances | CalDavProperty::MaxAttendeesPerInstance, ) => match self.parse_value()? { Some(Ok(value)) => DavValue::Uint64(value), Some(Err(value)) => DavValue::String(value), None => DavValue::Null, }, _ => self .collect_string_value()? .map(DavValue::String) .unwrap_or(DavValue::Null), }; elements.push(DavPropertyValue { property, value }); } else { // Ignore unknown elements self.seek_element_end()?; } } Token::ElementEnd | Token::Eof => { break; } Token::UnknownElement(raw) => { elements.push(DavPropertyValue { property: DavProperty::DeadProperty((&raw).into()), value: DavValue::DeadProperty(DeadProperty::parse(self)?), }); } token => return Err(token.into_unexpected()), } } Ok(()) } } pub(crate) trait TimeRangeFromRaw { fn from_raw(raw: &RawElement<'_>) -> super::Result>; } impl TimeRangeFromRaw for TimeRange { fn from_raw(raw: &RawElement<'_>) -> super::Result> { let mut range = TimeRange { start: i64::MIN, end: i64::MAX, }; for attribute in raw.attributes::() { match attribute? { Attribute::Start(start) => { range.start = start.0; } Attribute::End(end) => { range.end = end.0; } _ => {} } } if range.end < range.start { range.end = i64::MAX; } if range.start != i64::MIN || range.end != i64::MAX { Ok(Some(range)) } else { Ok(None) } } } impl DavProperty { pub(crate) fn from_element(element: NamedElement) -> Option { match (element.ns, element.element) { (Namespace::Dav, Element::Creationdate) => { Some(DavProperty::WebDav(WebDavProperty::CreationDate)) } (Namespace::Dav, Element::Displayname) => { Some(DavProperty::WebDav(WebDavProperty::DisplayName)) } (Namespace::Dav, Element::Getcontentlanguage) => { Some(DavProperty::WebDav(WebDavProperty::GetContentLanguage)) } (Namespace::Dav, Element::Getcontentlength) => { Some(DavProperty::WebDav(WebDavProperty::GetContentLength)) } (Namespace::Dav, Element::Getcontenttype) => { Some(DavProperty::WebDav(WebDavProperty::GetContentType)) } (Namespace::Dav, Element::Getetag) => { Some(DavProperty::WebDav(WebDavProperty::GetETag)) } (Namespace::Dav, Element::Getlastmodified) => { Some(DavProperty::WebDav(WebDavProperty::GetLastModified)) } (Namespace::Dav, Element::Resourcetype) => { Some(DavProperty::WebDav(WebDavProperty::ResourceType)) } (Namespace::Dav, Element::Lockdiscovery) => { Some(DavProperty::WebDav(WebDavProperty::LockDiscovery)) } (Namespace::Dav, Element::Supportedlock) => { Some(DavProperty::WebDav(WebDavProperty::SupportedLock)) } (Namespace::Dav, Element::CurrentUserPrincipal) => { Some(DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal)) } (Namespace::Dav, Element::QuotaAvailableBytes) => { Some(DavProperty::WebDav(WebDavProperty::QuotaAvailableBytes)) } (Namespace::Dav, Element::QuotaUsedBytes) => { Some(DavProperty::WebDav(WebDavProperty::QuotaUsedBytes)) } (Namespace::Dav, Element::SupportedReportSet) => { Some(DavProperty::WebDav(WebDavProperty::SupportedReportSet)) } (Namespace::Dav, Element::SyncToken) => { Some(DavProperty::WebDav(WebDavProperty::SyncToken)) } (Namespace::Dav, Element::AlternateUriSet) => { Some(DavProperty::Principal(PrincipalProperty::AlternateURISet)) } (Namespace::Dav, Element::PrincipalUrl) => { Some(DavProperty::Principal(PrincipalProperty::PrincipalURL)) } (Namespace::Dav, Element::GroupMemberSet) => { Some(DavProperty::Principal(PrincipalProperty::GroupMemberSet)) } (Namespace::Dav, Element::GroupMembership) => { Some(DavProperty::Principal(PrincipalProperty::GroupMembership)) } (Namespace::Dav, Element::Owner) => Some(DavProperty::WebDav(WebDavProperty::Owner)), (Namespace::Dav, Element::Group) => Some(DavProperty::WebDav(WebDavProperty::Group)), (Namespace::Dav, Element::SupportedPrivilegeSet) => { Some(DavProperty::WebDav(WebDavProperty::SupportedPrivilegeSet)) } (Namespace::Dav, Element::CurrentUserPrivilegeSet) => { Some(DavProperty::WebDav(WebDavProperty::CurrentUserPrivilegeSet)) } (Namespace::Dav, Element::Acl) => Some(DavProperty::WebDav(WebDavProperty::Acl)), (Namespace::Dav, Element::AclRestrictions) => { Some(DavProperty::WebDav(WebDavProperty::AclRestrictions)) } (Namespace::Dav, Element::InheritedAclSet) => { Some(DavProperty::WebDav(WebDavProperty::InheritedAclSet)) } (Namespace::Dav, Element::PrincipalCollectionSet) => { Some(DavProperty::WebDav(WebDavProperty::PrincipalCollectionSet)) } (Namespace::CardDav, Element::AddressbookDescription) => Some(DavProperty::CardDav( CardDavProperty::AddressbookDescription, )), (Namespace::CardDav, Element::SupportedAddressData) => { Some(DavProperty::CardDav(CardDavProperty::SupportedAddressData)) } (Namespace::CardDav, Element::SupportedCollationSet) => { Some(DavProperty::CardDav(CardDavProperty::SupportedCollationSet)) } (Namespace::CardDav, Element::AddressbookHomeSet) => Some(DavProperty::Principal( PrincipalProperty::AddressbookHomeSet, )), (Namespace::CardDav, Element::PrincipalAddress) => { Some(DavProperty::Principal(PrincipalProperty::PrincipalAddress)) } (Namespace::CardDav, Element::AddressData) => Some(DavProperty::CardDav( CardDavProperty::AddressData(Default::default()), )), (Namespace::CardDav, Element::MaxResourceSize) => { Some(DavProperty::CardDav(CardDavProperty::MaxResourceSize)) } (Namespace::CalDav, Element::CalendarDescription) => { Some(DavProperty::CalDav(CalDavProperty::CalendarDescription)) } (Namespace::CalDav, Element::CalendarTimezone) => { Some(DavProperty::CalDav(CalDavProperty::CalendarTimezone)) } (Namespace::CalDav, Element::SupportedCalendarComponentSet) => Some( DavProperty::CalDav(CalDavProperty::SupportedCalendarComponentSet), ), (Namespace::CalDav, Element::SupportedCollationSet) => { Some(DavProperty::CalDav(CalDavProperty::SupportedCollationSet)) } (Namespace::CalDav, Element::SupportedCalendarData) => { Some(DavProperty::CalDav(CalDavProperty::SupportedCalendarData)) } (Namespace::CalDav, Element::MaxResourceSize) => { Some(DavProperty::CalDav(CalDavProperty::MaxResourceSize)) } (Namespace::CalDav, Element::MinDateTime) => { Some(DavProperty::CalDav(CalDavProperty::MinDateTime)) } (Namespace::CalDav, Element::MaxDateTime) => { Some(DavProperty::CalDav(CalDavProperty::MaxDateTime)) } (Namespace::CalDav, Element::MaxInstances) => { Some(DavProperty::CalDav(CalDavProperty::MaxInstances)) } (Namespace::CalDav, Element::MaxAttendeesPerInstance) => { Some(DavProperty::CalDav(CalDavProperty::MaxAttendeesPerInstance)) } (Namespace::CalDav, Element::ScheduleDefaultCalendarUrl) => Some(DavProperty::CalDav( CalDavProperty::ScheduleDefaultCalendarURL, )), (Namespace::CalDav, Element::ScheduleTag) => { Some(DavProperty::CalDav(CalDavProperty::ScheduleTag)) } (Namespace::CalDav, Element::ScheduleCalendarTransp) => { Some(DavProperty::CalDav(CalDavProperty::ScheduleCalendarTransp)) } (Namespace::CalDav, Element::CalendarHomeSet) => { Some(DavProperty::Principal(PrincipalProperty::CalendarHomeSet)) } (Namespace::CalDav, Element::CalendarUserAddressSet) => Some(DavProperty::Principal( PrincipalProperty::CalendarUserAddressSet, )), (Namespace::CalDav, Element::CalendarUserType) => { Some(DavProperty::Principal(PrincipalProperty::CalendarUserType)) } (Namespace::CalDav, Element::ScheduleInboxUrl) => { Some(DavProperty::Principal(PrincipalProperty::ScheduleInboxURL)) } (Namespace::CalDav, Element::ScheduleOutboxUrl) => { Some(DavProperty::Principal(PrincipalProperty::ScheduleOutboxURL)) } (Namespace::CalDav, Element::CalendarData) => Some(DavProperty::CalDav( CalDavProperty::CalendarData(Default::default()), )), (Namespace::CalDav, Element::TimezoneServiceSet) => { Some(DavProperty::CalDav(CalDavProperty::TimezoneServiceSet)) } (Namespace::CalDav, Element::CalendarTimezoneId) => { Some(DavProperty::CalDav(CalDavProperty::TimezoneId)) } (Namespace::CalendarServer, Element::Getctag) => { Some(DavProperty::WebDav(WebDavProperty::GetCTag)) } _ => None, } } } impl TryFrom for ResourceType { type Error = (); fn try_from(value: NamedElement) -> Result { match (value.ns, value.element) { (Namespace::Dav, Element::Collection) => Ok(ResourceType::Collection), (Namespace::Dav, Element::Principal) => Ok(ResourceType::Principal), (Namespace::CardDav, Element::Addressbook) => Ok(ResourceType::AddressBook), (Namespace::CalDav, Element::Calendar) => Ok(ResourceType::Calendar), (Namespace::CalDav, Element::ScheduleInbox) => Ok(ResourceType::ScheduleInbox), (Namespace::CalDav, Element::ScheduleOutbox) => Ok(ResourceType::ScheduleOutbox), _ => Err(()), } } } struct ICalendarDateTime(i64); impl AttributeValue for ICalendarDateTime { fn from_str(s: &str) -> Option where Self: Sized, { let mut dt = PartialDateTime::default(); dt.parse_timestamp(&mut s.as_bytes().iter().peekable(), true); dt.to_timestamp().map(ICalendarDateTime) } } impl AttributeValue for ICalendarComponentType { fn from_str(s: &str) -> Option where Self: Sized, { ICalendarComponentType::parse(s.as_bytes()) } } impl AttributeValue for ICalendarProperty { fn from_str(s: &str) -> Option where Self: Sized, { ICalendarProperty::parse(s.as_bytes()) .unwrap_or_else(|| ICalendarProperty::Other(s.to_string())) .into() } } impl AttributeValue for ICalendarParameterName { fn from_str(s: &str) -> Option where Self: Sized, { ICalendarParameterName::parse(s).into() } } impl AttributeValue for VCardPropertyWithGroup { fn from_str(s: &str) -> Option where Self: Sized, { if let Some((group, s)) = s.split_once('.') { VCardPropertyWithGroup { name: VCardProperty::parse(s.as_bytes()) .unwrap_or_else(|| VCardProperty::Other(s.to_string())), group: group.to_string().into(), } .into() } else { VCardPropertyWithGroup { name: VCardProperty::parse(s.as_bytes()) .unwrap_or_else(|| VCardProperty::Other(s.to_string())), group: None, } .into() } } } impl AttributeValue for VCardParameterName { fn from_str(s: &str) -> Option where Self: Sized, { VCardParameterName::parse(s).into() } } impl XmlValueParser for ICalendar { fn parse_bytes(bytes: &[u8]) -> Option { let text = String::from_utf8_lossy(bytes); let mut parser = Parser::new(&text); if let Entry::ICalendar(ical) = parser.entry() { Some(ical) } else { None } } fn parse_str(text: &str) -> Option { let mut parser = Parser::new(text); if let Entry::ICalendar(ical) = parser.entry() { Some(ical) } else { None } } } impl XmlValueParser for u64 { fn parse_bytes(bytes: &[u8]) -> Option { std::str::from_utf8(bytes).ok().and_then(|s| s.parse().ok()) } fn parse_str(text: &str) -> Option { text.parse().ok() } } impl XmlValueParser for u32 { fn parse_bytes(bytes: &[u8]) -> Option { std::str::from_utf8(bytes).ok().and_then(|s| s.parse().ok()) } fn parse_str(text: &str) -> Option { text.parse().ok() } } impl XmlValueParser for DateTime { fn parse_bytes(bytes: &[u8]) -> Option { std::str::from_utf8(bytes) .ok() .and_then(DateTime::parse_rfc3339) } fn parse_str(text: &str) -> Option { DateTime::parse_rfc3339(text) } } ================================================ FILE: crates/dav-proto/src/parser/tokenizer.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{Error, RawElement, Token, UnexpectedToken, XmlValueParser}; use crate::schema::{Attribute, AttributeValue, Element, NamedElement, Namespace}; use quick_xml::{ NsReader, events::{Event, attributes::AttrError}, name::ResolveResult, }; pub struct Tokenizer<'x> { xml: NsReader<&'x [u8]>, last_is_end: bool, } impl<'x> Tokenizer<'x> { pub fn new(input: &'x [u8]) -> Self { let mut xml = NsReader::from_reader(input); xml.config_mut(); Self { xml, last_is_end: false, } } pub fn token(&'_ mut self) -> super::Result> { loop { if self.last_is_end { self.last_is_end = false; return Ok(Token::ElementEnd); } let (resolve_result, event) = self.xml.read_resolved_event()?; let tag = match event { Event::Start(tag) => tag, Event::Empty(tag) => { self.last_is_end = true; tag } Event::End(_) => { return Ok(Token::ElementEnd); } Event::Text(text) if text.iter().any(|ch| !ch.is_ascii_whitespace()) => { return text .xml_content() .map(Token::Text) .map_err(|err| Error::Xml(Box::new(err.into()))); } Event::GeneralRef(entity) => { hashify::fnc_map!(entity.as_ref(), b"lt" => { return Ok(Token::Text("<".into())); }, b"gt" => { return Ok(Token::Text(">".into())); }, b"amp" => { return Ok(Token::Text("&".into())); }, b"apos" => { return Ok(Token::Text("'".into())); }, b"quot" => { return Ok(Token::Text("\"".into())); }, _ => { if let Ok(Some(gr)) = entity.resolve_char_ref() { return Ok(Token::Text(gr.to_string().into())); } } ); return entity .xml_content() .map(Token::Text) .map_err(|err| Error::Xml(Box::new(err.into()))); } Event::CData(bytes) => return Ok(Token::Bytes(bytes.into_inner())), Event::Eof => return Ok(Token::Eof), _ => { continue; } }; // Parse element let name = tag.name(); match resolve_result { ResolveResult::Bound(raw_ns) if !raw_ns.as_ref().is_empty() => { if let (Some(ns), Some(element)) = ( Namespace::try_parse(raw_ns.as_ref()), Element::try_parse(name.local_name().as_ref()).copied(), ) { return Ok(Token::ElementStart { name: NamedElement { ns, element }, raw: RawElement::new(tag) .with_namespace_static(ns.namespace().as_bytes()), }); } else { return Ok(Token::UnknownElement( RawElement::new(tag).with_namespace(raw_ns), )); } } ResolveResult::Unknown(p) => { return Err(Error::Xml(Box::new(quick_xml::Error::Namespace( quick_xml::name::NamespaceError::UnknownPrefix(p), )))); } _ => { return Ok(Token::UnknownElement(RawElement::new(tag))); } } } } pub fn unwrap_named_element(&mut self) -> super::Result { match self.token()? { Token::ElementStart { name, .. } => Ok(name), found => Err(Error::UnexpectedToken(Box::new(UnexpectedToken { expected: None, found: found.into_owned(), }))), } } pub fn expect_named_element(&mut self, expected: NamedElement) -> super::Result<()> { match self.token()? { Token::ElementStart { name, .. } if name == expected => Ok(()), found => Err(Error::UnexpectedToken(Box::new(UnexpectedToken { expected: Token::ElementStart { name: expected, raw: RawElement::default(), } .into(), found: found.into_owned(), }))), } } pub fn expect_named_element_or_eof(&mut self, expected: NamedElement) -> super::Result { match self.token()? { Token::ElementStart { name, .. } if name == expected => Ok(true), Token::Eof => Ok(false), found => Err(Error::UnexpectedToken(Box::new(UnexpectedToken { expected: Token::ElementStart { name: expected, raw: RawElement::default(), } .into(), found: found.into_owned(), }))), } } pub fn expect_element_end(&mut self) -> super::Result<()> { match self.token()? { Token::ElementEnd => Ok(()), found => Err(Error::UnexpectedToken(Box::new(UnexpectedToken { expected: Token::ElementEnd.into(), found: found.into_owned(), }))), } } pub fn seek_element_end(&mut self) -> super::Result<()> { let mut depth = 1; loop { match self.token()? { Token::ElementStart { .. } | Token::UnknownElement(_) => depth += 1, Token::ElementEnd => { depth -= 1; if depth == 0 { return Ok(()); } } Token::Eof => return Err(Token::Eof.into_unexpected()), _ => {} } } } pub fn collect_string_value(&mut self) -> super::Result> { let mut depth = 1; let mut value: Option = None; loop { match self.token()? { Token::ElementStart { .. } | Token::UnknownElement(_) => depth += 1, Token::ElementEnd => { depth -= 1; if depth == 0 { break; } } Token::Text(text) => { if let Some(ref mut v) = value { v.push_str(&text); } else { value = Some(text.into_owned()); } } Token::Bytes(bytes) => { if let Some(ref mut v) = value { v.push_str(&String::from_utf8_lossy(&bytes)); } else { value = Some(String::from_utf8_lossy(&bytes).into_owned()); } } Token::Eof => return Err(Token::Eof.into_unexpected()), } } Ok(value) } pub fn parse_value(&mut self) -> super::Result>> { let mut depth = 1; let mut result: Option> = None; loop { match self.token()? { Token::ElementStart { .. } | Token::UnknownElement(_) => depth += 1, Token::ElementEnd => { depth -= 1; if depth == 0 { break; } } Token::Text(text) => { if let Some(value) = T::parse_str(&text) { result = Some(Ok(value)); } else { result = Some(Err(text.into_owned())); } } Token::Bytes(bytes) => { if let Some(value) = T::parse_bytes(&bytes) { result = Some(Ok(value)); } else { result = Some(Err(String::from_utf8_lossy(&bytes).into_owned())); } } Token::Eof => return Err(Token::Eof.into_unexpected()), } } Ok(result) } pub fn collect_elements(&mut self) -> super::Result> where T: TryFrom, { let mut elements = Vec::with_capacity(2); let mut depth = 1; loop { match self.token()? { Token::ElementStart { name, .. } => { if depth == 1 && let Ok(element) = T::try_from(name) { elements.push(element); } depth += 1; } Token::UnknownElement(_) => { depth += 1; } Token::ElementEnd => { depth -= 1; if depth == 0 { break; } } Token::Eof => break, _ => {} } } Ok(elements) } } impl RawElement<'_> { pub fn attributes( &self, ) -> impl Iterator>> + '_ { self.element.attributes().filter_map(|attr| match attr { Ok(attr) => match attr.unescape_value() { Ok(value) => Attribute::from_param(attr.key.as_ref(), value).map(Ok), Err(err) => Some(Err(err.into())), }, Err(err) => Some(Err(err.into())), }) } } impl From for Error { fn from(err: quick_xml::Error) -> Self { Error::Xml(Box::new(err)) } } impl From for Error { fn from(err: AttrError) -> Self { Error::Xml(Box::new(err.into())) } } #[cfg(test)] mod tests { use std::borrow::Cow; use crate::schema::{Collation, MatchType}; use super::*; #[derive(Debug, PartialEq, Eq)] pub enum TestToken<'x> { ElementStart(NamedElement), ElementEnd, Attribute(Attribute), Bytes(Cow<'x, [u8]>), Text(Cow<'x, str>), } #[test] fn test_tokenizer() { for (input, expected) in [ ( r#" "#, vec![ TestToken::ElementStart(NamedElement { ns: Namespace::CalDav, element: Element::CalendarQuery, }), TestToken::ElementStart(NamedElement { ns: Namespace::Dav, element: Element::Prop, }), TestToken::ElementStart(NamedElement { ns: Namespace::Dav, element: Element::Getetag, }), TestToken::ElementEnd, TestToken::ElementStart(NamedElement { ns: Namespace::CalDav, element: Element::CalendarData, }), TestToken::ElementEnd, TestToken::ElementEnd, TestToken::ElementStart(NamedElement { ns: Namespace::CalDav, element: Element::Filter, }), TestToken::ElementStart(NamedElement { ns: Namespace::CalDav, element: Element::CompFilter, }), TestToken::Attribute(Attribute::Name("VCALENDAR".to_string())), TestToken::ElementEnd, TestToken::ElementEnd, TestToken::ElementEnd, ], ), ( r#" me "#, vec![ TestToken::ElementStart(NamedElement { ns: Namespace::CardDav, element: Element::AddressbookQuery, }), TestToken::ElementStart(NamedElement { ns: Namespace::Dav, element: Element::Prop, }), TestToken::ElementStart(NamedElement { ns: Namespace::Dav, element: Element::Getetag, }), TestToken::ElementEnd, TestToken::ElementStart(NamedElement { ns: Namespace::CardDav, element: Element::AddressData, }), TestToken::ElementStart(NamedElement { ns: Namespace::CardDav, element: Element::Prop, }), TestToken::Attribute(Attribute::Name("VERSION".to_string())), TestToken::ElementEnd, TestToken::ElementStart(NamedElement { ns: Namespace::CardDav, element: Element::Prop, }), TestToken::Attribute(Attribute::Name("UID".to_string())), TestToken::ElementEnd, TestToken::ElementStart(NamedElement { ns: Namespace::CardDav, element: Element::Prop, }), TestToken::Attribute(Attribute::Name("NICKNAME".to_string())), TestToken::ElementEnd, TestToken::ElementStart(NamedElement { ns: Namespace::CardDav, element: Element::Prop, }), TestToken::Attribute(Attribute::Name("EMAIL".to_string())), TestToken::ElementEnd, TestToken::ElementStart(NamedElement { ns: Namespace::CardDav, element: Element::Prop, }), TestToken::Attribute(Attribute::Name("FN".to_string())), TestToken::ElementEnd, TestToken::ElementEnd, TestToken::ElementEnd, TestToken::ElementStart(NamedElement { ns: Namespace::CardDav, element: Element::Filter, }), TestToken::ElementStart(NamedElement { ns: Namespace::CardDav, element: Element::PropFilter, }), TestToken::Attribute(Attribute::Name("NICKNAME".to_string())), TestToken::ElementStart(NamedElement { ns: Namespace::CardDav, element: Element::TextMatch, }), TestToken::Attribute(Attribute::Collation(Collation::UnicodeCasemap)), TestToken::Attribute(Attribute::MatchType(MatchType::Equals)), TestToken::Text("me".into()), TestToken::ElementEnd, TestToken::ElementEnd, TestToken::ElementEnd, TestToken::ElementEnd, ], ), ] { let mut tokenizer = Tokenizer::new(input.as_bytes()); let mut result = vec![]; loop { match tokenizer.token() { Ok(token) => match token { Token::ElementStart { name, raw } => { result.push(TestToken::ElementStart(name)); for attr in raw.attributes::() { result.push(TestToken::Attribute(attr.unwrap())); } } Token::ElementEnd => { result.push(TestToken::ElementEnd); } Token::Bytes(cow) => { result.push(TestToken::Bytes(cow.into_owned().into())); } Token::Text(cow) => { result.push(TestToken::Text(cow.into_owned().into())); } Token::UnknownElement(_) => { //result.push(TestToken::UnknownElement(unknown_element)); } Token::Eof => break, }, Err(err) => { panic!("Error: {:?}", err); } } } assert_eq!(result, expected); } } } ================================================ FILE: crates/dav-proto/src/requests/acl.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ parser::{tokenizer::Tokenizer, DavParser, Token}, schema::{ property::{DavValue, Privilege}, request::{ Acl, AclPrincipalPropSet, DavPropertyValue, PrincipalMatch, PrincipalMatchProperties, PrincipalPropertySearch, PropertySearch, }, response::{Ace, GrantDeny, Href, List, Principal}, Element, NamedElement, Namespace, }, }; impl DavParser for Acl { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { stream.expect_named_element(NamedElement::dav(Element::Acl))?; let mut acl = Acl { aces: vec![] }; loop { match stream.token()? { Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Ace, }, .. } => { acl.aces.push(Ace::parse(stream)?); } Token::ElementEnd => { break; } Token::UnknownElement(_) => { stream.seek_element_end()?; } other => { return Err(other.into_unexpected()); } } } Ok(acl) } } impl DavParser for Ace { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { let mut ace = Ace { principal: Principal::All, invert: false, grant_deny: GrantDeny::Grant(List(vec![])), protected: false, inherited: None, }; let mut depth = 1; loop { match stream.token()? { Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Principal, }, .. } => { ace.principal = Principal::parse(stream)?; } Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Invert, }, .. } if depth == 1 => { ace.invert = true; depth += 1; } Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Protected, }, .. } if depth == 1 => { ace.protected = true; stream.expect_element_end()?; } Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Inherited, }, .. } if depth == 1 => { stream.expect_named_element(NamedElement::dav(Element::Href))?; ace.inherited = stream.collect_string_value()?.map(Href); stream.expect_element_end()?; } Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Grant, }, .. } if depth == 1 => { ace.grant_deny = GrantDeny::Grant(List(stream.collect_privileges()?)); } Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Deny, }, .. } if depth == 1 => { ace.grant_deny = GrantDeny::Deny(List(stream.collect_privileges()?)); } Token::ElementEnd => { depth -= 1; if depth == 0 { break; } } Token::UnknownElement(_) => { stream.seek_element_end()?; } other => { return Err(other.into_unexpected()); } } } Ok(ace) } } impl DavParser for Principal { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { let result = match stream.unwrap_named_element()? { NamedElement { ns: Namespace::Dav, element: Element::Href, } => Principal::Href(Href(stream.collect_string_value()?.unwrap_or_default())), NamedElement { ns: Namespace::Dav, element: Element::All, } => { stream.expect_element_end()?; Principal::All } NamedElement { ns: Namespace::Dav, element: Element::Authenticated, } => { stream.expect_element_end()?; Principal::Authenticated } NamedElement { ns: Namespace::Dav, element: Element::Unauthenticated, } => { stream.expect_element_end()?; Principal::Unauthenticated } NamedElement { ns: Namespace::Dav, element: Element::Property, } => { let property = stream.collect_properties(Vec::new())?; Principal::Property(List( property .into_iter() .map(|prop| DavPropertyValue::new(prop, DavValue::Null)) .collect(), )) } NamedElement { ns: Namespace::Dav, element: Element::Self_, } => { stream.expect_element_end()?; Principal::Self_ } other => return Err(other.into_unexpected()), }; stream.expect_element_end()?; Ok(result) } } impl Tokenizer<'_> { pub fn collect_privileges(&mut self) -> crate::parser::Result> { let mut privileges = Vec::new(); let mut depth = 1; loop { match self.token()? { Token::ElementStart { name, .. } => { if let Some(privilege) = Privilege::from_element(name) { privileges.push(privilege); self.expect_element_end()?; } else { depth += 1; } } Token::ElementEnd => { depth -= 1; if depth == 0 { break; } } Token::UnknownElement(_) => { self.seek_element_end()?; } other => { return Err(other.into_unexpected()); } } } Ok(privileges) } } impl Privilege { pub fn from_element(element: NamedElement) -> Option { match (element.ns, element.element) { (Namespace::Dav, Element::Read) => Some(Privilege::Read), (Namespace::Dav, Element::Write) => Some(Privilege::Write), (Namespace::Dav, Element::WriteProperties) => Some(Privilege::WriteProperties), (Namespace::Dav, Element::WriteContent) => Some(Privilege::WriteContent), (Namespace::Dav, Element::Unlock) => Some(Privilege::Unlock), (Namespace::Dav, Element::ReadAcl) => Some(Privilege::ReadAcl), (Namespace::Dav, Element::ReadCurrentUserPrivilegeSet) => { Some(Privilege::ReadCurrentUserPrivilegeSet) } (Namespace::Dav, Element::WriteAcl) => Some(Privilege::WriteAcl), (Namespace::Dav, Element::Bind) => Some(Privilege::Bind), (Namespace::Dav, Element::Unbind) => Some(Privilege::Unbind), (Namespace::Dav, Element::All) => Some(Privilege::All), (Namespace::CalDav, Element::ReadFreeBusy) => Some(Privilege::ReadFreeBusy), (Namespace::CalDav, Element::ScheduleDeliver) => Some(Privilege::ScheduleDeliver), (Namespace::CalDav, Element::ScheduleDeliverInvite) => { Some(Privilege::ScheduleDeliverInvite) } (Namespace::CalDav, Element::ScheduleDeliverReply) => { Some(Privilege::ScheduleDeliverReply) } (Namespace::CalDav, Element::ScheduleQueryFreebusy) => { Some(Privilege::ScheduleQueryFreeBusy) } (Namespace::CalDav, Element::ScheduleSend) => Some(Privilege::ScheduleSend), (Namespace::CalDav, Element::ScheduleSendInvite) => Some(Privilege::ScheduleSendInvite), (Namespace::CalDav, Element::ScheduleSendReply) => Some(Privilege::ScheduleSendReply), (Namespace::CalDav, Element::ScheduleSendFreebusy) => { Some(Privilege::ScheduleSendFreeBusy) } _ => None, } } } impl DavParser for AclPrincipalPropSet { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { let mut acps = AclPrincipalPropSet { properties: vec![] }; loop { match stream.token()? { Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Prop, }, .. } => { acps.properties = stream.collect_properties(acps.properties)?; } Token::ElementEnd => { break; } Token::UnknownElement(_) => { stream.seek_element_end()?; } other => { return Err(other.into_unexpected()); } } } Ok(acps) } } impl DavParser for PrincipalMatch { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { let mut pm = PrincipalMatch { principal_properties: PrincipalMatchProperties::Self_, properties: vec![], }; loop { match stream.token()? { Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::PrincipalProperty, }, .. } => { pm.principal_properties = PrincipalMatchProperties::Properties( stream.collect_properties(Vec::new())?, ); } Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Self_, }, .. } => { pm.principal_properties = PrincipalMatchProperties::Self_; stream.expect_element_end()?; } Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Prop, }, .. } => { pm.properties = stream.collect_properties(pm.properties)?; } Token::ElementEnd => { break; } Token::UnknownElement(_) => { stream.seek_element_end()?; } other => { return Err(other.into_unexpected()); } } } Ok(pm) } } impl DavParser for PrincipalPropertySearch { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { let mut pps = PrincipalPropertySearch { property_search: vec![], properties: vec![], apply_to_principal_collection_set: false, }; loop { match stream.token()? { Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::PropertySearch, }, .. } => { if let Some(prop) = PropertySearch::parse(stream)? { pps.property_search.push(prop); } } Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Prop, }, .. } => { pps.properties = stream.collect_properties(pps.properties)?; } Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::ApplyToPrincipalCollectionSet, }, .. } => { stream.expect_element_end()?; pps.apply_to_principal_collection_set = true; } Token::ElementEnd => { break; } Token::UnknownElement(_) => { stream.seek_element_end()?; } other => { return Err(other.into_unexpected()); } } } Ok(pps) } } impl PropertySearch { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result> { let mut property = None; let mut match_ = None; loop { match stream.token()? { Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Prop, }, .. } => { property = stream.collect_properties(Vec::new())?.into_iter().next(); } Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Match, }, .. } => { match_ = stream.collect_string_value()?; } Token::ElementEnd => { break; } Token::UnknownElement(_) => { stream.seek_element_end()?; } other => { return Err(other.into_unexpected()); } } } Ok(property.map(|property| PropertySearch { property, match_: match_.unwrap_or_default(), })) } } ================================================ FILE: crates/dav-proto/src/requests/lockinfo.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use types::dead_property::DeadProperty; use crate::{ parser::{tokenizer::Tokenizer, DavParser, Token}, schema::{ property::{LockScope, LockType}, request::LockInfo, Element, NamedElement, Namespace, }, }; impl DavParser for LockInfo { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { let mut lockinfo = LockInfo { lock_scope: LockScope::Exclusive, lock_type: LockType::Write, owner: None, }; if stream.expect_named_element_or_eof(NamedElement::dav(Element::Lockinfo))? { loop { match stream.token()? { Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Lockscope, }, .. } => { lockinfo.lock_scope = LockScope::parse(stream)?; } Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Locktype, }, .. } => { lockinfo.lock_type = LockType::parse(stream)?; } Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Owner, }, .. } => { lockinfo.owner = Some(DeadProperty::parse(stream)?); } Token::ElementEnd | Token::Eof => { break; } other => { return Err(other.into_unexpected()); } } } } Ok(lockinfo) } } impl DavParser for LockScope { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { match stream.unwrap_named_element()? { NamedElement { ns: Namespace::Dav, element: Element::Exclusive, } => { stream.expect_element_end()?; stream.expect_element_end()?; Ok(LockScope::Exclusive) } NamedElement { ns: Namespace::Dav, element: Element::Shared, } => { stream.expect_element_end()?; stream.expect_element_end()?; Ok(LockScope::Shared) } other => Err(other.into_unexpected()), } } } impl DavParser for LockType { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { match stream.unwrap_named_element()? { NamedElement { ns: Namespace::Dav, element: Element::Write, } => { stream.expect_element_end()?; stream.expect_element_end()?; Ok(LockType::Write) } other => Err(other.into_unexpected()), } } } ================================================ FILE: crates/dav-proto/src/requests/mkcol.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ parser::{tokenizer::Tokenizer, DavParser, Token}, schema::{request::MkCol, Element, NamedElement, Namespace}, }; impl DavParser for MkCol { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { let mut mkcol = MkCol { is_mkcalendar: false, props: Vec::new(), }; match stream.token()? { Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Mkcol, }, .. } => {} Token::ElementStart { name: NamedElement { ns: Namespace::CalDav, element: Element::Mkcalendar, }, .. } => { mkcol.is_mkcalendar = true; } Token::Eof => { return Ok(mkcol); } other => return Err(other.into_unexpected()), }; loop { match stream.token()? { Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Set, }, .. } => { stream.expect_named_element(NamedElement::dav(Element::Prop))?; stream.collect_property_values(&mut mkcol.props)?; stream.expect_element_end()?; } Token::ElementEnd | Token::Eof => { break; } token => return Err(token.into_unexpected()), } } Ok(mkcol) } } ================================================ FILE: crates/dav-proto/src/requests/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ parser::{tokenizer::Tokenizer, DavParser, RawElement, Token}, schema::Namespace, }; use types::dead_property::{DeadElementTag, DeadProperty, DeadPropertyTag}; pub mod acl; pub mod lockinfo; pub mod mkcol; pub mod propertyupdate; pub mod propfind; pub mod report; impl DavParser for DeadProperty { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { let mut depth = 1; let mut items = DeadProperty::default(); loop { match stream.token()? { Token::ElementStart { raw, .. } | Token::UnknownElement(raw) => { items.0.push(DeadPropertyTag::ElementStart((&raw).into())); depth += 1; } Token::ElementEnd => { depth -= 1; if depth == 0 { break; } items.0.push(DeadPropertyTag::ElementEnd); } Token::Text(text) => { items.0.push(DeadPropertyTag::Text(text.into_owned())); } Token::Bytes(bytes) => { items.0.push(DeadPropertyTag::Text( String::from_utf8_lossy(&bytes).into_owned(), )); } Token::Eof => { break; } } } Ok(items) } } pub trait NsDeadProperty { fn single_with_ns(namespace: Namespace, name: &str) -> Self; } impl NsDeadProperty for DeadProperty { fn single_with_ns(namespace: Namespace, name: &str) -> Self { DeadProperty(vec![ DeadPropertyTag::ElementStart(DeadElementTag { name: format!("{}:{name}", namespace.prefix()), attrs: None, }), DeadPropertyTag::ElementEnd, ]) } } impl From<&RawElement<'_>> for DeadElementTag { fn from(raw: &RawElement<'_>) -> Self { let name = std::str::from_utf8(raw.element.local_name().as_ref()) .unwrap_or("invalid-utf8") .trim_ascii() .to_string(); let mut attrs = String::with_capacity(raw.element.attributes_raw().len()); if let Some(namespace) = &raw.namespace { attrs.push_str("xmlns=\""); attrs.push_str(std::str::from_utf8(namespace).unwrap_or("invalid-utf8")); attrs.push('"'); } for attr in raw.element.attributes().flatten() { if attr.key.as_ref() == b"xmlns" || attr.key.as_ref().starts_with(b"xmlns:") { // Skip namespace attributes continue; } if let (Ok(key), Ok(value)) = ( std::str::from_utf8(attr.key.as_ref()), std::str::from_utf8(attr.value.as_ref()), ) { if !attrs.is_empty() { attrs.push(' '); } attrs.push_str(key); attrs.push('='); attrs.push('"'); attrs.push_str(value); attrs.push('"'); } } DeadElementTag { name, attrs: (!attrs.is_empty()).then_some(attrs), } } } #[cfg(test)] mod tests { use crate::{ parser::{tokenizer::Tokenizer, DavParser}, schema::request::{Acl, LockInfo, MkCol, PropFind, PropertyUpdate, Report}, }; #[test] fn parse_requests() { for entry in std::fs::read_dir("resources/requests").unwrap() { let entry = entry.unwrap(); let path = entry.path(); if path.extension().map(|ext| ext == "xml").unwrap_or(false) { println!("Parsing: {:?}", path); let filename = path.file_name().unwrap().to_str().unwrap(); let xml = std::fs::read_to_string(&path).unwrap(); let mut tokenizer = Tokenizer::new(xml.as_bytes()); let json_path = path.with_extension("json"); let json_output = match filename.split_once('-').unwrap().0 { "propfind" => match PropFind::parse(&mut tokenizer) { Ok(propfind) => serde_json::to_string_pretty(&propfind).unwrap(), Err(_) => String::new(), }, "propertyupdate" => serde_json::to_string_pretty( &PropertyUpdate::parse(&mut tokenizer).unwrap(), ) .unwrap(), "mkcol" => serde_json::to_string_pretty(&MkCol::parse(&mut tokenizer).unwrap()) .unwrap(), "lockinfo" => { serde_json::to_string_pretty(&LockInfo::parse(&mut tokenizer).unwrap()) .unwrap() } "report" => { serde_json::to_string_pretty(&Report::parse(&mut tokenizer).unwrap()) .unwrap() } "acl" => { serde_json::to_string_pretty(&Acl::parse(&mut tokenizer).unwrap()).unwrap() } _ => { panic!("Unknown method: {}", filename); } }; /*if json_path.exists() { let expected = std::fs::read_to_string(json_path).unwrap(); assert_eq!(json_output, expected); } else {*/ std::fs::write(json_path, json_output).unwrap(); //} } } } } ================================================ FILE: crates/dav-proto/src/requests/propertyupdate.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ parser::{tokenizer::Tokenizer, DavParser, Token}, schema::{request::PropertyUpdate, Element, NamedElement, Namespace}, }; impl DavParser for PropertyUpdate { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { stream.expect_named_element(NamedElement::dav(Element::Propertyupdate))?; let mut update = PropertyUpdate { set: Vec::with_capacity(4), remove: Vec::with_capacity(4), set_first: true, }; loop { match stream.token()? { Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Set, }, .. } => { stream.expect_named_element(NamedElement::dav(Element::Prop))?; stream.collect_property_values(&mut update.set)?; stream.expect_element_end()?; update.set_first = update.remove.is_empty(); } Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Remove, }, .. } => { stream.expect_named_element(NamedElement::dav(Element::Prop))?; update.remove = stream.collect_properties(update.remove)?; stream.expect_element_end()?; } Token::ElementEnd | Token::Eof => { break; } Token::UnknownElement(_) => { // Ignore unknown elements stream.seek_element_end()?; } token => return Err(token.into_unexpected()), } } Ok(update) } } ================================================ FILE: crates/dav-proto/src/requests/propfind.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ parser::{tokenizer::Tokenizer, DavParser, Token}, schema::{request::PropFind, Element, NamedElement, Namespace}, }; impl DavParser for PropFind { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { if stream.expect_named_element_or_eof(NamedElement::dav(Element::Propfind))? { match stream.unwrap_named_element()? { NamedElement { ns: Namespace::Dav, element: Element::Propname, } => Ok(PropFind::PropName), NamedElement { ns: Namespace::Dav, element: Element::Allprop, } => { stream.expect_element_end()?; if matches!( stream.token()?, Token::ElementStart { name: NamedElement { ns: Namespace::Dav, element: Element::Include }, .. } ) { stream.collect_properties(Vec::new()).map(PropFind::AllProp) } else { Ok(PropFind::AllProp(vec![])) } } NamedElement { ns: Namespace::Dav, element: Element::Prop, } => stream.collect_properties(Vec::new()).map(PropFind::Prop), element => Err(element.into_unexpected()), } } else { Ok(PropFind::AllProp(vec![])) } } } ================================================ FILE: crates/dav-proto/src/requests/report.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ parser::{ property::TimeRangeFromRaw, tokenizer::Tokenizer, DavParser, RawElement, Token, XmlValueParser, }, schema::{ property::DavProperty, request::{ AclPrincipalPropSet, AddressbookQuery, CalendarQuery, ExpandProperty, ExpandPropertyItem, Filter, FilterOp, FreeBusyQuery, MultiGet, PrincipalMatch, PrincipalPropertySearch, PropFind, Report, SyncCollection, TextMatch, Timezone, VCardPropertyWithGroup, }, Attribute, Collation, Element, MatchType, NamedElement, Namespace, }, Depth, }; use calcard::{ icalendar::{ICalendarComponentType, ICalendarParameterName, ICalendarProperty}, vcard::VCardParameterName, }; use types::{dead_property::DeadElementTag, TimeRange}; impl DavParser for Report { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { match stream.unwrap_named_element()? { NamedElement { ns: Namespace::CalDav, element: Element::CalendarQuery, } => CalendarQuery::parse(stream).map(Report::CalendarQuery), NamedElement { ns: Namespace::CalDav, element: Element::FreeBusyQuery, } => FreeBusyQuery::parse(stream).map(Report::FreeBusyQuery), NamedElement { ns: Namespace::CalDav, element: Element::CalendarMultiget, } => MultiGet::parse(stream).map(Report::CalendarMultiGet), NamedElement { ns: Namespace::CardDav, element: Element::AddressbookQuery, } => AddressbookQuery::parse(stream).map(Report::AddressbookQuery), NamedElement { ns: Namespace::CardDav, element: Element::AddressbookMultiget, } => MultiGet::parse(stream).map(Report::AddressbookMultiGet), NamedElement { ns: Namespace::Dav, element: Element::SyncCollection, } => SyncCollection::parse(stream).map(Report::SyncCollection), NamedElement { ns: Namespace::Dav, element: Element::AclPrincipalPropSet, } => AclPrincipalPropSet::parse(stream).map(Report::AclPrincipalPropSet), NamedElement { ns: Namespace::Dav, element: Element::PrincipalMatch, } => PrincipalMatch::parse(stream).map(Report::PrincipalMatch), NamedElement { ns: Namespace::Dav, element: Element::PrincipalPropertySearch, } => PrincipalPropertySearch::parse(stream).map(Report::PrincipalPropertySearch), NamedElement { ns: Namespace::Dav, element: Element::PrincipalSearchPropertySet, } => stream .expect_element_end() .map(|_| Report::PrincipalSearchPropertySet), NamedElement { ns: Namespace::Dav, element: Element::ExpandProperty, } => ExpandProperty::parse(stream).map(Report::ExpandProperty), other => Err(other.into_unexpected()), } } } impl DavParser for CalendarQuery { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { let mut cq = CalendarQuery { properties: PropFind::AllProp(vec![]), filters: vec![], timezone: Timezone::None, }; let mut depth = 1; let mut components = Vec::with_capacity(3); let mut property = None; let mut parameter = None; loop { match stream.token()? { Token::ElementStart { name, raw } => match name { NamedElement { ns: Namespace::Dav, element: Element::Propname, } if depth == 1 => { cq.properties = PropFind::PropName; stream.expect_element_end()?; } NamedElement { ns: Namespace::Dav, element: Element::Allprop, } if depth == 1 => { stream.expect_element_end()?; } NamedElement { ns: Namespace::Dav, element: Element::Prop, } if depth == 1 => { cq.properties = PropFind::Prop(stream.collect_properties(Vec::new())?); } NamedElement { ns: Namespace::CalDav, element: Element::Filter, } if depth == 1 => { depth += 1; } NamedElement { ns: Namespace::CalDav, element: Element::Timezone, } if depth == 1 => { cq.timezone = Timezone::Name(stream.collect_string_value()?.unwrap_or_default()); } NamedElement { ns: Namespace::CalDav, element: Element::TimezoneId, } if depth == 1 => { cq.timezone = Timezone::Id(stream.collect_string_value()?.unwrap_or_default()); } NamedElement { ns: Namespace::CalDav, element: Element::CompFilter, } if depth >= 2 => { for attribute in raw.attributes::() { if let Attribute::Name(name) = attribute? { components.push((name, depth)); } } depth += 1; } NamedElement { ns: Namespace::CalDav, element: Element::PropFilter, } if depth >= 3 => { for attribute in raw.attributes::() { if let Attribute::Name(name) = attribute? { property = Some(name); } } depth += 1; } NamedElement { ns: Namespace::CalDav, element: Element::ParamFilter, } if depth >= 4 => { for attribute in raw.attributes::() { if let Attribute::Name(name) = attribute? { parameter = Some(name); } } depth += 1; } NamedElement { ns: Namespace::CalDav, element: Element::IsNotDefined, } => { stream.expect_element_end()?; if let Some(filter) = Filter::from_parts( components.iter().map(|(c, _)| c.clone()).collect(), property.clone(), parameter.clone(), FilterOp::Undefined, ) { cq.filters.push(filter); } } NamedElement { ns: Namespace::CalDav, element: Element::TextMatch, } => { let mut tm = TextMatch::parse(raw)?; tm.value = stream.collect_string_value()?.unwrap_or_default(); if let Some(filter) = Filter::from_parts( components.iter().map(|(c, _)| c.clone()).collect(), property.clone(), parameter.clone(), FilterOp::TextMatch(tm), ) { cq.filters.push(filter); } } NamedElement { ns: Namespace::CalDav, element: Element::TimeRange, } => { let range = TimeRange::from_raw(&raw)?; stream.expect_element_end()?; if let Some(filter) = range.and_then(|range| { Filter::from_parts( components.iter().map(|(c, _)| c.clone()).collect(), property.clone(), parameter.clone(), FilterOp::TimeRange(range), ) }) { cq.filters.push(filter); } } name => return Err(name.into_unexpected()), }, Token::ElementEnd => { depth -= 1; if depth == 0 { break; } if matches!(components.last(), Some((_, d)) if *d == depth) { if components.len() > 1 && cq .filters .last() .and_then(|c| c.components()) .is_none_or(|c| c.len() < components.len()) { cq.filters.push(Filter::Component { comp: components.iter().map(|(c, _)| c.clone()).collect(), op: FilterOp::Exists, }); } components.pop(); } } Token::UnknownElement(_) => { stream.seek_element_end()?; } element => return Err(element.into_unexpected()), } } Ok(cq) } } impl DavParser for AddressbookQuery { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { let mut aq = AddressbookQuery { properties: PropFind::AllProp(vec![]), filters: vec![], limit: None, }; let mut depth = 1; let mut property = None; let mut parameter = None; loop { match stream.token()? { Token::ElementStart { name, raw } => match name { NamedElement { ns: Namespace::Dav, element: Element::Propname, } if depth == 1 => { aq.properties = PropFind::PropName; stream.expect_element_end()?; } NamedElement { ns: Namespace::Dav, element: Element::Allprop, } if depth == 1 => { stream.expect_element_end()?; } NamedElement { ns: Namespace::Dav, element: Element::Prop, } if depth == 1 => { aq.properties = PropFind::Prop(stream.collect_properties(Vec::new())?); } NamedElement { ns: Namespace::CardDav, element: Element::Filter, } if depth == 1 => { if let Some(filter) = Filter::parse(raw)? { aq.filters.push(filter); } depth += 1; } NamedElement { ns: Namespace::CardDav, element: Element::Limit, } if depth == 1 => { stream.expect_named_element(NamedElement::carddav(Element::Nresults))?; if let Some(Ok(limit)) = stream.parse_value::()? { aq.limit = limit.into(); } stream.expect_element_end()?; } NamedElement { ns: Namespace::CardDav, element: Element::PropFilter, } if depth == 2 => { let mut filter = None; for attribute in raw.attributes::() { match attribute? { Attribute::Name(name) => { property = Some(name); } Attribute::TestAllOf(all_of) => { filter = (if all_of { Filter::AllOf } else { Filter::AnyOf }).into(); } _ => {} } } if let Some(filter) = filter { aq.filters.push(filter); } depth += 1; } NamedElement { ns: Namespace::CardDav, element: Element::ParamFilter, } if depth == 3 => { for attribute in raw.attributes::() { if let Attribute::Name(name) = attribute? { parameter = Some(name); } } depth += 1; } NamedElement { ns: Namespace::CardDav, element: Element::IsNotDefined, } => { stream.expect_element_end()?; if let Some(filter) = Filter::from_parts( (), property.clone(), parameter.clone(), FilterOp::Undefined, ) { aq.filters.push(filter); } } NamedElement { ns: Namespace::CardDav, element: Element::TextMatch, } => { let mut tm = TextMatch::parse(raw)?; tm.value = stream.collect_string_value()?.unwrap_or_default(); if let Some(filter) = Filter::from_parts( (), property.clone(), parameter.clone(), FilterOp::TextMatch(tm), ) { aq.filters.push(filter); } } name => return Err(name.into_unexpected()), }, Token::ElementEnd => { depth -= 1; if depth == 0 { break; } } Token::UnknownElement(_) => { stream.seek_element_end()?; } element => return Err(element.into_unexpected()), } } Ok(aq) } } impl DavParser for FreeBusyQuery { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { match stream.token()? { Token::ElementStart { name: NamedElement { ns: Namespace::CalDav, element: Element::TimeRange, }, raw, } => TimeRange::from_raw(&raw).map(|range| FreeBusyQuery { range }), other => Err(other.into_unexpected()), } } } impl DavParser for MultiGet { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { let mut mg = MultiGet { properties: PropFind::AllProp(vec![]), hrefs: vec![], }; loop { match stream.token()? { Token::ElementStart { name, .. } => match name { NamedElement { ns: Namespace::Dav, element: Element::Propname, } => { mg.properties = PropFind::PropName; stream.expect_element_end()?; } NamedElement { ns: Namespace::Dav, element: Element::Allprop, } => { stream.expect_element_end()?; } NamedElement { ns: Namespace::Dav, element: Element::Prop, } => { mg.properties = PropFind::Prop(stream.collect_properties(Vec::new())?); } NamedElement { ns: Namespace::Dav, element: Element::Href, } => { if let Some(href) = stream.collect_string_value()? { mg.hrefs.push(href); } } name => return Err(name.into_unexpected()), }, Token::ElementEnd => { break; } element => return Err(element.into_unexpected()), } } Ok(mg) } } impl DavParser for SyncCollection { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { let mut sc = SyncCollection { properties: PropFind::AllProp(vec![]), limit: None, sync_token: None, depth: Depth::None, }; loop { match stream.token()? { Token::ElementStart { name, .. } => match name { NamedElement { ns: Namespace::Dav, element: Element::Prop, } => { sc.properties = PropFind::Prop(stream.collect_properties(Vec::new())?); } NamedElement { ns: Namespace::Dav, element: Element::Limit, } => { stream.expect_named_element(NamedElement::dav(Element::Nresults))?; if let Some(Ok(limit)) = stream.parse_value::()? { sc.limit = limit.into(); } stream.expect_element_end()?; } NamedElement { ns: Namespace::Dav, element: Element::SyncToken, } => { sc.sync_token = stream.collect_string_value()?; } NamedElement { ns: Namespace::Dav, element: Element::SyncLevel, } => { if let Some(Ok(depth)) = stream.parse_value::()? { sc.depth = depth; } } name => return Err(name.into_unexpected()), }, Token::ElementEnd => { break; } Token::UnknownElement(_) => { stream.seek_element_end()?; } element => return Err(element.into_unexpected()), } } Ok(sc) } } impl DavParser for ExpandProperty { fn parse(stream: &mut Tokenizer<'_>) -> crate::parser::Result { let mut ep = ExpandProperty { properties: vec![] }; let mut depth = 1; loop { match stream.token()? { Token::ElementStart { name, raw } => match name { NamedElement { ns, element: Element::Property, } => { for attribute in raw.attributes::() { if let Attribute::Name(name) = attribute? { if let Some(property) = Element::try_parse(name.as_bytes()) .copied() .and_then(|element| { DavProperty::from_element(NamedElement { ns, element }) }) { ep.properties.push(ExpandPropertyItem { property, depth: depth - 1, }); } else { let attrs = raw.element.attributes_raw().trim_ascii(); ep.properties.push(ExpandPropertyItem { property: DavProperty::DeadProperty(DeadElementTag { name, attrs: (!attrs.is_empty()).then(|| { String::from_utf8_lossy(attrs).into_owned() }), }), depth: depth - 1, }); } break; } } depth += 1; } name => return Err(name.into_unexpected()), }, Token::ElementEnd => { depth -= 1; if depth == 0 { break; } } Token::UnknownElement(_) => { stream.seek_element_end()?; } element => return Err(element.into_unexpected()), } } Ok(ep) } } impl TextMatch { fn parse(raw: RawElement<'_>) -> crate::parser::Result { let mut tm = TextMatch { match_type: MatchType::Contains, value: String::new(), collation: Collation::AsciiCasemap, negate: false, }; for attribute in raw.attributes::() { match attribute? { Attribute::MatchType(match_type) => { tm.match_type = match_type; } Attribute::NegateCondition(negate) => { tm.negate = negate; } Attribute::Collation(collation) => { tm.collation = collation; } _ => {} } } Ok(tm) } } impl Filter { fn from_parts(comp: A, prop: Option, param: Option, op: FilterOp) -> Option { match (prop, param) { (Some(prop), Some(param)) => Some(Filter::Parameter { comp, prop, param, op, }), (Some(prop), None) => Some(Filter::Property { comp, prop, op }), (None, None) => Some(Filter::Component { comp, op }), _ => None, } } fn components(&self) -> Option<&A> { match self { Filter::Component { comp, .. } => Some(comp), Filter::Property { comp, .. } => Some(comp), Filter::Parameter { comp, .. } => Some(comp), _ => None, } } fn parse(raw: RawElement<'_>) -> crate::parser::Result> { for attribute in raw.attributes::() { if let Attribute::TestAllOf(all_of) = attribute? { return Ok(Some(if all_of { Filter::AllOf } else { Filter::AnyOf })); } } Ok(None) } } impl XmlValueParser for Depth { fn parse_bytes(bytes: &[u8]) -> Option { Depth::parse(bytes) } fn parse_str(text: &str) -> Option { Depth::parse(text.as_bytes()) } } ================================================ FILE: crates/dav-proto/src/responses/acl.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::fmt::Display; use crate::{ responses::XmlEscape, schema::{ property::{DavProperty, Privilege}, response::{ Ace, AclRestrictions, GrantDeny, Href, List, Principal, PrincipalSearchProperty, PrincipalSearchPropertySet, RequiredPrincipal, Resource, SupportedPrivilege, }, Namespace, Namespaces, }, }; impl Display for SupportedPrivilege { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.privilege)?; if self.abstract_ { write!(f, "")?; } write!(f, "")?; self.description.write_escaped_to(f)?; write!( f, "{}", self.supported_privilege ) } } impl SupportedPrivilege { pub fn new(privilege: Privilege, description: impl Into) -> Self { SupportedPrivilege { privilege, abstract_: false, description: description.into(), supported_privilege: List(vec![]), } } pub fn with_abstract(mut self) -> Self { self.abstract_ = true; self } pub fn with_supported_privilege(mut self, supported_privilege: SupportedPrivilege) -> Self { self.supported_privilege.0.push(supported_privilege); self } pub fn with_opt_supported_privilege( mut self, supported_privilege: Option, ) -> Self { if let Some(supported_privilege) = supported_privilege { self.supported_privilege.0.push(supported_privilege); } self } pub fn all_privileges(is_calendar: bool) -> SupportedPrivilege { SupportedPrivilege::new(Privilege::All, "Any operation") .with_abstract() .with_supported_privilege( SupportedPrivilege::new(Privilege::Read, "Read objects").with_supported_privilege( SupportedPrivilege::new( Privilege::ReadCurrentUserPrivilegeSet, "Read current user privileges", ), ), ) .with_supported_privilege( SupportedPrivilege::new(Privilege::Write, "Write objects") .with_supported_privilege(SupportedPrivilege::new( Privilege::WriteProperties, "Write properties", )) .with_supported_privilege(SupportedPrivilege::new( Privilege::WriteContent, "Write object contents", )) .with_supported_privilege(SupportedPrivilege::new( Privilege::Bind, "Add resources to a collection", )) .with_supported_privilege(SupportedPrivilege::new( Privilege::Unbind, "Remove resources from a collection", )) .with_supported_privilege(SupportedPrivilege::new( Privilege::Unlock, "Unlock resources", )), ) .with_supported_privilege(SupportedPrivilege::new(Privilege::ReadAcl, "Read ACL")) .with_supported_privilege(SupportedPrivilege::new(Privilege::WriteAcl, "Write ACL")) .with_opt_supported_privilege((is_calendar).then(|| { SupportedPrivilege::new(Privilege::ReadFreeBusy, "Read free/busy information") })) } pub fn all_scheduling_privileges(is_inbox: bool) -> SupportedPrivilege { let privilege = SupportedPrivilege::new(Privilege::All, "Any operation") .with_abstract() .with_supported_privilege( SupportedPrivilege::new(Privilege::Read, "Read objects").with_supported_privilege( SupportedPrivilege::new( Privilege::ReadCurrentUserPrivilegeSet, "Read current user privileges", ), ), ); if is_inbox { privilege.with_supported_privilege( SupportedPrivilege::new( Privilege::ScheduleDeliver, "Deliver calendar scheduling messages", ) .with_supported_privilege(SupportedPrivilege::new( Privilege::ScheduleDeliverInvite, "Deliver calendar scheduling invites", )) .with_supported_privilege(SupportedPrivilege::new( Privilege::ScheduleDeliverReply, "Deliver calendar scheduling replies", )) .with_supported_privilege(SupportedPrivilege::new( Privilege::ScheduleQueryFreeBusy, "Query free/busy information", )), ) } else { privilege.with_supported_privilege( SupportedPrivilege::new( Privilege::ScheduleSend, "Send calendar scheduling messages", ) .with_supported_privilege(SupportedPrivilege::new( Privilege::ScheduleSendInvite, "Send calendar scheduling invites", )) .with_supported_privilege(SupportedPrivilege::new( Privilege::ScheduleSendReply, "Send calendar scheduling replies", )) .with_supported_privilege(SupportedPrivilege::new( Privilege::ScheduleSendFreeBusy, "Send free/busy information", )), ) } } } impl Display for Ace { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "")?; if self.invert { write!(f, "")?; } self.principal.fmt(f)?; if self.invert { write!(f, "")?; } self.grant_deny.fmt(f)?; if self.protected { write!(f, "")?; } if let Some(inherited) = &self.inherited { write!(f, "")?; inherited.fmt(f)?; write!(f, "")?; } write!(f, "") } } impl Display for Principal { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "")?; match self { Principal::Href(href) => href.fmt(f), Principal::Response(response) => response.fmt(f), Principal::All => "".fmt(f), Principal::Authenticated => "".fmt(f), Principal::Unauthenticated => "".fmt(f), Principal::Property(property) => { write!(f, "{}", property) } Principal::Self_ => "".fmt(f), }?; write!(f, "") } } impl Display for GrantDeny { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { GrantDeny::Grant(privileges) => { write!(f, "")?; privileges.fmt(f)?; write!(f, "") } GrantDeny::Deny(privileges) => { write!(f, "")?; privileges.fmt(f)?; write!(f, "") } } } } impl Display for AclRestrictions { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { if self.grant_only { write!(f, "")?; } if self.no_invert { write!(f, "")?; } if self.deny_before_grant { write!(f, "")?; } if let Some(required_principal) = &self.required_principal { required_principal.fmt(f)?; } Ok(()) } } impl Display for RequiredPrincipal { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "")?; match self { RequiredPrincipal::All => "".fmt(f)?, RequiredPrincipal::Authenticated => "".fmt(f)?, RequiredPrincipal::Unauthenticated => "".fmt(f)?, RequiredPrincipal::Self_ => "".fmt(f)?, RequiredPrincipal::Href(hrefs) => hrefs.fmt(f)?, RequiredPrincipal::Property(properties) => { for property in properties { write!(f, "{}", property)?; } } } write!(f, "") } } impl Display for Privilege { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Privilege::Read => "".fmt(f), Privilege::Write => "".fmt(f), Privilege::WriteProperties => "".fmt(f), Privilege::WriteContent => "".fmt(f), Privilege::Unlock => "".fmt(f), Privilege::ReadAcl => "".fmt(f), Privilege::ReadCurrentUserPrivilegeSet => { "".fmt(f) } Privilege::WriteAcl => "".fmt(f), Privilege::Bind => "".fmt(f), Privilege::Unbind => "".fmt(f), Privilege::All => "".fmt(f), Privilege::ReadFreeBusy => "".fmt(f), Privilege::ScheduleDeliver => "".fmt(f), Privilege::ScheduleDeliverInvite => { "".fmt(f) } Privilege::ScheduleDeliverReply => { "".fmt(f) } Privilege::ScheduleQueryFreeBusy => { "".fmt(f) } Privilege::ScheduleSend => "".fmt(f), Privilege::ScheduleSendInvite => { "".fmt(f) } Privilege::ScheduleSendReply => { "".fmt(f) } Privilege::ScheduleSendFreeBusy => { "".fmt(f) } } } } impl Display for Resource { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( f, "{}{}", self.href, self.privilege ) } } impl Display for PrincipalSearchPropertySet { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "")?; write!( f, "{}", self.namespaces, self.properties ) } } impl Display for PrincipalSearchProperty { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( f, "{}", self.name )?; write!( f, "{}", self.description ) } } impl Resource { pub fn new(href: impl Into, privilege: Privilege) -> Self { Resource { href: Href(href.into()), privilege, } } } impl PrincipalSearchPropertySet { pub fn new(properties: Vec) -> Self { PrincipalSearchPropertySet { namespaces: Namespaces::default(), properties: List(properties), } } pub fn with_namespace(mut self, namespace: Namespace) -> Self { self.namespaces.set(namespace); self } } impl PrincipalSearchProperty { pub fn new(name: impl Into, description: impl Into) -> Self { PrincipalSearchProperty { name: name.into(), description: description.into(), } } } impl Ace { pub fn new(principal: Principal, grant_deny: GrantDeny) -> Self { Ace { principal, invert: false, grant_deny, protected: false, inherited: None, } } pub fn with_invert(mut self) -> Self { self.invert = true; self } pub fn with_protected(mut self) -> Self { self.protected = true; self } pub fn with_inherited(mut self, inherited: impl Into) -> Self { self.inherited = Some(Href(inherited.into())); self } } impl GrantDeny { pub fn grant(privileges: Vec) -> Self { GrantDeny::Grant(List(privileges)) } pub fn deny(privileges: Vec) -> Self { GrantDeny::Deny(List(privileges)) } } impl AclRestrictions { pub fn new() -> Self { Self::default() } pub fn with_grant_only(mut self) -> Self { self.grant_only = true; self } pub fn with_no_invert(mut self) -> Self { self.no_invert = true; self } pub fn with_deny_before_grant(mut self) -> Self { self.deny_before_grant = true; self } pub fn with_required_principal(mut self, required_principal: RequiredPrincipal) -> Self { self.required_principal = Some(required_principal); self } } ================================================ FILE: crates/dav-proto/src/responses/error.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::fmt::Display; use crate::schema::{ response::{BaseCondition, CalCondition, CardCondition, Condition, ErrorResponse}, Namespace, Namespaces, }; impl Display for ErrorResponse { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( f, "", self.namespaces )?; match &self.error { Condition::Base(e) => e.fmt(f)?, Condition::Cal(e) => e.fmt(f)?, Condition::Card(e) => e.fmt(f)?, } write!(f, "") } } impl Display for Condition { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "")?; match self { Condition::Base(e) => e.fmt(f)?, Condition::Cal(e) => e.fmt(f)?, Condition::Card(e) => e.fmt(f)?, } write!(f, "") } } impl Display for BaseCondition { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { BaseCondition::NoConflictingLock(items) => { write!(f, "{items}") } BaseCondition::LockTokenSubmitted(items) => write!( f, "{items}" ), BaseCondition::LockTokenMatchesRequestUri => { write!(f, "") } BaseCondition::CannotModifyProtectedProperty => { write!(f, "") } BaseCondition::NoExternalEntities => write!(f, ""), BaseCondition::PreservedLiveProperties => write!(f, ""), BaseCondition::PropFindFiniteDepth => write!(f, ""), BaseCondition::ResourceMustBeNull => write!(f, ""), BaseCondition::NeedPrivileges(resources) => { write!(f, "{resources}") } BaseCondition::NumberOfMatchesWithinLimit => { write!(f, "") } BaseCondition::QuotaNotExceeded => write!(f, ""), BaseCondition::ValidResourceType => write!(f, ""), BaseCondition::ValidSyncToken => write!(f, ""), BaseCondition::NoAceConflict => write!(f, ""), BaseCondition::NoProtectedAceConflict => write!(f, ""), BaseCondition::NoInheritedAceConflict => write!(f, ""), BaseCondition::LimitedNumberOfAces => write!(f, ""), BaseCondition::DenyBeforeGrant => write!(f, ""), BaseCondition::GrantOnly => write!(f, ""), BaseCondition::NoInvert => write!(f, ""), BaseCondition::NoAbstract => write!(f, ""), BaseCondition::NotSupportedPrivilege => write!(f, ""), BaseCondition::MissingRequiredPrincipal => write!(f, ""), BaseCondition::RecognizedPrincipal => write!(f, ""), BaseCondition::AllowedPrincipal => write!(f, ""), } } } impl Display for CalCondition { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { CalCondition::CalendarCollectionLocationOk => { write!(f, "") } CalCondition::ValidCalendarData => write!(f, ""), CalCondition::ValidFilter => write!(f, ""), CalCondition::ValidTimezone => write!(f, ""), CalCondition::ValidCalendarObjectResource => { write!(f, "") } CalCondition::NoUidConflict(uid) => { write!(f, "{uid}") } CalCondition::InitializeCalendarCollection => { write!(f, "") } CalCondition::SupportedCalendarData => write!(f, ""), CalCondition::SupportedFilter(_) => write!(f, ""), CalCondition::SupportedCollation(c) => { write!(f, "{c}") } CalCondition::MinDateTime => write!(f, ""), CalCondition::MaxDateTime => write!(f, ""), CalCondition::MaxResourceSize(l) => { write!(f, "{l}") } CalCondition::MaxInstances => write!(f, ""), CalCondition::MaxAttendeesPerInstance => write!(f, ""), CalCondition::UniqueSchedulingObjectResource(href) => write!( f, "{href}" ), CalCondition::SameOrganizerInAllComponents => { write!(f, "") } CalCondition::AllowedOrganizerObjectChange => { write!(f, "") } CalCondition::AllowedAttendeeObjectChange => { write!(f, "") } CalCondition::DefaultCalendarNeeded => write!(f, ""), CalCondition::ValidScheduleDefaultCalendarUrl => { write!(f, "") } CalCondition::ValidSchedulingMessage => write!(f, ""), CalCondition::ValidOrganizer => write!(f, ""), CalCondition::SupportedCalendarComponent => { write!(f, "") } } } } impl Display for CardCondition { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { CardCondition::SupportedAddressData => write!(f, ""), CardCondition::SupportedAddressDataConversion => { write!(f, "") } CardCondition::SupportedFilter(_) => write!(f, ""), CardCondition::SupportedCollation(c) => { write!(f, "{c}") } CardCondition::ValidAddressData => write!(f, ""), CardCondition::NoUidConflict(uid) => { write!(f, "{uid}") } CardCondition::MaxResourceSize(l) => { write!(f, "{l}") } CardCondition::AddressBookCollectionLocationOk => { write!(f, "") } } } } impl From for Condition { fn from(error: CalCondition) -> Self { Condition::Cal(error) } } impl From for Condition { fn from(error: CardCondition) -> Self { Condition::Card(error) } } impl From for Condition { fn from(error: BaseCondition) -> Self { Condition::Base(error) } } impl ErrorResponse { pub fn new(error: impl Into) -> Self { ErrorResponse { namespaces: Namespaces::default(), error: error.into(), } } pub fn with_namespace(mut self, namespace: impl Into) -> Self { self.namespaces.set(namespace.into()); self } } ================================================ FILE: crates/dav-proto/src/responses/lock.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::fmt::Display; use types::dead_property::DeadProperty; use crate::{ responses::DeadPropertyFormat, schema::{ property::{ActiveLock, LockDiscovery, LockEntry, LockScope, LockType, SupportedLock}, request::LockInfo, response::{Href, List}, }, Depth, Timeout, }; impl Display for SupportedLock { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.0) } } impl Display for LockDiscovery { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.0) } } impl Display for ActiveLock { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( f, "{}{}{}", self.lock_scope, self.lock_type, self.depth )?; if let Some(owner) = &self.owner { f.write_str("")?; owner.fmt(f)?; f.write_str("")?; } write!(f, "{}", self.timeout)?; if let Some(lock_token) = &self.lock_token { write!(f, "{}", lock_token)?; } write!( f, "{}", self.lock_root ) } } impl Display for Depth { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Depth::Zero => write!(f, "0"), Depth::One => write!(f, "1"), Depth::Infinity => write!(f, "infinity"), Depth::None => write!(f, ""), } } } impl Display for Timeout { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Timeout::Infinite => write!(f, "Infinite"), Timeout::Second(s) => write!(f, "Second-{}", s), Timeout::None => Ok(()), } } } impl Display for LockInfo { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}{}", self.lock_scope, self.lock_type)?; if let Some(owner) = &self.owner { f.write_str("")?; owner.fmt(f)?; f.write_str("")?; } write!(f, "",) } } impl Display for LockEntry { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( f, "{}{}", self.lock_scope, self.lock_type ) } } impl Display for LockScope { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { LockScope::Exclusive => write!(f, ""), LockScope::Shared => write!(f, ""), } } } impl Display for LockType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { LockType::Write => write!(f, ""), LockType::Other => write!(f, ""), } } } impl ActiveLock { pub fn new(href: impl Into, lock_scope: LockScope) -> Self { Self { lock_scope, lock_type: LockType::Write, depth: Depth::Infinity, owner: None, timeout: Timeout::Infinite, lock_token: None, lock_root: Href(href.into()), } } pub fn with_depth(mut self, depth: Depth) -> Self { self.depth = depth; self } pub fn with_timeout(mut self, timeout: u64) -> Self { self.timeout = Timeout::Second(timeout); self } pub fn with_owner_opt(mut self, owner: Option) -> Self { self.owner = owner; self } pub fn with_owner(mut self, owner: DeadProperty) -> Self { self.owner = Some(owner); self } pub fn with_lock_token(mut self, token: impl Into) -> Self { self.lock_token = Some(Href(token.into())); self } } impl Default for SupportedLock { fn default() -> Self { Self(List(vec![ LockEntry { lock_scope: LockScope::Exclusive, lock_type: LockType::Write, }, LockEntry { lock_scope: LockScope::Shared, lock_type: LockType::Write, }, ])) } } ================================================ FILE: crates/dav-proto/src/responses/mkcol.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::fmt::Display; use crate::schema::{ response::{List, MkColResponse, PropStat}, Namespace, Namespaces, }; impl Display for MkColResponse { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "")?; if !self.mkcalendar { write!( f, "{}", self.namespaces, self.propstat ) } else { write!( f, "{}", self.namespaces, self.propstat ) } } } impl MkColResponse { pub fn new(propstat: Vec) -> Self { Self { namespaces: Namespaces::default(), propstat: List(propstat), mkcalendar: false, } } pub fn with_mkcalendar(mut self, mkcalendar: bool) -> Self { self.mkcalendar = mkcalendar; if mkcalendar { self.namespaces.set(Namespace::CalDav); } self } pub fn with_namespace(mut self, namespace: Namespace) -> Self { self.namespaces.set(namespace); self } } ================================================ FILE: crates/dav-proto/src/responses/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod acl; pub mod error; pub mod lock; pub mod mkcol; pub mod multistatus; pub mod property; pub mod propstat; pub mod schedule; use types::dead_property::{DeadProperty, DeadPropertyTag}; use crate::schema::{ property::{Comp, ResourceType, SupportedCollation}, response::{Href, List, Location, ResponseDescription, Status, SyncToken}, Namespaces, }; use std::fmt::{Display, Write}; trait XmlEscape { fn write_escaped_to(&self, out: &mut impl Write) -> std::fmt::Result; } trait XmlCdataEscape { fn write_cdata_escaped_to(&self, out: &mut impl Write) -> std::fmt::Result; } impl> XmlEscape for T { fn write_escaped_to(&self, out: &mut impl Write) -> std::fmt::Result { let str = self.as_ref(); for c in str.chars() { match c { '<' => out.write_str("<")?, '>' => out.write_str(">")?, '&' => out.write_str("&")?, '"' => out.write_str(""")?, '\'' => out.write_str("'")?, _ => out.write_char(c)?, } } Ok(()) } } impl> XmlCdataEscape for T { fn write_cdata_escaped_to(&self, out: &mut impl Write) -> std::fmt::Result { let str = self.as_ref(); let mut last_ch = '\0'; let mut last_ch2 = '\0'; out.write_str("' if last_ch == ']' && last_ch2 == ']' => { out.write_str("]]>")?; } _ => out.write_char(ch)?, } last_ch2 = last_ch; last_ch = ch; } out.write_str("]]>") } } impl Display for Namespaces { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("xmlns:D=\"DAV:\"")?; if self.cal { f.write_str(" xmlns:A=\"urn:ietf:params:xml:ns:caldav\"")?; } if self.card { f.write_str(" xmlns:B=\"urn:ietf:params:xml:ns:carddav\"")?; } if self.cs { f.write_str(" xmlns:C=\"http://calendarserver.org/ns/\"")?; } Ok(()) } } impl Display for Href { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "")?; self.0.write_escaped_to(f)?; write!(f, "") } } impl Display for List { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { for item in &self.0 { item.fmt(f)?; } Ok(()) } } impl Display for Status { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "")?; write!(f, "HTTP/1.1 {}", self.0)?; write!(f, "") } } impl Display for ResponseDescription { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "")?; self.0.write_escaped_to(f)?; write!(f, "") } } impl Display for Location { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "")?; self.0.fmt(f)?; write!(f, "") } } impl Display for SyncToken { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "")?; self.0.write_escaped_to(f)?; write!(f, "") } } impl Display for Comp { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "", self.0.as_str()) } } impl Display for ResourceType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { ResourceType::Collection => write!(f, ""), ResourceType::Principal => write!(f, ""), ResourceType::AddressBook => write!(f, ""), ResourceType::Calendar => write!(f, ""), ResourceType::ScheduleInbox => write!(f, ""), ResourceType::ScheduleOutbox => write!(f, ""), } } } impl Display for SupportedCollation { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let ns = self.namespace.prefix(); write!( f, "<{ns}:supported-collation>{}", self.collation.as_str() ) } } pub trait DeadPropertyFormat { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result; } impl DeadPropertyFormat for DeadProperty { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let mut last_tag = ""; for item in &self.0 { match item { DeadPropertyTag::ElementStart(tag) => { let name = &tag.name; if let Some(attrs) = &tag.attrs { write!(f, "<{name} {attrs}>")?; } else { write!(f, "<{name}>")?; } last_tag = name; } DeadPropertyTag::ElementEnd => { write!(f, "", last_tag)?; } DeadPropertyTag::Text(text) => { text.write_escaped_to(f)?; } } } Ok(()) } } #[cfg(test)] mod tests { use std::fmt::Display; use calcard::{icalendar::ICalendar, vcard::VCard}; use hyper::StatusCode; use mail_parser::DateTime; use types::dead_property::{DeadElementTag, DeadProperty, DeadPropertyTag}; use crate::{ parser::{tokenizer::Tokenizer, Token}, responses::XmlCdataEscape, schema::{ property::{ ActiveLock, CalDavProperty, CardDavProperty, DavValue, LockScope, Privilege, ResourceType, Rfc1123DateTime, SupportedLock, WebDavProperty, }, request::DavPropertyValue, response::{ Ace, AclRestrictions, BaseCondition, ErrorResponse, GrantDeny, Href, List, MkColResponse, MultiStatus, Principal, PrincipalSearchProperty, PrincipalSearchPropertySet, PropResponse, PropStat, RequiredPrincipal, Resource, Response, ScheduleResponse, ScheduleResponseItem, SupportedPrivilege, }, Namespace, }, Depth, }; impl List { pub fn new(vec: impl IntoIterator) -> Self { List(vec.into_iter().collect()) } } impl From for DavValue { fn from(v: ICalendar) -> Self { DavValue::ICalendar(v) } } impl From for DavValue { fn from(v: VCard) -> Self { DavValue::VCard(v) } } #[test] fn parse_responses() { for (num, test) in [ // 001.xml ErrorResponse::new(BaseCondition::LockTokenSubmitted(List::new([Href( "/locked/".to_string(), )]))) .to_string(), // 002.xml MultiStatus::new(vec![Response::new_propstat( "http://www.example.com/file", vec![ PropStat::new(DavPropertyValue::new( WebDavProperty::DisplayName, "Box type A", )), PropStat::new(DavPropertyValue::new( WebDavProperty::DisplayName, "Box type B", )) .with_status(StatusCode::FORBIDDEN) .with_response_description( "The user does not have access to the DingALing property.", ), ], )]) .with_response_description("There has been an access violation error.") .to_string(), // 003.xml MultiStatus::new(vec![ Response::new_propstat( "/container/", vec![PropStat::new_list(vec![ DavPropertyValue::new( WebDavProperty::CreationDate, DateTime::parse_rfc3339("1997-12-01T17:42:21-08:00Z").unwrap(), ), DavPropertyValue::new(WebDavProperty::DisplayName, "Example collection"), DavPropertyValue::new( WebDavProperty::ResourceType, vec![ResourceType::Collection], ), DavPropertyValue::new( WebDavProperty::SupportedLock, SupportedLock::default(), ), ])], ), Response::new_propstat( "/container/front.html", vec![PropStat::new_list(vec![ DavPropertyValue::new( WebDavProperty::CreationDate, DateTime::parse_rfc3339("1997-12-01T18:27:21-08:00").unwrap(), ), DavPropertyValue::new(WebDavProperty::DisplayName, "Example HTML resource"), DavPropertyValue::new(WebDavProperty::GetContentLength, 4525u64), DavPropertyValue::new(WebDavProperty::GetContentType, "text/html"), DavPropertyValue::new(WebDavProperty::GetETag, "\"zzyzx\""), DavPropertyValue::new( WebDavProperty::GetLastModified, DavValue::Rfc1123Date(Rfc1123DateTime::new( DateTime::parse_rfc822("Mon, 12 Jan 1998 09:25:56 GMT") .unwrap() .to_timestamp(), )), ), DavPropertyValue::new(WebDavProperty::ResourceType, DavValue::Null), DavPropertyValue::new( WebDavProperty::SupportedLock, SupportedLock::default(), ), ])], ), ]) .to_string(), // 004.xml MultiStatus::new(vec![Response::new_status( ["http://www.example.com/container/resource3"], StatusCode::LOCKED, ) .with_error(BaseCondition::LockTokenSubmitted(List(vec![])))]) .to_string(), // 005.xml PropResponse::new(vec![DavPropertyValue::new( WebDavProperty::LockDiscovery, vec![ActiveLock::new( "http://example.com/workspace/webdav/proposal.doc", LockScope::Exclusive, ) .with_owner(DeadProperty(vec![ DeadPropertyTag::ElementStart(DeadElementTag { name: "D:href".to_string(), attrs: None, }), DeadPropertyTag::Text("http://example.org/~ejw/contact.html".to_string()), DeadPropertyTag::ElementEnd, ])) .with_timeout(604800) .with_lock_token("urn:uuid:e71d4fae-5dec-22d6-fea5-00a0c91e6be4")], )]) .to_string(), // 006.xml MultiStatus::new(vec![Response::new_propstat( "http://www.example.com/container/", vec![PropStat::new_list(vec![DavPropertyValue::new( WebDavProperty::LockDiscovery, vec![ ActiveLock::new("http://www.example.com/container/", LockScope::Shared) .with_owner(DeadProperty(vec![DeadPropertyTag::Text( "Jane Smith".to_string(), )])) .with_depth(Depth::Zero) .with_lock_token("urn:uuid:f81de2ad-7f3d-a1b2-4f3c-00a0c91a9d76"), ], )])], )]) .to_string(), // 007.xml ErrorResponse::new(BaseCondition::LockTokenSubmitted(List(vec![Href( "/workspace/webdav/".to_string(), )]))) .to_string(), // 008.xml MultiStatus::new(vec![ Response::new_propstat( "http://cal.example.com/bernard/work/abcd2.ics", vec![PropStat::new_list(vec![ DavPropertyValue::new(WebDavProperty::GetETag, "\"fffff-abcd2\""), DavPropertyValue::new( CalDavProperty::CalendarData(Default::default()), DavValue::CData( r#"BEGIN:VCALENDAR VERSION:2.0 BEGIN:VEVENT DTSTART;TZID=US/Eastern:20060106T140000 DURATION:PT1H RECURRENCE-ID;TZID=US/Eastern:20060106T120000 SUMMARY:Event #2 bis bis UID:00959BC664CA650E933C892C@example.com END:VEVENT END:VCALENDAR "# .to_string(), ), ), ])], ), Response::new_propstat( "http://cal.example.com/bernard/work/abcd3.ics", vec![PropStat::new_list(vec![ DavPropertyValue::new(WebDavProperty::GetETag, "\"fffff-abcd3\""), DavPropertyValue::new( CalDavProperty::CalendarData(Default::default()), DavValue::CData( r#"BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VEVENT DTSTART;TZID=US/Eastern:20060104T100000 DURATION:PT1H SUMMARY:Event #3 UID:DC6C50A017428C5216A2F1CD@example.com END:VEVENT END:VCALENDAR "# .to_string(), ), ), ])], ), ]) .with_namespace(Namespace::CalDav) .to_string(), // 009.xml MkColResponse::new(vec![PropStat::new_list(vec![ DavPropertyValue::new(WebDavProperty::ResourceType, DavValue::Null), DavPropertyValue::new(WebDavProperty::DisplayName, DavValue::Null), DavPropertyValue::new(CardDavProperty::AddressbookDescription, DavValue::Null), ])]) .with_namespace(Namespace::CardDav) .to_string(), // 010.xml MultiStatus::new(vec![Response::new_propstat( "/home/bernard/addressbook/v102.vcf", vec![PropStat::new_list(vec![ DavPropertyValue::new(WebDavProperty::GetETag, "\"23ba4d-ff11fb\""), DavPropertyValue::new( CardDavProperty::AddressData(Default::default()), DavValue::CData( r#"BEGIN:VCARD VERSION:3.0 NICKNAME:me UID:34222-232@example.com FN:Cyrus Daboo EMAIL:daboo@example.com END:VCARD "# .to_string(), ), ), ])], )]) .with_namespace(Namespace::CardDav) .to_string(), // 011.xml MultiStatus::new(vec![ Response::new_status( ["/home/bernard/addressbook/"], StatusCode::INSUFFICIENT_STORAGE, ) .with_error(BaseCondition::NumberOfMatchesWithinLimit) .with_response_description("Only two matching records were returned"), Response::new_propstat( "/home/bernard/addressbook/v102.vcf", vec![PropStat::new_list(vec![DavPropertyValue::new( WebDavProperty::GetETag, "\"23ba4d-ff11fb\"", )])], ), Response::new_propstat( "/home/bernard/addressbook/v104.vcf", vec![PropStat::new_list(vec![DavPropertyValue::new( WebDavProperty::GetETag, "\"23ba4d-ff11fc\"", )])], ), ]) .with_namespace(Namespace::CardDav) .to_string(), // 012.xml ErrorResponse::new(BaseCondition::NeedPrivileges(List(vec![ Resource::new("/a", Privilege::Unbind), Resource::new("/c", Privilege::Bind), ]))) .to_string(), // 013.xml PrincipalSearchPropertySet::new(vec![ PrincipalSearchProperty::new(WebDavProperty::DisplayName, "Full name"), PrincipalSearchProperty::new(WebDavProperty::DisplayName, "Job title"), ]) .to_string(), // 014.xml MultiStatus::new(vec![Response::new_propstat( "http://www.example.com/papers/", vec![PropStat::new_list(vec![DavPropertyValue::new( WebDavProperty::SupportedPrivilegeSet, vec![SupportedPrivilege::new(Privilege::All, "Any operation") .with_abstract() .with_supported_privilege( SupportedPrivilege::new(Privilege::Read, "Read any object") .with_supported_privilege( SupportedPrivilege::new(Privilege::ReadAcl, "Read ACL") .with_abstract(), ) .with_supported_privilege( SupportedPrivilege::new( Privilege::ReadCurrentUserPrivilegeSet, "Read current user privilege set property", ) .with_abstract(), ), ) .with_supported_privilege( SupportedPrivilege::new(Privilege::Write, "Write any object") .with_supported_privilege( SupportedPrivilege::new(Privilege::WriteAcl, "Write ACL") .with_abstract(), ) .with_supported_privilege(SupportedPrivilege::new( Privilege::WriteProperties, "Write properties", )) .with_supported_privilege(SupportedPrivilege::new( Privilege::WriteContent, "Write resource content", )), ) .with_supported_privilege(SupportedPrivilege::new( Privilege::Unlock, "Unlock resource", ))], )])], )]) .to_string(), // 015.xml MultiStatus::new(vec![Response::new_propstat( "http://www.example.com/papers/", vec![PropStat::new_list(vec![DavPropertyValue::new( WebDavProperty::CurrentUserPrivilegeSet, vec![Privilege::Read], )])], )]) .to_string(), // 016.xml MultiStatus::new(vec![Response::new_propstat( "http://www.example.com/papers/", vec![PropStat::new_list(vec![DavPropertyValue::new( WebDavProperty::Acl, vec![ Ace::new( Principal::Href(Href( "http://www.example.com/acl/groups/maintainers".to_string(), )), GrantDeny::grant(vec![Privilege::Write]), ), Ace::new(Principal::All, GrantDeny::grant(vec![Privilege::Read])), ], )])], )]) .to_string(), // 017.xml MultiStatus::new(vec![Response::new_propstat( "http://www.example.com/papers/", vec![PropStat::new_list(vec![DavPropertyValue::new( WebDavProperty::AclRestrictions, AclRestrictions::new() .with_grant_only() .with_required_principal(RequiredPrincipal::All), )])], )]) .to_string(), // 018.xml MultiStatus::new(vec![Response::new_propstat( "http://www.example.com/papers/", vec![PropStat::new_list(vec![DavPropertyValue::new( WebDavProperty::PrincipalCollectionSet, vec![ Href("http://www.example.com/acl/users/".to_string()), Href("http://www.example.com/acl/groups/".to_string()), ], )])], )]) .to_string(), // 019.xml MultiStatus::new(vec![Response::new_propstat( "http://www.example.com/top/container/", vec![PropStat::new_list(vec![ DavPropertyValue::new( WebDavProperty::Owner, vec![Href("http://www.example.com/users/gclemm".to_string())], ), DavPropertyValue::new( WebDavProperty::SupportedPrivilegeSet, vec![SupportedPrivilege::new(Privilege::All, "Any operation") .with_abstract() .with_supported_privilege(SupportedPrivilege::new( Privilege::Read, "Read any object", )) .with_supported_privilege( SupportedPrivilege::new(Privilege::Write, "Write any object") .with_abstract(), ) .with_supported_privilege(SupportedPrivilege::new( Privilege::ReadAcl, "Read the ACL", )) .with_supported_privilege(SupportedPrivilege::new( Privilege::WriteAcl, "Write the ACL", ))], ), DavPropertyValue::new( WebDavProperty::CurrentUserPrivilegeSet, vec![Privilege::Read, Privilege::ReadAcl], ), DavPropertyValue::new( WebDavProperty::Acl, vec![ Ace::new( Principal::Href(Href( "http://www.example.com/users/esedlar".to_string(), )), GrantDeny::grant(vec![ Privilege::Read, Privilege::Write, Privilege::ReadAcl, ]), ), Ace::new( Principal::Href(Href( "http://www.example.com/groups/mrktng".to_string(), )), GrantDeny::deny(vec![Privilege::Read]), ), Ace::new( Principal::Property(List(vec![DavPropertyValue::new( WebDavProperty::Owner, DavValue::Null, )])), GrantDeny::grant(vec![Privilege::ReadAcl, Privilege::WriteAcl]), ), Ace::new(Principal::All, GrantDeny::grant(vec![Privilege::Read])) .with_inherited("http://www.example.com/top"), ], ), ])], )]) .to_string(), // 020.xml ScheduleResponse { items: List(vec![ ScheduleResponseItem { recipient: Href("mailto:wilfredo@example.com".to_string()), request_status: "2.0;Success".into(), calendar_data: Some("BEGIN:VCALENDAR".to_string()), }, ScheduleResponseItem { recipient: Href("mailto:bernard@example.net".to_string()), request_status: "2.0;Success".into(), calendar_data: Some("END:VCALENDAR".to_string()), }, ScheduleResponseItem { recipient: Href("mailto:mike@example.org".to_string()), request_status: "3.7;Invalid calendar user".into(), calendar_data: None, }, ]), } .to_string(), ] .into_iter() .enumerate() { let xml = std::fs::read_to_string(format!("resources/responses/{:03}.xml", num + 1)).unwrap(); let mut output_token = Tokenizer::new(test.as_bytes()); let mut expected_token = Tokenizer::new(xml.as_bytes()); let mut output_tokens = Vec::new(); let mut expected_tokens = Vec::new(); for (tokens, tokenizer) in [ (&mut output_tokens, &mut output_token), (&mut expected_tokens, &mut expected_token), ] { while let Ok(token) = tokenizer.token() { if token == Token::Eof { break; } match (tokens.last_mut(), token) { (Some(Token::Text(text)), Token::Text(new_text)) => { *text = format!("{}{}", text, new_text).into(); } (_, element) => { tokens.push(element.into_owned()); } } } } assert!(!output_tokens.is_empty()); assert!(!expected_tokens.is_empty()); assert_eq!(output_tokens.len(), expected_tokens.len()); for (output, expected) in output_tokens.iter().zip(expected_tokens.iter()) { if output != expected { eprintln!("{test}"); } assert_eq!(output, expected, "failed for {:03}.xml", num + 1); } } } #[test] fn escape_cdata() { for (test, expected) in [ ("", ""), ("hello", ""), ("hello world", ""), ("", "]]>"), ("&hello;", ""), ("'hello'", ""), ("\"hello\"", ""), ("<>&'\"", "&'\"]]>"), (">", "]]>"), ("]]>]", "]]]>"), ("]]>", "]]>"), ("hello]]>world", "world]]>"), ( "hello]]>pure-evil", "pure-evil]]>", ), ] { let mut output = String::new(); test.write_cdata_escaped_to(&mut output).unwrap(); assert_eq!(output, expected, "failed for input: {test:?}"); } } } ================================================ FILE: crates/dav-proto/src/responses/multistatus.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::fmt::Display; use hyper::StatusCode; use crate::schema::{ response::{ Condition, Href, List, Location, MultiStatus, PropStat, Response, ResponseDescription, ResponseType, Status, SyncToken, }, Namespace, Namespaces, }; impl Display for MultiStatus { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( f, "{}", self.namespaces, self.response )?; if let Some(response_description) = &self.response_description { write!(f, "{response_description}")?; } if let Some(sync_token) = &self.sync_token { write!(f, "{sync_token}")?; } write!(f, "") } } impl Display for Response { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "")?; self.href.fmt(f)?; self.typ.fmt(f)?; if let Some(error) = &self.error { error.fmt(f)?; } if let Some(response_description) = &self.response_description { response_description.fmt(f)?; } if let Some(location) = &self.location { location.fmt(f)?; } write!(f, "") } } impl Display for ResponseType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { ResponseType::PropStat(list) => list.fmt(f), ResponseType::Status { href, status } => { href.fmt(f)?; status.fmt(f) } } } } impl MultiStatus { pub fn new(response: Vec) -> Self { MultiStatus { namespaces: Namespaces::default(), response: List(response), response_description: None, sync_token: None, } } pub fn with_response(mut self, response: Response) -> Self { self.response.0.push(response); self } pub fn not_found(href: impl Into) -> Self { let mut response = Self::new(Vec::with_capacity(1)); response.response.0.push( Response::new_status([href], StatusCode::NOT_FOUND) .with_response_description("No resources found"), ); response } pub fn add_response(&mut self, response: Response) { self.response.0.push(response); } pub fn with_response_description(mut self, response_description: impl Into) -> Self { self.response_description = Some(ResponseDescription(response_description.into())); self } pub fn with_namespace(mut self, namespace: Namespace) -> Self { self.namespaces.set(namespace); self } pub fn set_namespace(&mut self, namespace: Namespace) { self.namespaces.set(namespace); } pub fn with_sync_token(mut self, sync_token: impl Into) -> Self { self.sync_token = Some(SyncToken(sync_token.into())); self } pub fn set_sync_token(&mut self, sync_token: impl Into) { self.sync_token = Some(SyncToken(sync_token.into())); } } impl Response { pub fn new_propstat(href: impl Into, propstat: Vec) -> Self { Response { href: href.into(), typ: ResponseType::PropStat(List(propstat)), error: None, response_description: None, location: None, } } pub fn new_status(href: T, status: StatusCode) -> Self where T: IntoIterator, H: Into, { let mut href = href.into_iter().map(|h| Href(h.into())); Response { href: href.next().unwrap(), typ: ResponseType::Status { href: List(href.collect()), status: Status(status), }, error: None, response_description: None, location: None, } } pub fn with_error(mut self, error: impl Into) -> Self { self.error = Some(error.into()); self } pub fn with_response_description(mut self, response_description: impl Into) -> Self { self.response_description = Some(ResponseDescription(response_description.into())); self } pub fn with_location(mut self, location: impl Into) -> Self { self.location = Some(Location(Href(location.into()))); self } } ================================================ FILE: crates/dav-proto/src/responses/property.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{XmlCdataEscape, XmlEscape}; use crate::{ responses::DeadPropertyFormat, schema::{ property::{ ActiveLock, CalDavProperty, CardDavProperty, Comp, DavProperty, DavValue, LockDiscovery, LockEntry, PrincipalProperty, Privilege, ReportSet, ResourceType, Rfc1123DateTime, SupportedCollation, SupportedLock, WebDavProperty, }, request::DavPropertyValue, response::{Ace, AclRestrictions, Href, List, PropResponse, SupportedPrivilege}, Namespace, Namespaces, }, }; use calcard::icalendar::ICalendarComponentType; use mail_parser::{ parsers::fields::date::{DOW, MONTH}, DateTime, }; use std::fmt::Display; use types::dead_property::DeadProperty; impl Display for PropResponse { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( f, "{}", self.namespaces, self.properties ) } } impl Display for DavPropertyValue { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let (name, attrs) = self.property.tag_name(); write!(f, "<{}", name)?; if let Some(attrs) = attrs { write!(f, " {attrs}")?; } if !matches!(self.value, DavValue::Null) { write!(f, ">{}", self.value, name) } else { write!(f, "/>") } } } impl Display for Rfc1123DateTime { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let dt = DateTime::from_timestamp(self.0); write!( f, "{}, {} {} {:04} {:02}:{:02}:{:02} GMT", DOW[dt.day_of_week() as usize], dt.day, MONTH .get(dt.month.saturating_sub(1) as usize) .copied() .unwrap_or_default(), dt.year, dt.hour, dt.minute, dt.second, ) } } impl Display for DavValue { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { DavValue::Timestamp(v) => { let dt = DateTime::from_timestamp(*v); write!( f, "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, ) } DavValue::Rfc1123Date(v) => v.fmt(f), DavValue::Uint64(v) => v.fmt(f), DavValue::String(v) => v.write_escaped_to(f), DavValue::ResourceTypes(v) => v.fmt(f), DavValue::ActiveLocks(v) => v.fmt(f), DavValue::LockEntries(v) => v.fmt(f), DavValue::ReportSets(v) => v.fmt(f), DavValue::CData(v) => v.write_cdata_escaped_to(f), DavValue::Components(v) => v.fmt(f), DavValue::Collations(v) => v.fmt(f), DavValue::Href(v) => v.fmt(f), DavValue::PrivilegeSet(v) => v.fmt(f), DavValue::Privileges(v) => v.fmt(f), DavValue::Acl(v) => v.fmt(f), DavValue::AclRestrictions(v) => v.fmt(f), DavValue::DeadProperty(v) => v.fmt(f), DavValue::SupportedAddressData => { write!( f, concat!( "", "", "", ) ) } DavValue::SupportedCalendarData => { write!( f, concat!( "", "", ) ) } DavValue::Response(v) => v.fmt(f), DavValue::VCard(_) | DavValue::ICalendar(_) | DavValue::Null => Ok(()), } } } impl DavValue { pub fn all_calendar_components() -> Self { DavValue::Components(List(vec![ Comp(ICalendarComponentType::VEvent), Comp(ICalendarComponentType::VTodo), Comp(ICalendarComponentType::VJournal), Comp(ICalendarComponentType::VFreebusy), Comp(ICalendarComponentType::VTimezone), Comp(ICalendarComponentType::VAlarm), Comp(ICalendarComponentType::Standard), Comp(ICalendarComponentType::Daylight), Comp(ICalendarComponentType::VAvailability), Comp(ICalendarComponentType::Available), Comp(ICalendarComponentType::Participant), Comp(ICalendarComponentType::VLocation), Comp(ICalendarComponentType::VResource), ])) } } impl DavProperty { fn tag_name(&self) -> (&str, Option<&str>) { ( match self { DavProperty::WebDav(prop) => match prop { WebDavProperty::CreationDate => "D:creationdate", WebDavProperty::DisplayName => "D:displayname", WebDavProperty::GetContentLanguage => "D:getcontentlanguage", WebDavProperty::GetContentLength => "D:getcontentlength", WebDavProperty::GetContentType => "D:getcontenttype", WebDavProperty::GetETag => "D:getetag", WebDavProperty::GetLastModified => "D:getlastmodified", WebDavProperty::ResourceType => "D:resourcetype", WebDavProperty::LockDiscovery => "D:lockdiscovery", WebDavProperty::SupportedLock => "D:supportedlock", WebDavProperty::CurrentUserPrincipal => "D:current-user-principal", WebDavProperty::QuotaAvailableBytes => "D:quota-available-bytes", WebDavProperty::QuotaUsedBytes => "D:quota-used-bytes", WebDavProperty::SupportedReportSet => "D:supported-report-set", WebDavProperty::SyncToken => "D:sync-token", WebDavProperty::Owner => "D:owner", WebDavProperty::Group => "D:group", WebDavProperty::SupportedPrivilegeSet => "D:supported-privilege-set", WebDavProperty::CurrentUserPrivilegeSet => "D:current-user-privilege-set", WebDavProperty::Acl => "D:acl", WebDavProperty::AclRestrictions => "D:acl-restrictions", WebDavProperty::InheritedAclSet => "D:inherited-acl-set", WebDavProperty::PrincipalCollectionSet => "D:principal-collection-set", WebDavProperty::GetCTag => "C:getctag", }, DavProperty::CardDav(prop) => match prop { CardDavProperty::AddressbookDescription => "B:addressbook-description", CardDavProperty::SupportedAddressData => "B:supported-address-data", CardDavProperty::SupportedCollationSet => "B:supported-collation-set", CardDavProperty::MaxResourceSize => "B:max-resource-size", CardDavProperty::AddressData(_) => "B:address-data", }, DavProperty::CalDav(prop) => match prop { CalDavProperty::CalendarDescription => "A:calendar-description", CalDavProperty::CalendarTimezone => "A:calendar-timezone", CalDavProperty::SupportedCalendarComponentSet => { "A:supported-calendar-component-set" } CalDavProperty::SupportedCalendarData => "A:supported-calendar-data", CalDavProperty::SupportedCollationSet => "A:supported-collation-set", CalDavProperty::MaxResourceSize => "A:max-resource-size", CalDavProperty::MinDateTime => "A:min-date-time", CalDavProperty::MaxDateTime => "A:max-date-time", CalDavProperty::MaxInstances => "A:max-instances", CalDavProperty::MaxAttendeesPerInstance => "A:max-attendees-per-instance", CalDavProperty::CalendarData(_) => "A:calendar-data", CalDavProperty::TimezoneServiceSet => "A:timezone-service-set", CalDavProperty::TimezoneId => "A:calendar-timezone-id", CalDavProperty::ScheduleDefaultCalendarURL => "A:schedule-default-calendar-URL", CalDavProperty::ScheduleTag => "A:schedule-tag", CalDavProperty::ScheduleCalendarTransp => "A:schedule-calendar-transp", }, DavProperty::Principal(prop) => match prop { PrincipalProperty::AlternateURISet => "D:alternate-URI-set", PrincipalProperty::PrincipalURL => "D:principal-URL", PrincipalProperty::GroupMemberSet => "D:group-member-set", PrincipalProperty::GroupMembership => "D:group-membership", PrincipalProperty::CalendarHomeSet => "A:calendar-home-set", PrincipalProperty::AddressbookHomeSet => "B:addressbook-home-set", PrincipalProperty::PrincipalAddress => "B:principal-address", PrincipalProperty::CalendarUserAddressSet => "A:calendar-user-address-set", PrincipalProperty::CalendarUserType => "A:calendar-user-type", PrincipalProperty::ScheduleInboxURL => "A:schedule-inbox-URL", PrincipalProperty::ScheduleOutboxURL => "A:schedule-outbox-URL", }, DavProperty::DeadProperty(dead) => { return (dead.name.as_str(), dead.attrs.as_deref()) } }, None, ) } pub fn namespace(&self) -> Namespace { match self { DavProperty::WebDav(WebDavProperty::GetCTag) => Namespace::CalendarServer, DavProperty::CardDav(_) | DavProperty::Principal(PrincipalProperty::AddressbookHomeSet) => Namespace::CardDav, DavProperty::CalDav(_) | DavProperty::Principal( PrincipalProperty::CalendarHomeSet | PrincipalProperty::CalendarUserAddressSet | PrincipalProperty::CalendarUserType | PrincipalProperty::ScheduleInboxURL | PrincipalProperty::ScheduleOutboxURL, ) => Namespace::CalDav, _ => Namespace::Dav, } } } impl AsRef for DavProperty { fn as_ref(&self) -> &str { self.tag_name().0 } } impl Display for ReportSet { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("")?; match self { ReportSet::SyncCollection => write!(f, ""), ReportSet::ExpandProperty => write!(f, ""), ReportSet::AddressbookQuery => write!(f, ""), ReportSet::AddressbookMultiGet => write!(f, ""), ReportSet::CalendarQuery => write!(f, ""), ReportSet::CalendarMultiGet => write!(f, ""), ReportSet::FreeBusyQuery => write!(f, ""), ReportSet::AclPrincipalPropSet => write!(f, ""), ReportSet::PrincipalMatch => write!(f, ""), ReportSet::PrincipalPropertySearch => write!(f, ""), ReportSet::PrincipalSearchPropertySet => { write!(f, "") } }?; f.write_str("") } } impl Display for DavProperty { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let (name, attrs) = self.tag_name(); if let Some(attrs) = attrs { write!(f, "<{name} {attrs}/>") } else { write!(f, "<{name}/>") } } } impl PropResponse { pub fn new(properties: Vec) -> Self { PropResponse { namespaces: Namespaces::default(), properties: List(properties), } } pub fn with_namespace(mut self, namespace: Namespace) -> Self { self.namespaces.set(namespace); self } } impl From for DavProperty { fn from(prop: WebDavProperty) -> Self { DavProperty::WebDav(prop) } } impl From for DavProperty { fn from(prop: CardDavProperty) -> Self { DavProperty::CardDav(prop) } } impl From for DavProperty { fn from(prop: CalDavProperty) -> Self { DavProperty::CalDav(prop) } } impl From for DavValue { fn from(v: String) -> Self { DavValue::String(v) } } impl From<&str> for DavValue { fn from(v: &str) -> Self { DavValue::String(v.to_string()) } } impl From for DavValue { fn from(v: u64) -> Self { DavValue::Uint64(v) } } impl From for DavValue { fn from(v: DateTime) -> Self { DavValue::Timestamp(v.to_timestamp()) } } impl From> for DavValue { fn from(v: Vec) -> Self { DavValue::ResourceTypes(List(v)) } } impl From> for DavValue { fn from(v: Vec) -> Self { DavValue::ReportSets(List(v)) } } impl From> for DavValue { fn from(v: Vec) -> Self { DavValue::Components(List(v)) } } impl From> for DavValue { fn from(v: Vec) -> Self { DavValue::Collations(List(v)) } } impl From for DavValue { fn from(v: SupportedLock) -> Self { DavValue::LockEntries(v.0) } } impl From> for DavValue { fn from(v: Vec) -> Self { DavValue::LockEntries(List(v)) } } impl From> for DavValue { fn from(v: Vec) -> Self { DavValue::ActiveLocks(List(v)) } } impl From for DavValue { fn from(v: LockDiscovery) -> Self { DavValue::ActiveLocks(v.0) } } impl From> for DavValue { fn from(v: Vec) -> Self { DavValue::PrivilegeSet(List(v)) } } impl From> for DavValue { fn from(v: Vec) -> Self { DavValue::Privileges(List(v)) } } impl From> for DavValue { fn from(v: Vec) -> Self { DavValue::Href(List(v)) } } impl From> for DavValue { fn from(v: Vec) -> Self { DavValue::Acl(List(v)) } } impl From for DavValue { fn from(v: AclRestrictions) -> Self { DavValue::AclRestrictions(v) } } impl From for DavValue { fn from(v: DeadProperty) -> Self { DavValue::DeadProperty(v) } } impl DavPropertyValue { pub fn new(property: impl Into, value: impl Into) -> Self { DavPropertyValue { property: property.into(), value: value.into(), } } pub fn empty(property: impl Into) -> Self { DavPropertyValue { property: property.into(), value: DavValue::Null, } } } ================================================ FILE: crates/dav-proto/src/responses/propstat.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::fmt::Display; use hyper::StatusCode; use crate::schema::{ request::DavPropertyValue, response::{Condition, List, Prop, PropStat, ResponseDescription, Status}, }; impl Display for PropStat { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "")?; self.prop.fmt(f)?; self.status.fmt(f)?; if let Some(error) = &self.error { error.fmt(f)?; } if let Some(response_description) = &self.response_description { response_description.fmt(f)?; } write!(f, "") } } impl Display for Prop { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.0) } } impl PropStat { #[cfg(test)] pub(crate) fn new(prop: impl Into) -> Self { PropStat { prop: Prop(List(vec![prop.into()])), status: Status(StatusCode::OK), error: None, response_description: None, } } pub fn new_list(props: Vec) -> Self { PropStat { prop: Prop(List(props)), status: Status(StatusCode::OK), error: None, response_description: None, } } pub fn with_prop(mut self, prop: impl Into) -> Self { self.prop.0 .0.push(prop.into()); self } pub fn with_status(mut self, status: StatusCode) -> Self { self.status = Status(status); self } pub fn with_error(mut self, error: impl Into) -> Self { self.error = Some(error.into()); self } pub fn with_response_description(mut self, response_description: impl Into) -> Self { self.response_description = Some(ResponseDescription(response_description.into())); self } } ================================================ FILE: crates/dav-proto/src/responses/schedule.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ responses::{XmlCdataEscape, XmlEscape}, schema::{ response::{ScheduleResponse, ScheduleResponseItem}, Namespaces, }, }; use std::fmt::Display; const NAMESPACE: Namespaces = Namespaces { cal: true, card: false, cs: false, }; impl Display for ScheduleResponse { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "")?; write!( f, "{}", self.items ) } } impl Display for ScheduleResponseItem { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "")?; write!(f, "{}", self.recipient)?; write!(f, "")?; self.request_status.write_escaped_to(f)?; write!(f, "")?; if let Some(calendar_data) = &self.calendar_data { write!(f, "")?; calendar_data.write_cdata_escaped_to(f)?; write!(f, "")?; } write!(f, "") } } ================================================ FILE: crates/dav-proto/src/schema/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::borrow::Cow; use request::TextMatch; pub mod property; pub mod request; pub mod response; #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct NamedElement { pub ns: Namespace, pub element: Element, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] #[repr(u8)] pub enum Namespace { Dav, CalDav, CardDav, CalendarServer, } #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct Namespaces { pub(crate) cal: bool, pub(crate) card: bool, pub(crate) cs: bool, } impl Namespaces { pub fn set(&mut self, ns: Namespace) { match ns { Namespace::CalDav => self.cal = true, Namespace::CardDav => self.card = true, Namespace::CalendarServer => self.cs = true, Namespace::Dav => {} } } } impl Namespace { pub fn try_parse(value: &[u8]) -> Option { hashify::tiny_map!(value, "DAV:" => Namespace::Dav, "urn:ietf:params:xml:ns:caldav" => Namespace::CalDav, "urn:ietf:params:xml:ns:carddav" => Namespace::CardDav, "http://calendarserver.org/ns/" => Namespace::CalendarServer, "http://calendarserver.org/ns" => Namespace::CalendarServer ) } pub fn prefix(&self) -> &str { match self { Namespace::Dav => "D", Namespace::CalDav => "A", Namespace::CardDav => "B", Namespace::CalendarServer => "C", } } pub fn namespace(&self) -> &'static str { match self { Namespace::Dav => "DAV:", Namespace::CalDav => "urn:ietf:params:xml:ns:caldav", Namespace::CardDav => "urn:ietf:params:xml:ns:carddav", Namespace::CalendarServer => "http://calendarserver.org/ns/", } } } impl AsRef for Namespace { fn as_ref(&self) -> &str { self.namespace() } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub enum Element { Abstract, Ace, Acl, AclPrincipalPropSet, AclRestrictions, Activelock, ActivityCheckoutSet, ActivityCollectionSet, ActivitySet, ActivityVersionSet, Add, AddMember, AddedVersion, AddressData, AddressDataType, Addressbook, AddressbookDescription, AddressbookHomeSet, AddressbookMultiget, AddressbookQuery, After, All, Allcomp, AllowClientDefinedUri, AllowedAttendeeSchedulingObjectChange, AllowedOrganizerSchedulingObjectChange, AllowedPrincipal, Allprop, AlternateUriSet, And, AnyOtherProperty, ApplyToVersion, ApplyToPrincipalCollectionSet, Ascending, Authenticated, AutoMergeSet, AutoUpdate, AutoVersion, Baseline, BaselineCollection, BaselineControl, BaselineControlResponse, BaselineControlledCollection, BaselineControlledCollectionSet, Basicsearch, Basicsearchschema, Before, Bind, BindResponse, BindingName, Calendar, CalendarAvailability, CalendarData, CalendarDescription, CalendarHomeSet, CalendarMultiget, CalendarQuery, CalendarTimezone, CalendarTimezoneId, CalendarUserAddressSet, CalendarUserType, Caseless, ChangedVersion, CheckedIn, CheckedOut, Checkin, CheckinActivity, CheckinFork, CheckinResponse, Checkout, CheckoutCheckin, CheckoutFork, CheckoutResponse, CheckoutSet, CheckoutUnlockedCheckin, Collection, Comment, CommonAncestor, Comp, CompFilter, CompareBaseline, CompareBaselineReport, ConflictPreview, Contains, Creationdate, CreatorDisplayname, CurrentActivitySet, CurrentUserPrincipal, CurrentUserPrivilegeSet, CurrentWorkspaceSet, Datatype, DefaultCalendarNeeded, DeletedVersion, Deny, DenyBeforeGrant, Depth, Descending, Description, Discouraged, Displayname, Eq, Error, Exclusive, Expand, ExpandProperty, Filter, First, Forbidden, ForkOk, FreeBusyQuery, From, Getcontentlanguage, Getcontentlength, Getcontenttype, Getctag, Getetag, Getlastmodified, Grammar, Grant, GrantOnly, Group, GroupMemberSet, GroupMembership, Gt, Gte, Href, IgnorePreview, Include, IncludeVersions, Inherited, InheritedAclSet, Invert, IsCollection, IsDefined, IsNotDefined, KeepCheckedOut, Label, LabelName, LabelNameSet, LabelResponse, LanguageDefined, LanguageMatches, Last, LatestActivityVersion, LatestActivityVersionReport, Like, Limit, LimitFreebusySet, LimitRecurrenceSet, LimitedNumberOfAces, Literal, LocateByHistory, Location, LockTokenSubmitted, Lockdiscovery, LockedCheckout, Lockentry, Lockinfo, Lockroot, Lockscope, Locktoken, Locktype, Lt, Lte, ManagedAttachmentsServerUrl, Match, MaxAttachmentSize, MaxAttachmentsPerResource, MaxAttendeesPerInstance, MaxDateTime, MaxInstances, MaxResourceSize, Merge, MergePreview, MergePreviewReport, MergeSet, MinDateTime, MissingRequiredPrincipal, Mkactivity, MkactivityResponse, Mkcalendar, MkcalendarResponse, Mkcol, MkcolResponse, Mkredirectref, MkredirectrefResponse, Mkworkspace, MkworkspaceResponse, Mount, Multistatus, NeedPrivileges, New, NoAbstract, NoAceConflict, NoAutoMerge, NoCheckout, NoConflictingLock, NoInheritedAceConflict, NoInvert, NoProtectedAceConflict, NoUidConflict, Not, NotSupportedPrivilege, Nresults, Opaque, Opdesc, Open, OperandLiteral, OperandProperty, OperandTypedLiteral, Operators, Options, OptionsResponse, Or, Order, OrderMember, Orderby, OrderingType, Orderpatch, OrderpatchResponse, Owner, ParamFilter, Parent, ParentSet, Permanent, Position, PredecessorSet, Principal, PrincipalUrl, PrincipalAddress, PrincipalCollectionSet, PrincipalMatch, PrincipalProperty, PrincipalPropertySearch, PrincipalSearchProperty, PrincipalSearchPropertySet, Privilege, Prop, PropFilter, Propdesc, Properties, Property, PropertySearch, Propertyupdate, Propfind, Propname, Propstat, Protected, QuerySchema, QuerySchemaDiscovery, QuotaAvailableBytes, QuotaUsedBytes, Read, ReadAcl, ReadCurrentUserPrivilegeSet, ReadFreeBusy, Rebind, RebindResponse, Recipient, RecognizedPrincipal, RedirectLifetime, Redirectref, Reftarget, Remove, Report, RequestStatus, RequiredPrincipal, Resource, ResourceId, Resourcetype, Response, Responsedescription, RootVersion, SameOrganizerInAllComponents, ScheduleCalendarTransp, ScheduleDefaultCalendarUrl, ScheduleDeliver, ScheduleDeliverInvite, ScheduleDeliverReply, ScheduleInbox, ScheduleInboxUrl, ScheduleOutbox, ScheduleOutboxUrl, ScheduleQueryFreebusy, ScheduleResponse, ScheduleSend, ScheduleSendFreebusy, ScheduleSendInvite, ScheduleSendReply, ScheduleTag, Scope, Score, Searchable, Segment, Select, Selectable, Self_, Set, Shared, Sortable, Source, Status, SubactivitySet, SubbaselineSet, SuccessorSet, SupportedAddressData, SupportedCalendarComponentSet, SupportedCalendarData, SupportedCollation, SupportedCollationSet, SupportedFilter, SupportedLiveProperty, SupportedLivePropertySet, SupportedMethod, SupportedMethodSet, SupportedPrivilege, SupportedPrivilegeSet, SupportedQueryGrammar, SupportedQueryGrammarSet, SupportedReport, SupportedReportSet, SupportedRscale, SupportedRscaleSet, Supportedlock, SyncCollection, SyncLevel, SyncToken, Target, Temporary, TextMatch, TimeRange, Timeout, Timezone, TimezoneId, TimezoneServiceSet, Transparent, TypedLiteral, Unauthenticated, Unbind, UnbindResponse, Uncheckout, UncheckoutResponse, UniqueSchedulingObjectResource, Unlock, Unreserved, Update, UpdatePreview, Updateredirectref, UpdateredirectrefResponse, Url, Username, ValidOrganizer, ValidScheduleDefaultCalendarUrl, ValidSchedulingMessage, Version, VersionControl, VersionControlResponse, VersionControlledBinding, VersionControlledBindingSet, VersionControlledConfiguration, VersionHistory, VersionHistoryCollectionSet, VersionHistorySet, VersionName, VersionSet, VersionTree, Where, Workspace, WorkspaceCheckoutSet, WorkspaceCollectionSet, Write, WriteAcl, WriteContent, WriteProperties, } impl Element { pub fn try_parse(value: &[u8]) -> Option<&Self> { hashify::map!(value, Element, "abstract" => Element::Abstract, "ace" => Element::Ace, "acl" => Element::Acl, "acl-principal-prop-set" => Element::AclPrincipalPropSet, "acl-restrictions" => Element::AclRestrictions, "activelock" => Element::Activelock, "activity-checkout-set" => Element::ActivityCheckoutSet, "activity-collection-set" => Element::ActivityCollectionSet, "activity-set" => Element::ActivitySet, "activity-version-set" => Element::ActivityVersionSet, "add" => Element::Add, "add-member" => Element::AddMember, "added-version" => Element::AddedVersion, "address-data" => Element::AddressData, "address-data-type" => Element::AddressDataType, "addressbook" => Element::Addressbook, "addressbook-description" => Element::AddressbookDescription, "addressbook-home-set" => Element::AddressbookHomeSet, "addressbook-multiget" => Element::AddressbookMultiget, "addressbook-query" => Element::AddressbookQuery, "after" => Element::After, "all" => Element::All, "allcomp" => Element::Allcomp, "allow-client-defined-uri" => Element::AllowClientDefinedUri, "allowed-attendee-scheduling-object-change" => Element::AllowedAttendeeSchedulingObjectChange, "allowed-organizer-scheduling-object-change" => Element::AllowedOrganizerSchedulingObjectChange, "allowed-principal" => Element::AllowedPrincipal, "allprop" => Element::Allprop, "alternate-URI-set" => Element::AlternateUriSet, "and" => Element::And, "any-other-property" => Element::AnyOtherProperty, "apply-to-version" => Element::ApplyToVersion, "apply-to-principal-collection-set" => Element::ApplyToPrincipalCollectionSet, "ascending" => Element::Ascending, "authenticated" => Element::Authenticated, "auto-merge-set" => Element::AutoMergeSet, "auto-update" => Element::AutoUpdate, "auto-version" => Element::AutoVersion, "baseline" => Element::Baseline, "baseline-collection" => Element::BaselineCollection, "baseline-control" => Element::BaselineControl, "baseline-control-response" => Element::BaselineControlResponse, "baseline-controlled-collection" => Element::BaselineControlledCollection, "baseline-controlled-collection-set" => Element::BaselineControlledCollectionSet, "basicsearch" => Element::Basicsearch, "basicsearchschema" => Element::Basicsearchschema, "before" => Element::Before, "bind" => Element::Bind, "bind-response" => Element::BindResponse, "binding-name" => Element::BindingName, "calendar" => Element::Calendar, "calendar-availability" => Element::CalendarAvailability, "calendar-data" => Element::CalendarData, "calendar-description" => Element::CalendarDescription, "calendar-home-set" => Element::CalendarHomeSet, "calendar-multiget" => Element::CalendarMultiget, "calendar-query" => Element::CalendarQuery, "calendar-timezone" => Element::CalendarTimezone, "calendar-timezone-id" => Element::CalendarTimezoneId, "calendar-user-address-set" => Element::CalendarUserAddressSet, "calendar-user-type" => Element::CalendarUserType, "caseless" => Element::Caseless, "changed-version" => Element::ChangedVersion, "checked-in" => Element::CheckedIn, "checked-out" => Element::CheckedOut, "checkin" => Element::Checkin, "checkin-activity" => Element::CheckinActivity, "checkin-fork" => Element::CheckinFork, "checkin-response" => Element::CheckinResponse, "checkout" => Element::Checkout, "checkout-checkin" => Element::CheckoutCheckin, "checkout-fork" => Element::CheckoutFork, "checkout-response" => Element::CheckoutResponse, "checkout-set" => Element::CheckoutSet, "checkout-unlocked-checkin" => Element::CheckoutUnlockedCheckin, "collection" => Element::Collection, "comment" => Element::Comment, "common-ancestor" => Element::CommonAncestor, "comp" => Element::Comp, "comp-filter" => Element::CompFilter, "compare-baseline" => Element::CompareBaseline, "compare-baseline-report" => Element::CompareBaselineReport, "conflict-preview" => Element::ConflictPreview, "contains" => Element::Contains, "creationdate" => Element::Creationdate, "creator-displayname" => Element::CreatorDisplayname, "current-activity-set" => Element::CurrentActivitySet, "current-user-principal" => Element::CurrentUserPrincipal, "current-user-privilege-set" => Element::CurrentUserPrivilegeSet, "current-workspace-set" => Element::CurrentWorkspaceSet, "datatype" => Element::Datatype, "default-calendar-needed" => Element::DefaultCalendarNeeded, "deleted-version" => Element::DeletedVersion, "deny" => Element::Deny, "deny-before-grant" => Element::DenyBeforeGrant, "depth" => Element::Depth, "descending" => Element::Descending, "description" => Element::Description, "discouraged" => Element::Discouraged, "displayname" => Element::Displayname, "eq" => Element::Eq, "error" => Element::Error, "exclusive" => Element::Exclusive, "expand" => Element::Expand, "expand-property" => Element::ExpandProperty, "filter" => Element::Filter, "first" => Element::First, "forbidden" => Element::Forbidden, "fork-ok" => Element::ForkOk, "free-busy-query" => Element::FreeBusyQuery, "from" => Element::From, "getcontentlanguage" => Element::Getcontentlanguage, "getcontentlength" => Element::Getcontentlength, "getcontenttype" => Element::Getcontenttype, "getetag" => Element::Getetag, "getctag" => Element::Getctag, "getlastmodified" => Element::Getlastmodified, "grammar" => Element::Grammar, "grant" => Element::Grant, "grant-only" => Element::GrantOnly, "group" => Element::Group, "group-member-set" => Element::GroupMemberSet, "group-membership" => Element::GroupMembership, "gt" => Element::Gt, "gte" => Element::Gte, "href" => Element::Href, "ignore-preview" => Element::IgnorePreview, "include" => Element::Include, "include-versions" => Element::IncludeVersions, "inherited" => Element::Inherited, "inherited-acl-set" => Element::InheritedAclSet, "invert" => Element::Invert, "is-collection" => Element::IsCollection, "is-defined" => Element::IsDefined, "is-not-defined" => Element::IsNotDefined, "keep-checked-out" => Element::KeepCheckedOut, "label" => Element::Label, "label-name" => Element::LabelName, "label-name-set" => Element::LabelNameSet, "label-response" => Element::LabelResponse, "language-defined" => Element::LanguageDefined, "language-matches" => Element::LanguageMatches, "last" => Element::Last, "latest-activity-version" => Element::LatestActivityVersion, "latest-activity-version-report" => Element::LatestActivityVersionReport, "like" => Element::Like, "limit" => Element::Limit, "limit-freebusy-set" => Element::LimitFreebusySet, "limit-recurrence-set" => Element::LimitRecurrenceSet, "limited-number-of-aces" => Element::LimitedNumberOfAces, "literal" => Element::Literal, "locate-by-history" => Element::LocateByHistory, "location" => Element::Location, "lock-token-submitted" => Element::LockTokenSubmitted, "lockdiscovery" => Element::Lockdiscovery, "locked-checkout" => Element::LockedCheckout, "lockentry" => Element::Lockentry, "lockinfo" => Element::Lockinfo, "lockroot" => Element::Lockroot, "lockscope" => Element::Lockscope, "locktoken" => Element::Locktoken, "locktype" => Element::Locktype, "lt" => Element::Lt, "lte" => Element::Lte, "managed-attachments-server-URL" => Element::ManagedAttachmentsServerUrl, "match" => Element::Match, "max-attachment-size" => Element::MaxAttachmentSize, "max-attachments-per-resource" => Element::MaxAttachmentsPerResource, "max-attendees-per-instance" => Element::MaxAttendeesPerInstance, "max-date-time" => Element::MaxDateTime, "max-instances" => Element::MaxInstances, "max-resource-size" => Element::MaxResourceSize, "merge" => Element::Merge, "merge-preview" => Element::MergePreview, "merge-preview-report" => Element::MergePreviewReport, "merge-set" => Element::MergeSet, "min-date-time" => Element::MinDateTime, "missing-required-principal" => Element::MissingRequiredPrincipal, "mkactivity" => Element::Mkactivity, "mkactivity-response" => Element::MkactivityResponse, "mkcalendar" => Element::Mkcalendar, "mkcalendar-response" => Element::MkcalendarResponse, "mkcol" => Element::Mkcol, "mkcol-response" => Element::MkcolResponse, "mkredirectref" => Element::Mkredirectref, "mkredirectref-response" => Element::MkredirectrefResponse, "mkworkspace" => Element::Mkworkspace, "mkworkspace-response" => Element::MkworkspaceResponse, "mount" => Element::Mount, "multistatus" => Element::Multistatus, "need-privileges" => Element::NeedPrivileges, "new" => Element::New, "no-abstract" => Element::NoAbstract, "no-ace-conflict" => Element::NoAceConflict, "no-auto-merge" => Element::NoAutoMerge, "no-checkout" => Element::NoCheckout, "no-conflicting-lock" => Element::NoConflictingLock, "no-inherited-ace-conflict" => Element::NoInheritedAceConflict, "no-invert" => Element::NoInvert, "no-protected-ace-conflict" => Element::NoProtectedAceConflict, "no-uid-conflict" => Element::NoUidConflict, "not" => Element::Not, "not-supported-privilege" => Element::NotSupportedPrivilege, "nresults" => Element::Nresults, "opaque" => Element::Opaque, "opdesc" => Element::Opdesc, "open" => Element::Open, "operand-literal" => Element::OperandLiteral, "operand-property" => Element::OperandProperty, "operand-typed-literal" => Element::OperandTypedLiteral, "operators" => Element::Operators, "options" => Element::Options, "options-response" => Element::OptionsResponse, "or" => Element::Or, "order" => Element::Order, "order-member" => Element::OrderMember, "orderby" => Element::Orderby, "ordering-type" => Element::OrderingType, "orderpatch" => Element::Orderpatch, "orderpatch-response" => Element::OrderpatchResponse, "owner" => Element::Owner, "param-filter" => Element::ParamFilter, "parent" => Element::Parent, "parent-set" => Element::ParentSet, "permanent" => Element::Permanent, "position" => Element::Position, "predecessor-set" => Element::PredecessorSet, "principal" => Element::Principal, "principal-URL" => Element::PrincipalUrl, "principal-address" => Element::PrincipalAddress, "principal-collection-set" => Element::PrincipalCollectionSet, "principal-match" => Element::PrincipalMatch, "principal-property" => Element::PrincipalProperty, "principal-property-search" => Element::PrincipalPropertySearch, "principal-search-property" => Element::PrincipalSearchProperty, "principal-search-property-set" => Element::PrincipalSearchPropertySet, "privilege" => Element::Privilege, "prop" => Element::Prop, "prop-filter" => Element::PropFilter, "propdesc" => Element::Propdesc, "properties" => Element::Properties, "property" => Element::Property, "property-search" => Element::PropertySearch, "propertyupdate" => Element::Propertyupdate, "propfind" => Element::Propfind, "propname" => Element::Propname, "propstat" => Element::Propstat, "protected" => Element::Protected, "query-schema" => Element::QuerySchema, "query-schema-discovery" => Element::QuerySchemaDiscovery, "quota-available-bytes" => Element::QuotaAvailableBytes, "quota-used-bytes" => Element::QuotaUsedBytes, "read" => Element::Read, "read-acl" => Element::ReadAcl, "read-current-user-privilege-set" => Element::ReadCurrentUserPrivilegeSet, "read-free-busy" => Element::ReadFreeBusy, "rebind" => Element::Rebind, "rebind-response" => Element::RebindResponse, "recipient" => Element::Recipient, "recognized-principal" => Element::RecognizedPrincipal, "redirect-lifetime" => Element::RedirectLifetime, "redirectref" => Element::Redirectref, "reftarget" => Element::Reftarget, "remove" => Element::Remove, "report" => Element::Report, "request-status" => Element::RequestStatus, "required-principal" => Element::RequiredPrincipal, "resource" => Element::Resource, "resource-id" => Element::ResourceId, "resourcetype" => Element::Resourcetype, "response" => Element::Response, "responsedescription" => Element::Responsedescription, "root-version" => Element::RootVersion, "same-organizer-in-all-components" => Element::SameOrganizerInAllComponents, "schedule-calendar-transp" => Element::ScheduleCalendarTransp, "schedule-default-calendar-URL" => Element::ScheduleDefaultCalendarUrl, "schedule-deliver" => Element::ScheduleDeliver, "schedule-deliver-invite" => Element::ScheduleDeliverInvite, "schedule-deliver-reply" => Element::ScheduleDeliverReply, "schedule-inbox" => Element::ScheduleInbox, "schedule-inbox-URL" => Element::ScheduleInboxUrl, "schedule-outbox" => Element::ScheduleOutbox, "schedule-outbox-URL" => Element::ScheduleOutboxUrl, "schedule-query-freebusy" => Element::ScheduleQueryFreebusy, "schedule-response" => Element::ScheduleResponse, "schedule-send" => Element::ScheduleSend, "schedule-send-freebusy" => Element::ScheduleSendFreebusy, "schedule-send-invite" => Element::ScheduleSendInvite, "schedule-send-reply" => Element::ScheduleSendReply, "schedule-tag" => Element::ScheduleTag, "scope" => Element::Scope, "score" => Element::Score, "searchable" => Element::Searchable, "segment" => Element::Segment, "select" => Element::Select, "selectable" => Element::Selectable, "self" => Element::Self_, "set" => Element::Set, "shared" => Element::Shared, "sortable" => Element::Sortable, "source" => Element::Source, "status" => Element::Status, "subactivity-set" => Element::SubactivitySet, "subbaseline-set" => Element::SubbaselineSet, "successor-set" => Element::SuccessorSet, "supported-address-data" => Element::SupportedAddressData, "supported-calendar-component-set" => Element::SupportedCalendarComponentSet, "supported-calendar-data" => Element::SupportedCalendarData, "supported-collation" => Element::SupportedCollation, "supported-collation-set" => Element::SupportedCollationSet, "supported-filter" => Element::SupportedFilter, "supported-live-property" => Element::SupportedLiveProperty, "supported-live-property-set" => Element::SupportedLivePropertySet, "supported-method" => Element::SupportedMethod, "supported-method-set" => Element::SupportedMethodSet, "supported-privilege" => Element::SupportedPrivilege, "supported-privilege-set" => Element::SupportedPrivilegeSet, "supported-query-grammar" => Element::SupportedQueryGrammar, "supported-query-grammar-set" => Element::SupportedQueryGrammarSet, "supported-report" => Element::SupportedReport, "supported-report-set" => Element::SupportedReportSet, "supported-rscale" => Element::SupportedRscale, "supported-rscale-set" => Element::SupportedRscaleSet, "supportedlock" => Element::Supportedlock, "sync-collection" => Element::SyncCollection, "sync-level" => Element::SyncLevel, "sync-token" => Element::SyncToken, "target" => Element::Target, "temporary" => Element::Temporary, "text-match" => Element::TextMatch, "time-range" => Element::TimeRange, "timeout" => Element::Timeout, "timezone" => Element::Timezone, "timezone-id" => Element::TimezoneId, "timezone-service-set" => Element::TimezoneServiceSet, "transparent" => Element::Transparent, "typed-literal" => Element::TypedLiteral, "unauthenticated" => Element::Unauthenticated, "unbind" => Element::Unbind, "unbind-response" => Element::UnbindResponse, "uncheckout" => Element::Uncheckout, "uncheckout-response" => Element::UncheckoutResponse, "unique-scheduling-object-resource" => Element::UniqueSchedulingObjectResource, "unlock" => Element::Unlock, "unreserved" => Element::Unreserved, "update" => Element::Update, "update-preview" => Element::UpdatePreview, "updateredirectref" => Element::Updateredirectref, "updateredirectref-response" => Element::UpdateredirectrefResponse, "url" => Element::Url, "username" => Element::Username, "valid-organizer" => Element::ValidOrganizer, "valid-schedule-default-calendar-URL" => Element::ValidScheduleDefaultCalendarUrl, "valid-scheduling-message" => Element::ValidSchedulingMessage, "version" => Element::Version, "version-control" => Element::VersionControl, "version-control-response" => Element::VersionControlResponse, "version-controlled-binding" => Element::VersionControlledBinding, "version-controlled-binding-set" => Element::VersionControlledBindingSet, "version-controlled-configuration" => Element::VersionControlledConfiguration, "version-history" => Element::VersionHistory, "version-history-collection-set" => Element::VersionHistoryCollectionSet, "version-history-set" => Element::VersionHistorySet, "version-name" => Element::VersionName, "version-set" => Element::VersionSet, "version-tree" => Element::VersionTree, "where" => Element::Where, "workspace" => Element::Workspace, "workspace-checkout-set" => Element::WorkspaceCheckoutSet, "workspace-collection-set" => Element::WorkspaceCollectionSet, "write" => Element::Write, "write-acl" => Element::WriteAcl, "write-content" => Element::WriteContent, "write-properties" => Element::WriteProperties, ) } } impl AsRef for Element { fn as_ref(&self) -> &str { match self { Element::Abstract => "abstract", Element::Ace => "ace", Element::Acl => "acl", Element::AclPrincipalPropSet => "acl-principal-prop-set", Element::AclRestrictions => "acl-restrictions", Element::Activelock => "activelock", Element::ActivityCheckoutSet => "activity-checkout-set", Element::ActivityCollectionSet => "activity-collection-set", Element::ActivitySet => "activity-set", Element::ActivityVersionSet => "activity-version-set", Element::Add => "add", Element::AddMember => "add-member", Element::AddedVersion => "added-version", Element::AddressData => "address-data", Element::AddressDataType => "address-data-type", Element::Addressbook => "addressbook", Element::AddressbookDescription => "addressbook-description", Element::AddressbookHomeSet => "addressbook-home-set", Element::AddressbookMultiget => "addressbook-multiget", Element::AddressbookQuery => "addressbook-query", Element::After => "after", Element::All => "all", Element::Allcomp => "allcomp", Element::AllowClientDefinedUri => "allow-client-defined-uri", Element::AllowedAttendeeSchedulingObjectChange => { "allowed-attendee-scheduling-object-change" } Element::AllowedOrganizerSchedulingObjectChange => { "allowed-organizer-scheduling-object-change" } Element::AllowedPrincipal => "allowed-principal", Element::Allprop => "allprop", Element::AlternateUriSet => "alternate-URI-set", Element::And => "and", Element::AnyOtherProperty => "any-other-property", Element::ApplyToVersion => "apply-to-version", Element::ApplyToPrincipalCollectionSet => "apply-to-principal-collection-set", Element::Ascending => "ascending", Element::Authenticated => "authenticated", Element::AutoMergeSet => "auto-merge-set", Element::AutoUpdate => "auto-update", Element::AutoVersion => "auto-version", Element::Baseline => "baseline", Element::BaselineCollection => "baseline-collection", Element::BaselineControl => "baseline-control", Element::BaselineControlResponse => "baseline-control-response", Element::BaselineControlledCollection => "baseline-controlled-collection", Element::BaselineControlledCollectionSet => "baseline-controlled-collection-set", Element::Basicsearch => "basicsearch", Element::Basicsearchschema => "basicsearchschema", Element::Before => "before", Element::Bind => "bind", Element::BindResponse => "bind-response", Element::BindingName => "binding-name", Element::Calendar => "calendar", Element::CalendarAvailability => "calendar-availability", Element::CalendarData => "calendar-data", Element::CalendarDescription => "calendar-description", Element::CalendarHomeSet => "calendar-home-set", Element::CalendarMultiget => "calendar-multiget", Element::CalendarQuery => "calendar-query", Element::CalendarTimezone => "calendar-timezone", Element::CalendarTimezoneId => "calendar-timezone-id", Element::CalendarUserAddressSet => "calendar-user-address-set", Element::CalendarUserType => "calendar-user-type", Element::Caseless => "caseless", Element::ChangedVersion => "changed-version", Element::CheckedIn => "checked-in", Element::CheckedOut => "checked-out", Element::Checkin => "checkin", Element::CheckinActivity => "checkin-activity", Element::CheckinFork => "checkin-fork", Element::CheckinResponse => "checkin-response", Element::Checkout => "checkout", Element::CheckoutCheckin => "checkout-checkin", Element::CheckoutFork => "checkout-fork", Element::CheckoutResponse => "checkout-response", Element::CheckoutSet => "checkout-set", Element::CheckoutUnlockedCheckin => "checkout-unlocked-checkin", Element::Collection => "collection", Element::Comment => "comment", Element::CommonAncestor => "common-ancestor", Element::Comp => "comp", Element::CompFilter => "comp-filter", Element::CompareBaseline => "compare-baseline", Element::CompareBaselineReport => "compare-baseline-report", Element::ConflictPreview => "conflict-preview", Element::Contains => "contains", Element::Creationdate => "creationdate", Element::CreatorDisplayname => "creator-displayname", Element::CurrentActivitySet => "current-activity-set", Element::CurrentUserPrincipal => "current-user-principal", Element::CurrentUserPrivilegeSet => "current-user-privilege-set", Element::CurrentWorkspaceSet => "current-workspace-set", Element::Datatype => "datatype", Element::DefaultCalendarNeeded => "default-calendar-needed", Element::DeletedVersion => "deleted-version", Element::Deny => "deny", Element::DenyBeforeGrant => "deny-before-grant", Element::Depth => "depth", Element::Descending => "descending", Element::Description => "description", Element::Discouraged => "discouraged", Element::Displayname => "displayname", Element::Eq => "eq", Element::Error => "error", Element::Exclusive => "exclusive", Element::Expand => "expand", Element::ExpandProperty => "expand-property", Element::Filter => "filter", Element::First => "first", Element::Forbidden => "forbidden", Element::ForkOk => "fork-ok", Element::FreeBusyQuery => "free-busy-query", Element::From => "from", Element::Getcontentlanguage => "getcontentlanguage", Element::Getcontentlength => "getcontentlength", Element::Getcontenttype => "getcontenttype", Element::Getetag => "getetag", Element::Getctag => "getctag", Element::Getlastmodified => "getlastmodified", Element::Grammar => "grammar", Element::Grant => "grant", Element::GrantOnly => "grant-only", Element::Group => "group", Element::GroupMemberSet => "group-member-set", Element::GroupMembership => "group-membership", Element::Gt => "gt", Element::Gte => "gte", Element::Href => "href", Element::IgnorePreview => "ignore-preview", Element::Include => "include", Element::IncludeVersions => "include-versions", Element::Inherited => "inherited", Element::InheritedAclSet => "inherited-acl-set", Element::Invert => "invert", Element::IsCollection => "is-collection", Element::IsDefined => "is-defined", Element::IsNotDefined => "is-not-defined", Element::KeepCheckedOut => "keep-checked-out", Element::Label => "label", Element::LabelName => "label-name", Element::LabelNameSet => "label-name-set", Element::LabelResponse => "label-response", Element::LanguageDefined => "language-defined", Element::LanguageMatches => "language-matches", Element::Last => "last", Element::LatestActivityVersion => "latest-activity-version", Element::LatestActivityVersionReport => "latest-activity-version-report", Element::Like => "like", Element::Limit => "limit", Element::LimitFreebusySet => "limit-freebusy-set", Element::LimitRecurrenceSet => "limit-recurrence-set", Element::LimitedNumberOfAces => "limited-number-of-aces", Element::Literal => "literal", Element::LocateByHistory => "locate-by-history", Element::Location => "location", Element::LockTokenSubmitted => "lock-token-submitted", Element::Lockdiscovery => "lockdiscovery", Element::LockedCheckout => "locked-checkout", Element::Lockentry => "lockentry", Element::Lockinfo => "lockinfo", Element::Lockroot => "lockroot", Element::Lockscope => "lockscope", Element::Locktoken => "locktoken", Element::Locktype => "locktype", Element::Lt => "lt", Element::Lte => "lte", Element::ManagedAttachmentsServerUrl => "managed-attachments-server-URL", Element::Match => "match", Element::MaxAttachmentSize => "max-attachment-size", Element::MaxAttachmentsPerResource => "max-attachments-per-resource", Element::MaxAttendeesPerInstance => "max-attendees-per-instance", Element::MaxDateTime => "max-date-time", Element::MaxInstances => "max-instances", Element::MaxResourceSize => "max-resource-size", Element::Merge => "merge", Element::MergePreview => "merge-preview", Element::MergePreviewReport => "merge-preview-report", Element::MergeSet => "merge-set", Element::MinDateTime => "min-date-time", Element::MissingRequiredPrincipal => "missing-required-principal", Element::Mkactivity => "mkactivity", Element::MkactivityResponse => "mkactivity-response", Element::Mkcalendar => "mkcalendar", Element::MkcalendarResponse => "mkcalendar-response", Element::Mkcol => "mkcol", Element::MkcolResponse => "mkcol-response", Element::Mkredirectref => "mkredirectref", Element::MkredirectrefResponse => "mkredirectref-response", Element::Mkworkspace => "mkworkspace", Element::MkworkspaceResponse => "mkworkspace-response", Element::Mount => "mount", Element::Multistatus => "multistatus", Element::NeedPrivileges => "need-privileges", Element::New => "new", Element::NoAbstract => "no-abstract", Element::NoAceConflict => "no-ace-conflict", Element::NoAutoMerge => "no-auto-merge", Element::NoCheckout => "no-checkout", Element::NoConflictingLock => "no-conflicting-lock", Element::NoInheritedAceConflict => "no-inherited-ace-conflict", Element::NoInvert => "no-invert", Element::NoProtectedAceConflict => "no-protected-ace-conflict", Element::NoUidConflict => "no-uid-conflict", Element::Not => "not", Element::NotSupportedPrivilege => "not-supported-privilege", Element::Nresults => "nresults", Element::Opaque => "opaque", Element::Opdesc => "opdesc", Element::Open => "open", Element::OperandLiteral => "operand-literal", Element::OperandProperty => "operand-property", Element::OperandTypedLiteral => "operand-typed-literal", Element::Operators => "operators", Element::Options => "options", Element::OptionsResponse => "options-response", Element::Or => "or", Element::Order => "order", Element::OrderMember => "order-member", Element::Orderby => "orderby", Element::OrderingType => "ordering-type", Element::Orderpatch => "orderpatch", Element::OrderpatchResponse => "orderpatch-response", Element::Owner => "owner", Element::ParamFilter => "param-filter", Element::Parent => "parent", Element::ParentSet => "parent-set", Element::Permanent => "permanent", Element::Position => "position", Element::PredecessorSet => "predecessor-set", Element::Principal => "principal", Element::PrincipalUrl => "principal-URL", Element::PrincipalAddress => "principal-address", Element::PrincipalCollectionSet => "principal-collection-set", Element::PrincipalMatch => "principal-match", Element::PrincipalProperty => "principal-property", Element::PrincipalPropertySearch => "principal-property-search", Element::PrincipalSearchProperty => "principal-search-property", Element::PrincipalSearchPropertySet => "principal-search-property-set", Element::Privilege => "privilege", Element::Prop => "prop", Element::PropFilter => "prop-filter", Element::Propdesc => "propdesc", Element::Properties => "properties", Element::Property => "property", Element::PropertySearch => "property-search", Element::Propertyupdate => "propertyupdate", Element::Propfind => "propfind", Element::Propname => "propname", Element::Propstat => "propstat", Element::Protected => "protected", Element::QuerySchema => "query-schema", Element::QuerySchemaDiscovery => "query-schema-discovery", Element::QuotaAvailableBytes => "quota-available-bytes", Element::QuotaUsedBytes => "quota-used-bytes", Element::Read => "read", Element::ReadAcl => "read-acl", Element::ReadCurrentUserPrivilegeSet => "read-current-user-privilege-set", Element::ReadFreeBusy => "read-free-busy", Element::Rebind => "rebind", Element::RebindResponse => "rebind-response", Element::Recipient => "recipient", Element::RecognizedPrincipal => "recognized-principal", Element::RedirectLifetime => "redirect-lifetime", Element::Redirectref => "redirectref", Element::Reftarget => "reftarget", Element::Remove => "remove", Element::Report => "report", Element::RequestStatus => "request-status", Element::RequiredPrincipal => "required-principal", Element::Resource => "resource", Element::ResourceId => "resource-id", Element::Resourcetype => "resourcetype", Element::Response => "response", Element::Responsedescription => "responsedescription", Element::RootVersion => "root-version", Element::SameOrganizerInAllComponents => "same-organizer-in-all-components", Element::ScheduleCalendarTransp => "schedule-calendar-transp", Element::ScheduleDefaultCalendarUrl => "schedule-default-calendar-URL", Element::ScheduleDeliver => "schedule-deliver", Element::ScheduleDeliverInvite => "schedule-deliver-invite", Element::ScheduleDeliverReply => "schedule-deliver-reply", Element::ScheduleInbox => "schedule-inbox", Element::ScheduleInboxUrl => "schedule-inbox-URL", Element::ScheduleOutbox => "schedule-outbox", Element::ScheduleOutboxUrl => "schedule-outbox-URL", Element::ScheduleQueryFreebusy => "schedule-query-freebusy", Element::ScheduleResponse => "schedule-response", Element::ScheduleSend => "schedule-send", Element::ScheduleSendFreebusy => "schedule-send-freebusy", Element::ScheduleSendInvite => "schedule-send-invite", Element::ScheduleSendReply => "schedule-send-reply", Element::ScheduleTag => "schedule-tag", Element::Scope => "scope", Element::Score => "score", Element::Searchable => "searchable", Element::Segment => "segment", Element::Select => "select", Element::Selectable => "selectable", Element::Self_ => "self", Element::Set => "set", Element::Shared => "shared", Element::Sortable => "sortable", Element::Source => "source", Element::Status => "status", Element::SubactivitySet => "subactivity-set", Element::SubbaselineSet => "subbaseline-set", Element::SuccessorSet => "successor-set", Element::SupportedAddressData => "supported-address-data", Element::SupportedCalendarComponentSet => "supported-calendar-component-set", Element::SupportedCalendarData => "supported-calendar-data", Element::SupportedCollation => "supported-collation", Element::SupportedCollationSet => "supported-collation-set", Element::SupportedFilter => "supported-filter", Element::SupportedLiveProperty => "supported-live-property", Element::SupportedLivePropertySet => "supported-live-property-set", Element::SupportedMethod => "supported-method", Element::SupportedMethodSet => "supported-method-set", Element::SupportedPrivilege => "supported-privilege", Element::SupportedPrivilegeSet => "supported-privilege-set", Element::SupportedQueryGrammar => "supported-query-grammar", Element::SupportedQueryGrammarSet => "supported-query-grammar-set", Element::SupportedReport => "supported-report", Element::SupportedReportSet => "supported-report-set", Element::SupportedRscale => "supported-rscale", Element::SupportedRscaleSet => "supported-rscale-set", Element::Supportedlock => "supportedlock", Element::SyncCollection => "sync-collection", Element::SyncLevel => "sync-level", Element::SyncToken => "sync-token", Element::Target => "target", Element::Temporary => "temporary", Element::TextMatch => "text-match", Element::TimeRange => "time-range", Element::Timeout => "timeout", Element::Timezone => "timezone", Element::TimezoneId => "timezone-id", Element::TimezoneServiceSet => "timezone-service-set", Element::Transparent => "transparent", Element::TypedLiteral => "typed-literal", Element::Unauthenticated => "unauthenticated", Element::Unbind => "unbind", Element::UnbindResponse => "unbind-response", Element::Uncheckout => "uncheckout", Element::UncheckoutResponse => "uncheckout-response", Element::UniqueSchedulingObjectResource => "unique-scheduling-object-resource", Element::Unlock => "unlock", Element::Unreserved => "unreserved", Element::Update => "update", Element::UpdatePreview => "update-preview", Element::Updateredirectref => "updateredirectref", Element::UpdateredirectrefResponse => "updateredirectref-response", Element::Url => "url", Element::Username => "username", Element::ValidOrganizer => "valid-organizer", Element::ValidScheduleDefaultCalendarUrl => "valid-schedule-default-calendar-URL", Element::ValidSchedulingMessage => "valid-scheduling-message", Element::Version => "version", Element::VersionControl => "version-control", Element::VersionControlResponse => "version-control-response", Element::VersionControlledBinding => "version-controlled-binding", Element::VersionControlledBindingSet => "version-controlled-binding-set", Element::VersionControlledConfiguration => "version-controlled-configuration", Element::VersionHistory => "version-history", Element::VersionHistoryCollectionSet => "version-history-collection-set", Element::VersionHistorySet => "version-history-set", Element::VersionName => "version-name", Element::VersionSet => "version-set", Element::VersionTree => "version-tree", Element::Where => "where", Element::Workspace => "workspace", Element::WorkspaceCheckoutSet => "workspace-checkout-set", Element::WorkspaceCollectionSet => "workspace-collection-set", Element::Write => "write", Element::WriteAcl => "write-acl", Element::WriteContent => "write-content", Element::WriteProperties => "write-properties", } } } #[derive(Debug, Clone, PartialEq, Eq)] pub enum Attribute { Caseless(bool), XsiType(XsiType), AllowPCData(bool), Name(T), Namespace(Namespace), ContentType(String), XmlLanguage(String), Version(String), NoValue(bool), TestAllOf(bool), MatchType(MatchType), NegateCondition(bool), Collation(Collation), Start(T), End(T), Unknown { param: String, value: String }, } pub trait AttributeValue { fn from_str(s: &str) -> Option where Self: Sized; } impl Attribute { pub fn from_param(param: &[u8], value: Cow<'_, str>) -> Option> { hashify::fnc_map!(param, "caseless" => { if let Some(b) = YesNo::from_str(value.as_ref()) { return Some(Attribute::Caseless(b)); } }, "xsi:type" => { return Some(Attribute::XsiType(XsiType::from_str(value.as_ref()).unwrap_or(XsiType::Unsupported))); }, "allow-pcdata" => { if let Some(b) = YesNo::from_str(value.as_ref()) { return Some(Attribute::AllowPCData(b)); } }, "novalue" => { if let Some(b) = YesNo::from_str(value.as_ref()) { return Some(Attribute::NoValue(b)); } }, "negate-condition" => { if let Some(b) = YesNo::from_str(value.as_ref()) { return Some(Attribute::NegateCondition(b)); } }, "name" => { if let Some(value) = T::from_str(value.as_ref()) { return Some(Attribute::Name(value)); } }, "namespace" => { if let Some(ns) = Namespace::try_parse(value.as_bytes()) { return Some(Attribute::Namespace(ns)); } }, "content-type" => { return Some(Attribute::ContentType(value.into_owned())); }, "version" => { return Some(Attribute::Version(value.into_owned())); }, "test" => { return Some(Attribute::TestAllOf(value.eq("allof"))); }, "match-type" => { if let Some(mt) = MatchType::try_parse(value.as_ref()) { return Some(Attribute::MatchType(mt)); } }, "collation" => { if let Some(c) = Collation::try_parse(value.as_ref()) { return Some(Attribute::Collation(c)); } }, "start" => { if let Some(value) = T::from_str(value.as_ref()) { return Some(Attribute::Start(value)); } }, "end" => { if let Some(value) = T::from_str(value.as_ref()) { return Some(Attribute::End(value)); } }, "xml:lang" => { return Some(Attribute::XmlLanguage(value.into_owned())); }, "xmlns" => { return None; }, _ => { if param.starts_with(b"xmlns:") { return None; } } ); Some(Attribute::Unknown { param: String::from_utf8_lossy(param).into_owned(), value: value.into_owned(), }) } } impl AttributeValue for String { fn from_str(s: &str) -> Option { Some(s.to_string()) } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub enum Collation { AsciiNumeric, AsciiCasemap, Octet, UnicodeCasemap, } impl Collation { pub fn try_parse(s: &str) -> Option { hashify::tiny_map!(s.as_bytes(), "i;ascii-numeric" => Collation::AsciiNumeric, "i;ascii-casemap" => Collation::AsciiCasemap, "i;octet" => Collation::Octet, "i;unicode-casemap" => Collation::UnicodeCasemap, ) } pub fn as_str(&self) -> &'static str { match self { Collation::AsciiNumeric => "i;ascii-numeric", Collation::AsciiCasemap => "i;ascii-casemap", Collation::Octet => "i;octet", Collation::UnicodeCasemap => "i;unicode-casemap", } } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub enum MatchType { Equals, Contains, StartsWith, EndsWith, } impl MatchType { pub fn try_parse(s: &str) -> Option { hashify::tiny_map!(s.as_bytes(), "equals" => MatchType::Equals, "contains" => MatchType::Contains, "starts-with" => MatchType::StartsWith, "ends-with" => MatchType::EndsWith, ) } } #[derive(Debug, Clone, PartialEq, Eq)] pub enum XsiType { String, Boolean, Decimal, Float, Double, Duration, DateTime, Time, Date, GYearMonth, GYear, GMonthDay, GDay, GMonth, HexBinary, Base64Binary, AnyUri, QName, Notation, Unsupported, } impl XsiType { fn from_str(s: &str) -> Option { hashify::tiny_map!(s.as_bytes(), "xs:string" => XsiType::String, "xs:boolean" => XsiType::Boolean, "xs:decimal" => XsiType::Decimal, "xs:float" => XsiType::Float, "xs:double" => XsiType::Double, "xs:duration" => XsiType::Duration, "xs:dateTime" => XsiType::DateTime, "xs:time" => XsiType::Time, "xs:date" => XsiType::Date, "xs:gYearMonth" => XsiType::GYearMonth, "xs:gYear" => XsiType::GYear, "xs:gMonthDay" => XsiType::GMonthDay, "xs:gDay" => XsiType::GDay, "xs:gMonth" => XsiType::GMonth, "xs:hexBinary" => XsiType::HexBinary, "xs:base64Binary" => XsiType::Base64Binary, "xs:anyURI" => XsiType::AnyUri, "xs:QName" => XsiType::QName, "xs:NOTATION" => XsiType::Notation, ) } } struct YesNo; impl YesNo { fn from_str(s: &str) -> Option { hashify::tiny_map!(s.as_bytes(), "yes" => true, "no" => false, ) } } impl TextMatch { pub fn matches(&self, text: &str) -> bool { match self.collation { Collation::Octet => { (match self.match_type { MatchType::Equals => text == self.value, MatchType::Contains => text.contains(&self.value), MatchType::StartsWith => text.starts_with(&self.value), MatchType::EndsWith => text.ends_with(&self.value), }) ^ self.negate } _ => { (match self.match_type { MatchType::Equals => text.to_lowercase() == self.value.to_lowercase(), MatchType::Contains => text.to_lowercase().contains(&self.value.to_lowercase()), MatchType::StartsWith => { text.to_lowercase().starts_with(&self.value.to_lowercase()) } MatchType::EndsWith => { text.to_lowercase().ends_with(&self.value.to_lowercase()) } }) ^ self.negate } } } } ================================================ FILE: crates/dav-proto/src/schema/property.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ request::DavPropertyValue, response::{Ace, AclRestrictions, Href, List, Response, SupportedPrivilege}, Collation, Namespace, }; use crate::{Depth, Timeout}; use calcard::{ icalendar::{ICalendar, ICalendarComponentType, ICalendarProperty}, vcard::{VCard, VCardProperty}, }; use types::{ dead_property::{DeadElementTag, DeadProperty}, TimeRange, }; #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(test, serde(tag = "type", content = "data"))] pub enum DavProperty { WebDav(WebDavProperty), CardDav(CardDavProperty), CalDav(CalDavProperty), Principal(PrincipalProperty), DeadProperty(DeadElementTag), } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(test, serde(tag = "type", content = "data"))] pub enum WebDavProperty { CreationDate, DisplayName, GetContentLanguage, GetContentLength, GetContentType, GetETag, GetLastModified, ResourceType, LockDiscovery, SupportedLock, SupportedReportSet, CurrentUserPrincipal, // Quota properties QuotaAvailableBytes, QuotaUsedBytes, // Sync properties SyncToken, // ACL properties (all protected) Owner, Group, SupportedPrivilegeSet, CurrentUserPrivilegeSet, Acl, AclRestrictions, InheritedAclSet, PrincipalCollectionSet, // Apple proprietary properties GetCTag, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(test, serde(tag = "type", content = "data"))] pub enum CardDavProperty { AddressbookDescription, SupportedAddressData, SupportedCollationSet, MaxResourceSize, AddressData(Vec), } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct CardDavPropertyName { pub group: Option, pub name: VCardProperty, pub no_value: bool, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(test, serde(tag = "type", content = "data"))] pub enum CalDavProperty { CalendarDescription, CalendarTimezone, SupportedCalendarComponentSet, SupportedCalendarData, SupportedCollationSet, MaxResourceSize, MinDateTime, MaxDateTime, MaxInstances, MaxAttendeesPerInstance, CalendarData(CalendarData), TimezoneServiceSet, TimezoneId, ScheduleDefaultCalendarURL, ScheduleTag, ScheduleCalendarTransp, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(test, serde(tag = "type", content = "data"))] pub enum PrincipalProperty { AlternateURISet, PrincipalURL, GroupMemberSet, GroupMembership, CalendarHomeSet, AddressbookHomeSet, PrincipalAddress, CalendarUserAddressSet, CalendarUserType, ScheduleInboxURL, ScheduleOutboxURL, } #[derive(Debug, Default, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct CalendarData { pub properties: Vec, pub expand: Option, pub limit_recurrence: Option, pub limit_freebusy: Option, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct CalDavPropertyName { pub component: Option, pub name: Option, pub no_value: bool, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] #[repr(transparent)] pub struct Rfc1123DateTime(pub(crate) i64); #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub enum DavValue { Timestamp(i64), Rfc1123Date(Rfc1123DateTime), Uint64(u64), String(String), CData(String), ResourceTypes(List), ActiveLocks(List), LockEntries(List), ReportSets(List), ICalendar(ICalendar), VCard(VCard), Components(List), Collations(List), PrivilegeSet(List), Privileges(List), Href(List), Acl(List), AclRestrictions(AclRestrictions), Response(Response), DeadProperty(DeadProperty), SupportedAddressData, SupportedCalendarData, Null, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub enum ReportSet { SyncCollection, ExpandProperty, AddressbookQuery, AddressbookMultiGet, CalendarQuery, CalendarMultiGet, FreeBusyQuery, AclPrincipalPropSet, PrincipalMatch, PrincipalPropertySearch, PrincipalSearchPropertySet, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct Comp(pub ICalendarComponentType); #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct SupportedCollation { pub collation: Collation, pub namespace: Namespace, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub enum ResourceType { Collection, Principal, AddressBook, Calendar, ScheduleInbox, ScheduleOutbox, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct LockDiscovery(pub List); #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct ActiveLock { pub lock_scope: LockScope, pub lock_type: LockType, pub depth: Depth, pub owner: Option, pub timeout: Timeout, pub lock_token: Option, pub lock_root: Href, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct SupportedLock(pub List); #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct LockEntry { pub lock_scope: LockScope, pub lock_type: LockType, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub enum LockType { Write, Other, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub enum LockScope { Exclusive, Shared, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub enum Privilege { Read, Write, WriteProperties, WriteContent, Unlock, ReadAcl, ReadCurrentUserPrivilegeSet, WriteAcl, Bind, Unbind, All, ReadFreeBusy, ScheduleDeliver, ScheduleDeliverInvite, ScheduleDeliverReply, ScheduleQueryFreeBusy, ScheduleSend, ScheduleSendInvite, ScheduleSendReply, ScheduleSendFreeBusy, } impl Privilege { pub fn all(is_calendar: bool) -> Vec { if is_calendar { vec![ Privilege::All, Privilege::Read, Privilege::Write, Privilege::WriteProperties, Privilege::WriteContent, Privilege::Unlock, Privilege::ReadAcl, Privilege::ReadCurrentUserPrivilegeSet, Privilege::WriteAcl, Privilege::Bind, Privilege::Unbind, Privilege::ReadFreeBusy, ] } else { vec![ Privilege::All, Privilege::Read, Privilege::Write, Privilege::WriteProperties, Privilege::WriteContent, Privilege::Unlock, Privilege::ReadAcl, Privilege::ReadCurrentUserPrivilegeSet, Privilege::WriteAcl, Privilege::Bind, Privilege::Unbind, ] } } pub fn scheduling(is_inbox: bool, is_owner: bool) -> Vec { let mut privileges = if is_inbox { vec![ Privilege::Read, Privilege::ReadCurrentUserPrivilegeSet, Privilege::ScheduleDeliver, Privilege::ScheduleDeliverInvite, Privilege::ScheduleDeliverReply, Privilege::ScheduleQueryFreeBusy, ] } else { vec![ Privilege::Read, Privilege::ReadCurrentUserPrivilegeSet, Privilege::ScheduleSend, Privilege::ScheduleSendInvite, Privilege::ScheduleSendReply, Privilege::ScheduleSendFreeBusy, ] }; if is_owner { privileges.extend([ Privilege::All, Privilege::Write, Privilege::WriteProperties, Privilege::WriteContent, Privilege::ReadAcl, Privilege::WriteAcl, ]); } privileges } } impl From for DavPropertyValue { fn from(value: DavProperty) -> Self { DavPropertyValue { property: value, value: DavValue::Null, } } } impl Rfc1123DateTime { pub fn new(timestamp: i64) -> Self { Self(timestamp) } } impl DavProperty { pub const ALL_PROPS: [DavProperty; 11] = [ DavProperty::WebDav(WebDavProperty::CreationDate), DavProperty::WebDav(WebDavProperty::DisplayName), DavProperty::WebDav(WebDavProperty::GetETag), DavProperty::WebDav(WebDavProperty::GetLastModified), DavProperty::WebDav(WebDavProperty::ResourceType), DavProperty::WebDav(WebDavProperty::LockDiscovery), DavProperty::WebDav(WebDavProperty::SupportedLock), DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal), DavProperty::WebDav(WebDavProperty::GetContentLanguage), DavProperty::WebDav(WebDavProperty::GetContentLength), DavProperty::WebDav(WebDavProperty::GetContentType), ]; pub fn is_all_prop(&self) -> bool { matches!( self, DavProperty::WebDav(WebDavProperty::CreationDate) | DavProperty::WebDav(WebDavProperty::DisplayName) | DavProperty::WebDav(WebDavProperty::GetETag) | DavProperty::WebDav(WebDavProperty::GetLastModified) | DavProperty::WebDav(WebDavProperty::ResourceType) | DavProperty::WebDav(WebDavProperty::LockDiscovery) | DavProperty::WebDav(WebDavProperty::SupportedLock) | DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal) | DavProperty::WebDav(WebDavProperty::GetContentLanguage) | DavProperty::WebDav(WebDavProperty::GetContentLength) | DavProperty::WebDav(WebDavProperty::GetContentType) | DavProperty::DeadProperty(_) ) } } impl ReportSet { pub fn calendar() -> Vec { vec![ ReportSet::SyncCollection, ReportSet::AclPrincipalPropSet, ReportSet::PrincipalMatch, ReportSet::ExpandProperty, ReportSet::CalendarQuery, ReportSet::CalendarMultiGet, ReportSet::FreeBusyQuery, ] } pub fn addressbook() -> Vec { vec![ ReportSet::SyncCollection, ReportSet::AclPrincipalPropSet, ReportSet::PrincipalMatch, ReportSet::ExpandProperty, ReportSet::AddressbookQuery, ReportSet::AddressbookMultiGet, ] } pub fn file() -> Vec { vec![ ReportSet::SyncCollection, ReportSet::AclPrincipalPropSet, ReportSet::PrincipalMatch, ] } pub fn principal() -> Vec { vec![ ReportSet::PrincipalPropertySearch, ReportSet::PrincipalSearchPropertySet, ReportSet::PrincipalMatch, ] } } ================================================ FILE: crates/dav-proto/src/schema/request.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ Collation, MatchType, property::{DavProperty, DavValue, LockScope, LockType}, response::Ace, }; use crate::{Condition, Depth}; use calcard::{ icalendar::{ICalendarComponentType, ICalendarParameterName, ICalendarProperty}, vcard::{VCardParameterName, VCardProperty}, }; use types::{ TimeRange, dead_property::{ArchivedDeadProperty, ArchivedDeadPropertyTag, DeadElementTag, DeadProperty}, }; #[derive(Debug, Clone, PartialEq, Eq, Default)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(test, serde(tag = "type", content = "data"))] pub enum PropFind { #[default] PropName, AllProp(Vec), Prop(Vec), } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct PropertyUpdate { pub set: Vec, pub remove: Vec, pub set_first: bool, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct DavPropertyValue { pub property: DavProperty, pub value: DavValue, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct MkCol { pub is_mkcalendar: bool, pub props: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct LockInfo { pub lock_scope: LockScope, pub lock_type: LockType, pub owner: Option, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(test, serde(tag = "type"))] pub enum Report { AddressbookQuery(AddressbookQuery), AddressbookMultiGet(MultiGet), CalendarQuery(CalendarQuery), CalendarMultiGet(MultiGet), FreeBusyQuery(FreeBusyQuery), SyncCollection(SyncCollection), ExpandProperty(ExpandProperty), AclPrincipalPropSet(AclPrincipalPropSet), PrincipalMatch(PrincipalMatch), PrincipalPropertySearch(PrincipalPropertySearch), PrincipalSearchPropertySet, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct ExpandProperty { pub properties: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct ExpandPropertyItem { pub property: DavProperty, pub depth: u32, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct AddressbookQuery { pub properties: PropFind, pub filters: Vec>, pub limit: Option, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct VCardPropertyWithGroup { pub name: VCardProperty, pub group: Option, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct CalendarQuery { pub properties: PropFind, pub filters: Vec, ICalendarProperty, ICalendarParameterName>>, pub timezone: Timezone, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(test, serde(tag = "type"))] pub enum Timezone { Name(String), Id(String), None, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct FreeBusyQuery { pub range: Option, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct MultiGet { pub properties: PropFind, pub hrefs: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct SyncCollection { pub sync_token: Option, pub properties: PropFind, pub depth: Depth, pub limit: Option, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(test, serde(tag = "type"))] pub enum Filter { AnyOf, AllOf, Component { comp: A, op: FilterOp, }, Property { comp: A, prop: B, op: FilterOp, }, Parameter { comp: A, prop: B, param: C, op: FilterOp, }, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(test, serde(tag = "type", content = "data"))] pub enum FilterOp { Exists, Undefined, TimeRange(TimeRange), TextMatch(TextMatch), } #[derive(Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(test, serde(tag = "type"))] pub struct TextMatch { pub match_type: MatchType, pub value: String, pub collation: Collation, pub negate: bool, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct Acl { pub aces: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct AclPrincipalPropSet { pub properties: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct PrincipalMatch { pub principal_properties: PrincipalMatchProperties, pub properties: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub enum PrincipalMatchProperties { Properties(Vec), Self_, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct PrincipalPropertySearch { pub property_search: Vec, pub properties: Vec, pub apply_to_principal_collection_set: bool, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct PropertySearch { pub property: DavProperty, pub match_: String, } impl PropertyUpdate { pub fn has_changes(&self) -> bool { !self.set.is_empty() || !self.remove.is_empty() } } impl FreeBusyQuery { pub fn new(start: i64, end: i64) -> Self { FreeBusyQuery { range: Some(TimeRange { start, end }), } } } pub trait DavDeadProperty { fn to_dav_values(&self, output: &mut Vec); } impl DavDeadProperty for ArchivedDeadProperty { fn to_dav_values(&self, output: &mut Vec) { let mut depth: u32 = 0; let mut tags = Vec::new(); let mut tag_start = None; for tag in self.0.iter() { match tag { ArchivedDeadPropertyTag::ElementStart(start) => { if depth == 0 { tag_start = Some(DeadElementTag::from(start)); } else { tags.push(tag.into()); } depth += 1; } ArchivedDeadPropertyTag::ElementEnd => { depth = depth.saturating_sub(1); if depth > 0 { tags.push(tag.into()); } else if let Some(tag_start) = tag_start.take() { output.push(DavPropertyValue::new( DavProperty::DeadProperty(tag_start), DavValue::DeadProperty(DeadProperty(std::mem::take(&mut tags))), )); } } ArchivedDeadPropertyTag::Text(_) => { if tag_start.is_some() { tags.push(tag.into()); } } } } } } impl Condition<'_> { pub fn is_none_match(&self) -> bool { match self { Condition::ETag { is_not, .. } | Condition::Exists { is_not } => *is_not, Condition::StateToken { .. } => false, } } } ================================================ FILE: crates/dav-proto/src/schema/response.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{borrow::Cow, fmt::Display}; use calcard::{ icalendar::{ICalendarComponentType, ICalendarParameterName, ICalendarProperty}, vcard::{VCardParameterName, VCardProperty}, }; use hyper::StatusCode; use super::{ property::{DavProperty, Privilege}, request::{DavPropertyValue, Filter}, Namespaces, }; pub struct MultiStatus { pub namespaces: Namespaces, pub response: List, pub response_description: Option, pub sync_token: Option, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct Response { pub href: Href, pub typ: ResponseType, pub error: Option, pub response_description: Option, pub location: Option, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub enum ResponseType { PropStat(List), Status { href: List, status: Status }, } #[derive(Debug, Clone, PartialEq, Eq)] #[repr(transparent)] pub struct Status(pub StatusCode); #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] #[repr(transparent)] pub struct Location(pub Href); #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] #[repr(transparent)] pub struct ResponseDescription(pub String); #[repr(transparent)] pub struct SyncToken(pub String); #[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] #[repr(transparent)] pub struct Href(pub String); #[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] #[repr(transparent)] pub struct List(pub Vec); pub struct MkColResponse { pub namespaces: Namespaces, pub propstat: List, pub mkcalendar: bool, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct PropStat { pub prop: Prop, pub status: Status, pub error: Option, pub response_description: Option, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] #[repr(transparent)] pub struct Prop(pub List); pub struct PropResponse { pub namespaces: Namespaces, pub properties: List, } #[derive(Default)] pub struct ScheduleResponse { pub items: List, } #[derive(Default)] pub struct ScheduleResponseItem { pub recipient: Href, pub request_status: Cow<'static, str>, pub calendar_data: Option, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct SupportedPrivilege { pub privilege: Privilege, pub abstract_: bool, pub description: String, pub supported_privilege: List, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct Ace { pub principal: Principal, pub invert: bool, pub grant_deny: GrantDeny, pub protected: bool, pub inherited: Option, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub enum GrantDeny { Grant(List), Deny(List), } #[derive(Debug, Default, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub enum Principal { Href(Href), Response(Response), All, #[default] Authenticated, Unauthenticated, Property(List), Self_, } #[derive(Debug, Default, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct AclRestrictions { pub grant_only: bool, pub no_invert: bool, pub deny_before_grant: bool, pub required_principal: Option, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub enum RequiredPrincipal { All, Authenticated, Unauthenticated, Self_, Href(List), Property(Vec), } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct PrincipalSearchPropertySet { pub namespaces: Namespaces, pub properties: List, } #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct PrincipalSearchProperty { pub name: DavProperty, pub description: String, } pub struct ErrorResponse { pub namespaces: Namespaces, pub error: Condition, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub enum Condition { Base(BaseCondition), Cal(CalCondition), Card(CardCondition), } #[derive(Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub enum BaseCondition { NoConflictingLock(List), LockTokenSubmitted(List), LockTokenMatchesRequestUri, CannotModifyProtectedProperty, NoExternalEntities, PreservedLiveProperties, PropFindFiniteDepth, ResourceMustBeNull, NeedPrivileges(List), NoAceConflict, NoProtectedAceConflict, NoInheritedAceConflict, LimitedNumberOfAces, DenyBeforeGrant, GrantOnly, NoInvert, NoAbstract, NotSupportedPrivilege, MissingRequiredPrincipal, RecognizedPrincipal, AllowedPrincipal, NumberOfMatchesWithinLimit, QuotaNotExceeded, ValidResourceType, ValidSyncToken, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct Resource { pub href: Href, pub privilege: Privilege, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub enum CalCondition { CalendarCollectionLocationOk, ValidCalendarData, ValidFilter, ValidCalendarObjectResource, ValidTimezone, NoUidConflict(Href), InitializeCalendarCollection, SupportedCalendarData, SupportedFilter( Vec, ICalendarProperty, ICalendarParameterName>>, ), SupportedCollation(String), SupportedCalendarComponent, MinDateTime, MaxDateTime, MaxResourceSize(u32), MaxInstances, MaxAttendeesPerInstance, UniqueSchedulingObjectResource(Href), SameOrganizerInAllComponents, AllowedOrganizerObjectChange, AllowedAttendeeObjectChange, DefaultCalendarNeeded, ValidScheduleDefaultCalendarUrl, ValidSchedulingMessage, ValidOrganizer, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub enum CardCondition { SupportedAddressData, SupportedAddressDataConversion, SupportedFilter(Vec>), SupportedCollation(String), ValidAddressData, NoUidConflict(Href), MaxResourceSize(u32), AddressBookCollectionLocationOk, } impl BaseCondition { pub fn status(&self) -> StatusCode { match self { BaseCondition::NoConflictingLock(_) => StatusCode::LOCKED, BaseCondition::CannotModifyProtectedProperty => StatusCode::FORBIDDEN, BaseCondition::LockTokenSubmitted(_) => StatusCode::LOCKED, BaseCondition::LockTokenMatchesRequestUri => StatusCode::CONFLICT, BaseCondition::NoExternalEntities => StatusCode::FORBIDDEN, BaseCondition::PreservedLiveProperties => StatusCode::CONFLICT, BaseCondition::PropFindFiniteDepth => StatusCode::FORBIDDEN, BaseCondition::ResourceMustBeNull => StatusCode::CONFLICT, BaseCondition::NeedPrivileges(_) => StatusCode::FORBIDDEN, BaseCondition::NumberOfMatchesWithinLimit => StatusCode::FORBIDDEN, _ => StatusCode::FORBIDDEN, } } } impl From for Href { fn from(value: String) -> Self { Self(value) } } impl From<&str> for Href { fn from(value: &str) -> Self { Self(value.to_string()) } } impl MultiStatus { pub fn is_empty(&self) -> bool { self.response.0.is_empty() } } impl BaseCondition { pub fn display_name(&self) -> &'static str { match self { BaseCondition::NoConflictingLock(_) => "NoConflictingLock", BaseCondition::CannotModifyProtectedProperty => "CannotModifyProtectedProperty", BaseCondition::LockTokenSubmitted(_) => "LockTokenSubmitted", BaseCondition::LockTokenMatchesRequestUri => "LockTokenMatchesRequestUri", BaseCondition::NoExternalEntities => "NoExternalEntities", BaseCondition::PreservedLiveProperties => "PreservedLiveProperties", BaseCondition::PropFindFiniteDepth => "PropFindFiniteDepth", BaseCondition::ResourceMustBeNull => "ResourceMustBeNull", BaseCondition::NeedPrivileges(_) => "NeedPrivileges", BaseCondition::NoAceConflict => "NoAceConflict", BaseCondition::NoProtectedAceConflict => "NoProtectedAceConflict", BaseCondition::NoInheritedAceConflict => "NoInheritedAceConflict", BaseCondition::LimitedNumberOfAces => "LimitedNumberOfAces", BaseCondition::DenyBeforeGrant => "DenyBeforeGrant", BaseCondition::GrantOnly => "GrantOnly", BaseCondition::NoInvert => "NoInvert", BaseCondition::NoAbstract => "NoAbstract", BaseCondition::NotSupportedPrivilege => "NotSupportedPrivilege", BaseCondition::MissingRequiredPrincipal => "MissingRequiredPrincipal", BaseCondition::RecognizedPrincipal => "RecognizedPrincipal", BaseCondition::AllowedPrincipal => "AllowedPrincipal", BaseCondition::NumberOfMatchesWithinLimit => "NumberOfMatchesWithinLimit", BaseCondition::QuotaNotExceeded => "QuotaNotExceeded", BaseCondition::ValidResourceType => "ValidResourceType", BaseCondition::ValidSyncToken => "ValidSyncToken", } } } impl CalCondition { pub fn display_name(&self) -> &'static str { match self { CalCondition::CalendarCollectionLocationOk => "CalendarCollectionLocationOk", CalCondition::ValidCalendarData => "ValidCalendarData", CalCondition::ValidFilter => "ValidFilter", CalCondition::ValidCalendarObjectResource => "ValidCalendarObjectResource", CalCondition::ValidTimezone => "ValidTimezone", CalCondition::NoUidConflict(_) => "NoUidConflict", CalCondition::InitializeCalendarCollection => "InitializeCalendarCollection", CalCondition::SupportedCalendarData => "SupportedCalendarData", CalCondition::SupportedFilter(_) => "SupportedFilter", CalCondition::SupportedCollation(_) => "SupportedCollation", CalCondition::MinDateTime => "MinDateTime", CalCondition::MaxDateTime => "MaxDateTime", CalCondition::MaxResourceSize(_) => "MaxResourceSize", CalCondition::MaxInstances => "MaxInstances", CalCondition::MaxAttendeesPerInstance => "MaxAttendeesPerInstance", CalCondition::UniqueSchedulingObjectResource(_) => "UniqueSchedulingObjectResource", CalCondition::SameOrganizerInAllComponents => "SameOrganizerInAllComponents", CalCondition::AllowedOrganizerObjectChange => "AllowedOrganizerObjectChange", CalCondition::AllowedAttendeeObjectChange => "AllowedAttendeeObjectChange", CalCondition::DefaultCalendarNeeded => "DefaultCalendarNeeded", CalCondition::ValidScheduleDefaultCalendarUrl => "ValidScheduleDefaultCalendarUrl", CalCondition::ValidSchedulingMessage => "ValidSchedulingMessage", CalCondition::ValidOrganizer => "ValidOrganizer", CalCondition::SupportedCalendarComponent => "SupportedCalendarComponent", } } } impl CardCondition { pub fn display_name(&self) -> &'static str { match self { CardCondition::SupportedAddressData => "SupportedAddressData", CardCondition::SupportedAddressDataConversion => "SupportedAddressDataConversion", CardCondition::SupportedFilter(_) => "SupportedFilter", CardCondition::SupportedCollation(_) => "SupportedCollation", CardCondition::ValidAddressData => "ValidAddressData", CardCondition::NoUidConflict(_) => "NoUidConflict", CardCondition::MaxResourceSize(_) => "MaxResourceSize", CardCondition::AddressBookCollectionLocationOk => "AddressBookCollectionLocationOk", } } } impl Condition { pub fn display_name(&self) -> &'static str { match self { Condition::Base(base) => base.display_name(), Condition::Cal(cal) => cal.display_name(), Condition::Card(card) => card.display_name(), } } } #[cfg(test)] mod serde_impl { use super::Status; use hyper::StatusCode; use serde::{Deserialize, Deserializer, Serialize, Serializer}; impl Serialize for Status { fn serialize(&self, serializer: S) -> Result where S: Serializer, { // Serialize the status code as a u16 serializer.serialize_u16(self.0.as_u16()) } } impl<'de> Deserialize<'de> for Status { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { // Deserialize as u16 let status_value = u16::deserialize(deserializer)?; // Convert u16 to StatusCode let status_code = StatusCode::try_from(status_value).map_err(|_| { serde::de::Error::custom(format!("Invalid status code: {}", status_value)) })?; Ok(Status(status_code)) } } } ================================================ FILE: crates/directory/Cargo.toml ================================================ [package] name = "directory" version = "0.15.5" edition = "2024" [dependencies] utils = { path = "../utils" } proc_macros = { path = "../utils/proc-macros" } store = { path = "../store" } trc = { path = "../trc" } nlp = { path = "../nlp" } types = { path = "../types" } smtp-proto = { version = "0.2" } mail-parser = { version = "0.11", features = ["full_encoding", "rkyv"] } mail-send = { version = "0.5", default-features = false, features = ["cram-md5", "ring", "tls12"] } mail-builder = { version = "0.4" } tokio = { version = "1.47", features = ["net"] } tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } rustls = { version = "0.23.5", default-features = false, features = ["std", "ring", "tls12"] } rustls-pki-types = { version = "1" } ldap3 = { version = "0.12", default-features = false, features = ["tls-rustls-ring"] } deadpool = { version = "0.10", features = ["managed", "rt_tokio_1"] } async-trait = "0.1.68" ahash = { version = "0.8" } pwhash = "1" password-hash = "0.5.0" argon2 = "0.5.0" pbkdf2 = {version = "0.12.1", features = ["simple"] } scrypt = "0.11.0" sha1 = "0.10.5" sha2 = "0.10.6" md5 = "0.8.0" futures = "0.3" regex = "1.7.0" serde = { version = "1.0", features = ["derive"]} totp-rs = { version = "5.5.1", features = ["otpauth"] } reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-webpki-roots", "http2"] } serde_json = "1.0" base64 = "0.22" rkyv = { version = "0.8.10", features = ["little_endian"] } compact_str = { version = "0.9.0", features = ["rkyv", "serde"] } [dev-dependencies] tokio = { version = "1.47", features = ["full"] } [features] test_mode = [] enterprise = [] ================================================ FILE: crates/directory/src/backend/imap/client.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use mail_send::Credentials; use smtp_proto::{ AUTH_OAUTHBEARER, AUTH_PLAIN, AUTH_XOAUTH2, IntoString, request::{AUTH, parser::Rfc5321Parser}, response::generate::BitToString, }; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use super::{ImapClient, ImapError}; impl ImapClient { pub async fn authenticate( &mut self, mechanism: u64, credentials: &Credentials, ) -> Result<(), ImapError> { if (mechanism & (AUTH_PLAIN | AUTH_XOAUTH2 | AUTH_OAUTHBEARER)) != 0 { self.write( format!( "C3 AUTHENTICATE {} {}\r\n", mechanism.to_mechanism(), credentials .encode(mechanism, "") .map_err(|err| ImapError::InvalidChallenge(err.to_string()))? ) .as_bytes(), ) .await?; } else { self.write(format!("C3 AUTHENTICATE {}\r\n", mechanism.to_mechanism()).as_bytes()) .await?; } let mut line = self.read_line().await?; for _ in 0..3 { if matches!(line.first(), Some(b'+')) { self.write( format!( "{}\r\n", credentials .encode( mechanism, std::str::from_utf8(line.get(2..).unwrap_or_default()) .unwrap_or_default() ) .map_err(|err| ImapError::InvalidChallenge(err.to_string()))? ) .as_bytes(), ) .await?; line = self.read_line().await?; } else if matches!(line.get(..5), Some(b"C3 OK")) { return Ok(()); } else if matches!(line.get(..5), Some(b"C3 NO")) || matches!(line.get(..6), Some(b"C3 BAD")) { return Err(ImapError::AuthenticationFailed); } else { return Err(ImapError::InvalidResponse(line.into_string())); } } Err(ImapError::InvalidResponse(line.into_string())) } pub async fn authentication_mechanisms(&mut self) -> Result { tokio::time::timeout(self.timeout, async { self.write(b"C0 CAPABILITY\r\n").await?; let mut line = self.read_line().await?.into_string(); if !line.starts_with("* CAPABILITY") { return Err(ImapError::InvalidResponse(line)); } while !line.contains("C0 ") { line.push_str(&self.read_line().await?.into_string()); } let mut line_iter = line.as_bytes().iter(); let mut parser = Rfc5321Parser::new(&mut line_iter); let mut mechanisms = 0; 'outer: while let Ok(ch) = parser.read_char() { if ch == b' ' { loop { if parser.hashed_value().unwrap_or(0) == AUTH && parser.stop_char == b'=' { if let Ok(Some(mechanism)) = parser.mechanism() { mechanisms |= mechanism; } match parser.stop_char { b' ' => (), b'\n' => break 'outer, _ => break, } } } } else if ch == b'\n' { break; } } Ok(mechanisms) }) .await .map_err(|_| ImapError::Timeout)? } pub async fn noop(&mut self) -> Result<(), ImapError> { tokio::time::timeout(self.timeout, async { self.write(b"C8 NOOP\r\n").await?; self.read_line().await?; Ok(()) }) .await .map_err(|_| ImapError::Timeout)? } pub async fn logout(&mut self) -> Result<(), ImapError> { tokio::time::timeout(self.timeout, async { self.write(b"C9 LOGOUT\r\n").await?; Ok(()) }) .await .map_err(|_| ImapError::Timeout)? } pub async fn expect_greeting(&mut self) -> Result<(), ImapError> { tokio::time::timeout(self.timeout, async { let line = self.read_line().await?; if matches!(line.get(..4), Some(b"* OK")) { Ok(()) } else { Err(ImapError::InvalidResponse(line.into_string())) } }) .await .map_err(|_| ImapError::Timeout)? } pub async fn read_line(&mut self) -> Result, ImapError> { let mut buf = vec![0u8; 1024]; let mut buf_extended = Vec::with_capacity(0); loop { let br = self.stream.read(&mut buf).await?; if br > 0 { if matches!(buf.get(br - 1), Some(b'\n')) { //println!("{:?}", std::str::from_utf8(&buf[..br]).unwrap()); return Ok(if buf_extended.is_empty() { buf.truncate(br); buf } else { buf_extended.extend_from_slice(&buf[..br]); buf_extended }); } else if buf_extended.is_empty() { buf_extended = buf[..br].to_vec(); } else { buf_extended.extend_from_slice(&buf[..br]); } } else { return Err(ImapError::Disconnected); } } } pub async fn write(&mut self, bytes: &[u8]) -> Result<(), std::io::Error> { self.stream.write_all(bytes).await?; self.stream.flush().await } } #[cfg(test)] mod test { use mail_send::smtp::tls::build_tls_connector; use smtp_proto::{AUTH_OAUTHBEARER, AUTH_PLAIN, AUTH_XOAUTH, AUTH_XOAUTH2}; use std::time::Duration; use crate::backend::imap::ImapClient; #[ignore] #[tokio::test] async fn imap_auth() { let connector = build_tls_connector(false); let mut client = ImapClient::connect( "imap.gmail.com:993", Duration::from_secs(5), &connector, "imap.gmail.com", true, ) .await .unwrap(); assert_eq!( AUTH_PLAIN | AUTH_XOAUTH | AUTH_XOAUTH2 | AUTH_OAUTHBEARER, client.authentication_mechanisms().await.unwrap() ); client.logout().await.unwrap(); } } ================================================ FILE: crates/directory/src/backend/imap/config.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use mail_send::smtp::tls::build_tls_connector; use utils::config::{Config, utils::AsKey}; use crate::core::config::build_pool; use super::{ImapConnectionManager, ImapDirectory}; impl ImapDirectory { pub fn from_config(config: &mut Config, prefix: impl AsKey) -> Option { let prefix = prefix.as_key(); let address = config.value_require((&prefix, "host"))?.to_string(); let tls_implicit: bool = config .property_or_default((&prefix, "tls.enable"), "false") .unwrap_or_default(); let port: u16 = config .property_or_default((&prefix, "port"), if tls_implicit { "993" } else { "143" }) .unwrap_or(if tls_implicit { 993 } else { 143 }); let manager = ImapConnectionManager { addr: format!("{address}:{port}"), timeout: config .property_or_default((&prefix, "timeout"), "30s") .unwrap_or_else(|| Duration::from_secs(30)), tls_connector: build_tls_connector( config .property_or_default((&prefix, "tls.allow-invalid-certs"), "false") .unwrap_or_default(), ), tls_hostname: address.to_string(), tls_implicit, mechanisms: 0.into(), }; Some(ImapDirectory { pool: build_pool(config, &prefix, manager) .map_err(|e| { config.new_parse_error( prefix.as_str(), format!("Failed to build IMAP pool: {e:?}"), ) }) .ok()?, domains: config .values((&prefix, "lookup.domains")) .map(|(_, v)| v.to_lowercase()) .collect(), }) } } ================================================ FILE: crates/directory/src/backend/imap/lookup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use mail_send::Credentials; use smtp_proto::{AUTH_CRAM_MD5, AUTH_LOGIN, AUTH_OAUTHBEARER, AUTH_PLAIN, AUTH_XOAUTH2}; use crate::{IntoError, Principal, QueryBy, Type, backend::RcptType}; use super::{ImapDirectory, ImapError}; impl ImapDirectory { pub async fn query(&self, query: QueryBy<'_>) -> trc::Result> { if let QueryBy::Credentials(credentials) = query { let mut client = self .pool .get() .await .map_err(|err| err.into_error().caused_by(trc::location!()))?; let mechanism = match credentials { Credentials::Plain { .. } if (client.mechanisms & (AUTH_PLAIN | AUTH_LOGIN | AUTH_CRAM_MD5)) != 0 => { if client.mechanisms & AUTH_CRAM_MD5 != 0 { AUTH_CRAM_MD5 } else if client.mechanisms & AUTH_PLAIN != 0 { AUTH_PLAIN } else { AUTH_LOGIN } } Credentials::OAuthBearer { .. } if client.mechanisms & AUTH_OAUTHBEARER != 0 => { AUTH_OAUTHBEARER } Credentials::XOauth2 { .. } if client.mechanisms & AUTH_XOAUTH2 != 0 => { AUTH_XOAUTH2 } _ => { trc::bail!(trc::StoreEvent::NotSupported.ctx( trc::Key::Reason, "IMAP server does not offer any supported auth mechanisms." )); } }; match client.authenticate(mechanism, credentials).await { Ok(_) => { client.is_valid = false; Ok(Some(Principal::new(u32::MAX, Type::Individual))) } Err(err) => match &err { ImapError::AuthenticationFailed => Ok(None), _ => Err(err.into_error()), }, } } else { Err(trc::StoreEvent::NotSupported.caused_by(trc::location!())) } } pub async fn email_to_id(&self, _address: &str) -> trc::Result> { Err(trc::StoreEvent::NotSupported.caused_by(trc::location!())) } pub async fn rcpt(&self, _address: &str) -> trc::Result { Err(trc::StoreEvent::NotSupported.caused_by(trc::location!())) } pub async fn vrfy(&self, _address: &str) -> trc::Result> { Err(trc::StoreEvent::NotSupported.caused_by(trc::location!())) } pub async fn expn(&self, _address: &str) -> trc::Result> { Err(trc::StoreEvent::NotSupported.caused_by(trc::location!())) } pub async fn is_local_domain(&self, domain: &str) -> trc::Result { Ok(self.domains.contains(domain)) } } ================================================ FILE: crates/directory/src/backend/imap/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod client; pub mod config; pub mod lookup; pub mod pool; pub mod tls; use std::{fmt::Display, sync::atomic::AtomicU64, time::Duration}; use ahash::AHashSet; use deadpool::managed::Pool; use tokio::io::{AsyncRead, AsyncWrite}; use tokio_rustls::TlsConnector; pub struct ImapDirectory { pool: Pool, domains: AHashSet, } pub struct ImapConnectionManager { addr: String, timeout: Duration, tls_connector: TlsConnector, tls_hostname: String, tls_implicit: bool, mechanisms: AtomicU64, } pub struct ImapClient { stream: T, mechanisms: u64, is_valid: bool, timeout: Duration, } #[derive(Debug)] pub enum ImapError { Io(std::io::Error), Timeout, InvalidResponse(String), InvalidChallenge(String), AuthenticationFailed, TLSInvalidName, Disconnected, } impl From for ImapError { fn from(error: std::io::Error) -> Self { ImapError::Io(error) } } impl Display for ImapError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { ImapError::Io(io) => write!(f, "I/O error: {io}"), ImapError::Timeout => f.write_str("Connection time-out"), ImapError::InvalidResponse(response) => write!(f, "Unexpected response: {response:?}"), ImapError::InvalidChallenge(response) => { write!(f, "Invalid auth challenge: {response}") } ImapError::TLSInvalidName => f.write_str("Invalid TLS name"), ImapError::Disconnected => f.write_str("Connection disconnected by peer"), ImapError::AuthenticationFailed => f.write_str("Authentication failed"), } } } ================================================ FILE: crates/directory/src/backend/imap/pool.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::atomic::Ordering; use async_trait::async_trait; use deadpool::managed; use tokio::net::TcpStream; use tokio_rustls::client::TlsStream; use super::{ImapClient, ImapConnectionManager, ImapError}; #[async_trait] impl managed::Manager for ImapConnectionManager { type Type = ImapClient>; type Error = ImapError; async fn create(&self) -> Result>, ImapError> { let mut conn = ImapClient::connect( &self.addr, self.timeout, &self.tls_connector, &self.tls_hostname, self.tls_implicit, ) .await?; // Obtain the list of supported authentication mechanisms. conn.mechanisms = self.mechanisms.load(Ordering::Relaxed); if conn.mechanisms == 0 { conn.mechanisms = conn.authentication_mechanisms().await?; self.mechanisms.store(conn.mechanisms, Ordering::Relaxed); } Ok(conn) } async fn recycle( &self, conn: &mut ImapClient>, _: &managed::Metrics, ) -> managed::RecycleResult { conn.noop() .await .map(|_| ()) .map_err(managed::RecycleError::Backend) } } ================================================ FILE: crates/directory/src/backend/imap/tls.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use rustls_pki_types::ServerName; use smtp_proto::IntoString; use tokio::net::{TcpStream, ToSocketAddrs}; use tokio_rustls::{TlsConnector, client::TlsStream}; use super::{ImapClient, ImapError}; impl ImapClient { async fn start_tls( mut self, tls_connector: &TlsConnector, tls_hostname: &str, ) -> Result>, ImapError> { let line = tokio::time::timeout(self.timeout, async { self.write(b"C7 STARTTLS\r\n").await?; self.read_line().await }) .await .map_err(|_| ImapError::Timeout)??; if matches!(line.get(..5), Some(b"C7 OK")) { self.into_tls(tls_connector, tls_hostname).await } else { Err(ImapError::InvalidResponse(line.into_string())) } } async fn into_tls( self, tls_connector: &TlsConnector, tls_hostname: &str, ) -> Result>, ImapError> { tokio::time::timeout(self.timeout, async { Ok(ImapClient { stream: tls_connector .connect( ServerName::try_from(tls_hostname.to_string()) .map_err(|_| ImapError::TLSInvalidName)?, self.stream, ) .await?, timeout: self.timeout, mechanisms: self.mechanisms, is_valid: true, }) }) .await .map_err(|_| ImapError::Timeout)? } } impl ImapClient> { pub async fn connect( addr: impl ToSocketAddrs, timeout: Duration, tls_connector: &TlsConnector, tls_hostname: &str, tls_implicit: bool, ) -> Result { let mut client: ImapClient = tokio::time::timeout(timeout, async { match TcpStream::connect(addr).await { Ok(stream) => Ok(ImapClient { stream, timeout, mechanisms: 0, is_valid: true, }), Err(err) => Err(ImapError::Io(err)), } }) .await .map_err(|_| ImapError::Timeout)??; if tls_implicit { let mut client = client.into_tls(tls_connector, tls_hostname).await?; client.expect_greeting().await?; Ok(client) } else { client.expect_greeting().await?; client.start_tls(tls_connector, tls_hostname).await } } } ================================================ FILE: crates/directory/src/backend/internal/lookup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{PrincipalInfo, manage::ManageDirectory}; use crate::{Principal, PrincipalData, QueryBy, QueryParams, Type, backend::RcptType}; use mail_send::Credentials; use store::{ Deserialize, IterateParams, Store, ValueKey, write::{DirectoryClass, ValueClass}, }; use trc::AddContext; use utils::DomainPart; #[allow(async_fn_in_trait)] pub trait DirectoryStore: Sync + Send { async fn query(&self, by: QueryParams<'_>) -> trc::Result>; async fn email_to_id(&self, address: &str) -> trc::Result>; async fn is_local_domain(&self, domain: &str) -> trc::Result; async fn rcpt(&self, address: &str) -> trc::Result; async fn vrfy(&self, address: &str) -> trc::Result>; async fn expn(&self, address: &str) -> trc::Result>; async fn expn_by_id(&self, id: u32) -> trc::Result>; } impl DirectoryStore for Store { async fn query(&self, by: QueryParams<'_>) -> trc::Result> { let (account_id, secret) = match by.by { QueryBy::Name(name) => (self.get_principal_id(name).await?, None), QueryBy::Id(account_id) => (account_id.into(), None), QueryBy::Credentials(credentials) => match credentials { Credentials::Plain { username, secret } => ( self.get_principal_id(username).await?, secret.as_str().into(), ), Credentials::OAuthBearer { token } => { (self.get_principal_id(token).await?, token.as_str().into()) } Credentials::XOauth2 { username, secret } => ( self.get_principal_id(username).await?, secret.as_str().into(), ), }, }; if let Some(account_id) = account_id && let Some(mut principal) = self.get_principal(account_id).await? { if let Some(secret) = secret && !principal .verify_secret(secret, by.only_app_pass, true) .await? { return Ok(None); } if by.return_member_of { for member in self.get_member_of(principal.id).await? { match member.typ { Type::List => principal .data .push(PrincipalData::List(member.principal_id)), Type::Role => principal .data .push(PrincipalData::Role(member.principal_id)), _ => principal .data .push(PrincipalData::MemberOf(member.principal_id)), } } } return Ok(Some(principal)); } Ok(None) } async fn email_to_id(&self, address: &str) -> trc::Result> { self.get_value::(ValueKey::from(ValueClass::Directory( DirectoryClass::EmailToId(address.as_bytes().to_vec()), ))) .await .map(|ptype| ptype.map(|ptype| ptype.id)) } async fn is_local_domain(&self, domain: &str) -> trc::Result { self.get_value::(ValueKey::from(ValueClass::Directory( DirectoryClass::NameToId(domain.as_bytes().to_vec()), ))) .await .map(|p| p.is_some_and(|p| p.typ == Type::Domain)) } async fn rcpt(&self, address: &str) -> trc::Result { if let Some(pinfo) = self .get_value::(ValueKey::from(ValueClass::Directory( DirectoryClass::EmailToId(address.as_bytes().to_vec()), ))) .await? { if pinfo.typ != Type::List { Ok(RcptType::Mailbox) } else { self.expn_by_id(pinfo.id).await.map(RcptType::List) } } else { Ok(RcptType::Invalid) } } async fn vrfy(&self, address: &str) -> trc::Result> { let mut results = Vec::new(); let address = address.try_local_part().unwrap_or(address); if address.len() > 3 { self.iterate( IterateParams::new( ValueKey::from(ValueClass::Directory(DirectoryClass::EmailToId(vec![0u8]))), ValueKey::from(ValueClass::Directory(DirectoryClass::EmailToId( vec![u8::MAX; 10], ))), ), |key, value| { let key = std::str::from_utf8(key.get(1..).unwrap_or_default()).unwrap_or_default(); if key.try_local_part().unwrap_or(key).contains(address) && PrincipalInfo::deserialize(value) .caused_by(trc::location!())? .typ != Type::List { results.push(key.into()); } Ok(true) }, ) .await .caused_by(trc::location!())?; } Ok(results) } async fn expn(&self, address: &str) -> trc::Result> { if let Some(ptype) = self .get_value::(ValueKey::from(ValueClass::Directory( DirectoryClass::EmailToId(address.as_bytes().to_vec()), ))) .await? .filter(|p| p.typ == Type::List) { self.expn_by_id(ptype.id).await } else { Ok(vec![]) } } async fn expn_by_id(&self, list_id: u32) -> trc::Result> { let mut results = Vec::new(); for account_id in self.get_members(list_id).await? { if let Some(email) = self.get_principal(account_id).await?.and_then(|p| { p.data.into_iter().find_map(|data| { if let PrincipalData::PrimaryEmail(email) | PrincipalData::EmailAlias(email) = data { Some(email) } else { None } }) }) { results.push(email); } } if let Some(principal) = self.get_principal(list_id).await? { results.extend(principal.data.into_iter().filter_map(|data| { if let PrincipalData::ExternalMember(member) = data { Some(member) } else { None } })); } Ok(results) } } ================================================ FILE: crates/directory/src/backend/internal/manage.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ PrincipalAction, PrincipalField, PrincipalInfo, PrincipalSet, PrincipalUpdate, PrincipalValue, SpecialSecrets, lookup::DirectoryStore, }; use crate::{ ArchivedPrincipalData, FALLBACK_ADMIN_ID, MemberOf, Permission, PermissionGrant, Permissions, Principal, PrincipalData, QueryBy, QueryParams, ROLE_ADMIN, ROLE_TENANT_ADMIN, ROLE_USER, Type, backend::RcptType, core::principal::build_search_index, }; use ahash::{AHashMap, AHashSet}; use compact_str::CompactString; use nlp::tokenizers::word::WordTokenizer; use store::{ Deserialize, IterateParams, Serialize, SerializeInfallible, Store, U32_LEN, ValueKey, backend::MAX_TOKEN_LENGTH, roaring::RoaringBitmap, write::{ AlignedBytes, Archive, Archiver, BatchBuilder, DirectoryClass, ValueClass, key::DeserializeBigEndian, }, }; use trc::AddContext; use types::{ collection::Collection, field::{self}, }; use utils::{DomainPart, sanitize_email}; #[derive(Debug, Default, serde::Serialize, serde::Deserialize)] pub struct PrincipalList { pub items: Vec, pub total: u64, } pub struct UpdatePrincipal<'x> { query: QueryBy<'x>, allowed_permissions: Option<&'x Permissions>, changes: Vec, tenant_id: Option, create_domains: bool, } #[derive(Debug, Default, PartialEq, Eq)] #[repr(transparent)] pub struct ChangedPrincipals(AHashMap); #[derive(Debug, Default, PartialEq, Eq)] pub struct ChangedPrincipal { pub typ: Type, pub name_change: bool, pub member_change: bool, } #[derive(Debug, Default, PartialEq, Eq)] pub struct CreatedPrincipal { pub id: u32, pub changed_principals: ChangedPrincipals, } #[allow(async_fn_in_trait)] pub trait ManageDirectory: Sized { async fn get_principal_id(&self, name: &str) -> trc::Result>; async fn get_principal_info(&self, name: &str) -> trc::Result>; async fn get_or_create_principal_id(&self, name: &str, typ: Type) -> trc::Result; async fn get_principal(&self, principal_id: u32) -> trc::Result>; async fn get_principal_name(&self, principal_id: u32) -> trc::Result>; async fn get_member_of(&self, principal_id: u32) -> trc::Result>; async fn get_members(&self, principal_id: u32) -> trc::Result>; async fn create_principal( &self, principal: PrincipalSet, tenant_id: Option, allowed_permissions: Option<&Permissions>, ) -> trc::Result; async fn update_principal(&self, params: UpdatePrincipal<'_>) -> trc::Result; async fn delete_principal(&self, by: QueryBy<'_>) -> trc::Result; async fn list_principals( &self, filter: Option<&str>, tenant_id: Option, types: &[Type], fetch: bool, page: usize, limit: usize, ) -> trc::Result>; async fn count_principals( &self, filter: Option<&str>, typ: Option, tenant_id: Option, ) -> trc::Result; async fn principal_ids( &self, typ: Option, tenant_id: Option, ) -> trc::Result; async fn map_principal( &self, principal: Principal, fields: &[PrincipalField], ) -> trc::Result; } #[allow(async_fn_in_trait)] trait ValidateDirectory: Sized { async fn validate_email( &self, email: &str, tenant_id: Option, create_if_missing: bool, ) -> trc::Result<()>; } impl ManageDirectory for Store { async fn get_principal(&self, principal_id: u32) -> trc::Result> { let archive = self .get_value::>(ValueKey::from(ValueClass::Directory( DirectoryClass::Principal(principal_id), ))) .await .caused_by(trc::location!())?; if let Some(archive) = archive { let mut principal = archive .deserialize::() .caused_by(trc::location!())?; principal.id = principal_id; Ok(Some(principal)) } else { Ok(None) } } async fn get_principal_name(&self, principal_id: u32) -> trc::Result> { let archive = self .get_value::>(ValueKey::from(ValueClass::Directory( DirectoryClass::Principal(principal_id), ))) .await .caused_by(trc::location!())?; if let Some(archive) = archive { let principal = archive .unarchive::() .caused_by(trc::location!())?; Ok(Some(principal.name.as_str().into())) } else { Ok(None) } } async fn get_principal_id(&self, name: &str) -> trc::Result> { self.get_principal_info(name).await.map(|v| v.map(|v| v.id)) } async fn get_principal_info(&self, name: &str) -> trc::Result> { self.get_value::(ValueKey::from(ValueClass::Directory( DirectoryClass::NameToId(name.as_bytes().to_vec()), ))) .await .caused_by(trc::location!()) } // Used by all directories except internal async fn get_or_create_principal_id(&self, name: &str, typ: Type) -> trc::Result { let mut try_count = 0; let name = name.to_lowercase(); let mut principal_id = None; loop { // Try to obtain ID if let Some(principal_id) = self .get_principal_id(&name) .await .caused_by(trc::location!())? { return Ok(principal_id); } let principal_id = if let Some(principal_id) = principal_id { principal_id } else { let principal_id_ = self .assign_document_ids(u32::MAX, Collection::Principal, 1) .await .caused_by(trc::location!())?; if principal_id_ == FALLBACK_ADMIN_ID { return Err(trc::StoreEvent::UnexpectedError .into_err() .details("ID assignment failed") .caused_by(trc::location!())); } principal_id = Some(principal_id_); principal_id_ }; // Prepare principal let mut principal = Principal::new(principal_id, typ); principal.name = name.as_str().into(); // Write principal ID let name_key = ValueClass::Directory(DirectoryClass::NameToId(name.as_bytes().to_vec())); let mut batch = BatchBuilder::new(); batch .with_account_id(u32::MAX) .with_collection(Collection::Principal) .assert_value(name_key.clone(), ()) .with_document(principal_id); build_search_index(&mut batch, principal_id, None, Some(&principal)); principal.sort(); batch .set( name_key, PrincipalInfo::new(principal_id, typ, None).serialize(), ) .set( ValueClass::Directory(DirectoryClass::Principal(principal_id)), Archiver::new(principal) .serialize() .caused_by(trc::location!())?, ); // Add default user role if typ == Type::Individual { batch .set( ValueClass::Directory(DirectoryClass::MemberOf { principal_id, member_of: ROLE_USER, }), vec![Type::Role as u8], ) .set( ValueClass::Directory(DirectoryClass::Members { principal_id: ROLE_USER, has_member: principal_id, }), vec![], ); } match self.write(batch.build_all()).await { Ok(_) => { return Ok(principal_id); } Err(err) => { if err.is_assertion_failure() && try_count < 3 { try_count += 1; continue; } else { return Err(err.caused_by(trc::location!())); } } } } } async fn create_principal( &self, mut principal_set: PrincipalSet, mut tenant_id: Option, allowed_permissions: Option<&Permissions>, ) -> trc::Result { // Make sure the principal has a name let name = principal_set.name().to_lowercase(); if name.is_empty() { return Err(err_missing(PrincipalField::Name)); } let mut valid_domains: AHashSet = AHashSet::new(); // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL // Validate tenant #[cfg(feature = "enterprise")] if let Some(tenant_id) = tenant_id { let tenant = self .query(crate::QueryParams::id(tenant_id).with_return_member_of(false)) .await? .ok_or_else(|| { trc::ManageEvent::NotFound .into_err() .id(tenant_id) .details("Tenant not found") .caused_by(trc::location!()) })?; // Enforce tenant quotas if let Some(limit) = tenant .directory_quota(&principal_set.typ()) .filter(|q| *q > 0) { // Obtain number of principals let total = self .count_principals(None, principal_set.typ().into(), tenant_id.into()) .await .caused_by(trc::location!())? as u32; if total >= limit { trc::bail!( trc::LimitEvent::TenantQuota .into_err() .details("Tenant principal quota exceeded") .ctx(trc::Key::Details, principal_set.typ().description()) .ctx(trc::Key::Limit, limit) .ctx(trc::Key::Total, total) ); } } } // SPDX-SnippetEnd // Make sure new name is not taken if self .get_principal_id(&name) .await .caused_by(trc::location!())? .is_some() { return Err(err_exists(PrincipalField::Name, name)); } let mut create_principal = Principal::new(0, principal_set.typ()); // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL // Obtain tenant id, only if no default tenant is provided #[cfg(feature = "enterprise")] if let (Some(tenant_name), None) = (principal_set.take_str(PrincipalField::Tenant), tenant_id) { tenant_id = self .get_principal_info(&tenant_name) .await .caused_by(trc::location!())? .filter(|v| v.typ == Type::Tenant) .ok_or_else(|| not_found(tenant_name.clone()))? .id .into(); } // Tenants must provide principal names including a valid domain #[cfg(feature = "enterprise")] if let Some(tenant_id) = tenant_id { if matches!(principal_set.typ, Type::Tenant) { return Err(error( "Invalid field", "Tenants cannot contain a tenant field".into(), )); } create_principal.data.push(PrincipalData::Tenant(tenant_id)); if !matches!(create_principal.typ, Type::Tenant | Type::Domain) { if let Some(domain) = name.try_domain_part() && self .get_principal_info(domain) .await .caused_by(trc::location!())? .filter(|v| v.typ == Type::Domain && v.has_tenant_access(tenant_id.into())) .is_some() { valid_domains.insert(domain.into()); } if valid_domains.is_empty() { return Err(error( "Invalid principal name", "Principal name must include a valid domain assigned to the tenant".into(), )); } } } // SPDX-SnippetEnd // Set fields create_principal.name = name; let mut has_secret = false; for secret in principal_set .take_str_array(PrincipalField::Secrets) .unwrap_or_default() { if secret.is_otp_secret() { create_principal.data.push(PrincipalData::OtpAuth(secret)); } else if secret.is_app_secret() { create_principal .data .push(PrincipalData::AppPassword(secret)); } else if !has_secret { has_secret = true; create_principal.data.push(PrincipalData::Password(secret)); } } if let Some(description) = principal_set.take_str(PrincipalField::Description) { create_principal .data .push(PrincipalData::Description(description)); } if let Some(picture) = principal_set.take_str(PrincipalField::Picture) { create_principal.data.push(PrincipalData::Picture(picture)); } if let Some(picture) = principal_set.take_str(PrincipalField::Locale) { create_principal.data.push(PrincipalData::Locale(picture)); } for url in principal_set .take_str_array(PrincipalField::Urls) .unwrap_or_default() { create_principal.data.push(PrincipalData::Url(url)); } for member in principal_set .take_str_array(PrincipalField::ExternalMembers) .unwrap_or_default() { create_principal .data .push(PrincipalData::ExternalMember(member)); } if let Some(quotas) = principal_set.take_int_array(PrincipalField::Quota) { for (idx, quota) in quotas.into_iter().take(Type::MAX_ID + 2).enumerate() { if quota != 0 { if idx != 0 { create_principal.data.push(PrincipalData::DirectoryQuota { quota: quota as u32, typ: Type::from_u8((idx - 1) as u8), }); } else { create_principal.data.push(PrincipalData::DiskQuota(quota)); } } } } // Map member names let mut members = Vec::new(); let mut member_of = Vec::new(); let mut changed_principals = ChangedPrincipals::default(); for (field, expected_type) in [ (PrincipalField::Members, None), (PrincipalField::MemberOf, Some(Type::Group)), (PrincipalField::Lists, Some(Type::List)), (PrincipalField::Roles, Some(Type::Role)), ] { if let Some(names) = principal_set.take_str_array(field) { let list = if field == PrincipalField::Members { &mut members } else { &mut member_of }; for name in names { let item = match ( self.get_principal_info(&name) .await .caused_by(trc::location!())? .filter(|v| { expected_type.is_none_or(|t| v.typ == t) && v.has_tenant_access(tenant_id) }), field.map_internal_roles(&name), ) { (_, Some(v)) => v, (Some(v), _) => { if field == PrincipalField::Members { // Update principal members changed_principals.add_change( v.id, v.typ, PrincipalField::MemberOf, ); } v } _ => { return Err(not_found(name)); } }; list.push(item); } } } // Map permissions let mut permissions = AHashMap::new(); for field in [ PrincipalField::EnabledPermissions, PrincipalField::DisabledPermissions, ] { let is_disabled = field == PrincipalField::DisabledPermissions; if let Some(names) = principal_set.take_str_array(field) { for name in names { let permission = Permission::from_name(&name).ok_or_else(|| { error( format!("Invalid {} value", field.as_str()), format!("Permission {name:?} is invalid").into(), ) })?; if !permissions.contains_key(&permission) { if allowed_permissions .as_ref() .is_none_or(|p| p.get(permission as usize)) || is_disabled { permissions.insert(permission, is_disabled); } else { return Err(error( "Invalid permission", format!("Your account cannot grant the {name:?} permission").into(), )); } } } } } if !permissions.is_empty() { for (permission, v) in permissions { create_principal.data.push(PrincipalData::Permission { permission_id: permission.id(), grant: !v, }); } } // Make sure the e-mail is not taken and validate domain if create_principal.typ != Type::OauthClient { for (idx, email) in principal_set .take_str_array(PrincipalField::Emails) .unwrap_or_default() .into_iter() .enumerate() { let email = email.to_lowercase(); if self.rcpt(&email).await.caused_by(trc::location!())? != RcptType::Invalid { return Err(err_exists(PrincipalField::Emails, email.to_string())); } if let Some(domain) = email.try_domain_part() && valid_domains.insert(domain.into()) { self.get_principal_info(domain) .await .caused_by(trc::location!())? .filter(|v| v.typ == Type::Domain && v.has_tenant_access(tenant_id)) .ok_or_else(|| not_found(domain.to_string()))?; } if idx == 0 { create_principal .data .push(PrincipalData::PrimaryEmail(email)); } else { create_principal.data.push(PrincipalData::EmailAlias(email)); } } } // Write principal let principal_id = self .assign_document_ids(u32::MAX, Collection::Principal, 1) .await .caused_by(trc::location!())?; if principal_id == FALLBACK_ADMIN_ID { return Err(trc::StoreEvent::UnexpectedError .into_err() .details("ID assignment failed") .caused_by(trc::location!())); } create_principal.id = principal_id; let mut batch = BatchBuilder::new(); let pinfo_name = PrincipalInfo::new(principal_id, create_principal.typ, tenant_id); let pinfo_email = PrincipalInfo::new(principal_id, create_principal.typ, None); // Validate object size if create_principal.object_size() > 100_000 { return Err(error( "Invalid parameter", "Principal object size exceeds 100kb safety limit.".into(), )); } // Serialize create_principal.sort(); let archiver = Archiver::new(create_principal); let principal_bytes = archiver.serialize().caused_by(trc::location!())?; let create_principal = archiver.into_inner(); batch .with_account_id(u32::MAX) .with_collection(Collection::Principal) .with_document(principal_id) .assert_value( ValueClass::Directory(DirectoryClass::NameToId( create_principal.name().as_bytes().to_vec(), )), (), ); build_search_index(&mut batch, principal_id, None, Some(&create_principal)); batch .set( ValueClass::Directory(DirectoryClass::Principal(principal_id)), principal_bytes, ) .set( ValueClass::Directory(DirectoryClass::NameToId( create_principal.name.as_bytes().to_vec(), )), pinfo_name.serialize(), ); // Write email to id mapping for email in create_principal.email_addresses() { batch.set( ValueClass::Directory(DirectoryClass::EmailToId(email.as_bytes().to_vec())), pinfo_email.serialize(), ); } // Write membership for member_of in member_of { batch.set( ValueClass::Directory(DirectoryClass::MemberOf { principal_id, member_of: member_of.id, }), vec![member_of.typ as u8], ); batch.set( ValueClass::Directory(DirectoryClass::Members { principal_id: member_of.id, has_member: principal_id, }), vec![], ); } for member in members { batch.set( ValueClass::Directory(DirectoryClass::MemberOf { principal_id: member.id, member_of: principal_id, }), vec![create_principal.typ as u8], ); batch.set( ValueClass::Directory(DirectoryClass::Members { principal_id, has_member: member.id, }), vec![], ); } self.write(batch.build_all()) .await .map(|_| CreatedPrincipal { id: principal_id, changed_principals, }) } async fn delete_principal(&self, by: QueryBy<'_>) -> trc::Result { // Obtain principal let principal_id = match by { QueryBy::Name(name) => self .get_principal_id(name) .await .caused_by(trc::location!())? .ok_or_else(|| not_found(name.to_string()))?, QueryBy::Id(principal_id) => principal_id, QueryBy::Credentials(_) => unreachable!(), }; let principal_ = self .get_value::>(ValueKey::from(ValueClass::Directory( DirectoryClass::Principal(principal_id), ))) .await .caused_by(trc::location!())? .ok_or_else(|| not_found(principal_id.to_string()))?; let principal = principal_ .unarchive::() .caused_by(trc::location!())?; let typ = Type::from(&principal.typ); let mut batch = BatchBuilder::new(); batch .with_account_id(u32::MAX) .with_collection(Collection::Principal); let tenant = principal.data.iter().find_map(|data| { if let ArchivedPrincipalData::Tenant(tenant_id) = data { Some(tenant_id.to_native()) } else { None } }); // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL // Make sure tenant has no data #[cfg(feature = "enterprise")] match typ { Type::Individual | Type::Group => { // Update tenant quota if let Some(tenant_id) = tenant { let quota = self .get_counter(DirectoryClass::UsedQuota(principal_id)) .await .caused_by(trc::location!())?; if quota > 0 { batch.add(DirectoryClass::UsedQuota(tenant_id), -quota); } } } Type::Tenant => { let tenant_members = self .list_principals( None, principal_id.into(), &[ Type::Individual, Type::Group, Type::Role, Type::List, Type::Resource, Type::Other, Type::Location, Type::Domain, Type::ApiKey, ], false, 0, 0, ) .await .caused_by(trc::location!())?; if tenant_members.total > 0 { let mut message = String::from("Tenant must have no members to be deleted: Found: "); for (num, principal) in tenant_members.items.iter().enumerate() { if num > 0 { message.push_str(", "); } message.push_str(principal.name()); } if tenant_members.total > 5 { message.push_str(" and "); message.push_str(&(tenant_members.total - 5).to_string()); message.push_str(" others"); } return Err(error("Tenant has members", message.into())); } } Type::Domain => { if let Some(tenant_id) = tenant { let name = principal.name.as_str(); let tenant_members = self .list_principals( None, tenant_id.into(), &[ Type::Individual, Type::Group, Type::Role, Type::List, Type::Resource, Type::Other, Type::Location, ], false, 0, 0, ) .await .caused_by(trc::location!())?; let domain_members = tenant_members .items .iter() .filter(|v| { v.name() .rsplit_once('@') .is_some_and(|(_, d)| d.eq_ignore_ascii_case(name)) }) .collect::>(); let total_domain_members = domain_members.len(); if total_domain_members > 0 { let mut message = String::from("Domains must have no members to be deleted: Found: "); for (num, principal) in domain_members.iter().enumerate() { if num > 0 { message.push_str(", "); } message.push_str(principal.name()); } if total_domain_members > 5 { message.push_str(" and "); message.push_str(&(total_domain_members - 5).to_string()); message.push_str(" others"); } return Err(error("Domain has members", message.into())); } } } _ => {} } // SPDX-SnippetEnd // Revoke ACLs, obtain all changed principals let mut changed_principals = ChangedPrincipals::default(); for member_id in self .acl_revoke_all(principal_id) .await .caused_by(trc::location!())? { changed_principals.add_change( member_id, Type::Individual, PrincipalField::EnabledPermissions, ); } // Delete principal batch .with_document(principal_id) .clear(DirectoryClass::NameToId(principal.name.as_bytes().to_vec())) .clear(DirectoryClass::Principal(principal_id)) .clear(DirectoryClass::UsedQuota(principal_id)); for email in principal.data.iter() { if let ArchivedPrincipalData::PrimaryEmail(email) | ArchivedPrincipalData::EmailAlias(email) = email { batch.clear(DirectoryClass::EmailToId(email.as_bytes().to_vec())); } } build_search_index(&mut batch, principal_id, Some(principal), None); for member in self .get_member_of(principal_id) .await .caused_by(trc::location!())? { // Update changed principals changed_principals.add_member_change( principal_id, typ, member.principal_id, member.typ, ); // Remove memberOf batch.clear(DirectoryClass::MemberOf { principal_id, member_of: member.principal_id, }); batch.clear(DirectoryClass::Members { principal_id: member.principal_id, has_member: principal_id, }); } for member_id in self .get_members(principal_id) .await .caused_by(trc::location!())? { // Update changed principals if let Some(member_info) = self .get_principal(member_id) .await .caused_by(trc::location!())? { changed_principals.add_member_change(member_id, member_info.typ, principal_id, typ); } // Remove members batch.clear(DirectoryClass::MemberOf { principal_id: member_id, member_of: principal_id, }); batch.clear(DirectoryClass::Members { principal_id, has_member: member_id, }); } // Delete push subscriptions if matches!(typ, Type::Individual) { batch.untag(field::PrincipalField::PushSubscriptions); } self.write(batch.build_all()) .await .caused_by(trc::location!())?; changed_principals.add_deletion(principal_id, typ); Ok(changed_principals) } async fn update_principal( &self, params: UpdatePrincipal<'_>, ) -> trc::Result { let principal_id = match params.query { QueryBy::Name(name) => self .get_principal_id(name) .await .caused_by(trc::location!())? .ok_or_else(|| not_found(name.to_string()))?, QueryBy::Id(principal_id) => principal_id, QueryBy::Credentials(_) => unreachable!(), }; let changes = params.changes; let tenant_id = params.tenant_id; // Fetch principal let principal_ = self .get_value::>(ValueKey::from(ValueClass::Directory( DirectoryClass::Principal(principal_id), ))) .await .caused_by(trc::location!())? .ok_or_else(|| not_found(principal_id))?; let prev_principal = principal_ .to_unarchived::() .caused_by(trc::location!())?; let mut principal = prev_principal .deserialize::() .caused_by(trc::location!())?; principal.id = principal_id; let principal_type = principal.typ; let validate_emails = principal_type != Type::OauthClient; // Keep track of changed principals let mut changed_principals = ChangedPrincipals::default(); // Obtain members and memberOf let mut member_of = self .get_member_of(principal_id) .await .caused_by(trc::location!())?; let mut members = self .get_members(principal_id) .await .caused_by(trc::location!())?; // Prepare changes let mut batch = BatchBuilder::new(); let mut pinfo_name = PrincipalInfo::new(principal_id, principal_type, principal.tenant()).serialize(); let pinfo_email = PrincipalInfo::new(principal_id, principal_type, None).serialize(); let update_principal = !changes.is_empty() && !changes.iter().all(|c| { matches!( c.field, PrincipalField::MemberOf | PrincipalField::Members | PrincipalField::Lists | PrincipalField::Roles ) }); let mut used_quota: Option = None; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL // Obtain used quota #[cfg(feature = "enterprise")] if tenant_id.is_none() && changes .iter() .any(|c| matches!(c.field, PrincipalField::Tenant)) { let quota = self .get_counter(DirectoryClass::UsedQuota(principal_id)) .await .caused_by(trc::location!())?; if quota > 0 { used_quota = Some(quota); } } // SPDX-SnippetEnd // Allowed principal types for Member fields let allowed_member_types = match principal_type { Type::Group => &[Type::Individual, Type::Group][..], Type::Resource => &[Type::Resource][..], Type::Location => &[ Type::Location, Type::Resource, Type::Individual, Type::Group, Type::Other, ][..], Type::List => &[Type::Individual, Type::Group][..], Type::Other | Type::Domain | Type::Tenant | Type::Individual | Type::ApiKey | Type::OauthClient => &[][..], Type::Role => &[Type::Role][..], }; let mut valid_domains = AHashSet::new(); // Process changes for change in changes { match (change.action, change.field, change.value) { (PrincipalAction::Set, PrincipalField::Name, PrincipalValue::String(new_name)) => { // Make sure new name is not taken let new_name = new_name.to_lowercase(); if principal.name() != new_name { if tenant_id.is_some() && !matches!(principal_type, Type::Tenant | Type::Domain) { if let Some(domain) = new_name.try_domain_part() && self .get_principal_info(domain) .await .caused_by(trc::location!())? .filter(|v| { v.typ == Type::Domain && v.has_tenant_access(tenant_id) }) .is_some() { valid_domains.insert(domain.to_string()); } if valid_domains.is_empty() { return Err(error( "Invalid principal name", "Principal name must include a valid domain assigned to the tenant".into(), )); } } if self .get_principal_id(&new_name) .await .caused_by(trc::location!())? .is_some() { return Err(err_exists(PrincipalField::Name, new_name)); } batch.clear(ValueClass::Directory(DirectoryClass::NameToId( principal.name().as_bytes().to_vec(), ))); batch.set( ValueClass::Directory(DirectoryClass::NameToId( new_name.as_bytes().to_vec(), )), pinfo_name.clone(), ); principal.name = new_name; // Name changed, update changed principals changed_principals.add_change(principal_id, principal_type, change.field); } } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] ( PrincipalAction::Set, PrincipalField::Tenant, PrincipalValue::String(tenant_name), ) if tenant_id.is_none() => { if !tenant_name.is_empty() { let tenant_info = self .get_principal_info(&tenant_name) .await .caused_by(trc::location!())? .ok_or_else(|| not_found(tenant_name.clone()))?; if tenant_info.typ != Type::Tenant { return Err(error( "Not a tenant", format!("Principal {tenant_name:?} is not a tenant").into(), )); } if principal.tenant() == Some(tenant_info.id) { continue; } // Update quota if let Some(used_quota) = used_quota { if let Some(old_tenant_id) = principal.tenant() { batch.add(DirectoryClass::UsedQuota(old_tenant_id), -used_quota); } batch.add(DirectoryClass::UsedQuota(tenant_info.id), used_quota); } // Tenant changed, update changed principals changed_principals.add_change(principal_id, principal_type, change.field); principal .data .retain(|v| !matches!(v, PrincipalData::Tenant(_))); principal.data.push(PrincipalData::Tenant(tenant_info.id)); pinfo_name = PrincipalInfo::new(principal_id, principal_type, tenant_info.id.into()) .serialize(); } else if let Some(tenant_id) = principal.tenant() { // Update quota if let Some(used_quota) = used_quota { batch.add(DirectoryClass::UsedQuota(tenant_id), -used_quota); } // Tenant changed, update changed principals changed_principals.add_change(principal_id, principal_type, change.field); principal .data .retain(|v| !matches!(v, PrincipalData::Tenant(_))); pinfo_name = PrincipalInfo::new(principal_id, principal_type, None).serialize(); } else { continue; } batch.set( ValueClass::Directory(DirectoryClass::NameToId( principal.name().as_bytes().to_vec(), )), pinfo_name.clone(), ); } // SPDX-SnippetEnd ( PrincipalAction::Set, PrincipalField::Secrets, value @ (PrincipalValue::StringList(_) | PrincipalValue::String(_)), ) => { // Password changed, update changed principals changed_principals.add_change(principal_id, principal_type, change.field); principal.data.retain(|v| { !matches!( v, PrincipalData::Password(_) | PrincipalData::AppPassword(_) | PrincipalData::OtpAuth(_) ) }); let mut has_secret = false; for secret in value.into_str_array() { if secret.is_otp_secret() { principal.data.push(PrincipalData::OtpAuth(secret)); } else if secret.is_app_secret() { principal.data.push(PrincipalData::AppPassword(secret)); } else if !has_secret { has_secret = true; principal.data.push(PrincipalData::Password(secret)); } } } ( PrincipalAction::AddItem, PrincipalField::Secrets, PrincipalValue::String(secret), ) => { if !principal.data.iter().any(|v| match v { PrincipalData::Password(v) | PrincipalData::AppPassword(v) | PrincipalData::OtpAuth(v) => *v == secret, _ => false, }) { if secret.is_app_secret() { principal.data.push(PrincipalData::AppPassword(secret)); } else if secret.is_otp_secret() { principal .data .retain(|v| !matches!(v, PrincipalData::OtpAuth(_))); principal.data.push(PrincipalData::OtpAuth(secret)); } else { principal .data .retain(|v| !matches!(v, PrincipalData::Password(_))); principal.data.push(PrincipalData::Password(secret)); } // Password changed, update changed principals changed_principals.add_change(principal_id, principal_type, change.field); } } ( PrincipalAction::RemoveItem, PrincipalField::Secrets, PrincipalValue::String(secret), ) => { // Password changed, update changed principals changed_principals.add_change(principal_id, principal_type, change.field); if secret.is_app_secret() || secret.is_otp_secret() { principal.data.retain(|v| match v { PrincipalData::AppPassword(v) | PrincipalData::OtpAuth(v) => { *v != secret && !v.starts_with(secret.as_str()) } _ => true, }); } else if !secret.is_empty() { principal.data.retain(|v| match v { PrincipalData::Password(v) => *v != secret, _ => true, }); } else { principal.data.retain(|v| { !matches!(v, PrincipalData::AppPassword(_) | PrincipalData::OtpAuth(_)) }); } } ( PrincipalAction::Set, PrincipalField::Description, PrincipalValue::String(value), ) => { principal .data .retain(|v| !matches!(v, PrincipalData::Description(_))); if !value.is_empty() { principal.data.push(PrincipalData::Description(value)); } } (PrincipalAction::Set, PrincipalField::Picture, PrincipalValue::String(value)) => { principal .data .retain(|v| !matches!(v, PrincipalData::Picture(_))); if !value.is_empty() { principal.data.push(PrincipalData::Picture(value)); } } (PrincipalAction::Set, PrincipalField::Locale, PrincipalValue::String(value)) => { principal .data .retain(|v| !matches!(v, PrincipalData::Locale(_))); if !value.is_empty() { principal.data.push(PrincipalData::Locale(value)); } } (PrincipalAction::Set, PrincipalField::Quota, PrincipalValue::Integer(quota)) if matches!( principal_type, Type::Individual | Type::Group | Type::Tenant ) => { // Quota changed, update changed principals changed_principals.add_change(principal_id, principal_type, change.field); principal .data .retain(|v| !matches!(v, PrincipalData::DiskQuota(_))); principal.data.push(PrincipalData::DiskQuota(quota)); } (PrincipalAction::Set, PrincipalField::Quota, PrincipalValue::String(quota)) if matches!( principal_type, Type::Individual | Type::Group | Type::Tenant ) && quota.is_empty() => { // Quota changed, update changed principals changed_principals.add_change(principal_id, principal_type, change.field); principal .data .retain(|v| !matches!(v, PrincipalData::DiskQuota(_))); } ( PrincipalAction::Set, PrincipalField::Quota, PrincipalValue::IntegerList(quotas), ) if matches!(principal_type, Type::Tenant) && quotas.len() <= (Type::MAX_ID + 2) => { let mut new_quota = None; principal.data.retain(|v| { !matches!( v, PrincipalData::DiskQuota(_) | PrincipalData::DirectoryQuota { .. } ) }); for (idx, quota) in quotas.into_iter().enumerate() { if quota != 0 { if idx != 0 { principal.data.push(PrincipalData::DirectoryQuota { quota: quota as u32, typ: Type::from_u8((idx - 1) as u8), }); } else { new_quota = Some(quota); } } } if let Some(new_quota) = new_quota { principal.data.push(PrincipalData::DiskQuota(new_quota)); } } // Emails ( PrincipalAction::Set, PrincipalField::Emails, PrincipalValue::StringList(emails), ) => { // Validate unique emails let emails = emails .into_iter() .map(|v| v.to_lowercase()) .collect::>(); for email in &emails { if !principal.email_addresses().any(|v| v == email) { if validate_emails { self.validate_email(email, tenant_id, params.create_domains) .await?; } batch.set( ValueClass::Directory(DirectoryClass::EmailToId( email.as_bytes().to_vec(), )), pinfo_email.clone(), ); } } for email in principal.email_addresses() { if !emails.iter().any(|v| v == email) { batch.clear(ValueClass::Directory(DirectoryClass::EmailToId( email.as_bytes().to_vec(), ))); } } // Emails changed, update changed principals changed_principals.add_change(principal_id, principal_type, change.field); principal.data.retain(|v| { !matches!( v, PrincipalData::PrimaryEmail(_) | PrincipalData::EmailAlias(_) ) }); for (idx, email) in emails.into_iter().enumerate() { if idx == 0 { principal.data.push(PrincipalData::PrimaryEmail(email)); } else { principal.data.push(PrincipalData::EmailAlias(email)); } } } ( PrincipalAction::AddItem, PrincipalField::Emails, PrincipalValue::String(email), ) => { let email = email.to_lowercase(); let mut emails_iter = principal.email_addresses().peekable(); let has_emails = emails_iter.peek().is_some(); let email_exists = emails_iter.any(|v| v == email); drop(emails_iter); if !email_exists { if validate_emails { self.validate_email(&email, tenant_id, params.create_domains) .await?; } batch.set( ValueClass::Directory(DirectoryClass::EmailToId( email.as_bytes().to_vec(), )), pinfo_email.clone(), ); if has_emails { principal.data.push(PrincipalData::EmailAlias(email)); } else { principal.data.push(PrincipalData::PrimaryEmail(email)); } // Emails changed, update changed principals changed_principals.add_change(principal_id, principal_type, change.field); } } ( PrincipalAction::RemoveItem, PrincipalField::Emails, PrincipalValue::String(email), ) => { let email = email.to_lowercase(); if principal.email_addresses().any(|v| v == email) { let mut deleted_primary = false; principal.data.retain(|v| match v { PrincipalData::EmailAlias(v) => v != &email, PrincipalData::PrimaryEmail(v) => { if v == &email { deleted_primary = true; false } else { true } } _ => true, }); batch.clear(ValueClass::Directory(DirectoryClass::EmailToId( email.as_bytes().to_vec(), ))); if deleted_primary { for data in &mut principal.data { if let PrincipalData::EmailAlias(email) = data { *data = PrincipalData::PrimaryEmail(std::mem::take(email)); break; } } } // Emails changed, update changed principals changed_principals.add_change(principal_id, principal_type, change.field); } } // MemberOf ( PrincipalAction::Set, PrincipalField::MemberOf | PrincipalField::Lists | PrincipalField::Roles, PrincipalValue::StringList(members), ) => { let mut new_member_of = Vec::new(); for member in members { let member_info = match ( self.get_principal_info(&member) .await .caused_by(trc::location!())? .filter(|p| p.has_tenant_access(tenant_id)), change.field.map_internal_roles(&member), ) { (_, Some(v)) => v, (Some(v), _) => v, _ => { return Err(not_found(member.clone())); } }; validate_member_of(change.field, principal_type, member_info.typ, &member)?; if !member_of.iter().any(|v| v.principal_id == member_info.id) { // Update changed principal ids changed_principals.add_member_change( principal_id, principal_type, member_info.id, member_info.typ, ); batch.set( ValueClass::Directory(DirectoryClass::MemberOf { principal_id, member_of: member_info.id, }), vec![member_info.typ as u8], ); batch.set( ValueClass::Directory(DirectoryClass::Members { principal_id: member_info.id, has_member: principal_id, }), vec![], ); } new_member_of.push(MemberOf { principal_id: member_info.id, typ: member_info.typ, }); } for member in &member_of { if !new_member_of .iter() .any(|v| v.principal_id == member.principal_id) { // Update changed principal ids changed_principals.add_member_change( principal_id, principal_type, member.principal_id, member.typ, ); batch.clear(ValueClass::Directory(DirectoryClass::MemberOf { principal_id, member_of: member.principal_id, })); batch.clear(ValueClass::Directory(DirectoryClass::Members { principal_id: member.principal_id, has_member: principal_id, })); } } member_of = new_member_of; } ( PrincipalAction::AddItem, PrincipalField::MemberOf | PrincipalField::Lists | PrincipalField::Roles, PrincipalValue::String(member), ) => { let member_info = match ( self.get_principal_info(&member) .await .caused_by(trc::location!())? .filter(|p| p.has_tenant_access(tenant_id)), change.field.map_internal_roles(&member), ) { (_, Some(v)) => v, (Some(v), _) => v, _ => { return Err(not_found(member.clone())); } }; if !member_of.iter().any(|v| v.principal_id == member_info.id) { validate_member_of(change.field, principal_type, member_info.typ, &member)?; // Update changed principal ids changed_principals.add_member_change( principal_id, principal_type, member_info.id, member_info.typ, ); batch.set( ValueClass::Directory(DirectoryClass::MemberOf { principal_id, member_of: member_info.id, }), vec![member_info.typ as u8], ); batch.set( ValueClass::Directory(DirectoryClass::Members { principal_id: member_info.id, has_member: principal_id, }), vec![], ); member_of.push(MemberOf { principal_id: member_info.id, typ: member_info.typ, }); } } ( PrincipalAction::RemoveItem, PrincipalField::MemberOf | PrincipalField::Lists | PrincipalField::Roles, PrincipalValue::String(member), ) => { if let Some(member_info) = self.get_principal_info(&member) .await .caused_by(trc::location!())? .or_else(|| { change.field.map_internal_role_name(&member).map(|id| { PrincipalInfo { id, typ: Type::Role, tenant: None, } }) }) { for (pos, member) in member_of.iter().enumerate() { if member.principal_id == member_info.id { // Update changed principal ids changed_principals.add_member_change( principal_id, principal_type, member_info.id, member_info.typ, ); batch.clear(ValueClass::Directory(DirectoryClass::MemberOf { principal_id, member_of: member_info.id, })); batch.clear(ValueClass::Directory(DirectoryClass::Members { principal_id: member_info.id, has_member: principal_id, })); member_of.remove(pos); break; } } } } ( PrincipalAction::Set, PrincipalField::Members, PrincipalValue::StringList(members_), ) => { let mut new_members = Vec::new(); for member in members_ { let member_info = self .get_principal_info(&member) .await .caused_by(trc::location!())? .filter(|p| p.has_tenant_access(tenant_id)) .ok_or_else(|| not_found(member.clone()))?; if !allowed_member_types.contains(&member_info.typ) { return Err(error( "Invalid members value", format!( "Principal {member:?} is not one of {}.", allowed_member_types .iter() .map(|v| v.description()) .collect::>() .join(", ") ) .into(), )); } if !members.contains(&member_info.id) { // Update changed principal ids changed_principals.add_member_change( member_info.id, member_info.typ, principal_id, principal_type, ); batch.set( ValueClass::Directory(DirectoryClass::MemberOf { principal_id: member_info.id, member_of: principal_id, }), vec![principal_type as u8], ); batch.set( ValueClass::Directory(DirectoryClass::Members { principal_id, has_member: member_info.id, }), vec![], ); } new_members.push(member_info.id); } for member_id in &members { if !new_members.contains(member_id) { // Update changed principal ids if principal_type != Type::List && let Some(member_info) = self .get_principal(*member_id) .await .caused_by(trc::location!())? { changed_principals.add_member_change( *member_id, member_info.typ, principal_id, principal_type, ); } batch.clear(ValueClass::Directory(DirectoryClass::MemberOf { principal_id: *member_id, member_of: principal_id, })); batch.clear(ValueClass::Directory(DirectoryClass::Members { principal_id, has_member: *member_id, })); } } members = new_members; } ( PrincipalAction::AddItem, PrincipalField::Members, PrincipalValue::String(member), ) => { let member_info = self .get_principal_info(&member) .await .caused_by(trc::location!())? .filter(|p| p.has_tenant_access(tenant_id)) .ok_or_else(|| not_found(member.clone()))?; if !members.contains(&member_info.id) { if !allowed_member_types.contains(&member_info.typ) { return Err(error( "Invalid members value", format!( "Principal {member:?} is not one of {}.", allowed_member_types .iter() .map(|v| v.description()) .collect::>() .join(", ") ) .into(), )); } // Update changed principal ids changed_principals.add_member_change( member_info.id, member_info.typ, principal_id, principal_type, ); batch.set( ValueClass::Directory(DirectoryClass::MemberOf { principal_id: member_info.id, member_of: principal_id, }), vec![principal_type as u8], ); batch.set( ValueClass::Directory(DirectoryClass::Members { principal_id, has_member: member_info.id, }), vec![], ); members.push(member_info.id); } } ( PrincipalAction::RemoveItem, PrincipalField::Members, PrincipalValue::String(member), ) => { if let Some(member_info) = self .get_principal_info(&member) .await .caused_by(trc::location!())? { for (pos, member_id) in members.iter().enumerate() { if *member_id == member_info.id { // Update changed principal ids changed_principals.add_member_change( member_info.id, member_info.typ, principal_id, principal_type, ); batch.clear(ValueClass::Directory(DirectoryClass::MemberOf { principal_id: member_info.id, member_of: principal_id, })); batch.clear(ValueClass::Directory(DirectoryClass::Members { principal_id, has_member: member_info.id, })); members.remove(pos); break; } } } } ( PrincipalAction::Set, PrincipalField::EnabledPermissions | PrincipalField::DisabledPermissions, PrincipalValue::StringList(names), ) => { let is_disabled = change.field == PrincipalField::DisabledPermissions; let mut permissions = AHashSet::with_capacity(names.len()); for name in names { let permission = Permission::from_name(&name).ok_or_else(|| { error( format!("Invalid {} value", change.field.as_str()), format!("Permission {name:?} is invalid").into(), ) })?; if !permissions.contains(&permission) { if params .allowed_permissions .as_ref() .is_none_or(|p| p.get(permission as usize)) || is_disabled { permissions.insert(permission); } else { return Err(error( "Invalid permission", format!("Your account cannot grant the {name:?} permission") .into(), )); } } } principal.remove_permissions(!is_disabled); if !permissions.is_empty() { principal.add_permissions(permissions.into_iter().map(|permission| { PermissionGrant { permission, grant: !is_disabled, } })); } // Permissions changed, update changed principals changed_principals.add_change(principal_id, principal_type, change.field); } ( PrincipalAction::AddItem, PrincipalField::EnabledPermissions | PrincipalField::DisabledPermissions, PrincipalValue::String(name), ) => { let permission = Permission::from_name(&name).ok_or_else(|| { error( format!("Invalid {} value", change.field.as_str()), format!("Permission {name:?} is invalid").into(), ) })?; if params .allowed_permissions .as_ref() .is_none_or(|p| p.get(permission as usize)) || change.field == PrincipalField::DisabledPermissions { principal.add_permission( permission, change.field == PrincipalField::EnabledPermissions, ); // Permissions changed, update changed principals changed_principals.add_change(principal_id, principal_type, change.field); } else { return Err(error( "Invalid permission", format!("Your account cannot grant the {name:?} permission").into(), )); } } ( PrincipalAction::RemoveItem, PrincipalField::EnabledPermissions | PrincipalField::DisabledPermissions, PrincipalValue::String(name), ) => { let permission = Permission::from_name(&name).ok_or_else(|| { error( format!("Invalid {} value", change.field.as_str()), format!("Permission {name:?} is invalid").into(), ) })?; principal.remove_permission( permission, change.field == PrincipalField::EnabledPermissions, ); // Permissions changed, update changed principals changed_principals.add_change(principal_id, principal_type, change.field); } ( PrincipalAction::Set, PrincipalField::ExternalMembers, PrincipalValue::StringList(items), ) => { principal .data .retain(|v| !matches!(v, PrincipalData::ExternalMember(_))); if !items.is_empty() { principal.data.extend( items .into_iter() .map(|item| { sanitize_email(&item) .map(PrincipalData::ExternalMember) .ok_or_else(|| { error( "Invalid email address", format!( "Invalid value {:?} for {}", item, change.field.as_str() ) .into(), ) }) }) .collect::>>()?, ); } } (PrincipalAction::Set, PrincipalField::Urls, PrincipalValue::StringList(items)) => { principal .data .retain(|v| !matches!(v, PrincipalData::Url(_))); if !items.is_empty() { principal .data .extend(items.into_iter().map(PrincipalData::Url)); } } ( PrincipalAction::AddItem, PrincipalField::Urls | PrincipalField::ExternalMembers, PrincipalValue::String(mut item), ) => { if matches!(change.field, PrincipalField::ExternalMembers) { item = sanitize_email(&item).ok_or_else(|| { error( "Invalid email address", format!("Invalid value {:?} for {}", item, change.field.as_str()) .into(), ) })? } let mut found = false; for data in &principal.data { match (data, change.field) { (PrincipalData::Url(url), PrincipalField::Urls) => { if url == &item { found = true; break; } } ( PrincipalData::ExternalMember(email), PrincipalField::ExternalMembers, ) => { if email == &item { found = true; break; } } _ => {} } } if !found { match change.field { PrincipalField::Urls => principal.data.push(PrincipalData::Url(item)), PrincipalField::ExternalMembers => { principal.data.push(PrincipalData::ExternalMember(item)) } _ => {} } } } ( PrincipalAction::RemoveItem, PrincipalField::Urls, PrincipalValue::String(item), ) => { principal.data.retain(|v| match v { PrincipalData::Url(v) => v != &item, _ => true, }); } ( PrincipalAction::RemoveItem, PrincipalField::ExternalMembers, PrincipalValue::String(item), ) => { principal.data.retain(|v| match v { PrincipalData::ExternalMember(v) => v != &item, _ => true, }); } (_, field, value) => { return Err(error( "Invalid parameter", format!("Invalid value {:?} for {}", value, field.as_str()).into(), )); } } } // Validate object size if principal.object_size() > 100_000 { return Err(error( "Invalid parameter", "Principal object size exceeds 100kb safety limit.".into(), )); } if update_principal { principal.sort(); build_search_index( &mut batch, principal_id, Some(prev_principal.inner), Some(&principal), ); batch .assert_value( ValueClass::Directory(DirectoryClass::Principal(principal_id)), prev_principal, ) .set( ValueClass::Directory(DirectoryClass::Principal(principal_id)), Archiver::new(principal) .serialize() .caused_by(trc::location!())?, ); } self.write(batch.build_all()) .await .caused_by(trc::location!())?; Ok(changed_principals) } async fn list_principals( &self, filter: Option<&str>, tenant_id: Option, types: &[Type], fetch: bool, page: usize, limit: usize, ) -> trc::Result> { let filter = if let Some(filter) = filter.filter(|f| !f.trim().is_empty()) { let mut matches = RoaringBitmap::new(); for token in WordTokenizer::new(filter, MAX_TOKEN_LENGTH) { let word_bytes = token.word.as_bytes(); let from_key = ValueKey::from(ValueClass::Directory(DirectoryClass::Index { word: word_bytes.to_vec(), principal_id: 0, })); let to_key = ValueKey::from(ValueClass::Directory(DirectoryClass::Index { word: word_bytes.to_vec(), principal_id: u32::MAX, })); let mut word_matches = RoaringBitmap::new(); self.iterate( IterateParams::new(from_key, to_key).no_values(), |key, _| { let id_pos = key.len() - U32_LEN; if key.get(1..id_pos).is_some_and(|v| v == word_bytes) { word_matches.insert(key.deserialize_be_u32(id_pos)?); Ok(true) } else { Ok(false) } }, ) .await .caused_by(trc::location!())?; if matches.is_empty() { matches = word_matches; } else { matches &= word_matches; if matches.is_empty() { break; } } } if !matches.is_empty() { Some(matches) } else { return Ok(PrincipalList { total: 0, items: vec![], }); } } else { None }; let from_key = ValueKey::from(ValueClass::Directory(DirectoryClass::NameToId(vec![]))); let to_key = ValueKey::from(ValueClass::Directory(DirectoryClass::NameToId(vec![ u8::MAX; 10 ]))); let max_items = if limit > 0 { limit } else { usize::MAX }; let mut offset = page.saturating_sub(1) * limit; let mut result = PrincipalList { items: Vec::new(), total: 0, }; self.iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { let pt = PrincipalInfo::deserialize(value).caused_by(trc::location!())?; if (types.is_empty() || types.contains(&pt.typ)) && pt.has_tenant_access(tenant_id) && filter.as_ref().is_none_or(|filter| filter.contains(pt.id)) { result.total += 1; if offset == 0 { if result.items.len() < max_items { let mut principal = Principal::new(pt.id, pt.typ); principal.name = String::from_utf8_lossy(key.get(1..).unwrap_or_default()) .into_owned(); result.items.push(principal); } } else { offset -= 1; } } Ok(true) }, ) .await .caused_by(trc::location!())?; if fetch && !result.items.is_empty() { let mut items = Vec::with_capacity(result.items.len()); for principal in result.items { items.push( self.query(QueryParams::id(principal.id).with_return_member_of(fetch)) .await .caused_by(trc::location!())? .ok_or_else(|| not_found(principal.name().to_string()))?, ); } result.items = items; Ok(result) } else { Ok(result) } } async fn count_principals( &self, filter: Option<&str>, typ: Option, tenant_id: Option, ) -> trc::Result { let from_key = ValueKey::from(ValueClass::Directory(DirectoryClass::NameToId(vec![]))); let to_key = ValueKey::from(ValueClass::Directory(DirectoryClass::NameToId(vec![ u8::MAX; 10 ]))); let mut count = 0; self.iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { let pt = PrincipalInfo::deserialize(value).caused_by(trc::location!())?; let name = std::str::from_utf8(key.get(1..).unwrap_or_default()).unwrap_or_default(); if typ.is_none_or(|t| pt.typ == t) && pt.has_tenant_access(tenant_id) && filter.is_none_or(|f| name.contains(f)) { count += 1; } Ok(true) }, ) .await .caused_by(trc::location!()) .map(|_| count) } async fn principal_ids( &self, typ: Option, tenant_id: Option, ) -> trc::Result { let mut results = RoaringBitmap::new(); self.iterate( IterateParams::new( ValueKey::from(ValueClass::Directory(DirectoryClass::NameToId(vec![0u8]))), ValueKey::from(ValueClass::Directory(DirectoryClass::NameToId(vec![ u8::MAX; 10 ]))), ), |_, value| { let pt = PrincipalInfo::deserialize(value).caused_by(trc::location!())?; if typ.is_none_or(|t| pt.typ == t) && pt.has_tenant_access(tenant_id) { results.insert(pt.id); } Ok(true) }, ) .await .caused_by(trc::location!()) .map(|_| results) } async fn get_member_of(&self, principal_id: u32) -> trc::Result> { let from_key = ValueKey::from(ValueClass::Directory(DirectoryClass::MemberOf { principal_id, member_of: 0, })); let to_key = ValueKey::from(ValueClass::Directory(DirectoryClass::MemberOf { principal_id, member_of: u32::MAX, })); let mut results = Vec::new(); self.iterate(IterateParams::new(from_key, to_key), |key, value| { results.push(MemberOf { principal_id: key.deserialize_be_u32(key.len() - U32_LEN)?, typ: value .first() .map(|v| Type::from_u8(*v)) .unwrap_or(Type::Group), }); Ok(true) }) .await .caused_by(trc::location!())?; Ok(results) } async fn get_members(&self, principal_id: u32) -> trc::Result> { let from_key = ValueKey::from(ValueClass::Directory(DirectoryClass::Members { principal_id, has_member: 0, })); let to_key = ValueKey::from(ValueClass::Directory(DirectoryClass::Members { principal_id, has_member: u32::MAX, })); let mut results = Vec::new(); self.iterate( IterateParams::new(from_key, to_key).no_values(), |key, _| { results.push(key.deserialize_be_u32(key.len() - U32_LEN)?); Ok(true) }, ) .await .caused_by(trc::location!())?; Ok(results) } async fn map_principal( &self, principal: Principal, fields: &[PrincipalField], ) -> trc::Result { let mut result = PrincipalSet::new(principal.id, principal.typ); let has_enabled = fields.is_empty() || fields.contains(&PrincipalField::EnabledPermissions); let has_disabled = fields.is_empty() || fields.contains(&PrincipalField::DisabledPermissions); let mut directory_quotas = Vec::new(); let mut quota = None; let mut tenant_id = None; for data in principal.data { match data { PrincipalData::MemberOf(principal_id) if fields.is_empty() || fields.contains(&PrincipalField::MemberOf) => { if let Some(name) = self .get_principal_name(principal_id) .await .caused_by(trc::location!())? { result.append_str(PrincipalField::MemberOf, name); } } PrincipalData::Role(principal_id) if fields.is_empty() || fields.contains(&PrincipalField::Roles) => { match principal_id { ROLE_ADMIN => { result.append_str(PrincipalField::Roles, "admin"); } ROLE_TENANT_ADMIN => { result.append_str(PrincipalField::Roles, "tenant-admin"); } ROLE_USER => { result.append_str(PrincipalField::Roles, "user"); } principal_id => { if let Some(name) = self .get_principal_name(principal_id) .await .caused_by(trc::location!())? { result.append_str(PrincipalField::Roles, name); } } } } PrincipalData::List(principal_id) if fields.is_empty() || fields.contains(&PrincipalField::Lists) => { if let Some(name) = self .get_principal_name(principal_id) .await .caused_by(trc::location!())? { result.append_str(PrincipalField::Lists, name); } } PrincipalData::Permission { permission_id, grant, } if has_enabled || has_disabled => { if grant { if has_enabled { result.append_str( PrincipalField::EnabledPermissions, Permission::from_id(permission_id) .map(|f| f.name()) .unwrap_or("unknown"), ); } } else if has_disabled { result.append_str( PrincipalField::DisabledPermissions, Permission::from_id(permission_id) .map(|f| f.name()) .unwrap_or("unknown"), ); } } PrincipalData::DiskQuota(q) => { quota = Some(q); } PrincipalData::Tenant(tid) => { tenant_id = Some(tid); } PrincipalData::Description(description) => { if fields.is_empty() || fields.contains(&PrincipalField::Description) { result.set(PrincipalField::Description, description); } } PrincipalData::Password(secret) | PrincipalData::AppPassword(secret) | PrincipalData::OtpAuth(secret) => { if fields.is_empty() || fields.contains(&PrincipalField::Secrets) { result.append_str(PrincipalField::Secrets, secret); } } PrincipalData::PrimaryEmail(email) | PrincipalData::EmailAlias(email) => { if fields.is_empty() || fields.contains(&PrincipalField::Emails) { result.append_str(PrincipalField::Emails, email); } } PrincipalData::Picture(picture) => { if fields.is_empty() || fields.contains(&PrincipalField::Picture) { result.set(PrincipalField::Picture, picture); } } PrincipalData::Locale(locale) => { if fields.is_empty() || fields.contains(&PrincipalField::Locale) { result.set(PrincipalField::Locale, locale); } } PrincipalData::ExternalMember(member) => { if fields.is_empty() || fields.contains(&PrincipalField::ExternalMembers) { result.append_str(PrincipalField::ExternalMembers, member); } } PrincipalData::Url(url) => { if fields.is_empty() || fields.contains(&PrincipalField::Urls) { result.append_str(PrincipalField::Urls, url); } } PrincipalData::DirectoryQuota { quota, typ } => { directory_quotas.push((typ, quota)); } _ => (), } } // Obtain member names if fields.is_empty() || fields.contains(&PrincipalField::Members) { match principal.typ { Type::Group | Type::List | Type::Role => { for member_id in self.get_members(principal.id).await? { if let Some(member_principal) = self .query(QueryParams::id(member_id).with_return_member_of(false)) .await? { result.append_str(PrincipalField::Members, member_principal.name); } } } Type::Domain => { let from_key = ValueKey::from(ValueClass::Directory(DirectoryClass::EmailToId(vec![]))); let to_key = ValueKey::from(ValueClass::Directory(DirectoryClass::EmailToId( vec![u8::MAX; 10], ))); let domain_name = &principal.name; let mut total: u64 = 0; self.iterate( IterateParams::new(from_key, to_key).no_values(), |key, _| { if std::str::from_utf8(key.get(1..).unwrap_or_default()) .unwrap_or_default() .rsplit_once('@') .is_some_and(|(_, domain)| domain == domain_name) { total += 1; } Ok(true) }, ) .await .caused_by(trc::location!())?; result.set(PrincipalField::Members, total); } Type::Tenant => { let from_key = ValueKey::from(ValueClass::Directory(DirectoryClass::NameToId(vec![]))); let to_key = ValueKey::from(ValueClass::Directory(DirectoryClass::NameToId( vec![u8::MAX; 10], ))); let mut total: u64 = 0; self.iterate(IterateParams::new(from_key, to_key), |_, value| { let pinfo = PrincipalInfo::deserialize(value).caused_by(trc::location!())?; if pinfo.typ == Type::Individual && pinfo.has_tenant_access(Some(principal.id)) { total += 1; } Ok(true) }) .await .caused_by(trc::location!())?; result.set(PrincipalField::Members, total); } _ => {} } } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL // Map tenant name #[cfg(feature = "enterprise")] if let Some(tenant_id) = tenant_id && (fields.is_empty() || fields.contains(&PrincipalField::Tenant)) && let Some(name) = self .get_principal_name(tenant_id) .await .caused_by(trc::location!())? { result.set(PrincipalField::Tenant, name); } // SPDX-SnippetEnd // Map fields if fields.is_empty() || fields.contains(&PrincipalField::Name) { result.set(PrincipalField::Name, principal.name); } if fields.is_empty() || fields.contains(&PrincipalField::Quota) { if !directory_quotas.is_empty() { let mut quotas = vec![0u64; Type::MAX_ID + 2]; if let Some(quota) = quota { quotas[0] = quota; } for (typ, quota) in directory_quotas { quotas[(typ as usize) + 1] = quota as u64; } result.set(PrincipalField::Quota, quotas); } else if let Some(quota) = quota { result.set(PrincipalField::Quota, quota); } } // Obtain used quota if matches!(principal.typ, Type::Individual | Type::Group | Type::Tenant) && (fields.is_empty() || fields.contains(&PrincipalField::UsedQuota)) { let quota = self .get_counter(DirectoryClass::UsedQuota(principal.id)) .await .caused_by(trc::location!())?; if quota > 0 { result.set(PrincipalField::UsedQuota, quota as u64); } } Ok(result) } } impl ValidateDirectory for Store { async fn validate_email( &self, email: &str, tenant_id: Option, create_if_missing: bool, ) -> trc::Result<()> { if self.rcpt(email).await.caused_by(trc::location!())? != RcptType::Invalid { Err(err_exists(PrincipalField::Emails, email.to_string())) } else if let Some(domain) = email.try_domain_part() { match self .get_principal_info(domain) .await .caused_by(trc::location!())? { Some(v) if v.typ == Type::Domain && v.has_tenant_access(tenant_id) => Ok(()), None if create_if_missing => self .create_principal( PrincipalSet::new(0, Type::Domain) .with_field(PrincipalField::Name, domain) .with_field(PrincipalField::Description, domain), tenant_id, None, ) .await .caused_by(trc::location!()) .map(|_| ()), _ => Err(not_found(domain.to_string())), } } else { Err(error("Invalid email", "Email address is invalid".into())) } } } impl PrincipalField { pub fn map_internal_role_name(&self, name: &str) -> Option { match (self, name) { (PrincipalField::Roles, "admin") => Some(ROLE_ADMIN), (PrincipalField::Roles, "tenant-admin") => Some(ROLE_TENANT_ADMIN), (PrincipalField::Roles, "user") => Some(ROLE_USER), _ => None, } } pub fn map_internal_roles(&self, name: &str) -> Option { self.map_internal_role_name(name) .map(|role_id| PrincipalInfo::new(role_id, Type::Role, None)) } } impl<'x> UpdatePrincipal<'x> { pub fn by_id(id: u32) -> Self { Self { query: QueryBy::Id(id), changes: Vec::new(), create_domains: false, tenant_id: None, allowed_permissions: None, } } pub fn by_name(name: &'x str) -> Self { Self { query: QueryBy::Name(name), changes: Vec::new(), create_domains: false, tenant_id: None, allowed_permissions: None, } } pub fn with_tenant(mut self, tenant_id: Option) -> Self { self.tenant_id = tenant_id; self } pub fn with_updates(mut self, changes: Vec) -> Self { self.changes = changes; self } pub fn with_allowed_permissions(mut self, permissions: &'x Permissions) -> Self { self.allowed_permissions = permissions.into(); self } pub fn create_domains(mut self) -> Self { self.create_domains = true; self } } fn validate_member_of( field: PrincipalField, typ: Type, member_type: Type, member_name: &str, ) -> trc::Result<()> { let expected_types = match (field, typ) { (PrincipalField::MemberOf, Type::Individual) => &[Type::Group, Type::Individual][..], (PrincipalField::MemberOf, Type::Group) => &[Type::Group][..], (PrincipalField::Lists, Type::Individual | Type::Group) => &[Type::List][..], (PrincipalField::Roles, Type::Individual | Type::Tenant | Type::Role) => &[Type::Role][..], _ => &[][..], }; if expected_types.is_empty() || !expected_types.contains(&member_type) { Err(error( format!("Invalid {} value", field.as_str()), if !expected_types.is_empty() { format!( "Principal {member_name:?} is not a {}.", expected_types .iter() .map(|t| t.description().to_string()) .collect::>() .join(", ") ) .into() } else { format!("Principal {member_name:?} cannot be added as a member.").into() }, )) } else { Ok(()) } } impl ChangedPrincipals { pub fn new() -> Self { Self::default() } pub fn from_change(principal_id: u32, principal_type: Type, field: PrincipalField) -> Self { let mut set = Self::default(); set.add_change(principal_id, principal_type, field); set } pub fn add_change(&mut self, principal_id: u32, principal_type: Type, field: PrincipalField) { if matches!( (principal_type, field), ( Type::Individual | Type::Group, PrincipalField::Name | PrincipalField::Quota | PrincipalField::Secrets | PrincipalField::Emails | PrincipalField::MemberOf | PrincipalField::Members | PrincipalField::Tenant | PrincipalField::Roles | PrincipalField::EnabledPermissions | PrincipalField::DisabledPermissions, ) | ( Type::Tenant | Type::Role | Type::ApiKey | Type::OauthClient, PrincipalField::MemberOf | PrincipalField::Members | PrincipalField::Secrets | PrincipalField::Tenant | PrincipalField::Roles | PrincipalField::EnabledPermissions | PrincipalField::DisabledPermissions, ) ) && principal_id < ROLE_USER { self.0 .entry(principal_id) .or_insert_with(|| ChangedPrincipal::new(principal_type)) .update_member_change(matches!( (field, principal_type), ( PrincipalField::EnabledPermissions | PrincipalField::DisabledPermissions, Type::Role | Type::Tenant ) )) .update_name_change(matches!(field, PrincipalField::Name)); } } pub fn add_member_change( &mut self, principal_id: u32, principal_type: Type, member_id: u32, member_type: Type, ) { match (principal_type, member_type) { (Type::Group | Type::Role, Type::Individual | Type::ApiKey | Type::OauthClient) => { self.0 .entry(member_id) .or_insert_with(|| ChangedPrincipal::new(member_type)); } (Type::Individual | Type::ApiKey | Type::OauthClient, Type::Group | Type::Role) => { self.0 .entry(principal_id) .or_insert_with(|| ChangedPrincipal::new(principal_type)); } ( Type::Group | Type::Tenant | Type::Role, Type::Individual | Type::Group | Type::Tenant | Type::Role, ) => { if principal_id < ROLE_USER { self.0 .entry(principal_id) .or_insert_with(|| ChangedPrincipal::new(principal_type)) .update_member_change(matches!(member_type, Type::Role)); } if member_id < ROLE_USER { self.0 .entry(member_id) .or_insert_with(|| ChangedPrincipal::new(member_type)) .update_member_change(matches!(principal_type, Type::Role)); } } _ => {} } } pub fn add_deletion(&mut self, principal_id: u32, principal_type: Type) { if matches!( principal_type, Type::Individual | Type::Group | Type::Tenant | Type::Role | Type::ApiKey | Type::OauthClient ) { self.0 .entry(principal_id) .or_insert_with(|| ChangedPrincipal::new(principal_type)); } } pub fn contains(&self, principal_id: u32) -> bool { self.0.contains_key(&principal_id) } pub fn iter(&'_ self) -> std::collections::hash_map::Iter<'_, u32, ChangedPrincipal> { self.0.iter() } pub fn is_empty(&self) -> bool { self.0.is_empty() } } impl ChangedPrincipal { pub fn new(typ: Type) -> Self { Self { typ, member_change: false, name_change: false, } } pub fn update_member_change(&mut self, member_change: bool) -> &mut Self { self.member_change |= member_change; self } pub fn update_name_change(&mut self, name_change: bool) -> &mut Self { self.name_change |= name_change; self } } pub fn err_missing(field: impl Into) -> trc::Error { trc::ManageEvent::MissingParameter.ctx(trc::Key::Key, field) } pub fn err_exists(field: impl Into, value: impl Into) -> trc::Error { trc::ManageEvent::AlreadyExists .ctx(trc::Key::Key, field) .ctx(trc::Key::Value, value) } pub fn not_found(value: impl Into) -> trc::Error { trc::ManageEvent::NotFound.ctx(trc::Key::Key, value) } pub fn unsupported(details: impl Into) -> trc::Error { trc::ManageEvent::NotSupported.ctx(trc::Key::Details, details) } pub fn enterprise() -> trc::Error { trc::ManageEvent::NotSupported.ctx(trc::Key::Details, "Enterprise feature") } pub fn error(details: impl Into, reason: Option>) -> trc::Error { trc::ManageEvent::Error .ctx(trc::Key::Details, details) .ctx_opt(trc::Key::Reason, reason) } impl From for trc::Value { fn from(value: PrincipalField) -> Self { trc::Value::String(CompactString::const_new(value.as_str())) } } ================================================ FILE: crates/directory/src/backend/internal/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod lookup; pub mod manage; use crate::Type; use ahash::AHashMap; use std::fmt::Display; use store::{Deserialize, SerializeInfallible, U32_LEN, write::key::KeySerializer}; use utils::codec::leb128::Leb128Iterator; pub struct PrincipalInfo { pub id: u32, pub typ: Type, pub tenant: Option, } impl PrincipalInfo { // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] pub fn has_tenant_access(&self, tenant_id: Option) -> bool { tenant_id.is_none_or(|tenant_id| { self.tenant.is_some_and(|t| tenant_id == t) || (self.typ == Type::Tenant && self.id == tenant_id) }) } // SPDX-SnippetEnd #[cfg(not(feature = "enterprise"))] pub fn has_tenant_access(&self, _tenant_id: Option) -> bool { true } } impl SerializeInfallible for PrincipalInfo { fn serialize(&self) -> Vec { if let Some(tenant) = self.tenant { KeySerializer::new((U32_LEN * 2) + 1) .write_leb128(self.id) .write(self.typ as u8) .write_leb128(tenant) .finalize() } else { KeySerializer::new(U32_LEN + 1) .write_leb128(self.id) .write(self.typ as u8) .finalize() } } } impl Deserialize for PrincipalInfo { fn deserialize(bytes_: &[u8]) -> trc::Result { let mut bytes = bytes_.iter(); Ok(PrincipalInfo { id: bytes.next_leb128().ok_or_else(|| { trc::StoreEvent::DataCorruption .caused_by(trc::location!()) .ctx(trc::Key::Value, bytes_) })?, typ: Type::from_u8(*bytes.next().ok_or_else(|| { trc::StoreEvent::DataCorruption .caused_by(trc::location!()) .ctx(trc::Key::Value, bytes_) })?), tenant: bytes.next_leb128(), }) } } impl PrincipalInfo { pub fn new(principal_id: u32, typ: Type, tenant: Option) -> Self { Self { id: principal_id, typ, tenant, } } } #[derive( Debug, Clone, Copy, PartialEq, Hash, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize, )] #[serde(rename_all = "camelCase")] pub enum PrincipalField { Name, Type, Quota, UsedQuota, Description, Secrets, Emails, MemberOf, Members, Tenant, Roles, Lists, EnabledPermissions, DisabledPermissions, Picture, Urls, ExternalMembers, Locale, } #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct PrincipalSet { pub id: u32, pub typ: Type, pub fields: AHashMap, } #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct PrincipalUpdate { pub action: PrincipalAction, pub field: PrincipalField, pub value: PrincipalValue, } #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub enum PrincipalAction { #[serde(rename = "set")] Set, #[serde(rename = "addItem")] AddItem, #[serde(rename = "removeItem")] RemoveItem, } #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] #[serde(untagged)] pub enum PrincipalValue { String(String), StringList(Vec), Integer(u64), IntegerList(Vec), } impl PrincipalUpdate { pub fn set(field: PrincipalField, value: PrincipalValue) -> PrincipalUpdate { PrincipalUpdate { action: PrincipalAction::Set, field, value, } } pub fn add_item(field: PrincipalField, value: PrincipalValue) -> PrincipalUpdate { PrincipalUpdate { action: PrincipalAction::AddItem, field, value, } } pub fn remove_item(field: PrincipalField, value: PrincipalValue) -> PrincipalUpdate { PrincipalUpdate { action: PrincipalAction::RemoveItem, field, value, } } } impl Display for PrincipalField { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { self.as_str().fmt(f) } } impl PrincipalField { pub fn id(&self) -> u8 { match self { PrincipalField::Name => 0, PrincipalField::Type => 1, PrincipalField::Quota => 2, PrincipalField::Description => 3, PrincipalField::Secrets => 4, PrincipalField::Emails => 5, PrincipalField::MemberOf => 6, PrincipalField::Members => 7, PrincipalField::Tenant => 8, PrincipalField::Roles => 9, PrincipalField::Lists => 10, PrincipalField::EnabledPermissions => 11, PrincipalField::DisabledPermissions => 12, PrincipalField::UsedQuota => 13, PrincipalField::Picture => 14, PrincipalField::Urls => 15, PrincipalField::ExternalMembers => 16, PrincipalField::Locale => 17, } } pub fn from_id(id: u8) -> Option { match id { 0 => Some(PrincipalField::Name), 1 => Some(PrincipalField::Type), 2 => Some(PrincipalField::Quota), 3 => Some(PrincipalField::Description), 4 => Some(PrincipalField::Secrets), 5 => Some(PrincipalField::Emails), 6 => Some(PrincipalField::MemberOf), 7 => Some(PrincipalField::Members), 8 => Some(PrincipalField::Tenant), 9 => Some(PrincipalField::Roles), 10 => Some(PrincipalField::Lists), 11 => Some(PrincipalField::EnabledPermissions), 12 => Some(PrincipalField::DisabledPermissions), 13 => Some(PrincipalField::UsedQuota), 14 => Some(PrincipalField::Picture), 15 => Some(PrincipalField::Urls), 16 => Some(PrincipalField::ExternalMembers), 17 => Some(PrincipalField::Locale), _ => None, } } pub fn as_str(&self) -> &'static str { match self { PrincipalField::Name => "name", PrincipalField::Type => "type", PrincipalField::Quota => "quota", PrincipalField::UsedQuota => "usedQuota", PrincipalField::Description => "description", PrincipalField::Secrets => "secrets", PrincipalField::Emails => "emails", PrincipalField::MemberOf => "memberOf", PrincipalField::Members => "members", PrincipalField::Tenant => "tenant", PrincipalField::Roles => "roles", PrincipalField::Lists => "lists", PrincipalField::EnabledPermissions => "enabledPermissions", PrincipalField::DisabledPermissions => "disabledPermissions", PrincipalField::Picture => "picture", PrincipalField::Urls => "urls", PrincipalField::ExternalMembers => "externalMembers", PrincipalField::Locale => "locale", } } pub fn try_parse(s: &str) -> Option { match s { "name" => Some(PrincipalField::Name), "type" => Some(PrincipalField::Type), "quota" => Some(PrincipalField::Quota), "usedQuota" => Some(PrincipalField::UsedQuota), "description" => Some(PrincipalField::Description), "secrets" => Some(PrincipalField::Secrets), "emails" => Some(PrincipalField::Emails), "memberOf" => Some(PrincipalField::MemberOf), "members" => Some(PrincipalField::Members), "tenant" => Some(PrincipalField::Tenant), "roles" => Some(PrincipalField::Roles), "lists" => Some(PrincipalField::Lists), "enabledPermissions" => Some(PrincipalField::EnabledPermissions), "disabledPermissions" => Some(PrincipalField::DisabledPermissions), "picture" => Some(PrincipalField::Picture), "urls" => Some(PrincipalField::Urls), "externalMembers" => Some(PrincipalField::ExternalMembers), "locale" => Some(PrincipalField::Locale), _ => None, } } } pub trait SpecialSecrets { fn is_otp_secret(&self) -> bool; fn is_app_secret(&self) -> bool; } impl SpecialSecrets for T where T: AsRef, { fn is_otp_secret(&self) -> bool { self.as_ref().starts_with("otpauth://") } fn is_app_secret(&self) -> bool { self.as_ref().starts_with("$app$") } } ================================================ FILE: crates/directory/src/backend/ldap/config.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use ldap3::LdapConnSettings; use store::Store; use utils::config::{Config, utils::AsKey}; use crate::core::config::build_pool; use super::{ AuthBind, Bind, LdapConnectionManager, LdapDirectory, LdapFilter, LdapFilterItem, LdapMappings, }; impl LdapDirectory { pub fn from_config(config: &mut Config, prefix: impl AsKey, data_store: Store) -> Option { let prefix = prefix.as_key(); let bind_dn = if let Some(dn) = config.value((&prefix, "bind.dn")) { Bind::new( dn.to_string(), config.value_require((&prefix, "bind.secret"))?.to_string(), ) .into() } else { None }; let manager = LdapConnectionManager::new( config.value_require((&prefix, "url"))?.to_string(), LdapConnSettings::new() .set_conn_timeout( config .property_or_default((&prefix, "timeout"), "30s") .unwrap_or_else(|| Duration::from_secs(30)), ) .set_starttls( config .property_or_default((&prefix, "tls.enable"), "false") .unwrap_or_default(), ) .set_no_tls_verify( config .property_or_default((&prefix, "tls.allow-invalid-certs"), "false") .unwrap_or_default(), ), bind_dn, ); let mut mappings = LdapMappings { base_dn: config.value_require((&prefix, "base-dn"))?.to_string(), filter_name: LdapFilter::from_config(config, (&prefix, "filter.name")), filter_email: LdapFilter::from_config(config, (&prefix, "filter.email")), attr_name: config .values((&prefix, "attributes.name")) .map(|(_, v)| v.to_lowercase()) .collect(), attr_groups: config .values((&prefix, "attributes.groups")) .map(|(_, v)| v.to_lowercase()) .collect(), attr_type: config .values((&prefix, "attributes.class")) .map(|(_, v)| v.to_lowercase()) .collect(), attr_description: config .values((&prefix, "attributes.description")) .map(|(_, v)| v.to_lowercase()) .collect(), attr_secret: config .values((&prefix, "attributes.secret")) .map(|(_, v)| v.to_lowercase()) .collect(), attr_secret_changed: config .values((&prefix, "attributes.secret-changed")) .map(|(_, v)| v.to_lowercase()) .collect(), attr_email_address: config .values((&prefix, "attributes.email")) .map(|(_, v)| v.to_lowercase()) .collect(), attr_quota: config .values((&prefix, "attributes.quota")) .map(|(_, v)| v.to_lowercase()) .collect(), attr_email_alias: config .values((&prefix, "attributes.email-alias")) .map(|(_, v)| v.to_lowercase()) .collect(), attrs_principal: vec!["objectClass".to_lowercase()], }; for attr in [ &mappings.attr_name, &mappings.attr_type, &mappings.attr_description, &mappings.attr_secret, &mappings.attr_secret_changed, &mappings.attr_quota, &mappings.attr_groups, &mappings.attr_email_address, &mappings.attr_email_alias, ] { mappings .attrs_principal .extend(attr.iter().filter(|a| !a.is_empty()).cloned()); } let auth_bind = match config .value((&prefix, "bind.auth.method")) .unwrap_or("default") { "template" => AuthBind::Template { template: LdapFilter::from_config(config, (&prefix, "bind.auth.template")), can_search: config .property_or_default::((&prefix, "bind.auth.search"), "true") .unwrap_or(true), }, "lookup" => AuthBind::Lookup, "default" => AuthBind::None, unknown => { config.new_parse_error( (&prefix, "bind.auth.method"), format!("Unknown LDAP bind method: {unknown}"), ); return None; } }; Some(LdapDirectory { mappings, pool: build_pool(config, &prefix, manager) .map_err(|e| { config.new_parse_error(prefix, format!("Failed to build LDAP pool: {e:?}")) }) .ok()?, auth_bind, data_store, }) } } impl LdapFilter { fn from_config(config: &mut Config, key: impl AsKey) -> Self { if let Some(value) = config.value(key.clone()) { let mut filter = Vec::new(); let mut token = String::new(); let mut value = value.chars(); while let Some(ch) = value.next() { match ch { '?' => { // For backwards compatibility, we treat '?' as a placeholder for the full value. if !token.is_empty() { filter.push(LdapFilterItem::Static(token)); token = String::new(); } filter.push(LdapFilterItem::Full); } '{' => { if !token.is_empty() { filter.push(LdapFilterItem::Static(token)); token = String::new(); } for ch in value.by_ref() { if ch == '}' { break; } else { token.push(ch); } } match token.as_str() { "user" | "username" | "email" => filter.push(LdapFilterItem::Full), "local" => filter.push(LdapFilterItem::LocalPart), "domain" => filter.push(LdapFilterItem::DomainPart), _ => { config.new_parse_error( key, format!("Unknown LDAP filter placeholder: {}", token), ); return Self::default(); } } token.clear(); } _ => token.push(ch), } } if !token.is_empty() { filter.push(LdapFilterItem::Static(token)); } if filter.len() >= 2 { return LdapFilter { filter }; } else { config.new_parse_error( key, format!("Missing parameter placeholders in value {:?}", value), ); } } Self::default() } } ================================================ FILE: crates/directory/src/backend/ldap/lookup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{AuthBind, LdapDirectory, LdapMappings}; use crate::{ IntoError, Principal, PrincipalData, QueryBy, QueryParams, ROLE_ADMIN, ROLE_USER, Type, backend::{ RcptType, internal::{ SpecialSecrets, lookup::DirectoryStore, manage::{self, ManageDirectory, UpdatePrincipal}, }, }, }; use ldap3::{Ldap, LdapConnAsync, ResultEntry, Scope, SearchEntry}; use mail_send::Credentials; use store::xxhash_rust; use trc::AddContext; impl LdapDirectory { pub async fn query(&self, by: QueryParams<'_>) -> trc::Result> { let mut conn = self.pool.get().await.map_err(|err| err.into_error())?; let (mut external_principal, member_of, stored_principal) = match by.by { QueryBy::Name(username) => { let filter = self.mappings.filter_name.build(username); if let Some(mut result) = self.find_principal(&mut conn, &filter).await? { if result.principal.name.is_empty() { result.principal.name = username.into(); } (result.principal, result.member_of, None) } else { trc::event!( Store(trc::StoreEvent::LdapWarning), Reason = "Name filter yielded no results", Details = filter ); return Ok(None); } } QueryBy::Id(uid) => { if let Some(stored_principal_) = self .data_store .query(QueryParams::id(uid).with_return_member_of(by.return_member_of)) .await? { if let Some(result) = self .find_principal( &mut conn, &self.mappings.filter_name.build(stored_principal_.name()), ) .await? { (result.principal, result.member_of, Some(stored_principal_)) } else { return Ok(None); } } else { return Ok(None); } } QueryBy::Credentials(credentials) => { let (username, secret) = match credentials { Credentials::Plain { username, secret } => (username, secret), Credentials::OAuthBearer { token } => (token, token), Credentials::XOauth2 { username, secret } => (username, secret), }; match &self.auth_bind { AuthBind::Template { template, can_search, } => { let (auth_bind_conn, mut ldap) = LdapConnAsync::with_settings( self.pool.manager().settings.clone(), &self.pool.manager().address, ) .await .map_err(|err| err.into_error().caused_by(trc::location!()))?; ldap3::drive!(auth_bind_conn); let dn = template.build(username); if ldap .simple_bind(&dn, secret) .await .map_err(|err| err.into_error().caused_by(trc::location!()))? .success() .is_err() { trc::event!( Store(trc::StoreEvent::LdapWarning), Reason = "Secret rejected during auth bind using template", Details = dn ); return Ok(None); } let filter = self.mappings.filter_name.build(username); let result = if *can_search { self.find_principal(&mut ldap, &filter).await } else { self.find_principal(&mut conn, &filter).await }; match result { Ok(Some(mut result)) => { if result.principal.name.is_empty() { result.principal.name = username.into(); } (result.principal, result.member_of, None) } Err(err) if err .matches(trc::EventType::Store(trc::StoreEvent::LdapError)) && err .value(trc::Key::Code) .and_then(|v| v.to_uint()) .is_some_and(|rc| [49, 50].contains(&rc)) => { trc::event!( Store(trc::StoreEvent::LdapWarning), Reason = "Error codes 49 or 50 returned by LDAP server", Details = vec![dn, filter] ); return Ok(None); } Ok(None) => { trc::event!( Store(trc::StoreEvent::LdapWarning), Reason = "Auth bind successful but filter yielded no results", Details = vec![dn, filter] ); return Ok(None); } Err(err) => return Err(err), } } AuthBind::Lookup => { let filter = self.mappings.filter_name.build(username); if let Some(mut result) = self.find_principal(&mut conn, &filter).await? { // Perform bind auth using the found dn let (auth_bind_conn, mut ldap) = LdapConnAsync::with_settings( self.pool.manager().settings.clone(), &self.pool.manager().address, ) .await .map_err(|err| err.into_error().caused_by(trc::location!()))?; ldap3::drive!(auth_bind_conn); if ldap .simple_bind(&result.dn, secret) .await .map_err(|err| err.into_error().caused_by(trc::location!()))? .success() .is_ok() { if result.principal.name.is_empty() { result.principal.name = username.into(); } (result.principal, result.member_of, None) } else { trc::event!( Store(trc::StoreEvent::LdapWarning), Reason = "Secret rejected during auth bind using lookup filter", Details = vec![result.dn, filter] ); return Ok(None); } } else { trc::event!( Store(trc::StoreEvent::LdapWarning), Reason = "Auth bind lookup filter yielded no results", Details = filter ); return Ok(None); } } AuthBind::None => { let filter = self.mappings.filter_name.build(username); if let Some(mut result) = self.find_principal(&mut conn, &filter).await? { if result.principal.verify_secret(secret, false, false).await? { if result.principal.name.is_empty() { result.principal.name = username.into(); } (result.principal, result.member_of, None) } else { trc::event!( Store(trc::StoreEvent::LdapWarning), Reason = "Password verification failed", Details = vec![result.dn, filter] ); return Ok(None); } } else { trc::event!( Store(trc::StoreEvent::LdapWarning), Reason = "Authentication filter yielded no results", Details = filter ); return Ok(None); } } } } }; // Query groups if !member_of.is_empty() && by.return_member_of { for mut name in member_of { if name.contains('=') { let (rs, _res) = conn .search( &name, Scope::Base, "objectClass=*", &self.mappings.attr_name, ) .await .map_err(|err| err.into_error().caused_by(trc::location!()))? .success() .map_err(|err| err.into_error().caused_by(trc::location!()))?; for entry in rs { 'outer: for (attr, value) in SearchEntry::construct(entry).attrs { if self.mappings.attr_name.contains(&attr.to_lowercase()) && let Some(group) = value.into_iter().next() && !group.is_empty() { name = group; break 'outer; } } } } let account_id = self .data_store .get_or_create_principal_id(&name, Type::Group) .await .caused_by(trc::location!())?; external_principal .data .push(PrincipalData::MemberOf(account_id)); } } // Obtain account ID if not available let mut principal = if let Some(stored_principal) = stored_principal { stored_principal } else { let id = self .data_store .get_or_create_principal_id(external_principal.name(), Type::Individual) .await .caused_by(trc::location!())?; self.data_store .query(QueryParams::id(id).with_return_member_of(by.return_member_of)) .await .caused_by(trc::location!())? .ok_or_else(|| manage::not_found(id).caused_by(trc::location!()))? }; // Keep the internal store up to date with the LDAP server let changes = principal.update_external(external_principal); if !changes.is_empty() { self.data_store .update_principal( UpdatePrincipal::by_id(principal.id) .with_updates(changes) .create_domains(), ) .await .caused_by(trc::location!())?; } Ok(Some(principal)) } pub async fn email_to_id(&self, address: &str) -> trc::Result> { let filter = self.mappings.filter_email.build(address.as_ref()); let rs = self .pool .get() .await .map_err(|err| err.into_error().caused_by(trc::location!()))? .search( &self.mappings.base_dn, Scope::Subtree, &filter, &self.mappings.attr_name, ) .await .map_err(|err| err.into_error().caused_by(trc::location!()))? .success() .map(|(rs, _res)| rs) .map_err(|err| err.into_error().caused_by(trc::location!()))?; trc::event!( Store(trc::StoreEvent::LdapQuery), Details = filter, Result = rs.iter().map(result_to_trace).collect::>() ); for entry in rs { for (attr, value) in SearchEntry::construct(entry).attrs { if self.mappings.attr_name.contains(&attr.to_lowercase()) && let Some(name) = value.into_iter().find(|name| !name.is_empty()) { return self .data_store .get_or_create_principal_id(&name, Type::Individual) .await .map(Some); } } } Ok(None) } pub async fn rcpt(&self, address: &str) -> trc::Result { let filter = self.mappings.filter_email.build(address.as_ref()); let result = self .pool .get() .await .map_err(|err| err.into_error().caused_by(trc::location!()))? .streaming_search( &self.mappings.base_dn, Scope::Subtree, &filter, &self.mappings.attr_email_address, ) .await .map_err(|err| err.into_error().caused_by(trc::location!()))? .next() .await .map(|entry| { let result = if entry.is_some() { RcptType::Mailbox } else { RcptType::Invalid }; trc::event!( Store(trc::StoreEvent::LdapQuery), Details = filter, Result = entry.as_ref().map(result_to_trace).unwrap_or_default() ); result }) .map_err(|err| err.into_error().caused_by(trc::location!()))?; if result != RcptType::Invalid { Ok(result) } else { self.data_store.rcpt(address).await.map(|result| { if matches!(result, RcptType::List(_)) { result } else { RcptType::Invalid } }) } } pub async fn vrfy(&self, address: &str) -> trc::Result> { self.data_store.vrfy(address).await } pub async fn expn(&self, address: &str) -> trc::Result> { self.data_store.expn(address).await } pub async fn is_local_domain(&self, domain: &str) -> trc::Result { self.data_store.is_local_domain(domain).await } } impl LdapDirectory { async fn find_principal( &self, conn: &mut Ldap, filter: &str, ) -> trc::Result> { conn.search( &self.mappings.base_dn, Scope::Subtree, filter, &self.mappings.attrs_principal, ) .await .map_err(|err| err.into_error().caused_by(trc::location!()))? .success() .map(|(rs, _)| { trc::event!( Store(trc::StoreEvent::LdapQuery), Details = filter.to_string(), Result = rs.first().map(result_to_trace).unwrap_or_default() ); rs.into_iter().next().map(|entry| { self.mappings .entry_to_principal(SearchEntry::construct(entry)) }) }) .map_err(|err| err.into_error().caused_by(trc::location!())) } } struct LdapResult { dn: String, principal: Principal, member_of: Vec, } impl LdapMappings { fn entry_to_principal(&self, entry: SearchEntry) -> LdapResult { let mut principal = Principal::new(0, Type::Individual); let mut role = ROLE_USER; let mut member_of = vec![]; let mut description = None; let mut secret = None; let mut otp_secret = None; let mut email = None; let mut email_aliases = Vec::new(); for (attr, value) in entry.attrs { let attr = attr.to_lowercase(); if self.attr_name.contains(&attr) { if !self.attr_email_address.contains(&attr) { principal.name = value.into_iter().next().unwrap_or_default(); } else { for (idx, item) in value.into_iter().enumerate() { if email.is_none() { email = Some(item.to_lowercase()); } if idx == 0 { principal.name = item; } } } } else if self.attr_secret.contains(&attr) { for item in value { if item.is_otp_secret() { otp_secret = Some(item); } else if item.is_app_secret() { principal.data.push(PrincipalData::AppPassword(item)); } else if secret.is_none() { secret = Some(item); } } } else if self.attr_secret_changed.contains(&attr) { // Create a disabled AppPassword, used to indicate that the password has been changed // but cannot be used for authentication. if secret.is_none() { secret = value.into_iter().next().map(|item| { format!("$app${}$", xxhash_rust::xxh3::xxh3_64(item.as_bytes())) }); } } else if self.attr_email_address.contains(&attr) { for item in value { if email.is_some() { email_aliases.push(item.to_lowercase()); } else { email = Some(item.to_lowercase()); } } } else if self.attr_email_alias.contains(&attr) { for item in value { email_aliases.push(item.to_lowercase()); } } else if let Some(idx) = self.attr_description.iter().position(|a| a == &attr) { if (description.is_none() || idx == 0) && let Some(desc) = value.into_iter().next() { description = Some(desc); } } else if self.attr_groups.contains(&attr) { member_of.extend(value); } else if self.attr_quota.contains(&attr) { if let Ok(quota) = value.into_iter().next().unwrap_or_default().parse::() && quota > 0 { principal.data.push(PrincipalData::DiskQuota(quota)); } } else if self.attr_type.contains(&attr) { for value in value { match value.to_ascii_lowercase().as_str() { "admin" | "administrator" | "root" | "superuser" => { role = ROLE_ADMIN; principal.typ = Type::Individual } "posixaccount" | "individual" | "person" | "inetorgperson" => { principal.typ = Type::Individual } "posixgroup" | "groupofuniquenames" | "group" => { principal.typ = Type::Group } _ => continue, } break; } } } for alias in email_aliases { if email.as_ref().is_none_or(|email| email != &alias) { principal.data.push(PrincipalData::EmailAlias(alias)); } } if let Some(email) = email { principal.data.push(PrincipalData::PrimaryEmail(email)); } if let Some(secret) = secret { principal.data.push(PrincipalData::Password(secret)); } if let Some(otp_secret) = otp_secret { principal.data.push(PrincipalData::OtpAuth(otp_secret)); } if let Some(desc) = description { principal.data.push(PrincipalData::Description(desc)); } principal.data.push(PrincipalData::Role(role)); LdapResult { dn: entry.dn, principal, member_of, } } } fn result_to_trace(rs: &ResultEntry) -> trc::Value { let se = SearchEntry::construct(rs.clone()); se.attrs .into_iter() .map(|(k, v)| trc::Value::Array(vec![trc::Value::from(k), trc::Value::from(v.join(", "))])) .chain([trc::Value::from(se.dn)]) .collect::>() .into() } ================================================ FILE: crates/directory/src/backend/ldap/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use deadpool::managed::Pool; use ldap3::{LdapConnSettings, ldap_escape}; use store::Store; pub mod config; pub mod lookup; pub mod pool; pub struct LdapDirectory { pool: Pool, mappings: LdapMappings, auth_bind: AuthBind, pub(crate) data_store: Store, } #[derive(Debug, Default)] pub struct LdapMappings { base_dn: String, filter_name: LdapFilter, filter_email: LdapFilter, attr_name: Vec, attr_type: Vec, attr_groups: Vec, attr_description: Vec, attr_secret: Vec, attr_secret_changed: Vec, attr_email_address: Vec, attr_email_alias: Vec, attr_quota: Vec, attrs_principal: Vec, } #[derive(Debug, Default)] pub(crate) struct LdapFilter { filter: Vec, } #[derive(Debug)] enum LdapFilterItem { Static(String), Full, LocalPart, DomainPart, } impl LdapFilter { pub fn build(&self, value: &str) -> String { let mut result = String::with_capacity(value.len() + 16); for item in &self.filter { match item { LdapFilterItem::Static(s) => result.push_str(s), LdapFilterItem::Full => result.push_str(ldap_escape(value).as_ref()), LdapFilterItem::LocalPart => { result.push_str( value .rsplit_once('@') .map(|(local, _)| local) .unwrap_or(value), ); } LdapFilterItem::DomainPart => { if let Some((_, domain)) = value.rsplit_once('@') { result.push_str(domain); } } } } result } } pub(crate) struct LdapConnectionManager { address: String, settings: LdapConnSettings, bind_dn: Option, } pub(crate) struct Bind { dn: String, password: String, } impl LdapConnectionManager { pub fn new(address: String, settings: LdapConnSettings, bind_dn: Option) -> Self { Self { address, settings, bind_dn, } } } impl Bind { pub fn new(dn: String, password: String) -> Self { Self { dn, password } } } pub(crate) enum AuthBind { Template { template: LdapFilter, can_search: bool, }, Lookup, None, } ================================================ FILE: crates/directory/src/backend/ldap/pool.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use async_trait::async_trait; use deadpool::managed; use ldap3::{Ldap, LdapConnAsync, LdapError, exop::WhoAmI}; use super::LdapConnectionManager; #[async_trait] impl managed::Manager for LdapConnectionManager { type Type = Ldap; type Error = LdapError; async fn create(&self) -> Result { let (conn, mut ldap) = LdapConnAsync::with_settings(self.settings.clone(), &self.address).await?; ldap3::drive!(conn); if let Some(bind) = &self.bind_dn { ldap.simple_bind(&bind.dn, &bind.password) .await? .success()?; } Ok(ldap) } async fn recycle( &self, conn: &mut Ldap, _: &managed::Metrics, ) -> managed::RecycleResult { conn.extended(WhoAmI) .await .map(|_| ()) .map_err(managed::RecycleError::Backend) } } ================================================ FILE: crates/directory/src/backend/memory/config.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use store::Store; use utils::config::{Config, utils::AsKey}; use crate::{ Principal, PrincipalData, ROLE_ADMIN, ROLE_USER, Type, backend::internal::manage::ManageDirectory, }; use super::{EmailType, MemoryDirectory}; impl MemoryDirectory { pub async fn from_config( config: &mut Config, prefix: impl AsKey, data_store: Store, ) -> Option { let prefix = prefix.as_key(); let mut directory = MemoryDirectory { data_store, principals: Default::default(), emails_to_ids: Default::default(), domains: Default::default(), }; for lookup_id in config.sub_keys((prefix.as_str(), "principals"), ".name") { let lookup_id = lookup_id.as_str(); let name = config .value_require((prefix.as_str(), "principals", lookup_id, "name"))? .to_string(); let (typ, is_superuser) = match config.value((prefix.as_str(), "principals", lookup_id, "class")) { Some("individual") => (Type::Individual, false), Some("admin") => (Type::Individual, true), Some("group") => (Type::Group, false), _ => (Type::Individual, false), }; // Obtain id let id = directory .data_store .get_or_create_principal_id(&name, Type::Individual) .await .map_err(|err| { config.new_build_error( prefix.as_str(), format!( "Failed to obtain id for principal {} ({}): {:?}", name, lookup_id, err ), ) }) .ok()?; // Create principal let mut principal = Principal::new(id, typ); principal.data.push(PrincipalData::Role(if is_superuser { ROLE_ADMIN } else { ROLE_USER })); // Obtain group ids for group in config .values((prefix.as_str(), "principals", lookup_id, "member-of")) .map(|(_, s)| s.to_string()) .collect::>() { principal.data.push(PrincipalData::MemberOf( directory .data_store .get_or_create_principal_id(&group, Type::Group) .await .map_err(|err| { config.new_build_error( prefix.as_str(), format!( "Failed to obtain id for principal {} ({}): {:?}", name, lookup_id, err ), ) }) .ok()?, )); } // Parse email addresses for (pos, (_, email)) in config .values((prefix.as_str(), "principals", lookup_id, "email")) .enumerate() { directory .emails_to_ids .entry(email.to_string()) .or_default() .push(if pos > 0 { EmailType::Alias(id) } else { EmailType::Primary(id) }); if let Some((_, domain)) = email.rsplit_once('@') { directory.domains.insert(domain.to_lowercase()); } if pos == 0 { principal .data .push(PrincipalData::PrimaryEmail(email.to_lowercase())); } else { principal .data .push(PrincipalData::EmailAlias(email.to_lowercase())); } } // Parse mailing lists for (_, email) in config.values((prefix.as_str(), "principals", lookup_id, "email-list")) { directory .emails_to_ids .entry(email.to_lowercase()) .or_default() .push(EmailType::List(id)); if let Some((_, domain)) = email.rsplit_once('@') { directory.domains.insert(domain.to_lowercase()); } } principal.name = name.as_str().into(); for (_, secret) in config.values((prefix.as_str(), "principals", lookup_id, "secret")) { principal.data.push(PrincipalData::Password(secret.into())); } if let Some(description) = config.value((prefix.as_str(), "principals", lookup_id, "description")) { principal .data .push(PrincipalData::Description(description.into())); } if let Some(quota) = config.property::((prefix.as_str(), "principals", lookup_id, "quota")) { principal.data.push(PrincipalData::DiskQuota(quota)); } directory.principals.push(principal); } Some(directory) } } ================================================ FILE: crates/directory/src/backend/memory/lookup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{EmailType, MemoryDirectory}; use crate::{Principal, QueryBy, QueryParams, backend::RcptType}; use mail_send::Credentials; impl MemoryDirectory { pub async fn query(&self, by: QueryParams<'_>) -> trc::Result> { match by.by { QueryBy::Name(name) => { for principal in &self.principals { if principal.name() == name { return Ok(Some(principal.clone())); } } } QueryBy::Id(uid) => { for principal in &self.principals { if principal.id == uid { return Ok(Some(principal.clone())); } } } QueryBy::Credentials(credentials) => { let (username, secret) = match credentials { Credentials::Plain { username, secret } => (username, secret), Credentials::OAuthBearer { token } => (token, token), Credentials::XOauth2 { username, secret } => (username, secret), }; for principal in &self.principals { if principal.name() == username { return if principal.verify_secret(secret, false, false).await? { Ok(Some(principal.clone())) } else { Ok(None) }; } } } } Ok(None) } pub async fn email_to_id(&self, address: &str) -> trc::Result> { Ok(self.emails_to_ids.get(address).and_then(|names| { names .iter() .map(|t| match t { EmailType::Primary(uid) | EmailType::Alias(uid) | EmailType::List(uid) => *uid, }) .next() })) } pub async fn rcpt(&self, address: &str) -> trc::Result { Ok(self.emails_to_ids.contains_key(address).into()) } pub async fn vrfy(&self, address: &str) -> trc::Result> { let mut result = Vec::new(); for (key, value) in &self.emails_to_ids { if key.contains(address) && value.iter().any(|t| matches!(t, EmailType::Primary(_))) { result.push(key.into()) } } Ok(result) } pub async fn expn(&self, address: &str) -> trc::Result> { let mut result = Vec::new(); for (key, value) in &self.emails_to_ids { if key == address { for item in value { if let EmailType::List(uid) = item { for principal in &self.principals { if principal.id == *uid { if let Some(addr) = principal.primary_email() { result.push(addr.to_string()) } break; } } } } } } Ok(result) } pub async fn is_local_domain(&self, domain: &str) -> trc::Result { Ok(self.domains.contains(domain)) } } ================================================ FILE: crates/directory/src/backend/memory/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use ahash::{AHashMap, AHashSet}; use store::Store; use crate::Principal; pub mod config; pub mod lookup; #[derive(Debug)] pub struct MemoryDirectory { principals: Vec, emails_to_ids: AHashMap>, pub(crate) data_store: Store, domains: AHashSet, } #[derive(Debug)] enum EmailType { Primary(u32), Alias(u32), List(u32), } ================================================ FILE: crates/directory/src/backend/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod imap; pub mod internal; pub mod ldap; pub mod memory; pub mod oidc; pub mod smtp; pub mod sql; #[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] pub enum RcptType { Mailbox, List(Vec), #[default] Invalid, } impl From for RcptType { fn from(value: bool) -> Self { if value { RcptType::Mailbox } else { RcptType::Invalid } } } ================================================ FILE: crates/directory/src/backend/oidc/config.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use base64::{Engine, engine::general_purpose}; use store::Store; use utils::config::{Config, utils::AsKey}; use super::{Authentication, EndpointType, OpenIdConfig, OpenIdDirectory}; impl OpenIdDirectory { pub fn from_config(config: &mut Config, prefix: impl AsKey, data_store: Store) -> Option { let prefix = prefix.as_key(); let endpoint_type = match config.value_require((&prefix, "endpoint.method"))? { "introspect" => match config.value_require((&prefix, "auth.method"))? { #[allow(clippy::to_string_in_format_args)] "basic" => EndpointType::Introspect(Authentication::Header(format!( "Basic {}", general_purpose::STANDARD.encode( format!( "{}:{}", config .value_require((&prefix, "auth.username"))? .to_string(), config.value_require((&prefix, "auth.secret"))? ) .as_bytes() ) ))), "token" => EndpointType::Introspect(Authentication::Header(format!( "Bearer {}", config.value_require((&prefix, "auth.token"))? ))), "user-token" => EndpointType::Introspect(Authentication::Bearer), "none" => EndpointType::Introspect(Authentication::None), _ => { config.new_build_error( (&prefix, "auth.method"), "Invalid authentication method, must be 'header', 'bearer' or 'none'", ); return None; } }, "userinfo" => EndpointType::UserInfo, _ => { config.new_build_error( (&prefix, "endpoint.method"), "Invalid endpoint method, must be 'introspect' or 'userinfo'", ); return None; } }; let email_field = config.value_require((&prefix, "fields.email"))?.to_string(); Some(OpenIdDirectory { config: OpenIdConfig { endpoint: config.value_require((&prefix, "endpoint.url"))?.to_string(), endpoint_type, endpoint_timeout: config .property_or_default::((&prefix, "timeout"), "30s") .unwrap_or_else(|| Duration::from_secs(30)), username_field: config .value((&prefix, "fields.username")) .filter(|&v| v != email_field) .map(|v| v.to_string()), email_field, full_name_field: config .value((&prefix, "fields.full-name")) .map(|v| v.to_string()), }, data_store, }) } } ================================================ FILE: crates/directory/src/backend/oidc/lookup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use ahash::HashMap; use mail_send::Credentials; use reqwest::{StatusCode, header::AUTHORIZATION}; use trc::{AddContext, AuthEvent}; use crate::{ Principal, PrincipalData, QueryBy, QueryParams, ROLE_USER, Type, backend::{ RcptType, internal::{ lookup::DirectoryStore, manage::{self, ManageDirectory, UpdatePrincipal}, }, oidc::{Authentication, EndpointType}, }, }; use super::{OpenIdConfig, OpenIdDirectory}; type OpenIdResponse = HashMap; impl OpenIdDirectory { pub async fn query(&self, by: QueryParams<'_>) -> trc::Result> { match &by.by { QueryBy::Credentials(Credentials::OAuthBearer { token }) => { // Send request #[cfg(feature = "test_mode")] let client = reqwest::Client::builder().danger_accept_invalid_certs(true); #[cfg(not(feature = "test_mode"))] let client = reqwest::Client::builder(); let client = client .timeout(self.config.endpoint_timeout) .build() .map_err(|err| { AuthEvent::Error .into_err() .reason(err) .details("Failed to build client") })?; let client = match &self.config.endpoint_type { EndpointType::UserInfo => client.get(&self.config.endpoint).bearer_auth(token), EndpointType::Introspect(authentication) => { let client = client.post(&self.config.endpoint).form(&[ ("token", token.as_str()), ("token_type_hint", "access_token"), ]); match authentication { Authentication::Header(header) => client.header(AUTHORIZATION, header), Authentication::Bearer => client.bearer_auth(token), Authentication::None => client, } } }; let response = client.send().await.map_err(|err| { AuthEvent::Error .into_err() .reason(err) .details("HTTP request failed") })?; match response.status() { StatusCode::OK => { // Fetch response let response = response.bytes().await.map_err(|err| { AuthEvent::Error .into_err() .reason(err) .details("Failed to read OIDC response") })?; // Deserialize response let external_principal = serde_json::from_slice::(&response) .map_err(|err| { AuthEvent::Error .into_err() .reason(err) .details("Failed to deserialize OIDC response") })? .build_principal(&self.config)?; // Fetch principal let id = self .data_store .get_or_create_principal_id(external_principal.name(), Type::Individual) .await .caused_by(trc::location!())?; let mut principal = self .data_store .query(QueryParams::id(id).with_return_member_of(by.return_member_of)) .await .caused_by(trc::location!())? .ok_or_else(|| manage::not_found(id).caused_by(trc::location!()))?; // Keep the internal store up to date with the OIDC server let changes = principal.update_external(external_principal); if !changes.is_empty() { self.data_store .update_principal( UpdatePrincipal::by_id(principal.id) .with_updates(changes) .create_domains(), ) .await .caused_by(trc::location!())?; } Ok(Some(principal)) } StatusCode::UNAUTHORIZED => Err(trc::AuthEvent::Failed .into_err() .code(401) .details("Unauthorized")), other => Err(trc::AuthEvent::Error .into_err() .code(other.as_u16()) .ctx(trc::Key::Reason, response.text().await.unwrap_or_default()) .details("Unexpected status code")), } } _ => self.data_store.query(by.with_only_app_pass(true)).await, } } pub async fn email_to_id(&self, address: &str) -> trc::Result> { self.data_store.email_to_id(address).await } pub async fn rcpt(&self, address: &str) -> trc::Result { self.data_store.rcpt(address).await } pub async fn vrfy(&self, address: &str) -> trc::Result> { self.data_store.vrfy(address).await } pub async fn expn(&self, address: &str) -> trc::Result> { self.data_store.expn(address).await } pub async fn is_local_domain(&self, domain: &str) -> trc::Result { self.data_store.is_local_domain(domain).await } } trait BuildPrincipal { fn build_principal(&mut self, config: &OpenIdConfig) -> trc::Result; fn take_required_field(&mut self, field: &str) -> trc::Result; fn take_field(&mut self, field: &str) -> Option; } impl BuildPrincipal for OpenIdResponse { fn build_principal(&mut self, config: &OpenIdConfig) -> trc::Result { let email = self .take_required_field(&config.email_field)? .to_lowercase(); let username = if let Some(username_field) = &config.username_field { self.take_required_field(username_field)?.to_lowercase() } else { email.clone() }; if !email.contains('@') && !email.contains('.') { return Err(AuthEvent::Error .into_err() .details("Email field is not valid") .ctx(trc::Key::Key, email)); } let full_name = config .full_name_field .as_ref() .and_then(|field| self.take_field(field)); // Build principal let mut data = Vec::with_capacity(3); data.push(PrincipalData::PrimaryEmail(email)); if let Some(name) = full_name { data.push(PrincipalData::Description(name)); } data.push(PrincipalData::Role(ROLE_USER)); Ok(Principal { id: u32::MAX, typ: Type::Individual, name: username, data, }) } fn take_required_field(&mut self, field: &str) -> trc::Result { match self.remove(field) { Some(serde_json::Value::String(value)) if !value.is_empty() => Ok(value), other => Err(trc::AuthEvent::Error .into_err() .details("Unexpected field type in OIDC response") .ctx(trc::Key::Key, field.to_string()) .ctx( trc::Key::Value, serde_json::to_string(&other.unwrap_or(serde_json::Value::Null)) .unwrap_or_default(), )), } } fn take_field(&mut self, field: &str) -> Option { match self.remove(field) { Some(serde_json::Value::String(value)) if !value.is_empty() => Some(value), _ => None, } } } ================================================ FILE: crates/directory/src/backend/oidc/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod config; pub mod lookup; use std::time::Duration; use store::Store; pub struct OpenIdDirectory { config: OpenIdConfig, pub(crate) data_store: Store, } struct OpenIdConfig { pub endpoint: String, pub endpoint_type: EndpointType, pub endpoint_timeout: Duration, pub email_field: String, pub username_field: Option, pub full_name_field: Option, } #[derive(Debug)] pub enum EndpointType { Introspect(Authentication), UserInfo, } #[derive(Debug)] pub enum Authentication { Header(String), Bearer, None, } ================================================ FILE: crates/directory/src/backend/smtp/config.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use mail_send::{SmtpClientBuilder, smtp::tls::build_tls_connector}; use utils::config::{Config, utils::AsKey}; use crate::core::config::build_pool; use super::{SmtpConnectionManager, SmtpDirectory}; impl SmtpDirectory { pub fn from_config(config: &mut Config, prefix: impl AsKey, is_lmtp: bool) -> Option { let prefix = prefix.as_key(); let address = config.value_require((&prefix, "host"))?.to_string(); let tls_implicit: bool = config .property_or_default((&prefix, "tls.enable"), "false") .unwrap_or_default(); let port: u16 = config .property_or_default((&prefix, "port"), if tls_implicit { "465" } else { "25" }) .unwrap_or(if tls_implicit { 465 } else { 25 }); let manager = SmtpConnectionManager { builder: SmtpClientBuilder { addr: format!("{address}:{port}"), timeout: config .property_or_default((&prefix, "timeout"), "30s") .unwrap_or_else(|| Duration::from_secs(30)), tls_connector: build_tls_connector( config .property_or_default((&prefix, "tls.allow-invalid-certs"), "false") .unwrap_or_default(), ), tls_hostname: address.to_string(), tls_implicit, is_lmtp, credentials: None, local_host: config .value("server.hostname") .unwrap_or("[127.0.0.1]") .to_string(), say_ehlo: false, local_ip: None, }, max_rcpt: config .property_or_default((&prefix, "limits.rcpt"), "10") .unwrap_or(10), max_auth_errors: config .property_or_default((&prefix, "limits.auth-errors"), "3") .unwrap_or(10), }; Some(SmtpDirectory { pool: build_pool(config, &prefix, manager) .map_err(|e| { config.new_parse_error( prefix.as_str(), format!("Failed to build SMTP pool: {e:?}"), ) }) .ok()?, domains: config .values((&prefix, "lookup.domains")) .map(|(_, v)| v.to_lowercase()) .collect(), }) } } ================================================ FILE: crates/directory/src/backend/smtp/lookup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use mail_send::{Credentials, smtp::AssertReply}; use smtp_proto::Severity; use crate::{IntoError, Principal, QueryBy, Type, backend::RcptType}; use super::{SmtpClient, SmtpDirectory}; impl SmtpDirectory { pub async fn query(&self, query: QueryBy<'_>) -> trc::Result> { if let QueryBy::Credentials(credentials) = query { self.pool .get() .await .map_err(|err| err.into_error().caused_by(trc::location!()))? .authenticate(credentials) .await } else { Err(trc::StoreEvent::NotSupported.caused_by(trc::location!())) } } pub async fn email_to_id(&self, _address: &str) -> trc::Result> { Err(trc::StoreEvent::NotSupported.caused_by(trc::location!())) } pub async fn rcpt(&self, address: &str) -> trc::Result { let mut conn = self .pool .get() .await .map_err(|err| err.into_error().caused_by(trc::location!()))?; if !conn.sent_mail_from { conn.client .cmd(b"MAIL FROM:<>\r\n") .await .map_err(|err| err.into_error().caused_by(trc::location!()))? .assert_positive_completion() .map_err(|err| err.into_error().caused_by(trc::location!()))?; conn.sent_mail_from = true; } let reply = conn .client .cmd(format!("RCPT TO:<{address}>\r\n").as_bytes()) .await .map_err(|err| err.into_error().caused_by(trc::location!()))?; match reply.severity() { Severity::PositiveCompletion => { conn.num_rcpts += 1; if conn.num_rcpts >= conn.max_rcpt { let _ = conn.client.rset().await; conn.num_rcpts = 0; conn.sent_mail_from = false; } Ok(RcptType::Mailbox) } Severity::PermanentNegativeCompletion => Ok(RcptType::Invalid), _ => Err(trc::StoreEvent::UnexpectedError .ctx(trc::Key::Code, reply.code()) .ctx(trc::Key::Details, reply.message)), } } pub async fn vrfy(&self, address: &str) -> trc::Result> { self.pool .get() .await .map_err(|err| err.into_error().caused_by(trc::location!()))? .expand(&format!("VRFY {address}\r\n")) .await } pub async fn expn(&self, address: &str) -> trc::Result> { self.pool .get() .await .map_err(|err| err.into_error().caused_by(trc::location!()))? .expand(&format!("EXPN {address}\r\n")) .await } pub async fn is_local_domain(&self, domain: &str) -> trc::Result { Ok(self.domains.contains(domain)) } } impl SmtpClient { async fn authenticate( &mut self, credentials: &Credentials, ) -> trc::Result> { match self .client .authenticate(credentials, &self.capabilities) .await { Ok(_) => Ok(Some(Principal::new(u32::MAX, Type::Individual))), Err(err) => match &err { mail_send::Error::AuthenticationFailed(err) if err.code() == 535 => { self.num_auth_failures += 1; Ok(None) } _ => Err(err.into_error()), }, } } async fn expand(&mut self, command: &str) -> trc::Result> { let reply = self .client .cmd(command.as_bytes()) .await .map_err(|err| err.into_error().caused_by(trc::location!()))?; match reply.code() { 250 | 251 => Ok(reply .message() .split('\n') .map(|p| p.into()) .collect::>()), code @ (550 | 551 | 553 | 500 | 502) => { Err(trc::StoreEvent::NotSupported.ctx(trc::Key::Code, code)) } code => Err(trc::StoreEvent::UnexpectedError .ctx(trc::Key::Code, code) .ctx(trc::Key::Details, reply.message)), } } } ================================================ FILE: crates/directory/src/backend/smtp/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod config; pub mod lookup; pub mod pool; use ahash::AHashSet; use deadpool::managed::Pool; use mail_send::SmtpClientBuilder; use smtp_proto::EhloResponse; use tokio::net::TcpStream; use tokio_rustls::client::TlsStream; pub struct SmtpDirectory { pool: Pool, domains: AHashSet, } pub struct SmtpConnectionManager { builder: SmtpClientBuilder, max_rcpt: usize, max_auth_errors: usize, } pub struct SmtpClient { client: mail_send::SmtpClient>, capabilities: EhloResponse, max_rcpt: usize, max_auth_errors: usize, num_rcpts: usize, num_auth_failures: usize, sent_mail_from: bool, } ================================================ FILE: crates/directory/src/backend/smtp/pool.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use async_trait::async_trait; use deadpool::managed; use mail_send::{Error, smtp::AssertReply}; use super::{SmtpClient, SmtpConnectionManager}; #[async_trait] impl managed::Manager for SmtpConnectionManager { type Type = SmtpClient; type Error = Error; async fn create(&self) -> Result { let mut client = self.builder.connect().await?; let capabilities = client .capabilities(&self.builder.local_host, self.builder.is_lmtp) .await?; Ok(SmtpClient { capabilities, client, max_auth_errors: self.max_auth_errors, max_rcpt: self.max_rcpt, num_rcpts: 0, num_auth_failures: 0, sent_mail_from: false, }) } async fn recycle( &self, conn: &mut SmtpClient, _: &managed::Metrics, ) -> managed::RecycleResult { if conn.num_auth_failures < conn.max_auth_errors { conn.client .cmd(b"NOOP\r\n") .await? .assert_positive_completion() .map(|_| ()) .map_err(managed::RecycleError::Backend) } else { Err(managed::RecycleError::Message( "No longer valid: Too many authentication failures".to_string(), )) } } } ================================================ FILE: crates/directory/src/backend/sql/config.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use store::{Store, Stores}; use utils::config::{Config, utils::AsKey}; use super::{SqlDirectory, SqlMappings}; impl SqlDirectory { pub fn from_config( config: &mut Config, prefix: impl AsKey, stores: &Stores, data_store: Store, ) -> Option { let prefix = prefix.as_key(); let store_id = config.value_require((&prefix, "store"))?.to_string(); let sql_store = if let Some(sql_store) = stores.stores.get(&store_id).filter(|store| store.is_sql()) { sql_store.clone() } else { let err = format!("Directory references a non-existent store {store_id:?}"); config.new_build_error((&prefix, "store"), err); return None; }; let mut mappings = SqlMappings { column_description: config .value((&prefix, "columns.description")) .unwrap_or_default() .to_string(), column_secret: config .value((&prefix, "columns.secret")) .unwrap_or_default() .to_string(), column_email: config .value((&prefix, "columns.email")) .unwrap_or_default() .to_string(), column_quota: config .value((&prefix, "columns.quota")) .unwrap_or_default() .to_string(), column_type: config .value((&prefix, "columns.class")) .unwrap_or_default() .to_string(), ..Default::default() }; for (query_id, query) in [ ("name", &mut mappings.query_name), ("members", &mut mappings.query_members), ("emails", &mut mappings.query_emails), ("recipients", &mut mappings.query_recipients), ("secrets", &mut mappings.query_secrets), ] { *query = config .value(("store", store_id.as_str(), "query", query_id)) .unwrap_or_default() .to_string(); } Some(SqlDirectory { sql_store, mappings, data_store, }) } } ================================================ FILE: crates/directory/src/backend/sql/lookup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{SqlDirectory, SqlMappings}; use crate::{ Principal, PrincipalData, QueryBy, QueryParams, ROLE_ADMIN, ROLE_USER, Type, backend::{ RcptType, internal::{ SpecialSecrets, lookup::DirectoryStore, manage::{self, ManageDirectory, UpdatePrincipal}, }, }, }; use mail_send::Credentials; use store::{NamedRows, Rows, Value}; use trc::AddContext; impl SqlDirectory { pub async fn query(&self, by: QueryParams<'_>) -> trc::Result> { let (external_principal, stored_principal) = match by.by { QueryBy::Name(username) => ( self.mappings .row_to_principal( self.sql_store .sql_query::( &self.mappings.query_name, vec![username.into()], ) .await .caused_by(trc::location!())?, ) .caused_by(trc::location!())? .map(|mut p| { p.name = username.into(); p }), None, ), QueryBy::Id(uid) => { if let Some(principal) = self .data_store .query(QueryParams::id(uid).with_return_member_of(by.return_member_of)) .await .caused_by(trc::location!())? { ( self.mappings .row_to_principal( self.sql_store .sql_query::( &self.mappings.query_name, vec![principal.name().into()], ) .await .caused_by(trc::location!())?, ) .caused_by(trc::location!())?, Some(principal), ) } else { return Ok(None); } } QueryBy::Credentials(credentials) => { let (username, secret) = match credentials { Credentials::Plain { username, secret } => (username, secret), Credentials::OAuthBearer { token } => (token, token), Credentials::XOauth2 { username, secret } => (username, secret), }; match self .mappings .row_to_principal( self.sql_store .sql_query::( &self.mappings.query_name, vec![username.into()], ) .await .caused_by(trc::location!())?, ) .caused_by(trc::location!())? { Some(mut principal) => { // Obtain secrets if !self.mappings.query_secrets.is_empty() { let secrets = self .sql_store .sql_query::( &self.mappings.query_secrets, vec![username.into()], ) .await .caused_by(trc::location!())?; for row in secrets.rows { for value in row.values { if let Value::Text(secret) = value { let secret = secret.into_owned(); if secret.is_otp_secret() { if !principal.data.iter().any(|data| { matches!(data, PrincipalData::OtpAuth(_)) }) { principal.data.push(PrincipalData::OtpAuth(secret)); } } else if secret.is_app_secret() { principal.data.push(PrincipalData::AppPassword(secret)); } else if !principal .data .iter() .any(|data| matches!(data, PrincipalData::Password(_))) { principal.data.push(PrincipalData::Password(secret)); } } } } } if principal .verify_secret(secret, false, false) .await .caused_by(trc::location!())? { principal.name = username.into(); (Some(principal), None) } else { (None, None) } } _ => (None, None), } } }; let mut external_principal = if let Some(external_principal) = external_principal { external_principal } else { return Ok(None); }; // Obtain members if by.return_member_of && !self.mappings.query_members.is_empty() { for row in self .sql_store .sql_query::( &self.mappings.query_members, vec![external_principal.name().into()], ) .await .caused_by(trc::location!())? .rows { if let Some(Value::Text(account_name)) = row.values.first() { let account_id = self .data_store .get_or_create_principal_id(account_name, Type::Group) .await .caused_by(trc::location!())?; external_principal .data .push(PrincipalData::MemberOf(account_id)); } } } // Obtain emails if !self.mappings.query_emails.is_empty() { let mut rows = self .sql_store .sql_query::( &self.mappings.query_emails, vec![external_principal.name().into()], ) .await .caused_by(trc::location!())? .rows .into_iter() .flat_map(|v| v.values.into_iter().map(|v| v.into_lower_string())); if external_principal.primary_email().is_none() && let Some(email) = rows.next() { external_principal .data .push(PrincipalData::PrimaryEmail(email)); } external_principal .data .extend(rows.map(PrincipalData::EmailAlias)); } // Obtain account ID if not available let mut principal = if let Some(stored_principal) = stored_principal { stored_principal } else { let id = self .data_store .get_or_create_principal_id(external_principal.name(), Type::Individual) .await .caused_by(trc::location!())?; self.data_store .query(QueryParams::id(id).with_return_member_of(by.return_member_of)) .await .caused_by(trc::location!())? .ok_or_else(|| manage::not_found(id).caused_by(trc::location!()))? }; // Keep the internal store up to date with the SQL server let changes = principal.update_external(external_principal); if !changes.is_empty() { self.data_store .update_principal( UpdatePrincipal::by_id(principal.id) .with_updates(changes) .create_domains(), ) .await .caused_by(trc::location!())?; } Ok(Some(principal)) } pub async fn email_to_id(&self, address: &str) -> trc::Result> { let names = self .sql_store .sql_query::(&self.mappings.query_recipients, vec![address.into()]) .await .caused_by(trc::location!())?; for row in names.rows { if let Some(Value::Text(name)) = row.values.first() { return self .data_store .get_or_create_principal_id(name, Type::Individual) .await .caused_by(trc::location!()) .map(Some); } } Ok(None) } pub async fn rcpt(&self, address: &str) -> trc::Result { let result = self .sql_store .sql_query::( &self.mappings.query_recipients, vec![address.to_string().into()], ) .await?; if result { Ok(RcptType::Mailbox) } else { self.data_store.rcpt(address).await.map(|result| { if matches!(result, RcptType::List(_)) { result } else { RcptType::Invalid } }) } } pub async fn vrfy(&self, address: &str) -> trc::Result> { self.data_store.vrfy(address).await } pub async fn expn(&self, address: &str) -> trc::Result> { self.data_store.expn(address).await } pub async fn is_local_domain(&self, domain: &str) -> trc::Result { self.data_store.is_local_domain(domain).await } } impl SqlMappings { pub fn row_to_principal(&self, rows: NamedRows) -> trc::Result> { if rows.rows.is_empty() { return Ok(None); } let mut principal = Principal::new(u32::MAX, Type::Individual); let mut role = ROLE_USER; let mut has_primary_email = false; let mut secret = None; if let Some(row) = rows.rows.into_iter().next() { for (name, value) in rows.names.into_iter().zip(row.values) { if name.eq_ignore_ascii_case(&self.column_secret) { if let Value::Text(text) = value { secret = Some(text.into_owned()); } } else if name.eq_ignore_ascii_case(&self.column_type) { match value.to_str().as_ref() { "individual" | "person" | "user" => { principal.typ = Type::Individual; } "group" => principal.typ = Type::Group, "admin" | "superuser" | "administrator" => { principal.typ = Type::Individual; role = ROLE_ADMIN; } _ => (), } } else if name.eq_ignore_ascii_case(&self.column_description) { if let Value::Text(text) = value { principal .data .push(PrincipalData::Description(text.as_ref().into())); } } else if name.eq_ignore_ascii_case(&self.column_email) { if let Value::Text(text) = value { if !has_primary_email { has_primary_email = true; principal .data .push(PrincipalData::PrimaryEmail(text.to_lowercase())); } else { principal .data .push(PrincipalData::EmailAlias(text.to_lowercase())); } } } else if name.eq_ignore_ascii_case(&self.column_quota) && let Value::Integer(quota) = value && quota > 0 { principal.data.push(PrincipalData::DiskQuota(quota as u64)); } } } if let Some(secret) = secret { principal.data.push(PrincipalData::Password(secret)); } principal.data.push(PrincipalData::Role(role)); Ok(Some(principal)) } } ================================================ FILE: crates/directory/src/backend/sql/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use store::Store; pub mod config; pub mod lookup; pub struct SqlDirectory { sql_store: Store, mappings: SqlMappings, pub(crate) data_store: Store, } #[derive(Debug, Default)] pub(crate) struct SqlMappings { query_name: String, query_members: String, query_emails: String, query_recipients: String, query_secrets: String, column_description: String, column_secret: String, column_email: String, column_quota: String, column_type: String, } ================================================ FILE: crates/directory/src/core/cache.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use utils::{ cache::CacheWithTtl, config::{Config, utils::AsKey}, }; use crate::backend::RcptType; pub struct CachedDirectory { cached_domains: CacheWithTtl, cached_rcpts: CacheWithTtl, ttl_pos: Duration, ttl_neg: Duration, } impl CachedDirectory { pub fn try_from_config(config: &mut Config, prefix: impl AsKey) -> Option { let prefix = prefix.as_key(); let cached_size = config .property_or_default::>((&prefix, "cache.size"), "1048576") .unwrap_or_default()?; Some(CachedDirectory { cached_domains: CacheWithTtl::new(50, cached_size), cached_rcpts: CacheWithTtl::new(100, cached_size), ttl_pos: config .property((&prefix, "cache.ttl.positive")) .unwrap_or(Duration::from_secs(86400)), ttl_neg: config .property((&prefix, "cache.ttl.negative")) .unwrap_or_else(|| Duration::from_secs(3600)), }) } pub fn get_rcpt(&self, address: &str) -> Option { self.cached_rcpts.get(address).map(Into::into) } pub fn set_rcpt(&self, address: &str, exists: &RcptType) { let (exists, ttl) = match exists { RcptType::Mailbox => (true, self.ttl_pos), RcptType::Invalid => (false, self.ttl_neg), RcptType::List(_) => return, }; self.cached_rcpts.insert(address.to_string(), exists, ttl); } pub fn get_domain(&self, domain: &str) -> Option { self.cached_domains.get(domain) } pub fn set_domain(&self, domain: &str, exists: bool) { self.cached_domains.insert( domain.to_string(), exists, if exists { self.ttl_pos } else { self.ttl_neg }, ); } } ================================================ FILE: crates/directory/src/core/config.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use deadpool::{ Runtime, managed::{Manager, Pool}, }; use std::{sync::Arc, time::Duration}; use store::{Store, Stores}; use utils::config::Config; use ahash::AHashMap; use crate::{ Directories, Directory, DirectoryInner, backend::{ imap::ImapDirectory, ldap::LdapDirectory, memory::MemoryDirectory, oidc::OpenIdDirectory, smtp::SmtpDirectory, sql::SqlDirectory, }, }; use super::cache::CachedDirectory; impl Directories { pub async fn parse( config: &mut Config, stores: &Stores, data_store: Store, is_enterprise: bool, ) -> Self { let mut directories = AHashMap::new(); for id in config.sub_keys("directory", ".type") { // Parse directory let id = id.as_str(); #[cfg(feature = "test_mode")] { if config .property_or_default::(("directory", id, "disable"), "false") .unwrap_or(false) { continue; } } let protocol = config .value_require(("directory", id, "type")) .unwrap() .to_string(); let prefix = ("directory", id); let store = match protocol.as_str() { "internal" => Some(DirectoryInner::Internal( if let Some(store_id) = config.value_require(("directory", id, "store")) { if let Some(data) = stores.stores.get(store_id) { data.clone() } else { config.new_parse_error( ("directory", id, "store"), "Store does not exist", ); continue; } } else { continue; }, )), "ldap" => LdapDirectory::from_config(config, prefix, data_store.clone()) .map(DirectoryInner::Ldap), "sql" => SqlDirectory::from_config(config, prefix, stores, data_store.clone()) .map(DirectoryInner::Sql), "imap" => ImapDirectory::from_config(config, prefix).map(DirectoryInner::Imap), "smtp" => { SmtpDirectory::from_config(config, prefix, false).map(DirectoryInner::Smtp) } "lmtp" => { SmtpDirectory::from_config(config, prefix, true).map(DirectoryInner::Smtp) } "memory" => MemoryDirectory::from_config(config, prefix, data_store.clone()) .await .map(DirectoryInner::Memory), "oidc" => OpenIdDirectory::from_config(config, prefix, data_store.clone()) .map(DirectoryInner::OpenId), unknown => { let err = format!("Unknown directory type: {unknown:?}"); config.new_parse_error(("directory", id, "type"), err); continue; } }; // Build directory if let Some(store) = store { // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] if store.is_enterprise_directory() && !is_enterprise { let message = format!("Directory {protocol:?} is an Enterprise Edition feature"); config.new_parse_error(("directory", id, "type"), message); continue; } // SPDX-SnippetEnd let directory = Arc::new(Directory { store, cache: CachedDirectory::try_from_config(config, ("directory", id)), }); // Add directory directories.insert(id.to_string(), directory); } } Directories { directories } } } pub(crate) fn build_pool( config: &mut Config, prefix: &str, manager: M, ) -> Result, String> { Pool::builder(manager) .runtime(Runtime::Tokio1) .max_size( config .property_or_default((prefix, "pool.max-connections"), "10") .unwrap_or(10), ) .create_timeout( config .property_or_default::((prefix, "pool.timeout.create"), "30s") .unwrap_or_else(|| Duration::from_secs(30)) .into(), ) .wait_timeout(config.property_or_default((prefix, "pool.timeout.wait"), "30s")) .recycle_timeout(config.property_or_default((prefix, "pool.timeout.recycle"), "30s")) .build() .map_err(|err| { format!( "Failed to build pool for {prefix:?}: {err}", prefix = prefix, err = err ) }) } ================================================ FILE: crates/directory/src/core/dispatch.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use trc::AddContext; use crate::{ Directory, DirectoryInner, Principal, QueryParams, backend::{RcptType, internal::lookup::DirectoryStore}, }; impl Directory { pub async fn query(&self, by: QueryParams<'_>) -> trc::Result> { match &self.store { DirectoryInner::Internal(store) => store.query(by).await, DirectoryInner::Ldap(store) => store.query(by).await, DirectoryInner::Sql(store) => store.query(by).await, DirectoryInner::Imap(store) => store.query(by.by).await, DirectoryInner::Smtp(store) => store.query(by.by).await, DirectoryInner::Memory(store) => store.query(by).await, DirectoryInner::OpenId(store) => store.query(by).await, } .caused_by(trc::location!()) } pub async fn email_to_id(&self, address: &str) -> trc::Result> { match &self.store { DirectoryInner::Internal(store) => store.email_to_id(address).await, DirectoryInner::Ldap(store) => store.email_to_id(address).await, DirectoryInner::Sql(store) => store.email_to_id(address).await, DirectoryInner::Imap(store) => store.email_to_id(address).await, DirectoryInner::Smtp(store) => store.email_to_id(address).await, DirectoryInner::Memory(store) => store.email_to_id(address).await, DirectoryInner::OpenId(store) => store.email_to_id(address).await, } .caused_by(trc::location!()) } pub async fn is_local_domain(&self, domain: &str) -> trc::Result { // Check cache if let Some(cache) = &self.cache && let Some(result) = cache.get_domain(domain) { return Ok(result); } let result = match &self.store { DirectoryInner::Internal(store) => store.is_local_domain(domain).await, DirectoryInner::Ldap(store) => store.is_local_domain(domain).await, DirectoryInner::Sql(store) => store.is_local_domain(domain).await, DirectoryInner::Imap(store) => store.is_local_domain(domain).await, DirectoryInner::Smtp(store) => store.is_local_domain(domain).await, DirectoryInner::Memory(store) => store.is_local_domain(domain).await, DirectoryInner::OpenId(store) => store.is_local_domain(domain).await, } .caused_by(trc::location!())?; // Update cache if let Some(cache) = &self.cache { cache.set_domain(domain, result); } Ok(result) } pub async fn rcpt(&self, email: &str) -> trc::Result { // Check cache if let Some(cache) = &self.cache && let Some(result) = cache.get_rcpt(email) { return Ok(result); } let result = match &self.store { DirectoryInner::Internal(store) => store.rcpt(email).await, DirectoryInner::Ldap(store) => store.rcpt(email).await, DirectoryInner::Sql(store) => store.rcpt(email).await, DirectoryInner::Imap(store) => store.rcpt(email).await, DirectoryInner::Smtp(store) => store.rcpt(email).await, DirectoryInner::Memory(store) => store.rcpt(email).await, DirectoryInner::OpenId(store) => store.rcpt(email).await, } .caused_by(trc::location!())?; // Update cache if let Some(cache) = &self.cache { cache.set_rcpt(email, &result); } Ok(result) } pub async fn vrfy(&self, address: &str) -> trc::Result> { match &self.store { DirectoryInner::Internal(store) => store.vrfy(address).await, DirectoryInner::Ldap(store) => store.vrfy(address).await, DirectoryInner::Sql(store) => store.vrfy(address).await, DirectoryInner::Imap(store) => store.vrfy(address).await, DirectoryInner::Smtp(store) => store.vrfy(address).await, DirectoryInner::Memory(store) => store.vrfy(address).await, DirectoryInner::OpenId(store) => store.vrfy(address).await, } .caused_by(trc::location!()) } pub async fn expn(&self, address: &str) -> trc::Result> { match &self.store { DirectoryInner::Internal(store) => store.expn(address).await, DirectoryInner::Ldap(store) => store.expn(address).await, DirectoryInner::Sql(store) => store.expn(address).await, DirectoryInner::Imap(store) => store.expn(address).await, DirectoryInner::Smtp(store) => store.expn(address).await, DirectoryInner::Memory(store) => store.expn(address).await, DirectoryInner::OpenId(store) => store.expn(address).await, } .caused_by(trc::location!()) } pub fn has_bearer_token_support(&self) -> bool { match &self.store { DirectoryInner::Internal(_) | DirectoryInner::Ldap(_) | DirectoryInner::Sql(_) | DirectoryInner::Imap(_) | DirectoryInner::Smtp(_) | DirectoryInner::Memory(_) => false, DirectoryInner::OpenId(_) => true, } } } impl DirectoryInner { pub fn is_enterprise_directory(&self) -> bool { false } } ================================================ FILE: crates/directory/src/core/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::Permission; pub mod cache; pub mod config; pub mod dispatch; pub mod principal; pub mod secret; impl Permission { pub fn description(&self) -> &'static str { match self { Permission::Impersonate => "Act on behalf of another user", Permission::UnlimitedRequests => "Perform unlimited requests", Permission::UnlimitedUploads => "Upload unlimited data", Permission::DeleteSystemFolders_ => "", Permission::MessageQueueList => "View message queue", Permission::MessageQueueGet => "Retrieve specific messages from the queue", Permission::MessageQueueUpdate => "Modify queued messages", Permission::MessageQueueDelete => "Remove messages from the queue", Permission::OutgoingReportList => "View outgoing DMARC and TLS reports", Permission::OutgoingReportGet => "Retrieve specific outgoing DMARC and TLS reports", Permission::OutgoingReportDelete => "Remove outgoing DMARC and TLS reports", Permission::IncomingReportList => "View incoming DMARC, TLS and ARF reports", Permission::IncomingReportGet => { "Retrieve specific incoming DMARC, TLS and ARF reports" } Permission::IncomingReportDelete => "Remove incoming DMARC, TLS and ARF reports", Permission::SettingsList => "View system settings", Permission::SettingsUpdate => "Modify system settings", Permission::SettingsDelete => "Remove system settings", Permission::SettingsReload => "Refresh system settings", Permission::IndividualList => "View list of user accounts", Permission::IndividualGet => "Retrieve specific account information", Permission::IndividualUpdate => "Modify user account information", Permission::IndividualDelete => "Remove user accounts", Permission::IndividualCreate => "Add new user accounts", Permission::GroupList => "View list of user groups", Permission::GroupGet => "Retrieve specific group information", Permission::GroupUpdate => "Modify group information", Permission::GroupDelete => "Remove user groups", Permission::GroupCreate => "Add new user groups", Permission::DomainList => "View list of email domains", Permission::DomainGet => "Retrieve specific domain information", Permission::DomainCreate => "Add new email domains", Permission::DomainUpdate => "Modify domain information", Permission::DomainDelete => "Remove email domains", Permission::TenantList => "View list of tenants", Permission::TenantGet => "Retrieve specific tenant information", Permission::TenantCreate => "Add new tenants", Permission::TenantUpdate => "Modify tenant information", Permission::TenantDelete => "Remove tenants", Permission::MailingListList => "View list of mailing lists", Permission::MailingListGet => "Retrieve specific mailing list information", Permission::MailingListCreate => "Create new mailing lists", Permission::MailingListUpdate => "Modify mailing list information", Permission::MailingListDelete => "Remove mailing lists", Permission::RoleList => "View list of roles", Permission::RoleGet => "Retrieve specific role information", Permission::RoleCreate => "Create new roles", Permission::RoleUpdate => "Modify role information", Permission::RoleDelete => "Remove roles", Permission::PrincipalList => "View list of principals", Permission::PrincipalGet => "Retrieve specific principal information", Permission::PrincipalCreate => "Create new principals", Permission::PrincipalUpdate => "Modify principal information", Permission::PrincipalDelete => "Remove principals", Permission::BlobFetch => "Retrieve arbitrary blobs", Permission::PurgeBlobStore => "Purge the blob storage", Permission::PurgeDataStore => "Purge the data storage", Permission::PurgeInMemoryStore => "Purge the in-memory storage", Permission::PurgeAccount => "Purge user accounts", Permission::FtsReindex => "Rebuild the full-text search index", Permission::Undelete => "Restore deleted items", Permission::DkimSignatureCreate => "Create DKIM signatures for email authentication", Permission::DkimSignatureGet => "Retrieve DKIM signature information", Permission::SpamFilterUpdate => "Modify spam filter settings", Permission::WebadminUpdate => "Modify web admin interface settings", Permission::LogsView => "Access system logs", Permission::SpamFilterTrain => "Train the spam filter", Permission::SpamFilterTest => "Test the spam filter", Permission::Restart => "Restart the email server", Permission::TracingList => "View stored traces", Permission::TracingGet => "Retrieve specific trace information", Permission::TracingLive => "Perform real-time tracing", Permission::MetricsList => "View stored metrics", Permission::MetricsLive => "View real-time metrics", Permission::Authenticate => "Authenticate", Permission::AuthenticateOauth => "Authenticate via OAuth", Permission::EmailSend => "Send emails", Permission::EmailReceive => "Receive emails", Permission::ManageEncryption => "Manage encryption-at-rest settings", Permission::ManagePasswords => "Manage account passwords", Permission::JmapEmailGet => "Retrieve emails via JMAP", Permission::JmapMailboxGet => "Retrieve mailboxes via JMAP", Permission::JmapThreadGet => "Retrieve email threads via JMAP", Permission::JmapIdentityGet => "Retrieve user identities via JMAP", Permission::JmapEmailSubmissionGet => "Retrieve email submission info via JMAP", Permission::JmapPushSubscriptionGet => "Retrieve push subscriptions via JMAP", Permission::JmapSieveScriptGet => "Retrieve Sieve scripts via JMAP", Permission::JmapVacationResponseGet => "Retrieve vacation responses via JMAP", Permission::JmapPrincipalGet => "Retrieve principal information via JMAP", Permission::JmapQuotaGet => "Retrieve quota information via JMAP", Permission::JmapBlobGet => "Retrieve blobs via JMAP", Permission::JmapEmailSet => "Modify emails via JMAP", Permission::JmapMailboxSet => "Modify mailboxes via JMAP", Permission::JmapIdentitySet => "Modify user identities via JMAP", Permission::JmapEmailSubmissionSet => "Modify email submission settings via JMAP", Permission::JmapPushSubscriptionSet => "Modify push subscriptions via JMAP", Permission::JmapSieveScriptSet => "Modify Sieve scripts via JMAP", Permission::JmapVacationResponseSet => "Modify vacation responses via JMAP", Permission::JmapEmailChanges => "Track email changes via JMAP", Permission::JmapMailboxChanges => "Track mailbox changes via JMAP", Permission::JmapThreadChanges => "Track thread changes via JMAP", Permission::JmapIdentityChanges => "Track identity changes via JMAP", Permission::JmapEmailSubmissionChanges => "Track email submission changes via JMAP", Permission::JmapQuotaChanges => "Track quota changes via JMAP", Permission::JmapEmailCopy => "Copy emails via JMAP", Permission::JmapBlobCopy => "Copy blobs via JMAP", Permission::JmapEmailImport => "Import emails via JMAP", Permission::JmapEmailParse => "Parse emails via JMAP", Permission::JmapEmailQueryChanges => "Track email query changes via JMAP", Permission::JmapMailboxQueryChanges => "Track mailbox query changes via JMAP", Permission::JmapEmailSubmissionQueryChanges => { "Track email submission query changes via JMAP" } Permission::JmapSieveScriptQueryChanges => "Track Sieve script query changes via JMAP", Permission::JmapPrincipalQueryChanges => "Track principal query changes via JMAP", Permission::JmapQuotaQueryChanges => "Track quota query changes via JMAP", Permission::JmapEmailQuery => "Perform email queries via JMAP", Permission::JmapMailboxQuery => "Perform mailbox queries via JMAP", Permission::JmapEmailSubmissionQuery => "Perform email submission queries via JMAP", Permission::JmapSieveScriptQuery => "Perform Sieve script queries via JMAP", Permission::JmapPrincipalQuery => "Perform principal queries via JMAP", Permission::JmapQuotaQuery => "Perform quota queries via JMAP", Permission::JmapSearchSnippet => "Retrieve search snippets via JMAP", Permission::JmapSieveScriptValidate => "Validate Sieve scripts via JMAP", Permission::JmapBlobLookup => "Look up blobs via JMAP", Permission::JmapBlobUpload => "Upload blobs via JMAP", Permission::JmapEcho => "Perform JMAP echo requests", Permission::ImapAuthenticate => "Authenticate via IMAP", Permission::ImapAclGet => "Retrieve ACLs via IMAP", Permission::ImapAclSet => "Set ACLs via IMAP", Permission::ImapMyRights => "Retrieve own rights via IMAP", Permission::ImapListRights => "List rights via IMAP", Permission::ImapAppend => "Append messages via IMAP", Permission::ImapCapability => "Retrieve server capabilities via IMAP", Permission::ImapId => "Retrieve server ID via IMAP", Permission::ImapCopy => "Copy messages via IMAP", Permission::ImapMove => "Move messages via IMAP", Permission::ImapCreate => "Create mailboxes via IMAP", Permission::ImapDelete => "Delete mailboxes or messages via IMAP", Permission::ImapEnable => "Enable IMAP extensions", Permission::ImapExpunge => "Expunge deleted messages via IMAP", Permission::ImapFetch => "Fetch messages or metadata via IMAP", Permission::ImapIdle => "Use IMAP IDLE command", Permission::ImapList => "List mailboxes via IMAP", Permission::ImapLsub => "List subscribed mailboxes via IMAP", Permission::ImapNamespace => "Retrieve namespaces via IMAP", Permission::ImapRename => "Rename mailboxes via IMAP", Permission::ImapSearch => "Search messages via IMAP", Permission::ImapSort => "Sort messages via IMAP", Permission::ImapSelect => "Select mailboxes via IMAP", Permission::ImapExamine => "Examine mailboxes via IMAP", Permission::ImapStatus => "Retrieve mailbox status via IMAP", Permission::ImapStore => "Modify message flags via IMAP", Permission::ImapSubscribe => "Subscribe to mailboxes via IMAP", Permission::ImapThread => "Thread messages via IMAP", Permission::Pop3Authenticate => "Authenticate via POP3", Permission::Pop3List => "List messages via POP3", Permission::Pop3Uidl => "Retrieve unique IDs via POP3", Permission::Pop3Stat => "Retrieve mailbox statistics via POP3", Permission::Pop3Retr => "Retrieve messages via POP3", Permission::Pop3Dele => "Mark messages for deletion via POP3", Permission::SieveAuthenticate => "Authenticate for Sieve script management", Permission::SieveListScripts => "List Sieve scripts", Permission::SieveSetActive => "Set active Sieve script", Permission::SieveGetScript => "Retrieve Sieve scripts", Permission::SievePutScript => "Upload Sieve scripts", Permission::SieveDeleteScript => "Delete Sieve scripts", Permission::SieveRenameScript => "Rename Sieve scripts", Permission::SieveCheckScript => "Validate Sieve scripts", Permission::SieveHaveSpace => "Check available space for Sieve scripts", Permission::OauthClientRegistration => "Register OAuth clients", Permission::OauthClientOverride => "Override OAuth client settings", Permission::ApiKeyList => "View API keys", Permission::ApiKeyGet => "Retrieve specific API keys", Permission::ApiKeyCreate => "Create new API keys", Permission::ApiKeyUpdate => "Modify API keys", Permission::ApiKeyDelete => "Remove API keys", Permission::OauthClientList => "View OAuth clients", Permission::OauthClientGet => "Retrieve specific OAuth clients", Permission::OauthClientCreate => "Create new OAuth clients", Permission::OauthClientUpdate => "Modify OAuth clients", Permission::OauthClientDelete => "Remove OAuth clients", Permission::AiModelInteract => "Interact with AI models", Permission::Troubleshoot => "Perform troubleshooting", Permission::DavSyncCollection => "Synchronize collection changes with client", Permission::DavPrincipalAcl => "Set principal properties for access control", Permission::DavPrincipalMatch => "Match principals based on specified criteria", Permission::DavPrincipalSearch => "Search for principals by property values", Permission::DavPrincipalSearchPropSet => "Define property sets for principal searches", Permission::DavExpandProperty => "Expand properties that reference other resources", Permission::DavPrincipalList => "List available principals in the system", Permission::DavFilePropFind => "Retrieve properties of file resources", Permission::DavFilePropPatch => "Modify properties of file resources", Permission::DavFileGet => "Download file resources", Permission::DavFileMkCol => "Create new file collections or directories", Permission::DavFileDelete => "Remove file resources", Permission::DavFilePut => "Upload or modify file resources", Permission::DavFileCopy => "Copy file resources to new locations", Permission::DavFileMove => "Move file resources to new locations", Permission::DavFileLock => "Lock file resources to prevent concurrent modifications", Permission::DavFileAcl => "Manage access control lists for file resources", Permission::DavCardPropFind => "Retrieve properties of address book entries", Permission::DavCardPropPatch => "Modify properties of address book entries", Permission::DavCardGet => "Download address book entries", Permission::DavCardMkCol => "Create new address book collections", Permission::DavCardDelete => "Remove address book entries or collections", Permission::DavCardPut => "Upload or modify address book entries", Permission::DavCardCopy => "Copy address book entries to new locations", Permission::DavCardMove => "Move address book entries to new locations", Permission::DavCardLock => { "Lock address book entries to prevent concurrent modifications" } Permission::DavCardAcl => "Manage access control lists for address book entries", Permission::DavCardQuery => "Search for address book entries matching criteria", Permission::DavCardMultiGet => { "Retrieve multiple address book entries in a single request" } Permission::DavCalPropFind => "Retrieve properties of calendar entries", Permission::DavCalPropPatch => "Modify properties of calendar entries", Permission::DavCalGet => "Download calendar entries", Permission::DavCalMkCol => "Create new calendar collections", Permission::DavCalDelete => "Remove calendar entries or collections", Permission::DavCalPut => "Upload or modify calendar entries", Permission::DavCalCopy => "Copy calendar entries to new locations", Permission::DavCalMove => "Move calendar entries to new locations", Permission::DavCalLock => "Lock calendar entries to prevent concurrent modifications", Permission::DavCalAcl => "Manage access control lists for calendar entries", Permission::DavCalQuery => "Search for calendar entries matching criteria", Permission::DavCalMultiGet => "Retrieve multiple calendar entries in a single request", Permission::DavCalFreeBusyQuery => "Query free/busy time information for scheduling", Permission::CalendarAlarms => "Receive calendar alarms via e-mail", Permission::CalendarSchedulingSend => "Send calendar scheduling requests via e-mail", Permission::CalendarSchedulingReceive => { "Receive calendar scheduling requests via e-mail" } Permission::JmapAddressBookGet => "Retrieve address books via JMAP", Permission::JmapAddressBookSet => "Create or update address books via JMAP", Permission::JmapAddressBookChanges => "Track address book changes via JMAP", Permission::JmapContactCardGet => "Retrieve contact cards via JMAP", Permission::JmapContactCardChanges => "Track contact card changes via JMAP", Permission::JmapContactCardQuery => { "Search for contact cards matching criteria via JMAP" } Permission::JmapContactCardQueryChanges => "Track contact card query changes via JMAP", Permission::JmapContactCardSet => "Create or update contact cards via JMAP", Permission::JmapContactCardCopy => "Copy contact cards to new locations via JMAP", Permission::JmapContactCardParse => "Parse contact cards via JMAP", Permission::JmapFileNodeGet => "Retrieve file nodes via JMAP", Permission::JmapFileNodeSet => "Create or update file nodes via JMAP", Permission::JmapFileNodeChanges => "Track file node changes via JMAP", Permission::JmapFileNodeQuery => "Search for file nodes matching criteria via JMAP", Permission::JmapFileNodeQueryChanges => "Track file node query changes via JMAP", Permission::JmapPrincipalGetAvailability => { "Retrieve availability information via JMAP" } Permission::JmapPrincipalChanges => "Track principal changes via JMAP", Permission::JmapShareNotificationGet => "Retrieve share notifications via JMAP", Permission::JmapShareNotificationSet => "Create or update share notifications via JMAP", Permission::JmapShareNotificationChanges => "Track share notification changes via JMAP", Permission::JmapShareNotificationQuery => { "Search for share notifications matching criteria via JMAP" } Permission::JmapShareNotificationQueryChanges => { "Track share notification query changes via JMAP" } Permission::JmapCalendarGet => "Retrieve calendars via JMAP", Permission::JmapCalendarSet => "Create or update calendars via JMAP", Permission::JmapCalendarChanges => "Track calendar changes via JMAP", Permission::JmapCalendarEventGet => "Retrieve calendar events via JMAP", Permission::JmapCalendarEventSet => "Create or update calendar events via JMAP", Permission::JmapCalendarEventChanges => "Track calendar event changes via JMAP", Permission::JmapCalendarEventQuery => { "Search for calendar events matching criteria via JMAP" } Permission::JmapCalendarEventQueryChanges => { "Track calendar event query changes via JMAP" } Permission::JmapCalendarEventCopy => "Copy calendar events to new locations via JMAP", Permission::JmapCalendarEventParse => "Parse calendar events via JMAP", Permission::JmapCalendarEventNotificationGet => { "Retrieve calendar event notifications via JMAP" } Permission::JmapCalendarEventNotificationSet => { "Create or update calendar event notifications via JMAP" } Permission::JmapCalendarEventNotificationChanges => { "Track calendar event notification changes via JMAP" } Permission::JmapCalendarEventNotificationQuery => { "Search for calendar event notifications matching criteria via JMAP" } Permission::JmapCalendarEventNotificationQueryChanges => { "Track calendar event notification query changes via JMAP" } Permission::JmapParticipantIdentityGet => { "Retrieve participant identity information via JMAP" } Permission::JmapParticipantIdentitySet => { "Create or update participant identities via JMAP" } Permission::JmapParticipantIdentityChanges => { "Track participant identity changes via JMAP" } } } } #[cfg(test)] mod test { use crate::Permission; #[test] #[ignore] #[allow(clippy::obfuscated_if_else)] fn print_permissions() { const CHECK: &str = ":white_check_mark:"; let mut permissions = Permission::all().collect::>(); permissions.sort_by(|a, b| a.name().cmp(b.name())); for permission in permissions { println!( "|`{}`|{}|{}|{}|{}|", permission.name(), permission.description(), CHECK, permission .is_tenant_admin_permission() .then_some(CHECK) .unwrap_or_default(), permission .is_user_permission() .then_some(CHECK) .unwrap_or_default() ); //println!("({:?},{:?}),", permission.name(), permission.description(),); } } } ================================================ FILE: crates/directory/src/core/principal.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ ArchivedPrincipal, ArchivedPrincipalData, FALLBACK_ADMIN_ID, Permission, PermissionGrant, Principal, PrincipalData, ROLE_ADMIN, Type, backend::internal::{PrincipalField, PrincipalSet, PrincipalUpdate, PrincipalValue}, }; use ahash::AHashSet; use nlp::tokenizers::word::WordTokenizer; use serde::{ Deserializer, Serializer, de::{self, IgnoredAny, Visitor}, ser::SerializeMap, }; use serde_json::Value; use std::{cmp::Ordering, collections::hash_map::Entry, fmt, str::FromStr}; use store::{ U32_LEN, U64_LEN, backend::MAX_TOKEN_LENGTH, write::{BatchBuilder, DirectoryClass}, }; impl Principal { pub fn new(id: u32, typ: Type) -> Self { Self { id, typ, name: "".into(), data: Default::default(), } } pub fn id(&self) -> u32 { self.id } pub fn typ(&self) -> Type { self.typ } pub fn name(&self) -> &str { self.name.as_str() } pub fn quota(&self) -> Option { self.data.iter().find_map(|d| { if let PrincipalData::DiskQuota(quota) = d { if *quota > 0 { Some(*quota) } else { None } } else { None } }) } pub fn directory_quota(&self, typ: &Type) -> Option { self.data.iter().find_map(|d| { if let PrincipalData::DirectoryQuota { quota, typ: qtyp } = d && qtyp == typ { Some(*quota) } else { None } }) } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] pub fn tenant(&self) -> Option { self.data.iter().find_map(|item| { if let PrincipalData::Tenant(tenant) = item { Some(*tenant) } else { None } }) } // SPDX-SnippetEnd #[cfg(not(feature = "enterprise"))] pub fn tenant(&self) -> Option { None } pub fn description(&self) -> Option<&str> { self.data.iter().find_map(|item| { if let PrincipalData::Description(description) = item { if !description.is_empty() { Some(description.as_str()) } else { None } } else { None } }) } pub fn secret(&self) -> Option<&str> { if let Some(PrincipalData::Password(password)) = self.data.first() { Some(password.as_str()) } else if let Some(PrincipalData::Password(password)) = self.data.get(1) { Some(password.as_str()) } else { None } } pub fn primary_email(&self) -> Option<&str> { self.data.iter().find_map(|item| { if let PrincipalData::PrimaryEmail(email) = item { Some(email.as_str()) } else { None } }) } pub fn email_addresses(&self) -> impl Iterator { let mut found_email = false; self.data .iter() .take_while(move |item| { if matches!( item, PrincipalData::PrimaryEmail(_) | PrincipalData::EmailAlias(_) ) { found_email = true; true } else { !found_email } }) .filter_map(|item| { if let PrincipalData::PrimaryEmail(email) | PrincipalData::EmailAlias(email) = item { Some(email.as_str()) } else { None } }) } pub fn into_primary_email(self) -> Option { self.data.into_iter().find_map(|item| { if let PrincipalData::PrimaryEmail(email) = item { Some(email) } else { None } }) } pub fn into_email_addresses(self) -> impl Iterator { self.data.into_iter().filter_map(|item| { if let PrincipalData::PrimaryEmail(email) | PrincipalData::EmailAlias(email) = item { Some(email) } else { None } }) } pub fn member_of(&self) -> impl Iterator { self.data.iter().filter_map(|item| { if let PrincipalData::MemberOf(item) = item { Some(*item) } else { None } }) } pub fn roles(&self) -> impl Iterator { self.data.iter().filter_map(|item| { if let PrincipalData::Role(item) = item { Some(*item) } else { None } }) } pub fn permissions(&self) -> impl Iterator { self.data.iter().filter_map(|item| { if let PrincipalData::Permission { permission_id, grant, } = item { Permission::from_id(*permission_id).map(|permission| PermissionGrant { permission, grant: *grant, }) } else { None } }) } pub fn urls(&self) -> impl Iterator { self.data.iter().filter_map(|item| { if let PrincipalData::Url(item) = item { Some(item) } else { None } }) } pub fn lists(&self) -> impl Iterator { self.data.iter().filter_map(|item| { if let PrincipalData::List(item) = item { Some(item) } else { None } }) } pub fn picture(&self) -> Option<&String> { self.data.iter().find_map(|item| { if let PrincipalData::Picture(picture) = item { picture.into() } else { None } }) } pub fn picture_mut(&mut self) -> Option<&mut String> { self.data.iter_mut().find_map(|item| { if let PrincipalData::Picture(picture) = item { picture.into() } else { None } }) } pub fn add_permission(&mut self, permission: Permission, grant: bool) { let permission = permission.id(); if let Some(permissions) = self.data.iter_mut().find_map(|item| { if let PrincipalData::Permission { permission_id, grant, } = item { if *permission_id == permission { Some(grant) } else { None } } else { None } }) { *permissions = grant; } else { self.data.push(PrincipalData::Permission { permission_id: permission, grant, }); } } pub fn add_permissions(&mut self, iter: impl Iterator) { for grant in iter { self.add_permission(grant.permission, grant.grant); } } pub fn remove_permission(&mut self, permission: Permission, grant: bool) { let permission = permission.id(); self.data.retain(|data| { if let PrincipalData::Permission { permission_id: p, grant: g, } = data { *p != permission || *g != grant } else { true } }); } pub fn remove_permissions(&mut self, grant: bool) { self.data.retain(|data| { if let PrincipalData::Permission { grant: g, .. } = data { *g != grant } else { true } }); } pub fn update_external(&mut self, external: Principal) -> Vec { let mut updates = Vec::new(); let mut external_data = AHashSet::with_capacity(external.data.len()); let mut has_role = false; let mut has_member_of = false; let mut has_quota = false; let mut has_otp_auth = false; let mut has_app_password = false; for item in external.data { match item { PrincipalData::DiskQuota(_) => { has_quota = true; external_data.insert(item); } PrincipalData::MemberOf(_) => { has_member_of = true; external_data.insert(item); } PrincipalData::Role(_) => { has_role = true; external_data.insert(item); } PrincipalData::OtpAuth(_) => { has_otp_auth = true; external_data.insert(item); } PrincipalData::AppPassword(_) => { has_app_password = true; external_data.insert(item); } PrincipalData::Password(_) | PrincipalData::Description(_) | PrincipalData::PrimaryEmail(_) | PrincipalData::EmailAlias(_) => { external_data.insert(item); } _ => {} } } let mut old_data = Vec::new(); let data_len = self.data.len(); for item in std::mem::replace(&mut self.data, Vec::with_capacity(data_len)) { match item { PrincipalData::Password(_) | PrincipalData::AppPassword(_) | PrincipalData::OtpAuth(_) | PrincipalData::Description(_) | PrincipalData::PrimaryEmail(_) | PrincipalData::EmailAlias(_) | PrincipalData::DiskQuota(_) | PrincipalData::MemberOf(_) | PrincipalData::Role(_) => { if external_data.remove(&item) || match item { PrincipalData::EmailAlias(_) => true, PrincipalData::AppPassword(_) => !has_app_password, PrincipalData::OtpAuth(_) => !has_otp_auth, PrincipalData::Role(_) => !has_role, PrincipalData::MemberOf(_) => !has_member_of, PrincipalData::DiskQuota(_) => !has_quota, _ => false, } { self.data.push(item); } else if matches!( item, PrincipalData::Password(_) | PrincipalData::AppPassword(_) | PrincipalData::OtpAuth(_) | PrincipalData::PrimaryEmail(_) | PrincipalData::EmailAlias(_) ) { old_data.push(item); } } _ => { self.data.push(item); } } } // Add new data let mut has_password = false; let mut has_email = false; for item in external_data { match &item { PrincipalData::Description(value) => { updates.push(PrincipalUpdate::set( PrincipalField::Description, PrincipalValue::String(value.to_string()), )); } PrincipalData::DiskQuota(value) => { updates.push(PrincipalUpdate::set( PrincipalField::Quota, PrincipalValue::Integer(*value), )); } PrincipalData::Password(value) | PrincipalData::AppPassword(value) | PrincipalData::OtpAuth(value) => { let item = PrincipalUpdate::add_item( PrincipalField::Secrets, PrincipalValue::String(value.to_string()), ); if !has_password && !updates.is_empty() { updates.insert(0, item); } else { updates.push(item); } has_password = true; } PrincipalData::PrimaryEmail(value) => { let item = PrincipalUpdate::add_item( PrincipalField::Emails, PrincipalValue::String(value.to_string()), ); if !has_email && !updates.is_empty() { updates.insert(0, item); } else { updates.push(item); } has_email = true; } PrincipalData::EmailAlias(value) => { updates.push(PrincipalUpdate::add_item( PrincipalField::Emails, PrincipalValue::String(value.to_string()), )); } _ => (), } self.data.push(item); } // Remove old data for item in old_data { match item { PrincipalData::Password(value) | PrincipalData::AppPassword(value) | PrincipalData::OtpAuth(value) => { updates.push(PrincipalUpdate::remove_item( PrincipalField::Secrets, PrincipalValue::String(value), )); } PrincipalData::PrimaryEmail(value) | PrincipalData::EmailAlias(value) => { updates.push(PrincipalUpdate::remove_item( PrincipalField::Emails, PrincipalValue::String(value), )); } _ => (), } } self.sort(); updates } pub fn object_size(&self) -> usize { self.name.len() + self .data .iter() .map(|item| item.object_size()) .sum::() } pub fn fallback_admin(fallback_pass: impl Into) -> Self { Principal { id: FALLBACK_ADMIN_ID, typ: Type::Individual, name: "Fallback Administrator".into(), data: vec![ PrincipalData::Role(ROLE_ADMIN), PrincipalData::Password(fallback_pass.into()), ], } } pub fn sort(&mut self) { self.data.sort_unstable(); } } impl PrincipalData { fn rank(&self) -> u8 { match self { PrincipalData::OtpAuth(_) => 0, PrincipalData::Password(_) => 1, PrincipalData::AppPassword(_) => 2, PrincipalData::PrimaryEmail(_) => 3, PrincipalData::EmailAlias(_) => 4, _ => 5, } } fn rank_string(&self) -> Option<&str> { match self { PrincipalData::OtpAuth(s) | PrincipalData::Password(s) | PrincipalData::AppPassword(s) | PrincipalData::PrimaryEmail(s) | PrincipalData::EmailAlias(s) => Some(s), _ => None, } } } impl PartialOrd for PrincipalData { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } impl Ord for PrincipalData { fn cmp(&self, other: &Self) -> Ordering { match self.rank().cmp(&other.rank()) { Ordering::Equal => match (self.rank_string(), other.rank_string()) { (Some(a), Some(b)) => a.cmp(b), _ => Ordering::Equal, }, other => other, } } } impl PrincipalData { pub fn object_size(&self) -> usize { match self { PrincipalData::Password(v) | PrincipalData::AppPassword(v) | PrincipalData::OtpAuth(v) | PrincipalData::Description(v) | PrincipalData::PrimaryEmail(v) | PrincipalData::EmailAlias(v) | PrincipalData::Picture(v) | PrincipalData::ExternalMember(v) | PrincipalData::Url(v) | PrincipalData::Locale(v) => v.len(), PrincipalData::DiskQuota(_) => U64_LEN, PrincipalData::Permission { .. } => U32_LEN + 1, PrincipalData::DirectoryQuota { .. } | PrincipalData::ObjectQuota { .. } => U64_LEN + 1, PrincipalData::Tenant(_) | PrincipalData::MemberOf(_) | PrincipalData::Role(_) | PrincipalData::List(_) => U32_LEN, } } } impl PrincipalSet { pub fn new(id: u32, typ: Type) -> Self { Self { id, typ, ..Default::default() } } pub fn id(&self) -> u32 { self.id } pub fn typ(&self) -> Type { self.typ } pub fn name(&self) -> &str { self.get_str(PrincipalField::Name).unwrap_or_default() } pub fn has_name(&self) -> bool { self.fields.contains_key(&PrincipalField::Name) } pub fn quota(&self) -> u64 { self.get_int(PrincipalField::Quota).unwrap_or_default() } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] pub fn tenant(&self) -> Option { self.get_int(PrincipalField::Tenant).map(|v| v as u32) } // SPDX-SnippetEnd #[cfg(not(feature = "enterprise"))] pub fn tenant(&self) -> Option { None } pub fn description(&self) -> Option<&str> { self.get_str(PrincipalField::Description) } pub fn get_str(&self, key: PrincipalField) -> Option<&str> { self.fields.get(&key).and_then(|v| v.as_str()) } pub fn get_int(&self, key: PrincipalField) -> Option { self.fields.get(&key).and_then(|v| v.as_int()) } pub fn get_str_array(&self, key: PrincipalField) -> Option<&[String]> { self.fields.get(&key).and_then(|v| match v { PrincipalValue::StringList(v) => Some(v.as_slice()), PrincipalValue::String(v) => Some(std::slice::from_ref(v)), PrincipalValue::Integer(_) | PrincipalValue::IntegerList(_) => None, }) } pub fn get_int_array(&self, key: PrincipalField) -> Option<&[u64]> { self.fields.get(&key).and_then(|v| match v { PrincipalValue::IntegerList(v) => Some(v.as_slice()), PrincipalValue::Integer(v) => Some(std::slice::from_ref(v)), PrincipalValue::String(_) | PrincipalValue::StringList(_) => None, }) } pub fn take(&mut self, key: PrincipalField) -> Option { self.fields.remove(&key) } pub fn take_str(&mut self, key: PrincipalField) -> Option { self.take(key).and_then(|v| match v { PrincipalValue::String(s) => Some(s), PrincipalValue::StringList(l) => l.into_iter().next(), PrincipalValue::Integer(i) => Some(i.to_string()), PrincipalValue::IntegerList(l) => l.into_iter().next().map(|i| i.to_string()), }) } pub fn take_int(&mut self, key: PrincipalField) -> Option { self.take(key).and_then(|v| match v { PrincipalValue::Integer(i) => Some(i), PrincipalValue::IntegerList(l) => l.into_iter().next(), PrincipalValue::String(s) => s.parse().ok(), PrincipalValue::StringList(l) => l.into_iter().next().and_then(|s| s.parse().ok()), }) } pub fn take_str_array(&mut self, key: PrincipalField) -> Option> { self.take(key).map(|v| v.into_str_array()) } pub fn take_int_array(&mut self, key: PrincipalField) -> Option> { self.take(key).map(|v| v.into_int_array()) } pub fn iter_str( &self, key: PrincipalField, ) -> Box + Sync + Send + '_> { self.fields .get(&key) .map(|v| v.iter_str()) .unwrap_or_else(|| Box::new(std::iter::empty())) } pub fn iter_mut_str( &mut self, key: PrincipalField, ) -> Box + Sync + Send + '_> { self.fields .get_mut(&key) .map(|v| v.iter_mut_str()) .unwrap_or_else(|| Box::new(std::iter::empty())) } pub fn iter_int( &self, key: PrincipalField, ) -> Box + Sync + Send + '_> { self.fields .get(&key) .map(|v| v.iter_int()) .unwrap_or_else(|| Box::new(std::iter::empty())) } pub fn iter_mut_int( &mut self, key: PrincipalField, ) -> Box + Sync + Send + '_> { self.fields .get_mut(&key) .map(|v| v.iter_mut_int()) .unwrap_or_else(|| Box::new(std::iter::empty())) } pub fn append_int(&mut self, key: PrincipalField, value: impl Into) -> &mut Self { let value = value.into(); match self.fields.entry(key) { Entry::Occupied(v) => { let v = v.into_mut(); match v { PrincipalValue::IntegerList(v) => { if !v.contains(&value) { v.push(value); } } PrincipalValue::Integer(i) => { if value != *i { *v = PrincipalValue::IntegerList(vec![*i, value]); } } PrincipalValue::String(s) => { *v = PrincipalValue::IntegerList(vec![s.parse().unwrap_or_default(), value]); } PrincipalValue::StringList(l) => { *v = PrincipalValue::IntegerList( l.iter() .map(|s| s.parse().unwrap_or_default()) .chain(std::iter::once(value)) .collect(), ); } } } Entry::Vacant(v) => { v.insert(PrincipalValue::IntegerList(vec![value])); } } self } pub fn append_str(&mut self, key: PrincipalField, value: impl Into) -> &mut Self { let value = value.into(); match self.fields.entry(key) { Entry::Occupied(v) => { let v = v.into_mut(); match v { PrincipalValue::StringList(v) => { if !v.contains(&value) { v.push(value); } } PrincipalValue::String(s) => { if s != &value { *v = PrincipalValue::StringList(vec![std::mem::take(s), value]); } } PrincipalValue::Integer(i) => { *v = PrincipalValue::StringList(vec![i.to_string(), value]); } PrincipalValue::IntegerList(l) => { *v = PrincipalValue::StringList( l.iter() .map(|i| i.to_string()) .chain(std::iter::once(value)) .collect(), ); } } } Entry::Vacant(v) => { v.insert(PrincipalValue::StringList(vec![value])); } } self } pub fn prepend_str(&mut self, key: PrincipalField, value: impl Into) -> &mut Self { let value = value.into(); match self.fields.entry(key) { Entry::Occupied(v) => { let v = v.into_mut(); match v { PrincipalValue::StringList(v) => { if !v.contains(&value) { v.insert(0, value); } } PrincipalValue::String(s) => { if s != &value { *v = PrincipalValue::StringList(vec![value, std::mem::take(s)]); } } PrincipalValue::Integer(i) => { *v = PrincipalValue::StringList(vec![value, i.to_string()]); } PrincipalValue::IntegerList(l) => { *v = PrincipalValue::StringList( std::iter::once(value) .chain(l.iter().map(|i| i.to_string())) .collect(), ); } } } Entry::Vacant(v) => { v.insert(PrincipalValue::StringList(vec![value])); } } self } pub fn set(&mut self, key: PrincipalField, value: impl Into) -> &mut Self { self.fields.insert(key, value.into()); self } pub fn with_field(mut self, key: PrincipalField, value: impl Into) -> Self { self.set(key, value); self } pub fn with_opt_field( mut self, key: PrincipalField, value: Option>, ) -> Self { if let Some(value) = value { self.set(key, value); } self } pub fn has_field(&self, key: PrincipalField) -> bool { self.fields.contains_key(&key) } pub fn has_str_value(&self, key: PrincipalField, value: &str) -> bool { self.fields.get(&key).is_some_and(|v| match v { PrincipalValue::String(v) => v == value, PrincipalValue::StringList(l) => l.iter().any(|v| v == value), PrincipalValue::Integer(_) | PrincipalValue::IntegerList(_) => false, }) } pub fn has_int_value(&self, key: PrincipalField, value: u64) -> bool { self.fields.get(&key).is_some_and(|v| match v { PrincipalValue::Integer(v) => *v == value, PrincipalValue::IntegerList(l) => l.contains(&value), PrincipalValue::String(_) | PrincipalValue::StringList(_) => false, }) } pub fn find_str(&self, value: &str) -> bool { self.fields.values().any(|v| v.find_str(value)) } pub fn field_len(&self, key: PrincipalField) -> usize { self.fields.get(&key).map_or(0, |v| match v { PrincipalValue::String(_) => 1, PrincipalValue::StringList(l) => l.len(), PrincipalValue::Integer(_) => 1, PrincipalValue::IntegerList(l) => l.len(), }) } pub fn remove(&mut self, key: PrincipalField) -> Option { self.fields.remove(&key) } pub fn retain_str(&mut self, key: PrincipalField, mut f: F) where F: FnMut(&String) -> bool, { if let Some(value) = self.fields.get_mut(&key) { match value { PrincipalValue::String(s) => { if !f(s) { self.fields.remove(&key); } } PrincipalValue::StringList(l) => { l.retain(f); if l.is_empty() { self.fields.remove(&key); } } _ => {} } } } pub fn retain_int(&mut self, key: PrincipalField, mut f: F) where F: FnMut(&u64) -> bool, { if let Some(value) = self.fields.get_mut(&key) { match value { PrincipalValue::Integer(i) => { if !f(i) { self.fields.remove(&key); } } PrincipalValue::IntegerList(l) => { l.retain(f); if l.is_empty() { self.fields.remove(&key); } } _ => {} } } } } impl PrincipalValue { pub fn as_str(&self) -> Option<&str> { match self { PrincipalValue::String(v) => Some(v.as_str()), PrincipalValue::StringList(v) => v.first().map(|s| s.as_str()), _ => None, } } pub fn as_int(&self) -> Option { match self { PrincipalValue::Integer(v) => Some(*v), PrincipalValue::IntegerList(v) => v.first().copied(), _ => None, } } pub fn iter_str(&self) -> Box + Sync + Send + '_> { match self { PrincipalValue::String(v) => Box::new(std::iter::once(v)), PrincipalValue::StringList(v) => Box::new(v.iter()), _ => Box::new(std::iter::empty()), } } pub fn iter_mut_str(&mut self) -> Box + Sync + Send + '_> { match self { PrincipalValue::String(v) => Box::new(std::iter::once(v)), PrincipalValue::StringList(v) => Box::new(v.iter_mut()), _ => Box::new(std::iter::empty()), } } pub fn iter_int(&self) -> Box + Sync + Send + '_> { match self { PrincipalValue::Integer(v) => Box::new(std::iter::once(*v)), PrincipalValue::IntegerList(v) => Box::new(v.iter().copied()), _ => Box::new(std::iter::empty()), } } pub fn iter_mut_int(&mut self) -> Box + Sync + Send + '_> { match self { PrincipalValue::Integer(v) => Box::new(std::iter::once(v)), PrincipalValue::IntegerList(v) => Box::new(v.iter_mut()), _ => Box::new(std::iter::empty()), } } pub fn into_array(self) -> Self { match self { PrincipalValue::String(v) => PrincipalValue::StringList(vec![v]), PrincipalValue::Integer(v) => PrincipalValue::IntegerList(vec![v]), v => v, } } pub fn into_str_array(self) -> Vec { match self { PrincipalValue::StringList(v) => v, PrincipalValue::String(v) => vec![v], PrincipalValue::Integer(v) => vec![v.to_string()], PrincipalValue::IntegerList(v) => v.into_iter().map(|v| v.to_string()).collect(), } } pub fn into_int_array(self) -> Vec { match self { PrincipalValue::IntegerList(v) => v, PrincipalValue::Integer(v) => vec![v], PrincipalValue::String(v) => vec![v.parse().unwrap_or_default()], PrincipalValue::StringList(v) => v .into_iter() .map(|v| v.parse().unwrap_or_default()) .collect(), } } pub fn serialized_size(&self) -> usize { match self { PrincipalValue::String(s) => s.len() + 2, PrincipalValue::StringList(s) => s.iter().map(|s| s.len() + 2).sum(), PrincipalValue::Integer(_) => U64_LEN, PrincipalValue::IntegerList(l) => l.len() * U64_LEN, } } pub fn find_str(&self, value: &str) -> bool { match self { PrincipalValue::String(s) => s.to_lowercase().contains(value), PrincipalValue::StringList(l) => l.iter().any(|s| s.to_lowercase().contains(value)), _ => false, } } } impl From for PrincipalValue { fn from(v: u64) -> Self { Self::Integer(v) } } impl From for PrincipalValue { fn from(v: String) -> Self { Self::String(v) } } impl From<&str> for PrincipalValue { fn from(v: &str) -> Self { Self::String(v.into()) } } impl From> for PrincipalValue { fn from(v: Vec) -> Self { Self::StringList(v) } } impl From> for PrincipalValue { fn from(v: Vec) -> Self { Self::IntegerList(v) } } impl From for PrincipalValue { fn from(v: u32) -> Self { Self::Integer(v as u64) } } impl From> for PrincipalValue { fn from(v: Vec) -> Self { Self::IntegerList(v.into_iter().map(|v| v as u64).collect()) } } pub(crate) fn build_search_index( batch: &mut BatchBuilder, principal_id: u32, current: Option<&ArchivedPrincipal>, new: Option<&Principal>, ) { let mut current_words = AHashSet::new(); let mut new_words = AHashSet::new(); if let Some(current) = current { for word in [Some(current.name.as_str())] .into_iter() .chain(current.data.iter().map(|s| match s { ArchivedPrincipalData::Description(v) | ArchivedPrincipalData::PrimaryEmail(v) | ArchivedPrincipalData::EmailAlias(v) => Some(v.as_str()), _ => None, })) .flatten() { current_words.extend(WordTokenizer::new(word, MAX_TOKEN_LENGTH).map(|t| t.word)); } } if let Some(new) = new { for word in [Some(new.name.as_str())] .into_iter() .chain(new.data.iter().map(|s| match s { PrincipalData::Description(v) | PrincipalData::PrimaryEmail(v) | PrincipalData::EmailAlias(v) => Some(v.as_str()), _ => None, })) .flatten() { new_words.extend(WordTokenizer::new(word, MAX_TOKEN_LENGTH).map(|t| t.word)); } } for word in new_words.difference(¤t_words) { batch.set( DirectoryClass::Index { word: word.as_bytes().to_vec(), principal_id, }, vec![], ); } for word in current_words.difference(&new_words) { batch.clear(DirectoryClass::Index { word: word.as_bytes().to_vec(), principal_id, }); } } impl Type { pub fn as_str(&self) -> &'static str { match self { Self::Individual => "individual", Self::Group => "group", Self::Resource => "resource", Self::Location => "location", Self::Other => "other", Self::List => "list", Self::Tenant => "tenant", Self::Role => "role", Self::Domain => "domain", Self::ApiKey => "apiKey", Self::OauthClient => "oauthClient", } } pub fn description(&self) -> &'static str { match self { Self::Individual => "Individual", Self::Group => "Group", Self::Resource => "Resource", Self::Location => "Location", Self::Tenant => "Tenant", Self::List => "List", Self::Other => "Other", Self::Role => "Role", Self::Domain => "Domain", Self::ApiKey => "API Key", Self::OauthClient => "OAuth Client", } } pub fn parse(value: &str) -> Option { match value { "individual" => Some(Type::Individual), "group" => Some(Type::Group), "resource" => Some(Type::Resource), "location" => Some(Type::Location), "list" => Some(Type::List), "tenant" => Some(Type::Tenant), "superuser" => Some(Type::Individual), // legacy "role" => Some(Type::Role), "domain" => Some(Type::Domain), "apiKey" => Some(Type::ApiKey), "oauthClient" => Some(Type::OauthClient), _ => None, } } pub const MAX_ID: usize = 11; pub fn from_u8(value: u8) -> Self { match value { 0 => Type::Individual, 1 => Type::Group, 2 => Type::Resource, 3 => Type::Location, 4 => Type::Other, // legacy 5 => Type::List, 6 => Type::Other, 7 => Type::Domain, 8 => Type::Tenant, 9 => Type::Role, 10 => Type::ApiKey, 11 => Type::OauthClient, _ => Type::Other, } } } impl FromStr for Type { type Err = (); fn from_str(s: &str) -> Result { Type::parse(s).ok_or(()) } } impl serde::Serialize for PrincipalSet { fn serialize(&self, serializer: S) -> Result where S: Serializer, { let mut map = serializer.serialize_map(None)?; map.serialize_entry("id", &self.id)?; map.serialize_entry("type", &self.typ.as_str())?; for (key, value) in &self.fields { match value { PrincipalValue::String(v) => map.serialize_entry(key.as_str(), v)?, PrincipalValue::StringList(v) => map.serialize_entry(key.as_str(), v)?, PrincipalValue::Integer(v) => map.serialize_entry(key.as_str(), v)?, PrincipalValue::IntegerList(v) => map.serialize_entry(key.as_str(), v)?, }; } map.end() } } const MAX_STRING_LEN: usize = 512; impl<'de> serde::Deserialize<'de> for PrincipalValue { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { struct PrincipalValueVisitor; impl<'de> Visitor<'de> for PrincipalValueVisitor { type Value = PrincipalValue; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("an optional values or a sequence of values") } fn visit_none(self) -> Result where E: de::Error, { Ok(PrincipalValue::String("".into())) } fn visit_some(self, deserializer: D) -> Result where D: Deserializer<'de>, { deserializer.deserialize_any(self) } fn visit_u64(self, value: u64) -> Result where E: de::Error, { Ok(PrincipalValue::Integer(value)) } fn visit_string(self, value: String) -> Result where E: de::Error, { if value.len() <= MAX_STRING_LEN { Ok(PrincipalValue::String(value)) } else { Err(serde::de::Error::custom("string too long")) } } fn visit_str(self, value: &str) -> Result where E: de::Error, { if value.len() <= MAX_STRING_LEN { Ok(PrincipalValue::String(value.into())) } else { Err(serde::de::Error::custom("string too long")) } } fn visit_seq(self, mut seq: A) -> Result where A: de::SeqAccess<'de>, { let mut vec_u64 = Vec::new(); let mut vec_string = Vec::new(); while let Some(value) = seq.next_element::()? { match value { StringOrU64::String(s) => { if s.len() <= MAX_STRING_LEN { vec_string.push(s); } else { return Err(serde::de::Error::custom("string too long")); } } StringOrU64::U64(u) => vec_u64.push(u), } } match (vec_u64.is_empty(), vec_string.is_empty()) { (true, false) => Ok(PrincipalValue::StringList(vec_string)), (false, true) => Ok(PrincipalValue::IntegerList(vec_u64)), (true, true) => Ok(PrincipalValue::StringList(vec_string)), _ => Err(serde::de::Error::custom("invalid principal value")), } } } deserializer.deserialize_any(PrincipalValueVisitor) } } impl<'de> serde::Deserialize<'de> for PrincipalSet { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { struct PrincipalVisitor; // Deserialize the principal impl<'de> Visitor<'de> for PrincipalVisitor { type Value = PrincipalSet; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("a valid principal") } fn visit_map(self, mut map: A) -> Result where A: de::MapAccess<'de>, { let mut principal = PrincipalSet::default(); while let Some(key) = map.next_key::<&str>()? { if key == "id" { // Ignored map.next_value::()?; continue; } let key = PrincipalField::try_parse(key).ok_or_else(|| { serde::de::Error::custom(format!("invalid principal field: {}", key)) })?; let value = match key { PrincipalField::Name => { PrincipalValue::String(map.next_value::().and_then(|v| { if v.len() <= MAX_STRING_LEN { Ok(v) } else { Err(serde::de::Error::custom("string too long")) } })?) } PrincipalField::Description | PrincipalField::Tenant | PrincipalField::Picture | PrincipalField::Locale => { if let Some(v) = map.next_value::>()? { if v.len() <= MAX_STRING_LEN { PrincipalValue::String(v) } else { return Err(serde::de::Error::custom("string too long")); } } else { continue; } } PrincipalField::Type => { principal.typ = Type::parse(map.next_value()?).ok_or_else(|| { serde::de::Error::custom("invalid principal type") })?; continue; } PrincipalField::Quota => map.next_value::()?, PrincipalField::Secrets | PrincipalField::Emails | PrincipalField::MemberOf | PrincipalField::Members | PrincipalField::Roles | PrincipalField::Lists | PrincipalField::EnabledPermissions | PrincipalField::DisabledPermissions | PrincipalField::Urls | PrincipalField::ExternalMembers => match map.next_value::()? { Value::String(v) => { if v.len() <= MAX_STRING_LEN { PrincipalValue::StringList(vec![v]) } else { return Err(serde::de::Error::custom("string too long")); } } Value::Array(v) => { if !v.is_empty() { PrincipalValue::StringList( v.into_iter() .filter_map(|item| { if let Value::String(s) = item { if s.len() <= MAX_STRING_LEN { Some(s) } else { None } } else { None } }) .collect(), ) } else { continue; } } _ => continue, }, PrincipalField::UsedQuota => { // consume and ignore map.next_value::()?; continue; } }; principal.fields.insert(key, value); } Ok(principal) } } deserializer.deserialize_map(PrincipalVisitor) } } #[derive(Debug)] enum StringOrU64 { String(String), U64(u64), } impl<'de> serde::Deserialize<'de> for StringOrU64 { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { struct StringOrU64Visitor; impl Visitor<'_> for StringOrU64Visitor { type Value = StringOrU64; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("a string or u64") } fn visit_str(self, value: &str) -> Result where E: de::Error, { if value.len() <= MAX_STRING_LEN { Ok(StringOrU64::String(value.to_string())) } else { Err(serde::de::Error::custom("string too long")) } } fn visit_string(self, v: String) -> Result where E: de::Error, { if v.len() <= MAX_STRING_LEN { Ok(StringOrU64::String(v)) } else { Err(serde::de::Error::custom("string too long")) } } fn visit_u64(self, value: u64) -> Result where E: de::Error, { Ok(StringOrU64::U64(value)) } } deserializer.deserialize_any(StringOrU64Visitor) } } impl Permission { pub fn all() -> impl Iterator { (0..Permission::COUNT as u32).filter_map(Permission::from_id) } pub const fn is_user_permission(&self) -> bool { matches!( self, Permission::Authenticate | Permission::AuthenticateOauth | Permission::EmailSend | Permission::EmailReceive | Permission::ManageEncryption | Permission::ManagePasswords | Permission::JmapEmailGet | Permission::JmapMailboxGet | Permission::JmapThreadGet | Permission::JmapIdentityGet | Permission::JmapEmailSubmissionGet | Permission::JmapPushSubscriptionGet | Permission::JmapSieveScriptGet | Permission::JmapVacationResponseGet | Permission::JmapQuotaGet | Permission::JmapBlobGet | Permission::JmapEmailSet | Permission::JmapMailboxSet | Permission::JmapIdentitySet | Permission::JmapEmailSubmissionSet | Permission::JmapPushSubscriptionSet | Permission::JmapSieveScriptSet | Permission::JmapVacationResponseSet | Permission::JmapEmailChanges | Permission::JmapMailboxChanges | Permission::JmapThreadChanges | Permission::JmapIdentityChanges | Permission::JmapEmailSubmissionChanges | Permission::JmapQuotaChanges | Permission::JmapEmailCopy | Permission::JmapBlobCopy | Permission::JmapEmailImport | Permission::JmapEmailParse | Permission::JmapEmailQueryChanges | Permission::JmapMailboxQueryChanges | Permission::JmapEmailSubmissionQueryChanges | Permission::JmapSieveScriptQueryChanges | Permission::JmapQuotaQueryChanges | Permission::JmapEmailQuery | Permission::JmapMailboxQuery | Permission::JmapEmailSubmissionQuery | Permission::JmapSieveScriptQuery | Permission::JmapQuotaQuery | Permission::JmapSearchSnippet | Permission::JmapSieveScriptValidate | Permission::JmapBlobLookup | Permission::JmapBlobUpload | Permission::JmapEcho | Permission::ImapAuthenticate | Permission::ImapAclGet | Permission::ImapAclSet | Permission::ImapMyRights | Permission::ImapListRights | Permission::ImapAppend | Permission::ImapCapability | Permission::ImapId | Permission::ImapCopy | Permission::ImapMove | Permission::ImapCreate | Permission::ImapDelete | Permission::ImapEnable | Permission::ImapExpunge | Permission::ImapFetch | Permission::ImapIdle | Permission::ImapList | Permission::ImapLsub | Permission::ImapNamespace | Permission::ImapRename | Permission::ImapSearch | Permission::ImapSort | Permission::ImapSelect | Permission::ImapExamine | Permission::ImapStatus | Permission::ImapStore | Permission::ImapSubscribe | Permission::ImapThread | Permission::Pop3Authenticate | Permission::Pop3List | Permission::Pop3Uidl | Permission::Pop3Stat | Permission::Pop3Retr | Permission::Pop3Dele | Permission::SieveAuthenticate | Permission::SieveListScripts | Permission::SieveSetActive | Permission::SieveGetScript | Permission::SievePutScript | Permission::SieveDeleteScript | Permission::SieveRenameScript | Permission::SieveCheckScript | Permission::SieveHaveSpace | Permission::DavSyncCollection | Permission::DavExpandProperty | Permission::DavPrincipalAcl | Permission::DavPrincipalList | Permission::DavPrincipalSearch | Permission::DavPrincipalMatch | Permission::DavPrincipalSearchPropSet | Permission::DavFilePropFind | Permission::DavFilePropPatch | Permission::DavFileGet | Permission::DavFileMkCol | Permission::DavFileDelete | Permission::DavFilePut | Permission::DavFileCopy | Permission::DavFileMove | Permission::DavFileLock | Permission::DavFileAcl | Permission::DavCardPropFind | Permission::DavCardPropPatch | Permission::DavCardGet | Permission::DavCardMkCol | Permission::DavCardDelete | Permission::DavCardPut | Permission::DavCardCopy | Permission::DavCardMove | Permission::DavCardLock | Permission::DavCardAcl | Permission::DavCardQuery | Permission::DavCardMultiGet | Permission::DavCalPropFind | Permission::DavCalPropPatch | Permission::DavCalGet | Permission::DavCalMkCol | Permission::DavCalDelete | Permission::DavCalPut | Permission::DavCalCopy | Permission::DavCalMove | Permission::DavCalLock | Permission::DavCalAcl | Permission::DavCalQuery | Permission::DavCalMultiGet | Permission::DavCalFreeBusyQuery | Permission::CalendarAlarms | Permission::CalendarSchedulingSend | Permission::CalendarSchedulingReceive | Permission::JmapAddressBookGet | Permission::JmapAddressBookSet | Permission::JmapAddressBookChanges | Permission::JmapContactCardGet | Permission::JmapContactCardChanges | Permission::JmapContactCardQuery | Permission::JmapContactCardQueryChanges | Permission::JmapContactCardSet | Permission::JmapContactCardCopy | Permission::JmapContactCardParse | Permission::JmapFileNodeGet | Permission::JmapFileNodeSet | Permission::JmapFileNodeChanges | Permission::JmapFileNodeQuery | Permission::JmapFileNodeQueryChanges | Permission::JmapPrincipalGetAvailability | Permission::JmapPrincipalChanges | Permission::JmapPrincipalQuery | Permission::JmapPrincipalGet | Permission::JmapPrincipalQueryChanges | Permission::JmapShareNotificationGet | Permission::JmapShareNotificationSet | Permission::JmapShareNotificationChanges | Permission::JmapShareNotificationQuery | Permission::JmapShareNotificationQueryChanges | Permission::JmapCalendarGet | Permission::JmapCalendarSet | Permission::JmapCalendarChanges | Permission::JmapCalendarEventGet | Permission::JmapCalendarEventSet | Permission::JmapCalendarEventChanges | Permission::JmapCalendarEventQuery | Permission::JmapCalendarEventQueryChanges | Permission::JmapCalendarEventCopy | Permission::JmapCalendarEventParse | Permission::JmapCalendarEventNotificationGet | Permission::JmapCalendarEventNotificationSet | Permission::JmapCalendarEventNotificationChanges | Permission::JmapCalendarEventNotificationQuery | Permission::JmapCalendarEventNotificationQueryChanges | Permission::JmapParticipantIdentityGet | Permission::JmapParticipantIdentitySet | Permission::JmapParticipantIdentityChanges ) } #[cfg(not(feature = "enterprise"))] pub const fn is_tenant_admin_permission(&self) -> bool { false } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] pub const fn is_tenant_admin_permission(&self) -> bool { matches!( self, Permission::MessageQueueList | Permission::MessageQueueGet | Permission::MessageQueueUpdate | Permission::MessageQueueDelete | Permission::OutgoingReportList | Permission::OutgoingReportGet | Permission::OutgoingReportDelete | Permission::IncomingReportList | Permission::IncomingReportGet | Permission::IncomingReportDelete | Permission::IndividualList | Permission::IndividualGet | Permission::IndividualUpdate | Permission::IndividualDelete | Permission::IndividualCreate | Permission::GroupList | Permission::GroupGet | Permission::GroupUpdate | Permission::GroupDelete | Permission::GroupCreate | Permission::DomainList | Permission::DomainGet | Permission::DomainCreate | Permission::DomainUpdate | Permission::DomainDelete | Permission::MailingListList | Permission::MailingListGet | Permission::MailingListCreate | Permission::MailingListUpdate | Permission::MailingListDelete | Permission::RoleList | Permission::RoleGet | Permission::RoleCreate | Permission::RoleUpdate | Permission::RoleDelete | Permission::PrincipalList | Permission::PrincipalGet | Permission::PrincipalCreate | Permission::PrincipalUpdate | Permission::PrincipalDelete | Permission::Undelete | Permission::DkimSignatureCreate | Permission::DkimSignatureGet | Permission::ApiKeyList | Permission::ApiKeyGet | Permission::ApiKeyCreate | Permission::ApiKeyUpdate | Permission::ApiKeyDelete | Permission::SpamFilterTrain | Permission::SpamFilterTest ) || self.is_user_permission() } // SPDX-SnippetEnd } ================================================ FILE: crates/directory/src/core/secret.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::Principal; use crate::PrincipalData; use argon2::Argon2; use compact_str::ToCompactString; use mail_builder::encoders::base64::base64_encode; use mail_parser::decoders::base64::base64_decode; use password_hash::PasswordHash; use pbkdf2::Pbkdf2; use pwhash::{bcrypt, bsdi_crypt, md5_crypt, sha1_crypt, sha256_crypt, sha512_crypt, unix_crypt}; use scrypt::Scrypt; use sha1::Digest; use sha1::Sha1; use sha2::Sha256; use sha2::Sha512; use tokio::sync::oneshot; use totp_rs::TOTP; impl Principal { pub async fn verify_secret( &self, code: &str, only_app_pass: bool, is_ordered: bool, ) -> trc::Result { let mut seen_password = false; let mut password = None; let mut otp_auth = None; for item in &self.data { match item { PrincipalData::OtpAuth(secret) => { if !only_app_pass { otp_auth = Some(secret); } seen_password = true; } PrincipalData::Password(secret) => { if !only_app_pass { password = Some(secret); } seen_password = true; } PrincipalData::AppPassword(secret) => { // App passwords do not require TOTP if let Some((_, app_secret)) = secret.strip_prefix("$app$").and_then(|s| s.split_once('$')) && verify_secret_hash(app_secret, code).await? { return Ok(true); } seen_password = true; } _ => { if seen_password && is_ordered { // Password-related secrets are expected to be at the beginning of the list break; } } } } // Validate TOTP match (otp_auth, password) { (Some(otp_auth), Some(password)) => { if let Some((code, totp_token)) = code.rsplit_once('$').filter(|(c, t)| { !c.is_empty() && (6..=8).contains(&t.len()) && t.as_bytes().iter().all(|b| b.is_ascii_digit()) }) { let result = verify_secret_hash(password, code).await? && TOTP::from_url(otp_auth) .map_err(|err| { trc::AuthEvent::Error .reason(err) .details(otp_auth.to_compact_string()) })? .check_current(totp_token) .unwrap_or(false); Ok(result) } else if verify_secret_hash(password, code).await? { // Only let the client know if the TOTP code is missing // if the password is correct Err(trc::AuthEvent::MissingTotp.into_err()) } else { Ok(false) } } (None, Some(password)) => verify_secret_hash(password, code).await, _ => Ok(false), } } } async fn verify_hash_prefix(hashed_secret: &str, secret: &str) -> trc::Result { if hashed_secret.starts_with("$argon2") || hashed_secret.starts_with("$pbkdf2") || hashed_secret.starts_with("$scrypt") { let (tx, rx) = oneshot::channel(); let secret = secret.to_string(); let hashed_secret = hashed_secret.to_string(); tokio::task::spawn_blocking(move || match PasswordHash::new(&hashed_secret) { Ok(hash) => { tx.send(Ok(hash .verify_password(&[&Argon2::default(), &Pbkdf2, &Scrypt], &secret) .is_ok())) .ok(); } Err(err) => { tx.send(Err(trc::AuthEvent::Error .reason(err) .details(hashed_secret))) .ok(); } }); match rx.await { Ok(result) => result, Err(err) => Err(trc::EventType::Server(trc::ServerEvent::ThreadError) .caused_by(trc::location!()) .reason(err)), } } else if hashed_secret.starts_with("$2") { // Blowfish crypt Ok(bcrypt::verify(secret, hashed_secret)) } else if hashed_secret.starts_with("$6$") { // SHA-512 crypt Ok(sha512_crypt::verify(secret, hashed_secret)) } else if hashed_secret.starts_with("$5$") { // SHA-256 crypt Ok(sha256_crypt::verify(secret, hashed_secret)) } else if hashed_secret.starts_with("$sha1") { // SHA-1 crypt Ok(sha1_crypt::verify(secret, hashed_secret)) } else if hashed_secret.starts_with("$1") { // MD5 based hash Ok(md5_crypt::verify(secret, hashed_secret)) } else { Err(trc::AuthEvent::Error .into_err() .details(hashed_secret.to_string())) } } pub async fn verify_secret_hash(hashed_secret: &str, secret: &str) -> trc::Result { if hashed_secret.starts_with('$') { verify_hash_prefix(hashed_secret, secret).await } else if hashed_secret.starts_with('_') { // Enhanced DES-based hash Ok(bsdi_crypt::verify(secret, hashed_secret)) } else if let Some(hashed_secret) = hashed_secret.strip_prefix('{') { if let Some((algo, hashed_secret)) = hashed_secret.split_once('}') { match algo { "ARGON2" | "ARGON2I" | "ARGON2ID" | "PBKDF2" => { verify_hash_prefix(hashed_secret, secret).await } "SHA" => { // SHA-1 let mut hasher = Sha1::new(); hasher.update(secret.as_bytes()); Ok( String::from_utf8( base64_encode(&hasher.finalize()[..]).unwrap_or_default(), ) .unwrap() == hashed_secret, ) } "SSHA" => { // Salted SHA-1 let decoded = base64_decode(hashed_secret.as_bytes()).unwrap_or_default(); let hash = decoded.get(..20).unwrap_or_default(); let salt = decoded.get(20..).unwrap_or_default(); let mut hasher = Sha1::new(); hasher.update(secret.as_bytes()); hasher.update(salt); Ok(&hasher.finalize()[..] == hash) } "SHA256" => { // Verify hash let mut hasher = Sha256::new(); hasher.update(secret.as_bytes()); Ok( String::from_utf8( base64_encode(&hasher.finalize()[..]).unwrap_or_default(), ) .unwrap() == hashed_secret, ) } "SSHA256" => { // Salted SHA-256 let decoded = base64_decode(hashed_secret.as_bytes()).unwrap_or_default(); let hash = decoded.get(..32).unwrap_or_default(); let salt = decoded.get(32..).unwrap_or_default(); let mut hasher = Sha256::new(); hasher.update(secret.as_bytes()); hasher.update(salt); Ok(&hasher.finalize()[..] == hash) } "SHA512" => { // SHA-512 let mut hasher = Sha512::new(); hasher.update(secret.as_bytes()); Ok( String::from_utf8( base64_encode(&hasher.finalize()[..]).unwrap_or_default(), ) .unwrap() == hashed_secret, ) } "SSHA512" => { // Salted SHA-512 let decoded = base64_decode(hashed_secret.as_bytes()).unwrap_or_default(); let hash = decoded.get(..64).unwrap_or_default(); let salt = decoded.get(64..).unwrap_or_default(); let mut hasher = Sha512::new(); hasher.update(secret.as_bytes()); hasher.update(salt); Ok(&hasher.finalize()[..] == hash) } "MD5" => { // MD5 let digest = md5::compute(secret.as_bytes()); Ok( String::from_utf8(base64_encode(&digest[..]).unwrap_or_default()).unwrap() == hashed_secret, ) } "CRYPT" | "crypt" => { if hashed_secret.starts_with('$') { verify_hash_prefix(hashed_secret, secret).await } else { // Unix crypt Ok(unix_crypt::verify(secret, hashed_secret)) } } "PLAIN" | "plain" | "CLEAR" | "clear" => Ok(hashed_secret == secret), _ => Err(trc::AuthEvent::Error .ctx(trc::Key::Reason, "Unsupported algorithm") .details(hashed_secret.to_string())), } } else { Err(trc::AuthEvent::Error .into_err() .details(hashed_secret.to_string())) } } else if !hashed_secret.is_empty() { Ok(hashed_secret == secret) } else { Ok(false) } } ================================================ FILE: crates/directory/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ #![warn(clippy::large_futures)] use ahash::AHashMap; use backend::{ imap::{ImapDirectory, ImapError}, ldap::LdapDirectory, memory::MemoryDirectory, smtp::SmtpDirectory, sql::SqlDirectory, }; use core::cache::CachedDirectory; use deadpool::managed::PoolError; use ldap3::LdapError; use mail_send::Credentials; use proc_macros::EnumMethods; use std::{fmt::Debug, sync::Arc}; use store::Store; use trc::ipc::bitset::Bitset; use types::collection::Collection; pub mod backend; pub mod core; pub struct Directory { pub store: DirectoryInner, pub cache: Option, } pub const FALLBACK_ADMIN_ID: u32 = u32::MAX; #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] pub struct Principal { pub id: u32, pub typ: Type, pub name: String, pub data: Vec, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq, Hash)] pub enum PrincipalData { Password(String), // Permissions and memberships Tenant(u32), MemberOf(u32), Role(u32), List(u32), Permission { permission_id: u32, grant: bool }, // Quotas DiskQuota(u64), DirectoryQuota { quota: u32, typ: Type }, ObjectQuota { quota: u32, typ: Collection }, // Profile data Description(String), PrimaryEmail(String), EmailAlias(String), Picture(String), ExternalMember(String), Url(String), Locale(String), // Secrets AppPassword(String), OtpAuth(String), } #[derive(Debug, Clone, PartialEq, Eq)] pub struct PermissionGrant { pub permission: Permission, pub grant: bool, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct MemberOf { pub principal_id: u32, pub typ: Type, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, Hash, )] #[serde(rename_all = "camelCase")] pub enum Type { #[default] Individual = 0, Group = 1, Resource = 2, Location = 3, List = 5, Other = 6, Domain = 7, Tenant = 8, Role = 9, ApiKey = 10, OauthClient = 11, } #[derive( Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, EnumMethods, )] #[serde(rename_all = "kebab-case")] pub enum Permission { // WARNING: add new ids at the end (TODO: use static ids) // Admin Impersonate, UnlimitedRequests, UnlimitedUploads, DeleteSystemFolders_, MessageQueueList, MessageQueueGet, MessageQueueUpdate, MessageQueueDelete, OutgoingReportList, OutgoingReportGet, OutgoingReportDelete, IncomingReportList, IncomingReportGet, IncomingReportDelete, SettingsList, SettingsUpdate, SettingsDelete, SettingsReload, IndividualList, IndividualGet, IndividualUpdate, IndividualDelete, IndividualCreate, GroupList, GroupGet, GroupUpdate, GroupDelete, GroupCreate, DomainList, DomainGet, DomainCreate, DomainUpdate, DomainDelete, TenantList, TenantGet, TenantCreate, TenantUpdate, TenantDelete, MailingListList, MailingListGet, MailingListCreate, MailingListUpdate, MailingListDelete, RoleList, RoleGet, RoleCreate, RoleUpdate, RoleDelete, PrincipalList, PrincipalGet, PrincipalCreate, PrincipalUpdate, PrincipalDelete, BlobFetch, PurgeBlobStore, PurgeDataStore, PurgeInMemoryStore, PurgeAccount, FtsReindex, Undelete, DkimSignatureCreate, DkimSignatureGet, SpamFilterUpdate, WebadminUpdate, LogsView, SpamFilterTrain, Restart, TracingList, TracingGet, TracingLive, MetricsList, MetricsLive, // Generic Authenticate, AuthenticateOauth, EmailSend, EmailReceive, // Account Management ManageEncryption, ManagePasswords, // JMAP JmapEmailGet, JmapMailboxGet, JmapThreadGet, JmapIdentityGet, JmapEmailSubmissionGet, JmapPushSubscriptionGet, JmapSieveScriptGet, JmapVacationResponseGet, JmapPrincipalGet, JmapQuotaGet, JmapBlobGet, JmapEmailSet, JmapMailboxSet, JmapIdentitySet, JmapEmailSubmissionSet, JmapPushSubscriptionSet, JmapSieveScriptSet, JmapVacationResponseSet, JmapEmailChanges, JmapMailboxChanges, JmapThreadChanges, JmapIdentityChanges, JmapEmailSubmissionChanges, JmapQuotaChanges, JmapEmailCopy, JmapBlobCopy, JmapEmailImport, JmapEmailParse, JmapEmailQueryChanges, JmapMailboxQueryChanges, JmapEmailSubmissionQueryChanges, JmapSieveScriptQueryChanges, JmapPrincipalQueryChanges, JmapQuotaQueryChanges, JmapEmailQuery, JmapMailboxQuery, JmapEmailSubmissionQuery, JmapSieveScriptQuery, JmapPrincipalQuery, JmapQuotaQuery, JmapSearchSnippet, JmapSieveScriptValidate, JmapBlobLookup, JmapBlobUpload, JmapEcho, // IMAP ImapAuthenticate, ImapAclGet, ImapAclSet, ImapMyRights, ImapListRights, ImapAppend, ImapCapability, ImapId, ImapCopy, ImapMove, ImapCreate, ImapDelete, ImapEnable, ImapExpunge, ImapFetch, ImapIdle, ImapList, ImapLsub, ImapNamespace, ImapRename, ImapSearch, ImapSort, ImapSelect, ImapExamine, ImapStatus, ImapStore, ImapSubscribe, ImapThread, // POP3 Pop3Authenticate, Pop3List, Pop3Uidl, Pop3Stat, Pop3Retr, Pop3Dele, // ManageSieve SieveAuthenticate, SieveListScripts, SieveSetActive, SieveGetScript, SievePutScript, SieveDeleteScript, SieveRenameScript, SieveCheckScript, SieveHaveSpace, // API keys ApiKeyList, ApiKeyGet, ApiKeyCreate, ApiKeyUpdate, ApiKeyDelete, // OAuth clients OauthClientList, OauthClientGet, OauthClientCreate, OauthClientUpdate, OauthClientDelete, // OAuth client registration OauthClientRegistration, OauthClientOverride, AiModelInteract, Troubleshoot, SpamFilterTest, // WebDAV permissions DavSyncCollection, DavExpandProperty, DavPrincipalAcl, DavPrincipalList, DavPrincipalMatch, DavPrincipalSearch, DavPrincipalSearchPropSet, DavFilePropFind, DavFilePropPatch, DavFileGet, DavFileMkCol, DavFileDelete, DavFilePut, DavFileCopy, DavFileMove, DavFileLock, DavFileAcl, DavCardPropFind, DavCardPropPatch, DavCardGet, DavCardMkCol, DavCardDelete, DavCardPut, DavCardCopy, DavCardMove, DavCardLock, DavCardAcl, DavCardQuery, DavCardMultiGet, DavCalPropFind, DavCalPropPatch, DavCalGet, DavCalMkCol, DavCalDelete, DavCalPut, DavCalCopy, DavCalMove, DavCalLock, DavCalAcl, DavCalQuery, DavCalMultiGet, DavCalFreeBusyQuery, CalendarAlarms, CalendarSchedulingSend, CalendarSchedulingReceive, JmapAddressBookGet, JmapAddressBookSet, JmapAddressBookChanges, JmapContactCardGet, JmapContactCardChanges, JmapContactCardQuery, JmapContactCardQueryChanges, JmapContactCardSet, JmapContactCardCopy, JmapContactCardParse, JmapFileNodeGet, JmapFileNodeSet, JmapFileNodeChanges, JmapFileNodeQuery, JmapFileNodeQueryChanges, JmapPrincipalGetAvailability, JmapPrincipalChanges, JmapShareNotificationGet, JmapShareNotificationSet, JmapShareNotificationChanges, JmapShareNotificationQuery, JmapShareNotificationQueryChanges, JmapCalendarGet, JmapCalendarSet, JmapCalendarChanges, JmapCalendarEventGet, JmapCalendarEventSet, JmapCalendarEventChanges, JmapCalendarEventQuery, JmapCalendarEventQueryChanges, JmapCalendarEventCopy, JmapCalendarEventParse, JmapCalendarEventNotificationGet, JmapCalendarEventNotificationSet, JmapCalendarEventNotificationChanges, JmapCalendarEventNotificationQuery, JmapCalendarEventNotificationQueryChanges, JmapParticipantIdentityGet, JmapParticipantIdentitySet, JmapParticipantIdentityChanges, // TODO: Reuse _ suffixes for new permissions // WARNING: add new ids at the end (TODO: use static ids) } pub const PERMISSIONS_BITSET_SIZE: usize = Permission::COUNT.div_ceil(std::mem::size_of::()); pub type Permissions = Bitset; pub const ROLE_ADMIN: u32 = u32::MAX; pub const ROLE_TENANT_ADMIN: u32 = u32::MAX - 1; pub const ROLE_USER: u32 = u32::MAX - 2; pub enum DirectoryInner { Internal(Store), Ldap(LdapDirectory), Sql(SqlDirectory), OpenId(backend::oidc::OpenIdDirectory), Imap(ImapDirectory), Smtp(SmtpDirectory), Memory(MemoryDirectory), } pub enum QueryBy<'x> { Name(&'x str), Id(u32), Credentials(&'x Credentials), } pub struct QueryParams<'x> { pub by: QueryBy<'x>, pub return_member_of: bool, pub only_app_pass: bool, } impl Default for Directory { fn default() -> Self { Self { store: DirectoryInner::Internal(Store::None), cache: None, } } } impl Debug for Directory { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Directory").finish() } } #[derive(Default, Clone, Debug)] pub struct Directories { pub directories: AHashMap>, } trait IntoError { fn into_error(self) -> trc::Error; } impl IntoError for PoolError { fn into_error(self) -> trc::Error { match self { PoolError::Backend(error) => error.into_error(), PoolError::Timeout(_) => trc::StoreEvent::PoolError .into_err() .details("Connection timed out"), err => trc::StoreEvent::PoolError.reason(err), } } } impl IntoError for PoolError { fn into_error(self) -> trc::Error { match self { PoolError::Backend(error) => error.into_error(), PoolError::Timeout(_) => trc::StoreEvent::PoolError .into_err() .details("Connection timed out"), err => trc::StoreEvent::PoolError.reason(err), } } } impl IntoError for PoolError { fn into_error(self) -> trc::Error { match self { PoolError::Backend(error) => error.into_error(), PoolError::Timeout(_) => trc::StoreEvent::PoolError .into_err() .details("Connection timed out"), err => trc::StoreEvent::PoolError.reason(err), } } } impl IntoError for ImapError { fn into_error(self) -> trc::Error { trc::ImapEvent::Error.into_err().reason(self) } } impl IntoError for mail_send::Error { fn into_error(self) -> trc::Error { trc::SmtpEvent::Error.into_err().reason(self) } } impl IntoError for LdapError { fn into_error(self) -> trc::Error { if let LdapError::LdapResult { result } = &self { trc::StoreEvent::LdapError .ctx(trc::Key::Code, result.rc) .reason(self) } else { trc::StoreEvent::LdapError.reason(self) } } } impl From<&ArchivedType> for Type { fn from(archived: &ArchivedType) -> Self { match archived { ArchivedType::Individual => Type::Individual, ArchivedType::Group => Type::Group, ArchivedType::Resource => Type::Resource, ArchivedType::Location => Type::Location, ArchivedType::List => Type::List, ArchivedType::Other => Type::Other, ArchivedType::Domain => Type::Domain, ArchivedType::Tenant => Type::Tenant, ArchivedType::Role => Type::Role, ArchivedType::ApiKey => Type::ApiKey, ArchivedType::OauthClient => Type::OauthClient, } } } impl<'x> QueryParams<'x> { pub fn name(name: &'x str) -> Self { QueryParams { by: QueryBy::Name(name), return_member_of: false, only_app_pass: false, } } pub fn credentials(credentials: &'x Credentials) -> Self { QueryParams { by: QueryBy::Credentials(credentials), return_member_of: false, only_app_pass: false, } } pub fn id(id: u32) -> Self { QueryParams { by: QueryBy::Id(id), return_member_of: false, only_app_pass: false, } } pub fn by(by: QueryBy<'x>) -> Self { QueryParams { by, return_member_of: false, only_app_pass: false, } } pub fn with_return_member_of(mut self, return_member_of: bool) -> Self { self.return_member_of = return_member_of; self } pub fn with_only_app_pass(mut self, only_app_pass: bool) -> Self { self.only_app_pass = only_app_pass; self } } ================================================ FILE: crates/email/Cargo.toml ================================================ [package] name = "email" version = "0.15.5" edition = "2024" [dependencies] utils = { path = "../utils" } nlp = { path = "../nlp" } store = { path = "../store" } trc = { path = "../trc" } types = { path = "../types" } common = { path = "../common" } directory = { path = "../directory" } groupware = { path = "../groupware" } spam-filter = { path = "../spam-filter" } smtp-proto = { version = "0.2", features = ["rkyv"] } mail-parser = { version = "0.11", features = ["full_encoding"] } mail-builder = { version = "0.4" } sieve-rs = { version = "0.7", features = ["rkyv"] } tokio = { version = "1.47", features = ["net", "macros"] } serde = { version = "1.0", features = ["derive"]} serde_json = "1.0" aes = "0.8.3" aes-gcm = "0.10.1" aes-gcm-siv = "0.11.1" cbc = { version = "0.1.2", features = ["alloc"] } rasn = "0.10" rasn-cms = "0.10" rasn-pkix = "0.10" rsa = "0.9.2" rand = "0.8" sequoia-openpgp = { version = "2.0", default-features = false, features = ["crypto-rust", "allow-experimental-crypto", "allow-variable-time-crypto"] } hashify = "0.2" rkyv = { version = "0.8.10", features = ["little_endian"] } compact_str = "0.9.0" [features] test_mode = [] enterprise = [] [dev-dependencies] tokio = { version = "1.47", features = ["full"] } ================================================ FILE: crates/email/src/cache/email.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::message::metadata::{ArchivedMessageData, MessageData}; use common::{ MessageCache, MessageStoreCache, MessageUidCache, MessagesCache, Server, auth::AccessToken, sharing::EffectiveAcl, }; use store::write::{AlignedBytes, Archive}; use store::{ValueKey, ahash::AHashMap, roaring::RoaringBitmap}; use trc::AddContext; use types::{ acl::Acl, collection::Collection, keyword::{Keyword, OTHER}, }; use utils::map::bitmap::Bitmap; struct MessagesCacheBuilder { pub change_id: u64, pub items: Vec, pub index: AHashMap, pub keywords: Vec>, pub size: u64, } pub(crate) async fn update_email_cache( server: &Server, account_id: u32, changed_ids: &AHashMap, store_cache: &MessageStoreCache, ) -> trc::Result { let mut new_cache = MessagesCacheBuilder { index: AHashMap::with_capacity(store_cache.emails.items.len()), items: Vec::with_capacity(store_cache.emails.items.len()), size: 0, change_id: 0, keywords: store_cache.emails.keywords.to_vec(), }; for (document_id, is_update) in changed_ids { if *is_update && let Some(archive) = server .store() .get_value::>(ValueKey::archive( account_id, Collection::Email, *document_id, )) .await .caused_by(trc::location!())? { insert_item( &mut new_cache, *document_id, archive.to_unarchived::()?, ); } } for item in &store_cache.emails.items { if !changed_ids.contains_key(&item.document_id) { email_insert(&mut new_cache, item.clone()); } } Ok(new_cache.build()) } pub(crate) async fn full_email_cache_build( server: &Server, account_id: u32, ) -> trc::Result { // Build cache let mut cache = MessagesCacheBuilder { items: Vec::with_capacity(16), index: AHashMap::with_capacity(16), keywords: Vec::new(), size: 0, change_id: 0, }; server .archives( account_id, Collection::Email, &(), |document_id, archive| { insert_item( &mut cache, document_id, archive.to_unarchived::()?, ); Ok(true) }, ) .await .caused_by(trc::location!())?; Ok(cache.build()) } fn insert_item( cache: &mut MessagesCacheBuilder, document_id: u32, archive: Archive<&ArchivedMessageData>, ) { let message = archive.inner; let mut item = MessageCache { mailboxes: message .mailboxes .iter() .map(|m| MessageUidCache { mailbox_id: m.mailbox_id.to_native(), uid: m.uid.to_native(), }) .collect(), keywords: 0, thread_id: message.thread_id.to_native(), change_id: archive.version.change_id().unwrap_or_default(), document_id, size: message.size.to_native(), }; for keyword in message.keywords.iter() { match keyword.id() { Ok(id) => { item.keywords |= 1 << id; } Err(custom) => { if let Some(idx) = cache.keywords.iter().position(|k| **k == *custom) { item.keywords |= 1 << (OTHER + idx); } else if cache.keywords.len() < (128 - OTHER) { cache.keywords.push(custom.into()); item.keywords |= 1 << (OTHER + cache.keywords.len() - 1); } } } } email_insert(cache, item); } impl MessagesCacheBuilder { pub fn build(mut self) -> MessagesCache { self.index.shrink_to_fit(); MessagesCache { change_id: self.change_id, items: self.items.into_boxed_slice(), index: self.index, keywords: self.keywords.into_boxed_slice(), size: self.size, } } } pub trait MessageCacheAccess { fn email_by_id(&self, id: &u32) -> Option<&MessageCache>; fn has_email_id(&self, id: &u32) -> bool; fn in_mailbox(&self, mailbox_id: u32) -> impl Iterator; fn in_mailboxes(&self, mailbox_ids: &[u32]) -> impl Iterator; fn in_thread(&self, thread_id: u32) -> impl Iterator; fn with_keyword(&self, keyword: &Keyword) -> impl Iterator; fn without_keyword(&self, keyword: &Keyword) -> impl Iterator; fn in_mailbox_with_keyword( &self, mailbox_id: u32, keyword: &Keyword, ) -> impl Iterator; fn in_mailbox_without_keyword( &self, mailbox_id: u32, keyword: &Keyword, ) -> impl Iterator; fn email_document_ids(&self) -> RoaringBitmap; fn shared_messages( &self, access_token: &AccessToken, check_acls: impl Into> + Sync + Send, ) -> RoaringBitmap; fn expand_keywords(&self, message: &MessageCache) -> impl Iterator; fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool; } impl MessageCacheAccess for MessageStoreCache { fn in_mailbox(&self, mailbox_id: u32) -> impl Iterator { self.emails .items .iter() .filter(move |m| m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id)) } fn in_mailboxes(&self, mailbox_ids: &[u32]) -> impl Iterator { self.emails.items.iter().filter(move |m| { m.mailboxes .iter() .any(|mb| mailbox_ids.contains(&mb.mailbox_id)) }) } fn in_thread(&self, thread_id: u32) -> impl Iterator { self.emails .items .iter() .filter(move |m| m.thread_id == thread_id) } fn with_keyword(&self, keyword: &Keyword) -> impl Iterator { let keyword_id = keyword_to_id(self, keyword); self.emails .items .iter() .filter(move |m| keyword_id.is_some_and(|id| m.keywords & (1 << id) != 0)) } fn without_keyword(&self, keyword: &Keyword) -> impl Iterator { let keyword_id = keyword_to_id(self, keyword); self.emails .items .iter() .filter(move |m| keyword_id.is_none_or(|id| m.keywords & (1 << id) == 0)) } fn in_mailbox_with_keyword( &self, mailbox_id: u32, keyword: &Keyword, ) -> impl Iterator { let keyword_id = keyword_to_id(self, keyword); self.emails.items.iter().filter(move |m| { m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) && keyword_id.is_some_and(|id| m.keywords & (1 << id) != 0) }) } fn in_mailbox_without_keyword( &self, mailbox_id: u32, keyword: &Keyword, ) -> impl Iterator { let keyword_id = keyword_to_id(self, keyword); self.emails.items.iter().filter(move |m| { m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) && keyword_id.is_none_or(|id| m.keywords & (1 << id) == 0) }) } fn shared_messages( &self, access_token: &AccessToken, check_acls: impl Into> + Sync + Send, ) -> RoaringBitmap { let check_acls = check_acls.into(); let mut shared_messages = RoaringBitmap::new(); for mailbox in &self.mailboxes.items { if mailbox .acls .as_slice() .effective_acl(access_token) .contains_all(check_acls) { shared_messages.extend( self.in_mailbox(mailbox.document_id) .map(|item| item.document_id), ); } } shared_messages } fn email_document_ids(&self) -> RoaringBitmap { RoaringBitmap::from_iter(self.emails.index.keys()) } fn email_by_id(&self, id: &u32) -> Option<&MessageCache> { self.emails .index .get(id) .and_then(|idx| self.emails.items.get(*idx as usize)) } fn has_email_id(&self, id: &u32) -> bool { self.emails.index.contains_key(id) } fn expand_keywords(&self, message: &MessageCache) -> impl Iterator { KeywordsIter(message.keywords).map(move |id| match Keyword::try_from_id(id) { Ok(keyword) => keyword, Err(id) => Keyword::Other(self.emails.keywords[id - OTHER].clone()), }) } fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool { keyword_to_id(self, keyword).is_some_and(|id| message.keywords & (1 << id) != 0) } } fn email_insert(cache: &mut MessagesCacheBuilder, item: MessageCache) { let id = item.document_id; if let Some(idx) = cache.index.get(&id) { cache.items[*idx as usize] = item; } else { cache.size += (std::mem::size_of::() + (std::mem::size_of::() * 2) + (item.mailboxes.len() * std::mem::size_of::())) as u64; let idx = cache.items.len() as u32; cache.items.push(item); cache.index.insert(id, idx); } } #[inline] fn keyword_to_id(cache: &MessageStoreCache, keyword: &Keyword) -> Option { match keyword.id() { Ok(id) => Some(id), Err(name) => cache .emails .keywords .iter() .position(|k| **k == *name) .map(|idx| (OTHER + idx) as u32), } } #[derive(Clone, Copy, Debug)] struct KeywordsIter(u128); impl Iterator for KeywordsIter { type Item = usize; fn next(&mut self) -> Option { if self.0 != 0 { let item = 127 - self.0.leading_zeros(); self.0 ^= 1 << item; Some(item as usize) } else { None } } } ================================================ FILE: crates/email/src/cache/mailbox.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::mailbox::{ArchivedMailbox, Mailbox, manage::MailboxFnc}; use common::{ MailboxCache, MailboxesCache, MessageStoreCache, Server, auth::AccessToken, sharing::EffectiveAcl, }; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use store::{ahash::AHashMap, roaring::RoaringBitmap}; use trc::AddContext; use types::{ acl::{Acl, AclGrant}, collection::Collection, special_use::SpecialUse, }; use utils::{map::bitmap::Bitmap, topological::TopologicalSort}; struct MailboxesCacheBuilder { pub change_id: u64, pub index: AHashMap, pub items: Vec, pub size: u64, } pub(crate) async fn update_mailbox_cache( server: &Server, account_id: u32, changed_ids: &AHashMap, store_cache: &MessageStoreCache, ) -> trc::Result { let mut new_cache = MailboxesCacheBuilder { items: Vec::with_capacity(store_cache.mailboxes.items.len()), index: AHashMap::with_capacity(store_cache.mailboxes.items.len()), size: 0, change_id: 0, }; for (document_id, is_update) in changed_ids { if *is_update && let Some(archive) = server .store() .get_value::>(ValueKey::archive( account_id, Collection::Mailbox, *document_id, )) .await .caused_by(trc::location!())? { insert_item( &mut new_cache, *document_id, archive.unarchive::()?, ); } } for item in store_cache.mailboxes.items.iter() { if !changed_ids.contains_key(&item.document_id) { mailbox_insert(&mut new_cache, item.clone()); } } build_tree(&mut new_cache); Ok(new_cache.build()) } pub(crate) async fn full_mailbox_cache_build( server: &Server, account_id: u32, ) -> trc::Result { // Build cache let mut cache = MailboxesCacheBuilder { items: Default::default(), index: Default::default(), size: 0, change_id: 0, }; server .archives( account_id, Collection::Mailbox, &(), |document_id, archive| { insert_item(&mut cache, document_id, archive.unarchive::()?); Ok(true) }, ) .await .caused_by(trc::location!())?; if cache.items.is_empty() { server .create_system_folders(account_id) .await .caused_by(trc::location!())?; server .archives( account_id, Collection::Mailbox, &(), |document_id, archive| { insert_item(&mut cache, document_id, archive.unarchive::()?); Ok(true) }, ) .await .caused_by(trc::location!())?; } build_tree(&mut cache); Ok(cache.build()) } fn insert_item(cache: &mut MailboxesCacheBuilder, document_id: u32, mailbox: &ArchivedMailbox) { let parent_id = mailbox.parent_id.to_native(); let item = MailboxCache { document_id, name: mailbox.name.as_str().into(), path: "".into(), role: (&mailbox.role).into(), parent_id: if parent_id > 0 { parent_id - 1 } else { u32::MAX }, sort_order: mailbox .sort_order .as_ref() .map(|s| s.to_native()) .unwrap_or(u32::MAX), subscribers: mailbox.subscribers.iter().map(|s| s.to_native()).collect(), uid_validity: mailbox.uid_validity.to_native(), acls: mailbox .acls .iter() .map(|acl| AclGrant { account_id: acl.account_id.to_native(), grants: Bitmap::from(&acl.grants), }) .collect(), }; mailbox_insert(cache, item); } fn build_tree(cache: &mut MailboxesCacheBuilder) { cache.size = 0; let mut topological_sort = TopologicalSort::with_capacity(cache.items.len()); for (idx, mailbox) in cache.items.iter_mut().enumerate() { topological_sort.insert( if mailbox.parent_id == u32::MAX { 0 } else { mailbox.parent_id + 1 }, mailbox.document_id + 1, ); mailbox.path = if matches!(mailbox.role, SpecialUse::Inbox) { "INBOX".into() } else if mailbox.is_root() && mailbox.name.as_str().eq_ignore_ascii_case("inbox") { format!("INBOX {}", idx + 1) } else { mailbox.name.clone() }; cache.size += item_size(mailbox); } for folder_id in topological_sort.into_iterator() { if folder_id != 0 { let folder_id = folder_id - 1; if let Some((path, parent_path)) = by_id(cache, &folder_id) .and_then(|folder| { folder .parent_id() .map(|parent_id| (&folder.path, parent_id)) }) .and_then(|(path, parent_id)| { by_id(cache, &parent_id).map(|folder| (path, &folder.path)) }) { let mut new_path = String::with_capacity(parent_path.len() + path.len() + 1); new_path.push_str(parent_path.as_str()); new_path.push('/'); new_path.push_str(path.as_str()); let folder = by_id_mut(cache, &folder_id).unwrap(); folder.path = new_path; } } } } impl MailboxesCacheBuilder { fn build(mut self) -> MailboxesCache { self.index.shrink_to_fit(); MailboxesCache { change_id: self.change_id, index: self.index, items: self.items.into_boxed_slice(), size: self.size, } } } pub trait MailboxCacheAccess { fn mailbox_by_id(&self, id: &u32) -> Option<&MailboxCache>; fn mailbox_by_name(&self, name: &str) -> Option<&MailboxCache>; fn mailbox_by_path(&self, name: &str) -> Option<&MailboxCache>; fn mailbox_by_role(&self, role: &SpecialUse) -> Option<&MailboxCache>; fn shared_mailboxes( &self, access_token: &AccessToken, check_acls: impl Into> + Sync + Send, ) -> RoaringBitmap; fn has_mailbox_id(&self, id: &u32) -> bool; } impl MailboxCacheAccess for MessageStoreCache { fn mailbox_by_name(&self, name: &str) -> Option<&MailboxCache> { self.mailboxes .items .iter() .find(|m| m.name.eq_ignore_ascii_case(name)) } fn mailbox_by_path(&self, path: &str) -> Option<&MailboxCache> { self.mailboxes .items .iter() .find(|m| m.path.eq_ignore_ascii_case(path)) } fn mailbox_by_role(&self, role: &SpecialUse) -> Option<&MailboxCache> { self.mailboxes.items.iter().find(|m| &m.role == role) } fn shared_mailboxes( &self, access_token: &AccessToken, check_acls: impl Into> + Sync + Send, ) -> RoaringBitmap { let check_acls = check_acls.into(); RoaringBitmap::from_iter( self.mailboxes .items .iter() .filter(|m| { m.acls .as_slice() .effective_acl(access_token) .contains_all(check_acls) }) .map(|m| m.document_id), ) } fn mailbox_by_id(&self, id: &u32) -> Option<&MailboxCache> { self.mailboxes .index .get(id) .and_then(|idx| self.mailboxes.items.get(*idx as usize)) } fn has_mailbox_id(&self, id: &u32) -> bool { self.mailboxes.index.contains_key(id) } } #[inline(always)] fn by_id<'x>(cache: &'x MailboxesCacheBuilder, id: &u32) -> Option<&'x MailboxCache> { cache .index .get(id) .and_then(|idx| cache.items.get(*idx as usize)) } #[inline(always)] fn by_id_mut<'x>(cache: &'x mut MailboxesCacheBuilder, id: &u32) -> Option<&'x mut MailboxCache> { cache .index .get(id) .and_then(|idx| cache.items.get_mut(*idx as usize)) } fn mailbox_insert(cache: &mut MailboxesCacheBuilder, item: MailboxCache) { let id = item.document_id; if let Some(idx) = cache.index.get(&id) { cache.items[*idx as usize] = item; } else { let idx = cache.items.len() as u32; cache.items.push(item); cache.index.insert(id, idx); } } #[inline(always)] fn item_size(item: &MailboxCache) -> u64 { (std::mem::size_of::() + (if item.name.len() > std::mem::size_of::() { item.name.len() } else { 0 }) + (if item.path.len() > std::mem::size_of::() { item.path.len() } else { 0 })) as u64 } ================================================ FILE: crates/email/src/cache/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{CacheSwap, MessageStoreCache, Server}; use email::{full_email_cache_build, update_email_cache}; use mailbox::{full_mailbox_cache_build, update_mailbox_cache}; use std::{collections::hash_map::Entry, sync::Arc, time::Instant}; use store::{ ahash::AHashMap, query::log::{Change, Query}, }; use tokio::sync::Semaphore; use trc::{AddContext, StoreEvent}; use types::collection::SyncCollection; pub mod email; pub mod mailbox; pub trait MessageCacheFetch: Sync + Send { fn get_cached_messages( &self, account_id: u32, ) -> impl Future>> + Send; } impl MessageCacheFetch for Server { async fn get_cached_messages(&self, account_id: u32) -> trc::Result> { let cache_ = match self .inner .cache .messages .get_value_or_guard_async(&account_id) .await { Ok(cache) => cache, Err(guard) => { let start_time = Instant::now(); let cache = full_cache_build(self, account_id, Arc::new(Semaphore::new(1))).await?; if guard.insert(CacheSwap::new(cache.clone())).is_err() { self.inner .cache .messages .insert(account_id, CacheSwap::new(cache.clone())); } trc::event!( Store(StoreEvent::CacheMiss), AccountId = account_id, Collection = SyncCollection::Email.as_str(), Total = vec![cache.emails.items.len(), cache.mailboxes.items.len()], ChangeId = cache.last_change_id, Elapsed = start_time.elapsed(), ); return Ok(cache); } }; // Obtain current state let cache = cache_.load_full(); let start_time = Instant::now(); let changes = self .core .storage .data .changes( account_id, SyncCollection::Email.into(), Query::Since(cache.last_change_id), ) .await .caused_by(trc::location!())?; // Regenerate cache if the change log has been truncated if changes.is_truncated { let cache = full_cache_build(self, account_id, cache.update_lock.clone()).await?; cache_.update(cache.clone()); trc::event!( Store(StoreEvent::CacheStale), AccountId = account_id, Collection = SyncCollection::Email.as_str(), ChangeId = cache.last_change_id, Total = vec![cache.emails.items.len(), cache.mailboxes.items.len()], Elapsed = start_time.elapsed(), ); return Ok(cache); } // Verify changes if changes.changes.is_empty() { trc::event!( Store(StoreEvent::CacheHit), AccountId = account_id, Collection = SyncCollection::Email.as_str(), ChangeId = cache.last_change_id, Elapsed = start_time.elapsed(), ); return Ok(cache); } // Lock for updates let _permit = cache.update_lock.acquire().await; let cache = cache_.0.load(); let mut cache = if cache.last_change_id >= changes.to_change_id { trc::event!( Store(StoreEvent::CacheHit), AccountId = account_id, Collection = SyncCollection::Email.as_str(), ChangeId = cache.last_change_id, Elapsed = start_time.elapsed(), ); return Ok(cache.clone()); } else { cache.as_ref().clone() }; let mut changed_items: AHashMap = AHashMap::with_capacity(changes.changes.len()); let mut changed_containers: AHashMap = AHashMap::with_capacity(changes.changes.len()); let mut has_container_property_changes = false; for change in changes.changes { match change { Change::InsertItem(id) => match changed_items.entry(id as u32) { Entry::Occupied(mut entry) => { *entry.get_mut() = true; } Entry::Vacant(entry) => { entry.insert(true); } }, Change::UpdateItem(id) => { changed_items.insert(id as u32, true); } Change::DeleteItem(id) => { match changed_items.entry(id as u32) { Entry::Occupied(mut entry) => { // Thread reassignment *entry.get_mut() = true; } Entry::Vacant(entry) => { entry.insert(false); } } } Change::InsertContainer(id) | Change::UpdateContainer(id) => { changed_containers.insert(id as u32, true); } Change::DeleteContainer(id) => { changed_containers.insert(id as u32, false); } Change::UpdateContainerProperty(_) => { has_container_property_changes = true; } } } if !changed_items.is_empty() { let mut email_cache = update_email_cache(self, account_id, &changed_items, &cache).await?; email_cache.change_id = changes.item_change_id.unwrap_or(changes.to_change_id); cache.emails = Arc::new(email_cache); } if !changed_containers.is_empty() { let mut mailbox_cache = update_mailbox_cache(self, account_id, &changed_containers, &cache).await?; mailbox_cache.change_id = changes.container_change_id.unwrap_or(changes.to_change_id); cache.mailboxes = Arc::new(mailbox_cache); } else if has_container_property_changes { let mut mailbox_cache = cache.mailboxes.as_ref().clone(); mailbox_cache.change_id = changes.container_change_id.unwrap_or(changes.to_change_id); cache.mailboxes = Arc::new(mailbox_cache); } cache.size = cache.emails.size + cache.mailboxes.size; cache.last_change_id = changes.to_change_id; let cache = Arc::new(cache); cache_.update(cache.clone()); trc::event!( Store(StoreEvent::CacheUpdate), AccountId = account_id, Collection = SyncCollection::Email.as_str(), ChangeId = cache.last_change_id, Details = vec![changed_items.len(), changed_containers.len()], Total = vec![cache.emails.items.len(), cache.mailboxes.items.len()], Elapsed = start_time.elapsed(), ); Ok(cache) } } async fn full_cache_build( server: &Server, account_id: u32, update_lock: Arc, ) -> trc::Result> { let last_change_id = server .core .storage .data .get_last_change_id(account_id, SyncCollection::Email.into()) .await .caused_by(trc::location!())? .unwrap_or_default(); let mut emails = full_email_cache_build(server, account_id).await?; let mut mailboxes = full_mailbox_cache_build(server, account_id).await?; let size = emails.size + mailboxes.size; emails.change_id = last_change_id; mailboxes.change_id = last_change_id; Ok(Arc::new(MessageStoreCache { update_lock, emails: Arc::new(emails), mailboxes: Arc::new(mailboxes), last_change_id, size, })) } ================================================ FILE: crates/email/src/identity/index.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ArchivedIdentity, Identity}; use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; use types::collection::SyncCollection; impl IndexableObject for Identity { fn index_values(&self) -> impl Iterator> { [IndexValue::LogItem { sync_collection: SyncCollection::Identity, prefix: None, }] .into_iter() } } impl IndexableObject for &ArchivedIdentity { fn index_values(&self) -> impl Iterator> { [IndexValue::LogItem { sync_collection: SyncCollection::Identity, prefix: None, }] .into_iter() } } impl IndexableAndSerializableObject for Identity { fn is_versioned() -> bool { false } } ================================================ FILE: crates/email/src/identity/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod index; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct Identity { pub name: String, pub email: String, pub reply_to: Option>, pub bcc: Option>, pub text_signature: String, pub html_signature: String, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] pub struct EmailAddress { pub name: Option, pub email: String, } ================================================ FILE: crates/email/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod cache; pub mod identity; pub mod mailbox; pub mod message; pub mod push; pub mod sieve; pub mod submission; ================================================ FILE: crates/email/src/mailbox/destroy.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::*; use crate::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, message::metadata::MessageData, }; use common::{ Server, auth::AccessToken, sharing::EffectiveAcl, storage::index::ObjectIndexBuilder, }; use store::{ SerializeInfallible, roaring::RoaringBitmap, write::{BatchBuilder, SearchIndex, TaskEpoch, TaskQueueClass, ValueClass}, }; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, VanishedCollection}, field::MailboxField, }; pub trait MailboxDestroy: Sync + Send { fn mailbox_destroy( &self, account_id: u32, document_id: u32, access_token: &AccessToken, remove_emails: bool, ) -> impl Future, MailboxDestroyError>>> + Send; } pub enum MailboxDestroyError { CannotDestroy, Forbidden, HasChildren, HasEmails, NotFound, AssertionFailed, } impl MailboxDestroy for Server { async fn mailbox_destroy( &self, account_id: u32, document_id: u32, access_token: &AccessToken, remove_emails: bool, ) -> trc::Result, MailboxDestroyError>> { // Internal folders cannot be deleted #[cfg(not(feature = "test_mode"))] if [INBOX_ID, TRASH_ID, JUNK_ID].contains(&document_id) { return Ok(Err(MailboxDestroyError::CannotDestroy)); } // Verify that this mailbox does not have sub-mailboxes let cache = self .get_cached_messages(account_id) .await .caused_by(trc::location!())?; if cache .mailboxes .items .iter() .any(|item| item.parent_id == document_id) { return Ok(Err(MailboxDestroyError::HasChildren)); } // Verify that the mailbox is empty let mut batch = BatchBuilder::new(); batch.with_account_id(account_id); let message_ids = RoaringBitmap::from_iter(cache.in_mailbox(document_id).map(|m| m.document_id)); if !message_ids.is_empty() { if remove_emails { // If the message is in multiple mailboxes, untag it from the current mailbox, // otherwise delete it. self.archives( account_id, Collection::Email, &message_ids, |message_id, message_data_| { // Remove mailbox from list let prev_message_data = message_data_ .to_unarchived::() .caused_by(trc::location!())?; if !prev_message_data .inner .mailboxes .iter() .any(|id| id.mailbox_id == document_id) { return Ok(true); } if prev_message_data.inner.mailboxes.len() == 1 { // Delete message for mailbox in prev_message_data.inner.mailboxes.iter() { batch.log_vanished_item( VanishedCollection::Email, (mailbox.mailbox_id.to_native(), mailbox.uid.to_native()), ); } batch .with_collection(Collection::Email) .with_document(message_id) .custom( ObjectIndexBuilder::<_, ()>::new() .with_access_token(access_token) .with_current(prev_message_data), ) .caused_by(trc::location!())? .set( ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { index: SearchIndex::Email, due: TaskEpoch::now(), is_insert: false, }), 0u64.serialize(), ) .commit_point(); } else { let new_message_data = MessageData { mailboxes: prev_message_data .inner .mailboxes .iter() .filter(|m| m.mailbox_id != document_id) .map(|m| m.to_native()) .collect(), keywords: prev_message_data .inner .keywords .iter() .map(|k| k.to_native()) .collect(), thread_id: prev_message_data.inner.thread_id.to_native(), size: prev_message_data.inner.size.to_native(), }; // Untag message from mailbox batch .with_collection(Collection::Email) .with_document(message_id) .custom( ObjectIndexBuilder::new() .with_access_token(access_token) .with_changes(new_message_data) .with_current(prev_message_data), ) .caused_by(trc::location!())? .commit_point(); } Ok(true) }, ) .await .caused_by(trc::location!())?; } else { return Ok(Err(MailboxDestroyError::HasEmails)); } } // Obtain mailbox if let Some(mailbox_) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::Mailbox, document_id, )) .await .caused_by(trc::location!())? { let mailbox = mailbox_ .to_unarchived::() .caused_by(trc::location!())?; // Validate ACLs if access_token.is_shared(account_id) { let acl = mailbox.inner.acls.effective_acl(access_token); if !acl.contains(Acl::Delete) || (remove_emails && !acl.contains(Acl::RemoveItems)) { return Ok(Err(MailboxDestroyError::Forbidden)); } } batch .with_account_id(account_id) .with_collection(Collection::Mailbox) .with_document(document_id) .clear(MailboxField::UidCounter) .custom(ObjectIndexBuilder::<_, ()>::new().with_current(mailbox)) .caused_by(trc::location!())?; } else { return Ok(Err(MailboxDestroyError::NotFound)); }; if !batch.is_empty() { match self .commit_batch(batch) .await .and_then(|ids| ids.last_change_id(account_id)) { Ok(change_id) => { self.notify_task_queue(); Ok(Ok(Some(change_id))) } Err(err) if err.is_assertion_failure() => { Ok(Err(MailboxDestroyError::AssertionFailed)) } Err(err) => Err(err.caused_by(trc::location!())), } } else { Ok(Ok(None)) } } } ================================================ FILE: crates/email/src/mailbox/index.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ArchivedMailbox, Mailbox}; use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; use types::{acl::AclGrant, collection::SyncCollection}; impl IndexableObject for Mailbox { fn index_values(&self) -> impl Iterator> { [ IndexValue::LogContainer { sync_collection: SyncCollection::Email, }, IndexValue::Acl { value: (&self.acls).into(), }, ] .into_iter() } } impl IndexableObject for &ArchivedMailbox { fn index_values(&self) -> impl Iterator> { [ IndexValue::LogContainer { sync_collection: SyncCollection::Email, }, IndexValue::Acl { value: self .acls .iter() .map(AclGrant::from) .collect::>() .into(), }, ] .into_iter() } } impl IndexableAndSerializableObject for Mailbox { fn is_versioned() -> bool { false } } ================================================ FILE: crates/email/src/mailbox/manage.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::*; use crate::cache::MessageCacheFetch; use common::{Server, storage::index::ObjectIndexBuilder}; use std::future::Future; use store::write::BatchBuilder; use trc::AddContext; use types::collection::Collection; pub trait MailboxFnc: Sync + Send { fn create_system_folders( &self, account_id: u32, ) -> impl Future> + Send; fn mailbox_create_path( &self, account_id: u32, path: &str, ) -> impl Future>> + Send; } impl MailboxFnc for Server { async fn create_system_folders(&self, account_id: u32) -> trc::Result<()> { #[cfg(feature = "test_mode")] if account_id == 0 { return Ok(()); } let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Mailbox); // Create mailboxes let mut last_document_id = ARCHIVE_ID; for folder in &self.core.jmap.default_folders { let document_id = match folder.special_use { SpecialUse::Inbox => INBOX_ID, SpecialUse::Trash => TRASH_ID, SpecialUse::Junk => JUNK_ID, SpecialUse::Drafts => DRAFTS_ID, SpecialUse::Sent => SENT_ID, SpecialUse::Archive => ARCHIVE_ID, SpecialUse::None | SpecialUse::Important | SpecialUse::Memos | SpecialUse::Scheduled | SpecialUse::Snoozed => { last_document_id += 1; last_document_id } SpecialUse::Shared => unreachable!(), }; let mut object = Mailbox::new(folder.name.clone()).with_role(folder.special_use); if folder.subscribe { object.add_subscriber(account_id); } batch .with_document(document_id) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(object)) .caused_by(trc::location!())?; } self.store() .assign_document_ids(account_id, Collection::Mailbox, (ARCHIVE_ID + 1) as u64) .await .caused_by(trc::location!())?; self.core .storage .data .write(batch.build_all()) .await .caused_by(trc::location!())?; Ok(()) } async fn mailbox_create_path(&self, account_id: u32, path: &str) -> trc::Result> { let cache = self .get_cached_messages(account_id) .await .caused_by(trc::location!())?; let mut next_parent_id = 0; let mut create_paths = Vec::with_capacity(2); let mut path = path.split('/').map(|v| v.trim()); let mut found_path = String::with_capacity(16); { while let Some(name) = path.next() { if !found_path.is_empty() { found_path.push('/'); } for ch in name.chars() { for ch in ch.to_lowercase() { found_path.push(ch); } } if let Some(item) = cache .mailboxes .items .iter() .find(|item| item.path.to_lowercase() == found_path) { next_parent_id = item.document_id + 1; } else { create_paths.push(name.to_string()); create_paths.extend(path.map(|v| v.to_string())); break; } } } // Create missing folders if !create_paths.is_empty() { if create_paths .iter() .any(|name| name.len() > self.core.jmap.mailbox_name_max_len) { return Ok(None); } let mut next_document_id = self .store() .assign_document_ids(account_id, Collection::Mailbox, create_paths.len() as u64) .await .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); for name in create_paths { let document_id = next_document_id; next_document_id -= 1; batch .with_account_id(account_id) .with_collection(Collection::Mailbox) .with_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(Mailbox::new(name).with_parent_id(next_parent_id)), ) .caused_by(trc::location!())?; next_parent_id = document_id + 1; } self.commit_batch(batch).await.caused_by(trc::location!())?; } Ok(Some(next_parent_id - 1)) } } ================================================ FILE: crates/email/src/mailbox/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use types::{acl::AclGrant, special_use::SpecialUse}; pub mod destroy; pub mod index; pub mod manage; pub const INBOX_ID: u32 = 0; pub const TRASH_ID: u32 = 1; pub const JUNK_ID: u32 = 2; pub const DRAFTS_ID: u32 = 3; pub const SENT_ID: u32 = 4; pub const ARCHIVE_ID: u32 = 5; #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] #[rkyv(derive(Debug))] pub struct Mailbox { pub name: String, pub role: SpecialUse, pub parent_id: u32, pub sort_order: Option, pub uid_validity: u32, pub subscribers: Vec, pub acls: Vec, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, Copy)] #[rkyv(derive(Debug, Clone, Copy))] pub struct UidMailbox { pub mailbox_id: u32, pub uid: u32, } impl Mailbox { pub fn new(name: impl Into) -> Self { Mailbox { name: name.into(), role: SpecialUse::None, parent_id: 0, sort_order: None, uid_validity: rand::random::(), subscribers: vec![], acls: vec![], } } pub fn with_role(mut self, role: SpecialUse) -> Self { self.role = role; self } pub fn with_parent_id(mut self, parent_id: u32) -> Self { self.parent_id = parent_id; self } pub fn with_sort_order(mut self, sort_order: u32) -> Self { self.sort_order = Some(sort_order); self } pub fn with_subscriber(mut self, subscriber: u32) -> Self { self.subscribers.push(subscriber); self } pub fn add_subscriber(&mut self, subscriber: u32) -> bool { if !self.subscribers.contains(&subscriber) { self.subscribers.push(subscriber); true } else { false } } pub fn remove_subscriber(&mut self, subscriber: u32) { self.subscribers.retain(|&x| x != subscriber); } pub fn is_subscribed(&self, subscriber: u32) -> bool { self.subscribers.contains(&subscriber) } } impl ArchivedMailbox { pub fn is_subscribed(&self, subscriber: u32) -> bool { self.subscribers.iter().any(|x| u32::from(x) == subscriber) } } impl PartialEq for UidMailbox { fn eq(&self, other: &Self) -> bool { self.mailbox_id == other.mailbox_id } } impl Eq for UidMailbox {} impl UidMailbox { pub fn new(mailbox_id: u32, uid: u32) -> Self { UidMailbox { mailbox_id, uid } } pub fn new_unassigned(mailbox_id: u32) -> Self { UidMailbox { mailbox_id, uid: 0 } } } ================================================ FILE: crates/email/src/message/copy.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ ingest::{EmailIngest, IngestedEmail}, metadata::{MessageData, MessageMetadata}, }; use crate::{ mailbox::UidMailbox, message::{ index::extractors::VisitTextArchived, ingest::{MergeThreadIds, ThreadInfo}, metadata::{ MESSAGE_HAS_ATTACHMENT, MESSAGE_RECEIVED_MASK, MetadataHeaderName, MetadataHeaderValue, }, }, }; use common::{Server, auth::ResourceToken, storage::index::ObjectIndexBuilder}; use mail_parser::parsers::fields::thread::thread_name; use store::write::{ BatchBuilder, IndexPropertyClass, SearchIndex, TaskEpoch, TaskQueueClass, ValueClass, }; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{ blob::{BlobClass, BlobId}, collection::{Collection, SyncCollection}, field::EmailField, keyword::Keyword, }; use utils::cheeky_hash::CheekyHash; pub enum CopyMessageError { NotFound, OverQuota, } pub trait EmailCopy: Sync + Send { #[allow(clippy::too_many_arguments)] fn copy_message( &self, from_account_id: u32, from_message_id: u32, resource_token: &ResourceToken, mailboxes: Vec, keywords: Vec, received_at: Option, session_id: u64, ) -> impl Future>> + Send; } impl EmailCopy for Server { #[allow(clippy::too_many_arguments)] async fn copy_message( &self, from_account_id: u32, from_message_id: u32, resource_token: &ResourceToken, mailboxes: Vec, keywords: Vec, received_at: Option, session_id: u64, ) -> trc::Result> { // Obtain metadata let account_id = resource_token.account_id; let mut metadata = if let Some(metadata) = self .store() .get_value::>(ValueKey::property( from_account_id, Collection::Email, from_message_id, EmailField::Metadata, )) .await? { metadata .deserialize::() .caused_by(trc::location!())? } else { return Ok(Err(CopyMessageError::NotFound)); }; // Check quota let size = metadata.root_part().offset_end; match self.has_available_quota(resource_token, size as u64).await { Ok(_) => (), Err(err) => { if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) || err.matches(trc::EventType::Limit(trc::LimitEvent::TenantQuota)) { trc::error!(err.account_id(account_id).span_id(session_id)); return Ok(Err(CopyMessageError::OverQuota)); } else { return Err(err); } } } // Set receivedAt if let Some(received_at) = received_at { metadata.rcvd_attach = (metadata.rcvd_attach & MESSAGE_HAS_ATTACHMENT) | (received_at & MESSAGE_RECEIVED_MASK); } // Obtain threadId let mut message_ids = Vec::new(); let mut subject = ""; for header in &metadata.contents[0].parts[0].headers { match &header.name { MetadataHeaderName::MessageId => { header.value.visit_text(|id| { if !id.is_empty() { message_ids.push(CheekyHash::new(id.as_bytes())); } }); } MetadataHeaderName::InReplyTo | MetadataHeaderName::References | MetadataHeaderName::ResentMessageId => { header.value.visit_text(|id| { if !id.is_empty() { message_ids.push(CheekyHash::new(id.as_bytes())); } }); } MetadataHeaderName::Subject if subject.is_empty() => { subject = thread_name(match &header.value { MetadataHeaderValue::Text(text) => text.as_ref(), MetadataHeaderValue::TextList(list) if !list.is_empty() => { list.first().unwrap().as_ref() } _ => "", }); } _ => (), } } // Obtain threadId let thread_result = self .find_thread_id(account_id, subject, &message_ids) .await .caused_by(trc::location!())?; // Assign id let mut email = IngestedEmail { size: size as usize, ..Default::default() }; let blob_hash = metadata.blob_hash.clone(); // Assign IMAP UIDs let mut mailbox_ids = Vec::with_capacity(mailboxes.len()); email.imap_uids = Vec::with_capacity(mailboxes.len()); let mut ids = self .assign_email_ids(account_id, mailboxes.iter().copied(), true) .await .caused_by(trc::location!())?; let document_id = ids.next().unwrap(); for (uid, mailbox_id) in ids.zip(mailboxes.iter().copied()) { mailbox_ids.push(UidMailbox::new(mailbox_id, uid)); email.imap_uids.push(uid); } // Prepare batch let mut batch = BatchBuilder::new(); batch.with_account_id(account_id); // Determine thread id let thread_id = if let Some(thread_id) = thread_result.thread_id { thread_id } else { batch .with_collection(Collection::Thread) .with_document(document_id) .log_container_insert(SyncCollection::Thread); document_id }; batch .with_collection(Collection::Email) .with_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_tenant_id(resource_token.tenant.map(|t| t.id)) .with_changes(MessageData { mailboxes: mailbox_ids.into_boxed_slice(), keywords: keywords.into_boxed_slice(), thread_id, size, }), ) .caused_by(trc::location!())? .set( ValueClass::IndexProperty(IndexPropertyClass::Hash { property: EmailField::Threading.into(), hash: thread_result.thread_hash, }), ThreadInfo::serialize(thread_id, &message_ids), ) .set( ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { index: SearchIndex::Email, due: TaskEpoch::now(), is_insert: true, }), vec![], ); // Merge threads if necessary if let Some(merge_threads) = MergeThreadIds::new(thread_result).serialize() { batch.set( ValueClass::TaskQueue(TaskQueueClass::MergeThreads { due: TaskEpoch::now(), }), merge_threads, ); } metadata .index(&mut batch, true) .caused_by(trc::location!())?; // Insert and obtain ids let change_id = self .store() .write(batch.build_all()) .await .caused_by(trc::location!())? .last_change_id(account_id)?; // Request indexing self.notify_task_queue(); // Update response email.document_id = document_id; email.thread_id = thread_id; email.change_id = change_id; email.blob_id = BlobId::new( blob_hash, BlobClass::Linked { account_id, collection: Collection::Email.into(), document_id, }, ); Ok(Ok(email)) } } ================================================ FILE: crates/email/src/message/crypto.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{borrow::Cow, collections::BTreeSet, fmt::Display, io::Cursor}; use aes::cipher::{BlockEncryptMut, KeyIvInit, block_padding::Pkcs7}; use mail_builder::{encoders::base64::base64_encode_mime, mime::make_boundary}; use mail_parser::{Message, MimeHeaders, PartType, decoders::base64::base64_decode}; use openpgp::{ parse::Parse, serialize::stream, types::{KeyFlags, SymmetricAlgorithm}, }; use rand::{RngCore, SeedableRng, rngs::StdRng}; use rasn::types::{ObjectIdentifier, OctetString}; use rasn_cms::{ AlgorithmIdentifier, CONTENT_DATA, CONTENT_ENVELOPED_DATA, EncryptedContent, EncryptedContentInfo, EncryptedKey, EnvelopedData, IssuerAndSerialNumber, KeyTransRecipientInfo, RecipientIdentifier, RecipientInfo, algorithms::{AES128_CBC, AES256_CBC, RSA}, pkcs7_compat::EncapsulatedContentInfo, }; use rsa::{Pkcs1v15Encrypt, RsaPublicKey, pkcs1::DecodeRsaPublicKey}; use sequoia_openpgp as openpgp; use store::{Deserialize, write::Archive}; const P: openpgp::policy::StandardPolicy<'static> = openpgp::policy::StandardPolicy::new(); #[derive(Debug)] pub enum EncryptMessageError { AlreadyEncrypted, Error(String), } #[derive( rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Clone, Copy, serde::Serialize, serde::Deserialize, )] #[rkyv(derive(Clone, Copy))] pub enum Algorithm { Aes128, Aes256, } #[derive( rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, )] pub enum EncryptionMethod { PGP, SMIME, } #[derive( Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, serde::Serialize, serde::Deserialize, )] pub struct EncryptionParams { pub certs: Box<[Box<[u8]>]>, pub flags: u64, } pub const ENCRYPT_TRAIN_SPAM_FILTER: u64 = 1; pub const ENCRYPT_METHOD_SMIME: u64 = 1 << 1; pub const ENCRYPT_METHOD_PGP: u64 = 1 << 2; pub const ENCRYPT_ALGO_AES256: u64 = 1 << 3; pub const ENCRYPT_ALGO_AES128: u64 = 1 << 4; #[derive( rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, serde::Serialize, serde::Deserialize, Default, )] #[serde(tag = "type")] #[serde(rename_all = "camelCase")] pub enum EncryptionType { PGP { algo: Algorithm, certs: String, allow_spam_training: bool, }, SMIME { algo: Algorithm, certs: String, allow_spam_training: bool, }, #[default] Disabled, } #[allow(async_fn_in_trait)] pub trait EncryptMessage { async fn encrypt( &self, params: &ArchivedEncryptionParams, ) -> Result, EncryptMessageError>; fn is_encrypted(&self) -> bool; } impl EncryptMessage for Message<'_> { async fn encrypt( &self, params: &ArchivedEncryptionParams, ) -> Result, EncryptMessageError> { let root = self.root_part(); let raw_message = self.raw_message(); let mut outer_message = Vec::with_capacity((raw_message.len() as f64 * 1.5) as usize); let mut inner_message = Vec::with_capacity(raw_message.len()); // Move MIME headers and body to inner message for header in root.headers() { (if header.name.is_mime_header() { &mut inner_message } else { &mut outer_message }) .extend_from_slice( &raw_message[header.offset_field() as usize..header.offset_end() as usize], ); } inner_message.extend_from_slice(b"\r\n"); inner_message.extend_from_slice(&raw_message[root.raw_body_offset() as usize..]); // Encrypt inner message match params.method() { EncryptionMethod::PGP => { // Prepare encrypted message let boundary = make_boundary("_"); outer_message.extend_from_slice( concat!( "Content-Type: multipart/encrypted;\r\n\t", "protocol=\"application/pgp-encrypted\";\r\n\t", "boundary=\"" ) .as_bytes(), ); outer_message.extend_from_slice(boundary.as_bytes()); outer_message.extend_from_slice( concat!( "\"\r\n\r\n", "OpenPGP/MIME message (Automatically encrypted by Stalwart)\r\n\r\n", "--" ) .as_bytes(), ); outer_message.extend_from_slice(boundary.as_bytes()); outer_message.extend_from_slice( concat!( "\r\nContent-Type: application/pgp-encrypted\r\n\r\n", "Version: 1\r\n\r\n--" ) .as_bytes(), ); outer_message.extend_from_slice(boundary.as_bytes()); outer_message.extend_from_slice( concat!( "\r\nContent-Type: application/octet-stream; name=\"encrypted.asc\"\r\n", "Content-Disposition: inline; filename=\"encrypted.asc\"\r\n\r\n" ) .as_bytes(), ); let certs = params .certs .iter() .map(openpgp::Cert::from_bytes) .collect::, _>>() .map_err(|err| { EncryptMessageError::Error(format!( "Failed to parse OpenPGP public key: {}", err )) })?; // Encrypt contents (TODO: use rayon) let algo = params.algo(); let encrypted_contents = tokio::task::spawn_blocking(move || { // Parse public key let mut keys = Vec::with_capacity(certs.len()); let policy = openpgp::policy::StandardPolicy::new(); for cert in &certs { for key in cert .keys() .with_policy(&policy, None) .supported() .alive() .revoked(false) .key_flags(KeyFlags::empty().set_transport_encryption()) { keys.push(key); } } // Compose a writer stack corresponding to the output format and // packet structure we want. let mut sink = Vec::with_capacity(inner_message.len()); // Stream an OpenPGP message. let message = stream::Armorer::new(stream::Message::new(&mut sink)) .build() .map_err(|err| { EncryptMessageError::Error(format!("Failed to create armorer: {}", err)) })?; let message = stream::Encryptor::for_recipients(message, keys) .symmetric_algo(match algo { Algorithm::Aes128 => SymmetricAlgorithm::AES128, Algorithm::Aes256 => SymmetricAlgorithm::AES256, }) .build() .map_err(|err| { EncryptMessageError::Error(format!( "Failed to build encryptor: {}", err )) })?; let mut message = stream::LiteralWriter::new(message).build().map_err(|err| { EncryptMessageError::Error(format!( "Failed to create literal writer: {}", err )) })?; std::io::copy(&mut Cursor::new(inner_message), &mut message).map_err( |err| { EncryptMessageError::Error(format!( "Failed to encrypt message: {}", err )) }, )?; message.finalize().map_err(|err| { EncryptMessageError::Error(format!("Failed to finalize message: {}", err)) })?; String::from_utf8(sink).map_err(|err| { EncryptMessageError::Error(format!( "Failed to convert encrypted message to UTF-8: {}", err )) }) }) .await .map_err(|err| { EncryptMessageError::Error(format!("Failed to encrypt message: {}", err)) })??; outer_message.extend_from_slice(encrypted_contents.as_bytes()); outer_message.extend_from_slice(b"\r\n--"); outer_message.extend_from_slice(boundary.as_bytes()); outer_message.extend_from_slice(b"--\r\n"); } EncryptionMethod::SMIME => { // Generate random IV let mut rng = StdRng::from_entropy(); let mut iv = vec![0u8; 16]; rng.fill_bytes(&mut iv); // Generate random key let mut key = vec![0u8; params.key_size()]; rng.fill_bytes(&mut key); // Encrypt contents (TODO: use rayon) let algo = params.algo(); let (encrypted_contents, key, iv) = tokio::task::spawn_blocking(move || { (algo.encrypt(&key, &iv, &inner_message), key, iv) }) .await .map_err(|err| { EncryptMessageError::Error(format!("Failed to encrypt message: {}", err)) })?; // Encrypt key using public keys #[allow(clippy::mutable_key_type)] let mut recipient_infos = BTreeSet::new(); for cert in params.certs.iter() { let cert = rasn::der::decode::(cert).map_err(|err| { EncryptMessageError::Error(format!( "Failed to parse certificate: {}", err )) })?; let public_key = RsaPublicKey::from_pkcs1_der( cert.tbs_certificate .subject_public_key_info .subject_public_key .as_raw_slice(), ) .map_err(|err| { EncryptMessageError::Error(format!("Failed to parse public key: {}", err)) })?; let encrypted_key = public_key .encrypt(&mut rng, Pkcs1v15Encrypt, &key[..]) .map_err(|err| { EncryptMessageError::Error(format!("Failed to encrypt key: {}", err)) }) .unwrap(); recipient_infos.insert(RecipientInfo::KeyTransRecipientInfo( KeyTransRecipientInfo { version: 0.into(), rid: RecipientIdentifier::IssuerAndSerialNumber( IssuerAndSerialNumber { issuer: cert.tbs_certificate.issuer, serial_number: cert.tbs_certificate.serial_number, }, ), key_encryption_algorithm: AlgorithmIdentifier { algorithm: RSA.into(), parameters: Some( rasn::der::encode(&()) .map_err(|err| { EncryptMessageError::Error(format!( "Failed to encode RSA algorithm identifier: {}", err )) })? .into(), ), }, encrypted_key: EncryptedKey::from(encrypted_key), }, )); } let pkcs7 = rasn::der::encode(&EncapsulatedContentInfo { content_type: CONTENT_ENVELOPED_DATA.into(), content: Some( rasn::der::encode(&EnvelopedData { version: 0.into(), originator_info: None, recipient_infos, encrypted_content_info: EncryptedContentInfo { content_type: CONTENT_DATA.into(), content_encryption_algorithm: AlgorithmIdentifier { algorithm: params.to_algorithm_identifier(), parameters: Some( rasn::der::encode(&OctetString::from(iv)) .map_err(|err| { EncryptMessageError::Error(format!( "Failed to encode IV: {}", err )) })? .into(), ), }, encrypted_content: Some(EncryptedContent::from(encrypted_contents)), }, unprotected_attrs: None, }) .map_err(|err| { EncryptMessageError::Error(format!( "Failed to encode EnvelopedData: {}", err )) })? .into(), ), }) .map_err(|err| { EncryptMessageError::Error(format!("Failed to encode ContentInfo: {}", err)) })?; // Generate message outer_message.extend_from_slice( concat!( "Content-Type: application/pkcs7-mime;\r\n", "\tname=\"smime.p7m\";\r\n", "\tsmime-type=enveloped-data\r\n", "Content-Disposition: attachment;\r\n", "\tfilename=\"smime.p7m\"\r\n", "Content-Transfer-Encoding: base64\r\n\r\n" ) .as_bytes(), ); base64_encode_mime(&pkcs7, &mut outer_message, false).map_err(|err| { EncryptMessageError::Error(format!("Failed to base64 encode PKCS7: {}", err)) })?; } } Ok(outer_message) } fn is_encrypted(&self) -> bool { if self.content_type().is_some_and(|ct| { let main_type = ct.c_type.as_ref(); let sub_type = ct .c_subtype .as_ref() .map(|s| s.as_ref()) .unwrap_or_default(); (main_type.eq_ignore_ascii_case("application") && (sub_type.eq_ignore_ascii_case("pkcs7-mime") || sub_type.eq_ignore_ascii_case("pkcs7-signature") || (sub_type.eq_ignore_ascii_case("octet-stream") && self.attachment_name().is_some_and(|name| { name.rsplit_once('.') .is_some_and(|(_, ext)| ["p7m", "p7s", "p7c", "p7z"].contains(&ext)) })))) || (main_type.eq_ignore_ascii_case("multipart") && sub_type.eq_ignore_ascii_case("encrypted")) }) { return true; } if self.parts.len() <= 2 { let mut text_part = None; let mut is_multipart = false; for part in &self.parts { match &part.body { PartType::Text(text) => { text_part = Some(text.as_ref()); } PartType::Multipart(_) => { is_multipart = true; } _ => (), } } match text_part { Some(text) if self.parts.len() == 1 || is_multipart => { if text.trim_start().starts_with("-----BEGIN PGP MESSAGE-----") { return true; } } _ => (), } } false } } impl ArchivedEncryptionParams { pub fn method(&self) -> EncryptionMethod { if self.flags & ENCRYPT_METHOD_PGP != 0 { EncryptionMethod::PGP } else { EncryptionMethod::SMIME } } pub fn algo(&self) -> Algorithm { if self.flags & ENCRYPT_ALGO_AES256 != 0 { Algorithm::Aes256 } else { Algorithm::Aes128 } } fn key_size(&self) -> usize { if self.flags & ENCRYPT_ALGO_AES256 != 0 { 32 } else { 16 } } fn to_algorithm_identifier(&self) -> ObjectIdentifier { if self.flags & ENCRYPT_ALGO_AES256 != 0 { AES256_CBC.into() } else { AES128_CBC.into() } } pub fn can_train_spam_filter(&self) -> bool { self.flags & ENCRYPT_TRAIN_SPAM_FILTER != 0 } } impl Algorithm { fn encrypt(&self, key: &[u8], iv: &[u8], contents: &[u8]) -> Vec { match self { Algorithm::Aes128 => cbc::Encryptor::::new(key.into(), iv.into()) .encrypt_padded_vec_mut::(contents), Algorithm::Aes256 => cbc::Encryptor::::new(key.into(), iv.into()) .encrypt_padded_vec_mut::(contents), } } } #[allow(clippy::type_complexity)] pub fn try_parse_certs( expected_method: EncryptionMethod, cert: Vec, ) -> Result]>, Cow<'static, str>> { // Check if it's a PEM file let (flags, certs) = if let Some(result) = try_parse_pem(&cert)? { (result.flags, result.certs) } else if rasn::der::decode::(&cert[..]).is_ok() { ( ENCRYPT_METHOD_SMIME, Box::from_iter([cert.into_boxed_slice()]), ) } else if let Ok(cert_) = openpgp::Cert::from_bytes(&cert[..]) { if !has_pgp_keys(cert_) { ( ENCRYPT_METHOD_PGP, Box::from_iter([cert.into_boxed_slice()]), ) } else { return Err("Could not find any suitable keys in certificate".into()); } } else { return Err("Could not find any valid certificates".into()); }; if expected_method.flags() & flags != 0 { Ok(certs) } else { Err("No valid certificates found for the selected encryption".into()) } } fn has_pgp_keys(cert: openpgp::Cert) -> bool { cert.keys() .with_policy(&P, None) .supported() .alive() .revoked(false) .key_flags(KeyFlags::empty().set_transport_encryption()) .next() .is_some() } #[allow(clippy::type_complexity)] fn try_parse_pem(bytes_: &[u8]) -> Result, Cow<'static, str>> { if let Some(internal) = std::str::from_utf8(bytes_) .ok() .and_then(|cert| cert.strip_prefix("-----STALWART CERTIFICATE-----")) { return base64_decode(internal.as_bytes()) .ok_or(Cow::from("Failed to decode base64")) .and_then(|bytes| { Archive::deserialize_owned(bytes) .and_then(|arch| arch.deserialize::()) .map_err(|_| Cow::from("Failed to deserialize internal certificate")) }) .map(Some); } let mut bytes = bytes_.iter().enumerate(); let mut buf = vec![]; let mut method = None; let mut certs: Vec> = vec![]; loop { // Find start of PEM block let mut start_pos = 0; for (pos, &ch) in bytes.by_ref() { if ch.is_ascii_whitespace() { continue; } else if ch == b'-' { start_pos = pos; break; } else { return Ok(None); } } // Find block type for (_, &ch) in bytes.by_ref() { match ch { b'-' => (), b'\n' => break, _ => { if ch.is_ascii() { buf.push(ch.to_ascii_uppercase()); } else { return Ok(None); } } } } if buf.is_empty() { break; } // Find type let tag = std::str::from_utf8(&buf).unwrap(); if tag.contains("CERTIFICATE") { if method.is_some_and(|m| m == EncryptionMethod::PGP) { return Err("Cannot mix OpenPGP and S/MIME certificates".into()); } else { method = Some(EncryptionMethod::SMIME); } } else if tag.contains("PGP") { if method.is_some_and(|m| m == EncryptionMethod::SMIME) { return Err("Cannot mix OpenPGP and S/MIME certificates".into()); } else { method = Some(EncryptionMethod::PGP); } } else { // Ignore block let mut found_end = false; for (_, &ch) in bytes.by_ref() { if ch == b'-' { found_end = true; } else if ch == b'\n' && found_end { break; } } buf.clear(); continue; } // Collect base64 buf.clear(); let mut found_end = false; let mut end_pos = 0; for (pos, &ch) in bytes.by_ref() { match ch { b'-' => { found_end = true; } b'\n' => { if found_end { end_pos = pos; break; } } _ => { if !ch.is_ascii_whitespace() { buf.push(ch); } } } } // Decode base64 let cert = base64_decode(&buf) .ok_or_else(|| Cow::from("Failed to decode base64 certificate."))? .into_boxed_slice(); match method.unwrap() { EncryptionMethod::PGP => match openpgp::Cert::from_bytes(bytes_) { Ok(cert) => { if !has_pgp_keys(cert) { return Err("Could not find any suitable keys in OpenPGP public key".into()); } certs.push( bytes_ .get(start_pos..end_pos + 1) .unwrap_or_default() .into(), ); } Err(err) => { return Err(format!("Failed to decode OpenPGP public key: {err}").into()); } }, EncryptionMethod::SMIME => { if let Err(err) = rasn::der::decode::(&cert) { return Err(format!("Failed to decode X509 certificate: {err}").into()); } certs.push(cert); } } buf.clear(); } Ok(method.map(|method| EncryptionParams { flags: method.flags(), certs: certs.into_boxed_slice(), })) } impl EncryptionMethod { pub fn flags(&self) -> u64 { match self { EncryptionMethod::PGP => ENCRYPT_METHOD_PGP, EncryptionMethod::SMIME => ENCRYPT_METHOD_SMIME, } } } impl Algorithm { pub fn flags(&self) -> u64 { match self { Algorithm::Aes128 => ENCRYPT_ALGO_AES128, Algorithm::Aes256 => ENCRYPT_ALGO_AES256, } } } impl Display for EncryptionMethod { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { EncryptionMethod::PGP => write!(f, "OpenPGP"), EncryptionMethod::SMIME => write!(f, "S/MIME"), } } } impl Display for Algorithm { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Algorithm::Aes128 => write!(f, "AES-128"), Algorithm::Aes256 => write!(f, "AES-256"), } } } ================================================ FILE: crates/email/src/message/delete.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::metadata::MessageData; use common::{KV_LOCK_PURGE_ACCOUNT, Server, storage::index::ObjectIndexBuilder}; use directory::backend::internal::manage::ManageDirectory; use groupware::calendar::storage::ItipAutoExpunge; use std::future::Future; use store::rand::prelude::SliceRandom; use store::write::key::DeserializeBigEndian; use store::write::{IndexPropertyClass, SearchIndex, TaskEpoch, TaskQueueClass, now}; use store::{IterateParams, SerializeInfallible, U32_LEN, U64_LEN, ValueKey}; use store::{ roaring::RoaringBitmap, write::{BatchBuilder, ValueClass}, }; use trc::AddContext; use types::collection::{Collection, VanishedCollection}; use types::field::{EmailField, EmailSubmissionField}; pub trait EmailDeletion: Sync + Send { fn emails_delete( &self, account_id: u32, tenant_id: Option, batch: &mut BatchBuilder, document_ids: RoaringBitmap, ) -> impl Future> + Send; fn purge_accounts(&self, use_roles: bool) -> impl Future + Send; fn purge_account(&self, account_id: u32) -> impl Future + Send; fn purge_email_submissions( &self, account_id: u32, hold_period: u64, ) -> impl Future> + Send; fn emails_auto_expunge( &self, account_id: u32, hold_period: u64, ) -> impl Future> + Send; } impl EmailDeletion for Server { async fn emails_delete( &self, account_id: u32, tenant_id: Option, batch: &mut BatchBuilder, document_ids: RoaringBitmap, ) -> trc::Result { let mut deleted_ids = RoaringBitmap::new(); batch .with_account_id(account_id) .with_collection(Collection::Email); self.archives( account_id, Collection::Email, &document_ids, |document_id, data_| { // Add changes to batch let metadata = data_ .to_unarchived::() .caused_by(trc::location!())?; for mailbox in metadata.inner.mailboxes.iter() { batch.log_vanished_item( VanishedCollection::Email, (mailbox.mailbox_id.to_native(), mailbox.uid.to_native()), ); } batch .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() .with_tenant_id(tenant_id) .with_current(metadata), ) .caused_by(trc::location!())? .set( ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { index: SearchIndex::Email, due: TaskEpoch::now(), is_insert: false, }), 0u64.serialize(), ) .commit_point(); deleted_ids.insert(document_id); Ok(true) }, ) .await?; let not_destroyed = if document_ids.len() == deleted_ids.len() { RoaringBitmap::new() } else { deleted_ids ^= document_ids; deleted_ids }; Ok(not_destroyed) } async fn purge_accounts(&self, use_roles: bool) { if let Ok(account_ids) = self.store().principal_ids(None, None).await { let mut account_ids: Vec = account_ids .into_iter() .filter(|id| { !use_roles || self .core .network .roles .purge_accounts .is_enabled_for_integer(*id) }) .collect(); // Shuffle account ids account_ids.shuffle(&mut store::rand::rng()); for account_id in account_ids { self.purge_account(account_id).await; } } } async fn purge_account(&self, account_id: u32) { // Lock account match self .core .storage .lookup .try_lock(KV_LOCK_PURGE_ACCOUNT, &account_id.to_be_bytes(), 3600) .await { Ok(true) => (), Ok(false) => { trc::event!(Purge(trc::PurgeEvent::InProgress), AccountId = account_id,); return; } Err(err) => { trc::error!( err.details("Failed to lock account.") .account_id(account_id) ); return; } } // Auto-expunge deleted and junk messages if let Some(hold_period) = self.core.jmap.mail_autoexpunge_after && let Err(err) = self.emails_auto_expunge(account_id, hold_period).await { trc::error!( err.details("Failed to auto-expunge e-mail messages.") .account_id(account_id) ); } // Auto-expunge iMIP messages if let Some(hold_period) = self.core.groupware.itip_inbox_auto_expunge && let Err(err) = self.itip_auto_expunge(account_id, hold_period).await { trc::error!( err.details("Failed to auto-expunge iTIP messages.") .account_id(account_id) ); } // Delete old e-mail submissions if let Some(hold_period) = self.core.jmap.email_submission_autoexpunge_after && let Err(err) = self.purge_email_submissions(account_id, hold_period).await { trc::error!( err.details("Failed to auto-expunge e-mail submissions.") .account_id(account_id) ); } // Purge changelogs if let Err(err) = self .delete_changes( account_id, self.core.jmap.changes_max_history, self.core.jmap.share_notification_max_history, ) .await { trc::error!( err.details("Failed to purge changes.") .account_id(account_id) ); } // Delete lock if let Err(err) = self .in_memory_store() .remove_lock(KV_LOCK_PURGE_ACCOUNT, &account_id.to_be_bytes()) .await { trc::error!(err.details("Failed to delete lock.").account_id(account_id)); } } async fn emails_auto_expunge(&self, account_id: u32, hold_period: u64) -> trc::Result<()> { // Filter messages by received date let mut destroy_ids = RoaringBitmap::new(); let cutoff = now().saturating_sub(hold_period); self.store() .iterate( IterateParams::new( ValueKey { account_id, collection: Collection::Email.into(), document_id: 0, class: ValueClass::Property(EmailField::DeletedAt.into()), }, ValueKey { account_id, collection: Collection::Email.into(), document_id: u32::MAX, class: ValueClass::Property(EmailField::DeletedAt.into()), }, ) .ascending(), |key, value| { let deleted_at = value.deserialize_be_u64(0)?; if deleted_at <= cutoff { destroy_ids.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); } Ok(true) }, ) .await .caused_by(trc::location!())?; if destroy_ids.is_empty() { return Ok(()); } trc::event!( Purge(trc::PurgeEvent::AutoExpunge), Collection = Collection::Email.as_str(), AccountId = account_id, Total = destroy_ids.len(), ); // Delete messages let mut batch = BatchBuilder::new(); let tenant_id = self .store() .get_principal(account_id) .await .caused_by(trc::location!())? .and_then(|p| p.tenant()); self.emails_delete(account_id, tenant_id, &mut batch, destroy_ids) .await?; self.commit_batch(batch).await?; self.notify_task_queue(); Ok(()) } async fn purge_email_submissions(&self, account_id: u32, hold_period: u64) -> trc::Result<()> { // Filter messages by received date let mut destroy_ids = Vec::new(); self.store() .iterate( IterateParams::new( ValueKey { account_id, collection: Collection::EmailSubmission.into(), document_id: 0, class: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: EmailSubmissionField::Metadata.into(), value: 0, }), }, ValueKey { account_id, collection: Collection::Email.into(), document_id: u32::MAX, class: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: EmailSubmissionField::Metadata.into(), value: now().saturating_sub(hold_period), }), }, ) .ascending() .no_values(), |key, _| { destroy_ids.push(( key.deserialize_be_u32(key.len() - U32_LEN)?, key.deserialize_be_u64(key.len() - U32_LEN - U64_LEN)?, )); Ok(true) }, ) .await .caused_by(trc::location!())?; if destroy_ids.is_empty() { return Ok(()); } trc::event!( Purge(trc::PurgeEvent::AutoExpunge), Collection = Collection::EmailSubmission.as_str(), AccountId = account_id, Total = destroy_ids.len(), ); // Delete messages let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::EmailSubmission); for (document_id, send_at) in destroy_ids { batch .with_document(document_id) .clear(EmailSubmissionField::Metadata) .clear(ValueClass::IndexProperty(IndexPropertyClass::Integer { property: EmailSubmissionField::Metadata.into(), value: send_at, })) .commit_point(); } self.commit_batch(batch).await?; Ok(()) } } ================================================ FILE: crates/email/src/message/delivery.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::ingest::{EmailIngest, IngestEmail, IngestSource}; use crate::{mailbox::INBOX_ID, sieve::ingest::SieveScriptIngest}; use common::{ Server, ipc::{EmailPush, PushNotification}, }; use directory::Permission; use mail_parser::MessageParser; use std::{borrow::Cow, future::Future}; use store::ahash::AHashMap; use types::blob_hash::BlobHash; #[derive(Debug)] pub struct IngestMessage { pub sender_address: String, pub sender_authenticated: bool, pub recipients: Vec, pub message_blob: BlobHash, pub message_size: u64, pub session_id: u64, } #[derive(Debug)] pub struct IngestRecipient { pub address: String, pub is_spam: bool, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum LocalDeliveryStatus { Success, TemporaryFailure { reason: Cow<'static, str>, }, PermanentFailure { code: [u8; 3], reason: Cow<'static, str>, }, } pub struct LocalDeliveryResult { pub status: Vec, pub autogenerated: Vec, } pub struct AutogeneratedMessage { pub sender_address: String, pub recipients: Vec, pub message: Vec, } pub trait MailDelivery: Sync + Send { fn deliver_message( &self, message: IngestMessage, ) -> impl Future + Send; } impl MailDelivery for Server { async fn deliver_message(&self, message: IngestMessage) -> LocalDeliveryResult { // Read message let raw_message = match self .core .storage .blob .get_blob(message.message_blob.as_slice(), 0..usize::MAX) .await { Ok(Some(raw_message)) => raw_message, Ok(None) => { trc::event!( MessageIngest(trc::MessageIngestEvent::Error), Reason = "Blob not found.", SpanId = message.session_id, CausedBy = trc::location!() ); return LocalDeliveryResult { status: (0..message.recipients.len()) .map(|_| LocalDeliveryStatus::TemporaryFailure { reason: "Blob not found.".into(), }) .collect::>(), autogenerated: vec![], }; } Err(err) => { trc::error!( err.details("Failed to fetch message blob.") .span_id(message.session_id) .caused_by(trc::location!()) ); return LocalDeliveryResult { status: (0..message.recipients.len()) .map(|_| LocalDeliveryStatus::TemporaryFailure { reason: "Temporary I/O error.".into(), }) .collect::>(), autogenerated: vec![], }; } }; // Obtain the account IDs for each recipient let mut account_ids: AHashMap = AHashMap::with_capacity(message.recipients.len()); let mut result = LocalDeliveryResult { status: Vec::with_capacity(message.recipients.len()), autogenerated: Vec::new(), }; for rcpt in message.recipients { let account_id = match self .email_to_id( &self.core.storage.directory, &rcpt.address, message.session_id, ) .await { Ok(Some(account_id)) => account_id, Ok(None) => { // Something went wrong result.status.push(LocalDeliveryStatus::PermanentFailure { code: [5, 5, 0], reason: "Mailbox not found.".into(), }); continue; } Err(err) => { trc::error!( err.details("Failed to lookup recipient.") .ctx(trc::Key::To, rcpt.address.to_string()) .span_id(message.session_id) .caused_by(trc::location!()) ); result.status.push(LocalDeliveryStatus::TemporaryFailure { reason: "Address lookup failed.".into(), }); continue; } }; if let Some(status) = account_ids .get(&account_id) .and_then(|pos| result.status.get(*pos)) { result.status.push(status.clone()); continue; } // Obtain access token let status = match self.get_access_token(account_id).await.and_then(|token| { token .assert_has_permission(Permission::EmailReceive) .map(|_| token) }) { Ok(access_token) => { // Check if there is an active sieve script match self.sieve_script_get_active(account_id).await { Ok(None) => { // Ingest message self.email_ingest(IngestEmail { raw_message: &raw_message, blob_hash: Some(&message.message_blob), message: MessageParser::new().parse(&raw_message), access_token: &access_token, mailbox_ids: vec![INBOX_ID], keywords: vec![], received_at: None, source: IngestSource::Smtp { deliver_to: &rcpt.address, is_sender_authenticated: message.sender_authenticated, is_spam: rcpt.is_spam, }, session_id: message.session_id, }) .await } Ok(Some(active_script)) => { self.sieve_script_ingest( &access_token, &message.message_blob, &raw_message, &message.sender_address, message.sender_authenticated, &rcpt, message.session_id, active_script, &mut result.autogenerated, ) .await } Err(err) => Err(err), } } Err(err) => Err(err), }; let status = match status { Ok(ingested_message) => { // Notify state change if ingested_message.change_id != u64::MAX { self.broadcast_push_notification(PushNotification::EmailPush(EmailPush { account_id, email_id: ingested_message.document_id, change_id: ingested_message.change_id, })) .await; } LocalDeliveryStatus::Success } Err(err) => { let status = match err.as_ref() { trc::EventType::Limit(trc::LimitEvent::Quota) => { LocalDeliveryStatus::TemporaryFailure { reason: "Mailbox over quota.".into(), } } trc::EventType::Limit(trc::LimitEvent::TenantQuota) => { LocalDeliveryStatus::TemporaryFailure { reason: "Organization over quota.".into(), } } trc::EventType::Security(trc::SecurityEvent::Unauthorized) => { LocalDeliveryStatus::PermanentFailure { code: [5, 5, 0], reason: "This account is not authorized to receive email.".into(), } } trc::EventType::MessageIngest(trc::MessageIngestEvent::Error) => { LocalDeliveryStatus::PermanentFailure { code: err .value(trc::Key::Code) .and_then(|v| v.to_uint()) .map(|n| { [(n / 100) as u8, ((n % 100) / 10) as u8, (n % 10) as u8] }) .unwrap_or([5, 5, 0]), reason: err .value_as_str(trc::Key::Reason) .unwrap_or_default() .to_string() .into(), } } _ => LocalDeliveryStatus::TemporaryFailure { reason: "Transient server failure.".into(), }, }; trc::error!( err.ctx(trc::Key::To, rcpt.address.to_string()) .span_id(message.session_id) ); status } }; // Cache response for UID to avoid duplicate deliveries account_ids.insert(account_id, result.status.len()); result.status.push(status); } result } } ================================================ FILE: crates/email/src/message/index/extractors.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::message::metadata::{ ArchivedMessageMetadataContents, ArchivedMessageMetadataPart, ArchivedMetadataHeaderValue, MetadataHeaderName, MetadataHeaderValue, }; use mail_parser::{Addr, Address, Group, HeaderValue}; use nlp::language::Language; use rkyv::option::ArchivedOption; use std::borrow::Cow; impl ArchivedMessageMetadataContents { pub fn is_html_part(&self, part_id: u16) -> bool { self.html_body.iter().any(|&id| id == part_id) } pub fn is_text_part(&self, part_id: u16) -> bool { self.text_body.iter().any(|&id| id == part_id) } } impl ArchivedMessageMetadataPart { pub fn language(&self) -> Option { self.header_value(&MetadataHeaderName::ContentLanguage) .and_then(|v| { Language::from_iso_639(v.as_text()?) .unwrap_or(Language::Unknown) .into() }) } } #[derive(Debug, PartialEq, Eq)] pub enum AddressElement { Name, Address, GroupName, } pub trait VisitText { fn visit_addresses(&self, visitor: impl FnMut(AddressElement, &str)); fn visit_text<'x>(&'x self, visitor: impl FnMut(&'x str)); fn into_visit_text(self, visitor: impl FnMut(String)); } impl VisitText for HeaderValue<'_> { fn visit_addresses(&self, mut visitor: impl FnMut(AddressElement, &str)) { match self { HeaderValue::Address(Address::List(addr_list)) => { for addr in addr_list { if let Some(name) = &addr.name { visitor(AddressElement::Name, name); } if let Some(addr) = &addr.address { visitor(AddressElement::Address, addr); } } } HeaderValue::Address(Address::Group(groups)) => { for group in groups { if let Some(name) = &group.name { visitor(AddressElement::GroupName, name); } for addr in &group.addresses { if let Some(name) = &addr.name { visitor(AddressElement::Name, name); } if let Some(addr) = &addr.address { visitor(AddressElement::Address, addr); } } } } _ => (), } } fn visit_text<'x>(&'x self, mut visitor: impl FnMut(&'x str)) { match &self { HeaderValue::Text(text) => { visitor(text.as_ref()); } HeaderValue::TextList(texts) => { for text in texts { visitor(text.as_ref()); } } _ => (), } } fn into_visit_text(self, mut visitor: impl FnMut(String)) { match self { HeaderValue::Text(text) => { visitor(text.into_owned()); } HeaderValue::TextList(texts) => { for text in texts { visitor(text.into_owned()); } } _ => (), } } } pub trait VisitTextArchived { fn visit_addresses(&self, visitor: impl FnMut(AddressElement, &str)); fn visit_text(&self, visitor: impl FnMut(&str)); } impl VisitTextArchived for MetadataHeaderValue { fn visit_addresses(&self, mut visitor: impl FnMut(AddressElement, &str)) { match self { MetadataHeaderValue::AddressList(addr_list) => { for addr in addr_list.iter() { if let Some(name) = &addr.name { visitor(AddressElement::Name, name); } if let Some(addr) = &addr.address { visitor(AddressElement::Address, addr); } } } MetadataHeaderValue::AddressGroup(groups) => { for group in groups.iter() { if let Some(name) = &group.name { visitor(AddressElement::GroupName, name); } for addr in group.addresses.iter() { if let Some(name) = &addr.name { visitor(AddressElement::Name, name); } if let Some(addr) = &addr.address { visitor(AddressElement::Address, addr); } } } } _ => (), } } fn visit_text(&self, mut visitor: impl FnMut(&str)) { match &self { MetadataHeaderValue::Text(text) => { visitor(text.as_ref()); } MetadataHeaderValue::TextList(texts) => { for text in texts.iter() { visitor(text.as_ref()); } } _ => (), } } } impl VisitTextArchived for ArchivedMetadataHeaderValue { fn visit_addresses(&self, mut visitor: impl FnMut(AddressElement, &str)) { match self { ArchivedMetadataHeaderValue::AddressList(addr_list) => { for addr in addr_list.iter() { if let ArchivedOption::Some(name) = &addr.name { visitor(AddressElement::Name, name); } if let ArchivedOption::Some(addr) = &addr.address { visitor(AddressElement::Address, addr); } } } ArchivedMetadataHeaderValue::AddressGroup(groups) => { for group in groups.iter() { if let ArchivedOption::Some(name) = &group.name { visitor(AddressElement::GroupName, name); } for addr in group.addresses.iter() { if let ArchivedOption::Some(name) = &addr.name { visitor(AddressElement::Name, name); } if let ArchivedOption::Some(addr) = &addr.address { visitor(AddressElement::Address, addr); } } } } _ => (), } } fn visit_text(&self, mut visitor: impl FnMut(&str)) { match &self { ArchivedMetadataHeaderValue::Text(text) => { visitor(text.as_ref()); } ArchivedMetadataHeaderValue::TextList(texts) => { for text in texts.iter() { visitor(text.as_ref()); } } _ => (), } } } pub trait TrimTextValue { fn trim_text(self, length: usize) -> Self; } impl TrimTextValue for HeaderValue<'_> { fn trim_text(self, length: usize) -> Self { match self { HeaderValue::Address(Address::List(v)) => { HeaderValue::Address(Address::List(v.trim_text(length))) } HeaderValue::Address(Address::Group(v)) => { HeaderValue::Address(Address::Group(v.trim_text(length))) } HeaderValue::Text(v) => HeaderValue::Text(v.trim_text(length)), HeaderValue::TextList(v) => HeaderValue::TextList(v.trim_text(length)), v => v, } } } impl TrimTextValue for Addr<'_> { fn trim_text(self, length: usize) -> Self { Self { name: self.name.map(|v| v.trim_text(length)), address: self.address.map(|v| v.trim_text(length)), } } } impl TrimTextValue for Group<'_> { fn trim_text(self, length: usize) -> Self { Self { name: self.name.map(|v| v.trim_text(length)), addresses: self.addresses.trim_text(length), } } } impl TrimTextValue for &str { fn trim_text(self, length: usize) -> Self { if self.len() < length { self } else { let mut index = 0; for (i, _) in self.char_indices() { if i > length { break; } index = i; } &self[..index] } } } impl TrimTextValue for Cow<'_, str> { fn trim_text(self, length: usize) -> Self { if self.len() < length { self } else { let mut result = String::with_capacity(length); for (i, c) in self.char_indices() { if i > length { break; } result.push(c); } result.into() } } } impl TrimTextValue for Vec { fn trim_text(self, length: usize) -> Self { self.into_iter().map(|v| v.trim_text(length)).collect() } } ================================================ FILE: crates/email/src/message/index/metadata.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::message::{ index::{IndexMessage, MAX_MESSAGE_PARTS, PREVIEW_LENGTH}, metadata::{ ArchivedMessageMetadata, ArchivedMessageMetadataPart, ArchivedMetadataHeaderName, MESSAGE_HAS_ATTACHMENT, MESSAGE_RECEIVED_MASK, MessageData, MessageMetadata, MessageMetadataPart, build_metadata_contents, }, }; use common::storage::index::ObjectIndexBuilder; use mail_parser::{ PartType, decoders::html::html_to_text, parsers::{fields::thread::thread_name, preview::preview_text}, }; use store::{ Serialize, write::{Archiver, BatchBuilder, BlobLink, BlobOp, IndexPropertyClass, ValueClass}, }; use trc::AddContext; use types::{blob_hash::BlobHash, field::EmailField}; use utils::cheeky_hash::CheekyHash; impl MessageMetadata { #[inline(always)] pub fn root_part(&self) -> &MessageMetadataPart { &self.contents[0].parts[0] } pub fn index(self, batch: &mut BatchBuilder, set: bool) -> trc::Result<()> { if set { batch .set( BlobOp::Link { hash: self.blob_hash.clone(), to: BlobLink::Document, }, Vec::new(), ) .set(EmailField::Metadata, Archiver::new(self).serialize()?); } else { batch .clear(BlobOp::Link { hash: self.blob_hash.clone(), to: BlobLink::Document, }) .clear(EmailField::Metadata); } Ok(()) } } impl ArchivedMessageMetadata { #[inline(always)] pub fn root_part(&self) -> &ArchivedMessageMetadataPart { &self.contents[0].parts[0] } pub fn unindex(&self, batch: &mut BatchBuilder) { // Delete metadata let thread_name = self .contents .first() .and_then(|c| c.parts.first()) .and_then(|p| { p.headers.iter().rev().find_map(|h| { if let ArchivedMetadataHeaderName::Subject = &h.name { h.value.as_text() } else { None } }) }) .map(thread_name) .unwrap_or_default(); batch .clear(EmailField::Metadata) .clear(ValueClass::IndexProperty(IndexPropertyClass::Hash { property: EmailField::Threading.into(), hash: CheekyHash::new(if !thread_name.is_empty() { thread_name } else { "!" }), })) .clear(BlobOp::Link { hash: BlobHash::from(&self.blob_hash), to: BlobLink::Document, }); } } impl IndexMessage for BatchBuilder { fn index_message<'x>( &mut self, tenant_id: Option, mut message: mail_parser::Message<'x>, extra_headers: Vec, mut extra_headers_parsed: Vec>, blob_hash: BlobHash, data: MessageData, received_at: u64, ) -> trc::Result<&mut Self> { let mut has_attachments = false; let mut preview = None; let preview_part_id = message .text_body .first() .or_else(|| message.html_body.first()) .copied() .unwrap_or(u32::MAX); for (part_id, part) in message.parts.iter().take(MAX_MESSAGE_PARTS).enumerate() { let part_id = part_id as u32; match &part.body { mail_parser::PartType::Text(text) => { if part_id == preview_part_id { preview = preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH).into(); } if !message.text_body.contains(&part_id) && !message.html_body.contains(&part_id) { has_attachments = true; } } mail_parser::PartType::Html(html) => { let text = html_to_text(html); if part_id == preview_part_id { preview = preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH).into(); } if !message.text_body.contains(&part_id) && !message.html_body.contains(&part_id) { has_attachments = true; } } mail_parser::PartType::Binary(_) | mail_parser::PartType::Message(_) if !has_attachments => { has_attachments = true; } _ => {} } } // Build raw headers let root_part = message.root_part(); let mut raw_headers = Vec::with_capacity( (root_part.offset_body - root_part.offset_header) as usize + extra_headers.len(), ); raw_headers.extend_from_slice(&extra_headers); raw_headers.extend_from_slice( message .raw_message .as_ref() .get(root_part.offset_header as usize..root_part.offset_body as usize) .unwrap_or_default(), ); // Add additional headers to message let blob_body_offset = if !extra_headers.is_empty() { // Add extra headers to root part let offset_start = extra_headers.len() as u32; let mut part_iter_stack = Vec::new(); let mut part_iter = message.parts.iter_mut(); loop { if let Some(part) = part_iter.next() { // Increment header offsets for header in part.headers.iter_mut() { header.offset_field += offset_start; header.offset_start += offset_start; header.offset_end += offset_start; } // Adjust part offsets part.offset_body += offset_start; part.offset_end += offset_start; part.offset_header += offset_start; if let PartType::Message(sub_message) = &mut part.body && sub_message.root_part().offset_header != 0 { part_iter_stack.push(part_iter); part_iter = sub_message.parts.iter_mut(); } } else if let Some(iter) = part_iter_stack.pop() { part_iter = iter; } else { break; } } // Add extra headers to root part let root_part = &mut message.parts[0]; extra_headers_parsed.append(&mut root_part.headers); root_part.offset_header = 0; root_part.headers = extra_headers_parsed; root_part.offset_body - offset_start } else { message.root_part().offset_body }; // Build metadata let metadata = MessageMetadata { preview: preview.unwrap_or_default().into_owned().into_boxed_str(), raw_headers: raw_headers.into_boxed_slice(), contents: build_metadata_contents(message), blob_hash, blob_body_offset, rcvd_attach: (if has_attachments { MESSAGE_HAS_ATTACHMENT } else { 0 }) | (received_at & MESSAGE_RECEIVED_MASK), }; self.set( BlobOp::Link { hash: metadata.blob_hash.clone(), to: BlobLink::Document, }, Vec::new(), ) .custom( ObjectIndexBuilder::<(), _>::new() .with_tenant_id(tenant_id) .with_changes(data), ) .caused_by(trc::location!())? .set( EmailField::Metadata, Archiver::new(metadata) .serialize() .caused_by(trc::location!())?, ); Ok(self) } } ================================================ FILE: crates/email/src/message/index/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ mailbox::{JUNK_ID, TRASH_ID}, message::metadata::{ArchivedMessageData, MessageData}, }; use common::storage::index::{IndexItem, IndexValue, IndexableObject}; use store::write::now; use types::{blob_hash::BlobHash, collection::SyncCollection, field::EmailField}; pub mod extractors; pub mod metadata; pub mod search; pub(super) const MAX_MESSAGE_PARTS: usize = 1000; pub const PREVIEW_LENGTH: usize = 256; impl IndexableObject for MessageData { fn index_values(&self) -> impl Iterator> { let mut mailboxes = Vec::with_capacity(self.mailboxes.len()); let mut is_in_trash = false; for mailbox in &self.mailboxes { mailboxes.push(mailbox.mailbox_id); is_in_trash |= mailbox.mailbox_id == TRASH_ID || mailbox.mailbox_id == JUNK_ID; } [ IndexValue::Property { field: EmailField::DeletedAt.into(), value: if is_in_trash { IndexItem::from(now()) } else { IndexItem::None }, }, IndexValue::Quota { used: self.size }, IndexValue::LogItem { sync_collection: SyncCollection::Email, prefix: self.thread_id.into(), }, IndexValue::LogContainerProperty { sync_collection: SyncCollection::Thread, ids: vec![self.thread_id], }, IndexValue::LogContainerProperty { sync_collection: SyncCollection::Email, ids: mailboxes, }, ] .into_iter() } } impl IndexableObject for &ArchivedMessageData { fn index_values(&self) -> impl Iterator> { let mut mailboxes = Vec::with_capacity(self.mailboxes.len()); let mut is_in_trash = false; for mailbox in self.mailboxes.iter() { let mailbox_id = mailbox.mailbox_id.to_native(); mailboxes.push(mailbox_id); is_in_trash |= mailbox_id == TRASH_ID || mailbox_id == JUNK_ID; } [ IndexValue::Property { field: EmailField::DeletedAt.into(), value: if is_in_trash { IndexItem::from(now()) } else { IndexItem::None }, }, IndexValue::Quota { used: self.size.to_native(), }, IndexValue::LogItem { sync_collection: SyncCollection::Email, prefix: self.thread_id.to_native().into(), }, IndexValue::LogContainerProperty { sync_collection: SyncCollection::Thread, ids: vec![self.thread_id.to_native()], }, IndexValue::LogContainerProperty { sync_collection: SyncCollection::Email, ids: mailboxes, }, ] .into_iter() } } pub(super) trait IndexMessage { #[allow(clippy::too_many_arguments)] fn index_message<'x>( &mut self, tenant_id: Option, message: mail_parser::Message<'x>, extra_headers: Vec, extra_headers_parsed: Vec>, blob_hash: BlobHash, data: MessageData, received_at: u64, ) -> trc::Result<&mut Self>; } ================================================ FILE: crates/email/src/message/index/search.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::message::{ index::{MAX_MESSAGE_PARTS, extractors::VisitTextArchived}, metadata::{ ArchivedMessageMetadata, ArchivedMetadataHeaderName, ArchivedMetadataHeaderValue, ArchivedMetadataPartType, DecodedPartContent, MESSAGE_RECEIVED_MASK, MetadataHeaderName, }, }; use mail_parser::{DateTime, decoders::html::html_to_text, parsers::fields::thread::thread_name}; use nlp::{ language::{ Language, detect::{LanguageDetector, MIN_LANGUAGE_SCORE}, }, tokenizers::word::WordTokenizer, }; use store::{ ahash::AHashSet, backend::MAX_TOKEN_LENGTH, search::{EmailSearchField, IndexDocument, SearchField}, write::SearchIndex, }; use utils::chained_bytes::ChainedBytes; impl ArchivedMessageMetadata { pub fn index_document( &self, account_id: u32, document_id: u32, raw_message: &[u8], index_fields: &AHashSet, default_language: Language, ) -> IndexDocument { let mut detector = LanguageDetector::new(); let mut language = Language::Unknown; let message_contents = &self.contents[0]; let mut document = IndexDocument::new(SearchIndex::Email) .with_account_id(account_id) .with_document_id(document_id); let raw_message = ChainedBytes::new(self.raw_headers.as_ref()).with_last( raw_message .get(self.blob_body_offset.to_native() as usize..) .unwrap_or_default(), ); if index_fields.is_empty() || index_fields.contains(&SearchField::Email(EmailSearchField::ReceivedAt)) { document.index_unsigned( SearchField::Email(EmailSearchField::ReceivedAt), self.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK, ); } if index_fields.is_empty() || index_fields.contains(&SearchField::Email(EmailSearchField::Size)) { document.index_unsigned( SearchField::Email(EmailSearchField::Size), raw_message.len() as u32, ); } for (part_id, part) in message_contents .parts .iter() .take(MAX_MESSAGE_PARTS) .enumerate() { let part_language = part.language().unwrap_or(language); if part_id == 0 { language = part_language; for header in part.headers.iter().rev() { match &header.name { ArchivedMetadataHeaderName::From => { if index_fields.is_empty() || index_fields .contains(&SearchField::Email(EmailSearchField::From)) { header.value.visit_addresses(|_, value| { document.index_text( SearchField::Email(EmailSearchField::From), value, Language::None, ); }); } } ArchivedMetadataHeaderName::To => { if index_fields.is_empty() || index_fields.contains(&SearchField::Email(EmailSearchField::To)) { header.value.visit_addresses(|_, value| { document.index_text( SearchField::Email(EmailSearchField::To), value, Language::None, ); }); } } ArchivedMetadataHeaderName::Cc => { if index_fields.is_empty() || index_fields.contains(&SearchField::Email(EmailSearchField::Cc)) { header.value.visit_addresses(|_, value| { document.index_text( SearchField::Email(EmailSearchField::Cc), value, Language::None, ); }); } } ArchivedMetadataHeaderName::Bcc => { if index_fields.is_empty() || index_fields.contains(&SearchField::Email(EmailSearchField::Bcc)) { header.value.visit_addresses(|_, value| { document.index_text( SearchField::Email(EmailSearchField::Bcc), value, Language::None, ); }); } } ArchivedMetadataHeaderName::Subject => { if (index_fields.is_empty() || index_fields .contains(&SearchField::Email(EmailSearchField::Subject))) && let Some(subject) = header.value.as_text() { let subject = thread_name(subject); if part_language.is_unknown() { detector.detect(subject, MIN_LANGUAGE_SCORE); } document.index_text( SearchField::Email(EmailSearchField::Subject), subject, part_language, ); } } ArchivedMetadataHeaderName::Date => { if (index_fields.is_empty() || index_fields .contains(&SearchField::Email(EmailSearchField::SentAt))) && let Some(date) = header.value.as_datetime() { document.index_integer( SearchField::Email(EmailSearchField::SentAt), DateTime::from(date).to_timestamp(), ); } } _ => { #[cfg(not(feature = "test_mode"))] let index_headers = index_fields .contains(&SearchField::Email(EmailSearchField::Headers)); #[cfg(feature = "test_mode")] let index_headers = true; if index_headers { let mut value = String::new(); match &header.value { ArchivedMetadataHeaderValue::AddressList(_) | ArchivedMetadataHeaderValue::AddressGroup(_) => { header.value.visit_addresses(|_, addr| { if !value.is_empty() { value.push(' '); } value.push_str(addr); }); } ArchivedMetadataHeaderValue::Text(_) | ArchivedMetadataHeaderValue::TextList(_) => { header.value.visit_text(|text| { if !value.is_empty() { value.push(' '); } value.push_str(text); }); } _ => { if (matches!( header.value, ArchivedMetadataHeaderValue::ContentType(_) ) || matches!( header.name, ArchivedMetadataHeaderName::Received )) && let Some(header) = raw_message.get(header.value_range()) { let header = std::str::from_utf8(header.as_ref()) .unwrap_or_default(); for word in WordTokenizer::new(header, MAX_TOKEN_LENGTH) { if !value.is_empty() { value.push(' '); } value.push_str(word.word.as_ref()); } } } } document.insert_key_value( EmailSearchField::Headers, header.name.as_str(), value, ); } } } } } let part_id = part_id as u16; match &part.body { ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => { let text = match (part.decode_contents(&raw_message), &part.body) { (DecodedPartContent::Text(text), ArchivedMetadataPartType::Text) => text, (DecodedPartContent::Text(html), ArchivedMetadataPartType::Html) => { html_to_text(html.as_ref()).into() } _ => unreachable!(), }; if message_contents.is_html_part(part_id) || message_contents.is_text_part(part_id) { if index_fields.is_empty() || index_fields.contains(&SearchField::Email(EmailSearchField::Body)) { if part_language.is_unknown() { detector.detect(text.as_ref(), MIN_LANGUAGE_SCORE); } document.index_text( SearchField::Email(EmailSearchField::Body), text.as_ref(), part_language, ); } } else if index_fields.is_empty() || index_fields.contains(&SearchField::Email(EmailSearchField::Attachment)) { if part_language.is_unknown() { detector.detect(text.as_ref(), MIN_LANGUAGE_SCORE); } document.index_text( SearchField::Email(EmailSearchField::Attachment), text.as_ref(), part_language, ); } } ArchivedMetadataPartType::Message(nested_message_id) if index_fields.is_empty() || index_fields .contains(&SearchField::Email(EmailSearchField::Attachment)) => { let nested_message = self.message_id(*nested_message_id); let nested_message_language = nested_message .root_part() .language() .unwrap_or(Language::Unknown); if let Some(ArchivedMetadataHeaderValue::Text(subject)) = nested_message .root_part() .header_value(&MetadataHeaderName::Subject) { if nested_message_language.is_unknown() { detector.detect(subject.as_ref(), MIN_LANGUAGE_SCORE); } document.index_text( SearchField::Email(EmailSearchField::Attachment), subject.as_ref(), nested_message_language, ); } for sub_part in nested_message.parts.iter().take(MAX_MESSAGE_PARTS) { let language = sub_part.language().unwrap_or(nested_message_language); match &sub_part.body { ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => { let text = match ( sub_part.decode_contents(&raw_message), &sub_part.body, ) { ( DecodedPartContent::Text(text), ArchivedMetadataPartType::Text, ) => text, ( DecodedPartContent::Text(html), ArchivedMetadataPartType::Html, ) => html_to_text(html.as_ref()).into(), _ => unreachable!(), }; if language.is_unknown() { detector.detect(text.as_ref(), MIN_LANGUAGE_SCORE); } document.index_text( SearchField::Email(EmailSearchField::Attachment), text.as_ref(), language, ); } _ => (), } } } _ => {} } } #[cfg(not(feature = "test_mode"))] document.set_unknown_language( detector .most_frequent_language() .unwrap_or(default_language), ); #[cfg(feature = "test_mode")] document.set_unknown_language(default_language); let has_attachment = document.has_field(&(SearchField::Email(EmailSearchField::Attachment))); document.index_bool(EmailSearchField::HasAttachment, has_attachment); document } } ================================================ FILE: crates/email/src/message/ingest.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::crypto::{EncryptMessage, EncryptMessageError}; use crate::{ cache::{MessageCacheFetch, email::MessageCacheAccess, mailbox::MailboxCacheAccess}, mailbox::{INBOX_ID, JUNK_ID, SENT_ID, TRASH_ID, UidMailbox}, message::{ crypto::EncryptionParams, index::{IndexMessage, extractors::VisitText}, metadata::{MessageData, MessageMetadata}, }, }; use common::{Server, auth::AccessToken}; use directory::Permission; use groupware::{ calendar::itip::{ItipIngest, ItipIngestError}, scheduling::{ItipError, ItipMessages}, }; use mail_parser::{ DateTime, Header, HeaderName, HeaderValue, Message, MessageParser, MimeHeaders, PartType, parsers::fields::thread::thread_name, }; use std::{borrow::Cow, cmp::Ordering, fmt::Write, time::Instant}; use std::{future::Future, hash::Hasher}; use store::write::{AlignedBytes, Archive}; use store::{ IndexKeyPrefix, IterateParams, U32_LEN, ValueKey, ahash::{AHashMap, AHashSet}, write::{ AssignedId, AssignedIds, BatchBuilder, BlobLink, BlobOp, IndexPropertyClass, SearchIndex, TaskEpoch, TaskQueueClass, ValueClass, key::DeserializeBigEndian, now, }, }; use trc::{AddContext, MessageIngestEvent, SpamEvent}; use types::{ blob::{BlobClass, BlobId}, blob_hash::BlobHash, collection::{Collection, SyncCollection}, field::{ContactField, EmailField, MailboxField, PrincipalField}, keyword::Keyword, special_use::SpecialUse, }; use utils::{cheeky_hash::CheekyHash, sanitize_email}; #[derive(Default)] pub struct IngestedEmail { pub document_id: u32, pub thread_id: u32, pub change_id: u64, pub blob_id: BlobId, pub size: usize, pub imap_uids: Vec, } pub struct IngestEmail<'x> { pub raw_message: &'x [u8], pub blob_hash: Option<&'x BlobHash>, pub message: Option>, pub access_token: &'x AccessToken, pub mailbox_ids: Vec, pub keywords: Vec, pub received_at: Option, pub source: IngestSource<'x>, pub session_id: u64, } #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum IngestSource<'x> { Smtp { deliver_to: &'x str, is_sender_authenticated: bool, is_spam: bool, }, Jmap { train_classifier: bool, }, Imap { train_classifier: bool, }, Restore, } pub trait EmailIngest: Sync + Send { fn email_ingest( &self, params: IngestEmail, ) -> impl Future> + Send; fn find_thread_id( &self, account_id: u32, thread_name: &str, message_ids: &[CheekyHash], ) -> impl Future> + Send; fn assign_email_ids( &self, account_id: u32, mailbox_ids: impl IntoIterator + Sync + Send, generate_email_id: bool, ) -> impl Future + 'static>> + Send; fn add_account_spam_sample( &self, batch: &mut BatchBuilder, account_id: u32, document_id: u32, is_spam: bool, span_id: u64, ) -> impl Future> + Send; fn add_spam_sample( &self, batch: &mut BatchBuilder, hash: BlobHash, is_spam: bool, hold_sample: bool, span_id: u64, ); } pub struct ThreadResult { pub thread_id: Option, pub thread_hash: CheekyHash, pub merge_ids: Vec, pub duplicate_ids: Vec, } impl EmailIngest for Server { #[allow(clippy::blocks_in_conditions)] async fn email_ingest(&self, mut params: IngestEmail<'_>) -> trc::Result { // Check quota let start_time = Instant::now(); let account_id = params.access_token.primary_id; let tenant_id = params.access_token.tenant.map(|t| t.id); let mut raw_message_len = params.raw_message.len() as u64; let resource_token = params.access_token.as_resource_token(); self.has_available_quota(&resource_token, raw_message_len) .await .caused_by(trc::location!())?; // Parse message let mut raw_message = Cow::from(params.raw_message); let mut message = params.message.ok_or_else(|| { trc::EventType::MessageIngest(trc::MessageIngestEvent::Error) .ctx(trc::Key::Code, 550) .ctx(trc::Key::Reason, "Failed to parse e-mail message.") })?; // Obtain message references and thread name let mut message_id = None; let mut message_ids = Vec::new(); let thread_result = { let mut subject = ""; for header in message.root_part().headers().iter().rev() { match &header.name { HeaderName::MessageId => header.value.visit_text(|id| { if !id.is_empty() { if message_id.is_none() { message_id = id.to_string().into(); } message_ids.push(CheekyHash::new(id.as_bytes())); } }), HeaderName::InReplyTo | HeaderName::References | HeaderName::ResentMessageId => { header.value.visit_text(|id| { if !id.is_empty() { message_ids.push(CheekyHash::new(id.as_bytes())); } }); } HeaderName::Subject if subject.is_empty() => { subject = thread_name(match &header.value { HeaderValue::Text(text) => text.as_ref(), HeaderValue::TextList(list) if !list.is_empty() => { list.first().unwrap().as_ref() } _ => "", }); } _ => (), } } message_ids.sort_unstable(); message_ids.dedup(); self.find_thread_id(account_id, subject, &message_ids) .await? }; // Skip duplicate messages for SMTP ingestion if !thread_result.duplicate_ids.is_empty() && params.source.is_smtp() { // Fetch cached messages let cache = self .get_cached_messages(account_id) .await .caused_by(trc::location!())?; // Skip duplicate messages let target_mailbox_id = params.mailbox_ids.first().copied().unwrap_or(INBOX_ID); if !cache .in_mailboxes(&[target_mailbox_id, JUNK_ID]) .any(|m| thread_result.duplicate_ids.contains(&m.document_id)) { trc::event!( MessageIngest(MessageIngestEvent::Duplicate), SpanId = params.session_id, AccountId = account_id, MessageId = message_id, ); return Ok(IngestedEmail { document_id: 0, thread_id: 0, change_id: u64::MAX, blob_id: BlobId::default(), imap_uids: Vec::new(), size: 0, }); } } // Spam classification and training let mut train_spam = None; let mut extra_headers = String::new(); let mut extra_headers_parsed = Vec::new(); let mut itip_messages = Vec::new(); let is_spam = match params.source { IngestSource::Smtp { deliver_to, is_sender_authenticated, mut is_spam, } => { // Add delivered to header if self.core.smtp.session.data.add_delivered_to { extra_headers = format!("Delivered-To: {deliver_to}\r\n"); extra_headers_parsed.push(Header { name: HeaderName::Other("Delivered-To".into()), value: HeaderValue::Text(deliver_to.into()), offset_field: 0, offset_start: 13, offset_end: extra_headers.len() as u32, }); } // Spam training on confirmed false positives if self.core.spam.enabled { let mut overridden = None; // If the message is classified as spam, check whether the // sender address is present in the user's address book. if is_spam && self.core.spam.card_is_ham && let Some(sender) = message .from() .and_then(|s| s.first()) .and_then(|s| s.address()) .and_then(sanitize_email) && sender != deliver_to && is_sender_authenticated && self .document_exists( account_id, Collection::ContactCard, ContactField::Email, sender.as_bytes(), ) .await .caused_by(trc::location!())? { is_spam = false; if self .core .spam .classifier .as_ref() .is_some_and(|c| c.auto_learn_card_is_ham) { train_spam = Some(false); } overridden = Some("card-exists"); } // Check if the message is a trusted reply to a previous message if is_spam && self.core.spam.trusted_reply && let Some(thread_id) = thread_result.thread_id { let cache = self .get_cached_messages(account_id) .await .caused_by(trc::location!())?; let sent_folder_id = cache .mailbox_by_role(&SpecialUse::Sent) .map(|m| m.document_id) .unwrap_or(SENT_ID); if cache .in_thread(thread_id) .any(|m| m.mailboxes.iter().any(|mb| mb.mailbox_id == sent_folder_id)) { is_spam = false; if self .core .spam .classifier .as_ref() .is_some_and(|c| c.auto_learn_reply_ham) { train_spam = Some(false); } overridden = Some("trusted-reply"); } } // Add Spam-Status header const HEADER: &str = "X-Spam-Status"; let offset_field = extra_headers.len(); let offset_start = offset_field + HEADER.len() + 1; let result = if is_spam { "Yes" } else { "No" }; if let Some(reason) = overridden { let _ = write!( &mut extra_headers, "{HEADER}: {result}, reason={reason}\r\n", ); } else { let _ = write!(&mut extra_headers, "{HEADER}: {result}\r\n",); } extra_headers_parsed.push(Header { name: HeaderName::Other(HEADER.into()), value: HeaderValue::Text( extra_headers[offset_start + 1..extra_headers.len() - 2] .to_string() .into(), ), offset_field: offset_field as u32, offset_start: offset_start as u32, offset_end: extra_headers.len() as u32, }); if is_spam && params.mailbox_ids == [INBOX_ID] { params.mailbox_ids[0] = JUNK_ID; params.keywords.push(Keyword::Junk); } } // iMIP processing if self.core.groupware.itip_enabled && !is_spam && is_sender_authenticated && params .access_token .has_permission(Permission::CalendarSchedulingReceive) { let mut sender = None; for part in &message.parts { if part.content_type().is_some_and(|ct| { ct.ctype().eq_ignore_ascii_case("text") && ct .subtype() .is_some_and(|st| st.eq_ignore_ascii_case("calendar")) && ct.has_attribute("method") }) && let Some(itip_message) = part.text_contents() { if itip_message.len() < self.core.groupware.itip_inbound_max_ical_size { if let Some(sender) = sender.get_or_insert_with(|| { message .from() .and_then(|s| s.first()) .and_then(|s| s.address()) .and_then(sanitize_email) }) { match self .itip_ingest( params.access_token, &resource_token, sender, deliver_to, itip_message, ) .await { Ok(message) => { if let Some(message) = message { itip_messages.push(message); } trc::event!( Calendar(trc::CalendarEvent::ItipMessageReceived), SpanId = params.session_id, From = sender.to_string(), AccountId = account_id, ); } Err(ItipIngestError::Message(itip_error)) => { match itip_error { ItipError::NothingToSend | ItipError::OtherSchedulingAgent => (), err => { trc::event!( Calendar( trc::CalendarEvent::ItipMessageError ), SpanId = params.session_id, From = sender.to_string(), AccountId = account_id, Details = err.to_string(), ) } } } Err(ItipIngestError::Internal(err)) => { trc::error!(err.caused_by(trc::location!())); } } } } else { trc::event!( Calendar(trc::CalendarEvent::ItipMessageError), SpanId = params.session_id, From = message .from() .and_then(|a| a.first()) .and_then(|a| a.address()) .map(|a| a.to_string()), AccountId = account_id, Details = "iMIP message too large", Limit = self.core.groupware.itip_inbound_max_ical_size, Size = itip_message.len(), ) } } } } is_spam } IngestSource::Jmap { train_classifier } | IngestSource::Imap { train_classifier } => { // Determine spam training if train_classifier && self.core.spam.enabled { if params.keywords.contains(&Keyword::Junk) { train_spam = Some(true); } else if params.keywords.contains(&Keyword::NotJunk) { if !params.mailbox_ids.contains(&TRASH_ID) { train_spam = Some(false); } } else if params.mailbox_ids[0] == JUNK_ID { train_spam = Some(true); } else if params.mailbox_ids[0] == INBOX_ID { train_spam = Some(false); } } // Set receivedAt if not present if params.received_at.is_none() { params.received_at = message .root_part() .headers() .iter() .filter_map(|header| { if let (HeaderName::Received, HeaderValue::Received(received)) = (&header.name, &header.value) { received.date.map(|dt| dt.to_timestamp() as u64) } else { None } }) .max(); } false } _ => false, }; // Encrypt message let do_encrypt = match params.source { IngestSource::Jmap { .. } | IngestSource::Imap { .. } => { self.core.jmap.encrypt && self.core.jmap.encrypt_append } IngestSource::Smtp { .. } => self.core.jmap.encrypt, IngestSource::Restore => false, }; let is_encrypted = if do_encrypt && !message.is_encrypted() && let Some(encrypt_params_) = self .store() .get_value::>(ValueKey::property( account_id, Collection::Principal, 0, PrincipalField::EncryptionKeys, )) .await .caused_by(trc::location!())? { let encrypt_params = encrypt_params_ .unarchive::() .caused_by(trc::location!())?; match message.encrypt(encrypt_params).await { Ok(new_raw_message) => { raw_message = Cow::from(new_raw_message); raw_message_len = raw_message.len() as u64; message = MessageParser::default() .parse(raw_message.as_ref()) .ok_or_else(|| { trc::EventType::MessageIngest(trc::MessageIngestEvent::Error) .ctx(trc::Key::Code, 550) .ctx( trc::Key::Reason, "Failed to parse encrypted e-mail message.", ) })?; // Disable spam training if requested if !encrypt_params.can_train_spam_filter() { train_spam = None; } // Remove contents from parsed message for part in &mut message.parts { match &mut part.body { PartType::Text(txt) | PartType::Html(txt) => { *txt = Cow::from(""); } PartType::Binary(bin) | PartType::InlineBinary(bin) => { *bin = Cow::from(&[][..]); } PartType::Message(_) => { part.body = PartType::Binary(Cow::from(&[][..])); } PartType::Multipart(_) => (), } } true } Err(EncryptMessageError::Error(err)) => { trc::bail!( trc::StoreEvent::CryptoError .into_err() .caused_by(trc::location!()) .reason(err) ); } _ => unreachable!(), } } else { false }; // Store blob let (blob_hash, blob_hold) = if !is_encrypted && let Some(blob_hash) = params.blob_hash { (blob_hash.clone(), None) } else { self.put_temporary_blob(account_id, raw_message.as_ref(), 60) .await .map(|(hash, op)| (hash, Some(op))) .caused_by(trc::location!())? }; // Assign IMAP UIDs let mut mailbox_ids = Vec::with_capacity(params.mailbox_ids.len()); let mut imap_uids = Vec::with_capacity(params.mailbox_ids.len()); let mut ids = self .assign_email_ids(account_id, params.mailbox_ids.iter().copied(), true) .await .caused_by(trc::location!())?; let document_id = ids.next().unwrap(); for (uid, mailbox_id) in ids.zip(params.mailbox_ids.iter().copied()) { mailbox_ids.push(UidMailbox::new(mailbox_id, uid)); imap_uids.push(uid); } // Build write batch let mut batch = BatchBuilder::new(); let mailbox_ids_event = mailbox_ids .iter() .map(|m| trc::Value::from(m.mailbox_id)) .collect::>(); batch.with_account_id(account_id); // Determine thread id let thread_id = if let Some(thread_id) = thread_result.thread_id { thread_id } else { batch .with_collection(Collection::Thread) .with_document(document_id) .log_container_insert(SyncCollection::Thread); document_id }; let data = MessageData { mailboxes: mailbox_ids.into_boxed_slice(), keywords: params.keywords.into_boxed_slice(), thread_id, size: (message.raw_message.len() + extra_headers.len()) as u32, }; batch .with_collection(Collection::Email) .with_document(document_id) .index_message( tenant_id, message, extra_headers.into_bytes(), extra_headers_parsed, blob_hash.clone(), data, params.received_at.unwrap_or_else(now), ) .caused_by(trc::location!())? .set( ValueClass::IndexProperty(IndexPropertyClass::Hash { property: EmailField::Threading.into(), hash: thread_result.thread_hash, }), ThreadInfo::serialize(thread_id, &message_ids), ) .set( ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { index: SearchIndex::Email, due: TaskEpoch::now(), is_insert: true, }), vec![], ); if let Some(blob_hold) = blob_hold { batch.clear(blob_hold); } // Merge threads if necessary if let Some(merge_threads) = MergeThreadIds::new(thread_result).serialize() { batch.set( ValueClass::TaskQueue(TaskQueueClass::MergeThreads { due: TaskEpoch::now(), }), merge_threads, ); } // Request spam training if let Some(learn_spam) = train_spam { self.add_spam_sample( &mut batch, params.blob_hash.unwrap_or(&blob_hash).clone(), learn_spam, !is_encrypted, params.session_id, ); } // Add iTIP responses to batch if !itip_messages.is_empty() { ItipMessages::new(itip_messages) .queue(&mut batch) .caused_by(trc::location!())?; } // Insert and obtain ids let change_id = self .store() .write(batch.build_all()) .await .caused_by(trc::location!())? .last_change_id(account_id)?; // Request FTS index self.notify_task_queue(); trc::event!( MessageIngest(match params.source { IngestSource::Smtp { .. } => if !is_spam { MessageIngestEvent::Ham } else { MessageIngestEvent::Spam }, IngestSource::Jmap { .. } | IngestSource::Restore => MessageIngestEvent::JmapAppend, IngestSource::Imap { .. } => MessageIngestEvent::ImapAppend, }), SpanId = params.session_id, AccountId = account_id, DocumentId = document_id, MailboxId = mailbox_ids_event, BlobId = blob_hash.to_hex(), ChangeId = change_id, MessageId = message_id, Size = raw_message_len, Elapsed = start_time.elapsed(), ); Ok(IngestedEmail { document_id, thread_id, change_id, blob_id: BlobId { hash: blob_hash, class: BlobClass::Linked { account_id, collection: Collection::Email.into(), document_id, }, section: None, }, size: raw_message_len as usize, imap_uids, }) } async fn find_thread_id( &self, account_id: u32, thread_name: &str, message_ids: &[CheekyHash], ) -> trc::Result { let mut result = ThreadResult { thread_id: None, thread_hash: CheekyHash::new(if !thread_name.is_empty() { thread_name } else { "!" }), merge_ids: vec![], duplicate_ids: vec![], }; if message_ids.is_empty() { return Ok(result); } // Find thread ids let key_len = IndexKeyPrefix::len() + result.thread_hash.len() + U32_LEN; let document_id_pos = key_len - U32_LEN; let mut thread_merge = ThreadMerge::new(); self.store() .iterate( IterateParams::new( ValueKey { account_id, collection: Collection::Email.into(), document_id: 0, class: ValueClass::IndexProperty(IndexPropertyClass::Hash { property: EmailField::Threading.into(), hash: result.thread_hash, }), }, ValueKey { account_id, collection: Collection::Email.into(), document_id: u32::MAX, class: ValueClass::IndexProperty(IndexPropertyClass::Hash { property: EmailField::Threading.into(), hash: result.thread_hash, }), }, ) .ascending(), |key, value| { if key.len() == key_len { // Find matching references let references = value.get(U32_LEN..).unwrap_or_default(); if has_message_id(message_ids, references) { let document_id = key.deserialize_be_u32(document_id_pos)?; let thread_id = value.deserialize_be_u32(0)?; if message_ids.len() == 1 || (message_ids.len() == references.len() / CheekyHash::HASH_SIZE && references .chunks_exact(CheekyHash::HASH_SIZE) .zip(message_ids.iter()) .all(|(a, b)| a == b.as_raw_bytes())) { result.duplicate_ids.push(document_id); } thread_merge.add(thread_id, document_id); } } Ok(true) }, ) .await .caused_by(trc::location!())?; match thread_merge.num_thread_ids() { 0 => Ok(result), 1 => { // Happy path, only one thread id result.thread_id = thread_merge.thread_ids().next().copied(); Ok(result) } _ => { // Multiple thread ids that this message belongs to, merge them let thread_merge = thread_merge.merge(); result.merge_ids = thread_merge.merge_ids; result.thread_id = Some(thread_merge.thread_id); Ok(result) } } } async fn assign_email_ids( &self, account_id: u32, mailbox_ids: impl IntoIterator + Sync + Send, generate_email_id: bool, ) -> trc::Result + 'static> { // Increment UID next let mut batch = BatchBuilder::new(); batch.with_account_id(account_id); let mut expected_ids = 0; if generate_email_id { batch .with_collection(Collection::Email) .add_and_get(ValueClass::DocumentId, 1); expected_ids += 1; } batch.with_collection(Collection::Mailbox); for mailbox_id in mailbox_ids { batch .with_document(mailbox_id) .add_and_get(MailboxField::UidCounter, 1); expected_ids += 1; } let ids = if expected_ids > 0 { self.core.storage.data.write(batch.build_all()).await? } else { AssignedIds::default() }; if ids.ids.len() == expected_ids { Ok(ids.ids.into_iter().map(|id| match id { AssignedId::Counter(id) => id as u32, AssignedId::ChangeId(_) => unreachable!(), })) } else { Err(trc::StoreEvent::UnexpectedError .caused_by(trc::location!()) .ctx(trc::Key::Reason, "No all document ids were generated")) } } async fn add_account_spam_sample( &self, batch: &mut BatchBuilder, account_id: u32, document_id: u32, is_spam: bool, span_id: u64, ) -> trc::Result<()> { if self.core.spam.classifier.is_some() && let Some(archive) = self .store() .get_value::>(ValueKey::property( account_id, Collection::Email, document_id, EmailField::Metadata, )) .await .caused_by(trc::location!())? { let metadata = archive .to_unarchived::() .caused_by(trc::location!())?; self.add_spam_sample( batch, (&metadata.inner.blob_hash).into(), is_spam, true, span_id, ); } Ok(()) } fn add_spam_sample( &self, batch: &mut BatchBuilder, hash: BlobHash, is_spam: bool, hold_sample: bool, span_id: u64, ) { if let Some(config) = &self.core.spam.classifier { let mut dt = DateTime::from_timestamp(now() as i64); dt.hour = 0; dt.minute = 0; dt.second = 0; let until = dt.to_timestamp() as u64 + config.hold_samples_for; batch .set( BlobOp::Link { hash: hash.clone(), to: BlobLink::Temporary { until }, }, vec![BlobLink::SPAM_SAMPLE_LINK], ) .set( BlobOp::SpamSample { hash, until }, vec![u8::from(is_spam), u8::from(hold_sample)], ); trc::event!( Spam(SpamEvent::TrainSampleAdded), AccountId = batch.last_account_id(), Details = if is_spam { "spam" } else { "ham" }, Expires = trc::Value::Timestamp(until), SpanId = span_id, ); } } } fn has_message_id(a: &[CheekyHash], b: &[u8]) -> bool { let mut i = 0; let mut j = 0; let a_len = a.len(); let b_len = b.len() / CheekyHash::HASH_SIZE; while i < a_len && j < b_len { match a[i] .as_raw_bytes() .as_slice() .cmp(&b[j * CheekyHash::HASH_SIZE..(j + 1) * CheekyHash::HASH_SIZE]) { std::cmp::Ordering::Equal => return true, std::cmp::Ordering::Less => i += 1, std::cmp::Ordering::Greater => j += 1, } } false } impl IngestSource<'_> { pub fn is_smtp(&self) -> bool { matches!(self, Self::Smtp { .. }) } } #[derive(Debug, Clone, PartialEq, Eq)] pub struct MergeThreadIds { pub thread_hash: CheekyHash, pub merge_ids: T, } impl MergeThreadIds> { pub(crate) fn new(thread_result: ThreadResult) -> Self { Self { thread_hash: thread_result.thread_hash, merge_ids: thread_result.merge_ids, } } pub(crate) fn serialize(&self) -> Option> { if !self.merge_ids.is_empty() { let mut buf = Vec::with_capacity(self.thread_hash.len() + self.merge_ids.len() * U32_LEN); buf.extend_from_slice(self.thread_hash.as_bytes()); for id in &self.merge_ids { buf.extend_from_slice(&id.to_be_bytes()); } Some(buf) } else { None } } } impl MergeThreadIds> { pub fn deserialize(bytes: &[u8]) -> Option { if !bytes.is_empty() { let thread_hash = CheekyHash::deserialize(bytes)?; let mut merge_ids = AHashSet::with_capacity(((bytes.len() - thread_hash.len()) / U32_LEN) + 1); let mut start_offset = thread_hash.len(); while let Some(id_bytes) = bytes.get(start_offset..start_offset + U32_LEN) { merge_ids.insert(u32::from_be_bytes(id_bytes.try_into().ok()?)); start_offset += U32_LEN; } Some(Self { thread_hash, merge_ids, }) } else { None } } } impl std::hash::Hash for MergeThreadIds> { fn hash(&self, state: &mut H) { self.thread_hash.hash(state); self.merge_ids.len().hash(state); } } pub struct ThreadInfo; impl ThreadInfo { pub fn serialize(thread_id: u32, ref_ids: &[CheekyHash]) -> Vec { let mut buf = Vec::with_capacity(U32_LEN + 1 + ref_ids.len() * CheekyHash::HASH_SIZE); buf.extend_from_slice(&thread_id.to_be_bytes()); for ref_id in ref_ids { buf.extend_from_slice(ref_id.as_raw_bytes()); } buf } } pub struct ThreadMerge { entries: AHashMap>, } pub struct ThreadMergeResult { pub thread_id: u32, pub merge_ids: Vec, } impl ThreadMerge { #[allow(clippy::new_without_default)] pub fn new() -> Self { Self { entries: AHashMap::with_capacity(8), } } pub fn add(&mut self, thread_id: u32, document_id: u32) { self.entries.entry(thread_id).or_default().push(document_id); } pub fn num_thread_ids(&self) -> usize { self.entries.len() } pub fn thread_ids(&self) -> impl Iterator { self.entries.keys() } pub fn thread_groups(&self) -> impl Iterator)> { self.entries.iter() } pub fn merge_thread_id(&self) -> u32 { let mut max_thread_id = u32::MAX; let mut max_count = 0; for (thread_id, ids) in &self.entries { match ids.len().cmp(&max_count) { Ordering::Greater => { max_count = ids.len(); max_thread_id = *thread_id; } Ordering::Equal => { if *thread_id < max_thread_id { max_thread_id = *thread_id; } } Ordering::Less => (), } } max_thread_id } pub fn merge(self) -> ThreadMergeResult { let mut max_thread_id = u32::MAX; let mut max_count = 0; let mut merge_ids = Vec::with_capacity(self.entries.len()); for (thread_id, ids) in self.entries { match ids.len().cmp(&max_count) { Ordering::Greater => { max_count = ids.len(); max_thread_id = thread_id; } Ordering::Equal => { if thread_id < max_thread_id { max_thread_id = thread_id; } } Ordering::Less => (), } merge_ids.push(thread_id); } ThreadMergeResult { thread_id: max_thread_id, merge_ids, } } } ================================================ FILE: crates/email/src/message/metadata.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::mailbox::{ArchivedUidMailbox, UidMailbox}; use common::storage::index::IndexableAndSerializableObject; use mail_parser::{ Addr, Address, Attribute, ContentType, DateTime, Encoding, Group, HeaderName, HeaderValue, PartType, decoders::{ base64::base64_decode, charsets::map::charset_decoder, quoted_printable::quoted_printable_decode, }, }; use rkyv::{boxed::ArchivedBox, rend::u16_le}; use std::{borrow::Cow, collections::VecDeque, ops::Range}; use types::{ blob_hash::BlobHash, keyword::{ArchivedKeyword, Keyword}, }; use utils::chained_bytes::ChainedBytes; #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] pub struct MessageData { pub mailboxes: Box<[UidMailbox]>, pub keywords: Box<[Keyword]>, pub thread_id: u32, pub size: u32, } #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] pub struct MessageMetadata { pub contents: Box<[MessageMetadataContents]>, pub rcvd_attach: u64, pub blob_hash: BlobHash, pub blob_body_offset: u32, pub preview: Box, pub raw_headers: Box<[u8]>, } pub const MESSAGE_HAS_ATTACHMENT: u64 = 1 << 63; pub const MESSAGE_RECEIVED_MASK: u64 = !MESSAGE_HAS_ATTACHMENT; impl IndexableAndSerializableObject for MessageData { fn is_versioned() -> bool { true } } #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] pub struct MessageMetadataContents { pub html_body: Box<[u16]>, pub text_body: Box<[u16]>, pub attachments: Box<[u16]>, pub parts: Box<[MessageMetadataPart]>, } #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] pub struct MessageMetadataPart { pub headers: Box<[MetadataHeader]>, pub body: MetadataPartType, pub flags: u32, pub offset_header: u32, pub offset_body: u32, pub offset_end: u32, } pub const PART_ENCODING_BASE64: u32 = 1 << 31; pub const PART_ENCODING_QP: u32 = 1 << 30; pub const PART_ENCODING_PROBLEM: u32 = 1 << 29; pub const PART_SIZE_MASK: u32 = !(PART_ENCODING_BASE64 | PART_ENCODING_QP | PART_ENCODING_PROBLEM); #[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] #[rkyv(compare(PartialEq))] pub struct MetadataHeader { pub name: MetadataHeaderName, pub value: MetadataHeaderValue, pub base_offset: u32, pub start: u16, pub end: u16, } #[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] #[rkyv(compare(PartialEq))] pub enum MetadataHeaderName { Other(Box), Subject, From, To, Cc, Date, Bcc, ReplyTo, Sender, Comments, InReplyTo, Keywords, Received, MessageId, References, ReturnPath, MimeVersion, ContentDescription, ContentId, ContentLanguage, ContentLocation, ContentTransferEncoding, ContentType, ContentDisposition, ResentTo, ResentFrom, ResentBcc, ResentCc, ResentSender, ResentDate, ResentMessageId, ListArchive, ListHelp, ListId, ListOwner, ListPost, ListSubscribe, ListUnsubscribe, DkimSignature, ArcAuthenticationResults, ArcMessageSignature, ArcSeal, // Delivery/Routing DeliveredTo, XOriginalTo, ReturnReceiptTo, DispositionNotificationTo, ErrorsTo, // Authentication AuthenticationResults, ReceivedSpf, // Spam/Virus XSpamStatus, XSpamScore, XSpamFlag, XSpamResult, // Priority Importance, Priority, XPriority, XMSMailPriority, // Client/Agent XMailer, UserAgent, XMimeOLE, // Network/Origin XOriginatingIp, XForwardedTo, XForwardedFor, // Auto-response AutoSubmitted, XAutoResponseSuppress, Precedence, // Organization/Threading Organization, ThreadIndex, ThreadTopic, // List (additional) ListUnsubscribePost, FeedbackId, } #[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] #[rkyv(compare(PartialEq))] pub enum MetadataHeaderValue { AddressList(Box<[MetadataAddress]>), AddressGroup(Box<[MetadataAddressGroup]>), Text(Box), TextList(Box<[Box]>), DateTime(MetadataDateTime), ContentType(MetadataContentType), Empty, } #[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] #[rkyv(compare(PartialEq))] pub struct MetadataDateTime { pub year: u16, pub month: u8, pub day: u8, pub hour: u8, pub minute: u8, pub second: u8, pub tz_hour: i8, pub tz_minute: u8, } #[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] #[rkyv(compare(PartialEq))] pub struct MetadataAddress { pub name: Option>, pub address: Option>, } #[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] #[rkyv(compare(PartialEq))] pub struct MetadataAddressGroup { pub name: Option>, pub addresses: Box<[MetadataAddress]>, } #[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] #[rkyv(compare(PartialEq))] pub struct MetadataContentType { pub c_type: Box, pub c_subtype: Option>, pub attributes: Box<[MetadataAttribute]>, } #[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] #[rkyv(compare(PartialEq))] pub struct MetadataAttribute { pub name: Box, pub value: Box, } #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] pub enum MetadataPartType { Text, Html, Binary, InlineBinary, Message(u16), Multipart(Box<[u16]>), } impl MessageMetadataContents { pub fn root_part(&self) -> &MessageMetadataPart { &self.parts[0] } } #[derive(Debug)] pub struct DecodedParts<'x> { pub raw_messages: Vec>, pub parts: Vec>, } #[derive(Debug)] pub enum DecodedRawMessage<'x> { Borrowed(ChainedBytes<'x>), Owned(Vec), } #[derive(Debug)] pub struct DecodedPart<'x> { pub message_id: usize, pub part_offset: usize, pub content: DecodedPartContent<'x>, } #[derive(Debug)] pub enum DecodedPartContent<'x> { Text(Cow<'x, str>), Binary(Cow<'x, [u8]>), } impl<'x> DecodedParts<'x> { #[inline] pub fn raw_message(&self, message_id: usize) -> Option<&DecodedRawMessage<'x>> { self.raw_messages.get(message_id) } #[inline] pub fn raw_message_section( &'_ self, message_id: usize, range: Range, ) -> Option> { self.raw_messages.get(message_id).and_then(|m| m.get(range)) } #[inline] pub fn part(&self, message_id: usize, part_offset: usize) -> Option<&DecodedPartContent<'x>> { self.parts .iter() .find(|p| p.message_id == message_id && p.part_offset == part_offset) .map(|p| &p.content) } #[inline] pub fn text_part(&self, message_id: usize, part_offset: usize) -> Option<&str> { self.part(message_id, part_offset).and_then(|p| match p { DecodedPartContent::Text(text) => Some(text.as_ref()), DecodedPartContent::Binary(_) => None, }) } #[inline] pub fn binary_part(&self, message_id: usize, part_offset: usize) -> Option<&[u8]> { self.part(message_id, part_offset).map(|p| match p { DecodedPartContent::Text(part) => part.as_bytes(), DecodedPartContent::Binary(binary) => binary.as_ref(), }) } } impl DecodedPartContent<'_> { pub fn as_bytes(&self) -> &[u8] { match self { DecodedPartContent::Text(text) => text.as_bytes(), DecodedPartContent::Binary(binary) => binary, } } #[allow(clippy::len_without_is_empty)] pub fn len(&self) -> usize { match self { DecodedPartContent::Text(text) => text.len(), DecodedPartContent::Binary(binary) => binary.len(), } } pub fn as_str(&self) -> &str { match self { DecodedPartContent::Text(text) => text, DecodedPartContent::Binary(binary) => std::str::from_utf8(binary).unwrap_or_default(), } } } impl<'x> DecodedRawMessage<'x> { pub fn get(&'_ self, index: Range) -> Option> { match self { DecodedRawMessage::Borrowed(bytes) => bytes.get(index), DecodedRawMessage::Owned(vec) => vec.get(index).map(Cow::Borrowed), } } } impl ArchivedMessageMetadata { #[inline(always)] pub fn message_id(&self, message_id: u16_le) -> &ArchivedMessageMetadataContents { &self.contents[u16::from(message_id) as usize] } pub fn decode_contents<'x>(&self, raw: ChainedBytes<'x>) -> DecodedParts<'x> { let mut result = DecodedParts { raw_messages: Vec::with_capacity(self.contents.len()), parts: Vec::new(), }; for _ in 0..self.contents.len() { result .raw_messages .push(DecodedRawMessage::Borrowed(raw.clone())); } for (message_id, contents) in self.contents.iter().enumerate() { for part in contents.parts.iter() { let part_offset = u32::from(part.offset_header) as usize; match &part.body { ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html | ArchivedMetadataPartType::Binary | ArchivedMetadataPartType::InlineBinary => { match result.raw_messages.get(message_id).unwrap() { DecodedRawMessage::Borrowed(bytes) => { result.parts.push(DecodedPart { message_id, part_offset, content: part.decode_contents(bytes), }); } DecodedRawMessage::Owned(bytes) => { result.parts.push(DecodedPart { message_id, part_offset, content: match part.decode_contents(&ChainedBytes::new(bytes)) { DecodedPartContent::Text(text) => { DecodedPartContent::Text(text.into_owned().into()) } DecodedPartContent::Binary(binary) => { DecodedPartContent::Binary(binary.into_owned().into()) } }, }); } } } ArchivedMetadataPartType::Message(nested_message_id) => { let sub_contents = if (part.flags & (PART_ENCODING_BASE64 | PART_ENCODING_QP)) != 0 { match result.raw_messages.get(message_id).unwrap() { DecodedRawMessage::Borrowed(bytes) => { part.contents(bytes).into_owned() } DecodedRawMessage::Owned(bytes) => { let bytes = ChainedBytes::new(bytes); part.contents(&bytes).into_owned() } } } else if let Some(DecodedRawMessage::Owned(bytes)) = result.raw_messages.get(message_id) { bytes.clone() } else { continue; }; result.raw_messages[usize::from(*nested_message_id)] = DecodedRawMessage::Owned(sub_contents); } _ => {} } } } result } } impl ArchivedMessageMetadataPart { pub fn contents<'x>(&self, raw_message: &ChainedBytes<'x>) -> Cow<'x, [u8]> { let bytes = raw_message.get(self.body_to_end()).unwrap_or_default(); if (self.flags & PART_ENCODING_BASE64) != 0 { base64_decode(bytes.as_ref()).unwrap_or_default().into() } else if (self.flags & PART_ENCODING_QP) != 0 { quoted_printable_decode(bytes.as_ref()) .unwrap_or_default() .into() } else { bytes } } #[inline(always)] pub fn body_to_end(&self) -> Range { (self.offset_body.to_native() as usize)..(self.offset_end.to_native() as usize) } #[inline(always)] pub fn header_to_end(&self) -> Range { self.offset_header.to_native() as usize..self.offset_end.to_native() as usize } #[inline(always)] pub fn header_to_body(&self) -> Range { self.offset_header.to_native() as usize..self.offset_body.to_native() as usize } pub fn decode_contents<'x>(&self, raw_message: &ChainedBytes<'x>) -> DecodedPartContent<'x> { let bytes = self.contents(raw_message); match self.body { ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => { DecodedPartContent::Text( match ( bytes, self.header_value(&MetadataHeaderName::ContentType) .and_then(|c| c.as_content_type()) .and_then(|ct| { ct.attribute("charset") .and_then(|c| charset_decoder(c.as_bytes())) }), ) { (Cow::Owned(vec), Some(charset_decoder)) => charset_decoder(&vec).into(), (Cow::Owned(vec), None) => String::from_utf8(vec) .unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned()) .into(), (Cow::Borrowed(bytes), Some(charset_decoder)) => { charset_decoder(bytes).into() } (Cow::Borrowed(bytes), None) => String::from_utf8_lossy(bytes), }, ) } ArchivedMetadataPartType::Binary => DecodedPartContent::Binary(bytes), ArchivedMetadataPartType::InlineBinary => DecodedPartContent::Binary(bytes), ArchivedMetadataPartType::Message(_) | ArchivedMetadataPartType::Multipart(_) => { unreachable!() } } } } pub fn build_metadata_contents( message: mail_parser::Message<'_>, ) -> Box<[MessageMetadataContents]> { let mut messages = VecDeque::from([message]); let mut message_id = 0; let mut contents = Vec::new(); while let Some(message) = messages.pop_front() { let mut parts = Vec::with_capacity(message.parts.len()); for part in message.parts { let (size, body) = match part.body { PartType::Text(contents) => (contents.len(), MetadataPartType::Text), PartType::Html(contents) => (contents.len(), MetadataPartType::Html), PartType::Binary(contents) => (contents.len(), MetadataPartType::Binary), PartType::InlineBinary(contents) => { (contents.len(), MetadataPartType::InlineBinary) } PartType::Message(message) => { let message_len = message.root_part().raw_len(); messages.push_back(message); message_id += 1; (message_len as usize, MetadataPartType::Message(message_id)) } PartType::Multipart(parts) => ( 0, MetadataPartType::Multipart(parts.into_iter().map(|p| p as u16).collect()), ), }; let flags = match part.encoding { Encoding::None => 0, Encoding::QuotedPrintable => PART_ENCODING_QP, Encoding::Base64 => PART_ENCODING_BASE64, } | (if part.is_encoding_problem { PART_ENCODING_PROBLEM } else { 0 }) | (size as u32 & PART_SIZE_MASK); parts.push(MessageMetadataPart { headers: part .headers .into_iter() .map(|hdr| MetadataHeader { value: if matches!( &hdr.name, HeaderName::Subject | HeaderName::From | HeaderName::To | HeaderName::Cc | HeaderName::Date | HeaderName::Bcc | HeaderName::ReplyTo | HeaderName::Sender | HeaderName::Comments | HeaderName::InReplyTo | HeaderName::Keywords | HeaderName::MessageId | HeaderName::References | HeaderName::ResentMessageId | HeaderName::ContentDescription | HeaderName::ContentId | HeaderName::ContentLanguage | HeaderName::ContentLocation | HeaderName::ContentTransferEncoding | HeaderName::ContentType | HeaderName::ContentDisposition | HeaderName::ListId ) { hdr.value } else { HeaderValue::Empty } .into(), name: hdr.name.into(), base_offset: hdr.offset_field, start: (hdr.offset_start - hdr.offset_field) as u16, end: (hdr.offset_end - hdr.offset_field) as u16, }) .collect(), body, flags, offset_header: part.offset_header, offset_body: part.offset_body, offset_end: part.offset_end, }); } contents.push(MessageMetadataContents { html_body: message.html_body.into_iter().map(|c| c as u16).collect(), text_body: message.text_body.into_iter().map(|c| c as u16).collect(), attachments: message.attachments.into_iter().map(|c| c as u16).collect(), parts: parts.into_boxed_slice(), }); } contents.into_boxed_slice() } impl ArchivedMessageMetadataPart { pub fn is_message(&self) -> bool { matches!(self.body, ArchivedMetadataPartType::Message(_)) } pub fn sub_parts(&self) -> Option<&ArchivedBox<[u16_le]>> { if let ArchivedMetadataPartType::Multipart(parts) = &self.body { Some(parts) } else { None } } pub fn raw_len(&self) -> usize { (u32::from(self.offset_end)).saturating_sub(u32::from(self.offset_header)) as usize } pub fn header_values( &self, name: &MetadataHeaderName, ) -> impl Iterator + Sync + Send { self.headers.iter().filter_map(move |header| { if &header.name == name { Some(&header.value) } else { None } }) } pub fn header_value(&self, name: &MetadataHeaderName) -> Option<&ArchivedMetadataHeaderValue> { self.headers.iter().rev().find_map(move |header| { if &header.name == name { Some(&header.value) } else { None } }) } pub fn subject(&self) -> Option<&str> { self.header_value(&MetadataHeaderName::Subject) .and_then(|header| header.as_text()) } pub fn date(&self) -> Option { self.header_value(&MetadataHeaderName::Date) .and_then(|header| header.as_datetime()) .map(|dt| dt.into()) } pub fn message_id(&self) -> Option<&str> { self.header_value(&MetadataHeaderName::MessageId) .and_then(|header| header.as_text()) } pub fn in_reply_to(&self) -> &ArchivedMetadataHeaderValue { self.header_value(&MetadataHeaderName::InReplyTo) .unwrap_or(&ArchivedMetadataHeaderValue::Empty) } pub fn content_description(&self) -> Option<&str> { self.header_value(&MetadataHeaderName::ContentDescription) .and_then(|header| header.as_text()) } pub fn content_disposition(&self) -> Option<&ArchivedMetadataContentType> { self.header_value(&MetadataHeaderName::ContentDisposition) .and_then(|header| header.as_content_type()) } pub fn content_id(&self) -> Option<&str> { self.header_value(&MetadataHeaderName::ContentId) .and_then(|header| header.as_text()) } pub fn content_transfer_encoding(&self) -> Option<&str> { self.header_value(&MetadataHeaderName::ContentTransferEncoding) .and_then(|header| header.as_text()) } pub fn content_type(&self) -> Option<&ArchivedMetadataContentType> { self.header_value(&MetadataHeaderName::ContentType) .and_then(|header| header.as_content_type()) } pub fn content_language(&self) -> &ArchivedMetadataHeaderValue { self.header_value(&MetadataHeaderName::ContentLanguage) .unwrap_or(&ArchivedMetadataHeaderValue::Empty) } pub fn content_location(&self) -> Option<&str> { self.header_value(&MetadataHeaderName::ContentLocation) .and_then(|header| header.as_text()) } pub fn attachment_name(&self) -> Option<&str> { self.content_disposition() .and_then(|cd| cd.attribute("filename")) .or_else(|| self.content_type().and_then(|ct| ct.attribute("name"))) } } impl From> for MetadataHeaderName { fn from(value: HeaderName<'_>) -> Self { match value { HeaderName::Subject => MetadataHeaderName::Subject, HeaderName::From => MetadataHeaderName::From, HeaderName::To => MetadataHeaderName::To, HeaderName::Cc => MetadataHeaderName::Cc, HeaderName::Date => MetadataHeaderName::Date, HeaderName::Bcc => MetadataHeaderName::Bcc, HeaderName::ReplyTo => MetadataHeaderName::ReplyTo, HeaderName::Sender => MetadataHeaderName::Sender, HeaderName::Comments => MetadataHeaderName::Comments, HeaderName::InReplyTo => MetadataHeaderName::InReplyTo, HeaderName::Keywords => MetadataHeaderName::Keywords, HeaderName::Received => MetadataHeaderName::Received, HeaderName::MessageId => MetadataHeaderName::MessageId, HeaderName::References => MetadataHeaderName::References, HeaderName::ReturnPath => MetadataHeaderName::ReturnPath, HeaderName::MimeVersion => MetadataHeaderName::MimeVersion, HeaderName::ContentDescription => MetadataHeaderName::ContentDescription, HeaderName::ContentId => MetadataHeaderName::ContentId, HeaderName::ContentLanguage => MetadataHeaderName::ContentLanguage, HeaderName::ContentLocation => MetadataHeaderName::ContentLocation, HeaderName::ContentTransferEncoding => MetadataHeaderName::ContentTransferEncoding, HeaderName::ContentType => MetadataHeaderName::ContentType, HeaderName::ContentDisposition => MetadataHeaderName::ContentDisposition, HeaderName::ResentTo => MetadataHeaderName::ResentTo, HeaderName::ResentFrom => MetadataHeaderName::ResentFrom, HeaderName::ResentBcc => MetadataHeaderName::ResentBcc, HeaderName::ResentCc => MetadataHeaderName::ResentCc, HeaderName::ResentSender => MetadataHeaderName::ResentSender, HeaderName::ResentDate => MetadataHeaderName::ResentDate, HeaderName::ResentMessageId => MetadataHeaderName::ResentMessageId, HeaderName::ListArchive => MetadataHeaderName::ListArchive, HeaderName::ListHelp => MetadataHeaderName::ListHelp, HeaderName::ListId => MetadataHeaderName::ListId, HeaderName::ListOwner => MetadataHeaderName::ListOwner, HeaderName::ListPost => MetadataHeaderName::ListPost, HeaderName::ListSubscribe => MetadataHeaderName::ListSubscribe, HeaderName::ListUnsubscribe => MetadataHeaderName::ListUnsubscribe, HeaderName::DkimSignature => MetadataHeaderName::DkimSignature, HeaderName::ArcAuthenticationResults => MetadataHeaderName::ArcAuthenticationResults, HeaderName::ArcMessageSignature => MetadataHeaderName::ArcMessageSignature, HeaderName::ArcSeal => MetadataHeaderName::ArcSeal, HeaderName::Other(value) => { let name = hashify::tiny_map_ignore_case!(value.as_bytes(), // Delivery/Routing "Delivered-To" => MetadataHeaderName::DeliveredTo, "X-Original-To" => MetadataHeaderName::XOriginalTo, "Return-Receipt-To" => MetadataHeaderName::ReturnReceiptTo, "Disposition-Notification-To" => MetadataHeaderName::DispositionNotificationTo, "Errors-To" => MetadataHeaderName::ErrorsTo, // Authentication "Authentication-Results" => MetadataHeaderName::AuthenticationResults, "Received-SPF" => MetadataHeaderName::ReceivedSpf, // Spam/Virus "X-Spam-Status" => MetadataHeaderName::XSpamStatus, "X-Spam-Score" => MetadataHeaderName::XSpamScore, "X-Spam-Flag" => MetadataHeaderName::XSpamFlag, "X-Spam-Result" => MetadataHeaderName::XSpamResult, // Priority "Importance" => MetadataHeaderName::Importance, "Priority" => MetadataHeaderName::Priority, "X-Priority" => MetadataHeaderName::XPriority, "X-MSMail-Priority" => MetadataHeaderName::XMSMailPriority, // Client/Agent "X-Mailer" => MetadataHeaderName::XMailer, "User-Agent" => MetadataHeaderName::UserAgent, "X-MimeOLE" => MetadataHeaderName::XMimeOLE, // Network/Origin "X-Originating-IP" => MetadataHeaderName::XOriginatingIp, "X-Forwarded-To" => MetadataHeaderName::XForwardedTo, "X-Forwarded-For" => MetadataHeaderName::XForwardedFor, // Auto-response "Auto-Submitted" => MetadataHeaderName::AutoSubmitted, "X-Auto-Response-Suppress" => MetadataHeaderName::XAutoResponseSuppress, "Precedence" => MetadataHeaderName::Precedence, // Organization/Threading "Organization" => MetadataHeaderName::Organization, "Thread-Index" => MetadataHeaderName::ThreadIndex, "Thread-Topic" => MetadataHeaderName::ThreadTopic, // List (additional) "List-Unsubscribe-Post" => MetadataHeaderName::ListUnsubscribePost, "Feedback-ID" => MetadataHeaderName::FeedbackId, ); name.unwrap_or_else(|| { MetadataHeaderName::Other(value.into_owned().into_boxed_str()) }) } other => MetadataHeaderName::Other(other.as_str().to_string().into_boxed_str()), } } } impl From> for MetadataHeaderValue { fn from(value: HeaderValue<'_>) -> Self { match value { HeaderValue::Address(address) => match address { Address::List(address) => MetadataHeaderValue::AddressList( address .into_iter() .map(|a| MetadataAddress { name: a.name.map(|a| a.into_owned().into_boxed_str()), address: a.address.map(|a| a.into_owned().into_boxed_str()), }) .collect(), ), Address::Group(groups) => MetadataHeaderValue::AddressGroup( groups .into_iter() .map(|g| MetadataAddressGroup { name: g.name.map(|a| a.into_owned().into_boxed_str()), addresses: g .addresses .into_iter() .map(|a| MetadataAddress { name: a.name.map(|a| a.into_owned().into_boxed_str()), address: a.address.map(|a| a.into_owned().into_boxed_str()), }) .collect(), }) .collect(), ), }, HeaderValue::Text(text) => { MetadataHeaderValue::Text(text.into_owned().into_boxed_str()) } HeaderValue::TextList(texts) => MetadataHeaderValue::TextList( texts .into_iter() .map(|v| v.into_owned().into_boxed_str()) .collect(), ), HeaderValue::DateTime(dt) => MetadataHeaderValue::DateTime(MetadataDateTime { year: dt.year, month: dt.month, day: dt.day, hour: dt.hour, minute: dt.minute, second: dt.second, tz_hour: (if dt.tz_before_gmt { -1 } else { 1 }) * dt.tz_hour as i8, tz_minute: dt.tz_minute, }), HeaderValue::ContentType(ct) => MetadataHeaderValue::ContentType(MetadataContentType { c_type: ct.c_type.into_owned().into_boxed_str(), c_subtype: ct.c_subtype.map(|v| v.into_owned().into_boxed_str()), attributes: ct .attributes .unwrap_or_default() .into_iter() .map(|a| MetadataAttribute { name: a.name.into_owned().into_boxed_str(), value: a.value.into_owned().into_boxed_str(), }) .collect(), }), HeaderValue::Received(_) | HeaderValue::Empty => MetadataHeaderValue::Empty, } } } impl From<&ArchivedMetadataDateTime> for DateTime { fn from(dt: &ArchivedMetadataDateTime) -> Self { DateTime { year: dt.year.to_native(), month: dt.month, day: dt.day, hour: dt.hour, minute: dt.minute, second: dt.second, tz_before_gmt: dt.tz_hour < 0, tz_hour: dt.tz_hour.unsigned_abs(), tz_minute: dt.tz_minute, } } } impl ArchivedMessageMetadataContents { pub fn root_part(&self) -> &ArchivedMessageMetadataPart { &self.parts[0] } } #[derive(Default)] pub struct MessageDataBuilder { pub mailboxes: Vec, pub keywords: Vec, pub thread_id: u32, pub size: u32, } impl MessageDataBuilder { pub fn set_keywords(&mut self, keywords: Vec) { self.keywords = keywords; } pub fn add_keyword(&mut self, keyword: Keyword) -> bool { if !self.keywords.contains(&keyword) { self.keywords.push(keyword); true } else { false } } pub fn remove_keyword(&mut self, keyword: &Keyword) -> bool { let prev_len = self.keywords.len(); self.keywords.retain(|k| k != keyword); self.keywords.len() != prev_len } pub fn set_mailboxes(&mut self, mailboxes: Vec) { self.mailboxes = mailboxes; } pub fn add_mailbox(&mut self, mailbox: UidMailbox) { if !self.mailboxes.contains(&mailbox) { self.mailboxes.push(mailbox); } } pub fn remove_mailbox(&mut self, mailbox: u32) { self.mailboxes.retain(|m| m.mailbox_id != mailbox); } pub fn has_keyword(&self, keyword: &Keyword) -> bool { self.keywords.iter().any(|k| k == keyword) } pub fn has_keyword_changes(&self, prev_data: &ArchivedMessageData) -> bool { self.keywords.len() != prev_data.keywords.len() || !self .keywords .iter() .all(|k| prev_data.keywords.iter().any(|pk| pk == k)) } pub fn added_keywords( &self, prev_data: &ArchivedMessageData, ) -> impl Iterator { self.keywords .iter() .filter(|k| prev_data.keywords.iter().all(|pk| pk != *k)) } pub fn removed_keywords<'x>( &'x self, prev_data: &'x ArchivedMessageData, ) -> impl Iterator { prev_data .keywords .iter() .filter(|k| self.keywords.iter().all(|pk| pk != *k)) } pub fn added_mailboxes( &self, prev_data: &ArchivedMessageData, ) -> impl Iterator { self.mailboxes.iter().filter(|m| { prev_data .mailboxes .iter() .all(|pm| pm.mailbox_id != m.mailbox_id) }) } pub fn removed_mailboxes<'x>( &'x self, prev_data: &'x ArchivedMessageData, ) -> impl Iterator { prev_data.mailboxes.iter().filter(|m| { self.mailboxes .iter() .all(|pm| pm.mailbox_id != m.mailbox_id) }) } pub fn has_mailbox_changes(&self, prev_data: &ArchivedMessageData) -> bool { self.mailboxes.len() != prev_data.mailboxes.len() || !self.mailboxes.iter().all(|m| { prev_data .mailboxes .iter() .any(|pm| pm.mailbox_id == m.mailbox_id) }) } pub fn seal(self) -> MessageData { MessageData { mailboxes: self.mailboxes.into_boxed_slice(), keywords: self.keywords.into_boxed_slice(), thread_id: self.thread_id, size: self.size, } } } impl MessageData { pub fn has_mailbox_id(&self, mailbox_id: u32) -> bool { self.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) } } impl ArchivedMessageData { pub fn has_mailbox_id(&self, mailbox_id: u32) -> bool { self.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) } pub fn message_uid(&self, mailbox_id: u32) -> Option { self.mailboxes .iter() .find(|m| m.mailbox_id == mailbox_id) .map(|m| m.uid.to_native()) } pub fn to_builder(&self) -> MessageDataBuilder { MessageDataBuilder { mailboxes: self.mailboxes.iter().map(|m| m.to_native()).collect(), keywords: self.keywords.iter().map(|k| k.to_native()).collect(), thread_id: self.thread_id.to_native(), size: self.size.to_native(), } } } impl ArchivedMetadataContentType { pub fn ctype(&self) -> &str { &self.c_type } pub fn subtype(&self) -> Option<&str> { self.c_subtype.as_ref().map(|s| s.as_ref()) } pub fn attribute(&self, name: &str) -> Option<&str> { self.attributes .iter() .find(|a| *a.name == *name) .map(|a| a.value.as_ref()) } /// Returns `true` when the provided attribute name is present pub fn has_attribute(&self, name: &str) -> bool { self.attributes.iter().any(|a| *a.name == *name) } pub fn is_attachment(&self) -> bool { self.c_type.eq_ignore_ascii_case("attachment") } pub fn is_inline(&self) -> bool { self.c_type.eq_ignore_ascii_case("inline") } } impl ArchivedMetadataHeaderValue { pub fn is_empty(&self) -> bool { self == &MetadataHeaderValue::Empty } pub fn as_text(&self) -> Option<&str> { match self { ArchivedMetadataHeaderValue::Text(s) => Some(s.as_ref()), ArchivedMetadataHeaderValue::TextList(l) => l.last().map(|v| v.as_ref()), _ => None, } } pub fn as_text_list(&self) -> Option<&[ArchivedBox]> { match self { ArchivedMetadataHeaderValue::Text(s) => Some(std::slice::from_ref(s)), ArchivedMetadataHeaderValue::TextList(l) => Some(l.as_ref()), _ => None, } } pub fn as_content_type(&self) -> Option<&ArchivedMetadataContentType> { match self { ArchivedMetadataHeaderValue::ContentType(c) => Some(c), _ => None, } } pub fn as_datetime(&self) -> Option<&ArchivedMetadataDateTime> { match self { ArchivedMetadataHeaderValue::DateTime(d) => Some(d), _ => None, } } pub fn as_single_address(&self) -> Option<&ArchivedMetadataAddress> { match self { ArchivedMetadataHeaderValue::AddressList(list) => list.first(), ArchivedMetadataHeaderValue::AddressGroup(groups) => { groups.first().and_then(|g| g.addresses.first()) } _ => None, } } } impl ArchivedUidMailbox { pub fn to_native(&self) -> UidMailbox { UidMailbox { mailbox_id: self.mailbox_id.to_native(), uid: self.uid.to_native(), } } } impl ArchivedMetadataHeader { #[inline(always)] pub fn value_range(&self) -> Range { (self.base_offset.to_native() as usize + self.start.to_native() as usize) ..(self.base_offset.to_native() as usize + self.end.to_native() as usize) } #[inline(always)] pub fn name_value_range(&self) -> Range { (self.base_offset.to_native() as usize) ..(self.base_offset.to_native() as usize + self.end.to_native() as usize) } } impl ArchivedMetadataHeaderName { pub fn is_mime_header(&self) -> bool { matches!( self, ArchivedMetadataHeaderName::ContentDescription | ArchivedMetadataHeaderName::ContentId | ArchivedMetadataHeaderName::ContentLanguage | ArchivedMetadataHeaderName::ContentLocation | ArchivedMetadataHeaderName::ContentTransferEncoding | ArchivedMetadataHeaderName::ContentType | ArchivedMetadataHeaderName::ContentDisposition ) } pub fn as_str(&self) -> &str { match self { ArchivedMetadataHeaderName::Subject => "Subject", ArchivedMetadataHeaderName::From => "From", ArchivedMetadataHeaderName::To => "To", ArchivedMetadataHeaderName::Cc => "Cc", ArchivedMetadataHeaderName::Date => "Date", ArchivedMetadataHeaderName::Bcc => "Bcc", ArchivedMetadataHeaderName::ReplyTo => "Reply-To", ArchivedMetadataHeaderName::Sender => "Sender", ArchivedMetadataHeaderName::Comments => "Comments", ArchivedMetadataHeaderName::InReplyTo => "In-Reply-To", ArchivedMetadataHeaderName::Keywords => "Keywords", ArchivedMetadataHeaderName::Received => "Received", ArchivedMetadataHeaderName::MessageId => "Message-ID", ArchivedMetadataHeaderName::References => "References", ArchivedMetadataHeaderName::ReturnPath => "Return-Path", ArchivedMetadataHeaderName::MimeVersion => "MIME-Version", ArchivedMetadataHeaderName::ContentDescription => "Content-Description", ArchivedMetadataHeaderName::ContentId => "Content-ID", ArchivedMetadataHeaderName::ContentLanguage => "Content-Language", ArchivedMetadataHeaderName::ContentLocation => "Content-Location", ArchivedMetadataHeaderName::ContentTransferEncoding => "Content-Transfer-Encoding", ArchivedMetadataHeaderName::ContentType => "Content-Type", ArchivedMetadataHeaderName::ContentDisposition => "Content-Disposition", ArchivedMetadataHeaderName::ResentTo => "Resent-To", ArchivedMetadataHeaderName::ResentFrom => "Resent-From", ArchivedMetadataHeaderName::ResentBcc => "Resent-Bcc", ArchivedMetadataHeaderName::ResentCc => "Resent-Cc", ArchivedMetadataHeaderName::ResentSender => "Resent-Sender", ArchivedMetadataHeaderName::ResentDate => "Resent-Date", ArchivedMetadataHeaderName::ResentMessageId => "Resent-Message-ID", ArchivedMetadataHeaderName::ListArchive => "List-Archive", ArchivedMetadataHeaderName::ListHelp => "List-Help", ArchivedMetadataHeaderName::ListId => "List-ID", ArchivedMetadataHeaderName::ListOwner => "List-Owner", ArchivedMetadataHeaderName::ListPost => "List-Post", ArchivedMetadataHeaderName::ListSubscribe => "List-Subscribe", ArchivedMetadataHeaderName::ListUnsubscribe => "List-Unsubscribe", ArchivedMetadataHeaderName::ArcAuthenticationResults => "ARC-Authentication-Results", ArchivedMetadataHeaderName::ArcMessageSignature => "ARC-Message-Signature", ArchivedMetadataHeaderName::ArcSeal => "ARC-Seal", ArchivedMetadataHeaderName::DkimSignature => "DKIM-Signature", ArchivedMetadataHeaderName::DeliveredTo => "Delivered-To", ArchivedMetadataHeaderName::XOriginalTo => "X-Original-To", ArchivedMetadataHeaderName::ReturnReceiptTo => "Return-Receipt-To", ArchivedMetadataHeaderName::DispositionNotificationTo => "Disposition-Notification-To", ArchivedMetadataHeaderName::ErrorsTo => "Errors-To", ArchivedMetadataHeaderName::AuthenticationResults => "Authentication-Results", ArchivedMetadataHeaderName::ReceivedSpf => "Received-SPF", ArchivedMetadataHeaderName::XSpamStatus => "X-Spam-Status", ArchivedMetadataHeaderName::XSpamScore => "X-Spam-Score", ArchivedMetadataHeaderName::XSpamFlag => "X-Spam-Flag", ArchivedMetadataHeaderName::XSpamResult => "X-Spam-Result", ArchivedMetadataHeaderName::Importance => "Importance", ArchivedMetadataHeaderName::Priority => "Priority", ArchivedMetadataHeaderName::XPriority => "X-Priority", ArchivedMetadataHeaderName::XMSMailPriority => "X-MSMail-Priority", ArchivedMetadataHeaderName::XMailer => "X-Mailer", ArchivedMetadataHeaderName::UserAgent => "User-Agent", ArchivedMetadataHeaderName::XMimeOLE => "X-MimeOLE", ArchivedMetadataHeaderName::XOriginatingIp => "X-Originating-IP", ArchivedMetadataHeaderName::XForwardedTo => "X-Forwarded-To", ArchivedMetadataHeaderName::XForwardedFor => "X-Forwarded-For", ArchivedMetadataHeaderName::AutoSubmitted => "Auto-Submitted", ArchivedMetadataHeaderName::XAutoResponseSuppress => "X-Auto-Response-Suppress", ArchivedMetadataHeaderName::Precedence => "Precedence", ArchivedMetadataHeaderName::Organization => "Organization", ArchivedMetadataHeaderName::ThreadIndex => "Thread-Index", ArchivedMetadataHeaderName::ThreadTopic => "Thread-Topic", ArchivedMetadataHeaderName::ListUnsubscribePost => "List-Unsubscribe-Post", ArchivedMetadataHeaderName::FeedbackId => "Feedback-ID", ArchivedMetadataHeaderName::Other(name) => name.as_ref(), } } } impl From<&ArchivedMetadataHeaderValue> for HeaderValue<'static> { fn from(value: &ArchivedMetadataHeaderValue) -> Self { match value { ArchivedMetadataHeaderValue::AddressList(addr) => HeaderValue::Address(Address::List( addr.as_ref().iter().map(Into::into).collect(), )), ArchivedMetadataHeaderValue::AddressGroup(addr) => HeaderValue::Address( Address::Group(addr.as_ref().iter().map(Into::into).collect()), ), ArchivedMetadataHeaderValue::Text(text) => HeaderValue::Text(text.to_string().into()), ArchivedMetadataHeaderValue::TextList(textlist) => HeaderValue::TextList( textlist .as_ref() .iter() .map(|s| s.to_string().into()) .collect(), ), ArchivedMetadataHeaderValue::DateTime(dt) => HeaderValue::DateTime(dt.into()), ArchivedMetadataHeaderValue::ContentType(ct) => HeaderValue::ContentType(ct.into()), ArchivedMetadataHeaderValue::Empty => HeaderValue::Empty, } } } impl From<&ArchivedMetadataAddress> for Addr<'static> { fn from(value: &ArchivedMetadataAddress) -> Self { Addr { name: value.name.as_ref().map(|n| n.to_string().into()), address: value.address.as_ref().map(|a| a.to_string().into()), } } } impl From<&ArchivedMetadataAddressGroup> for Group<'static> { fn from(value: &ArchivedMetadataAddressGroup) -> Self { Group { name: value.name.as_ref().map(|n| n.to_string().into()), addresses: value.addresses.as_ref().iter().map(Into::into).collect(), } } } impl From<&ArchivedMetadataContentType> for ContentType<'static> { fn from(value: &ArchivedMetadataContentType) -> Self { ContentType { c_type: value.ctype().to_string().into(), c_subtype: value.subtype().map(|s| s.to_string().into()), attributes: Some( value .attributes .iter() .map(|a| Attribute { name: a.name.to_string().into(), value: a.value.to_string().into(), }) .collect(), ), } } } ================================================ FILE: crates/email/src/message/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod copy; pub mod crypto; pub mod delete; pub mod delivery; pub mod index; pub mod ingest; pub mod metadata; ================================================ FILE: crates/email/src/push/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use types::type_state::DataType; use utils::map::bitmap::Bitmap; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Default, Debug, Clone, PartialEq, Eq, )] pub struct PushSubscription { pub id: u32, pub url: String, pub device_client_id: String, pub expires: u64, pub verification_code: String, pub verified: bool, pub types: Bitmap, pub keys: Option, pub email_push: Vec, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] pub struct Keys { pub p256dh: Vec, pub auth: Vec, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Default, Debug, Clone, PartialEq, Eq, )] pub struct PushSubscriptions { pub subscriptions: Vec, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Default, Debug, Clone, PartialEq, Eq, )] pub struct EmailPush { pub account_id: u32, pub properties: u64, pub filters: Vec, pub flags: u16, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] pub enum EmailPushFilter { Condition { field: u8, value: EmailPushValue }, And, Or, Not, End, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] pub enum EmailPushValue { Text(String), Number(u64), TextList(Vec), NumberList(Vec), } ================================================ FILE: crates/email/src/sieve/delete.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::SieveScript; use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; use store::write::BatchBuilder; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{collection::Collection, field::SieveField}; pub trait SieveScriptDelete: Sync + Send { fn sieve_script_delete( &self, account_id: u32, document_id: u32, access_token: &AccessToken, batch: &mut BatchBuilder, ) -> impl Future> + Send; } impl SieveScriptDelete for Server { async fn sieve_script_delete( &self, account_id: u32, document_id: u32, access_token: &AccessToken, batch: &mut BatchBuilder, ) -> trc::Result { // Fetch record if let Some(obj_) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, document_id, )) .await? { // Delete record batch .with_account_id(account_id) .with_collection(Collection::SieveScript) .with_document(document_id) .clear(SieveField::Ids) .custom( ObjectIndexBuilder::<_, ()>::new() .with_current( obj_.to_unarchived::() .caused_by(trc::location!())?, ) .with_access_token(access_token), ) .caused_by(trc::location!())? .commit_point(); Ok(true) } else { Ok(false) } } } ================================================ FILE: crates/email/src/sieve/index.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ArchivedSieveScript, SieveScript}; use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; use types::{collection::SyncCollection, field::SieveField}; impl IndexableObject for SieveScript { fn index_values(&self) -> impl Iterator> { [ IndexValue::Index { field: SieveField::Name.into(), value: self.name.as_str().to_lowercase().into(), }, IndexValue::Blob { value: self.blob_hash.clone(), }, IndexValue::LogItem { sync_collection: SyncCollection::SieveScript, prefix: None, }, IndexValue::Quota { used: self.size }, ] .into_iter() } } impl IndexableAndSerializableObject for SieveScript { fn is_versioned() -> bool { false } } impl IndexableObject for &ArchivedSieveScript { fn index_values(&self) -> impl Iterator> { [ IndexValue::Index { field: SieveField::Name.into(), value: self.name.to_lowercase().into(), }, IndexValue::Blob { value: (&self.blob_hash).into(), }, IndexValue::LogItem { sync_collection: SyncCollection::SieveScript, prefix: None, }, IndexValue::Quota { used: u32::from(self.size), }, ] .into_iter() } } ================================================ FILE: crates/email/src/sieve/ingest.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ActiveScript, SeenIdHash, SieveScript}; use crate::{ cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}, mailbox::{INBOX_ID, TRASH_ID, manage::MailboxFnc}, message::{ delivery::{AutogeneratedMessage, IngestRecipient}, ingest::{EmailIngest, IngestEmail, IngestSource, IngestedEmail}, }, }; use common::{Server, auth::AccessToken, scripts::plugins::PluginContext}; use directory::QueryParams; use mail_parser::MessageParser; use sieve::{Envelope, Event, Input, Mailbox, Recipient, Sieve, SpamStatus}; use std::{borrow::Cow, sync::Arc}; use std::{future::Future, str::FromStr}; use store::{ Deserialize, Serialize, ValueKey, ahash::AHashMap, dispatch::lookup::KeyValue, write::{ AlignedBytes, Archive, ArchiveVersion, Archiver, BatchBuilder, BlobLink, BlobOp, ValueClass, }, }; use trc::{AddContext, SieveEvent}; use types::{ blob_hash::BlobHash, collection::Collection, field::{PrincipalField, SieveField}, id::Id, keyword::Keyword, special_use::SpecialUse, }; use utils::config::utils::ParseValue; struct SieveMessage<'x> { pub raw_message: Cow<'x, [u8]>, pub file_into: Vec, pub did_file_into: bool, pub flags: Vec, } pub trait SieveScriptIngest: Sync + Send { #[allow(clippy::too_many_arguments)] fn sieve_script_ingest( &self, access_token: &AccessToken, blob_hash: &BlobHash, raw_message: &[u8], envelope_from: &str, envelope_from_authenticated: bool, envelope_to: &IngestRecipient, session_id: u64, active_script: ActiveScript, autogenerated: &mut Vec, ) -> impl Future> + Send; fn sieve_script_get_active_id( &self, account_id: u32, ) -> impl Future>> + Send; fn sieve_script_get_active( &self, account_id: u32, ) -> impl Future>> + Send; fn sieve_script_get_by_name( &self, account_id: u32, name: &str, ) -> impl Future>> + Send; fn sieve_script_compile( &self, account_id: u32, document_id: u32, ) -> impl Future>> + Send; } impl SieveScriptIngest for Server { #[allow(clippy::blocks_in_conditions)] async fn sieve_script_ingest( &self, access_token: &AccessToken, blob_hash: &BlobHash, raw_message: &[u8], envelope_from: &str, envelope_from_authenticated: bool, envelope_to: &IngestRecipient, session_id: u64, active_script: ActiveScript, autogenerated: &mut Vec, ) -> trc::Result { // Parse message let message = if let Some(message) = MessageParser::new().parse(raw_message) { message } else { return Err( trc::EventType::MessageIngest(trc::MessageIngestEvent::Error) .ctx(trc::Key::Code, 550) .ctx(trc::Key::Reason, "Failed to parse e-mail message."), ); }; // Obtain mailboxIds let account_id = access_token.primary_id; let mut cache = self .get_cached_messages(account_id) .await .caused_by(trc::location!())?; // Create Sieve instance let mut instance = self.core.sieve.untrusted_runtime.filter_parsed(message); // Set account name and email let mail_from = self .core .storage .directory .query(QueryParams::id(account_id).with_return_member_of(false)) .await .caused_by(trc::location!())? .and_then(|p| { instance.set_user_full_name(p.description().unwrap_or_else(|| p.name())); p.into_primary_email() }); // Set account address let mail_from = mail_from.unwrap_or_else(|| envelope_to.address.as_str().into()); instance.set_user_address(&mail_from); // Set envelope instance.set_envelope(Envelope::From, envelope_from); instance.set_envelope(Envelope::To, envelope_to.address.as_str()); instance.set_spam_status(if envelope_to.is_spam { SpamStatus::Spam } else { SpamStatus::Ham }); let mut input = Input::script( active_script.script_name.to_string(), active_script.script.clone(), ); let mut do_discard = false; let mut do_deliver = false; let mut reject_reason = None; let mut messages: Vec = vec![SieveMessage { raw_message: raw_message.into(), file_into: Vec::new(), flags: Vec::new(), did_file_into: false, }]; let mut ingested_message = IngestedEmail { document_id: 0, thread_id: 0, change_id: u64::MAX, blob_id: Default::default(), size: raw_message.len(), imap_uids: Vec::new(), }; let mut checked_ids: AHashMap = AHashMap::new(); while let Some(event) = instance.run(input) { match event { Ok(event) => match event { Event::IncludeScript { name, .. } => match &name { sieve::Script::Personal(name_) => { if let Ok(Some(script)) = self.sieve_script_get_by_name(account_id, name_).await { input = Input::script(name, script); } else { input = false.into(); } } sieve::Script::Global(name_) => { if let Some(script) = self.get_untrusted_sieve_script(&name_.to_lowercase(), session_id) { input = Input::script(name, script.clone()); } else { input = false.into(); } } }, Event::MailboxExists { mailboxes, special_use, } => { if !mailboxes.is_empty() { let mut special_use_ids = Vec::with_capacity(special_use.len()); for role in special_use { special_use_ids.push(if role.eq_ignore_ascii_case("inbox") { INBOX_ID } else if role.eq_ignore_ascii_case("trash") { TRASH_ID } else { let mut mailbox_id = u32::MAX; if let Ok(role) = SpecialUse::parse_value(&role) && let Some(m) = cache.mailbox_by_role(&role) { mailbox_id = m.document_id; } mailbox_id }); } let mut result = true; for mailbox in mailboxes { match mailbox { Mailbox::Name(name) => { if !matches!( cache.mailbox_by_path(&name), Some(item) if special_use_ids.is_empty() || special_use_ids.contains(&item.document_id) ) { result = false; break; } } Mailbox::Id(id) => { if !matches!(Id::from_str(&id), Ok(id) if cache.has_mailbox_id(&id.document_id()) && (special_use_ids.is_empty() || special_use_ids.contains(&id.document_id()))) { result = false; break; } } } } input = result.into(); } else if !special_use.is_empty() { let mut result = true; for role in special_use { if !role.eq_ignore_ascii_case("inbox") && !role.eq_ignore_ascii_case("trash") { let role = SpecialUse::parse_value(&role); if role.is_err() || cache.mailbox_by_role(&role.unwrap()).is_none() { result = false; break; } } } input = result.into(); } else { input = false.into(); } } Event::DuplicateId { id, expiry, last } => { let id_hash = SeenIdHash::new( account_id, active_script.version.hash().unwrap_or_default(), &id, ); if let Some(result) = checked_ids.get(&id_hash) { input = (*result).into(); } else { let exists = self .in_memory_store() .key_get::<()>(id_hash.key()) .await .caused_by(trc::location!())? .is_some(); if !exists || last { self.in_memory_store() .key_set(KeyValue::new(id_hash.key(), vec![]).expires(expiry)) .await .caused_by(trc::location!())?; } checked_ids.insert(id_hash, exists); input = exists.into(); } } Event::Discard => { do_discard = true; input = true.into(); } Event::Reject { reason, .. } => { reject_reason = reason.into(); do_discard = true; input = true.into(); } Event::Keep { flags, message_id } => { if let Some(message) = messages.get_mut(message_id) { message.flags = flags.into_iter().map(Keyword::from).collect(); if !message.file_into.contains(&INBOX_ID) { message.file_into.push(INBOX_ID); } do_deliver = true; } else { trc::event!( Sieve(SieveEvent::UnexpectedError), Details = "Unknown message id.", MessageId = message_id, SpanId = session_id ); } input = true.into(); } Event::FileInto { folder, flags, mailbox_id, special_use, create, message_id, } => { let mut target_id = u32::MAX; // Find mailbox by Id if let Some(mailbox_id) = mailbox_id.and_then(|m| Id::from_str(&m).ok()) { let mailbox_id = mailbox_id.document_id(); if cache.has_mailbox_id(&mailbox_id) { target_id = mailbox_id; } } // Find mailbox by role if let Some(special_use) = special_use && target_id == u32::MAX { if special_use.eq_ignore_ascii_case("inbox") { target_id = INBOX_ID; } else if special_use.eq_ignore_ascii_case("trash") { target_id = TRASH_ID; } else if let Ok(role) = SpecialUse::parse_value(&special_use) && let Some(item) = cache.mailbox_by_role(&role) { target_id = item.document_id; } } // Find mailbox by name if target_id == u32::MAX { if !create { if let Some(m) = cache.mailbox_by_path(&folder) { target_id = m.document_id; } } else if let Some(document_id) = self .mailbox_create_path(account_id, &folder) .await .caused_by(trc::location!())? { cache = self .get_cached_messages(account_id) .await .caused_by(trc::location!())?; target_id = document_id; } } // Default to Inbox if target_id == u32::MAX { target_id = INBOX_ID; } if let Some(message) = messages.get_mut(message_id) { message.flags = flags.into_iter().map(Keyword::from).collect(); if !message.file_into.contains(&target_id) { message.file_into.push(target_id); } message.did_file_into = true; do_deliver = true; } else { trc::event!( Sieve(SieveEvent::UnexpectedError), Details = "Unknown message id.", MessageId = message_id, SpanId = session_id ); } input = true.into(); } Event::SendMessage { recipient, message_id, .. } => { input = true.into(); if let Some(message) = messages.get(message_id) { let recipients: Vec = match recipient { Recipient::Address(rcpt) => vec![rcpt], Recipient::Group(rcpts) => rcpts, Recipient::List(_) => { // Not yet implemented continue; } }; if message.raw_message.len() <= self.core.jmap.mail_max_size { trc::event!( Sieve(SieveEvent::SendMessage), From = mail_from.clone(), To = recipients .iter() .map(|r| trc::Value::String(r.as_str().into())) .collect::>(), Size = message.raw_message.len(), SpanId = session_id ); autogenerated.push(AutogeneratedMessage { sender_address: mail_from.clone(), recipients, message: message.raw_message.to_vec(), }); } else { trc::event!( Sieve(SieveEvent::MessageTooLarge), From = mail_from.clone(), To = recipients .iter() .map(|r| trc::Value::String(r.as_str().into())) .collect::>(), Size = message.raw_message.len(), Limit = self.core.jmap.mail_max_size, SpanId = session_id, ); } } else { trc::event!( Sieve(SieveEvent::UnexpectedError), Details = "Unknown message id.", MessageId = message_id, SpanId = session_id ); continue; } } Event::ListContains { .. } | Event::Notify { .. } | Event::SetEnvelope { .. } => { // Not allowed input = false.into(); } Event::Function { id, arguments } => { input = self .core .run_plugin( id, PluginContext { session_id, server: self, message: instance.message(), modifications: &mut Vec::new(), access_token: access_token.into(), arguments, }, ) .await; } Event::CreatedMessage { message, .. } => { messages.push(SieveMessage { raw_message: message.into(), file_into: Vec::new(), flags: Vec::new(), did_file_into: false, }); input = true.into(); } }, #[cfg(feature = "test_mode")] Err(sieve::runtime::RuntimeError::ScriptErrorMessage(err)) => { panic!("Sieve test failed: {}", err); } Err(err) => { trc::event!( Sieve(SieveEvent::RuntimeError), Reason = err.to_string(), SpanId = session_id ); input = true.into(); } } } // Fail-safe, no discard and no keep seen, assume that something went wrong and file anyway. if !do_deliver && !do_discard { messages[0].file_into.push(INBOX_ID); } // Deliver messages let mut last_temp_error = None; let mut has_delivered = false; for (message_id, sieve_message) in messages.into_iter().enumerate() { if !sieve_message.file_into.is_empty() { // Parse message if needed let (blob_hash, message) = if message_id == 0 && !instance.has_message_changed() { (blob_hash.into(), instance.take_message()) } else if let Some(message) = MessageParser::new().parse(sieve_message.raw_message.as_ref()) { (None, message) } else { trc::event!( Sieve(SieveEvent::UnexpectedError), Details = "Failed to parse Sieve generated message.", SpanId = session_id ); continue; }; // Deliver message match self .email_ingest(IngestEmail { raw_message: &sieve_message.raw_message, blob_hash, message: message.into(), access_token, mailbox_ids: sieve_message.file_into, keywords: sieve_message.flags, received_at: None, source: IngestSource::Smtp { deliver_to: envelope_to.address.as_str(), is_sender_authenticated: envelope_from_authenticated, is_spam: envelope_to.is_spam, }, session_id, }) .await { Ok(ingested_message_) => { has_delivered = true; ingested_message = ingested_message_; } Err(err) => { last_temp_error = err.into(); } } } } if let Some(reject_reason) = reject_reason { Err( trc::EventType::MessageIngest(trc::MessageIngestEvent::Error) .ctx(trc::Key::Code, 571) .ctx(trc::Key::Reason, reject_reason), ) } else if has_delivered || last_temp_error.is_none() { Ok(ingested_message) } else { // There were problems during delivery #[allow(clippy::unnecessary_unwrap)] Err(last_temp_error.unwrap()) } } async fn sieve_script_get_active_id(&self, account_id: u32) -> trc::Result> { self.store() .get_value::(ValueKey { account_id, collection: Collection::Principal.into(), document_id: 0, class: ValueClass::Property(PrincipalField::ActiveScriptId.into()), }) .await .caused_by(trc::location!()) } async fn sieve_script_get_active(&self, account_id: u32) -> trc::Result> { // Find the currently active script if let Some(document_id) = self .store() .get_value::(ValueKey { account_id, collection: Collection::Principal.into(), document_id: 0, class: ValueClass::Property(PrincipalField::ActiveScriptId.into()), }) .await .caused_by(trc::location!())? { if let Some(script) = self.sieve_script_compile(account_id, document_id).await? { Ok(Some(ActiveScript { document_id, script: Arc::new(script.script), script_name: script.name, version: script.version, })) } else { Ok(None) } } else { Ok(None) } } async fn sieve_script_get_by_name( &self, account_id: u32, name: &str, ) -> trc::Result> { // Find the script by name if let Some(document_id) = self .document_ids_matching( account_id, Collection::SieveScript, SieveField::Name, name.as_bytes(), ) .await .caused_by(trc::location!())? .min() { self.sieve_script_compile(account_id, document_id) .await .map(|script| script.map(|s| s.script)) } else { Ok(None) } } #[allow(clippy::blocks_in_conditions)] async fn sieve_script_compile( &self, account_id: u32, document_id: u32, ) -> trc::Result> { // Obtain script object let Some(script_object) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, document_id, )) .await? else { return Ok(None); }; // Obtain the sieve script length let version = script_object.version; let unarchived_script = script_object .unarchive::() .caused_by(trc::location!())?; let script_offset = u32::from(unarchived_script.size) as usize; // Obtain the sieve script blob let script_bytes = self .core .storage .blob .get_blob(unarchived_script.blob_hash.0.as_ref(), 0..usize::MAX) .await .caused_by(trc::location!())? .ok_or_else(|| { trc::StoreEvent::NotFound .into_err() .caused_by(trc::location!()) .document_id(document_id) })?; // Obtain the precompiled script if let Some(script) = script_bytes.get(script_offset..).and_then(|bytes| { as Deserialize>::deserialize(bytes) .ok()? .deserialize::() .ok() }) { Ok(Some(CompiledScript { script, name: unarchived_script.name.as_str().into(), version, })) } else { // Deserialization failed, probably because the script compiler version changed match self.core.sieve.untrusted_compiler.compile( script_bytes.get(0..script_offset).ok_or_else(|| { trc::StoreEvent::NotFound .into_err() .caused_by(trc::location!()) .document_id(document_id) })?, ) { Ok(sieve) => { // Store updated compiled sieve script let sieve = Archiver::new(sieve).untrusted(); let compiled_bytes = sieve.serialize().caused_by(trc::location!())?; let mut updated_sieve_bytes = Vec::with_capacity(script_offset + compiled_bytes.len()); updated_sieve_bytes.extend_from_slice(&script_bytes[0..script_offset]); updated_sieve_bytes.extend_from_slice(&compiled_bytes); // Store updated blob let (new_blob_hash, new_blob_hold) = self .put_temporary_blob(account_id, &updated_sieve_bytes, 60) .await?; let mut new_script_object = rkyv::deserialize(unarchived_script).caused_by(trc::location!())?; let blob_hash = std::mem::replace(&mut new_script_object.blob_hash, new_blob_hash.clone()); let new_archive = Archiver::new(new_script_object); // Update script object let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::SieveScript) .with_document(document_id) .assert_value(SieveField::Archive, &script_object) .set( SieveField::Archive, new_archive.serialize().caused_by(trc::location!())?, ) .clear(BlobOp::Link { hash: blob_hash, to: BlobLink::Document, }) .set( BlobOp::Link { hash: new_blob_hash, to: BlobLink::Document, }, Vec::new(), ) .clear(new_blob_hold); self.store() .write(batch.build_all()) .await .caused_by(trc::location!())?; Ok(Some(CompiledScript { script: sieve.into_inner(), name: new_archive.into_inner().name, version, })) } Err(error) => Err(trc::StoreEvent::UnexpectedError .caused_by(trc::location!()) .reason(error) .details("Failed to compile Sieve script")), } } } } pub struct CompiledScript { pub script: Sieve, pub name: String, pub version: ArchiveVersion, } ================================================ FILE: crates/email/src/sieve/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::KV_SIEVE_ID; use sieve::Sieve; use std::sync::Arc; use store::{blake3, write::ArchiveVersion}; use types::blob_hash::BlobHash; pub mod delete; pub mod index; pub mod ingest; #[derive(Debug, Clone)] pub struct ActiveScript { pub document_id: u32, pub version: ArchiveVersion, pub script_name: String, pub script: Arc, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] #[rkyv(derive(Debug))] pub struct SieveScript { pub name: String, pub blob_hash: BlobHash, pub size: u32, pub vacation_response: Option, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] #[rkyv(derive(Debug))] pub struct VacationResponse { pub from_date: Option, pub to_date: Option, pub subject: Option, pub text_body: Option, pub html_body: Option, } impl SieveScript { pub fn new(name: impl Into, blob_hash: BlobHash) -> Self { SieveScript { name: name.into(), blob_hash, vacation_response: None, size: 0, } } pub fn with_name(mut self, name: impl Into) -> Self { self.name = name.into(); self } pub fn with_blob_hash(mut self, blob_hash: BlobHash) -> Self { self.blob_hash = blob_hash; self } pub fn with_size(mut self, size: u32) -> Self { self.size = size; self } pub fn with_vacation_response(mut self, vacation_response: VacationResponse) -> Self { self.vacation_response = Some(vacation_response); self } } #[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] #[repr(transparent)] pub struct SeenIdHash(pub [u8; 32]); impl SeenIdHash { pub fn new(account_id: u32, hash: u32, id: &str) -> Self { let mut hasher = blake3::Hasher::new(); hasher.update(&account_id.to_be_bytes()); hasher.update(&hash.to_be_bytes()); hasher.update(id.as_bytes()); SeenIdHash(hasher.finalize().into()) } pub fn key(&self) -> Vec { let mut result = Vec::with_capacity(self.0.len() + 1); result.push(KV_SIEVE_ID); result.extend_from_slice(&self.0); result } } impl AsRef<[u8]> for SeenIdHash { fn as_ref(&self) -> &[u8] { &self.0 } } ================================================ FILE: crates/email/src/submission/index.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ArchivedEmailSubmission, EmailSubmission}; use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; use store::{ U32_LEN, write::{IndexPropertyClass, ValueClass, key::KeySerializer}, }; use types::{collection::SyncCollection, field::EmailSubmissionField}; impl IndexableObject for EmailSubmission { fn index_values(&self) -> impl Iterator> { [ IndexValue::Property { field: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: EmailSubmissionField::Metadata.into(), value: self.send_at, }), value: KeySerializer::new(U32_LEN * 3 + 1) .write(self.email_id) .write(self.thread_id) .write(self.identity_id) .write(self.undo_status.as_index()) .finalize() .into(), }, IndexValue::LogItem { sync_collection: SyncCollection::EmailSubmission, prefix: None, }, ] .into_iter() } } impl IndexableObject for &ArchivedEmailSubmission { fn index_values(&self) -> impl Iterator> { [ IndexValue::Property { field: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: EmailSubmissionField::Metadata.into(), value: self.send_at.to_native(), }), value: KeySerializer::new(U32_LEN * 3 + 1) .write(self.email_id.to_native()) .write(self.thread_id.to_native()) .write(self.identity_id.to_native()) .write(self.undo_status.as_index()) .finalize() .into(), }, IndexValue::LogItem { sync_collection: SyncCollection::EmailSubmission, prefix: None, }, ] .into_iter() } } impl IndexableAndSerializableObject for EmailSubmission { fn is_versioned() -> bool { false } } ================================================ FILE: crates/email/src/submission/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use utils::map::vec_map::VecMap; pub mod index; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct EmailSubmission { pub email_id: u32, pub thread_id: u32, pub identity_id: u32, pub send_at: u64, pub queue_id: Option, pub undo_status: UndoStatus, pub envelope: Envelope, pub delivery_status: VecMap, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct Envelope { pub mail_from: Address, pub rcpt_to: Vec
, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct Address { pub email: String, pub parameters: Option>>, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct DeliveryStatus { pub smtp_reply: String, pub delivered: Delivered, pub displayed: bool, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub enum Delivered { Queued, Yes, No, #[default] Unknown, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub enum UndoStatus { #[default] Pending, Final, Canceled, } impl UndoStatus { pub fn parse(s: &str) -> Option { hashify::tiny_map!(s.as_bytes(), "pending" => UndoStatus::Pending, "final" => UndoStatus::Final, "canceled" => UndoStatus::Canceled, "cancelled" => UndoStatus::Canceled, ) } pub fn as_str(&self) -> &'static str { match self { UndoStatus::Pending => "pending", UndoStatus::Final => "final", UndoStatus::Canceled => "canceled", } } pub fn as_index(&self) -> u8 { match self { UndoStatus::Pending => b'p', UndoStatus::Final => b'f', UndoStatus::Canceled => b'c', } } } impl ArchivedUndoStatus { pub fn as_str(&self) -> &'static str { match self { ArchivedUndoStatus::Pending => "pending", ArchivedUndoStatus::Final => "final", ArchivedUndoStatus::Canceled => "canceled", } } pub fn as_index(&self) -> u8 { match self { ArchivedUndoStatus::Pending => b'p', ArchivedUndoStatus::Final => b'f', ArchivedUndoStatus::Canceled => b'c', } } } impl From<&ArchivedDeliveryStatus> for DeliveryStatus { fn from(value: &ArchivedDeliveryStatus) -> Self { DeliveryStatus { smtp_reply: value.smtp_reply.to_string(), delivered: match value.delivered { ArchivedDelivered::Queued => Delivered::Queued, ArchivedDelivered::Yes => Delivered::Yes, ArchivedDelivered::No => Delivered::No, ArchivedDelivered::Unknown => Delivered::Unknown, }, displayed: value.displayed, } } } impl Delivered { pub fn as_str(&self) -> &'static str { match self { Delivered::Queued => "queued", Delivered::Yes => "yes", Delivered::No => "no", Delivered::Unknown => "unknown", } } } impl ArchivedDelivered { pub fn as_str(&self) -> &'static str { match self { ArchivedDelivered::Queued => "queued", ArchivedDelivered::Yes => "yes", ArchivedDelivered::No => "no", ArchivedDelivered::Unknown => "unknown", } } } ================================================ FILE: crates/groupware/Cargo.toml ================================================ [package] name = "groupware" version = "0.15.5" edition = "2024" [dependencies] utils = { path = "../utils" } store = { path = "../store" } common = { path = "../common" } types = { path = "../types" } trc = { path = "../trc" } nlp = { path = "../nlp" } directory = { path = "../directory" } calcard = { version = "0.3", features = ["rkyv"] } hashify = "0.2" tokio = { version = "1.47", features = ["net", "macros"] } rkyv = { version = "0.8.10", features = ["little_endian"] } percent-encoding = "2.3.1" compact_str = "0.9.0" ahash = { version = "0.8" } chrono = "0.4.40" [features] test_mode = [] enterprise = [] [dev-dependencies] tokio = { version = "1.47", features = ["full"] } ================================================ FILE: crates/groupware/src/cache/calcard.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::GroupwareCache; use crate::{ DavResourceName, RFC_3986, calendar::{ ArchivedCalendar, ArchivedCalendarEvent, Calendar, CalendarEvent, SCHEDULE_INBOX_ID, SCHEDULE_OUTBOX_ID, storage::ItipAutoExpunge, }, contact::{AddressBook, ArchivedAddressBook, ArchivedContactCard, ContactCard}, }; use calcard::common::timezone::Tz; use common::{ DavName, DavPath, DavResource, DavResourceMetadata, DavResources, Server, TinyCalendarPreferences, auth::AccessToken, }; use directory::backend::internal::manage::ManageDirectory; use std::sync::Arc; use store::ahash::{AHashMap, AHashSet}; use tokio::sync::Semaphore; use trc::AddContext; use types::{ acl::AclGrant, collection::{Collection, SyncCollection}, }; use utils::map::bitmap::Bitmap; pub(super) async fn build_calcard_resources( server: &Server, access_token: &AccessToken, account_id: u32, sync_collection: SyncCollection, container_collection: Collection, item_collection: Collection, update_lock: Arc, ) -> trc::Result { let is_calendar = matches!(sync_collection, SyncCollection::Calendar); let name = server .store() .get_principal_name(account_id) .await .caused_by(trc::location!())? .unwrap_or_else(|| format!("_{account_id}")); let mut cache = DavResources { base_path: format!( "{}/{}/", if is_calendar { DavResourceName::Cal } else { DavResourceName::Card } .base_path(), percent_encoding::utf8_percent_encode(&name, RFC_3986), ), paths: AHashSet::with_capacity(16), resources: Vec::with_capacity(16), item_change_id: 0, container_change_id: 0, highest_change_id: 0, size: std::mem::size_of::() as u64, update_lock, }; let mut is_first_check = true; loop { let last_change_id = server .core .storage .data .get_last_change_id(account_id, sync_collection.into()) .await .caused_by(trc::location!())? .unwrap_or_default(); cache.item_change_id = last_change_id; cache.container_change_id = last_change_id; cache.highest_change_id = last_change_id; server .archives( account_id, container_collection, &(), |document_id, archive| { let resource = if is_calendar { resource_from_calendar(archive.unarchive::()?, document_id) } else { resource_from_addressbook(archive.unarchive::()?, document_id) }; let path = DavPath { path: resource.container_name().unwrap().to_string(), parent_id: None, hierarchy_seq: 1, resource_idx: cache.resources.len(), }; cache.size += (std::mem::size_of::() + std::mem::size_of::() + (path.path.len()) * 2) as u64; cache.paths.insert(path); cache.resources.push(resource); Ok(true) }, ) .await .caused_by(trc::location!())?; if cache.paths.is_empty() { if is_first_check { if is_calendar { server .create_default_calendar(access_token, account_id, &name) .await?; } else { server .create_default_addressbook(access_token, account_id, &name) .await?; } is_first_check = false; continue; } else { return Ok(cache); } } let parent_range = cache.resources.len(); server .archives(account_id, item_collection, &(), |document_id, archive| { let resource = if is_calendar { resource_from_event(archive.unarchive::()?, document_id) } else { resource_from_card(archive.unarchive::()?, document_id) }; let resource_idx = cache.resources.len(); for name in resource.child_names().unwrap_or_default().iter() { if let Some(parent) = cache.resources.get(..parent_range).and_then(|resources| { resources.iter().find(|r| r.document_id == name.parent_id) }) { let path = DavPath { path: format!("{}/{}", parent.container_name().unwrap(), name.name), parent_id: Some(name.parent_id), hierarchy_seq: 0, resource_idx, }; cache.size += (std::mem::size_of::() + name.name.len() + path.path.len()) as u64; cache.paths.insert(path); } } cache.size += std::mem::size_of::() as u64; cache.resources.push(resource); Ok(true) }) .await .caused_by(trc::location!())?; return Ok(cache); } } pub(super) async fn build_scheduling_resources( server: &Server, account_id: u32, update_lock: Arc, ) -> trc::Result { let last_change_id = server .core .storage .data .get_last_change_id(account_id, SyncCollection::CalendarEventNotification.into()) .await .caused_by(trc::location!())? .unwrap_or_default(); let name = server .store() .get_principal_name(account_id) .await .caused_by(trc::location!())? .unwrap_or_else(|| format!("_{account_id}")); let item_ids = server .itip_ids(account_id) .await .caused_by(trc::location!())?; let mut cache = DavResources { base_path: format!( "{}/{}/", DavResourceName::Scheduling.base_path(), percent_encoding::utf8_percent_encode(&name, RFC_3986), ), paths: AHashSet::with_capacity((2 + item_ids.len()) as usize), resources: Vec::with_capacity((2 + item_ids.len()) as usize), item_change_id: last_change_id, container_change_id: last_change_id, highest_change_id: last_change_id, size: std::mem::size_of::() as u64, update_lock, }; for (document_id, is_container) in item_ids .into_iter() .map(|document_id| (document_id, false)) .chain([(SCHEDULE_INBOX_ID, true), (SCHEDULE_OUTBOX_ID, true)]) { let path = path_from_scheduling(document_id, cache.resources.len(), is_container); cache.size += (std::mem::size_of::() + (path.path.len() * 2)) as u64 + std::mem::size_of::() as u64; cache.paths.insert(path); cache .resources .push(resource_from_scheduling(document_id, is_container)); } Ok(cache) } pub(super) fn build_simple_hierarchy(cache: &mut DavResources) { cache.paths = AHashSet::with_capacity(cache.resources.len()); let name_idx = cache .resources .iter() .filter_map(|resource| { resource .container_name() .map(|name| (resource.document_id, name)) }) .collect::>(); for (resource_idx, resource) in cache.resources.iter().enumerate() { match &resource.data { DavResourceMetadata::Calendar { name, .. } | DavResourceMetadata::AddressBook { name, .. } => { let path = DavPath { path: name.to_string(), parent_id: None, hierarchy_seq: 1, resource_idx, }; cache.size += (std::mem::size_of::() + name.len() + path.path.len()) as u64; cache.paths.insert(path); } DavResourceMetadata::CalendarEvent { names, .. } | DavResourceMetadata::ContactCard { names } => { for name in names { if let Some(parent_name) = name_idx.get(&name.parent_id) { let path = DavPath { path: format!("{parent_name}/{}", name.name), parent_id: Some(name.parent_id), hierarchy_seq: 0, resource_idx, }; cache.size += (std::mem::size_of::() + name.name.len() + path.path.len()) as u64; cache.paths.insert(path); } } } _ => unreachable!(), } cache.size += std::mem::size_of::() as u64; } } pub(super) fn resource_from_calendar(calendar: &ArchivedCalendar, document_id: u32) -> DavResource { DavResource { document_id, data: DavResourceMetadata::Calendar { name: calendar.name.to_string(), acls: calendar .acls .iter() .map(|acl| AclGrant { account_id: acl.account_id.to_native(), grants: Bitmap::from(&acl.grants), }) .collect(), preferences: calendar .preferences .iter() .map(|pref| TinyCalendarPreferences { account_id: pref.account_id.to_native(), flags: pref.flags.to_native(), tz: pref.time_zone.tz().unwrap_or(Tz::UTC), }) .collect(), }, } } pub(super) fn resource_from_event(event: &ArchivedCalendarEvent, document_id: u32) -> DavResource { let (start, duration) = event.data.event_range().unwrap_or_default(); DavResource { document_id, data: DavResourceMetadata::CalendarEvent { names: event .names .iter() .map(|name| DavName { name: name.name.to_string(), parent_id: name.parent_id.to_native(), }) .collect(), start, duration, }, } } pub(super) fn resource_from_scheduling(document_id: u32, is_container: bool) -> DavResource { DavResource { document_id, data: DavResourceMetadata::CalendarEventNotification { names: if !is_container { [DavName { name: format!("{document_id}.ics"), parent_id: SCHEDULE_INBOX_ID, }] .into_iter() .collect() } else { Default::default() }, }, } } pub(super) fn path_from_scheduling( document_id: u32, resource_idx: usize, is_container: bool, ) -> DavPath { if is_container { DavPath { path: if document_id == SCHEDULE_INBOX_ID { "inbox".to_string() } else { "outbox".to_string() }, parent_id: None, hierarchy_seq: 1, resource_idx, } } else { DavPath { path: format!("inbox/{document_id}.ics"), parent_id: Some(SCHEDULE_INBOX_ID), hierarchy_seq: 0, resource_idx, } } } pub(super) fn resource_from_addressbook( book: &ArchivedAddressBook, document_id: u32, ) -> DavResource { DavResource { document_id, data: DavResourceMetadata::AddressBook { name: book.name.to_string(), acls: book .acls .iter() .map(|acl| AclGrant { account_id: acl.account_id.to_native(), grants: Bitmap::from(&acl.grants), }) .collect(), }, } } pub(super) fn resource_from_card(card: &ArchivedContactCard, document_id: u32) -> DavResource { DavResource { document_id, data: DavResourceMetadata::ContactCard { names: card .names .iter() .map(|name| DavName { name: name.name.to_string(), parent_id: name.parent_id.to_native(), }) .collect(), }, } } ================================================ FILE: crates/groupware/src/cache/file.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ DavResourceName, RFC_3986, file::{ArchivedFileNode, FileNode}, }; use common::{DavPath, DavResource, DavResourceMetadata, DavResources, Server}; use directory::backend::internal::manage::ManageDirectory; use std::sync::Arc; use store::ahash::{AHashMap, AHashSet}; use tokio::sync::Semaphore; use trc::AddContext; use types::{ acl::AclGrant, collection::{Collection, SyncCollection}, }; use utils::{map::bitmap::Bitmap, topological::TopologicalSort}; pub(super) async fn build_file_resources( server: &Server, account_id: u32, update_lock: Arc, ) -> trc::Result { let last_change_id = server .core .storage .data .get_last_change_id(account_id, SyncCollection::FileNode.into()) .await .caused_by(trc::location!())? .unwrap_or_default(); let name = server .store() .get_principal_name(account_id) .await .caused_by(trc::location!())? .unwrap_or_else(|| format!("_{account_id}")); let mut resources = Vec::with_capacity(16); server .archives( account_id, Collection::FileNode, &(), |document_id, archive| { resources.push(resource_from_file( archive.unarchive::()?, document_id, )); Ok(true) }, ) .await .caused_by(trc::location!())?; let mut files = DavResources { base_path: format!( "{}/{}/", DavResourceName::File.base_path(), percent_encoding::utf8_percent_encode(&name, RFC_3986), ), size: std::mem::size_of::() as u64, paths: AHashSet::with_capacity(resources.len()), resources, item_change_id: last_change_id, container_change_id: last_change_id, highest_change_id: last_change_id, update_lock, }; build_nested_hierarchy(&mut files); Ok(files) } pub(super) fn build_nested_hierarchy(resources: &mut DavResources) { let mut topological_sort = TopologicalSort::with_capacity(resources.resources.len()); let mut names = AHashMap::with_capacity(resources.resources.len()); for (resource_idx, resource) in resources.resources.iter().enumerate() { if let DavResourceMetadata::File { parent_id, .. } = resource.data { topological_sort.insert( parent_id.map(|id| id + 1).unwrap_or_default(), resource.document_id + 1, ); names.insert( resource.document_id, DavPath { path: resource.container_name().unwrap().to_string(), parent_id, hierarchy_seq: 0, resource_idx, }, ); } } for (hierarchy_sequence, folder_id) in topological_sort.into_iterator().enumerate() { if folder_id != 0 { let folder_id = folder_id - 1; if let Some((name, parent_name)) = names .get(&folder_id) .and_then(|folder| folder.parent_id.map(|parent_id| (&folder.path, parent_id))) .and_then(|(name, parent_id)| { names.get(&parent_id).map(|folder| (name, &folder.path)) }) { let name = format!("{parent_name}/{name}"); let folder = names.get_mut(&folder_id).unwrap(); folder.path = name; folder.hierarchy_seq = hierarchy_sequence as u32; } else { names.get_mut(&folder_id).unwrap().hierarchy_seq = hierarchy_sequence as u32; } } } resources.paths = names .into_values() .inspect(|v| { resources.size += (std::mem::size_of::() + std::mem::size_of::() + std::mem::size_of::() + std::mem::size_of::() + v.path.len()) as u64; }) .collect(); } pub(super) fn resource_from_file(node: &ArchivedFileNode, document_id: u32) -> DavResource { let parent_id = node.parent_id.to_native(); DavResource { document_id, data: DavResourceMetadata::File { name: node.name.as_str().to_string(), size: node.file.as_ref().map(|f| f.size.to_native()), parent_id: if parent_id > 0 { Some(parent_id - 1) } else { None }, acls: node .acls .iter() .map(|acl| AclGrant { account_id: acl.account_id.to_native(), grants: Bitmap::from(&acl.grants), }) .collect(), }, } } ================================================ FILE: crates/groupware/src/cache/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ cache::calcard::{build_scheduling_resources, path_from_scheduling, resource_from_scheduling}, calendar::{Calendar, CalendarEvent, CalendarPreferences}, contact::{AddressBook, AddressBookPreferences, ContactCard}, file::FileNode, }; use ahash::AHashSet; use calcard::{ build_calcard_resources, build_simple_hierarchy, resource_from_addressbook, resource_from_calendar, resource_from_card, resource_from_event, }; use common::{CacheSwap, DavResource, DavResources, Server, auth::AccessToken}; use file::{build_file_resources, build_nested_hierarchy, resource_from_file}; use std::{sync::Arc, time::Instant}; use store::{ SerializeInfallible, ValueKey, ahash::AHashMap, query::log::{Change, Query}, write::{AlignedBytes, Archive, BatchBuilder, ValueClass}, }; use tokio::sync::Semaphore; use trc::{AddContext, StoreEvent}; use types::{ collection::{Collection, SyncCollection}, field::PrincipalField, }; pub mod calcard; pub mod file; pub trait GroupwareCache: Sync + Send { fn fetch_dav_resources( &self, access_token: &AccessToken, account_id: u32, collection: SyncCollection, ) -> impl Future>> + Send; fn create_default_addressbook( &self, access_token: &AccessToken, account_id: u32, account_name: &str, ) -> impl Future>> + Send; fn create_default_calendar( &self, access_token: &AccessToken, account_id: u32, account_name: &str, ) -> impl Future>> + Send; fn get_or_create_default_calendar( &self, access_token: &AccessToken, account_id: u32, ) -> impl Future>> + Send; fn cached_dav_resources( &self, account_id: u32, collection: SyncCollection, ) -> Option>; } impl GroupwareCache for Server { async fn fetch_dav_resources( &self, access_token: &AccessToken, account_id: u32, collection: SyncCollection, ) -> trc::Result> { let cache_store = match collection { SyncCollection::Calendar => &self.inner.cache.events, SyncCollection::AddressBook => &self.inner.cache.contacts, SyncCollection::FileNode => &self.inner.cache.files, SyncCollection::CalendarEventNotification => &self.inner.cache.scheduling, _ => unreachable!(), }; let cache_ = match cache_store.get_value_or_guard_async(&account_id).await { Ok(cache) => cache, Err(guard) => { let start_time = Instant::now(); let cache = full_cache_build( self, account_id, collection, Arc::new(Semaphore::new(1)), access_token, ) .await?; if guard.insert(CacheSwap::new(cache.clone())).is_err() { cache_store.insert(account_id, CacheSwap::new(cache.clone())); } trc::event!( Store(StoreEvent::CacheMiss), AccountId = account_id, Collection = collection.as_str(), Total = cache.resources.len(), ChangeId = cache.highest_change_id, Elapsed = start_time.elapsed(), ); return Ok(cache); } }; // Obtain current state let cache = cache_.load_full(); let start_time = Instant::now(); let changes = self .core .storage .data .changes( account_id, collection.into(), Query::Since(cache.highest_change_id), ) .await .caused_by(trc::location!())?; // Regenerate cache if the change log has been truncated if changes.is_truncated { let cache = full_cache_build( self, account_id, collection, cache.update_lock.clone(), access_token, ) .await?; cache_.update(cache.clone()); trc::event!( Store(StoreEvent::CacheStale), AccountId = account_id, Collection = collection.as_str(), ChangeId = cache.highest_change_id, Total = cache.resources.len(), Elapsed = start_time.elapsed(), ); return Ok(cache); } // Verify changes if changes.changes.is_empty() { trc::event!( Store(StoreEvent::CacheHit), AccountId = account_id, Collection = collection.as_str(), ChangeId = cache.highest_change_id, Elapsed = start_time.elapsed(), ); return Ok(cache); } // Lock for updates let _permit = cache.update_lock.acquire().await; let cache = cache_.load_full(); if cache.highest_change_id >= changes.to_change_id { trc::event!( Store(StoreEvent::CacheHit), AccountId = account_id, Collection = collection.as_str(), ChangeId = cache.highest_change_id, Elapsed = start_time.elapsed(), ); return Ok(cache); } let num_changes = changes.changes.len(); let cache = if !matches!(collection, SyncCollection::CalendarEventNotification) { let mut updated_resources = AHashMap::with_capacity(8); let has_no_children = collection == SyncCollection::FileNode; process_changes( self, account_id, collection, has_no_children, &mut updated_resources, changes.changes, ) .await?; let mut rebuild_hierarchy = false; let mut resources = Vec::with_capacity(cache.resources.len()); for resource in &cache.resources { let is_container = has_no_children || resource.is_container(); if let Some(updated_resource) = updated_resources.remove(&(is_container, resource.document_id)) { if let Some(updated_resource) = updated_resource { rebuild_hierarchy = rebuild_hierarchy || updated_resource.has_hierarchy_changes(resource); resources.push(updated_resource); } else { // Deleted resource rebuild_hierarchy = true; } } else { resources.push(resource.clone()); } } // Add new resources for resource in updated_resources.into_values().flatten() { resources.push(resource); rebuild_hierarchy = true; } if rebuild_hierarchy { let mut cache = DavResources { base_path: cache.base_path.clone(), paths: Default::default(), resources, item_change_id: changes.item_change_id.unwrap_or(cache.item_change_id), container_change_id: changes .container_change_id .unwrap_or(cache.container_change_id), highest_change_id: changes.to_change_id, size: std::mem::size_of::() as u64, update_lock: cache.update_lock.clone(), }; if matches!(collection, SyncCollection::FileNode) { build_nested_hierarchy(&mut cache); } else { build_simple_hierarchy(&mut cache); } cache } else { DavResources { base_path: cache.base_path.clone(), paths: cache.paths.clone(), resources, item_change_id: changes.item_change_id.unwrap_or(cache.item_change_id), container_change_id: changes .container_change_id .unwrap_or(cache.container_change_id), highest_change_id: changes.to_change_id, size: cache.size, update_lock: cache.update_lock.clone(), } } } else { let mut delete_ids = AHashSet::with_capacity(changes.changes.len()); let mut resources = Vec::with_capacity(cache.resources.len()); let mut paths = AHashSet::with_capacity(cache.paths.len()); for change in changes.changes { match change { Change::InsertItem(document_id) => { let document_id = document_id as u32; paths.insert(path_from_scheduling(document_id, resources.len(), false)); resources.push(resource_from_scheduling(document_id, false)); } Change::DeleteItem(document_id) => { delete_ids.insert(document_id as u32); } _ => {} } } for resource in &cache.resources { if !delete_ids.contains(&resource.document_id) { paths.insert(path_from_scheduling( resource.document_id, resources.len(), resource.is_container(), )); resources.push(resource.clone()); } } DavResources { base_path: cache.base_path.clone(), paths, resources, item_change_id: changes.item_change_id.unwrap_or(cache.item_change_id), container_change_id: changes .container_change_id .unwrap_or(cache.container_change_id), highest_change_id: changes.to_change_id, size: cache.size, update_lock: cache.update_lock.clone(), } }; let cache = Arc::new(cache); cache_.update(cache.clone()); trc::event!( Store(StoreEvent::CacheUpdate), AccountId = account_id, Collection = collection.as_str(), ChangeId = cache.highest_change_id, Details = num_changes, Total = cache.resources.len(), Elapsed = start_time.elapsed(), ); Ok(cache) } async fn create_default_addressbook( &self, access_token: &AccessToken, account_id: u32, account_name: &str, ) -> trc::Result> { if let Some(name) = &self.core.groupware.default_addressbook_name { let mut batch = BatchBuilder::new(); let document_id = self .store() .assign_document_ids(account_id, Collection::AddressBook, 1) .await?; AddressBook { name: name.clone(), preferences: vec![AddressBookPreferences { account_id, name: format!( "{} ({})", self.core .groupware .default_addressbook_display_name .as_ref() .unwrap_or(name), account_name ), ..Default::default() }], ..Default::default() } .insert(access_token, account_id, document_id, &mut batch)?; self.commit_batch(batch).await?; Ok(Some(document_id)) } else { Ok(None) } } async fn create_default_calendar( &self, access_token: &AccessToken, account_id: u32, account_name: &str, ) -> trc::Result> { if let Some(name) = &self.core.groupware.default_calendar_name { let mut batch = BatchBuilder::new(); let document_id = self .store() .assign_document_ids(account_id, Collection::Calendar, 1) .await?; Calendar { name: name.clone(), preferences: vec![CalendarPreferences { account_id, name: format!( "{} ({})", self.core .groupware .default_calendar_display_name .as_ref() .unwrap_or(name), account_name ), ..Default::default() }], ..Default::default() } .insert(access_token, account_id, document_id, &mut batch)?; // Set default calendar batch .with_collection(Collection::Principal) .with_document(0) .set(PrincipalField::DefaultCalendarId, document_id.serialize()); self.commit_batch(batch).await?; Ok(Some(document_id)) } else { Ok(None) } } async fn get_or_create_default_calendar( &self, access_token: &AccessToken, account_id: u32, ) -> trc::Result> { let default_calendar_id = self .store() .get_value::(ValueKey { account_id, collection: Collection::Principal.into(), document_id: 0, class: ValueClass::Property(PrincipalField::DefaultCalendarId.into()), }) .await .caused_by(trc::location!())?; if default_calendar_id.is_some() { Ok(default_calendar_id) } else { self.fetch_dav_resources(access_token, account_id, SyncCollection::Calendar) .await .map(|c| c.document_ids(true).next()) } } fn cached_dav_resources( &self, account_id: u32, collection: SyncCollection, ) -> Option> { (match collection { SyncCollection::Calendar => &self.inner.cache.events, SyncCollection::AddressBook => &self.inner.cache.contacts, SyncCollection::FileNode => &self.inner.cache.files, _ => unreachable!(), }) .get(&account_id) .map(|cache| cache.load_full()) } } async fn process_changes( server: &Server, account_id: u32, collection: SyncCollection, has_no_children: bool, updated_resources: &mut AHashMap<(bool, u32), Option>, changes: Vec, ) -> trc::Result<()> { for change in changes { match change { Change::InsertItem(id) | Change::UpdateItem(id) => { let document_id = id as u32; if let Some(archive) = server .store() .get_value::>(ValueKey::archive( account_id, collection.collection(false), document_id, )) .await .caused_by(trc::location!())? { updated_resources.insert( (has_no_children, document_id), Some(resource_from_archive( archive, document_id, collection, false, )?), ); } else { updated_resources.insert((has_no_children, document_id), None); } } Change::DeleteItem(id) => { updated_resources.insert((has_no_children, id as u32), None); } Change::InsertContainer(id) | Change::UpdateContainer(id) => { let document_id = id as u32; if let Some(archive) = server .store() .get_value::>(ValueKey::archive( account_id, collection.collection(true), document_id, )) .await .caused_by(trc::location!())? { updated_resources.insert( (true, document_id), Some(resource_from_archive( archive, document_id, collection, true, )?), ); } else { updated_resources.insert((true, document_id), None); } } Change::DeleteContainer(id) => { updated_resources.insert((true, id as u32), None); } Change::UpdateContainerProperty(_) => (), } } Ok(()) } async fn full_cache_build( server: &Server, account_id: u32, collection: SyncCollection, update_lock: Arc, access_token: &AccessToken, ) -> trc::Result> { match collection { SyncCollection::Calendar => { build_calcard_resources( server, access_token, account_id, SyncCollection::Calendar, Collection::Calendar, Collection::CalendarEvent, update_lock, ) .await } SyncCollection::AddressBook => { build_calcard_resources( server, access_token, account_id, SyncCollection::AddressBook, Collection::AddressBook, Collection::ContactCard, update_lock, ) .await } SyncCollection::FileNode => build_file_resources(server, account_id, update_lock).await, SyncCollection::CalendarEventNotification => { build_scheduling_resources(server, account_id, update_lock).await } _ => unreachable!(), } .map(Arc::new) } fn resource_from_archive( archive: Archive, document_id: u32, collection: SyncCollection, is_container: bool, ) -> trc::Result { Ok(match collection { SyncCollection::Calendar => { if is_container { resource_from_calendar( archive .unarchive::() .caused_by(trc::location!())?, document_id, ) } else { resource_from_event( archive .unarchive::() .caused_by(trc::location!())?, document_id, ) } } SyncCollection::AddressBook => { if is_container { resource_from_addressbook( archive .unarchive::() .caused_by(trc::location!())?, document_id, ) } else { resource_from_card( archive .unarchive::() .caused_by(trc::location!())?, document_id, ) } } SyncCollection::FileNode => resource_from_file( archive .unarchive::() .caused_by(trc::location!())?, document_id, ), _ => unreachable!(), }) } ================================================ FILE: crates/groupware/src/calendar/alarm.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{Alarm, AlarmDelta, ArchivedAlarmDelta, ArchivedCalendarEventData}; use calcard::{ common::timezone::Tz, icalendar::{ ICalendarComponent, ICalendarParameterName, ICalendarParameterValue, ICalendarProperty, ICalendarRelated, ICalendarValue, }, }; use chrono::{DateTime, TimeZone}; use std::str::FromStr; use store::write::bitpack::BitpackIterator; use utils::codec::leb128::Leb128Reader; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct CalendarAlarm { pub alarm_id: u16, pub event_id: u16, pub alarm_time: i64, pub typ: CalendarAlarmType, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum CalendarAlarmType { Email { event_start: i64, event_start_tz: u16, event_end: i64, event_end_tz: u16, }, Display { recurrence_id: Option, }, } impl ArchivedCalendarEventData { pub fn next_alarm(&self, start_time: i64, default_tz: Tz) -> Option { if self.alarms.is_empty() { return None; } let base_offset = self.base_offset.to_native(); let mut next_alarm: Option = None; 'outer: for range in self.time_ranges.iter() { let comp_id = range.id.to_native(); let Some(alarm) = self.alarms.iter().find(|a| a.parent_id == comp_id) else { continue; }; let instances = range.instances.as_ref(); let (offset_or_count, bytes_read) = instances.read_leb128::()?; let duration = range.duration.to_native() as i64; let mut start_tz = Tz::from_id(range.start_tz.to_native())?; let mut end_tz = Tz::from_id(range.end_tz.to_native())?; if start_tz.is_floating() && !default_tz.is_floating() { start_tz = default_tz; } if end_tz.is_floating() && !default_tz.is_floating() { end_tz = default_tz; } if instances.len() > bytes_read { // Recurring event let unpacker = BitpackIterator::from_bytes_and_offset(instances, bytes_read, offset_or_count); for start_offset in unpacker { let start_date_naive = start_offset as i64 + base_offset; let end_date_naive = start_date_naive + duration; let start = start_tz .from_local_datetime( &DateTime::from_timestamp(start_date_naive, 0)?.naive_local(), ) .single()? .timestamp(); let end = end_tz .from_local_datetime( &DateTime::from_timestamp(end_date_naive, 0)?.naive_local(), ) .single()? .timestamp(); if let Some(alarm_time) = alarm.delta.to_timestamp(start, end, default_tz) && alarm_time > start_time && next_alarm .as_ref() .is_none_or(|next| alarm_time < next.alarm_time) { next_alarm = Some(CalendarAlarm { alarm_id: alarm.id.to_native(), event_id: alarm.parent_id.to_native(), alarm_time, typ: if alarm.is_email_alert { CalendarAlarmType::Email { event_start: start_date_naive, event_start_tz: start_tz.as_id(), event_end: end_date_naive, event_end_tz: end_tz.as_id(), } } else { let comp = &self.event.components[alarm.parent_id.to_native() as usize]; CalendarAlarmType::Display { recurrence_id: if comp.is_recurrent_or_override() { start_date_naive.into() } else { None }, } }, }); continue 'outer; } } } else { // Single event let start_date_naive = offset_or_count as i64 + base_offset; let end_date_naive = start_date_naive + duration; let start = start_tz .from_local_datetime( &DateTime::from_timestamp(start_date_naive, 0)?.naive_local(), ) .single()? .timestamp(); let end = end_tz .from_local_datetime( &DateTime::from_timestamp(end_date_naive, 0)?.naive_local(), ) .single()? .timestamp(); if let Some(alarm_time) = alarm.delta.to_timestamp(start, end, default_tz) && alarm_time > start_time && next_alarm .as_ref() .is_none_or(|next| alarm_time < next.alarm_time) { next_alarm = Some(CalendarAlarm { alarm_id: alarm.id.to_native(), event_id: alarm.parent_id.to_native(), alarm_time, typ: if alarm.is_email_alert { CalendarAlarmType::Email { event_start: start_date_naive, event_start_tz: start_tz.as_id(), event_end: end_date_naive, event_end_tz: end_tz.as_id(), } } else { let comp = &self.event.components[alarm.parent_id.to_native() as usize]; CalendarAlarmType::Display { recurrence_id: if comp.is_recurrent_or_override() { start_date_naive.into() } else { None }, } }, }); } } } next_alarm } } pub trait ExpandAlarm { fn expand_alarm(&self, id: u16, parent_id: u16) -> Option; } impl ExpandAlarm for ICalendarComponent { fn expand_alarm(&self, id: u16, parent_id: u16) -> Option { let mut trigger = None; let mut is_email_alert = false; for entry in self.entries.iter() { match &entry.name { ICalendarProperty::Trigger => { let mut tz = None; let mut trigger_start = true; for param in entry.params.iter() { match (¶m.name, ¶m.value) { ( ICalendarParameterName::Related, ICalendarParameterValue::Related(related), ) => { trigger_start = matches!(related, ICalendarRelated::Start); } ( ICalendarParameterName::Tzid, ICalendarParameterValue::Text(tz_id), ) => { tz = Tz::from_str(tz_id).ok(); } _ => {} } } trigger = match entry.values.first()? { ICalendarValue::PartialDateTime(dt) => { let tz = tz.unwrap_or(Tz::Floating); dt.to_date_time_with_tz(tz).map(|dt| { let timestamp = dt.timestamp(); if !dt.timezone().is_floating() { AlarmDelta::FixedUtc(timestamp) } else { AlarmDelta::FixedFloating(timestamp) } }) } ICalendarValue::Duration(duration) => { if trigger_start { Some(AlarmDelta::Start(duration.as_seconds())) } else { Some(AlarmDelta::End(duration.as_seconds())) } } _ => None, }; } ICalendarProperty::Action => { is_email_alert = is_email_alert || entry .values .first() .and_then(|v| v.as_text()) .is_some_and(|v| v.eq_ignore_ascii_case("email")); } ICalendarProperty::Summary | ICalendarProperty::Description => { is_email_alert = is_email_alert || entry .values .first() .and_then(|v| v.as_text()) .is_some_and(|v| v.contains("@email")); } _ => {} } } trigger.map(|delta| Alarm { id, parent_id, delta, is_email_alert, }) } } impl AlarmDelta { pub fn to_timestamp(&self, start: i64, end: i64, default_tz: Tz) -> Option { match self { AlarmDelta::Start(delta) => Some(start + delta), AlarmDelta::End(delta) => Some(end + delta), AlarmDelta::FixedUtc(timestamp) => Some(*timestamp), AlarmDelta::FixedFloating(timestamp) => default_tz .from_local_datetime(&DateTime::from_timestamp(*timestamp, 0)?.naive_local()) .single() .map(|dt| dt.timestamp()), } } } impl ArchivedAlarmDelta { pub fn to_timestamp(&self, start: i64, end: i64, default_tz: Tz) -> Option { match self { ArchivedAlarmDelta::Start(delta) => Some(start + delta.to_native()), ArchivedAlarmDelta::End(delta) => Some(end + delta.to_native()), ArchivedAlarmDelta::FixedUtc(timestamp) => Some(timestamp.to_native()), ArchivedAlarmDelta::FixedFloating(timestamp) => default_tz .from_local_datetime( &DateTime::from_timestamp(timestamp.to_native(), 0)?.naive_local(), ) .single() .map(|dt| dt.timestamp()), } } } ================================================ FILE: crates/groupware/src/calendar/dates.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ ArchivedCalendarEventData, ArchivedTimezone, CalendarEventData, Timezone, alarm::{CalendarAlarm, ExpandAlarm}, }; use crate::calendar::{ComponentTimeRange, alarm::CalendarAlarmType}; use calcard::{ common::timezone::Tz, icalendar::{ICalendar, ICalendarComponentType, dates::TimeOrDelta}, }; use compact_str::ToCompactString; use store::{ ahash::AHashMap, write::{key::KeySerializer, now}, }; impl CalendarEventData { pub fn new( ical: ICalendar, default_tz: Tz, max_expansions: usize, next_email_alarm: &mut Option, ) -> Self { let mut ranges = TimeRanges::default(); let now = now() as i64; let expanded = ical.expand_dates(default_tz, max_expansions); let mut groups: AHashMap<(u16, u16, u16, i32), Vec> = AHashMap::with_capacity(16); let mut alarms = AHashMap::with_capacity(16); for event in expanded.events { let start_naive = event.start.naive_local(); let start_tz = event.start.timezone().as_id(); let start_timestamp_utc = event.start.timestamp(); let start_timestamp_naive = start_naive.and_utc().timestamp(); let (end_timestamp_utc, end_timestamp_naive, end_tz) = match event.end { TimeOrDelta::Time(time) => { let end_naive = time.naive_local(); let end_timestamp_utc = time.timestamp(); let end_timestamp_naive = end_naive.and_utc().timestamp(); ( end_timestamp_utc, end_timestamp_naive, time.timezone().as_id(), ) } TimeOrDelta::Delta(delta) => { let delta = delta.num_seconds(); ( start_timestamp_utc + delta, start_timestamp_naive + delta, start_tz, ) } }; // Expand alarms let mut min = std::cmp::min(start_timestamp_utc, end_timestamp_utc); let mut max = std::cmp::max(start_timestamp_utc, end_timestamp_utc); for alarm in alarms.entry(event.comp_id).or_insert_with(|| { ical.component_by_id(event.comp_id) .map_or(&[][..], |c| c.component_ids.as_slice()) .iter() .filter_map(|alarm_id| { ical.component_by_id(*alarm_id).and_then(|alarm| { if alarm.component_type == ICalendarComponentType::VAlarm { alarm.expand_alarm(*alarm_id as u16, event.comp_id as u16) } else { None } }) }) .collect::>() }) { if let Some(alarm_time) = alarm .delta .to_timestamp(start_timestamp_utc, end_timestamp_utc, default_tz) { if alarm_time < min { min = alarm_time; } if alarm_time > max { max = alarm_time; } if alarm_time > now && next_email_alarm .as_ref() .is_none_or(|next| alarm_time < next.alarm_time) { *next_email_alarm = Some(CalendarAlarm { alarm_id: alarm.id, event_id: alarm.parent_id, alarm_time, typ: if alarm.is_email_alert { CalendarAlarmType::Email { event_start: start_timestamp_naive, event_end: end_timestamp_naive, event_start_tz: start_tz, event_end_tz: end_tz, } } else { CalendarAlarmType::Display { recurrence_id: if ical.components[alarm.parent_id as usize] .is_recurrent_or_override() { start_timestamp_naive.into() } else { None }, } }, }); } } } ranges.update_base_offset(start_timestamp_naive, end_timestamp_naive); ranges.update_utc_min_max(min, max); groups .entry(( start_tz, end_tz, event.comp_id as u16, (end_timestamp_naive - start_timestamp_naive) as i32, )) .or_default() .push(start_timestamp_naive); } let mut events = Vec::with_capacity(groups.len()); for ((start_tz, end_tz, id, duration), mut instances) in groups { let instances = if instances.len() > 1 { instances.sort_unstable(); // Bitpack instances let mut instance_offsets = Vec::with_capacity(instances.len()); for instance in instances { debug_assert!(instance >= ranges.base_offset); instance_offsets.push((instance - ranges.base_offset) as u32); } KeySerializer::new(instance_offsets.len() * std::mem::size_of::()) .bitpack_sorted(&instance_offsets) .finalize() } else { KeySerializer::new(std::mem::size_of::()) .write_leb128((instances.first().unwrap() - ranges.base_offset) as u32) .finalize() }; events.push(ComponentTimeRange { id, start_tz, end_tz, duration, instances: instances.into_boxed_slice(), }); } if !expanded.errors.is_empty() { trc::event!( Calendar(trc::CalendarEvent::RuleExpansionError), Reason = expanded .errors .into_iter() .map(|e| e.error.to_compact_string()) .collect::>(), Details = ical.to_string(), Limit = max_expansions, ); } CalendarEventData { event: ical, time_ranges: events.into_boxed_slice(), alarms: alarms .into_values() .flatten() .collect::>() .into_boxed_slice(), base_offset: ranges.base_offset, base_time_utc: (ranges.min_time_utc - ranges.base_offset) as u32, duration: (ranges.max_time_utc - ranges.min_time_utc) as u32, } } pub fn event_range(&self) -> Option<(i64, u32)> { if self.base_offset != 0 { Some((self.base_offset + self.base_time_utc as i64, self.duration)) } else { None } } } #[derive(Default, Debug)] struct TimeRanges { max_time_utc: i64, min_time_utc: i64, base_offset: i64, } impl TimeRanges { pub fn update_base_offset(&mut self, t1: i64, t2: i64) { let offset = std::cmp::min(t1, t2); if offset < self.base_offset || self.base_offset == 0 { self.base_offset = offset; } } pub fn update_utc_min_max(&mut self, min: i64, max: i64) { if min < self.min_time_utc || self.min_time_utc == 0 { self.min_time_utc = min; } if max > self.max_time_utc { self.max_time_utc = max; } if min < self.base_offset || self.base_offset == 0 { self.base_offset = min; } } } impl ArchivedCalendarEventData { pub fn event_range(&self) -> Option<(i64, u32)> { if self.base_offset != 0 { Some(( self.base_offset.to_native() + self.base_time_utc.to_native() as i64, self.duration.to_native(), )) } else { None } } pub fn event_range_start(&self) -> i64 { self.base_offset.to_native() + self.base_time_utc.to_native() as i64 } pub fn event_range_end(&self) -> i64 { self.base_offset.to_native() + self.base_time_utc.to_native() as i64 + self.duration.to_native() as i64 } } impl CalendarEventData { pub fn event_range_start(&self) -> i64 { self.base_offset + self.base_time_utc as i64 } pub fn event_range_end(&self) -> i64 { self.base_offset + self.base_time_utc as i64 + self.duration as i64 } } impl Timezone { pub fn tz(&self) -> Option { match self { Timezone::IANA(iana) => Tz::from_id(*iana), Timezone::Custom(icalendar) => icalendar .timezones() .filter_map(|t| t.timezone().map(|x| x.1)) .next(), Timezone::Default => None, } } } impl ArchivedTimezone { pub fn tz(&self) -> Option { match self { ArchivedTimezone::IANA(iana) => Tz::from_id(iana.to_native()), ArchivedTimezone::Custom(icalendar) => icalendar .timezones() .filter_map(|t| t.timezone().map(|x| x.1)) .next(), ArchivedTimezone::Default => None, } } } ================================================ FILE: crates/groupware/src/calendar/expand.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::ArchivedCalendarEventData; use crate::calendar::CalendarEventData; use ahash::AHashSet; use calcard::common::timezone::Tz; use chrono::{DateTime, TimeZone}; use store::write::bitpack::BitpackIterator; use types::TimeRange; use utils::codec::leb128::Leb128Reader; #[derive(Debug, Clone, PartialEq, Eq)] pub struct CalendarEventExpansion { pub comp_id: u32, pub expansion_id: u32, pub start: i64, pub end: i64, } impl ArchivedCalendarEventData { pub fn expand(&self, default_tz: Tz, limit: TimeRange) -> Option> { let mut expansion = Vec::with_capacity(self.time_ranges.len()); let base_offset = self.base_offset.to_native(); let mut base_expansion_id = 0; 'outer: for range in self.time_ranges.iter() { let instances = range.instances.as_ref(); let (offset_or_count, bytes_read) = instances.read_leb128::()?; let comp_id = range.id.to_native() as u32; let duration = range.duration.to_native() as i64; let mut start_tz = Tz::from_id(range.start_tz.to_native())?; let mut end_tz = Tz::from_id(range.end_tz.to_native())?; let is_todo = self.event.components[comp_id as usize] .component_type .is_todo(); if start_tz.is_floating() && !default_tz.is_floating() { start_tz = default_tz; } if end_tz.is_floating() && !default_tz.is_floating() { end_tz = default_tz; } if instances.len() > bytes_read { // Recurring event let unpacker = BitpackIterator::from_bytes_and_offset(instances, bytes_read, offset_or_count); let mut expansion_id = base_expansion_id; base_expansion_id += offset_or_count; for start_offset in unpacker { let start_date_naive = start_offset as i64 + base_offset; let end_date_naive = start_date_naive + duration; let start = start_tz .from_local_datetime( &DateTime::from_timestamp(start_date_naive, 0)?.naive_local(), ) .single()? .timestamp(); let end = end_tz .from_local_datetime( &DateTime::from_timestamp(end_date_naive, 0)?.naive_local(), ) .single()? .timestamp(); if limit.is_in_range(is_todo, start, end) { expansion.push(CalendarEventExpansion { comp_id, expansion_id, start, end, }); } else if start > limit.end { continue 'outer; } expansion_id += 1; } } else { // Single event let start_date_naive = offset_or_count as i64 + base_offset; let end_date_naive = start_date_naive + duration; let start = start_tz .from_local_datetime( &DateTime::from_timestamp(start_date_naive, 0)?.naive_local(), ) .single()? .timestamp(); let end = end_tz .from_local_datetime( &DateTime::from_timestamp(end_date_naive, 0)?.naive_local(), ) .single()? .timestamp(); if limit.is_in_range(is_todo, start, end) { expansion.push(CalendarEventExpansion { comp_id, expansion_id: base_expansion_id, start, end, }); } base_expansion_id += 1; } } Some(expansion) } } impl CalendarEventData { pub fn expand_from_ids( &self, expansion_ids: &mut AHashSet, default_tz: Tz, ) -> Option> { let mut expansion = Vec::with_capacity(expansion_ids.len()); let base_offset = self.base_offset; let mut base_expansion_id = 0; 'outer: for range in self.time_ranges.iter() { let instances = range.instances.as_ref(); let (offset_or_count, bytes_read) = instances.read_leb128::()?; let mut start_tz = Tz::from_id(range.start_tz)?; let mut end_tz = Tz::from_id(range.end_tz)?; if start_tz.is_floating() && !default_tz.is_floating() { start_tz = default_tz; } if end_tz.is_floating() && !default_tz.is_floating() { end_tz = default_tz; } if instances.len() > bytes_read { let match_range = base_expansion_id..base_expansion_id + offset_or_count; let mut match_count = expansion_ids .iter() .filter(|id| match_range.contains(id)) .count(); let mut expansion_id = base_expansion_id; base_expansion_id += offset_or_count; if match_count > 0 { let unpacker = BitpackIterator::from_bytes_and_offset( instances, bytes_read, offset_or_count, ); for start_offset in unpacker { if expansion_ids.remove(&expansion_id) { let start_date_naive = start_offset as i64 + base_offset; let end_date_naive = start_date_naive + range.duration as i64; let start = start_tz .from_local_datetime( &DateTime::from_timestamp(start_date_naive, 0)?.naive_local(), ) .single()? .timestamp(); let end = end_tz .from_local_datetime( &DateTime::from_timestamp(end_date_naive, 0)?.naive_local(), ) .single()? .timestamp(); expansion.push(CalendarEventExpansion { comp_id: range.id as u32, expansion_id, start, end, }); match_count -= 1; if match_count == 0 { if expansion_ids.is_empty() { break 'outer; } else { continue 'outer; } } } expansion_id += 1; } } } else { if expansion_ids.remove(&base_expansion_id) { // Single event let start_date_naive = offset_or_count as i64 + base_offset; let end_date_naive = start_date_naive + range.duration as i64; let start = start_tz .from_local_datetime( &DateTime::from_timestamp(start_date_naive, 0)?.naive_local(), ) .single()? .timestamp(); let end = end_tz .from_local_datetime( &DateTime::from_timestamp(end_date_naive, 0)?.naive_local(), ) .single()? .timestamp(); expansion.push(CalendarEventExpansion { comp_id: range.id as u32, expansion_id: base_expansion_id, start, end, }); if expansion_ids.is_empty() { break 'outer; } } base_expansion_id += 1; } } if !expansion_ids.is_empty() { expansion.extend( expansion_ids .drain() .map(|expansion_id| CalendarEventExpansion { comp_id: u32::MAX, expansion_id, start: i64::MAX, end: i64::MAX, }), ); } Some(expansion) } pub fn expand_single(&self, comp_id: u32, default_tz: Tz) -> Option { let range = self.time_ranges.iter().find(|r| r.id as u32 == comp_id)?; let instances = range.instances.as_ref(); let (offset_or_count, bytes_read) = instances.read_leb128::()?; let mut start_tz = Tz::from_id(range.start_tz)?; let mut end_tz = Tz::from_id(range.end_tz)?; if start_tz.is_floating() && !default_tz.is_floating() { start_tz = default_tz; } if end_tz.is_floating() && !default_tz.is_floating() { end_tz = default_tz; } let start_offset = if instances.len() > bytes_read { // Recurring event let mut unpacker = BitpackIterator::from_bytes_and_offset(instances, bytes_read, offset_or_count); unpacker.next()? } else { // Single event offset_or_count }; let start_date_naive = start_offset as i64 + self.base_offset; let end_date_naive = start_date_naive + range.duration as i64; let start = start_tz .from_local_datetime(&DateTime::from_timestamp(start_date_naive, 0)?.naive_local()) .single()? .timestamp(); let end = end_tz .from_local_datetime(&DateTime::from_timestamp(end_date_naive, 0)?.naive_local()) .single()? .timestamp(); Some(CalendarEventExpansion { comp_id, expansion_id: u32::MAX, start, end, }) } } impl Default for CalendarEventExpansion { fn default() -> Self { Self { comp_id: u32::MAX, expansion_id: u32::MAX, start: i64::MAX, end: i64::MAX, } } } impl CalendarEventExpansion { pub fn is_valid(&self) -> bool { self.comp_id != u32::MAX && self.start != i64::MAX && self.end != i64::MAX } } ================================================ FILE: crates/groupware/src/calendar/index.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ ArchivedCalendar, ArchivedCalendarEvent, ArchivedCalendarPreferences, ArchivedDefaultAlert, ArchivedTimezone, Calendar, CalendarEvent, CalendarPreferences, DefaultAlert, Timezone, }; use crate::calendar::{ ArchivedCalendarEventNotification, ArchivedChangedBy, ArchivedEventPreferences, CalendarEventNotification, ChangedBy, EventPreferences, }; use ahash::AHashSet; use calcard::icalendar::{ ArchivedICalendarParameterValue, ArchivedICalendarProperty, ArchivedICalendarValue, ICalendarParameterValue, ICalendarProperty, ICalendarValue, }; use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; use nlp::language::{ Language, detect::{LanguageDetector, MIN_LANGUAGE_SCORE}, }; use store::{ U32_LEN, search::{CalendarSearchField, IndexDocument, SearchField}, write::{IndexPropertyClass, SearchIndex, ValueClass}, xxhash_rust::xxh3, }; use types::{ acl::AclGrant, collection::SyncCollection, field::{CalendarEventField, CalendarNotificationField}, }; impl IndexableObject for Calendar { fn index_values(&self) -> impl Iterator> { [ IndexValue::Acl { value: (&self.acls).into(), }, IndexValue::Quota { used: self.size() as u32, }, IndexValue::LogContainer { sync_collection: SyncCollection::Calendar, }, ] .into_iter() } } impl IndexableObject for &ArchivedCalendar { fn index_values(&self) -> impl Iterator> { [ IndexValue::Acl { value: self .acls .iter() .map(AclGrant::from) .collect::>() .into(), }, IndexValue::Quota { used: self.size() as u32, }, IndexValue::LogContainer { sync_collection: SyncCollection::Calendar, }, ] .into_iter() } } impl IndexableAndSerializableObject for Calendar { fn is_versioned() -> bool { true } } impl IndexableObject for CalendarEvent { fn index_values(&self) -> impl Iterator> { [ IndexValue::SearchIndex { index: SearchIndex::Calendar, hash: self .hashes() .chain([self.data.event_range_start() as u64]) .fold(0, |acc, hash| acc ^ hash), }, IndexValue::Index { field: CalendarEventField::Uid.into(), value: self.data.event.uids().next().into(), }, IndexValue::Quota { used: self.size() as u32, }, IndexValue::LogItem { sync_collection: SyncCollection::Calendar, prefix: None, }, ] .into_iter() } } impl IndexableObject for &ArchivedCalendarEvent { fn index_values(&self) -> impl Iterator> { [ IndexValue::SearchIndex { index: SearchIndex::Calendar, hash: self .hashes() .chain([self.data.event_range_start() as u64]) .fold(0, |acc, hash| acc ^ hash), }, IndexValue::Index { field: CalendarEventField::Uid.into(), value: self.data.event.uids().next().into(), }, IndexValue::Quota { used: self.size() as u32, }, IndexValue::LogItem { sync_collection: SyncCollection::Calendar, prefix: None, }, ] .into_iter() } } impl IndexableAndSerializableObject for CalendarEvent { fn is_versioned() -> bool { true } } impl IndexableObject for CalendarEventNotification { fn index_values(&self) -> impl Iterator> { [ IndexValue::Quota { used: self.size() as u32, }, IndexValue::Property { field: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: CalendarNotificationField::CreatedToId.into(), value: self.created as u64, }), value: self.event_id.unwrap_or(u32::MAX).into(), }, IndexValue::LogItem { sync_collection: SyncCollection::CalendarEventNotification, prefix: None, }, ] .into_iter() } } impl IndexableObject for &ArchivedCalendarEventNotification { fn index_values(&self) -> impl Iterator> { [ IndexValue::Quota { used: self.size() as u32, }, IndexValue::Property { field: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: CalendarNotificationField::CreatedToId.into(), value: self.created.to_native() as u64, }), value: self .event_id .as_ref() .map(|v| v.to_native()) .unwrap_or(u32::MAX) .into(), }, IndexValue::LogItem { sync_collection: SyncCollection::CalendarEventNotification, prefix: None, }, ] .into_iter() } } impl IndexableAndSerializableObject for CalendarEventNotification { fn is_versioned() -> bool { false } } impl Calendar { pub fn size(&self) -> usize { self.dead_properties.size() + self.preferences.iter().map(|p| p.size()).sum::() + self.name.len() + std::mem::size_of::() } } impl ArchivedCalendar { pub fn size(&self) -> usize { self.dead_properties.size() + self.preferences.iter().map(|p| p.size()).sum::() + self.name.len() + std::mem::size_of::() } } impl CalendarEvent { pub fn size(&self) -> usize { self.dead_properties.size() + self.display_name.as_ref().map_or(0, |n| n.len()) + self.names.iter().map(|n| n.name.len()).sum::() + self.preferences.iter().map(|p| p.size()).sum::() + self.size as usize + std::mem::size_of::() } } impl ArchivedCalendarEvent { pub fn size(&self) -> usize { self.dead_properties.size() + self.display_name.as_ref().map_or(0, |n| n.len()) + self.names.iter().map(|n| n.name.len()).sum::() + self.preferences.iter().map(|p| p.size()).sum::() + self.size.to_native() as usize + std::mem::size_of::() } } impl CalendarEventNotification { pub fn size(&self) -> usize { (match &self.changed_by { ChangedBy::PrincipalId(_) => U32_LEN, ChangedBy::CalendarAddress(v) => v.len(), }) + std::mem::size_of::() + self.size as usize } } impl ArchivedCalendarEventNotification { pub fn size(&self) -> usize { (match &self.changed_by { ArchivedChangedBy::PrincipalId(_) => U32_LEN, ArchivedChangedBy::CalendarAddress(v) => v.len(), }) + std::mem::size_of::() + self.size.to_native() as usize } } impl CalendarPreferences { pub fn size(&self) -> usize { self.name.len() + self.default_alerts.iter().map(|a| a.size()).sum::() + self.description.as_ref().map_or(0, |n| n.len()) + self.color.as_ref().map_or(0, |n| n.len()) + self.time_zone.size() + std::mem::size_of::() } } impl ArchivedCalendarPreferences { pub fn size(&self) -> usize { self.name.len() + self.default_alerts.iter().map(|a| a.size()).sum::() + self.description.as_ref().map_or(0, |n| n.len()) + self.color.as_ref().map_or(0, |n| n.len()) + self.time_zone.size() + std::mem::size_of::() } } impl EventPreferences { pub fn size(&self) -> usize { self.alerts.iter().map(|a| a.size()).sum::() + self.properties.iter().map(|p| p.size()).sum::() + std::mem::size_of::() } } impl ArchivedEventPreferences { pub fn size(&self) -> usize { self.alerts.iter().map(|a| a.size()).sum::() + self.properties.iter().map(|p| p.size()).sum::() + std::mem::size_of::() } } impl Timezone { pub fn size(&self) -> usize { match self { Timezone::IANA(_) => 2, Timezone::Custom(c) => c.size(), Timezone::Default => 0, } } } impl ArchivedTimezone { pub fn size(&self) -> usize { match self { ArchivedTimezone::IANA(_) => 2, ArchivedTimezone::Custom(c) => c.size(), ArchivedTimezone::Default => 0, } } } impl DefaultAlert { pub fn size(&self) -> usize { std::mem::size_of::() + self.id.len() } } impl ArchivedDefaultAlert { pub fn size(&self) -> usize { std::mem::size_of::() + self.id.len() } } impl CalendarEvent { pub fn hashes(&self) -> impl Iterator { self.data .event .components .iter() .filter(|e| e.component_type.is_scheduling_object()) .flat_map(|e| { e.entries.iter().filter(|e| { matches!( e.name, ICalendarProperty::Summary | ICalendarProperty::Location | ICalendarProperty::Description | ICalendarProperty::Categories | ICalendarProperty::Comment | ICalendarProperty::Attendee | ICalendarProperty::Organizer | ICalendarProperty::Uid ) }) }) .flat_map(|e| { e.values .iter() .filter_map(|v| match v { ICalendarValue::Text(v) => Some(v.as_str()), ICalendarValue::Uri(uri) => uri.as_str(), _ => None, }) .chain(e.params.iter().filter_map(|p| match &p.value { ICalendarParameterValue::Text(v) => Some(v.as_str()), ICalendarParameterValue::Uri(uri) => uri.as_str(), _ => None, })) }) .map(|v| xxh3::xxh3_64(v.as_bytes())) } } impl ArchivedCalendarEvent { pub fn hashes(&self) -> impl Iterator { self.data .event .components .iter() .filter(|e| e.component_type.is_scheduling_object()) .flat_map(|e| { e.entries.iter().filter(|e| { matches!( e.name, ArchivedICalendarProperty::Summary | ArchivedICalendarProperty::Location | ArchivedICalendarProperty::Description | ArchivedICalendarProperty::Categories | ArchivedICalendarProperty::Comment | ArchivedICalendarProperty::Attendee | ArchivedICalendarProperty::Organizer | ArchivedICalendarProperty::Uid ) }) }) .flat_map(|e| { e.values .iter() .filter_map(|v| match v { ArchivedICalendarValue::Text(v) => Some(v.as_str()), ArchivedICalendarValue::Uri(uri) => uri.as_str(), _ => None, }) .chain(e.params.iter().filter_map(|p| match &p.value { ArchivedICalendarParameterValue::Text(v) => Some(v.as_str()), ArchivedICalendarParameterValue::Uri(uri) => uri.as_str(), _ => None, })) }) .map(|v| xxh3::xxh3_64(v.as_bytes())) } } impl ArchivedCalendarEvent { pub fn index_document( &self, account_id: u32, document_id: u32, index_fields: &AHashSet, default_language: Language, ) -> IndexDocument { let mut document = IndexDocument::new(SearchIndex::Calendar) .with_account_id(account_id) .with_document_id(document_id); if index_fields.is_empty() || index_fields.contains(&SearchField::Calendar(CalendarSearchField::Start)) { document.index_integer(CalendarSearchField::Start, self.data.event_range_start()); } let mut detector = LanguageDetector::new(); for component in self .data .event .components .iter() .filter(|e| e.component_type.is_scheduling_object()) { for entry in component.entries.iter() { let (is_lang, is_keyword, field) = match entry.name { ArchivedICalendarProperty::Summary => (true, false, CalendarSearchField::Title), ArchivedICalendarProperty::Description => { (true, false, CalendarSearchField::Description) } ArchivedICalendarProperty::Location => { (false, false, CalendarSearchField::Location) } ArchivedICalendarProperty::Organizer => { (false, false, CalendarSearchField::Owner) } ArchivedICalendarProperty::Attendee => { (false, false, CalendarSearchField::Attendee) } ArchivedICalendarProperty::Uid => (false, true, CalendarSearchField::Uid), _ => continue, }; let field = SearchField::Calendar(field); if index_fields.is_empty() || index_fields.contains(&field) { for value in entry .values .iter() .filter_map(|v| match v { ArchivedICalendarValue::Text(v) => Some(v.as_str()), ArchivedICalendarValue::Uri(uri) => uri.as_str(), _ => None, }) .chain(entry.params.iter().filter_map(|p| match &p.value { ArchivedICalendarParameterValue::Text(v) => Some(v.as_str()), ArchivedICalendarParameterValue::Uri(uri) => uri.as_str(), _ => None, })) { let value = value.strip_prefix("mailto:").unwrap_or(value).trim(); let lang = if is_lang { detector.detect(value, MIN_LANGUAGE_SCORE); Language::Unknown } else { Language::None }; if !is_keyword { document.index_text(field.clone(), value, lang); } else { document.index_keyword(field.clone(), value); } } } } } document.set_unknown_language( detector .most_frequent_language() .unwrap_or(default_language), ); document } } ================================================ FILE: crates/groupware/src/calendar/itip.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ RFC_3986, cache::GroupwareCache, calendar::{ CalendarEvent, CalendarEventData, CalendarEventNotification, ChangedBy, EVENT_NOTIFICATION_IS_CHANGE, }, scheduling::{ ItipError, ItipMessage, inbound::{ MergeResult, itip_import_message, itip_merge_changes, itip_method, itip_process_message, }, snapshot::itip_snapshot, }, }; use calcard::{ common::{IanaString, timezone::Tz}, icalendar::{ ICalendar, ICalendarComponentType, ICalendarEntry, ICalendarMethod, ICalendarParameter, ICalendarParameterName, ICalendarParameterValue, ICalendarParticipationStatus, ICalendarProperty, ICalendarValue, }, }; use common::{ DavName, Server, auth::{AccessToken, ResourceToken, oauth::GrantType}, config::groupware::CalendarTemplateVariable, i18n, }; use store::{ ValueKey, rand, write::{AlignedBytes, Archive, BatchBuilder, now}, }; use trc::AddContext; use types::{ collection::Collection, field::{CalendarEventField, ContactField}, }; use utils::{template::Variables, url_params::UrlParams}; pub enum ItipIngestError { Message(ItipError), Internal(trc::Error), } #[derive(Default)] pub struct ItipRsvpUrl(String); pub trait ItipIngest: Sync + Send { fn itip_ingest( &self, access_token: &AccessToken, resource_token: &ResourceToken, sender: &str, recipient: &str, itip_message: &str, ) -> impl Future>, ItipIngestError>> + Send; fn http_rsvp_url( &self, account_id: u32, document_id: u32, attendee: &str, ) -> impl Future> + Send; fn http_rsvp_handle( &self, query: &str, language: &str, ) -> impl Future> + Send; } impl ItipIngest for Server { async fn itip_ingest( &self, access_token: &AccessToken, resource_token: &ResourceToken, sender: &str, recipient: &str, itip_message: &str, ) -> Result>, ItipIngestError> { // Parse and validate the iTIP message let mut itip = ICalendar::parse(itip_message) .map_err(|_| ItipIngestError::Message(ItipError::ICalendarParseError)) .and_then(|ical| { if ical.components.len() > 1 && ical.components[0].component_type == ICalendarComponentType::VCalendar { Ok(ical) } else { Err(ItipIngestError::Message(ItipError::ICalendarParseError)) } })?; // Microsoft Exchange does not include the organizer in REPLY, assume it is the recipient. // This will be validated against the stored event anyway. if itip.components[0] .property(&ICalendarProperty::Method) .and_then(|v| v.values.first()) .is_some_and(|v| { matches!( v, ICalendarValue::Method(ICalendarMethod::Reply | ICalendarMethod::Request) ) }) { for comp in &mut itip.components { if comp.component_type.is_scheduling_object() { let mut has_organizer = false; let mut has_attendee = false; for entry in &comp.entries { match entry.name { ICalendarProperty::Organizer => has_organizer = true, ICalendarProperty::Attendee => has_attendee = true, _ => {} } } if has_attendee && !has_organizer { comp.entries.push(ICalendarEntry { name: ICalendarProperty::Organizer, params: vec![], values: vec![ICalendarValue::Text(format!("mailto:{recipient}"))], }); } } } } let itip_snapshots = itip_snapshot(&itip, access_token.emails.as_slice(), false)?; if !itip_snapshots.sender_is_organizer_or_attendee(sender) { return Err(ItipIngestError::Message( ItipError::SenderIsNotOrganizerNorAttendee, )); } // Obtain changedBy let changed_by = if let Some(id) = self.email_to_id(self.directory(), sender, 0).await? { ChangedBy::PrincipalId(id) } else { ChangedBy::CalendarAddress(sender.into()) }; // Find event by UID let account_id = access_token.primary_id; let document_id = self .document_ids_matching( account_id, Collection::CalendarEvent, CalendarEventField::Uid, itip_snapshots.uid.as_bytes(), ) .await .caused_by(trc::location!())? .iter() .next(); if let Some(document_id) = document_id { if let Some(archive) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, )) .await .caused_by(trc::location!())? { let event_ = archive .to_unarchived::() .caused_by(trc::location!())?; let mut event = event_ .deserialize::() .caused_by(trc::location!())?; // Process the iTIP message let snapshots = itip_snapshot(&event.data.event, access_token.emails.as_slice(), false)?; let is_organizer_update = !itip_snapshots.organizer.email.is_local; match itip_process_message( &event.data.event, snapshots, &itip, itip_snapshots, sender.to_string(), )? { MergeResult::Actions(changes) => { // Merge changes itip_merge_changes(&mut event.data.event, changes); // Calculate the new ical size event.size = event.data.event.to_string().len() as u32; if event.size > self.core.groupware.max_ical_size as u32 { return Err(ItipIngestError::Message(ItipError::EventTooLarge)); } // Validate quota let extra_bytes = (event.size as u64) .saturating_sub(event_.inner.size.to_native() as u64); if extra_bytes > 0 && self .has_available_quota(resource_token, extra_bytes) .await .is_err() { return Err(ItipIngestError::Message(ItipError::QuotaExceeded)); } // Build event let now = now() as i64; let prev_email_alarm = event_.inner.data.next_alarm(now, Tz::Floating); let mut next_email_alarm = None; event.data = CalendarEventData::new( event.data.event, Tz::Floating, self.core.groupware.max_ical_instances, &mut next_email_alarm, ); if is_organizer_update { if let Some(schedule_tag) = &mut event.schedule_tag { *schedule_tag += 1; } else { event.schedule_tag = Some(1); } } // Build event for schedule inbox let itip_document_id = self .store() .assign_document_ids( account_id, Collection::CalendarEventNotification, 1, ) .await .caused_by(trc::location!())?; let itip_message = CalendarEventNotification { event: itip, changed_by, event_id: Some(document_id), flags: EVENT_NOTIFICATION_IS_CHANGE, size: itip_message.len() as u32, ..Default::default() }; // Prepare write batch let mut batch = BatchBuilder::new(); event .update(access_token, event_, account_id, document_id, &mut batch) .caused_by(trc::location!())?; if prev_email_alarm != next_email_alarm { if let Some(prev_alarm) = prev_email_alarm { prev_alarm.delete_task(&mut batch); } if let Some(next_alarm) = next_email_alarm { next_alarm.write_task(&mut batch); } } itip_message .insert(access_token, account_id, itip_document_id, &mut batch) .caused_by(trc::location!())?; self.commit_batch(batch).await.caused_by(trc::location!())?; Ok(None) } MergeResult::Message(itip_message) => Ok(Some(itip_message)), MergeResult::None => Ok(None), } } else { Err(ItipIngestError::Message(ItipError::EventNotFound)) } } else { // Verify that auto-adding invitations is allowed if !self.core.groupware.itip_auto_add && !matches!(changed_by, ChangedBy::PrincipalId(_)) && !self .document_exists( account_id, Collection::ContactCard, ContactField::Email, sender.as_bytes(), ) .await .caused_by(trc::location!())? { return Err(ItipIngestError::Message(ItipError::AutoAddDisabled)); } else if itip_method(&itip)? != &ICalendarMethod::Request { return Err(ItipIngestError::Message(ItipError::EventNotFound)); } // Import the iTIP message let mut ical = itip.clone(); itip_import_message(&mut ical)?; // Validate quota if self .has_available_quota(resource_token, itip_message.len() as u64) .await .is_err() { return Err(ItipIngestError::Message(ItipError::QuotaExceeded)); } // Obtain parent calendar let Some(parent_id) = self .get_or_create_default_calendar(access_token, account_id) .await .caused_by(trc::location!())? else { return Err(ItipIngestError::Message(ItipError::NoDefaultCalendar)); }; // Build event let mut next_email_alarm = None; let now = now() as i64; let event = CalendarEvent { names: vec![DavName { name: format!("{}_{}.ics", now, rand::random::()), parent_id, }], data: CalendarEventData::new( ical, Tz::Floating, self.core.groupware.max_ical_instances, &mut next_email_alarm, ), size: itip_message.len() as u32, schedule_tag: Some(1), ..Default::default() }; // Obtain document ids let document_id = self .store() .assign_document_ids(account_id, Collection::CalendarEvent, 1) .await .caused_by(trc::location!())?; let itip_document_id = self .store() .assign_document_ids(account_id, Collection::CalendarEventNotification, 1) .await .caused_by(trc::location!())?; let itip_message = CalendarEventNotification { event: itip, event_id: Some(document_id), changed_by, size: itip_message.len() as u32, ..Default::default() }; // Prepare write batch let mut batch = BatchBuilder::new(); event .insert( access_token, account_id, document_id, next_email_alarm, &mut batch, ) .caused_by(trc::location!())?; itip_message .insert(access_token, account_id, itip_document_id, &mut batch) .caused_by(trc::location!())?; self.commit_batch(batch).await.caused_by(trc::location!())?; Ok(None) } } async fn http_rsvp_url( &self, account_id: u32, document_id: u32, attendee: &str, ) -> Option { if let Some(base_url) = &self.core.groupware.itip_http_rsvp_url { match self .encode_access_token( GrantType::Rsvp, account_id, &format!("{attendee};{document_id}"), self.core.groupware.itip_http_rsvp_expiration, ) .await { Ok(access_token) => Some(ItipRsvpUrl(format!( "{base_url}?i={}", percent_encoding::percent_encode(access_token.as_bytes(), RFC_3986) ))), Err(err) => { trc::error!(err.caused_by(trc::location!())); None } } } else { None } } async fn http_rsvp_handle(&self, query: &str, language: &str) -> trc::Result { let response = if let Some(rsvp) = decode_rsvp_response(self, query).await { if let Some(archive) = self .store() .get_value::>(ValueKey::archive( rsvp.account_id, Collection::CalendarEvent, rsvp.document_id, )) .await .caused_by(trc::location!())? { let event = archive .to_unarchived::() .caused_by(trc::location!())?; let mut new_event = event .deserialize::() .caused_by(trc::location!())?; let mut did_change = false; let mut summary = None; let mut description = None; let mut found_participant = false; for component in &mut new_event.data.event.components { if component.component_type.is_scheduling_object() { 'outer: for entry in &mut component.entries { if entry.name == ICalendarProperty::Attendee && entry .calendar_address() .is_some_and(|v| v.eq_ignore_ascii_case(&rsvp.attendee)) { let mut add_partstat = true; for param in &mut entry.params { if let ( ICalendarParameterName::Partstat, ICalendarParameterValue::Partstat(partstat), ) = (¶m.name, &mut param.value) { if partstat != &rsvp.partstat { *partstat = rsvp.partstat.clone(); add_partstat = false; } else { continue 'outer; } } } if add_partstat { entry .params .push(ICalendarParameter::partstat(rsvp.partstat.clone())); } found_participant = true; did_change = true; } else if summary.is_none() && entry.name == ICalendarProperty::Summary { summary = entry .values .first() .and_then(|v| v.as_text()) .map(|s| s.to_string()); } else if description.is_none() && entry.name == ICalendarProperty::Description { description = entry .values .first() .and_then(|v| v.as_text()) .map(|s| s.to_string()); } } } } if did_change { // Prepare write batch let access_token = self .get_access_token(rsvp.account_id) .await .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); new_event .update( &access_token, event, rsvp.account_id, rsvp.document_id, &mut batch, ) .caused_by(trc::location!())?; self.commit_batch(batch).await.caused_by(trc::location!())?; } if found_participant { Response::Success { summary, description, } } else { Response::NoLongerParticipant } } else { Response::EventNotFound } } else { Response::ParseError }; Ok(render_response(self, response, language)) } } struct RsvpResponse { account_id: u32, document_id: u32, attendee: String, partstat: ICalendarParticipationStatus, } async fn decode_rsvp_response(server: &Server, query: &str) -> Option { let params = UrlParams::new(query.into()); let token = params.get("i")?; let method = params.get("m").and_then(|m| { hashify::tiny_map_ignore_case!(m.as_bytes(), "ACCEPTED" => ICalendarParticipationStatus::Accepted, "DECLINED" => ICalendarParticipationStatus::Declined, "TENTATIVE" => ICalendarParticipationStatus::Tentative, "COMPLETED" => ICalendarParticipationStatus::Completed, "IN-PROCESS" => ICalendarParticipationStatus::InProcess, ) })?; let token = server .validate_access_token(GrantType::Rsvp.into(), token) .await .ok()?; let (attendee, document_id) = token .client_id .rsplit_once(';') .and_then(|(attendee, doc_id)| { doc_id .parse::() .ok() .map(|doc_id| (attendee.to_string(), doc_id)) })?; RsvpResponse { account_id: token.account_id, document_id, attendee, partstat: method, } .into() } enum Response { Success { summary: Option, description: Option, }, EventNotFound, ParseError, NoLongerParticipant, } fn render_response(server: &Server, response: Response, language: &str) -> String { // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] let template = server .core .enterprise .as_ref() .and_then(|e| e.template_scheduling_web.as_ref()) .unwrap_or(&server.core.groupware.itip_template); // SPDX-SnippetEnd #[cfg(not(feature = "enterprise"))] let template = &server.core.groupware.itip_template; let locale = i18n::locale_or_default(language); let mut variables = Variables::new(); match response { Response::Success { summary, description, } => { variables.insert_single( CalendarTemplateVariable::PageTitle, locale.calendar_rsvp_recorded.to_string(), ); variables.insert_single( CalendarTemplateVariable::Header, locale.calendar_rsvp_recorded.to_string(), ); variables.insert_block( CalendarTemplateVariable::EventDetails, [ summary.map(|summary| { [ ( CalendarTemplateVariable::Key, locale.calendar_summary.to_string(), ), (CalendarTemplateVariable::Value, summary), ] }), description.map(|description| { [ ( CalendarTemplateVariable::Key, locale.calendar_description.to_string(), ), (CalendarTemplateVariable::Value, description), ] }), ] .into_iter() .flatten(), ); variables.insert_single(CalendarTemplateVariable::Color, "info".to_string()); } Response::EventNotFound => { variables.insert_single( CalendarTemplateVariable::PageTitle, locale.calendar_rsvp_failed.to_string(), ); variables.insert_single( CalendarTemplateVariable::Header, locale.calendar_event_not_found.to_string(), ); variables.insert_single(CalendarTemplateVariable::Color, "danger".to_string()); } Response::ParseError => { variables.insert_single( CalendarTemplateVariable::PageTitle, locale.calendar_rsvp_failed.to_string(), ); variables.insert_single( CalendarTemplateVariable::Header, locale.calendar_invalid_rsvp.to_string(), ); variables.insert_single(CalendarTemplateVariable::Color, "danger".to_string()); } Response::NoLongerParticipant => { variables.insert_single( CalendarTemplateVariable::PageTitle, locale.calendar_rsvp_failed.to_string(), ); variables.insert_single( CalendarTemplateVariable::Header, locale.calendar_not_participant.to_string(), ); variables.insert_single(CalendarTemplateVariable::Color, "warning".to_string()); } } variables.insert_single(CalendarTemplateVariable::LogoCid, "/logo.svg".to_string()); template.eval(&variables) } impl ItipRsvpUrl { pub fn url(&self, partstat: &ICalendarParticipationStatus) -> String { format!("{}&m={}", self.0, partstat.as_str()) } } impl From for ItipIngestError { fn from(err: ItipError) -> Self { ItipIngestError::Message(err) } } impl From for ItipIngestError { fn from(err: trc::Error) -> Self { ItipIngestError::Internal(err) } } ================================================ FILE: crates/groupware/src/calendar/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod alarm; pub mod dates; pub mod expand; pub mod index; pub mod itip; pub mod storage; use calcard::icalendar::{ ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarDuration, ICalendarEntry, }; use common::{DavName, auth::AccessToken}; use types::{acl::AclGrant, dead_property::DeadProperty}; use utils::map::bitmap::BitmapItem; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct Calendar { pub name: String, pub preferences: Vec, pub acls: Vec, pub supported_components: u64, pub dead_properties: DeadProperty, pub created: i64, pub modified: i64, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SupportedComponent { VCalendar, // [RFC5545, Section 3.4] VEvent, // [RFC5545, Section 3.6.1] VTodo, // [RFC5545, Section 3.6.2] VJournal, // [RFC5545, Section 3.6.3] VFreebusy, // [RFC5545, Section 3.6.4] VTimezone, // [RFC5545, Section 3.6.5] VAlarm, // [RFC5545, Section 3.6.6] Standard, // [RFC5545, Section 3.6.5] Daylight, // [RFC5545, Section 3.6.5] VAvailability, // [RFC7953, Section 3.1] Available, // [RFC7953, Section 3.1] Participant, // [RFC9073, Section 7.1] VLocation, // [RFC9073, Section 7.2] [RFC Errata 7381] VResource, // [RFC9073, Section 7.3] VStatus, // draft-ietf-calext-ical-tasks-14 Other, } pub const CALENDAR_SUBSCRIBED: u16 = 1; pub const CALENDAR_INVISIBLE: u16 = 1 << 1; pub const CALENDAR_AVAILABILITY_NONE: u16 = 1 << 2; pub const CALENDAR_AVAILABILITY_ATTENDING: u16 = 1 << 3; pub const CALENDAR_AVAILABILITY_ALL: u16 = 1 << 4; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct CalendarPreferences { pub account_id: u32, pub name: String, pub description: Option, pub sort_order: u32, pub color: Option, pub flags: u16, pub time_zone: Timezone, pub default_alerts: Vec, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct DefaultAlert { pub id: String, pub offset: ICalendarDuration, pub flags: u16, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct ParticipantIdentities { pub identities: Vec, pub default_name: String, pub default: u32, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct ParticipantIdentity { pub id: u32, pub name: Option, pub calendar_address: String, } pub const ALERT_WITH_TIME: u16 = 1; pub const ALERT_EMAIL: u16 = 1 << 1; pub const ALERT_RELATIVE_TO_END: u16 = 1 << 2; pub const SCHEDULE_INBOX_ID: u32 = u32::MAX - 1; pub const SCHEDULE_OUTBOX_ID: u32 = u32::MAX - 2; pub const EVENT_INVITE_SELF: u16 = 1; pub const EVENT_INVITE_OTHERS: u16 = 1 << 1; pub const EVENT_HIDE_ATTENDEES: u16 = 1 << 2; pub const EVENT_DRAFT: u16 = 1 << 3; pub const EVENT_NOTIFICATION_IS_DRAFT: u16 = 1; pub const EVENT_NOTIFICATION_IS_CHANGE: u16 = 1 << 1; pub const PREF_USE_DEFAULT_ALERTS: u16 = 1; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct CalendarEvent { pub names: Vec, pub display_name: Option, pub data: CalendarEventData, pub preferences: Vec, pub flags: u16, pub dead_properties: DeadProperty, pub size: u32, pub created: i64, pub modified: i64, pub schedule_tag: Option, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct CalendarEventNotification { pub event: ICalendar, pub event_id: Option, pub changed_by: ChangedBy, pub flags: u16, pub size: u32, pub created: i64, pub modified: i64, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] pub enum ChangedBy { PrincipalId(u32), CalendarAddress(String), } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct CalendarEventData { pub event: ICalendar, pub time_ranges: Box<[ComponentTimeRange]>, pub alarms: Box<[Alarm]>, pub base_offset: i64, pub base_time_utc: u32, pub duration: u32, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] #[rkyv(compare(PartialEq), derive(Debug))] pub struct Alarm { pub id: u16, pub parent_id: u16, pub delta: AlarmDelta, pub is_email_alert: bool, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] #[rkyv(compare(PartialEq), derive(Debug))] pub enum AlarmDelta { Start(i64), End(i64), FixedUtc(i64), FixedFloating(i64), } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct ComponentTimeRange { pub id: u16, pub start_tz: u16, pub end_tz: u16, pub duration: i32, pub instances: Box<[u8]>, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct EventPreferences { pub account_id: u32, pub flags: u16, pub properties: Vec, pub alerts: Vec, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub enum Timezone { IANA(u16), Custom(ICalendar), #[default] Default, } impl Calendar { pub fn preferences(&self, access_token: &AccessToken) -> &CalendarPreferences { if self.preferences.len() == 1 { &self.preferences[0] } else { let account_id = access_token.primary_id(); self.preferences .iter() .find(|p| p.account_id == account_id) .or_else(|| self.preferences.first()) .unwrap() } } pub fn preferences_mut(&mut self, access_token: &AccessToken) -> &mut CalendarPreferences { let account_id = access_token.primary_id(); let idx = if let Some(idx) = self .preferences .iter() .position(|p| p.account_id == account_id) { idx } else { let mut preferences = self.preferences[0].clone(); preferences.account_id = account_id; self.preferences.push(preferences); self.preferences.len() - 1 }; &mut self.preferences[idx] } } impl ArchivedCalendar { pub fn default_alerts( &self, access_token: &AccessToken, with_time: bool, ) -> impl Iterator { self.preferences(access_token) .default_alerts .iter() .filter(move |a| (a.flags & ALERT_WITH_TIME != 0) == with_time) } pub fn preferences(&self, access_token: &AccessToken) -> &ArchivedCalendarPreferences { if self.preferences.len() == 1 { &self.preferences[0] } else { let account_id = access_token.primary_id(); self.preferences .iter() .find(|p| p.account_id == account_id) .or_else(|| self.preferences.first()) .unwrap() } } } impl CalendarEvent { pub fn preferences(&self, access_token: &AccessToken) -> Option<&EventPreferences> { self.preferences .iter() .find(|p| p.account_id == access_token.primary_id()) } pub fn preferences_mut(&mut self, access_token: &AccessToken) -> &mut EventPreferences { let account_id = access_token.primary_id(); let idx = if let Some(idx) = self .preferences .iter() .position(|p| p.account_id == account_id) { idx } else { self.preferences.push(EventPreferences { account_id, flags: PREF_USE_DEFAULT_ALERTS, properties: Vec::new(), alerts: Vec::new(), }); self.preferences.len() - 1 }; &mut self.preferences[idx] } pub fn added_calendar_ids( &self, prev_data: &ArchivedCalendarEvent, ) -> impl Iterator { self.names .iter() .filter(|m| prev_data.names.iter().all(|pm| pm.parent_id != m.parent_id)) .map(|m| m.parent_id) } pub fn removed_calendar_ids( &self, prev_data: &ArchivedCalendarEvent, ) -> impl Iterator { prev_data .names .iter() .filter(|m| self.names.iter().all(|pm| pm.parent_id != m.parent_id)) .map(|m| m.parent_id.to_native()) } pub fn unchanged_calendar_ids( &self, prev_data: &ArchivedCalendarEvent, ) -> impl Iterator { self.names .iter() .filter(|m| prev_data.names.iter().any(|pm| pm.parent_id == m.parent_id)) .map(|m| m.parent_id) } } impl ArchivedCalendarEvent { pub fn preferences(&self, access_token: &AccessToken) -> Option<&ArchivedEventPreferences> { self.preferences .iter() .find(|p| p.account_id == access_token.primary_id()) } } impl Default for ChangedBy { fn default() -> Self { ChangedBy::CalendarAddress("".into()) } } impl From for SupportedComponent { fn from(value: u64) -> Self { match value { 0 => SupportedComponent::VCalendar, 1 => SupportedComponent::VEvent, 2 => SupportedComponent::VTodo, 3 => SupportedComponent::VJournal, 4 => SupportedComponent::VFreebusy, 5 => SupportedComponent::VTimezone, 6 => SupportedComponent::VAlarm, 7 => SupportedComponent::Standard, 8 => SupportedComponent::Daylight, 9 => SupportedComponent::VAvailability, 10 => SupportedComponent::Available, 11 => SupportedComponent::Participant, 12 => SupportedComponent::VLocation, 13 => SupportedComponent::VResource, 14 => SupportedComponent::VStatus, _ => SupportedComponent::Other, } } } impl From for u64 { fn from(value: SupportedComponent) -> Self { match value { SupportedComponent::VCalendar => 0, SupportedComponent::VEvent => 1, SupportedComponent::VTodo => 2, SupportedComponent::VJournal => 3, SupportedComponent::VFreebusy => 4, SupportedComponent::VTimezone => 5, SupportedComponent::VAlarm => 6, SupportedComponent::Standard => 7, SupportedComponent::Daylight => 8, SupportedComponent::VAvailability => 9, SupportedComponent::Available => 10, SupportedComponent::Participant => 11, SupportedComponent::VLocation => 12, SupportedComponent::VResource => 13, SupportedComponent::VStatus => 14, SupportedComponent::Other => 15, } } } impl BitmapItem for SupportedComponent { fn max() -> u64 { u64::from(SupportedComponent::Other) } fn is_valid(&self) -> bool { !matches!(self, SupportedComponent::Other) } } impl From for SupportedComponent { fn from(value: ICalendarComponentType) -> Self { match value { ICalendarComponentType::VCalendar => SupportedComponent::VCalendar, ICalendarComponentType::VEvent => SupportedComponent::VEvent, ICalendarComponentType::VTodo => SupportedComponent::VTodo, ICalendarComponentType::VJournal => SupportedComponent::VJournal, ICalendarComponentType::VFreebusy => SupportedComponent::VFreebusy, ICalendarComponentType::VTimezone => SupportedComponent::VTimezone, ICalendarComponentType::VAlarm => SupportedComponent::VAlarm, ICalendarComponentType::Standard => SupportedComponent::Standard, ICalendarComponentType::Daylight => SupportedComponent::Daylight, ICalendarComponentType::VAvailability => SupportedComponent::VAvailability, ICalendarComponentType::Available => SupportedComponent::Available, ICalendarComponentType::Participant => SupportedComponent::Participant, ICalendarComponentType::VLocation => SupportedComponent::VLocation, ICalendarComponentType::VResource => SupportedComponent::VResource, ICalendarComponentType::VStatus => SupportedComponent::VStatus, _ => SupportedComponent::Other, } } } impl From for ICalendarComponentType { fn from(value: SupportedComponent) -> Self { match value { SupportedComponent::VCalendar => ICalendarComponentType::VCalendar, SupportedComponent::VEvent => ICalendarComponentType::VEvent, SupportedComponent::VTodo => ICalendarComponentType::VTodo, SupportedComponent::VJournal => ICalendarComponentType::VJournal, SupportedComponent::VFreebusy => ICalendarComponentType::VFreebusy, SupportedComponent::VTimezone => ICalendarComponentType::VTimezone, SupportedComponent::VAlarm => ICalendarComponentType::VAlarm, SupportedComponent::Standard => ICalendarComponentType::Standard, SupportedComponent::Daylight => ICalendarComponentType::Daylight, SupportedComponent::VAvailability => ICalendarComponentType::VAvailability, SupportedComponent::Available => ICalendarComponentType::Available, SupportedComponent::Participant => ICalendarComponentType::Participant, SupportedComponent::VLocation => ICalendarComponentType::VLocation, SupportedComponent::VResource => ICalendarComponentType::VResource, SupportedComponent::VStatus => ICalendarComponentType::VStatus, SupportedComponent::Other => ICalendarComponentType::Other(Default::default()), } } } ================================================ FILE: crates/groupware/src/calendar/storage.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ ArchivedCalendar, ArchivedCalendarEvent, Calendar, CalendarEvent, CalendarPreferences, alarm::CalendarAlarm, }; use crate::{ DavResourceName, DestroyArchive, RFC_3986, calendar::{ ArchivedCalendarEventNotification, CalendarEventNotification, alarm::CalendarAlarmType, }, scheduling::{ItipMessages, event_cancel::itip_cancel}, }; use calcard::common::timezone::Tz; use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; use store::{ IterateParams, U16_LEN, U32_LEN, U64_LEN, ValueKey, roaring::RoaringBitmap, write::{ AlignedBytes, Archive, BatchBuilder, IndexPropertyClass, TaskEpoch, TaskQueueClass, ValueClass, key::{DeserializeBigEndian, KeySerializer}, now, }, }; use trc::AddContext; use types::{ collection::{Collection, VanishedCollection}, field::CalendarNotificationField, }; pub trait ItipAutoExpunge: Sync + Send { fn itip_ids(&self, account_id: u32) -> impl Future> + Send; fn itip_auto_expunge( &self, account_id: u32, hold_period: u64, ) -> impl Future> + Send; } impl ItipAutoExpunge for Server { async fn itip_ids(&self, account_id: u32) -> trc::Result { let mut document_ids = RoaringBitmap::new(); self.store() .iterate( IterateParams::new( ValueKey { account_id, collection: Collection::CalendarEventNotification.into(), document_id: 0, class: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: CalendarNotificationField::CreatedToId.into(), value: 0, }), }, ValueKey { account_id, collection: Collection::CalendarEventNotification.into(), document_id: 0, class: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: CalendarNotificationField::CreatedToId.into(), value: u64::MAX, }), }, ) .no_values() .ascending(), |key, _| { document_ids.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); Ok(true) }, ) .await .caused_by(trc::location!()) .map(|_| document_ids) } async fn itip_auto_expunge(&self, account_id: u32, hold_period: u64) -> trc::Result<()> { let mut destroy_ids = RoaringBitmap::new(); self.store() .iterate( IterateParams::new( ValueKey { account_id, collection: Collection::CalendarEventNotification.into(), document_id: 0, class: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: CalendarNotificationField::CreatedToId.into(), value: 0, }), }, ValueKey { account_id, collection: Collection::CalendarEventNotification.into(), document_id: 0, class: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: CalendarNotificationField::CreatedToId.into(), value: now().saturating_sub(hold_period), }), }, ) .no_values() .ascending(), |key, _| { destroy_ids.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); Ok(true) }, ) .await .caused_by(trc::location!())?; if destroy_ids.is_empty() { return Ok(()); } trc::event!( Purge(trc::PurgeEvent::AutoExpunge), AccountId = account_id, Collection = Collection::CalendarEventNotification.as_str(), Total = destroy_ids.len(), ); // Tombstone messages let mut batch = BatchBuilder::new(); let access_token = self .get_access_token(account_id) .await .caused_by(trc::location!())?; for document_id in destroy_ids { // Fetch event if let Some(event_) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEventNotification, document_id, )) .await .caused_by(trc::location!())? { let event = event_ .to_unarchived::() .caused_by(trc::location!())?; DestroyArchive(event) .delete(&access_token, account_id, document_id, &mut batch) .caused_by(trc::location!())?; } } self.commit_batch(batch).await.caused_by(trc::location!())?; Ok(()) } } impl CalendarEvent { pub fn update<'x>( self, access_token: &AccessToken, event: Archive<&ArchivedCalendarEvent>, account_id: u32, document_id: u32, batch: &'x mut BatchBuilder, ) -> trc::Result<&'x mut BatchBuilder> { let mut new_event = self; // Build event new_event.modified = now() as i64; // Prepare write batch batch .with_account_id(account_id) .with_collection(Collection::CalendarEvent) .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_current(event) .with_changes(new_event) .with_access_token(access_token), ) .map(|b| b.commit_point()) } pub fn insert<'x>( self, access_token: &AccessToken, account_id: u32, document_id: u32, next_alarm: Option, batch: &'x mut BatchBuilder, ) -> trc::Result<&'x mut BatchBuilder> { // Build event let mut event = self; let now = now() as i64; event.modified = now; event.created = now; // Prepare write batch batch .with_account_id(account_id) .with_collection(Collection::CalendarEvent) .with_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(event) .with_access_token(access_token), ) .map(|batch| { if let Some(next_alarm) = next_alarm { next_alarm.write_task(batch); } batch.commit_point() }) } } impl Calendar { pub fn insert<'x>( self, access_token: &AccessToken, account_id: u32, document_id: u32, batch: &'x mut BatchBuilder, ) -> trc::Result<&'x mut BatchBuilder> { // Build address calendar let mut calendar = self; let now = now() as i64; calendar.modified = now; calendar.created = now; if calendar.preferences.is_empty() { calendar.preferences.push(CalendarPreferences { account_id, name: "default".to_string(), ..Default::default() }); } // Prepare write batch batch .with_account_id(account_id) .with_collection(Collection::Calendar) .with_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(calendar) .with_access_token(access_token), ) .map(|b| b.commit_point()) } pub fn update<'x>( self, access_token: &AccessToken, calendar: Archive<&ArchivedCalendar>, account_id: u32, document_id: u32, batch: &'x mut BatchBuilder, ) -> trc::Result<&'x mut BatchBuilder> { // Build address calendar let mut new_calendar = self; new_calendar.modified = now() as i64; // Prepare write batch batch .with_account_id(account_id) .with_collection(Collection::Calendar) .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_current(calendar) .with_changes(new_calendar) .with_access_token(access_token), ) .map(|b| b.commit_point()) } } impl CalendarEventNotification { pub fn insert<'x>( self, access_token: &AccessToken, account_id: u32, document_id: u32, batch: &'x mut BatchBuilder, ) -> trc::Result<&'x mut BatchBuilder> { // Build event let mut event = self; let now = now() as i64; event.modified = now; event.created = now; // Prepare write batch batch .with_account_id(account_id) .with_collection(Collection::CalendarEventNotification) .with_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(event) .with_access_token(access_token), ) .map(|batch| batch.commit_point()) } } impl DestroyArchive> { #[allow(clippy::too_many_arguments)] pub async fn delete_with_events( self, server: &Server, access_token: &AccessToken, account_id: u32, document_id: u32, children_ids: Vec, delete_path: Option, send_itip: bool, batch: &mut BatchBuilder, ) -> trc::Result<()> { // Process deletions let calendar_id = document_id; for document_id in children_ids { if let Some(event_) = server .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, )) .await? { DestroyArchive( event_ .to_unarchived::() .caused_by(trc::location!())?, ) .delete( access_token, account_id, document_id, calendar_id, None, send_itip, batch, )?; } } self.delete(access_token, account_id, document_id, delete_path, batch) } pub fn delete( self, access_token: &AccessToken, account_id: u32, document_id: u32, delete_path: Option, batch: &mut BatchBuilder, ) -> trc::Result<()> { let calendar = self.0; // Delete calendar batch .with_account_id(account_id) .with_collection(Collection::Calendar) .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() .with_access_token(access_token) .with_current(calendar), ) .caused_by(trc::location!())?; if let Some(delete_path) = delete_path { batch.log_vanished_item(VanishedCollection::Calendar, delete_path); } batch.commit_point(); Ok(()) } } impl DestroyArchive> { #[allow(clippy::too_many_arguments)] pub fn delete( self, access_token: &AccessToken, account_id: u32, document_id: u32, calendar_id: u32, delete_path: Option, send_itip: bool, batch: &mut BatchBuilder, ) -> trc::Result<()> { if let Some(delete_idx) = self .0 .inner .names .iter() .position(|name| name.parent_id == calendar_id) { if self.0.inner.names.len() > 1 { // Unlink calendar id from event let event = self.0; let mut new_event = event .deserialize::() .caused_by(trc::location!())?; new_event.names.swap_remove(delete_idx); batch .with_account_id(account_id) .with_collection(Collection::CalendarEvent) .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_access_token(access_token) .with_current(event) .with_changes(new_event), ) .caused_by(trc::location!())?; } else { self.delete_all(access_token, account_id, document_id, send_itip, batch)?; } if let Some(delete_path) = delete_path { batch.log_vanished_item(VanishedCollection::Calendar, delete_path); } batch.commit_point(); } Ok(()) } #[allow(clippy::too_many_arguments)] pub fn delete_all( self, access_token: &AccessToken, account_id: u32, document_id: u32, send_itip: bool, batch: &mut BatchBuilder, ) -> trc::Result<()> { let event = self.0; // Delete event batch .with_account_id(account_id) .with_collection(Collection::CalendarEvent) .with_document(document_id); // Remove next alarm if it exists let now = now() as i64; if let Some(next_alarm) = event.inner.data.next_alarm(now, Tz::Floating) { next_alarm.delete_task(batch); } // Scheduling if send_itip && event.inner.schedule_tag.is_some() && event.inner.data.event_range_end() > now { let event = event .deserialize::() .caused_by(trc::location!())?; if let Ok(messages) = itip_cancel(&event.data.event, access_token.emails.as_slice(), true) { ItipMessages::new(vec![messages]) .queue(batch) .caused_by(trc::location!())?; } } batch .custom( ObjectIndexBuilder::<_, ()>::new() .with_access_token(access_token) .with_current(event), ) .caused_by(trc::location!())?; Ok(()) } } impl DestroyArchive> { #[allow(clippy::too_many_arguments)] pub fn delete( self, access_token: &AccessToken, account_id: u32, document_id: u32, batch: &mut BatchBuilder, ) -> trc::Result<()> { // Delete event batch .with_account_id(account_id) .with_collection(Collection::CalendarEventNotification) .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() .with_access_token(access_token) .with_current(self.0), ) .caused_by(trc::location!())? .commit_point(); Ok(()) } } impl CalendarAlarm { pub fn write_task(&self, batch: &mut BatchBuilder) { match &self.typ { CalendarAlarmType::Email { event_start, event_start_tz, event_end, event_end_tz, } => { batch.set( ValueClass::TaskQueue(TaskQueueClass::SendAlarm { due: TaskEpoch::new(self.alarm_time as u64), event_id: self.event_id, alarm_id: self.alarm_id, is_email_alert: true, }), KeySerializer::new((U64_LEN * 2) + (U16_LEN * 2)) .write(*event_start as u64) .write(*event_end as u64) .write(*event_start_tz) .write(*event_end_tz) .finalize(), ); } CalendarAlarmType::Display { recurrence_id } => { batch.set( ValueClass::TaskQueue(TaskQueueClass::SendAlarm { due: TaskEpoch::new(self.alarm_time as u64), event_id: self.event_id, alarm_id: self.alarm_id, is_email_alert: false, }), KeySerializer::new(U64_LEN) .write(recurrence_id.unwrap_or_default() as u64) .finalize(), ); } } } pub fn delete_task(&self, batch: &mut BatchBuilder) { batch.clear(ValueClass::TaskQueue(TaskQueueClass::SendAlarm { due: TaskEpoch::new(self.alarm_time as u64), event_id: self.event_id, alarm_id: self.alarm_id, is_email_alert: matches!(self.typ, CalendarAlarmType::Email { .. }), })); } } impl ArchivedCalendarEvent { pub async fn webcal_uri( &self, server: &Server, access_token: &AccessToken, ) -> trc::Result { for event_name in self.names.iter() { if let Some(calendar_) = server .store() .get_value::>(ValueKey::archive( access_token.primary_id, Collection::Calendar, event_name.parent_id.to_native(), )) .await .caused_by(trc::location!())? { let calendar = calendar_ .unarchive::() .caused_by(trc::location!())?; return Ok(format!( "webcal://{}{}/{}/{}/{}", server.core.network.server_name, DavResourceName::Cal.base_path(), percent_encoding::utf8_percent_encode(&access_token.name, RFC_3986), calendar.name, event_name.name )); } } Err(trc::StoreEvent::UnexpectedError .into_err() .details("Event is not linked to any calendar")) } } ================================================ FILE: crates/groupware/src/contact/index.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{AddressBook, ArchivedAddressBook, ArchivedContactCard, ContactCard}; use ahash::AHashSet; use calcard::{ common::IanaString, vcard::{ArchivedVCardProperty, ArchivedVCardValue, VCardProperty}, }; use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; use nlp::language::{ Language, detect::{LanguageDetector, MIN_LANGUAGE_SCORE}, }; use store::{ search::{ContactSearchField, IndexDocument, SearchField}, write::{IndexPropertyClass, SearchIndex, ValueClass}, xxhash_rust::xxh3, }; use types::{acl::AclGrant, collection::SyncCollection, field::ContactField}; use utils::sanitize_email; impl IndexableObject for AddressBook { fn index_values(&self) -> impl Iterator> { [ IndexValue::Acl { value: (&self.acls).into(), }, IndexValue::Quota { used: self.size() as u32, }, IndexValue::LogContainer { sync_collection: SyncCollection::AddressBook, }, ] .into_iter() } } impl IndexableObject for &ArchivedAddressBook { fn index_values(&self) -> impl Iterator> { [ IndexValue::Acl { value: self .acls .iter() .map(AclGrant::from) .collect::>() .into(), }, IndexValue::Quota { used: self.size() as u32, }, IndexValue::LogContainer { sync_collection: SyncCollection::AddressBook, }, ] .into_iter() } } impl IndexableAndSerializableObject for AddressBook { fn is_versioned() -> bool { true } } impl IndexableObject for ContactCard { fn index_values(&self) -> impl Iterator> { [ IndexValue::Index { field: ContactField::Uid.into(), value: self.card.uid().into(), }, IndexValue::Index { field: ContactField::Email.into(), value: self.emails().next().into(), }, IndexValue::Property { field: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: ContactField::CreatedToUpdated.into(), value: self.created as u64, }), value: self.modified.into(), }, IndexValue::SearchIndex { index: SearchIndex::Contacts, hash: self.hashes().fold(0, |acc, hash| acc ^ hash), }, IndexValue::Quota { used: self.size() as u32, }, IndexValue::LogItem { sync_collection: SyncCollection::AddressBook, prefix: None, }, ] .into_iter() } } impl IndexableObject for &ArchivedContactCard { fn index_values(&self) -> impl Iterator> { [ IndexValue::Index { field: ContactField::Uid.into(), value: self.card.uid().into(), }, IndexValue::Index { field: ContactField::Email.into(), value: self.emails().next().into(), }, IndexValue::Property { field: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: ContactField::CreatedToUpdated.into(), value: self.created.to_native() as u64, }), value: (self.modified.to_native() as u64).into(), }, IndexValue::SearchIndex { index: SearchIndex::Contacts, hash: self.hashes().fold(0, |acc, hash| acc ^ hash), }, IndexValue::Quota { used: self.size() as u32, }, IndexValue::LogItem { sync_collection: SyncCollection::AddressBook, prefix: None, }, ] .into_iter() } } impl IndexableAndSerializableObject for ContactCard { fn is_versioned() -> bool { true } } impl AddressBook { pub fn size(&self) -> usize { self.dead_properties.size() + self .preferences .iter() .map(|p| p.name.len() + p.description.as_ref().map_or(0, |n| n.len())) .sum::() + self.name.len() + std::mem::size_of::() } } impl ArchivedAddressBook { pub fn size(&self) -> usize { self.dead_properties.size() + self .preferences .iter() .map(|p| p.name.len() + p.description.as_ref().map_or(0, |n| n.len())) .sum::() + self.name.len() + std::mem::size_of::() } } impl ContactCard { pub fn size(&self) -> usize { self.dead_properties.size() + self.display_name.as_ref().map_or(0, |n| n.len()) + self.names.iter().map(|n| n.name.len()).sum::() + self.size as usize + std::mem::size_of::() } pub fn hashes(&self) -> impl Iterator { self.card .entries .iter() .filter(|e| { matches!( e.name, VCardProperty::Adr | VCardProperty::N | VCardProperty::Fn | VCardProperty::Title | VCardProperty::Org | VCardProperty::Note | VCardProperty::Nickname | VCardProperty::Email | VCardProperty::Kind | VCardProperty::Uid | VCardProperty::Member | VCardProperty::Impp | VCardProperty::Socialprofile | VCardProperty::Tel ) }) .flat_map(|e| e.values.iter().filter_map(|v| v.as_text())) .map(|v| xxh3::xxh3_64(v.as_bytes())) } pub fn emails(&self) -> impl Iterator { self.card.properties(&VCardProperty::Email).flat_map(|e| { e.values .iter() .filter_map(|v| v.as_text().and_then(sanitize_email)) }) } } impl ArchivedContactCard { pub fn size(&self) -> usize { self.dead_properties.size() + self.display_name.as_ref().map_or(0, |n| n.len()) + self.names.iter().map(|n| n.name.len()).sum::() + self.size.to_native() as usize + std::mem::size_of::() } pub fn hashes(&self) -> impl Iterator { self.card .entries .iter() .filter(|e| { matches!( e.name, ArchivedVCardProperty::Adr | ArchivedVCardProperty::N | ArchivedVCardProperty::Fn | ArchivedVCardProperty::Title | ArchivedVCardProperty::Org | ArchivedVCardProperty::Note | ArchivedVCardProperty::Nickname | ArchivedVCardProperty::Email | ArchivedVCardProperty::Kind | ArchivedVCardProperty::Uid | ArchivedVCardProperty::Member | ArchivedVCardProperty::Impp | ArchivedVCardProperty::Socialprofile | ArchivedVCardProperty::Tel ) }) .flat_map(|e| e.values.iter().filter_map(|v| v.as_text())) .map(|v| xxh3::xxh3_64(v.as_bytes())) } pub fn emails(&self) -> impl Iterator { self.card.properties(&VCardProperty::Email).flat_map(|e| { e.values .iter() .filter_map(|v| v.as_text().and_then(sanitize_email)) }) } pub fn index_document( &self, account_id: u32, document_id: u32, index_fields: &AHashSet, default_language: Language, ) -> IndexDocument { let mut document = IndexDocument::new(SearchIndex::Contacts) .with_account_id(account_id) .with_document_id(document_id); let mut detector = LanguageDetector::new(); for entry in self.card.entries.iter() { let (is_text, is_keyword, field) = match entry.name { ArchivedVCardProperty::N => (false, false, ContactSearchField::Name), ArchivedVCardProperty::Nickname => (false, false, ContactSearchField::Nickname), ArchivedVCardProperty::Org => (false, false, ContactSearchField::Organization), ArchivedVCardProperty::Email => (false, false, ContactSearchField::Email), ArchivedVCardProperty::Tel => (false, false, ContactSearchField::Phone), ArchivedVCardProperty::Impp | ArchivedVCardProperty::Socialprofile => { (false, false, ContactSearchField::OnlineService) } ArchivedVCardProperty::Adr => (false, false, ContactSearchField::Address), ArchivedVCardProperty::Note => (true, false, ContactSearchField::Note), ArchivedVCardProperty::Kind => (false, true, ContactSearchField::Kind), ArchivedVCardProperty::Uid => (false, true, ContactSearchField::Uid), ArchivedVCardProperty::Member => (false, false, ContactSearchField::Member), _ => continue, }; let field = SearchField::Contact(field); if index_fields.is_empty() || index_fields.contains(&field) { for value in entry.values.iter() { match value { ArchivedVCardValue::Text(v) => { if !is_keyword { let lang = if is_text { detector.detect(v.as_str().trim(), MIN_LANGUAGE_SCORE); Language::Unknown } else { Language::None }; document.index_text(field.clone(), v, lang); } else { document.index_keyword(field.clone(), v.as_str()); } } ArchivedVCardValue::Kind(v) => { document.index_keyword(field.clone(), v.as_str()); } ArchivedVCardValue::Component(v) => { for item in v.iter() { document.index_text(field.clone(), item.trim(), Language::None); } } _ => (), } } /*for param in entry.params.iter() { if let ArchivedVCardParameterValue::Text(value) = ¶m.value { let lang = if is_text { detector.detect(value.as_str(), MIN_LANGUAGE_SCORE); Language::Unknown } else { Language::None }; document.index_text(field.clone(), value, lang); } }*/ } } document.set_unknown_language( detector .most_frequent_language() .unwrap_or(default_language), ); document } } ================================================ FILE: crates/groupware/src/contact/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod index; pub mod storage; use calcard::vcard::VCard; use common::{DavName, auth::AccessToken}; use types::{acl::AclGrant, dead_property::DeadProperty}; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] #[rkyv(derive(Debug))] pub struct AddressBook { pub name: String, pub preferences: Vec, pub subscribers: Vec, pub dead_properties: DeadProperty, pub acls: Vec, pub created: i64, pub modified: i64, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] #[rkyv(derive(Debug))] pub struct AddressBookPreferences { pub account_id: u32, pub name: String, pub description: Option, pub sort_order: u32, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct ContactCard { pub names: Vec, pub display_name: Option, pub card: VCard, pub dead_properties: DeadProperty, pub created: i64, pub modified: i64, pub size: u32, } impl AddressBook { pub fn preferences(&self, access_token: &AccessToken) -> &AddressBookPreferences { if self.preferences.len() == 1 { &self.preferences[0] } else { let account_id = access_token.primary_id(); self.preferences .iter() .find(|p| p.account_id == account_id) .or_else(|| self.preferences.first()) .unwrap() } } pub fn preferences_mut(&mut self, access_token: &AccessToken) -> &mut AddressBookPreferences { let account_id = access_token.primary_id(); let idx = if let Some(idx) = self .preferences .iter() .position(|p| p.account_id == account_id) { idx } else { let mut preferences = self.preferences[0].clone(); preferences.account_id = account_id; self.preferences.push(preferences); self.preferences.len() - 1 }; &mut self.preferences[idx] } } impl ArchivedAddressBook { pub fn preferences(&self, access_token: &AccessToken) -> &ArchivedAddressBookPreferences { if self.preferences.len() == 1 { &self.preferences[0] } else { let account_id = access_token.primary_id(); self.preferences .iter() .find(|p| p.account_id == account_id) .or_else(|| self.preferences.first()) .unwrap() } } } impl ContactCard { pub fn added_addressbook_ids( &self, prev_data: &ArchivedContactCard, ) -> impl Iterator { self.names .iter() .filter(|m| prev_data.names.iter().all(|pm| pm.parent_id != m.parent_id)) .map(|m| m.parent_id) } pub fn removed_addressbook_ids( &self, prev_data: &ArchivedContactCard, ) -> impl Iterator { prev_data .names .iter() .filter(|m| self.names.iter().all(|pm| pm.parent_id != m.parent_id)) .map(|m| m.parent_id.to_native()) } pub fn unchanged_addressbook_ids( &self, prev_data: &ArchivedContactCard, ) -> impl Iterator { self.names .iter() .filter(|m| prev_data.names.iter().any(|pm| pm.parent_id == m.parent_id)) .map(|m| m.parent_id) } } ================================================ FILE: crates/groupware/src/contact/storage.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{AddressBook, ArchivedAddressBook, ArchivedContactCard, ContactCard}; use crate::DestroyArchive; use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; use store::{ ValueKey, write::{AlignedBytes, Archive, BatchBuilder, now}, }; use trc::AddContext; use types::collection::{Collection, VanishedCollection}; impl ContactCard { pub fn update<'x>( self, access_token: &AccessToken, card: Archive<&ArchivedContactCard>, account_id: u32, document_id: u32, batch: &'x mut BatchBuilder, ) -> trc::Result<&'x mut BatchBuilder> { let mut new_card = self; // Build card new_card.modified = now() as i64; // Prepare write batch batch .with_account_id(account_id) .with_collection(Collection::ContactCard) .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_current(card) .with_changes(new_card) .with_access_token(access_token), ) .map(|b| b.commit_point()) } pub fn insert<'x>( self, access_token: &AccessToken, account_id: u32, document_id: u32, batch: &'x mut BatchBuilder, ) -> trc::Result<&'x mut BatchBuilder> { // Build card let mut card = self; let now = now() as i64; card.modified = now; card.created = now; // Prepare write batch batch .with_account_id(account_id) .with_collection(Collection::ContactCard) .with_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(card) .with_access_token(access_token), ) .map(|b| b.commit_point()) } } impl AddressBook { pub fn insert<'x>( self, access_token: &AccessToken, account_id: u32, document_id: u32, batch: &'x mut BatchBuilder, ) -> trc::Result<&'x mut BatchBuilder> { // Build address book let mut book = self; let now = now() as i64; book.modified = now; book.created = now; // Prepare write batch batch .with_account_id(account_id) .with_collection(Collection::AddressBook) .with_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(book) .with_access_token(access_token), ) .map(|b| b.commit_point()) } pub fn update<'x>( self, access_token: &AccessToken, book: Archive<&ArchivedAddressBook>, account_id: u32, document_id: u32, batch: &'x mut BatchBuilder, ) -> trc::Result<&'x mut BatchBuilder> { // Build address book let mut new_book = self; new_book.modified = now() as i64; // Prepare write batch batch .with_account_id(account_id) .with_collection(Collection::AddressBook) .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_current(book) .with_changes(new_book) .with_access_token(access_token), ) .map(|b| b.commit_point()) } } impl DestroyArchive> { #[allow(clippy::too_many_arguments)] pub async fn delete_with_cards( self, server: &Server, access_token: &AccessToken, account_id: u32, document_id: u32, children_ids: Vec, delete_path: Option, batch: &mut BatchBuilder, ) -> trc::Result<()> { // Process deletions let addressbook_id = document_id; for document_id in children_ids { if let Some(card_) = server .store() .get_value::>(ValueKey::archive( account_id, Collection::ContactCard, document_id, )) .await? { DestroyArchive( card_ .to_unarchived::() .caused_by(trc::location!())?, ) .delete( access_token, account_id, document_id, addressbook_id, None, batch, )?; } } self.delete(access_token, account_id, document_id, delete_path, batch) } pub fn delete( self, access_token: &AccessToken, account_id: u32, document_id: u32, delete_path: Option, batch: &mut BatchBuilder, ) -> trc::Result<()> { let book = self.0; // Delete addressbook batch .with_account_id(account_id) .with_collection(Collection::AddressBook) .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() .with_access_token(access_token) .with_current(book), ) .caused_by(trc::location!())?; if let Some(delete_path) = delete_path { batch.log_vanished_item(VanishedCollection::AddressBook, delete_path); } batch.commit_point(); Ok(()) } } impl DestroyArchive> { pub fn delete( self, access_token: &AccessToken, account_id: u32, document_id: u32, addressbook_id: u32, delete_path: Option, batch: &mut BatchBuilder, ) -> trc::Result<()> { let card = self.0; if let Some(delete_idx) = card .inner .names .iter() .position(|name| name.parent_id == addressbook_id) { batch .with_account_id(account_id) .with_collection(Collection::ContactCard); if card.inner.names.len() > 1 { // Unlink addressbook id from card let mut new_card = card .deserialize::() .caused_by(trc::location!())?; new_card.names.swap_remove(delete_idx); batch .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_access_token(access_token) .with_current(card) .with_changes(new_card), ) .caused_by(trc::location!())?; } else { // Delete card batch .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() .with_access_token(access_token) .with_current(card), ) .caused_by(trc::location!())?; } if let Some(delete_path) = delete_path { batch.log_vanished_item(VanishedCollection::AddressBook, delete_path); } batch.commit_point(); } Ok(()) } pub fn delete_all( self, access_token: &AccessToken, account_id: u32, document_id: u32, batch: &mut BatchBuilder, ) -> trc::Result<()> { batch .with_account_id(account_id) .with_collection(Collection::ContactCard) .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() .with_access_token(access_token) .with_current(self.0), ) .caused_by(trc::location!()) .map(|b| { b.commit_point(); }) } } ================================================ FILE: crates/groupware/src/file/index.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ArchivedFileNode, FileNode}; use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; use types::{acl::AclGrant, collection::SyncCollection}; impl IndexableObject for FileNode { fn index_values(&self) -> impl Iterator> { let mut values = Vec::with_capacity(6); values.extend([ IndexValue::Acl { value: (&self.acls).into(), }, IndexValue::LogItem { prefix: None, sync_collection: SyncCollection::FileNode, }, IndexValue::Quota { used: self.size() as u32, }, ]); if let Some(file) = &self.file { values.extend([IndexValue::Blob { value: file.blob_hash.clone(), }]); } values.into_iter() } } impl IndexableObject for &ArchivedFileNode { fn index_values(&self) -> impl Iterator> { let mut values = Vec::with_capacity(6); values.extend([ IndexValue::Acl { value: self .acls .iter() .map(AclGrant::from) .collect::>() .into(), }, IndexValue::LogItem { prefix: None, sync_collection: SyncCollection::FileNode, }, IndexValue::Quota { used: self.size() as u32, }, ]); if let Some(file) = self.file.as_ref() { values.extend([IndexValue::Blob { value: (&file.blob_hash).into(), }]); } values.into_iter() } } impl IndexableAndSerializableObject for FileNode { fn is_versioned() -> bool { true } } impl FileNode { pub fn size(&self) -> usize { self.dead_properties.size() + self.display_name.as_ref().map_or(0, |n| n.len()) + self.name.len() + self.file.as_ref().map_or(0, |f| f.size as usize) + std::mem::size_of::() } } impl ArchivedFileNode { pub fn size(&self) -> usize { self.dead_properties.size() + self.display_name.as_ref().map_or(0, |n| n.len()) + self.name.len() + self .file .as_ref() .map_or(0, |f| f.size.to_native() as usize) + std::mem::size_of::() } } ================================================ FILE: crates/groupware/src/file/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod index; pub mod storage; use types::{acl::AclGrant, blob_hash::BlobHash, dead_property::DeadProperty}; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] #[rkyv(derive(Debug))] pub struct FileNode { pub parent_id: u32, pub name: String, pub display_name: Option, pub file: Option, pub created: i64, pub modified: i64, pub dead_properties: DeadProperty, pub acls: Vec, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] #[rkyv(derive(Debug))] pub struct FileProperties { pub blob_hash: BlobHash, pub size: u32, pub media_type: Option, pub executable: bool, } ================================================ FILE: crates/groupware/src/file/storage.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ArchivedFileNode, FileNode}; use crate::DestroyArchive; use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; use store::{ ValueKey, write::{AlignedBytes, Archive, BatchBuilder, now}, }; use trc::AddContext; use types::collection::{Collection, VanishedCollection}; impl FileNode { pub fn insert<'x>( self, access_token: &AccessToken, account_id: u32, document_id: u32, batch: &'x mut BatchBuilder, ) -> trc::Result<&'x mut BatchBuilder> { // Build node let mut node = self; let now = now() as i64; node.modified = now; node.created = now; // Prepare write batch batch .with_account_id(account_id) .with_collection(Collection::FileNode) .with_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(node) .with_access_token(access_token), ) .map(|b| b.commit_point()) } pub fn update<'x>( self, access_token: &AccessToken, node: Archive<&ArchivedFileNode>, account_id: u32, document_id: u32, batch: &'x mut BatchBuilder, ) -> trc::Result<&'x mut BatchBuilder> { // Build node let mut new_node = self; new_node.modified = now() as i64; batch .with_account_id(account_id) .with_collection(Collection::FileNode) .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_current(node) .with_changes(new_node) .with_access_token(access_token), ) .map(|b| b.commit_point()) } } impl DestroyArchive> { pub fn delete( self, access_token: &AccessToken, account_id: u32, document_id: u32, batch: &mut BatchBuilder, path: String, ) -> trc::Result<()> { // Prepare write batch batch .with_account_id(account_id) .with_collection(Collection::FileNode) .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() .with_current(self.0) .with_access_token(access_token), )? .log_vanished_item(VanishedCollection::FileNode, path) .commit_point(); Ok(()) } } impl DestroyArchive> { pub async fn delete( self, server: &Server, access_token: &AccessToken, account_id: u32, delete_path: Option, ) -> trc::Result<()> { // Process deletions let mut batch = BatchBuilder::new(); self.delete_batch(server, access_token, account_id, delete_path, &mut batch) .await?; // Write changes if !batch.is_empty() { server .commit_batch(batch) .await .caused_by(trc::location!())?; } Ok(()) } pub async fn delete_batch( self, server: &Server, access_token: &AccessToken, account_id: u32, delete_path: Option, batch: &mut BatchBuilder, ) -> trc::Result<()> { // Process deletions batch .with_account_id(account_id) .with_collection(Collection::FileNode); for document_id in self.0 { if let Some(node) = server .store() .get_value::>(ValueKey::archive( account_id, Collection::FileNode, document_id, )) .await? { // Delete record batch .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() .with_access_token(access_token) .with_current( node.to_unarchived::() .caused_by(trc::location!())?, ), ) .caused_by(trc::location!())? .commit_point(); } } if !batch.is_empty() && let Some(delete_path) = delete_path { batch.log_vanished_item(VanishedCollection::FileNode, delete_path); } Ok(()) } } ================================================ FILE: crates/groupware/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use calcard::common::timezone::Tz; use common::DavResources; use percent_encoding::{AsciiSet, CONTROLS}; use types::collection::{Collection, SyncCollection}; pub mod cache; pub mod calendar; pub mod contact; pub mod file; pub mod scheduling; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DavResourceName { Card, Cal, File, Principal, Scheduling, } pub const RFC_3986: &AsciiSet = &CONTROLS .add(b' ') .add(b'!') .add(b'"') .add(b'#') .add(b'$') .add(b'%') .add(b'&') .add(b'\'') .add(b'(') .add(b')') .add(b'*') .add(b'+') .add(b',') .add(b'/') .add(b':') .add(b';') .add(b'<') .add(b'=') .add(b'>') .add(b'?') .add(b'@') .add(b'[') .add(b'\\') .add(b']') .add(b'^') .add(b'`') .add(b'{') .add(b'|') .add(b'}'); pub struct DestroyArchive(pub T); impl DavResourceName { pub fn parse(service: &str) -> Option { hashify::tiny_map!(service.as_bytes(), "card" => DavResourceName::Card, "cal" => DavResourceName::Cal, "file" => DavResourceName::File, "pal" => DavResourceName::Principal, "itip" => DavResourceName::Scheduling, ) } pub fn base_path(&self) -> &'static str { match self { DavResourceName::Card => "/dav/card", DavResourceName::Cal => "/dav/cal", DavResourceName::File => "/dav/file", DavResourceName::Principal => "/dav/pal", DavResourceName::Scheduling => "/dav/itip", } } pub fn collection_path(&self) -> &'static str { match self { DavResourceName::Card => "/dav/card/", DavResourceName::Cal => "/dav/cal/", DavResourceName::File => "/dav/file/", DavResourceName::Principal => "/dav/pal/", DavResourceName::Scheduling => "/dav/itip/", } } pub fn name(&self) -> &'static str { match self { DavResourceName::Card => "CardDAV", DavResourceName::Cal => "CalDAV", DavResourceName::File => "WebDAV", DavResourceName::Principal => "Principal", DavResourceName::Scheduling => "Scheduling", } } } impl From for Collection { fn from(value: DavResourceName) -> Self { match value { DavResourceName::Card => Collection::AddressBook, DavResourceName::Cal => Collection::Calendar, DavResourceName::File => Collection::FileNode, DavResourceName::Principal => Collection::Principal, DavResourceName::Scheduling => Collection::CalendarEventNotification, } } } impl From for DavResourceName { fn from(value: Collection) -> Self { match value { Collection::AddressBook => DavResourceName::Card, Collection::Calendar => DavResourceName::Cal, Collection::FileNode => DavResourceName::File, Collection::Principal => DavResourceName::Principal, Collection::CalendarEventNotification => DavResourceName::Scheduling, _ => unreachable!(), } } } impl From for DavResourceName { fn from(value: SyncCollection) -> Self { match value { SyncCollection::AddressBook => DavResourceName::Card, SyncCollection::Calendar => DavResourceName::Cal, SyncCollection::FileNode => DavResourceName::File, SyncCollection::CalendarEventNotification => DavResourceName::Scheduling, _ => unreachable!(), } } } pub trait DavCalendarResource { fn calendar_default_tz(&self, calendar_id: u32, account_id: u32) -> Option; } impl DavCalendarResource for DavResources { fn calendar_default_tz(&self, calendar_id: u32, account_id: u32) -> Option { self.container_resource_by_id(calendar_id) .and_then(|c| c.calendar_preferences(account_id)) .map(|p| p.tz) } } ================================================ FILE: crates/groupware/src/scheduling/attendee.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::scheduling::{ Email, InstanceId, ItipEntryValue, ItipError, ItipMessage, ItipSnapshot, ItipSnapshots, ItipSummary, itip::{ ItipExportAs, can_attendee_modify_property, itip_add_tz, itip_build_envelope, itip_export_component, }, organizer::organizer_request_full, }; use ahash::AHashSet; use calcard::{ common::PartialDateTime, icalendar::{ ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarMethod, ICalendarParameter, ICalendarParticipationStatus, ICalendarProperty, ICalendarValue, }, }; pub(crate) fn attendee_handle_update( new_ical: &ICalendar, old_itip: ItipSnapshots<'_>, new_itip: ItipSnapshots<'_>, ) -> Result>, ItipError> { let dt_stamp = PartialDateTime::now(); let mut message = ICalendar { components: Vec::with_capacity(2), }; message .components .push(itip_build_envelope(ICalendarMethod::Reply)); let mut mail_from = None; let mut email_rcpt = AHashSet::new(); let mut new_delegates = AHashSet::new(); let mut part_stat = &ICalendarParticipationStatus::NeedsAction; for (instance_id, instance) in &new_itip.components { if let Some(old_instance) = old_itip.components.get(instance_id) { match (instance.local_attendee(), old_instance.local_attendee()) { (Some(local_attendee), Some(old_local_attendee)) if local_attendee.email == old_local_attendee.email => { // Check added fields let mut send_update = false; for new_entry in instance.entries.difference(&old_instance.entries) { match (new_entry.name, &new_entry.value) { (ICalendarProperty::Exdate, ItipEntryValue::DateTime(date)) if instance_id == &InstanceId::Main => { if let Some((mut cancel_comp, attendee_email)) = attendee_decline( instance_id, &old_itip, old_instance, &dt_stamp, &mut email_rcpt, false, ) { // Add EXDATE as RECURRENCE-ID cancel_comp .entries .push(date.to_entry(ICalendarProperty::RecurrenceId)); part_stat = &ICalendarParticipationStatus::Declined; // Add cancel component let comp_id = message.components.len() as u32; message.components[0].component_ids.push(comp_id); message.components.push(cancel_comp); mail_from = Some(&attendee_email.email); } } _ => { // Changing these properties is not allowed if !can_attendee_modify_property( &instance.comp.component_type, new_entry.name, ) { return Err(ItipError::CannotModifyProperty( new_entry.name.clone(), )); } else { send_update = send_update || (instance.comp.component_type == ICalendarComponentType::VTodo && matches!( new_entry.name, ICalendarProperty::Status | ICalendarProperty::PercentComplete | ICalendarProperty::Completed )); } } } } // Send participation status update if local_attendee.is_server_scheduling && ((local_attendee.part_stat != old_local_attendee.part_stat) || local_attendee.force_send.is_some() || send_update) { // Build the attendee list if let Some(new_partstat) = local_attendee.part_stat { part_stat = new_partstat; } let mut attendee_entry_uids = vec![local_attendee.entry_id]; let old_delegates = old_instance .external_attendees() .filter(|a| a.is_delegated_from(old_local_attendee)) .map(|a| a.email.email.as_str()) .collect::>(); for external_attendee in instance.external_attendees() { if external_attendee.is_delegated_from(local_attendee) { if external_attendee.send_invite_messages() && !old_delegates .contains(&external_attendee.email.email.as_str()) { new_delegates.insert(external_attendee.email.email.as_str()); } } else if external_attendee.is_delegated_to(local_attendee) { if external_attendee.send_update_messages() { email_rcpt.insert(external_attendee.email.email.as_str()); } } else { continue; } attendee_entry_uids.push(external_attendee.entry_id); } let comp_id = message.components.len() as u32; message.components[0].component_ids.push(comp_id); message.components.push(itip_export_component( instance.comp, new_itip.uid, &dt_stamp, instance.sequence.unwrap_or_default(), ItipExportAs::Attendee(attendee_entry_uids), )); mail_from = Some(&local_attendee.email.email); } // Check removed fields for removed_entry in old_instance.entries.difference(&instance.entries) { if !can_attendee_modify_property( &instance.comp.component_type, removed_entry.name, ) { // Removing these properties is not allowed return Err(ItipError::CannotModifyProperty( removed_entry.name.clone(), )); } } } _ => { // Change in local attendee email is not allowed return Err(ItipError::CannotModifyAddress); } } } else if let Some(local_attendee) = instance .local_attendee() .filter(|_| instance_id != &InstanceId::Main) { let mut attendee_entry_uids = vec![local_attendee.entry_id]; for external_attendee in instance.external_attendees() { if external_attendee.is_delegated_from(local_attendee) { if external_attendee.send_invite_messages() { new_delegates.insert(external_attendee.email.email.as_str()); } } else if external_attendee.is_delegated_to(local_attendee) { if external_attendee.send_update_messages() { email_rcpt.insert(external_attendee.email.email.as_str()); } } else { continue; } attendee_entry_uids.push(external_attendee.entry_id); } // A new instance has been added let comp_id = message.components.len() as u32; message.components[0].component_ids.push(comp_id); message.components.push(itip_export_component( instance.comp, new_itip.uid, &dt_stamp, instance.sequence.unwrap_or_default(), ItipExportAs::Attendee(attendee_entry_uids), )); mail_from = Some(&local_attendee.email.email); } else { return Err(ItipError::CannotModifyInstance); } } for (instance_id, old_instance) in &old_itip.components { if !new_itip.components.contains_key(instance_id) { if instance_id != &InstanceId::Main && old_instance.has_local_attendee() { // Send cancel message for removed instances if let Some((cancel_comp, attendee_email)) = attendee_decline( instance_id, &old_itip, old_instance, &dt_stamp, &mut email_rcpt, false, ) { // Add cancel component let comp_id = message.components.len() as u32; message.components[0].component_ids.push(comp_id); message.components.push(cancel_comp); mail_from = Some(&attendee_email.email); } } else { // Removing instances is not allowed return Err(ItipError::CannotModifyInstance); } } } if let Some(from) = mail_from { email_rcpt.insert(&new_itip.organizer.email.email); // Add timezones if needed itip_add_tz(&mut message, new_ical); let mut responses = vec![ItipMessage { from: from.to_string(), from_organizer: false, to: email_rcpt.into_iter().map(|e| e.to_string()).collect(), summary: ItipSummary::Rsvp { part_stat: part_stat.clone(), current: new_itip .main_instance_or_default() .build_summary(Some(&new_itip.organizer), &[]), }, message, }]; // Invite new delegates if !new_delegates.is_empty() { let from = from.to_string(); let new_delegates = new_delegates .into_iter() .map(|e| e.to_string()) .collect::>(); if let Ok(messages_) = organizer_request_full(new_ical, &new_itip, None, true) { for mut message in messages_ { message.from = from.clone(); message.to = new_delegates.clone(); message.from_organizer = false; responses.push(message); } } } Ok(responses) } else { Err(ItipError::NothingToSend) } } pub(crate) fn attendee_decline<'x>( instance_id: &'x InstanceId, itip: &'x ItipSnapshots<'x>, comp: &'x ItipSnapshot<'x>, dt_stamp: &'x PartialDateTime, email_rcpt: &mut AHashSet<&'x str>, skip_needs_action: bool, ) -> Option<(ICalendarComponent, &'x Email)> { let component = comp.comp; let mut cancel_comp = ICalendarComponent { component_type: component.component_type.clone(), entries: Vec::with_capacity(5), component_ids: vec![], }; let mut local_attendee = None; let mut delegated_from = None; for attendee in &comp.attendees { if attendee.email.is_local { if attendee.is_server_scheduling && attendee.rsvp.is_none_or(|rsvp| rsvp) && match attendee.part_stat { Some( ICalendarParticipationStatus::Declined | ICalendarParticipationStatus::Delegated, ) => attendee.force_send.is_some(), Some(ICalendarParticipationStatus::NeedsAction) => !skip_needs_action, _ => true, } { local_attendee = Some(attendee); } } else if attendee.delegated_to.iter().any(|d| d.is_local) { cancel_comp .entries .push(component.entries[attendee.entry_id as usize].clone()); delegated_from = Some(&attendee.email.email); } } local_attendee.map(|local_attendee| { cancel_comp.add_property( ICalendarProperty::Organizer, ICalendarValue::Text(itip.organizer.email.to_string()), ); cancel_comp.add_property_with_params( ICalendarProperty::Attendee, [ICalendarParameter::partstat( ICalendarParticipationStatus::Declined, )], ICalendarValue::Text(local_attendee.email.to_string()), ); cancel_comp.add_uid(itip.uid); cancel_comp.add_dtstamp(dt_stamp.clone()); cancel_comp.add_sequence(comp.sequence.unwrap_or_default()); cancel_comp.entries.extend( component .entries .iter() .filter(|e| { matches!( e.name, ICalendarProperty::Dtstart | ICalendarProperty::Dtend | ICalendarProperty::Duration | ICalendarProperty::Due | ICalendarProperty::Description | ICalendarProperty::Summary ) }) .cloned(), ); if let InstanceId::Recurrence(recurrence_id) = instance_id { cancel_comp .entries .push(component.entries[recurrence_id.entry_id as usize].clone()); } if let Some(delegated_from) = delegated_from { email_rcpt.insert(delegated_from); } (cancel_comp, &local_attendee.email) }) } ================================================ FILE: crates/groupware/src/scheduling/event_cancel.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::scheduling::{ InstanceId, ItipError, ItipMessage, ItipSummary, attendee::attendee_decline, itip::{itip_add_tz, itip_build_envelope}, snapshot::itip_snapshot, }; use ahash::AHashSet; use calcard::{ common::PartialDateTime, icalendar::{ ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarMethod, ICalendarParticipationStatus, ICalendarProperty, ICalendarStatus, ICalendarValue, }, }; pub fn itip_cancel( ical: &ICalendar, account_emails: &[String], is_deletion: bool, ) -> Result, ItipError> { // Prepare iTIP message let itip = itip_snapshot(ical, account_emails, false)?; let dt_stamp = PartialDateTime::now(); let mut message = ICalendar { components: Vec::with_capacity(2), }; if itip.organizer.email.is_local { // Send cancel message let mut comp = itip_build_envelope(ICalendarMethod::Cancel); comp.component_ids.push(1); message.components.push(comp); // Fetch guest emails let mut recipients = AHashSet::new(); let mut cancel_guests = AHashSet::new(); let mut component_type = &ICalendarComponentType::VEvent; let mut sequence = 0; for (instance_id, comp) in &itip.components { component_type = &comp.comp.component_type; for attendee in &comp.attendees { if attendee.send_update_messages() { recipients.insert(attendee.email.email.clone()); } cancel_guests.insert(&attendee.email); } // Increment sequence if needed if instance_id == &InstanceId::Main { sequence = comp.sequence.unwrap_or_default() + 1; } } if !recipients.is_empty() && component_type != &ICalendarComponentType::VFreebusy { let instance = itip.main_instance_or_default(); message.components.push(build_cancel_component( instance.comp, sequence, dt_stamp, &[], )); // Add timezones itip_add_tz(&mut message, ical); Ok(ItipMessage { to: recipients.into_iter().collect(), summary: ItipSummary::Cancel(instance.build_summary(None, &[])), from: itip.organizer.email.email, from_organizer: true, message, }) } else { Err(ItipError::NothingToSend) } } else { // Send decline message message .components .push(itip_build_envelope(ICalendarMethod::Reply)); // Decline attendance for all instances that have local attendees let mut mail_from = None; let mut email_rcpt = AHashSet::new(); for (instance_id, comp) in &itip.components { if let Some((cancel_comp, attendee_email)) = attendee_decline( instance_id, &itip, comp, &dt_stamp, &mut email_rcpt, is_deletion, ) { // Add cancel component let comp_id = message.components.len() as u32; message.components[0].component_ids.push(comp_id); message.components.push(cancel_comp); mail_from = Some(&attendee_email.email); } } if let Some(from) = mail_from { // Add timezone information if needed itip_add_tz(&mut message, ical); email_rcpt.insert(&itip.organizer.email.email); Ok(ItipMessage { from: from.to_string(), from_organizer: false, to: email_rcpt.into_iter().map(|e| e.to_string()).collect(), summary: ItipSummary::Rsvp { part_stat: ICalendarParticipationStatus::Declined, current: itip.main_instance_or_default().build_summary(None, &[]), }, message, }) } else { Err(ItipError::NothingToSend) } } } pub(crate) fn build_cancel_component( component: &ICalendarComponent, sequence: i64, dt_stamp: PartialDateTime, attendees: &[&str], ) -> ICalendarComponent { let mut cancel_comp = ICalendarComponent { component_type: component.component_type.clone(), entries: Vec::with_capacity(7), component_ids: vec![], }; cancel_comp.add_property( ICalendarProperty::Status, ICalendarValue::Status(ICalendarStatus::Cancelled), ); cancel_comp.add_dtstamp(dt_stamp); cancel_comp.add_sequence(sequence); cancel_comp.entries.extend( component .entries .iter() .filter(|e| match e.name { ICalendarProperty::Organizer | ICalendarProperty::Uid | ICalendarProperty::Summary | ICalendarProperty::Dtstart | ICalendarProperty::Dtend | ICalendarProperty::Duration | ICalendarProperty::Due | ICalendarProperty::RecurrenceId | ICalendarProperty::Created | ICalendarProperty::LastModified | ICalendarProperty::Description | ICalendarProperty::Location => true, ICalendarProperty::Attendee => { attendees.is_empty() || e.values .first() .and_then(|v| v.as_text()) .is_some_and(|email| { attendees.iter().any(|attendee| { email .strip_suffix(attendee) .is_some_and(|v| v.ends_with(':') || v.is_empty()) }) }) } _ => false, }) .cloned(), ); cancel_comp } ================================================ FILE: crates/groupware/src/scheduling/event_create.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::scheduling::{ ItipError, ItipMessage, itip::itip_finalize, organizer::organizer_request_full, snapshot::itip_snapshot, }; use calcard::icalendar::ICalendar; pub fn itip_create( ical: &mut ICalendar, account_emails: &[String], ) -> Result>, ItipError> { let itip = itip_snapshot(ical, account_emails, false)?; if !itip.organizer.is_server_scheduling { Err(ItipError::OtherSchedulingAgent) } else if !itip.organizer.email.is_local { Err(ItipError::NotOrganizer) } else { let mut sequences = Vec::new(); organizer_request_full(ical, &itip, Some(&mut sequences), true).inspect(|_| { itip_finalize(ical, &sequences); }) } } ================================================ FILE: crates/groupware/src/scheduling/event_update.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::scheduling::{ ItipError, ItipMessage, attendee::attendee_handle_update, event_cancel::itip_cancel, itip::itip_finalize, organizer::organizer_handle_update, snapshot::itip_snapshot, }; use calcard::icalendar::ICalendar; pub fn itip_update( ical: &mut ICalendar, old_ical: &ICalendar, account_emails: &[String], ) -> Result>, ItipError> { let old_itip = itip_snapshot(old_ical, account_emails, false)?; match itip_snapshot(ical, account_emails, false) { Ok(new_itip) => { let mut sequences = Vec::new(); if old_itip.organizer.email != new_itip.organizer.email { // RFC 6638 does not support replacing the organizer Err(ItipError::OrganizerMismatch) } else if old_itip.organizer.email.is_local { organizer_handle_update(old_ical, ical, old_itip, new_itip, &mut sequences) } else { attendee_handle_update(ical, old_itip, new_itip) } .inspect(|_| { itip_finalize(ical, &sequences); }) } Err(err) => { match &err { ItipError::NoSchedulingInfo | ItipError::NotOrganizer | ItipError::NotOrganizerNorAttendee | ItipError::OtherSchedulingAgent => { if old_itip.organizer.email.is_local { // RFC 6638 does not support replacing the organizer, so we cancel the event itip_cancel(old_ical, account_emails, false).map(|message| vec![message]) } else { Err(ItipError::CannotModifyAddress) } } _ => Err(err), } } } } ================================================ FILE: crates/groupware/src/scheduling/inbound.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::scheduling::{ InstanceId, ItipError, ItipMessage, ItipSnapshots, organizer::organizer_request_full, }; use ahash::AHashSet; use calcard::icalendar::{ ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarEntry, ICalendarMethod, ICalendarParameter, ICalendarParameterName, ICalendarProperty, ICalendarStatus, ICalendarValue, Uri, }; #[derive(Debug)] pub enum MergeAction { AddEntries { component_id: u16, entries: Vec, }, RemoveEntries { component_id: u16, entries: AHashSet, }, AddParameters { component_id: u16, entry_id: u16, parameters: Vec, }, RemoveParameters { component_id: u16, entry_id: u16, parameters: Vec, }, AddComponent { component: ICalendarComponent, }, RemoveComponent { component_id: u16, }, } pub enum MergeResult { Actions(Vec), Message(ItipMessage), None, } pub fn itip_process_message( ical: &ICalendar, snapshots: ItipSnapshots<'_>, itip: &ICalendar, itip_snapshots: ItipSnapshots<'_>, sender: String, ) -> Result { if snapshots.organizer.email != itip_snapshots.organizer.email { return Err(ItipError::OrganizerMismatch); } let method = itip_method(itip)?; let mut merge_actions = Vec::new(); if snapshots.organizer.email.is_local { // Handle attendee updates if snapshots.organizer.email.email == sender { return Err(ItipError::OrganizerIsLocalAddress); } match method { ICalendarMethod::Reply => { handle_reply(&snapshots, &itip_snapshots, &sender, &mut merge_actions)?; } ICalendarMethod::Refresh => { return organizer_request_full(ical, &snapshots, None, false).and_then( |messages| { messages .into_iter() .next() .map(|mut message| { message.to = vec![sender]; MergeResult::Message(message) }) .ok_or(ItipError::NothingToSend) }, ); } _ => return Err(ItipError::UnsupportedMethod(method.clone())), } } else { // Handle organizer and attendees updates match method { ICalendarMethod::Request => { let mut is_full_update = false; for (instance_id, itip_snapshot) in &itip_snapshots.components { is_full_update = is_full_update || instance_id == &InstanceId::Main; let itip_component = &itip.components[itip_snapshot.comp_id as usize]; if let Some(snapshot) = snapshots.components.get(instance_id) { // Merge instances if itip_snapshot.sequence.unwrap_or_default() >= snapshot.sequence.unwrap_or_default() { let mut changed_entries = itip_snapshot .entries .symmetric_difference(&snapshot.entries) .map(|entry| entry.name.clone()) .collect::>(); if itip_snapshot.attendees != snapshot.attendees { changed_entries.insert(ICalendarProperty::Attendee); } if itip_snapshot.dtstamp.is_some() && itip_snapshot.dtstamp != snapshot.dtstamp { changed_entries.insert(ICalendarProperty::Dtstamp); } changed_entries.insert(ICalendarProperty::Sequence); if !changed_entries.is_empty() { let entries = itip_component .entries .iter() .filter(|entry| changed_entries.contains(&entry.name)) .cloned() .collect(); merge_actions.push(MergeAction::RemoveEntries { component_id: snapshot.comp_id, entries: changed_entries, }); merge_actions.push(MergeAction::AddEntries { component_id: snapshot.comp_id, entries, }); } } else { return Err(ItipError::OutOfSequence); } } else { // Add instance merge_actions.push(MergeAction::AddComponent { component: ICalendarComponent { component_type: itip_component.component_type.clone(), entries: itip_component .entries .iter() .filter(|entry| { !matches!(entry.name, ICalendarProperty::Other(_)) }) .cloned() .collect(), component_ids: vec![], }, }); } } if is_full_update { for (instance_id, snapshot) in &snapshots.components { if !itip_snapshots.components.contains_key(instance_id) { // Remove instance merge_actions.push(MergeAction::RemoveComponent { component_id: snapshot.comp_id, }); } } } } ICalendarMethod::Add => { for (instance_id, itip_snapshot) in &itip_snapshots.components { if !snapshots.components.contains_key(instance_id) { let itip_component = &itip.components[itip_snapshot.comp_id as usize]; merge_actions.push(MergeAction::AddComponent { component: ICalendarComponent { component_type: itip_component.component_type.clone(), entries: itip_component .entries .iter() .filter(|entry| { !matches!(entry.name, ICalendarProperty::Other(_)) }) .cloned() .collect(), component_ids: vec![], }, }); } } } ICalendarMethod::Cancel => { let mut cancel_all_instances = false; for (instance_id, itip_snapshot) in &itip_snapshots.components { if let Some(snapshot) = snapshots.components.get(instance_id) { if itip_snapshot.sequence.unwrap_or_default() >= snapshot.sequence.unwrap_or_default() { // Cancel instance let itip_component = itip_snapshot.comp; merge_actions.push(MergeAction::RemoveEntries { component_id: snapshot.comp_id, entries: [ ICalendarProperty::Organizer, ICalendarProperty::Attendee, ICalendarProperty::Status, ICalendarProperty::Sequence, ] .into_iter() .collect(), }); merge_actions.push(MergeAction::AddEntries { component_id: snapshot.comp_id, entries: itip_component .entries .iter() .filter(|entry| { matches!( entry.name, ICalendarProperty::Organizer | ICalendarProperty::Attendee ) }) .cloned() .chain([ICalendarEntry { name: ICalendarProperty::Status, params: vec![], values: vec![ICalendarValue::Status( ICalendarStatus::Cancelled, )], }]) .collect(), }); cancel_all_instances = cancel_all_instances || instance_id == &InstanceId::Main; } else { return Err(ItipError::OutOfSequence); } } else { let itip_component = itip_snapshot.comp; merge_actions.push(MergeAction::AddComponent { component: ICalendarComponent { component_type: itip_component.component_type.clone(), entries: itip_component .entries .iter() .filter(|entry| { !matches!( entry.name, ICalendarProperty::Status | ICalendarProperty::Other(_) ) }) .cloned() .chain([ICalendarEntry { name: ICalendarProperty::Status, params: vec![], values: vec![ICalendarValue::Status( ICalendarStatus::Cancelled, )], }]) .collect(), component_ids: vec![], }, }); } } if cancel_all_instances { // Remove all instances let itip_main = itip_snapshots.components.get(&InstanceId::Main).unwrap(); let itip_component = itip_main.comp; for (instance_id, snapshot) in &snapshots.components { if !itip_snapshots.components.contains_key(instance_id) { merge_actions.push(MergeAction::RemoveEntries { component_id: snapshot.comp_id, entries: [ ICalendarProperty::Organizer, ICalendarProperty::Attendee, ICalendarProperty::Status, ] .into_iter() .collect(), }); merge_actions.push(MergeAction::AddEntries { component_id: snapshot.comp_id, entries: itip_component .entries .iter() .filter(|entry| { matches!( entry.name, ICalendarProperty::Organizer | ICalendarProperty::Attendee ) }) .cloned() .chain([ICalendarEntry { name: ICalendarProperty::Status, params: vec![], values: vec![ICalendarValue::Status( ICalendarStatus::Cancelled, )], }]) .collect(), }); } } } } ICalendarMethod::Reply if itip_snapshots.components.values().any(|snapshot| { snapshot.external_attendees().any(|a| { a.email.email == sender && a.delegated_from.iter().any(|a| a.is_local) }) }) => { handle_reply(&snapshots, &itip_snapshots, &sender, &mut merge_actions)?; } _ => return Err(ItipError::UnsupportedMethod(method.clone())), } } if !merge_actions.is_empty() { Ok(MergeResult::Actions(merge_actions)) } else { Ok(MergeResult::None) } } pub fn itip_import_message(ical: &mut ICalendar) -> Result<(), ItipError> { let mut expect_object_type = None; for comp in ical.components.iter_mut() { if comp.component_type.is_scheduling_object() { match expect_object_type { Some(expected) if expected != &comp.component_type => { return Err(ItipError::MultipleObjectTypes); } None => { expect_object_type = Some(&comp.component_type); } _ => {} } } else if comp.component_type == ICalendarComponentType::VCalendar { comp.entries .retain(|entry| !matches!(entry.name, ICalendarProperty::Method)); } } Ok(()) } fn handle_reply( snapshots: &ItipSnapshots<'_>, itip_snapshots: &ItipSnapshots<'_>, sender: &str, merge_actions: &mut Vec, ) -> Result<(), ItipError> { for (instance_id, itip_snapshot) in &itip_snapshots.components { if let Some(snapshot) = snapshots.components.get(instance_id) { if let (Some(attendee), Some(updated_attendee)) = ( snapshot.attendee_by_email(sender), itip_snapshot.attendee_by_email(sender), ) { let itip_component = itip_snapshot.comp; let changed_part_stat = attendee.part_stat != updated_attendee.part_stat; let changed_rsvp = attendee.rsvp != updated_attendee.rsvp; let changed_delegated_to = attendee.delegated_to != updated_attendee.delegated_to; let has_request_status = !itip_snapshot.request_status.is_empty(); if changed_part_stat || changed_rsvp || changed_delegated_to || has_request_status { // Update participant status let mut add_parameters = Vec::new(); let mut remove_parameters = Vec::new(); if changed_part_stat { remove_parameters.push(ICalendarParameterName::Partstat); if let Some(part_stat) = updated_attendee.part_stat { add_parameters.push(ICalendarParameter::partstat(part_stat.clone())); } } if changed_rsvp { remove_parameters.push(ICalendarParameterName::Rsvp); if let Some(rsvp) = updated_attendee.rsvp { add_parameters.push(ICalendarParameter::rsvp(rsvp)); } } if changed_delegated_to { remove_parameters.push(ICalendarParameterName::DelegatedTo); if !updated_attendee.delegated_to.is_empty() { add_parameters.extend(updated_attendee.delegated_to.iter().map( |email| { ICalendarParameter::delegated_to(Uri::Location( email.to_string(), )) }, )); } } if has_request_status { remove_parameters.push(ICalendarParameterName::ScheduleStatus); add_parameters.push(ICalendarParameter::schedule_status( itip_snapshot.request_status.join(","), )); } merge_actions.push(MergeAction::RemoveParameters { component_id: snapshot.comp_id, entry_id: attendee.entry_id, parameters: remove_parameters, }); merge_actions.push(MergeAction::AddParameters { component_id: snapshot.comp_id, entry_id: attendee.entry_id, parameters: add_parameters, }); // Add unknown delegated attendees for delegated_to in &updated_attendee.delegated_to { if let Some(itip_delegated) = itip_snapshot.attendee_by_email(&delegated_to.email) { if let Some(delegated) = snapshot.attendee_by_email(&delegated_to.email) { if delegated != itip_delegated { merge_actions.push(MergeAction::RemoveParameters { component_id: snapshot.comp_id, entry_id: delegated.entry_id, parameters: vec![ ICalendarParameterName::DelegatedTo, ICalendarParameterName::DelegatedFrom, ICalendarParameterName::Partstat, ICalendarParameterName::Rsvp, ICalendarParameterName::ScheduleStatus, ICalendarParameterName::Role, ], }); merge_actions.push(MergeAction::AddParameters { component_id: snapshot.comp_id, entry_id: delegated.entry_id, parameters: itip_component.entries [itip_delegated.entry_id as usize] .params .iter() .filter(|param| { matches!( param.name, ICalendarParameterName::DelegatedTo | ICalendarParameterName::DelegatedFrom | ICalendarParameterName::Partstat | ICalendarParameterName::Rsvp | ICalendarParameterName::ScheduleStatus | ICalendarParameterName::Role ) }) .cloned() .collect(), }); } } else { merge_actions.push(MergeAction::AddEntries { component_id: snapshot.comp_id, entries: vec![ itip_component.entries[itip_delegated.entry_id as usize] .clone(), ], }); } } } } // Add changed properties for VTODO if snapshot.comp.component_type == ICalendarComponentType::VTodo { let mut remove_entries = AHashSet::new(); let mut add_entries = Vec::new(); for entry in itip_component.entries.iter() { if matches!( entry.name, ICalendarProperty::PercentComplete | ICalendarProperty::Status | ICalendarProperty::Completed ) { remove_entries.insert(entry.name.clone()); add_entries.push(entry.clone()); } } if !add_entries.is_empty() { merge_actions.push(MergeAction::RemoveEntries { component_id: snapshot.comp_id, entries: remove_entries, }); merge_actions.push(MergeAction::AddEntries { component_id: snapshot.comp_id, entries: add_entries, }); } } } else { return Err(ItipError::SenderIsNotParticipant(sender.to_string())); } } else if itip_snapshot.attendee_by_email(sender).is_some() { // Add component let itip_component = itip_snapshot.comp; let is_todo = itip_component.component_type == ICalendarComponentType::VTodo; merge_actions.push(MergeAction::AddComponent { component: ICalendarComponent { component_type: itip_component.component_type.clone(), entries: itip_component .entries .iter() .filter(|entry| { matches!( entry.name, ICalendarProperty::Organizer | ICalendarProperty::Attendee | ICalendarProperty::Uid | ICalendarProperty::Dtstamp | ICalendarProperty::Sequence | ICalendarProperty::RecurrenceId ) || (is_todo && matches!( entry.name, ICalendarProperty::PercentComplete | ICalendarProperty::Status | ICalendarProperty::Completed )) }) .cloned() .collect(), component_ids: vec![], }, }); } else { return Err(ItipError::SenderIsNotParticipant(sender.to_string())); } } Ok(()) } pub fn itip_merge_changes(ical: &mut ICalendar, changes: Vec) { let mut remove_component_ids: Vec = Vec::new(); for action in changes { match action { MergeAction::AddEntries { component_id, entries, } => { let component = &mut ical.components[component_id as usize]; component.entries.extend(entries); } MergeAction::RemoveEntries { component_id, entries, } => { let component = &mut ical.components[component_id as usize]; component .entries .retain(|entry| !entries.contains(&entry.name)); } MergeAction::AddParameters { component_id, entry_id, parameters, } => { ical.components[component_id as usize].entries[entry_id as usize] .params .extend(parameters); } MergeAction::RemoveParameters { component_id, entry_id, parameters, } => { ical.components[component_id as usize].entries[entry_id as usize] .params .retain(|param| !parameters.contains(¶m.name)); } MergeAction::AddComponent { component } => { let comp_id = ical.components.len() as u32; if let Some(root) = ical .components .get_mut(0) .filter(|c| c.component_type == ICalendarComponentType::VCalendar) { root.component_ids.push(comp_id); ical.components.push(component); } } MergeAction::RemoveComponent { component_id } => { remove_component_ids.push(component_id as u32); } } } if !remove_component_ids.is_empty() { ical.remove_component_ids(&remove_component_ids); } } pub fn itip_method(ical: &ICalendar) -> Result<&ICalendarMethod, ItipError> { ical.components .first() .and_then(|comp| { comp.entries.iter().find_map(|entry| { if entry.name == ICalendarProperty::Method { entry.values.first().and_then(|value| { if let ICalendarValue::Method(method) = value { Some(method) } else { None } }) } else { None } }) }) .ok_or(ItipError::MissingMethod) } ================================================ FILE: crates/groupware/src/scheduling/itip.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::scheduling::{ArchivedItipSummary, ItipMessage, ItipMessages}; use calcard::{ common::{IanaString, PartialDateTime}, icalendar::{ ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarEntry, ICalendarMethod, ICalendarParameter, ICalendarParameterName, ICalendarParameterValue, ICalendarParticipationStatus, ICalendarProperty, ICalendarValue, }, }; use common::PROD_ID; use store::{ Serialize, write::{Archiver, BatchBuilder, TaskEpoch, TaskQueueClass, ValueClass}, }; use trc::AddContext; pub(crate) fn itip_build_envelope(method: ICalendarMethod) -> ICalendarComponent { ICalendarComponent { component_type: ICalendarComponentType::VCalendar, entries: vec![ ICalendarEntry { name: ICalendarProperty::Version, params: vec![], values: vec![ICalendarValue::Text("2.0".to_string())], }, ICalendarEntry { name: ICalendarProperty::Prodid, params: vec![], values: vec![ICalendarValue::Text(PROD_ID.to_string())], }, ICalendarEntry { name: ICalendarProperty::Method, params: vec![], values: vec![ICalendarValue::Method(method)], }, ], component_ids: Default::default(), } } pub(crate) enum ItipExportAs<'x> { Organizer(&'x ICalendarParticipationStatus), Attendee(Vec), } pub(crate) fn itip_export_component( component: &ICalendarComponent, uid: &str, dt_stamp: &PartialDateTime, sequence: i64, export_as: ItipExportAs<'_>, ) -> ICalendarComponent { let is_todo = component.component_type == ICalendarComponentType::VTodo; let mut comp = ICalendarComponent { component_type: component.component_type.clone(), entries: Vec::with_capacity(component.entries.len() + 1), component_ids: Default::default(), }; comp.add_dtstamp(dt_stamp.clone()); comp.add_sequence(sequence); comp.add_uid(uid); for (entry_id, entry) in component.entries.iter().enumerate() { match (&entry.name, &export_as) { ( ICalendarProperty::Organizer | ICalendarProperty::Attendee, ItipExportAs::Organizer(partstat), ) => { let mut new_entry = ICalendarEntry { name: entry.name.clone(), params: Vec::with_capacity(entry.params.len()), values: entry.values.clone(), }; let mut has_partstat = false; let mut rsvp = true; for entry in &entry.params { match &entry.name { ICalendarParameterName::ScheduleStatus | ICalendarParameterName::ScheduleAgent | ICalendarParameterName::ScheduleForceSend => {} _ => { match &entry.name { ICalendarParameterName::Rsvp => { rsvp = !matches!( entry.value, ICalendarParameterValue::Bool(false) ); } ICalendarParameterName::Partstat => { has_partstat = true; } _ => {} } new_entry.params.push(entry.clone()) } } } if !has_partstat && rsvp && entry.name == ICalendarProperty::Attendee { new_entry .params .push(ICalendarParameter::partstat((*partstat).clone())); } comp.entries.push(new_entry); } ( ICalendarProperty::Organizer | ICalendarProperty::Attendee, ItipExportAs::Attendee(attendee_entry_ids), ) => { if attendee_entry_ids.contains(&(entry_id as u16)) || entry.name == ICalendarProperty::Organizer { comp.entries.push(ICalendarEntry { name: entry.name.clone(), params: entry .params .iter() .filter(|param| { !matches!( ¶m.name, ICalendarParameterName::ScheduleStatus | ICalendarParameterName::ScheduleAgent | ICalendarParameterName::ScheduleForceSend ) }) .cloned() .collect(), values: entry.values.clone(), }); } } ( ICalendarProperty::RequestStatus | ICalendarProperty::Dtstamp | ICalendarProperty::Sequence | ICalendarProperty::Uid, _, ) => {} (_, ItipExportAs::Organizer(_)) | ( ICalendarProperty::RecurrenceId | ICalendarProperty::Dtstart | ICalendarProperty::Dtend | ICalendarProperty::Duration | ICalendarProperty::Due | ICalendarProperty::Description | ICalendarProperty::Summary, _, ) => { comp.entries.push(entry.clone()); } ( ICalendarProperty::Status | ICalendarProperty::PercentComplete | ICalendarProperty::Completed, _, ) if is_todo => { comp.entries.push(entry.clone()); } _ => {} } } if matches!(export_as, ItipExportAs::Attendee(_)) { comp.entries.push(ICalendarEntry { name: ICalendarProperty::RequestStatus, params: vec![], values: vec![ ICalendarValue::Text("2.0".to_string()), ICalendarValue::Text("Success".to_string()), ], }); } comp } pub(crate) fn itip_finalize(ical: &mut ICalendar, scheduling_object_ids: &[u16]) { for comp in ical.components.iter_mut() { if comp.component_type.is_scheduling_object() { // Remove scheduling info from non-updated components for entry in comp.entries.iter_mut() { if matches!( entry.name, ICalendarProperty::Organizer | ICalendarProperty::Attendee ) { entry.params.retain(|param| { !matches!(param.name, ICalendarParameterName::ScheduleForceSend) }); } } } } for comp_id in scheduling_object_ids { let comp = &mut ical.components[*comp_id as usize]; let mut found_sequence = false; for entry in &mut comp.entries { if entry.name == ICalendarProperty::Sequence { if let Some(ICalendarValue::Integer(seq)) = entry.values.first_mut() { *seq += 1; } else { entry.values = vec![ICalendarValue::Integer(1)]; } found_sequence = true; break; } } if !found_sequence { comp.add_sequence(1); } } } pub(crate) fn itip_add_tz(message: &mut ICalendar, ical: &ICalendar) { let mut has_timezones = false; if message.components.iter().any(|c| { has_timezones = has_timezones || c.component_type == ICalendarComponentType::VTimezone; !has_timezones && c.entries.iter().any(|e| { e.params .iter() .any(|p| matches!(p.name, ICalendarParameterName::Tzid)) }) }) && !has_timezones { message.copy_timezones(ical); } } #[inline] pub(crate) fn can_attendee_modify_property( component_type: &ICalendarComponentType, property: &ICalendarProperty, ) -> bool { match component_type { ICalendarComponentType::VEvent | ICalendarComponentType::VJournal => { matches!( property, ICalendarProperty::Exdate | ICalendarProperty::Summary | ICalendarProperty::Description | ICalendarProperty::Comment ) } ICalendarComponentType::VTodo => matches!( property, ICalendarProperty::Exdate | ICalendarProperty::Summary | ICalendarProperty::Description | ICalendarProperty::Status | ICalendarProperty::PercentComplete | ICalendarProperty::Completed | ICalendarProperty::Comment ), _ => false, } } impl ItipMessages { pub fn new(messages: Vec>) -> Self { ItipMessages { messages: messages.into_iter().map(|m| m.into()).collect(), } } pub fn queue(self, batch: &mut BatchBuilder) -> trc::Result<()> { let due = TaskEpoch::now().with_random_sequence_id(); batch.set( ValueClass::TaskQueue(TaskQueueClass::SendImip { due, is_payload: false, }), vec![], ); batch.set( ValueClass::TaskQueue(TaskQueueClass::SendImip { due, is_payload: true, }), Archiver::new(self) .serialize() .caused_by(trc::location!())?, ); Ok(()) } } impl From> for ItipMessage { fn from(message: ItipMessage) -> Self { ItipMessage { from: message.from, from_organizer: message.from_organizer, to: message.to, summary: message.summary, message: message.message.to_string(), } } } impl ArchivedItipSummary { pub fn method(&self) -> &str { match self { ArchivedItipSummary::Invite(_) => ICalendarMethod::Request.as_str(), ArchivedItipSummary::Update { method, .. } => method.as_str(), ArchivedItipSummary::Cancel(_) => ICalendarMethod::Cancel.as_str(), ArchivedItipSummary::Rsvp { .. } => ICalendarMethod::Reply.as_str(), } } } ================================================ FILE: crates/groupware/src/scheduling/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use ahash::{AHashMap, AHashSet}; use calcard::{ common::{IanaString, PartialDateTime}, icalendar::{ ICalendarComponent, ICalendarDuration, ICalendarEntry, ICalendarMethod, ICalendarParameter, ICalendarParticipationRole, ICalendarParticipationStatus, ICalendarPeriod, ICalendarProperty, ICalendarRecurrenceRule, ICalendarScheduleForceSendValue, ICalendarStatus, ICalendarUserTypes, ICalendarValue, Uri, }, }; use std::{fmt::Display, hash::Hash}; pub mod attendee; pub mod event_cancel; pub mod event_create; pub mod event_update; pub mod inbound; pub mod itip; pub mod organizer; pub mod snapshot; #[derive(Debug)] pub struct ItipSnapshots<'x> { pub organizer: Organizer<'x>, pub uid: &'x str, pub components: AHashMap>, } #[derive(Debug)] pub struct ItipSnapshot<'x> { pub comp_id: u16, pub comp: &'x ICalendarComponent, pub attendees: AHashSet>, pub dtstamp: Option<&'x PartialDateTime>, pub entries: AHashSet>, pub sequence: Option, pub request_status: Vec<&'x str>, } #[derive(Debug, PartialEq, Eq, Hash)] pub struct ItipEntry<'x> { pub name: &'x ICalendarProperty, pub value: ItipEntryValue<'x>, } #[derive(Debug, PartialEq, Eq, Hash)] pub enum ItipEntryValue<'x> { DateTime(ItipDateTime<'x>), Period(&'x ICalendarPeriod), Duration(&'x ICalendarDuration), Status(&'x ICalendarStatus), RRule(&'x ICalendarRecurrenceRule), Text(&'x str), Integer(i64), } #[derive(Debug)] pub struct ItipDateTime<'x> { pub date: &'x PartialDateTime, pub tz_id: Option<&'x str>, pub tz_code: u16, pub timestamp: i64, } #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum InstanceId { Main, Recurrence(RecurrenceId), } #[derive(Debug, PartialOrd, Ord)] pub struct RecurrenceId { pub entry_id: u16, pub date: i64, pub this_and_future: bool, } #[derive(Debug)] pub struct Attendee<'x> { pub entry_id: u16, pub email: Email, pub name: Option<&'x str>, pub part_stat: Option<&'x ICalendarParticipationStatus>, pub delegated_from: Vec, pub delegated_to: Vec, pub role: Option<&'x ICalendarParticipationRole>, pub cu_type: Option<&'x ICalendarUserTypes>, pub sent_by: Option, pub rsvp: Option, pub is_server_scheduling: bool, pub force_send: Option<&'x ICalendarScheduleForceSendValue>, } #[derive(Debug)] pub struct Organizer<'x> { pub entry_id: u16, pub email: Email, pub name: Option<&'x str>, pub is_server_scheduling: bool, pub force_send: Option<&'x ICalendarScheduleForceSendValue>, } #[derive(Debug)] pub struct Email { pub email: String, pub is_local: bool, } #[derive(Debug)] pub enum ItipError { NoSchedulingInfo, OtherSchedulingAgent, NotOrganizer, NotOrganizerNorAttendee, NothingToSend, MissingUid, MultipleUid, MultipleOrganizer, MultipleObjectTypes, MultipleObjectInstances, CannotModifyProperty(ICalendarProperty), CannotModifyInstance, CannotModifyAddress, OrganizerMismatch, MissingMethod, InvalidComponentType, OutOfSequence, OrganizerIsLocalAddress, SenderIsNotOrganizerNorAttendee, SenderIsNotParticipant(String), UnknownParticipant(String), UnsupportedMethod(ICalendarMethod), ICalendarParseError, EventNotFound, EventTooLarge, QuotaExceeded, NoDefaultCalendar, AutoAddDisabled, } #[derive(Debug, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)] pub struct ItipMessage { pub from: String, pub from_organizer: bool, pub to: Vec, pub summary: ItipSummary, pub message: T, } #[derive(Debug, Clone, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)] pub enum ItipSummary { Invite(Vec), Update { method: ICalendarMethod, current: Vec, previous: Vec, }, Cancel(Vec), Rsvp { part_stat: ICalendarParticipationStatus, current: Vec, }, } #[derive(Debug, Clone, Hash, PartialEq, Eq, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)] pub struct ItipField { pub name: ICalendarProperty, pub value: ItipValue, } #[derive(Debug, Clone, Hash, PartialEq, Eq, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)] pub enum ItipValue { Text(String), Time(ItipTime), Rrule(Box), Participants(Vec), } #[derive(Debug, Clone, Hash, PartialEq, Eq, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)] pub struct ItipTime { pub start: i64, pub tz_id: u16, } #[derive(Debug, Clone, Hash, PartialEq, Eq, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)] pub struct ItipParticipant { pub email: String, pub name: Option, pub is_organizer: bool, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)] pub struct ItipMessages { pub messages: Vec>, } impl Attendee<'_> { pub fn send_invite_messages(&self) -> bool { !self.email.is_local && self.is_server_scheduling && self.rsvp.is_none_or(|rsvp| rsvp) && (self.force_send.is_some() || self.part_stat.is_none_or(|part_stat| { part_stat == &ICalendarParticipationStatus::NeedsAction })) } pub fn send_update_messages(&self) -> bool { !self.email.is_local && self.is_server_scheduling && self.rsvp.is_none_or(|rsvp| rsvp) && (self.force_send.is_some() || self .part_stat .is_none_or(|part_stat| part_stat != &ICalendarParticipationStatus::Declined)) } pub fn is_delegated_from(&self, attendee: &Attendee<'_>) -> bool { self.delegated_from .iter() .any(|d| d.email == attendee.email.email) } pub fn is_delegated_to(&self, attendee: &Attendee<'_>) -> bool { self.delegated_to .iter() .any(|d| d.email == attendee.email.email) } } impl Email { pub fn new(email: &str, local_addresses: &[String]) -> Option { email.contains('@').then(|| { let email = email.trim().trim_start_matches("mailto:").to_lowercase(); let is_local = local_addresses.contains(&email); Email { email, is_local } }) } pub fn from_uri(uri: &Uri, local_addresses: &[String]) -> Option { if let Uri::Location(uri) = uri { Email::new(uri.as_str(), local_addresses) } else { None } } } impl PartialEq for Attendee<'_> { fn eq(&self, other: &Self) -> bool { self.email == other.email && self.part_stat == other.part_stat && self.delegated_from == other.delegated_from && self.delegated_to == other.delegated_to && self.role == other.role && self.cu_type == other.cu_type && self.sent_by == other.sent_by } } impl Eq for Attendee<'_> {} impl Hash for Attendee<'_> { fn hash(&self, state: &mut H) { self.email.hash(state); self.part_stat.hash(state); self.delegated_from.hash(state); self.delegated_to.hash(state); self.role.hash(state); self.cu_type.hash(state); self.sent_by.hash(state); } } impl Display for Email { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "mailto:{}", self.email) } } impl Hash for Email { fn hash(&self, state: &mut H) { self.email.hash(state); } } impl PartialEq for Email { fn eq(&self, other: &Self) -> bool { self.email == other.email } } impl Eq for Email {} impl PartialEq for RecurrenceId { fn eq(&self, other: &Self) -> bool { self.date == other.date && self.this_and_future == other.this_and_future } } impl Eq for RecurrenceId {} impl Hash for RecurrenceId { fn hash(&self, state: &mut H) { self.date.hash(state); self.this_and_future.hash(state); } } impl PartialEq for ItipDateTime<'_> { fn eq(&self, other: &Self) -> bool { self.timestamp == other.timestamp } } impl Eq for ItipDateTime<'_> {} impl Hash for ItipDateTime<'_> { fn hash(&self, state: &mut H) { self.timestamp.hash(state); } } impl ItipDateTime<'_> { pub fn to_entry(&self, name: ICalendarProperty) -> ICalendarEntry { ICalendarEntry { name, params: self .tz_id .map(|tz_id| vec![ICalendarParameter::tzid(tz_id.to_string())]) .unwrap_or_default(), values: vec![ICalendarValue::PartialDateTime(Box::new(self.date.clone()))], } } } impl ItipError { pub fn is_jmap_error(&self) -> bool { matches!( self, ItipError::MultipleOrganizer | ItipError::OrganizerIsLocalAddress | ItipError::SenderIsNotParticipant(_) | ItipError::OrganizerMismatch | ItipError::CannotModifyProperty(_) | ItipError::CannotModifyInstance | ItipError::CannotModifyAddress //| ItipError::MissingUid | ItipError::MultipleUid | ItipError::MultipleObjectTypes | ItipError::MultipleObjectInstances | ItipError::MissingMethod | ItipError::InvalidComponentType | ItipError::OutOfSequence | ItipError::UnknownParticipant(_) | ItipError::UnsupportedMethod(_) ) } } impl Display for ItipError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { ItipError::NoSchedulingInfo => write!(f, "No scheduling information found"), ItipError::OtherSchedulingAgent => write!(f, "Other scheduling agent"), ItipError::NotOrganizer => write!(f, "Not the organizer of the event"), ItipError::NotOrganizerNorAttendee => write!(f, "Not an organizer or attendee"), ItipError::NothingToSend => write!(f, "No iTIP messages to send"), ItipError::MissingUid => write!(f, "Missing UID in iCalendar object"), ItipError::MultipleUid => write!(f, "Multiple UIDs found in iCalendar object"), ItipError::MultipleOrganizer => { write!(f, "Multiple organizers found in iCalendar object") } ItipError::MultipleObjectTypes => { write!(f, "Multiple object types found in iCalendar object") } ItipError::MultipleObjectInstances => { write!(f, "Multiple object instances found in iCalendar object") } ItipError::CannotModifyProperty(prop) => { write!(f, "Cannot modify property {}", prop.as_str()) } ItipError::CannotModifyInstance => write!(f, "Cannot modify instance of the event"), ItipError::CannotModifyAddress => write!(f, "Cannot modify address of the event"), ItipError::OrganizerMismatch => write!(f, "Organizer mismatch in iCalendar object"), ItipError::MissingMethod => write!(f, "Missing method in the iTIP message"), ItipError::InvalidComponentType => { write!(f, "Invalid component type in iCalendar object") } ItipError::OutOfSequence => write!(f, "Old sequence number found"), ItipError::OrganizerIsLocalAddress => { write!( f, "Organizer matches one of the recipient's account addresses" ) } ItipError::SenderIsNotParticipant(participant) => { write!(f, "Sender {participant:?} is not a participant") } ItipError::SenderIsNotOrganizerNorAttendee => { write!(f, "Sender is neither organizer nor attendee") } ItipError::UnknownParticipant(participant) => { write!(f, "Unknown participant: {}", participant) } ItipError::UnsupportedMethod(method) => { write!(f, "Unsupported method: {}", method.as_str()) } ItipError::ICalendarParseError => write!(f, "Failed to parse iCalendar object"), ItipError::EventNotFound => write!(f, "Event found in index but not in database"), ItipError::EventTooLarge => write!( f, "Applying the iTIP message would exceed the maximum event size" ), ItipError::QuotaExceeded => write!(f, "Quota exceeded"), ItipError::NoDefaultCalendar => write!(f, "No default calendar found for the account"), ItipError::AutoAddDisabled => { write!(f, "Auto-adding events is disabled for this account") } } } } ================================================ FILE: crates/groupware/src/scheduling/organizer.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::scheduling::{ InstanceId, ItipError, ItipMessage, ItipSnapshots, ItipSummary, event_cancel::build_cancel_component, itip::{ItipExportAs, itip_add_tz, itip_build_envelope, itip_export_component}, }; use ahash::{AHashMap, AHashSet}; use calcard::{ common::PartialDateTime, icalendar::{ ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarMethod, ICalendarParticipationStatus, ICalendarProperty, ICalendarStatus, }, }; use std::collections::hash_map::Entry; pub(crate) fn organizer_handle_update( old_ical: &ICalendar, new_ical: &ICalendar, old_itip: ItipSnapshots<'_>, new_itip: ItipSnapshots<'_>, increment_sequences: &mut Vec, ) -> Result>, ItipError> { let mut changed_instances: Vec<(&InstanceId, &str, &ICalendarMethod)> = Vec::new(); let mut increment_sequence = false; let mut changed_properties = AHashSet::new(); for (instance_id, instance) in &new_itip.components { if let Some(old_instance) = old_itip.components.get(instance_id) { let changed_entries = instance.entries != old_instance.entries; let changed_attendees = instance.attendees != old_instance.attendees; if changed_entries || changed_attendees { if changed_entries { for entry in instance.entries.symmetric_difference(&old_instance.entries) { increment_sequence = increment_sequence || matches!( entry.name, ICalendarProperty::Dtstart | ICalendarProperty::Dtend | ICalendarProperty::Duration | ICalendarProperty::Due | ICalendarProperty::Rrule | ICalendarProperty::Rdate | ICalendarProperty::Exdate | ICalendarProperty::Status | ICalendarProperty::Location ); changed_properties.insert(entry.name); } } if changed_attendees { changed_instances.extend( old_instance .external_attendees() .filter(|attendee| attendee.send_update_messages()) .map(|attendee| attendee.email.email.as_str()) .collect::>() .difference( &instance .external_attendees() .map(|attendee| attendee.email.email.as_str()) .collect::>(), ) .map(|attendee| (instance_id, *attendee, &ICalendarMethod::Cancel)), ); changed_properties.insert(&ICalendarProperty::Attendee); increment_sequence = true; } changed_instances.extend(instance.attendees.iter().filter_map(|attendee| { if attendee.send_update_messages() { Some(( instance_id, attendee.email.email.as_str(), &ICalendarMethod::Request, )) } else { None } })); } } else if instance_id != &InstanceId::Main { changed_properties.insert(&ICalendarProperty::Exdate); let method = if matches!(instance.comp.status(), Some(ICalendarStatus::Cancelled)) { &ICalendarMethod::Cancel } else { &ICalendarMethod::Add }; changed_instances.extend(instance.attendees.iter().filter_map(|attendee| { if attendee.send_invite_messages() { Some((instance_id, attendee.email.email.as_str(), method)) } else { None } })); increment_sequence = true; } else { return Err(ItipError::CannotModifyInstance); } } for (instance_id, old_instance) in &old_itip.components { if !new_itip.components.contains_key(instance_id) { if instance_id != &InstanceId::Main { changed_instances.extend(old_instance.attendees.iter().filter_map(|attendee| { if attendee.send_update_messages() { Some(( instance_id, attendee.email.email.as_str(), &ICalendarMethod::Cancel, )) } else { None } })); changed_properties.insert(&ICalendarProperty::Exdate); increment_sequence = true; } else { return Err(ItipError::CannotModifyInstance); } } } if changed_instances.is_empty() { return Err(ItipError::NothingToSend); } // Remove partial notifications for attendees that receive a full update for the main instance // or, that will receive both add and remove messages let mut send_full_update: AHashSet<&str> = AHashSet::new(); let mut send_partial_update: AHashMap<&str, AHashMap<&ICalendarMethod, Vec<&InstanceId>>> = AHashMap::new(); for (instance_id, email, method) in &changed_instances { if *instance_id == &InstanceId::Main && *method == &ICalendarMethod::Request { send_full_update.insert(*email); send_partial_update.remove(email); } else if !send_full_update.contains(email) { match send_partial_update.entry(email) { Entry::Occupied(mut entry) => { let entry = entry.get_mut(); let is_empty = entry.is_empty(); match entry.entry(method) { Entry::Occupied(mut method_entry) => { method_entry.get_mut().push(*instance_id); } Entry::Vacant(method_entry) if is_empty => { method_entry.insert(vec![*instance_id]); } _ => { // Switch to full update for this participant send_full_update.insert(*email); send_partial_update.remove(email); } } } Entry::Vacant(entry) => { entry.insert(AHashMap::from_iter([(*method, vec![*instance_id])])); } } } } // Build summary of changed properties let new_summary = new_itip .main_instance_or_default() .build_summary(Some(&new_itip.organizer), &[]); let old_summary = old_itip .main_instance_or_default() .build_summary(Some(&old_itip.organizer), &new_summary); // Prepare full updates let mut messages = Vec::new(); if !send_full_update.is_empty() { match organizer_request_full( new_ical, &new_itip, increment_sequence.then_some(increment_sequences), false, ) { Ok(messages_) => { for mut message in messages_ { message.summary = ItipSummary::Update { method: ICalendarMethod::Request, current: new_summary.clone(), previous: old_summary.clone(), }; messages.push(message); } } Err(err) => { if send_partial_update.is_empty() { return Err(err); } } } } // Prepare partial updates if !send_partial_update.is_empty() { // Group updates by email and method let mut updates: AHashMap<(&ICalendarMethod, Vec<&InstanceId>), Vec<&str>> = AHashMap::new(); for (email, partial_updates) in send_partial_update { for (method, mut instances) in partial_updates { instances.sort_unstable(); instances.dedup(); updates.entry((method, instances)).or_default().push(email); } } let dt_stamp = PartialDateTime::now(); for ((method, instances), emails) in updates { let (mut ical, mut itip, is_cancel) = if matches!(method, ICalendarMethod::Cancel) { (old_ical, &old_itip, true) } else { (new_ical, &new_itip, false) }; // Prepare iTIP message let mut message = ICalendar { components: Vec::with_capacity(instances.len() + 1), }; message.components.push(itip_build_envelope(method.clone())); let mut increment_sequences = Vec::new(); for instance_id in instances { let comp = match itip.components.get(instance_id) { Some(comp) => comp, None => { // New component added with CANCELLED status ical = new_ical; itip = &new_itip; itip.components.get(instance_id).unwrap() } }; // Prepare component for iTIP let sequence = if increment_sequence { comp.sequence.unwrap_or_default() + 1 } else { comp.sequence.unwrap_or_default() }; let orig_component = comp.comp; let component = if !is_cancel { if increment_sequence { increment_sequences.push(comp.comp_id); } // Export component with updated sequence and participation status itip_export_component( orig_component, itip.uid, &dt_stamp, sequence, ItipExportAs::Organizer(&ICalendarParticipationStatus::NeedsAction), ) } else { build_cancel_component(orig_component, sequence, dt_stamp.clone(), &emails) }; // Add component to message let comp_id = message.components.len() as u32; message.components.push(component); message.components[0].component_ids.push(comp_id); } // Add timezones itip_add_tz(&mut message, ical); messages.push(ItipMessage { from: itip.organizer.email.email.clone(), from_organizer: true, to: emails.into_iter().map(|e| e.to_string()).collect(), summary: if method == &ICalendarMethod::Cancel { ItipSummary::Cancel( new_summary .iter() .chain(old_summary.iter()) .map(|summary| (&summary.name, summary)) .collect::>() .into_values() .cloned() .collect(), ) } else { ItipSummary::Update { method: method.clone(), current: new_summary.clone(), previous: old_summary.clone(), } }, message, }); } } Ok(messages) } pub(crate) fn organizer_request_full( ical: &ICalendar, itip: &ItipSnapshots<'_>, mut increment_sequence: Option<&mut Vec>, is_first_request: bool, ) -> Result>, ItipError> { // Prepare iTIP message let dt_stamp = PartialDateTime::now(); let mut message = ICalendar { components: vec![ICalendarComponent::default(); ical.components.len()], }; message.components[0] = itip_build_envelope(ICalendarMethod::Request); let mut recipients = AHashSet::new(); let mut copy_components = AHashSet::new(); for comp in itip.components.values() { // Skip private components if comp.attendees.is_empty() { continue; } // Prepare component for iTIP let sequence = if let Some(increment_sequence) = &mut increment_sequence { increment_sequence.push(comp.comp_id); comp.sequence.unwrap_or_default() + 1 } else { comp.sequence.unwrap_or_default() }; let orig_component = &ical.components[comp.comp_id as usize]; let mut component = itip_export_component( orig_component, itip.uid, &dt_stamp, sequence, ItipExportAs::Organizer(&ICalendarParticipationStatus::NeedsAction), ); // Add VALARM sub-components if is_first_request { for sub_comp_id in &orig_component.component_ids { if matches!( ical.components[*sub_comp_id as usize].component_type, ICalendarComponentType::VAlarm ) { copy_components.insert(*sub_comp_id); component.component_ids.push(*sub_comp_id); } } } // Add component to message message.components[comp.comp_id as usize] = component; message.components[0] .component_ids .push(comp.comp_id as u32); // Add attendees for attendee in &comp.attendees { if (is_first_request && attendee.send_invite_messages()) || (!is_first_request && attendee.send_update_messages()) { recipients.insert(&attendee.email.email); } } } // Copy timezones and alarms for (comp_id, comp) in ical.components.iter().enumerate() { if matches!(comp.component_type, ICalendarComponentType::VTimezone) { copy_components.extend(comp.component_ids.iter().copied()); message.components[0].component_ids.push(comp_id as u32); } else if !copy_components.contains(&(comp_id as u32)) { continue; } message.components[comp_id] = comp.clone(); } message.components[0].component_ids.sort_unstable(); if !recipients.is_empty() { Ok(vec![ItipMessage { from: itip.organizer.email.email.clone(), from_organizer: true, to: recipients.into_iter().map(|e| e.to_string()).collect(), summary: ItipSummary::Invite( itip.main_instance_or_default() .build_summary(Some(&itip.organizer), &[]), ), message, }]) } else { Err(ItipError::NothingToSend) } } ================================================ FILE: crates/groupware/src/scheduling/snapshot.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::scheduling::{ Attendee, Email, InstanceId, ItipDateTime, ItipEntry, ItipEntryValue, ItipError, ItipField, ItipParticipant, ItipSnapshot, ItipSnapshots, ItipTime, ItipValue, Organizer, RecurrenceId, }; use ahash::AHashMap; use calcard::icalendar::{ ICalendar, ICalendarParameterName, ICalendarParameterValue, ICalendarProperty, ICalendarScheduleAgentValue, ICalendarValue, Uri, }; pub fn itip_snapshot<'x, 'y>( ical: &'x ICalendar, account_emails: &'y [String], force_add_client_scheduling: bool, ) -> Result, ItipError> { if !ical.components.iter().any(|comp| { comp.component_type.is_scheduling_object() && comp .entries .iter() .any(|e| matches!(e.name, ICalendarProperty::Organizer)) }) { return Err(ItipError::NoSchedulingInfo); } let mut organizer: Option> = None; let mut uid: Option<&'x str> = None; let mut components = AHashMap::new(); let mut expect_object_type = None; let mut has_local_emails = false; let mut tz_resolver = None; for (comp_id, comp) in ical.components.iter().enumerate() { if comp.component_type.is_scheduling_object() { match expect_object_type { Some(expected) if expected != &comp.component_type => { return Err(ItipError::MultipleObjectTypes); } None => { expect_object_type = Some(&comp.component_type); } _ => {} } let mut sched_comp = ItipSnapshot { comp_id: comp_id as u16, comp, attendees: Default::default(), dtstamp: Default::default(), entries: Default::default(), sequence: Default::default(), request_status: Default::default(), }; let mut instance_id = InstanceId::Main; for (entry_id, entry) in comp.entries.iter().enumerate() { match &entry.name { ICalendarProperty::Organizer => { if let Some(email) = entry .values .first() .and_then(|v| v.as_text()) .and_then(|v| Email::new(v, account_emails)) { let mut part = Organizer { entry_id: entry_id as u16, email, is_server_scheduling: true, name: None, force_send: None, }; has_local_emails |= part.email.is_local; for param in &entry.params { match (¶m.name, ¶m.value) { ( ICalendarParameterName::ScheduleAgent, ICalendarParameterValue::ScheduleAgent( ICalendarScheduleAgentValue::Client | ICalendarScheduleAgentValue::None, ), ) => { part.is_server_scheduling = false; } ( ICalendarParameterName::ScheduleForceSend, ICalendarParameterValue::ScheduleForceSend(force_send), ) => { part.force_send = Some(force_send); } ( ICalendarParameterName::Cn, ICalendarParameterValue::Text(name), ) => { part.name = Some(name.as_str()); } _ => {} } } if !part.is_server_scheduling && !force_add_client_scheduling { return Err(ItipError::OtherSchedulingAgent); } match organizer { Some(existing_organizer) if existing_organizer.email.email != part.email.email => { return Err(ItipError::MultipleOrganizer); } None => { organizer = Some(part); } _ => {} } } } ICalendarProperty::Attendee => { if let Some(email) = entry .values .first() .and_then(|v| v.as_text()) .and_then(|v| Email::new(v, account_emails)) { let mut part = Attendee { entry_id: entry_id as u16, email, name: None, rsvp: None, is_server_scheduling: true, force_send: None, part_stat: None, delegated_from: vec![], delegated_to: vec![], cu_type: None, role: None, sent_by: None, }; for param in &entry.params { match (¶m.name, ¶m.value) { ( ICalendarParameterName::ScheduleAgent, ICalendarParameterValue::ScheduleAgent(agent), ) => { part.is_server_scheduling = agent == &ICalendarScheduleAgentValue::Server; } ( ICalendarParameterName::Rsvp, ICalendarParameterValue::Bool(rsvp), ) => { part.rsvp = Some(*rsvp); } ( ICalendarParameterName::ScheduleForceSend, ICalendarParameterValue::ScheduleForceSend(force_send), ) => { part.force_send = Some(force_send); } ( ICalendarParameterName::Partstat, ICalendarParameterValue::Partstat(value), ) => { part.part_stat = Some(value); } ( ICalendarParameterName::Cutype, ICalendarParameterValue::Cutype(value), ) => { part.cu_type = Some(value); } ( ICalendarParameterName::DelegatedFrom, ICalendarParameterValue::Uri(uri), ) => { if let Some(uri) = Email::from_uri(uri, account_emails) { part.delegated_from.push(uri); } } ( ICalendarParameterName::DelegatedTo, ICalendarParameterValue::Uri(uri), ) => { if let Some(uri) = Email::from_uri(uri, account_emails) { part.delegated_to.push(uri); } } ( ICalendarParameterName::Role, ICalendarParameterValue::Role(value), ) => { part.role = Some(value); } ( ICalendarParameterName::SentBy, ICalendarParameterValue::Uri(value), ) => { part.sent_by = Email::from_uri(value, account_emails); } ( ICalendarParameterName::Cn, ICalendarParameterValue::Text(name), ) => { part.name = Some(name.as_str()); } _ => {} } } has_local_emails |= part.email.is_local && (force_add_client_scheduling || part.is_server_scheduling); sched_comp.attendees.insert(part); } } ICalendarProperty::Uid => { if let Some(uid_) = entry .values .first() .and_then(|v| v.as_text()) .map(|v| v.trim()) .filter(|v| !v.is_empty()) { match uid { Some(existing_uid) if existing_uid != uid_ => { return Err(ItipError::MultipleUid); } None => { uid = Some(uid_); } _ => {} } } } ICalendarProperty::Sequence => { if let Some(sequence) = entry.values.first().and_then(|v| v.as_integer()) { sched_comp.sequence = Some(sequence); } } ICalendarProperty::RecurrenceId => { if let Some(date) = entry.values.first().and_then(|v| v.as_partial_date_time()) { let mut this_and_future = false; let mut tz_id = None; for param in &entry.params { match (¶m.name, ¶m.value) { ( ICalendarParameterName::Tzid, ICalendarParameterValue::Text(id), ) => { tz_id = Some(id.as_str()); } (ICalendarParameterName::Range, _) => { this_and_future = true; } _ => (), } } instance_id = InstanceId::Recurrence(RecurrenceId { entry_id: entry_id as u16, date: date .to_date_time_with_tz( tz_resolver .get_or_insert_with(|| ical.build_tz_resolver()) .resolve_or_default(tz_id), ) .map(|dt| dt.timestamp()) .unwrap_or_else(|| date.to_timestamp().unwrap_or_default()), this_and_future, }); } } ICalendarProperty::RequestStatus => { if let Some(value) = entry.values.first().and_then(|v| v.as_text()) { sched_comp.request_status.push(value); } } ICalendarProperty::Dtstamp => { sched_comp.dtstamp = entry.values.first().and_then(|v| v.as_partial_date_time()); } ICalendarProperty::Dtstart | ICalendarProperty::Dtend | ICalendarProperty::Duration | ICalendarProperty::Due | ICalendarProperty::Rrule | ICalendarProperty::Rdate | ICalendarProperty::Exdate | ICalendarProperty::Status | ICalendarProperty::Location | ICalendarProperty::Summary | ICalendarProperty::Description | ICalendarProperty::Priority | ICalendarProperty::PercentComplete | ICalendarProperty::Completed => { let tz_id = entry.tz_id(); for value in &entry.values { let value = match value { ICalendarValue::Uri(Uri::Location(v)) => { ItipEntryValue::Text(v.as_str()) } ICalendarValue::PartialDateTime(date) => { let tz = tz_resolver .get_or_insert_with(|| ical.build_tz_resolver()) .resolve_or_default(tz_id); ItipEntryValue::DateTime(ItipDateTime { date: date.as_ref(), tz_id, tz_code: tz.as_id(), timestamp: date .to_date_time_with_tz(tz) .map(|dt| dt.timestamp()) .unwrap_or_else(|| { date.to_timestamp().unwrap_or_default() }), }) } ICalendarValue::Duration(v) => ItipEntryValue::Duration(v), ICalendarValue::RecurrenceRule(v) => ItipEntryValue::RRule(v), ICalendarValue::Period(v) => ItipEntryValue::Period(v), ICalendarValue::Integer(v) => ItipEntryValue::Integer(*v), ICalendarValue::Text(v) => ItipEntryValue::Text(v.as_str()), ICalendarValue::Status(v) => ItipEntryValue::Status(v), _ => continue, }; sched_comp.entries.insert(ItipEntry { name: &entry.name, value, }); } } _ => {} } } if components.insert(instance_id, sched_comp).is_some() { return Err(ItipError::MultipleObjectInstances); } } } if has_local_emails { Ok(ItipSnapshots { organizer: organizer.ok_or(ItipError::NoSchedulingInfo)?, uid: uid.ok_or(ItipError::MissingUid)?, components, }) } else { Err(ItipError::NotOrganizerNorAttendee) } } impl ItipSnapshots<'_> { pub fn sender_is_organizer_or_attendee(&self, email: &str) -> bool { self.organizer.email.email == email || self.components.values().any(|snapshot| { snapshot .attendees .iter() .any(|attendee| attendee.email.email == email) }) } pub fn main_instance(&self) -> Option<&ItipSnapshot<'_>> { self.components.get(&InstanceId::Main) } pub fn main_instance_or_default(&self) -> &ItipSnapshot<'_> { self.main_instance() .unwrap_or_else(|| self.components.values().next().unwrap()) } } impl ItipSnapshot<'_> { pub fn has_local_attendee(&self) -> bool { self.attendees .iter() .any(|attendee| attendee.email.is_local) } pub fn local_attendee(&self) -> Option<&Attendee<'_>> { self.attendees .iter() .find(|attendee| attendee.email.is_local) } pub fn external_attendees(&self) -> impl Iterator> + '_ { self.attendees.iter().filter(|item| !item.email.is_local) } pub fn attendee_by_email(&self, email: &str) -> Option<&Attendee<'_>> { self.attendees .iter() .find(|attendee| attendee.email.email == email) } pub fn build_summary( &self, include_guests: Option<&Organizer<'_>>, skip_fields: &[ItipField], ) -> Vec { let mut fields = Vec::with_capacity(5); for entry in &self.entries { if matches!( entry.name, ICalendarProperty::Summary | ICalendarProperty::Description | ICalendarProperty::Dtstart | ICalendarProperty::Location | ICalendarProperty::Rrule ) { let value = match &entry.value { ItipEntryValue::DateTime(dt) => ItipValue::Time(ItipTime { start: dt.timestamp, tz_id: dt.tz_code, }), ItipEntryValue::RRule(rule) => ItipValue::Rrule(Box::new((*rule).clone())), ItipEntryValue::Text(value) => ItipValue::Text(value.to_string()), _ => continue, }; let field = ItipField { name: entry.name.clone(), value, }; if !skip_fields.contains(&field) { fields.push(field); } } } if let Some(organizer) = include_guests { let mut attendees = Vec::with_capacity(self.attendees.len()); for attendee in &self.attendees { if attendee.email.email != organizer.email.email { attendees.push(ItipParticipant { email: attendee.email.email.to_string(), name: attendee.name.map(|n| n.to_string()), is_organizer: false, }); } } attendees.push(ItipParticipant { email: organizer.email.email.to_string(), name: organizer.name.map(|n| n.to_string()), is_organizer: true, }); attendees.sort_by(|a, b| { if a.is_organizer && !b.is_organizer { std::cmp::Ordering::Less } else if !a.is_organizer && b.is_organizer { std::cmp::Ordering::Greater } else if let (Some(a_name), Some(b_name)) = (a.name.as_deref(), b.name.as_deref()) { match a_name.cmp(b_name) { std::cmp::Ordering::Equal => a.email.cmp(&b.email), ord => ord, } } else { a.email.cmp(&b.email) } }); let field = ItipField { name: ICalendarProperty::Attendee, value: ItipValue::Participants(attendees), }; if !skip_fields.contains(&field) { fields.push(field); } } fields } } ================================================ FILE: crates/http/Cargo.toml ================================================ [package] name = "http" version = "0.15.5" edition = "2024" [dependencies] store = { path = "../store" } common = { path = "../common" } utils = { path = "../utils" } trc = { path = "../trc" } email = { path = "../email" } smtp = { path = "../smtp" } jmap = { path = "../jmap" } dav = { path = "../dav" } groupware = { path = "../groupware" } spam-filter = { path = "../spam-filter" } http_proto = { path = "../http-proto" } jmap_proto = { path = "../jmap-proto" } types = { path = "../types" } directory = { path = "../directory" } services = { path = "../services" } smtp-proto = { version = "0.2" } mail-parser = { version = "0.11", features = ["full_encoding", "rkyv"] } mail-builder = { version = "0.4" } mail-auth = { version = "0.7.1", features = ["generate"] } mail-send = { version = "0.5", default-features = false, features = ["cram-md5", "ring", "tls12"] } tokio = { version = "1.47", features = ["rt"] } hyper = { version = "1.0.1", features = ["server", "http1", "http2"] } hyper-util = { version = "0.1.1", features = ["tokio"] } http-body-util = "0.1.0" async-stream = "0.3.5" quick-xml = "0.38" serde = { version = "1.0", features = ["derive"]} serde_json = "1.0" x509-parser = "0.18" chrono = "0.4" base64 = "0.22" pkcs8 = { version = "0.10.2", features = ["alloc", "std"] } rsa = "0.9.2" sha1 = "0.10" sha2 = "0.10" rev_lines = "0.3.0" rkyv = { version = "0.8.10", features = ["little_endian"] } form-data = { version = "0.6.0", features = ["sync"], default-features = false } mime = "0.3.17" compact_str = "0.9.0" [dev-dependencies] [features] test_mode = [] enterprise = [] ================================================ FILE: crates/http/src/auth/authenticate.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::auth::AccessToken; use common::{HttpAuthCache, Server, auth::AuthRequest, listener::limiter::InFlight}; use http_proto::{HttpRequest, HttpSessionData}; use hyper::header; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; use std::future::Future; use std::sync::Arc; use std::time::{Duration, Instant}; pub trait Authenticator: Sync + Send { fn authenticate_headers( &self, req: &HttpRequest, session: &HttpSessionData, allow_api_access: bool, ) -> impl Future, Arc)>> + Send; } impl Authenticator for Server { async fn authenticate_headers( &self, req: &HttpRequest, session: &HttpSessionData, allow_api_access: bool, ) -> trc::Result<(Option, Arc)> { if let Some((mechanism, token)) = req.authorization() { // Check if the credentials are cached if let Some(http_cache) = self.inner.cache.http_auth.get(token) { // Make sure the revision is still valid if http_cache.expires <= Instant::now() { let access_token = self.get_access_token(http_cache.account_id).await?; if access_token.revision == http_cache.revision { // Enforce authenticated rate limit return self .is_http_authenticated_request_allowed(&access_token) .await .map(|in_flight| (in_flight, access_token)); } } // If the revision is not valid, remove the cached credentials self.inner.cache.http_auth.remove(token); } let credentials = if mechanism.eq_ignore_ascii_case("basic") { // Decode the base64 encoded credentials decode_plain_auth(token).ok_or_else(|| { trc::AuthEvent::Error .into_err() .details("Failed to decode Basic auth request.") .id(token.to_string()) .caused_by(trc::location!()) })? } else if mechanism.eq_ignore_ascii_case("bearer") { // Enforce anonymous rate limit self.is_http_anonymous_request_allowed(&session.remote_ip) .await?; decode_bearer_token(token, allow_api_access).ok_or_else(|| { trc::AuthEvent::Error .into_err() .details("Failed to decode Bearer token.") .id(token.to_string()) .caused_by(trc::location!()) })? } else { // Enforce anonymous rate limit self.is_http_anonymous_request_allowed(&session.remote_ip) .await?; return Err(trc::AuthEvent::Error .into_err() .reason("Unsupported authentication mechanism.") .details(token.to_string()) .caused_by(trc::location!())); }; // Authenticate let access_token = self .authenticate( &AuthRequest::from_credentials( credentials, session.session_id, session.remote_ip, ) .with_api_access(allow_api_access), ) .await?; // Cache credentials self.inner.cache.http_auth.insert( token.to_string(), HttpAuthCache { account_id: access_token.primary_id(), revision: access_token.revision, expires: Instant::now() + Duration::from_secs(self.core.oauth.oauth_expiry_token), }, ); // Enforce authenticated rate limit self.is_http_authenticated_request_allowed(&access_token) .await .map(|in_flight| (in_flight, access_token)) } else { // Enforce anonymous rate limit self.is_http_anonymous_request_allowed(&session.remote_ip) .await?; Err(trc::AuthEvent::Failed .into_err() .details("Missing Authorization header.") .caused_by(trc::location!())) } } } pub trait HttpHeaders { fn authorization(&self) -> Option<(&str, &str)>; fn authorization_basic(&self) -> Option<&str>; } impl HttpHeaders for HttpRequest { fn authorization(&self) -> Option<(&str, &str)> { self.headers() .get(header::AUTHORIZATION) .and_then(|h| h.to_str().ok()) .and_then(|h| h.split_once(' ').map(|(l, t)| (l, t.trim()))) } fn authorization_basic(&self) -> Option<&str> { self.authorization().and_then(|(l, t)| { if l.eq_ignore_ascii_case("basic") { Some(t) } else { None } }) } } fn decode_plain_auth(token: &str) -> Option> { base64_decode(token.as_bytes()) .and_then(|token| String::from_utf8(token).ok()) .and_then(|token| { token .split_once(':') .map(|(login, secret)| Credentials::Plain { username: login.trim().to_lowercase(), secret: secret.to_string(), }) }) } fn decode_bearer_token(token: &str, allow_api_access: bool) -> Option> { if allow_api_access && let Some(token) = token.strip_prefix("api_").and_then(decode_plain_auth) { return Some(token); } Some(Credentials::OAuthBearer { token: token.to_string(), }) } ================================================ FILE: crates/http/src/auth/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod authenticate; pub mod oauth; ================================================ FILE: crates/http/src/auth/oauth/auth.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::auth::oauth::OAuthStatus; use common::{ KV_OAUTH, Server, auth::{ AccessToken, oauth::{CLIENT_ID_MAX_LEN, DEVICE_CODE_LEN, USER_CODE_ALPHABET, USER_CODE_LEN}, }, }; use http_proto::*; use serde::Deserialize; use serde_json::json; use std::future::Future; use std::sync::Arc; use store::{ Serialize, dispatch::lookup::KeyValue, write::{Archive, Archiver}, }; use store::{ rand::{ Rng, distr::{Alphanumeric, StandardUniform}, rng, }, write::AlignedBytes, }; use trc::AddContext; use super::{DeviceAuthResponse, FormData, MAX_POST_LEN, OAuthCode, OAuthCodeRequest}; #[derive(Debug, serde::Serialize, Deserialize)] pub struct OAuthMetadata { pub issuer: String, pub token_endpoint: String, pub authorization_endpoint: String, pub device_authorization_endpoint: String, pub registration_endpoint: String, pub introspection_endpoint: String, pub grant_types_supported: Vec, pub response_types_supported: Vec, pub scopes_supported: Vec, } pub trait OAuthApiHandler: Sync + Send { fn handle_oauth_api_request( &self, access_token: Arc, body: Option>, ) -> impl Future> + Send; fn handle_device_auth( &self, req: &mut HttpRequest, session: HttpSessionData, ) -> impl Future> + Send; fn handle_oauth_metadata( &self, req: HttpRequest, session: HttpSessionData, ) -> impl Future> + Send; } impl OAuthApiHandler for Server { async fn handle_oauth_api_request( &self, access_token: Arc, body: Option>, ) -> trc::Result { let request = serde_json::from_slice::(body.as_deref().unwrap_or_default()) .map_err(|err| { trc::EventType::Resource(trc::ResourceEvent::BadParameters).from_json_error(err) })?; let response = match request { OAuthCodeRequest::Code { client_id, redirect_uri, nonce, } => { // Validate clientId if client_id.len() > CLIENT_ID_MAX_LEN { return Err(trc::ManageEvent::Error .into_err() .details("Client ID is invalid.")); } else if redirect_uri .as_ref() .is_some_and(|uri| uri.starts_with("http://")) { return Err(trc::ManageEvent::Error .into_err() .details("Redirect URI must be HTTPS.")); } // Generate client code let client_code = rng() .sample_iter(Alphanumeric) .take(DEVICE_CODE_LEN) .map(char::from) .collect::(); // Serialize OAuth code let value = Archiver::new(OAuthCode { status: OAuthStatus::Authorized, account_id: access_token.primary_id(), client_id, nonce, params: redirect_uri.unwrap_or_default(), }) .untrusted() .serialize() .caused_by(trc::location!())?; // Insert client code self.core .storage .lookup .key_set( KeyValue::with_prefix(KV_OAUTH, client_code.as_bytes(), value) .expires(self.core.oauth.oauth_expiry_auth_code), ) .await?; #[cfg(not(feature = "enterprise"))] let is_enterprise = false; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] let is_enterprise = self.core.is_enterprise_edition(); // SPDX-SnippetEnd json!({ "data": { "code": client_code, "permissions": access_token.permissions(), "version": env!("CARGO_PKG_VERSION"), "isEnterprise": is_enterprise, }, }) } OAuthCodeRequest::Device { code } => { let mut success = false; // Obtain code if let Some(auth_code_) = self .core .storage .lookup .key_get::>(KeyValue::<()>::build_key( KV_OAUTH, code.as_bytes(), )) .await? { let oauth = auth_code_ .unarchive::() .caused_by(trc::location!())?; if oauth.status == OAuthStatus::Pending { let new_oauth_code = OAuthCode { status: OAuthStatus::Authorized, account_id: access_token.primary_id(), client_id: oauth.client_id.to_string(), nonce: oauth.nonce.as_ref().map(|s| s.to_string()), params: Default::default(), }; success = true; // Delete issued user code self.core .storage .lookup .key_delete(KeyValue::<()>::build_key(KV_OAUTH, code.as_bytes())) .await?; // Update device code status self.core .storage .lookup .key_set( KeyValue::with_prefix( KV_OAUTH, oauth.params.as_bytes(), Archiver::new(new_oauth_code) .untrusted() .serialize() .caused_by(trc::location!())?, ) .expires(self.core.oauth.oauth_expiry_auth_code), ) .await?; } } json!({ "data": success, }) } }; Ok(JsonResponse::new(response).no_cache().into_http_response()) } async fn handle_device_auth( &self, req: &mut HttpRequest, session: HttpSessionData, ) -> trc::Result { // Parse form let mut form_data = FormData::from_request(req, MAX_POST_LEN, session.session_id).await?; let client_id = form_data .remove("client_id") .filter(|client_id| client_id.len() <= CLIENT_ID_MAX_LEN) .ok_or_else(|| { trc::ResourceEvent::BadParameters .into_err() .details("Client ID is missing.") })?; let nonce = form_data.remove("nonce"); // Generate device code let device_code = rng() .sample_iter(Alphanumeric) .take(DEVICE_CODE_LEN) .map(char::from) .collect::(); // Generate user code let mut user_code = String::with_capacity(USER_CODE_LEN + 1); for (pos, ch) in rng() .sample_iter(StandardUniform) .take(USER_CODE_LEN) .map(|v: u64| char::from(USER_CODE_ALPHABET[v as usize % USER_CODE_ALPHABET.len()])) .enumerate() { if pos == USER_CODE_LEN / 2 { user_code.push('-'); } user_code.push(ch); } // Add OAuth status let oauth_code = Archiver::new(OAuthCode { status: OAuthStatus::Pending, account_id: u32::MAX, client_id, nonce, params: device_code.clone(), }) .untrusted() .serialize() .caused_by(trc::location!())?; // Insert device code self.core .storage .lookup .key_set( KeyValue::with_prefix(KV_OAUTH, device_code.as_bytes(), oauth_code.clone()) .expires(self.core.oauth.oauth_expiry_user_code), ) .await?; // Insert user code self.core .storage .lookup .key_set( KeyValue::with_prefix(KV_OAUTH, user_code.as_bytes(), oauth_code) .expires(self.core.oauth.oauth_expiry_user_code), ) .await?; // Build response let base_url = HttpContext::new(&session, req) .resolve_response_url(self) .await; Ok(JsonResponse::new(DeviceAuthResponse { verification_uri: format!("{base_url}/authorize"), verification_uri_complete: format!("{base_url}/authorize/?code={user_code}"), device_code, user_code, expires_in: self.core.oauth.oauth_expiry_user_code, interval: 5, }) .no_cache() .into_http_response()) } async fn handle_oauth_metadata( &self, req: HttpRequest, session: HttpSessionData, ) -> trc::Result { let base_url = HttpContext::new(&session, &req) .resolve_response_url(self) .await .to_string(); Ok(JsonResponse::new(OAuthMetadata { authorization_endpoint: format!("{base_url}/authorize/code",), token_endpoint: format!("{base_url}/auth/token"), device_authorization_endpoint: format!("{base_url}/auth/device"), introspection_endpoint: format!("{base_url}/auth/introspect"), registration_endpoint: format!("{base_url}/auth/register"), grant_types_supported: vec![ "authorization_code".to_string(), "implicit".to_string(), "urn:ietf:params:oauth:grant-type:device_code".to_string(), ], response_types_supported: vec![ "code".to_string(), "id_token".to_string(), "code token".to_string(), "id_token token".to_string(), ], scopes_supported: vec![ "openid".to_string(), "offline_access".to_string(), "urn:ietf:params:jmap:core".to_string(), "urn:ietf:params:jmap:mail".to_string(), "urn:ietf:params:jmap:submission".to_string(), "urn:ietf:params:jmap:vacationresponse".to_string(), ], issuer: base_url, }) .into_http_response()) } } ================================================ FILE: crates/http/src/auth/oauth/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use http_proto::{HttpRequest, request::fetch_body}; use hyper::header::CONTENT_TYPE; use serde::{Deserialize, Serialize}; use utils::map::vec_map::VecMap; pub mod auth; pub mod openid; pub mod registration; pub mod token; #[derive( rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, )] #[rkyv(compare(PartialEq))] pub enum OAuthStatus { Authorized, TokenIssued, Pending, } const MAX_POST_LEN: usize = 2048; pub struct OAuth { pub key: String, pub expiry_user_code: u64, pub expiry_auth_code: u64, pub expiry_token: u64, pub expiry_refresh_token: u64, pub expiry_refresh_token_renew: u64, pub max_auth_attempts: u32, pub metadata: String, } #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] pub struct OAuthCode { pub status: OAuthStatus, pub account_id: u32, pub client_id: String, pub nonce: Option, pub params: String, } #[derive(Debug, Serialize, Deserialize)] pub struct DeviceAuthGet { code: Option, } #[derive(Debug, Serialize, Deserialize)] pub struct DeviceAuthPost { code: Option, email: Option, password: Option, } #[derive(Debug, Serialize, Deserialize)] pub struct DeviceAuthRequest { client_id: String, } #[derive(Debug, Serialize, Deserialize)] pub struct DeviceAuthResponse { pub device_code: String, pub user_code: String, pub verification_uri: String, pub verification_uri_complete: String, pub expires_in: u64, pub interval: u64, } #[derive(Debug, Serialize, Deserialize)] pub struct CodeAuthRequest { response_type: String, client_id: String, redirect_uri: String, scope: Option, state: Option, } #[derive(Debug, Serialize, Deserialize)] pub struct CodeAuthForm { code: String, email: Option, password: Option, } #[derive(Debug, Serialize, Deserialize)] pub struct TokenRequest { pub grant_type: String, pub code: Option, pub device_code: Option, pub client_id: Option, pub refresh_token: Option, pub redirect_uri: Option, } #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(untagged)] pub enum TokenResponse { Granted(OAuthResponse), Error { error: ErrorType }, } #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] pub struct OAuthResponse { pub access_token: String, pub token_type: String, pub expires_in: u64, #[serde(skip_serializing_if = "Option::is_none")] pub refresh_token: Option, #[serde(skip_serializing_if = "Option::is_none")] pub scope: Option, #[serde(skip_serializing_if = "Option::is_none")] pub id_token: Option, } #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] pub enum ErrorType { #[serde(rename = "invalid_grant")] InvalidGrant, #[serde(rename = "invalid_client")] InvalidClient, #[serde(rename = "invalid_scope")] InvalidScope, #[serde(rename = "invalid_request")] InvalidRequest, #[serde(rename = "unauthorized_client")] UnauthorizedClient, #[serde(rename = "unsupported_grant_type")] UnsupportedGrantType, #[serde(rename = "authorization_pending")] AuthorizationPending, #[serde(rename = "slow_down")] SlowDown, #[serde(rename = "access_denied")] AccessDenied, #[serde(rename = "expired_token")] ExpiredToken, } #[derive(Debug, Serialize, Deserialize)] #[serde(tag = "type")] #[serde(rename_all = "camelCase")] pub enum OAuthCodeRequest { Code { client_id: String, redirect_uri: Option, #[serde(default)] nonce: Option, }, Device { code: String, }, } impl TokenResponse { pub fn error(error: ErrorType) -> Self { TokenResponse::Error { error } } pub fn is_error(&self) -> bool { matches!(self, TokenResponse::Error { .. }) } } #[derive(Debug)] pub struct FormData { fields: VecMap, } impl FormData { pub async fn from_request( req: &mut HttpRequest, max_len: usize, session_id: u64, ) -> trc::Result { match ( req.headers() .get(CONTENT_TYPE) .and_then(|h| h.to_str().ok()) .and_then(|val| val.parse::().ok()), fetch_body(req, max_len, session_id).await, ) { (Some(content_type), Some(body)) => { let mut fields = VecMap::new(); if let Some(boundary) = content_type.get_param(mime::BOUNDARY) { for mut field in form_data::FormData::new(&body[..], boundary.as_str()).flatten() { let value = String::from_utf8_lossy(&field.bytes().unwrap_or_default()) .into_owned(); fields.append(field.name, value); } } else { for (key, value) in http_proto::form_urlencoded::parse(&body) { fields.append(key.into_owned(), value.into_owned()); } } Ok(FormData { fields }) } _ => Err(trc::ResourceEvent::BadParameters .into_err() .details("Invalid post request")), } } pub fn get(&self, key: &str) -> Option<&str> { self.fields.get(key).map(|v| v.as_str()) } pub fn remove(&mut self, key: &str) -> Option { self.fields.remove(key) } pub fn has_field(&self, key: &str) -> bool { self.fields.get(key).is_some_and(|v| !v.is_empty()) } pub fn fields(&self) -> impl Iterator { self.fields.iter() } } ================================================ FILE: crates/http/src/auth/oauth/openid.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::future::Future; use common::{ Server, auth::{AccessToken, oauth::oidc::Userinfo}, }; use serde::{Deserialize, Serialize}; use http_proto::*; #[derive(Debug, Serialize, Deserialize)] pub struct OpenIdMetadata { pub issuer: String, pub authorization_endpoint: String, pub token_endpoint: String, pub userinfo_endpoint: String, pub jwks_uri: String, pub registration_endpoint: String, pub device_authorization_endpoint: String, pub scopes_supported: Vec, pub response_types_supported: Vec, pub subject_types_supported: Vec, pub grant_types_supported: Vec, pub id_token_signing_alg_values_supported: Vec, pub claims_supported: Vec, } pub trait OpenIdHandler: Sync + Send { fn handle_userinfo_request( &self, access_token: &AccessToken, ) -> impl Future> + Send; fn handle_oidc_metadata( &self, req: HttpRequest, session: HttpSessionData, ) -> impl Future> + Send; } impl OpenIdHandler for Server { async fn handle_userinfo_request( &self, access_token: &AccessToken, ) -> trc::Result { Ok(JsonResponse::new(Userinfo { sub: Some(access_token.primary_id.to_string()), name: access_token.description.clone(), preferred_username: Some(access_token.name.clone()), email: access_token.emails.first().cloned(), email_verified: !access_token.emails.is_empty(), ..Default::default() }) .no_cache() .into_http_response()) } async fn handle_oidc_metadata( &self, req: HttpRequest, session: HttpSessionData, ) -> trc::Result { let base_url = HttpContext::new(&session, &req) .resolve_response_url(self) .await; Ok(JsonResponse::new(OpenIdMetadata { authorization_endpoint: format!("{base_url}/authorize/code",), token_endpoint: format!("{base_url}/auth/token"), userinfo_endpoint: format!("{base_url}/auth/userinfo"), jwks_uri: format!("{base_url}/auth/jwks.json"), registration_endpoint: format!("{base_url}/auth/register"), device_authorization_endpoint: format!("{base_url}/auth/device"), response_types_supported: vec![ "code".into(), "id_token".into(), "id_token token".into(), ], grant_types_supported: vec![ "authorization_code".into(), "implicit".into(), "urn:ietf:params:oauth:grant-type:device_code".into(), ], scopes_supported: vec!["openid".into(), "offline_access".into()], subject_types_supported: vec!["public".into()], id_token_signing_alg_values_supported: vec![ "RS256".into(), "RS384".into(), "RS512".into(), "ES256".into(), "ES384".into(), "PS256".into(), "PS384".into(), "PS512".into(), "HS256".into(), "HS384".into(), "HS512".into(), ], claims_supported: vec![ "sub".into(), "name".into(), "preferred_username".into(), "email".into(), "email_verified".into(), ], issuer: base_url, }) .into_http_response()) } } ================================================ FILE: crates/http/src/auth/oauth/registration.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::future::Future; use common::{ Server, auth::oauth::registration::{ClientRegistrationRequest, ClientRegistrationResponse}, }; use directory::{ Permission, QueryParams, Type, backend::internal::{ PrincipalField, PrincipalSet, lookup::DirectoryStore, manage::ManageDirectory, }, }; use store::rand::{Rng, distr::Alphanumeric, rng}; use trc::{AddContext, AuthEvent}; use crate::auth::authenticate::Authenticator; use http_proto::{request::fetch_body, *}; use super::ErrorType; pub trait ClientRegistrationHandler: Sync + Send { fn handle_oauth_registration_request( &self, req: &mut HttpRequest, session: HttpSessionData, ) -> impl Future> + Send; fn validate_client_registration( &self, client_id: &str, redirect_uri: Option<&str>, account_id: u32, ) -> impl Future>> + Send; } impl ClientRegistrationHandler for Server { async fn handle_oauth_registration_request( &self, req: &mut HttpRequest, session: HttpSessionData, ) -> trc::Result { if !self.core.oauth.allow_anonymous_client_registration { // Authenticate request let (_, access_token) = self.authenticate_headers(req, &session, true).await?; // Validate permissions access_token.assert_has_permission(Permission::OauthClientRegistration)?; } else { self.is_http_anonymous_request_allowed(&session.remote_ip) .await?; } // Parse request let body = fetch_body(req, 20 * 1024, session.session_id).await; let request = serde_json::from_slice::( body.as_deref().unwrap_or_default(), ) .map_err(|err| { trc::EventType::Resource(trc::ResourceEvent::BadParameters).from_json_error(err) })?; // Generate client ID let client_id = rng() .sample_iter(Alphanumeric) .take(20) .map(|ch| char::from(ch.to_ascii_lowercase())) .collect::(); self.store() .create_principal( PrincipalSet::new(u32::MAX, Type::OauthClient) .with_field(PrincipalField::Name, client_id.clone()) .with_field(PrincipalField::Urls, request.redirect_uris.clone()) .with_opt_field(PrincipalField::Description, request.client_name.clone()) .with_field(PrincipalField::Emails, request.contacts.clone()) .with_opt_field(PrincipalField::Picture, request.logo_uri.clone()), None, None, ) .await .caused_by(trc::location!())?; trc::event!( Auth(AuthEvent::ClientRegistration), Id = client_id.to_string(), RemoteIp = session.remote_ip ); Ok(JsonResponse::new(ClientRegistrationResponse { client_id, request, ..Default::default() }) .no_cache() .into_http_response()) } async fn validate_client_registration( &self, client_id: &str, redirect_uri: Option<&str>, account_id: u32, ) -> trc::Result> { if !self.core.oauth.require_client_authentication { return Ok(None); } // Fetch client registration let found_registration = if let Some(client) = self .store() .query(QueryParams::name(client_id).with_return_member_of(false)) .await .caused_by(trc::location!())? .filter(|p| p.typ() == Type::OauthClient) { if let Some(redirect_uri) = redirect_uri { if client.urls().any(|uri| uri == redirect_uri) { return Ok(None); } } else { // Device flow does not require a redirect URI return Ok(None); } true } else { false }; // Check if the account is allowed to override client registration if self .get_access_token(account_id) .await .caused_by(trc::location!())? .has_permission(Permission::OauthClientOverride) { return Ok(None); } Ok(Some(if found_registration { ErrorType::InvalidClient } else { ErrorType::InvalidRequest })) } } ================================================ FILE: crates/http/src/auth/oauth/token.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ ArchivedOAuthStatus, ErrorType, FormData, MAX_POST_LEN, OAuthCode, OAuthResponse, OAuthStatus, TokenResponse, registration::ClientRegistrationHandler, }; use common::{ KV_OAUTH, Server, auth::{ AccessToken, oauth::{GrantType, oidc::StandardClaims}, }, }; use http_proto::*; use hyper::StatusCode; use std::future::Future; use store::{ dispatch::lookup::KeyValue, write::{AlignedBytes, Archive}, }; use trc::AddContext; pub trait TokenHandler: Sync + Send { fn handle_token_request( &self, req: &mut HttpRequest, session: HttpSessionData, ) -> impl Future> + Send; fn handle_token_introspect( &self, req: &mut HttpRequest, access_token: &AccessToken, session_id: u64, ) -> impl Future> + Send; fn issue_token( &self, account_id: u32, client_id: &str, issuer: String, nonce: Option, with_refresh_token: bool, with_id_token: bool, ) -> impl Future> + Send; } impl TokenHandler for Server { // Token endpoint async fn handle_token_request( &self, req: &mut HttpRequest, session: HttpSessionData, ) -> trc::Result { // Parse form let params = FormData::from_request(req, MAX_POST_LEN, session.session_id).await?; let grant_type = params.get("grant_type").unwrap_or_default(); let mut response = TokenResponse::error(ErrorType::InvalidGrant); let issuer = HttpContext::new(&session, req) .resolve_response_url(self) .await; if grant_type.eq_ignore_ascii_case("authorization_code") { response = if let (Some(code), Some(client_id), Some(redirect_uri)) = ( params.get("code"), params.get("client_id"), params.get("redirect_uri"), ) { // Obtain code match self .core .storage .lookup .key_get::>(KeyValue::<()>::build_key( KV_OAUTH, code.as_bytes(), )) .await? { Some(auth_code_) => { let oauth = auth_code_ .unarchive::() .caused_by(trc::location!())?; if client_id != oauth.client_id || redirect_uri != oauth.params { TokenResponse::error(ErrorType::InvalidClient) } else if oauth.status == OAuthStatus::Authorized { // Validate client id if let Some(error) = self .validate_client_registration( client_id, redirect_uri.into(), oauth.account_id.into(), ) .await? { TokenResponse::error(error) } else { // Mark this token as issued self.core .storage .lookup .key_delete(KeyValue::<()>::build_key( KV_OAUTH, code.as_bytes(), )) .await?; // Issue token self.issue_token( oauth.account_id.into(), &oauth.client_id, issuer, oauth.nonce.as_ref().map(|s| s.as_str().into()), true, true, ) .await .map(TokenResponse::Granted) .map_err(|err| { trc::AuthEvent::Error .into_err() .details(err) .caused_by(trc::location!()) })? } } else { TokenResponse::error(ErrorType::InvalidGrant) } } None => TokenResponse::error(ErrorType::AccessDenied), } } else { TokenResponse::error(ErrorType::InvalidClient) }; } else if grant_type.eq_ignore_ascii_case("urn:ietf:params:oauth:grant-type:device_code") { response = TokenResponse::error(ErrorType::ExpiredToken); if let (Some(device_code), Some(client_id)) = (params.get("device_code"), params.get("client_id")) { // Obtain code if let Some(auth_code_) = self .core .storage .lookup .key_get::>(KeyValue::<()>::build_key( KV_OAUTH, device_code.as_bytes(), )) .await? { let oauth = auth_code_ .unarchive::() .caused_by(trc::location!())?; response = if oauth.client_id != client_id { TokenResponse::error(ErrorType::InvalidClient) } else { match oauth.status { ArchivedOAuthStatus::Authorized => { if let Some(error) = self .validate_client_registration( client_id, None, oauth.account_id.into(), ) .await? { TokenResponse::error(error) } else { // Mark this token as issued self.core .storage .lookup .key_delete(KeyValue::<()>::build_key( KV_OAUTH, device_code.as_bytes(), )) .await?; // Issue token self.issue_token( oauth.account_id.into(), &oauth.client_id, issuer, oauth.nonce.as_ref().map(|s| s.as_str().into()), true, true, ) .await .map(TokenResponse::Granted) .map_err(|err| { trc::AuthEvent::Error .into_err() .details(err) .caused_by(trc::location!()) })? } } ArchivedOAuthStatus::Pending => { TokenResponse::error(ErrorType::AuthorizationPending) } ArchivedOAuthStatus::TokenIssued => { TokenResponse::error(ErrorType::ExpiredToken) } } }; } } } else if grant_type.eq_ignore_ascii_case("refresh_token") { if let Some(refresh_token) = params.get("refresh_token") { response = match self .validate_access_token(GrantType::RefreshToken.into(), refresh_token) .await { Ok(token_info) => self .issue_token( token_info.account_id, &token_info.client_id, issuer, None, token_info.expires_in <= self.core.oauth.oauth_expiry_refresh_token_renew, false, ) .await .map(TokenResponse::Granted) .map_err(|err| { trc::AuthEvent::Error .into_err() .details(err) .caused_by(trc::location!()) })?, Err(err) => { trc::error!( err.caused_by(trc::location!()) .details("Failed to validate refresh token") .span_id(session.session_id) ); TokenResponse::error(ErrorType::InvalidGrant) } }; } else { response = TokenResponse::error(ErrorType::InvalidRequest); } } Ok(JsonResponse::with_status( if response.is_error() { StatusCode::BAD_REQUEST } else { StatusCode::OK }, response, ) .into_http_response()) } async fn handle_token_introspect( &self, req: &mut HttpRequest, access_token: &AccessToken, session_id: u64, ) -> trc::Result { // Parse token let token = FormData::from_request(req, 1024, session_id) .await? .remove("token") .ok_or_else(|| { trc::ResourceEvent::BadParameters .into_err() .details("Client ID is missing.") })?; self.introspect_access_token(&token, access_token) .await .map(|response| JsonResponse::new(response).no_cache().into_http_response()) } async fn issue_token( &self, account_id: u32, client_id: &str, issuer: String, nonce: Option, with_refresh_token: bool, with_id_token: bool, ) -> trc::Result { Ok(OAuthResponse { access_token: self .encode_access_token( GrantType::AccessToken, account_id, client_id, self.core.oauth.oauth_expiry_token, ) .await?, token_type: "bearer".to_string(), expires_in: self.core.oauth.oauth_expiry_token, refresh_token: if with_refresh_token { self.encode_access_token( GrantType::RefreshToken, account_id, client_id, self.core.oauth.oauth_expiry_refresh_token, ) .await? .into() } else { None }, id_token: if with_id_token { // Obtain access token let access_token = self .get_access_token(account_id) .await .caused_by(trc::location!())?; match self.issue_id_token( account_id.to_string(), issuer, client_id, StandardClaims { nonce, preferred_username: access_token.name.clone().into(), email: access_token.emails.first().cloned(), description: access_token.description.clone(), }, ) { Ok(id_token) => Some(id_token), Err(err) => { trc::error!(err); None } } } else { None }, scope: None, }) } } ================================================ FILE: crates/http/src/autoconfig/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{Server, manager::webadmin::Resource}; use directory::QueryParams; use http_proto::*; use quick_xml::Reader; use quick_xml::events::Event; use std::fmt::Write; use std::future::Future; use trc::AddContext; use utils::url_params::UrlParams; pub trait Autoconfig: Sync + Send { fn handle_autoconfig_request( &self, req: &HttpRequest, ) -> impl Future> + Send; fn handle_autodiscover_request( &self, body: Option>, ) -> impl Future> + Send; fn autoconfig_parameters<'x>( &'x self, emailaddress: &'x str, fail_if_invalid: bool, ) -> impl Future> + Send; } impl Autoconfig for Server { async fn handle_autoconfig_request(&self, req: &HttpRequest) -> trc::Result { // Obtain parameters let params = UrlParams::new(req.uri().query()); let emailaddress = params .get("emailaddress") .unwrap_or_default() .to_lowercase(); let (account_name, server_name, domain) = self.autoconfig_parameters(&emailaddress, false).await?; let services = self.core.storage.config.get_services().await?; // Build XML response let mut config = String::with_capacity(1024); config.push_str("\n"); config.push_str("\n"); let _ = writeln!(&mut config, "\t"); let _ = writeln!(&mut config, "\t\t{domain}"); let _ = writeln!(&mut config, "\t\t{emailaddress}"); let _ = writeln!( &mut config, "\t\t{domain}" ); for (protocol, port, is_tls) in services { let tag = match protocol.as_str() { "imap" | "pop3" => "incomingServer", "smtp" if port != 25 => "outgoingServer", _ => continue, }; let _ = writeln!(&mut config, "\t\t<{tag} type=\"{protocol}\">"); let _ = writeln!(&mut config, "\t\t\t{server_name}"); let _ = writeln!(&mut config, "\t\t\t{port}"); let _ = writeln!( &mut config, "\t\t\t{}", if is_tls { "SSL" } else { "STARTTLS" } ); let _ = writeln!(&mut config, "\t\t\t{account_name}"); let _ = writeln!( &mut config, "\t\t\tpassword-cleartext" ); let _ = writeln!(&mut config, "\t\t"); } config.push_str("\t\n"); for (tag, protocol, url) in [ ("addressBook", "carddav", "card"), ("calendar", "caldav", "cal"), ("fileShare", "webdav", "file"), ] { let _ = writeln!(&mut config, "\t<{tag} type=\"{protocol}\">"); let _ = writeln!(&mut config, "\t\t{account_name}"); let _ = writeln!( &mut config, "\t\thttp-basic" ); let _ = writeln!( &mut config, "\t\thttps://{server_name}/dav/{url}" ); let _ = writeln!(&mut config, "\t"); } let _ = writeln!( &mut config, "\t" ); config.push_str("\n"); Ok( Resource::new("application/xml; charset=utf-8", config.into_bytes()) .into_http_response(), ) } async fn handle_autodiscover_request( &self, body: Option>, ) -> trc::Result { // Obtain parameters let emailaddress = parse_autodiscover_request(body.as_deref().unwrap_or_default()) .map_err(|err| { trc::ResourceEvent::BadParameters .into_err() .details("Failed to parse autodiscover request") .ctx(trc::Key::Reason, err) })?; let (account_name, server_name, _) = self.autoconfig_parameters(&emailaddress, true).await?; let services = self.core.storage.config.get_services().await?; // Build XML response let mut config = String::with_capacity(1024); let _ = writeln!(&mut config, ""); let _ = writeln!( &mut config, "" ); let _ = writeln!( &mut config, "\t" ); let _ = writeln!(&mut config, "\t\t"); let _ = writeln!( &mut config, "\t\t\t{emailaddress}" ); let _ = writeln!( &mut config, "\t\t\t{emailaddress}" ); // DeploymentId is a required field of User but we are not a MS Exchange server so use a random value let _ = writeln!( &mut config, "\t\t\t644560b8-a1ce-429c-8ace-23395843f701" ); let _ = writeln!(&mut config, "\t\t"); let _ = writeln!(&mut config, "\t\t"); let _ = writeln!(&mut config, "\t\t\temail"); let _ = writeln!(&mut config, "\t\t\tsettings"); for (protocol, port, is_tls) in services { match protocol.as_str() { "imap" | "pop3" => (), "smtp" if port != 25 => (), _ => continue, } let _ = writeln!(&mut config, "\t\t\t"); let _ = writeln!( &mut config, "\t\t\t\t{}", protocol.to_uppercase() ); let _ = writeln!(&mut config, "\t\t\t\t{server_name}"); let _ = writeln!(&mut config, "\t\t\t\t{port}"); let _ = writeln!(&mut config, "\t\t\t\t{account_name}"); let _ = writeln!(&mut config, "\t\t\t\ton"); let _ = writeln!(&mut config, "\t\t\t\t0"); let _ = writeln!(&mut config, "\t\t\t\t0"); let _ = writeln!( &mut config, "\t\t\t\t{}", if is_tls { "on" } else { "off" } ); if is_tls { let _ = writeln!(&mut config, "\t\t\t\tTLS"); } let _ = writeln!(&mut config, "\t\t\t\toff"); let _ = writeln!(&mut config, "\t\t\t"); } let _ = writeln!(&mut config, "\t\t"); let _ = writeln!(&mut config, "\t"); let _ = writeln!(&mut config, ""); Ok( Resource::new("application/xml; charset=utf-8", config.into_bytes()) .into_http_response(), ) } async fn autoconfig_parameters<'x>( &'x self, emailaddress: &'x str, fail_if_invalid: bool, ) -> trc::Result<(String, String, &'x str)> { // Return EMAILADDRESS let Some((_, domain)) = emailaddress.rsplit_once('@') else { return if !fail_if_invalid { Ok(( "%EMAILADDRESS%".to_string(), self.core.network.server_name.clone(), &self.core.network.report_domain, )) } else { Err(trc::ResourceEvent::BadParameters .into_err() .details("Missing domain in email address")) }; }; // Find the account name by e-mail address let mut account_name = emailaddress.into(); if let Some(id) = self .core .storage .directory .email_to_id(emailaddress) .await .caused_by(trc::location!())? && let Ok(Some(principal)) = self .core .storage .directory .query(QueryParams::id(id).with_return_member_of(false)) .await && principal .primary_email() .is_some_and(|email| email.eq_ignore_ascii_case(emailaddress)) { account_name = principal.name; } Ok((account_name, self.core.network.server_name.clone(), domain)) } } fn parse_autodiscover_request(bytes: &[u8]) -> Result { if bytes.is_empty() { return Err("Empty request body".to_string()); } let mut reader = Reader::from_reader(bytes); reader.config_mut().trim_text(true); let mut buf = Vec::with_capacity(128); 'outer: for tag_name in ["Autodiscover", "Request", "EMailAddress"] { loop { match reader.read_event_into(&mut buf) { Ok(Event::Start(e)) => { let found_tag_name = e.name(); if tag_name .as_bytes() .eq_ignore_ascii_case(found_tag_name.as_ref()) { continue 'outer; } else if tag_name == "EMailAddress" { // Skip unsupported tags under Request, such as AcceptableResponseSchema let mut tag_count = 0; loop { match reader.read_event_into(&mut buf) { Ok(Event::End(_)) => { if tag_count == 0 { break; } else { tag_count -= 1; } } Ok(Event::Start(_)) => { tag_count += 1; } Ok(Event::Eof) => { return Err(format!( "Expected value, found unexpected EOF at position {}.", reader.buffer_position() )); } _ => (), } } } else { return Err(format!( "Expected tag {}, found unexpected tag {} at position {}.", tag_name, String::from_utf8_lossy(found_tag_name.as_ref()), reader.buffer_position() )); } } Ok(Event::Decl(_) | Event::Text(_)) => (), Err(e) => { return Err(format!( "Error at position {}: {:?}", reader.buffer_position(), e )); } Ok(event) => { return Err(format!( "Expected tag {}, found unexpected event {event:?} at position {}.", tag_name, reader.buffer_position() )); } } } } if let Ok(Event::Text(text)) = reader.read_event_into(&mut buf) && let Ok(text) = text.xml_content() && text.contains('@') { return Ok(text.trim().to_lowercase()); } Err(format!( "Expected email address, found unexpected value at position {}.", reader.buffer_position() )) } #[cfg(test)] mod tests { #[test] fn parse_autodiscover() { let r = r#" email@example.com http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a "#; assert_eq!( super::parse_autodiscover_request(r.as_bytes()).unwrap(), "email@example.com" ); } } ================================================ FILE: crates/http/src/form/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::auth::oauth::FormData; use chrono::Utc; use common::{ KV_RATE_LIMIT_CONTACT, Server, config::network::{ContactForm, FieldOrDefault}, ip_to_bytes, psl, }; use email::message::delivery::{IngestMessage, IngestRecipient, LocalDeliveryStatus, MailDelivery}; use http_proto::*; use hyper::StatusCode; use mail_auth::common::cache::NoCache; use mail_builder::{ MessageBuilder, headers::{ HeaderType, address::{Address, EmailAddress}, }, mime::make_boundary, }; use serde_json::json; use std::{borrow::Cow, fmt::Write, future::Future}; use store::write::BatchBuilder; use trc::AddContext; pub trait FormHandler: Sync + Send { fn handle_contact_form( &self, session: &HttpSessionData, form: &ContactForm, form_data: FormData, ) -> impl Future> + Send; } impl FormHandler for Server { async fn handle_contact_form( &self, session: &HttpSessionData, form: &ContactForm, form_data: FormData, ) -> trc::Result { // Validate rate if let Some(rate) = &form.rate && !session.remote_ip.is_loopback() && self .core .storage .lookup .is_rate_allowed( KV_RATE_LIMIT_CONTACT, &ip_to_bytes(&session.remote_ip), rate, false, ) .await .caused_by(trc::location!())? .is_some() { return Err(trc::LimitEvent::TooManyRequests.into_err()); } // Validate honeypot if form .field_honey_pot .as_ref() .is_some_and(|field| form_data.has_field(field)) { return Err(trc::ResourceEvent::BadParameters .into_err() .details("Honey pot field present")); } // Obtain fields let from_email = form_data .get_or_default(&form.from_email) .trim() .to_lowercase(); let from_subject = form_data.get_or_default(&form.from_subject).trim(); let from_name = form_data.get_or_default(&form.from_name).trim(); // Validate email let mut failure = None; let mut has_success = false; if form.validate_domain && from_email != form.from_email.default { if let Some(domain) = from_email.rsplit_once('@').and_then(|(local, domain)| { if !local.is_empty() && domain.contains('.') && psl::domain(domain.as_bytes()).is_some_and(|d| d.suffix().typ().is_some()) { Some(domain) } else { None } }) { if self .core .smtp .resolvers .dns .mx_lookup(domain, None::<&NoCache<_, _>>) .await .is_err() { failure = Some(format!("No MX records found for domain {domain:?}. Please enter a valid email address.", ).into()); } } else { failure = Some(Cow::Borrowed("Please enter a valid email address.")); } } // Discard empty forms if failure.is_none() && form_data.fields().all(|(_, value)| value.trim().is_empty()) { failure = Some(Cow::Borrowed("Empty form")); } if failure.is_none() { // Build body let mut body = String::with_capacity(1024); for (field, value) in form_data.fields() { if !value.is_empty() { body.push_str(field); body.push_str(": "); body.push_str(value); body.push_str("\r\n"); } } let _ = write!( &mut body, "Date: {}\r\n", Utc::now().format("%a, %d %b %Y %T %z") ); let _ = write!( &mut body, "IP: {}:{}\r\n", session.remote_ip, session.remote_port ); // Build message let message = MessageBuilder::new() .from((from_name, from_email.as_str())) .header( "To", HeaderType::Address(Address::List( form.rcpt_to .iter() .map(|rcpt| { Address::Address(EmailAddress { name: None, email: rcpt.into(), }) }) .collect(), )), ) .header("Auto-Submitted", HeaderType::Text("auto-generated".into())) .message_id(format!( "<{}@{}.{}>", make_boundary("."), session.remote_ip, session.remote_port )) .subject(from_subject) .text_body(body) .write_to_vec() .unwrap_or_default(); // Reserve and write blob let (message_blob, blob_hold) = self .put_temporary_blob(u32::MAX, &message, 60) .await .caused_by(trc::location!())?; for result in self .deliver_message(IngestMessage { sender_address: from_email, sender_authenticated: false, recipients: form .rcpt_to .iter() .map(|address| IngestRecipient { address: address.clone(), is_spam: false, }) .collect(), message_blob, message_size: message.len() as u64, session_id: session.session_id, }) .await .status { match result { LocalDeliveryStatus::Success => { has_success = true; } LocalDeliveryStatus::TemporaryFailure { reason } | LocalDeliveryStatus::PermanentFailure { reason, .. } => { failure = Some(reason) } } } // Remove blob hold let mut batch = BatchBuilder::new(); batch.clear(blob_hold); self.store() .write(batch.build_all()) .await .caused_by(trc::location!())?; // Suppress errors if there is at least one success if has_success { failure = None; } } Ok(JsonResponse::with_status( if has_success { StatusCode::OK } else { StatusCode::BAD_REQUEST }, json!({ "data": { "success": has_success, "details": failure, }, }), ) .into_http_response()) } } impl FormData { pub fn get_or_default<'x>(&'x self, field: &'x FieldOrDefault) -> &'x str { if let Some(field_name) = &field.field { self.get(field_name) .filter(|f| !f.is_empty()) .unwrap_or(field.default.as_str()) } else { field.default.as_str() } } } ================================================ FILE: crates/http/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod auth; pub mod autoconfig; pub mod form; pub mod management; pub mod request; use std::sync::Arc; use common::Inner; #[derive(Clone)] pub struct HttpSessionManager { pub inner: Arc, } impl HttpSessionManager { pub fn new(inner: Arc) -> Self { Self { inner } } } ================================================ FILE: crates/http/src/management/crypto.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{Server, auth::AccessToken}; use directory::backend::internal::manage; use email::message::crypto::{ ENCRYPT_TRAIN_SPAM_FILTER, EncryptMessage, EncryptMessageError, EncryptionMethod, EncryptionParams, EncryptionType, try_parse_certs, }; use http_proto::*; use mail_builder::encoders::base64::base64_encode_mime; use mail_parser::MessageParser; use serde_json::json; use std::{future::Future, sync::Arc}; use store::{ Deserialize, Serialize, ValueKey, write::{AlignedBytes, Archive, Archiver, BatchBuilder}, }; use trc::AddContext; use types::{collection::Collection, field::PrincipalField}; pub trait CryptoHandler: Sync + Send { fn handle_crypto_get( &self, access_token: Arc, ) -> impl Future> + Send; fn handle_crypto_post( &self, access_token: Arc, body: Option>, ) -> impl Future> + Send; } impl CryptoHandler for Server { async fn handle_crypto_get(&self, access_token: Arc) -> trc::Result { let ec = if let Some(params_) = self .store() .get_value::>(ValueKey::property( access_token.primary_id(), Collection::Principal, 0, PrincipalField::EncryptionKeys, )) .await? { let params = params_ .unarchive::() .caused_by(trc::location!())?; let algo = params.algo(); let method = params.method(); let allow_spam_training = params.can_train_spam_filter(); let mut certs = Vec::new(); certs.extend_from_slice(b"-----STALWART CERTIFICATE-----\r\n"); let _ = base64_encode_mime(¶ms_.into_inner(), &mut certs, false); certs.extend_from_slice(b"\r\n"); let certs = String::from_utf8(certs).unwrap_or_default(); match method { EncryptionMethod::PGP => EncryptionType::PGP { algo, certs, allow_spam_training, }, EncryptionMethod::SMIME => EncryptionType::SMIME { algo, certs, allow_spam_training, }, } } else { EncryptionType::Disabled }; Ok(JsonResponse::new(json!({ "data": ec, })) .into_http_response()) } async fn handle_crypto_post( &self, access_token: Arc, body: Option>, ) -> trc::Result { let request = serde_json::from_slice::(body.as_deref().unwrap_or_default()) .map_err(|err| trc::ResourceEvent::BadParameters.into_err().reason(err))?; let (method, algo, mut certs, allow_spam_training) = match request { EncryptionType::PGP { algo, certs, allow_spam_training, } => (EncryptionMethod::PGP, algo, certs, allow_spam_training), EncryptionType::SMIME { algo, certs, allow_spam_training, } => (EncryptionMethod::SMIME, algo, certs, allow_spam_training), EncryptionType::Disabled => { // Disable encryption at rest let mut batch = BatchBuilder::new(); batch .with_account_id(access_token.primary_id()) .with_collection(Collection::Principal) .with_document(0) .clear(PrincipalField::EncryptionKeys); self.core.storage.data.write(batch.build_all()).await?; return Ok(JsonResponse::new(json!({ "data": (), })) .into_http_response()); } }; if !certs.ends_with("\n") { certs.push('\n'); } // Make sure Encryption is enabled if !self.core.jmap.encrypt { return Err(manage::unsupported( "Encryption-at-rest has been disabled by the system administrator", )); } // Parse certificates let certs = try_parse_certs(method, certs.into_bytes()) .map_err(|err| manage::error(err, None::))?; let num_certs = certs.len(); let params = Archiver::new(EncryptionParams { flags: method.flags() | algo.flags() | if allow_spam_training { ENCRYPT_TRAIN_SPAM_FILTER } else { 0 }, certs, }) .serialize() .caused_by(trc::location!())?; // Try a test encryption if let Err(EncryptMessageError::Error(message)) = MessageParser::new() .parse("Subject: test\r\ntest\r\n".as_bytes()) .unwrap() .encrypt( as Deserialize>::deserialize(params.as_slice())? .unarchive::()?, ) .await { return Err(manage::error(message, None::)); } // Save encryption params let mut batch = BatchBuilder::new(); batch .with_account_id(access_token.primary_id()) .with_collection(Collection::Principal) .with_document(0) .set(PrincipalField::EncryptionKeys, params); self.core.storage.data.write(batch.build_all()).await?; Ok(JsonResponse::new(json!({ "data": num_certs, })) .into_http_response()) } } ================================================ FILE: crates/http/src/management/dkim.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::str::FromStr; use common::{Server, auth::AccessToken, config::smtp::auth::simple_pem_parse}; use directory::{Permission, backend::internal::manage}; use hyper::Method; use mail_auth::{ common::crypto::{Ed25519Key, RsaKey, Sha256}, dkim::generate::DkimKeyPair, }; use mail_builder::encoders::base64::base64_encode; use mail_parser::DateTime; use pkcs8::Document; use rsa::pkcs1::DecodeRsaPublicKey; use serde::{Deserialize, Serialize}; use serde_json::json; use store::write::now; use http_proto::{request::decode_path_element, *}; use std::future::Future; #[derive(Debug, Serialize, Deserialize, Copy, Clone, PartialEq, Eq)] pub enum Algorithm { Rsa, Ed25519, } #[derive(Debug, Serialize, Deserialize)] struct DkimSignature { id: Option, algorithm: Algorithm, domain: String, selector: Option, } pub trait DkimManagement: Sync + Send { fn handle_manage_dkim( &self, req: &HttpRequest, path: Vec<&str>, body: Option>, access_token: &AccessToken, ) -> impl Future> + Send; fn handle_get_public_key( &self, path: Vec<&str>, ) -> impl Future> + Send; fn handle_create_signature( &self, body: Option>, ) -> impl Future> + Send; fn create_dkim_key( &self, algo: Algorithm, id: impl AsRef + Send, domain: impl Into + Send, selector: impl Into + Send, ) -> impl Future> + Send; } impl DkimManagement for Server { async fn handle_manage_dkim( &self, req: &HttpRequest, path: Vec<&str>, body: Option>, access_token: &AccessToken, ) -> trc::Result { match *req.method() { Method::GET => { // Validate the access token access_token.assert_has_permission(Permission::DkimSignatureGet)?; self.handle_get_public_key(path).await } Method::POST => { // Validate the access token access_token.assert_has_permission(Permission::DkimSignatureCreate)?; self.handle_create_signature(body).await } _ => Err(trc::ResourceEvent::NotFound.into_err()), } } async fn handle_get_public_key(&self, path: Vec<&str>) -> trc::Result { let signature_id = match path.get(1) { Some(signature_id) => decode_path_element(signature_id), None => { return Err(trc::ResourceEvent::NotFound.into_err()); } }; let (pk, algo) = match ( self.core .storage .config .get(&format!("signature.{signature_id}.private-key")) .await, self.core .storage .config .get(&format!("signature.{signature_id}.algorithm")) .await .map(|algo| algo.and_then(|algo| algo.parse::().ok())), ) { (Ok(Some(pk)), Ok(Some(algorithm))) => (pk, algorithm), (Err(err), _) | (_, Err(err)) => return Err(err.caused_by(trc::location!())), _ => return Err(trc::ResourceEvent::NotFound.into_err()), }; Ok(JsonResponse::new(json!({ "data": obtain_dkim_public_key(algo, &pk)?, })) .into_http_response()) } async fn handle_create_signature(&self, body: Option>) -> trc::Result { let request = match serde_json::from_slice::(body.as_deref().unwrap_or_default()) { Ok(request) => request, Err(err) => { return Err( trc::EventType::Resource(trc::ResourceEvent::BadParameters).reason(err) ); } }; let algo_str = match request.algorithm { Algorithm::Rsa => "rsa", Algorithm::Ed25519 => "ed25519", }; let id = request .id .unwrap_or_else(|| format!("{algo_str}-{}", request.domain)); let selector = request.selector.unwrap_or_else(|| { let dt = DateTime::from_timestamp(now() as i64); format!( "{:04}{:02}{}", dt.year, dt.month, if Algorithm::Rsa == request.algorithm { "r" } else { "e" } ) }); // Make sure the signature does not exist already if let Some(value) = self .core .storage .config .get(&format!("signature.{id}.private-key")) .await? { return Err(manage::err_exists( format!("signature.{id}.private-key"), value, )); } // Create signature self.create_dkim_key(request.algorithm, id, request.domain, selector) .await?; Ok(JsonResponse::new(json!({ "data": (), })) .into_http_response()) } async fn create_dkim_key( &self, algo: Algorithm, id: impl AsRef, domain: impl Into, selector: impl Into, ) -> trc::Result<()> { let id = id.as_ref(); let (algorithm, pk_type) = match algo { Algorithm::Rsa => ("rsa-sha256", "RSA PRIVATE KEY"), Algorithm::Ed25519 => ("ed25519-sha256", "PRIVATE KEY"), }; let mut pk = format!("-----BEGIN {pk_type}-----\n").into_bytes(); let mut lf_count = 65; for ch in base64_encode( match algo { Algorithm::Rsa => DkimKeyPair::generate_rsa(2048), Algorithm::Ed25519 => DkimKeyPair::generate_ed25519(), } .map_err(|err| { manage::error("Failed to generate key", err.to_string().into()) .caused_by(trc::location!()) })? .private_key(), ) .unwrap_or_default() { pk.push(ch); lf_count -= 1; if lf_count == 0 { pk.push(b'\n'); lf_count = 65; } } if lf_count != 65 { pk.push(b'\n'); } pk.extend_from_slice(format!("-----END {pk_type}-----\n").as_bytes()); self.core .storage .config .set( [ ( format!("signature.{id}.private-key"), String::from_utf8(pk).unwrap(), ), (format!("signature.{id}.domain"), domain.into()), (format!("signature.{id}.selector"), selector.into()), (format!("signature.{id}.algorithm"), algorithm.to_string()), ( format!("signature.{id}.canonicalization"), "relaxed/relaxed".to_string(), ), (format!("signature.{id}.headers.0"), "From".to_string()), (format!("signature.{id}.headers.1"), "To".to_string()), (format!("signature.{id}.headers.2"), "Date".to_string()), (format!("signature.{id}.headers.3"), "Subject".to_string()), ( format!("signature.{id}.headers.4"), "Message-ID".to_string(), ), (format!("signature.{id}.report"), "false".to_string()), ], true, ) .await } } pub fn obtain_dkim_public_key(algo: Algorithm, pk: &str) -> trc::Result { match simple_pem_parse(pk) { Some(der) => match algo { Algorithm::Rsa => match RsaKey::::from_der(&der).and_then(|key| { Document::from_pkcs1_der(&key.public_key()) .map_err(|err| mail_auth::Error::CryptoError(err.to_string())) }) { Ok(pk) => Ok( String::from_utf8(base64_encode(pk.as_bytes()).unwrap_or_default()) .unwrap_or_default(), ), Err(err) => Err(manage::error( "Failed to read RSA DER", err.to_string().into(), )), }, Algorithm::Ed25519 => { match Ed25519Key::from_pkcs8_maybe_unchecked_der(&der) .map_err(|err| mail_auth::Error::CryptoError(err.to_string())) { Ok(pk) => Ok(String::from_utf8( base64_encode(&pk.public_key()).unwrap_or_default(), ) .unwrap_or_default()), Err(err) => Err(manage::error("Crypto error", err.to_string().into())), } } }, None => Err(manage::error("Failed to decode private key", None::)), } } impl FromStr for Algorithm { type Err = (); fn from_str(s: &str) -> Result { match s.split_once('-').map(|(algo, _)| algo) { Some("rsa") => Ok(Algorithm::Rsa), Some("ed25519") => Ok(Algorithm::Ed25519), _ => Err(()), } } } ================================================ FILE: crates/http/src/management/dns.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{Server, auth::AccessToken}; use directory::{ Permission, backend::internal::manage::{self}, }; use hyper::Method; use serde::{Deserialize, Serialize}; use serde_json::json; use sha1::Digest; use utils::config::Config; use x509_parser::parse_x509_certificate; use crate::management::dkim::{Algorithm, obtain_dkim_public_key}; use http_proto::{request::decode_path_element, *}; use std::future::Future; #[derive(Debug, Serialize, Deserialize)] pub struct DnsRecord { #[serde(rename = "type")] typ: String, name: String, content: String, } pub trait DnsManagement: Sync + Send { fn handle_manage_dns( &self, req: &HttpRequest, path: Vec<&str>, access_token: &AccessToken, ) -> impl Future> + Send; fn build_dns_records( &self, domain_name: &str, ) -> impl Future>> + Send; } impl DnsManagement for Server { async fn handle_manage_dns( &self, req: &HttpRequest, path: Vec<&str>, access_token: &AccessToken, ) -> trc::Result { match ( path.get(1).copied().unwrap_or_default(), path.get(2), req.method(), ) { ("records", Some(domain), &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::DomainGet)?; // Obtain DNS records let domain = decode_path_element(domain); Ok(JsonResponse::new(json!({ "data": self.build_dns_records(domain.as_ref()).await?, })) .into_http_response()) } _ => Err(trc::ResourceEvent::NotFound.into_err()), } } async fn build_dns_records(&self, domain_name: &str) -> trc::Result> { // Obtain server name let server_name = &self.core.network.server_name; let mut records = Vec::new(); // Obtain DKIM keys let mut keys = Config::default(); let mut signature_ids = Vec::new(); let mut has_macros = false; for (key, value) in self.core.storage.config.list("signature.", true).await? { match key.strip_suffix(".domain") { Some(key_id) if value == domain_name => { signature_ids.push(key_id.to_string()); } _ => (), } if !has_macros && value.contains("%{") { has_macros = true; } keys.keys.insert(key, value); } // Add MX and CNAME records records.push(DnsRecord { typ: "MX".to_string(), name: format!("{domain_name}."), content: format!("10 {server_name}."), }); if server_name.strip_prefix("mail.") != Some(domain_name) { records.push(DnsRecord { typ: "CNAME".to_string(), name: format!("mail.{domain_name}."), content: format!("{server_name}."), }); } // Process DKIM keys if has_macros { keys.resolve_macros(&["env", "file", "cfg"]).await; keys.log_errors(); } for signature_id in signature_ids { if let (Some(algo), Some(pk), Some(selector)) = ( keys.value(format!("{signature_id}.algorithm")) .and_then(|algo| algo.parse::().ok()), keys.value(format!("{signature_id}.private-key")), keys.value(format!("{signature_id}.selector")), ) { match obtain_dkim_public_key(algo, pk) { Ok(public) => { records.push(DnsRecord { typ: "TXT".to_string(), name: format!("{selector}._domainkey.{domain_name}.",), content: match algo { Algorithm::Rsa => format!("v=DKIM1; k=rsa; h=sha256; p={public}"), Algorithm::Ed25519 => { format!("v=DKIM1; k=ed25519; h=sha256; p={public}") } }, }); } Err(err) => { trc::error!(err); } } } } // Add SPF records if server_name.ends_with(&format!(".{domain_name}")) || server_name == domain_name { records.push(DnsRecord { typ: "TXT".to_string(), name: format!("{server_name}."), content: "v=spf1 a ra=postmaster -all".to_string(), }); } records.push(DnsRecord { typ: "TXT".to_string(), name: format!("{domain_name}."), content: "v=spf1 mx ra=postmaster -all".to_string(), }); let mut has_https = false; for (protocol, port, is_tls) in self .core .storage .config .get_services() .await .unwrap_or_default() { match (protocol.as_str(), port) { ("smtp", port @ 26..=u16::MAX) => { records.push(DnsRecord { typ: "SRV".to_string(), name: format!( "_submission{}._tcp.{domain_name}.", if is_tls { "s" } else { "" } ), content: format!("0 1 {port} {server_name}."), }); } ("imap" | "pop3", port @ 1..=u16::MAX) => { records.push(DnsRecord { typ: "SRV".to_string(), name: format!( "_{protocol}{}._tcp.{domain_name}.", if is_tls { "s" } else { "" } ), content: format!("0 1 {port} {server_name}."), }); } ("http", _) if is_tls => { has_https = true; for service in ["jmap", "caldavs", "carddavs"] { records.push(DnsRecord { typ: "SRV".to_string(), name: format!("_{service}._tcp.{domain_name}.",), content: format!("0 1 {port} {server_name}."), }); } } _ => (), } } if has_https { // Add autoconfig and autodiscover records records.push(DnsRecord { typ: "CNAME".to_string(), name: format!("autoconfig.{domain_name}."), content: format!("{server_name}."), }); records.push(DnsRecord { typ: "CNAME".to_string(), name: format!("autodiscover.{domain_name}."), content: format!("{server_name}."), }); // Add MTA-STS records if let Some(policy) = self.build_mta_sts_policy() { records.push(DnsRecord { typ: "CNAME".to_string(), name: format!("mta-sts.{domain_name}."), content: format!("{server_name}."), }); records.push(DnsRecord { typ: "TXT".to_string(), name: format!("_mta-sts.{domain_name}."), content: format!("v=STSv1; id={}", policy.id), }); } } // Add DMARC record records.push(DnsRecord { typ: "TXT".to_string(), name: format!("_dmarc.{domain_name}."), content: format!("v=DMARC1; p=reject; rua=mailto:postmaster@{domain_name}; ruf=mailto:postmaster@{domain_name}",), }); // Add TLS reporting record records.push(DnsRecord { typ: "TXT".to_string(), name: format!("_smtp._tls.{domain_name}."), content: format!("v=TLSRPTv1; rua=mailto:postmaster@{domain_name}",), }); // Add TLSA records for (name, key) in self.inner.data.tls_certificates.load().iter() { if !name.ends_with(domain_name) || name.starts_with("mta-sts.") || name.starts_with("autoconfig.") || name.starts_with("autodiscover.") { continue; } for (cert_num, cert) in key.cert.iter().enumerate() { let parsed_cert = match parse_x509_certificate(cert) { Ok((_, parsed_cert)) => parsed_cert, Err(err) => { trc::error!(manage::error( "Failed to parse certificate", err.to_string().into() )); continue; } }; let name = if !name.starts_with('.') { format!("_25._tcp.{name}.") } else { format!("_25._tcp.mail.{name}.") }; let cu = if cert_num == 0 { 3 } else { 2 }; for (s, cert) in [cert, parsed_cert.subject_pki.raw].into_iter().enumerate() { for (m, hash) in [ format!("{:x}", sha2::Sha256::digest(cert)), format!("{:x}", sha2::Sha512::digest(cert)), ] .into_iter() .enumerate() { records.push(DnsRecord { typ: "TLSA".to_string(), name: name.clone(), content: format!("{} {} {} {}", cu, s, m + 1, hash), }); } } } } Ok(records) } } ================================================ FILE: crates/http/src/management/enterprise/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: LicenseRef-SEL * * This file is subject to the Stalwart Enterprise License Agreement (SEL) and * is NOT open source software. * */ pub mod telemetry; pub mod undelete; ================================================ FILE: crates/http/src/management/enterprise/telemetry.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: LicenseRef-SEL * * This file is subject to the Stalwart Enterprise License Agreement (SEL) and * is NOT open source software. * */ use std::{ fmt::Write, time::{Duration, Instant}, }; use common::{ Server, auth::{AccessToken, oauth::GrantType}, telemetry::{ metrics::store::{Metric, MetricsStore}, tracers::store::TracingStore, }, }; use directory::{Permission, backend::internal::manage}; use http_body_util::{StreamBody, combinators::BoxBody}; use http_proto::*; use hyper::{ Method, StatusCode, body::{Bytes, Frame}, }; use mail_parser::DateTime; use serde_json::json; use std::future::Future; use store::{ ahash::{AHashMap, AHashSet}, search::{SearchComparator, SearchField, SearchFilter, SearchQuery, TracingSearchField}, write::{SearchIndex, now}, }; use trc::{ Collector, DeliveryEvent, EventType, Key, MetricType, QueueEvent, Value, ipc::{bitset::Bitset, subscriber::SubscriberBuilder}, serializers::json::JsonEventSerializer, }; use utils::{snowflake::SnowflakeIdGenerator, url_params::UrlParams}; use crate::management::Timestamp; pub trait TelemetryApi: Sync + Send { fn handle_telemetry_api_request( &self, req: &HttpRequest, path: Vec<&str>, access_token: &AccessToken, ) -> impl Future> + Send; } impl TelemetryApi for Server { async fn handle_telemetry_api_request( &self, req: &HttpRequest, path: Vec<&str>, access_token: &AccessToken, ) -> trc::Result { let params = UrlParams::new(req.uri().query()); let account_id = access_token.primary_id(); match ( path.get(1).copied().unwrap_or_default(), path.get(2).copied(), req.method(), ) { ("traces", None, &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::TracingList)?; let page: usize = params.parse("page").unwrap_or(0); let limit: usize = params.parse("limit").unwrap_or(0); let mut tracing_query = Vec::new(); tracing_query.push(SearchFilter::And); if let Some(typ) = params.parse::("type") { tracing_query.push(SearchFilter::eq(TracingSearchField::EventType, typ.code())); } if let Some(queue_id) = params.parse::("queue_id") { tracing_query.push(SearchFilter::eq(TracingSearchField::QueueId, queue_id)); } if let Some(query) = params.get("filter") { let mut buf = String::with_capacity(query.len()); let mut in_quote = false; for ch in query.chars() { if ch.is_ascii_whitespace() { if in_quote { buf.push(' '); } else if !buf.is_empty() { tracing_query.push(SearchFilter::has_keyword( TracingSearchField::Keywords, buf, )); buf = String::new(); } } else if ch == '"' { buf.push(ch); if in_quote { if !buf.is_empty() { tracing_query.push(SearchFilter::has_keyword( TracingSearchField::Keywords, buf, )); buf = String::new(); } in_quote = false; } else { in_quote = true; } } else { buf.push(ch); } } if !buf.is_empty() { tracing_query .push(SearchFilter::has_keyword(TracingSearchField::Keywords, buf)); } } let values = params.get("values").is_some(); if let Some(before) = params .parse::("before") .map(|t| t.into_inner()) .and_then(SnowflakeIdGenerator::from_timestamp) { tracing_query.push(SearchFilter::lt(SearchField::Id, before)); } if let Some(after) = params .parse::("after") .map(|t| t.into_inner()) .and_then(SnowflakeIdGenerator::from_timestamp) { tracing_query.push(SearchFilter::gt(SearchField::Id, after)); } if !tracing_query.iter().any(|f| { matches!( f, SearchFilter::Operator { field: SearchField::Tracing( TracingSearchField::Keywords | TracingSearchField::QueueId ) | SearchField::Id, .. } ) }) { tracing_query.push(SearchFilter::gt( SearchField::Id, SnowflakeIdGenerator::from_timestamp(now() - 86400).unwrap_or_default(), )); } tracing_query.push(SearchFilter::End); let store = &self .core .enterprise .as_ref() .and_then(|e| e.trace_store.as_ref()) .ok_or_else(|| manage::unsupported("No tracing store has been configured"))? .store; let span_ids = self .search_store() .query_global( SearchQuery::new(SearchIndex::Tracing) .with_filters(tracing_query) .with_comparator(SearchComparator::Field { field: SearchField::Id, ascending: false, }), ) .await?; let (total, span_ids) = if limit > 0 { let offset = page.saturating_sub(1) * limit; ( span_ids.len(), span_ids.into_iter().skip(offset).take(limit).collect(), ) } else { (span_ids.len(), span_ids) }; if values && !span_ids.is_empty() { let mut values = Vec::with_capacity(span_ids.len()); for span_id in span_ids { for event in store.get_span(span_id).await? { if matches!( event.inner.typ, EventType::Delivery(DeliveryEvent::AttemptStart) | EventType::Queue( QueueEvent::QueueMessage | QueueEvent::QueueMessageAuthenticated ) ) { values.push(event); break; } } } Ok(JsonResponse::new(json!({ "data": { "items": JsonEventSerializer::new(values).with_spans(), "total": total, }, })) .into_http_response()) } else { Ok(JsonResponse::new(json!({ "data": { "items": span_ids, "total": total, }, })) .into_http_response()) } } ("traces", Some("live"), &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::TracingLive)?; let mut key_filters = AHashMap::new(); let mut filter = None; for (key, value) in params.into_inner() { if key == "filter" { filter = value.into_owned().into(); } else if let Some(key) = Key::try_parse(key.to_ascii_lowercase().as_str()) { key_filters.insert(key, value.into_owned()); } } let (_, mut rx) = SubscriberBuilder::new("live-tracer".to_string()) .with_interests(Box::new(Bitset::all())) .with_lossy(false) .register(); let throttle = Duration::from_secs(1); let ping_interval = Duration::from_secs(30); let ping_payload = Bytes::from(format!( "event: ping\ndata: {{\"interval\": {}}}\n\n", ping_interval.as_millis() )); let mut last_ping = Instant::now(); let mut events = Vec::new(); let mut active_span_ids = AHashSet::new(); Ok(HttpResponse::new(StatusCode::OK) .with_content_type("text/event-stream") .with_cache_control("no-store") .with_stream_body(BoxBody::new(StreamBody::new( async_stream::stream! { let mut last_message = Instant::now() - throttle; let mut timeout = ping_interval; loop { match tokio::time::timeout(timeout, rx.recv()).await { Ok(Some(event_batch)) => { for event in event_batch { if (filter.is_none() && key_filters.is_empty()) || event .span_id() .is_some_and(|span_id| active_span_ids.contains(&span_id)) { events.push(event); } else { let mut matched_keys = AHashSet::new(); for (key, value) in event .keys .iter() .chain(event.inner.span.as_ref().map_or(([]).iter(), |s| s.keys.iter())) { if let Some(needle) = key_filters.get(key).or(filter.as_ref()) { let matches = match value { Value::String(haystack) => haystack.contains(needle), Value::Timestamp(haystack) => { DateTime::from_timestamp(*haystack as i64) .to_rfc3339() .contains(needle) } Value::Bool(true) => needle == "true", Value::Bool(false) => needle == "false", Value::Ipv4(haystack) => haystack.to_string().contains(needle), Value::Ipv6(haystack) => haystack.to_string().contains(needle), Value::Event(_) | Value::Array(_) | Value::UInt(_) | Value::Int(_) | Value::Float(_) | Value::Duration(_) | Value::Bytes(_) | Value::None => false, }; if matches { matched_keys.insert(*key); if filter.is_some() || matched_keys.len() == key_filters.len() { if let Some(span_id) = event.span_id() { active_span_ids.insert(span_id); } events.push(event); break; } } } } } } } Ok(None) => { break; } Err(_) => (), } timeout = if !events.is_empty() { let elapsed = last_message.elapsed(); if elapsed >= throttle { last_message = Instant::now(); yield Ok(Frame::data(Bytes::from(format!( "event: trace\ndata: {}\n\n", serde_json::to_string( &JsonEventSerializer::new(std::mem::take(&mut events)) .with_description() .with_explanation()).unwrap_or_default() )))); ping_interval } else { throttle - elapsed } } else { let elapsed = last_ping.elapsed(); if elapsed >= ping_interval { last_ping = Instant::now(); yield Ok(Frame::data(ping_payload.clone())); ping_interval } else { ping_interval - elapsed } }; } }, )))) } ("trace", id, &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::TracingGet)?; let store = &self .core .enterprise .as_ref() .and_then(|e| e.trace_store.as_ref()) .ok_or_else(|| manage::unsupported("No tracing store has been configured"))? .store; let mut events = Vec::new(); for span_id in id .or_else(|| params.get("id")) .unwrap_or_default() .split(',') { if let Ok(span_id) = span_id.parse::() { events.push( JsonEventSerializer::new(store.get_span(span_id).await?) .with_description() .with_explanation(), ); } else { events.push(JsonEventSerializer::new(Vec::new())); } } if events.len() == 1 && id.is_some() { Ok(JsonResponse::new(json!({ "data": events.into_iter().next().unwrap(), })) .into_http_response()) } else { Ok(JsonResponse::new(json!({ "data": events, })) .into_http_response()) } } ("live", Some("tracing-token"), &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::TracingLive)?; // Issue a live telemetry token valid for 60 seconds Ok(JsonResponse::new(json!({ "data": self.encode_access_token(GrantType::LiveTracing, account_id, "web", 60).await?, })) .into_http_response()) } ("live", Some("metrics-token"), &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::MetricsLive)?; // Issue a live telemetry token valid for 60 seconds Ok(JsonResponse::new(json!({ "data": self.encode_access_token(GrantType::LiveMetrics, account_id, "web", 60).await?, })) .into_http_response()) } ("metrics", None, &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::MetricsList)?; let before = params .parse::("before") .map(|t| t.into_inner()) .unwrap_or(u64::MAX); let after = params .parse::("after") .map(|t| t.into_inner()) .unwrap_or(0); let results = self .core .enterprise .as_ref() .and_then(|e| e.metrics_store.as_ref()) .ok_or_else(|| { manage::error( "No metrics store has been defined", "You need to configure a metrics store in order to use this feature." .into(), ) })? .store .query_metrics(after, before) .await?; let mut metrics = Vec::with_capacity(results.len()); for metric in results { metrics.push(match metric { Metric::Counter { id, timestamp, value, } => Metric::Counter { id: id.name(), timestamp: DateTime::from_timestamp(timestamp as i64).to_rfc3339(), value, }, Metric::Histogram { id, timestamp, count, sum, } => Metric::Histogram { id: id.name(), timestamp: DateTime::from_timestamp(timestamp as i64).to_rfc3339(), count, sum, }, Metric::Gauge { id, timestamp, value, } => Metric::Gauge { id: id.name(), timestamp: DateTime::from_timestamp(timestamp as i64).to_rfc3339(), value, }, }); } Ok(JsonResponse::new(json!({ "data": metrics, })) .into_http_response()) } ("metrics", Some("live"), &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::MetricsLive)?; let interval = Duration::from_secs( params .parse::("interval") .filter(|interval| *interval >= 1) .unwrap_or(30), ); let mut event_types = AHashSet::new(); let mut metric_types = AHashSet::new(); for metric_name in params.get("metrics").unwrap_or_default().split(',') { let metric_name = metric_name.trim(); if !metric_name.is_empty() { if let Some(event_type) = EventType::try_parse(metric_name) { event_types.insert(event_type); } else if let Some(metric_type) = MetricType::try_parse(metric_name) { metric_types.insert(metric_type); } } } // Refresh expensive metrics for metric_type in [ MetricType::QueueCount, MetricType::UserCount, MetricType::DomainCount, ] { if metric_types.contains(&metric_type) { let value = match metric_type { MetricType::QueueCount => self.total_queued_messages().await?, MetricType::UserCount => self.total_accounts().await?, MetricType::DomainCount => self.total_domains().await?, _ => unreachable!(), }; Collector::update_gauge(metric_type, value); } } Ok(HttpResponse::new(StatusCode::OK) .with_content_type("text/event-stream") .with_cache_control("no-store") .with_stream_body(BoxBody::new(StreamBody::new( async_stream::stream! { loop { let mut metrics = String::with_capacity(512); metrics.push_str("event: metrics\ndata: ["); let mut is_first = true; for counter in Collector::collect_counters(true) { if event_types.is_empty() || event_types.contains(&counter.id()) { if !is_first { metrics.push(','); } else { is_first = false; } let _ = write!( &mut metrics, "{{\"id\":\"{}\",\"type\":\"counter\",\"value\":{}}}", counter.id().name(), counter.value() ); } } for gauge in Collector::collect_gauges(true) { if metric_types.is_empty() || metric_types.contains(&gauge.id()) { if !is_first { metrics.push(','); } else { is_first = false; } let _ = write!( &mut metrics, "{{\"id\":\"{}\",\"type\":\"gauge\",\"value\":{}}}", gauge.id().name(), gauge.get() ); } } for histogram in Collector::collect_histograms(true) { if metric_types.is_empty() || metric_types.contains(&histogram.id()) { if !is_first { metrics.push(','); } else { is_first = false; } let _ = write!( &mut metrics, "{{\"id\":\"{}\",\"type\":\"histogram\",\"count\":{},\"sum\":{}}}", histogram.id().name(), histogram.count(), histogram.sum() ); } } metrics.push_str("]\n\n"); yield Ok(Frame::data(Bytes::from(metrics))); tokio::time::sleep(interval).await; } }, )))) } _ => Err(trc::ResourceEvent::NotFound.into_err()), } } } ================================================ FILE: crates/http/src/management/enterprise/undelete.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: LicenseRef-SEL * * This file is subject to the Stalwart Enterprise License Agreement (SEL) and * is NOT open source software. * */ use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; use common::{Server, enterprise::undelete::DeletedItemType}; use directory::backend::internal::manage::ManageDirectory; use email::{ mailbox::INBOX_ID, message::ingest::{EmailIngest, IngestEmail, IngestSource}, }; use http_proto::{request::decode_path_element, *}; use hyper::Method; use mail_parser::{DateTime, MessageParser}; use serde_json::json; use std::future::Future; use std::str::FromStr; use store::write::{BatchBuilder, BlobLink, BlobOp}; use trc::AddContext; use types::{blob_hash::BlobHash, collection::Collection}; use utils::url_params::UrlParams; #[derive(serde::Deserialize, serde::Serialize, Debug)] pub struct UndeleteRequest { pub hash: H, pub collection: C, #[serde(rename = "restoreTime")] pub time: T, #[serde(rename = "cancelDeletion")] #[serde(default)] pub cancel_deletion: Option, } #[derive(serde::Serialize, serde::Deserialize, PartialEq, Eq, Debug)] #[serde(tag = "type")] #[serde(rename_all = "camelCase")] pub enum UndeleteResponse { Success, NotFound, Error { reason: String }, } #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct DeletedBlobResponse { pub hash: String, pub size: u32, #[serde(rename = "deletedAt")] pub deleted_at: String, #[serde(rename = "expiresAt")] pub expires_at: String, pub item: DeletedItemResponse, } #[derive(Debug, serde::Serialize, serde::Deserialize)] #[serde(tag = "type")] #[serde(rename_all = "camelCase")] pub enum DeletedItemResponse { Email { from: Box, subject: Box, received_at: String, }, FileNode { name: Box, }, CalendarEvent { title: Box, start_time: String, }, ContactCard { name: Box, }, SieveScript { name: Box, }, } pub trait UndeleteApi: Sync + Send { fn handle_undelete_api_request( &self, req: &HttpRequest, path: Vec<&str>, body: Option>, session: &HttpSessionData, ) -> impl Future> + Send; } impl UndeleteApi for Server { async fn handle_undelete_api_request( &self, req: &HttpRequest, path: Vec<&str>, body: Option>, session: &HttpSessionData, ) -> trc::Result { match (path.get(2).copied(), req.method()) { (Some(account_name), &Method::GET) => { let account_name = decode_path_element(account_name); let account_id = self .core .storage .data .get_principal_id(account_name.as_ref()) .await? .ok_or_else(|| trc::ResourceEvent::NotFound.into_err())?; let mut deleted = self.core.list_deleted(account_id).await?; let params = UrlParams::new(req.uri().query()); let limit = params.parse::("limit").unwrap_or_default(); let mut offset = params .parse::("page") .unwrap_or_default() .saturating_sub(1) * limit; // Sort ascending by deleted_at let total = deleted.len(); deleted.sort_by(|a, b| a.item.deleted_at.cmp(&b.item.deleted_at)); let mut results = Vec::with_capacity(if limit > 0 { limit } else { total }); for blob in deleted { if offset == 0 { results.push(DeletedBlobResponse { hash: URL_SAFE_NO_PAD.encode(blob.hash.as_slice()), size: blob.item.size, deleted_at: DateTime::from_timestamp(blob.item.deleted_at as i64) .to_rfc3339(), expires_at: DateTime::from_timestamp(blob.expires_at as i64) .to_rfc3339(), item: match blob.item.typ { DeletedItemType::Email { from, subject, received_at, } => DeletedItemResponse::Email { from, subject, received_at: DateTime::from_timestamp(received_at as i64) .to_rfc3339(), }, DeletedItemType::FileNode { name } => { DeletedItemResponse::FileNode { name } } DeletedItemType::CalendarEvent { title, start_time } => { DeletedItemResponse::CalendarEvent { title, start_time: DateTime::from_timestamp(start_time as i64) .to_rfc3339(), } } DeletedItemType::ContactCard { name } => { DeletedItemResponse::ContactCard { name } } DeletedItemType::SieveScript { name } => { DeletedItemResponse::SieveScript { name } } }, }); if results.len() == limit { break; } } else { offset -= 1; } } Ok(JsonResponse::new(json!({ "data":{ "items": results, "total": total, }, })) .into_http_response()) } (Some(account_name), &Method::POST) => { let account_name = decode_path_element(account_name); let account_id = self .core .storage .data .get_principal_id(account_name.as_ref()) .await? .ok_or_else(|| trc::ResourceEvent::NotFound.into_err())?; let requests: Vec> = match serde_json::from_slice::< Option>>, >(body.as_deref().unwrap_or_default()) { Ok(Some(requests)) => requests .into_iter() .map(|request| { UndeleteRequest { hash: BlobHash::try_from_hash_slice( URL_SAFE_NO_PAD .decode(request.hash.as_bytes()) .ok()? .as_slice(), ) .ok()?, collection: Collection::from_str(request.collection.as_str()) .ok()?, time: DateTime::parse_rfc3339(request.time.as_str())? .to_timestamp() as u64, cancel_deletion: if let Some(cancel_deletion) = request.cancel_deletion { (DateTime::parse_rfc3339(cancel_deletion.as_str())? .to_timestamp() as u64) .into() } else { None }, } .into() }) .collect::>>() .ok_or_else(|| trc::ResourceEvent::BadParameters.into_err())?, Ok(None) => { let deleted = self.core.list_deleted(account_id).await?; let mut results = Vec::with_capacity(deleted.len()); for blob in deleted { results.push(UndeleteRequest { hash: blob.hash, collection: match blob.item.typ { DeletedItemType::Email { .. } => Collection::Email, DeletedItemType::FileNode { .. } => Collection::FileNode, DeletedItemType::CalendarEvent { .. } => { Collection::CalendarEvent } DeletedItemType::ContactCard { .. } => { Collection::ContactCard } DeletedItemType::SieveScript { .. } => { Collection::SieveScript } }, time: blob.item.deleted_at, cancel_deletion: blob.expires_at.into(), }); } results } Err(_) => { return Err(trc::ResourceEvent::BadParameters.into_err()); } }; let access_token = self .get_access_token(account_id) .await .caused_by(trc::location!())?; let mut results = Vec::with_capacity(requests.len()); let mut batch = BatchBuilder::new(); batch.with_account_id(account_id); for request in requests { match request.collection { Collection::Email => { match self .blob_store() .get_blob(request.hash.as_slice(), 0..usize::MAX) .await? { Some(bytes) => { match self .email_ingest(IngestEmail { raw_message: &bytes, message: MessageParser::new().parse(&bytes), blob_hash: Some(&request.hash), access_token: access_token.as_ref(), mailbox_ids: vec![INBOX_ID], keywords: vec![], received_at: request.time.into(), source: IngestSource::Restore, session_id: session.session_id, }) .await { Ok(_) => { results.push(UndeleteResponse::Success); if let Some(cancel_deletion) = request.cancel_deletion { batch .clear(BlobOp::Link { hash: request.hash.clone(), to: BlobLink::Temporary { until: cancel_deletion, }, }) .clear(BlobOp::Undelete { hash: request.hash, until: cancel_deletion, }); } } Err(mut err) if err.matches(trc::EventType::MessageIngest( trc::MessageIngestEvent::Error, )) => { results.push(UndeleteResponse::Error { reason: err .take_value(trc::Key::Reason) .and_then(|v| v.into_string()) .unwrap() .to_string(), }); } Err(err) => { return Err(err.caused_by(trc::location!())); } } } None => { results.push(UndeleteResponse::NotFound); } } } _ => { results.push(UndeleteResponse::Error { reason: "Unsupported collection".to_string(), }); } } } // Commit batch if !batch.is_empty() { self.core .storage .data .write(batch.build_all()) .await .caused_by(trc::location!())?; } Ok(JsonResponse::new(json!({ "data": results, })) .into_http_response()) } _ => Err(trc::ResourceEvent::NotFound.into_err()), } } } ================================================ FILE: crates/http/src/management/log.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ fs::{self, File}, io, path::Path, }; use chrono::DateTime; use common::{Server, auth::AccessToken}; use directory::{Permission, backend::internal::manage}; use rev_lines::RevLines; use serde::Serialize; use serde_json::json; use std::future::Future; use tokio::sync::oneshot; use utils::url_params::UrlParams; use http_proto::*; #[derive(Serialize)] struct LogEntry { timestamp: String, level: String, event: String, event_id: String, details: String, } pub trait LogManagement: Sync + Send { fn handle_view_logs( &self, req: &HttpRequest, access_token: &AccessToken, ) -> impl Future> + Send; } impl LogManagement for Server { async fn handle_view_logs( &self, req: &HttpRequest, access_token: &AccessToken, ) -> trc::Result { // Validate the access token access_token.assert_has_permission(Permission::LogsView)?; let path = self .core .metrics .log_path .clone() .ok_or_else(|| manage::unsupported("Tracer log path not configured"))?; let params = UrlParams::new(req.uri().query()); let filter = params.get("filter").unwrap_or_default().to_string(); let page: usize = params.parse("page").unwrap_or(0); let limit: usize = params.parse("limit").unwrap_or(100); let offset = page.saturating_sub(1) * limit; // TODO: Use worker pool let (tx, rx) = oneshot::channel(); tokio::task::spawn_blocking(move || { let _ = tx.send(read_log_files(path, &filter, offset, limit)); }); let (total, items) = rx .await .map_err(|err| { trc::EventType::Server(trc::ServerEvent::ThreadError) .reason(err) .caused_by(trc::location!()) })? .map_err(|err| { trc::ManageEvent::Error .reason(err) .details("Failed to read log files") .caused_by(trc::location!()) })?; Ok(JsonResponse::new(json!({ "data": { "items": items, "total": total, }, })) .into_http_response()) } } fn read_log_files( path: impl AsRef, filter: &str, mut offset: usize, limit: usize, ) -> io::Result<(usize, Vec)> { let mut logs = fs::read_dir(path)?.collect::, _>>()?; let mut total = 0; // Sort the entries by file name in reverse order. logs.sort_by_key(|b| std::cmp::Reverse(b.file_name())); // Iterate and print the file names. let mut entries = Vec::with_capacity(limit); let mut logs = logs.into_iter(); while let Some(log) = logs.next() { if log.file_type()?.is_file() { let mut rev_lines = RevLines::new(File::open(log.path())?); while let Some(line) = rev_lines.next() { let line = line.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; if filter.is_empty() || line.contains(filter) { total += 1; if offset == 0 { if let Some(entry) = LogEntry::from_line(&line) { entries.push(entry); if entries.len() == limit { if rev_lines.next().is_some() || logs.next().is_some() { total += limit; } return Ok((total, entries)); } } } else { offset -= 1; } } } } } Ok((total, entries)) } impl LogEntry { fn from_line(line: &str) -> Option { let (timestamp, rest) = line.split_once(' ')?; let timestamp = DateTime::parse_from_rfc3339(timestamp).ok()?; let (level, rest) = rest.trim().split_once(' ')?; let (event, rest) = rest.trim().split_once(" (")?; let (event_id, details) = rest.split_once(")")?; Some(Self { timestamp: timestamp.to_rfc3339_opts(chrono::SecondsFormat::Secs, true), level: level.to_string(), event: event.to_string(), event_id: event_id.to_string(), details: details.trim().to_string(), }) } } ================================================ FILE: crates/http/src/management/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod crypto; pub mod dkim; pub mod dns; pub mod log; pub mod principal; pub mod queue; pub mod reload; pub mod report; pub mod settings; pub mod spam; pub mod stores; pub mod troubleshoot; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] pub mod enterprise; #[cfg(feature = "enterprise")] use enterprise::telemetry::TelemetryApi; // SPDX-SnippetEnd use crate::auth::oauth::auth::OAuthApiHandler; use common::{Server, auth::AccessToken}; use crypto::CryptoHandler; use directory::{Permission, backend::internal::manage}; use dkim::DkimManagement; use dns::DnsManagement; use http_proto::{request::fetch_body, *}; use hyper::{Method, StatusCode, header}; use jmap::api::{ToJmapHttpResponse, ToRequestError}; use jmap_proto::error::request::RequestError; use log::LogManagement; use mail_parser::DateTime; use principal::PrincipalManager; use queue::QueueManagement; use reload::ManageReload; use report::ManageReports; use serde::Serialize; use settings::ManageSettings; use spam::ManageSpamHandler; use std::future::Future; use std::{str::FromStr, sync::Arc}; use store::write::now; use stores::ManageStore; use troubleshoot::TroubleshootApi; #[derive(Serialize)] #[serde(tag = "error")] #[serde(rename_all = "camelCase")] pub enum ManagementApiError<'x> { FieldAlreadyExists { field: &'x str, value: &'x str, }, FieldMissing { field: &'x str, }, NotFound { item: &'x str, }, Unsupported { details: &'x str, }, AssertFailed, Other { details: &'x str, reason: Option<&'x str>, }, } pub trait ManagementApi: Sync + Send { fn handle_api_manage_request( &self, req: &mut HttpRequest, access_token: Arc, session: &HttpSessionData, ) -> impl Future> + Send; } impl ManagementApi for Server { #[allow(unused_variables)] async fn handle_api_manage_request( &self, req: &mut HttpRequest, access_token: Arc, session: &HttpSessionData, ) -> trc::Result { let body = fetch_body(req, 1024 * 1024, session.session_id).await; let path = req.uri().path().split('/').skip(2).collect::>(); match path.first().copied().unwrap_or_default() { "queue" => self.handle_manage_queue(req, path, &access_token).await, "settings" => { self.handle_manage_settings(req, path, body, &access_token) .await } "reports" => self.handle_manage_reports(req, path, &access_token).await, "principal" => { self.handle_manage_principal(req, path, body, &access_token) .await } "dns" => self.handle_manage_dns(req, path, &access_token).await, "store" => { self.handle_manage_store(req, path, body, session, &access_token) .await } "reload" => self.handle_manage_reload(req, path, &access_token).await, "dkim" => { self.handle_manage_dkim(req, path, body, &access_token) .await } "update" => self.handle_manage_update(req, path, &access_token).await, "logs" if req.method() == Method::GET => { self.handle_view_logs(req, &access_token).await } "spam-filter" => { self.handle_manage_spam(req, path, body, session, &access_token) .await } "restart" if req.method() == Method::GET => { // Validate the access token access_token.assert_has_permission(Permission::Restart)?; Err(manage::unsupported("Restart is not yet supported")) } "oauth" => { // Validate the access token access_token.assert_has_permission(Permission::AuthenticateOauth)?; self.handle_oauth_api_request(access_token, body).await } "account" => match (path.get(1).copied().unwrap_or_default(), req.method()) { ("crypto", &Method::POST) => { // Validate the access token access_token.assert_has_permission(Permission::ManageEncryption)?; self.handle_crypto_post(access_token, body).await } ("crypto", &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::ManageEncryption)?; self.handle_crypto_get(access_token).await } ("auth", &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::ManagePasswords)?; self.handle_account_auth_get(access_token).await } ("auth", &Method::POST) => { // Validate the access token access_token.assert_has_permission(Permission::ManagePasswords)?; self.handle_account_auth_post(req, access_token, body).await } _ => Err(trc::ResourceEvent::NotFound.into_err()), }, "troubleshoot" => { // Validate the access token access_token.assert_has_permission(Permission::Troubleshoot)?; self.handle_troubleshoot_api_request(req, path, &access_token, body) .await } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] "telemetry" => { // WARNING: TAMPERING WITH THIS FUNCTION IS STRICTLY PROHIBITED // Any attempt to modify, bypass, or disable this license validation mechanism // constitutes a severe violation of the Stalwart Enterprise License Agreement. // Such actions may result in immediate termination of your license, legal action, // and substantial financial penalties. Stalwart Labs LLC actively monitors for // unauthorized modifications and will pursue all available legal remedies against // violators to the fullest extent of the law, including but not limited to claims // for copyright infringement, breach of contract, and fraud. if self.core.is_enterprise_edition() { self.handle_telemetry_api_request(req, path, &access_token) .await } else { Err(manage::enterprise()) } } // SPDX-SnippetEnd _ => Err(trc::ResourceEvent::NotFound.into_err()), } } } pub(super) struct FutureTimestamp(u64); pub(super) struct Timestamp(u64); impl FromStr for Timestamp { type Err = (); fn from_str(s: &str) -> Result { if let Some(dt) = DateTime::parse_rfc3339(s) { Ok(Timestamp(dt.to_timestamp() as u64)) } else { Err(()) } } } impl FromStr for FutureTimestamp { type Err = (); fn from_str(s: &str) -> Result { if let Some(dt) = DateTime::parse_rfc3339(s) { let instant = dt.to_timestamp() as u64; if instant >= now() { return Ok(FutureTimestamp(instant)); } } Err(()) } } impl FutureTimestamp { pub fn into_inner(self) -> u64 { self.0 } } impl Timestamp { pub fn into_inner(self) -> u64 { self.0 } } pub trait ToManageHttpResponse { fn into_http_response(self) -> HttpResponse; } impl ToManageHttpResponse for &trc::Error { fn into_http_response(self) -> HttpResponse { match self.as_ref() { trc::EventType::Manage(cause) => { match cause { trc::ManageEvent::MissingParameter => ManagementApiError::FieldMissing { field: self.value_as_str(trc::Key::Key).unwrap_or_default(), }, trc::ManageEvent::AlreadyExists => ManagementApiError::FieldAlreadyExists { field: self.value_as_str(trc::Key::Key).unwrap_or_default(), value: self.value_as_str(trc::Key::Value).unwrap_or_default(), }, trc::ManageEvent::NotFound => ManagementApiError::NotFound { item: self.value_as_str(trc::Key::Key).unwrap_or_default(), }, trc::ManageEvent::NotSupported => ManagementApiError::Unsupported { details: self .value(trc::Key::Details) .or_else(|| self.value(trc::Key::Reason)) .and_then(|v| v.as_str()) .unwrap_or("Requested action is unsupported"), }, trc::ManageEvent::AssertFailed => ManagementApiError::AssertFailed, trc::ManageEvent::Error => ManagementApiError::Other { reason: self.value_as_str(trc::Key::Reason), details: self .value_as_str(trc::Key::Details) .unwrap_or("Unknown error"), }, } } .into_http_response(), trc::EventType::Auth( trc::AuthEvent::Failed | trc::AuthEvent::Error | trc::AuthEvent::TokenExpired, ) => HttpResponse::unauthorized(true), _ => self.to_request_error().into_http_response(), } } } pub trait UnauthorizedResponse { fn unauthorized(include_realms: bool) -> Self; } impl UnauthorizedResponse for HttpResponse { fn unauthorized(include_realms: bool) -> Self { (if include_realms { HttpResponse::new(StatusCode::UNAUTHORIZED) .with_header(header::WWW_AUTHENTICATE, "Bearer realm=\"Stalwart Server\"") .with_header(header::WWW_AUTHENTICATE, "Basic realm=\"Stalwart Server\"") } else { HttpResponse::new(StatusCode::UNAUTHORIZED) }) .with_content_type("application/problem+json") .with_text_body(serde_json::to_string(&RequestError::unauthorized()).unwrap_or_default()) } } impl ManagementApiError<'_> { fn into_http_response(self) -> HttpResponse { JsonResponse::new(self).into_http_response() } } ================================================ FILE: crates/http/src/management/principal.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::management::stores::destroy_account_data; use common::{Server, auth::AccessToken}; use directory::{ DirectoryInner, Permission, PrincipalData, QueryBy, QueryParams, Type, backend::internal::{ PrincipalAction, PrincipalField, PrincipalSet, PrincipalUpdate, PrincipalValue, lookup::DirectoryStore, manage::{ self, ChangedPrincipals, ManageDirectory, PrincipalList, UpdatePrincipal, not_found, }, }, }; use http_proto::{request::decode_path_element, *}; use hyper::{Method, header}; use serde_json::json; use std::future::Future; use std::sync::Arc; use trc::AddContext; use utils::url_params::UrlParams; #[derive(Debug, serde::Serialize, serde::Deserialize)] #[serde(tag = "type")] #[serde(rename_all = "camelCase")] pub enum AccountAuthRequest { SetPassword { password: String }, EnableOtpAuth { url: String }, DisableOtpAuth { url: Option }, AddAppPassword { name: String, password: String }, RemoveAppPassword { name: Option }, } #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct AccountAuthResponse { #[serde(rename = "otpEnabled")] pub otp_auth: bool, #[serde(rename = "appPasswords")] pub app_passwords: Vec, } pub trait PrincipalManager: Sync + Send { fn handle_manage_principal( &self, req: &HttpRequest, path: Vec<&str>, body: Option>, access_token: &AccessToken, ) -> impl Future> + Send; fn handle_account_auth_get( &self, access_token: Arc, ) -> impl Future> + Send; fn handle_account_auth_post( &self, req: &HttpRequest, access_token: Arc, body: Option>, ) -> impl Future> + Send; fn assert_supported_directory(&self, override_: bool) -> trc::Result<()>; } impl PrincipalManager for Server { async fn handle_manage_principal( &self, req: &HttpRequest, path: Vec<&str>, body: Option>, access_token: &AccessToken, ) -> trc::Result { match (path.get(1).copied(), req.method()) { (None | Some("deploy"), &Method::POST) => { // Parse principal let principal = serde_json::from_slice::(body.as_deref().unwrap_or_default()) .map_err(|err| { trc::EventType::Resource(trc::ResourceEvent::BadParameters) .from_json_error(err) })?; // Validate the access token access_token.assert_has_permission(match principal.typ() { Type::Individual => Permission::IndividualCreate, Type::Group => Permission::GroupCreate, Type::List => Permission::MailingListCreate, Type::Domain => Permission::DomainCreate, Type::Tenant => Permission::TenantCreate, Type::Role => Permission::RoleCreate, Type::ApiKey => Permission::ApiKeyCreate, Type::OauthClient => Permission::OauthClientCreate, Type::Resource | Type::Location | Type::Other => Permission::PrincipalCreate, })?; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] { if (matches!(principal.typ(), Type::Tenant) || principal.has_field(PrincipalField::Tenant)) && !self.core.is_enterprise_edition() { return Err(manage::enterprise()); } if matches!(principal.typ(), Type::Individual) && self.core.is_enterprise_edition() && !self.can_create_account().await? { return Err(manage::error( "License account limit reached", format!( "Enterprise licensed account limit reached: {} accounts licensed.", self.licensed_accounts() ) .into(), )); } } // SPDX-SnippetEnd // Make sure the current directory supports updates if matches!(principal.typ(), Type::Individual) { self.assert_supported_directory(path.get(1).copied() == Some("deploy"))?; } // Validate roles let tenant_id = access_token.tenant.map(|t| t.id); for name in principal .get_str_array(PrincipalField::Roles) .unwrap_or_default() { if let Some(pinfo) = self .store() .get_principal_info(name) .await .caused_by(trc::location!())? .filter(|v| v.typ == Type::Role && v.has_tenant_access(tenant_id)) .or_else(|| PrincipalField::Roles.map_internal_roles(name)) { let role_permissions = self.get_role_permissions(pinfo.id).await?.finalize_as_ref(); let mut allowed_permissions = role_permissions.clone(); allowed_permissions.intersection(&access_token.permissions); if allowed_permissions != role_permissions { return Err(manage::error( "Invalid role", format!("Your account cannot grant the {name:?} role").into(), )); } } } // Set default report domain if missing let report_domain = if principal.typ() == Type::Domain && self .core .storage .config .get("report.domain") .await .is_ok_and(|v| v.is_none()) { principal.name().to_lowercase().into() } else { None }; // Create principal let result = self .core .storage .data .create_principal(principal, tenant_id, Some(&access_token.permissions)) .await?; // Set report domain if let Some(report_domain) = report_domain && let Err(err) = self .core .storage .config .set([("report.domain", report_domain)], true) .await { trc::error!(err.details("Failed to set report domain")); } // Increment revision self.invalidate_principal_caches(result.changed_principals) .await; Ok(JsonResponse::new(json!({ "data": result.id, })) .into_http_response()) } (None, &Method::GET) => { // List principal ids let params = UrlParams::new(req.uri().query()); let filter = params.get("filter"); let page: usize = params.parse("page").unwrap_or(0); let limit: usize = params.parse("limit").unwrap_or(0); let count = params.get("count").is_some(); // Parse types let mut types = Vec::new(); for typ in params .get("types") .or_else(|| params.get("type")) .unwrap_or_default() .split(',') { if let Some(typ) = Type::parse(typ) && !types.contains(&typ) { types.push(typ); } } // Parse fields let mut fields = Vec::new(); for field in params.get("fields").unwrap_or_default().split(',') { if let Some(field) = PrincipalField::try_parse(field) && !fields.contains(&field) { fields.push(field); } } // Validate the access token let validate_types = if !types.is_empty() { types.as_slice() } else { &[ Type::Individual, Type::Group, Type::List, Type::Domain, Type::Tenant, Type::Role, Type::Other, Type::ApiKey, Type::OauthClient, ] }; for typ in validate_types { access_token.assert_has_permission(match typ { Type::Individual => Permission::IndividualList, Type::Group => Permission::GroupList, Type::List => Permission::MailingListList, Type::Domain => Permission::DomainList, Type::Tenant => Permission::TenantList, Type::Role => Permission::RoleList, Type::ApiKey => Permission::ApiKeyList, Type::OauthClient => Permission::OauthClientList, Type::Resource | Type::Location | Type::Other => Permission::PrincipalList, })?; } let mut tenant = access_token.tenant.map(|t| t.id); // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] { if self.core.is_enterprise_edition() { if tenant.is_none() { // Limit search to current tenant if let Some(tenant_name) = params.get("tenant") { tenant = self .core .storage .data .get_principal_info(tenant_name) .await? .filter(|p| p.typ == Type::Tenant) .map(|p| p.id); } } } else if types.contains(&Type::Tenant) { return Err(manage::enterprise()); } } // SPDX-SnippetEnd let principals = self .store() .list_principals( filter, tenant, &types, fields.len() != 1 || fields.first().is_none_or(|v| v != &PrincipalField::Name), page, limit, ) .await?; let principals: PrincipalList = if !count { let mut expanded = PrincipalList { items: Vec::with_capacity(principals.items.len()), total: principals.total, }; for principal in principals.items { expanded .items .push(self.store().map_principal(principal, &fields).await?); } expanded } else { PrincipalList { items: vec![], total: principals.total, } }; Ok(JsonResponse::new(json!({ "data": principals, })) .into_http_response()) } (None, &Method::DELETE) => { // List principal ids let params = UrlParams::new(req.uri().query()); let filter = params.get("filter"); let typ = params.parse::("type").ok_or_else(|| { trc::EventType::Resource(trc::ResourceEvent::BadParameters) .into_err() .details("Invalid type") })?; if params.get("confirm") != Some("true") { return Err(trc::EventType::Resource(trc::ResourceEvent::BadParameters) .into_err() .details("Missing confirmation parameter")); } // Validate the access token access_token.assert_has_permission(match typ { Type::Individual => Permission::IndividualDelete, Type::Group => Permission::GroupDelete, Type::List => Permission::MailingListDelete, Type::Domain => Permission::DomainDelete, Type::Tenant => Permission::TenantDelete, Type::Role => Permission::RoleDelete, Type::ApiKey => Permission::ApiKeyDelete, Type::OauthClient => Permission::OauthClientDelete, Type::Resource | Type::Location | Type::Other => Permission::PrincipalDelete, })?; let mut tenant = access_token.tenant.map(|t| t.id); // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] { if self.core.is_enterprise_edition() { if tenant.is_none() { // Limit search to current tenant if let Some(tenant_name) = params.get("tenant") { tenant = self .core .storage .data .get_principal_info(tenant_name) .await? .filter(|p| p.typ == Type::Tenant) .map(|p| p.id); } } } else if typ == Type::Tenant { return Err(manage::enterprise()); } } // SPDX-SnippetEnd let principals = self .store() .list_principals(filter, tenant, &[typ], false, 0, 0) .await?; let found = !principals.items.is_empty(); if found { let server = self.clone(); tokio::spawn(async move { for principal in principals.items { // Delete account match server .store() .delete_principal(QueryBy::Id(principal.id())) .await { Ok(changed_principals) => { // Increment revision server.invalidate_principal_caches(changed_principals).await; } Err(err) => { trc::error!(err.details("Failed to delete principal")); continue; } } if let Err(err) = destroy_account_data( &server, principal.id(), matches!(typ, Type::Individual | Type::Group), ) .await { trc::error!(err.details("Failed to delete principal")); } } }); } Ok(JsonResponse::new(json!({ "data": found, })) .into_http_response()) } (Some(name), method) => { // Fetch, update or delete principal let name = decode_path_element(name); let (account_id, typ) = self .core .storage .data .get_principal_info(name.as_ref()) .await? .filter(|p| p.has_tenant_access(access_token.tenant.map(|t| t.id))) .map(|p| (p.id, p.typ)) .ok_or_else(|| not_found(name.to_string()))?; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] { if matches!(typ, Type::Tenant) && !self.core.is_enterprise_edition() { return Err(manage::enterprise()); } } // SPDX-SnippetEnd match *method { Method::GET => { // Validate the access token access_token.assert_has_permission(match typ { Type::Individual => Permission::IndividualGet, Type::Group => Permission::GroupGet, Type::List => Permission::MailingListGet, Type::Domain => Permission::DomainGet, Type::Tenant => Permission::TenantGet, Type::Role => Permission::RoleGet, Type::ApiKey => Permission::ApiKeyGet, Type::OauthClient => Permission::OauthClientGet, Type::Resource | Type::Location | Type::Other => { Permission::PrincipalGet } })?; let principal = self .store() .query(QueryParams::id(account_id).with_return_member_of(true)) .await? .ok_or_else(|| trc::ManageEvent::NotFound.into_err())?; // Map fields let principal = self .core .storage .data .map_principal(principal, &[]) .await .caused_by(trc::location!())?; Ok(JsonResponse::new(json!({ "data": principal, })) .into_http_response()) } Method::DELETE => { // Validate the access token access_token.assert_has_permission(match typ { Type::Individual => Permission::IndividualDelete, Type::Group => Permission::GroupDelete, Type::List => Permission::MailingListDelete, Type::Domain => Permission::DomainDelete, Type::Tenant => Permission::TenantDelete, Type::Role => Permission::RoleDelete, Type::ApiKey => Permission::ApiKeyDelete, Type::OauthClient => Permission::OauthClientDelete, Type::Resource | Type::Location | Type::Other => { Permission::PrincipalDelete } })?; // Delete account let changed_principals = self .store() .delete_principal(QueryBy::Id(account_id)) .await?; if let Err(err) = destroy_account_data( self, account_id, matches!(typ, Type::Individual | Type::Group), ) .await { trc::error!(err.details("Failed to delete principal")); } // Increment revision self.invalidate_principal_caches(changed_principals).await; Ok(JsonResponse::new(json!({ "data": (), })) .into_http_response()) } Method::PATCH => { // Validate the access token let permission_needed = match typ { Type::Individual => Permission::IndividualUpdate, Type::Group => Permission::GroupUpdate, Type::List => Permission::MailingListUpdate, Type::Domain => Permission::DomainUpdate, Type::Tenant => Permission::TenantUpdate, Type::Role => Permission::RoleUpdate, Type::ApiKey => Permission::ApiKeyUpdate, Type::OauthClient => Permission::OauthClientUpdate, Type::Resource | Type::Location | Type::Other => { Permission::PrincipalUpdate } }; access_token.assert_has_permission(permission_needed)?; let changes = serde_json::from_slice::>( body.as_deref().unwrap_or_default(), ) .map_err(|err| { trc::EventType::Resource(trc::ResourceEvent::BadParameters) .from_json_error(err) })?; // Validate changes let mut invalidate_logo_cache = false; for change in &changes { match change.field { PrincipalField::Secrets | PrincipalField::Name | PrincipalField::Emails | PrincipalField::Quota | PrincipalField::UsedQuota | PrincipalField::Description | PrincipalField::Type | PrincipalField::MemberOf | PrincipalField::Members | PrincipalField::Lists | PrincipalField::Urls | PrincipalField::ExternalMembers | PrincipalField::Locale => (), PrincipalField::Picture => { invalidate_logo_cache |= matches!(typ, Type::Domain | Type::Tenant); } PrincipalField::Tenant => { // Tenants are not allowed to change their tenantId if access_token.tenant.is_some() { trc::bail!( trc::SecurityEvent::Unauthorized .into_err() .details(permission_needed.name()) .ctx( trc::Key::Reason, "Tenants cannot change their tenantId" ) ); } } PrincipalField::Roles | PrincipalField::EnabledPermissions | PrincipalField::DisabledPermissions => { if change.field == PrincipalField::Roles && matches!( change.action, PrincipalAction::AddItem | PrincipalAction::Set ) { let roles = match &change.value { PrincipalValue::String(v) => std::slice::from_ref(v), PrincipalValue::StringList(vec) => vec, PrincipalValue::Integer(_) | PrincipalValue::IntegerList(_) => continue, }; // Validate roles let tenant_id = access_token.tenant.map(|t| t.id); for name in roles { if let Some(pinfo) = self .store() .get_principal_info(name) .await .caused_by(trc::location!())? .filter(|v| { v.typ == Type::Role && v.has_tenant_access(tenant_id) }) .or_else(|| { PrincipalField::Roles.map_internal_roles(name) }) { let role_permissions = self .get_role_permissions(pinfo.id) .await? .finalize_as_ref(); let mut allowed_permissions = role_permissions.clone(); allowed_permissions .intersection(&access_token.permissions); if allowed_permissions != role_permissions { return Err(manage::error( "Invalid role", format!("Your account cannot grant the {name:?} role").into(), )); } } } } } } } // Update principal let changed_principals = self .core .storage .data .update_principal( UpdatePrincipal::by_id(account_id) .with_updates(changes) .with_tenant(access_token.tenant.map(|t| t.id)) .with_allowed_permissions(&access_token.permissions), ) .await?; // Increment revision self.invalidate_principal_caches(changed_principals).await; // Invalidate logo cache if needed if invalidate_logo_cache { self.inner.data.logos.lock().clear(); } Ok(JsonResponse::new(json!({ "data": (), })) .into_http_response()) } _ => Err(trc::ResourceEvent::NotFound.into_err()), } } _ => Err(trc::ResourceEvent::NotFound.into_err()), } } async fn handle_account_auth_get( &self, access_token: Arc, ) -> trc::Result { let mut response = AccountAuthResponse { otp_auth: false, app_passwords: Vec::new(), }; if access_token.primary_id() != u32::MAX { let principal = self .directory() .query(QueryParams::id(access_token.primary_id()).with_return_member_of(false)) .await? .ok_or_else(|| trc::ManageEvent::NotFound.into_err())?; for data in &principal.data { match data { PrincipalData::OtpAuth(_) => { response.otp_auth = true; } PrincipalData::AppPassword(secret) => { if let Some((app_name, _)) = secret.strip_prefix("$app$").and_then(|s| s.split_once('$')) { response.app_passwords.push(app_name.into()); } } _ => {} } } } Ok(JsonResponse::new(json!({ "data": response, })) .into_http_response()) } async fn handle_account_auth_post( &self, req: &HttpRequest, access_token: Arc, body: Option>, ) -> trc::Result { // Parse request let requests = serde_json::from_slice::>(body.as_deref().unwrap_or_default()) .map_err(|err| { trc::EventType::Resource(trc::ResourceEvent::BadParameters).from_json_error(err) })?; if requests.is_empty() { return Err(trc::EventType::Resource(trc::ResourceEvent::BadParameters) .into_err() .details("Empty request")); } // Make sure the user authenticated using Basic auth if requests.iter().any(|r| { matches!( r, AccountAuthRequest::DisableOtpAuth { .. } | AccountAuthRequest::EnableOtpAuth { .. } | AccountAuthRequest::SetPassword { .. } ) }) && req .headers() .get(header::AUTHORIZATION) .and_then(|h| h.to_str().ok()) .is_none_or(|header| !header.to_lowercase().starts_with("basic ")) { return Err(manage::error( "Password changes only allowed using Basic auth", None::, )); } // Handle Fallback admin password changes if access_token.primary_id() == u32::MAX { match requests.into_iter().next().unwrap() { AccountAuthRequest::SetPassword { password } => { self.core .storage .config .set( [("authentication.fallback-admin.secret", password.to_string())], true, ) .await?; // Increment revision self.invalidate_principal_caches(ChangedPrincipals::from_change( access_token.primary_id(), Type::Individual, PrincipalField::Secrets, )) .await; return Ok(JsonResponse::new(json!({ "data": (), })) .into_http_response()); } _ => { return Err(manage::error( "Fallback administrator accounts do not support 2FA or AppPasswords", None::, )); } } } // Make sure the current directory supports updates if requests.iter().any(|r| { matches!( r, AccountAuthRequest::SetPassword { .. } | AccountAuthRequest::EnableOtpAuth { .. } | AccountAuthRequest::DisableOtpAuth { .. } ) }) { self.assert_supported_directory(false)?; } // Build actions let mut actions = Vec::with_capacity(requests.len()); for request in requests { let (action, secret) = match request { AccountAuthRequest::SetPassword { password } => { actions.push(PrincipalUpdate { action: PrincipalAction::RemoveItem, field: PrincipalField::Secrets, value: PrincipalValue::String(String::new()), }); (PrincipalAction::AddItem, password) } AccountAuthRequest::EnableOtpAuth { url } => (PrincipalAction::AddItem, url), AccountAuthRequest::DisableOtpAuth { url } => ( PrincipalAction::RemoveItem, url.unwrap_or_else(|| "otpauth://".into()), ), AccountAuthRequest::AddAppPassword { name, password } => { (PrincipalAction::AddItem, format!("$app${name}${password}")) } AccountAuthRequest::RemoveAppPassword { name } => ( PrincipalAction::RemoveItem, format!("$app${}", name.unwrap_or_default()), ), }; actions.push(PrincipalUpdate { action, field: PrincipalField::Secrets, value: PrincipalValue::String(secret), }); } // Update password let changed_principals = self .core .storage .data .update_principal( UpdatePrincipal::by_id(access_token.primary_id()) .with_updates(actions) .with_tenant(access_token.tenant.map(|t| t.id)), ) .await?; // Increment revision self.invalidate_principal_caches(changed_principals).await; Ok(JsonResponse::new(json!({ "data": (), })) .into_http_response()) } fn assert_supported_directory(&self, override_: bool) -> trc::Result<()> { let class = match &self.core.storage.directory.store { DirectoryInner::Internal(_) => return Ok(()), DirectoryInner::Ldap(_) => "LDAP", DirectoryInner::Sql(_) => "SQL", DirectoryInner::Imap(_) => "IMAP", DirectoryInner::Smtp(_) => "SMTP", DirectoryInner::Memory(_) => "In-Memory", DirectoryInner::OpenId(_) => "OpenID", }; if !override_ { Err(manage::unsupported(format!( concat!( "{} directory cannot be managed. ", "Only internal directories support inserts ", "and update operations." ), class ))) } else { Ok(()) } } } ================================================ FILE: crates/http/src/management/queue.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::FutureTimestamp; use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; use common::{ Server, auth::AccessToken, config::smtp::queue::{ArchivedQueueExpiry, QueueExpiry, QueueName}, ipc::QueueEvent, }; use directory::{Permission, Type, backend::internal::manage::ManageDirectory}; use http_proto::{request::decode_path_element, *}; use hyper::Method; use mail_auth::{ dmarc::URI, mta_sts::ReportUri, report::{self, tlsrpt::TlsReport}, }; use mail_parser::DateTime; use serde::{Deserializer, Serializer}; use serde_json::json; use smtp::{ queue::{ self, ArchivedMessage, ArchivedStatus, ErrorDetails, QueueId, Status, spool::SmtpSpool, }, reporting::{dmarc::DmarcReporting, tls::TlsReporting}, }; use std::{future::Future, sync::atomic::Ordering}; use store::{ Deserialize, IterateParams, ValueKey, write::{ AlignedBytes, Archive, QueueClass, ReportEvent, ValueClass, key::DeserializeBigEndian, now, }, }; use trc::AddContext; use utils::url_params::UrlParams; #[derive(Debug, serde::Serialize, serde::Deserialize, PartialEq, Eq)] pub struct Message { pub id: QueueId, pub return_path: String, pub recipients: Vec, #[serde(deserialize_with = "deserialize_datetime")] #[serde(serialize_with = "serialize_datetime")] pub created: DateTime, pub size: u64, #[serde(skip_serializing_if = "is_zero")] #[serde(default)] pub priority: i16, #[serde(skip_serializing_if = "Option::is_none")] pub env_id: Option, pub blob_hash: String, } #[derive(Debug, serde::Serialize, serde::Deserialize, PartialEq, Eq)] pub struct Recipient { pub address: String, pub queue: String, pub status: Status, pub retry_num: u32, #[serde(skip_serializing_if = "Option::is_none")] #[serde(deserialize_with = "deserialize_maybe_datetime")] #[serde(serialize_with = "serialize_maybe_datetime")] pub next_retry: Option, #[serde(skip_serializing_if = "Option::is_none")] #[serde(deserialize_with = "deserialize_maybe_datetime")] #[serde(serialize_with = "serialize_maybe_datetime")] pub next_notify: Option, #[serde(skip_serializing_if = "Option::is_none")] #[serde(deserialize_with = "deserialize_maybe_datetime")] #[serde(serialize_with = "serialize_maybe_datetime")] pub expires: Option, #[serde(skip_serializing_if = "Option::is_none")] pub orcpt: Option, } #[derive(Debug, serde::Serialize, serde::Deserialize)] #[serde(tag = "type")] #[serde(rename_all = "camelCase")] pub enum Report { Tls { id: String, domain: String, #[serde(deserialize_with = "deserialize_datetime")] #[serde(serialize_with = "serialize_datetime")] range_from: DateTime, #[serde(deserialize_with = "deserialize_datetime")] #[serde(serialize_with = "serialize_datetime")] range_to: DateTime, report: TlsReport, rua: Vec, }, Dmarc { id: String, domain: String, #[serde(deserialize_with = "deserialize_datetime")] #[serde(serialize_with = "serialize_datetime")] range_from: DateTime, #[serde(deserialize_with = "deserialize_datetime")] #[serde(serialize_with = "serialize_datetime")] range_to: DateTime, report: report::Report, rua: Vec, }, } pub trait QueueManagement: Sync + Send { fn handle_manage_queue( &self, req: &HttpRequest, path: Vec<&str>, access_token: &AccessToken, ) -> impl Future> + Send; } impl QueueManagement for Server { async fn handle_manage_queue( &self, req: &HttpRequest, path: Vec<&str>, access_token: &AccessToken, ) -> trc::Result { let params = UrlParams::new(req.uri().query()); let mut tenant_domains: Option> = None; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL // Limit to tenant domains #[cfg(feature = "enterprise")] if self.core.is_enterprise_edition() && let Some(tenant) = access_token.tenant { tenant_domains = self .core .storage .data .list_principals(None, tenant.id.into(), &[Type::Domain], false, 0, 0) .await .map(|principals| { principals .items .into_iter() .map(|p| p.name) .collect::>() }) .caused_by(trc::location!())? .into(); } // SPDX-SnippetEnd match ( path.get(1).copied().unwrap_or_default(), path.get(2).copied().map(decode_path_element), req.method(), ) { ("messages", None, &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::MessageQueueList)?; let result = fetch_queued_messages(self, ¶ms, &tenant_domains).await?; let queue_status = self.inner.data.queue_status.load(Ordering::Relaxed); Ok(if !result.values.is_empty() { JsonResponse::new(json!({ "data":{ "items": result.values, "total": result.total, "status": queue_status, }, })) } else { JsonResponse::new(json!({ "data": { "items": result.ids, "total": result.total, "status": queue_status, }, })) } .into_http_response()) } ("messages", Some(queue_id), &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::MessageQueueGet)?; let queue_id = queue_id.parse().unwrap_or_default(); if let Some(message_) = self.read_message_archive(queue_id).await? { let message = message_.unarchive::()?; if message.is_tenant_domain(&tenant_domains) { return Ok(JsonResponse::new(json!({ "data": Message::from_archive(queue_id, message), })) .into_http_response()); } } Err(trc::ResourceEvent::NotFound.into_err()) } ("messages", None, &Method::PATCH) => { // Validate the access token access_token.assert_has_permission(Permission::MessageQueueUpdate)?; let time = params .parse::("at") .map(|t| t.into_inner()) .unwrap_or_else(now); let result = fetch_queued_messages(self, ¶ms, &tenant_domains).await?; let found = !result.ids.is_empty(); if found { let server = self.clone(); tokio::spawn(async move { for id in result.ids { if let Some(mut message) = server.read_message(id, QueueName::default()).await { let mut has_changes = false; for recipient in &mut message.message.recipients { if matches!( recipient.status, Status::Scheduled | Status::TemporaryFailure(_) ) { recipient.retry.due = time; if recipient .expiration_time(message.message.created) .is_some_and(|expires| expires > time) { recipient.expires = QueueExpiry::Attempts(recipient.retry.inner + 10); } has_changes = true; } } if has_changes { message.save_changes(&server, None).await; } } } let _ = server.inner.ipc.queue_tx.send(QueueEvent::Refresh).await; }); } Ok(JsonResponse::new(json!({ "data": found, })) .into_http_response()) } ("messages", Some(queue_id), &Method::PATCH) => { // Validate the access token access_token.assert_has_permission(Permission::MessageQueueUpdate)?; let time = params .parse::("at") .map(|t| t.into_inner()) .unwrap_or_else(now); let item = params.get("filter"); if let Some(mut message) = self .read_message(queue_id.parse().unwrap_or_default(), QueueName::default()) .await .filter(|message| { tenant_domains .as_ref() .is_none_or(|domains| message.has_domain(domains)) }) { let mut found = false; for recipient in &mut message.message.recipients { if matches!( recipient.status, Status::Scheduled | Status::TemporaryFailure(_) ) && item .as_ref() .is_none_or(|item| recipient.address().contains(item)) { recipient.retry.due = time; if recipient .expiration_time(message.message.created) .is_some_and(|expires| expires > time) { recipient.expires = QueueExpiry::Attempts(recipient.retry.inner + 10); } found = true; } } if found { message.save_changes(self, None).await; let _ = self.inner.ipc.queue_tx.send(QueueEvent::Refresh).await; } Ok(JsonResponse::new(json!({ "data": found, })) .into_http_response()) } else { Err(trc::ResourceEvent::NotFound.into_err()) } } ("messages", None, &Method::DELETE) => { // Validate the access token access_token.assert_has_permission(Permission::MessageQueueDelete)?; let result = fetch_queued_messages(self, ¶ms, &tenant_domains).await?; let found = !result.ids.is_empty(); if found { let server = self.clone(); tokio::spawn(async move { let is_active = server.inner.data.queue_status.load(Ordering::Relaxed); if is_active { let _ = server .inner .ipc .queue_tx .send(QueueEvent::Paused(true)) .await; } for id in result.ids { if let Some(message) = server.read_message(id, QueueName::default()).await { message.remove(&server, None).await; } } if is_active { let _ = server .inner .ipc .queue_tx .send(QueueEvent::Paused(false)) .await; } }); } Ok(JsonResponse::new(json!({ "data": found, })) .into_http_response()) } ("messages", Some(queue_id), &Method::DELETE) => { // Validate the access token access_token.assert_has_permission(Permission::MessageQueueDelete)?; if let Some(mut message) = self .read_message(queue_id.parse().unwrap_or_default(), QueueName::default()) .await .filter(|message| { tenant_domains .as_ref() .is_none_or(|domains| message.has_domain(domains)) }) { let mut found = false; if let Some(item) = params.get("filter") { // Cancel delivery for all recipients that match for rcpt in &mut message.message.recipients { if rcpt.address().contains(item) { rcpt.status = Status::PermanentFailure(ErrorDetails { entity: "localhost".into(), details: queue::Error::Io("Delivery canceled.".into()), }); found = true; } } if found { // Delete message if there are no pending deliveries if message.message.recipients.iter().any(|recipient| { matches!( recipient.status, Status::TemporaryFailure(_) | Status::Scheduled ) }) { message.save_changes(self, None).await; } else { message.remove(self, None).await; } } } else { message.remove(self, None).await; found = true; } Ok(JsonResponse::new(json!({ "data": found, })) .into_http_response()) } else { Err(trc::ResourceEvent::NotFound.into_err()) } } ("reports", None, &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::OutgoingReportList)?; let result = fetch_queued_reports(self, ¶ms, &tenant_domains).await?; Ok(JsonResponse::new(json!({ "data": { "items": result.ids.into_iter().map(|id| id.queue_id()).collect::>(), "total": result.total, }, })) .into_http_response()) } ("reports", Some(report_id), &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::OutgoingReportGet)?; let mut result = None; if let Some(report_id) = parse_queued_report_id(report_id.as_ref()) { match report_id { QueueClass::DmarcReportHeader(event) if tenant_domains.as_ref().is_none_or(|domains| { domains.iter().any(|dd| dd == &event.domain) }) => { let mut rua = Vec::new(); if let Some(report) = self .generate_dmarc_aggregate_report(&event, &mut rua, None, 0) .await? { result = Report::dmarc(event, report, rua).into(); } } QueueClass::TlsReportHeader(event) if tenant_domains.as_ref().is_none_or(|domains| { domains.iter().any(|dd| dd == &event.domain) }) => { let mut rua = Vec::new(); if let Some(report) = self .generate_tls_aggregate_report( std::slice::from_ref(&event), &mut rua, None, 0, ) .await? { result = Report::tls(event, report, rua).into(); } } _ => (), } } if let Some(result) = result { Ok(JsonResponse::new(json!({ "data": result, })) .into_http_response()) } else { Err(trc::ResourceEvent::NotFound.into_err()) } } ("reports", None, &Method::DELETE) => { // Validate the access token access_token.assert_has_permission(Permission::OutgoingReportDelete)?; let result = fetch_queued_reports(self, ¶ms, &tenant_domains).await?; let found = !result.ids.is_empty(); if found { let server = self.clone(); tokio::spawn(async move { for id in result.ids { match id { QueueClass::DmarcReportHeader(event) => { server.delete_dmarc_report(event).await; } QueueClass::TlsReportHeader(event) => { server.delete_tls_report(vec![event]).await; } _ => (), } } }); } Ok(JsonResponse::new(json!({ "data": found, })) .into_http_response()) } ("reports", Some(report_id), &Method::DELETE) => { // Validate the access token access_token.assert_has_permission(Permission::OutgoingReportDelete)?; if let Some(report_id) = parse_queued_report_id(report_id.as_ref()) { let result = match report_id { QueueClass::DmarcReportHeader(event) if tenant_domains.as_ref().is_none_or(|domains| { domains.iter().any(|dd| dd == &event.domain) }) => { self.delete_dmarc_report(event).await; true } QueueClass::TlsReportHeader(event) if tenant_domains.as_ref().is_none_or(|domains| { domains.iter().any(|dd| dd == &event.domain) }) => { self.delete_tls_report(vec![event]).await; true } _ => false, }; Ok(JsonResponse::new(json!({ "data": result, })) .into_http_response()) } else { Err(trc::ResourceEvent::NotFound.into_err()) } } ("status", None, &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::MessageQueueGet)?; Ok(JsonResponse::new(json!({ "data": self.inner.data.queue_status.load(Ordering::Relaxed), })) .into_http_response()) } ("status", Some(action), &Method::PATCH) => { // Validate the access token access_token.assert_has_permission(Permission::MessageQueueUpdate)?; let prev_status = self.inner.data.queue_status.load(Ordering::Relaxed); let _ = self .inner .ipc .queue_tx .send(QueueEvent::Paused(action == "stop")) .await; Ok(JsonResponse::new(json!({ "data": prev_status, })) .into_http_response()) } _ => Err(trc::ResourceEvent::NotFound.into_err()), } } } impl Message { fn from_archive(id: u64, message: &ArchivedMessage) -> Self { let now = now(); Message { id, return_path: message.return_path.to_string(), created: DateTime::from_timestamp(u64::from(message.created) as i64), size: message.size.into(), priority: message.priority.into(), env_id: message.env_id.as_ref().map(|id| id.to_string()), recipients: message .recipients .iter() .map(|rcpt| Recipient { address: rcpt.address().to_string(), queue: rcpt.queue.to_string(), status: match &rcpt.status { ArchivedStatus::Scheduled => Status::Scheduled, ArchivedStatus::Completed(status) => { Status::Completed(status.response.to_string()) } ArchivedStatus::TemporaryFailure(status) => { Status::TemporaryFailure(status.to_string()) } ArchivedStatus::PermanentFailure(status) => { Status::PermanentFailure(status.to_string()) } }, retry_num: rcpt.retry.inner.into(), next_retry: Some(DateTime::from_timestamp(u64::from(rcpt.retry.due) as i64)), next_notify: if rcpt.notify.due > now { DateTime::from_timestamp(u64::from(rcpt.notify.due) as i64).into() } else { None }, expires: if let ArchivedQueueExpiry::Ttl(time) = &rcpt.expires { DateTime::from_timestamp((u64::from(*time) + message.created) as i64).into() } else { None }, orcpt: rcpt.orcpt.as_ref().map(|orcpt| orcpt.to_string()), }) .collect(), blob_hash: URL_SAFE_NO_PAD.encode::<&[u8]>(message.blob_hash.0.as_slice()), } } } struct QueuedMessages { ids: Vec, values: Vec, total: usize, } async fn fetch_queued_messages( server: &Server, params: &UrlParams<'_>, tenant_domains: &Option>, ) -> trc::Result { let queue = params.get("queue").and_then(QueueName::new); let text = params.get("text"); let from = params.get("from"); let to = params.get("to"); let before = params .parse::("before") .map(|t| t.into_inner()); let after = params .parse::("after") .map(|t| t.into_inner()); let page = params.parse::("page").unwrap_or_default(); let limit = params.parse::("limit").unwrap_or_default(); let values = params.has_key("values"); let range_start = params.parse::("range-start").unwrap_or_default(); let range_end = params.parse::("range-end").unwrap_or(u64::MAX); let max_total = params.parse::("max-total").unwrap_or_default(); let mut result = QueuedMessages { ids: Vec::new(), values: Vec::new(), total: 0, }; let from_key = ValueKey::from(ValueClass::Queue(QueueClass::Message(range_start))); let to_key = ValueKey::from(ValueClass::Queue(QueueClass::Message(range_end))); let has_filters = text.is_some() || from.is_some() || to.is_some() || before.is_some() || after.is_some() || queue.is_some(); let mut offset = page.saturating_sub(1) * limit; let mut total_returned = 0; server .core .storage .data .iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { let message_ = as Deserialize>::deserialize(value) .add_context(|ctx| ctx.ctx(trc::Key::Key, key))?; let message = message_ .unarchive::() .add_context(|ctx| ctx.ctx(trc::Key::Key, key))?; let matches = tenant_domains .as_ref() .is_none_or(|domains| message.has_domain(domains)) && (!has_filters || (text .as_ref() .map(|text| { message.return_path.contains(text) || message .recipients .iter() .any(|r| r.address().contains(text)) }) .unwrap_or_else(|| { from.as_ref() .is_none_or(|from| message.return_path.contains(from)) && to.as_ref().is_none_or(|to| { message.recipients.iter().any(|r| r.address().contains(to)) }) }) && before.as_ref().is_none_or(|before| { message .next_delivery_event(queue) .is_some_and(|next| next < *before) }) && after.as_ref().is_none_or(|after| { message .next_delivery_event(queue) .is_some_and(|next| next > *after) }) && queue .as_ref() .is_none_or(|q| message.recipients.iter().any(|r| &r.queue == q)))); if matches { if offset == 0 { if limit == 0 || total_returned < limit { let queue_id = key.deserialize_be_u64(0)?; if values { result.values.push(Message::from_archive(queue_id, message)); } else { result.ids.push(queue_id); } total_returned += 1; } } else { offset -= 1; } result.total += 1; } Ok(max_total == 0 || result.total < max_total) }, ) .await .caused_by(trc::location!()) .map(|_| result) } struct QueuedReports { ids: Vec, total: usize, } async fn fetch_queued_reports( server: &Server, params: &UrlParams<'_>, tenant_domains: &Option>, ) -> trc::Result { let domain = params.get("domain").map(|d| d.to_lowercase()); let type_ = params.get("type").and_then(|t| match t { "dmarc" => 0u8.into(), "tls" => 1u8.into(), _ => None, }); let page: usize = params.parse("page").unwrap_or_default(); let limit: usize = params.parse("limit").unwrap_or_default(); let range_start = params.parse::("range-start").unwrap_or_default(); let range_end = params.parse::("range-end").unwrap_or(u64::MAX); let max_total = params.parse::("max-total").unwrap_or_default(); let mut result = QueuedReports { ids: Vec::new(), total: 0, }; let from_key = ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportHeader( ReportEvent { due: range_start, policy_hash: 0, seq_id: 0, domain: String::new(), }, ))); let to_key = ValueKey::from(ValueClass::Queue(QueueClass::TlsReportHeader( ReportEvent { due: range_end, policy_hash: 0, seq_id: 0, domain: String::new(), }, ))); let mut offset = page.saturating_sub(1) * limit; let mut total_returned = 0; server .core .storage .data .iterate( IterateParams::new(from_key, to_key).ascending().no_values(), |key, _| { if type_.is_none_or(|t| t == *key.last().unwrap()) { let event = ReportEvent::deserialize(key)?; if tenant_domains .as_ref() .is_none_or(|domains| domains.iter().any(|dd| dd == &event.domain)) && event.seq_id != 0 && domain.as_ref().is_none_or(|d| event.domain.contains(d)) { if offset == 0 { if limit == 0 || total_returned < limit { result.ids.push(if *key.last().unwrap() == 0 { QueueClass::DmarcReportHeader(event) } else { QueueClass::TlsReportHeader(event) }); total_returned += 1; } } else { offset -= 1; } result.total += 1; } } Ok(max_total == 0 || result.total < max_total) }, ) .await .caused_by(trc::location!()) .map(|_| result) } impl Report { fn dmarc(event: ReportEvent, report: report::Report, rua: Vec) -> Self { Self::Dmarc { domain: event.domain.clone(), range_from: DateTime::from_timestamp(event.seq_id as i64), range_to: DateTime::from_timestamp(event.due as i64), id: QueueClass::DmarcReportHeader(event).queue_id(), report, rua, } } fn tls(event: ReportEvent, report: TlsReport, rua: Vec) -> Self { Self::Tls { domain: event.domain.clone(), range_from: DateTime::from_timestamp(event.seq_id as i64), range_to: DateTime::from_timestamp(event.due as i64), id: QueueClass::TlsReportHeader(event).queue_id(), report, rua, } } } trait GenerateQueueId { fn queue_id(&self) -> String; } impl GenerateQueueId for QueueClass { fn queue_id(&self) -> String { match self { QueueClass::DmarcReportHeader(h) => { format!("d!{}!{}!{}!{}", h.domain, h.policy_hash, h.seq_id, h.due) } QueueClass::TlsReportHeader(h) => { format!("t!{}!{}!{}!{}", h.domain, h.policy_hash, h.seq_id, h.due) } _ => unreachable!(), } } } fn parse_queued_report_id(id: &str) -> Option { let mut parts = id.split('!'); let type_ = parts.next()?; let event = ReportEvent { domain: parts.next()?.to_string(), policy_hash: parts.next().and_then(|p| p.parse::().ok())?, seq_id: parts.next().and_then(|p| p.parse::().ok())?, due: parts.next().and_then(|p| p.parse::().ok())?, }; match type_ { "d" => Some(QueueClass::DmarcReportHeader(event)), "t" => Some(QueueClass::TlsReportHeader(event)), _ => None, } } fn serialize_maybe_datetime(value: &Option, serializer: S) -> Result where S: Serializer, { match value { Some(value) => serializer.serialize_some(&value.to_rfc3339()), None => serializer.serialize_none(), } } fn deserialize_maybe_datetime<'de, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, { if let Some(value) = as serde::Deserialize>::deserialize(deserializer)? { if let Some(value) = DateTime::parse_rfc3339(value) { Ok(Some(value)) } else { Err(serde::de::Error::custom( "Failed to parse RFC3339 timestamp", )) } } else { Ok(None) } } fn serialize_datetime(value: &DateTime, serializer: S) -> Result where S: Serializer, { serializer.serialize_str(&value.to_rfc3339()) } fn deserialize_datetime<'de, D>(deserializer: D) -> Result where D: Deserializer<'de>, { use serde::Deserialize; if let Some(value) = DateTime::parse_rfc3339(<&str>::deserialize(deserializer)?) { Ok(value) } else { Err(serde::de::Error::custom( "Failed to parse RFC3339 timestamp", )) } } fn is_zero(num: &i16) -> bool { *num == 0 } trait IsTenantDomain { fn is_tenant_domain(&self, tenant_domains: &Option>) -> bool; } impl IsTenantDomain for ArchivedMessage { fn is_tenant_domain(&self, tenant_domains: &Option>) -> bool { tenant_domains .as_ref() .is_none_or(|domains| self.has_domain(domains)) } } ================================================ FILE: crates/http/src/management/reload.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{ Server, auth::AccessToken, ipc::{BroadcastEvent, HousekeeperEvent}, }; use directory::Permission; use hyper::Method; use serde_json::json; use std::future::Future; use utils::url_params::UrlParams; use http_proto::*; pub trait ManageReload: Sync + Send { fn handle_manage_reload( &self, req: &HttpRequest, path: Vec<&str>, access_token: &AccessToken, ) -> impl Future> + Send; fn handle_manage_update( &self, req: &HttpRequest, path: Vec<&str>, access_token: &AccessToken, ) -> impl Future> + Send; } impl ManageReload for Server { async fn handle_manage_reload( &self, req: &HttpRequest, path: Vec<&str>, access_token: &AccessToken, ) -> trc::Result { // Validate the access token access_token.assert_has_permission(Permission::SettingsReload)?; match (path.get(1).copied(), req.method()) { (Some("lookup"), &Method::GET) => { let result = self.reload_lookups().await?; // Update core if let Some(core) = result.new_core { self.inner.shared_core.store(core.into()); } Ok(JsonResponse::new(json!({ "data": result.config, })) .into_http_response()) } (Some("certificate"), &Method::GET) => Ok(JsonResponse::new(json!({ "data": self.reload_certificates().await?.config, })) .into_http_response()), (Some("server.blocked-ip"), &Method::GET) => { let result = self.reload_blocked_ips().await?; self.cluster_broadcast(BroadcastEvent::ReloadBlockedIps) .await; Ok(JsonResponse::new(json!({ "data": result.config, })) .into_http_response()) } (_, &Method::GET) => { let result = self.reload().await?; if !UrlParams::new(req.uri().query()).has_key("dry-run") { if let Some(core) = result.new_core { // Update core self.inner.shared_core.store(core.into()); self.cluster_broadcast(BroadcastEvent::ReloadSettings).await; } if let Some(tracers) = result.tracers { // Update tracers // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] tracers.update(self.inner.shared_core.load().is_enterprise_edition()); // SPDX-SnippetEnd #[cfg(not(feature = "enterprise"))] tracers.update(false); } // Reload settings self.inner .ipc .housekeeper_tx .send(HousekeeperEvent::ReloadSettings) .await .map_err(|err| { trc::EventType::Server(trc::ServerEvent::ThreadError) .reason(err) .details(concat!( "Failed to send settings reload ", "event to housekeeper" )) .caused_by(trc::location!()) })?; } Ok(JsonResponse::new(json!({ "data": result.config, })) .into_http_response()) } _ => Err(trc::ResourceEvent::NotFound.into_err()), } } async fn handle_manage_update( &self, req: &HttpRequest, path: Vec<&str>, access_token: &AccessToken, ) -> trc::Result { match (path.get(1).copied(), req.method()) { (Some("spam-filter"), &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::SpamFilterUpdate)?; let params = UrlParams::new(req.uri().query()); let overwrite = params.has_key("overwrite"); let force = params.has_key("force"); Ok(JsonResponse::new(json!({ "data": self .core .storage .config .update_spam_rules(force, overwrite) .await? .map(|v| v.to_string()), })) .into_http_response()) } (Some("webadmin"), &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::WebadminUpdate)?; self.inner .data .webadmin .update_and_unpack(&self.core) .await?; Ok(JsonResponse::new(json!({ "data": (), })) .into_http_response()) } _ => Err(trc::ResourceEvent::NotFound.into_err()), } } } ================================================ FILE: crates/http/src/management/report.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{Server, auth::AccessToken}; use directory::{Permission, Type, backend::internal::manage::ManageDirectory}; use http_proto::{request::decode_path_element, *}; use hyper::Method; use mail_auth::report::{ Feedback, tlsrpt::{FailureDetails, Policy, TlsReport}, }; use serde_json::json; use smtp::reporting::analysis::IncomingReport; use std::future::Future; use store::{ Deserialize, IterateParams, Key, U64_LEN, ValueKey, write::{ AlignedBytes, Archive, BatchBuilder, ReportClass, ValueClass, key::DeserializeBigEndian, }, }; use trc::AddContext; use utils::url_params::UrlParams; enum ReportType { Dmarc, Tls, Arf, } pub trait ManageReports: Sync + Send { fn handle_manage_reports( &self, req: &HttpRequest, path: Vec<&str>, access_token: &AccessToken, ) -> impl Future> + Send; } impl ManageReports for Server { async fn handle_manage_reports( &self, req: &HttpRequest, path: Vec<&str>, access_token: &AccessToken, ) -> trc::Result { let mut tenant_domains: Option> = None; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL // Limit to tenant domains #[cfg(feature = "enterprise")] if self.core.is_enterprise_edition() && let Some(tenant) = access_token.tenant { tenant_domains = self .core .storage .data .list_principals(None, tenant.id.into(), &[Type::Domain], false, 0, 0) .await .map(|principals| { principals .items .into_iter() .map(|p| p.name) .collect::>() }) .caused_by(trc::location!())? .into(); } // SPDX-SnippetEnd match ( path.get(1).copied().unwrap_or_default(), path.get(2).copied().map(decode_path_element), req.method(), ) { (class @ ("dmarc" | "tls" | "arf"), None, &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::IncomingReportList)?; let params = UrlParams::new(req.uri().query()); let IncomingReports { ids, total } = fetch_incoming_reports(self, class, ¶ms, &tenant_domains).await?; Ok(JsonResponse::new(json!({ "data": { "items": ids.into_iter().map(|(id, expires)| { format!("{id}_{expires}") }).collect::>(), "total": total, }, })) .into_http_response()) } (class @ ("dmarc" | "tls" | "arf"), Some(report_id), &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::IncomingReportGet)?; if let Some(report_id) = parse_incoming_report_id(class, report_id.as_ref()) { match &report_id { ReportClass::Tls { .. } => match fetch_report::>( self, ValueKey::from(ValueClass::Report(report_id)), ) .await? { Some(report) if tenant_domains .as_ref() .is_none_or(|domains| report.has_domain(domains)) => { Ok(JsonResponse::new(json!({ "data": report, })) .into_http_response()) } _ => Err(trc::ResourceEvent::NotFound.into_err()), }, ReportClass::Dmarc { .. } => { match fetch_report::>( self, ValueKey::from(ValueClass::Report(report_id)), ) .await? { Some(report) if tenant_domains .as_ref() .is_none_or(|domains| report.has_domain(domains)) => { Ok(JsonResponse::new(json!({ "data": report, })) .into_http_response()) } _ => Err(trc::ResourceEvent::NotFound.into_err()), } } ReportClass::Arf { .. } => match fetch_report::>( self, ValueKey::from(ValueClass::Report(report_id)), ) .await? { Some(report) if tenant_domains .as_ref() .is_none_or(|domains| report.has_domain(domains)) => { Ok(JsonResponse::new(json!({ "data": report, })) .into_http_response()) } _ => Err(trc::ResourceEvent::NotFound.into_err()), }, } } else { Err(trc::ResourceEvent::NotFound.into_err()) } } (class @ ("dmarc" | "tls" | "arf"), None, &Method::DELETE) => { // Validate the access token access_token.assert_has_permission(Permission::IncomingReportDelete)?; let params = UrlParams::new(req.uri().query()); let IncomingReports { ids, .. } = fetch_incoming_reports(self, class, ¶ms, &tenant_domains).await?; let found = !ids.is_empty(); if found { let class = match class { "dmarc" => ReportClass::Dmarc { id: 0, expires: 0 }, "tls" => ReportClass::Tls { id: 0, expires: 0 }, "arf" => ReportClass::Arf { id: 0, expires: 0 }, _ => unreachable!(), }; let server = self.clone(); tokio::spawn(async move { let mut batch = BatchBuilder::new(); for (id, expires) in ids { let report_id = match &class { ReportClass::Dmarc { .. } => ReportClass::Dmarc { id, expires }, ReportClass::Tls { .. } => ReportClass::Tls { id, expires }, ReportClass::Arf { .. } => ReportClass::Arf { id, expires }, }; batch.clear(ValueClass::Report(report_id)); if batch.is_large_batch() { if let Err(err) = server.core.storage.data.write(batch.build_all()).await { trc::error!(err.caused_by(trc::location!())); } batch = BatchBuilder::new(); } } if !batch.is_empty() && let Err(err) = server.core.storage.data.write(batch.build_all()).await { trc::error!(err.caused_by(trc::location!())); } }); } Ok(JsonResponse::new(json!({ "data": found, })) .into_http_response()) } (class @ ("dmarc" | "tls" | "arf"), Some(report_id), &Method::DELETE) => { // Validate the access token access_token.assert_has_permission(Permission::IncomingReportDelete)?; if let Some(report_id) = parse_incoming_report_id(class, report_id.as_ref()) { if let Some(domains) = &tenant_domains { let is_tenant_report = match &report_id { ReportClass::Tls { .. } => fetch_report::>( self, ValueKey::from(ValueClass::Report(report_id.clone())), ) .await? .is_none_or(|report| report.has_domain(domains)), ReportClass::Dmarc { .. } => { fetch_report::>( self, ValueKey::from(ValueClass::Report(report_id.clone())), ) .await? .is_none_or(|report| report.has_domain(domains)) } ReportClass::Arf { .. } => fetch_report::>( self, ValueKey::from(ValueClass::Report(report_id.clone())), ) .await? .is_none_or(|report| report.has_domain(domains)), }; if !is_tenant_report { return Err(trc::ResourceEvent::NotFound.into_err()); } } let mut batch = BatchBuilder::new(); batch.clear(ValueClass::Report(report_id)); self.core.storage.data.write(batch.build_all()).await?; Ok(JsonResponse::new(json!({ "data": true, })) .into_http_response()) } else { Err(trc::ResourceEvent::NotFound.into_err()) } } _ => Err(trc::ResourceEvent::NotFound.into_err()), } } } async fn fetch_report(server: &Server, key: impl Key) -> trc::Result> where T: rkyv::Archive + for<'a> rkyv::Serialize< rkyv::api::high::HighSerializer< rkyv::util::AlignedVec, rkyv::ser::allocator::ArenaHandle<'a>, rkyv::rancor::Error, >, >, T::Archived: for<'a> rkyv::bytecheck::CheckBytes> + rkyv::Deserialize>, { if let Some(tls) = server .store() .get_value::>(key) .await? { tls.deserialize::().map(Some) } else { Ok(None) } } struct IncomingReports { ids: Vec<(u64, u64)>, total: usize, } async fn fetch_incoming_reports( server: &Server, class: &str, params: &UrlParams<'_>, tenant_domains: &Option>, ) -> trc::Result { let filter = params.get("text"); let page: usize = params.parse::("page").unwrap_or_default(); let limit: usize = params.parse::("limit").unwrap_or_default(); let range_start = params.parse::("range-start").unwrap_or_default(); let range_end = params.parse::("range-end").unwrap_or(u64::MAX); let max_total = params.parse::("max-total").unwrap_or_default(); let (from_key, to_key, typ) = match class { "dmarc" => ( ValueKey::from(ValueClass::Report(ReportClass::Dmarc { id: range_start, expires: 0, })), ValueKey::from(ValueClass::Report(ReportClass::Dmarc { id: range_end, expires: u64::MAX, })), ReportType::Dmarc, ), "tls" => ( ValueKey::from(ValueClass::Report(ReportClass::Tls { id: range_start, expires: 0, })), ValueKey::from(ValueClass::Report(ReportClass::Tls { id: range_end, expires: u64::MAX, })), ReportType::Tls, ), "arf" => ( ValueKey::from(ValueClass::Report(ReportClass::Arf { id: range_start, expires: 0, })), ValueKey::from(ValueClass::Report(ReportClass::Arf { id: range_end, expires: u64::MAX, })), ReportType::Arf, ), _ => unreachable!(), }; let mut results = IncomingReports { ids: Vec::new(), total: 0, }; let mut offset = page.saturating_sub(1) * limit; let mut last_id = 0; let has_filters = filter.is_some() || tenant_domains.is_some(); server .core .storage .data .iterate( IterateParams::new(from_key, to_key) .set_values(has_filters) .descending(), |key, value| { // Skip chunked records let id = key.deserialize_be_u64(U64_LEN + 1)?; if id == last_id { return Ok(true); } last_id = id; // TODO: Support filtering chunked records (over 10MB) on FDB let matches = if has_filters { let archive = as Deserialize>::deserialize(value)?; match typ { ReportType::Dmarc => { let report = archive .deserialize::>() .caused_by(trc::location!())?; filter.is_none_or(|f| report.contains(f)) && tenant_domains .as_ref() .is_none_or(|domains| report.has_domain(domains)) } ReportType::Tls => { let report = archive .deserialize::>() .caused_by(trc::location!())?; filter.is_none_or(|f| report.contains(f)) && tenant_domains .as_ref() .is_none_or(|domains| report.has_domain(domains)) } ReportType::Arf => { let report = archive .deserialize::>() .caused_by(trc::location!())?; filter.is_none_or(|f| report.contains(f)) && tenant_domains .as_ref() .is_none_or(|domains| report.has_domain(domains)) } } } else { true }; if matches { if offset == 0 { if limit == 0 || results.ids.len() < limit { results.ids.push((id, key.deserialize_be_u64(1)?)); } } else { offset -= 1; } results.total += 1; } Ok(max_total == 0 || results.total < max_total) }, ) .await .caused_by(trc::location!()) .map(|_| results) } fn parse_incoming_report_id(class: &str, id: &str) -> Option { let mut parts = id.split('_'); let id = parts.next()?.parse().ok()?; let expires = parts.next()?.parse().ok()?; match class { "dmarc" => Some(ReportClass::Dmarc { id, expires }), "tls" => Some(ReportClass::Tls { id, expires }), "arf" => Some(ReportClass::Arf { id, expires }), _ => None, } } impl From<&str> for ReportType { fn from(s: &str) -> Self { match s { "dmarc" => Self::Dmarc, "tls" => Self::Tls, "arf" => Self::Arf, _ => unreachable!(), } } } trait Contains { fn contains(&self, text: &str) -> bool; } impl Contains for mail_auth::report::Report { fn contains(&self, text: &str) -> bool { self.domain().contains(text) || self.org_name().to_lowercase().contains(text) || self.report_id().contains(text) || self .extra_contact_info() .is_some_and(|c| c.to_lowercase().contains(text)) || self.records().iter().any(|record| record.contains(text)) } } impl Contains for mail_auth::report::Record { fn contains(&self, filter: &str) -> bool { self.envelope_from().contains(filter) || self.header_from().contains(filter) || self.envelope_to().is_some_and(|to| to.contains(filter)) || self.dkim_auth_result().iter().any(|dkim| { dkim.domain().contains(filter) || dkim.selector().contains(filter) || dkim .human_result() .as_ref() .is_some_and(|r| r.contains(filter)) }) || self.spf_auth_result().iter().any(|spf| { spf.domain().contains(filter) || spf.human_result().is_some_and(|r| r.contains(filter)) }) || self .source_ip() .is_some_and(|ip| ip.to_string().contains(filter)) } } impl Contains for TlsReport { fn contains(&self, text: &str) -> bool { self.organization_name .as_ref() .is_some_and(|o| o.to_lowercase().contains(text)) || self .contact_info .as_ref() .is_some_and(|c| c.to_lowercase().contains(text)) || self.report_id.contains(text) || self.policies.iter().any(|p| p.contains(text)) } } impl Contains for Policy { fn contains(&self, filter: &str) -> bool { self.policy.policy_domain.contains(filter) || self .policy .policy_string .iter() .any(|s| s.to_lowercase().contains(filter)) || self .policy .mx_host .iter() .any(|s| s.to_lowercase().contains(filter)) || self.failure_details.iter().any(|f| f.contains(filter)) } } impl Contains for FailureDetails { fn contains(&self, filter: &str) -> bool { self.sending_mta_ip .is_some_and(|s| s.to_string().contains(filter)) || self .receiving_ip .is_some_and(|s| s.to_string().contains(filter)) || self .receiving_mx_hostname .as_ref() .is_some_and(|s| s.contains(filter)) || self .receiving_mx_helo .as_ref() .is_some_and(|s| s.contains(filter)) || self .additional_information .as_ref() .is_some_and(|s| s.contains(filter)) || self .failure_reason_code .as_ref() .is_some_and(|s| s.contains(filter)) } } impl Contains for Feedback<'_> { fn contains(&self, text: &str) -> bool { // Check if any of the string fields contain the filter self.authentication_results() .iter() .any(|s| s.contains(text)) || self .original_envelope_id() .is_some_and(|s| s.contains(text)) || self.original_mail_from().is_some_and(|s| s.contains(text)) || self.original_rcpt_to().is_some_and(|s| s.contains(text)) || self.reported_domain().iter().any(|s| s.contains(text)) || self.reported_uri().iter().any(|s| s.contains(text)) || self.reporting_mta().is_some_and(|s| s.contains(text)) || self.user_agent().is_some_and(|s| s.contains(text)) || self.dkim_adsp_dns().is_some_and(|s| s.contains(text)) || self .dkim_canonicalized_body() .is_some_and(|s| s.contains(text)) || self .dkim_canonicalized_header() .is_some_and(|s| s.contains(text)) || self.dkim_domain().is_some_and(|s| s.contains(text)) || self.dkim_identity().is_some_and(|s| s.contains(text)) || self.dkim_selector().is_some_and(|s| s.contains(text)) || self.dkim_selector_dns().is_some_and(|s| s.contains(text)) || self.spf_dns().is_some_and(|s| s.contains(text)) || self.message().is_some_and(|s| s.contains(text)) || self.headers().is_some_and(|s| s.contains(text)) } } impl Contains for IncomingReport { fn contains(&self, text: &str) -> bool { self.from.to_lowercase().contains(text) || self.to.iter().any(|to| to.to_lowercase().contains(text)) || self.subject.to_lowercase().contains(text) || self.report.contains(text) } } ================================================ FILE: crates/http/src/management/settings.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{Server, auth::AccessToken}; use directory::Permission; use hyper::Method; use serde_json::json; use store::ahash::AHashMap; use utils::{config::ConfigKey, map::vec_map::VecMap, url_params::UrlParams}; use http_proto::{request::decode_path_element, *}; use std::future::Future; #[derive(Debug, serde::Serialize, serde::Deserialize)] #[serde(tag = "type")] #[serde(rename_all = "camelCase")] pub enum UpdateSettings { Delete { keys: Vec, }, Clear { prefix: String, #[serde(default)] filter: Option, }, Insert { prefix: Option, values: Vec<(String, String)>, assert_empty: bool, }, } pub trait ManageSettings: Sync + Send { fn handle_manage_settings( &self, req: &HttpRequest, path: Vec<&str>, body: Option>, access_token: &AccessToken, ) -> impl Future> + Send; } impl ManageSettings for Server { async fn handle_manage_settings( &self, req: &HttpRequest, path: Vec<&str>, body: Option>, access_token: &AccessToken, ) -> trc::Result { match (path.get(1).copied(), req.method()) { (Some("group"), &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::SettingsList)?; // List settings let params = UrlParams::new(req.uri().query()); let prefix = params .get("prefix") .map(|p| { if !p.ends_with('.') { format!("{p}.") } else { p.to_string() } }) .unwrap_or_default(); let suffix = params .get("suffix") .map(|s| { if !s.starts_with('.') { format!(".{s}") } else { s.to_string() } }) .unwrap_or_default(); let field = params.get("field"); let filter = params.get("filter").unwrap_or_default().to_lowercase(); let limit: usize = params.parse("limit").unwrap_or(0); let mut offset = params.parse::("page").unwrap_or(0).saturating_sub(1) * limit; let has_filter = !filter.is_empty(); let settings = self.core.storage.config.list(&prefix, true).await?; if !suffix.is_empty() && !settings.is_empty() { // Obtain record ids let mut total = 0; let mut ids = Vec::new(); for key in settings.keys() { if let Some(id) = key.strip_suffix(&suffix) && !id.is_empty() { if !has_filter { if offset == 0 { if limit == 0 || ids.len() < limit { ids.push(id); } } else { offset -= 1; } total += 1; } else { ids.push(id); } } } // Group settings by record id let mut records = Vec::new(); for id in ids { let mut record = AHashMap::new(); let prefix = format!("{id}."); record.insert("_id".to_string(), id.to_string()); for (k, v) in &settings { if let Some(k) = k.strip_prefix(&prefix) { if field.is_none_or(|field| field == k) { record.insert(k.to_string(), v.to_string()); } } else if record.len() > 1 { break; } } if has_filter { if record .iter() .any(|(_, v)| v.to_lowercase().contains(&filter)) { if offset == 0 { if limit == 0 || records.len() < limit { records.push(record); } } else { offset -= 1; } total += 1; } } else { records.push(record); } } Ok(JsonResponse::new(json!({ "data": { "total": total, "items": records, }, })) .into_http_response()) } else { let mut total = 0; let mut items = Vec::new(); for (k, v) in settings { if filter.is_empty() || k.to_lowercase().contains(&filter) || v.to_lowercase().contains(&filter) { if offset == 0 { if limit == 0 || items.len() < limit { let k = k.strip_prefix(&prefix).map(|k| k.to_string()).unwrap_or(k); items.push(json!({ "_id": k, "_value": v, })); } } else { offset -= 1; } total += 1; } } Ok(JsonResponse::new(json!({ "data": { "total": total, "items": items, }, })) .into_http_response()) } } (Some("list"), &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::SettingsList)?; // List settings let params = UrlParams::new(req.uri().query()); let prefix = params .get("prefix") .map(|p| { if !p.ends_with('.') { format!("{p}.") } else { p.to_string() } }) .unwrap_or_default(); let limit: usize = params.parse("limit").unwrap_or(0); let offset = params.parse::("page").unwrap_or(0).saturating_sub(1) * limit; let settings = self.core.storage.config.list(&prefix, true).await?; let total = settings.len(); let items = settings .into_iter() .skip(offset) .take(if limit == 0 { total } else { limit }) .collect::>(); Ok(JsonResponse::new(json!({ "data": { "total": total, "items": items, }, })) .into_http_response()) } (Some("keys"), &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::SettingsList)?; // Obtain keys let params = UrlParams::new(req.uri().query()); let keys = params .get("keys") .map(|s| s.split(',').collect::>()) .unwrap_or_default(); let prefixes = params .get("prefixes") .map(|s| s.split(',').collect::>()) .unwrap_or_default(); let mut results = AHashMap::with_capacity(keys.len()); for key in keys { if let Some(value) = self.core.storage.config.get(key).await? { results.insert(key.to_string(), value); } } for prefix in prefixes { let prefix = if !prefix.ends_with('.') { format!("{prefix}.") } else { prefix.to_string() }; results.extend(self.core.storage.config.list(&prefix, false).await?); } Ok(JsonResponse::new(json!({ "data": results, })) .into_http_response()) } (Some(prefix), &Method::DELETE) if !prefix.is_empty() => { // Validate the access token access_token.assert_has_permission(Permission::SettingsDelete)?; let prefix = decode_path_element(prefix); self.core.storage.config.clear(prefix.as_ref()).await?; Ok(JsonResponse::new(json!({ "data": (), })) .into_http_response()) } (None, &Method::POST) => { // Validate the access token access_token.assert_has_permission(Permission::SettingsUpdate)?; let changes = serde_json::from_slice::>( body.as_deref().unwrap_or_default(), ) .map_err(|err| { trc::EventType::Resource(trc::ResourceEvent::BadParameters).from_json_error(err) })?; for change in changes { match change { UpdateSettings::Delete { keys } => { for key in keys { self.core.storage.config.clear(key).await?; } } UpdateSettings::Clear { prefix, filter } => { if let Some(filter) = filter { for (key, value) in self.core.storage.config.list(&prefix, false).await? { if value.to_lowercase().contains(&filter) || key.to_lowercase().contains(&filter) { self.core.storage.config.clear(key).await?; } } } else { self.core.storage.config.clear_prefix(&prefix).await?; } } UpdateSettings::Insert { prefix, values, assert_empty, } => { if assert_empty { if let Some(prefix) = &prefix { if !self .core .storage .config .list(&format!("{prefix}."), true) .await? .is_empty() { return Err(trc::ManageEvent::AssertFailed.into_err()); } } else if let Some((key, _)) = values.first() && self.core.storage.config.get(key).await?.is_some() { return Err(trc::ManageEvent::AssertFailed.into_err()); } } self.core .storage .config .set( values.into_iter().map(|(key, value)| ConfigKey { key: if let Some(prefix) = &prefix { format!("{prefix}.{key}") } else { key }, value, }), true, ) .await?; } } } Ok(JsonResponse::new(json!({ "data": (), })) .into_http_response()) } _ => Err(trc::ResourceEvent::NotFound.into_err()), } } } ================================================ FILE: crates/http/src/management/spam.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{ Server, auth::AccessToken, config::spamfilter::SpamFilterAction, manager::{SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY}, psl, }; use directory::{ Permission, backend::internal::manage::{self, ManageDirectory}, }; use email::message::ingest::EmailIngest; use http_proto::{request::decode_path_element, *}; use hyper::Method; use mail_auth::{ AuthenticatedMessage, DmarcResult, dmarc::verify::DmarcParameters, spf::verify::SpfParameters, }; use mail_parser::MessageParser; use serde::{Deserialize, Serialize}; use serde_json::json; use spam_filter::{ SpamFilterInput, analysis::{init::SpamFilterInit, score::SpamFilterAnalyzeScore}, modules::classifier::SpamClassifier, }; use std::future::Future; use std::net::IpAddr; use store::{ahash::AHashMap, write::BatchBuilder}; pub trait ManageSpamHandler: Sync + Send { fn handle_manage_spam( &self, req: &HttpRequest, path: Vec<&str>, body: Option>, session: &HttpSessionData, access_token: &AccessToken, ) -> impl Future> + Send; } #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SpamClassifyRequest { pub message: String, // Session details pub remote_ip: IpAddr, #[serde(default)] pub ehlo_domain: String, #[serde(default)] pub authenticated_as: Option, // TLS #[serde(default)] pub is_tls: bool, // Envelope pub env_from: String, pub env_from_flags: u64, pub env_rcpt_to: Vec, } #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SpamClassifyResponse { pub score: f32, pub tags: AHashMap>, pub disposition: SpamFilterDisposition, } #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] #[serde(tag = "action")] pub enum SpamFilterDisposition { Allow { value: T }, Discard, Reject, } impl ManageSpamHandler for Server { async fn handle_manage_spam( &self, req: &HttpRequest, path: Vec<&str>, body: Option>, session: &HttpSessionData, access_token: &AccessToken, ) -> trc::Result { match (path.get(1).copied(), path.get(2).copied(), req.method()) { (Some("upload"), Some(class @ ("ham" | "spam")), &Method::POST) => { // Validate the access token access_token.assert_has_permission(Permission::SpamFilterTrain)?; let message = body.ok_or_else(|| manage::error("Failed to parse message.", None::))?; let account_id = if let Some(account) = path.get(3).copied().filter(|a| !a.is_empty()) { let principal = self .store() .get_principal_info(decode_path_element(account).as_ref()) .await? .ok_or_else(|| manage::not_found(account.to_string()))?; if access_token.tenant.is_some() && principal.tenant != access_token.tenant_id() { return Err(manage::error( "Account does not belong to this tenant.", None::, )); } principal.id } else if access_token.tenant.is_none() { u32::MAX } else { return Err(manage::error( "Account ID is required for tenants.", None::, )); }; // Write sample let (blob_hash, blob_hold) = self.put_temporary_blob(account_id, &message, 60).await?; let mut batch = BatchBuilder::new(); batch.with_account_id(account_id).clear(blob_hold); self.add_spam_sample( &mut batch, blob_hash, class == "spam", true, session.session_id, ); self.store().write(batch.build_all()).await?; Ok(JsonResponse::new(json!({ "data": (), })) .into_http_response()) } (Some("train"), request, &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::SpamFilterTrain)?; let result = match request { Some("start") | Some("reset") => { if !self.inner.ipc.train_task_controller.is_running() { let reset = matches!(request, Some("reset")); let server = self.clone(); tokio::spawn(async move { if let Err(err) = server.spam_train(reset).await { trc::error!(err.caused_by(trc::location!())); } }); true } else { false } } Some("stop") => { if self.inner.ipc.train_task_controller.is_running() { self.inner.ipc.train_task_controller.stop(); true } else { false } } Some("delete") => { for key in [SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY] { self.blob_store().delete_blob(key).await?; } true } Some("status") => self.inner.ipc.train_task_controller.is_running(), _ => { return Err(trc::ResourceEvent::NotFound.into_err()); } }; Ok(JsonResponse::new(json!({ "data": result, })) .into_http_response()) } (Some("classify"), _, &Method::POST) => { // Validate the access token access_token.assert_has_permission(Permission::SpamFilterTest)?; // Parse request let request = serde_json::from_slice::( body.as_deref().unwrap_or_default(), ) .map_err(|err| { trc::EventType::Resource(trc::ResourceEvent::BadParameters).from_json_error(err) })?; // Built spam filter input let message = MessageParser::new() .parse(request.message.as_bytes()) .filter(|m| m.root_part().headers().iter().any(|h| !h.name.is_other())) .ok_or_else(|| manage::error("Failed to parse message.", None::))?; let remote_ip = request.remote_ip; let ehlo_domain = request.ehlo_domain.to_lowercase(); let mail_from = request.env_from.to_lowercase(); let mail_from_domain = mail_from.rsplit_once('@').map(|(_, domain)| domain); let local_host = &self.core.network.server_name; let spf_ehlo_result = self.core .smtp .resolvers .dns .verify_spf(self.inner.cache.build_auth_parameters( SpfParameters::verify_ehlo(remote_ip, &ehlo_domain, local_host), )) .await; let iprev_result = self .core .smtp .resolvers .dns .verify_iprev(self.inner.cache.build_auth_parameters(remote_ip)) .await; let spf_mail_from_result = if let Some(mail_from_domain) = mail_from_domain { self.core .smtp .resolvers .dns .check_host(self.inner.cache.build_auth_parameters(SpfParameters::new( remote_ip, mail_from_domain, &ehlo_domain, local_host, &mail_from, ))) .await } else { self.core .smtp .resolvers .dns .check_host(self.inner.cache.build_auth_parameters(SpfParameters::new( remote_ip, &ehlo_domain, &ehlo_domain, local_host, &format!("postmaster@{ehlo_domain}"), ))) .await }; let auth_message = AuthenticatedMessage::from_parsed(&message, true); let dkim_output = self .core .smtp .resolvers .dns .verify_dkim(self.inner.cache.build_auth_parameters(&auth_message)) .await; let arc_output = self .core .smtp .resolvers .dns .verify_arc(self.inner.cache.build_auth_parameters(&auth_message)) .await; let dmarc_output = self .core .smtp .resolvers .dns .verify_dmarc(self.inner.cache.build_auth_parameters(DmarcParameters { message: &auth_message, dkim_output: &dkim_output, rfc5321_mail_from_domain: mail_from_domain.unwrap_or(ehlo_domain.as_str()), spf_output: &spf_mail_from_result, domain_suffix_fn: |domain| psl::domain_str(domain).unwrap_or(domain), })) .await; let dmarc_pass = matches!(dmarc_output.spf_result(), DmarcResult::Pass) || matches!(dmarc_output.dkim_result(), DmarcResult::Pass); let dmarc_result = if dmarc_pass { DmarcResult::Pass } else if dmarc_output.spf_result() != &DmarcResult::None { dmarc_output.spf_result().clone() } else if dmarc_output.dkim_result() != &DmarcResult::None { dmarc_output.dkim_result().clone() } else { DmarcResult::None }; let dmarc_policy = dmarc_output.policy(); let asn_geo = self.lookup_asn_country(remote_ip).await; let input = SpamFilterInput { message: &message, span_id: session.session_id, arc_result: Some(&arc_output), spf_ehlo_result: Some(&spf_ehlo_result), spf_mail_from_result: Some(&spf_mail_from_result), dkim_result: dkim_output.as_slice(), dmarc_result: Some(&dmarc_result), dmarc_policy: Some(&dmarc_policy), iprev_result: Some(&iprev_result), remote_ip: request.remote_ip, ehlo_domain: Some(ehlo_domain.as_str()), authenticated_as: request.authenticated_as.as_deref(), asn: asn_geo.asn.as_ref().map(|a| a.id), country: asn_geo.country.as_ref().map(|c| c.as_str()), is_tls: request.is_tls, env_from: &request.env_from, env_from_flags: request.env_from_flags, env_rcpt_to: request.env_rcpt_to.iter().map(String::as_str).collect(), is_test: true, is_train: false, }; // Classify let mut ctx = self.spam_filter_init(input); let result = self.spam_filter_classify(&mut ctx).await; // Build response let mut response = SpamClassifyResponse { score: ctx.result.score, tags: AHashMap::with_capacity(ctx.result.tags.len()), disposition: match result { SpamFilterAction::Allow(value) => SpamFilterDisposition::Allow { value: value.headers, }, SpamFilterAction::Discard => SpamFilterDisposition::Discard, SpamFilterAction::Reject => SpamFilterDisposition::Reject, SpamFilterAction::Disabled => SpamFilterDisposition::Allow { value: String::new(), }, }, }; for tag in ctx.result.tags { let disposition = match self.core.spam.lists.scores.get(&tag) { Some(SpamFilterAction::Allow(score)) => { SpamFilterDisposition::Allow { value: *score } } Some(SpamFilterAction::Discard) => SpamFilterDisposition::Discard, Some(SpamFilterAction::Reject) => SpamFilterDisposition::Reject, Some(SpamFilterAction::Disabled) | None => { SpamFilterDisposition::Allow { value: 0.0 } } }; response.tags.insert(tag, disposition); } Ok(JsonResponse::new(json!({ "data": response, })) .into_http_response()) } _ => Err(trc::ResourceEvent::NotFound.into_err()), } } } ================================================ FILE: crates/http/src/management/stores.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; use common::{ auth::AccessToken, ipc::{HousekeeperEvent, PurgeType}, manager::webadmin::Resource, storage::index::ObjectIndexBuilder, *, }; use directory::{ Permission, backend::internal::manage::{self, ManageDirectory}, }; use email::{ cache::MessageCacheFetch, message::{ ingest::EmailIngest, metadata::{MessageData, MessageMetadata}, }, sieve::SieveScript, }; use groupware::{ calendar::{Calendar, CalendarEvent, CalendarEventNotification}, contact::{AddressBook, ContactCard}, file::FileNode, }; use http_proto::{request::decode_path_element, *}; use hyper::Method; use serde_json::json; use services::task_manager::index::ReindexIndexTask; use std::future::Future; use store::{ Serialize, ValueKey, rand, search::SearchQuery, write::{ AlignedBytes, Archive, Archiver, BatchBuilder, BlobLink, BlobOp, DirectoryClass, SearchIndex, ValueClass, }, }; use trc::AddContext; use types::{ blob_hash::BlobHash, collection::Collection, field::{EmailField, Field, MailboxField}, }; use utils::url_params::UrlParams; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] use super::enterprise::undelete::UndeleteApi; // SPDX-SnippetEnd pub trait ManageStore: Sync + Send { fn handle_manage_store( &self, req: &HttpRequest, path: Vec<&str>, body: Option>, session: &HttpSessionData, access_token: &AccessToken, ) -> impl Future> + Send; fn housekeeper_request( &self, event: HousekeeperEvent, ) -> impl Future> + Send; } impl ManageStore for Server { async fn handle_manage_store( &self, req: &HttpRequest, path: Vec<&str>, body: Option>, session: &HttpSessionData, access_token: &AccessToken, ) -> trc::Result { match ( path.get(1).copied(), path.get(2).copied(), path.get(3).copied(), req.method(), ) { (Some("blobs"), Some(blob_hash), _, &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::BlobFetch)?; let blob_hash = URL_SAFE_NO_PAD .decode(decode_path_element(blob_hash).as_bytes()) .map_err(|err| { trc::EventType::Resource(trc::ResourceEvent::BadParameters) .from_base64_error(err) })?; let contents = self .core .storage .blob .get_blob(&blob_hash, 0..usize::MAX) .await? .ok_or_else(|| trc::ManageEvent::NotFound.into_err())?; let params = UrlParams::new(req.uri().query()); let offset = params.parse("offset").unwrap_or(0); let limit = params.parse("limit").unwrap_or(usize::MAX); let contents = if offset == 0 && limit == usize::MAX { contents } else { contents .get(offset..std::cmp::min(offset + limit, contents.len())) .unwrap_or_default() .to_vec() }; Ok(Resource::new("application/octet-stream", contents).into_http_response()) } (Some("purge"), Some("blob"), _, &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::PurgeBlobStore)?; self.housekeeper_request(HousekeeperEvent::Purge(PurgeType::Blobs { store: self.core.storage.data.clone(), blob_store: self.core.storage.blob.clone(), })) .await } (Some("purge"), Some("data"), id, &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::PurgeDataStore)?; let store = if let Some(id) = id.filter(|id| *id != "default") { if let Some(store) = self.core.storage.stores.get(id) { store.clone() } else { return Err(trc::ResourceEvent::NotFound.into_err()); } } else { self.core.storage.data.clone() }; self.housekeeper_request(HousekeeperEvent::Purge(PurgeType::Data(store))) .await } (Some("purge"), Some("in-memory"), id, &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::PurgeInMemoryStore)?; let store = if let Some(id) = id.filter(|id| *id != "default") { if let Some(store) = self.core.storage.lookups.get(id) { store.clone() } else { return Err(trc::ResourceEvent::NotFound.into_err()); } } else { self.core.storage.lookup.clone() }; let prefix = match path.get(4).copied() { Some("acme") => vec![KV_ACME].into(), Some("oauth") => vec![KV_OAUTH].into(), Some("rate-rcpt") => vec![KV_RATE_LIMIT_RCPT].into(), Some("rate-scan") => vec![KV_RATE_LIMIT_SCAN].into(), Some("rate-loiter") => vec![KV_RATE_LIMIT_LOITER].into(), Some("rate-auth") => vec![KV_RATE_LIMIT_AUTH].into(), Some("rate-smtp") => vec![KV_RATE_LIMIT_SMTP].into(), Some("rate-contact") => vec![KV_RATE_LIMIT_CONTACT].into(), Some("rate-http-authenticated") => { vec![KV_RATE_LIMIT_HTTP_AUTHENTICATED].into() } Some("rate-http-anonymous") => vec![KV_RATE_LIMIT_HTTP_ANONYMOUS].into(), Some("rate-imap") => vec![KV_RATE_LIMIT_IMAP].into(), Some("greylist") => vec![KV_GREYLIST].into(), Some("lock-purge-account") => vec![KV_LOCK_PURGE_ACCOUNT].into(), Some("lock-queue-message") => vec![KV_LOCK_QUEUE_MESSAGE].into(), Some("lock-queue-report") => vec![KV_LOCK_QUEUE_REPORT].into(), Some("lock-email-task") => vec![KV_LOCK_TASK].into(), Some("lock-housekeeper") => vec![KV_LOCK_HOUSEKEEPER].into(), _ => None, }; self.housekeeper_request(HousekeeperEvent::Purge(PurgeType::Lookup { store, prefix, })) .await } (Some("purge"), Some("account"), id, &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::PurgeAccount)?; let account_id = if let Some(id) = id { self.core .storage .data .get_principal_id(decode_path_element(id).as_ref()) .await? .ok_or_else(|| trc::ManageEvent::NotFound.into_err())? .into() } else { None }; self.housekeeper_request(HousekeeperEvent::Purge(PurgeType::Account { account_id, use_roles: false, })) .await } (Some("reindex"), Some(index), id, &Method::GET) => { // Validate the access token access_token.assert_has_permission(Permission::FtsReindex)?; let account_id = if let Some(id) = id { self.core .storage .data .get_principal_id(decode_path_element(id).as_ref()) .await? .ok_or_else(|| trc::ManageEvent::NotFound.into_err())? .into() } else { None }; let tenant_id = access_token.tenant.map(|t| t.id); let index = SearchIndex::try_from_str(index).ok_or_else(|| { trc::ResourceEvent::BadParameters.reason("Invalid search index specified") })?; let jmap = self.clone(); tokio::spawn(async move { if let Err(err) = jmap.reindex(index, account_id, tenant_id).await { trc::error!(err.details("Failed to reindex FTS")); } }); Ok(JsonResponse::new(json!({ "data": (), })) .into_http_response()) } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] (Some("undelete"), _, _, _) => { // WARNING: TAMPERING WITH THIS FUNCTION IS STRICTLY PROHIBITED // Any attempt to modify, bypass, or disable this license validation mechanism // constitutes a severe violation of the Stalwart Enterprise License Agreement. // Such actions may result in immediate termination of your license, legal action, // and substantial financial penalties. Stalwart Labs LLC actively monitors for // unauthorized modifications and will pursue all available legal remedies against // violators to the fullest extent of the law, including but not limited to claims // for copyright infringement, breach of contract, and fraud. // Validate the access token access_token.assert_has_permission(Permission::Undelete)?; if self.core.is_enterprise_edition() { self.handle_undelete_api_request(req, path, body, session) .await } else { Err(manage::enterprise()) } } // SPDX-SnippetEnd (Some("uids"), Some(account_id), None, &Method::DELETE) => { let account_id = self .core .storage .data .get_principal_id(decode_path_element(account_id).as_ref()) .await? .ok_or_else(|| trc::ManageEvent::NotFound.into_err())?; let result = reset_imap_uids(self, account_id).await?; Ok(JsonResponse::new(json!({ "data": result, })) .into_http_response()) } (Some("quota"), Some(account_id), None, method @ (&Method::GET | &Method::DELETE)) => { let account_id = self .core .storage .data .get_principal_id(decode_path_element(account_id).as_ref()) .await? .ok_or_else(|| trc::ManageEvent::NotFound.into_err())?; if method == Method::DELETE { recalculate_quota(self, account_id).await?; } let result = self.get_used_quota(account_id).await?; Ok(JsonResponse::new(json!({ "data": result, })) .into_http_response()) } _ => Err(trc::ResourceEvent::NotFound.into_err()), } } async fn housekeeper_request(&self, event: HousekeeperEvent) -> trc::Result { self.inner .ipc .housekeeper_tx .send(event) .await .map_err(|err| { trc::EventType::Server(trc::ServerEvent::ThreadError) .reason(err) .details("Failed to send housekeeper event") })?; Ok(JsonResponse::new(json!({ "data": (), })) .into_http_response()) } } pub async fn recalculate_quota(server: &Server, account_id: u32) -> trc::Result<()> { let mut quota = 0; for collection in [ Collection::Email, Collection::Calendar, Collection::CalendarEvent, Collection::CalendarEventNotification, Collection::AddressBook, Collection::ContactCard, Collection::FileNode, ] { server .archives(account_id, collection, &(), |_, archive| { match collection { Collection::Email => { quota += archive.unarchive::()?.size.to_native() as i64; } Collection::Calendar => { quota += archive.unarchive::()?.size() as i64; } Collection::CalendarEvent => { quota += archive.unarchive::()?.size() as i64; } Collection::CalendarEventNotification => { quota += archive.unarchive::()?.size() as i64; } Collection::AddressBook => { quota += archive.unarchive::()?.size() as i64; } Collection::ContactCard => { quota += archive.unarchive::()?.size() as i64; } Collection::FileNode => { quota += archive.unarchive::()?.size() as i64; } _ => {} } Ok(true) }) .await .caused_by(trc::location!())?; } let mut batch = BatchBuilder::new(); batch .clear(DirectoryClass::UsedQuota(account_id)) .add(DirectoryClass::UsedQuota(account_id), quota); server .store() .write(batch.build_all()) .await .caused_by(trc::location!()) .map(|_| ()) } pub async fn destroy_account_blobs(server: &Server, account_id: u32) -> trc::Result<()> { let mut delete_keys = Vec::new(); for (collection, field) in [ (Collection::Email, u8::from(EmailField::Metadata)), (Collection::FileNode, u8::from(Field::ARCHIVE)), (Collection::SieveScript, u8::from(Field::ARCHIVE)), ] { server .all_archives(account_id, collection, field, |document_id, archive| { match collection { Collection::Email => { let message = archive.unarchive::()?; delete_keys.push(( collection, document_id, BlobHash::from(&message.blob_hash), )); } Collection::FileNode => { if let Some(file) = archive.unarchive::()?.file.as_ref() { delete_keys.push(( collection, document_id, BlobHash::from(&file.blob_hash), )); } } Collection::SieveScript => { let sieve = archive.unarchive::()?; delete_keys.push(( collection, document_id, BlobHash::from(&sieve.blob_hash), )); } _ => {} } Ok(()) }) .await .caused_by(trc::location!())?; } let mut batch = BatchBuilder::new(); batch.with_account_id(account_id); for (collection, document_id, hash) in delete_keys { if batch.is_large_batch() { server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); batch.with_account_id(account_id); } batch .with_collection(collection) .with_document(document_id) .clear(ValueClass::Blob(BlobOp::Link { hash, to: BlobLink::Document, })); } if !batch.is_empty() { server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } Ok(()) } pub async fn destroy_account_data( server: &Server, account_id: u32, has_data: bool, ) -> trc::Result<()> { // Unlink all accounts's blobs if has_data { destroy_account_blobs(server, account_id).await?; } // Destroy account data server .store() .danger_destroy_account(account_id) .await .caused_by(trc::location!())?; if has_data { // Remove search index for index in [ SearchIndex::Email, SearchIndex::Contacts, SearchIndex::Calendar, ] { if let Err(err) = server .core .storage .fts .unindex(SearchQuery::new(index).with_account_id(account_id)) .await { trc::error!(err.details("Failed to delete FTS index")); } } } Ok(()) } pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u32, u32)> { let mut mailbox_count = 0; let mut email_count = 0; let cache = server .get_cached_messages(account_id) .await .caused_by(trc::location!())?; for &mailbox_id in cache.mailboxes.index.keys() { let mailbox = server .store() .get_value::>(ValueKey::archive( account_id, Collection::Mailbox, mailbox_id, )) .await .caused_by(trc::location!())? .ok_or_else(|| trc::ImapEvent::Error.into_err().caused_by(trc::location!()))? .into_deserialized::() .caused_by(trc::location!())?; let mut new_mailbox = mailbox.inner.clone(); new_mailbox.uid_validity = rand::random::(); let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Mailbox) .with_document(mailbox_id) .custom( ObjectIndexBuilder::new() .with_current(mailbox) .with_changes(new_mailbox), ) .caused_by(trc::location!())? .clear(MailboxField::UidCounter); server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; mailbox_count += 1; } // Reset all UIDs for message_id in cache.emails.items.iter().map(|i| i.document_id) { let data = server .store() .get_value::>(ValueKey::archive( account_id, Collection::Email, message_id, )) .await .caused_by(trc::location!())?; let data_ = if let Some(data) = data { data } else { continue; }; let data = data_ .to_unarchived::() .caused_by(trc::location!())?; let mut new_data = data .deserialize::() .caused_by(trc::location!())?; let ids = server .assign_email_ids( account_id, new_data.mailboxes.iter().map(|m| m.mailbox_id), false, ) .await .caused_by(trc::location!())?; for (uid_mailbox, uid) in new_data.mailboxes.iter_mut().zip(ids) { uid_mailbox.uid = uid; } // Prepare write batch let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Email) .with_document(message_id) .assert_value(ValueClass::Property(EmailField::Archive.into()), &data) .set( EmailField::Archive, Archiver::new(new_data) .serialize() .caused_by(trc::location!())?, ); server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; email_count += 1; } Ok((mailbox_count, email_count)) } ================================================ FILE: crates/http/src/management/troubleshoot.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ future::Future, net::{IpAddr, SocketAddr}, time::{Duration, Instant}, }; use common::{ Server, auth::{AccessToken, oauth::GrantType}, config::smtp::{ queue::MxConfig, resolver::{Policy, Tlsa}, }, psl, }; use directory::backend::internal::manage; use http_body_util::{StreamBody, combinators::BoxBody}; use hyper::{ Method, StatusCode, body::{Bytes, Frame}, }; use mail_auth::{ AuthenticatedMessage, DkimResult, DmarcResult, IpLookupStrategy, IprevOutput, IprevResult, SpfOutput, SpfResult, dmarc::{self, verify::DmarcParameters}, mta_sts::TlsRpt, spf::verify::SpfParameters, }; use serde::{Deserialize, Serialize}; use serde_json::json; use smtp::outbound::{ client::{SmtpClient, StartTlsResult}, dane::{dnssec::TlsaLookup, verify::TlsaVerify}, lookup::{DnsLookup, ToNextHop}, mta_sts::{lookup::MtaStsLookup, verify::VerifyPolicy}, }; use tokio::{io::AsyncWriteExt, sync::mpsc}; use utils::url_params::UrlParams; use http_proto::{request::decode_path_element, *}; pub trait TroubleshootApi: Sync + Send { fn handle_troubleshoot_api_request( &self, req: &HttpRequest, path: Vec<&str>, access_token: &AccessToken, body: Option>, ) -> impl Future> + Send; } impl TroubleshootApi for Server { async fn handle_troubleshoot_api_request( &self, req: &HttpRequest, path: Vec<&str>, access_token: &AccessToken, body: Option>, ) -> trc::Result { let params = UrlParams::new(req.uri().query()); let account_id = access_token.primary_id(); match ( path.get(1).copied().unwrap_or_default(), path.get(2).copied(), req.method(), ) { ("token", None, &Method::GET) => { // Issue a live telemetry token valid for 60 seconds Ok(JsonResponse::new(json!({ "data": self.encode_access_token(GrantType::Troubleshoot, account_id, "web", 60).await?, })) .into_http_response()) } ("delivery", Some(target), &Method::GET) => { let timeout = Duration::from_secs( params .parse::("timeout") .filter(|interval| *interval >= 1) .unwrap_or(30), ); let mut rx = spawn_delivery_troubleshoot( self.clone(), decode_path_element(target).to_lowercase(), timeout, ); Ok(HttpResponse::new(StatusCode::OK) .with_content_type("text/event-stream") .with_cache_control("no-store") .with_stream_body(BoxBody::new(StreamBody::new(async_stream::stream! { while let Some(stage) = rx.recv().await { yield Ok(stage.to_frame()); } yield Ok(DeliveryStage::Completed.to_frame()); })))) } ("dmarc", None, &Method::POST) => { let request = serde_json::from_slice::( body.as_deref().unwrap_or_default(), ) .map_err(|err| { trc::EventType::Resource(trc::ResourceEvent::BadParameters).from_json_error(err) })?; let response = dmarc_troubleshoot(self, request).await.ok_or_else(|| { manage::error( "Invalid message body", "Failed to parse message body".into(), ) })?; Ok(JsonResponse::new(json!({ "data": response, })) .into_http_response()) } _ => Err(trc::ResourceEvent::NotFound.into_err()), } } } #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] #[serde(tag = "type")] enum DeliveryStage { MxLookupStart { domain: String, }, MxLookupSuccess { mxs: Vec, elapsed: u64, }, MxLookupError { reason: String, elapsed: u64, }, MtaStsFetchStart, MtaStsFetchSuccess { policy: Policy, elapsed: u64, }, MtaStsFetchError { reason: String, elapsed: u64, }, MtaStsNotFound { elapsed: u64, }, TlsRptLookupStart, TlsRptLookupSuccess { rua: Vec, elapsed: u64, }, TlsRptLookupError { reason: String, elapsed: u64, }, TlsRptNotFound { elapsed: u64, }, DeliveryAttemptStart { hostname: String, }, MtaStsVerifySuccess, MtaStsVerifyError { reason: String, }, TlsaLookupStart, TlsaLookupSuccess { record: Tlsa, elapsed: u64, }, TlsaNotFound { elapsed: u64, reason: String, }, TlsaLookupError { elapsed: u64, reason: String, }, IpLookupStart, IpLookupSuccess { remote_ips: Vec, elapsed: u64, }, IpLookupError { reason: String, elapsed: u64, }, ConnectionStart { remote_ip: IpAddr, }, ConnectionSuccess { elapsed: u64, }, ConnectionError { elapsed: u64, reason: String, }, ReadGreetingStart, ReadGreetingSuccess { elapsed: u64, }, ReadGreetingError { elapsed: u64, reason: String, }, EhloStart, EhloSuccess { elapsed: u64, }, EhloError { elapsed: u64, reason: String, }, StartTlsStart, StartTlsSuccess { elapsed: u64, }, StartTlsError { elapsed: u64, reason: String, }, DaneVerifySuccess, DaneVerifyError { reason: String, }, MailFromStart, MailFromSuccess { elapsed: u64, }, MailFromError { reason: String, elapsed: u64, }, RcptToStart, RcptToSuccess { elapsed: u64, }, RcptToError { reason: String, elapsed: u64, }, QuitStart, QuitCompleted { elapsed: u64, }, Completed, } #[derive(Debug, Serialize, Deserialize)] struct MX { pub exchanges: Vec, pub preference: u16, } #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] #[serde(tag = "type")] pub enum ReportUri { Mail { email: String }, Http { url: String }, } impl DeliveryStage { fn to_frame(&self) -> Frame { let payload = format!( "event: event\ndata: [{}]\n\n", serde_json::to_string(self).unwrap_or_default() ); Frame::data(Bytes::from(payload)) } } trait ElapsedMs { fn elapsed_ms(&self) -> u64; } impl ElapsedMs for Instant { fn elapsed_ms(&self) -> u64 { self.elapsed().as_millis() as u64 } } fn spawn_delivery_troubleshoot( server: Server, domain_or_email: String, timeout: Duration, ) -> mpsc::Receiver { let (tx, rx) = mpsc::channel(10); tokio::spawn(async move { let _ = delivery_troubleshoot(tx, server, domain_or_email, timeout).await; }); rx } async fn delivery_troubleshoot( tx: mpsc::Sender, server: Server, domain_or_email: String, timeout: Duration, ) -> Result<(), mpsc::error::SendError> { let (domain, email) = if let Some((_, domain)) = domain_or_email.rsplit_once('@') { (domain.to_string(), Some(domain_or_email)) } else { (domain_or_email, None) }; let local_host = &server.core.network.server_name; tx.send(DeliveryStage::MxLookupStart { domain: domain.to_string(), }) .await?; // Lookup MX let now = Instant::now(); let mxs = match server .core .smtp .resolvers .dns .mx_lookup(&domain, Some(&server.inner.cache.dns_mx)) .await { Ok(mxs) => mxs, Err(err) => { tx.send(DeliveryStage::MxLookupError { reason: err.to_string(), elapsed: now.elapsed_ms(), }) .await?; return Ok(()); } }; // Obtain remote host list let mx_config = MxConfig { max_mx: mxs.len(), max_multi_homed: 10, ip_lookup_strategy: IpLookupStrategy::Ipv4thenIpv6, }; let hosts = if let Some(hosts) = mxs.to_remote_hosts(&domain, &mx_config) { tx.send(DeliveryStage::MxLookupSuccess { mxs: mxs .iter() .map(|mx| MX { exchanges: mx.exchanges.clone(), preference: mx.preference, }) .collect(), elapsed: now.elapsed_ms(), }) .await?; hosts } else { tx.send(DeliveryStage::MxLookupError { reason: "Null MX record".to_string(), elapsed: now.elapsed_ms(), }) .await?; return Ok(()); }; // Fetch MTA-STS policy let now = Instant::now(); tx.send(DeliveryStage::MtaStsFetchStart).await?; let mta_sts_policy = match server.lookup_mta_sts_policy(&domain, timeout).await { Ok(policy) => { tx.send(DeliveryStage::MtaStsFetchSuccess { policy: policy.as_ref().clone(), elapsed: now.elapsed_ms(), }) .await?; Some(policy) } Err(err) => { if matches!( &err, smtp::outbound::mta_sts::Error::Dns(mail_auth::Error::DnsRecordNotFound(_)) ) { tx.send(DeliveryStage::MtaStsNotFound { elapsed: now.elapsed_ms(), }) .await?; } else { tx.send(DeliveryStage::MtaStsFetchError { reason: err.to_string(), elapsed: now.elapsed_ms(), }) .await?; } None } }; // Fetch TLS-RPT settings let now = Instant::now(); tx.send(DeliveryStage::TlsRptLookupStart).await?; match server .core .smtp .resolvers .dns .txt_lookup::( format!("_smtp._tls.{domain}."), Some(&server.inner.cache.dns_txt), ) .await { Ok(record) => { tx.send(DeliveryStage::TlsRptLookupSuccess { rua: record .rua .iter() .map(|r| match r { mail_auth::mta_sts::ReportUri::Mail(email) => ReportUri::Mail { email: email.clone(), }, mail_auth::mta_sts::ReportUri::Http(url) => { ReportUri::Http { url: url.clone() } } }) .collect(), elapsed: now.elapsed_ms(), }) .await?; } Err(err) => { if matches!(&err, mail_auth::Error::DnsRecordNotFound(_)) { tx.send(DeliveryStage::TlsRptNotFound { elapsed: now.elapsed_ms(), }) .await?; } else { tx.send(DeliveryStage::TlsRptLookupError { reason: err.to_string(), elapsed: now.elapsed_ms(), }) .await?; } } } // Try with each host 'outer: for host in hosts { let hostname = host.hostname(); tx.send(DeliveryStage::DeliveryAttemptStart { hostname: hostname.to_string(), }) .await?; // Verify MTA-STS policy if let Some(mta_sts_policy) = &mta_sts_policy { if mta_sts_policy.verify(hostname) { tx.send(DeliveryStage::MtaStsVerifySuccess).await?; } else { tx.send(DeliveryStage::MtaStsVerifyError { reason: "Not authorized by policy".to_string(), }) .await?; continue; } } // Fetch TLSA record tx.send(DeliveryStage::TlsaLookupStart).await?; let now = Instant::now(); let dane_policy = match server.tlsa_lookup(format!("_25._tcp.{hostname}.")).await { Ok(Some(tlsa)) if tlsa.has_end_entities => { tx.send(DeliveryStage::TlsaLookupSuccess { record: tlsa.as_ref().clone(), elapsed: now.elapsed_ms(), }) .await?; Some(tlsa) } Ok(Some(_)) => { tx.send(DeliveryStage::TlsaLookupError { elapsed: now.elapsed_ms(), reason: "TLSA record does not have end entities".to_string(), }) .await?; None } Ok(None) => { tx.send(DeliveryStage::TlsaNotFound { elapsed: now.elapsed_ms(), reason: "No TLSA DNSSEC records found".to_string(), }) .await?; None } Err(err) => { if matches!(&err, mail_auth::Error::DnsRecordNotFound(_)) { tx.send(DeliveryStage::TlsaNotFound { elapsed: now.elapsed_ms(), reason: "No TLSA records found for MX".to_string(), }) .await?; } else { tx.send(DeliveryStage::TlsaLookupError { elapsed: now.elapsed_ms(), reason: err.to_string(), }) .await?; } None } }; tx.send(DeliveryStage::IpLookupStart).await?; let now = Instant::now(); match server .ip_lookup( host.fqdn_hostname().as_ref(), IpLookupStrategy::Ipv4thenIpv6, usize::MAX, ) .await { Ok(remote_ips) if !remote_ips.is_empty() => { tx.send(DeliveryStage::IpLookupSuccess { remote_ips: remote_ips.clone(), elapsed: now.elapsed_ms(), }) .await?; for remote_ip in remote_ips { // Start connection tx.send(DeliveryStage::ConnectionStart { remote_ip }) .await?; let now = Instant::now(); match SmtpClient::connect(SocketAddr::new(remote_ip, 25), timeout, 0).await { Ok(mut client) => { tx.send(DeliveryStage::ConnectionSuccess { elapsed: now.elapsed_ms(), }) .await?; // Read greeting tx.send(DeliveryStage::ReadGreetingStart).await?; let now = Instant::now(); if let Err(status) = client.read_greeting(hostname).await { tx.send(DeliveryStage::ReadGreetingError { elapsed: now.elapsed_ms(), reason: status.to_string(), }) .await?; continue; } tx.send(DeliveryStage::ReadGreetingSuccess { elapsed: now.elapsed_ms(), }) .await?; // Say EHLO tx.send(DeliveryStage::EhloStart).await?; let now = Instant::now(); let capabilities = match tokio::time::timeout(timeout, async { client .stream .write_all(format!("EHLO {local_host}\r\n",).as_bytes()) .await?; client.stream.flush().await?; client.read_ehlo().await }) .await { Ok(Ok(capabilities)) => { tx.send(DeliveryStage::EhloSuccess { elapsed: now.elapsed_ms(), }) .await?; capabilities } Ok(Err(err)) => { tx.send(DeliveryStage::EhloError { elapsed: now.elapsed_ms(), reason: err.to_string(), }) .await?; continue; } Err(_) => { tx.send(DeliveryStage::EhloError { elapsed: now.elapsed_ms(), reason: "Timed out reading response".to_string(), }) .await?; continue; } }; // Start TLS tx.send(DeliveryStage::StartTlsStart).await?; let now = Instant::now(); let mut client = match client .try_start_tls( &server.inner.data.smtp_connectors.pki_verify, hostname, &capabilities, ) .await { StartTlsResult::Success { smtp_client } => { tx.send(DeliveryStage::StartTlsSuccess { elapsed: now.elapsed_ms(), }) .await?; smtp_client } StartTlsResult::Error { error } => { tx.send(DeliveryStage::StartTlsError { elapsed: now.elapsed_ms(), reason: error.to_string(), }) .await?; continue; } StartTlsResult::Unavailable { response, .. } => { tx.send(DeliveryStage::StartTlsError { elapsed: now.elapsed_ms(), reason: response.map(|r| r.to_string()).unwrap_or_else( || "STARTTLS not advertised by host".to_string(), ), }) .await?; continue; } }; // Verify DANE policy if let Some(dane_policy) = &dane_policy { if let Err(err) = dane_policy.verify( 0, hostname, client.tls_connection().peer_certificates(), ) { tx.send(DeliveryStage::DaneVerifyError { reason: err.to_string(), }) .await?; } else { tx.send(DeliveryStage::DaneVerifySuccess).await?; } } // Say EHLO again (some SMTP servers require this) tx.send(DeliveryStage::EhloStart).await?; let now = Instant::now(); match tokio::time::timeout(timeout, async { client .stream .write_all(format!("EHLO {local_host}\r\n",).as_bytes()) .await?; client.stream.flush().await?; client.read_ehlo().await }) .await { Ok(Ok(_)) => { tx.send(DeliveryStage::EhloSuccess { elapsed: now.elapsed_ms(), }) .await?; } Ok(Err(err)) => { tx.send(DeliveryStage::EhloError { elapsed: now.elapsed_ms(), reason: err.to_string(), }) .await?; continue; } Err(_) => { tx.send(DeliveryStage::EhloError { elapsed: now.elapsed_ms(), reason: "Timed out reading response".to_string(), }) .await?; continue; } } // Verify recipient let mut is_success = email.is_none(); if let Some(email) = &email { // MAIL FROM tx.send(DeliveryStage::MailFromStart).await?; let now = Instant::now(); match client.cmd(b"MAIL FROM:<>\r\n").await.and_then(|r| { if r.is_positive_completion() { Ok(r) } else { Err(mail_send::Error::UnexpectedReply(r)) } }) { Ok(_) => { tx.send(DeliveryStage::MailFromSuccess { elapsed: now.elapsed_ms(), }) .await?; // RCPT TO tx.send(DeliveryStage::RcptToStart).await?; let now = Instant::now(); match client .cmd(format!("RCPT TO:<{email}>\r\n").as_bytes()) .await .and_then(|r| { if r.is_positive_completion() { Ok(r) } else { Err(mail_send::Error::UnexpectedReply(r)) } }) { Ok(_) => { is_success = true; tx.send(DeliveryStage::RcptToSuccess { elapsed: now.elapsed_ms(), }) .await?; } Err(err) => { tx.send(DeliveryStage::RcptToError { reason: err.to_string(), elapsed: now.elapsed_ms(), }) .await?; } } } Err(err) => { tx.send(DeliveryStage::MailFromError { reason: err.to_string(), elapsed: now.elapsed_ms(), }) .await?; } } } // QUIT tx.send(DeliveryStage::QuitStart).await?; let now = Instant::now(); client.quit().await; tx.send(DeliveryStage::QuitCompleted { elapsed: now.elapsed_ms(), }) .await?; if is_success { break 'outer; } } Err(err) => { tx.send(DeliveryStage::ConnectionError { elapsed: now.elapsed_ms(), reason: err.to_string(), }) .await?; } } } } Ok(_) => { tx.send(DeliveryStage::IpLookupError { reason: "No IP addresses found for host".to_string(), elapsed: now.elapsed_ms(), }) .await?; } Err(err) => { tx.send(DeliveryStage::IpLookupError { reason: err.to_string(), elapsed: now.elapsed_ms(), }) .await?; } } } Ok(()) } #[derive(Debug, Serialize, Deserialize)] struct DmarcTroubleshootRequest { #[serde(rename = "remoteIp")] remote_ip: IpAddr, #[serde(rename = "ehloDomain")] ehlo_domain: String, #[serde(rename = "mailFrom")] mail_from: String, body: Option, } #[derive(Debug, Serialize, Deserialize)] struct DmarcTroubleshootResponse { #[serde(rename = "spfEhloDomain")] spf_ehlo_domain: String, #[serde(rename = "spfEhloResult")] spf_ehlo_result: AuthResult, #[serde(rename = "spfMailFromDomain")] spf_mail_from_domain: String, #[serde(rename = "spfMailFromResult")] spf_mail_from_result: AuthResult, #[serde(rename = "ipRevResult")] ip_rev_result: AuthResult, #[serde(rename = "ipRevPtr")] ip_rev_ptr: Vec, #[serde(rename = "dkimResults")] dkim_results: Vec, #[serde(rename = "dkimPass")] dkim_pass: bool, #[serde(rename = "arcResult")] arc_result: AuthResult, #[serde(rename = "dmarcResult")] dmarc_result: AuthResult, #[serde(rename = "dmarcPass")] dmarc_pass: bool, #[serde(rename = "dmarcPolicy")] dmarc_policy: DmarcPolicy, elapsed: u64, } #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] #[serde(tag = "type")] pub enum AuthResult { Pass, Fail { details: Option }, SoftFail { details: Option }, TempError { details: Option }, PermError { details: Option }, Neutral { details: Option }, None, } #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum DmarcPolicy { None, Quarantine, Reject, Unspecified, } async fn dmarc_troubleshoot( server: &Server, request: DmarcTroubleshootRequest, ) -> Option { let remote_ip = request.remote_ip; let ehlo_domain = request.ehlo_domain.to_lowercase(); let mail_from = request.mail_from.to_lowercase(); let mail_from_domain = mail_from.rsplit_once('@').map(|(_, domain)| domain); let local_host = &server.core.network.server_name; let now = Instant::now(); let ehlo_spf_output = server .core .smtp .resolvers .dns .verify_spf( server .inner .cache .build_auth_parameters(SpfParameters::verify_ehlo( remote_ip, &ehlo_domain, local_host, )), ) .await; let iprev = server .core .smtp .resolvers .dns .verify_iprev(server.inner.cache.build_auth_parameters(remote_ip)) .await; let mail_spf_output = if let Some(mail_from_domain) = mail_from_domain { server .core .smtp .resolvers .dns .check_host(server.inner.cache.build_auth_parameters(SpfParameters::new( remote_ip, mail_from_domain, &ehlo_domain, local_host, &mail_from, ))) .await } else { server .core .smtp .resolvers .dns .check_host(server.inner.cache.build_auth_parameters(SpfParameters::new( remote_ip, &ehlo_domain, &ehlo_domain, local_host, &format!("postmaster@{ehlo_domain}"), ))) .await }; let body = request .body .unwrap_or_else(|| format!("From: {mail_from}\r\nSubject: test\r\n\r\ntest")); let auth_message = AuthenticatedMessage::parse_with_opts(body.as_bytes(), true)?; let dkim_output = server .core .smtp .resolvers .dns .verify_dkim(server.inner.cache.build_auth_parameters(&auth_message)) .await; let dkim_pass = dkim_output .iter() .any(|d| matches!(d.result(), DkimResult::Pass)); let arc_output = server .core .smtp .resolvers .dns .verify_arc(server.inner.cache.build_auth_parameters(&auth_message)) .await; let dmarc_output = server .core .smtp .resolvers .dns .verify_dmarc(server.inner.cache.build_auth_parameters(DmarcParameters { message: &auth_message, dkim_output: &dkim_output, rfc5321_mail_from_domain: mail_from_domain.unwrap_or(ehlo_domain.as_str()), spf_output: &mail_spf_output, domain_suffix_fn: |domain| psl::domain_str(domain).unwrap_or(domain), })) .await; let dmarc_pass = matches!(dmarc_output.spf_result(), DmarcResult::Pass) || matches!(dmarc_output.dkim_result(), DmarcResult::Pass); let dmarc_result = if dmarc_pass { DmarcResult::Pass } else if dmarc_output.spf_result() != &DmarcResult::None { dmarc_output.spf_result().clone() } else if dmarc_output.dkim_result() != &DmarcResult::None { dmarc_output.dkim_result().clone() } else { DmarcResult::None }; Some(DmarcTroubleshootResponse { spf_ehlo_domain: ehlo_spf_output.domain().to_string(), spf_ehlo_result: (&ehlo_spf_output).into(), spf_mail_from_domain: mail_spf_output.domain().to_string(), spf_mail_from_result: (&mail_spf_output).into(), ip_rev_ptr: iprev .ptr .as_ref() .map(|ptr| ptr.as_ref().clone()) .unwrap_or_default(), ip_rev_result: (&iprev).into(), dkim_pass, dkim_results: dkim_output .iter() .map(|result| result.result().into()) .collect(), arc_result: arc_output.result().into(), dmarc_result: (&dmarc_result).into(), dmarc_policy: (&dmarc_output.policy()).into(), dmarc_pass, elapsed: now.elapsed_ms(), }) } impl From<&SpfOutput> for AuthResult { fn from(value: &SpfOutput) -> Self { match value.result() { SpfResult::Pass => AuthResult::Pass, SpfResult::Fail => AuthResult::Fail { details: value.explanation().map(|e| e.to_string()), }, SpfResult::SoftFail => AuthResult::SoftFail { details: value.explanation().map(|e| e.to_string()), }, SpfResult::Neutral => AuthResult::Neutral { details: value.explanation().map(|e| e.to_string()), }, SpfResult::TempError => AuthResult::TempError { details: value.explanation().map(|e| e.to_string()), }, SpfResult::PermError => AuthResult::PermError { details: value.explanation().map(|e| e.to_string()), }, SpfResult::None => AuthResult::None, } } } impl From for SpfOutput { fn from(value: AuthResult) -> Self { match value { AuthResult::Pass => SpfOutput::new(String::new()).with_result(SpfResult::Pass), AuthResult::Fail { .. } => SpfOutput::new(String::new()).with_result(SpfResult::Fail), AuthResult::SoftFail { .. } => { SpfOutput::new(String::new()).with_result(SpfResult::SoftFail) } AuthResult::Neutral { .. } => { SpfOutput::new(String::new()).with_result(SpfResult::Neutral) } AuthResult::TempError { .. } => { SpfOutput::new(String::new()).with_result(SpfResult::TempError) } AuthResult::PermError { .. } => { SpfOutput::new(String::new()).with_result(SpfResult::PermError) } AuthResult::None => SpfOutput::new(String::new()).with_result(SpfResult::None), } } } impl From<&IprevOutput> for AuthResult { fn from(value: &IprevOutput) -> Self { match &value.result { IprevResult::Pass => AuthResult::Pass, IprevResult::Fail(error) => AuthResult::Fail { details: error.to_string().into(), }, IprevResult::TempError(error) => AuthResult::TempError { details: error.to_string().into(), }, IprevResult::PermError(error) => AuthResult::PermError { details: error.to_string().into(), }, IprevResult::None => AuthResult::None, } } } impl From for IprevResult { fn from(value: AuthResult) -> Self { match value { AuthResult::Pass => IprevResult::Pass, AuthResult::Fail { details } => { IprevResult::Fail(mail_auth::Error::Io(details.unwrap_or_default())) } AuthResult::TempError { details } => { IprevResult::TempError(mail_auth::Error::Io(details.unwrap_or_default())) } AuthResult::PermError { details } => { IprevResult::PermError(mail_auth::Error::Io(details.unwrap_or_default())) } AuthResult::None => IprevResult::None, _ => IprevResult::None, } } } impl From<&DkimResult> for AuthResult { fn from(value: &DkimResult) -> Self { match value { DkimResult::Pass => AuthResult::Pass, DkimResult::Neutral(error) => AuthResult::Neutral { details: error.to_string().into(), }, DkimResult::Fail(error) => AuthResult::Fail { details: error.to_string().into(), }, DkimResult::PermError(error) => AuthResult::PermError { details: error.to_string().into(), }, DkimResult::TempError(error) => AuthResult::TempError { details: error.to_string().into(), }, DkimResult::None => AuthResult::None, } } } impl From for DkimResult { fn from(value: AuthResult) -> Self { match value { AuthResult::Pass => DkimResult::Pass, AuthResult::Neutral { details } => { DkimResult::Neutral(mail_auth::Error::Io(details.unwrap_or_default())) } AuthResult::Fail { details } => { DkimResult::Fail(mail_auth::Error::Io(details.unwrap_or_default())) } AuthResult::PermError { details } => { DkimResult::PermError(mail_auth::Error::Io(details.unwrap_or_default())) } AuthResult::TempError { details } => { DkimResult::TempError(mail_auth::Error::Io(details.unwrap_or_default())) } _ => DkimResult::None, } } } impl From<&DmarcResult> for AuthResult { fn from(value: &DmarcResult) -> Self { match value { DmarcResult::Pass => AuthResult::Pass, DmarcResult::Fail(error) => AuthResult::Fail { details: error.to_string().into(), }, DmarcResult::TempError(error) => AuthResult::TempError { details: error.to_string().into(), }, DmarcResult::PermError(error) => AuthResult::PermError { details: error.to_string().into(), }, DmarcResult::None => AuthResult::None, } } } impl From for DmarcResult { fn from(value: AuthResult) -> Self { match value { AuthResult::Pass => DmarcResult::Pass, AuthResult::Fail { details } => { DmarcResult::Fail(mail_auth::Error::Io(details.unwrap_or_default())) } AuthResult::TempError { details } => { DmarcResult::TempError(mail_auth::Error::Io(details.unwrap_or_default())) } AuthResult::PermError { details } => { DmarcResult::PermError(mail_auth::Error::Io(details.unwrap_or_default())) } AuthResult::None => DmarcResult::None, _ => DmarcResult::None, } } } impl From<&dmarc::Policy> for DmarcPolicy { fn from(value: &dmarc::Policy) -> Self { match value { dmarc::Policy::None => DmarcPolicy::None, dmarc::Policy::Quarantine => DmarcPolicy::Quarantine, dmarc::Policy::Reject => DmarcPolicy::Reject, dmarc::Policy::Unspecified => DmarcPolicy::Unspecified, } } } impl From for dmarc::Policy { fn from(value: DmarcPolicy) -> Self { match value { DmarcPolicy::None => dmarc::Policy::None, DmarcPolicy::Quarantine => dmarc::Policy::Quarantine, DmarcPolicy::Reject => dmarc::Policy::Reject, DmarcPolicy::Unspecified => dmarc::Policy::Unspecified, } } } ================================================ FILE: crates/http/src/request.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ HttpSessionManager, auth::{ authenticate::{Authenticator, HttpHeaders}, oauth::{ FormData, auth::OAuthApiHandler, openid::OpenIdHandler, registration::ClientRegistrationHandler, token::TokenHandler, }, }, autoconfig::Autoconfig, form::FormHandler, management::{ ManagementApi, ToManageHttpResponse, UnauthorizedResponse, troubleshoot::TroubleshootApi, }, }; use common::{ Inner, KV_ACME, Server, auth::{AccessToken, oauth::GrantType}, core::BuildServer, ipc::PushEvent, listener::{SessionData, SessionManager, SessionStream}, manager::webadmin::Resource, }; use dav::{DavMethod, request::DavRequestHandler}; use directory::Permission; use groupware::{DavResourceName, calendar::itip::ItipIngest}; use http_proto::{ DownloadResponse, HtmlResponse, HttpContext, HttpRequest, HttpResponse, HttpResponseBody, HttpSessionData, JsonProblemResponse, ToHttpResponse, form_urlencoded, request::fetch_body, }; use hyper::{ Method, StatusCode, body, header::{self, CONTENT_TYPE}, server::conn::http1, service::service_fn, }; use hyper_util::rt::TokioIo; use jmap::{ api::{ ToJmapHttpResponse, event_source::EventSourceHandler, request::RequestHandler, session::SessionHandler, }, blob::{download::BlobDownload, upload::BlobUpload}, websocket::upgrade::WebSocketUpgrade, }; use jmap_proto::request::{Request, capability::Session}; use std::{net::IpAddr, str::FromStr, sync::Arc}; use store::dispatch::lookup::KeyValue; use trc::SecurityEvent; use types::{blob::BlobId, id::Id}; use utils::url_params::UrlParams; pub trait ParseHttp: Sync + Send { fn parse_http_request( &self, req: HttpRequest, session: HttpSessionData, ) -> impl Future> + Send; } impl ParseHttp for Server { async fn parse_http_request( &self, mut req: HttpRequest, session: HttpSessionData, ) -> trc::Result { let mut path = req.uri().path().split('/'); path.next(); // Validate endpoint access let ctx = HttpContext::new(&session, &req); match ctx.has_endpoint_access(self).await { StatusCode::OK => (), status => { // Allow loopback address to avoid lockouts if !session.remote_ip.is_loopback() { return Ok(JsonProblemResponse(status).into_http_response()); } } } match path.next().unwrap_or_default() { "jmap" => { match (path.next().unwrap_or_default(), req.method()) { ("", &Method::POST) => { // Authenticate request let (_in_flight, access_token) = self.authenticate_headers(&req, &session, false).await?; let bytes = fetch_body( &mut req, if !access_token.has_permission(Permission::UnlimitedUploads) { self.core.jmap.upload_max_size } else { 0 }, session.session_id, ) .await .ok_or_else(|| trc::LimitEvent::SizeRequest.into_err())?; return Ok(self .handle_jmap_request( Request::parse( &bytes, self.core.jmap.request_max_calls, self.core.jmap.request_max_size, )?, access_token, &session, ) .await .into_http_response()); } ("download", &Method::GET) => { // Authenticate request let (_in_flight, access_token) = self.authenticate_headers(&req, &session, false).await?; if let (Some(_), Some(blob_id), Some(name)) = ( path.next().and_then(|p| Id::from_str(p).ok()), path.next().and_then(BlobId::from_base32), path.next(), ) { return match self.blob_download(&blob_id, &access_token).await? { Some(blob) => Ok(DownloadResponse { filename: name.to_string(), content_type: req .uri() .query() .and_then(|q| { form_urlencoded::parse(q.as_bytes()) .find(|(k, _)| k == "accept") .map(|(_, v)| v.into_owned()) }) .unwrap_or("application/octet-stream".to_string()), blob, } .into_http_response()), None => Err(trc::ResourceEvent::NotFound.into_err()), }; } } ("upload", &Method::POST) => { // Authenticate request let (_in_flight, access_token) = self.authenticate_headers(&req, &session, false).await?; if let Some(account_id) = path.next().and_then(|p| Id::from_str(p).ok()) { return match fetch_body( &mut req, if !access_token.has_permission(Permission::UnlimitedUploads) { self.core.jmap.upload_max_size } else { 0 }, session.session_id, ) .await { Some(bytes) => Ok(self .blob_upload( account_id, req.headers() .get(CONTENT_TYPE) .and_then(|h| h.to_str().ok()) .unwrap_or("application/octet-stream"), &bytes, access_token, ) .await? .into_http_response()), None => Err(trc::LimitEvent::SizeUpload.into_err()), }; } } ("eventsource", &Method::GET) => { // Authenticate request let (_in_flight, access_token) = self.authenticate_headers(&req, &session, false).await?; return self.handle_event_source(req, access_token).await; } ("ws", &Method::GET) => { // Authenticate request let (_in_flight, access_token) = self.authenticate_headers(&req, &session, false).await?; return self .upgrade_websocket_connection(req, access_token, session) .await; } ("session", &Method::GET) => { return if req.headers().contains_key(header::AUTHORIZATION) { // Authenticate request let (_in_flight, access_token) = self.authenticate_headers(&req, &session, false).await?; self.handle_session_resource( ctx.resolve_response_url(self).await, access_token, ) .await .map(|s| s.into_http_response()) } else { Ok(Session::new( ctx.resolve_response_url(self).await, &self.core.jmap.capabilities, ) .into_http_response()) }; } (_, &Method::OPTIONS) => { return Ok(JsonProblemResponse(StatusCode::NO_CONTENT).into_http_response()); } _ => (), } } "dav" => { let response = match ( path.next().and_then(DavResourceName::parse), DavMethod::parse(req.method()), ) { (Some(_), Some(DavMethod::OPTIONS)) => HttpResponse::new(StatusCode::OK) .with_header( "DAV", concat!( "1, 2, 3, access-control, extended-mkcol, calendar-access, ", "calendar-auto-schedule, calendar-no-timezone, addressbook" ), ) .with_header( "Allow", concat!( "OPTIONS, GET, HEAD, POST, PUT, DELETE, COPY, MOVE, MKCALENDAR, ", "MKCOL, PROPFIND, PROPPATCH, LOCK, UNLOCK, REPORT, ACL" ), ), (Some(resource), Some(method)) => { // Authenticate request let (_in_flight, access_token) = self.authenticate_headers(&req, &session, false).await?; self.handle_dav_request(req, access_token, &session, resource, method) .await } (_, None) => HttpResponse::new(StatusCode::METHOD_NOT_ALLOWED), (None, _) => HttpResponse::new(StatusCode::NOT_FOUND), }; return Ok(response); } ".well-known" => match (path.next().unwrap_or_default(), req.method()) { ("jmap", &Method::GET) => { return Ok(HttpResponse::new(StatusCode::TEMPORARY_REDIRECT) .with_no_cache() .with_location("/jmap/session")); } ("caldav", _) => { return Ok(HttpResponse::new(StatusCode::TEMPORARY_REDIRECT) .with_no_cache() .with_location(DavResourceName::Cal.base_path())); } ("carddav", _) => { return Ok(HttpResponse::new(StatusCode::TEMPORARY_REDIRECT) .with_no_cache() .with_location(DavResourceName::Card.base_path())); } ("oauth-authorization-server", &Method::GET) => { // Limit anonymous requests self.is_http_anonymous_request_allowed(&session.remote_ip) .await?; return self.handle_oauth_metadata(req, session).await; } ("openid-configuration", &Method::GET) => { // Limit anonymous requests self.is_http_anonymous_request_allowed(&session.remote_ip) .await?; return self.handle_oidc_metadata(req, session).await; } ("acme-challenge", &Method::GET) if self.has_acme_http_providers() => { if let Some(token) = path.next() { return match self .core .storage .lookup .key_get::(KeyValue::<()>::build_key(KV_ACME, token)) .await? { Some(proof) => Ok(Resource::new("text/plain", proof.into_bytes()) .into_http_response()), None => Err(trc::ResourceEvent::NotFound.into_err()), }; } } ("mta-sts.txt", &Method::GET) => { // Limit anonymous requests self.is_http_anonymous_request_allowed(&session.remote_ip) .await?; return if let Some(policy) = self.build_mta_sts_policy() { Ok(Resource::new("text/plain", policy.to_string().into_bytes()) .into_http_response()) } else { Err(trc::ResourceEvent::NotFound.into_err()) }; } ("mail-v1.xml", &Method::GET) => { // Limit anonymous requests self.is_http_anonymous_request_allowed(&session.remote_ip) .await?; return self.handle_autoconfig_request(&req).await; } ("autoconfig", &Method::GET) => { if path.next().unwrap_or_default() == "mail" && path.next().unwrap_or_default() == "config-v1.1.xml" { // Limit anonymous requests self.is_http_anonymous_request_allowed(&session.remote_ip) .await?; return self.handle_autoconfig_request(&req).await; } } (_, &Method::OPTIONS) => { return Ok(JsonProblemResponse(StatusCode::NO_CONTENT).into_http_response()); } _ => (), }, "auth" => match (path.next().unwrap_or_default(), req.method()) { ("device", &Method::POST) => { self.is_http_anonymous_request_allowed(&session.remote_ip) .await?; return self.handle_device_auth(&mut req, session).await; } ("token", &Method::POST) => { self.is_http_anonymous_request_allowed(&session.remote_ip) .await?; return self.handle_token_request(&mut req, session).await; } ("introspect", &Method::POST) => { // Authenticate request let (_in_flight, access_token) = self.authenticate_headers(&req, &session, false).await?; return self .handle_token_introspect(&mut req, &access_token, session.session_id) .await; } ("userinfo", &Method::GET) => { // Authenticate request let (_in_flight, access_token) = self.authenticate_headers(&req, &session, false).await?; return self.handle_userinfo_request(&access_token).await; } ("register", &Method::POST) => { return self .handle_oauth_registration_request(&mut req, session) .await; } ("jwks.json", &Method::GET) => { // Limit anonymous requests self.is_http_anonymous_request_allowed(&session.remote_ip) .await?; return Ok(self.core.oauth.oidc_jwks.clone().into_http_response()); } (_, &Method::OPTIONS) => { return Ok(JsonProblemResponse(StatusCode::NO_CONTENT).into_http_response()); } _ => (), }, "api" => { // Allow CORS preflight requests if req.method() == Method::OPTIONS { return Ok(JsonProblemResponse(StatusCode::NO_CONTENT).into_http_response()); } // Authenticate user match self.authenticate_headers(&req, &session, true).await { Ok((_, access_token)) => { return self .handle_api_manage_request(&mut req, access_token, &session) .await; } Err(err) => { if err.matches(trc::EventType::Auth(trc::AuthEvent::Failed)) { let params = UrlParams::new(req.uri().query()); let path = req.uri().path().split('/').skip(2).collect::>(); let (grant_type, token) = match ( path.first().copied(), path.get(1).copied(), params.get("token"), ) { // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] (Some("telemetry"), Some("traces"), Some(token)) if self.core.is_enterprise_edition() => { (GrantType::LiveTracing, token) } #[cfg(feature = "enterprise")] (Some("telemetry"), Some("metrics"), Some(token)) if self.core.is_enterprise_edition() => { (GrantType::LiveMetrics, token) } // SPDX-SnippetEnd (Some("troubleshoot"), _, Some(token)) => { (GrantType::Troubleshoot, token) } _ => return Ok(HttpResponse::unauthorized(false)), }; let token_info = self.validate_access_token(grant_type.into(), token).await?; return match grant_type { // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] GrantType::LiveTracing | GrantType::LiveMetrics => { use crate::management::enterprise::telemetry::TelemetryApi; self.handle_telemetry_api_request( &req, path, &AccessToken::from_id(token_info.account_id) .with_permission(Permission::MetricsLive) .with_permission(Permission::TracingLive), ) .await } // SPDX-SnippetEnd GrantType::Troubleshoot => { self.handle_troubleshoot_api_request( &req, path, &AccessToken::from_id(token_info.account_id) .with_permission(Permission::Troubleshoot), None, ) .await } _ => unreachable!(), }; } return Err(err); } } } "mail" => { if req.method() == Method::GET && path.next().unwrap_or_default() == "config-v1.1.xml" { // Limit anonymous requests self.is_http_anonymous_request_allowed(&session.remote_ip) .await?; return self.handle_autoconfig_request(&req).await; } } "calendar" => { // Limit anonymous requests self.is_http_anonymous_request_allowed(&session.remote_ip) .await?; if self.core.groupware.itip_http_rsvp_url.is_some() && req.method() == Method::GET && path.next().unwrap_or_default() == "rsvp" { return self .http_rsvp_handle( req.uri().query().unwrap_or_default(), req.headers() .get(header::ACCEPT_LANGUAGE) .and_then(|v| v.to_str().ok()) .map(|lang| { let lang = lang.split_once(',').map_or(lang, |(l, _)| l); lang.split_once(';').map_or(lang, |(l, _)| l) }) .unwrap_or("en"), ) .await .map(|response| { HtmlResponse::new(response) .into_http_response() .with_no_store() }); } } "autodiscover" | "Autodiscover" => { if req.method() == Method::POST && path .next() .unwrap_or_default() .eq_ignore_ascii_case("autodiscover.xml") { // Limit anonymous requests self.is_http_anonymous_request_allowed(&session.remote_ip) .await?; return self .handle_autodiscover_request( fetch_body(&mut req, 8192, session.session_id).await, ) .await; } } "robots.txt" => { // Limit anonymous requests self.is_http_anonymous_request_allowed(&session.remote_ip) .await?; return Ok( Resource::new("text/plain", b"User-agent: *\nDisallow: /\n".to_vec()) .into_http_response(), ); } "healthz" => { // Limit anonymous requests self.is_http_anonymous_request_allowed(&session.remote_ip) .await?; match path.next().unwrap_or_default() { "live" => { return Ok(JsonProblemResponse(StatusCode::OK).into_http_response()); } "ready" => { return Ok(JsonProblemResponse({ if !self.core.storage.data.is_none() { StatusCode::OK } else { StatusCode::SERVICE_UNAVAILABLE } }) .into_http_response()); } _ => (), } } "metrics" => match path.next().unwrap_or_default() { "prometheus" => { if let Some(prometheus) = &self.core.metrics.prometheus { if let Some(auth) = &prometheus.auth && req .authorization_basic() .is_none_or(|secret| secret != auth) { return Err(trc::AuthEvent::Failed .into_err() .details("Invalid or missing credentials.") .caused_by(trc::location!())); } return Ok(Resource::new( "text/plain; version=0.0.4", self.export_prometheus_metrics().await?.into_bytes(), ) .into_http_response()); } } "otel" => { // Reserved for future use } _ => (), }, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] "logo.svg" if self.is_enterprise_edition() => { match self .logo_resource( req.headers() .get(header::HOST) .and_then(|h| h.to_str().ok()) .map(|h| h.rsplit_once(':').map_or(h, |(h, _)| h)) .unwrap_or_default(), ) .await { Ok(Some(resource)) => { return Ok(resource.into_http_response()); } Ok(None) => (), Err(err) => { trc::error!(err.span_id(session.session_id)); } } let resource = self.inner.data.webadmin.get("logo.svg").await?; if !resource.is_empty() { return Ok(resource.into_http_response()); } } // SPDX-SnippetEnd "form" => { if let Some(form) = &self.core.network.contact_form { match *req.method() { Method::POST => { self.is_http_anonymous_request_allowed(&session.remote_ip) .await?; let form_data = FormData::from_request(&mut req, form.max_size, session.session_id) .await?; return self.handle_contact_form(&session, form, form_data).await; } Method::OPTIONS => { return Ok( JsonProblemResponse(StatusCode::NO_CONTENT).into_http_response() ); } _ => {} } } } _ => { let path = req.uri().path(); let resource = self .inner .data .webadmin .get(path.strip_prefix('/').unwrap_or(path)) .await?; if !resource.is_empty() { return Ok(resource.into_http_response()); } } } // Block dangerous URLs let path = req.uri().path(); if self.is_http_banned_path(path, session.remote_ip).await? { trc::event!( Security(SecurityEvent::ScanBan), SpanId = session.session_id, RemoteIp = session.remote_ip, Path = path.to_string(), ); } Err(trc::ResourceEvent::NotFound.into_err()) } } async fn handle_session(inner: Arc, session: SessionData) { let _in_flight = session.in_flight; let is_tls = session.stream.is_tls(); if let Err(http_err) = http1::Builder::new() .keep_alive(true) .serve_connection( TokioIo::new(session.stream), service_fn(|req: hyper::Request| { let instance = session.instance.clone(); let inner = inner.clone(); async move { let server = inner.build_server(); // Obtain remote IP let remote_ip = if !server.core.jmap.http_use_forwarded { trc::event!( Http(trc::HttpEvent::RequestUrl), SpanId = session.session_id, Url = req.uri().to_string(), ); session.remote_ip } else if let Some(forwarded_for) = req .headers() .get(header::FORWARDED) .and_then(|h| h.to_str().ok()) .and_then(|h| { let h = h.to_ascii_lowercase(); h.split_once("for=").and_then(|(_, rest)| { let mut start_ip = usize::MAX; let mut end_ip = usize::MAX; for (pos, ch) in rest.char_indices() { match ch { '0'..='9' | 'a'..='f' | ':' | '.' => { if start_ip == usize::MAX { start_ip = pos; } end_ip = pos; } '"' | '[' | ' ' if start_ip == usize::MAX => {} _ => { break; } } } rest.get(start_ip..=end_ip) .and_then(|h| h.parse::().ok()) }) }) .or_else(|| { req.headers() .get("X-Forwarded-For") .and_then(|h| h.to_str().ok()) .map(|h| h.split_once(',').map_or(h, |(ip, _)| ip).trim()) .and_then(|h| h.parse::().ok()) }) { // Check if the forwarded IP has been blocked if server.is_ip_blocked(&forwarded_for) { trc::event!( Security(trc::SecurityEvent::IpBlocked), ListenerId = instance.id.clone(), RemoteIp = forwarded_for, SpanId = session.session_id, ); return Ok::<_, hyper::Error>( JsonProblemResponse(StatusCode::FORBIDDEN) .into_http_response() .build(), ); } trc::event!( Http(trc::HttpEvent::RequestUrl), SpanId = session.session_id, RemoteIp = forwarded_for, Url = req.uri().to_string(), ); forwarded_for } else { trc::event!( Http(trc::HttpEvent::XForwardedMissing), SpanId = session.session_id, ); session.remote_ip }; // Parse HTTP request let response = match Box::pin(server.parse_http_request( req, HttpSessionData { instance, local_ip: session.local_ip, local_port: session.local_port, remote_ip, remote_port: session.remote_port, is_tls, session_id: session.session_id, }, )) .await { Ok(response) => response, Err(err) => { let response = err.into_http_response(); trc::error!(err.span_id(session.session_id)); response } }; trc::event!( Http(trc::HttpEvent::ResponseBody), SpanId = session.session_id, Contents = match response.body() { HttpResponseBody::Text(value) => trc::Value::String(value.as_str().into()), HttpResponseBody::Binary(_) => trc::Value::String("[binary data]".into()), HttpResponseBody::Stream(_) => trc::Value::String("[stream]".into()), _ => trc::Value::None, }, Code = response.status().as_u16(), Size = response.size(), ); // Build response let mut response = response.build(); // Add custom headers if !server.core.jmap.http_headers.is_empty() { let headers = response.headers_mut(); for (header, value) in &server.core.jmap.http_headers { headers.insert(header.clone(), value.clone()); } } Ok::<_, hyper::Error>(response) } }), ) .with_upgrades() .await { if http_err.is_parse() { let server = inner.build_server(); if !server.core.jmap.http_use_forwarded { match server.is_scanner_fail2banned(session.remote_ip).await { Ok(true) => { trc::event!( Security(SecurityEvent::ScanBan), SpanId = session.session_id, RemoteIp = session.remote_ip, Reason = http_err.to_string(), ); return; } Ok(false) => {} Err(err) => { trc::error!( err.span_id(session.session_id) .details("Failed to check for fail2ban") ); } } } } trc::event!( Http(trc::HttpEvent::Error), SpanId = session.session_id, Reason = http_err.to_string(), ); } } impl SessionManager for HttpSessionManager { fn handle(self, session: SessionData) -> impl Future + Send { handle_session(self.inner, session) } #[allow(clippy::manual_async_fn)] fn shutdown(&self) -> impl std::future::Future + Send { async { let _ = self.inner.ipc.push_tx.send(PushEvent::Stop).await; } } } ================================================ FILE: crates/http-proto/Cargo.toml ================================================ [package] name = "http_proto" version = "0.15.5" edition = "2024" [dependencies] common = { path = "../common" } trc = { path = "../trc" } serde = { version = "1.0", features = ["derive"]} serde_json = "1.0" hyper = { version = "1.0.1", features = ["server", "http1", "http2"] } hyper-util = { version = "0.1.1", features = ["tokio"] } http-body-util = "0.1.0" form_urlencoded = "1.1.0" percent-encoding = "2.3.1" compact_str = "0.9.0" [dev-dependencies] [features] test_mode = [] enterprise = [] ================================================ FILE: crates/http-proto/src/context.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{ Server, expr::{functions::ResolveVariable, *}, }; use compact_str::{ToCompactString, format_compact}; use hyper::StatusCode; use crate::{HttpContext, HttpRequest, HttpSessionData}; impl<'x> HttpContext<'x> { pub fn new(session: &'x HttpSessionData, req: &'x HttpRequest) -> Self { Self { session, req } } pub async fn resolve_response_url(&self, server: &Server) -> String { server .eval_if( &server.core.network.http_response_url, self, self.session.session_id, ) .await .unwrap_or_else(|| { format!( "http{}://{}:{}", if self.session.is_tls { "s" } else { "" }, self.session.local_ip, self.session.local_port ) }) } pub async fn has_endpoint_access(&self, server: &Server) -> StatusCode { server .eval_if( &server.core.network.http_allowed_endpoint, self, self.session.session_id, ) .await .unwrap_or(StatusCode::OK) } } impl ResolveVariable for HttpContext<'_> { fn resolve_variable(&self, variable: u32) -> Variable<'_> { match variable { V_REMOTE_IP => self.session.remote_ip.to_compact_string().into(), V_REMOTE_PORT => self.session.remote_port.into(), V_LOCAL_IP => self.session.local_ip.to_compact_string().into(), V_LOCAL_PORT => self.session.local_port.into(), V_TLS => self.session.is_tls.into(), V_PROTOCOL => if self.session.is_tls { "https" } else { "http" }.into(), V_LISTENER => self.session.instance.id.as_str().into(), V_URL => self.req.uri().to_compact_string().into(), V_URL_PATH => self.req.uri().path().into(), V_METHOD => self.req.method().as_str().into(), V_HEADERS => self .req .headers() .iter() .map(|(h, v)| { Variable::String( format_compact!("{}: {}", h.as_str(), v.to_str().unwrap_or_default()) .into(), ) }) .collect::>() .into(), _ => Variable::default(), } } fn resolve_global(&self, _: &str) -> Variable<'_> { Variable::Integer(0) } } ================================================ FILE: crates/http-proto/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod context; pub mod request; pub mod response; pub use form_urlencoded; use std::{net::IpAddr, sync::Arc}; use common::listener::ServerInstance; use hyper::StatusCode; pub type HttpRequest = hyper::Request; pub struct JsonResponse { status: StatusCode, inner: T, no_cache: bool, } pub struct HtmlResponse { status: StatusCode, body: String, } pub enum HttpResponseBody { Text(String), Binary(Vec), Stream(http_body_util::combinators::BoxBody), WebsocketUpgrade(String), Empty, } pub struct HttpResponse { status: StatusCode, builder: hyper::http::response::Builder, body: HttpResponseBody, } pub struct HttpContext<'x> { pub session: &'x HttpSessionData, pub req: &'x HttpRequest, } pub struct HttpSessionData { pub instance: Arc, pub local_ip: IpAddr, pub local_port: u16, pub remote_ip: IpAddr, pub remote_port: u16, pub is_tls: bool, pub session_id: u64, } pub struct DownloadResponse { pub filename: String, pub content_type: String, pub blob: Vec, } pub struct JsonProblemResponse(pub StatusCode); impl JsonResponse { pub fn new(inner: T) -> Self { JsonResponse { inner, status: StatusCode::OK, no_cache: false, } } pub fn with_status(status: StatusCode, inner: T) -> Self { JsonResponse { inner, status, no_cache: false, } } pub fn no_cache(mut self) -> Self { self.no_cache = true; self } } impl HtmlResponse { pub fn new(body: String) -> Self { HtmlResponse { body, status: StatusCode::OK, } } pub fn with_status(status: StatusCode, body: String) -> Self { HtmlResponse { body, status } } } pub trait ToHttpResponse { fn into_http_response(self) -> HttpResponse; } ================================================ FILE: crates/http-proto/src/request.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::borrow::Cow; use compact_str::ToCompactString; use http_body_util::BodyExt; use crate::HttpRequest; #[inline] pub fn decode_path_element(item: &str) -> Cow<'_, str> { percent_encoding::percent_decode_str(item) .decode_utf8() .unwrap_or_else(|_| item.into()) } pub async fn fetch_body( req: &mut HttpRequest, max_size: usize, session_id: u64, ) -> Option> { let mut bytes = Vec::with_capacity(1024); while let Some(Ok(frame)) = req.frame().await { if let Some(data) = frame.data_ref() { if bytes.len() + data.len() <= max_size || max_size == 0 { bytes.extend_from_slice(data); } else { trc::event!( Http(trc::HttpEvent::RequestBody), SpanId = session_id, Details = req .headers() .iter() .map(|(k, v)| trc::Value::Array(vec![ k.as_str().to_compact_string().into(), v.to_str().unwrap_or_default().to_compact_string().into() ])) .collect::>(), Contents = std::str::from_utf8(&bytes) .unwrap_or("[binary data]") .to_string(), Size = bytes.len(), Limit = max_size, ); return None; } } } trc::event!( Http(trc::HttpEvent::RequestBody), SpanId = session_id, Details = req .headers() .iter() .map(|(k, v)| trc::Value::Array(vec![ k.as_str().to_compact_string().into(), v.to_str().unwrap_or_default().to_compact_string().into() ])) .collect::>(), Contents = std::str::from_utf8(&bytes) .unwrap_or("[binary data]") .to_string(), Size = bytes.len(), ); bytes.into() } ================================================ FILE: crates/http-proto/src/response.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::manager::webadmin::Resource; use http_body_util::{BodyExt, Full}; use hyper::{ StatusCode, body::Bytes, header::{self, HeaderName, HeaderValue}, }; use serde_json::json; use crate::{ DownloadResponse, HtmlResponse, HttpResponse, HttpResponseBody, JsonProblemResponse, JsonResponse, ToHttpResponse, }; impl HttpResponse { pub fn new(status: StatusCode) -> Self { HttpResponse { status, builder: hyper::Response::builder().status(status), body: HttpResponseBody::Empty, } } pub fn with_content_type(mut self, content_type: V) -> Self where V: TryInto, >::Error: Into, { self.builder = self.builder.header(header::CONTENT_TYPE, content_type); self } pub fn with_status_code(mut self, status: StatusCode) -> Self { self.status = status; self.builder = self.builder.status(status); self } pub fn with_content_length(mut self, content_length: usize) -> Self { self.builder = self.builder.header(header::CONTENT_LENGTH, content_length); self } pub fn with_etag(mut self, etag: String) -> Self { self.builder = self.builder.header(header::ETAG, etag); self } pub fn with_etag_opt(self, etag: Option) -> Self { if let Some(etag) = etag { self.with_etag(etag) } else { self } } pub fn with_schedule_tag_opt(mut self, tag: Option) -> Self { if let Some(tag) = tag { self.builder = self.builder.header("Schedule-Tag", format!("\"{tag}\"")); self } else { self } } pub fn with_last_modified(mut self, last_modified: String) -> Self { self.builder = self.builder.header(header::LAST_MODIFIED, last_modified); self } pub fn with_lock_token(mut self, token_uri: &str) -> Self { self.builder = self.builder.header("Lock-Token", format!("<{token_uri}>")); self } pub fn with_header(mut self, name: K, value: V) -> Self where K: TryInto, >::Error: Into, V: TryInto, >::Error: Into, { self.builder = self.builder.header(name, value); self } pub fn with_xml_body(self, body: impl Into) -> Self { self.with_text_body(body) .with_content_type("application/xml; charset=utf-8") } pub fn with_text_body(mut self, body: impl Into) -> Self { let body = body.into(); let body_len = body.len(); self.body = HttpResponseBody::Text(body); self.with_content_length(body_len) } pub fn with_binary_body(mut self, body: impl Into>) -> Self { let body = body.into(); let body_len = body.len(); self.body = HttpResponseBody::Binary(body); self.with_content_length(body_len) } pub fn with_stream_body( mut self, stream: http_body_util::combinators::BoxBody, ) -> Self { self.body = HttpResponseBody::Stream(stream); self } pub fn with_websocket_upgrade(mut self, derived_key: String) -> Self { self.body = HttpResponseBody::WebsocketUpgrade(derived_key); self } pub fn with_content_disposition(mut self, content_disposition: V) -> Self where V: TryInto, >::Error: Into, { self.builder = self .builder .header(header::CONTENT_DISPOSITION, content_disposition); self } pub fn with_cache_control(mut self, cache_control: V) -> Self where V: TryInto, >::Error: Into, { self.builder = self.builder.header(header::CACHE_CONTROL, cache_control); self } pub fn with_no_store(mut self) -> Self { self.builder = self .builder .header(header::CACHE_CONTROL, "no-store, no-cache, must-revalidate"); self } pub fn with_no_cache(mut self) -> Self { self.builder = self.builder.header(header::CACHE_CONTROL, "no-cache"); self } pub fn with_location(mut self, location: V) -> Self where V: TryInto, >::Error: Into, { self.builder = self.builder.header(header::LOCATION, location); self } pub fn size(&self) -> usize { match &self.body { HttpResponseBody::Text(value) => value.len(), HttpResponseBody::Binary(value) => value.len(), _ => 0, } } pub fn build( self, ) -> hyper::Response> { match self.body { HttpResponseBody::Text(body) => self.builder.body( Full::new(Bytes::from(body)) .map_err(|never| match never {}) .boxed(), ), HttpResponseBody::Binary(body) => self.builder.body( Full::new(Bytes::from(body)) .map_err(|never| match never {}) .boxed(), ), HttpResponseBody::Empty => self.builder.header(header::CONTENT_LENGTH, 0).body( Full::new(Bytes::new()) .map_err(|never| match never {}) .boxed(), ), HttpResponseBody::Stream(stream) => self.builder.body(stream), HttpResponseBody::WebsocketUpgrade(derived_key) => self .builder .header(header::CONNECTION, "upgrade") .header(header::UPGRADE, "websocket") .header("Sec-WebSocket-Accept", &derived_key) .header("Sec-WebSocket-Protocol", "jmap") .body( Full::new(Bytes::from("Switching to WebSocket protocol")) .map_err(|never| match never {}) .boxed(), ), } .unwrap() } pub fn body(&self) -> &HttpResponseBody { &self.body } pub fn status(&self) -> StatusCode { self.status } pub fn headers(&self) -> Option<&hyper::HeaderMap> { self.builder.headers_ref() } } impl ToHttpResponse for JsonResponse { fn into_http_response(self) -> HttpResponse { let response = HttpResponse::new(self.status) .with_content_type("application/json; charset=utf-8") .with_text_body(serde_json::to_string(&self.inner).unwrap_or_default()); if self.no_cache { response.with_no_store() } else { response } } } impl ToHttpResponse for DownloadResponse { fn into_http_response(self) -> HttpResponse { HttpResponse::new(StatusCode::OK) .with_content_type(self.content_type) .with_content_disposition(format!( "attachment; filename=\"{}\"", self.filename.replace('\"', "\\\"") )) .with_cache_control("private, immutable, max-age=31536000") .with_binary_body(self.blob) } } impl ToHttpResponse for Resource> { fn into_http_response(self) -> HttpResponse { HttpResponse::new(StatusCode::OK) .with_content_type(self.content_type.as_ref()) .with_binary_body(self.contents) } } impl ToHttpResponse for HtmlResponse { fn into_http_response(self) -> HttpResponse { HttpResponse::new(self.status) .with_content_type("text/html; charset=utf-8") .with_text_body(self.body) } } impl ToHttpResponse for JsonProblemResponse { fn into_http_response(self) -> HttpResponse { HttpResponse::new(self.0) .with_content_type("application/problem+json") .with_text_body( serde_json::to_string(&json!( { "type": "about:blank", "title": self.0.canonical_reason().unwrap_or_default(), "status": self.0.as_u16(), "detail": self.0.canonical_reason().unwrap_or_default(), } )) .unwrap_or_default(), ) } } ================================================ FILE: crates/imap/Cargo.toml ================================================ [package] name = "imap" version = "0.15.5" edition = "2024" [dependencies] imap_proto = { path = "../imap-proto" } types = { path = "../types" } directory = { path = "../directory" } trc = { path = "../trc" } store = { path = "../store" } common = { path = "../common" } email = { path = "../email" } nlp = { path = "../nlp" } utils = { path = "../utils" } mail-parser = { version = "0.11", features = ["full_encoding"] } mail-send = { version = "0.5", default-features = false, features = ["cram-md5", "ring", "tls12"] } rustls = { version = "0.23.5", default-features = false, features = ["std", "ring", "tls12"] } rustls-pemfile = "2.0" tokio = { version = "1.47", features = ["full"] } tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } parking_lot = "0.12" ahash = { version = "0.8" } md5 = "0.8.0" rand = "0.9.0" indexmap = "2.7.1" compact_str = "0.9.0" [features] test_mode = [] ================================================ FILE: crates/imap/src/core/client.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{iter::Peekable, sync::Arc, vec::IntoIter}; use common::{ KV_RATE_LIMIT_IMAP, listener::{SessionResult, SessionStream}, }; use imap_proto::{ Command, ResponseType, StatusResponse, receiver::{self, Request}, }; use trc::SecurityEvent; use super::{SelectedMailbox, Session, SessionData, State}; impl Session { pub async fn ingest(&mut self, bytes: &[u8]) -> SessionResult { trc::event!( Imap(trc::ImapEvent::RawInput), SpanId = self.session_id, Size = bytes.len(), Contents = trc::Value::from_maybe_string(bytes), ); let mut bytes = bytes.iter(); let mut requests = Vec::with_capacity(2); let mut needs_literal = None; loop { match self.receiver.parse(&mut bytes) { Ok(request) => match self.is_allowed(request).await { Ok(request) => { requests.push(request); } Err(err) => { if !self.write_error(err).await { return SessionResult::Close; } } }, Err(receiver::Error::NeedsMoreData) => { break; } Err(receiver::Error::NeedsLiteral { size }) => { needs_literal = size.into(); break; } Err(receiver::Error::Error { response }) => { // Check for port scanners if matches!( (&self.state, response.key(trc::Key::Code)), ( State::NotAuthenticated { .. }, Some(trc::Value::String(v)) ) if v == "PARSE" ) { match self.server.is_scanner_fail2banned(self.remote_addr).await { Ok(true) => { trc::event!( Security(SecurityEvent::ScanBan), SpanId = self.session_id, RemoteIp = self.remote_addr, Reason = "Invalid IMAP command", ); return SessionResult::Close; } Ok(false) => {} Err(err) => { trc::error!( err.span_id(self.session_id) .details("Failed to check for fail2ban") ); } } } if !self.write_error(response).await { return SessionResult::Close; } break; } } } let mut requests = requests.into_iter().peekable(); while let Some(request) = requests.next() { let result = match request.command { Command::List | Command::Lsub => self .handle_list(request) .await .map(|_| SessionResult::Continue), Command::Select | Command::Examine => self .handle_select(request) .await .map(|_| SessionResult::Continue), Command::Create => self .handle_create(group_requests(&mut requests, vec![request])) .await .map(|_| SessionResult::Continue), Command::Delete => self .handle_delete(group_requests(&mut requests, vec![request])) .await .map(|_| SessionResult::Continue), Command::Rename => self .handle_rename(request) .await .map(|_| SessionResult::Continue), Command::Status => self .handle_status(group_requests(&mut requests, vec![request])) .await .map(|_| SessionResult::Continue), Command::Append => self .handle_append(request) .await .map(|_| SessionResult::Continue), Command::Close => self .handle_close(request) .await .map(|_| SessionResult::Continue), Command::Unselect => self .handle_unselect(request) .await .map(|_| SessionResult::Continue), Command::Expunge(is_uid) => self .handle_expunge(request, is_uid) .await .map(|_| SessionResult::Continue), Command::Search(is_uid) => self .handle_search(request, false, is_uid) .await .map(|_| SessionResult::Continue), Command::Fetch(_) => self .handle_fetch(group_requests(&mut requests, vec![request])) .await .map(|_| SessionResult::Continue), Command::Store(is_uid) => self .handle_store(request, is_uid) .await .map(|_| SessionResult::Continue), Command::Copy(is_uid) => self .handle_copy_move(request, false, is_uid) .await .map(|_| SessionResult::Continue), Command::Move(is_uid) => self .handle_copy_move(request, true, is_uid) .await .map(|_| SessionResult::Continue), Command::Sort(is_uid) => self .handle_search(request, true, is_uid) .await .map(|_| SessionResult::Continue), Command::Thread(is_uid) => self .handle_thread(request, is_uid) .await .map(|_| SessionResult::Continue), Command::Idle => self .handle_idle(request) .await .map(|_| SessionResult::Continue), Command::Subscribe => self .handle_subscribe(request, true) .await .map(|_| SessionResult::Continue), Command::Unsubscribe => self .handle_subscribe(request, false) .await .map(|_| SessionResult::Continue), Command::Namespace => self .handle_namespace(request) .await .map(|_| SessionResult::Continue), Command::Authenticate => self .handle_authenticate(request) .await .map(|_| SessionResult::Continue), Command::Login => self .handle_login(request) .await .map(|_| SessionResult::Continue), Command::Capability => self .handle_capability(request) .await .map(|_| SessionResult::Continue), Command::Enable => self .handle_enable(request) .await .map(|_| SessionResult::Continue), Command::StartTls => self .write_bytes( StatusResponse::ok("Begin TLS negotiation now") .with_tag(request.tag) .into_bytes(), ) .await .map(|_| SessionResult::UpgradeTls), Command::Noop => self .handle_noop(request) .await .map(|_| SessionResult::Continue), Command::Check => self .handle_noop(request) .await .map(|_| SessionResult::Continue), Command::Logout => self .handle_logout(request) .await .map(|_| SessionResult::Close), Command::SetAcl => self .handle_set_acl(request) .await .map(|_| SessionResult::Continue), Command::DeleteAcl => self .handle_set_acl(request) .await .map(|_| SessionResult::Continue), Command::GetAcl => self .handle_get_acl(request) .await .map(|_| SessionResult::Continue), Command::ListRights => self .handle_list_rights(request) .await .map(|_| SessionResult::Continue), Command::MyRights => self .handle_my_rights(request) .await .map(|_| SessionResult::Continue), Command::GetQuota => self .handle_get_quota(request) .await .map(|_| SessionResult::Continue), Command::GetQuotaRoot => self .handle_get_quota_root(request) .await .map(|_| SessionResult::Continue), Command::Unauthenticate => self .handle_unauthenticate(request) .await .map(|_| SessionResult::Continue), Command::Id => self .handle_id(request) .await .map(|_| SessionResult::Continue), }; match result { Ok(SessionResult::Continue) => (), Ok(result) => return result, Err(err) => { if !self.write_error(err).await { return SessionResult::Close; } } } } if let Some(needs_literal) = needs_literal && let Err(err) = self .write_bytes(format!("+ Ready for {} bytes.\r\n", needs_literal).into_bytes()) .await { self.write_error(err).await; return SessionResult::Close; } SessionResult::Continue } } pub fn group_requests( requests: &mut Peekable>>, mut grouped_requests: Vec>, ) -> Vec> { let last_command = grouped_requests.last().unwrap().command; loop { match requests.peek() { Some(request) if request.command == last_command => { grouped_requests.push(requests.next().unwrap()); } _ => break, } } grouped_requests } impl Session { async fn is_allowed(&self, request: Request) -> trc::Result> { let state = &self.state; // Rate limit request if let State::Authenticated { data } | State::Selected { data, .. } = state && let Some(rate) = &self.server.core.imap.rate_requests && data .server .core .storage .lookup .is_rate_allowed( KV_RATE_LIMIT_IMAP, &data.account_id.to_be_bytes(), rate, true, ) .await? .is_some() { return Err(trc::LimitEvent::TooManyRequests.into_err()); } match &request.command { Command::Capability | Command::Noop | Command::Logout | Command::Id => Ok(request), Command::StartTls => { if !self.is_tls { if self.instance.acceptor.is_tls() { Ok(request) } else { Err(trc::ImapEvent::Error .into_err() .details("TLS is not available.") .id(request.tag)) } } else { Err(trc::ImapEvent::Error .into_err() .details("Already in TLS mode.") .id(request.tag)) } } Command::Authenticate => { if let State::NotAuthenticated { .. } = state { Ok(request) } else { Err(trc::ImapEvent::Error .into_err() .details("Already authenticated.") .id(request.tag)) } } Command::Login => { if let State::NotAuthenticated { .. } = state { if self.is_tls || self.server.core.imap.allow_plain_auth { Ok(request) } else { Err(trc::ImapEvent::Error .into_err() .details("LOGIN is disabled on the clear-text port.") .id(request.tag)) } } else { Err(trc::ImapEvent::Error .into_err() .details("Already authenticated.") .id(request.tag)) } } Command::Enable | Command::Select | Command::Examine | Command::Create | Command::Delete | Command::Rename | Command::Subscribe | Command::Unsubscribe | Command::List | Command::Lsub | Command::Namespace | Command::Status | Command::Append | Command::Idle | Command::SetAcl | Command::DeleteAcl | Command::GetAcl | Command::ListRights | Command::MyRights | Command::Unauthenticate | Command::GetQuota | Command::GetQuotaRoot => { if let State::Authenticated { .. } | State::Selected { .. } = state { Ok(request) } else { Err(trc::ImapEvent::Error .into_err() .details("Not authenticated.") .id(request.tag)) } } Command::Close | Command::Unselect | Command::Expunge(_) | Command::Search(_) | Command::Fetch(_) | Command::Store(_) | Command::Copy(_) | Command::Move(_) | Command::Check | Command::Sort(_) | Command::Thread(_) => match state { State::Selected { mailbox, .. } => { if mailbox.is_select || !matches!( request.command, Command::Store(_) | Command::Expunge(_) | Command::Move(_), ) { Ok(request) } else { Err(trc::ImapEvent::Error .into_err() .details("Not permitted in EXAMINE state.") .id(request.tag)) } } State::Authenticated { .. } => Err(trc::ImapEvent::Error .into_err() .details("No mailbox is selected.") .ctx(trc::Key::Type, ResponseType::Bad) .id(request.tag)), State::NotAuthenticated { .. } => Err(trc::ImapEvent::Error .into_err() .details("Not authenticated.") .id(request.tag)), }, } } } impl State { pub fn auth_failures(&self) -> u32 { match self { State::NotAuthenticated { auth_failures, .. } => *auth_failures, _ => unreachable!(), } } pub fn session_data(&self) -> Arc> { match self { State::Authenticated { data } => data.clone(), State::Selected { data, .. } => data.clone(), _ => unreachable!(), } } pub fn mailbox_state(&self) -> (Arc>, Arc) { match self { State::Selected { data, mailbox, .. } => (data.clone(), mailbox.clone()), _ => unreachable!(), } } pub fn session_mailbox_state(&self) -> (Arc>, Option>) { match self { State::Authenticated { data } => (data.clone(), None), State::Selected { data, mailbox, .. } => (data.clone(), mailbox.clone().into()), _ => unreachable!(), } } pub fn select_data(&self) -> (Arc>, Arc) { match self { State::Selected { data, mailbox } => (data.clone(), mailbox.clone()), _ => unreachable!(), } } pub fn spawn_task(&self, params: P, fnc: F) -> trc::Result<()> where F: FnOnce(P, &super::SessionData) -> R + Send + 'static, P: Send + Sync + 'static, R: std::future::Future> + Send + 'static, { let data = self.session_data(); tokio::spawn(async move { if let Err(err) = fnc(params, &data).await { let _ = data.write_error(err).await; } }); Ok(()) } pub fn is_authenticated(&self) -> bool { matches!(self, State::Authenticated { .. } | State::Selected { .. }) } pub fn close_mailbox(&self) -> bool { matches!(self, State::Selected { .. }) } } ================================================ FILE: crates/imap/src/core/mailbox.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{Account, MailboxId, MailboxSync, Session, SessionData}; use crate::core::Mailbox; use ahash::AHashMap; use common::{ auth::AccessToken, listener::{SessionStream, limiter::InFlight}, sharing::EffectiveAcl, }; use directory::backend::internal::manage::ManageDirectory; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess, mailbox::MailboxCacheAccess}, mailbox::INBOX_ID, }; use imap_proto::protocol::list::Attribute; use parking_lot::Mutex; use std::{ collections::BTreeMap, sync::{Arc, atomic::Ordering}, }; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{acl::Acl, collection::Collection, id::Id, keyword::Keyword, special_use::SpecialUse}; impl SessionData { pub async fn new( session: &Session, access_token: Arc, in_flight: Option, ) -> trc::Result { let mut session = SessionData { stream_tx: session.stream_tx.clone(), server: session.server.clone(), account_id: access_token.primary_id(), session_id: session.session_id, mailboxes: Mutex::new(vec![]), state: access_token.state().into(), access_token, in_flight, }; let access_token = session.access_token.clone(); // Fetch mailboxes for the main account let mut mailboxes = vec![ session .fetch_account_mailboxes(session.account_id, None, &access_token, None) .await .caused_by(trc::location!())? .unwrap(), ]; // Fetch shared mailboxes for &account_id in access_token.shared_accounts(Collection::Mailbox) { let prefix: String = format!( "{}/{}", session.server.core.jmap.shared_folder, session .server .store() .get_principal_name(account_id) .await .caused_by(trc::location!())? .unwrap_or_else(|| Id::from(account_id).to_string()) ); mailboxes.push( session .fetch_account_mailboxes(account_id, prefix.into(), &access_token, None) .await .caused_by(trc::location!())? .unwrap(), ); } session.mailboxes = Mutex::new(mailboxes); Ok(session) } async fn fetch_account_mailboxes( &self, account_id: u32, mailbox_prefix: Option, access_token: &AccessToken, current_state: Option, ) -> trc::Result> { let cache = self .server .get_cached_messages(account_id) .await .caused_by(trc::location!())?; if current_state.is_some_and(|state| state == cache.last_change_id) { return Ok(None); } let shared_mailbox_ids = if access_token.is_primary_id(account_id) || access_token.member_of.contains(&account_id) { None } else { cache.shared_mailboxes(access_token, Acl::Read).into() }; // Build special uses let mut special_uses = AHashMap::new(); for mailbox in &cache.mailboxes.items { if shared_mailbox_ids .as_ref() .is_none_or(|ids| ids.contains(mailbox.document_id)) && !matches!(mailbox.role, SpecialUse::None) { special_uses.insert(mailbox.role, mailbox.document_id); } } // Build account let mut account = Account { account_id, prefix: mailbox_prefix, mailbox_names: BTreeMap::new(), mailbox_state: AHashMap::with_capacity(cache.mailboxes.items.len()), last_change_id: cache.last_change_id, }; for mailbox in &cache.mailboxes.items { if shared_mailbox_ids .as_ref() .is_some_and(|ids| !ids.contains(mailbox.document_id)) { continue; } // Build mailbox path and map it to its effective id let mailbox_name = if let Some(prefix) = &account.prefix { let mut name = String::with_capacity(prefix.len() + mailbox.path.len() + 1); name.push_str(prefix.as_str()); name.push('/'); name.push_str(mailbox.path.as_str()); name } else { mailbox.path.clone() }; let effective_mailbox_id = self .server .core .jmap .default_folders .iter() .find(|f| f.name == mailbox_name || f.aliases.iter().any(|a| a == &mailbox_name)) .and_then(|f| special_uses.get(&f.special_use)) .copied() .unwrap_or(mailbox.document_id); account .mailbox_names .insert(mailbox_name, effective_mailbox_id); account.mailbox_state.insert( mailbox.document_id, Mailbox { has_children: cache .mailboxes .items .iter() .any(|child| child.parent_id == mailbox.document_id), is_subscribed: mailbox.subscribers.contains(&access_token.primary_id()), special_use: match mailbox.role { SpecialUse::Trash => Some(Attribute::Trash), SpecialUse::Junk => Some(Attribute::Junk), SpecialUse::Drafts => Some(Attribute::Drafts), SpecialUse::Archive => Some(Attribute::Archive), SpecialUse::Sent => Some(Attribute::Sent), SpecialUse::Important => Some(Attribute::Important), SpecialUse::Memos => Some(Attribute::Memos), SpecialUse::Scheduled => Some(Attribute::Scheduled), SpecialUse::Snoozed => Some(Attribute::Snoozed), _ => None, }, total_messages: cache.in_mailbox(mailbox.document_id).count() as u64, total_unseen: cache .in_mailbox_without_keyword(mailbox.document_id, &Keyword::Seen) .count() as u64, total_deleted: cache .in_mailbox_with_keyword(mailbox.document_id, &Keyword::Deleted) .count() as u64, uid_validity: mailbox.uid_validity as u64, uid_next: self .get_uid_next(&MailboxId { account_id, mailbox_id: mailbox.document_id, }) .await .caused_by(trc::location!())? as u64, total_deleted_storage: None, size: None, }, ); } Ok(account.into()) } pub async fn synchronize_mailboxes( &self, return_changes: bool, ) -> trc::Result> { let mut changes = if return_changes { MailboxSync::default().into() } else { None }; // Obtain access token let access_token = self .server .get_access_token(self.account_id) .await .caused_by(trc::location!())?; let state = access_token.state(); // Shared mailboxes might have changed let mut added_accounts = Vec::new(); if self.state.load(Ordering::Relaxed) != state { // Remove unlinked shared accounts let mut added_account_ids = Vec::new(); { let mut mailboxes = self.mailboxes.lock(); let mut new_accounts = Vec::with_capacity(mailboxes.len()); let has_access_to = access_token .shared_accounts(Collection::Mailbox) .copied() .collect::>(); for account in mailboxes.drain(..) { if access_token.is_primary_id(account.account_id) || has_access_to.contains(&account.account_id) { new_accounts.push(account); } else { // Add unshared mailboxes to deleted list if let Some(changes) = &mut changes { for (mailbox_name, _) in account.mailbox_names { changes.deleted.push(mailbox_name); } } } } // Add new shared account ids for account_id in has_access_to { if !new_accounts .iter() .skip(1) .any(|m| m.account_id == account_id) { added_account_ids.push(account_id); } } *mailboxes = new_accounts; } // Fetch mailboxes for each new shared account for account_id in added_account_ids { let prefix: String = format!( "{}/{}", self.server.core.jmap.shared_folder, self.server .store() .get_principal_name(account_id) .await .caused_by(trc::location!())? .unwrap_or_else(|| Id::from(account_id).to_string()) ); added_accounts.push( self.fetch_account_mailboxes(account_id, prefix.into(), &access_token, None) .await? .unwrap(), ); } // Update state self.state.store(state, Ordering::Relaxed); } // Fetch mailbox changes for all accounts let mut changed_accounts = Vec::new(); let account_states = self .mailboxes .lock() .iter() .map(|m| (m.account_id, m.prefix.clone(), m.last_change_id)) .collect::>(); for (account_id, prefix, last_state) in account_states { if let Some(changed_account) = self .fetch_account_mailboxes(account_id, prefix, &access_token, last_state.into()) .await .caused_by(trc::location!())? { changed_accounts.push(changed_account); } } // Update mailboxes if !changed_accounts.is_empty() || !added_accounts.is_empty() { let mut mailboxes = self.mailboxes.lock(); for changed_account in changed_accounts { if let Some(pos) = mailboxes .iter() .position(|a| a.account_id == changed_account.account_id) { // Add changes and deletions if let Some(changes) = &mut changes { let old_account = &mailboxes[pos]; let new_account = &changed_account; // Add new mailboxes for (mailbox_name, mailbox_id) in new_account.mailbox_names.iter() { if let Some(old_mailbox) = old_account.mailbox_state.get(mailbox_id) { if let Some(mailbox) = new_account.mailbox_state.get(mailbox_id) && (mailbox.total_messages != old_mailbox.total_messages || mailbox.total_unseen != old_mailbox.total_unseen) { changes.changed.push(mailbox_name.clone()); } } else { changes.added.push(mailbox_name.clone()); } } // Add deleted mailboxes for (mailbox_name, mailbox_id) in &old_account.mailbox_names { if !new_account.mailbox_state.contains_key(mailbox_id) { changes.deleted.push(mailbox_name.clone()); } } } mailboxes[pos] = changed_account; } else { // Add newly shared accounts if let Some(changes) = &mut changes { changes .added .extend(changed_account.mailbox_names.keys().cloned()); } mailboxes.push(changed_account); } } if !added_accounts.is_empty() { // Add newly shared accounts if let Some(changes) = &mut changes { for added_account in &added_accounts { changes .added .extend(added_account.mailbox_names.keys().cloned()); } } mailboxes.extend(added_accounts); } } Ok(changes) } pub fn get_mailbox_by_name(&self, mailbox_name: &str) -> Option { let is_inbox = mailbox_name.eq_ignore_ascii_case("inbox"); for account in self.mailboxes.lock().iter() { if account .prefix .as_ref() .is_none_or(|p| mailbox_name.starts_with(p.as_str())) { for (mailbox_name_, mailbox_id_) in account.mailbox_names.iter() { if (!is_inbox && mailbox_name_ == mailbox_name) || (is_inbox && *mailbox_id_ == INBOX_ID) { return MailboxId { account_id: account.account_id, mailbox_id: *mailbox_id_, } .into(); } } } } None } pub async fn check_mailbox_acl( &self, account_id: u32, document_id: u32, item: Acl, ) -> trc::Result { let access_token = self.get_access_token().await?; Ok(access_token.is_member(account_id) || self .server .store() .get_value::>(ValueKey::archive( account_id, Collection::Mailbox, document_id, )) .await .and_then(|mailbox| { if let Some(mailbox) = mailbox { Ok(Some( mailbox .unarchive::()? .acls .effective_acl(&access_token) .contains(item), )) } else { Ok(None) } })? .ok_or_else(|| { trc::ImapEvent::Error .caused_by(trc::location!()) .details("Mailbox no longer exists.") })?) } } ================================================ FILE: crates/imap/src/core/message.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ ImapUidToId, Mailbox, MailboxId, MailboxState, NextMailboxState, SelectedMailbox, SessionData, }; use crate::core::ImapId; use ahash::AHashMap; use common::listener::SessionStream; use email::cache::MessageCacheFetch; use imap_proto::protocol::{Sequence, expunge, select::Exists}; use std::collections::BTreeMap; use store::{ValueKey, write::ValueClass}; use trc::AddContext; use types::{collection::Collection, field::MailboxField}; impl SessionData { pub async fn fetch_messages( &self, mailbox: &MailboxId, current_state: Option, ) -> trc::Result> { let cached_messages = self .server .get_cached_messages(mailbox.account_id) .await .caused_by(trc::location!())?; if current_state.is_some_and(|state| state == cached_messages.emails.change_id) { return Ok(None); } // Obtain UID next and assign UIDs let uid_map = cached_messages .emails .items .iter() .filter_map(|item| { item.mailboxes.iter().find_map(|m| { if m.mailbox_id == mailbox.mailbox_id { Some((m.uid, item.document_id)) } else { None } }) }) .collect::>(); let mut uid_max = 0; let mut id_to_imap = AHashMap::with_capacity(uid_map.len()); let mut uid_to_id = AHashMap::with_capacity(uid_map.len()); for (seqnum, (uid, message_id)) in uid_map.into_iter().enumerate() { if uid > uid_max { uid_max = uid; } id_to_imap.insert( message_id, ImapId { uid, seqnum: seqnum as u32 + 1, }, ); uid_to_id.insert(uid, message_id); } Ok(Some(MailboxState { total_messages: id_to_imap.len(), id_to_imap, uid_to_id, uid_max, modseq: cached_messages.emails.change_id, next_state: None, })) } pub async fn synchronize_messages(&self, mailbox: &SelectedMailbox) -> trc::Result { // Obtain current modseq let mut current_modseq = mailbox.state.lock().modseq; if let Some(new_state) = self .fetch_messages(&mailbox.id, current_modseq.into()) .await? { // Synchronize messages let mut current_state = mailbox.state.lock(); current_modseq = new_state.modseq; // Add missing uids let mut deletions = current_state .next_state .take() .map(|state| state.deletions) .unwrap_or_default(); let mut id_to_imap = AHashMap::with_capacity(current_state.id_to_imap.len()); for (id, imap_id) in std::mem::take(&mut current_state.id_to_imap) { if !new_state.uid_to_id.contains_key(&imap_id.uid) { // Add to deletions deletions.push(imap_id); // Invalidate entries current_state.uid_to_id.remove(&imap_id.uid); } else { id_to_imap.insert(id, imap_id); } } current_state.id_to_imap = id_to_imap; // Update state current_state.modseq = new_state.modseq; current_state.next_state = Some(Box::new(NextMailboxState { next_state: new_state, deletions, })); } Ok(current_modseq) } pub async fn write_mailbox_changes( &self, mailbox: &SelectedMailbox, is_qresync: bool, ) -> trc::Result { // Resync mailbox let modseq = self.synchronize_messages(mailbox).await?; let mut buf = Vec::new(); { let mut current_state = mailbox.state.lock(); if let Some(next_state) = current_state.next_state.take() { if !next_state.deletions.is_empty() { let mut ids = next_state .deletions .into_iter() .map(|id| if is_qresync { id.uid } else { id.seqnum }) .collect::>(); ids.sort_unstable(); expunge::Response { is_qresync, ids }.serialize_to(&mut buf); } if !buf.is_empty() || next_state .next_state .uid_max .saturating_sub(current_state.uid_max) > 0 { Exists { total_messages: next_state.next_state.total_messages, } .serialize(&mut buf); } *current_state = next_state.next_state; } } if !buf.is_empty() { self.write_bytes(buf).await?; } Ok(modseq) } pub async fn get_uid_next(&self, mailbox: &MailboxId) -> trc::Result { self.server .core .storage .data .get_counter(ValueKey { account_id: mailbox.account_id, collection: Collection::Mailbox.into(), document_id: mailbox.mailbox_id, class: ValueClass::Property(MailboxField::UidCounter.into()), }) .await .map(|v| (v + 1) as u32) } pub fn mailbox_state(&self, mailbox: &MailboxId) -> Option { self.mailboxes .lock() .iter() .find(|m| m.account_id == mailbox.account_id) .and_then(|m| m.mailbox_state.get(&mailbox.mailbox_id)) .cloned() } } impl SelectedMailbox { pub async fn sequence_to_ids( &self, sequence: &Sequence, is_uid: bool, ) -> trc::Result> { if !sequence.is_saved_search() { let mut ids = AHashMap::new(); let state = self.state.lock(); if is_uid { let id_to_imap = state .next_state .as_ref() .map(|s| &s.next_state.id_to_imap) .unwrap_or(&state.id_to_imap); if !state.id_to_imap.is_empty() { for (id, imap_id) in id_to_imap { if sequence.contains(imap_id.uid, state.uid_max) { ids.insert(*id, *imap_id); } } } } else if !state.id_to_imap.is_empty() { for (id, imap_id) in &state.id_to_imap { if sequence.contains(imap_id.seqnum, state.total_messages as u32) { ids.insert(*id, *imap_id); } } } Ok(ids) } else { let saved_ids = self.get_saved_search().await.ok_or_else(|| { trc::ImapEvent::Error .into_err() .details("No saved search found.") })?; let mut ids = AHashMap::with_capacity(saved_ids.len()); let state = self.state.lock(); for imap_id in saved_ids.iter() { if let Some(id) = state.uid_to_id.get(&imap_id.uid) { ids.insert(*id, *imap_id); } } Ok(ids) } } pub async fn sequence_expand_missing(&self, sequence: &Sequence, is_uid: bool) -> Vec { let mut deleted_ids = Vec::new(); if !sequence.is_saved_search() { let state = self.state.lock(); if is_uid { for uid in sequence.expand(state.uid_max) { if !state.uid_to_id.contains_key(&uid) { deleted_ids.push(uid); } } } else { for seqnum in sequence.expand(state.total_messages as u32) { if seqnum > state.total_messages as u32 { deleted_ids.push(seqnum); } } } } else if let Some(saved_ids) = self.get_saved_search().await { let state = self.state.lock(); for id in saved_ids.iter() { if !state.uid_to_id.contains_key(&id.uid) { deleted_ids.push(if is_uid { id.uid } else { id.seqnum }); } } } deleted_ids.sort_unstable(); deleted_ids } pub fn append_messages(&self, ids: Vec, modseq: Option) { let mut mailbox = self.state.lock(); if modseq.unwrap_or(0) > mailbox.modseq { let mut uid_max = 0; for id in ids { mailbox.total_messages += 1; let seqnum = mailbox.total_messages as u32; mailbox.uid_to_id.insert(id.uid, id.uid); mailbox.id_to_imap.insert( id.id, ImapId { uid: id.uid, seqnum, }, ); uid_max = id.uid; } mailbox.uid_max = uid_max; } } } ================================================ FILE: crates/imap/src/core/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ collections::BTreeMap, net::IpAddr, sync::{Arc, atomic::AtomicU32}, }; use ahash::AHashMap; use common::{ Inner, Server, auth::AccessToken, listener::{ServerInstance, SessionStream, limiter::InFlight}, }; use imap_proto::{ Command, protocol::{ProtocolVersion, list::Attribute}, receiver::Receiver, }; use tokio::{ io::{ReadHalf, WriteHalf}, sync::watch, }; use trc::AddContext; pub mod client; pub mod mailbox; pub mod message; pub mod session; #[derive(Clone)] pub struct ImapSessionManager { pub inner: Arc, } impl ImapSessionManager { pub fn new(inner: Arc) -> Self { Self { inner } } } pub struct Session { pub server: Server, pub instance: Arc, pub receiver: Receiver, pub version: ProtocolVersion, pub state: State, pub is_tls: bool, pub is_condstore: bool, pub is_qresync: bool, pub is_utf8: bool, pub stream_rx: ReadHalf, pub stream_tx: Arc>>, pub in_flight: InFlight, pub remote_addr: IpAddr, pub session_id: u64, } pub struct SessionData { pub account_id: u32, pub access_token: Arc, pub server: Server, pub session_id: u64, pub mailboxes: parking_lot::Mutex>, pub stream_tx: Arc>>, pub state: AtomicU32, pub in_flight: Option, } pub struct SelectedMailbox { pub id: MailboxId, pub state: parking_lot::Mutex, pub saved_search: parking_lot::Mutex, pub is_select: bool, pub is_condstore: bool, } #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] pub struct AccountId { pub account_id: u32, pub primary_id: u32, } #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] pub struct MailboxId { pub account_id: u32, pub mailbox_id: u32, } #[derive(Debug, Clone, Default)] pub struct Account { pub account_id: u32, pub prefix: Option, pub mailbox_names: BTreeMap, pub mailbox_state: AHashMap, pub last_change_id: u64, } #[derive(Debug, Default, Clone)] pub struct Mailbox { pub has_children: bool, pub is_subscribed: bool, pub special_use: Option, pub total_messages: u64, pub total_unseen: u64, pub total_deleted: u64, pub total_deleted_storage: Option, pub uid_validity: u64, pub uid_next: u64, pub size: Option, } #[derive(Debug, Clone, Default)] pub struct MailboxState { pub uid_max: u32, pub id_to_imap: AHashMap, pub uid_to_id: AHashMap, pub total_messages: usize, pub modseq: u64, pub next_state: Option>, } #[derive(Debug, Clone)] pub struct NextMailboxState { pub next_state: MailboxState, pub deletions: Vec, } #[derive(Debug, Clone, Copy, Default)] pub struct ImapId { pub uid: u32, pub seqnum: u32, } #[derive(Debug, Default)] pub struct MailboxSync { pub added: Vec, pub changed: Vec, pub deleted: Vec, } pub enum SavedSearch { InFlight { rx: watch::Receiver>>, }, Results { items: Arc>, }, None, } #[derive(Debug, Clone, Copy, Default)] pub struct ImapUidToId { pub uid: u32, pub id: u32, } pub enum State { NotAuthenticated { auth_failures: u32, }, Authenticated { data: Arc>, }, Selected { data: Arc>, mailbox: Arc, }, } impl State { pub fn try_replace_stream_tx( self, new_stream: Arc>>, ) -> Option> { match self { State::NotAuthenticated { auth_failures } => { State::NotAuthenticated { auth_failures }.into() } State::Authenticated { data } => { Arc::try_unwrap(data).ok().map(|data| State::Authenticated { data: Arc::new(data.replace_stream_tx(new_stream)), }) } State::Selected { data, mailbox } => { Arc::try_unwrap(data).ok().map(|data| State::Selected { data: Arc::new(data.replace_stream_tx(new_stream)), mailbox, }) } } } } impl SessionData { pub async fn get_access_token(&self) -> trc::Result> { self.server .get_access_token(self.account_id) .await .caused_by(trc::location!()) } pub fn replace_stream_tx( self, new_stream: Arc>>, ) -> SessionData { SessionData { account_id: self.account_id, server: self.server, session_id: self.session_id, mailboxes: self.mailboxes, stream_tx: new_stream, state: self.state, in_flight: self.in_flight, access_token: self.access_token, } } } impl MailboxState { pub fn map_result_id(&self, document_id: u32, is_uid: bool) -> Option<(u32, ImapId)> { if let Some(imap_id) = self.id_to_imap.get(&document_id) { Some((if is_uid { imap_id.uid } else { imap_id.seqnum }, *imap_id)) } else if is_uid { self.next_state.as_ref().and_then(|s| { s.next_state .id_to_imap .get(&document_id) .map(|imap_id| (imap_id.uid, *imap_id)) }) } else { None } } } ================================================ FILE: crates/imap/src/core/session.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::Arc; use common::{ core::BuildServer, listener::{SessionData, SessionManager, SessionResult, SessionStream, stream::NullIo}, }; use imap_proto::{ protocol::{ProtocolVersion, SerializeResponse}, receiver::Receiver, }; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio_rustls::server::TlsStream; use crate::{GREETING_WITH_TLS, GREETING_WITHOUT_TLS}; use super::{ImapSessionManager, Session, State}; impl SessionManager for ImapSessionManager { #[allow(clippy::manual_async_fn)] fn handle( self, session: SessionData, ) -> impl std::future::Future + Send { async move { if let Ok(mut session) = Session::new(session, self).await && session.handle_conn().await && session.instance.acceptor.is_tls() && let Ok(mut session) = session.into_tls().await { session.handle_conn().await; } } } #[allow(clippy::manual_async_fn)] fn shutdown(&self) -> impl std::future::Future + Send { async {} } } impl Session { pub async fn handle_conn(&mut self) -> bool { let mut buf = vec![0; 8192]; let mut shutdown_rx = self.instance.shutdown_rx.clone(); loop { tokio::select! { result = tokio::time::timeout( if !matches!(self.state, State::NotAuthenticated {..}) { self.server.core.imap.timeout_auth } else { self.server.core.imap.timeout_unauth }, self.stream_rx.read(&mut buf)) => { match result { Ok(Ok(bytes_read)) => { if bytes_read > 0 { match self.ingest(&buf[..bytes_read]).await { SessionResult::Continue => (), SessionResult::UpgradeTls => { return true; } SessionResult::Close => { break; } } } else { trc::event!( Network(trc::NetworkEvent::Closed), SpanId = self.session_id, CausedBy = trc::location!() ); break; } }, Ok(Err(err)) => { trc::event!( Network(trc::NetworkEvent::ReadError), SpanId = self.session_id, Reason = err.to_string(), CausedBy = trc::location!() ); break; }, Err(_) => { trc::event!( Network(trc::NetworkEvent::Timeout), SpanId = self.session_id, CausedBy = trc::location!() ); self.write_bytes(&b"* BYE Connection timed out.\r\n"[..]).await.ok(); break; } } }, _ = shutdown_rx.changed() => { trc::event!( Network(trc::NetworkEvent::Closed), SpanId = self.session_id, Reason = "Server shutting down", CausedBy = trc::location!() ); self.write_bytes(&b"* BYE Server shutting down.\r\n"[..]).await.ok(); break; } }; } false } pub async fn new( mut session: SessionData, manager: ImapSessionManager, ) -> Result, ()> { // Write greeting let is_tls = session.stream.is_tls(); let greeting = if !is_tls && session.instance.acceptor.is_tls() { &GREETING_WITH_TLS } else { &GREETING_WITHOUT_TLS }; if let Err(err) = session.stream.write_all(greeting).await { trc::event!( Network(trc::NetworkEvent::WriteError), Reason = err.to_string(), SpanId = session.session_id, Details = "Failed to write to stream" ); return Err(()); } let _ = session.stream.flush().await; // Split stream into read and write halves let (stream_rx, stream_tx) = tokio::io::split(session.stream); let server = manager.inner.build_server(); Ok(Session { receiver: Receiver::with_max_request_size(server.core.imap.max_request_size), version: ProtocolVersion::Rev1, state: State::NotAuthenticated { auth_failures: 0 }, is_tls, is_condstore: false, is_qresync: false, is_utf8: false, server, instance: session.instance, session_id: session.session_id, in_flight: session.in_flight, remote_addr: session.remote_ip, stream_rx, stream_tx: Arc::new(tokio::sync::Mutex::new(stream_tx)), }) } pub async fn into_tls(self) -> Result>, ()> { // Drop references to write half from state let state = if let Some(state) = self.state .try_replace_stream_tx(Arc::new(tokio::sync::Mutex::new( tokio::io::split(NullIo::default()).1, ))) { state } else { trc::event!( Network(trc::NetworkEvent::SplitError), SpanId = self.session_id, Details = "Failed to obtain write half state" ); return Err(()); }; // Take ownership of WriteHalf and unsplit it from ReadHalf let stream = if let Ok(stream_tx) = Arc::try_unwrap(self.stream_tx).map(|mutex| mutex.into_inner()) { self.stream_rx.unsplit(stream_tx) } else { trc::event!( Network(trc::NetworkEvent::SplitError), SpanId = self.session_id, Details = "Failed to take ownership of write half" ); return Err(()); }; // Upgrade to TLS let (stream_rx, stream_tx) = tokio::io::split(self.instance.tls_accept(stream, self.session_id).await?); let stream_tx = Arc::new(tokio::sync::Mutex::new(stream_tx)); Ok(Session { server: self.server, instance: self.instance, receiver: self.receiver, version: self.version, state: state.try_replace_stream_tx(stream_tx.clone()).unwrap(), is_tls: true, is_condstore: self.is_condstore, is_qresync: self.is_qresync, is_utf8: self.is_utf8, session_id: self.session_id, in_flight: self.in_flight, remote_addr: self.remote_addr, stream_rx, stream_tx, }) } } impl Session { pub async fn write_bytes(&self, bytes: impl AsRef<[u8]>) -> trc::Result<()> { let bytes = bytes.as_ref(); trc::event!( Imap(trc::ImapEvent::RawOutput), SpanId = self.session_id, Size = bytes.len(), Contents = trc::Value::from_maybe_string(bytes), ); let mut stream = self.stream_tx.lock().await; if let Err(err) = stream.write_all(bytes).await { Err(trc::NetworkEvent::WriteError .into_err() .reason(err) .details("Failed to write to stream")) } else { let _ = stream.flush().await; Ok(()) } } pub async fn write_error(&self, err: trc::Error) -> bool { if err.should_write_err() { let disconnect = err.must_disconnect(); let bytes = err.serialize(); trc::error!(err.span_id(self.session_id)); if let Err(err) = self.write_bytes(bytes).await { trc::error!(err.span_id(self.session_id)); false } else { !disconnect } } else { trc::error!(err); false } } } impl super::SessionData { pub async fn write_bytes(&self, bytes: impl AsRef<[u8]>) -> trc::Result<()> { let bytes = bytes.as_ref(); trc::event!( Imap(trc::ImapEvent::RawOutput), SpanId = self.session_id, Size = bytes.len(), Contents = trc::Value::from_maybe_string(bytes), ); let mut stream = self.stream_tx.lock().await; if let Err(err) = stream.write_all(bytes.as_ref()).await { Err(trc::NetworkEvent::WriteError .into_err() .reason(err) .details("Failed to write to stream")) } else { let _ = stream.flush().await; Ok(()) } } pub async fn write_error(&self, err: trc::Error) -> trc::Result<()> { if err.should_write_err() { let bytes = err.serialize(); trc::error!(err.span_id(self.session_id)); self.write_bytes(bytes).await } else { trc::error!(err.span_id(self.session_id)); Ok(()) } } } ================================================ FILE: crates/imap/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::LazyLock; use imap_proto::{ResponseCode, StatusResponse, protocol::capability::Capability}; pub mod core; pub mod op; static SERVER_GREETING: &str = "Stalwart IMAP4rev2 at your service."; pub(crate) static GREETING_WITH_TLS: LazyLock> = LazyLock::new(|| { StatusResponse::ok(SERVER_GREETING) .with_code(ResponseCode::Capability { capabilities: Capability::all_capabilities(false, true), }) .into_bytes() }); pub(crate) static GREETING_WITHOUT_TLS: LazyLock> = LazyLock::new(|| { StatusResponse::ok(SERVER_GREETING) .with_code(ResponseCode::Capability { capabilities: Capability::all_capabilities(false, false), }) .into_bytes() }); pub struct ImapError; ================================================ FILE: crates/imap/src/op/acl.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ core::{MailboxId, Session, SessionData, State}, op::ImapContext, spawn_op, }; use common::{ auth::AccessToken, listener::SessionStream, sharing::EffectiveAcl, storage::index::ObjectIndexBuilder, }; use compact_str::ToCompactString; use directory::{ Permission, QueryParams, Type, backend::internal::{ PrincipalField, manage::{ChangedPrincipals, ManageDirectory}, }, }; use imap_proto::{ Command, ResponseCode, StatusResponse, protocol::acl::{ Arguments, GetAclResponse, ListRightsResponse, ModRightsOp, MyRightsResponse, Rights, }, receiver::Request, }; use std::{sync::Arc, time::Instant}; use store::{ ValueKey, write::{AlignedBytes, Archive, BatchBuilder}, }; use trc::AddContext; use types::{ acl::{Acl, AclGrant}, collection::Collection, }; use utils::map::bitmap::Bitmap; impl Session { pub async fn handle_get_acl(&mut self, request: Request) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapAclGet)?; let op_start = Instant::now(); let arguments = request.parse_acl(self.is_utf8)?; let is_utf8 = self.version.is_rev2() || self.is_utf8; let data = self.state.session_data(); spawn_op!(data, { let (mailbox_id, mailbox_, _) = data .get_acl_mailbox(&arguments, true) .await .imap_ctx(&arguments.tag, trc::location!())?; let mut permissions = Vec::new(); let mailbox = mailbox_ .to_unarchived::() .imap_ctx(&arguments.tag, trc::location!())?; // Add the current user if they are the owner or a group member if data.access_token.is_member(mailbox_id.account_id) { permissions.push(( data.access_token.name.clone(), vec![ Rights::Read, Rights::Lookup, Rights::Insert, Rights::DeleteMessages, Rights::Expunge, Rights::Seen, Rights::Write, Rights::CreateMailbox, Rights::DeleteMailbox, Rights::Post, Rights::Administer, ], )); } for item in mailbox.inner.acls.iter() { if item.account_id == mailbox_id.account_id { // Skip the current user, as they are already added above continue; } if let Some(account_name) = data .server .store() .get_principal_name(item.account_id.into()) .await .imap_ctx(&arguments.tag, trc::location!())? { let mut rights = Vec::new(); for acl in Bitmap::from(&item.grants) { match acl { Acl::Read => { rights.push(Rights::Lookup); } Acl::Modify => { rights.push(Rights::CreateMailbox); } Acl::Delete => { rights.push(Rights::DeleteMailbox); } Acl::ReadItems => { rights.push(Rights::Read); } Acl::AddItems => { rights.push(Rights::Insert); } Acl::ModifyItems => { rights.push(Rights::Write); rights.push(Rights::Seen); } Acl::RemoveItems => { rights.push(Rights::DeleteMessages); rights.push(Rights::Expunge); } Acl::CreateChild => { rights.push(Rights::CreateMailbox); } Acl::Share => { rights.push(Rights::Administer); } Acl::Submit => { rights.push(Rights::Post); } _ => (), } } permissions.push((account_name, rights)); } } trc::event!( Imap(trc::ImapEvent::GetAcl), SpanId = data.session_id, MailboxName = arguments.mailbox_name.clone(), AccountId = mailbox_id.account_id, MailboxId = mailbox_id.mailbox_id, Total = permissions.len(), Elapsed = op_start.elapsed() ); data.write_bytes( StatusResponse::completed(Command::GetAcl) .with_tag(arguments.tag) .serialize( GetAclResponse { mailbox_name: arguments.mailbox_name.to_string(), permissions, } .into_bytes(is_utf8), ), ) .await }) } pub async fn handle_my_rights(&mut self, request: Request) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapMyRights)?; let op_start = Instant::now(); let arguments = request.parse_acl(self.is_utf8)?; let data = self.state.session_data(); let is_utf8 = self.version.is_rev2() || self.is_utf8; spawn_op!(data, { let (mailbox_id, mailbox_, access_token) = data .get_acl_mailbox(&arguments, false) .await .imap_ctx(&arguments.tag, trc::location!())?; let mailbox = mailbox_ .to_unarchived::() .imap_ctx(&arguments.tag, trc::location!())?; let rights = if access_token.is_shared(mailbox_id.account_id) { let acl = mailbox.inner.acls.effective_acl(&access_token); let mut rights = Vec::with_capacity(5); if acl.contains(Acl::ReadItems) { rights.push(Rights::Read); rights.push(Rights::Lookup); } if acl.contains(Acl::AddItems) { rights.push(Rights::Insert); } if acl.contains(Acl::RemoveItems) { rights.push(Rights::DeleteMessages); rights.push(Rights::Expunge); } if acl.contains(Acl::ModifyItems) { rights.push(Rights::Seen); rights.push(Rights::Write); } if acl.contains(Acl::CreateChild) { rights.push(Rights::CreateMailbox); } if acl.contains(Acl::Delete) { rights.push(Rights::DeleteMailbox); } if acl.contains(Acl::Submit) { rights.push(Rights::Post); } rights } else { vec![ Rights::Read, Rights::Lookup, Rights::Insert, Rights::DeleteMessages, Rights::Expunge, Rights::Seen, Rights::Write, Rights::CreateMailbox, Rights::DeleteMailbox, Rights::Post, Rights::Administer, ] }; trc::event!( Imap(trc::ImapEvent::MyRights), SpanId = data.session_id, MailboxName = arguments.mailbox_name.clone(), AccountId = mailbox_id.account_id, MailboxId = mailbox_id.mailbox_id, Details = rights .iter() .map(|r| trc::Value::String(r.to_compact_string())) .collect::>(), Elapsed = op_start.elapsed() ); data.write_bytes( StatusResponse::completed(Command::MyRights) .with_tag(arguments.tag) .serialize( MyRightsResponse { mailbox_name: arguments.mailbox_name.to_string(), rights, } .into_bytes(is_utf8), ), ) .await }) } pub async fn handle_set_acl(&mut self, request: Request) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapAclSet)?; let op_start = Instant::now(); let command = request.command; let arguments = request.parse_acl(self.is_utf8)?; let data = self.state.session_data(); spawn_op!(data, { // Validate mailbox let (mailbox_id, current_mailbox, _) = data .get_acl_mailbox(&arguments, false) .await .imap_ctx(&arguments.tag, trc::location!())?; let current_mailbox = current_mailbox .into_deserialized::() .imap_ctx(&arguments.tag, trc::location!())?; // Obtain principal id let acl_account_id = data .server .core .storage .directory .query( QueryParams::name(arguments.identifier.as_ref().unwrap()) .with_return_member_of(false), ) .await .imap_ctx(&arguments.tag, trc::location!())? .ok_or_else(|| { trc::ImapEvent::Error .into_err() .details("Account does not exist") .id(arguments.tag.to_string()) .caused_by(trc::location!()) })? .id(); // Prepare changes let mut mailbox = current_mailbox.inner.clone(); let (op, rights) = arguments .mod_rights .map(|mr| { ( mr.op, Bitmap::from_iter(mr.rights.into_iter().map(Acl::from)), ) }) .unwrap_or_else(|| (ModRightsOp::Replace, Bitmap::new())); if let Some(item) = mailbox .acls .iter_mut() .find(|item| item.account_id == acl_account_id) { match op { ModRightsOp::Replace => { if !rights.is_empty() { item.grants = rights; } else { mailbox .acls .retain(|item| item.account_id != acl_account_id); } } ModRightsOp::Add => { item.grants.union(&rights); } ModRightsOp::Remove => { for right in rights { item.grants.remove(right); } if item.grants.is_empty() { mailbox .acls .retain(|item| item.account_id != acl_account_id); } } } } else if !rights.is_empty() { match op { ModRightsOp::Add | ModRightsOp::Replace => { mailbox.acls.push(AclGrant { account_id: acl_account_id, grants: rights, }); } ModRightsOp::Remove => (), } } if mailbox.acls.len() > data.server.core.groupware.max_shares_per_item { return Err(trc::ImapEvent::Error .into_err() .details("Maximum shares per item exceeded") .caused_by(trc::location!())); } let grants = mailbox .acls .iter() .map(|r| trc::Value::from(r.account_id)) .collect::>(); // Write changes let mut batch = BatchBuilder::new(); batch .with_account_id(mailbox_id.account_id) .with_collection(Collection::Mailbox) .with_document(mailbox_id.mailbox_id) .custom( ObjectIndexBuilder::new() .with_changes(mailbox) .with_current(current_mailbox), ) .imap_ctx(&arguments.tag, trc::location!())?; if !batch.is_empty() { data.server .commit_batch(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; } // Invalidate ACLs data.server .invalidate_principal_caches(ChangedPrincipals::from_change( acl_account_id, Type::Individual, PrincipalField::EnabledPermissions, )) .await; trc::event!( Imap(trc::ImapEvent::SetAcl), SpanId = data.session_id, MailboxName = arguments.mailbox_name.clone(), AccountId = mailbox_id.account_id, MailboxId = mailbox_id.mailbox_id, Details = grants, Elapsed = op_start.elapsed() ); data.write_bytes( StatusResponse::completed(command) .with_tag(arguments.tag) .into_bytes(), ) .await }) } pub async fn handle_list_rights(&mut self, request: Request) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapListRights)?; let op_start = Instant::now(); let arguments = request.parse_acl(self.is_utf8)?; trc::event!( Imap(trc::ImapEvent::ListRights), SpanId = self.session_id, MailboxName = arguments.mailbox_name.clone(), Elapsed = op_start.elapsed() ); self.write_bytes( StatusResponse::completed(Command::ListRights) .with_tag(arguments.tag) .serialize( ListRightsResponse { mailbox_name: arguments.mailbox_name, identifier: arguments.identifier.unwrap(), permissions: vec![ vec![Rights::Read], vec![Rights::Lookup], vec![Rights::Write, Rights::Seen], vec![Rights::Insert], vec![Rights::Expunge, Rights::DeleteMessages], vec![Rights::CreateMailbox], vec![Rights::DeleteMailbox], vec![Rights::Post], vec![Rights::Administer], ], } .into_bytes(self.version.is_rev2() || self.is_utf8), ), ) .await } pub fn assert_has_permission(&self, permission: Permission) -> trc::Result { match &self.state { State::Authenticated { data } | State::Selected { data, .. } => { data.access_token.assert_has_permission(permission) } State::NotAuthenticated { .. } => Ok(false), } } } impl SessionData { async fn get_acl_mailbox( &self, arguments: &Arguments, validate: bool, ) -> trc::Result<(MailboxId, Archive, Arc)> { if let Some(mailbox) = self.get_mailbox_by_name(&arguments.mailbox_name) { if let Some(values) = self .server .store() .get_value::>(ValueKey::archive( mailbox.account_id, Collection::Mailbox, mailbox.mailbox_id, )) .await .caused_by(trc::location!())? { let access_token = self.get_access_token().await.caused_by(trc::location!())?; if !validate || access_token.is_member(mailbox.account_id) || values .unarchive::() .caused_by(trc::location!())? .acls .effective_acl(&access_token) .contains(Acl::Share) { Ok((mailbox, values, access_token)) } else { Err(trc::ImapEvent::Error .into_err() .details("You do not have enough permissions to perform this operation.") .code(ResponseCode::NoPerm)) } } else { Err(trc::ImapEvent::Error .caused_by(trc::location!()) .details("Mailbox does not exist.")) } } else { Err(trc::ImapEvent::Error .into_err() .details("Mailbox does not exist.")) } } } ================================================ FILE: crates/imap/src/op/append.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ImapContext, ToModSeq}; use crate::{ core::{ImapUidToId, MailboxId, SelectedMailbox, Session, SessionData}, spawn_op, }; use common::{ipc::PushNotification, listener::SessionStream}; use directory::Permission; use email::message::ingest::{EmailIngest, IngestEmail, IngestSource}; use imap_proto::{ Command, ResponseCode, StatusResponse, protocol::{append::Arguments, select::HighestModSeq}, receiver::Request, }; use mail_parser::MessageParser; use std::{sync::Arc, time::Instant}; use types::{ acl::Acl, keyword::Keyword, type_state::{DataType, StateChange}, }; impl Session { pub async fn handle_append(&mut self, request: Request) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapAppend)?; let op_start = Instant::now(); let arguments = request.parse_append(self.is_utf8)?; let (data, selected_mailbox) = self.state.session_mailbox_state(); // Refresh mailboxes data.synchronize_mailboxes(false) .await .imap_ctx(&arguments.tag, trc::location!())?; // Obtain mailbox let mailbox = if let Some(mailbox) = data.get_mailbox_by_name(&arguments.mailbox_name) { mailbox } else { return Err(trc::ImapEvent::Error .into_err() .details("Mailbox does not exist.") .code(ResponseCode::TryCreate) .id(arguments.tag)); }; let is_qresync = self.is_qresync; spawn_op!(data, { let response = data .append_messages(arguments, selected_mailbox, mailbox, is_qresync, op_start) .await? .into_bytes(); data.write_bytes(response).await }) } } impl SessionData { async fn append_messages( &self, arguments: Arguments, selected_mailbox: Option>, mailbox: MailboxId, is_qresync: bool, op_start: Instant, ) -> trc::Result { // Verify ACLs let account_id = mailbox.account_id; let mailbox_id = mailbox.mailbox_id; if !self .check_mailbox_acl(account_id, mailbox_id, Acl::AddItems) .await .imap_ctx(&arguments.tag, trc::location!())? { return Err(trc::ImapEvent::Error .into_err() .details( "You do not have the required permissions to append messages to this mailbox.", ) .code(ResponseCode::NoPerm) .id(arguments.tag)); } // Obtain access token let access_token = self .server .get_access_token(mailbox.account_id) .await .imap_ctx(&arguments.tag, trc::location!())?; // Append messages let mut response = StatusResponse::completed(Command::Append); let mut created_ids = Vec::with_capacity(arguments.messages.len()); let mut last_change_id = None; for message in arguments.messages { match self .server .email_ingest(IngestEmail { raw_message: &message.message, message: MessageParser::new().parse(&message.message), blob_hash: None, access_token: &access_token, mailbox_ids: vec![mailbox_id], keywords: message.flags.into_iter().map(Keyword::from).collect(), received_at: message.received_at.map(|d| d as u64), source: IngestSource::Imap { train_classifier: true, }, session_id: self.session_id, }) .await { Ok(email) => { created_ids.push(ImapUidToId { uid: email.imap_uids[0], id: email.document_id, }); last_change_id = Some(email.change_id); } Err(err) => { return Err( if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) { err.details("Disk quota exceeded.") .code(ResponseCode::OverQuota) } else if err.matches(trc::EventType::Limit(trc::LimitEvent::TenantQuota)) { err.details("Organization disk quota exceeded.") .code(ResponseCode::OverQuota) } else { err } .id(arguments.tag), ); } } } // Broadcast changes if let Some(change_id) = last_change_id { self.server .broadcast_push_notification(PushNotification::StateChange( StateChange::new(account_id) .with_change_id(change_id) .with_change(DataType::Email) .with_change(DataType::Mailbox) .with_change(DataType::Thread), )) .await; } trc::event!( Imap(trc::ImapEvent::Append), SpanId = self.session_id, MailboxName = arguments.mailbox_name.clone(), AccountId = account_id, MailboxId = mailbox_id, DocumentId = created_ids .iter() .map(|r| trc::Value::from(r.id)) .collect::>(), Elapsed = op_start.elapsed() ); if !created_ids.is_empty() { let uids = created_ids.iter().map(|id| id.uid).collect(); match selected_mailbox { Some(selected_mailbox) if selected_mailbox.id == mailbox => { // Write updated modseq if is_qresync { self.write_bytes( HighestModSeq::new(last_change_id.unwrap_or_default().to_modseq()) .into_bytes(), ) .await?; } selected_mailbox.append_messages(created_ids, last_change_id); } _ => {} }; let uid_validity = self .mailbox_state(&mailbox) .map(|m| m.uid_validity as u32) .unwrap_or_default(); response = response.with_code(ResponseCode::AppendUid { uid_validity, uids }); } Ok(response.with_tag(arguments.tag)) } } ================================================ FILE: crates/imap/src/op/authenticate.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{ auth::{ AuthRequest, sasl::{sasl_decode_challenge_oauth, sasl_decode_challenge_plain}, }, listener::{SessionStream, limiter::LimiterResult}, }; use directory::Permission; use imap_proto::{ Command, ResponseCode, StatusResponse, protocol::{authenticate::Mechanism, capability::Capability}, receiver::{self, Request}, }; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; use std::sync::Arc; use crate::core::{Session, SessionData, State}; impl Session { pub async fn handle_authenticate(&mut self, request: Request) -> trc::Result<()> { let mut args = request.parse_authenticate()?; match args.mechanism { Mechanism::Plain | Mechanism::OAuthBearer | Mechanism::XOauth2 => { if !args.params.is_empty() { let challenge = base64_decode(args.params.pop().unwrap().as_bytes()) .ok_or_else(|| { trc::AuthEvent::Error .into_err() .details("Failed to decode challenge.") .id(args.tag.clone()) .code(ResponseCode::Parse) })?; let credentials = if args.mechanism == Mechanism::Plain { sasl_decode_challenge_plain(&challenge) } else { sasl_decode_challenge_oauth(&challenge) } .ok_or_else(|| { trc::AuthEvent::Error .into_err() .details("Invalid SASL challenge.") .id(args.tag.clone()) })?; self.authenticate(credentials, args.tag).await } else { self.receiver.request = receiver::Request { tag: args.tag, command: Command::Authenticate, tokens: vec![receiver::Token::Argument(args.mechanism.into_bytes())], }; self.receiver.state = receiver::State::Argument { last_ch: b' ' }; self.write_bytes(b"+ \r\n".to_vec()).await } } _ => Err(trc::AuthEvent::Error .into_err() .details("Authentication mechanism not supported.") .id(args.tag) .code(ResponseCode::Cannot)), } } pub async fn authenticate( &mut self, credentials: Credentials, tag: String, ) -> trc::Result<()> { // Authenticate let access_token = self .server .authenticate(&AuthRequest::from_credentials( credentials, self.session_id, self.remote_addr, )) .await .map_err(|err| { if err.matches(trc::EventType::Auth(trc::AuthEvent::Failed)) { let auth_failures = self.state.auth_failures(); if auth_failures < self.server.core.imap.max_auth_failures { self.state = State::NotAuthenticated { auth_failures: auth_failures + 1, }; } else { return trc::AuthEvent::TooManyAttempts.into_err().caused_by(err); } } err.id(tag.clone()) }) .and_then(|token| { token .assert_has_permission(Permission::ImapAuthenticate) .map(|_| token) })?; // Enforce concurrency limits let in_flight = match access_token.is_imap_request_allowed() { LimiterResult::Allowed(in_flight) => Some(in_flight), LimiterResult::Forbidden => { return Err(trc::LimitEvent::ConcurrentRequest .into_err() .id(tag.clone())); } LimiterResult::Disabled => None, }; // Create session self.state = State::Authenticated { data: Arc::new( SessionData::new(self, access_token, in_flight) .await .map_err(|err| err.id(tag.clone()))?, ), }; self.write_bytes( StatusResponse::ok("Authentication successful") .with_code(ResponseCode::Capability { capabilities: Capability::all_capabilities( true, !self.is_tls && self.instance.acceptor.is_tls(), ), }) .with_tag(tag) .into_bytes(), ) .await } pub async fn handle_unauthenticate(&mut self, request: Request) -> trc::Result<()> { self.state = State::NotAuthenticated { auth_failures: 0 }; self.write_bytes( StatusResponse::completed(Command::Unauthenticate) .with_tag(request.tag) .into_bytes(), ) .await } } ================================================ FILE: crates/imap/src/op/capability.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Instant; use crate::core::Session; use common::listener::SessionStream; use directory::Permission; use imap_proto::{ Command, StatusResponse, protocol::{ ImapResponse, capability::{Capability, Response}, }, receiver::Request, }; impl Session { pub async fn handle_capability(&mut self, request: Request) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapCapability)?; let op_start = Instant::now(); trc::event!( Imap(trc::ImapEvent::Capabilities), SpanId = self.session_id, Tls = self.is_tls, Strict = !self.server.core.imap.allow_plain_auth, Elapsed = op_start.elapsed() ); self.write_bytes( StatusResponse::completed(Command::Capability) .with_tag(request.tag) .serialize( Response { capabilities: Capability::all_capabilities( self.state.is_authenticated(), !self.is_tls && self.instance.acceptor.is_tls(), ), } .serialize(), ), ) .await } pub async fn handle_id(&mut self, request: Request) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapId)?; let op_start = Instant::now(); trc::event!( Imap(trc::ImapEvent::Id), SpanId = self.session_id, Elapsed = op_start.elapsed() ); self.write_bytes( StatusResponse::completed(Command::Id) .with_tag(request.tag) .serialize( concat!( "* ID (\"name\" \"Stalwart\" \"version\" \"1.0.0\" \"vendor\" \"Stalwart Labs LLC\" ", "\"support-url\" \"https://stalw.art\")\r\n" ) .as_bytes() .to_vec(), ), ) .await } } ================================================ FILE: crates/imap/src/op/close.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Instant; use crate::core::{Session, State}; use common::listener::SessionStream; use imap_proto::{Command, StatusResponse, receiver::Request}; use trc::AddContext; impl Session { pub async fn handle_close(&mut self, request: Request) -> trc::Result<()> { let op_start = Instant::now(); let (data, mailbox) = self.state.select_data(); if mailbox.is_select { data.expunge(mailbox.clone(), None, op_start) .await .caused_by(trc::location!())?; } trc::event!( Imap(trc::ImapEvent::Close), SpanId = self.session_id, AccountId = mailbox.id.account_id, MailboxId = mailbox.id.mailbox_id, Elapsed = op_start.elapsed() ); self.state = State::Authenticated { data }; self.write_bytes( StatusResponse::completed(Command::Close) .with_tag(request.tag) .into_bytes(), ) .await } } ================================================ FILE: crates/imap/src/op/copy_move.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::ImapContext; use crate::{ core::{MailboxId, SelectedMailbox, Session, SessionData}, spawn_op, }; use common::{ipc::PushNotification, listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, mailbox::{JUNK_ID, TRASH_ID, UidMailbox}, message::{ copy::{CopyMessageError, EmailCopy}, ingest::EmailIngest, metadata::MessageData, }, }; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, protocol::copy_move::Arguments, receiver::Request, }; use std::{sync::Arc, time::Instant}; use store::{ ValueKey, roaring::RoaringBitmap, write::{AlignedBytes, Archive, BatchBuilder}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, VanishedCollection}, type_state::{DataType, StateChange}, }; impl Session { pub async fn handle_copy_move( &mut self, request: Request, is_move: bool, is_uid: bool, ) -> trc::Result<()> { // Validate access self.assert_has_permission(if is_move { Permission::ImapMove } else { Permission::ImapCopy })?; let op_start = Instant::now(); let arguments = request.parse_copy_move(self.is_utf8)?; let (data, src_mailbox) = self.state.mailbox_state(); let is_qresync = self.is_qresync; spawn_op!(data, { // Refresh mailboxes data.synchronize_mailboxes(false) .await .imap_ctx(&arguments.tag, trc::location!())?; // Make sure the mailbox exists. let dest_mailbox = if let Some(mailbox) = data.get_mailbox_by_name(&arguments.mailbox_name) { mailbox } else { return Err(trc::ImapEvent::Error .into_err() .details("Destination mailbox does not exist.") .code(ResponseCode::TryCreate) .id(arguments.tag)); }; // Check that the destination mailbox is not the same as the source mailbox. if src_mailbox.id.account_id == dest_mailbox.account_id && src_mailbox.id.mailbox_id == dest_mailbox.mailbox_id { return Err(trc::ImapEvent::Error .into_err() .details("Source and destination mailboxes are the same.") .code(ResponseCode::Cannot) .id(arguments.tag)); } data.copy_move( arguments, src_mailbox, dest_mailbox, is_move, is_uid, is_qresync, op_start, ) .await }) } } impl SessionData { #[allow(clippy::too_many_arguments)] pub async fn copy_move( &self, arguments: Arguments, src_mailbox: Arc, dest_mailbox: MailboxId, is_move: bool, is_uid: bool, is_qresync: bool, op_start: Instant, ) -> trc::Result<()> { self.synchronize_messages(&src_mailbox) .await .imap_ctx(&arguments.tag, trc::location!())?; // Convert IMAP ids to JMAP ids. let ids = src_mailbox .sequence_to_ids(&arguments.sequence_set, is_uid) .await .imap_ctx(&arguments.tag, trc::location!())?; if ids.is_empty() { trc::event!( Imap(if is_move { trc::ImapEvent::Move } else { trc::ImapEvent::Copy }), SpanId = self.session_id, Source = src_mailbox.id.account_id, Details = trc::Value::None, Uid = trc::Value::None, AccountId = dest_mailbox.account_id, MailboxId = dest_mailbox.mailbox_id, Elapsed = op_start.elapsed() ); return self .write_bytes( StatusResponse::ok(if is_move { "No messages were moved." } else { "No messages were copied." }) .with_tag(arguments.tag) .into_bytes(), ) .await; } // Verify that the user can delete messages from the source mailbox. if is_move && !self .check_mailbox_acl( src_mailbox.id.account_id, src_mailbox.id.mailbox_id, Acl::RemoveItems, ) .await .imap_ctx(&arguments.tag, trc::location!())? { return Err(trc::ImapEvent::Error .into_err() .details(concat!( "You do not have the required permissions to ", "remove messages from the source mailbox." )) .code(ResponseCode::NoPerm) .id(arguments.tag)); } // Verify that the user can append messages to the destination mailbox. let dest_mailbox_id = dest_mailbox.mailbox_id; if !self .check_mailbox_acl(dest_mailbox.account_id, dest_mailbox_id, Acl::AddItems) .await .imap_ctx(&arguments.tag, trc::location!())? { return Err(trc::ImapEvent::Error .into_err() .details(concat!( "You do not have the required permissions to ", "add messages to the destination mailbox." )) .code(ResponseCode::NoPerm) .id(arguments.tag)); } let mut response = StatusResponse::completed(if is_move { Command::Move(is_uid) } else { Command::Copy(is_uid) }); let mut did_move = false; let mut copied_ids = Vec::with_capacity(ids.len()); let access_token = self .server .get_access_token(dest_mailbox.account_id) .await .imap_ctx(&arguments.tag, trc::location!())?; if src_mailbox.id.account_id == dest_mailbox.account_id { // Mailboxes are in the same account let account_id = src_mailbox.id.account_id; let dest_mailbox_id = UidMailbox::new_unassigned(dest_mailbox_id); let mut batch = BatchBuilder::new(); for (id, imap_id) in ids { // Obtain mailbox tags let data_ = if let Some(result) = self .get_message_data(account_id, id) .await .imap_ctx(&arguments.tag, trc::location!())? { result } else { continue; }; // Deserialize let data = data_ .to_unarchived::() .imap_ctx(&arguments.tag, trc::location!())?; // Make sure the message still belongs to this mailbox if !data .inner .mailboxes .iter() .any(|mailbox| mailbox.mailbox_id == src_mailbox.id.mailbox_id) { continue; } // If the message is already in the destination mailbox, skip it. if let Some(mailbox) = data .inner .mailboxes .iter() .find(|mailbox| mailbox.mailbox_id == dest_mailbox_id.mailbox_id) { copied_ids.push((imap_id.uid, mailbox.uid.to_native())); if is_move { let mut new_data = data.inner.to_builder(); new_data.remove_mailbox(src_mailbox.id.mailbox_id); batch .with_account_id(account_id) .with_collection(Collection::Email) .with_document(id) .custom( ObjectIndexBuilder::new() .with_current(data) .with_changes(new_data.seal()), ) .imap_ctx(&arguments.tag, trc::location!())? .log_vanished_item( VanishedCollection::Email, (src_mailbox.id.mailbox_id, imap_id.uid), ) .commit_point(); did_move = true; } continue; } // Prepare changes let mut new_data = data.inner.to_builder(); // Add destination folder new_data.add_mailbox(dest_mailbox_id); if is_move { new_data.remove_mailbox(src_mailbox.id.mailbox_id); } // Assign IMAP UIDs let ids = self .server .assign_email_ids( account_id, new_data .mailboxes .iter() .filter(|m| m.uid == 0) .map(|m| m.mailbox_id), false, ) .await .caused_by(trc::location!())?; for (uid_mailbox, uid) in new_data .mailboxes .iter_mut() .filter(|m| m.uid == 0) .zip(ids) { copied_ids.push((imap_id.uid, uid)); uid_mailbox.uid = uid; } // Prepare write batch batch .with_account_id(account_id) .with_collection(Collection::Email) .with_document(id) .custom( ObjectIndexBuilder::new() .with_current(data) .with_changes(new_data.seal()), ) .imap_ctx(&arguments.tag, trc::location!())?; if is_move { batch.log_vanished_item( VanishedCollection::Email, (src_mailbox.id.mailbox_id, imap_id.uid), ); } // Add message to training queue if dest_mailbox_id.mailbox_id == JUNK_ID { self.server .add_account_spam_sample(&mut batch, account_id, id, true, self.session_id) .await .imap_ctx(&arguments.tag, trc::location!())?; } else if src_mailbox.id.mailbox_id == JUNK_ID && dest_mailbox_id.mailbox_id != TRASH_ID { self.server .add_account_spam_sample(&mut batch, account_id, id, false, self.session_id) .await .imap_ctx(&arguments.tag, trc::location!())?; } batch.commit_point(); // Update changelog if is_move { did_move = true; } } // Write changes self.server .commit_batch(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; } else { // Obtain quota for target account let src_account_id = src_mailbox.id.account_id; let mut dest_change_id = None; let dest_account_id = dest_mailbox.account_id; let resource_token = access_token.as_resource_token(); let mut destroy_ids = RoaringBitmap::new(); let cache = self .server .get_cached_messages(src_account_id) .await .imap_ctx(&arguments.tag, trc::location!())?; for (id, imap_id) in ids { match self .server .copy_message( src_account_id, id, &resource_token, vec![dest_mailbox_id], cache .email_by_id(&id) .map(|e| cache.expand_keywords(e).collect()) .unwrap_or_default(), None, self.session_id, ) .await .imap_ctx(&arguments.tag, trc::location!())? { Ok(email) => { dest_change_id = email.change_id.into(); if let Some(assigned_uid) = email.imap_uids.first() { debug_assert!(*assigned_uid > 0); copied_ids.push((imap_id.uid, *assigned_uid)); } } Err(err) => { match err { CopyMessageError::OverQuota => { response.rtype = ResponseType::No; response.code = Some(ResponseCode::OverQuota); response.message = "Mailbox quota exceeded".into(); } CopyMessageError::NotFound => (), } continue; } }; if is_move { destroy_ids.insert(id); } } // Untag or delete emails if !destroy_ids.is_empty() { let mut batch = BatchBuilder::new(); self.email_untag_or_delete( src_account_id, src_mailbox.id.mailbox_id, &destroy_ids, &mut batch, ) .await .imap_ctx(&arguments.tag, trc::location!())?; self.server .commit_batch(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; did_move = true; } // Broadcast changes on destination account if let Some(change_id) = dest_change_id { self.server .broadcast_push_notification(PushNotification::StateChange( StateChange::new(dest_account_id) .with_change_id(change_id) .with_change(DataType::Email) .with_change(DataType::Thread) .with_change(DataType::Mailbox), )) .await; } } // Map copied JMAP Ids to IMAP UIDs in the destination folder. if copied_ids.is_empty() { return if response.rtype != ResponseType::Ok { Err(trc::ImapEvent::Error .into_err() .details(response.message) .ctx_opt(trc::Key::Code, response.code) .id(arguments.tag)) } else { trc::event!( Imap(if is_move { trc::ImapEvent::Move } else { trc::ImapEvent::Copy }), SpanId = self.session_id, Source = src_mailbox.id.account_id, Details = trc::Value::None, Uid = trc::Value::None, AccountId = dest_mailbox.account_id, MailboxId = dest_mailbox.mailbox_id, Elapsed = op_start.elapsed() ); self.write_bytes( StatusResponse::ok(if is_move { "No messages were moved." } else { "No messages were copied." }) .with_tag(arguments.tag) .into_bytes(), ) .await }; } // Prepare response let uid_validity = self .mailbox_state(&dest_mailbox) .map(|m| m.uid_validity as u32) .unwrap_or_default(); let mut src_uids = Vec::with_capacity(copied_ids.len()); let mut dest_uids = Vec::with_capacity(copied_ids.len()); for (src_uid, dest_uid) in copied_ids { src_uids.push(src_uid); dest_uids.push(dest_uid); } src_uids.sort_unstable(); dest_uids.sort_unstable(); trc::event!( Imap(if is_move { trc::ImapEvent::Move } else { trc::ImapEvent::Copy }), SpanId = self.session_id, Source = src_mailbox.id.account_id, Details = src_uids .iter() .map(|r| trc::Value::from(*r)) .collect::>(), AccountId = dest_mailbox.account_id, MailboxId = dest_mailbox.mailbox_id, Uid = dest_uids .iter() .map(|r| trc::Value::from(*r)) .collect::>(), Elapsed = op_start.elapsed() ); let response = if is_move { self.write_bytes( StatusResponse::ok("Copied UIDs") .with_code(ResponseCode::CopyUid { uid_validity, src_uids, dest_uids, }) .into_bytes(), ) .await?; if did_move { // Resynchronize source mailbox on a successful move self.write_mailbox_changes(&src_mailbox, is_qresync) .await .imap_ctx(&arguments.tag, trc::location!())?; } response.with_tag(arguments.tag).into_bytes() } else { response .with_tag(arguments.tag) .with_code(ResponseCode::CopyUid { uid_validity, src_uids, dest_uids, }) .into_bytes() }; self.write_bytes(response).await } pub async fn get_message_data( &self, account_id: u32, id: u32, ) -> trc::Result>> { if let Some(data) = self .server .store() .get_value::>(ValueKey::archive( account_id, Collection::Email, id, )) .await? { Ok(Some(data)) } else { trc::event!( Store(trc::StoreEvent::NotFound), AccountId = account_id, Collection = Collection::Email, MessageId = id, SpanId = self.session_id, Details = "Message not found" ); Ok(None) } } } ================================================ FILE: crates/imap/src/op/create.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ core::{Session, SessionData}, op::ImapContext, spawn_op, }; use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; use email::cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}; use imap_proto::{ Command, ResponseCode, StatusResponse, protocol::{create::Arguments, list::Attribute}, receiver::Request, }; use std::time::Instant; use store::write::BatchBuilder; use trc::AddContext; use types::{acl::Acl, collection::Collection, id::Id, special_use::SpecialUse}; impl Session { pub async fn handle_create(&mut self, requests: Vec>) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapCreate)?; let data = self.state.session_data(); let is_utf8 = self.is_utf8; spawn_op!(data, { for request in requests { match request.parse_create(is_utf8) { Ok(argument) => match data.create_folder(argument).await { Ok(response) => { data.write_bytes(response.into_bytes()).await?; } Err(error) => { data.write_error(error).await?; } }, Err(err) => data.write_error(err).await?, } } Ok(()) }) } } impl SessionData { pub async fn create_folder(&self, arguments: Arguments) -> trc::Result { let op_start = Instant::now(); // Refresh mailboxes self.synchronize_mailboxes(false) .await .imap_ctx(&arguments.tag, trc::location!())?; // Validate mailbox name let params = self .validate_mailbox_create(&arguments.mailbox_name, arguments.mailbox_role) .await .imap_ctx(&arguments.tag, trc::location!())?; debug_assert!(!params.path.is_empty()); // Build batch let mut parent_id = params.parent_mailbox_id.map(|id| id + 1).unwrap_or(0); let mut create_ids = Vec::with_capacity(params.path.len()); let mut next_document_id = self .server .store() .assign_document_ids( params.account_id, Collection::Mailbox, params.path.len() as u64, ) .await .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); for (pos, &path_item) in params.path.iter().enumerate() { let mut mailbox = email::mailbox::Mailbox::new(path_item).with_parent_id(parent_id); if pos == params.path.len() - 1 && let Some(mailbox_role) = arguments.mailbox_role.map(attr_to_role) { mailbox.role = mailbox_role; } let mailbox_id = next_document_id; next_document_id -= 1; batch .with_account_id(params.account_id) .with_collection(Collection::Mailbox) .with_document(mailbox_id) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(mailbox)) .imap_ctx(&arguments.tag, trc::location!())? .commit_point(); parent_id = mailbox_id + 1; create_ids.push(mailbox_id); } self.server .commit_batch(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; trc::event!( Imap(trc::ImapEvent::CreateMailbox), SpanId = self.session_id, MailboxName = arguments.mailbox_name.clone(), AccountId = params.account_id, MailboxId = create_ids .iter() .map(|&id| trc::Value::from(id)) .collect::>(), Elapsed = op_start.elapsed() ); // Build response Ok(StatusResponse::ok("Mailbox created.") .with_code(ResponseCode::MailboxId { mailbox_id: Id::from_parts(params.account_id, parent_id - 1).to_string(), }) .with_tag(arguments.tag)) } pub async fn validate_mailbox_create<'x>( &self, mailbox_name: &'x str, mailbox_role: Option, ) -> trc::Result> { // Remove leading and trailing separators let mut name = mailbox_name.trim(); if let Some(suffix) = name.strip_prefix('/') { name = suffix.trim(); }; if let Some(prefix) = name.strip_suffix('/') { name = prefix.trim(); } if name.is_empty() { return Err(trc::ImapEvent::Error .into_err() .details(format!("Invalid folder name '{mailbox_name}'.",))); } // Build path let mut path = Vec::new(); if name.contains('/') { // Locate parent mailbox for path_item in name.split('/') { let path_item = path_item.trim(); if path_item.is_empty() { return Err(trc::ImapEvent::Error .into_err() .details("Invalid empty path item.")); } else if path_item.len() > self.server.core.jmap.mailbox_name_max_len { return Err(trc::ImapEvent::Error .into_err() .details("Mailbox name is too long.")); } path.push(path_item); } if path.len() > self.server.core.jmap.mailbox_max_depth { return Err(trc::ImapEvent::Error .into_err() .details("Mailbox path is too deep.")); } } else { path.push(name); } // Validate special folders let mut parent_mailbox_id = None; let mut parent_mailbox_name = None; let (account_id, path) = { let mailboxes = self.mailboxes.lock(); let (account, full_path, prefix) = if path.first() == Some(&self.server.core.jmap.shared_folder.as_str()) { // Shared Folders// if path.len() < 3 { return Err(trc::ImapEvent::Error .into_err() .details("Mailboxes under root shared folders are not allowed.") .code(ResponseCode::Cannot)); } // Build path let root = &mut path[2]; if root.eq_ignore_ascii_case("INBOX") { *root = "INBOX"; } let full_path = path.join("/"); let prefix = Some(format!("{}/{}", path[0], path[1])); // Locate account if let Some(account) = mailboxes .iter() .skip(1) .find(|account| account.prefix == prefix) { (account, full_path, prefix) } else { #[allow(clippy::unnecessary_literal_unwrap)] return Err(trc::ImapEvent::Error.into_err().details(format!( "Shared account '{}' not found.", prefix.unwrap_or_default() ))); } } else if let Some(account) = mailboxes.first() { let root = &mut path[0]; if root.eq_ignore_ascii_case("INBOX") { *root = "INBOX"; } (account, path.join("/"), None) } else { return Err(trc::ImapEvent::Error .into_err() .details("Internal server error.") .caused_by(trc::location!()) .code(ResponseCode::ContactAdmin)); }; // Locate parent mailbox if account.mailbox_names.contains_key(&full_path) { return Err(trc::ImapEvent::Error .into_err() .details(format!("Mailbox '{}' already exists.", full_path)) .code(ResponseCode::AlreadyExists)); } ( account.account_id, if path.len() > 1 { let mut create_path = Vec::with_capacity(path.len()); while !path.is_empty() { let mailbox_name: String = path.join("/"); if let Some(&mailbox_id) = account.mailbox_names.get(&mailbox_name) { parent_mailbox_id = mailbox_id.into(); parent_mailbox_name = mailbox_name.into(); break; } else if prefix .as_ref() .is_some_and(|prefix| prefix == &mailbox_name) { break; } else { create_path.push(path.pop().unwrap()); } } create_path.reverse(); create_path } else { path }, ) }; // Validate ACLs if let Some(parent_mailbox_id) = parent_mailbox_id { if !self .check_mailbox_acl(account_id, parent_mailbox_id, Acl::CreateChild) .await? { return Err(trc::ImapEvent::Error .into_err() .details("You are not allowed to create sub mailboxes under this mailbox.") .code(ResponseCode::NoPerm)); } } else if self.account_id != account_id && !self .get_access_token() .await .caused_by(trc::location!())? .is_member(account_id) { return Err(trc::ImapEvent::Error .into_err() .details("You are not allowed to create root folders under shared folders.") .code(ResponseCode::Cannot)); } Ok(CreateParams { account_id, path, parent_mailbox_id, parent_mailbox_name, special_use: if let Some(mailbox_role) = mailbox_role { // Make sure role is unique let special_use = attr_to_role(mailbox_role); if self .server .get_cached_messages(account_id) .await .caused_by(trc::location!())? .mailbox_by_role(&special_use) .is_some() { return Err(trc::ImapEvent::Error .into_err() .details(format!( "A mailbox with role '{}' already exists.", special_use.as_str().unwrap_or_default() )) .code(ResponseCode::UseAttr)); } Some(mailbox_role) } else { None }, is_rename: false, }) } } #[derive(Debug)] pub struct CreateParams<'x> { pub account_id: u32, pub path: Vec<&'x str>, pub parent_mailbox_id: Option, pub parent_mailbox_name: Option, pub special_use: Option, pub is_rename: bool, } #[inline] fn attr_to_role(attr: Attribute) -> SpecialUse { match attr { Attribute::Archive => SpecialUse::Archive, Attribute::Drafts => SpecialUse::Drafts, Attribute::Junk => SpecialUse::Junk, Attribute::Sent => SpecialUse::Sent, Attribute::Trash => SpecialUse::Trash, Attribute::Important => SpecialUse::Important, Attribute::Memos => SpecialUse::Memos, Attribute::Scheduled => SpecialUse::Scheduled, Attribute::Snoozed => SpecialUse::Snoozed, _ => SpecialUse::None, } } ================================================ FILE: crates/imap/src/op/delete.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::ImapContext; use crate::{ core::{Session, SessionData}, spawn_op, }; use common::listener::SessionStream; use directory::Permission; use email::mailbox::destroy::{MailboxDestroy, MailboxDestroyError}; use imap_proto::{ Command, ResponseCode, StatusResponse, protocol::delete::Arguments, receiver::Request, }; use std::time::Instant; impl Session { pub async fn handle_delete(&mut self, requests: Vec>) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapDelete)?; let data = self.state.session_data(); let is_utf8 = self.is_utf8; spawn_op!(data, { for request in requests { match request.parse_delete(is_utf8) { Ok(argument) => match data.delete_folder(argument).await { Ok(response) => { data.write_bytes(response.into_bytes()).await?; } Err(error) => { data.write_error(error).await?; } }, Err(response) => data.write_error(response).await?, } } Ok(()) }) } } impl SessionData { pub async fn delete_folder(&self, arguments: Arguments) -> trc::Result { let op_start = Instant::now(); // Refresh mailboxes self.synchronize_mailboxes(false) .await .imap_ctx(&arguments.tag, trc::location!())?; // Validate mailbox let (account_id, mailbox_id) = if let Some(mailbox) = self.get_mailbox_by_name(&arguments.mailbox_name) { (mailbox.account_id, mailbox.mailbox_id) } else { return Err(trc::ImapEvent::Error .into_err() .details("Mailbox does not exist.") .code(ResponseCode::TryCreate) .id(arguments.tag)); }; // Delete message let access_token = self .get_access_token() .await .imap_ctx(&arguments.tag, trc::location!())?; if let Err(err) = self .server .mailbox_destroy(account_id, mailbox_id, &access_token, true) .await .imap_ctx(&arguments.tag, trc::location!())? { let (code, message) = match err { MailboxDestroyError::CannotDestroy => { (ResponseCode::NoPerm, "You cannot delete system mailboxes") } MailboxDestroyError::Forbidden => ( ResponseCode::NoPerm, "You do not have enough permissions to delete this mailbox", ), MailboxDestroyError::HasChildren => { (ResponseCode::HasChildren, "Mailbox has children") } MailboxDestroyError::HasEmails => (ResponseCode::HasChildren, "Mailbox has emails"), MailboxDestroyError::NotFound => (ResponseCode::NonExistent, "Mailbox not found"), MailboxDestroyError::AssertionFailed => ( ResponseCode::Cannot, "Another process is accessing this mailbox", ), }; return Err(trc::ImapEvent::Error .into_err() .details(message) .code(code) .id(arguments.tag)); } // Update mailbox cache for account in self.mailboxes.lock().iter_mut() { if account.account_id == account_id { account.mailbox_names.remove(&arguments.mailbox_name); account.mailbox_state.remove(&mailbox_id); break; } } trc::event!( Imap(trc::ImapEvent::DeleteMailbox), SpanId = self.session_id, MailboxName = arguments.mailbox_name, AccountId = account_id, MailboxId = mailbox_id, Elapsed = op_start.elapsed() ); Ok(StatusResponse::ok("Mailbox deleted.").with_tag(arguments.tag)) } } ================================================ FILE: crates/imap/src/op/enable.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Instant; use crate::core::Session; use common::listener::SessionStream; use directory::Permission; use imap_proto::{ Command, StatusResponse, protocol::{ImapResponse, ProtocolVersion, capability::Capability, enable}, receiver::Request, }; impl Session { pub async fn handle_enable(&mut self, request: Request) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapEnable)?; let op_start = Instant::now(); let arguments = request.parse_enable()?; let mut response = enable::Response { enabled: Vec::with_capacity(arguments.capabilities.len()), }; for capability in arguments.capabilities { match capability { Capability::IMAP4rev2 => { self.version = ProtocolVersion::Rev2; self.is_utf8 = true; } Capability::IMAP4rev1 => { self.version = ProtocolVersion::Rev1; } Capability::CondStore => { self.is_condstore = true; } Capability::QResync => { self.is_qresync = true; self.is_condstore = true; } Capability::Utf8Accept => { self.is_utf8 = true; } _ => { continue; } } response.enabled.push(capability); } trc::event!( Imap(trc::ImapEvent::Enable), SpanId = self.session_id, Details = response .enabled .iter() .map(|c| trc::Value::from(format!("{c:?}"))) .collect::>(), Elapsed = op_start.elapsed() ); self.write_bytes( StatusResponse::ok("ENABLE successful.") .with_tag(arguments.tag) .serialize(response.serialize()), ) .await } } ================================================ FILE: crates/imap/src/op/expunge.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ImapContext, ToModSeq}; use crate::core::{ImapId, SavedSearch, SelectedMailbox, Session, SessionData}; use ahash::AHashMap; use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, message::metadata::MessageData, }; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, parser::parse_sequence_set, receiver::{Request, Token}, }; use std::{sync::Arc, time::Instant}; use store::{ SerializeInfallible, roaring::RoaringBitmap, write::{BatchBuilder, SearchIndex, TaskEpoch, TaskQueueClass, ValueClass}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, VanishedCollection}, keyword::Keyword, }; impl Session { pub async fn handle_expunge( &mut self, request: Request, is_uid: bool, ) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapExpunge)?; let op_start = Instant::now(); let (data, mailbox) = self.state.select_data(); // Validate ACL if !data .check_mailbox_acl( mailbox.id.account_id, mailbox.id.mailbox_id, Acl::RemoveItems, ) .await .imap_ctx(&request.tag, trc::location!())? { return Err(trc::ImapEvent::Error .into_err() .details(concat!( "You do not have the required permissions ", "to remove messages from this mailbox." )) .code(ResponseCode::NoPerm) .id(request.tag)); } // Parse sequence to operate on let sequence = match request.tokens.into_iter().next() { Some(Token::Argument(value)) if is_uid => { let sequence = parse_sequence_set(&value).map_err(|err| { trc::ImapEvent::Error .into_err() .details(err) .ctx(trc::Key::Type, ResponseType::Bad) .id(request.tag.clone()) })?; Some( mailbox .sequence_to_ids(&sequence, true) .await .map_err(|err| err.id(request.tag.clone()))?, ) } _ => None, }; // Expunge data.expunge(mailbox.clone(), sequence, op_start) .await .imap_ctx(&request.tag, trc::location!())?; // Clear saved searches *mailbox.saved_search.lock() = SavedSearch::None; // Synchronize messages let modseq = data .write_mailbox_changes(&mailbox, self.is_qresync) .await .imap_ctx(&request.tag, trc::location!())?; let mut response = StatusResponse::completed(Command::Expunge(is_uid)).with_tag(request.tag); if self.is_condstore { response = response.with_code(ResponseCode::HighestModseq { modseq: modseq.to_modseq(), }); } self.write_bytes(response.into_bytes()).await } } impl SessionData { pub async fn expunge( &self, mailbox: Arc, sequence: Option>, op_start: Instant, ) -> trc::Result<()> { // Obtain message ids let account_id = mailbox.id.account_id; let mut deleted_ids = RoaringBitmap::from_iter( self.server .get_cached_messages(account_id) .await .caused_by(trc::location!())? .in_mailbox_with_keyword(mailbox.id.mailbox_id, &Keyword::Deleted) .map(|m| m.document_id), ); // Filter by sequence if let Some(sequence) = &sequence { deleted_ids &= RoaringBitmap::from_iter(sequence.keys()); } // Delete ids let mut batch = BatchBuilder::new(); self.email_untag_or_delete(account_id, mailbox.id.mailbox_id, &deleted_ids, &mut batch) .await .caused_by(trc::location!())?; trc::event!( Imap(trc::ImapEvent::Expunge), SpanId = self.session_id, AccountId = account_id, MailboxId = mailbox.id.mailbox_id, DocumentId = deleted_ids.iter().map(trc::Value::from).collect::>(), Elapsed = op_start.elapsed() ); // Write changes on source account if !batch.is_empty() { self.server .commit_batch(batch) .await .caused_by(trc::location!())?; self.server.notify_task_queue(); } Ok(()) } pub async fn email_untag_or_delete( &self, account_id: u32, mailbox_id: u32, deleted_ids: &RoaringBitmap, batch: &mut BatchBuilder, ) -> trc::Result<()> { batch .with_account_id(account_id) .with_collection(Collection::Email); self.server .archives( account_id, Collection::Email, deleted_ids, |document_id, data_| { let metadata = data_ .to_unarchived::() .caused_by(trc::location!())?; if let Some(message_uid) = metadata.inner.message_uid(mailbox_id) { // Add vanished items batch.with_document(document_id); batch.log_vanished_item( VanishedCollection::Email, (mailbox_id, message_uid), ); if metadata.inner.mailboxes.len() == 1 { // Delete message batch .custom( ObjectIndexBuilder::<_, ()>::new() .with_access_token(&self.access_token) .with_current(metadata), ) .caused_by(trc::location!())? .set( ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { index: SearchIndex::Email, due: TaskEpoch::now(), is_insert: false, }), 0u64.serialize(), ) .commit_point(); } else { // Untag message from this mailbox and remove Deleted flag let mut new_metadata = metadata.inner.to_builder(); new_metadata.remove_mailbox(mailbox_id); new_metadata.remove_keyword(&Keyword::Deleted); // Write changes batch .custom( ObjectIndexBuilder::new() .with_current(metadata) .with_changes(new_metadata.seal()), ) .caused_by(trc::location!())? .commit_point(); } } Ok(true) }, ) .await .caused_by(trc::location!())?; Ok(()) } } ================================================ FILE: crates/imap/src/op/fetch.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{FromModSeq, ImapContext}; use crate::{ core::{SelectedMailbox, Session, SessionData}, spawn_op, }; use ahash::AHashMap; use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, message::metadata::{ ArchivedMessageMetadata, ArchivedMessageMetadataContents, ArchivedMetadataHeaderValue, ArchivedMetadataPartType, DecodedParts, MESSAGE_RECEIVED_MASK, MessageData, MessageMetadata, MetadataHeaderName, PART_ENCODING_PROBLEM, }, }; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, parser::PushUnique, protocol::{ Flag, expunge::Vanished, fetch::{ self, Arguments, Attribute, BodyContents, BodyPart, BodyPartExtension, BodyPartFields, DataItem, Envelope, FetchItem, Section, }, }, receiver::Request, }; use std::{borrow::Cow, sync::Arc, time::Instant}; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use store::{ query::log::{Change, Query}, rkyv::rend::u16_le, write::BatchBuilder, }; use types::{ acl::Acl, collection::{Collection, SyncCollection, VanishedCollection}, field::EmailField, id::Id, keyword::Keyword, }; use utils::chained_bytes::{ChainedBytes, SliceRange}; impl Session { pub async fn handle_fetch(&mut self, requests: Vec>) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapFetch)?; let (data, mailbox) = self.state.select_data(); let is_qresync = self.is_qresync; let mut ops = Vec::with_capacity(requests.len()); for request in requests { let is_uid = matches!(request.command, Command::Fetch(true)); match request.parse_fetch() { Ok(arguments) => { let enabled_condstore = if !self.is_condstore && arguments.changed_since.is_some() || arguments.attributes.contains(&Attribute::ModSeq) { self.is_condstore = true; true } else { false }; ops.push(Ok((is_uid, enabled_condstore, arguments))); } Err(err) => { ops.push(Err(err)); } } } spawn_op!(data, { for op in ops { match op { Ok((is_uid, enabled_condstore, arguments)) => { let response = data .fetch( arguments, mailbox.clone(), is_uid, is_qresync, enabled_condstore, Instant::now(), ) .await?; data.write_bytes(response.into_bytes()).await?; } Err(err) => data.write_error(err).await?, } } Ok(()) }) } } impl SessionData { #[allow(clippy::too_many_arguments)] pub async fn fetch( &self, mut arguments: Arguments, mailbox: Arc, is_uid: bool, is_qresync: bool, enabled_condstore: bool, op_start: Instant, ) -> trc::Result { // Validate VANISHED parameter if arguments.include_vanished { if !is_qresync { return Err(trc::ImapEvent::Error .into_err() .details("Enable QRESYNC first to use the VANISHED parameter.") .ctx(trc::Key::Type, ResponseType::Bad) .id(arguments.tag)); } else if !is_uid { return Err(trc::ImapEvent::Error .into_err() .details("VANISHED parameter is only available for UID FETCH.") .ctx(trc::Key::Type, ResponseType::Bad) .id(arguments.tag)); } } // Resync messages if needed let account_id = mailbox.id.account_id; let mut modseq = self .synchronize_messages(&mailbox) .await .imap_ctx(&arguments.tag, trc::location!())?; // Convert IMAP ids to JMAP ids. let mut ids = mailbox .sequence_to_ids(&arguments.sequence_set, is_uid) .await .imap_ctx(&arguments.tag, trc::location!())?; // Convert state to modseq if let Some(changed_since) = arguments.changed_since { // Obtain changes since the modseq. let changelog = self .server .store() .changes( account_id, SyncCollection::Email.into(), Query::from_modseq(changed_since), ) .await .imap_ctx(&arguments.tag, trc::location!())?; // Process changes let mut changed_ids = AHashMap::new(); let mut has_vanished = false; for change in changelog.changes { match change { Change::InsertItem(id) | Change::UpdateItem(id) => { let id = (id & u32::MAX as u64) as u32; if let Some(uid) = ids.get(&id) { changed_ids.insert(id, *uid); } if !has_vanished { has_vanished = matches!(change, Change::UpdateItem(_)); } } Change::DeleteItem(_) => { has_vanished = true; } _ => (), } } // Send vanished UIDs if arguments.include_vanished && has_vanished { // Add to vanished all known destroyed Ids let vanished = self .server .store() .vanished::<(u32, u32)>( account_id, VanishedCollection::Email.into(), Query::from_modseq(changed_since), ) .await .imap_ctx(&arguments.tag, trc::location!())? .into_iter() .filter_map(|(mailbox_id, uid)| { if mailbox.id.mailbox_id == mailbox_id { Some(uid) } else { None } }) .collect::>(); if !vanished.is_empty() { let mut buf = Vec::with_capacity(vanished.len() * 3); Vanished { earlier: true, ids: vanished, } .serialize(&mut buf); self.write_bytes(buf).await?; } } // Filter out ids without changes if changed_ids.is_empty() { // Condstore was just enabled, return highest modseq. if enabled_condstore { self.write_bytes( StatusResponse::ok("Highest Modseq") .with_code(ResponseCode::highest_modseq(modseq)) .into_bytes(), ) .await?; } trc::event!( Imap(trc::ImapEvent::Fetch), SpanId = self.session_id, AccountId = account_id, MailboxId = mailbox.id.mailbox_id, Elapsed = op_start.elapsed() ); return Ok( StatusResponse::completed(Command::Fetch(is_uid)).with_tag(arguments.tag) ); } ids = changed_ids; arguments.attributes.push_unique(Attribute::ModSeq); } // Build properties list let mut set_seen_flags = false; let mut needs_blobs = false; for attribute in &arguments.attributes { match attribute { Attribute::BodySection { sections, .. } if sections.first().is_some_and(|s| { matches!(s, Section::Header | Section::HeaderFields { .. }) }) => {} Attribute::Body | Attribute::BodyStructure | Attribute::BinarySize { .. } => { /* Note that this did not result in \Seen being set, because RFC822.HEADER response data occurs as a result of a FETCH of RFC822.HEADER. BODY[HEADER] response data occurs as a result of a FETCH of BODY[HEADER] (which sets \Seen) or BODY.PEEK[HEADER] (which does not set \Seen). */ needs_blobs = true; } Attribute::BodySection { peek, .. } | Attribute::Binary { peek, .. } => { if mailbox.is_select && !*peek { set_seen_flags = true; } needs_blobs = true; } Attribute::Rfc822Text | Attribute::Rfc822 => { if mailbox.is_select { set_seen_flags = true; } needs_blobs = true; } _ => (), } } if set_seen_flags && !self .check_mailbox_acl( mailbox.id.account_id, mailbox.id.mailbox_id, Acl::ModifyItems, ) .await .imap_ctx(&arguments.tag, trc::location!())? { set_seen_flags = false; } if is_uid { if arguments.attributes.is_empty() { arguments.attributes.push(Attribute::Flags); } else if !arguments.attributes.contains(&Attribute::Uid) { arguments.attributes.insert(0, Attribute::Uid); } } // Process each message let mut batch = BatchBuilder::new(); let mut ids = ids .into_iter() .map(|(id, imap_id)| (imap_id.seqnum, imap_id.uid, id)) .collect::>(); ids.sort_unstable_by_key(|(seqnum, _, _)| *seqnum); let fetched_ids = ids .iter() .map(|id| trc::Value::from(id.2)) .collect::>(); let message_cache = self .server .get_cached_messages(account_id) .await .imap_ctx(&arguments.tag, trc::location!())?; for (seqnum, uid, id) in ids { // Obtain attributes and keywords let (metadata_, data) = if let (Some(email), Some(data)) = ( self.server .store() .get_value::>(ValueKey::property( account_id, Collection::Email, id, EmailField::Metadata, )) .await .imap_ctx(&arguments.tag, trc::location!())?, message_cache.email_by_id(&id), ) { (email, data) } else { trc::event!( Store(trc::StoreEvent::NotFound), AccountId = account_id, DocumentId = id, Collection = Collection::Email, Details = "Message metadata not found.", CausedBy = trc::location!(), ); continue; }; let metadata = metadata_ .unarchive::() .imap_ctx(&arguments.tag, trc::location!())?; let raw_body; // Fetch and parse blob let mut raw_message = ChainedBytes::new(metadata.raw_headers.as_ref()); if needs_blobs { // Retrieve raw message if needed raw_body = self .server .blob_store() .get_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX) .await .imap_ctx(&arguments.tag, trc::location!())?; if let Some(raw_body) = &raw_body { raw_message.append( raw_body .get(metadata.blob_body_offset.to_native() as usize..) .unwrap_or_default(), ); } else { trc::event!( Store(trc::StoreEvent::NotFound), AccountId = account_id, DocumentId = id, Collection = Collection::Email, BlobId = metadata.blob_hash.0.as_slice(), Details = "Blob not found.", CausedBy = trc::location!(), ); continue; } } let message = &metadata.contents[0]; let decoded = metadata.decode_contents(raw_message.clone()); // Build response let mut items = Vec::with_capacity(arguments.attributes.len()); let set_seen_flag = set_seen_flags && !message_cache.has_keyword(data, &Keyword::Seen); for attribute in &arguments.attributes { match attribute { Attribute::Envelope => { items.push(DataItem::Envelope { envelope: message.envelope(), }); } Attribute::Flags => { let mut flags = message_cache .expand_keywords(data) .map(Flag::from) .collect::>(); if set_seen_flag { flags.push(Flag::Seen); } items.push(DataItem::Flags { flags }); } Attribute::InternalDate => { items.push(DataItem::InternalDate { date: (metadata.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK) as i64, }); } Attribute::Preview { .. } => { items.push(DataItem::Preview { contents: if !metadata.preview.is_empty() { Some(metadata.preview.as_bytes().into()) } else { None }, }); } Attribute::Rfc822Size => { items.push(DataItem::Rfc822Size { size: data.size as usize, }); } Attribute::Uid => { items.push(DataItem::Uid { uid }); } Attribute::Rfc822 => { items.push(DataItem::Rfc822 { contents: raw_message.get_full_range(), }); } Attribute::Rfc822Header => { let contents = raw_message.get_slice_range( 0..u32::from(metadata.root_part().offset_body) as usize, ); if contents != SliceRange::None { items.push(DataItem::Rfc822Header { contents }); } } Attribute::Rfc822Text => { items.push(DataItem::Rfc822Text { contents: raw_message.get_full_range(), }); } Attribute::Body => { items.push(DataItem::Body { part: metadata.body_structure(&decoded, false), }); } Attribute::BodyStructure => { items.push(DataItem::BodyStructure { part: metadata.body_structure(&decoded, true), }); } Attribute::BodySection { sections, partial, .. } => { if let Some(contents) = metadata.body_section(&decoded, sections, *partial) { items.push(DataItem::BodySection { sections: sections.to_vec(), origin_octet: partial.map(|(start, _)| start), contents, }); } } Attribute::Binary { sections, partial, .. } => match metadata.binary(&decoded, sections, *partial) { Ok(Some(contents)) => { items.push(DataItem::Binary { sections: sections.to_vec(), offset: partial.map(|(start, _)| start), contents, }); } Err(_) => { self.write_error( trc::ImapEvent::Error .into_err() .details(format!( "Failed to decode part {} of message {}.", sections .iter() .map(|s| s.to_string()) .collect::>() .join("."), if is_uid { uid } else { seqnum } )) .code(ResponseCode::UnknownCte), ) .await?; continue; } _ => (), }, Attribute::BinarySize { sections } => { if let Some(size) = metadata.binary_size(&decoded, sections) { items.push(DataItem::BinarySize { sections: sections.to_vec(), size, }); } } Attribute::ModSeq => { items.push(DataItem::ModSeq { modseq: data.change_id + 1, }); } Attribute::EmailId => { items.push(DataItem::EmailId { email_id: Id::from_parts(account_id, id).to_string(), }); } Attribute::ThreadId => { items.push(DataItem::ThreadId { thread_id: Id::from_parts(account_id, data.thread_id).to_string(), }); } } } // Add flags to the response if the message was unseen if set_seen_flag && !arguments.attributes.contains(&Attribute::Flags) { let mut flags = message_cache .expand_keywords(data) .map(Flag::from) .collect::>(); flags.push(Flag::Seen); items.push(DataItem::Flags { flags }); } // Serialize fetch item let mut buf = Vec::with_capacity(128); FetchItem { id: seqnum, items }.serialize(&mut buf); self.write_bytes(buf).await?; // Add to set flags if set_seen_flag && let Some(data_) = self .server .store() .get_value::>(ValueKey::archive( account_id, Collection::Email, id, )) .await .imap_ctx(&arguments.tag, trc::location!())? { let data = data_ .to_unarchived::() .imap_ctx(&arguments.tag, trc::location!())?; let mut new_data = data.inner.to_builder(); new_data.keywords.push(Keyword::Seen); batch .with_account_id(account_id) .with_collection(Collection::Email) .with_document(id) .custom( ObjectIndexBuilder::new() .with_current(data) .with_changes(new_data.seal()), ) .imap_ctx(&arguments.tag, trc::location!())? .commit_point(); } } // Set Seen ids if !batch.is_empty() { match self .server .commit_batch(batch) .await .and_then(|ids| ids.last_change_id(account_id)) .imap_ctx(&arguments.tag, trc::location!()) { Ok(change_id) => { modseq = change_id; } Err(err) => { if !err.is_assertion_failure() { return Err(err.id(arguments.tag)); } } } } trc::event!( Imap(trc::ImapEvent::Fetch), SpanId = self.session_id, AccountId = account_id, MailboxId = mailbox.id.mailbox_id, DocumentId = fetched_ids, Details = arguments .attributes .iter() .map(|c| trc::Value::from(format!("{c:?}"))) .collect::>(), Elapsed = op_start.elapsed() ); // Condstore was enabled with this command if enabled_condstore { self.write_bytes( StatusResponse::ok("Highest Modseq") .with_code(ResponseCode::highest_modseq(modseq)) .into_bytes(), ) .await?; } Ok(StatusResponse::completed(Command::Fetch(is_uid)).with_tag(arguments.tag)) } } #[allow(clippy::result_unit_err)] pub trait AsImapDataItem { fn body_structure(&'_ self, decoded: &DecodedParts<'_>, is_extended: bool) -> BodyPart<'_>; fn body_section<'x>( &self, decoded: &'x DecodedParts<'x>, sections: &[Section], partial: Option<(u32, u32)>, ) -> Option>; fn binary<'x>( &self, decoded: &'x DecodedParts<'x>, sections: &[u32], partial: Option<(u32, u32)>, ) -> Result>, ()>; fn binary_size(&self, decoded: &DecodedParts<'_>, sections: &[u32]) -> Option; } #[allow(clippy::result_unit_err)] pub trait AsImapDataItemPart { fn as_body_part( &'_ self, decoded: &DecodedParts<'_>, message_id: usize, part_id: usize, is_extended: bool, ) -> BodyPart<'_>; fn envelope(&'_ self) -> Envelope<'_>; } impl AsImapDataItemPart for ArchivedMessageMetadataContents { fn as_body_part( &'_ self, decoded: &DecodedParts<'_>, message_id: usize, part_id: usize, is_extended: bool, ) -> BodyPart<'_> { let part = &self.parts[part_id]; let body = decoded.raw_message_section(message_id, part.body_to_end()); let (is_multipart, is_text) = match &part.body { ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => (false, true), ArchivedMetadataPartType::Multipart(_) => (true, false), _ => (false, false), }; let content_type = part .header_value(&MetadataHeaderName::ContentType) .and_then(|ct| ct.as_content_type()); let mut body_md5 = None; let mut extension = BodyPartExtension::default(); let mut fields = BodyPartFields::default(); if !is_multipart || is_extended { fields.body_parameters = content_type .as_ref() .map(|ct| { ct.attributes .iter() .map(|k| (k.name.as_ref().into(), k.value.as_ref().into())) .collect::>() }) .filter(|p| !p.is_empty()) } if !is_multipart { fields.body_subtype = content_type .as_ref() .and_then(|ct| ct.c_subtype.as_ref().map(|cs| cs.as_ref().into())); fields.body_id = part .header_value(&MetadataHeaderName::ContentId) .and_then(|id| id.as_text().map(|id| format!("<{}>", id).into())); fields.body_description = part .header_value(&MetadataHeaderName::ContentDescription) .and_then(|ct| ct.as_text().map(|ct| ct.into())); fields.body_encoding = part .header_value(&MetadataHeaderName::ContentTransferEncoding) .and_then(|ct| ct.as_text().map(|ct| ct.into())); fields.body_size_octets = body.as_ref().map(|b| b.len()).unwrap_or(0); if is_text { if fields.body_subtype.is_none() { fields.body_subtype = Some("plain".into()); } if fields.body_encoding.is_none() { fields.body_encoding = Some("7bit".into()); } if fields.body_parameters.is_none() { fields.body_parameters = Some(vec![("charset".into(), "us-ascii".into())]); } } } if is_extended { if !is_multipart { body_md5 = body .as_ref() .map(|b| format!("{:x}", md5::compute(b)).into()); } extension.body_disposition = part .header_value(&MetadataHeaderName::ContentDisposition) .and_then(|cd| cd.as_content_type()) .map(|cd| { ( cd.c_type.as_ref().into(), cd.attributes .iter() .map(|k| (k.name.as_ref().into(), k.value.as_ref().into())) .collect::>(), ) }); extension.body_language = part .header_value(&MetadataHeaderName::ContentLanguage) .and_then(|hv| { hv.as_text_list() .map(|list| list.iter().map(|item| item.as_ref().into()).collect()) }); extension.body_location = part .header_value(&MetadataHeaderName::ContentLocation) .and_then(|ct| ct.as_text().map(|ct| ct.into())); } match &part.body { ArchivedMetadataPartType::Multipart(parts) => BodyPart::Multipart { body_parts: Vec::with_capacity(parts.len()), body_subtype: content_type .as_ref() .and_then(|ct| ct.c_subtype.as_ref().map(|cs| cs.as_ref().into())) .unwrap_or_else(|| "".into()), body_parameters: fields.body_parameters, extension, }, ArchivedMetadataPartType::Message(_) => BodyPart::Message { fields, envelope: None, body: None, body_size_lines: 0, body_md5, extension, }, _ => { if is_text { BodyPart::Text { fields, body_size_lines: body .as_ref() .map(|b| b.iter().filter(|&&ch| ch == b'\n').count()) .unwrap_or(0), body_md5, extension, } } else { BodyPart::Basic { body_type: content_type .as_ref() .map(|ct| Cow::from(ct.c_type.as_ref())), fields, body_md5, extension, } } } } } fn envelope(&'_ self) -> Envelope<'_> { let headers = self.root_part(); Envelope { date: headers.date(), subject: headers.subject().map(|s| s.into()), from: headers .header_values(&MetadataHeaderName::From) .flat_map(|a| a.as_imap_address()) .collect(), sender: headers .header_values(&MetadataHeaderName::Sender) .flat_map(|a| a.as_imap_address()) .collect(), reply_to: headers .header_values(&MetadataHeaderName::ReplyTo) .flat_map(|a| a.as_imap_address()) .collect(), to: headers .header_values(&MetadataHeaderName::To) .flat_map(|a| a.as_imap_address()) .collect(), cc: headers .header_values(&MetadataHeaderName::Cc) .flat_map(|a| a.as_imap_address()) .collect(), bcc: headers .header_values(&MetadataHeaderName::Bcc) .flat_map(|a| a.as_imap_address()) .collect(), in_reply_to: headers.in_reply_to().as_text_list().map(|list| { let mut irt = String::with_capacity(list.len() * 10); for (pos, l) in list.iter().enumerate() { if pos > 0 { irt.push(' '); } irt.push('<'); irt.push_str(l.as_ref()); irt.push('>'); } irt.into() }), message_id: headers.message_id().map(|id| format!("<{}>", id).into()), } } } impl AsImapDataItem for ArchivedMessageMetadata { fn body_structure(&'_ self, decoded: &DecodedParts<'_>, is_extended: bool) -> BodyPart<'_> { let mut stack = Vec::new(); let base_part = [u16_le::from_native(0)]; let mut parts = base_part.as_slice().iter(); let mut message = &self.contents[0]; let mut root_part = None; let mut message_id = 0; loop { while let Some(part_id) = parts.next() { let part_id = u16::from(part_id) as usize; let mut part = message.as_body_part(decoded, message_id, part_id, is_extended); match &message.parts[part_id].body { ArchivedMetadataPartType::Message(nested_message_id) => { let nested_message = self.message_id(*nested_message_id); part.set_envelope(nested_message.envelope()); if let Some(root_part) = root_part { if stack.len() == 10_000 { debug_assert!(false, "Too much nesting in message metadata"); return root_part; } stack.push((root_part, parts, (message, message_id).into())); } root_part = part.into(); parts = base_part.as_slice().iter(); message = nested_message; message_id = u16::from(*nested_message_id) as usize; continue; } ArchivedMetadataPartType::Multipart(subparts) => { if let Some(root_part) = root_part { if stack.len() == 10_000 { debug_assert!(false, "Too much nesting in message metadata"); return root_part; } stack.push((root_part, parts, None)); } root_part = part.into(); parts = subparts.iter(); continue; } _ => (), } if let Some(root_part) = &mut root_part { root_part.add_part(part); } else { return part; } } if let Some((mut prev_root_part, prev_parts, prev_message)) = stack.pop() { if let Some((prev_message, prev_message_id)) = prev_message { message = prev_message; message_id = prev_message_id; } prev_root_part.add_part(root_part.unwrap()); parts = prev_parts; root_part = prev_root_part.into(); } else { break; } } root_part.unwrap() } fn body_section<'x>( &self, decoded: &'x DecodedParts<'x>, sections: &[Section], partial: Option<(u32, u32)>, ) -> Option> { let mut part = self.root_part(); if sections.is_empty() { return Some(get_cow_partial_bytes( decoded.raw_message_section(0, part.header_to_end())?, partial, )); } let mut message = &self.contents[0]; let mut message_id = 0; let mut sections_iter = sections.iter().enumerate().peekable(); while let Some((section_num, section)) = sections_iter.next() { match section { Section::Part { num } => { part = if let Some(sub_part_ids) = part.sub_parts() { sub_part_ids .as_ref() .get((*num).saturating_sub(1) as usize) .and_then(|pos| message.parts.as_ref().get(u16::from(*pos) as usize)) } else if *num == 1 && (section_num == sections.len() - 1 || part.is_message()) { Some(part) } else { None }?; if let ArchivedMetadataPartType::Message(nested_message_id) = &part.body && let Some(( _, Section::Part { .. } | Section::Header | Section::HeaderFields { .. } | Section::Text, )) = sections_iter.peek() { message = self.message_id(*nested_message_id); part = message.root_part(); message_id = u16::from(nested_message_id) as usize; } } Section::Header => { return Some(get_cow_partial_bytes( decoded.raw_message_section(message_id, part.header_to_body())?, partial, )); } Section::HeaderFields { not, fields } => { let mut headers = Vec::with_capacity( u32::from(part.offset_body).saturating_sub(u32::from(part.offset_header)) as usize, ); for header in part.headers.iter() { let header_name = header.name.as_str(); if fields.iter().any(|f| header_name.eq_ignore_ascii_case(f)) != *not { headers.extend_from_slice(header_name.as_bytes()); headers.push(b':'); headers.extend_from_slice( &decoded .raw_message_section(message_id, header.value_range()) .unwrap_or_default(), ); } } headers.extend_from_slice(b"\r\n"); return Some(if partial.is_none() { headers.into() } else { get_partial_bytes(&headers, partial).to_vec().into() }); } Section::Text => { return Some(get_cow_partial_bytes( decoded.raw_message_section(message_id, part.body_to_end())?, partial, )); } Section::Mime => { let mut headers = Vec::with_capacity( u32::from(part.offset_body).saturating_sub(u32::from(part.offset_header)) as usize, ); for header in part.headers.iter() { if header.name.is_mime_header() || header.name.as_str().starts_with("Content-") { headers.extend_from_slice(header.name.as_str().as_bytes()); headers.extend_from_slice(b":"); headers.extend_from_slice( &decoded .raw_message_section(message_id, header.value_range()) .unwrap_or_default(), ); } } headers.extend_from_slice(b"\r\n"); return Some(if partial.is_none() { headers.into() } else { get_partial_bytes(&headers, partial).to_vec().into() }); } } } // BODY[x] should return both headers and body, but most clients // expect BODY[x] to return only the body, just like BOXY[x.TEXT] does. Some(get_cow_partial_bytes( decoded.raw_message_section(message_id, part.body_to_end())?, partial, )) } fn binary<'x>( &self, decoded: &'x DecodedParts<'x>, sections: &[u32], partial: Option<(u32, u32)>, ) -> Result>, ()> { let mut message = &self.contents[0]; let mut message_id = 0; let mut part = self.root_part(); let mut sections_iter = sections.iter().enumerate().peekable(); while let Some((section_num, num)) = sections_iter.next() { part = if let Some(sub_part_ids) = part.sub_parts() { if let Some(part) = sub_part_ids .as_ref() .get((*num).saturating_sub(1) as usize) .and_then(|pos| message.parts.as_ref().get(u16::from(*pos) as usize)) { part } else { return Ok(None); } } else if *num == 1 && (section_num == sections.len() - 1 || part.is_message()) { part } else { return Ok(None); }; if let (ArchivedMetadataPartType::Message(nested_message), Some(_)) = (&part.body, sections_iter.peek()) { message = self.message_id(*nested_message); part = message.root_part(); message_id = u16::from(nested_message) as usize; } } if (part.flags & PART_ENCODING_PROBLEM) == 0 { let part_offset = u32::from(part.offset_header) as usize; Ok(match &part.body { ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => { BodyContents::Text(String::from_utf8_lossy(get_partial_bytes( decoded .binary_part(message_id, part_offset) .unwrap_or_default(), partial, ))) .into() } ArchivedMetadataPartType::Binary | ArchivedMetadataPartType::InlineBinary => { BodyContents::Bytes( get_partial_bytes( decoded .binary_part(message_id, part_offset) .unwrap_or_default(), partial, ) .into(), ) .into() } ArchivedMetadataPartType::Message(message) => BodyContents::Bytes({ { let part = self.message_id(*message).root_part(); get_cow_partial_bytes( decoded .raw_message_section(message_id, part.header_to_end()) .unwrap_or_default(), partial, ) } }) .into(), ArchivedMetadataPartType::Multipart(_) => { BodyContents::Bytes(get_cow_partial_bytes( decoded .raw_message_section(message_id, part.header_to_end()) .unwrap_or_default(), partial, )) .into() } }) } else { Err(()) } } fn binary_size(&self, decoded: &DecodedParts<'_>, sections: &[u32]) -> Option { let mut message = &self.contents[0]; let mut message_id = 0; let mut part = self.root_part(); let mut sections_iter = sections.iter().enumerate().peekable(); while let Some((section_num, num)) = sections_iter.next() { part = if let Some(sub_part_ids) = part.sub_parts() { sub_part_ids .as_ref() .get((*num).saturating_sub(1) as usize) .and_then(|pos| message.parts.as_ref().get(u16::from(pos) as usize)) } else if *num == 1 && (section_num == sections.len() - 1 || part.is_message()) { Some(part) } else { None }?; if let (ArchivedMetadataPartType::Message(nested_message), Some(_)) = (&part.body, sections_iter.peek()) { message = self.message_id(*nested_message); message_id = u16::from(nested_message) as usize; part = message.root_part(); } } match &part.body { ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html | ArchivedMetadataPartType::Binary | ArchivedMetadataPartType::InlineBinary => decoded .part(message_id, u32::from(part.offset_header) as usize) .map(|p| p.len()) .unwrap_or_default(), ArchivedMetadataPartType::Message(message) => { self.message_id(*message).root_part().raw_len() } ArchivedMetadataPartType::Multipart(_) => part.raw_len(), } .into() } } #[inline(always)] fn get_partial_bytes(bytes: &[u8], partial: Option<(u32, u32)>) -> &[u8] { if let Some((start, end)) = partial { bytes .get(start as usize..std::cmp::min((start + end) as usize, bytes.len())) .unwrap_or_default() } else { bytes } } #[inline(always)] fn get_cow_partial_bytes(bytes: Cow<'_, [u8]>, partial: Option<(u32, u32)>) -> Cow<'_, [u8]> { if let Some((start, end)) = partial { let range = start as usize..std::cmp::min((start + end) as usize, bytes.len()); match bytes { Cow::Borrowed(bytes) => Cow::Borrowed(bytes.get(range).unwrap_or_default()), Cow::Owned(bytes) => Cow::Owned(bytes.get(range).unwrap_or_default().to_vec()), } } else { bytes } } trait AsImapAddress { fn as_imap_address(&'_ self) -> Vec>; } impl AsImapAddress for ArchivedMetadataHeaderValue { fn as_imap_address(&'_ self) -> Vec> { let mut addresses = Vec::new(); match self { ArchivedMetadataHeaderValue::AddressList(list) => { for addr in list.iter() { if let Some(email) = addr.address.as_ref() { addresses.push(fetch::Address::Single(fetch::EmailAddress { name: addr.name.as_ref().map(|n| n.as_ref().into()), address: email.as_ref().into(), })); } } } ArchivedMetadataHeaderValue::AddressGroup(list) => { for group in list.iter() { addresses.push(fetch::Address::Group(fetch::AddressGroup { name: group.name.as_ref().map(|n| n.as_ref().into()), addresses: group .addresses .iter() .filter_map(|addr| { fetch::EmailAddress { name: addr.name.as_ref().map(|n| n.as_ref().into()), address: addr.address.as_ref()?.as_ref().into(), } .into() }) .collect(), })); } } _ => (), } addresses } } ================================================ FILE: crates/imap/src/op/idle.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ core::{SelectedMailbox, Session, SessionData, State}, op::ImapContext, }; use ahash::AHashSet; use common::{ipc::PushNotification, listener::SessionStream}; use directory::Permission; use imap_proto::{ Command, StatusResponse, protocol::{ Sequence, fetch, list::{Attribute, ListItem}, status::Status, }, receiver::Request, }; use std::{sync::Arc, time::Instant}; use store::query::log::Query; use tokio::io::AsyncReadExt; use trc::AddContext; use types::{collection::SyncCollection, type_state::DataType}; use utils::map::bitmap::Bitmap; impl Session { pub async fn handle_idle(&mut self, request: Request) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapIdle)?; let op_start = Instant::now(); let (data, mailbox, types) = match &self.state { State::Authenticated { data, .. } => { (data.clone(), None, Bitmap::from_iter([DataType::Mailbox])) } State::Selected { data, mailbox, .. } => ( data.clone(), mailbox.clone().into(), Bitmap::from_iter([DataType::Email, DataType::Mailbox, DataType::EmailDelivery]), ), _ => unreachable!(), }; let is_rev2 = self.version.is_rev2(); let is_utf8 = self.is_utf8; let is_qresync = self.is_qresync; // Register with push manager let mut push_rx = self .server .subscribe_push_manager(&data.access_token, types) .await .imap_ctx(&request.tag, trc::location!())?; // Send continuation response self.write_bytes(b"+ Idling, send 'DONE' to stop.\r\n".to_vec()) .await?; trc::event!( Imap(trc::ImapEvent::IdleStart), SpanId = self.session_id, Elapsed = op_start.elapsed() ); let op_start = Instant::now(); let mut buf = vec![0; 4]; loop { tokio::select! { result = tokio::time::timeout(self.server.core.imap.timeout_idle, self.stream_rx.read_exact(&mut buf)) => { match result { Ok(Ok(bytes_read)) => { if bytes_read > 0 { if (buf[..bytes_read]).windows(4).any(|w| w == b"DONE") { trc::event!(Imap(trc::ImapEvent::IdleStop), SpanId = self.session_id, Elapsed = op_start.elapsed()); return self.write_bytes(StatusResponse::completed(Command::Idle) .with_tag(request.tag) .into_bytes()).await; } } else { return Err(trc::NetworkEvent::Closed.into_err().details("IMAP connection closed by client.").id(request.tag)); } }, Ok(Err(err)) => { return Err(trc::NetworkEvent::ReadError.into_err().reason(err).details("IMAP connection error.").id(request.tag)); }, Err(_) => { self.write_bytes(&b"* BYE IDLE timed out.\r\n"[..]).await.ok(); return Err(trc::NetworkEvent::Timeout.into_err().details("IMAP IDLE timed out.").id(request.tag)); } } } push_notification = push_rx.recv() => { if let Some(push_notification) = push_notification { let mut has_mailbox_changes = false; let mut has_email_changes = false; match push_notification { PushNotification::StateChange(state_change) => { for type_state in state_change.types { match type_state { DataType::Email | DataType::EmailDelivery => { has_email_changes = true; } DataType::Mailbox => { has_mailbox_changes = true; } _ => {} } } }, PushNotification::EmailPush(_) => { has_email_changes = true; has_mailbox_changes = true; }, PushNotification::CalendarAlert(_) => (), } if has_mailbox_changes || has_email_changes { data.write_changes(&mailbox, has_mailbox_changes, has_email_changes, is_qresync, is_rev2, is_utf8).await?; } } else { self.write_bytes(&b"* BYE Server shutting down.\r\n"[..]).await.ok(); return Err(trc::NetworkEvent::Closed.into_err().details("IDLE channel closed.").id(request.tag)); } } } } } } impl SessionData { pub async fn write_changes( &self, mailbox: &Option>, check_mailboxes: bool, check_emails: bool, is_qresync: bool, is_rev2: bool, is_utf8: bool, ) -> trc::Result<()> { // Fetch all changed mailboxes if check_mailboxes { let changes = self .synchronize_mailboxes(true) .await .caused_by(trc::location!())? .unwrap(); let mut buf = Vec::with_capacity(64); // List deleted mailboxes for mailbox_name in changes.deleted { ListItem { mailbox_name, attributes: vec![Attribute::NonExistent], tags: vec![], } .serialize(&mut buf, is_rev2, is_utf8, false); } // List added mailboxes for mailbox_name in changes.added { ListItem { mailbox_name, attributes: vec![], tags: vec![], } .serialize(&mut buf, is_rev2, is_utf8, false); } // Obtain status of changed mailboxes for mailbox_name in changes.changed { if let Ok(status) = self .status( mailbox_name, &[ Status::Messages, Status::Unseen, Status::UidNext, Status::UidValidity, ], ) .await { status.serialize(&mut buf, is_utf8); } } if !buf.is_empty() { self.write_bytes(buf).await?; } } // Fetch selected mailbox changes if check_emails { // Synchronize emails if let Some(mailbox) = mailbox { // Obtain changes since last sync let modseq = mailbox.state.lock().modseq; let new_state = self .write_mailbox_changes(mailbox, is_qresync) .await .caused_by(trc::location!())?; if new_state == modseq { return Ok(()); } // Obtain changed messages let changelog = self .server .store() .changes( mailbox.id.account_id, SyncCollection::Email.into(), Query::Since(modseq), ) .await .caused_by(trc::location!())?; let changed_ids = { let state = mailbox.state.lock(); changelog .changes .into_iter() .filter_map(|change| { change.try_unwrap_item_id().and_then(|item_id| { state .id_to_imap .get(&((item_id & u32::MAX as u64) as u32)) .map(|id| id.uid) }) }) .collect::>() }; if !changed_ids.is_empty() { let op_start = Instant::now(); return self .fetch( fetch::Arguments { tag: "".into(), sequence_set: Sequence::List { items: changed_ids .into_iter() .map(|uid| Sequence::Number { value: uid }) .collect(), }, attributes: vec![fetch::Attribute::Flags, fetch::Attribute::Uid], changed_since: None, include_vanished: false, }, mailbox.clone(), true, is_qresync, false, op_start, ) .await .caused_by(trc::location!()) .map(|_| ()); } } } Ok(()) } } ================================================ FILE: crates/imap/src/op/list.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Instant; use crate::{ core::{Session, SessionData}, spawn_op, }; use common::listener::SessionStream; use directory::Permission; use imap_proto::{ Command, StatusResponse, protocol::{ ImapResponse, ProtocolVersion, list::{ self, Arguments, Attribute, ChildInfo, ListItem, ReturnOption, SelectionOption, Tag, }, }, receiver::Request, }; use trc::StoreEvent; use super::ImapContext; impl Session { pub async fn handle_list(&mut self, request: Request) -> trc::Result<()> { let op_start = Instant::now(); let command = request.command; let is_lsub = command == Command::Lsub; let arguments = if !is_lsub { // Validate access self.assert_has_permission(Permission::ImapList)?; request.parse_list(self.is_utf8) } else { // Validate access self.assert_has_permission(Permission::ImapLsub)?; request.parse_lsub(self.is_utf8) }?; if !arguments.is_separator_query() { let data = self.state.session_data(); let version = self.version; let is_utf8 = self.is_utf8; spawn_op!( data, data.list(arguments, is_lsub, version, is_utf8, op_start) .await ) } else { self.write_bytes( StatusResponse::completed(command) .with_tag(arguments.unwrap_tag()) .serialize( list::Response { is_rev2: self.version.is_rev2(), is_utf8: self.is_utf8, is_lsub, list_items: vec![ListItem { mailbox_name: "".into(), attributes: vec![Attribute::NoSelect], tags: vec![], }], status_items: Vec::new(), } .serialize(), ), ) .await } } } impl SessionData { pub async fn list( &self, arguments: Arguments, is_lsub: bool, version: ProtocolVersion, is_utf8: bool, op_start: Instant, ) -> trc::Result<()> { let (tag, reference_name, mut patterns, selection_options, return_options) = match arguments { Arguments::Basic { tag, reference_name, mailbox_name, } => ( tag, reference_name, vec![mailbox_name], Vec::new(), Vec::new(), ), Arguments::Extended { tag, reference_name, mailbox_name, selection_options, return_options, } => ( tag, reference_name, mailbox_name, selection_options, return_options, ), }; // Refresh mailboxes self.synchronize_mailboxes(false) .await .imap_ctx(&tag, trc::location!())?; // Process arguments let mut filter_subscribed = false; let mut filter_special_use = false; let mut recursive_match = false; let mut include_special_use = true; let mut include_subscribed = false; let mut include_children = false; let mut include_status = None; for selection_option in &selection_options { match selection_option { SelectionOption::Subscribed => { filter_subscribed = true; include_subscribed = true; } SelectionOption::Remote => (), SelectionOption::SpecialUse => { filter_special_use = true; include_special_use = true; } SelectionOption::RecursiveMatch => { recursive_match = true; } } } for return_option in &return_options { match return_option { ReturnOption::Subscribed => { include_subscribed = true; } ReturnOption::Children => { include_children = true; } ReturnOption::Status(status) => { include_status = status.into(); } ReturnOption::SpecialUse => { include_special_use = true; } } } if recursive_match && !filter_subscribed { return Err(trc::ImapEvent::Error .into_err() .details("RECURSIVEMATCH requires the SUBSCRIBED selection option.") .id(tag)); } // Append reference name if !patterns.is_empty() && !reference_name.is_empty() { patterns.iter_mut().for_each(|item| { *item = format!("{}{}", reference_name, item); }) } let mut list_items = Vec::with_capacity(10); // Add mailboxes let mut added_shared_folder = false; for account in self.mailboxes.lock().iter() { if let Some(prefix) = &account.prefix { if !added_shared_folder { if !filter_subscribed && matches_pattern(&patterns, &self.server.core.jmap.shared_folder) { list_items.push(ListItem { mailbox_name: self.server.core.jmap.shared_folder.as_str().into(), attributes: if include_children { vec![Attribute::HasChildren, Attribute::NoSelect] } else { vec![Attribute::NoSelect] }, tags: vec![], }); } added_shared_folder = true; } if !filter_subscribed && matches_pattern(&patterns, prefix) { list_items.push(ListItem { mailbox_name: prefix.clone(), attributes: if include_children { vec![Attribute::HasChildren, Attribute::NoSelect] } else { vec![Attribute::NoSelect] }, tags: vec![], }); } } for (mailbox_name, mailbox_id) in &account.mailbox_names { if matches_pattern(&patterns, mailbox_name) { let mailbox = if let Some(mailbox) = account.mailbox_state.get(mailbox_id) { mailbox } else { trc::event!( Store(StoreEvent::UnexpectedError), Details = "IMAP mailbox no longer present in account state", Id = *mailbox_id, Details = account .mailbox_state .keys() .copied() .map(trc::Value::from) .collect::>() ); continue; }; let mut has_recursive_match = false; if recursive_match { let prefix = format!("{}/", mailbox_name); for (mailbox_name, mailbox_id) in &account.mailbox_names { if mailbox_name.starts_with(&prefix) && account.mailbox_state.get(mailbox_id).unwrap().is_subscribed { has_recursive_match = true; break; } } } if !filter_subscribed || mailbox.is_subscribed || has_recursive_match { let mut attributes = Vec::with_capacity(2); if include_children { attributes.push(if mailbox.has_children { Attribute::HasChildren } else { Attribute::HasNoChildren }); } if include_subscribed && mailbox.is_subscribed { attributes.push(Attribute::Subscribed); } if include_special_use { if let Some(special_use) = &mailbox.special_use { attributes.push(*special_use); } else if filter_special_use { continue; } } list_items.push(ListItem { mailbox_name: mailbox_name.clone(), attributes, tags: if !has_recursive_match { vec![] } else { vec![Tag::ChildInfo(vec![ChildInfo::Subscribed])] }, }); } } } } // Add status response let mut status_items = Vec::new(); if let Some(include_status) = include_status { for list_item in &list_items { match self .status(list_item.mailbox_name.clone(), include_status) .await .imap_ctx(&tag, trc::location!()) { Ok(status_item) => { status_items.push(status_item); } Err(err) => { self.write_error(err).await?; } } } } trc::event!( Imap(if !is_lsub { trc::ImapEvent::List } else { trc::ImapEvent::Lsub }), SpanId = self.session_id, Details = list_items .iter() .map(|item| trc::Value::from(item.mailbox_name.clone())) .collect::>(), Elapsed = op_start.elapsed() ); // Write response self.write_bytes( StatusResponse::completed(if !is_lsub { Command::List } else { Command::Lsub }) .with_tag(tag) .serialize( list::Response { is_rev2: version.is_rev2(), is_utf8, is_lsub, list_items, status_items, } .serialize(), ), ) .await } } #[allow(clippy::while_let_on_iterator)] pub fn matches_pattern(patterns: &[String], mailbox_name: &str) -> bool { if patterns.is_empty() { return true; } 'outer: for pattern in patterns { let mut pattern_bytes = pattern.as_bytes().iter().enumerate().peekable(); let mut mailbox_name = mailbox_name.as_bytes().iter().peekable(); 'inner: while let Some((pos, &ch)) = pattern_bytes.next() { if ch == b'%' || ch == b'*' { let mut end_pos = pos; while let Some(&(_, &next_ch)) = pattern_bytes.peek() { if next_ch == b'%' || next_ch == b'*' { break; } else { end_pos = pattern_bytes.next().unwrap().0; } } if end_pos > pos { let match_bytes = &pattern.as_bytes()[pos + 1..end_pos + 1]; let mut match_count = 0; let pattern_eof = end_pos == pattern.len() - 1; loop { match mailbox_name.next() { Some(&ch) => { if match_bytes[match_count] == ch { match_count += 1; if match_count == match_bytes.len() { if !pattern_eof { continue 'inner; } else if mailbox_name.peek().is_none() { return true; } else { // Match needs to be at the end of the string, // reset counter. match_count = 0; } } } else if match_count > 0 { match_count = 0; } } None => continue 'outer, } } } else if ch == b'*' || !mailbox_name.any(|&ch| ch == b'/') { return true; } else { continue 'outer; } } else { match mailbox_name.next() { Some(&mch) if mch == ch => (), _ => continue 'outer, } } } if mailbox_name.next().is_none() { return true; } } false } ================================================ FILE: crates/imap/src/op/login.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use imap_proto::{Command, receiver::Request}; use crate::core::Session; use common::listener::SessionStream; use mail_send::Credentials; impl Session { pub async fn handle_login(&mut self, request: Request) -> trc::Result<()> { let arguments = request.parse_login()?; self.authenticate( Credentials::Plain { username: arguments.username.to_string(), secret: arguments.password.to_string(), }, arguments.tag, ) .await } } ================================================ FILE: crates/imap/src/op/logout.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Instant; use crate::core::Session; use common::listener::SessionStream; use imap_proto::{Command, StatusResponse, receiver::Request}; impl Session { pub async fn handle_logout(&mut self, request: Request) -> trc::Result<()> { let op_start = Instant::now(); let mut response = StatusResponse::bye("Stalwart IMAP4rev2 bids you farewell.".to_string()).into_bytes(); trc::event!( Imap(trc::ImapEvent::Logout), SpanId = self.session_id, Elapsed = op_start.elapsed() ); response.extend( StatusResponse::completed(Command::Logout) .with_tag(request.tag) .into_bytes(), ); self.write_bytes(response).await } } ================================================ FILE: crates/imap/src/op/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use ::store::query::log::Query; use imap_proto::ResponseCode; pub mod acl; pub mod append; pub mod authenticate; pub mod capability; pub mod close; pub mod copy_move; pub mod create; pub mod delete; pub mod enable; pub mod expunge; pub mod fetch; pub mod idle; pub mod list; pub mod login; pub mod logout; pub mod namespace; pub mod noop; pub mod quota; pub mod rename; pub mod search; pub mod select; pub mod status; pub mod store; pub mod subscribe; pub mod thread; trait FromModSeq { fn from_modseq(modseq: u64) -> Self; } trait ToModSeq { fn to_modseq(&self) -> u64; } impl FromModSeq for Query { fn from_modseq(modseq: u64) -> Self { if modseq > 0 { Query::Since(modseq - 1) } else { Query::All } } } impl ToModSeq for u64 { fn to_modseq(&self) -> u64 { if *self > 0 { *self + 1 } else { 0 } } } #[macro_export] macro_rules! spawn_op { ($data:expr, $($code:tt)*) => { { tokio::spawn(async move { let data = &($data); if let Err(err) = (async { $($code)* }) .await { let _ = data.write_error(err).await; } }); Ok(())} }; } pub trait ImapContext { fn imap_ctx(self, tag: &str, location: &'static str) -> trc::Result; } impl ImapContext for trc::Result { fn imap_ctx(self, tag: &str, location: &'static str) -> trc::Result { match self { Ok(value) => Ok(value), Err(err) => Err( if !err.matches(trc::EventType::Imap(trc::ImapEvent::Error)) { err.ctx(trc::Key::Id, tag.to_string()) .ctx(trc::Key::Details, "Internal Server Error") .ctx(trc::Key::Code, ResponseCode::ContactAdmin) .ctx(trc::Key::CausedBy, location) } else { err.ctx(trc::Key::Id, tag.to_string()) }, ), } } } ================================================ FILE: crates/imap/src/op/namespace.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::core::Session; use common::listener::SessionStream; use directory::Permission; use imap_proto::{ Command, StatusResponse, protocol::{ImapResponse, namespace::Response}, receiver::Request, }; impl Session { pub async fn handle_namespace(&mut self, request: Request) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapNamespace)?; trc::event!( Imap(trc::ImapEvent::Namespace), SpanId = self.session_id, Elapsed = trc::Value::Duration(0) ); self.write_bytes( StatusResponse::completed(Command::Namespace) .with_tag(request.tag) .serialize( Response { shared_prefix: if self.state.session_data().mailboxes.lock().len() > 1 { Some(self.server.core.jmap.shared_folder.as_str().into()) } else { None }, } .serialize(), ), ) .await } } ================================================ FILE: crates/imap/src/op/noop.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Instant; use crate::core::{Session, State}; use common::listener::SessionStream; use imap_proto::{Command, StatusResponse, receiver::Request}; impl Session { pub async fn handle_noop(&mut self, request: Request) -> trc::Result<()> { let op_start = Instant::now(); if let State::Selected { data, mailbox, .. } = &self.state { data.write_changes( &Some(mailbox.clone()), false, true, self.is_qresync, self.version.is_rev2(), self.is_utf8, ) .await?; } trc::event!( Imap(trc::ImapEvent::Noop), SpanId = self.session_id, Elapsed = op_start.elapsed() ); self.write_bytes( StatusResponse::completed(request.command) .with_tag(request.tag) .into_bytes(), ) .await } } ================================================ FILE: crates/imap/src/op/quota.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ core::{Session, SessionData}, op::ImapContext, spawn_op, }; use common::listener::SessionStream; use directory::Permission; use imap_proto::{ Command, ResponseCode, StatusResponse, protocol::{ ImapResponse, capability::QuotaResourceName, quota::{Arguments, QuotaItem, QuotaResource, Response}, }, receiver::Request, }; use std::time::Instant; impl Session { pub async fn handle_get_quota(&mut self, request: Request) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapStatus)?; let data = self.state.session_data(); spawn_op!(data, { match request.parse_get_quota() { Ok(argument) => match data.get_quota(argument).await { Ok(response) => { data.write_bytes(response).await?; } Err(error) => { data.write_error(error).await?; } }, Err(err) => data.write_error(err).await?, } Ok(()) }) } pub async fn handle_get_quota_root(&mut self, request: Request) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapStatus)?; let data = self.state.session_data(); let is_utf8 = self.is_utf8; spawn_op!(data, { match request.parse_get_quota_root(is_utf8) { Ok(argument) => match data.get_quota_root(argument).await { Ok(response) => { data.write_bytes(response).await?; } Err(error) => { data.write_error(error).await?; } }, Err(err) => data.write_error(err).await?, } Ok(()) }) } } impl SessionData { pub async fn get_quota(&self, arguments: Arguments) -> trc::Result> { let op_start = Instant::now(); // Refresh mailboxes self.synchronize_mailboxes(false) .await .imap_ctx(&arguments.tag, trc::location!())?; // Validate quota root let account_id: u32 = arguments .name .strip_prefix("#") .and_then(|id| id.parse().ok()) .filter(|id| self.access_token.is_member(*id)) .ok_or_else(|| { trc::ImapEvent::Error .into_err() .details("Invalid quota root parameter.") .id(arguments.tag.to_string()) })?; // Obtain access token for mailbox let access_token = self .server .get_access_token(account_id) .await .imap_ctx(&arguments.tag, trc::location!())?; let used_quota = self .server .get_used_quota(account_id) .await .imap_ctx(&arguments.tag, trc::location!())?; trc::event!( Imap(trc::ImapEvent::GetQuota), SpanId = self.session_id, Id = arguments.name.clone(), Details = vec![ trc::Value::from(used_quota), trc::Value::from(access_token.quota) ], Elapsed = op_start.elapsed() ); // Build response let response = Response { quota_root_items: vec![], quota_items: vec![QuotaItem { name: arguments.name, resources: if access_token.quota > 0 { vec![QuotaResource { resource: QuotaResourceName::Storage, total: access_token.quota, used: used_quota as u64, }] } else { vec![] }, }], }; Ok(StatusResponse::ok("GETQUOTA successful.") .with_tag(arguments.tag) .serialize(response.serialize())) } pub async fn get_quota_root(&self, arguments: Arguments) -> trc::Result> { let op_start = Instant::now(); // Refresh mailboxes self.synchronize_mailboxes(false) .await .imap_ctx(&arguments.tag, trc::location!())?; // Validate mailbox let account_id = if let Some(mailbox) = self.get_mailbox_by_name(&arguments.name) { mailbox.account_id } else { return Err(trc::ImapEvent::Error .into_err() .details("Mailbox does not exist.") .code(ResponseCode::TryCreate) .id(arguments.tag)); }; // Obtain access token for mailbox let access_token = self .server .get_access_token(account_id) .await .imap_ctx(&arguments.tag, trc::location!())?; let used_quota = self .server .get_used_quota(account_id) .await .imap_ctx(&arguments.tag, trc::location!())?; trc::event!( Imap(trc::ImapEvent::GetQuota), SpanId = self.session_id, MailboxName = arguments.name.clone(), Details = vec![ trc::Value::from(used_quota), trc::Value::from(access_token.quota) ], Elapsed = op_start.elapsed() ); // Build response let response = Response { quota_root_items: vec![arguments.name, format!("#{account_id}")], quota_items: vec![QuotaItem { name: format!("#{account_id}"), resources: if access_token.quota > 0 { vec![QuotaResource { resource: QuotaResourceName::Storage, total: access_token.quota, used: used_quota as u64, }] } else { vec![] }, }], }; Ok(StatusResponse::ok("GETQUOTAROOT successful.") .with_tag(arguments.tag) .serialize(response.serialize())) } } ================================================ FILE: crates/imap/src/op/rename.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ core::{Session, SessionData}, spawn_op, }; use common::{listener::SessionStream, sharing::EffectiveAcl, storage::index::ObjectIndexBuilder}; use directory::Permission; use imap_proto::{ Command, ResponseCode, StatusResponse, protocol::rename::Arguments, receiver::Request, }; use std::time::Instant; use store::{ ValueKey, write::{AlignedBytes, Archive, BatchBuilder}, }; use trc::AddContext; use types::{acl::Acl, collection::Collection}; use super::ImapContext; impl Session { pub async fn handle_rename(&mut self, request: Request) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapRename)?; let op_start = Instant::now(); let arguments = request.parse_rename(self.is_utf8)?; let data = self.state.session_data(); spawn_op!(data, { let response = data.rename_folder(arguments, op_start).await?; data.write_bytes(response.into_bytes()).await }) } } impl SessionData { pub async fn rename_folder( &self, arguments: Arguments, op_start: Instant, ) -> trc::Result { // Refresh mailboxes self.synchronize_mailboxes(false) .await .imap_ctx(&arguments.tag, trc::location!())?; // Validate mailbox name let mut params = self .validate_mailbox_create(&arguments.new_mailbox_name, None) .await .add_context(|err| err.id(arguments.tag.clone()))?; params.is_rename = true; // Validate source mailbox let mailbox_id = { let mut mailbox_id = None; for account in self.mailboxes.lock().iter() { if let Some(mailbox_id_) = account.mailbox_names.get(&arguments.mailbox_name) { if account.account_id == params.account_id { mailbox_id = (*mailbox_id_).into(); break; } else { return Err(trc::ImapEvent::Error .into_err() .details("Cannot move mailboxes between accounts.") .code(ResponseCode::Cannot) .id(arguments.tag)); } } } if let Some(mailbox_id) = mailbox_id { mailbox_id } else { return Err(trc::ImapEvent::Error .into_err() .details(format!("Mailbox '{}' not found.", arguments.mailbox_name)) .code(ResponseCode::NonExistent) .id(arguments.tag)); } }; // Obtain mailbox let mailbox_ = self .server .store() .get_value::>(ValueKey::archive( params.account_id, Collection::Mailbox, mailbox_id, )) .await .imap_ctx(&arguments.tag, trc::location!())? .ok_or_else(|| { trc::ImapEvent::Error .into_err() .details(format!("Mailbox '{}' not found.", arguments.mailbox_name)) .caused_by(trc::location!()) .code(ResponseCode::NonExistent) .id(arguments.tag.clone()) })?; let mailbox = mailbox_ .to_unarchived::() .imap_ctx(&arguments.tag, trc::location!())?; // Validate ACL let access_token = self .get_access_token() .await .imap_ctx(&arguments.tag, trc::location!())?; if access_token.is_shared(params.account_id) && !mailbox .inner .acls .effective_acl(&access_token) .contains(Acl::Modify) { return Err(trc::ImapEvent::Error .into_err() .details("You are not allowed to rename this mailbox.") .code(ResponseCode::NoPerm) .id(arguments.tag)); } // Get new mailbox name from path let new_mailbox_name = params.path.pop().unwrap(); // Build batch let mut parent_id = params.parent_mailbox_id.map(|id| id + 1).unwrap_or(0); let mut create_ids = Vec::with_capacity(params.path.len()); let mut next_document_id = self .server .store() .assign_document_ids( params.account_id, Collection::Mailbox, params.path.len() as u64, ) .await .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); for &path_item in params.path.iter() { let mailbox_id = next_document_id; next_document_id -= 1; batch .with_account_id(params.account_id) .with_collection(Collection::Mailbox) .with_document(mailbox_id) .custom(ObjectIndexBuilder::<(), _>::new().with_changes( email::mailbox::Mailbox::new(path_item).with_parent_id(parent_id), )) .imap_ctx(&arguments.tag, trc::location!())? .commit_point(); parent_id = mailbox_id + 1; create_ids.push(mailbox_id); } let mut new_mailbox = mailbox .deserialize::() .caused_by(trc::location!())?; new_mailbox.name = new_mailbox_name.into(); new_mailbox.parent_id = parent_id; new_mailbox.uid_validity = rand::random::(); batch .with_account_id(params.account_id) .with_collection(Collection::Mailbox) .with_document(mailbox_id) .custom( ObjectIndexBuilder::new() .with_current(mailbox) .with_changes(new_mailbox), ) .imap_ctx(&arguments.tag, trc::location!())?; self.server .commit_batch(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; trc::event!( Imap(trc::ImapEvent::RenameMailbox), SpanId = self.session_id, AccountId = params.account_id, MailboxName = arguments.new_mailbox_name, MailboxId = mailbox_id, Elapsed = op_start.elapsed() ); Ok(StatusResponse::completed(Command::Rename).with_tag(arguments.tag)) } } ================================================ FILE: crates/imap/src/op/search.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{FromModSeq, ToModSeq}; use crate::{ core::{ImapId, SavedSearch, SelectedMailbox, Session, SessionData}, spawn_op, }; use common::listener::SessionStream; use directory::Permission; use email::cache::{MessageCacheFetch, email::MessageCacheAccess}; use imap_proto::{ Command, StatusResponse, protocol::{ Sequence, search::{self, Arguments, Comparator, Filter, Response, ResultOption}, }, receiver::Request, }; use mail_parser::HeaderName; use nlp::language::Language; use std::{str::FromStr, sync::Arc, time::Instant}; use store::{ query::log::Query, roaring::RoaringBitmap, search::{ EmailSearchField, SearchComparator, SearchFilter, SearchOperator, SearchQuery, SearchValue, }, write::{SearchIndex, now}, }; use tokio::sync::watch; use trc::AddContext; use types::{collection::SyncCollection, id::Id, keyword::Keyword}; use utils::map::vec_map::VecMap; impl Session { pub async fn handle_search( &mut self, request: Request, is_sort: bool, is_uid: bool, ) -> trc::Result<()> { let op_start = Instant::now(); let mut arguments = if !is_sort { // Validate access self.assert_has_permission(Permission::ImapSearch)?; request.parse_search(self.version) } else { // Validate access self.assert_has_permission(Permission::ImapSort)?; request.parse_sort() }?; let (data, mailbox) = self.state.mailbox_state(); // Create channel for results let (results_tx, prev_saved_search) = if arguments.result_options.contains(&ResultOption::Save) { let prev_saved_search = Some(mailbox.get_saved_search().await); let (tx, rx) = watch::channel(Arc::new(Vec::new())); *mailbox.saved_search.lock() = SavedSearch::InFlight { rx }; (tx.into(), prev_saved_search) } else { (None, None) }; spawn_op!(data, { let tag = std::mem::take(&mut arguments.tag); let bytes = match data .search( arguments, mailbox.clone(), results_tx, prev_saved_search.clone(), is_uid, op_start, ) .await { Ok(response) => { let response = response.serialize(&tag); StatusResponse::completed(if !is_sort { Command::Search(is_uid) } else { Command::Sort(is_uid) }) .with_tag(tag) .serialize(response) } Err(err) => { if let Some(prev_saved_search) = prev_saved_search { *mailbox.saved_search.lock() = prev_saved_search .map_or(SavedSearch::None, |s| SavedSearch::Results { items: s }); } return Err(err.id(tag)); } }; data.write_bytes(bytes).await }) } } impl SessionData { pub async fn search( &self, arguments: Arguments, mailbox: Arc, results_tx: Option>>>, prev_saved_search: Option>>>, is_uid: bool, op_start: Instant, ) -> trc::Result { // Run query let is_sort = arguments.sort.is_some(); let (result_set, include_highest_modseq) = self .query( arguments.filter, arguments.sort.unwrap_or_default(), &mailbox, &prev_saved_search, ) .await?; // Obtain modseq let highest_modseq = if include_highest_modseq { self.synchronize_messages(&mailbox) .await? .to_modseq() .into() } else { None }; // Sort and map ids let mut min: Option<(u32, ImapId)> = None; let mut max: Option<(u32, ImapId)> = None; let mut total = 0; let results_len = result_set.len(); let mut saved_results = if results_tx.is_some() { Some(Vec::with_capacity(results_len)) } else { None }; let mut imap_ids = Vec::with_capacity(results_len); mailbox.map_search_results( result_set.into_iter(), is_uid, arguments.result_options.contains(&ResultOption::Min), arguments.result_options.contains(&ResultOption::Max), &mut min, &mut max, &mut total, &mut imap_ids, &mut saved_results, ); if !is_sort { imap_ids.sort_unstable(); } // Save results if let (Some(results_tx), Some(saved_results)) = (results_tx, saved_results) { let saved_results = Arc::new(saved_results); *mailbox.saved_search.lock() = SavedSearch::Results { items: saved_results.clone(), }; results_tx.send(saved_results).ok(); } trc::event!( Imap(if !is_sort { trc::ImapEvent::Search } else { trc::ImapEvent::Sort }), SpanId = self.session_id, AccountId = mailbox.id.account_id, MailboxId = mailbox.id.mailbox_id, Total = total, Elapsed = op_start.elapsed() ); // Build response Ok(Response { is_uid, min: min.map(|(id, _)| id), max: max.map(|(id, _)| id), count: if arguments.result_options.contains(&ResultOption::Count) { Some(total) } else { None }, ids: if arguments.result_options.is_empty() || arguments.result_options.contains(&ResultOption::All) { imap_ids } else { vec![] }, is_sort, is_esearch: arguments.is_esearch, highest_modseq, }) } pub async fn query( &self, imap_filter: Vec, imap_comparator: Vec, mailbox: &SelectedMailbox, prev_saved_search: &Option>>>, ) -> trc::Result<(Vec, bool)> { // Obtain message ids let mut filters = Vec::with_capacity(imap_filter.len() + 1); let cache = self .server .get_cached_messages(mailbox.id.account_id) .await .caused_by(trc::location!())?; let message_ids = RoaringBitmap::from_iter( cache .in_mailbox(mailbox.id.mailbox_id) .map(|m| m.document_id), ); // Convert query let mut include_highest_modseq = false; for filter in imap_filter { match filter { Filter::Sequence(sequence, uid_filter) => { let mut set = RoaringBitmap::new(); if let (Sequence::SavedSearch, Some(prev_saved_search)) = (&sequence, &prev_saved_search) { if let Some(prev_saved_search) = prev_saved_search { let state = mailbox.state.lock(); for imap_id in prev_saved_search.iter() { if let Some(id) = state.uid_to_id.get(&imap_id.uid) { set.insert(*id); } } } else { return Err(trc::ImapEvent::Error .into_err() .details("No saved search found.")); } } else { for id in mailbox.sequence_to_ids(&sequence, uid_filter).await?.keys() { set.insert(*id); } } filters.push(SearchFilter::is_in_set(set)); } Filter::All => { filters.push(SearchFilter::is_in_set(message_ids.clone())); } Filter::Answered => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache .with_keyword(&Keyword::Answered) .map(|m| m.document_id), ))); } Filter::Before(date) => { filters.push(SearchFilter::lt(EmailSearchField::ReceivedAt, date)); } Filter::Deleted => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.with_keyword(&Keyword::Deleted).map(|m| m.document_id), ))); } Filter::Draft => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.with_keyword(&Keyword::Draft).map(|m| m.document_id), ))); } Filter::Flagged => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.with_keyword(&Keyword::Flagged).map(|m| m.document_id), ))); } Filter::Keyword(keyword) => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache .with_keyword(&Keyword::from(keyword)) .map(|m| m.document_id), ))); } Filter::Larger(size) => { filters.push(SearchFilter::gt(EmailSearchField::Size, size)); } Filter::On(date) => { filters.push(SearchFilter::And); filters.push(SearchFilter::ge(EmailSearchField::ReceivedAt, date)); filters.push(SearchFilter::lt(EmailSearchField::ReceivedAt, date + 86400)); filters.push(SearchFilter::End); } Filter::Seen => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.with_keyword(&Keyword::Seen).map(|m| m.document_id), ))); } Filter::SentBefore(date) => { filters.push(SearchFilter::lt(EmailSearchField::SentAt, date)); } Filter::SentOn(date) => { filters.push(SearchFilter::And); filters.push(SearchFilter::ge(EmailSearchField::SentAt, date)); filters.push(SearchFilter::lt(EmailSearchField::SentAt, date + 86400)); filters.push(SearchFilter::End); } Filter::SentSince(date) => { filters.push(SearchFilter::ge(EmailSearchField::SentAt, date)); } Filter::Since(date) => { filters.push(SearchFilter::ge(EmailSearchField::ReceivedAt, date)); } Filter::Smaller(size) => { filters.push(SearchFilter::lt(EmailSearchField::Size, size)); } Filter::Unanswered => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache .without_keyword(&Keyword::Answered) .map(|m| m.document_id), ))); } Filter::Undeleted => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache .without_keyword(&Keyword::Deleted) .map(|m| m.document_id), ))); } Filter::Undraft => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache .without_keyword(&Keyword::Draft) .map(|m| m.document_id), ))); } Filter::Unflagged => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache .without_keyword(&Keyword::Flagged) .map(|m| m.document_id), ))); } Filter::Unkeyword(keyword) => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache .without_keyword(&Keyword::from(keyword)) .map(|m| m.document_id), ))); } Filter::Unseen => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.without_keyword(&Keyword::Seen).map(|m| m.document_id), ))); } Filter::Recent => { //filters.push(SearchFilter::is_in_set(self.get_recent(&mailbox.id))); } Filter::New => { /*filters.push(SearchFilter::And); filters.push(SearchFilter::is_in_set(self.get_recent(&mailbox.id))); filters.push(SearchFilter::Not); filters.push(SearchFilter::is_in_bitmap( EmailSearchField::Keywords, Keyword::Seen, )); filters.push(SearchFilter::End); filters.push(SearchFilter::End);*/ } Filter::Old => { /*filters.push(SearchFilter::Not); filters.push(SearchFilter::is_in_set(self.get_recent(&mailbox.id))); filters.push(SearchFilter::End);*/ } Filter::Older(secs) => { filters.push(SearchFilter::le( EmailSearchField::ReceivedAt, now().saturating_sub(secs as u64), )); } Filter::Younger(secs) => { filters.push(SearchFilter::ge( EmailSearchField::ReceivedAt, now().saturating_sub(secs as u64), )); } Filter::ModSeq((modseq, _)) => { let mut set = RoaringBitmap::new(); for id in self .server .store() .changes( mailbox.id.account_id, SyncCollection::Email.into(), Query::from_modseq(modseq), ) .await? .changes .into_iter() .filter_map(|change| change.try_unwrap_item_id()) { let id = (id & u32::MAX as u64) as u32; if message_ids.contains(id) { set.insert(id); } } filters.push(SearchFilter::is_in_set(set)); include_highest_modseq = true; } Filter::EmailId(id) => { if let Ok(id) = Id::from_str(&id) { filters.push(SearchFilter::is_in_set( RoaringBitmap::from_sorted_iter([id.document_id()]).unwrap(), )); } else { return Err(trc::ImapEvent::Error .into_err() .details(format!("Failed to parse email id '{id}'.",))); } } Filter::ThreadId(id) => { if let Ok(id) = Id::from_str(&id) { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.in_thread(id.document_id()).map(|m| m.document_id), ))); } else { return Err(trc::ImapEvent::Error .into_err() .details(format!("Failed to parse thread id '{id}'.",))); } } Filter::Bcc(text) => { filters.push(SearchFilter::has_text( EmailSearchField::Bcc, text, Language::None, )); } Filter::Body(text) => { filters.push(SearchFilter::has_text_detect( EmailSearchField::Body, text, self.server.core.jmap.default_language, )); } Filter::Cc(text) => { filters.push(SearchFilter::has_text( EmailSearchField::Cc, text, Language::None, )); } Filter::From(text) => { filters.push(SearchFilter::has_text( EmailSearchField::From, text, Language::None, )); } Filter::Header(header, value) => { if let Some(header) = HeaderName::parse(header) { match header { HeaderName::Subject => { filters.push(SearchFilter::has_text_detect( EmailSearchField::Subject, value, self.server.core.jmap.default_language, )); } header @ (HeaderName::From | HeaderName::To | HeaderName::Cc | HeaderName::Bcc) => { filters.push(SearchFilter::has_text( match header { HeaderName::From => EmailSearchField::From, HeaderName::To => EmailSearchField::To, HeaderName::Cc => EmailSearchField::Cc, HeaderName::Bcc => EmailSearchField::Bcc, _ => unreachable!(), }, value, Language::None, )); } header => { let op = if matches!( header, HeaderName::MessageId | HeaderName::InReplyTo | HeaderName::References | HeaderName::ResentMessageId ) || value.is_empty() { SearchOperator::Equal } else { SearchOperator::Contains }; filters.push(SearchFilter::cond( EmailSearchField::Headers, op, SearchValue::KeyValues( VecMap::with_capacity(1) .with_append(header.as_str().to_lowercase(), value), ), )); } } } } Filter::Subject(text) => { filters.push(SearchFilter::has_text_detect( EmailSearchField::Subject, text, self.server.core.jmap.default_language, )); } Filter::Text(text) => { let (text, language) = Language::detect(text, self.server.core.jmap.default_language); filters.push(SearchFilter::Or); filters.push(SearchFilter::has_text( EmailSearchField::From, &text, Language::None, )); filters.push(SearchFilter::has_text( EmailSearchField::To, &text, Language::None, )); filters.push(SearchFilter::has_text( EmailSearchField::Cc, &text, Language::None, )); filters.push(SearchFilter::has_text( EmailSearchField::Bcc, &text, Language::None, )); filters.push(SearchFilter::has_text( EmailSearchField::Subject, &text, language, )); filters.push(SearchFilter::has_text( EmailSearchField::Body, &text, language, )); filters.push(SearchFilter::has_text( EmailSearchField::Attachment, text, language, )); filters.push(SearchFilter::End); } Filter::To(text) => { filters.push(SearchFilter::has_text( EmailSearchField::To, text, Language::None, )); } Filter::And => { filters.push(SearchFilter::And); } Filter::Or => { filters.push(SearchFilter::Or); } Filter::Not => { filters.push(SearchFilter::Not); } Filter::End => { filters.push(SearchFilter::End); } } } // Convert comparators let mut comparators = Vec::with_capacity(imap_comparator.len()); for comparator in imap_comparator { comparators.push(match comparator.sort { search::Sort::Arrival => { SearchComparator::field(EmailSearchField::ReceivedAt, comparator.ascending) } search::Sort::Cc => { return Err(trc::ImapEvent::Error .into_err() .details("Sorting by CC is not supported.")); } search::Sort::Date => { SearchComparator::field(EmailSearchField::SentAt, comparator.ascending) } search::Sort::From | search::Sort::DisplayFrom => { SearchComparator::field(EmailSearchField::From, comparator.ascending) } search::Sort::Size => { SearchComparator::field(EmailSearchField::Size, comparator.ascending) } search::Sort::Subject => { SearchComparator::field(EmailSearchField::Subject, comparator.ascending) } search::Sort::To | search::Sort::DisplayTo => { SearchComparator::field(EmailSearchField::To, comparator.ascending) } }); } // Run query self.server .search_store() .query_account( SearchQuery::new(SearchIndex::Email) .with_filters(filters) .with_comparators(comparators) .with_account_id(mailbox.id.account_id) .with_mask(message_ids), ) .await .map(|res| (res, include_highest_modseq)) .caused_by(trc::location!()) } } impl SelectedMailbox { pub async fn get_saved_search(&self) -> Option>> { let mut rx = match &*self.saved_search.lock() { SavedSearch::InFlight { rx } => rx.clone(), SavedSearch::Results { items } => { return Some(items.clone()); } SavedSearch::None => { return None; } }; rx.changed().await.ok(); let v = rx.borrow(); Some(v.clone()) } #[allow(clippy::too_many_arguments)] pub fn map_search_results( &self, ids: impl Iterator, is_uid: bool, find_min: bool, find_max: bool, min: &mut Option<(u32, ImapId)>, max: &mut Option<(u32, ImapId)>, total: &mut u32, imap_ids: &mut Vec, saved_results: &mut Option>, ) { let state = self.state.lock(); let find_min_or_max = find_min || find_max; for document_id in ids { if let Some((id, imap_id)) = state.map_result_id(document_id, is_uid) { if find_min_or_max { if find_min { if let Some((prev_min, _)) = min { if id < *prev_min { *min = Some((id, imap_id)); } } else { *min = Some((id, imap_id)); } } if find_max { if let Some((prev_max, _)) = max { if id > *prev_max { *max = Some((id, imap_id)); } } else { *max = Some((id, imap_id)); } } } else { imap_ids.push(id); if let Some(r) = saved_results.as_mut() { r.push(imap_id) } } *total += 1; } } if find_min || find_max { for (id, imap_id) in [min, max].into_iter().flatten() { imap_ids.push(*id); if let Some(r) = saved_results.as_mut() { r.push(*imap_id) } } } } } impl SavedSearch { pub async fn unwrap(&self) -> Option>> { match self { SavedSearch::InFlight { rx } => { let mut rx = rx.clone(); rx.changed().await.ok(); let v = rx.borrow(); Some(v.clone()) } SavedSearch::Results { items } => Some(items.clone()), SavedSearch::None => None, } } } ================================================ FILE: crates/imap/src/op/select.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ImapContext, ToModSeq}; use crate::core::{SavedSearch, SelectedMailbox, Session, State}; use common::listener::SessionStream; use directory::Permission; use imap_proto::{ Command, ResponseCode, StatusResponse, protocol::{ ImapResponse, Sequence, fetch, list::ListItem, select::{HighestModSeq, Response}, }, receiver::Request, }; use std::{sync::Arc, time::Instant}; use types::id::Id; impl Session { pub async fn handle_select(&mut self, request: Request) -> trc::Result<()> { // Validate access self.assert_has_permission(if request.command == Command::Select { Permission::ImapSelect } else { Permission::ImapExamine })?; let op_start = Instant::now(); let is_select = request.command == Command::Select; let command = request.command; let arguments = request.parse_select(self.is_utf8)?; let data = self.state.session_data(); // Refresh mailboxes data.synchronize_mailboxes(false) .await .imap_ctx(&arguments.tag, trc::location!())?; if let Some(mailbox) = data.get_mailbox_by_name(&arguments.mailbox_name) { // Try obtaining the mailbox from the cache let state = data .fetch_messages(&mailbox, None) .await .imap_ctx(&arguments.tag, trc::location!())? .unwrap(); // Synchronize messages let closed_previous = self.state.close_mailbox(); let is_condstore = self.is_condstore || arguments.condstore; // Build new state let is_rev2 = self.version.is_rev2(); let is_utf8 = self.is_utf8; let mailbox_state = data.mailbox_state(&mailbox).unwrap(); let total_messages = state.total_messages; let highest_modseq = if is_condstore { HighestModSeq::new(state.modseq.to_modseq()).into() } else { None }; let mailbox = Arc::new(SelectedMailbox { id: mailbox, state: parking_lot::Mutex::new(state), saved_search: parking_lot::Mutex::new(SavedSearch::None), is_select, is_condstore, }); // Validate QRESYNC arguments if let Some(qresync) = arguments.qresync { if !self.is_qresync { return Err(trc::ImapEvent::Error .into_err() .details("QRESYNC is not enabled.") .id(arguments.tag)); } if qresync.uid_validity == mailbox_state.uid_validity as u32 { // Send flags for changed messages data.fetch( fetch::Arguments { tag: "".into(), sequence_set: qresync .known_uids .or_else(|| qresync.seq_match.map(|(_, s)| s)) .unwrap_or(Sequence::Range { start: 1.into(), end: None, }), attributes: vec![fetch::Attribute::Flags], changed_since: qresync.modseq.into(), include_vanished: true, }, mailbox.clone(), true, true, false, Instant::now(), ) .await .imap_ctx(&arguments.tag, trc::location!())?; } } trc::event!( Imap(trc::ImapEvent::Select), SpanId = self.session_id, MailboxName = arguments.mailbox_name.clone(), AccountId = mailbox.id.account_id, MailboxId = mailbox.id.mailbox_id, Total = total_messages, UidNext = mailbox_state.uid_next, UidValidity = mailbox_state.uid_validity, Elapsed = op_start.elapsed() ); // Build response let response = Response { mailbox: ListItem::new(arguments.mailbox_name), total_messages, recent_messages: 0, unseen_seq: 0, uid_validity: mailbox_state.uid_validity as u32, uid_next: mailbox_state.uid_next as u32, closed_previous, is_rev2, is_utf8, highest_modseq, mailbox_id: Id::from_parts(mailbox.id.account_id, mailbox.id.mailbox_id) .to_string(), }; // Update state self.state = State::Selected { data, mailbox }; self.write_bytes( StatusResponse::completed(command) .with_tag(arguments.tag) .with_code(if is_select { ResponseCode::ReadWrite } else { ResponseCode::ReadOnly }) .serialize(response.serialize()), ) .await } else { Err(trc::ImapEvent::Error .into_err() .details("Mailbox does not exist.") .code(ResponseCode::NonExistent) .id(arguments.tag)) } } pub async fn handle_unselect(&mut self, request: Request) -> trc::Result<()> { self.state.close_mailbox(); self.state = State::Authenticated { data: self.state.session_data(), }; self.write_bytes( StatusResponse::completed(Command::Unselect) .with_tag(request.tag) .into_bytes(), ) .await } } ================================================ FILE: crates/imap/src/op/status.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::ToModSeq; use crate::{ core::{Mailbox, Session, SessionData}, op::ImapContext, spawn_op, }; use common::listener::SessionStream; use directory::Permission; use email::cache::{MessageCacheFetch, email::MessageCacheAccess}; use imap_proto::{ Command, ResponseCode, StatusResponse, parser::PushUnique, protocol::status::{Status, StatusItem, StatusItemType}, receiver::Request, }; use std::time::Instant; use trc::AddContext; use types::{id::Id, keyword::Keyword}; impl Session { pub async fn handle_status(&mut self, requests: Vec>) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapStatus)?; let is_utf8 = self.is_utf8; let data = self.state.session_data(); spawn_op!(data, { let mut did_sync = false; for request in requests.into_iter() { match request.parse_status(is_utf8) { Ok(arguments) => { let op_start = Instant::now(); if !did_sync { // Refresh mailboxes data.synchronize_mailboxes(false) .await .imap_ctx(&arguments.tag, trc::location!())?; did_sync = true; } // Fetch status let status = data .status(arguments.mailbox_name, &arguments.items) .await .imap_ctx(&arguments.tag, trc::location!())?; trc::event!( Imap(trc::ImapEvent::Status), SpanId = data.session_id, MailboxName = status.mailbox_name.clone(), Details = arguments .items .iter() .map(|c| trc::Value::from(format!("{c:?}"))) .collect::>(), Elapsed = op_start.elapsed() ); let mut buf = Vec::with_capacity(32); status.serialize(&mut buf, is_utf8); data.write_bytes( StatusResponse::completed(Command::Status) .with_tag(arguments.tag) .serialize(buf), ) .await?; } Err(err) => data.write_error(err).await?, } } Ok(()) }) } } impl SessionData { pub async fn status(&self, mailbox_name: String, items: &[Status]) -> trc::Result { // Get mailbox id let mailbox = if let Some(mailbox) = self.get_mailbox_by_name(&mailbox_name) { mailbox } else { // Some IMAP clients will try to get the status of a mailbox with the NoSelect flag return if mailbox_name == self.server.core.jmap.shared_folder || mailbox_name .split_once('/') .is_some_and(|(base_name, path)| { base_name == self.server.core.jmap.shared_folder && !path.contains('/') }) { Ok(StatusItem { mailbox_name, items: items .iter() .map(|item| { ( *item, match item { Status::Messages | Status::Size | Status::Unseen | Status::Recent | Status::Deleted | Status::HighestModSeq | Status::DeletedStorage => StatusItemType::Number(0), Status::UidNext | Status::UidValidity => { StatusItemType::Number(1) } Status::MailboxId => StatusItemType::String("none".into()), }, ) }) .collect(), }) } else { Err(trc::ImapEvent::Error .into_err() .details("Mailbox does not exist.") .code(ResponseCode::NonExistent)) }; }; // Make sure all requested fields are up to date let mut items_update = Vec::with_capacity(items.len()); let mut items_response = Vec::with_capacity(items.len()); for account in self.mailboxes.lock().iter_mut() { if account.account_id == mailbox.account_id { let mailbox_state = if let Some(mailbox_state) = account.mailbox_state.get(&mailbox.mailbox_id) { mailbox_state } else { continue; }; for item in items { match item { Status::Messages => { items_response.push(( *item, StatusItemType::Number(mailbox_state.total_messages), )); } Status::UidNext => { items_response .push((*item, StatusItemType::Number(mailbox_state.uid_next))); } Status::UidValidity => { items_response .push((*item, StatusItemType::Number(mailbox_state.uid_validity))); } Status::Unseen => { items_response .push((*item, StatusItemType::Number(mailbox_state.total_unseen))); } Status::Deleted => { items_response .push((*item, StatusItemType::Number(mailbox_state.total_deleted))); } Status::DeletedStorage => { if let Some(value) = mailbox_state.total_deleted_storage { items_response.push((*item, StatusItemType::Number(value))); } else { items_update.push_unique(*item); } } Status::Size => { if let Some(value) = mailbox_state.size { items_response.push((*item, StatusItemType::Number(value))); } else { items_update.push_unique(*item); } } Status::HighestModSeq => { items_response.push(( *item, StatusItemType::Number(account.last_change_id.to_modseq()), )); } Status::MailboxId => { items_response.push(( *item, StatusItemType::String( Id::from_parts(mailbox.account_id, mailbox.mailbox_id) .to_string(), ), )); } Status::Recent => { items_response.push((*item, StatusItemType::Number(0))); } } } break; } } if !items_update.is_empty() { // Retrieve latest values let mut values_update = Vec::with_capacity(items_update.len()); let cache = self .server .get_cached_messages(mailbox.account_id) .await .caused_by(trc::location!())?; for item in items_update { let result = match item { Status::DeletedStorage => cache .in_mailbox_with_keyword(mailbox.mailbox_id, &Keyword::Deleted) .map(|x| x.size) .sum::() as u64, Status::Size => cache .in_mailbox(mailbox.mailbox_id) .map(|x| x.size) .sum::() as u64, _ => { unreachable!() } }; items_response.push((item, StatusItemType::Number(result))); values_update.push((item, result)); } // Update cache for account in self.mailboxes.lock().iter_mut() { if account.account_id == mailbox.account_id { let mailbox_state = account .mailbox_state .entry(mailbox.mailbox_id) .or_insert_with(Mailbox::default); for (item, value) in values_update { match item { Status::DeletedStorage => { mailbox_state.total_deleted_storage = value.into() } Status::Size => mailbox_state.size = value.into(), Status::Recent => { items_response .iter_mut() .find(|(i, _)| *i == Status::Recent) .unwrap() .1 = StatusItemType::Number(0); } _ => { unreachable!() } } } break; } } } // Generate response Ok(StatusItem { mailbox_name, items: items_response, }) } } ================================================ FILE: crates/imap/src/op/store.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{FromModSeq, ImapContext}; use crate::{ core::{SelectedMailbox, Session, SessionData}, spawn_op, }; use ahash::AHashSet; use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; use email::{ mailbox::TRASH_ID, message::{ingest::EmailIngest, metadata::MessageData}, }; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, protocol::{ Flag, ImapResponse, fetch::{DataItem, FetchItem}, store::{Arguments, Operation, Response}, }, receiver::Request, }; use std::{sync::Arc, time::Instant}; use store::{ ValueKey, query::log::{Change, Query}, write::{AlignedBytes, Archive, BatchBuilder}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, keyword::Keyword, }; impl Session { pub async fn handle_store( &mut self, request: Request, is_uid: bool, ) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapStore)?; let op_start = Instant::now(); let arguments = request.parse_store()?; let (data, mailbox) = self.state.select_data(); let is_condstore = self.is_condstore || mailbox.is_condstore; spawn_op!(data, { let response = data .store(arguments, mailbox, is_uid, is_condstore, op_start) .await?; data.write_bytes(response).await }) } } impl SessionData { pub async fn store( &self, arguments: Arguments, mailbox: Arc, is_uid: bool, is_condstore: bool, op_start: Instant, ) -> trc::Result> { // Resync messages if needed let account_id = mailbox.id.account_id; self.synchronize_messages(&mailbox) .await .imap_ctx(&arguments.tag, trc::location!())?; // Convert IMAP ids to JMAP ids. let mut ids = mailbox .sequence_to_ids(&arguments.sequence_set, is_uid) .await .imap_ctx(&arguments.tag, trc::location!())?; if ids.is_empty() { return Ok(StatusResponse::completed(Command::Store(is_uid)) .with_tag(arguments.tag) .into_bytes()); } // Verify that the user can modify messages in this mailbox. if !self .check_mailbox_acl( mailbox.id.account_id, mailbox.id.mailbox_id, Acl::ModifyItems, ) .await .imap_ctx(&arguments.tag, trc::location!())? { return Err(trc::ImapEvent::Error .into_err() .details( "You do not have the required permissions to modify messages in this mailbox.", ) .id(arguments.tag) .code(ResponseCode::NoPerm) .caused_by(trc::location!())); } // Filter out unchanged since ids let mut response_code = None; let mut unchanged_failed = false; if let Some(unchanged_since) = arguments.unchanged_since { // Obtain changes since the modseq. let changelog = self .server .store() .changes( account_id, SyncCollection::Email.into(), Query::from_modseq(unchanged_since), ) .await .imap_ctx(&arguments.tag, trc::location!())?; let mut modified = mailbox .sequence_expand_missing(&arguments.sequence_set, is_uid) .await; // Add all IDs that changed in this mailbox for (id, is_delete) in changelog.changes.into_iter().filter_map(|change| { change.item_id().map(|id| { ( (id & u32::MAX as u64) as u32, matches!(change, Change::DeleteItem(_)), ) }) }) { if let Some(imap_id) = ids.remove(&id) { if is_uid { modified.push(imap_id.uid); } else { modified.push(imap_id.seqnum); if is_delete { unchanged_failed = true; } } } } if !modified.is_empty() { modified.sort_unstable(); response_code = ResponseCode::Modified { ids: modified }.into(); } } // Build response let mut response = if !unchanged_failed { StatusResponse::completed(Command::Store(is_uid)) } else { StatusResponse::no("Some of the messages no longer exist.") } .with_tag(arguments.tag); if let Some(response_code) = response_code { response = response.with_code(response_code) } if ids.is_empty() { trc::event!( Imap(trc::ImapEvent::Store), SpanId = self.session_id, AccountId = mailbox.id.account_id, MailboxId = mailbox.id.mailbox_id, Type = format!("{:?}", arguments.operation), Details = arguments .keywords .iter() .map(|c| trc::Value::from(format!("{c:?}"))) .collect::>(), Elapsed = op_start.elapsed() ); return Ok(response.into_bytes()); } let mut items = Response { items: Vec::with_capacity(ids.len()), }; // Process each change let set_keywords = arguments .keywords .iter() .map(|k| Keyword::from(k.clone())) .collect::>(); let mut changed_mailboxes = AHashSet::new(); let mut batch = BatchBuilder::new(); for (id, imap_id) in &ids { // Obtain message data let data_ = if let Some(data) = self .server .store() .get_value::>(ValueKey::archive( account_id, Collection::Email, *id, )) .await .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())? { data } else { continue; }; // Deserialize let data = data_ .to_unarchived::() .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?; let mut new_data = data.inner.to_builder(); // Apply changes let mut seen_changed = false; match arguments.operation { Operation::Set => { seen_changed = set_keywords.contains(&Keyword::Seen) != new_data.has_keyword(&Keyword::Seen); new_data.set_keywords(set_keywords.clone()); } Operation::Add => { for keyword in &set_keywords { if new_data.add_keyword(keyword.clone()) && keyword == &Keyword::Seen { seen_changed = true; } } } Operation::Clear => { for keyword in &set_keywords { if new_data.remove_keyword(keyword) && keyword == &Keyword::Seen { seen_changed = true; } } } } if !new_data.has_keyword_changes(data.inner) { continue; } // Train spam filter let mut train_spam = None; for keyword in new_data.added_keywords(data.inner) { if keyword == &Keyword::Junk { train_spam = Some(true); break; } else if keyword == &Keyword::NotJunk && !data.inner.has_mailbox_id(TRASH_ID) { // Only train as ham if not in Trash (Apple likes to add NotJunk to trashed items, which would be spammy) train_spam = Some(false); break; } } if train_spam.is_none() { for keyword in new_data.removed_keywords(data.inner) { if keyword == &Keyword::Junk { if !data.inner.has_mailbox_id(TRASH_ID) { train_spam = Some(false); } break; } } } // Convert keywords to flags let flags = if !arguments.is_silent { new_data .keywords .iter() .cloned() .map(Flag::from) .collect::>() } else { vec![] }; // Set all current mailboxes as changed if the Seen tag changed if seen_changed { for mailbox_id in new_data.mailboxes.iter() { changed_mailboxes.insert(mailbox_id.mailbox_id); } } // Write changes batch .with_account_id(account_id) .with_collection(Collection::Email) .with_document(*id) .custom( ObjectIndexBuilder::new() .with_current(data) .with_changes(new_data.seal()), ) .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?; // Add spam train task if let Some(learn_spam) = train_spam { self.server .add_account_spam_sample( &mut batch, account_id, *id, learn_spam, self.session_id, ) .await .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?; } // Set commit point batch.commit_point(); // Add item to response if !arguments.is_silent { let mut data_items = vec![DataItem::Flags { flags }]; if is_uid { data_items.push(DataItem::Uid { uid: imap_id.uid }); } items.items.push(FetchItem { id: imap_id.seqnum, items: data_items, }); } else if is_condstore { items.items.push(FetchItem { id: imap_id.seqnum, items: if is_uid { vec![DataItem::Uid { uid: imap_id.uid }] } else { vec![] }, }); } } // Log mailbox changes if !changed_mailboxes.is_empty() { for parent_id in changed_mailboxes { batch.log_container_property_change(SyncCollection::Email, parent_id); } } // Write changes if !batch.is_empty() { match self .server .commit_batch(batch) .await .and_then(|ids| ids.last_change_id(mailbox.id.account_id)) .caused_by(trc::location!()) { Ok(change_id) => { if is_condstore { let modseq = change_id + 1; for item in items.items.iter_mut() { item.items.push(DataItem::ModSeq { modseq }); } } } Err(err) if err.is_assertion_failure() => { items.items.clear(); response.rtype = ResponseType::No; response.message = "Some messages were modified by another process.".into(); } Err(err) => { return Err(err.id(response.tag.unwrap())); } } } trc::event!( Imap(trc::ImapEvent::Store), SpanId = self.session_id, AccountId = mailbox.id.account_id, MailboxId = mailbox.id.mailbox_id, DocumentId = ids .iter() .map(|id| trc::Value::from(*id.0)) .collect::>(), Type = format!("{:?}", arguments.operation), Details = arguments .keywords .iter() .map(|c| trc::Value::from(format!("{c:?}"))) .collect::>(), Elapsed = op_start.elapsed() ); // Send response Ok(response.serialize(items.serialize())) } } ================================================ FILE: crates/imap/src/op/subscribe.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::ImapContext; use crate::{ core::{Session, SessionData}, spawn_op, }; use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; use imap_proto::{Command, ResponseCode, StatusResponse, receiver::Request}; use std::time::Instant; use store::{ ValueKey, write::{AlignedBytes, Archive, BatchBuilder}, }; use types::collection::Collection; impl Session { pub async fn handle_subscribe( &mut self, request: Request, is_subscribe: bool, ) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapSubscribe)?; let op_start = Instant::now(); let arguments = request.parse_subscribe(self.is_utf8)?; let data = self.state.session_data(); spawn_op!(data, { let response = data .subscribe_folder( arguments.tag, arguments.mailbox_name, is_subscribe, op_start, ) .await?; data.write_bytes(response.into_bytes()).await }) } } impl SessionData { pub async fn subscribe_folder( &self, tag: String, mailbox_name: String, subscribe: bool, op_start: Instant, ) -> trc::Result { // Refresh mailboxes self.synchronize_mailboxes(false) .await .imap_ctx(&tag, trc::location!())?; // Validate mailbox let (account_id, mailbox_id) = match self.get_mailbox_by_name(&mailbox_name) { Some(mailbox) => (mailbox.account_id, mailbox.mailbox_id), None => { return Err(trc::ImapEvent::Error .into_err() .details("Mailbox does not exist.") .code(ResponseCode::NonExistent) .id(tag) .caused_by(trc::location!())); } }; // Verify if mailbox is already subscribed/unsubscribed for account in self.mailboxes.lock().iter_mut() { if account.account_id == account_id { if let Some(mailbox) = account.mailbox_state.get(&mailbox_id) && mailbox.is_subscribed == subscribe { return Err(trc::ImapEvent::Error .into_err() .details(if subscribe { "Mailbox is already subscribed." } else { "Mailbox is already unsubscribed." }) .id(tag)); } break; } } // Obtain mailbox let mailbox_ = self .server .store() .get_value::>(ValueKey::archive( account_id, Collection::Mailbox, mailbox_id, )) .await .imap_ctx(&tag, trc::location!())? .ok_or_else(|| { trc::ImapEvent::Error .into_err() .details("Mailbox does not exist.") .code(ResponseCode::NonExistent) .id(tag.clone()) .caused_by(trc::location!()) })?; let mailbox = mailbox_ .to_unarchived::() .imap_ctx(&tag, trc::location!())?; if (subscribe && !mailbox.inner.is_subscribed(self.account_id)) || (!subscribe && mailbox.inner.is_subscribed(self.account_id)) { // Build batch let mut new_mailbox = mailbox.deserialize().imap_ctx(&tag, trc::location!())?; if subscribe { new_mailbox.subscribers.push(self.account_id); } else { new_mailbox.remove_subscriber(self.account_id); } let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Mailbox) .with_document(mailbox_id) .custom( ObjectIndexBuilder::new() .with_current(mailbox) .with_changes(new_mailbox), ) .imap_ctx(&tag, trc::location!())?; self.server .commit_batch(batch) .await .imap_ctx(&tag, trc::location!())?; // Update mailbox cache for account in self.mailboxes.lock().iter_mut() { if account.account_id == account_id { if let Some(mailbox) = account.mailbox_state.get_mut(&mailbox_id) { mailbox.is_subscribed = subscribe; } break; } } } trc::event!( Imap(if subscribe { trc::ImapEvent::Subscribe } else { trc::ImapEvent::Unsubscribe }), SpanId = self.session_id, AccountId = account_id, MailboxId = mailbox_id, MailboxName = mailbox_name, Elapsed = op_start.elapsed() ); Ok(StatusResponse::ok(if subscribe { "Mailbox subscribed." } else { "Mailbox unsubscribed." }) .with_tag(tag)) } } ================================================ FILE: crates/imap/src/op/thread.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ core::{SelectedMailbox, Session, SessionData}, spawn_op, }; use ahash::AHashMap; use common::listener::SessionStream; use directory::Permission; use email::cache::{MessageCacheFetch, email::MessageCacheAccess}; use imap_proto::{ Command, StatusResponse, protocol::{ ImapResponse, thread::{Arguments, Response}, }, receiver::Request, }; use std::{sync::Arc, time::Instant}; use trc::AddContext; impl Session { pub async fn handle_thread( &mut self, request: Request, is_uid: bool, ) -> trc::Result<()> { // Validate access self.assert_has_permission(Permission::ImapThread)?; let op_start = Instant::now(); let command = request.command; let mut arguments = request.parse_thread()?; let (data, mailbox) = self.state.mailbox_state(); spawn_op!(data, { let tag = std::mem::take(&mut arguments.tag); match data.thread(arguments, mailbox, is_uid, op_start).await { Ok(response) => { data.write_bytes( StatusResponse::completed(command) .with_tag(tag) .serialize(response.serialize()), ) .await } Err(err) => Err(err.id(tag)), } }) } } impl SessionData { pub async fn thread( &self, arguments: Arguments, mailbox: Arc, is_uid: bool, op_start: Instant, ) -> trc::Result { // Run query let (result_set, _) = self .query(arguments.filter, vec![], &mailbox, &None) .await?; // Synchronize mailbox if !result_set.is_empty() { self.synchronize_messages(&mailbox) .await .caused_by(trc::location!())?; } else { return Ok(Response { is_uid, threads: vec![], }); } // Lock the cache let cache = self .server .get_cached_messages(mailbox.id.account_id) .await .caused_by(trc::location!())?; // Group messages by thread let mut threads: AHashMap> = AHashMap::new(); let state = mailbox.state.lock(); for document_id in result_set { if let Some(item) = cache.email_by_id(&document_id) && let Some((imap_id, _)) = state.map_result_id(document_id, is_uid) { threads.entry(item.thread_id).or_default().push(imap_id); } } let mut threads = threads .into_iter() .map(|(_, mut messages)| { messages.sort_unstable(); messages }) .collect::>(); threads.sort_unstable(); trc::event!( Imap(trc::ImapEvent::Thread), SpanId = self.session_id, AccountId = mailbox.id.account_id, MailboxId = mailbox.id.mailbox_id, Total = threads.len(), Elapsed = op_start.elapsed() ); // Build response Ok(Response { is_uid, threads }) } } ================================================ FILE: crates/imap-proto/Cargo.toml ================================================ [package] name = "imap_proto" version = "0.15.5" edition = "2024" [dependencies] types = { path = "../types" } utils = { path = "../utils" } mail-parser = { version = "0.11", features = ["full_encoding", "rkyv"] } ahash = { version = "0.8" } chrono = { version = "0.4"} trc = { path = "../trc" } hashify = { version = "0.2" } compact_str = "0.9.0" [dev-dependencies] tokio = { version = "1.47", features = ["full"] } ================================================ FILE: crates/imap-proto/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use protocol::capability::Capability; use std::borrow::Cow; pub mod parser; pub mod protocol; pub mod receiver; pub mod utf7; #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum Command { // Client Commands - Any State Capability, #[default] Noop, Logout, // Client Commands - Not Authenticated State StartTls, Authenticate, Login, // Client Commands - Authenticated State Enable, Select, Examine, Create, Delete, Rename, Subscribe, Unsubscribe, List, Namespace, Status, Append, Idle, // Client Commands - Selected State Close, Unselect, Expunge(bool), Search(bool), Fetch(bool), Store(bool), Copy(bool), Move(bool), // IMAP4rev1 Lsub, Check, // RFC 5256 Sort(bool), Thread(bool), // RFC 4314 SetAcl, DeleteAcl, GetAcl, ListRights, MyRights, // RFC 8437 Unauthenticate, // RFC 2971 Id, // RFC 9208 GetQuota, GetQuotaRoot, } impl Command { pub fn is_uid(&self) -> bool { matches!( self, Command::Fetch(true) | Command::Search(true) | Command::Copy(true) | Command::Move(true) | Command::Store(true) | Command::Expunge(true) | Command::Sort(true) | Command::Thread(true) ) } } #[derive(Debug, Clone, PartialEq, Eq)] pub enum ResponseCode { Alert, AlreadyExists, AppendUid { uid_validity: u32, uids: Vec, }, AuthenticationFailed, AuthorizationFailed, BadCharset, Cannot, Capability { capabilities: Vec, }, ClientBug, Closed, ContactAdmin, CopyUid { uid_validity: u32, src_uids: Vec, dest_uids: Vec, }, Corruption, Expired, ExpungeIssued, HasChildren, InUse, Limit, NonExistent, NoPerm, OverQuota, Parse, PermanentFlags, PrivacyRequired, ReadOnly, ReadWrite, ServerBug, TryCreate, UidNext, UidNotSticky, UidValidity, Unavailable, UnknownCte, // CONDSTORE Modified { ids: Vec, }, HighestModseq { modseq: u64, }, // ObjectID MailboxId { mailbox_id: String, }, // USEATTR UseAttr, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct StatusResponse { pub tag: Option, pub code: Option, pub message: Cow<'static, str>, pub rtype: ResponseType, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum ResponseType { Ok, No, Bad, PreAuth, Bye, } impl ResponseCode { pub fn highest_modseq(modseq: u64) -> Self { ResponseCode::HighestModseq { modseq: if modseq > 0 { modseq + 1 } else { 0 }, } } } impl StatusResponse { pub fn bad(message: impl Into>) -> Self { StatusResponse { tag: None, code: None, message: message.into(), rtype: ResponseType::Bad, } } pub fn parse_error(message: impl Into>) -> Self { StatusResponse { tag: None, code: ResponseCode::Parse.into(), message: message.into(), rtype: ResponseType::Bad, } } pub fn database_failure() -> Self { StatusResponse::no("Database failure.").with_code(ResponseCode::ContactAdmin) } pub fn completed(command: Command) -> Self { StatusResponse::ok(format!("{} completed", command)) } pub fn with_code(mut self, code: ResponseCode) -> Self { self.code = Some(code); self } pub fn with_tag(mut self, tag: impl Into) -> Self { self.tag = Some(tag.into()); self } pub fn no(message: impl Into>) -> Self { StatusResponse { tag: None, code: None, message: message.into(), rtype: ResponseType::No, } } pub fn ok(message: impl Into>) -> Self { StatusResponse { tag: None, code: None, message: message.into(), rtype: ResponseType::Ok, } } pub fn bye(message: impl Into>) -> Self { StatusResponse { tag: None, code: None, message: message.into(), rtype: ResponseType::Bye, } } } ================================================ FILE: crates/imap-proto/src/parser/acl.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::ToCompactString; use crate::{ Command, protocol::acl::{self, ModRights, ModRightsOp, Rights}, receiver::{Request, bad}, utf7::utf7_maybe_decode, }; use super::PushUnique; /* setacl = "SETACL" SP mailbox SP identifier SP mod-rights deleteacl = "DELETEACL" SP mailbox SP identifier getacl = "GETACL" SP mailbox listrights = "LISTRIGHTS" SP mailbox SP identifier myrights = "MYRIGHTS" SP mailbox */ impl Request { pub fn parse_acl(self, is_utf8: bool) -> trc::Result { let (has_identifier, has_mod_rights) = match self.command { Command::SetAcl => (true, true), Command::DeleteAcl | Command::ListRights => (true, false), Command::GetAcl | Command::MyRights => (false, false), _ => unreachable!(), }; let mut tokens = self.tokens.into_iter(); let mailbox_name = utf7_maybe_decode( tokens .next() .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing mailbox name."))? .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, is_utf8, ); let identifier = if has_identifier { tokens .next() .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing identifier."))? .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))? .into() } else { None }; let mod_rights = if has_mod_rights { ModRights::parse( &tokens .next() .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing rights."))? .unwrap_bytes(), ) .map_err(|v| bad(self.tag.to_compact_string(), v))? .into() } else { None }; Ok(acl::Arguments { tag: self.tag, mailbox_name, identifier, mod_rights, }) } } impl ModRights { pub fn parse(value: &[u8]) -> super::Result { let mut op = ModRightsOp::Replace; let mut rights = Vec::with_capacity(value.len()); for (pos, ch) in value.iter().enumerate() { rights.push_unique(match ch { b'l' => Rights::Lookup, b'r' => Rights::Read, b's' => Rights::Seen, b'w' => Rights::Write, b'i' => Rights::Insert, b'p' => Rights::Post, b'k' => Rights::CreateMailbox, b'x' => Rights::DeleteMailbox, b't' => Rights::DeleteMessages, b'e' => Rights::Expunge, b'a' => Rights::Administer, // RFC2086 b'd' => Rights::DeleteMessages, b'c' => Rights::CreateMailbox, b'+' if pos == 0 => { op = ModRightsOp::Add; continue; } b'-' if pos == 0 => { op = ModRightsOp::Remove; continue; } _ => { return Err( format!("Invalid character {:?} in rights.", char::from(*ch)).into(), ); } }) } if !rights.is_empty() { Ok(ModRights { op, rights }) } else { Err("At least one right has to be specified.".into()) } } } #[cfg(test)] mod tests { use crate::{ protocol::acl::{self, ModRights, ModRightsOp, Rights}, receiver::Receiver, }; #[test] fn parse_acl() { let mut receiver = Receiver::new(); for (command, arguments) in [ ( "A003 Setacl INBOX/Drafts Byron lrswikda\r\n", acl::Arguments { tag: "A003".into(), mailbox_name: "INBOX/Drafts".into(), identifier: Some("Byron".into()), mod_rights: ModRights { op: ModRightsOp::Replace, rights: vec![ Rights::Lookup, Rights::Read, Rights::Seen, Rights::Write, Rights::Insert, Rights::CreateMailbox, Rights::DeleteMessages, Rights::Administer, ], } .into(), }, ), ( "A002 SETACL INBOX/Drafts Chris +cda\r\n", acl::Arguments { tag: "A002".into(), mailbox_name: "INBOX/Drafts".into(), identifier: Some("Chris".into()), mod_rights: ModRights { op: ModRightsOp::Add, rights: vec![ Rights::CreateMailbox, Rights::DeleteMessages, Rights::Administer, ], } .into(), }, ), ( "A036 SETACL INBOX/Drafts John -lrswicda\r\n", acl::Arguments { tag: "A036".into(), mailbox_name: "INBOX/Drafts".into(), identifier: Some("John".into()), mod_rights: ModRights { op: ModRightsOp::Remove, rights: vec![ Rights::Lookup, Rights::Read, Rights::Seen, Rights::Write, Rights::Insert, Rights::CreateMailbox, Rights::DeleteMessages, Rights::Administer, ], } .into(), }, ), ( "A001 GETACL INBOX/Drafts\r\n", acl::Arguments { tag: "A001".into(), mailbox_name: "INBOX/Drafts".into(), identifier: None, mod_rights: None, }, ), ] { assert_eq!( receiver .parse(&mut command.as_bytes().iter()) .unwrap() .parse_acl(false) .unwrap(), arguments, "{:?}", command ); } } } ================================================ FILE: crates/imap-proto/src/parser/append.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::ToCompactString; use crate::{ Command, protocol::{ Flag, append::{self, Message}, }, receiver::{Request, Token, bad}, utf7::utf7_maybe_decode, }; use super::parse_datetime; enum State { None, Flags, UTF8, UTF8Data, } impl Request { pub fn parse_append(self, is_utf8: bool) -> trc::Result { match self.tokens.len() { 0 | 1 => Err(self.into_error("Missing arguments.")), _ => { // Obtain mailbox name let mut tokens = self.tokens.into_iter().peekable(); let mailbox_name = utf7_maybe_decode( tokens .next() .unwrap() .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, is_utf8, ); let mut messages = Vec::new(); while tokens.peek().is_some() { // Parse flags let mut message = Message { message: vec![], flags: vec![], received_at: None, }; let mut state = State::None; let mut seen_flags = false; while let Some(token) = tokens.next() { match token { Token::ParenthesisOpen => { state = match state { State::None if !seen_flags => { seen_flags = true; State::Flags } State::UTF8 => State::UTF8Data, _ => { return Err(bad( self.tag.to_compact_string(), "Invalid opening parenthesis found.", )); } }; } Token::ParenthesisClose => match state { State::None | State::UTF8 => { return Err(bad( self.tag.to_compact_string(), "Invalid closing parenthesis found.", )); } State::Flags => { state = State::None; } State::UTF8Data => { break; } }, Token::Argument(value) => match state { State::None => { if value.eq_ignore_ascii_case(b"utf8") { state = State::UTF8; } else if matches!(tokens.peek(), Some(Token::Argument(_))) && value.len() <= 28 && !value.contains(&b'\n') { if let Ok(date_time) = parse_datetime(&value) { message.received_at = Some(date_time); } else { return Err(bad( self.tag.to_compact_string(), "Failed to parse received time.", )); } } else { message.message = value; break; } } State::Flags => { message.flags.push( Flag::parse_imap(value) .map_err(|v| bad(self.tag.to_compact_string(), v))?, ); } State::UTF8 => { return Err(bad( self.tag.to_compact_string(), "Expected parenthesis after UTF8.", )); } State::UTF8Data => { if message.message.is_empty() { message.message = value; } else { return Err(bad( self.tag.to_compact_string(), "Invalid parameter after message literal.", )); } } }, _ => { return Err(bad( self.tag.to_compact_string(), "Invalid arguments.", )); } } } messages.push(message); } Ok(append::Arguments { tag: self.tag, mailbox_name, messages, }) } } } } #[cfg(test)] mod tests { use crate::{ protocol::{ Flag, append::{self, Message}, }, receiver::{Error, Receiver}, }; #[test] fn parse_append() { let mut receiver = Receiver::new(); for (command, arguments) in [ ( "A003 APPEND saved-messages (\\Seen) {1+}\r\na\r\n", append::Arguments { tag: "A003".into(), mailbox_name: "saved-messages".into(), messages: vec![Message { message: vec![b'a'], flags: vec![Flag::Seen], received_at: None, }], }, ), ( "A003 APPEND \"hello world\" (\\Seen \\Draft $MDNSent) {1+}\r\na\r\n", append::Arguments { tag: "A003".into(), mailbox_name: "hello world".into(), messages: vec![Message { message: vec![b'a'], flags: vec![Flag::Seen, Flag::Draft, Flag::MDNSent], received_at: None, }], }, ), ( "A003 APPEND \"hi\" ($Junk) \"7-Feb-1994 22:43:04 -0800\" {1+}\r\na\r\n", append::Arguments { tag: "A003".into(), mailbox_name: "hi".into(), messages: vec![Message { message: vec![b'a'], flags: vec![Flag::Junk], received_at: Some(760689784), }], }, ), ( "A003 APPEND \"hi\" \"20-Nov-2022 23:59:59 +0300\" {1+}\r\na\r\n", append::Arguments { tag: "A003".into(), mailbox_name: "hi".into(), messages: vec![Message { message: vec![b'a'], flags: vec![], received_at: Some(1668977999), }], }, ), ( "A003 APPEND \"hi\" \"20-Nov-2022 23:59:59 +0300\" ~{1+}\r\na\r\n", append::Arguments { tag: "A003".into(), mailbox_name: "hi".into(), messages: vec![Message { message: vec![b'a'], flags: vec![], received_at: Some(1668977999), }], }, ), ( "42 APPEND \"Drafts\" (\\Draft) UTF8 (~{5+}\r\nhello)\r\n", append::Arguments { tag: "42".into(), mailbox_name: "Drafts".into(), messages: vec![Message { message: vec![b'h', b'e', b'l', b'l', b'o'], flags: vec![Flag::Draft], received_at: None, }], }, ), ( "42 APPEND \"Drafts\" (\\Draft) \"20-Nov-2022 23:59:59 +0300\" UTF8 (~{5+}\r\nhello)\r\n", append::Arguments { tag: "42".into(), mailbox_name: "Drafts".into(), messages: vec![Message { message: vec![b'h', b'e', b'l', b'l', b'o'], flags: vec![Flag::Draft], received_at: Some(1668977999), }], }, ), ( "A003 APPEND \"&A8g- \\\"&A9QD1APUA9gD3APcA-+\\\"\" (\\Seen) \"7-Feb-1994 22:43:04 -0800\" {1+}\r\na\r\n", append::Arguments { tag: "A003".into(), mailbox_name: "ψ \"ϔϔϔϘϜϜ+\"".into(), messages: vec![Message { message: vec![b'a'], flags: vec![Flag::Seen], received_at: Some(760689784), }], }, ), ] { assert_eq!( receiver .parse(&mut command.as_bytes().iter()) .expect(command) .parse_append(false) .expect(command), arguments, "{:?}", command ); } // Multiappend for line in [ "A003 APPEND saved-messages (\\Seen) UTF8 ({329}\r\n", "Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST)\r\n", "From: Fred Foobar \r\n", "Subject: afternoon meeting\r\n", "To: mooch@owatagu.example.net\r\n", "Message-Id: \r\n", "MIME-Version: 1.0\r\n", "Content-Type: TEXT/PLAIN; CHARSET=US-ASCII\r\n", "\r\n", "Hello Joe, do you think we can meet at 3:30 tomorrow?\r\n)", " (\\Seen) \"7-Feb-1994 22:43:04 -0800\" {295}\r\n", "Date: Mon, 7 Feb 1994 22:43:04 -0800 (PST)\r\n", "From: Joe Mooch \r\n", "Subject: Re: afternoon meeting\r\n", "To: foobar@blurdybloop.example.com\r\n", "Message-Id: \r\n", "MIME-Version: 1.0\r\n", "Content-Type: TEXT/PLAIN; CHARSET=US-ASCII\r\n\r\n", "3:30 is fine with me.\r\n\r\n", ] { match receiver.parse(&mut line.as_bytes().iter()) { Ok(request) => { assert_eq!( request.parse_append(false).unwrap(), append::Arguments { tag: "A003".into(), mailbox_name: "saved-messages".into(), messages: vec![ Message { message: concat!( "Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST)\r\n", "From: Fred Foobar \r\n", "Subject: afternoon meeting\r\n", "To: mooch@owatagu.example.net\r\n", "Message-Id: \r\n", "MIME-Version: 1.0\r\n", "Content-Type: TEXT/PLAIN; CHARSET=US-ASCII\r\n", "\r\n", "Hello Joe, do you think we can meet at 3:30 tomorrow?\r\n", ) .as_bytes() .to_vec(), flags: vec![Flag::Seen], received_at: None, }, Message { message: concat!( "Date: Mon, 7 Feb 1994 22:43:04 -0800 (PST)\r\n", "From: Joe Mooch \r\n", "Subject: Re: afternoon meeting\r\n", "To: foobar@blurdybloop.example.com\r\n", "Message-Id: \r\n", "MIME-Version: 1.0\r\n", "Content-Type: TEXT/PLAIN; CHARSET=US-ASCII\r\n\r\n", "3:30 is fine with me.\r\n", ) .as_bytes() .to_vec(), flags: vec![Flag::Seen], received_at: Some(760689784), } ], }, ); } Err(err) => match err { Error::NeedsMoreData | Error::NeedsLiteral { .. } => (), Error::Error { response } => panic!("{:?}", response), }, } } } } ================================================ FILE: crates/imap-proto/src/parser/authenticate.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::ToCompactString; use crate::{ Command, protocol::authenticate::{self, Mechanism}, receiver::{Request, bad}, }; impl Request { pub fn parse_authenticate(self) -> trc::Result { if !self.tokens.is_empty() { let mut tokens = self.tokens.into_iter(); Ok(authenticate::Arguments { mechanism: Mechanism::parse(&tokens.next().unwrap().unwrap_bytes()) .map_err(|v| bad(self.tag.to_compact_string(), v))?, params: tokens .filter_map(|token| token.unwrap_string().ok()) .collect(), tag: self.tag, }) } else { Err(self.into_error("Authentication mechanism missing.")) } } } impl Mechanism { pub fn parse(value: &[u8]) -> super::Result { hashify::tiny_map_ignore_case!(value, "PLAIN" => Self::Plain, "CRAM-MD5" => Self::CramMd5, "DIGEST-MD5" => Self::DigestMd5, "SCRAM-SHA-1" => Self::ScramSha1, "SCRAM-SHA-256" => Self::ScramSha256, "APOP" => Self::Apop, "NTLM" => Self::Ntlm, "GSSAPI" => Self::Gssapi, "ANONYMOUS" => Self::Anonymous, "EXTERNAL" => Self::External, "OAUTHBEARER" => Self::OAuthBearer, "XOAUTH2" => Self::XOauth2, ) .ok_or_else(|| { format!( "Unsupported mechanism '{}'.", String::from_utf8_lossy(value) ) .into() }) } } #[cfg(test)] mod tests { use crate::{ protocol::authenticate::{self, Mechanism}, receiver::Receiver, }; #[test] fn parse_authenticate() { let mut receiver = Receiver::new(); for (command, arguments) in [ ( "a002 AUTHENTICATE \"EXTERNAL\" {16+}\r\nfred@example.com\r\n", authenticate::Arguments { tag: "a002".into(), mechanism: Mechanism::External, params: vec!["fred@example.com".into()], }, ), ( "A01 AUTHENTICATE PLAIN\r\n", authenticate::Arguments { tag: "A01".into(), mechanism: Mechanism::Plain, params: vec![], }, ), ] { assert_eq!( receiver .parse(&mut command.as_bytes().iter()) .unwrap() .parse_authenticate() .unwrap(), arguments ); } } } ================================================ FILE: crates/imap-proto/src/parser/copy_move.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::ToCompactString; use crate::{ Command, protocol::copy_move, receiver::{Request, bad}, utf7::utf7_maybe_decode, }; use super::parse_sequence_set; impl Request { pub fn parse_copy_move(self, is_utf8: bool) -> trc::Result { if self.tokens.len() > 1 { let mut tokens = self.tokens.into_iter(); Ok(copy_move::Arguments { sequence_set: parse_sequence_set( &tokens .next() .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing sequence set."))? .unwrap_bytes(), ) .map_err(|v| bad(self.tag.to_compact_string(), v))?, mailbox_name: utf7_maybe_decode( tokens .next() .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing mailbox name."))? .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, is_utf8, ), tag: self.tag, }) } else { Err(self.into_error("Missing arguments.")) } } } #[cfg(test)] mod tests { use crate::{ protocol::{Sequence, copy_move}, receiver::Receiver, }; #[test] fn parse_copy() { let mut receiver = Receiver::new(); assert_eq!( receiver .parse(&mut "A003 COPY 2:4 MEETING\r\n".as_bytes().iter()) .unwrap() .parse_copy_move(false) .unwrap(), copy_move::Arguments { sequence_set: Sequence::Range { start: 2.into(), end: 4.into(), }, mailbox_name: "MEETING".into(), tag: "A003".into(), } ); assert_eq!( receiver .parse(&mut "A003 COPY 2:4 \"You &- Me\"\r\n".as_bytes().iter()) .unwrap() .parse_copy_move(false) .unwrap(), copy_move::Arguments { sequence_set: Sequence::Range { start: 2.into(), end: 4.into(), }, mailbox_name: "You & Me".into(), tag: "A003".into(), } ); } } ================================================ FILE: crates/imap-proto/src/parser/create.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::{CompactString, ToCompactString, format_compact}; use crate::{ Command, protocol::{create, list::Attribute}, receiver::{Request, Token, bad}, utf7::utf7_maybe_decode, }; impl Request { pub fn parse_create(self, is_utf8: bool) -> trc::Result { if !self.tokens.is_empty() { let mut tokens = self.tokens.into_iter(); let mailbox_name = utf7_maybe_decode( tokens .next() .unwrap() .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, is_utf8, ); let mailbox_role = if let Some(Token::ParenthesisOpen) = tokens.next() { match tokens.next() { Some(Token::Argument(param)) if param.eq_ignore_ascii_case(b"USE") => (), _ => { return Err(bad( CompactString::from_string_buffer(self.tag), "Failed to parse, expected 'USE'.", )); } } if tokens .next() .is_none_or(|token| !token.is_parenthesis_open()) { return Err(bad( CompactString::from_string_buffer(self.tag), "Expected '(' after 'USE'.", )); } match tokens.next() { Some(Token::Argument(value)) => { let r = hashify::tiny_map_ignore_case!(value.as_slice(), "\\Archive" => Some(Attribute::Archive), "\\Drafts" => Some(Attribute::Drafts), "\\Junk" => Some(Attribute::Junk), "\\Sent" => Some(Attribute::Sent), "\\Trash" => Some(Attribute::Trash), "\\Important" => Some(Attribute::Important), "\\Memos" => Some(Attribute::Memos), "\\Scheduled" => Some(Attribute::Scheduled), "\\Snoozed" => Some(Attribute::Snoozed), "\\All" => None, ); match r { Some(Some(tag)) => Some(tag), Some(None) => { return Err(bad( CompactString::from_string_buffer(self.tag), "A mailbox with the \"\\All\" attribute already exists.", )); } None => { return Err(bad( CompactString::from_string_buffer(self.tag), format_compact!( "Special use attribute {:?} is not supported.", String::from_utf8_lossy(&value) ), )); } } } _ => { return Err(bad( CompactString::from_string_buffer(self.tag), "Invalid SPECIAL-USE attribute.", )); } } } else { None }; Ok(create::Arguments { mailbox_name, mailbox_role, tag: self.tag, }) } else { Err(self.into_error("Missing arguments.")) } } } #[cfg(test)] mod tests { use crate::{ protocol::{create, list::Attribute}, receiver::Receiver, }; #[test] fn parse_create() { let mut receiver = Receiver::new(); for (command, arguments) in [ ( "A142 CREATE 12345\r\n", create::Arguments { tag: "A142".into(), mailbox_name: "12345".into(), mailbox_role: None, }, ), ( "A142 CREATE \"my funky mailbox\"\r\n", create::Arguments { tag: "A142".into(), mailbox_name: "my funky mailbox".into(), mailbox_role: None, }, ), ( "t1 CREATE \"Important Messages\" (USE (\\Important))\r\n", create::Arguments { tag: "t1".into(), mailbox_name: "Important Messages".into(), mailbox_role: Some(Attribute::Important), }, ), ( "A142 CREATE \"Test-ąęć-Test\"\r\n", create::Arguments { tag: "A142".into(), mailbox_name: "Test-ąęć-Test".into(), mailbox_role: None, }, ), ] { assert_eq!( receiver .parse(&mut command.as_bytes().iter()) .unwrap() .parse_create(true) .unwrap(), arguments ); } } } ================================================ FILE: crates/imap-proto/src/parser/delete.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::ToCompactString; use crate::{ Command, protocol::delete, receiver::{Request, bad}, utf7::utf7_maybe_decode, }; impl Request { pub fn parse_delete(self, is_utf8: bool) -> trc::Result { match self.tokens.len() { 1 => Ok(delete::Arguments { mailbox_name: utf7_maybe_decode( self.tokens .into_iter() .next() .unwrap() .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, is_utf8, ), tag: self.tag, }), 0 => Err(self.into_error("Missing mailbox name.")), _ => Err(self.into_error("Too many arguments.")), } } } #[cfg(test)] mod tests { use crate::{protocol::delete, receiver::Receiver}; #[test] fn parse_delete() { let mut receiver = Receiver::new(); for (command, arguments) in [ ( "A142 DELETE INBOX\r\n", delete::Arguments { mailbox_name: "INBOX".into(), tag: "A142".into(), }, ), ( "A142 DELETE \"my funky mailbox\"\r\n", delete::Arguments { mailbox_name: "my funky mailbox".into(), tag: "A142".into(), }, ), ] { assert_eq!( receiver .parse(&mut command.as_bytes().iter()) .unwrap() .parse_delete(true) .unwrap(), arguments ); } } } ================================================ FILE: crates/imap-proto/src/parser/enable.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::ToCompactString; use crate::{ Command, protocol::{capability::Capability, enable}, receiver::{Request, bad}, }; impl Request { pub fn parse_enable(self) -> trc::Result { let len = self.tokens.len(); if len > 0 { let mut capabilities = Vec::with_capacity(len); for capability in self.tokens { capabilities.push( Capability::parse(&capability.unwrap_bytes()) .map_err(|v| bad(self.tag.to_compact_string(), v))?, ); } Ok(enable::Arguments { tag: self.tag, capabilities, }) } else { Err(self.into_error("Missing arguments.")) } } } impl Capability { pub fn parse(value: &[u8]) -> super::Result { hashify::tiny_map_ignore_case!(value, "IMAP4rev2" => Self::IMAP4rev2, "STARTTLS" => Self::StartTLS, "LOGINDISABLED" => Self::LoginDisabled, "CONDSTORE" => Self::CondStore, "QRESYNC" => Self::QResync, "UTF8=ACCEPT" => Self::Utf8Accept, ) .ok_or_else(|| { format!( "Unsupported capability '{}'.", String::from_utf8_lossy(value) ) .into() }) } } #[cfg(test)] mod tests { use crate::{ protocol::{capability::Capability, enable}, receiver::Receiver, }; #[test] fn parse_enable() { let mut receiver = Receiver::new(); assert_eq!( receiver .parse(&mut "t2 ENABLE IMAP4rev2 CONDSTORE\r\n".as_bytes().iter()) .unwrap() .parse_enable() .unwrap(), enable::Arguments { tag: "t2".into(), capabilities: vec![Capability::IMAP4rev2, Capability::CondStore], } ); } } ================================================ FILE: crates/imap-proto/src/parser/fetch.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::borrow::Cow; use std::iter::Peekable; use std::vec::IntoIter; use compact_str::{CompactString, ToCompactString, format_compact}; use crate::{ Command, protocol::fetch::{self, Attribute, Section}, receiver::{Request, Token, bad}, }; use super::{PushUnique, parse_number, parse_sequence_set}; impl Request { #[allow(clippy::while_let_on_iterator)] pub fn parse_fetch(self) -> trc::Result { if self.tokens.len() < 2 { return Err(self.into_error("Missing parameters.")); } let mut tokens = self.tokens.into_iter().peekable(); let mut attributes = Vec::new(); let sequence_set = parse_sequence_set( &tokens .next() .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing sequence set."))? .unwrap_bytes(), ) .map_err(|v| bad(self.tag.to_compact_string(), v))?; let mut in_parentheses = false; while let Some(token) = tokens.next() { match token { Token::Argument(value) => { hashify::fnc_map_ignore_case!(value.as_slice(), "ALL" => { attributes = vec![ Attribute::Flags, Attribute::InternalDate, Attribute::Rfc822Size, Attribute::Envelope, ]; break; }, "FULL" => { attributes = vec![ Attribute::Flags, Attribute::InternalDate, Attribute::Rfc822Size, Attribute::Envelope, Attribute::Body, ]; break; }, "FAST" => { attributes = vec![ Attribute::Flags, Attribute::InternalDate, Attribute::Rfc822Size, ]; break; }, "ENVELOPE" => { attributes.push_unique(Attribute::Envelope); }, "FLAGS" => { attributes.push_unique(Attribute::Flags); }, "INTERNALDATE" => { attributes.push_unique(Attribute::InternalDate); }, "BODYSTRUCTURE" => { attributes.push_unique(Attribute::BodyStructure); }, "UID" => { attributes.push_unique(Attribute::Uid); }, "RFC822" => { attributes.push_unique( if tokens.peek().is_some_and(|token| token.is_dot()) { tokens.next(); let rfc822 = tokens .next() .ok_or_else(|| { bad(self.tag.to_compact_string(), "Missing RFC822 parameter.") })? .unwrap_bytes(); if rfc822.eq_ignore_ascii_case(b"HEADER") { Attribute::Rfc822Header } else if rfc822.eq_ignore_ascii_case(b"SIZE") { Attribute::Rfc822Size } else if rfc822.eq_ignore_ascii_case(b"TEXT") { Attribute::Rfc822Text } else { return Err(bad( CompactString::from_string_buffer(self.tag), format_compact!( "Invalid RFC822 parameter {:?}.", String::from_utf8_lossy(&rfc822) ), )); } } else { Attribute::Rfc822 }, ); }, "BODY" => { let is_peek = match tokens.peek() { Some(Token::BracketOpen) => { tokens.next(); false } Some(Token::Dot) => { tokens.next(); if tokens .next() .is_none_or( |token| !token.eq_ignore_ascii_case(b"PEEK")) { return Err(bad( self.tag.to_compact_string(), "Expected 'PEEK' after '.'.", )); } if tokens.next().is_none_or( |token| !token.is_bracket_open()) { return Err(bad( self.tag.to_compact_string(), "Expected '[' after 'BODY.PEEK'", )); } true } _ => { attributes.push_unique(Attribute::Body); if !in_parentheses { break; } else { continue; } } }; // Parse section-spect let mut sections = Vec::new(); while let Some(token) = tokens.next() { match token { Token::BracketClose => break, Token::Argument(value) => { let section = if value.eq_ignore_ascii_case(b"HEADER") { if let Some(Token::Dot) = tokens.peek() { tokens.next(); if tokens.next().is_none_or( |token| { !token.eq_ignore_ascii_case(b"FIELDS") }) { return Err(bad( CompactString::from_string_buffer(self.tag), "Expected 'FIELDS' after 'HEADER.'.", )); } let is_not = if let Some(Token::Dot) = tokens.peek() { tokens.next(); if tokens.next().is_none_or( |token| { !token.eq_ignore_ascii_case(b"NOT") }) { return Err(bad( CompactString::from_string_buffer(self.tag), "Expected 'NOT' after 'HEADER.FIELDS.'.", )); } true } else { false }; if tokens .next() .is_none_or( |token| !token.is_parenthesis_open()) { return Err(bad( CompactString::from_string_buffer(self.tag), "Expected '(' after 'HEADER.FIELDS'.", )); } let mut fields = Vec::new(); while let Some(token) = tokens.next() { match token { Token::ParenthesisClose => break, Token::Argument(value) => { fields.push(String::from_utf8(value).map_err( |_| bad(self.tag.to_compact_string(),"Invalid UTF-8 in header field name."), )?); } _ => { return Err(bad( CompactString::from_string_buffer(self.tag), "Expected field name.", )) } } } Section::HeaderFields { not: is_not, fields, } } else { Section::Header } } else if value.eq_ignore_ascii_case(b"TEXT") { Section::Text } else if value.eq_ignore_ascii_case(b"MIME") { Section::Mime } else { Section::Part { num: parse_number::(&value) .map_err(|v| bad(self.tag.to_compact_string(), v))?, } }; sections.push(section); } Token::Dot => (), _ => { return Err(bad( CompactString::from_string_buffer(self.tag), format_compact!( "Invalid token {:?} found in section-spect.", token ), )) } } } attributes.push_unique(Attribute::BodySection { peek: is_peek, sections, partial: parse_partial(&mut tokens) .map_err(|v| bad(self.tag.to_compact_string(), v))?, }); }, "BINARY" => { let (is_peek, is_size) = if let Some(Token::Dot) = tokens.peek() { tokens.next(); let param = tokens .next() .ok_or({ bad(self.tag.to_compact_string(),"Missing parameter after 'BINARY.'.") })? .unwrap_bytes(); if param.eq_ignore_ascii_case(b"PEEK") { (true, false) } else if param.eq_ignore_ascii_case(b"SIZE") { (false, true) } else { return Err(bad( CompactString::from_string_buffer(self.tag), "Expected 'PEEK' or 'SIZE' after 'BINARY.'.", )); } } else { (false, false) }; // Parse section-part if tokens.next().is_none_or( |token| !token.is_bracket_open()) { return Err(bad(self.tag.to_compact_string(), "Expected '[' after 'BINARY'.")); } let mut sections = Vec::new(); while let Some(token) = tokens.next() { match token { Token::Argument(value) => { sections.push( parse_number::(&value) .map_err(|v| bad(self.tag.to_compact_string(), v))?, ); } Token::Dot => (), Token::BracketClose => break, _ => { return Err(bad( CompactString::from_string_buffer(self.tag), format_compact!( "Expected part section integer, got {:?}.", token.to_string() ), )) } } } attributes.push_unique(if !is_size { Attribute::Binary { peek: is_peek, sections, partial: parse_partial(&mut tokens) .map_err(|v| bad(self.tag.to_compact_string(), v))?, } } else { Attribute::BinarySize { sections } }); }, "PREVIEW" => { attributes.push_unique(Attribute::Preview { lazy: if let Some(Token::ParenthesisOpen) = tokens.peek() { tokens.next(); let mut is_lazy = false; while let Some(token) = tokens.next() { match token { Token::ParenthesisClose => break, Token::Argument(value) => { if value.eq_ignore_ascii_case(b"LAZY") { is_lazy = true; } } _ => (), } } is_lazy } else { false }, }); }, "MODSEQ" => { attributes.push_unique(Attribute::ModSeq); }, "EMAILID" => { attributes.push_unique(Attribute::EmailId); }, "THREADID" => { attributes.push_unique(Attribute::ThreadId); }, _ => { return Err(bad( CompactString::from_string_buffer(self.tag), format_compact!("Invalid attribute {:?}", String::from_utf8_lossy(&value)), )); } ); if !in_parentheses { break; } } Token::ParenthesisOpen => { if !in_parentheses { in_parentheses = true; } else { return Err(bad( self.tag.to_compact_string(), "Unexpected parenthesis open.", )); } } Token::ParenthesisClose => { if in_parentheses { break; } else { return Err(bad( self.tag.to_compact_string(), "Unexpected parenthesis close.", )); } } _ => { return Err(bad( CompactString::from_string_buffer(self.tag), format_compact!("Invalid fetch argument {:?}.", token.to_string()), )); } } } // CONDSTORE parameters let mut changed_since = None; let mut include_vanished = false; if let Some(Token::ParenthesisOpen) = tokens.peek() { tokens.next(); while let Some(token) = tokens.next() { match token { Token::Argument(param) if param.eq_ignore_ascii_case(b"CHANGEDSINCE") => { changed_since = parse_number::( &tokens .next() .ok_or_else(|| { bad( self.tag.to_compact_string(), "Missing CHANGEDSINCE parameter.", ) })? .unwrap_bytes(), ) .map_err(|v| bad(self.tag.to_compact_string(), v))? .into(); } Token::Argument(param) if param.eq_ignore_ascii_case(b"VANISHED") => { include_vanished = true; } Token::ParenthesisClose => { break; } _ => { return Err(bad( self.tag.to_compact_string(), format_compact!("Unsupported parameter '{}'.", token), )); } } } } if !attributes.is_empty() { Ok(fetch::Arguments { tag: self.tag, sequence_set, attributes, changed_since, include_vanished, }) } else { Err(bad( CompactString::from_string_buffer(self.tag), "No data items to fetch specified.", )) } } } pub fn parse_partial(tokens: &mut Peekable>) -> super::Result> { if tokens.peek().is_none_or(|token| !token.is_lt()) { return Ok(None); } tokens.next(); let start = parse_number::( &tokens .next() .ok_or_else(|| Cow::from("Missing partial start."))? .unwrap_bytes(), )?; if tokens.next().is_none_or(|token| !token.is_dot()) { return Err("Expected '.' after partial start.".into()); } let end = parse_number::( &tokens .next() .ok_or_else(|| Cow::from("Missing partial end."))? .unwrap_bytes(), )?; if end == 0 { return Err("Invalid partial range.".into()); } if tokens.next().is_none_or(|token| !token.is_gt()) { return Err("Expected '>' after range.".into()); } Ok(Some((start, end))) } /* fetch = "FETCH" SP sequence-set SP ( "ALL" / "FULL" / "FAST" / fetch-att / "(" fetch-att *(SP fetch-att) ")") fetch-att = "ENVELOPE" / "FLAGS" / "INTERNALDATE" / "RFC822" [".HEADER" / ".SIZE" / ".TEXT"] / "BODY" ["STRUCTURE"] / "UID" / "BODY" section [partial] / "BODY.PEEK" section [partial] / "BINARY" [".PEEK"] section-binary [partial] / "BINARY.SIZE" section-binary partial = "<" number64 "." nz-number64 ">" ; Partial FETCH request. 0-based offset of ; the first octet, followed by the number of ; octets in the fragment. section = "[" [section-spec] "]" section-binary = "[" [section-part] "]" section-msgtext = "HEADER" / "HEADER.FIELDS" [".NOT"] SP header-list / "TEXT" ; top-level or MESSAGE/RFC822 or ; MESSAGE/GLOBAL part section-part = nz-number *("." nz-number) ; body part reference. ; Allows for accessing nested body parts. section-spec = section-msgtext / (section-part ["." section-text]) section-text = section-msgtext / "MIME" ; text other than actual body part (headers, ; etc.) */ #[cfg(test)] mod tests { use crate::{ protocol::{ Sequence, fetch::{self, Attribute, Section}, }, receiver::Receiver, }; #[test] fn parse_fetch() { let mut receiver = Receiver::new(); for (command, arguments) in [ ( "A654 FETCH 2:4 (FLAGS BODY[HEADER.FIELDS (DATE FROM)])\r\n", fetch::Arguments { tag: "A654".into(), sequence_set: Sequence::range(2.into(), 4.into()), attributes: vec![ Attribute::Flags, Attribute::BodySection { peek: false, sections: vec![Section::HeaderFields { not: false, fields: vec!["DATE".into(), "FROM".into()], }], partial: None, }, ], changed_since: None, include_vanished: false, }, ), ( "A001 FETCH 1 BODY[]\r\n", fetch::Arguments { tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![Attribute::BodySection { peek: false, sections: vec![], partial: None, }], changed_since: None, include_vanished: false, }, ), ( "A001 FETCH 1 (BODY[HEADER])\r\n", fetch::Arguments { tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![Attribute::BodySection { peek: false, sections: vec![Section::Header], partial: None, }], changed_since: None, include_vanished: false, }, ), ( "A001 FETCH 1 (BODY.PEEK[HEADER.FIELDS (X-MAILER)] PREVIEW(LAZY))\r\n", fetch::Arguments { tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![ Attribute::BodySection { peek: true, sections: vec![Section::HeaderFields { not: false, fields: vec!["X-MAILER".into()], }], partial: None, }, Attribute::Preview { lazy: true }, ], changed_since: None, include_vanished: false, }, ), ( "A001 FETCH 1 (BODY[HEADER.FIELDS.NOT (FROM TO SUBJECT)])\r\n", fetch::Arguments { tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![Attribute::BodySection { peek: false, sections: vec![Section::HeaderFields { not: true, fields: vec!["FROM".into(), "TO".into(), "SUBJECT".into()], }], partial: None, }], changed_since: None, include_vanished: false, }, ), ( "A001 FETCH 1 (BODY[MIME] BODY[TEXT] PREVIEW)\r\n", fetch::Arguments { tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![ Attribute::BodySection { peek: false, sections: vec![Section::Mime], partial: None, }, Attribute::BodySection { peek: false, sections: vec![Section::Text], partial: None, }, Attribute::Preview { lazy: false }, ], changed_since: None, include_vanished: false, }, ), ( "A001 FETCH 1 (BODYSTRUCTURE ENVELOPE FLAGS INTERNALDATE UID)\r\n", fetch::Arguments { tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![ Attribute::BodyStructure, Attribute::Envelope, Attribute::Flags, Attribute::InternalDate, Attribute::Uid, ], changed_since: None, include_vanished: false, }, ), ( "A001 FETCH 1 (RFC822 RFC822.HEADER RFC822.SIZE RFC822.TEXT)\r\n", fetch::Arguments { tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![ Attribute::Rfc822, Attribute::Rfc822Header, Attribute::Rfc822Size, Attribute::Rfc822Text, ], changed_since: None, include_vanished: false, }, ), ( concat!( "A001 FETCH 1 (", "BODY[4.2.HEADER]<0.20> ", "BODY.PEEK[3.2.2.2] ", "BODY[4.2.TEXT]<4.100> ", "BINARY[1.2.3] ", "BINARY.PEEK[4] ", "BINARY[6.5.4]<100.200> ", "BINARY.PEEK[7]<9.88> ", "BINARY.SIZE[9.1]", ")\r\n" ), fetch::Arguments { tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![ Attribute::BodySection { peek: false, sections: vec![ Section::Part { num: 4 }, Section::Part { num: 2 }, Section::Header, ], partial: Some((0, 20)), }, Attribute::BodySection { peek: true, sections: vec![ Section::Part { num: 3 }, Section::Part { num: 2 }, Section::Part { num: 2 }, Section::Part { num: 2 }, ], partial: None, }, Attribute::BodySection { peek: false, sections: vec![ Section::Part { num: 4 }, Section::Part { num: 2 }, Section::Text, ], partial: Some((4, 100)), }, Attribute::Binary { peek: false, sections: vec![1, 2, 3], partial: None, }, Attribute::Binary { peek: true, sections: vec![4], partial: None, }, Attribute::Binary { peek: false, sections: vec![6, 5, 4], partial: Some((100, 200)), }, Attribute::Binary { peek: true, sections: vec![7], partial: Some((9, 88)), }, Attribute::BinarySize { sections: vec![9, 1], }, ], changed_since: None, include_vanished: false, }, ), ( "A001 FETCH 1 ALL\r\n", fetch::Arguments { tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![ Attribute::Flags, Attribute::InternalDate, Attribute::Rfc822Size, Attribute::Envelope, ], changed_since: None, include_vanished: false, }, ), ( "A001 FETCH 1 FULL\r\n", fetch::Arguments { tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![ Attribute::Flags, Attribute::InternalDate, Attribute::Rfc822Size, Attribute::Envelope, Attribute::Body, ], changed_since: None, include_vanished: false, }, ), ( "A001 FETCH 1 FAST\r\n", fetch::Arguments { tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![ Attribute::Flags, Attribute::InternalDate, Attribute::Rfc822Size, ], changed_since: None, include_vanished: false, }, ), ( "s100 UID FETCH 1:* (FLAGS MODSEQ) (CHANGEDSINCE 12345 VANISHED)\r\n", fetch::Arguments { tag: "s100".into(), sequence_set: Sequence::range(1.into(), None), attributes: vec![Attribute::Flags, Attribute::ModSeq], changed_since: 12345.into(), include_vanished: true, }, ), ( "9 UID FETCH 1:* UID (VANISHED CHANGEDSINCE 1)\r\n", fetch::Arguments { tag: "9".into(), sequence_set: Sequence::range(1.into(), None), attributes: vec![Attribute::Uid], changed_since: 1.into(), include_vanished: true, }, ), ] { assert_eq!( receiver .parse(&mut command.as_bytes().iter()) .unwrap() .parse_fetch() .expect(command), arguments, "{}", command ); } } } ================================================ FILE: crates/imap-proto/src/parser/list.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::{CompactString, ToCompactString}; use crate::{ Command, protocol::{ list::{self, ReturnOption, SelectionOption}, status::Status, }, receiver::{Request, Token, bad}, utf7::utf7_maybe_decode, }; impl Request { #[allow(clippy::while_let_on_iterator)] pub fn parse_list(self, is_utf8: bool) -> trc::Result { match self.tokens.len() { 0 | 1 => Err(self.into_error("Missing arguments.")), 2 => { let mut tokens = self.tokens.into_iter(); Ok(list::Arguments::Basic { reference_name: tokens .next() .unwrap() .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, mailbox_name: utf7_maybe_decode( tokens .next() .unwrap() .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, is_utf8, ), tag: self.tag, }) } _ => { let mut tokens = self.tokens.into_iter(); let mut selection_options = Vec::new(); let mut return_options = Vec::new(); let mut mailbox_name = Vec::new(); let reference_name = match tokens.next().unwrap() { Token::ParenthesisOpen => { while let Some(token) = tokens.next() { match token { Token::ParenthesisClose => break, Token::Argument(value) => { selection_options.push( SelectionOption::parse(&value) .map_err(|v| bad(self.tag.to_compact_string(), v))?, ); } _ => { return Err(bad( self.tag.to_compact_string(), "Invalid selection option argument.", )); } } } tokens .next() .ok_or_else(|| { bad(self.tag.to_compact_string(), "Missing reference name.") })? .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))? } token => token .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, }; match tokens .next() .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing mailbox name."))? { Token::ParenthesisOpen => { while let Some(token) = tokens.next() { match token { Token::ParenthesisClose => break, token => { mailbox_name.push( token .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, ); } } } } token => { mailbox_name.push(utf7_maybe_decode( token .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, is_utf8, )); } } if tokens .next() .is_some_and(|token| token.eq_ignore_ascii_case(b"return")) { if tokens .next() .is_none_or(|token| !token.is_parenthesis_open()) { return Err(bad( self.tag.to_compact_string(), "Invalid return option, expected parenthesis.", )); } while let Some(token) = tokens.next() { match token { Token::ParenthesisClose => break, Token::Argument(value) => { let mut return_option = ReturnOption::parse(&value) .map_err(|v| bad(self.tag.to_compact_string(), v))?; if let ReturnOption::Status(status) = &mut return_option { if tokens .next() .is_none_or(|token| !token.is_parenthesis_open()) { return Err(bad( CompactString::from_string_buffer(self.tag), "Invalid return option, expected parenthesis after STATUS.", )); } while let Some(token) = tokens.next() { match token { Token::ParenthesisClose => break, Token::Argument(value) => { status.push(Status::parse(&value).map_err( |v| bad(self.tag.to_compact_string(), v), )?); } _ => { return Err(bad( CompactString::from_string_buffer(self.tag), "Invalid status return option argument.", )); } } } } return_options.push(return_option); } _ => { return Err(bad( self.tag.to_compact_string(), "Invalid return option argument.", )); } } } } Ok(list::Arguments::Extended { tag: self.tag, reference_name, mailbox_name, selection_options, return_options, }) } } } } impl SelectionOption { pub fn parse(value: &[u8]) -> super::Result { hashify::tiny_map_ignore_case!(value, "SUBSCRIBED" => Self::Subscribed, "REMOTE" => Self::Remote, "RECURSIVEMATCH" => Self::RecursiveMatch, "SPECIAL-USE" => Self::SpecialUse, ) .ok_or_else(|| { format!( "Unsupported selection option '{}'.", String::from_utf8_lossy(value) ) .into() }) } } impl ReturnOption { pub fn parse(value: &[u8]) -> super::Result { hashify::tiny_map_ignore_case!(value, "SUBSCRIBED" => Self::Subscribed, "CHILDREN" => Self::Children, "STATUS" => Self::Status(Vec::with_capacity(2)), "SPECIAL-USE" => Self::SpecialUse, ) .ok_or_else(|| format!("Invalid return option {:?}", String::from_utf8_lossy(value)).into()) } } #[cfg(test)] mod tests { use crate::{ protocol::{ list::{self, ReturnOption, SelectionOption}, status::Status, }, receiver::Receiver, }; #[test] fn parse_list() { let mut receiver = Receiver::new(); for (command, arguments) in [ ( "A682 LIST \"\" *\r\n", list::Arguments::Basic { tag: "A682".into(), reference_name: "".into(), mailbox_name: "*".into(), }, ), ( "A02 LIST (SUBSCRIBED) \"\" \"*\"\r\n", list::Arguments::Extended { tag: "A02".into(), reference_name: "".into(), mailbox_name: vec!["*".into()], selection_options: vec![SelectionOption::Subscribed], return_options: vec![], }, ), ( "A03 LIST () \"\" \"%\" RETURN (CHILDREN)\r\n", list::Arguments::Extended { tag: "A03".into(), reference_name: "".into(), mailbox_name: vec!["%".into()], selection_options: vec![], return_options: vec![ReturnOption::Children], }, ), ( "A04 LIST (REMOTE) \"\" \"%\" RETURN (CHILDREN)\r\n", list::Arguments::Extended { tag: "A04".into(), reference_name: "".into(), mailbox_name: vec!["%".into()], selection_options: vec![SelectionOption::Remote], return_options: vec![ReturnOption::Children], }, ), ( "A05 LIST (REMOTE SUBSCRIBED) \"\" \"*\"\r\n", list::Arguments::Extended { tag: "A05".into(), reference_name: "".into(), mailbox_name: vec!["*".into()], selection_options: vec![SelectionOption::Remote, SelectionOption::Subscribed], return_options: vec![], }, ), ( "A06 LIST (REMOTE) \"\" \"*\" RETURN (SUBSCRIBED)\r\n", list::Arguments::Extended { tag: "A06".into(), reference_name: "".into(), mailbox_name: vec!["*".into()], selection_options: vec![SelectionOption::Remote], return_options: vec![ReturnOption::Subscribed], }, ), ( "C04 LIST (SUBSCRIBED RECURSIVEMATCH) \"\" \"%\"\r\n", list::Arguments::Extended { tag: "C04".into(), reference_name: "".into(), mailbox_name: vec!["%".into()], selection_options: vec![ SelectionOption::Subscribed, SelectionOption::RecursiveMatch, ], return_options: vec![], }, ), ( "C04 LIST (SUBSCRIBED RECURSIVEMATCH) \"\" \"%\" RETURN (CHILDREN)\r\n", list::Arguments::Extended { tag: "C04".into(), reference_name: "".into(), mailbox_name: vec!["%".into()], selection_options: vec![ SelectionOption::Subscribed, SelectionOption::RecursiveMatch, ], return_options: vec![ReturnOption::Children], }, ), ( "a1 LIST \"\" (\"foo\")\r\n", list::Arguments::Extended { tag: "a1".into(), reference_name: "".into(), mailbox_name: vec!["foo".into()], selection_options: vec![], return_options: vec![], }, ), ( "a3.1 LIST \"\" (% music/rock)\r\n", list::Arguments::Extended { tag: "a3.1".into(), reference_name: "".into(), mailbox_name: vec!["%".into(), "music/rock".into()], selection_options: vec![], return_options: vec![], }, ), ( "BBB LIST \"\" (\"INBOX\" \"Drafts\" \"Sent/%\")\r\n", list::Arguments::Extended { tag: "BBB".into(), reference_name: "".into(), mailbox_name: vec!["INBOX".into(), "Drafts".into(), "Sent/%".into()], selection_options: vec![], return_options: vec![], }, ), ( "A01 LIST \"\" % RETURN (STATUS (MESSAGES UNSEEN))\r\n", list::Arguments::Extended { tag: "A01".into(), reference_name: "".into(), mailbox_name: vec!["%".into()], selection_options: vec![], return_options: vec![ReturnOption::Status(vec![ Status::Messages, Status::Unseen, ])], }, ), ( concat!( "A02 LIST (SUBSCRIBED RECURSIVEMATCH) \"\" ", "% RETURN (CHILDREN STATUS (MESSAGES))\r\n" ), list::Arguments::Extended { tag: "A02".into(), reference_name: "".into(), mailbox_name: vec!["%".into()], selection_options: vec![ SelectionOption::Subscribed, SelectionOption::RecursiveMatch, ], return_options: vec![ ReturnOption::Children, ReturnOption::Status(vec![Status::Messages]), ], }, ), ] { assert_eq!( receiver .parse(&mut command.as_bytes().iter()) .unwrap() .parse_list(true) .unwrap(), arguments ); } } } ================================================ FILE: crates/imap-proto/src/parser/login.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::ToCompactString; use crate::{ Command, protocol::login, receiver::{Request, bad}, }; impl Request { pub fn parse_login(self) -> trc::Result { match self.tokens.len() { 2 => { let mut tokens = self.tokens.into_iter(); Ok(login::Arguments { username: tokens .next() .unwrap() .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, password: tokens .next() .unwrap() .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, tag: self.tag, }) } 0 => Err(self.into_error("Missing arguments.")), _ => Err(self.into_error("Too many arguments.")), } } } #[cfg(test)] mod tests { use crate::{protocol::login, receiver::Receiver}; #[test] fn parse_login() { let mut receiver = Receiver::new(); for (command, arguments) in [ ( "a001 LOGIN SMITH SESAME\r\n", login::Arguments { tag: "a001".into(), username: "SMITH".into(), password: "SESAME".into(), }, ), ( "A001 LOGIN {11+}\r\nFRED FOOBAR {7+}\r\nfat man\r\n", login::Arguments { tag: "A001".into(), username: "FRED FOOBAR".into(), password: "fat man".into(), }, ), ] { assert_eq!( receiver .parse(&mut command.as_bytes().iter()) .unwrap() .parse_login() .unwrap(), arguments ); } } } ================================================ FILE: crates/imap-proto/src/parser/lsub.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::ToCompactString; use crate::{ Command, protocol::list::{self, SelectionOption}, receiver::{Request, bad}, utf7::utf7_maybe_decode, }; impl Request { pub fn parse_lsub(self, is_utf8: bool) -> trc::Result { if self.tokens.len() > 1 { let mut tokens = self.tokens.into_iter(); Ok(list::Arguments::Extended { reference_name: tokens .next() .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing reference name."))? .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, mailbox_name: vec![utf7_maybe_decode( tokens .next() .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing mailbox name."))? .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, is_utf8, )], selection_options: vec![SelectionOption::Subscribed], return_options: vec![], tag: self.tag, }) } else { Err(self.into_error("Missing arguments.")) } } } #[cfg(test)] mod tests { use crate::{ protocol::list::{self, SelectionOption}, receiver::Receiver, }; #[test] fn parse_lsub() { let mut receiver = Receiver::new(); for (command, arguments) in [ ( "A002 LSUB \"#news.\" \"comp.mail.*\"\r\n", list::Arguments::Extended { tag: "A002".into(), reference_name: "#news.".into(), mailbox_name: vec!["comp.mail.*".into()], selection_options: vec![SelectionOption::Subscribed], return_options: vec![], }, ), ( "A002 LSUB \"#news.\" \"comp.%\"\r\n", list::Arguments::Extended { tag: "A002".into(), reference_name: "#news.".into(), mailbox_name: vec!["comp.%".into()], selection_options: vec![SelectionOption::Subscribed], return_options: vec![], }, ), ] { assert_eq!( receiver .parse(&mut command.as_bytes().iter()) .unwrap() .parse_lsub(false) .unwrap(), arguments ); } } } ================================================ FILE: crates/imap-proto/src/parser/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod acl; pub mod append; pub mod authenticate; pub mod copy_move; pub mod create; pub mod delete; pub mod enable; pub mod fetch; pub mod list; pub mod login; pub mod lsub; pub mod quota; pub mod rename; pub mod search; pub mod select; pub mod sort; pub mod status; pub mod store; pub mod subscribe; pub mod thread; use std::{borrow::Cow, str::FromStr}; use chrono::{DateTime, NaiveDate}; use crate::{ Command, protocol::{Flag, Sequence}, receiver::CommandParser, }; pub type Result = std::result::Result>; impl CommandParser for Command { fn parse(value: &[u8], uid: bool) -> Option { hashify::tiny_map!(value, "CAPABILITY" => Command::Capability, "NOOP" => Command::Noop, "LOGOUT" => Command::Logout, "STARTTLS" => Command::StartTls, "AUTHENTICATE" => Command::Authenticate, "LOGIN" => Command::Login, "ENABLE" => Command::Enable, "SELECT" => Command::Select, "EXAMINE" => Command::Examine, "CREATE" => Command::Create, "DELETE" => Command::Delete, "RENAME" => Command::Rename, "SUBSCRIBE" => Command::Subscribe, "UNSUBSCRIBE" => Command::Unsubscribe, "LIST" => Command::List, "NAMESPACE" => Command::Namespace, "STATUS" => Command::Status, "APPEND" => Command::Append, "IDLE" => Command::Idle, "CLOSE" => Command::Close, "UNSELECT" => Command::Unselect, "EXPUNGE" => Command::Expunge(uid), "SEARCH" => Command::Search(uid), "FETCH" => Command::Fetch(uid), "STORE" => Command::Store(uid), "COPY" => Command::Copy(uid), "MOVE" => Command::Move(uid), "SORT" => Command::Sort(uid), "THREAD" => Command::Thread(uid), "LSUB" => Command::Lsub, "CHECK" => Command::Check, "SETACL" => Command::SetAcl, "DELETEACL" => Command::DeleteAcl, "GETACL" => Command::GetAcl, "LISTRIGHTS" => Command::ListRights, "MYRIGHTS" => Command::MyRights, "UNAUTHENTICATE" => Command::Unauthenticate, "ID" => Command::Id, "GETQUOTA" => Command::GetQuota, "GETQUOTAROOT" => Command::GetQuotaRoot, ) } #[inline(always)] fn tokenize_brackets(&self) -> bool { matches!(self, Command::Fetch(_)) } } impl Flag { pub fn parse_imap(value: Vec) -> Result { if !value.is_empty() { let flag = hashify::tiny_map_ignore_case!(value.as_slice(), "\\Seen" => Flag::Seen, "\\Answered" => Flag::Answered, "\\Flagged" => Flag::Flagged, "\\Deleted" => Flag::Deleted, "\\Draft" => Flag::Draft, "\\Recent" => Flag::Recent, "\\Important" => Flag::Important, "$Forwarded" => Flag::Forwarded, "$MDNSent" => Flag::MDNSent, "$Junk" => Flag::Junk, "$NotJunk" => Flag::NotJunk, "$Phishing" => Flag::Phishing, "$Important" => Flag::Important, "$autosent" => Flag::Autosent, "$canunsubscribe" => Flag::CanUnsubscribe, "$followed" => Flag::Followed, "$hasattachment" => Flag::HasAttachment, "$hasmemo" => Flag::HasMemo, "$hasnoattachment" => Flag::HasNoAttachment, "$imported" => Flag::Imported, "$istrusted" => Flag::IsTrusted, "$MailFlagBit0" => Flag::MailFlagBit0, "$MailFlagBit1" => Flag::MailFlagBit1, "$MailFlagBit2" => Flag::MailFlagBit2, "$maskedemail" => Flag::MaskedEmail, "$memo" => Flag::Memo, "$muted" => Flag::Muted, "$new" => Flag::New, "$notify" => Flag::Notify, "$unsubscribed" => Flag::Unsubscribed, ); if let Some(flag) = flag { Ok(flag) } else { String::from_utf8(value) .map_err(|_| Cow::from("Invalid UTF-8.")) .map(|v| Flag::Keyword(v.into_boxed_str())) } } else { Err(Cow::from("Null flags are not allowed.")) } } pub fn parse_jmap(value: String) -> Self { if value.starts_with('$') { hashify::tiny_map_ignore_case!(value.as_bytes(), "$seen" => Flag::Seen, "$draft" => Flag::Draft, "$flagged" => Flag::Flagged, "$answered" => Flag::Answered, "$recent" => Flag::Recent, "$important" => Flag::Important, "$phishing" => Flag::Phishing, "$junk" => Flag::Junk, "$notjunk" => Flag::NotJunk, "$deleted" => Flag::Deleted, "$forwarded" => Flag::Forwarded, "$mdnsent" => Flag::MDNSent, "$autosent" => Flag::Autosent, "$canunsubscribe" => Flag::CanUnsubscribe, "$followed" => Flag::Followed, "$hasattachment" => Flag::HasAttachment, "$hasmemo" => Flag::HasMemo, "$hasnoattachment" => Flag::HasNoAttachment, "$imported" => Flag::Imported, "$istrusted" => Flag::IsTrusted, "$MailFlagBit0" => Flag::MailFlagBit0, "$MailFlagBit1" => Flag::MailFlagBit1, "$MailFlagBit2" => Flag::MailFlagBit2, "$maskedemail" => Flag::MaskedEmail, "$memo" => Flag::Memo, "$muted" => Flag::Muted, "$new" => Flag::New, "$notify" => Flag::Notify, "$unsubscribed" => Flag::Unsubscribed, ) .unwrap_or_else(|| Flag::Keyword(value.into_boxed_str())) } else { let mut keyword = String::with_capacity(value.len()); for c in value.chars() { if c.is_ascii_alphanumeric() { keyword.push(c); } else { keyword.push('_'); } } Flag::Keyword(keyword.into_boxed_str()) } } } pub fn parse_datetime(value: &[u8]) -> Result { std::str::from_utf8(value) .map_err(|_| Cow::from("Expected date/time, found an invalid UTF-8 string.")) .and_then(|datetime| { DateTime::parse_from_str(datetime.trim(), "%d-%b-%Y %H:%M:%S %z") .map_err(|_| Cow::from(format!("Failed to parse date/time '{}'.", datetime))) .map(|dt| dt.timestamp()) }) } pub fn parse_date(value: &[u8]) -> Result { std::str::from_utf8(value) .map_err(|_| Cow::from("Expected date, found an invalid UTF-8 string.")) .and_then(|date| { NaiveDate::parse_from_str(date.trim(), "%d-%b-%Y") .map_err(|_| Cow::from(format!("Failed to parse date '{}'.", date))) .map(|dt| { dt.and_hms_opt(0, 0, 0) .unwrap_or_default() .and_utc() .timestamp() }) }) } pub fn parse_number(value: &[u8]) -> Result { std::str::from_utf8(value) .map_err(|_| Cow::from("Expected a number, found an invalid UTF-8 string.")) .and_then(|string| { string .parse::() .map_err(|_| Cow::from(format!("Expected a number, found {:?}.", string))) }) } pub fn parse_sequence_set(value: &[u8]) -> Result { let mut sequence_set = Vec::new(); let mut range_start = None; let mut token_start = None; let mut is_wildcard = false; let mut is_range = false; let mut is_saved_search = false; for (mut pos, ch) in value.iter().enumerate() { let mut add_token = false; match ch { b',' => { add_token = true; } b':' => { if !is_range { if let Some(from_pos) = token_start { range_start = parse_number::(value.get(from_pos..pos).ok_or_else(|| { Cow::from(format!( "Invalid sequence set {:?}, parse error.", String::from_utf8_lossy(value) )) })?)? .into(); token_start = None; } else if is_wildcard { is_wildcard = false; } else { return Err(Cow::from(format!( "Invalid sequence set {:?}, number expected before ':'.", String::from_utf8_lossy(value) ))); } is_range = true; } else { return Err(Cow::from(format!( "Invalid sequence set {:?}, ':' appears multiple times.", String::from_utf8_lossy(value) ))); } } b'*' => { if !is_wildcard { if value.len() == 1 { return Ok(Sequence::Range { start: None, end: None, }); } else if token_start.is_none() { is_wildcard = true; } else { return Err(Cow::from(format!( "Invalid sequence set {:?}, invalid use of '*'.", String::from_utf8_lossy(value) ))); } } else { return Err(Cow::from(format!( "Invalid sequence set {:?}, '*' appears multiple times.", String::from_utf8_lossy(value) ))); } } b'$' => { if value.get(pos + 1).is_none_or(|&ch| ch == b',') { is_saved_search = true; } else { return Err(Cow::from(format!( "Invalid sequence set {:?}, unexpected token after '$'.", String::from_utf8_lossy(value) ))); } } _ => { if ch.is_ascii_digit() { if is_wildcard { return Err(Cow::from(format!( "Invalid sequence set {:?}, invalid use of '*'.", String::from_utf8_lossy(value) ))); } if token_start.is_none() { token_start = pos.into(); } } else { return Err(Cow::from(format!( "Invalid sequence set {:?}, found invalid character '{}' at position {}.", String::from_utf8_lossy(value), ch, pos ))); } } } if add_token || pos == value.len() - 1 { if is_range { sequence_set.push(Sequence::Range { start: range_start, end: if !is_wildcard { if !add_token { pos += 1; } parse_number::( value .get( token_start.ok_or_else(|| { Cow::from(format!( "Invalid sequence set {:?}, expected number.", String::from_utf8_lossy(value) )) })?..pos, ) .ok_or_else(|| { Cow::from(format!( "Invalid sequence set {:?}, parse error.", String::from_utf8_lossy(value) )) })?, )? .into() } else { is_wildcard = false; None }, }); is_range = false; range_start = None; } else { if !add_token { pos += 1; } if is_wildcard { sequence_set.push(Sequence::Range { start: None, end: None, }); is_wildcard = false; } else if is_saved_search { sequence_set.push(Sequence::SavedSearch); is_saved_search = false; } else { sequence_set.push(Sequence::Number { value: parse_number( value .get( token_start.ok_or_else(|| { Cow::from(format!( "Invalid sequence set {:?}, expected number.", String::from_utf8_lossy(value) )) })?..pos, ) .ok_or_else(|| { Cow::from(format!( "Invalid sequence set {:?}, parse error.", String::from_utf8_lossy(value) )) })?, )?, }); } } token_start = None; } } match sequence_set.len() { 1 => Ok(sequence_set.pop().unwrap()), 0 => Err(Cow::from("Invalid empty sequence set.")), _ => Ok(Sequence::List { items: sequence_set, }), } } pub trait PushUnique { fn push_unique(&mut self, value: T); } impl PushUnique for Vec { fn push_unique(&mut self, value: T) { if !self.contains(&value) { self.push(value); } } } #[cfg(test)] mod tests { use crate::protocol::Sequence; #[test] fn parse_sequence_set() { for (sequence, expected_result) in [ ("$", Sequence::SavedSearch), ( "*", Sequence::Range { start: None, end: None, }, ), ( "1,3000:3021", Sequence::List { items: vec![ Sequence::Number { value: 1 }, Sequence::Range { start: 3000.into(), end: 3021.into(), }, ], }, ), ( "2,4:7,9,12:*", Sequence::List { items: vec![ Sequence::Number { value: 2 }, Sequence::Range { start: 4.into(), end: 7.into(), }, Sequence::Number { value: 9 }, Sequence::Range { start: 12.into(), end: None, }, ], }, ), ( "*:4,5:7", Sequence::List { items: vec![ Sequence::Range { start: None, end: 4.into(), }, Sequence::Range { start: 5.into(), end: 7.into(), }, ], }, ), ( "2,4,5", Sequence::List { items: vec![ Sequence::Number { value: 2 }, Sequence::Number { value: 4 }, Sequence::Number { value: 5 }, ], }, ), ] { assert_eq!( super::parse_sequence_set(sequence.as_bytes()).unwrap(), expected_result ); } } } ================================================ FILE: crates/imap-proto/src/parser/quota.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::ToCompactString; use crate::{ Command, protocol::quota, receiver::{Request, bad}, utf7::utf7_maybe_decode, }; impl Request { pub fn parse_get_quota_root(self, is_utf8: bool) -> trc::Result { match self.tokens.len() { 1 => Ok(quota::Arguments { name: utf7_maybe_decode( self.tokens .into_iter() .next() .unwrap() .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, is_utf8, ), tag: self.tag, }), 0 => Err(self.into_error("Missing mailbox name.")), _ => Err(self.into_error("Too many arguments.")), } } pub fn parse_get_quota(self) -> trc::Result { match self.tokens.len() { 1 => Ok(quota::Arguments { name: self .tokens .into_iter() .next() .unwrap() .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, tag: self.tag, }), 0 => Err(self.into_error("Missing quota root.")), _ => Err(self.into_error("Too many arguments.")), } } } #[cfg(test)] mod tests { use crate::{protocol::quota, receiver::Receiver}; #[test] fn parse_quota() { let mut receiver = Receiver::new(); let (command, arguments) = ( "A142 GETQUOTAROOT INBOX\r\n", quota::Arguments { name: "INBOX".into(), tag: "A142".into(), }, ); assert_eq!( receiver .parse(&mut command.as_bytes().iter()) .unwrap() .parse_get_quota_root(true) .unwrap(), arguments ); let (command, arguments) = ( "A142 GETQUOTA \"my funky mailbox\"\r\n", quota::Arguments { name: "my funky mailbox".into(), tag: "A142".into(), }, ); assert_eq!( receiver .parse(&mut command.as_bytes().iter()) .unwrap() .parse_get_quota() .unwrap(), arguments ); } } ================================================ FILE: crates/imap-proto/src/parser/rename.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::ToCompactString; use crate::{ Command, protocol::rename, receiver::{Request, bad}, utf7::utf7_maybe_decode, }; impl Request { pub fn parse_rename(self, is_utf8: bool) -> trc::Result { match self.tokens.len() { 2 => { let mut tokens = self.tokens.into_iter(); Ok(rename::Arguments { mailbox_name: utf7_maybe_decode( tokens .next() .unwrap() .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, is_utf8, ), new_mailbox_name: utf7_maybe_decode( tokens .next() .unwrap() .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, is_utf8, ), tag: self.tag, }) } 0 => Err(self.into_error("Missing argument.")), 1 => Err(self.into_error("Missing new mailbox name.")), _ => Err(self.into_error("Too many arguments.")), } } } #[cfg(test)] mod tests { use crate::{protocol::rename, receiver::Receiver}; #[test] fn parse_rename() { let mut receiver = Receiver::new(); for (command, arguments) in [ ( "A142 RENAME \"my funky mailbox\" Private\r\n", rename::Arguments { mailbox_name: "my funky mailbox".into(), new_mailbox_name: "Private".into(), tag: "A142".into(), }, ), ( "A142 RENAME {1+}\r\na {1+}\r\nb\r\n", rename::Arguments { mailbox_name: "a".into(), new_mailbox_name: "b".into(), tag: "A142".into(), }, ), ] { assert_eq!( receiver .parse(&mut command.as_bytes().iter()) .unwrap() .parse_rename(true) .unwrap(), arguments ); } } } ================================================ FILE: crates/imap-proto/src/parser/search.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::borrow::Cow; use std::iter::Peekable; use std::vec::IntoIter; use compact_str::ToCompactString; use mail_parser::decoders::charsets::DecoderFnc; use mail_parser::decoders::charsets::map::charset_decoder; use crate::Command; use crate::protocol::search::{self, Filter}; use crate::protocol::search::{ModSeqEntry, ResultOption}; use crate::protocol::{Flag, ProtocolVersion}; use crate::receiver::{Request, Token, bad}; use super::{parse_date, parse_number, parse_sequence_set}; impl Request { #[allow(clippy::while_let_on_iterator)] pub fn parse_search(self, version: ProtocolVersion) -> trc::Result { if self.tokens.is_empty() { return Err(self.into_error("Missing search criteria.")); } let mut tokens = self.tokens.into_iter().peekable(); let mut result_options = Vec::new(); let mut decoder = None; let mut is_esearch = version.is_rev2(); loop { match tokens.peek() { Some(Token::Argument(value)) if value.eq_ignore_ascii_case(b"return") => { tokens.next(); is_esearch = true; result_options = parse_result_options(&mut tokens) .map_err(|v| bad(self.tag.to_compact_string(), v))?; } Some(Token::Argument(value)) if value.eq_ignore_ascii_case(b"charset") => { tokens.next(); decoder = charset_decoder( &tokens .next() .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing charset."))? .unwrap_bytes(), ); } _ => break, } } let filter = parse_filters(&mut tokens, decoder) .map_err(|v| bad(self.tag.to_compact_string(), v))?; match filter.len() { 0 => Err(bad( self.tag.to_compact_string(), "No filters found in command.", )), _ => Ok(search::Arguments { tag: self.tag, result_options, filter, sort: None, is_esearch, }), } } } pub fn parse_result_options( tokens: &mut Peekable>, ) -> super::Result> { let mut result_options = Vec::new(); if tokens .next() .is_none_or(|token| !token.is_parenthesis_open()) { return Err(Cow::from("Invalid result option, expected parenthesis.")); } for token in tokens { match token { Token::ParenthesisClose => break, Token::Argument(value) => { result_options.push(ResultOption::parse(&value)?); } _ => return Err(Cow::from("Invalid result option argument.")), } } Ok(result_options) } pub fn parse_filters( tokens: &mut Peekable>, decoder: Option, ) -> super::Result> { let mut filters = Vec::new(); let mut filters_len = 0; let mut filters_stack = Vec::new(); let mut operator = Filter::And; while let Some(token) = tokens.next() { let mut found_parenthesis = false; match token { Token::Argument(value) => { hashify::fnc_map_ignore_case!(value.as_slice(), "ALL" => { filters.push(Filter::All); }, "ANSWERED" => { filters.push(Filter::Answered); }, "BCC" => { filters.push(Filter::Bcc(decode_argument(tokens, decoder)?)); }, "BEFORE" => { filters.push(Filter::Before(parse_date( &tokens .next() .ok_or_else(|| Cow::from("Expected date"))? .unwrap_bytes(), )?)); }, "BODY" => { filters.push(Filter::Body(decode_argument(tokens, decoder)?)); }, "CC" => { filters.push(Filter::Cc(decode_argument(tokens, decoder)?)); }, "DELETED" => { filters.push(Filter::Deleted); }, "DRAFT" => { filters.push(Filter::Draft); }, "FLAGGED" => { filters.push(Filter::Flagged); }, "FROM" => { filters.push(Filter::From(decode_argument(tokens, decoder)?)); }, "HEADER" => { filters.push(Filter::Header( decode_argument(tokens, decoder)?, decode_argument(tokens, decoder)?, )); }, "KEYWORD" => { filters.push(Filter::Keyword(Flag::parse_imap( tokens .next() .ok_or_else(|| Cow::from("Expected keyword"))? .unwrap_bytes(), )?)); }, "LARGER" => { filters.push(Filter::Larger(parse_number::( &tokens .next() .ok_or_else(|| Cow::from("Expected integer"))? .unwrap_bytes(), )?)); }, "ON" => { filters.push(Filter::On(parse_date( &tokens .next() .ok_or_else(|| Cow::from("Expected date"))? .unwrap_bytes(), )?)); }, "SEEN" => { filters.push(Filter::Seen); }, "SENTBEFORE" => { filters.push(Filter::SentBefore(parse_date( &tokens .next() .ok_or_else(|| Cow::from("Expected date"))? .unwrap_bytes(), )?)); }, "SENTON" => { filters.push(Filter::SentOn(parse_date( &tokens .next() .ok_or_else(|| Cow::from("Expected date"))? .unwrap_bytes(), )?)); }, "SENTSINCE" => { filters.push(Filter::SentSince(parse_date( &tokens .next() .ok_or_else(|| Cow::from("Expected date"))? .unwrap_bytes(), )?)); }, "SINCE" => { filters.push(Filter::Since(parse_date( &tokens .next() .ok_or_else(|| Cow::from("Expected date"))? .unwrap_bytes(), )?)); }, "SMALLER" => { filters.push(Filter::Smaller(parse_number::( &tokens .next() .ok_or_else(|| Cow::from("Expected integer"))? .unwrap_bytes(), )?)); }, "SUBJECT" => { filters.push(Filter::Subject(decode_argument(tokens, decoder)?)); }, "TEXT" => { filters.push(Filter::Text(decode_argument(tokens, decoder)?)); }, "TO" => { filters.push(Filter::To(decode_argument(tokens, decoder)?)); }, "UID" => { filters.push(Filter::Sequence( parse_sequence_set( &tokens .next() .ok_or_else(|| Cow::from("Missing sequence set."))? .unwrap_bytes(), )?, true, )); }, "UNANSWERED" => { filters.push(Filter::Unanswered); }, "UNDELETED" => { filters.push(Filter::Undeleted); }, "UNDRAFT" => { filters.push(Filter::Undraft); }, "UNFLAGGED" => { filters.push(Filter::Unflagged); }, "UNKEYWORD" => { filters.push(Filter::Unkeyword(Flag::parse_imap( tokens .next() .ok_or_else(|| Cow::from("Expected keyword"))? .unwrap_bytes(), )?)); }, "UNSEEN" => { filters.push(Filter::Unseen); }, "OLDER" => { filters.push(Filter::Older(parse_number::( &tokens .next() .ok_or_else(|| Cow::from("Expected integer"))? .unwrap_bytes(), )?)); }, "YOUNGER" => { filters.push(Filter::Younger(parse_number::( &tokens .next() .ok_or_else(|| Cow::from("Expected integer"))? .unwrap_bytes(), )?)); }, "OLD" => { filters.push(Filter::Old); }, "NEW" => { filters.push(Filter::New); }, "RECENT" => { filters.push(Filter::Recent); }, "MODSEQ" => { let param = tokens .next() .ok_or_else(|| Cow::from("Missing MODSEQ parameters."))? .unwrap_bytes(); if param.is_empty() || param.iter().any(|ch| !ch.is_ascii_digit()) { if param.len() <= 7 || !param.starts_with(b"/flags/") { return Err(format!( "Unsupported MODSEQ parameter '{}'.", String::from_utf8_lossy(¶m) ) .into()); } let flag = Flag::parse_imap((param[7..]).to_vec())?; let mod_seq_entry = match tokens.next() { Some(Token::Argument(value)) if value.eq_ignore_ascii_case(b"all") => { ModSeqEntry::All(flag) } Some(Token::Argument(value)) if value.eq_ignore_ascii_case(b"shared") => { ModSeqEntry::Shared(flag) } Some(Token::Argument(value)) if value.eq_ignore_ascii_case(b"priv") => { ModSeqEntry::Private(flag) } Some(token) => { return Err( format!("Unsupported MODSEQ parameter '{}'.", token).into() ); } None => { return Err("Missing MODSEQ entry-type-req parameter.".into()); } }; filters.push(Filter::ModSeq(( parse_number::( &tokens .next() .ok_or_else(|| { Cow::from("Missing MODSEQ mod-sequence-valzer parameter.") })? .unwrap_bytes(), )?, mod_seq_entry, ))); } else { filters.push(Filter::ModSeq(( parse_number::(¶m)?, ModSeqEntry::None, ))); } }, "EMAILID" => { filters.push(Filter::EmailId( tokens .next() .ok_or_else(|| Cow::from("Expected an EMAILID value."))? .unwrap_string()?, )); }, "THREADID" => { filters.push(Filter::ThreadId( tokens .next() .ok_or_else(|| Cow::from("Expected an THREADID value."))? .unwrap_string()?, )); }, "OR" => { if filters_stack.len() > 10 { return Err(Cow::from("Too many nested filters")); } filters_stack.push((filters, operator, filters_len)); filters_len = 0; filters = Vec::with_capacity(2); operator = Filter::Or; continue; }, "NOT" => { if filters_stack.len() > 10 { return Err(Cow::from("Too many nested filters")); } filters_stack.push((filters, operator, filters_len)); filters_len = 0; filters = Vec::with_capacity(1); operator = Filter::Not; continue; }, _ => { filters.push(Filter::Sequence(parse_sequence_set(&value)?, false)); } ); filters_len += 1; } Token::ParenthesisOpen => { if filters_stack.len() > 10 { return Err(Cow::from("Too many nested filters")); } filters_stack.push((filters, operator, filters_len)); filters_len = 0; filters = Vec::with_capacity(5); operator = Filter::And; continue; } Token::ParenthesisClose => { if filters_stack.is_empty() { return Err(Cow::from("Unexpected parenthesis.")); } found_parenthesis = true; } token => return Err(format!("Unexpected token {:?}.", token.to_string()).into()), } if !filters_stack.is_empty() && (found_parenthesis || (operator == Filter::Or && filters_len == 2) || (operator == Filter::Not && filters_len == 1)) { while let Some((mut prev_filters, prev_operator, prev_filters_len)) = filters_stack.pop() { if operator == Filter::And && (prev_operator != Filter::Or || filters_len == 1) { prev_filters.extend(filters); filters_len += prev_filters_len; } else { prev_filters.push(operator); prev_filters.extend(filters); prev_filters.push(Filter::End); filters_len = prev_filters_len + 1; } operator = prev_operator; filters = prev_filters; if operator == Filter::And || (operator == Filter::Or && filters_len < 2) { break; } } } } Ok(filters) } pub fn decode_argument( tokens: &mut Peekable>, decoder: Option, ) -> super::Result { let argument = tokens .next() .ok_or_else(|| Cow::from("Expected string."))? .unwrap_bytes(); if let Some(decoder) = decoder { Ok(decoder(&argument)) } else { Ok(String::from_utf8(argument).map_err(|_| Cow::from("Invalid UTF-8 argument."))?) } } impl ResultOption { pub fn parse(value: &[u8]) -> super::Result { hashify::tiny_map_ignore_case!( value, "min" => Self::Min, "max" => Self::Max, "all" => Self::All, "count" => Self::Count, "save" => Self::Save, "context" => Self::Context, ) .ok_or_else(|| { format!( "Invalid result option '{}'.", String::from_utf8_lossy(value) ) .into() }) } } #[cfg(test)] mod tests { use crate::{ protocol::{ Flag, ProtocolVersion, Sequence, search::{self, Filter, ModSeqEntry, ResultOption}, }, receiver::Receiver, }; #[test] fn parse_search() { let mut receiver = Receiver::new(); for (command, arguments) in [ ( b"A282 SEARCH RETURN (MIN COUNT) FLAGGED SINCE 1-Feb-1994 NOT FROM \"Smith\"\r\n" .to_vec(), search::Arguments { tag: "A282".into(), result_options: vec![ResultOption::Min, ResultOption::Count], filter: vec![ Filter::Flagged, Filter::Since(760060800), Filter::Not, Filter::From("Smith".into()), Filter::End, ], is_esearch: true, sort: None, }, ), ( b"A283 SEARCH RETURN () FLAGGED SINCE 1-Feb-1994 NOT FROM \"Smith\"\r\n".to_vec(), search::Arguments { tag: "A283".into(), result_options: vec![], filter: vec![ Filter::Flagged, Filter::Since(760060800), Filter::Not, Filter::From("Smith".into()), Filter::End, ], is_esearch: true, sort: None, }, ), ( b"A301 SEARCH $ SMALLER 4096\r\n".to_vec(), search::Arguments { tag: "A301".into(), result_options: vec![], filter: vec![Filter::seq_saved_search(), Filter::Smaller(4096)], is_esearch: true, sort: None, }, ), ( "P283 SEARCH CHARSET UTF-8 (OR $ 1,3000:3021) TEXT {8+}\r\nмать\r\n" .as_bytes() .to_vec(), search::Arguments { tag: "P283".into(), result_options: vec![], filter: vec![ Filter::Or, Filter::seq_saved_search(), Filter::Sequence( Sequence::List { items: vec![ Sequence::number(1), Sequence::range(3000.into(), 3021.into()), ], }, false, ), Filter::End, Filter::Text("мать".into()), ], is_esearch: true, sort: None, }, ), ( b"F282 SEARCH RETURN (SAVE) KEYWORD $Junk\r\n".to_vec(), search::Arguments { tag: "F282".into(), result_options: vec![ResultOption::Save], filter: vec![Filter::Keyword(Flag::Junk)], is_esearch: true, sort: None, }, ), ( [ b"F282 SEARCH OR OR FROM hello@world.com TO ".to_vec(), b"test@example.com OR BCC jane@foobar.com ".to_vec(), b"CC john@doe.com\r\n".to_vec(), ] .concat(), search::Arguments { tag: "F282".into(), result_options: vec![], filter: vec![ Filter::Or, Filter::Or, Filter::From("hello@world.com".into()), Filter::To("test@example.com".into()), Filter::End, Filter::Or, Filter::Bcc("jane@foobar.com".into()), Filter::Cc("john@doe.com".into()), Filter::End, Filter::End, ], is_esearch: true, sort: None, }, ), ( [ b"abc SEARCH OR SMALLER 10000 OR ".to_vec(), b"HEADER Subject \"ravioli festival\" ".to_vec(), b"HEADER From \"dr. ravioli\"\r\n".to_vec(), ] .concat(), search::Arguments { tag: "abc".into(), result_options: vec![], filter: vec![ Filter::Or, Filter::Smaller(10000), Filter::Or, Filter::Header("Subject".into(), "ravioli festival".into()), Filter::Header("From".into(), "dr. ravioli".into()), Filter::End, Filter::End, ], is_esearch: true, sort: None, }, ), ( [ b"abc SEARCH (DELETED SEEN ANSWERED) ".to_vec(), b"NOT (FROM john TO jane BCC bill) ".to_vec(), b"(1,30:* UID 1,2,3,4 $)\r\n".to_vec(), ] .concat(), search::Arguments { tag: "abc".into(), result_options: vec![], filter: vec![ Filter::Deleted, Filter::Seen, Filter::Answered, Filter::Not, Filter::From("john".into()), Filter::To("jane".into()), Filter::Bcc("bill".into()), Filter::End, Filter::Sequence( Sequence::List { items: vec![Sequence::number(1), Sequence::range(30.into(), None)], }, false, ), Filter::Sequence( Sequence::List { items: vec![ Sequence::number(1), Sequence::number(2), Sequence::number(3), Sequence::number(4), ], }, true, ), Filter::seq_saved_search(), ], is_esearch: true, sort: None, }, ), ( [ b"abc SEARCH *:* UID *:100,100:* ".to_vec(), b"(FLAGGED (DRAFT (DELETED (ANSWERED)))) ".to_vec(), b"OR (SENTON 20-Nov-2022) (LARGER 8196)\r\n".to_vec(), ] .concat(), search::Arguments { tag: "abc".into(), result_options: vec![], filter: vec![ Filter::seq_range(None, None), Filter::Sequence( Sequence::List { items: vec![ Sequence::range(None, 100.into()), Sequence::range(100.into(), None), ], }, true, ), Filter::Flagged, Filter::Draft, Filter::Deleted, Filter::Answered, Filter::Or, Filter::SentOn(1668902400), Filter::Larger(8196), Filter::End, ], is_esearch: true, sort: None, }, ), ( [ b"abc SEARCH NOT (FROM john OR TO jane CC bill) ".to_vec(), b"OR (UNDELETED ALL) ($ NOT FLAGGED) ".to_vec(), b"(((KEYWORD \"tps report\")))\r\n".to_vec(), ] .concat(), search::Arguments { tag: "abc".into(), result_options: vec![], filter: vec![ Filter::Not, Filter::From("john".into()), Filter::Or, Filter::To("jane".into()), Filter::Cc("bill".into()), Filter::End, Filter::End, Filter::Or, Filter::And, Filter::Undeleted, Filter::All, Filter::End, Filter::And, Filter::seq_saved_search(), Filter::Not, Filter::Flagged, Filter::End, Filter::End, Filter::End, Filter::Keyword(Flag::Keyword("tps report".into())), ], is_esearch: true, sort: None, }, ), ( [ b"B283 SEARCH RETURN (SAVE MIN MAX) CHARSET KOI8-R TEXT ".to_vec(), b"{11+}\r\n\xf0\xd2\xc9\xd7\xc5\xd4, \xcd\xc9\xd2\r\n".to_vec(), ] .concat(), search::Arguments { tag: "B283".into(), result_options: vec![ResultOption::Save, ResultOption::Min, ResultOption::Max], filter: vec![Filter::Text("Привет, мир".into())], is_esearch: true, sort: None, }, ), ( b"B283 SEARCH CHARSET BIG5 FROM \"\xa7A\xa6n\xa1A\xa5@\xac\xc9\"\r\n".to_vec(), search::Arguments { tag: "B283".into(), result_options: vec![], filter: vec![Filter::From("你好，世界".into())], is_esearch: true, sort: None, }, ), ( b"a SEARCH MODSEQ \"/flags/\\draft\" all 620162338\r\n".to_vec(), search::Arguments { tag: "a".into(), result_options: vec![], filter: vec![Filter::ModSeq((620162338, ModSeqEntry::All(Flag::Draft)))], is_esearch: true, sort: None, }, ), ( b"t SEARCH OR NOT MODSEQ 720162338 LARGER 50000\r\n".to_vec(), search::Arguments { tag: "t".into(), result_options: vec![], filter: vec![ Filter::Or, Filter::Not, Filter::ModSeq((720162338, ModSeqEntry::None)), Filter::End, Filter::Larger(50000), Filter::End, ], is_esearch: true, sort: None, }, ), ( b"5 UID SEARCH BEFORE 1-Dec-2023\r\n".to_vec(), search::Arguments { tag: "5".into(), result_options: vec![], filter: vec![Filter::Before(1701388800)], is_esearch: true, sort: None, }, ), ] { let command_str = String::from_utf8_lossy(&command).into_owned(); assert_eq!( receiver .parse(&mut command.iter()) .unwrap() .parse_search(ProtocolVersion::Rev2) .expect(&command_str), arguments, "{}", command_str ); } } } ================================================ FILE: crates/imap-proto/src/parser/select.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::{CompactString, ToCompactString, format_compact}; use crate::{ Command, protocol::select::{self, QResync}, receiver::{Request, Token, bad}, utf7::utf7_maybe_decode, }; use super::{parse_number, parse_sequence_set}; impl Request { pub fn parse_select(self, is_utf8: bool) -> trc::Result { if !self.tokens.is_empty() { let mut tokens = self.tokens.into_iter().peekable(); // Mailbox name let mailbox_name = utf7_maybe_decode( tokens .next() .unwrap() .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, is_utf8, ); // CONDSTORE parameters let mut condstore = false; let mut qresync = None; match tokens.next() { Some(Token::ParenthesisOpen) => { while let Some(token) = tokens.next() { match token { Token::Argument(param) if param.eq_ignore_ascii_case(b"CONDSTORE") => { condstore = true; } Token::Argument(param) if param.eq_ignore_ascii_case(b"QRESYNC") => { if tokens .next() .is_none_or(|token| !token.is_parenthesis_open()) { return Err(bad( CompactString::from_string_buffer(self.tag), "Expected '(' after 'QRESYNC'.", )); } let uid_validity = parse_number::( &tokens .next() .ok_or_else(|| { bad( self.tag.to_compact_string(), "Missing uidvalidity parameter for QRESYNC.", ) })? .unwrap_bytes(), ) .map_err(|v| bad(self.tag.to_compact_string(), v))?; let modseq = parse_number::( &tokens .next() .ok_or_else(|| { bad( self.tag.to_compact_string(), "Missing modseq parameter for QRESYNC.", ) })? .unwrap_bytes(), ) .map_err(|v| bad(self.tag.to_compact_string(), v))?; let mut known_uids = None; let mut seq_match = None; let has_seq_match = match tokens.peek() { Some(Token::Argument(value)) => { known_uids = parse_sequence_set(value) .map_err(|v| bad(self.tag.to_compact_string(), v))? .into(); tokens.next(); if matches!(tokens.peek(), Some(Token::ParenthesisOpen)) { tokens.next(); true } else { false } } Some(Token::ParenthesisOpen) => { tokens.next(); true } _ => false, }; if has_seq_match { seq_match = Some(( parse_sequence_set( &tokens .next() .ok_or_else(|| { bad( self.tag.to_compact_string(), "Missing known-sequence-set parameter for QRESYNC.", ) })? .unwrap_bytes(), ) .map_err(|v| bad(self.tag.to_compact_string(), v))?, parse_sequence_set( &tokens .next() .ok_or_else(|| { bad( self.tag.to_compact_string(), "Missing known-uid-set parameter for QRESYNC.", ) })? .unwrap_bytes(), ) .map_err(|v| bad(self.tag.to_compact_string(), v))?, )); if tokens .next() .is_none_or(|token| !token.is_parenthesis_close()) { return Err(bad( CompactString::from_string_buffer(self.tag), "Missing ')' for 'QRESYNC'.", )); } } if tokens .next() .is_none_or(|token| !token.is_parenthesis_close()) { return Err(bad( CompactString::from_string_buffer(self.tag), "Missing ')' for 'QRESYNC'.", )); } qresync = QResync { uid_validity, modseq, known_uids, seq_match, } .into(); } Token::ParenthesisClose => { break; } _ => { return Err(bad( CompactString::from_string_buffer(self.tag), format_compact!("Unexpected value '{}'.", token), )); } } } } Some(token) => { return Err(bad( CompactString::from_string_buffer(self.tag), format_compact!("Unexpected value '{}'.", token), )); } None => (), } Ok(select::Arguments { mailbox_name, tag: self.tag, condstore, qresync, }) } else { Err(self.into_error("Missing mailbox name.")) } } } #[cfg(test)] mod tests { use crate::{ protocol::{ Sequence, select::{self, QResync}, }, receiver::Receiver, }; #[test] fn parse_select() { let mut receiver = Receiver::new(); for (command, arguments) in [ ( "A142 SELECT INBOX\r\n", select::Arguments { mailbox_name: "INBOX".into(), tag: "A142".into(), condstore: false, qresync: None, }, ), ( "A142 SELECT \"my funky mailbox\"\r\n", select::Arguments { mailbox_name: "my funky mailbox".into(), tag: "A142".into(), condstore: false, qresync: None, }, ), ( "A142 SELECT INBOX (CONDSTORE)\r\n", select::Arguments { mailbox_name: "INBOX".into(), tag: "A142".into(), condstore: true, qresync: None, }, ), ( "A142 SELECT INBOX (QRESYNC (3857529045 20010715194032001 1:198))\r\n", select::Arguments { mailbox_name: "INBOX".into(), tag: "A142".into(), condstore: false, qresync: QResync { uid_validity: 3857529045, modseq: 20010715194032001, known_uids: Some(Sequence::Range { start: Some(1), end: Some(198), }), seq_match: None, } .into(), }, ), ( concat!( "A03 SELECT INBOX (QRESYNC (67890007 90060115194045000 ", "41:211,214:541) CONDSTORE)\r\n" ), select::Arguments { mailbox_name: "INBOX".into(), tag: "A03".into(), condstore: true, qresync: QResync { uid_validity: 67890007, modseq: 90060115194045000, known_uids: Some(Sequence::List { items: vec![ Sequence::Range { start: Some(41), end: Some(211), }, Sequence::Range { start: Some(214), end: Some(541), }, ], }), seq_match: None, } .into(), }, ), ( concat!( "B04 SELECT INBOX (QRESYNC (67890007 ", "90060115194045000 1:29997 (5000,7500,9000,9990:9999 15000,", "22500,27000,29970,29973,29976,29979,29982,29985,29988,29991,", "29994,29997)))\r\n" ), select::Arguments { mailbox_name: "INBOX".into(), tag: "B04".into(), condstore: false, qresync: QResync { uid_validity: 67890007, modseq: 90060115194045000, known_uids: Some(Sequence::Range { start: Some(1), end: Some(29997), }), seq_match: Some(( Sequence::List { items: vec![ Sequence::Number { value: 5000 }, Sequence::Number { value: 7500 }, Sequence::Number { value: 9000 }, Sequence::Range { start: Some(9990), end: Some(9999), }, ], }, Sequence::List { items: vec![ Sequence::Number { value: 15000 }, Sequence::Number { value: 22500 }, Sequence::Number { value: 27000 }, Sequence::Number { value: 29970 }, Sequence::Number { value: 29973 }, Sequence::Number { value: 29976 }, Sequence::Number { value: 29979 }, Sequence::Number { value: 29982 }, Sequence::Number { value: 29985 }, Sequence::Number { value: 29988 }, Sequence::Number { value: 29991 }, Sequence::Number { value: 29994 }, Sequence::Number { value: 29997 }, ], }, )), } .into(), }, ), ( "A12 SELECT \"INBOX\" (QRESYNC (1693237464 16582))\r\n", select::Arguments { mailbox_name: "INBOX".into(), tag: "A12".into(), condstore: false, qresync: QResync { uid_validity: 1693237464, modseq: 16582, known_uids: None, seq_match: None, } .into(), }, ), ] { assert_eq!( receiver .parse(&mut command.as_bytes().iter()) .unwrap_or_else(|err| panic!( "Failed to parse command '{}': {:?}", command, err )) .parse_select(true) .unwrap_or_else(|err| panic!( "Failed to parse command '{}': {:?}", command, err )), arguments, "Failed to parse {}", command ); } } } ================================================ FILE: crates/imap-proto/src/parser/sort.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::ToCompactString; use mail_parser::decoders::charsets::map::charset_decoder; use crate::{ Command, protocol::search::{Arguments, Comparator, Sort}, receiver::{Request, Token, bad}, }; use super::search::{parse_filters, parse_result_options}; impl Request { #[allow(clippy::while_let_on_iterator)] pub fn parse_sort(self) -> trc::Result { if self.tokens.is_empty() { return Err(self.into_error("Missing sort criteria.")); } let mut tokens = self.tokens.into_iter().peekable(); let mut sort = Vec::new(); let (result_options, is_esearch) = match tokens.peek() { Some(Token::Argument(value)) if value.eq_ignore_ascii_case(b"return") => { tokens.next(); ( parse_result_options(&mut tokens) .map_err(|v| bad(self.tag.to_compact_string(), v))?, true, ) } _ => (Vec::new(), false), }; if tokens .next() .is_none_or(|token| !token.is_parenthesis_open()) { return Err(bad( self.tag.to_compact_string(), "Expected sort criteria between parentheses.", )); } let mut is_ascending = true; while let Some(token) = tokens.next() { match token { Token::ParenthesisClose => break, Token::Argument(value) => { if value.eq_ignore_ascii_case(b"REVERSE") { is_ascending = false; } else { sort.push(Comparator { sort: Sort::parse(&value) .map_err(|v| bad(self.tag.to_compact_string(), v))?, ascending: is_ascending, }); is_ascending = true; } } _ => { return Err(bad( self.tag.to_compact_string(), "Invalid result option argument.", )); } } } if sort.is_empty() { return Err(bad(self.tag.to_compact_string(), "Missing sort criteria.")); } let decoder = charset_decoder( &tokens .next() .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing charset."))? .unwrap_bytes(), ); let filter = parse_filters(&mut tokens, decoder) .map_err(|v| bad(self.tag.to_compact_string(), v))?; match filter.len() { 0 => Err(bad( self.tag.to_compact_string(), "No filters found in command.", )), _ => Ok(Arguments { sort: sort.into(), result_options, filter, is_esearch, tag: self.tag, }), } } } impl Sort { pub fn parse(value: &[u8]) -> super::Result { hashify::tiny_map_ignore_case!(value, "ARRIVAL" => Self::Arrival, "CC" => Self::Cc, "DATE" => Self::Date, "FROM" => Self::From, "SIZE" => Self::Size, "SUBJECT" => Self::Subject, "TO" => Self::To, "DISPLAYFROM" => Self::DisplayFrom, "DISPLAYTO" => Self::DisplayTo, ) .ok_or_else(|| format!("Invalid sort criteria {:?}", String::from_utf8_lossy(value)).into()) } } #[cfg(test)] mod tests { use crate::{ protocol::{ Flag, search::{Arguments, Comparator, Filter, ResultOption, Sort}, }, receiver::Receiver, }; #[test] fn parse_sort() { let mut receiver = Receiver::new(); for (command, arguments) in [ ( b"A282 SORT (SUBJECT) UTF-8 SINCE 1-Feb-1994\r\n".to_vec(), Arguments { sort: vec![Comparator { sort: Sort::Subject, ascending: true, }] .into(), filter: vec![Filter::Since(760060800)], result_options: Vec::new(), is_esearch: false, tag: "A282".into(), }, ), ( b"A283 SORT (SUBJECT REVERSE DATE) UTF-8 ALL\r\n".to_vec(), Arguments { sort: vec![ Comparator { sort: Sort::Subject, ascending: true, }, Comparator { sort: Sort::Date, ascending: false, }, ] .into(), filter: vec![Filter::All], result_options: Vec::new(), is_esearch: false, tag: "A283".into(), }, ), ( b"A284 SORT (SUBJECT) US-ASCII TEXT \"not in mailbox\"\r\n".to_vec(), Arguments { sort: vec![Comparator { sort: Sort::Subject, ascending: true, }] .into(), filter: vec![Filter::Text("not in mailbox".into())], result_options: Vec::new(), is_esearch: false, tag: "A284".into(), }, ), ( [ b"A284 SORT (REVERSE ARRIVAL FROM) iso-8859-6 SUBJECT ".to_vec(), b"\"\xe5\xd1\xcd\xc8\xc7 \xc8\xc7\xe4\xd9\xc7\xe4\xe5\"\r\n".to_vec(), ] .concat(), Arguments { sort: vec![ Comparator { sort: Sort::Arrival, ascending: false, }, Comparator { sort: Sort::From, ascending: true, }, ] .into(), filter: vec![Filter::Subject("مرحبا بالعالم".into())], result_options: Vec::new(), is_esearch: false, tag: "A284".into(), }, ), ( [ b"E01 UID SORT RETURN (COUNT) (REVERSE DATE) ".to_vec(), b"UTF-8 UNDELETED UNKEYWORD $Junk\r\n".to_vec(), ] .concat(), Arguments { sort: vec![Comparator { sort: Sort::Date, ascending: false, }] .into(), filter: vec![Filter::Undeleted, Filter::Unkeyword(Flag::Junk)], result_options: vec![ResultOption::Count], is_esearch: true, tag: "E01".into(), }, ), ] { let command_str = String::from_utf8_lossy(&command).into_owned(); assert_eq!( receiver .parse(&mut command.iter()) .unwrap() .parse_sort() .expect(&command_str), arguments, "{}", command_str ); } } } ================================================ FILE: crates/imap-proto/src/parser/status.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::{CompactString, ToCompactString}; use crate::Command; use crate::protocol::status; use crate::protocol::status::Status; use crate::receiver::{Request, Token, bad}; use crate::utf7::utf7_maybe_decode; impl Request { pub fn parse_status(self, is_utf8: bool) -> trc::Result { match self.tokens.len() { 0..=3 => Err(self.into_error("Missing arguments.")), len => { let mut tokens = self.tokens.into_iter(); let mailbox_name = utf7_maybe_decode( tokens .next() .unwrap() .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, is_utf8, ); let mut items = Vec::with_capacity(len - 2); if tokens .next() .is_none_or(|token| !token.is_parenthesis_open()) { return Err(bad( self.tag.to_compact_string(), "Expected parenthesis after mailbox name.", )); } #[allow(clippy::while_let_on_iterator)] while let Some(token) = tokens.next() { match token { Token::ParenthesisClose => break, Token::Argument(value) => { items.push( Status::parse(&value) .map_err(|v| bad(self.tag.to_compact_string(), v))?, ); } _ => { return Err(bad( self.tag.to_compact_string(), "Invalid status return option argument.", )); } } } if !items.is_empty() { Ok(status::Arguments { tag: self.tag, mailbox_name, items, }) } else { Err(bad( CompactString::from_string_buffer(self.tag), "At least one status item is required.", )) } } } } } impl Status { pub fn parse(value: &[u8]) -> super::Result { hashify::tiny_map_ignore_case!(value, "MESSAGES" => Self::Messages, "UIDNEXT" => Self::UidNext, "UIDVALIDITY" => Self::UidValidity, "UNSEEN" => Self::Unseen, "DELETED" => Self::Deleted, "SIZE" => Self::Size, "HIGHESTMODSEQ" => Self::HighestModSeq, "MAILBOXID" => Self::MailboxId, "RECENT" => Self::Recent, "DELETED-STORAGE" => Self::DeletedStorage ) .ok_or_else(|| { format!( "Invalid status option '{}'.", String::from_utf8_lossy(value) ) .into() }) } } #[cfg(test)] mod tests { use crate::{protocol::status, receiver::Receiver}; #[test] fn parse_status() { let mut receiver = Receiver::new(); assert_eq!( receiver .parse( &mut "A042 STATUS blurdybloop (UIDNEXT MESSAGES)\r\n" .as_bytes() .iter() ) .unwrap() .parse_status(true) .unwrap(), status::Arguments { tag: "A042".into(), mailbox_name: "blurdybloop".into(), items: vec![status::Status::UidNext, status::Status::Messages], } ); } } ================================================ FILE: crates/imap-proto/src/parser/store.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::{CompactString, ToCompactString, format_compact}; use crate::{ Command, protocol::{ Flag, store::{self, Operation}, }, receiver::{Request, Token, bad}, }; use super::{parse_number, parse_sequence_set}; impl Request { pub fn parse_store(self) -> trc::Result { let mut tokens = self.tokens.into_iter().peekable(); // Sequence set let sequence_set = parse_sequence_set( &tokens .next() .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing sequence set."))? .unwrap_bytes(), ) .map_err(|v| bad(self.tag.to_compact_string(), v))?; let mut unchanged_since = None; // CONDSTORE parameters if let Some(Token::ParenthesisOpen) = tokens.peek() { tokens.next(); while let Some(token) = tokens.next() { match token { Token::Argument(param) if param.eq_ignore_ascii_case(b"UNCHANGEDSINCE") => { unchanged_since = parse_number::( &tokens .next() .ok_or_else(|| { bad( self.tag.to_compact_string(), "Missing UNCHANGEDSINCE parameter.", ) })? .unwrap_bytes(), ) .map_err(|v| bad(self.tag.to_compact_string(), v))? .into(); } Token::ParenthesisClose => { break; } _ => { return Err(bad( self.tag.to_compact_string(), format_compact!("Unsupported parameter '{}'.", token), )); } } } } // Operation let operation = tokens .next() .ok_or_else(|| { bad( self.tag.to_compact_string(), "Missing message data item name.", ) })? .unwrap_bytes(); let (is_silent, operation) = hashify::tiny_map_ignore_case!(operation.as_slice(), "FLAGS" => (false, Operation::Set), "FLAGS.SILENT" => (true, Operation::Set), "+FLAGS" => (false, Operation::Add), "+FLAGS.SILENT" => (true, Operation::Add), "-FLAGS" => (false, Operation::Clear), "-FLAGS.SILENT" => (true, Operation::Clear), ) .ok_or_else(|| { bad( self.tag.to_compact_string(), format_compact!( "Unsupported message data item name: {:?}", String::from_utf8_lossy(&operation) ), ) })?; // Flags let mut keywords = Vec::new(); match tokens .next() .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing flags to set."))? { Token::ParenthesisOpen => { for token in tokens { match token { Token::Argument(flag) => { keywords.push( Flag::parse_imap(flag) .map_err(|v| bad(self.tag.to_compact_string(), v))?, ); } Token::ParenthesisClose => { break; } _ => { return Err(bad(self.tag.to_compact_string(), "Unsupported flag.")); } } } } Token::Argument(flag) => { keywords.push( Flag::parse_imap(flag).map_err(|v| bad(self.tag.to_compact_string(), v))?, ); } _ => { return Err(bad( CompactString::from_string_buffer(self.tag), "Invalid flags parameter.", )); } } if !keywords.is_empty() || operation == Operation::Set { Ok(store::Arguments { tag: self.tag, sequence_set, operation, is_silent, keywords, unchanged_since, }) } else { Err(bad(self.tag.to_compact_string(), "Missing flags to set.")) } } } #[cfg(test)] mod tests { use crate::{ protocol::{ Flag, Sequence, store::{self, Operation}, }, receiver::Receiver, }; #[test] fn parse_store() { let mut receiver = Receiver::new(); for (command, arguments) in [ ( "A003 STORE 2:4 +FLAGS (\\Deleted)\r\n", store::Arguments { sequence_set: Sequence::Range { start: 2.into(), end: 4.into(), }, is_silent: false, operation: Operation::Add, keywords: vec![Flag::Deleted], tag: "A003".into(), unchanged_since: None, }, ), ( "A004 STORE *:100 -FLAGS.SILENT ($Phishing $Junk)\r\n", store::Arguments { sequence_set: Sequence::Range { start: None, end: 100.into(), }, is_silent: true, operation: Operation::Clear, keywords: vec![Flag::Phishing, Flag::Junk], tag: "A004".into(), unchanged_since: None, }, ), ( "d105 STORE 7,5,9 (UNCHANGEDSINCE 320162338) +FLAGS.SILENT \\Deleted\r\n", store::Arguments { sequence_set: Sequence::List { items: vec![ Sequence::Number { value: 7 }, Sequence::Number { value: 5 }, Sequence::Number { value: 9 }, ], }, is_silent: true, operation: Operation::Add, keywords: vec![Flag::Deleted], tag: "d105".into(), unchanged_since: Some(320162338), }, ), ] { assert_eq!( receiver .parse(&mut command.as_bytes().iter()) .unwrap() .parse_store() .unwrap(), arguments ); } } } ================================================ FILE: crates/imap-proto/src/parser/subscribe.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::ToCompactString; use crate::{ Command, protocol::subscribe, receiver::{Request, bad}, utf7::utf7_maybe_decode, }; impl Request { pub fn parse_subscribe(self, is_utf8: bool) -> trc::Result { match self.tokens.len() { 1 => Ok(subscribe::Arguments { mailbox_name: utf7_maybe_decode( self.tokens .into_iter() .next() .unwrap() .unwrap_string() .map_err(|v| bad(self.tag.to_compact_string(), v))?, is_utf8, ), tag: self.tag, }), 0 => Err(self.into_error("Missing mailbox name.")), _ => Err(self.into_error("Too many arguments.")), } } } #[cfg(test)] mod tests { use crate::{protocol::subscribe, receiver::Receiver}; #[test] fn parse_subscribe() { let mut receiver = Receiver::new(); for (command, arguments) in [ ( "A142 SUBSCRIBE #news.comp.mail.mime\r\n", subscribe::Arguments { mailbox_name: "#news.comp.mail.mime".into(), tag: "A142".into(), }, ), ( "A142 SUBSCRIBE \"#news.comp.mail.mime\"\r\n", subscribe::Arguments { mailbox_name: "#news.comp.mail.mime".into(), tag: "A142".into(), }, ), ] { assert_eq!( receiver .parse(&mut command.as_bytes().iter()) .unwrap() .parse_subscribe(true) .unwrap(), arguments ); } } } ================================================ FILE: crates/imap-proto/src/parser/thread.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::ToCompactString; use mail_parser::decoders::charsets::map::charset_decoder; use crate::{ Command, protocol::thread::{self, Algorithm}, receiver::{Request, bad}, }; use super::search::parse_filters; impl Request { #[allow(clippy::while_let_on_iterator)] pub fn parse_thread(self) -> trc::Result { if self.tokens.is_empty() { return Err(self.into_error("Missing thread criteria.")); } let mut tokens = self.tokens.into_iter().peekable(); let algorithm = Algorithm::parse( &tokens .next() .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing threading algorithm."))? .unwrap_bytes(), ) .map_err(|v| bad(self.tag.to_compact_string(), v))?; let decoder = charset_decoder( &tokens .next() .ok_or_else(|| bad(self.tag.to_compact_string(), "Missing charset."))? .unwrap_bytes(), ); let filter = parse_filters(&mut tokens, decoder) .map_err(|v| bad(self.tag.to_compact_string(), v))?; match filter.len() { 0 => Err(bad( self.tag.to_compact_string(), "No filters found in command.", )), _ => Ok(thread::Arguments { algorithm, filter, tag: self.tag, }), } } } impl Algorithm { pub fn parse(value: &[u8]) -> super::Result { hashify::tiny_map_ignore_case!(value, "ORDEREDSUBJECT" => Self::OrderedSubject, "REFERENCES" => Self::References, ) .ok_or_else(|| { format!( "Invalid threading algorithm {:?}", String::from_utf8_lossy(value) ) .into() }) } } #[cfg(test)] mod tests { use crate::{ protocol::{ search::Filter, thread::{self, Algorithm}, }, receiver::Receiver, }; #[test] fn parse_thread() { let mut receiver = Receiver::new(); for (command, arguments) in [ ( b"A283 THREAD ORDEREDSUBJECT UTF-8 SINCE 5-MAR-2000\r\n".to_vec(), thread::Arguments { algorithm: Algorithm::OrderedSubject, filter: vec![Filter::Since(952214400)], tag: "A283".into(), }, ), ( b"A284 THREAD REFERENCES US-ASCII TEXT \"gewp\"\r\n".to_vec(), thread::Arguments { algorithm: Algorithm::References, filter: vec![Filter::Text("gewp".into())], tag: "A284".into(), }, ), ] { let command_str = String::from_utf8_lossy(&command).into_owned(); assert_eq!( receiver .parse(&mut command.iter()) .unwrap() .parse_thread() .expect(&command_str), arguments, "{}", command_str ); } } } ================================================ FILE: crates/imap-proto/src/protocol/acl.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ /* l - lookup (mailbox is visible to LIST/LSUB commands, SUBSCRIBE mailbox) r - read (SELECT the mailbox, perform STATUS) s - keep seen/unseen information across sessions (set or clear \SEEN flag via STORE, also set \SEEN during APPEND/COPY/ FETCH BODY[...]) w - write (set or clear flags other than \SEEN and \DELETED via STORE, also set them during APPEND/COPY) i - insert (perform APPEND, COPY into mailbox) p - post (send mail to submission address for mailbox, not enforced by IMAP4 itself) k - create mailboxes (CREATE new sub-mailboxes in any implementation-defined hierarchy, parent mailbox for the new mailbox name in RENAME) x - delete mailbox (DELETE mailbox, old mailbox name in RENAME) t - delete messages (set or clear \DELETED flag via STORE, set \DELETED flag during APPEND/COPY) e - perform EXPUNGE and expunge as a part of CLOSE a - administer (perform SETACL/DELETEACL/GETACL/LISTRIGHTS) // RFC2086 c - create (CREATE new sub-mailboxes in any implementation-defined hierarchy) d - delete (STORE DELETED flag, perform EXPUNGE) */ use types::acl::Acl; use super::quoted_string; use crate::utf7::utf7_encode; use std::fmt::Display; #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum Rights { Lookup, Read, Seen, Write, Insert, Post, CreateMailbox, DeleteMailbox, DeleteMessages, Expunge, Administer, } #[derive(Debug, PartialEq, Eq, Clone)] pub struct ModRights { pub op: ModRightsOp, pub rights: Vec, } #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum ModRightsOp { Add, Remove, Replace, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { pub tag: String, pub mailbox_name: String, pub identifier: Option, pub mod_rights: Option, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct GetAclResponse { pub mailbox_name: String, pub permissions: Vec<(String, Vec)>, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct ListRightsResponse { pub mailbox_name: String, pub identifier: String, pub permissions: Vec>, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct MyRightsResponse { pub mailbox_name: String, pub rights: Vec, } impl GetAclResponse { pub fn into_bytes(self, is_utf8: bool) -> Vec { let mut buf = Vec::with_capacity(self.mailbox_name.len() + 10 * self.permissions.len() * 5); buf.extend_from_slice(b"* ACL "); if is_utf8 { quoted_string(&mut buf, &self.mailbox_name); } else { quoted_string(&mut buf, &utf7_encode(&self.mailbox_name)); } for (identifier, rights) in self.permissions { buf.extend_from_slice(b" "); quoted_string(&mut buf, &identifier); buf.extend_from_slice(b" "); for right in rights { buf.push(right.to_char()); } } buf.extend_from_slice(b"\r\n"); buf } } impl ListRightsResponse { pub fn into_bytes(self, is_utf8: bool) -> Vec { let mut buf = Vec::with_capacity( self.mailbox_name.len() + self.identifier.len() + 10 * self.permissions.len() * 5, ); buf.extend_from_slice(b"* LISTRIGHTS "); if is_utf8 { quoted_string(&mut buf, &self.mailbox_name); } else { quoted_string(&mut buf, &utf7_encode(&self.mailbox_name)); } buf.extend_from_slice(b" "); quoted_string(&mut buf, &self.identifier); for rights in self.permissions { buf.extend_from_slice(b" "); for right in rights { buf.push(right.to_char()); } } buf.extend_from_slice(b"\r\n"); buf } } impl MyRightsResponse { pub fn into_bytes(self, is_utf8: bool) -> Vec { let mut buf = Vec::with_capacity(self.mailbox_name.len() + 10 + self.rights.len()); buf.extend_from_slice(b"* MYRIGHTS "); if is_utf8 { quoted_string(&mut buf, &self.mailbox_name); } else { quoted_string(&mut buf, &utf7_encode(&self.mailbox_name)); } buf.extend_from_slice(b" "); for right in self.rights { buf.push(right.to_char()); } buf.extend_from_slice(b"\r\n"); buf } } impl Rights { pub fn to_char(&self) -> u8 { match self { Rights::Lookup => b'l', Rights::Read => b'r', Rights::Seen => b's', Rights::Write => b'w', Rights::Insert => b'i', Rights::Post => b'p', Rights::CreateMailbox => b'k', Rights::DeleteMailbox => b'x', Rights::DeleteMessages => b't', Rights::Expunge => b'e', Rights::Administer => b'a', } } } impl Display for Rights { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Rights::Lookup => write!(f, "l"), Rights::Read => write!(f, "r"), Rights::Seen => write!(f, "s"), Rights::Write => write!(f, "w"), Rights::Insert => write!(f, "i"), Rights::Post => write!(f, "p"), Rights::CreateMailbox => write!(f, "k"), Rights::DeleteMailbox => write!(f, "x"), Rights::DeleteMessages => write!(f, "t"), Rights::Expunge => write!(f, "e"), Rights::Administer => write!(f, "a"), } } } impl From for Acl { fn from(value: Rights) -> Self { match value { Rights::Lookup => Acl::Read, Rights::Read => Acl::ReadItems, Rights::Seen => Acl::ModifyItems, Rights::Write => Acl::ModifyItems, Rights::Insert => Acl::AddItems, Rights::Post => Acl::Submit, Rights::CreateMailbox => Acl::CreateChild, Rights::DeleteMailbox => Acl::Delete, Rights::DeleteMessages => Acl::RemoveItems, Rights::Expunge => Acl::RemoveItems, Rights::Administer => Acl::Share, } } } #[cfg(test)] mod tests { use crate::protocol::acl::{GetAclResponse, ListRightsResponse, MyRightsResponse, Rights}; #[test] fn serialize_acl() { assert_eq!( String::from_utf8( GetAclResponse { mailbox_name: "INBOX".into(), permissions: vec![ ( "Fred".into(), vec![ Rights::Lookup, Rights::Read, Rights::Seen, Rights::Write, Rights::Insert, Rights::CreateMailbox, Rights::DeleteMessages, Rights::Administer, ] ), ( "David".into(), vec![ Rights::CreateMailbox, Rights::DeleteMessages, Rights::Administer, ] ) ] } .into_bytes(true) ) .unwrap(), "* ACL \"INBOX\" \"Fred\" lrswikta \"David\" kta\r\n" ); assert_eq!( String::from_utf8( ListRightsResponse { mailbox_name: "Deleted Items".into(), identifier: "Fred".into(), permissions: vec![ vec![Rights::Lookup, Rights::Read], vec![Rights::Administer], vec![Rights::DeleteMailbox] ] } .into_bytes(true) ) .unwrap(), "* LISTRIGHTS \"Deleted Items\" \"Fred\" lr a x\r\n" ); assert_eq!( String::from_utf8( MyRightsResponse { mailbox_name: "Important".into(), rights: vec![Rights::Lookup, Rights::Read, Rights::DeleteMailbox] } .into_bytes(true) ) .unwrap(), "* MYRIGHTS \"Important\" lrx\r\n" ); } } ================================================ FILE: crates/imap-proto/src/protocol/append.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::Flag; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { pub tag: String, pub mailbox_name: String, pub messages: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct Message { pub message: Vec, pub flags: Vec, pub received_at: Option, } ================================================ FILE: crates/imap-proto/src/protocol/authenticate.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { pub tag: String, pub mechanism: Mechanism, pub params: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum Mechanism { Plain, CramMd5, DigestMd5, ScramSha1, ScramSha256, Apop, Ntlm, Gssapi, Anonymous, External, OAuthBearer, XOauth2, } impl Mechanism { pub fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(match self { Mechanism::Plain => b"PLAIN", Mechanism::CramMd5 => b"CRAM-MD5", Mechanism::DigestMd5 => b"DIGEST-MD5", Mechanism::ScramSha1 => b"SCRAM-SHA-1", Mechanism::ScramSha256 => b"SCRAM-SHA-256", Mechanism::Apop => b"APOP", Mechanism::Ntlm => b"NTLM", Mechanism::Gssapi => b"GSSAPI", Mechanism::Anonymous => b"ANONYMOUS", Mechanism::External => b"EXTERNAL", Mechanism::OAuthBearer => b"OAUTHBEARER", Mechanism::XOauth2 => b"XOAUTH2", }); } pub fn into_bytes(self) -> Vec { let mut buf = Vec::with_capacity(10); self.serialize(&mut buf); buf } } ================================================ FILE: crates/imap-proto/src/protocol/capability.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ImapResponse, authenticate::Mechanism}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Response { pub capabilities: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum Capability { IMAP4rev2, IMAP4rev1, StartTLS, LoginDisabled, Idle, Namespace, Id, Children, MultiAppend, Binary, Unselect, ACL, UIDPlus, ESearch, SASLIR, //SASL-IR Within, Enable, SearchRes, Sort, Thread, //THREAD=REFERENCES ListExtended, //LIST-EXTENDED ListStatus, //LIST-STATUS ESort, SortDisplay, //SORT=DISPLAY SpecialUse, //SPECIAL-USE CreateSpecialUse, //CREATE-SPECIAL-USEE Move, CondStore, QResync, LiteralPlus, //LITERAL+ UnAuthenticate, StatusSize, //STATUS=SIZE ObjectId, Preview, Utf8Accept, Auth(Mechanism), Quota, QuotaResource(QuotaResourceName), QuotaSet, JmapAccess, } /* STORAGE The physical space estimate, in units of 1024 octets, of the mailboxes governed by the quota root. DELETED-STORAGE STATUS request data item and response data item N/A [Alexey_Melnikov] [IESG] [RFC9208, Section 5.1] MESSAGE The number of messages stored within the mailboxes governed by the quota root. DELETED STATUS request data item and response data item N/A [Alexey_Melnikov] [IESG] [RFC9208, Section 5.2] MAILBOX The number of mailboxes governed by the quota root. N/A N/A [Alexey_Melnikov] [IESG] [RFC9208, Section 5.3] ANNOTATION-STORAGE */ #[derive(Debug, Clone, PartialEq, Eq)] pub enum QuotaResourceName { Storage, Message, Mailbox, AnnotationStorage, } impl Capability { pub fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(match self { Capability::Auth(mechanism) => { buf.extend_from_slice(b"AUTH="); mechanism.serialize(buf); return; } Capability::IMAP4rev2 => b"IMAP4rev2", Capability::IMAP4rev1 => b"IMAP4rev1", Capability::StartTLS => b"STARTTLS", Capability::LoginDisabled => b"LOGINDISABLED", Capability::CondStore => b"CONDSTORE", Capability::QResync => b"QRESYNC", Capability::LiteralPlus => b"LITERAL+", Capability::UnAuthenticate => b"UNAUTHENTICATE", Capability::StatusSize => b"STATUS=SIZE", Capability::ObjectId => b"OBJECTID", Capability::Preview => b"PREVIEW", Capability::Idle => b"IDLE", Capability::Namespace => b"NAMESPACE", Capability::Id => b"ID", Capability::Children => b"CHILDREN", Capability::MultiAppend => b"MULTIAPPEND", Capability::Binary => b"BINARY", Capability::Unselect => b"UNSELECT", Capability::ACL => b"ACL", Capability::UIDPlus => b"UIDPLUS", Capability::ESearch => b"ESEARCH", Capability::SASLIR => b"SASL-IR", Capability::Within => b"WITHIN", Capability::Enable => b"ENABLE", Capability::SearchRes => b"SEARCHRES", Capability::Sort => b"SORT", Capability::Thread => b"THREAD=REFERENCES", Capability::ListExtended => b"LIST-EXTENDED", Capability::ListStatus => b"LIST-STATUS", Capability::ESort => b"ESORT", Capability::SortDisplay => b"SORT=DISPLAY", Capability::SpecialUse => b"SPECIAL-USE", Capability::CreateSpecialUse => b"CREATE-SPECIAL-USE", Capability::Move => b"MOVE", Capability::Utf8Accept => b"UTF8=ACCEPT", Capability::Quota => b"QUOTA", Capability::QuotaResource(quota_resource) => { buf.extend_from_slice(b"QUOTA=RES-"); buf.extend_from_slice(match quota_resource { QuotaResourceName::Storage => b"STORAGE", QuotaResourceName::Message => b"MESSAGE", QuotaResourceName::Mailbox => b"MAILBOX", QuotaResourceName::AnnotationStorage => b"ANNOTATION-STORAGE", }); return; } Capability::QuotaSet => b"QUOTA=SET", Capability::JmapAccess => b"JMAPACCESS", }); } pub fn all_capabilities(is_authenticated: bool, offer_tls: bool) -> Vec { let mut capabilities = vec![ Capability::IMAP4rev2, Capability::IMAP4rev1, Capability::Enable, Capability::SASLIR, Capability::LiteralPlus, Capability::Id, Capability::Utf8Accept, Capability::JmapAccess, ]; if is_authenticated { capabilities.extend([ Capability::Idle, Capability::Namespace, Capability::Children, Capability::MultiAppend, Capability::Binary, Capability::Unselect, Capability::ACL, Capability::UIDPlus, Capability::ESearch, Capability::Within, Capability::SearchRes, Capability::Sort, Capability::Thread, Capability::ListExtended, Capability::ListStatus, Capability::ESort, Capability::SortDisplay, Capability::SpecialUse, Capability::CreateSpecialUse, Capability::Move, Capability::CondStore, Capability::QResync, Capability::UnAuthenticate, Capability::StatusSize, Capability::ObjectId, Capability::Preview, Capability::Quota, Capability::QuotaResource(QuotaResourceName::Storage), ]); } else { capabilities.extend([ Capability::Auth(Mechanism::Plain), Capability::Auth(Mechanism::OAuthBearer), Capability::Auth(Mechanism::XOauth2), ]); } if offer_tls { capabilities.push(Capability::StartTLS); } capabilities } } impl ImapResponse for Response { fn serialize(self) -> Vec { let mut buf = Vec::with_capacity(64); buf.extend_from_slice(b"* CAPABILITY"); for capability in self.capabilities.iter() { buf.push(b' '); capability.serialize(&mut buf); } buf.extend_from_slice(b"\r\n"); buf } } #[cfg(test)] mod tests { use crate::protocol::{ ImapResponse, capability::{Capability, Response}, }; #[test] fn serialize_capability() { assert_eq!( &Response { capabilities: vec![ Capability::IMAP4rev2, Capability::StartTLS, Capability::LoginDisabled ], } .serialize(), "* CAPABILITY IMAP4rev2 STARTTLS LOGINDISABLED\r\n".as_bytes() ); } } ================================================ FILE: crates/imap-proto/src/protocol/copy_move.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::Sequence; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { pub tag: String, pub sequence_set: Sequence, pub mailbox_name: String, } ================================================ FILE: crates/imap-proto/src/protocol/create.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::list::Attribute; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { pub tag: String, pub mailbox_name: String, pub mailbox_role: Option, } ================================================ FILE: crates/imap-proto/src/protocol/delete.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { pub tag: String, pub mailbox_name: String, } ================================================ FILE: crates/imap-proto/src/protocol/enable.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ImapResponse, capability::Capability}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { pub tag: String, pub capabilities: Vec, } pub struct Response { pub enabled: Vec, } impl ImapResponse for Response { fn serialize(self) -> Vec { if !self.enabled.is_empty() { let mut buf = Vec::with_capacity(64); buf.extend(b"* ENABLED"); for capability in self.enabled { buf.push(b' '); capability.serialize(&mut buf); } buf.push(b'\r'); buf.push(b'\n'); buf } else { Vec::new() } } } ================================================ FILE: crates/imap-proto/src/protocol/expunge.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ImapResponse, serialize_sequence}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Response { pub is_qresync: bool, pub ids: Vec, } impl ImapResponse for Response { fn serialize(self) -> Vec { let mut buf = Vec::with_capacity(64); self.serialize_to(&mut buf); buf } } impl Response { pub fn serialize_to(self, buf: &mut Vec) { if !self.is_qresync { for (num_deletions, id) in self.ids.into_iter().enumerate() { buf.extend_from_slice(b"* "); buf.extend_from_slice( id.saturating_sub(num_deletions as u32) .to_string() .as_bytes(), ); buf.extend_from_slice(b" EXPUNGE\r\n"); } } else { Vanished { earlier: false, ids: self.ids, } .serialize(buf); } } } #[derive(Debug, Clone, PartialEq, Eq)] pub struct Vanished { pub earlier: bool, pub ids: Vec, } impl Vanished { pub fn serialize(&self, buf: &mut Vec) { if self.earlier { buf.extend_from_slice(b"* VANISHED (EARLIER) "); } else { buf.extend_from_slice(b"* VANISHED "); } serialize_sequence(buf, &self.ids); buf.extend_from_slice(b"\r\n"); } } #[cfg(test)] mod tests { use crate::protocol::ImapResponse; #[test] fn serialize_expunge() { assert_eq!( String::from_utf8( super::Response { is_qresync: false, ids: vec![3, 4, 5] } .serialize() ) .unwrap(), concat!("* 3 EXPUNGE\r\n", "* 3 EXPUNGE\r\n", "* 3 EXPUNGE\r\n",) ); assert_eq!( String::from_utf8( super::Response { is_qresync: false, ids: vec![3, 4, 7, 9, 11] } .serialize() ) .unwrap(), concat!( "* 3 EXPUNGE\r\n", "* 3 EXPUNGE\r\n", "* 5 EXPUNGE\r\n", "* 6 EXPUNGE\r\n", "* 7 EXPUNGE\r\n", ) ); assert_eq!( String::from_utf8( super::Response { is_qresync: true, ids: vec![3, 4, 5] } .serialize() ) .unwrap(), "* VANISHED 3:5\r\n" ); } } ================================================ FILE: crates/imap-proto/src/protocol/fetch.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::borrow::Cow; use mail_parser::DateTime; use utils::chained_bytes::SliceRange; use crate::protocol::literal_string_slice; use super::{ Flag, ImapResponse, Sequence, literal_string, quoted_or_literal_string, quoted_or_literal_string_or_nil, quoted_rfc2822_or_nil, quoted_timestamp, }; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { pub tag: String, pub sequence_set: Sequence, pub attributes: Vec, pub changed_since: Option, pub include_vanished: bool, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct Response<'x> { pub is_uid: bool, pub items: Vec>, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct FetchItem<'x> { pub id: u32, pub items: Vec>, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum Attribute { Envelope, Flags, InternalDate, Rfc822, Rfc822Size, Rfc822Header, Rfc822Text, Body, BodyStructure, BodySection { peek: bool, sections: Vec
, partial: Option<(u32, u32)>, }, Uid, Binary { peek: bool, sections: Vec, partial: Option<(u32, u32)>, }, BinarySize { sections: Vec, }, Preview { lazy: bool, }, ModSeq, EmailId, ThreadId, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum Section { Part { num: u32 }, Header, HeaderFields { not: bool, fields: Vec }, Text, Mime, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum DataItem<'x> { Binary { sections: Vec, offset: Option, contents: BodyContents<'x>, }, BinarySize { sections: Vec, size: usize, }, Body { part: BodyPart<'x>, }, BodyStructure { part: BodyPart<'x>, }, BodySection { sections: Vec
, origin_octet: Option, contents: Cow<'x, [u8]>, }, Envelope { envelope: Envelope<'x>, }, Flags { flags: Vec, }, InternalDate { date: i64, }, Uid { uid: u32, }, Rfc822 { contents: SliceRange<'x>, }, Rfc822Header { contents: SliceRange<'x>, }, Rfc822Size { size: usize, }, Rfc822Text { contents: SliceRange<'x>, }, Preview { contents: Option>, }, ModSeq { modseq: u64, }, EmailId { email_id: String, }, ThreadId { thread_id: String, }, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum Address<'x> { Single(EmailAddress<'x>), Group(AddressGroup<'x>), } #[derive(Debug, Clone, PartialEq, Eq)] pub struct AddressGroup<'x> { pub name: Option>, pub addresses: Vec>, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct EmailAddress<'x> { pub name: Option>, pub address: Cow<'x, str>, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum BodyContents<'x> { Text(Cow<'x, str>), Bytes(Cow<'x, [u8]>), } #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct Envelope<'x> { pub date: Option, pub subject: Option>, pub from: Vec>, pub sender: Vec>, pub reply_to: Vec>, pub to: Vec>, pub cc: Vec>, pub bcc: Vec>, pub in_reply_to: Option>, pub message_id: Option>, } #[derive(Debug, Clone, PartialEq, Eq)] #[allow(clippy::type_complexity)] pub enum BodyPart<'x> { Multipart { body_parts: Vec>, body_subtype: Cow<'x, str>, // Extension data body_parameters: Option, Cow<'x, str>)>>, extension: BodyPartExtension<'x>, }, Basic { body_type: Option>, fields: BodyPartFields<'x>, // Extension data body_md5: Option>, extension: BodyPartExtension<'x>, }, Text { fields: BodyPartFields<'x>, body_size_lines: usize, // Extension data body_md5: Option>, extension: BodyPartExtension<'x>, }, Message { fields: BodyPartFields<'x>, envelope: Option>>, body: Option>>, body_size_lines: usize, // Extension data body_md5: Option>, extension: BodyPartExtension<'x>, }, } #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct BodyPartFields<'x> { pub body_subtype: Option>, pub body_parameters: Option, Cow<'x, str>)>>, pub body_id: Option>, pub body_description: Option>, pub body_encoding: Option>, pub body_size_octets: usize, } #[derive(Debug, Clone, PartialEq, Eq, Default)] #[allow(clippy::type_complexity)] pub struct BodyPartExtension<'x> { pub body_disposition: Option<(Cow<'x, str>, Vec<(Cow<'x, str>, Cow<'x, str>)>)>, pub body_language: Option>>, pub body_location: Option>, } impl Address<'_> { pub fn serialize(&self, buf: &mut Vec) { match self { Address::Single(addr) => addr.serialize(buf), Address::Group(addr) => addr.serialize(buf), } } pub fn into_owned<'y>(self) -> Address<'y> { match self { Address::Single(addr) => Address::Single(addr.into_owned()), Address::Group(addr) => Address::Group(addr.into_owned()), } } } impl EmailAddress<'_> { pub fn serialize(&self, buf: &mut Vec) { buf.push(b'('); if let Some(name) = &self.name { quoted_or_literal_string(buf, name); } else { buf.extend_from_slice(b"NIL"); } let addr = if let Some((route, addr)) = self.address.split_once(':') { buf.push(b' '); quoted_or_literal_string(buf, route); buf.push(b' '); addr } else { buf.extend_from_slice(b" NIL "); &self.address }; if let Some((local, host)) = addr.rsplit_once('@') { quoted_or_literal_string(buf, local); buf.push(b' '); quoted_or_literal_string(buf, host); } else { quoted_or_literal_string(buf, &self.address); buf.extend_from_slice(b" \"\""); } buf.push(b')'); } pub fn into_owned<'y>(self) -> EmailAddress<'y> { EmailAddress { name: self.name.map(|n| n.into_owned().into()), address: self.address.into_owned().into(), } } } impl AddressGroup<'_> { pub fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(b"(NIL NIL "); if let Some(name) = &self.name { quoted_or_literal_string(buf, name); } else { buf.extend_from_slice(b"\"\""); } buf.extend_from_slice(b" NIL)"); for addr in &self.addresses { addr.serialize(buf); } buf.extend_from_slice(b"(NIL NIL NIL NIL)"); } pub fn into_owned<'y>(self) -> AddressGroup<'y> { AddressGroup { name: self.name.map(|n| n.into_owned().into()), addresses: self .addresses .into_iter() .map(|addr| addr.into_owned()) .collect(), } } } impl<'x> BodyPart<'x> { pub fn serialize(&self, buf: &mut Vec, is_extended: bool) { buf.push(b'('); match self { BodyPart::Multipart { body_parts, body_subtype, body_parameters, extension, } => { for part in body_parts.iter() { part.serialize(buf, is_extended); } buf.push(b' '); quoted_or_literal_string(buf, body_subtype); if is_extended { if let Some(body_parameters) = body_parameters { buf.extend_from_slice(b" ("); for (pos, (key, value)) in body_parameters.iter().enumerate() { if pos > 0 { buf.push(b' '); } quoted_or_literal_string(buf, key); buf.push(b' '); quoted_or_literal_string(buf, value); } buf.push(b')'); } else { buf.extend_from_slice(b" NIL"); } buf.push(b' '); extension.serialize(buf); } } BodyPart::Basic { body_type, fields, body_md5, extension, } => { quoted_or_literal_string_or_nil(buf, body_type.as_deref()); buf.push(b' '); fields.serialize(buf); if is_extended { buf.push(b' '); quoted_or_literal_string_or_nil(buf, body_md5.as_deref()); buf.push(b' '); extension.serialize(buf); } } BodyPart::Text { fields, body_size_lines, body_md5, extension, } => { buf.extend_from_slice(b"\"text\" "); fields.serialize(buf); buf.push(b' '); buf.extend_from_slice(body_size_lines.to_string().as_bytes()); if is_extended { buf.push(b' '); quoted_or_literal_string_or_nil(buf, body_md5.as_deref()); buf.push(b' '); extension.serialize(buf); } } BodyPart::Message { fields, envelope, body, body_size_lines, body_md5, extension, } => { buf.extend_from_slice(b"\"message\" "); fields.serialize(buf); buf.push(b' '); if let Some(envelope) = envelope { envelope.serialize(buf); } else { buf.extend_from_slice(b"NIL"); } buf.push(b' '); if let Some(body) = body { body.serialize(buf, is_extended); } else { buf.extend_from_slice(b"NIL"); } buf.push(b' '); buf.extend_from_slice(body_size_lines.to_string().as_bytes()); if is_extended { buf.push(b' '); quoted_or_literal_string_or_nil(buf, body_md5.as_deref()); buf.push(b' '); extension.serialize(buf); } } } buf.push(b')'); } pub fn add_part(&mut self, part: BodyPart<'x>) { match self { BodyPart::Multipart { body_parts, .. } => body_parts.push(part), BodyPart::Message { body, .. } => *body = Box::new(part).into(), _ => debug_assert!(false, "Cannot add a part to a non-multipart body part"), } } pub fn set_envelope(&mut self, envelope_: Envelope<'x>) { match self { BodyPart::Message { envelope, .. } => *envelope = Some(Box::new(envelope_)), _ => debug_assert!(false, "Cannot set envelope on a non-message body part"), } } pub fn into_owned<'y>(self) -> BodyPart<'y> { match self { BodyPart::Multipart { body_parts, body_subtype, body_parameters, extension, } => BodyPart::Multipart { body_parts: body_parts.into_iter().map(|v| v.into_owned()).collect(), body_subtype: body_subtype.into_owned().into(), body_parameters: body_parameters.map(|b| { b.into_iter() .map(|(k, v)| (k.into_owned().into(), v.into_owned().into())) .collect::>() }), extension: extension.into_owned(), }, BodyPart::Basic { body_type, fields, body_md5, extension, } => BodyPart::Basic { body_type: body_type.map(|v| v.into_owned().into()), fields: fields.into_owned(), body_md5: body_md5.map(|v| v.into_owned().into()), extension: extension.into_owned(), }, BodyPart::Text { fields, body_size_lines, body_md5, extension, } => BodyPart::Text { fields: fields.into_owned(), body_size_lines, body_md5: body_md5.map(|v| v.into_owned().into()), extension: extension.into_owned(), }, BodyPart::Message { fields, envelope, body, body_size_lines, body_md5, extension, } => BodyPart::Message { fields: fields.into_owned(), envelope: envelope.map(|v| Box::new(v.into_owned())), body: body.map(|b| Box::new(b.into_owned())), body_size_lines, body_md5: body_md5.map(|v| v.into_owned().into()), extension: extension.into_owned(), }, } } } impl BodyPartFields<'_> { pub fn serialize(&self, buf: &mut Vec) { quoted_or_literal_string_or_nil(buf, self.body_subtype.as_deref()); if let Some(body_parameters) = &self.body_parameters { buf.extend_from_slice(b" ("); for (pos, (key, value)) in body_parameters.iter().enumerate() { if pos > 0 { buf.push(b' '); } quoted_or_literal_string(buf, key); buf.push(b' '); quoted_or_literal_string(buf, value); } buf.push(b')'); } else { buf.extend_from_slice(b" NIL"); } for item in [&self.body_id, &self.body_description, &self.body_encoding] { buf.push(b' '); quoted_or_literal_string_or_nil(buf, item.as_deref()); } buf.push(b' '); buf.extend_from_slice(self.body_size_octets.to_string().as_bytes()); } pub fn into_owned<'y>(self) -> BodyPartFields<'y> { BodyPartFields { body_subtype: self.body_subtype.map(|v| v.into_owned().into()), body_parameters: self.body_parameters.map(|b| { b.into_iter() .map(|(k, v)| (k.into_owned().into(), v.into_owned().into())) .collect::>() }), body_id: self.body_id.map(|v| v.into_owned().into()), body_description: self.body_description.map(|v| v.into_owned().into()), body_encoding: self.body_encoding.map(|v| v.into_owned().into()), body_size_octets: self.body_size_octets, } } } impl BodyPartExtension<'_> { pub fn serialize(&self, buf: &mut Vec) { if let Some((disposition, parameters)) = &self.body_disposition { buf.push(b'('); quoted_or_literal_string(buf, disposition); if !parameters.is_empty() { buf.extend_from_slice(b" ("); for (pos, (key, value)) in parameters.iter().enumerate() { if pos > 0 { buf.push(b' '); } quoted_or_literal_string(buf, key); buf.push(b' '); quoted_or_literal_string(buf, value); } buf.extend_from_slice(b"))"); } else { buf.extend_from_slice(b" NIL)"); } } else { buf.extend_from_slice(b"NIL"); } if let Some(body_language) = &self.body_language { match body_language.len() { 0 => buf.extend_from_slice(b" NIL"), 1 => { buf.push(b' '); quoted_or_literal_string(buf, body_language.last().unwrap()); } _ => { buf.extend_from_slice(b" ("); for (pos, lang) in body_language.iter().enumerate() { if pos > 0 { buf.push(b' '); } quoted_or_literal_string(buf, lang); } buf.push(b')'); } } } else { buf.extend_from_slice(b" NIL"); } buf.push(b' '); quoted_or_literal_string_or_nil(buf, self.body_location.as_deref()); } pub fn into_owned<'y>(self) -> BodyPartExtension<'y> { BodyPartExtension { body_disposition: self.body_disposition.map(|(a, b)| { ( a.into_owned().into(), b.into_iter() .map(|(k, v)| (k.into_owned().into(), v.into_owned().into())) .collect::>(), ) }), body_language: self .body_language .map(|v| v.into_iter().map(|a| a.into_owned().into()).collect()), body_location: self.body_location.map(|v| v.into_owned().into()), } } } impl BodyContents<'_> { pub fn into_owned<'y>(self) -> BodyContents<'y> { match self { BodyContents::Text(text) => BodyContents::Text(text.into_owned().into()), BodyContents::Bytes(bytes) => BodyContents::Bytes(bytes.into_owned().into()), } } } impl Section { pub fn serialize(&self, buf: &mut Vec) { match self { Section::Part { num } => { buf.extend_from_slice(num.to_string().as_bytes()); } Section::Header => { buf.extend_from_slice(b"HEADER"); } Section::HeaderFields { not, fields } => { if !not { buf.extend_from_slice(b"HEADER.FIELDS "); } else { buf.extend_from_slice(b"HEADER.FIELDS.NOT "); } buf.push(b'('); for (pos, field) in fields.iter().enumerate() { if pos > 0 { buf.push(b' '); } buf.extend_from_slice(field.as_str().to_ascii_uppercase().as_bytes()); } buf.push(b')'); } Section::Text => { buf.extend_from_slice(b"TEXT"); } Section::Mime => { buf.extend_from_slice(b"MIME"); } }; } } static DUMMY_ADDRESS: [Address; 1] = [Address::Single(EmailAddress { name: None, address: Cow::Borrowed("unknown@localhost"), })]; impl Envelope<'_> { pub fn serialize(&self, buf: &mut Vec) { buf.push(b'('); quoted_rfc2822_or_nil(buf, &self.date); buf.push(b' '); quoted_or_literal_string_or_nil(buf, self.subject.as_deref()); // Note: [RFC-2822] requires that all messages have a valid // From header. Therefore, the from, sender, and reply-to // members in the envelope can not be NIL. let from = if !self.from.is_empty() { &self.from[..] } else { &DUMMY_ADDRESS[..] }; self.serialize_addresses(buf, from); self.serialize_addresses( buf, if !self.sender.is_empty() { &self.sender } else { from }, ); self.serialize_addresses( buf, if !self.reply_to.is_empty() { &self.reply_to } else { from }, ); self.serialize_addresses(buf, &self.to); self.serialize_addresses(buf, &self.cc); self.serialize_addresses(buf, &self.bcc); for item in [&self.in_reply_to, &self.message_id] { buf.push(b' '); quoted_or_literal_string_or_nil(buf, item.as_deref()); } buf.push(b')'); } fn serialize_addresses(&self, buf: &mut Vec, addresses: &[Address]) { buf.push(b' '); if !addresses.is_empty() { buf.push(b'('); for address in addresses { address.serialize(buf); } buf.push(b')'); } else { buf.extend_from_slice(b"NIL"); } } pub fn into_owned<'y>(self) -> Envelope<'y> { Envelope { date: self.date, subject: self.subject.map(|v| v.into_owned().into()), from: self.from.into_iter().map(|v| v.into_owned()).collect(), sender: self.sender.into_iter().map(|v| v.into_owned()).collect(), reply_to: self.reply_to.into_iter().map(|v| v.into_owned()).collect(), to: self.to.into_iter().map(|v| v.into_owned()).collect(), cc: self.cc.into_iter().map(|v| v.into_owned()).collect(), bcc: self.bcc.into_iter().map(|v| v.into_owned()).collect(), in_reply_to: self.in_reply_to.map(|v| v.into_owned().into()), message_id: self.message_id.map(|v| v.into_owned().into()), } } } impl DataItem<'_> { pub fn serialize(&self, buf: &mut Vec) { match self { DataItem::Binary { sections, offset, contents, } => { buf.extend_from_slice(b"BINARY["); for (pos, section) in sections.iter().enumerate() { if pos > 0 { buf.push(b'.'); } buf.extend_from_slice(section.to_string().as_bytes()); } if let Some(offset) = offset { buf.extend_from_slice(b"]<"); buf.extend_from_slice(offset.to_string().as_bytes()); buf.extend_from_slice(b"> "); } else { buf.extend_from_slice(b"] "); } match contents { BodyContents::Text(text) => { literal_string(buf, text.as_bytes()); } BodyContents::Bytes(bytes) => { buf.extend_from_slice(b"~{"); buf.extend_from_slice(bytes.len().to_string().as_bytes()); buf.extend_from_slice(b"}\r\n"); buf.extend_from_slice(bytes); } } } DataItem::BinarySize { sections, size } => { buf.extend_from_slice(b"BINARY.SIZE["); for (pos, section) in sections.iter().enumerate() { if pos > 0 { buf.push(b'.'); } buf.extend_from_slice(section.to_string().as_bytes()); } buf.extend_from_slice(b"] "); buf.extend_from_slice(size.to_string().as_bytes()); } DataItem::Body { part } => { buf.extend_from_slice(b"BODY "); part.serialize(buf, false); } DataItem::BodyStructure { part } => { buf.extend_from_slice(b"BODYSTRUCTURE "); part.serialize(buf, true); } DataItem::BodySection { sections, origin_octet, contents, } => { buf.extend_from_slice(b"BODY["); for (pos, section) in sections.iter().enumerate() { if pos > 0 { buf.push(b'.'); } section.serialize(buf); } if let Some(origin_octet) = origin_octet { buf.extend_from_slice(b"]<"); buf.extend_from_slice(origin_octet.to_string().as_bytes()); buf.extend_from_slice(b"> "); } else { buf.extend_from_slice(b"] "); } literal_string(buf, contents); } DataItem::Envelope { envelope } => { buf.extend_from_slice(b"ENVELOPE "); envelope.serialize(buf); } DataItem::Flags { flags } => { buf.extend_from_slice(b"FLAGS ("); for (pos, flag) in flags.iter().enumerate() { if pos > 0 { buf.push(b' '); } flag.serialize(buf); } buf.push(b')'); } DataItem::InternalDate { date } => { buf.extend_from_slice(b"INTERNALDATE "); quoted_timestamp(buf, *date); } DataItem::Uid { uid } => { buf.extend_from_slice(b"UID "); buf.extend_from_slice(uid.to_string().as_bytes()); } DataItem::Rfc822 { contents } => { buf.extend_from_slice(b"RFC822 "); literal_string_slice(buf, contents); } DataItem::Rfc822Header { contents } => { buf.extend_from_slice(b"RFC822.HEADER "); literal_string_slice(buf, contents); } DataItem::Rfc822Size { size } => { buf.extend_from_slice(b"RFC822.SIZE "); buf.extend_from_slice(size.to_string().as_bytes()); } DataItem::Rfc822Text { contents } => { buf.extend_from_slice(b"RFC822.TEXT "); literal_string_slice(buf, contents); } DataItem::Preview { contents } => { buf.extend_from_slice(b"PREVIEW "); if let Some(contents) = contents { literal_string(buf, contents); } else { buf.extend_from_slice(b"NIL"); } } DataItem::ModSeq { modseq } => { buf.extend_from_slice(b"MODSEQ ("); buf.extend_from_slice(modseq.to_string().as_bytes()); buf.push(b')'); } DataItem::EmailId { email_id } => { buf.extend_from_slice(b"EMAILID ("); buf.extend_from_slice(email_id.as_bytes()); buf.push(b')'); } DataItem::ThreadId { thread_id } => { buf.extend_from_slice(b"THREADID ("); buf.extend_from_slice(thread_id.as_bytes()); buf.push(b')'); } } } } impl FetchItem<'_> { pub fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(b"* "); buf.extend_from_slice(self.id.to_string().as_bytes()); buf.extend_from_slice(b" FETCH ("); for (pos, item) in self.items.iter().enumerate() { if pos > 0 { buf.push(b' '); } item.serialize(buf); } buf.extend_from_slice(b")\r\n"); } } impl ImapResponse for Response<'_> { fn serialize(self) -> Vec { let mut buf = Vec::with_capacity(128); for item in &self.items { item.serialize(&mut buf); } buf } } /* body = "(" (body-type-1part / body-type-mpart) ")" body-type-1part = (body-type-basic / body-type-msg / body-type-text) [SP body-ext-1part] body-type-basic = media-basic SP body-fields ; MESSAGE subtype MUST NOT be "RFC822" or ; "GLOBAL" body-type-mpart = 1*body SP media-subtype [SP body-ext-mpart] ; MULTIPART body part body-type-msg = media-message SP body-fields SP envelope SP body SP body-fld-lines body-type-text = media-text SP body-fields SP body-fld-lines body-fields = body-fld-param SP body-fld-id SP body-fld-desc SP body-fld-enc SP body-fld-octets media-message = DQUOTE "MESSAGE" DQUOTE SP DQUOTE ("RFC822" / "GLOBAL") DQUOTE ; Defined in [MIME-IMT] media-basic = ((DQUOTE ("APPLICATION" / "AUDIO" / "IMAGE" / "FONT" / "MESSAGE" / "MODEL" / "VIDEO" ) DQUOTE) / string) SP media-subtype envelope = "(" env-date SP env-subject SP env-from SP env-sender SP env-reply-to SP env-to SP env-cc SP env-bcc SP env-in-reply-to SP env-message-id ")" body-fld-lines = number64 */ #[cfg(test)] mod tests { use mail_parser::DateTime; use utils::chained_bytes::SliceRange; use crate::protocol::{Flag, ImapResponse}; use super::{ Address, AddressGroup, BodyPart, BodyPartExtension, BodyPartFields, DataItem, EmailAddress, Envelope, FetchItem, Response, Section, }; #[test] fn serialize_fetch_data_item() { for (item, expected_response) in [ ( super::DataItem::Envelope { envelope: Envelope { date: DateTime::from_timestamp(837570205).into(), subject: Some("IMAP4rev2 WG mtg summary and minutes".into()), from: vec![Address::Single(EmailAddress { name: Some("Terry Gray".into()), address: "gray@cac.washington.edu".into(), })], sender: vec![Address::Single(EmailAddress { name: Some("Terry Gray".into()), address: "gray@cac.washington.edu".into(), })], reply_to: vec![Address::Single(EmailAddress { name: Some("Terry Gray".into()), address: "gray@cac.washington.edu".into(), })], to: vec![Address::Single(EmailAddress { name: None, address: "imap@cac.washington.edu".into(), })], cc: vec![ Address::Single(EmailAddress { name: None, address: "minutes@CNRI.Reston.VA.US".into(), }), Address::Single(EmailAddress { name: Some("John Klensin".into()), address: "KLENSIN@MIT.EDU".into(), }), ], bcc: vec![], in_reply_to: None, message_id: Some("".into()), }, }, concat!( "ENVELOPE (\"Wed, 17 Jul 1996 02:23:25 +0000\" ", "\"IMAP4rev2 WG mtg summary and minutes\" ", "((\"Terry Gray\" NIL \"gray\" \"cac.washington.edu\")) ", "((\"Terry Gray\" NIL \"gray\" \"cac.washington.edu\")) ", "((\"Terry Gray\" NIL \"gray\" \"cac.washington.edu\")) ", "((NIL NIL \"imap\" \"cac.washington.edu\")) ", "((NIL NIL \"minutes\" \"CNRI.Reston.VA.US\")", "(\"John Klensin\" NIL \"KLENSIN\" \"MIT.EDU\")) NIL NIL ", "\"\")" ), ), ( super::DataItem::Envelope { envelope: Envelope { date: DateTime::from_timestamp(837570205).into(), subject: Some("Group test".into()), from: vec![Address::Single(EmailAddress { name: Some("Bill Foobar".into()), address: "foobar@example.com".into(), })], sender: vec![], reply_to: vec![], to: vec![Address::Group(AddressGroup { name: Some("Friends and Family".into()), addresses: vec![ EmailAddress { name: Some("John Doe".into()), address: "jdoe@example.com".into(), }, EmailAddress { name: Some("Jane Smith".into()), address: "jane.smith@example.com".into(), }, ], })], cc: vec![], bcc: vec![], in_reply_to: None, message_id: Some("".into()), }, }, concat!( "ENVELOPE (\"Wed, 17 Jul 1996 02:23:25 +0000\" ", "\"Group test\" ", "((\"Bill Foobar\" NIL \"foobar\" \"example.com\")) ", "((\"Bill Foobar\" NIL \"foobar\" \"example.com\")) ", "((\"Bill Foobar\" NIL \"foobar\" \"example.com\")) ", "((NIL NIL \"Friends and Family\" NIL)", "(\"John Doe\" NIL \"jdoe\" \"example.com\")", "(\"Jane Smith\" NIL \"jane.smith\" \"example.com\")", "(NIL NIL NIL NIL)) ", "NIL NIL NIL \"\")" ), ), ( super::DataItem::Body { part: BodyPart::Text { fields: BodyPartFields { body_subtype: Some("PLAIN".into()), body_parameters: vec![("CHARSET".into(), "US-ASCII".into())].into(), body_id: None, body_description: None, body_encoding: Some("7BIT".into()), body_size_octets: 2279, }, body_size_lines: 48, body_md5: None, extension: BodyPartExtension { body_disposition: None, body_language: None, body_location: None, }, }, }, "BODY (\"text\" \"PLAIN\" (\"CHARSET\" \"US-ASCII\") NIL NIL \"7BIT\" 2279 48)", ), ( super::DataItem::Body { part: BodyPart::Message { fields: BodyPartFields { body_subtype: Some("RFC822".into()), body_parameters: None, body_id: Some("".into()), body_description: Some("An attached email".into()), body_encoding: Some("quoted-printable".into()), body_size_octets: 9323, }, envelope: Box::new(Envelope { date: DateTime::from_timestamp(837570205).into(), subject: Some("Hello world!".into()), from: vec![Address::Single(EmailAddress { name: Some("Terry Gray".into()), address: "gray@cac.washington.edu".into(), })], sender: vec![Address::Single(EmailAddress { name: Some("Terry Gray".into()), address: "gray@cac.washington.edu".into(), })], reply_to: vec![Address::Single(EmailAddress { name: Some("Terry Gray".into()), address: "gray@cac.washington.edu".into(), })], to: vec![Address::Single(EmailAddress { name: None, address: "imap@cac.washington.edu".into(), })], cc: vec![], bcc: vec![], in_reply_to: None, message_id: Some("<4234324@domain.com>".into()), }) .into(), body: Box::new(BodyPart::Text { fields: BodyPartFields { body_subtype: Some("HTML".into()), body_parameters: None, body_id: None, body_description: None, body_encoding: Some("8BIT".into()), body_size_octets: 4234, }, body_size_lines: 431, body_md5: None, extension: BodyPartExtension { body_disposition: None, body_language: None, body_location: None, }, }) .into(), body_size_lines: 908, body_md5: None, extension: BodyPartExtension { body_disposition: None, body_language: None, body_location: None, }, }, }, concat!( "BODY (\"message\" \"RFC822\" NIL \"\" \"An attached email\" ", "\"quoted-printable\" 9323 (\"Wed, 17 Jul 1996 02:23:25 +0000\" ", "\"Hello world!\" ", "((\"Terry Gray\" NIL \"gray\" \"cac.washington.edu\")) ", "((\"Terry Gray\" NIL \"gray\" \"cac.washington.edu\")) ", "((\"Terry Gray\" NIL \"gray\" \"cac.washington.edu\")) ", "((NIL NIL \"imap\" \"cac.washington.edu\")) NIL NIL NIL ", "\"<4234324@domain.com>\") (\"text\" \"HTML\" NIL NIL NIL ", "\"8BIT\" 4234 431) 908)" ), ), ( super::DataItem::Body { part: BodyPart::Multipart { body_parts: vec![ BodyPart::Text { fields: BodyPartFields { body_subtype: Some("PLAIN".into()), body_parameters: vec![("CHARSET".into(), "US-ASCII".into())] .into(), body_id: None, body_description: None, body_encoding: Some("7BIT".into()), body_size_octets: 1152, }, body_size_lines: 23, body_md5: None, extension: BodyPartExtension { body_disposition: None, body_language: None, body_location: None, }, }, BodyPart::Text { fields: BodyPartFields { body_subtype: Some("PLAIN".into()), body_parameters: vec![ ("CHARSET".into(), "US-ASCII".into()), ("NAME".into(), "cc.diff".into()), ] .into(), body_id: Some( "<960723163407.20117h@cac.washington.edu>".into(), ), body_description: Some("Compiler diff".into()), body_encoding: Some("BASE64".into()), body_size_octets: 4554, }, body_size_lines: 73, body_md5: None, extension: BodyPartExtension { body_disposition: None, body_language: None, body_location: None, }, }, ], body_subtype: "MIXED".into(), body_parameters: None, extension: BodyPartExtension { body_disposition: None, body_language: None, body_location: None, }, }, }, concat!( "BODY ((\"text\" \"PLAIN\" (\"CHARSET\" \"US-ASCII\") ", "NIL NIL \"7BIT\" 1152 23)", "(\"text\" \"PLAIN\" (\"CHARSET\" \"US-ASCII\" \"NAME\" \"cc.diff\") ", "\"<960723163407.20117h@cac.washington.edu>\" \"Compiler diff\" ", "\"BASE64\" 4554 73) \"MIXED\")", ), ), ( DataItem::BodyStructure { part: BodyPart::Multipart { body_parts: vec![ BodyPart::Multipart { body_parts: vec![ BodyPart::Text { fields: BodyPartFields { body_subtype: Some("PLAIN".into()), body_parameters: vec![( "CHARSET".into(), "UTF-8".into(), )] .into(), body_id: Some("<111@domain.com>".into()), body_description: Some("Text part".into()), body_encoding: Some("7BIT".into()), body_size_octets: 1152, }, body_size_lines: 23, body_md5: Some("8o3456".into()), extension: BodyPartExtension { body_disposition: ("inline".into(), vec![]).into(), body_language: vec!["en-US".into()].into(), body_location: Some("right here".into()), }, }, BodyPart::Text { fields: BodyPartFields { body_subtype: Some("HTML".into()), body_parameters: vec![( "CHARSET".into(), "UTF-8".into(), )] .into(), body_id: Some("<54535@domain.com>".into()), body_description: Some("HTML part".into()), body_encoding: Some("8BIT".into()), body_size_octets: 45345, }, body_size_lines: 994, body_md5: Some("53454".into()), extension: BodyPartExtension { body_disposition: ( "attachment".into(), vec![("filename".into(), "myfile.txt".into())], ) .into(), body_language: vec!["en-US".into(), "de-DE".into()] .into(), body_location: Some("right there".into()), }, }, ], body_subtype: "ALTERNATIVE".into(), body_parameters: vec![( "x-param".into(), "a very special parameter".into(), )] .into(), extension: BodyPartExtension { body_disposition: None, body_language: vec!["en-US".into()].into(), body_location: Some("unknown".into()), }, }, BodyPart::Basic { body_type: Some("APPLICATION".into()), fields: BodyPartFields { body_subtype: Some("MSWORD".into()), body_parameters: vec![( "NAME".into(), "chimichangas.docx".into(), )] .into(), body_id: Some("<4444@chimi.changa>".into()), body_description: Some("Chimichangas recipe".into()), body_encoding: Some("base64".into()), body_size_octets: 84723, }, body_md5: Some("1234".into()), extension: BodyPartExtension { body_disposition: ( "attachment".into(), vec![("filename".into(), "chimichangas.docx".into())], ) .into(), body_language: vec!["en-MX".into()].into(), body_location: Some("secret location".into()), }, }, ], body_subtype: "MIXED".into(), body_parameters: None, extension: BodyPartExtension { body_disposition: None, body_language: None, body_location: None, }, }, }, concat!( "BODYSTRUCTURE (((\"text\" \"PLAIN\" (\"CHARSET\" \"UTF-8\") ", "\"<111@domain.com>\" \"Text part\" \"7BIT\" 1152 23 \"8o3456\" ", "(\"inline\" NIL) \"en-US\" \"right here\")", "(\"text\" \"HTML\" (\"CHARSET\" \"UTF-8\") ", "\"<54535@domain.com>\" \"HTML part\" \"8BIT\" 45345 994 \"53454\" ", "(\"attachment\" (\"filename\" \"myfile.txt\")) ", "(\"en-US\" \"de-DE\") ", "\"right there\") \"ALTERNATIVE\" (\"x-param\" ", "\"a very special parameter\") ", "NIL \"en-US\" \"unknown\")", "(\"APPLICATION\" \"MSWORD\" (\"NAME\" \"chimichangas.docx\") ", "\"<4444@chimi.changa>\" \"Chimichangas recipe\" \"base64\"", " 84723 \"1234\" ", "(\"attachment\" (\"filename\" \"chimichangas.docx\")) \"en-MX\" ", "\"secret location\") \"MIXED\" NIL NIL NIL NIL)", ), ), ( super::DataItem::Binary { sections: vec![1, 2, 3], offset: 10.into(), contents: super::BodyContents::Bytes(b"hello".to_vec().into()), }, "BINARY[1.2.3]<10> ~{5}\r\nhello", ), ( super::DataItem::Binary { sections: vec![1, 2, 3], offset: None, contents: super::BodyContents::Text("hello".into()), }, "BINARY[1.2.3] {5}\r\nhello", ), ( super::DataItem::BodySection { sections: vec![ Section::Part { num: 1 }, Section::Part { num: 2 }, Section::Mime, ], origin_octet: 11.into(), contents: b"howdy"[..].into(), }, "BODY[1.2.MIME]<11> {5}\r\nhowdy", ), ( super::DataItem::BodySection { sections: vec![Section::HeaderFields { not: true, fields: vec!["Subject".into(), "x-special".into()], }], origin_octet: None, contents: b"howdy"[..].into(), }, "BODY[HEADER.FIELDS.NOT (SUBJECT X-SPECIAL)] {5}\r\nhowdy", ), ( super::DataItem::BodySection { sections: vec![Section::HeaderFields { not: false, fields: vec!["From".into(), "List-Archive".into()], }], origin_octet: None, contents: b"howdy"[..].into(), }, "BODY[HEADER.FIELDS (FROM LIST-ARCHIVE)] {5}\r\nhowdy", ), ( super::DataItem::Flags { flags: vec![Flag::Seen], }, "FLAGS (\\Seen)", ), ( super::DataItem::InternalDate { date: 482374938 }, "INTERNALDATE \"15-Apr-1985 01:02:18 +0000\"", ), ] { let mut buf = Vec::with_capacity(100); item.serialize(&mut buf); assert_eq!(String::from_utf8(buf).unwrap(), expected_response); } } #[test] fn serialize_fetch() { assert_eq!( String::from_utf8( Response { is_uid: false, items: vec![FetchItem { id: 123, items: vec![ super::DataItem::Flags { flags: vec![Flag::Deleted, Flag::Flagged], }, super::DataItem::Uid { uid: 983 }, super::DataItem::Rfc822Size { size: 443 }, super::DataItem::Rfc822Text { contents: SliceRange::Single(&b"hi"[..]), }, super::DataItem::Rfc822Header { contents: SliceRange::Single(&b"header"[..]), }, ], }], } .serialize(), ) .unwrap(), concat!( "* 123 FETCH (FLAGS (\\Deleted \\Flagged) ", "UID 983 ", "RFC822.SIZE 443 ", "RFC822.TEXT {2}\r\nhi ", "RFC822.HEADER {6}\r\nheader)\r\n", ) ); } } ================================================ FILE: crates/imap-proto/src/protocol/list.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::utf7::utf7_encode; use super::{ ImapResponse, quoted_string, status::{Status, StatusItem}, }; #[derive(Debug, Clone, PartialEq, Eq)] pub enum Arguments { Basic { tag: String, reference_name: String, mailbox_name: String, }, Extended { tag: String, reference_name: String, mailbox_name: Vec, selection_options: Vec, return_options: Vec, }, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct Response { pub is_rev2: bool, pub is_utf8: bool, pub is_lsub: bool, pub list_items: Vec, pub status_items: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum SelectionOption { Subscribed, Remote, RecursiveMatch, SpecialUse, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum ReturnOption { Subscribed, Children, Status(Vec), SpecialUse, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Attribute { NoInferiors, NoSelect, Marked, Unmarked, NonExistent, HasChildren, HasNoChildren, Subscribed, Remote, All, Archive, Drafts, Flagged, Junk, Sent, Trash, Important, Memos, Scheduled, Snoozed, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum ChildInfo { Subscribed, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum Tag { ChildInfo(Vec), OldName(String), } #[derive(Debug, Clone, PartialEq, Eq)] pub struct ListItem { pub mailbox_name: String, pub attributes: Vec, pub tags: Vec, } impl Arguments { pub fn is_separator_query(&self) -> bool { match self { Arguments::Basic { mailbox_name, reference_name, .. } => mailbox_name.is_empty() && reference_name.is_empty(), Arguments::Extended { mailbox_name, reference_name, .. } => mailbox_name.is_empty() && reference_name.is_empty(), } } pub fn unwrap_tag(self) -> String { match self { Arguments::Basic { tag, .. } => tag, Arguments::Extended { tag, .. } => tag, } } } impl Attribute { pub fn is_rev1(&self) -> bool { matches!( self, Attribute::NoInferiors | Attribute::NoSelect | Attribute::Marked | Attribute::Unmarked ) } pub fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(match self { Attribute::NoInferiors => b"\\NoInferiors", Attribute::NoSelect => b"\\NoSelect", Attribute::Marked => b"\\Marked", Attribute::Unmarked => b"\\Unmarked", Attribute::NonExistent => b"\\NonExistent", Attribute::HasChildren => b"\\HasChildren", Attribute::HasNoChildren => b"\\HasNoChildren", Attribute::Subscribed => b"\\Subscribed", Attribute::Remote => b"\\Remote", Attribute::All => b"\\All", Attribute::Archive => b"\\Archive", Attribute::Drafts => b"\\Drafts", Attribute::Flagged => b"\\Flagged", Attribute::Junk => b"\\Junk", Attribute::Sent => b"\\Sent", Attribute::Trash => b"\\Trash", Attribute::Important => b"\\Important", Attribute::Memos => b"\\Memos", Attribute::Scheduled => b"\\Scheduled", Attribute::Snoozed => b"\\Snoozed", }); } } impl TryFrom<&str> for Attribute { type Error = (); fn try_from(value: &str) -> Result { hashify::tiny_map!(value.as_bytes(), "archive" => Attribute::Archive, "drafts" => Attribute::Drafts, "junk" => Attribute::Junk, "sent" => Attribute::Sent, "trash" => Attribute::Trash, "important" => Attribute::Important, "memos" => Attribute::Memos, "scheduled" => Attribute::Scheduled, "snoozed" => Attribute::Snoozed, ) .ok_or(()) } } impl ChildInfo { pub fn serialize(&self, buf: &mut Vec) { buf.push(b'\"'); buf.extend_from_slice(match self { ChildInfo::Subscribed => b"SUBSCRIBED", }); buf.push(b'\"'); } } impl Tag { pub fn serialize(&self, buf: &mut Vec) { match self { Tag::ChildInfo(child_info) => { buf.extend_from_slice(b"\"CHILDINFO\" ("); for (pos, child_info) in child_info.iter().enumerate() { if pos > 0 { buf.push(b' '); } child_info.serialize(buf); } buf.push(b')'); } Tag::OldName(old_name) => { buf.extend_from_slice(b"\"OLDNAME\" ("); quoted_string(buf, old_name); buf.push(b')'); } } } } impl ListItem { pub fn new(name: impl Into) -> Self { ListItem { mailbox_name: name.into(), attributes: Vec::new(), tags: Vec::new(), } } pub fn serialize(&self, buf: &mut Vec, is_rev2: bool, is_utf8: bool, is_lsub: bool) { let normalized_mailbox_name = utf7_encode(&self.mailbox_name); if !is_lsub { buf.extend_from_slice(b"* LIST ("); } else { buf.extend_from_slice(b"* LSUB ("); } for (pos, attr) in self.attributes.iter().enumerate() { if pos > 0 { buf.push(b' '); } attr.serialize(buf); } buf.extend_from_slice(b") \"/\" "); let mut extra_tags = Vec::new(); if normalized_mailbox_name != self.mailbox_name { if is_rev2 || is_utf8 { quoted_string(buf, &self.mailbox_name); if is_rev2 { extra_tags.push(Tag::OldName(normalized_mailbox_name)); } } else { quoted_string(buf, &normalized_mailbox_name); } } else { quoted_string(buf, &self.mailbox_name); } if !extra_tags.is_empty() || !self.tags.is_empty() { buf.extend_from_slice(b" ("); for (pos, tag) in extra_tags.iter().chain(self.tags.iter()).enumerate() { if pos > 0 { buf.push(b' '); } tag.serialize(buf); } buf.extend_from_slice(b")\r\n"); } else { buf.extend_from_slice(b"\r\n"); } } } impl ImapResponse for Response { fn serialize(self) -> Vec { let mut buf = Vec::with_capacity(100); match (self.list_items.is_empty(), self.status_items.is_empty()) { (false, false) => { for (list_item, status_item) in self.list_items.iter().zip(self.status_items.iter()) { list_item.serialize(&mut buf, self.is_rev2, self.is_utf8, self.is_lsub); status_item.serialize(&mut buf, self.is_rev2); } } (false, true) => { for list_item in &self.list_items { list_item.serialize(&mut buf, self.is_rev2, self.is_utf8, self.is_lsub); } } (true, false) => { for status_item in &self.status_items { status_item.serialize(&mut buf, self.is_rev2); } } _ => (), } buf } } #[cfg(test)] mod tests { use crate::protocol::{ ImapResponse, status::{Status, StatusItem, StatusItemType}, }; use super::{Attribute, ChildInfo, ListItem, Tag}; #[test] fn serialize_list_item() { for (response, expected_v2, expected_v1) in [ ( super::ListItem { mailbox_name: "".into(), attributes: vec![], tags: vec![], }, "* LIST () \"/\" \"\"\r\n", "* LIST () \"/\" \"\"\r\n", ), ( super::ListItem { mailbox_name: "中國書店".into(), attributes: vec![Attribute::NoInferiors, Attribute::Drafts], tags: vec![], }, concat!( "* LIST (\\NoInferiors \\Drafts) \"/\" \"中國書店\" ", "(\"OLDNAME\" (\"&Ti1XC2b4Xpc-\"))\r\n" ), "* LIST (\\NoInferiors \\Drafts) \"/\" \"&Ti1XC2b4Xpc-\"\r\n", ), ( super::ListItem { mailbox_name: "☺".into(), attributes: vec![Attribute::Subscribed, Attribute::Remote], tags: vec![Tag::ChildInfo(vec![ChildInfo::Subscribed])], }, concat!( "* LIST (\\Subscribed \\Remote) \"/\" \"☺\" ", "(\"OLDNAME\" (\"&Jjo-\") \"CHILDINFO\" (\"SUBSCRIBED\"))\r\n" ), concat!( "* LIST (\\Subscribed \\Remote) \"/\" \"&Jjo-\" ", "(\"CHILDINFO\" (\"SUBSCRIBED\"))\r\n" ), ), ( super::ListItem { mailbox_name: "foo".into(), attributes: vec![Attribute::HasNoChildren], tags: vec![Tag::ChildInfo(vec![ChildInfo::Subscribed])], }, "* LIST (\\HasNoChildren) \"/\" \"foo\" (\"CHILDINFO\" (\"SUBSCRIBED\"))\r\n", "* LIST (\\HasNoChildren) \"/\" \"foo\" (\"CHILDINFO\" (\"SUBSCRIBED\"))\r\n", ), ] { let mut buf_1 = Vec::with_capacity(100); let mut buf_2 = Vec::with_capacity(100); response.serialize(&mut buf_1, false, false, false); response.serialize(&mut buf_2, true, true, false); let response_v1 = String::from_utf8(buf_1).unwrap(); let response_v2 = String::from_utf8(buf_2).unwrap(); assert_eq!(response_v2, expected_v2); assert_eq!(response_v1, expected_v1); } } #[test] fn serialize_list() { let mut response = super::Response { list_items: vec![ ListItem { mailbox_name: "INBOX".into(), attributes: vec![Attribute::Subscribed], tags: vec![], }, ListItem { mailbox_name: "foo".into(), attributes: vec![], tags: vec![Tag::ChildInfo(vec![ChildInfo::Subscribed])], }, ], status_items: vec![ StatusItem { mailbox_name: "INBOX".into(), items: vec![(Status::Messages, StatusItemType::Number(17))], }, StatusItem { mailbox_name: "foo".into(), items: vec![ (Status::Messages, StatusItemType::Number(30)), (Status::Unseen, StatusItemType::Number(29)), ], }, ], is_lsub: false, is_rev2: true, is_utf8: true, }; let expected_v2 = concat!( "* LIST (\\Subscribed) \"/\" \"INBOX\"\r\n", "* STATUS \"INBOX\" (MESSAGES 17)\r\n", "* LIST () \"/\" \"foo\" (\"CHILDINFO\" (\"SUBSCRIBED\"))\r\n", "* STATUS \"foo\" (MESSAGES 30 UNSEEN 29)\r\n", ); let expected_v1 = concat!( "* LSUB (\\Subscribed) \"/\" \"INBOX\"\r\n", "* LSUB () \"/\" \"foo\" (\"CHILDINFO\" (\"SUBSCRIBED\"))\r\n", ); let response_v2 = String::from_utf8(response.clone().serialize()).unwrap(); response.is_rev2 = false; response.is_utf8 = false; response.is_lsub = true; response.status_items.clear(); let response_v1 = String::from_utf8(response.serialize()).unwrap(); assert_eq!(response_v2, expected_v2); assert_eq!(response_v1, expected_v1); } } ================================================ FILE: crates/imap-proto/src/protocol/login.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { pub tag: String, pub username: String, pub password: String, } ================================================ FILE: crates/imap-proto/src/protocol/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{Command, ResponseCode, ResponseType, StatusResponse}; use ahash::AHashSet; use chrono::{DateTime, Utc}; use compact_str::CompactString; use std::{cmp::Ordering, fmt::Display}; use types::keyword::{ArchivedKeyword, Keyword}; use utils::chained_bytes::SliceRange; pub mod acl; pub mod append; pub mod authenticate; pub mod capability; pub mod copy_move; pub mod create; pub mod delete; pub mod enable; pub mod expunge; pub mod fetch; pub mod list; pub mod login; pub mod namespace; pub mod quota; pub mod rename; pub mod search; pub mod select; pub mod status; pub mod store; pub mod subscribe; pub mod thread; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ProtocolVersion { Rev1, Rev2, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum Sequence { Number { value: u32, }, Range { start: Option, end: Option, }, SavedSearch, List { items: Vec, }, } impl Sequence { pub fn number(value: u32) -> Sequence { Sequence::Number { value } } pub fn range(start: Option, end: Option) -> Sequence { Sequence::Range { start, end } } pub fn contains(&self, value: u32, max_value: u32) -> bool { match self { Sequence::Number { value: number } => *number == value, Sequence::Range { start, end } => match (start, end) { (Some(start), Some(end)) => { value >= *start && value <= *end || value >= *end && value <= *start } (Some(range), None) | (None, Some(range)) => { value >= *range && value <= max_value || value >= max_value && value <= *range } (None, None) => value == max_value, }, Sequence::List { items } => { for item in items { if item.contains(value, max_value) { return true; } } false } Sequence::SavedSearch => false, } } pub fn is_saved_search(&self) -> bool { match self { Sequence::SavedSearch => true, Sequence::List { items } => items.iter().any(|s| s.is_saved_search()), _ => false, } } pub fn expand(&self, max_value: u32) -> AHashSet { match self { Sequence::Number { value } => AHashSet::from_iter([*value]), Sequence::List { items } => { let mut result = AHashSet::with_capacity(items.len()); for item in items { match item { Sequence::Number { value } => { result.insert(*value); } Sequence::Range { start, end } => { let start = start.unwrap_or(max_value); let end = end.unwrap_or(max_value); match start.cmp(&end) { Ordering::Equal => { result.insert(start); } Ordering::Less => { result.extend(start..=end); } Ordering::Greater => { result.extend(end..=start); } } } _ => (), } } result } Sequence::Range { start, end } => { let mut result = AHashSet::new(); let start = start.unwrap_or(max_value); let end = end.unwrap_or(max_value); match start.cmp(&end) { Ordering::Equal => { result.insert(start); } Ordering::Less => { result.extend(start..=end); } Ordering::Greater => { result.extend(end..=start); } } result } _ => AHashSet::new(), } } } pub trait ImapResponse { fn serialize(self) -> Vec; } pub fn quoted_string(buf: &mut Vec, text: &str) { buf.push(b'"'); for &c in text.as_bytes() { if c == b'\\' || c == b'"' { buf.push(b'\\'); } buf.push(c); } buf.push(b'"'); } pub fn quoted_or_literal_string(buf: &mut Vec, text: &str) { if text .as_bytes() .iter() .any(|ch| [b'\\', b'"', b'\r', b'\n'].contains(ch)) { literal_string(buf, text.as_bytes()) } else { buf.push(b'"'); buf.extend_from_slice(text.as_bytes()); buf.push(b'"'); } } pub fn quoted_or_literal_string_or_nil(buf: &mut Vec, text: Option<&str>) { if let Some(text) = text { quoted_or_literal_string(buf, text); } else { buf.extend_from_slice(b"NIL"); } } pub fn quoted_string_or_nil(buf: &mut Vec, text: Option<&str>) { if let Some(text) = text { quoted_string(buf, text); } else { buf.extend_from_slice(b"NIL"); } } pub fn literal_string(buf: &mut Vec, text: &[u8]) { buf.push(b'{'); buf.extend_from_slice(text.len().to_string().as_bytes()); buf.extend_from_slice(b"}\r\n"); buf.extend_from_slice(text); } pub fn literal_string_slice(buf: &mut Vec, text: &SliceRange<'_>) { buf.push(b'{'); buf.extend_from_slice(text.len().to_string().as_bytes()); buf.extend_from_slice(b"}\r\n"); buf.extend(*text); } pub fn quoted_timestamp(buf: &mut Vec, timestamp: i64) { buf.push(b'"'); buf.extend_from_slice( DateTime::::from_timestamp(timestamp, 0) .unwrap_or_default() .format("%d-%b-%Y %H:%M:%S %z") .to_string() .as_bytes(), ); buf.push(b'"'); } pub fn quoted_rfc2822(buf: &mut Vec, timestamp: &mail_parser::DateTime) { buf.push(b'"'); buf.extend_from_slice(timestamp.to_rfc822().as_bytes()); buf.push(b'"'); } pub fn quoted_rfc2822_or_nil(buf: &mut Vec, timestamp: &Option) { if let Some(timestamp) = timestamp { quoted_rfc2822(buf, timestamp); } else { buf.extend_from_slice(b"NIL"); } } #[derive(Debug, Clone, PartialEq, Eq)] pub enum Flag { Seen, Draft, Flagged, Answered, Recent, Important, Phishing, Junk, NotJunk, Deleted, Forwarded, MDNSent, Autosent, CanUnsubscribe, Followed, HasAttachment, HasMemo, HasNoAttachment, Imported, IsTrusted, MailFlagBit0, MailFlagBit1, MailFlagBit2, MaskedEmail, Memo, Muted, New, Notify, Unsubscribed, Keyword(Box), } impl Flag { pub fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(match self { Flag::Seen => b"\\Seen", Flag::Draft => b"\\Draft", Flag::Flagged => b"\\Flagged", Flag::Answered => b"\\Answered", Flag::Recent => b"\\Recent", Flag::Important => b"\\Important", Flag::Phishing => b"$Phishing", Flag::Junk => b"$Junk", Flag::NotJunk => b"$NotJunk", Flag::Deleted => b"\\Deleted", Flag::Forwarded => b"$Forwarded", Flag::MDNSent => b"$MDNSent", Flag::Autosent => b"$autosent", Flag::CanUnsubscribe => b"$canunsubscribe", Flag::Followed => b"$followed", Flag::HasAttachment => b"$hasattachment", Flag::HasMemo => b"$hasmemo", Flag::HasNoAttachment => b"$hasnoattachment", Flag::Imported => b"$imported", Flag::IsTrusted => b"$istrusted", Flag::MailFlagBit0 => b"$MailFlagBit0", Flag::MailFlagBit1 => b"$MailFlagBit1", Flag::MailFlagBit2 => b"$MailFlagBit2", Flag::MaskedEmail => b"$maskedemail", Flag::Memo => b"$memo", Flag::Muted => b"$muted", Flag::New => b"$new", Flag::Notify => b"$notify", Flag::Unsubscribed => b"$unsubscribed", Flag::Keyword(keyword) => keyword.as_bytes(), }); } } impl From for Flag { fn from(value: Keyword) -> Self { match value { Keyword::Seen => Flag::Seen, Keyword::Draft => Flag::Draft, Keyword::Flagged => Flag::Flagged, Keyword::Answered => Flag::Answered, Keyword::Recent => Flag::Recent, Keyword::Important => Flag::Important, Keyword::Phishing => Flag::Phishing, Keyword::Junk => Flag::Junk, Keyword::NotJunk => Flag::NotJunk, Keyword::Deleted => Flag::Deleted, Keyword::Forwarded => Flag::Forwarded, Keyword::MdnSent => Flag::MDNSent, Keyword::Autosent => Flag::Autosent, Keyword::CanUnsubscribe => Flag::CanUnsubscribe, Keyword::Followed => Flag::Followed, Keyword::HasAttachment => Flag::HasAttachment, Keyword::HasMemo => Flag::HasMemo, Keyword::HasNoAttachment => Flag::HasNoAttachment, Keyword::Imported => Flag::Imported, Keyword::IsTrusted => Flag::IsTrusted, Keyword::MailFlagBit0 => Flag::MailFlagBit0, Keyword::MailFlagBit1 => Flag::MailFlagBit1, Keyword::MailFlagBit2 => Flag::MailFlagBit2, Keyword::MaskedEmail => Flag::MaskedEmail, Keyword::Memo => Flag::Memo, Keyword::Muted => Flag::Muted, Keyword::New => Flag::New, Keyword::Notify => Flag::Notify, Keyword::Unsubscribed => Flag::Unsubscribed, Keyword::Other(value) => Flag::Keyword(value), } } } impl From<&ArchivedKeyword> for Flag { fn from(value: &ArchivedKeyword) -> Self { match value { ArchivedKeyword::Seen => Flag::Seen, ArchivedKeyword::Draft => Flag::Draft, ArchivedKeyword::Flagged => Flag::Flagged, ArchivedKeyword::Answered => Flag::Answered, ArchivedKeyword::Recent => Flag::Recent, ArchivedKeyword::Important => Flag::Important, ArchivedKeyword::Phishing => Flag::Phishing, ArchivedKeyword::Junk => Flag::Junk, ArchivedKeyword::NotJunk => Flag::NotJunk, ArchivedKeyword::Deleted => Flag::Deleted, ArchivedKeyword::Forwarded => Flag::Forwarded, ArchivedKeyword::MdnSent => Flag::MDNSent, ArchivedKeyword::Autosent => Flag::Autosent, ArchivedKeyword::CanUnsubscribe => Flag::CanUnsubscribe, ArchivedKeyword::Followed => Flag::Followed, ArchivedKeyword::HasAttachment => Flag::HasAttachment, ArchivedKeyword::HasMemo => Flag::HasMemo, ArchivedKeyword::HasNoAttachment => Flag::HasNoAttachment, ArchivedKeyword::Imported => Flag::Imported, ArchivedKeyword::IsTrusted => Flag::IsTrusted, ArchivedKeyword::MailFlagBit0 => Flag::MailFlagBit0, ArchivedKeyword::MailFlagBit1 => Flag::MailFlagBit1, ArchivedKeyword::MailFlagBit2 => Flag::MailFlagBit2, ArchivedKeyword::MaskedEmail => Flag::MaskedEmail, ArchivedKeyword::Memo => Flag::Memo, ArchivedKeyword::Muted => Flag::Muted, ArchivedKeyword::New => Flag::New, ArchivedKeyword::Notify => Flag::Notify, ArchivedKeyword::Unsubscribed => Flag::Unsubscribed, ArchivedKeyword::Other(value) => Flag::Keyword(value.as_ref().into()), } } } impl From for Keyword { fn from(value: Flag) -> Self { match value { Flag::Seen => Keyword::Seen, Flag::Draft => Keyword::Draft, Flag::Flagged => Keyword::Flagged, Flag::Answered => Keyword::Answered, Flag::Recent => Keyword::Recent, Flag::Important => Keyword::Important, Flag::Phishing => Keyword::Phishing, Flag::Junk => Keyword::Junk, Flag::NotJunk => Keyword::NotJunk, Flag::Deleted => Keyword::Deleted, Flag::Forwarded => Keyword::Forwarded, Flag::MDNSent => Keyword::MdnSent, Flag::Autosent => Keyword::Autosent, Flag::CanUnsubscribe => Keyword::CanUnsubscribe, Flag::Followed => Keyword::Followed, Flag::HasAttachment => Keyword::HasAttachment, Flag::HasMemo => Keyword::HasMemo, Flag::HasNoAttachment => Keyword::HasNoAttachment, Flag::Imported => Keyword::Imported, Flag::IsTrusted => Keyword::IsTrusted, Flag::MailFlagBit0 => Keyword::MailFlagBit0, Flag::MailFlagBit1 => Keyword::MailFlagBit1, Flag::MailFlagBit2 => Keyword::MailFlagBit2, Flag::MaskedEmail => Keyword::MaskedEmail, Flag::Memo => Keyword::Memo, Flag::Muted => Keyword::Muted, Flag::New => Keyword::New, Flag::Notify => Keyword::Notify, Flag::Unsubscribed => Keyword::Unsubscribed, Flag::Keyword(value) => Keyword::from_boxed_other(value), } } } impl ResponseCode { pub fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(match self { ResponseCode::Alert => b"ALERT", ResponseCode::AlreadyExists => b"ALREADYEXISTS", ResponseCode::AppendUid { uid_validity, uids } => { buf.extend_from_slice(b"APPENDUID "); buf.extend_from_slice(uid_validity.to_string().as_bytes()); buf.push(b' '); serialize_sequence(buf, uids); return; } ResponseCode::AuthenticationFailed => b"AUTHENTICATIONFAILED", ResponseCode::AuthorizationFailed => b"AUTHORIZATIONFAILED", ResponseCode::BadCharset => b"BADCHARSET", ResponseCode::Cannot => b"CANNOT", ResponseCode::Capability { capabilities } => { buf.extend_from_slice(b"CAPABILITY"); for capability in capabilities { buf.push(b' '); capability.serialize(buf); } return; } ResponseCode::ClientBug => b"CLIENTBUG", ResponseCode::Closed => b"CLOSED", ResponseCode::ContactAdmin => b"CONTACTADMIN", ResponseCode::CopyUid { uid_validity, src_uids, dest_uids, } => { buf.extend_from_slice(b"COPYUID "); buf.extend_from_slice(uid_validity.to_string().as_bytes()); buf.push(b' '); serialize_sequence(buf, src_uids); buf.push(b' '); serialize_sequence(buf, dest_uids); return; } ResponseCode::Corruption => b"CORRUPTION", ResponseCode::Expired => b"EXPIRED", ResponseCode::ExpungeIssued => b"EXPUNGEISSUED", ResponseCode::HasChildren => b"HASCHILDREN", ResponseCode::InUse => b"INUSE", ResponseCode::Limit => b"LIMIT", ResponseCode::NonExistent => b"NONEXISTENT", ResponseCode::NoPerm => b"NOPERM", ResponseCode::OverQuota => b"OVERQUOTA", ResponseCode::Parse => b"PARSE", ResponseCode::PermanentFlags => b"PERMANENTFLAGS", ResponseCode::PrivacyRequired => b"PRIVACYREQUIRED", ResponseCode::ReadOnly => b"READ-ONLY", ResponseCode::ReadWrite => b"READ-WRITE", ResponseCode::ServerBug => b"SERVERBUG", ResponseCode::TryCreate => b"TRYCREATE", ResponseCode::UidNext => b"UIDNEXT", ResponseCode::UidNotSticky => b"UIDNOTSTICKY", ResponseCode::UidValidity => b"UIDVALIDITY", ResponseCode::Unavailable => b"UNAVAILABLE", ResponseCode::UnknownCte => b"UNKNOWN-CTE", ResponseCode::Modified { ids } => { buf.extend_from_slice(b"MODIFIED "); serialize_sequence(buf, ids); return; } ResponseCode::MailboxId { mailbox_id } => { buf.extend_from_slice(b"MAILBOXID ("); buf.extend_from_slice(mailbox_id.as_bytes()); buf.push(b')'); return; } ResponseCode::HighestModseq { modseq } => { buf.extend_from_slice(b"HIGHESTMODSEQ "); buf.extend_from_slice(modseq.to_string().as_bytes()); return; } ResponseCode::UseAttr => b"USEATTR", }); } pub fn as_str(&self) -> &'static str { // Only returns the name without arguments match self { ResponseCode::Alert => "ALERT", ResponseCode::AlreadyExists => "ALREADYEXISTS", ResponseCode::AppendUid { .. } => "APPENDUID", ResponseCode::AuthenticationFailed => "AUTHENTICATIONFAILED", ResponseCode::AuthorizationFailed => "AUTHORIZATIONFAILED", ResponseCode::BadCharset => "BADCHARSET", ResponseCode::Cannot => "CANNOT", ResponseCode::Capability { .. } => "CAPABILITY", ResponseCode::ClientBug => "CLIENTBUG", ResponseCode::Closed => "CLOSED", ResponseCode::ContactAdmin => "CONTACTADMIN", ResponseCode::CopyUid { .. } => "COPYUID", ResponseCode::Corruption => "CORRUPTION", ResponseCode::Expired => "EXPIRED", ResponseCode::ExpungeIssued => "EXPUNGEISSUED", ResponseCode::HasChildren => "HASCHILDREN", ResponseCode::InUse => "INUSE", ResponseCode::Limit => "LIMIT", ResponseCode::NonExistent => "NONEXISTENT", ResponseCode::NoPerm => "NOPERM", ResponseCode::OverQuota => "OVERQUOTA", ResponseCode::Parse => "PARSE", ResponseCode::PermanentFlags => "PERMANENTFLAGS", ResponseCode::PrivacyRequired => "PRIVACYREQUIRED", ResponseCode::ReadOnly => "READ-ONLY", ResponseCode::ReadWrite => "READ-WRITE", ResponseCode::ServerBug => "SERVERBUG", ResponseCode::TryCreate => "TRYCREATE", ResponseCode::UidNext => "UIDNEXT", ResponseCode::UidNotSticky => "UIDNOTSTICKY", ResponseCode::UidValidity => "UIDVALIDITY", ResponseCode::Unavailable => "UNAVAILABLE", ResponseCode::UnknownCte => "UNKNOWN-CTE", ResponseCode::Modified { .. } => "MODIFIED", ResponseCode::MailboxId { .. } => "MAILBOXID", ResponseCode::HighestModseq { .. } => "HIGHESTMODSEQ", ResponseCode::UseAttr => "USEATTR", } } } impl ResponseType { pub fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(self.as_str().as_bytes()); } pub fn as_str(&self) -> &'static str { match self { ResponseType::Ok => "OK", ResponseType::No => "NO", ResponseType::Bad => "BAD", ResponseType::PreAuth => "PREAUTH", ResponseType::Bye => "BYE", } } } impl From for trc::Value { fn from(value: ResponseCode) -> Self { trc::Value::String(CompactString::const_new(value.as_str())) } } impl From for trc::Value { fn from(value: ResponseType) -> Self { trc::Value::String(CompactString::const_new(value.as_str())) } } impl StatusResponse { pub fn serialize(self, mut buf: Vec) -> Vec { if let Some(tag) = &self.tag { buf.extend_from_slice(tag.as_bytes()); } else { buf.push(b'*'); } buf.push(b' '); self.rtype.serialize(&mut buf); buf.push(b' '); if let Some(code) = &self.code { buf.push(b'['); code.serialize(&mut buf); buf.extend_from_slice(b"] "); } buf.extend_from_slice(self.message.as_bytes()); buf.extend_from_slice(b"\r\n"); buf } pub fn into_bytes(self) -> Vec { self.serialize(Vec::with_capacity(16)) } } pub trait SerializeResponse { fn serialize(&self) -> Vec; } impl SerializeResponse for trc::Error { fn serialize(&self) -> Vec { let mut buf = Vec::with_capacity(128); if let Some(tag) = self.value_as_str(trc::Key::Id) { buf.extend_from_slice(tag.as_bytes()); } else { buf.push(b'*'); } buf.push(b' '); buf.extend_from_slice(self.value_as_str(trc::Key::Type).unwrap_or("NO").as_bytes()); buf.push(b' '); if let Some(code) = self .value_as_str(trc::Key::Code) .or_else(|| match self.as_ref() { trc::EventType::Store(trc::StoreEvent::NotFound) => { Some(ResponseCode::NonExistent.as_str()) } trc::EventType::Store(_) => Some(ResponseCode::ContactAdmin.as_str()), trc::EventType::Limit(trc::LimitEvent::Quota) => { Some(ResponseCode::OverQuota.as_str()) } trc::EventType::Limit(_) => Some(ResponseCode::Limit.as_str()), trc::EventType::Auth(_) => Some(ResponseCode::AuthenticationFailed.as_str()), trc::EventType::Security(_) => Some(ResponseCode::AuthorizationFailed.as_str()), _ => None, }) { buf.push(b'['); buf.extend_from_slice(code.as_bytes()); buf.extend_from_slice(b"] "); } buf.extend_from_slice( self.value_as_str(trc::Key::Details) .unwrap_or_else(|| self.as_ref().message()) .as_bytes(), ); buf.extend_from_slice(b"\r\n"); buf } } impl ProtocolVersion { #[inline(always)] pub fn is_rev2(&self) -> bool { matches!(self, ProtocolVersion::Rev2) } #[inline(always)] pub fn is_rev1(&self) -> bool { matches!(self, ProtocolVersion::Rev1) } } pub fn serialize_sequence(buf: &mut Vec, list: &[u32]) { let mut ids = list.iter().peekable(); while let Some(&id) = ids.next() { buf.extend_from_slice(id.to_string().as_bytes()); let mut range_id = id; loop { match ids.peek() { Some(&&next_id) if next_id == range_id + 1 => { range_id += 1; ids.next(); } next => { if range_id != id { buf.push(b':'); buf.extend_from_slice(range_id.to_string().as_bytes()); } if next.is_some() { buf.push(b','); } break; } } } } } impl Display for Command { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { Command::Capability => write!(f, "CAPABILITY"), Command::Noop => write!(f, "NOOP"), Command::Logout => write!(f, "LOGOUT"), Command::StartTls => write!(f, "STARTTLS"), Command::Authenticate => write!(f, "AUTHENTICATE"), Command::Login => write!(f, "LOGIN"), Command::Enable => write!(f, "ENABLE"), Command::Select => write!(f, "SELECT"), Command::Examine => write!(f, "EXAMINE"), Command::Create => write!(f, "CREATE"), Command::Delete => write!(f, "DELETE"), Command::Rename => write!(f, "RENAME"), Command::Subscribe => write!(f, "SUBSCRIBE"), Command::Unsubscribe => write!(f, "UNSUBSCRIBE"), Command::List => write!(f, "LIST"), Command::Namespace => write!(f, "NAMESPACE"), Command::Status => write!(f, "STATUS"), Command::Append => write!(f, "APPEND"), Command::Idle => write!(f, "IDLE"), Command::Close => write!(f, "CLOSE"), Command::Unselect => write!(f, "UNSELECT"), Command::Expunge(false) => write!(f, "EXPUNGE"), Command::Search(false) => write!(f, "SEARCH"), Command::Fetch(false) => write!(f, "FETCH"), Command::Store(false) => write!(f, "STORE"), Command::Copy(false) => write!(f, "COPY"), Command::Move(false) => write!(f, "MOVE"), Command::Sort(false) => write!(f, "SORT"), Command::Thread(false) => write!(f, "THREAD"), Command::Expunge(true) => write!(f, "UID EXPUNGE"), Command::Search(true) => write!(f, "UID SEARCH"), Command::Fetch(true) => write!(f, "UID FETCH"), Command::Store(true) => write!(f, "UID STORE"), Command::Copy(true) => write!(f, "UID COPY"), Command::Move(true) => write!(f, "UID MOVE"), Command::Sort(true) => write!(f, "UID SORT"), Command::Thread(true) => write!(f, "UID THREAD"), Command::Lsub => write!(f, "LSUB"), Command::Check => write!(f, "CHECK"), Command::SetAcl => write!(f, "SETACL"), Command::DeleteAcl => write!(f, "DELETEACL"), Command::GetAcl => write!(f, "GETACL"), Command::ListRights => write!(f, "LISTRIGHTS"), Command::MyRights => write!(f, "MYRIGHTS"), Command::Unauthenticate => write!(f, "UNAUTHENTICATE"), Command::Id => write!(f, "ID"), Command::GetQuota => write!(f, "GETQUOTA"), Command::GetQuotaRoot => write!(f, "GETQUOTAROOT"), } } } #[cfg(test)] mod tests { use crate::parser::parse_sequence_set; #[test] fn sequence_set_contains() { for (sequence, expected_result, max_value) in [ ("1,5:10", vec![1, 5, 6, 7, 8, 9, 10], 10), ("2,4:7,9,12:*", vec![2, 4, 5, 6, 7, 9, 12, 13, 14, 15], 15), ("*:4,5:7", vec![4, 5, 6, 7], 7), ("2,4,5", vec![2, 4, 5], 5), ] { let sequence = parse_sequence_set(sequence.as_bytes()).unwrap(); assert_eq!( (1..=15) .filter(|num| sequence.contains(*num, max_value)) .collect::>(), expected_result ); } } } ================================================ FILE: crates/imap-proto/src/protocol/namespace.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ImapResponse, quoted_string}; pub struct Response { pub shared_prefix: Option, } impl ImapResponse for Response { fn serialize(self) -> Vec { let mut buf = Vec::with_capacity(64); if let Some(shared_prefix) = &self.shared_prefix { buf.extend_from_slice(b"* NAMESPACE ((\"\" \"/\")) (("); quoted_string(&mut buf, shared_prefix); buf.extend_from_slice(b" \"/\")) NIL\r\n"); } else { buf.extend_from_slice(b"* NAMESPACE ((\"\" \"/\")) NIL NIL\r\n"); } buf } } ================================================ FILE: crates/imap-proto/src/protocol/quota.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ImapResponse, capability::QuotaResourceName, quoted_string}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { pub tag: String, pub name: String, } pub struct QuotaItem { pub name: String, pub resources: Vec, } pub struct QuotaResource { pub resource: QuotaResourceName, pub total: u64, pub used: u64, } pub struct Response { pub quota_root_items: Vec, pub quota_items: Vec, } impl ImapResponse for Response { fn serialize(self) -> Vec { let mut buf = Vec::with_capacity(64); if !self.quota_root_items.is_empty() { buf.extend_from_slice(b"* QUOTAROOT"); for item in &self.quota_root_items { buf.push(b' '); quoted_string(&mut buf, item); } buf.extend_from_slice(b"\r\n"); } if !self.quota_items.is_empty() { for item in &self.quota_items { buf.extend_from_slice(b"* QUOTA "); quoted_string(&mut buf, &item.name); buf.extend_from_slice(b" ("); for (pos, resource) in item.resources.iter().enumerate() { if pos > 0 { buf.push(b' '); } let mut total = resource.total; let mut used = resource.used; match resource.resource { QuotaResourceName::Storage => { total /= 1024; used /= 1024; buf.extend_from_slice(b"STORAGE ") } QuotaResourceName::Message => buf.extend_from_slice(b"MESSAGE "), QuotaResourceName::Mailbox => buf.extend_from_slice(b"MAILBOX "), QuotaResourceName::AnnotationStorage => { buf.extend_from_slice(b"ANNOTATION-STORAGE ") } } buf.extend_from_slice(format!("{used} {total}").as_bytes()); } buf.extend_from_slice(b")\r\n"); } } buf } } #[cfg(test)] mod tests { use crate::protocol::{ImapResponse, capability::QuotaResourceName}; use super::{QuotaItem, QuotaResource}; #[test] fn serialize_quota() { for (response, expected) in [ ( super::Response { quota_root_items: vec!["INBOX".into(), "#test".into()], quota_items: vec![], }, "* QUOTAROOT \"INBOX\" \"#test\"\r\n", ), ( super::Response { quota_root_items: vec![], quota_items: vec![QuotaItem { name: "INBOX".into(), resources: vec![QuotaResource { resource: QuotaResourceName::Storage, total: 1073741824, used: 1048576, }], }], }, "* QUOTA \"INBOX\" (STORAGE 1024 1048576)\r\n", ), ( super::Response { quota_root_items: vec!["my mailbox".into(), "".into()], quota_items: vec![QuotaItem { name: "INBOX".into(), resources: vec![ QuotaResource { resource: QuotaResourceName::Storage, total: 1073741824, used: 1048576, }, QuotaResource { resource: QuotaResourceName::Message, total: 100, used: 2, }, ], }], }, concat!( "* QUOTAROOT \"my mailbox\" \"\"\r\n", "* QUOTA \"INBOX\" (STORAGE 1024 1048576 MESSAGE 2 100)\r\n" ), ), ] { assert_eq!(String::from_utf8(response.serialize()).unwrap(), expected); } } } ================================================ FILE: crates/imap-proto/src/protocol/rename.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { pub tag: String, pub mailbox_name: String, pub new_mailbox_name: String, } ================================================ FILE: crates/imap-proto/src/protocol/search.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{Flag, Sequence, quoted_string, serialize_sequence}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { pub tag: String, pub is_esearch: bool, pub sort: Option>, pub result_options: Vec, pub filter: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum Sort { Arrival, Cc, Date, From, DisplayFrom, Size, Subject, To, DisplayTo, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct Comparator { pub sort: Sort, pub ascending: bool, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct Response { pub is_uid: bool, pub is_esearch: bool, pub is_sort: bool, pub ids: Vec, pub min: Option, pub max: Option, pub count: Option, pub highest_modseq: Option, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum ResultOption { Min, Max, All, Count, Save, Context, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum Filter { Sequence(Sequence, bool), All, Answered, Bcc(String), Before(i64), Body(String), Cc(String), Deleted, Draft, Flagged, From(String), Header(String, String), Keyword(Flag), Larger(u32), On(i64), Seen, SentBefore(i64), SentOn(i64), SentSince(i64), Since(i64), Smaller(u32), Subject(String), Text(String), To(String), Unanswered, Undeleted, Undraft, Unflagged, Unkeyword(Flag), Unseen, // Logical operators And, Or, Not, End, // Imap4rev1 Recent, New, Old, // RFC 5032 - WITHIN Older(u32), Younger(u32), // RFC 4551 - CONDSTORE ModSeq((u64, ModSeqEntry)), // RFC 8474 - ObjectID EmailId(String), ThreadId(String), } #[derive(Debug, Clone, PartialEq, Eq)] pub enum ModSeqEntry { Shared(Flag), Private(Flag), All(Flag), None, } impl Filter { pub fn seq_saved_search() -> Filter { Filter::Sequence(Sequence::SavedSearch, false) } pub fn seq_range(start: Option, end: Option) -> Filter { Filter::Sequence(Sequence::Range { start, end }, false) } } impl Response { pub fn serialize(self, tag: &str) -> Vec { let mut buf = Vec::with_capacity(64); if self.is_esearch { buf.extend_from_slice(b"* ESEARCH (TAG "); quoted_string(&mut buf, tag); buf.extend_from_slice(b")"); if self.is_uid { buf.extend_from_slice(b" UID"); } if let Some(count) = &self.count { buf.extend_from_slice(b" COUNT "); buf.extend_from_slice(count.to_string().as_bytes()); } if let Some(min) = &self.min { buf.extend_from_slice(b" MIN "); buf.extend_from_slice(min.to_string().as_bytes()); } if let Some(max) = &self.max { buf.extend_from_slice(b" MAX "); buf.extend_from_slice(max.to_string().as_bytes()); } if !self.ids.is_empty() { buf.extend_from_slice(b" ALL "); serialize_sequence(&mut buf, &self.ids); } if let Some(highest_modseq) = self.highest_modseq { buf.extend_from_slice(b" MODSEQ "); buf.extend_from_slice(highest_modseq.to_string().as_bytes()); } } else { if !self.is_sort { buf.extend_from_slice(b"* SEARCH"); } else { buf.extend_from_slice(b"* SORT"); } if !self.ids.is_empty() { for id in &self.ids { buf.push(b' '); buf.extend_from_slice(id.to_string().as_bytes()); } } if let Some(highest_modseq) = self.highest_modseq { buf.extend_from_slice(b" (MODSEQ "); buf.extend_from_slice(highest_modseq.to_string().as_bytes()); buf.push(b')'); } } buf.extend_from_slice(b"\r\n"); buf } } #[cfg(test)] mod tests { #[test] fn serialize_search() { for (mut response, tag, expected_v2, expected_v1) in [ ( super::Response { is_uid: false, is_esearch: true, is_sort: false, ids: vec![2, 10, 11], min: 2.into(), max: 11.into(), count: 3.into(), highest_modseq: None, }, "A283", "* ESEARCH (TAG \"A283\") COUNT 3 MIN 2 MAX 11 ALL 2,10:11\r\n", "* SEARCH 2 10 11\r\n", ), ( super::Response { is_uid: false, is_esearch: true, is_sort: false, ids: vec![ 1, 2, 3, 5, 10, 11, 12, 13, 90, 92, 93, 94, 95, 96, 97, 98, 99, ], min: None, max: None, count: None, highest_modseq: None, }, "A283", "* ESEARCH (TAG \"A283\") ALL 1:3,5,10:13,90,92:99\r\n", "* SEARCH 1 2 3 5 10 11 12 13 90 92 93 94 95 96 97 98 99\r\n", ), ( super::Response { is_uid: false, is_esearch: true, is_sort: false, ids: vec![], min: None, max: None, count: None, highest_modseq: None, }, "A283", "* ESEARCH (TAG \"A283\")\r\n", "* SEARCH\r\n", ), ( super::Response { is_uid: false, is_esearch: true, is_sort: false, ids: vec![10, 11, 12, 13, 21], min: None, max: None, count: None, highest_modseq: 12345.into(), }, "A283", "* ESEARCH (TAG \"A283\") ALL 10:13,21 MODSEQ 12345\r\n", "* SEARCH 10 11 12 13 21 (MODSEQ 12345)\r\n", ), ] { let response_v2 = String::from_utf8(response.clone().serialize(tag)).unwrap(); response.is_esearch = false; let response_v1 = String::from_utf8(response.serialize(tag)).unwrap(); assert_eq!(response_v2, expected_v2); assert_eq!(response_v1, expected_v1); } } } ================================================ FILE: crates/imap-proto/src/protocol/select.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ResponseCode, StatusResponse}; use super::{ImapResponse, Sequence, list::ListItem}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { pub tag: String, pub mailbox_name: String, pub condstore: bool, pub qresync: Option, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct QResync { pub uid_validity: u32, pub modseq: u64, pub known_uids: Option, pub seq_match: Option<(Sequence, Sequence)>, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct HighestModSeq(u64); #[derive(Debug, Clone)] pub struct Response { pub mailbox: ListItem, pub total_messages: usize, pub recent_messages: usize, pub unseen_seq: u32, pub uid_validity: u32, pub uid_next: u32, pub is_rev2: bool, pub is_utf8: bool, pub closed_previous: bool, pub highest_modseq: Option, pub mailbox_id: String, } #[derive(Debug, Clone)] pub struct Exists { pub total_messages: usize, } impl ImapResponse for Response { fn serialize(self) -> Vec { let mut buf = Vec::with_capacity(100); if self.closed_previous { buf = StatusResponse::ok("Closed previous mailbox") .with_code(ResponseCode::Closed) .serialize(buf); } buf.extend_from_slice(b"* "); buf.extend_from_slice(self.total_messages.to_string().as_bytes()); if !self.is_rev2 && self.recent_messages > 0 { buf.extend_from_slice( b" EXISTS\r\n* FLAGS (\\Answered \\Flagged \\Deleted \\Seen \\Draft \\Recent)\r\n", ); } else { buf.extend_from_slice( b" EXISTS\r\n* FLAGS (\\Answered \\Flagged \\Deleted \\Seen \\Draft)\r\n", ); } if self.is_rev2 { self.mailbox .serialize(&mut buf, self.is_rev2, self.is_utf8, false); } else { buf.extend_from_slice(b"* "); buf.extend_from_slice(self.recent_messages.to_string().as_bytes()); buf.extend_from_slice(b" RECENT\r\n"); if self.unseen_seq > 0 { buf.extend_from_slice(b"* OK [UNSEEN "); buf.extend_from_slice(self.unseen_seq.to_string().as_bytes()); buf.extend_from_slice(b"] Unseen messages\r\n"); } } buf.extend_from_slice( b"* OK [PERMANENTFLAGS (\\Deleted \\Seen \\Answered \\Flagged \\Draft \\*)] All allowed\r\n", ); buf.extend_from_slice(b"* OK [UIDVALIDITY "); buf.extend_from_slice(self.uid_validity.to_string().as_bytes()); buf.extend_from_slice(b"] UIDs valid\r\n* OK [UIDNEXT "); buf.extend_from_slice(self.uid_next.to_string().as_bytes()); buf.extend_from_slice(b"] Next predicted UID\r\n"); if let Some(highest_modseq) = self.highest_modseq { highest_modseq.serialize(&mut buf); } buf.extend_from_slice(b"* OK [MAILBOXID ("); buf.extend_from_slice(self.mailbox_id.as_bytes()); buf.extend_from_slice(b")] Unique Mailbox ID\r\n"); buf } } impl HighestModSeq { pub fn new(modseq: u64) -> Self { Self(modseq) } pub fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(b"* OK [HIGHESTMODSEQ "); buf.extend_from_slice(self.0.to_string().as_bytes()); buf.extend_from_slice(b"] Highest Modseq\r\n"); } pub fn into_bytes(self) -> Vec { let mut buf = Vec::with_capacity(40); self.serialize(&mut buf); buf } } impl Exists { pub fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(b"* "); buf.extend_from_slice(self.total_messages.to_string().as_bytes()); buf.extend_from_slice(b" EXISTS\r\n"); } pub fn into_bytes(self) -> Vec { let mut buf = Vec::with_capacity(15); self.serialize(&mut buf); buf } } #[cfg(test)] mod tests { use crate::protocol::{ImapResponse, list::ListItem}; use super::HighestModSeq; #[test] fn serialize_select() { for (mut response, _tag, expected_v2, expected_v1) in [ ( super::Response { mailbox: ListItem::new("INBOX"), total_messages: 172, recent_messages: 5, unseen_seq: 3, uid_validity: 3857529045, uid_next: 4392, closed_previous: false, is_rev2: true, is_utf8: true, highest_modseq: HighestModSeq::new(100).into(), mailbox_id: "abc".into(), }, "A142", concat!( "* 172 EXISTS\r\n", "* FLAGS (\\Answered \\Flagged \\Deleted \\Seen \\Draft)\r\n", "* LIST () \"/\" \"INBOX\"\r\n", "* OK [PERMANENTFLAGS (\\Deleted \\Seen \\Answered \\Flagged \\Draft \\*)] All allowed\r\n", "* OK [UIDVALIDITY 3857529045] UIDs valid\r\n", "* OK [UIDNEXT 4392] Next predicted UID\r\n", "* OK [HIGHESTMODSEQ 100] Highest Modseq\r\n", "* OK [MAILBOXID (abc)] Unique Mailbox ID\r\n" ), concat!( "* 172 EXISTS\r\n", "* FLAGS (\\Answered \\Flagged \\Deleted \\Seen \\Draft \\Recent)\r\n", "* 5 RECENT\r\n", "* OK [UNSEEN 3] Unseen messages\r\n", "* OK [PERMANENTFLAGS (\\Deleted \\Seen \\Answered \\Flagged \\Draft \\*)] All allowed\r\n", "* OK [UIDVALIDITY 3857529045] UIDs valid\r\n", "* OK [UIDNEXT 4392] Next predicted UID\r\n", "* OK [HIGHESTMODSEQ 100] Highest Modseq\r\n", "* OK [MAILBOXID (abc)] Unique Mailbox ID\r\n" ), ), ( super::Response { mailbox: ListItem::new("~peter/mail/台北/日本語"), total_messages: 172, recent_messages: 5, unseen_seq: 3, uid_validity: 3857529045, uid_next: 4392, closed_previous: true, is_rev2: true, is_utf8: true, highest_modseq: None, mailbox_id: "abc".into(), }, "A142", concat!( "* OK [CLOSED] Closed previous mailbox\r\n", "* 172 EXISTS\r\n", "* FLAGS (\\Answered \\Flagged \\Deleted \\Seen \\Draft)\r\n", "* LIST () \"/\" \"~peter/mail/台北/日本語\" (\"OLDNAME\" ", "(\"~peter/mail/&U,BTFw-/&ZeVnLIqe-\"))\r\n", "* OK [PERMANENTFLAGS (\\Deleted \\Seen \\Answered \\Flagged \\Draft \\*)] All allowed\r\n", "* OK [UIDVALIDITY 3857529045] UIDs valid\r\n", "* OK [UIDNEXT 4392] Next predicted UID\r\n", "* OK [MAILBOXID (abc)] Unique Mailbox ID\r\n" ), concat!( "* OK [CLOSED] Closed previous mailbox\r\n", "* 172 EXISTS\r\n", "* FLAGS (\\Answered \\Flagged \\Deleted \\Seen \\Draft \\Recent)\r\n", "* 5 RECENT\r\n", "* OK [UNSEEN 3] Unseen messages\r\n", "* OK [PERMANENTFLAGS (\\Deleted \\Seen \\Answered \\Flagged \\Draft \\*)] All allowed\r\n", "* OK [UIDVALIDITY 3857529045] UIDs valid\r\n", "* OK [UIDNEXT 4392] Next predicted UID\r\n", "* OK [MAILBOXID (abc)] Unique Mailbox ID\r\n" ), ), ] { let response_v2 = String::from_utf8(response.clone().serialize()).unwrap(); response.is_rev2 = false; let response_v1 = String::from_utf8(response.serialize()).unwrap(); assert_eq!(response_v2, expected_v2); assert_eq!(response_v1, expected_v1); } } } ================================================ FILE: crates/imap-proto/src/protocol/status.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::utf7::utf7_encode; use super::quoted_string; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { pub tag: String, pub mailbox_name: String, pub items: Vec, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Status { Messages, UidNext, UidValidity, Unseen, Deleted, Size, Recent, HighestModSeq, MailboxId, DeletedStorage, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct StatusItem { pub mailbox_name: String, pub items: Vec<(Status, StatusItemType)>, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum StatusItemType { Number(u64), String(String), } impl StatusItem { pub fn serialize(&self, buf: &mut Vec, is_utf8: bool) { buf.extend_from_slice(b"* STATUS "); if is_utf8 { quoted_string(buf, &self.mailbox_name); } else { quoted_string(buf, &utf7_encode(&self.mailbox_name)); } buf.extend_from_slice(b" ("); for (pos, (status_item, value)) in self.items.iter().enumerate() { if pos > 0 { buf.push(b' '); } buf.extend_from_slice(match status_item { Status::Messages => b"MESSAGES ", Status::UidNext => b"UIDNEXT ", Status::UidValidity => b"UIDVALIDITY ", Status::Unseen => b"UNSEEN ", Status::Deleted => b"DELETED ", Status::Size => b"SIZE ", Status::HighestModSeq => b"HIGHESTMODSEQ ", Status::MailboxId => b"MAILBOXID ", Status::Recent => b"RECENT ", Status::DeletedStorage => b"DELETED-STORAGE ", }); match value { StatusItemType::Number(num) => { buf.extend_from_slice(num.to_string().as_bytes()); } StatusItemType::String(str) => { buf.push(b'('); buf.extend_from_slice(str.as_bytes()); buf.push(b')'); } } } buf.extend_from_slice(b")\r\n"); } } #[cfg(test)] mod tests { use crate::protocol::status::{Status, StatusItem, StatusItemType}; #[test] fn serialize_status() { let mut buf = Vec::new(); StatusItem { mailbox_name: "blurdybloop".into(), items: vec![ (Status::Messages, StatusItemType::Number(231)), (Status::UidNext, StatusItemType::Number(44292)), (Status::MailboxId, StatusItemType::String("abc-123".into())), ], } .serialize(&mut buf, true); assert_eq!( String::from_utf8(buf).unwrap(), "* STATUS \"blurdybloop\" (MESSAGES 231 UIDNEXT 44292 MAILBOXID (abc-123))\r\n" ); } } ================================================ FILE: crates/imap-proto/src/protocol/store.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{Flag, ImapResponse, Sequence, fetch::FetchItem}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { pub tag: String, pub sequence_set: Sequence, pub operation: Operation, pub is_silent: bool, pub keywords: Vec, pub unchanged_since: Option, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum Operation { Set, Add, Clear, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct Response<'x> { pub items: Vec>, } impl ImapResponse for Response<'_> { fn serialize(self) -> Vec { let mut buf = Vec::with_capacity(64); for item in &self.items { item.serialize(&mut buf); } buf } } ================================================ FILE: crates/imap-proto/src/protocol/subscribe.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { pub tag: String, pub mailbox_name: String, } ================================================ FILE: crates/imap-proto/src/protocol/thread.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ImapResponse, search::Filter}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { pub tag: String, pub filter: Vec, pub algorithm: Algorithm, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum Algorithm { OrderedSubject, References, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct Response { pub is_uid: bool, pub threads: Vec>, } impl ImapResponse for Response { fn serialize(self) -> Vec { let mut buf = Vec::with_capacity(64); buf.extend_from_slice(b"* THREAD "); for thread in &self.threads { buf.push(b'('); for (pos, id) in thread.iter().enumerate() { if pos > 0 { buf.push(b' '); } buf.extend_from_slice(id.to_string().as_bytes()); } buf.push(b')'); } buf.extend_from_slice(b"\r\n"); buf } } #[cfg(test)] mod tests { use crate::protocol::ImapResponse; #[test] fn serialize_thread() { assert_eq!( String::from_utf8( super::Response { is_uid: true, threads: vec![vec![2, 10, 11], vec![49], vec![1, 3]], } .serialize() ) .unwrap(), "* THREAD (2 10 11)(49)(1 3)\r\n" ); } } ================================================ FILE: crates/imap-proto/src/receiver.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ResponseCode, ResponseType}; use compact_str::{CompactString, format_compact}; use std::fmt::Display; #[derive(Debug, Clone)] pub enum Error { NeedsMoreData, NeedsLiteral { size: u32 }, Error { response: trc::Error }, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct Request { pub tag: String, pub command: T, pub tokens: Vec, } pub trait CommandParser: Sized + Default { fn parse(bytes: &[u8], is_uid: bool) -> Option; fn tokenize_brackets(&self) -> bool; } #[derive(Debug, Clone, PartialEq, Eq)] pub enum Token { Argument(Vec), ParenthesisOpen, // ( ParenthesisClose, // ) BracketOpen, // [ BracketClose, // ] Lt, // < Gt, // > Dot, // . Nil, // NIL } impl Default for Request { fn default() -> Self { Self { tag: String::new(), command: T::default(), tokens: Vec::new(), } } } #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum State { Start, Tag, Command { is_uid: bool }, Argument { last_ch: u8 }, ArgumentQuoted { escaped: bool }, Literal { non_sync: bool }, LiteralSeek { size: u32, non_sync: bool }, LiteralData { remaining: u32 }, } pub struct Receiver { buf: ArgumentBuffer, pub request: Request, pub state: State, pub max_request_size: usize, pub current_request_size: usize, pub start_state: State, } const ARG_MAX_LEN: usize = 4096; struct ArgumentBuffer { buf: Vec, } impl Receiver { pub fn new() -> Self { Receiver { max_request_size: 25 * 1024 * 1024, // 25MB ..Default::default() } } pub fn with_start_state(mut self, state: State) -> Self { self.state = state; self.start_state = state; self } pub fn with_max_request_size(max_request_size: usize) -> Self { Receiver { max_request_size, ..Default::default() } } pub fn error_reset(&mut self, message: impl Into) -> Error { let request = std::mem::take(&mut self.request); let err = Error::err( if !request.tag.is_empty() { request.tag.into() } else { None }, message, ); self.buf = ArgumentBuffer::default(); self.state = self.start_state; self.current_request_size = 0; err } fn push_argument(&mut self, in_quote: bool) -> Result<(), Error> { if !self.buf.is_empty() { self.current_request_size += self.buf.len(); if self.current_request_size > self.max_request_size { return Err(self.error_reset(format_compact!( "Request exceeds maximum limit of {} bytes.", self.max_request_size ))); } self.request.tokens.push(Token::Argument(self.buf.take())); } else if in_quote { self.request.tokens.push(Token::Nil); } Ok(()) } fn push_token(&mut self, token: Token) -> Result<(), Error> { self.current_request_size += 1; if self.current_request_size > self.max_request_size { return Err(self.error_reset(format_compact!( "Request exceeds maximum limit of {} bytes.", self.max_request_size ))); } self.request.tokens.push(token); Ok(()) } pub fn parse(&mut self, bytes: &mut std::slice::Iter<'_, u8>) -> Result, Error> { #[allow(clippy::while_let_on_iterator)] while let Some(&ch) = bytes.next() { match self.state { State::Start => { if !ch.is_ascii_whitespace() { // SAFETY: This called just once self.buf.push_unchecked(ch); self.state = State::Tag; } } State::Tag => match ch { b' ' => { if !self.buf.is_empty() { self.request.tag = String::from_utf8(self.buf.take()).map_err(|_| { self.error_reset("Tag is not a valid UTF-8 string.") })?; self.state = State::Command { is_uid: false }; } } b'\t' | b'\r' => {} b'\n' => { return Err(self.error_reset(format_compact!( "Missing command after tag {:?}, found CRLF instead.", self.buf.as_str() ))); } _ => { self.buf.push_checked(ch, 128).map_err(|_| { self.error_reset("Tag exceeds maximum length of 128 characters.") })?; } }, State::Command { is_uid } => { if ch.is_ascii_alphanumeric() { self.buf .push_checked(ch.to_ascii_uppercase(), 15) .map_err(|_| { self.error_reset("Command exceeds maximum length of 15 characters.") })?; } else if ch.is_ascii_whitespace() { if !self.buf.is_empty() { if !self.buf.as_ref().eq_ignore_ascii_case(b"UID") { self.request.command = T::parse(self.buf.as_ref(), is_uid) .ok_or_else(|| { let err = format_compact!( "Unrecognized command '{}'.", String::from_utf8_lossy(self.buf.as_ref()) ); self.error_reset(err) })?; self.buf.clear(); if ch != b'\n' { self.state = State::Argument { last_ch: b' ' }; } else { self.state = self.start_state; self.current_request_size = 0; return Ok(std::mem::take(&mut self.request)); } } else { self.buf.clear(); self.state = State::Command { is_uid: true }; } } } else { return Err(self.error_reset(format_compact!( "Invalid character {:?} in command name.", ch as char ))); } } State::Argument { last_ch } => match ch { b'\"' if last_ch.is_ascii_whitespace() => { self.push_argument(false)?; self.state = State::ArgumentQuoted { escaped: false }; } b'{' if last_ch.is_ascii_whitespace() || (last_ch == b'~' && self.buf.len() == 1) => { if last_ch != b'~' { self.push_argument(false)?; } else { self.buf.clear(); } self.state = State::Literal { non_sync: false }; } b'(' => { self.push_argument(false)?; self.push_token(Token::ParenthesisOpen)?; } b')' => { self.push_argument(false)?; self.push_token(Token::ParenthesisClose)?; } b'[' if self.request.command.tokenize_brackets() => { self.push_argument(false)?; self.push_token(Token::BracketOpen)?; } b']' if self.request.command.tokenize_brackets() => { self.push_argument(false)?; self.push_token(Token::BracketClose)?; } b'<' if self.request.command.tokenize_brackets() => { self.push_argument(false)?; self.push_token(Token::Lt)?; } b'>' if self.request.command.tokenize_brackets() => { self.push_argument(false)?; self.push_token(Token::Gt)?; } b'.' if self.request.command.tokenize_brackets() => { self.push_argument(false)?; self.push_token(Token::Dot)?; } b'\n' => { self.push_argument(false)?; self.state = self.start_state; self.current_request_size = 0; return Ok(std::mem::take(&mut self.request)); } _ if ch.is_ascii_whitespace() => { self.push_argument(false)?; self.state = State::Argument { last_ch: ch }; } _ => { self.buf.push_checked(ch, ARG_MAX_LEN).map_err(|_| { self.error_reset("Argument exceeds maximum length of 4096 bytes.") })?; self.state = State::Argument { last_ch: ch }; } }, State::ArgumentQuoted { escaped } => match ch { b'\"' => { if !escaped { self.push_argument(true)?; self.state = State::Argument { last_ch: b' ' }; } else { self.buf .push_checked(ch, ARG_MAX_LEN) .map_err(|_| self.error_reset("Quoted argument too long."))?; self.state = State::ArgumentQuoted { escaped: false }; } } b'\\' => { if escaped { self.buf .push_checked(ch, ARG_MAX_LEN) .map_err(|_| self.error_reset("Quoted argument too long."))?; } self.state = State::ArgumentQuoted { escaped: !escaped }; } b'\n' => { return Err(self.error_reset("Unterminated quoted argument.")); } _ => { if escaped { // SAFETY: We check the size below self.buf.push_unchecked(b'\\'); } self.buf .push_checked(ch, ARG_MAX_LEN) .map_err(|_| self.error_reset("Quoted argument too long."))?; self.state = State::ArgumentQuoted { escaped: false }; } }, State::Literal { non_sync } => { match ch { b'}' => { if !self.buf.is_empty() { let size = self.buf.as_str().parse::().map_err(|_| { self.error_reset("Literal size is not a valid number.") })?; if self.current_request_size + size as usize > self.max_request_size { return Err(self.error_reset(format_compact!( "Literal exceeds the maximum request size of {} bytes.", self.max_request_size ))); } self.state = State::LiteralSeek { size, non_sync }; self.buf.resize_buffer(size as usize); self.buf.clear(); } else { return Err(self.error_reset("Invalid empty literal.")); } } b'+' => { if !self.buf.is_empty() { self.state = State::Literal { non_sync: true }; } else { return Err(self.error_reset("Invalid non-sync literal.")); } } _ if ch.is_ascii_digit() => { if !non_sync { self.buf.push_checked(ch, 15).map_err(|_| { self.error_reset("Literal size exceeds maximum of 15 digits.") })?; } else { // Digit found after non-sync '+' flag return Err(self.error_reset("Invalid literal.")); } } _ => { return Err(self.error_reset(format_compact!( "Invalid character {:?} in literal.", ch as char ))); } } } State::LiteralSeek { size, non_sync } => { if ch == b'\n' { if size > 0 { self.state = State::LiteralData { remaining: size }; } else { self.state = State::Argument { last_ch: b' ' }; self.push_token(Token::Nil)?; } if !non_sync { return Err(Error::NeedsLiteral { size }); } } else if !ch.is_ascii_whitespace() { return Err( self.error_reset("Expected CRLF after literal, found an invalid char.") ); } } State::LiteralData { remaining } => { // SAFETY: We checked the size before entering this state self.buf.push_unchecked(ch); if remaining > 1 { self.state = State::LiteralData { remaining: remaining - 1, }; } else { self.push_argument(false)?; self.state = State::Argument { last_ch: b' ' }; } } } } Err(Error::NeedsMoreData) } } impl ArgumentBuffer { pub fn new() -> Self { ArgumentBuffer { buf: Vec::with_capacity(10), } } pub fn resize_buffer(&mut self, size: usize) { if self.buf.capacity() < size { self.buf.reserve(size - self.buf.capacity()); } } #[inline(always)] pub fn push_checked(&mut self, byte: u8, limit: usize) -> Result<(), ()> { if self.buf.len() < limit { self.buf.push(byte); Ok(()) } else { Err(()) } } #[inline(always)] pub fn push_unchecked(&mut self, byte: u8) { self.buf.push(byte); } pub fn take(&mut self) -> Vec { let buf = self.buf.clone(); self.buf.clear(); buf } #[inline(always)] pub fn len(&self) -> usize { self.buf.len() } #[inline(always)] pub fn is_empty(&self) -> bool { self.buf.is_empty() } #[inline(always)] pub fn clear(&mut self) { self.buf.clear(); } #[inline(always)] pub fn as_str(&self) -> &str { std::str::from_utf8(&self.buf).unwrap_or_default() } } impl Token { pub fn unwrap_string(self) -> crate::parser::Result { match self { Token::Argument(value) => { String::from_utf8(value).map_err(|_| "Invalid UTF-8 in argument.".into()) } other => Ok(other.to_string()), } } pub fn unwrap_bytes(self) -> Vec { match self { Token::Argument(value) => value, other => other.as_bytes().to_vec(), } } pub fn eq_ignore_ascii_case(&self, bytes: &[u8]) -> bool { match self { Token::Argument(argument) => argument.eq_ignore_ascii_case(bytes), Token::ParenthesisOpen => bytes.eq(b"("), Token::ParenthesisClose => bytes.eq(b")"), Token::BracketOpen => bytes.eq(b"["), Token::BracketClose => bytes.eq(b"]"), Token::Gt => bytes.eq(b">"), Token::Lt => bytes.eq(b"<"), Token::Dot => bytes.eq(b"."), Token::Nil => bytes.is_empty(), } } pub fn is_parenthesis_open(&self) -> bool { matches!(self, Token::ParenthesisOpen) } pub fn is_parenthesis_close(&self) -> bool { matches!(self, Token::ParenthesisClose) } pub fn is_bracket_open(&self) -> bool { matches!(self, Token::BracketOpen) } pub fn is_bracket_close(&self) -> bool { matches!(self, Token::BracketClose) } pub fn is_dot(&self) -> bool { matches!(self, Token::Dot) } pub fn is_lt(&self) -> bool { matches!(self, Token::Lt) } pub fn is_gt(&self) -> bool { matches!(self, Token::Gt) } } impl AsRef<[u8]> for ArgumentBuffer { fn as_ref(&self) -> &[u8] { &self.buf } } impl Default for ArgumentBuffer { fn default() -> Self { Self::new() } } impl Display for Token { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { f.write_str(&String::from_utf8_lossy(self.as_bytes())) } } impl Token { pub fn as_bytes(&self) -> &[u8] { match self { Token::Argument(value) => value, Token::ParenthesisOpen => b"(", Token::ParenthesisClose => b")", Token::BracketOpen => b"[", Token::BracketClose => b"]", Token::Gt => b">", Token::Lt => b"<", Token::Dot => b".", Token::Nil => b"", } } } impl Error { pub fn err(tag: Option>, message: impl Into) -> Self { Error::Error { response: trc::ImapEvent::Error .ctx(trc::Key::Details, message) .ctx_opt(trc::Key::Id, tag.map(Into::into)) .ctx(trc::Key::Type, ResponseType::Bad) .code(ResponseCode::Parse), } } } impl Default for Receiver { fn default() -> Self { Self { buf: Default::default(), request: Default::default(), state: State::Start, start_state: State::Start, max_request_size: 25 * 1024 * 1024, current_request_size: 0, } } } impl Request { pub fn into_error(self, message: impl Into) -> trc::Error { trc::ImapEvent::Error .ctx(trc::Key::Details, message) .ctx(trc::Key::Id, CompactString::from_string_buffer(self.tag)) } pub fn into_parse_error(self, message: impl Into) -> trc::Error { trc::ImapEvent::Error .ctx(trc::Key::Details, message) .ctx(trc::Key::Id, CompactString::from_string_buffer(self.tag)) .ctx(trc::Key::Code, ResponseCode::Parse) .ctx(trc::Key::Type, ResponseType::Bad) } } pub(crate) fn bad(tag: impl Into, message: impl Into) -> trc::Error { trc::ImapEvent::Error .ctx(trc::Key::Details, message) .ctx(trc::Key::Id, tag) .ctx(trc::Key::Type, ResponseType::Bad) } /* astring = 1*ASTRING-CHAR / string string = quoted / literal literal = "{" number64 ["+"] "}" CRLF *CHAR8 quoted = DQUOTE *QUOTED-CHAR DQUOTE ASTRING-CHAR = ATOM-CHAR / resp-specials atom = 1*ATOM-CHAR ATOM-CHAR = atom-specials = "(" / ")" / "{" / SP / CTL / list-wildcards / quoted-specials / resp-specials resp-specials = "]" list-wildcards = "%" / "*" quoted-specials = DQUOTE / "\" DQUOTE = %x22 ; " (Double Quote) */ #[cfg(test)] mod tests { use crate::Command; use super::{Error, Receiver, Request, Token}; #[test] fn receiver_parse_ok() { let mut receiver = Receiver::new(); for (frames, expected_requests) in [ ( vec!["abcd CAPABILITY\r\n"], vec![Request { tag: "abcd".into(), command: Command::Capability, tokens: vec![], }], ), ( vec!["A023 LO", "GOUT\r\n"], vec![Request { tag: "A023".into(), command: Command::Logout, tokens: vec![], }], ), ( vec![" A001 AUTHENTICATE GSSAPI \r\n"], vec![Request { tag: "A001".into(), command: Command::Authenticate, tokens: vec![Token::Argument(b"GSSAPI".to_vec())], }], ), ( vec!["A03 AUTHENTICATE ", "PLAIN dGVzdAB0ZXN", "0AHRlc3Q=\r\n"], vec![Request { tag: "A03".into(), command: Command::Authenticate, tokens: vec![ Token::Argument(b"PLAIN".to_vec()), Token::Argument(b"dGVzdAB0ZXN0AHRlc3Q=".to_vec()), ], }], ), ( vec!["A003 CREATE owatagusiam/\r\n"], vec![Request { tag: "A003".into(), command: Command::Create, tokens: vec![Token::Argument(b"owatagusiam/".to_vec())], }], ), ( vec!["A682 LIST \"\" *\r\n"], vec![Request { tag: "A682".into(), command: Command::List, tokens: vec![Token::Nil, Token::Argument(b"*".to_vec())], }], ), ( vec!["A03 LIST () \"\" \"%\" RETURN (CHILDREN)\r\n"], vec![Request { tag: "A03".into(), command: Command::List, tokens: vec![ Token::ParenthesisOpen, Token::ParenthesisClose, Token::Nil, Token::Argument(b"%".to_vec()), Token::Argument(b"RETURN".to_vec()), Token::ParenthesisOpen, Token::Argument(b"CHILDREN".to_vec()), Token::ParenthesisClose, ], }], ), ( vec!["A05 LIST (REMOTE SUBSCRIBED) \"\" \"*\"\r\n"], vec![Request { tag: "A05".into(), command: Command::List, tokens: vec![ Token::ParenthesisOpen, Token::Argument(b"REMOTE".to_vec()), Token::Argument(b"SUBSCRIBED".to_vec()), Token::ParenthesisClose, Token::Nil, Token::Argument(b"*".to_vec()), ], }], ), ( vec!["a1 list \"\" (\"foo\")\r\n"], vec![Request { tag: "a1".into(), command: Command::List, tokens: vec![ Token::Nil, Token::ParenthesisOpen, Token::Argument(b"foo".to_vec()), Token::ParenthesisClose, ], }], ), ( vec!["a3.1 LIST \"\" (% music/rock)\r\n"], vec![Request { tag: "a3.1".into(), command: Command::List, tokens: vec![ Token::Nil, Token::ParenthesisOpen, Token::Argument(b"%".to_vec()), Token::Argument(b"music/rock".to_vec()), Token::ParenthesisClose, ], }], ), ( vec!["A01 LIST \"\" % RETURN (STATUS (MESSAGES UNSEEN))\r\n"], vec![Request { tag: "A01".into(), command: Command::List, tokens: vec![ Token::Nil, Token::Argument(b"%".to_vec()), Token::Argument(b"RETURN".to_vec()), Token::ParenthesisOpen, Token::Argument(b"STATUS".to_vec()), Token::ParenthesisOpen, Token::Argument(b"MESSAGES".to_vec()), Token::Argument(b"UNSEEN".to_vec()), Token::ParenthesisClose, Token::ParenthesisClose, ], }], ), ( vec![" A01 LiSt \"\" % RETURN ( STATUS ( MESSAGES UNSEEN ) ) \r\n"], vec![Request { tag: "A01".into(), command: Command::List, tokens: vec![ Token::Nil, Token::Argument(b"%".to_vec()), Token::Argument(b"RETURN".to_vec()), Token::ParenthesisOpen, Token::Argument(b"STATUS".to_vec()), Token::ParenthesisOpen, Token::Argument(b"MESSAGES".to_vec()), Token::Argument(b"UNSEEN".to_vec()), Token::ParenthesisClose, Token::ParenthesisClose, ], }], ), ( vec!["A02 LIST (SUBSCRIBED RECURSIVEMATCH) \"\" % RETURN (STATUS (MESSAGES))\r\n"], vec![Request { tag: "A02".into(), command: Command::List, tokens: vec![ Token::ParenthesisOpen, Token::Argument(b"SUBSCRIBED".to_vec()), Token::Argument(b"RECURSIVEMATCH".to_vec()), Token::ParenthesisClose, Token::Nil, Token::Argument(b"%".to_vec()), Token::Argument(b"RETURN".to_vec()), Token::ParenthesisOpen, Token::Argument(b"STATUS".to_vec()), Token::ParenthesisOpen, Token::Argument(b"MESSAGES".to_vec()), Token::ParenthesisClose, Token::ParenthesisClose, ], }], ), ( vec!["A002 CREATE \"INBOX.Sent Mail\"\r\n"], vec![Request { tag: "A002".into(), command: Command::Create, tokens: vec![Token::Argument(b"INBOX.Sent Mail".to_vec())], }], ), ( vec!["A002 CREATE \"Maibox \\\"quo\\\\ted\\\" \"\r\n"], vec![Request { tag: "A002".into(), command: Command::Create, tokens: vec![Token::Argument(b"Maibox \"quo\\ted\" ".to_vec())], }], ), ( vec!["A004 COPY 2:4 meeting\r\n"], vec![Request { tag: "A004".into(), command: Command::Copy(false), tokens: vec![ Token::Argument(b"2:4".to_vec()), Token::Argument(b"meeting".to_vec()), ], }], ), ( vec![ "A282 SEARCH RETURN (MIN COU", "NT) FLAGGED SINCE 1-Feb-1994 ", "NOT FROM \"Smith\"\r\n", ], vec![Request { tag: "A282".into(), command: Command::Search(false), tokens: vec![ Token::Argument(b"RETURN".to_vec()), Token::ParenthesisOpen, Token::Argument(b"MIN".to_vec()), Token::Argument(b"COUNT".to_vec()), Token::ParenthesisClose, Token::Argument(b"FLAGGED".to_vec()), Token::Argument(b"SINCE".to_vec()), Token::Argument(b"1-Feb-1994".to_vec()), Token::Argument(b"NOT".to_vec()), Token::Argument(b"FROM".to_vec()), Token::Argument(b"Smith".to_vec()), ], }], ), ( vec!["F284 UID STORE $ +FLAGS.Silent (\\Deleted)\r\n"], vec![Request { tag: "F284".into(), command: Command::Store(true), tokens: vec![ Token::Argument(b"$".to_vec()), Token::Argument(b"+FLAGS.Silent".to_vec()), Token::ParenthesisOpen, Token::Argument(b"\\Deleted".to_vec()), Token::ParenthesisClose, ], }], ), ( vec!["A654 FETCH 2:4 (FLAGS BODY[HEADER.FIELDS (DATE FROM)])\r\n"], vec![Request { tag: "A654".into(), command: Command::Fetch(false), tokens: vec![ Token::Argument(b"2:4".to_vec()), Token::ParenthesisOpen, Token::Argument(b"FLAGS".to_vec()), Token::Argument(b"BODY".to_vec()), Token::BracketOpen, Token::Argument(b"HEADER".to_vec()), Token::Dot, Token::Argument(b"FIELDS".to_vec()), Token::ParenthesisOpen, Token::Argument(b"DATE".to_vec()), Token::Argument(b"FROM".to_vec()), Token::ParenthesisClose, Token::BracketClose, Token::ParenthesisClose, ], }], ), ( vec![ "B283 UID SEARCH RETURN (SAVE) CHARSET ", "KOI8-R (OR $ 1,3000:3021) TEXT \"hello world\"\r\n", ], vec![Request { tag: "B283".into(), command: Command::Search(true), tokens: vec![ Token::Argument(b"RETURN".to_vec()), Token::ParenthesisOpen, Token::Argument(b"SAVE".to_vec()), Token::ParenthesisClose, Token::Argument(b"CHARSET".to_vec()), Token::Argument(b"KOI8-R".to_vec()), Token::ParenthesisOpen, Token::Argument(b"OR".to_vec()), Token::Argument(b"$".to_vec()), Token::Argument(b"1,3000:3021".to_vec()), Token::ParenthesisClose, Token::Argument(b"TEXT".to_vec()), Token::Argument(b"hello world".to_vec()), ], }], ), ( vec![ "P283 SEARCH CHARSET UTF-8 (OR $ 1,3000:3021) ", "TEXT {8+}\r\nмать\r\n", ], vec![Request { tag: "P283".into(), command: Command::Search(false), tokens: vec![ Token::Argument(b"CHARSET".to_vec()), Token::Argument(b"UTF-8".to_vec()), Token::ParenthesisOpen, Token::Argument(b"OR".to_vec()), Token::Argument(b"$".to_vec()), Token::Argument(b"1,3000:3021".to_vec()), Token::ParenthesisClose, Token::Argument(b"TEXT".to_vec()), Token::Argument("мать".to_string().into_bytes()), ], }], ), ( vec!["A001 LOGIN {11}\r\n", "FRED FOOBAR {7}\r\n", "fat man\r\n"], vec![Request { tag: "A001".into(), command: Command::Login, tokens: vec![ Token::Argument(b"FRED FOOBAR".to_vec()), Token::Argument(b"fat man".to_vec()), ], }], ), ( vec!["TAG3 CREATE \"Test-ąęć-Test\"\r\n"], vec![Request { tag: "TAG3".into(), command: Command::Create, tokens: vec![Token::Argument("Test-ąęć-Test".as_bytes().to_vec())], }], ), ( vec!["abc LOGIN {0}\r\n", "\r\n"], vec![Request { tag: "abc".into(), command: Command::Login, tokens: vec![Token::Nil], }], ), ( vec!["abc LOGIN {0+}\r\n\r\n"], vec![Request { tag: "abc".into(), command: Command::Login, tokens: vec![Token::Nil], }], ), ( vec![ "A003 APPEND saved-messages (\\Seen) {297+}\r\n", "Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST)\r\n", "From: Fred Foobar \r\n", "Subject: afternoon meeting\r\n", "To: mooch@example.com\r\n", "Message-Id: \r\n", "MIME-Version: 1.0\r\n", "Content-Type: TEXT/PLAIN; CHARSET=US-ASCII\r\n", "\r\n", "Hello Joe, do you think we can meet at 3:30 tomorrow?\r\n\r\n", ], vec![Request { tag: "A003".into(), command: Command::Append, tokens: vec![ Token::Argument(b"saved-messages".to_vec()), Token::ParenthesisOpen, Token::Argument(b"\\Seen".to_vec()), Token::ParenthesisClose, Token::Argument( concat!( "Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST)\r\n", "From: Fred Foobar \r\n", "Subject: afternoon meeting\r\n", "To: mooch@example.com\r\n", "Message-Id: \r\n", "MIME-Version: 1.0\r\n", "Content-Type: TEXT/PLAIN; CHARSET=US-ASCII\r\n", "\r\n", "Hello Joe, do you think we can meet at 3:30 tomorrow?\r\n" ) .as_bytes() .to_vec(), ), ], }], ), ( vec![ "A003 APPEND saved-messages (\\Seen) {326}\r\n", "Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST)\r\n", "From: Fred Foobar \r\n", "Subject: afternoon meeting\r\n", "To: mooch@owatagu.siam.edu.example\r\n", "Message-Id: \r\n", "MIME-Version: 1.0\r\n", "Content-Type: TEXT/PLAIN; CHARSET=US-ASCII\r\n", "\r\n", "Hello Joe, do you think we can meet at 3:30 tomorrow?\r\n\r\n", ], vec![Request { tag: "A003".into(), command: Command::Append, tokens: vec![ Token::Argument(b"saved-messages".to_vec()), Token::ParenthesisOpen, Token::Argument(b"\\Seen".to_vec()), Token::ParenthesisClose, Token::Argument( concat!( "Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST)\r\n", "From: Fred Foobar \r\n", "Subject: afternoon meeting\r\n", "To: mooch@owatagu.siam.edu.example\r\n", "Message-Id: \r\n", "MIME-Version: 1.0\r\n", "Content-Type: TEXT/PLAIN; CHARSET=US-ASCII\r\n", "\r\n", "Hello Joe, do you think we can meet at 3:30 tomorrow?\r\n", ) .as_bytes() .to_vec(), ), ], }], ), ( vec!["001 NOOP\r\n002 CAPABILITY\r\nabc LOGIN hello world\r\n"], vec![ Request { tag: "001".into(), command: Command::Noop, tokens: vec![], }, Request { tag: "002".into(), command: Command::Capability, tokens: vec![], }, Request { tag: "abc".into(), command: Command::Login, tokens: vec![ Token::Argument(b"hello".to_vec()), Token::Argument(b"world".to_vec()), ], }, ], ), ] { let mut requests = Vec::new(); for frame in &frames { let mut bytes = frame.as_bytes().iter(); loop { match receiver.parse(&mut bytes) { Ok(request) => requests.push(request), Err(Error::NeedsMoreData | Error::NeedsLiteral { .. }) => break, Err(err) => panic!("{:?} for frames {:#?}", err, frames), } } } assert_eq!(requests, expected_requests, "{:#?}", frames); } } #[test] fn receiver_parse_invalid() { let mut receiver = Receiver::::new(); for invalid in [ //"\r\n", //" \r \n", "a001\r\n", "a001 unknown\r\n", "a001 login {abc}\r\n", "a001 login {+30}\r\n", "a001 login {30} junk\r\n", ] { match receiver.parse(&mut invalid.as_bytes().iter()) { Err(Error::Error { .. }) => {} result => panic!("Expecter error, got: {:?}", result), } } } } ================================================ FILE: crates/imap-proto/src/utf7.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ // Ported from https://github.com/jstedfast/MailKit/blob/master/MailKit/Net/Imap/ImapEncoding.cs // Author: Jeffrey Stedfast static UTF_7_RANK: &[u8] = &[ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 63, 255, 255, 255, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255, 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, 255, 255, ]; static UTF_7_MAP: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,"; pub fn utf7_decode(text: &str) -> Option { let mut bytes: Vec = Vec::with_capacity(text.len()); let mut bits = 0; let mut v: u32 = 0; let mut shifted = false; let mut text = text.chars().peekable(); while let Some(ch) = text.next() { if shifted { if ch == '-' { shifted = false; bits = 0; v = 0; } else if ch as usize > 127 { return None; } else { let rank = *UTF_7_RANK.get(ch as usize)?; if rank == 0xff { return None; } v = (v << 6) | rank as u32; bits += 6; if bits >= 16 { bytes.push(((v >> (bits - 16)) & 0xffff) as u16); bits -= 16; } } } else if ch == '&' { match text.peek() { Some('-') => { bytes.push(b'&' as u16); text.next(); } Some(_) => { shifted = true; } None => { bytes.push(ch as u16); } } } else { bytes.push(ch as u16); } } String::from_utf16(&bytes).ok() } pub fn utf7_encode(text: &str) -> String { let mut result = String::with_capacity(text.len()); let mut shifted = false; let mut bits = 0; let mut u: u32 = 0; for ch in text.encode_utf16() { if (0x20..0x7f).contains(&ch) { if shifted { if bits > 0 { result.push(char::from(UTF_7_MAP[((u << (6 - bits)) & 0x3f) as usize])); } result.push('-'); shifted = false; bits = 0; } if ch == 0x26 { result.push_str("&-"); } else { result.push((ch as u8) as char); } } else { if !shifted { result.push('&'); shifted = true; } u = (u << 16) | ch as u32; bits += 16; while bits >= 6 { result.push(char::from(UTF_7_MAP[((u >> (bits - 6)) & 0x3f) as usize])); bits -= 6; } } } if shifted { if bits > 0 { result.push(char::from(UTF_7_MAP[((u << (6 - bits)) & 0x3f) as usize])); } result.push('-'); } result } #[inline(always)] pub fn utf7_maybe_decode(text: String, is_utf8: bool) -> String { if is_utf8 { text } else { utf7_decode(&text).unwrap_or(text) } } #[cfg(test)] mod tests { #[test] fn utf7_decode() { for (input, expected_result) in [ ("~peter/mail/&U,BTFw-/&ZeVnLIqe-", "~peter/mail/台北/日本語"), ("&U,BTF2XlZyyKng-", "台北日本語"), ("Hello, World&ACE-", "Hello, World!"), ("Hi Mom -&Jjo--!", "Hi Mom -☺-!"), ("&ZeVnLIqe-", "日本語"), ("Item 3 is &AKM-1.", "Item 3 is £1."), ("Plus minus &- -&- &--", "Plus minus & -& &-"), ( "&APw-ber ihre mi&AN8-liche Lage&ADs- &ACI-wir", "über ihre mißliche Lage; \"wir", ), ( concat!( "&ACI-The sayings of Confucius,&ACI- James R. Ware, trans. &U,BTFw-:\n", "&ZYeB9FH6ckh5Pg-, 1980.\n", "&Vttm+E6UfZM-, &W4tRQ066bOg-, &UxdOrA-: &Ti1XC2b4Xpc-, 1990." ), concat!( "\"The sayings of Confucius,\" James R. Ware, trans. 台北:\n", "文致出版社, 1980.\n", "四書五經, 宋元人注, 北京: 中國書店, 1990." ), ), ("Test-ąęć-Test", "Test-ąęć-Test"), (r#"&A8g- "&A9QD1APUA9gD3APcA-+""#, "ψ \"ϔϔϔϘϜϜ+\""), ] { assert_eq!( super::utf7_decode(input).expect(input), expected_result, "while decoding {:?}", input ); } } #[test] fn utf7_encode() { for (expected_result, input) in [ ("~peter/mail/&U,BTFw-/&ZeVnLIqe-", "~peter/mail/台北/日本語"), ("&U,BTF2XlZyyKng-", "台北日本語"), ("Hi Mom -&Jjo--!", "Hi Mom -☺-!"), ("&ZeVnLIqe-", "日本語"), ("Item 3 is &AKM-1.", "Item 3 is £1."), ("Plus minus &- -&- &--", "Plus minus & -& &-"), ("&VMhUyNg93gQ-", "哈哈😄"), ] { assert_eq!( super::utf7_encode(input), expected_result, "while encoding {:?}", expected_result ); } } } ================================================ FILE: crates/jmap/Cargo.toml ================================================ [package] name = "jmap" version = "0.15.5" edition = "2024" [dependencies] store = { path = "../store" } nlp = { path = "../nlp" } http_proto = { path = "../http-proto" } jmap_proto = { path = "../jmap-proto" } types = { path = "../types" } smtp = { path = "../smtp" } utils = { path = "../utils" } common = { path = "../common" } services = { path = "../services" } directory = { path = "../directory" } trc = { path = "../trc" } spam-filter = { path = "../spam-filter" } email = { path = "../email" } groupware = { path = "../groupware" } calcard = { version = "0.3" } smtp-proto = { version = "0.2" } mail-parser = { version = "0.11", features = ["full_encoding", "rkyv"] } mail-builder = { version = "0.4" } mail-send = { version = "0.5", default-features = false, features = ["cram-md5", "ring", "tls12"] } mail-auth = { version = "0.7.1", features = ["generate"] } sieve-rs = { version = "0.7", features = ["rkyv"] } jmap-tools = { version = "0.1", features = ["rkyv"] } serde = { version = "1.0", features = ["derive"]} serde_json = "1.0" hyper = { version = "1.0.1", features = ["server", "http1", "http2"] } hyper-util = { version = "0.1.1", features = ["tokio"] } http-body-util = "0.1.0" tokio = { version = "1.47", features = ["rt"] } futures-util = "0.3.28" async-stream = "0.3.5" base64 = "0.22" p256 = { version = "0.13", features = ["ecdh"] } hkdf = "0.12.3" sha1 = "0.10" sha2 = "0.10" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-webpki-roots", "http2"]} tokio-tungstenite = "0.28" tungstenite = "0.28" chrono = "0.4" rand = "0.9.0" pkcs8 = { version = "0.10.2", features = ["alloc", "std"] } lz4_flex = { version = "0.12", default-features = false } aes-gcm = "0.10.1" aes-gcm-siv = "0.11.1" rsa = "0.9.2" rkyv = { version = "0.8.10", features = ["little_endian"] } compact_str = "0.9.0" hashify = "0.2" [features] test_mode = [] enterprise = [] ================================================ FILE: crates/jmap/src/addressbook/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{api::acl::JmapRights, changes::state::JmapCacheState}; use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; use groupware::{cache::GroupwareCache, contact::AddressBook}; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::addressbook::{self, AddressBookProperty, AddressBookValue}, }; use jmap_tools::{Map, Value}; use store::{ValueKey, roaring::RoaringBitmap, write::{AlignedBytes, Archive, ValueClass}}; use trc::AddContext; use types::{ acl::{Acl, AclGrant}, collection::{Collection, SyncCollection}, field::PrincipalField, }; pub trait AddressBookGet: Sync + Send { fn address_book_get( &self, request: GetRequest, access_token: &AccessToken, ) -> impl Future>> + Send; } impl AddressBookGet for Server { async fn address_book_get( &self, mut request: GetRequest, access_token: &AccessToken, ) -> trc::Result> { let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ AddressBookProperty::Id, AddressBookProperty::Name, AddressBookProperty::Description, AddressBookProperty::SortOrder, AddressBookProperty::IsDefault, AddressBookProperty::IsSubscribed, AddressBookProperty::MyRights, ]); let account_id = request.account_id.document_id(); let cache = self .fetch_dav_resources(access_token, account_id, SyncCollection::AddressBook) .await?; let address_book_ids = if access_token.is_member(account_id) { cache.document_ids(true).collect::() } else { cache.shared_containers(access_token, [Acl::Read, Acl::ReadItems], true) }; let default_address_book_id = self .store() .get_value::(ValueKey { account_id, collection: Collection::Principal.into(), document_id: 0, class: ValueClass::Property(PrincipalField::DefaultAddressBookId.into()), }) .await .caused_by(trc::location!())? .or_else(|| { if address_book_ids.len() == 1 { address_book_ids.iter().next() } else { None } }); let ids = if let Some(ids) = ids { ids } else { address_book_ids .iter() .take(self.core.jmap.get_max_objects) .map(Into::into) .collect::>() }; let mut response = GetResponse { account_id: request.account_id.into(), state: cache.get_state(true).into(), list: Vec::with_capacity(ids.len()), not_found: vec![], }; for id in ids { // Obtain the address_book object let document_id = id.document_id(); if !address_book_ids.contains(document_id) { response.not_found.push(id); continue; } let _address_book = if let Some(address_book) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::AddressBook, document_id, )) .await? { address_book } else { response.not_found.push(id); continue; }; let address_book = _address_book .unarchive::() .caused_by(trc::location!())?; let mut result = Map::with_capacity(properties.len()); for property in &properties { match property { AddressBookProperty::Id => { result.insert_unchecked(AddressBookProperty::Id, AddressBookValue::Id(id)); } AddressBookProperty::Name => { result.insert_unchecked( AddressBookProperty::Name, address_book.preferences(access_token).name.to_string(), ); } AddressBookProperty::Description => { result.insert_unchecked( AddressBookProperty::Description, address_book .preferences(access_token) .description .as_ref() .map(|v| v.to_string()), ); } AddressBookProperty::SortOrder => { result.insert_unchecked( AddressBookProperty::SortOrder, address_book .preferences(access_token) .sort_order .to_native(), ); } AddressBookProperty::IsDefault => { result.insert_unchecked( AddressBookProperty::IsDefault, default_address_book_id == Some(document_id), ); } AddressBookProperty::IsSubscribed => { result.insert_unchecked( AddressBookProperty::IsSubscribed, address_book .subscribers .iter() .any(|account_id| *account_id == access_token.primary_id()), ); } AddressBookProperty::ShareWith => { result.insert_unchecked( AddressBookProperty::ShareWith, JmapRights::share_with::( account_id, access_token, &address_book .acls .iter() .map(AclGrant::from) .collect::>(), ), ); } AddressBookProperty::MyRights => { result.insert_unchecked( AddressBookProperty::MyRights, if access_token.is_shared(account_id) { JmapRights::rights::( address_book.acls.effective_acl(access_token), ) } else { JmapRights::all_rights::() }, ); } property => { result.insert_unchecked(property.clone(), Value::Null); } } } response.list.push(result.into()); } Ok(response) } } ================================================ FILE: crates/jmap/src/addressbook/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod get; pub mod set; ================================================ FILE: crates/jmap/src/addressbook/set.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::api::acl::{JmapAcl, JmapRights}; use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; use groupware::{ DestroyArchive, cache::GroupwareCache, contact::{AddressBook, AddressBookPreferences, ContactCard}, }; use http_proto::HttpSessionData; use jmap_proto::{ error::set::SetError, method::set::{SetRequest, SetResponse}, object::addressbook::{self, AddressBookProperty, AddressBookValue}, request::{IntoValid, reference::MaybeIdReference}, types::state::State, }; use jmap_tools::{JsonPointerItem, Key, Value}; use rand::{Rng, distr::Alphanumeric}; use store::{ SerializeInfallible, ValueKey, ahash::AHashSet, write::{AlignedBytes, Archive, BatchBuilder, ValueClass}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, field::PrincipalField, }; pub trait AddressBookSet: Sync + Send { fn address_book_set( &self, request: SetRequest<'_, addressbook::AddressBook>, access_token: &AccessToken, session: &HttpSessionData, ) -> impl Future>> + Send; } impl AddressBookSet for Server { async fn address_book_set( &self, mut request: SetRequest<'_, addressbook::AddressBook>, access_token: &AccessToken, _session: &HttpSessionData, ) -> trc::Result> { let account_id = request.account_id.document_id(); let cache = self .fetch_dav_resources(access_token, account_id, SyncCollection::AddressBook) .await?; let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; let will_destroy = request.unwrap_destroy().into_valid().collect::>(); let is_shared = access_token.is_shared(account_id); let mut set_default = None; // Process creates let mut batch = BatchBuilder::new(); 'create: for (id, object) in request.unwrap_create() { if is_shared { response.not_created.append( id, SetError::forbidden() .with_description("Cannot create address books in a shared account."), ); continue 'create; } let mut address_book = AddressBook { name: rand::rng() .sample_iter(Alphanumeric) .take(10) .map(char::from) .collect::(), preferences: vec![AddressBookPreferences { account_id, name: "Address Book".to_string(), ..Default::default() }], ..Default::default() }; // Process changes if let Err(err) = update_address_book(object, &mut address_book, access_token) { response.not_created.append(id, err); continue 'create; } // Validate ACLs if !address_book.acls.is_empty() { if let Err(err) = self.acl_validate(&address_book.acls).await { response.not_created.append(id, err.into()); continue 'create; } self.refresh_acls(&address_book.acls, None).await; } // Insert record let document_id = self .store() .assign_document_ids(account_id, Collection::AddressBook, 1) .await .caused_by(trc::location!())?; address_book .insert(access_token, account_id, document_id, &mut batch) .caused_by(trc::location!())?; if let Some(MaybeIdReference::Reference(id_ref)) = &request.arguments.on_success_set_is_default && id_ref == &id { set_default = Some(document_id); } response.created(id, document_id); } // Process updates 'update: for (id, object) in request.unwrap_update().into_valid() { // Make sure id won't be destroyed if will_destroy.contains(&id) { response.not_updated.append(id, SetError::will_destroy()); continue 'update; } // Obtain address book let document_id = id.document_id(); let address_book_ = if let Some(address_book_) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::AddressBook, document_id, )) .await? { address_book_ } else { response.not_updated.append(id, SetError::not_found()); continue 'update; }; let address_book = address_book_ .to_unarchived::() .caused_by(trc::location!())?; let mut new_address_book = address_book .deserialize::() .caused_by(trc::location!())?; // Apply changes let has_acl_changes = match update_address_book(object, &mut new_address_book, access_token) { Ok(has_acl_changes_) => has_acl_changes_, Err(err) => { response.not_updated.append(id, err); continue 'update; } }; // Validate ACL if is_shared { let acl = address_book.inner.acls.effective_acl(access_token); if !acl.contains(Acl::Modify) || (has_acl_changes && !acl.contains(Acl::Share)) { response.not_updated.append( id, SetError::forbidden() .with_description("You are not allowed to modify this address book."), ); continue 'update; } } if has_acl_changes { if let Err(err) = self.acl_validate(&new_address_book.acls).await { response.not_updated.append(id, err.into()); continue 'update; } self.refresh_archived_acls( &new_address_book.acls, address_book.inner.acls.as_slice(), ) .await; } // Update record new_address_book .update( access_token, address_book, account_id, document_id, &mut batch, ) .caused_by(trc::location!())?; response.updated.append(id, None); } // Process deletions let mut reset_default_address_book = false; if !will_destroy.is_empty() { let mut destroy_children = AHashSet::new(); let mut destroy_parents = AHashSet::new(); let default_address_book_id = self .store() .get_value::(ValueKey { account_id, collection: Collection::Principal.into(), document_id: 0, class: ValueClass::Property(PrincipalField::DefaultAddressBookId.into()), }) .await .caused_by(trc::location!())?; let on_destroy_remove_contents = request .arguments .on_destroy_remove_contents .unwrap_or(false); for id in will_destroy { let document_id = id.document_id(); if !cache.has_container_id(&document_id) { response.not_destroyed.append(id, SetError::not_found()); continue; }; let Some(address_book_) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::AddressBook, document_id, )) .await .caused_by(trc::location!())? else { response.not_destroyed.append(id, SetError::not_found()); continue; }; let address_book = address_book_ .to_unarchived::() .caused_by(trc::location!())?; // Validate ACLs if is_shared && !address_book .inner .acls .effective_acl(access_token) .contains_all([Acl::Delete, Acl::RemoveItems].into_iter()) { response.not_destroyed.append( id, SetError::forbidden() .with_description("You are not allowed to delete this address book."), ); continue; } // Obtain children ids let children_ids = cache.children_ids(document_id).collect::>(); if !children_ids.is_empty() && !on_destroy_remove_contents { response .not_destroyed .append(id, SetError::address_book_has_contents()); continue; } destroy_children.extend(children_ids.iter().copied()); destroy_parents.insert(document_id); // Delete record DestroyArchive(address_book) .delete(access_token, account_id, document_id, None, &mut batch) .caused_by(trc::location!())?; if default_address_book_id == Some(document_id) { reset_default_address_book = true; } response.destroyed.push(id); } // Delete children if !destroy_children.is_empty() { for document_id in destroy_children { if let Some(card_) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::ContactCard, document_id, )) .await? { let card = card_ .to_unarchived::() .caused_by(trc::location!())?; if card .inner .names .iter() .all(|n| destroy_parents.contains(&n.parent_id.to_native())) { // Card only belongs to address books being deleted, delete it DestroyArchive(card).delete_all( access_token, account_id, document_id, &mut batch, )?; } else { // Unlink addressbook id from card let mut new_card = card .deserialize::() .caused_by(trc::location!())?; new_card .names .retain(|n| !destroy_parents.contains(&n.parent_id)); new_card.update( access_token, card, account_id, document_id, &mut batch, )?; } } } } } // Set default address book if let Some(MaybeIdReference::Id(id)) = &request.arguments.on_success_set_is_default { set_default = Some(id.document_id()); } if let Some(default_address_book_id) = set_default { if response.not_created.is_empty() && response.not_updated.is_empty() && response.not_destroyed.is_empty() { batch .with_account_id(account_id) .with_collection(Collection::Principal) .with_document(0) .set( PrincipalField::DefaultAddressBookId, default_address_book_id.serialize(), ); } } else if reset_default_address_book { batch .with_account_id(account_id) .with_collection(Collection::Principal) .with_document(0) .clear(PrincipalField::DefaultAddressBookId); } // Write changes if !batch.is_empty() && let Ok(change_id) = self .commit_batch(batch) .await .caused_by(trc::location!())? .last_change_id(account_id) { self.notify_task_queue(); response.new_state = State::Exact(change_id).into(); } Ok(response) } } fn update_address_book( updates: Value<'_, AddressBookProperty, AddressBookValue>, address_book: &mut AddressBook, access_token: &AccessToken, ) -> Result> { let mut has_acl_changes = false; for (property, value) in updates.into_expanded_object() { let Key::Property(property) = property else { return Err(SetError::invalid_properties() .with_property(property.to_owned()) .with_description("Invalid property.")); }; match (property, value) { (AddressBookProperty::Name, Value::Str(value)) if (1..=255).contains(&value.len()) => { address_book.preferences_mut(access_token).name = value.into_owned(); } (AddressBookProperty::Description, Value::Str(value)) if value.len() < 255 => { address_book.preferences_mut(access_token).description = value.into_owned().into(); } (AddressBookProperty::Description, Value::Null) => { address_book.preferences_mut(access_token).description = None; } (AddressBookProperty::SortOrder, Value::Number(value)) => { address_book.preferences_mut(access_token).sort_order = value.cast_to_u64() as u32; } (AddressBookProperty::IsSubscribed, Value::Bool(subscribe)) => { let account_id = access_token.primary_id(); if subscribe { if !address_book.subscribers.contains(&account_id) { address_book.subscribers.push(account_id); } } else { address_book.subscribers.retain(|id| *id != account_id); } } (AddressBookProperty::ShareWith, value) => { address_book.acls = JmapRights::acl_set::(value)?; has_acl_changes = true; } (AddressBookProperty::Pointer(pointer), value) if matches!( pointer.first(), Some(JsonPointerItem::Key(Key::Property( AddressBookProperty::ShareWith ))) ) => { let mut pointer = pointer.iter(); pointer.next(); address_book.acls = JmapRights::acl_patch::( std::mem::take(&mut address_book.acls), pointer, value, )?; has_acl_changes = true; } (property, _) => { return Err(SetError::invalid_properties() .with_property(property.clone()) .with_description("Field could not be set.")); } } } // Validate name if address_book.preferences(access_token).name.is_empty() { return Err(SetError::invalid_properties() .with_property(AddressBookProperty::Name) .with_description("Missing name.")); } Ok(has_acl_changes) } ================================================ FILE: crates/jmap/src/api/acl.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; use directory::backend::internal::manage::ManageDirectory; use jmap_proto::{ error::set::SetError, object::{JmapRight, JmapSharedObject}, }; use jmap_tools::{JsonPointerIter, Key, Map, Property, Value}; use types::{ acl::{Acl, AclGrant}, id::Id, }; use utils::map::bitmap::Bitmap; pub struct JmapRights; impl JmapRights { pub fn acl_set( value: Value<'_, T::Property, T::Element>, ) -> Result, SetError> where Id: TryFrom, T::Right: TryFrom, { let mut grants = Vec::new(); for (key, value) in value.into_expanded_object() { let account_id = key .try_into_property() .and_then(|p| Id::try_from(p).ok()) .ok_or_else(|| { SetError::invalid_properties() .with_property(T::SHARE_WITH_PROPERTY) .with_description("Invalid account id.") })? .document_id(); if !grants .iter() .any(|item: &AclGrant| item.account_id == account_id) { let acls = Self::map_acls::(value)?; if !acls.is_empty() { grants.push(AclGrant { account_id, grants: acls, }); } } } Ok(grants) } pub fn acl_patch( mut grants: Vec, mut path: JsonPointerIter<'_, T::Property>, value: Value<'_, T::Property, T::Element>, ) -> Result, SetError> where Id: TryFrom, T::Right: TryFrom, { let account_id = path .next() .and_then(|item| item.as_property_key()) .cloned() .and_then(|p| Id::try_from(p).ok()) .ok_or_else(|| { SetError::invalid_properties() .with_property(T::SHARE_WITH_PROPERTY) .with_description("Invalid account id.") })? .document_id(); if let Some(right) = path.next() { let is_set = match value { Value::Bool(is_set) => is_set, Value::Null => false, _ => { return Err(SetError::invalid_properties() .with_property(T::SHARE_WITH_PROPERTY) .with_description("Invalid ACL value.")); } }; let acl = right .as_property_key() .cloned() .and_then(|p| T::Right::try_from(p).ok()) .ok_or_else(|| { SetError::invalid_properties() .with_property(T::SHARE_WITH_PROPERTY) .with_description(format!( "Invalid permission {:?}.", right.to_cow().unwrap_or_default() )) })? .to_acl() .iter() .copied(); if let Some(acl_item) = grants.iter_mut().find(|item| item.account_id == account_id) { if is_set { acl_item.grants.insert_many(acl); } else { acl_item.grants.insert_many(acl); if acl_item.grants.is_empty() { grants.retain(|item| item.account_id != account_id); } } } else if is_set { grants.push(AclGrant { account_id, grants: Bitmap::from_iter(acl), }); } } else { let acls = Self::map_acls::(value)?; if !acls.is_empty() { if let Some(acl_item) = grants.iter_mut().find(|item| item.account_id == account_id) { acl_item.grants = acls; } else { grants.push(AclGrant { account_id, grants: acls, }); } } else { grants.retain(|item| item.account_id != account_id); } } Ok(grants) } fn map_acls( value: Value<'_, T::Property, T::Element>, ) -> Result, SetError> where Id: TryFrom, T::Right: TryFrom, { let mut acls = Bitmap::new(); for key in value.into_expanded_boolean_set() { acls.insert_many( key.as_property() .and_then(|p| T::Right::try_from(p.clone()).ok()) .ok_or_else(|| { SetError::invalid_properties() .with_property(T::SHARE_WITH_PROPERTY) .with_description(format!("Invalid permission {:?}.", key.to_string())) })? .to_acl() .iter() .copied(), ); } Ok(acls) } pub fn all_rights() -> Value<'static, T::Property, T::Element> { let rights = T::Right::all_rights(); let mut obj = Map::with_capacity(rights.len()); for right in rights { obj.insert_unchecked(Key::Property((*right).into()), Value::Bool(true)); } Value::Object(obj) } pub fn rights( acls: Bitmap, ) -> Value<'static, T::Property, T::Element> { let mut obj = Map::with_capacity(3); for right in T::Right::all_rights() { obj.insert_unchecked( Key::Property((*right).into()), Value::Bool(right.to_acl().iter().all(|acl| acls.contains(*acl))), ); } Value::Object(obj) } pub fn share_with( account_id: u32, access_token: &AccessToken, grants: &[AclGrant], ) -> Value<'static, T::Property, T::Element> where T::Property: From, { if access_token.is_member(account_id) || grants.effective_acl(access_token).contains(Acl::Share) { let mut share_with = Map::with_capacity(grants.len()); for grant in grants { share_with.insert_unchecked( Key::Property(Id::from(grant.account_id).into()), Self::rights::(grant.grants), ); } Value::Object(share_with) } else { Value::Null } } } pub trait JmapAcl { fn acl_validate( &self, grants: &[AclGrant], ) -> impl Future> + Send; } pub enum ShareValidationError { MaxSharesExceeded(usize), InvalidAccountId(Id), } impl JmapAcl for Server { async fn acl_validate(&self, grants: &[AclGrant]) -> Result<(), ShareValidationError> { if grants.len() > self.core.groupware.max_shares_per_item { return Err(ShareValidationError::MaxSharesExceeded( self.core.groupware.max_shares_per_item, )); } let principal_ids = self .store() .principal_ids(None, None) .await .unwrap_or_default(); for grant in grants { if !principal_ids.contains(grant.account_id) { return Err(ShareValidationError::InvalidAccountId(Id::from( grant.account_id, ))); } } Ok(()) } } impl From for SetError { fn from(err: ShareValidationError) -> Self { match err { ShareValidationError::MaxSharesExceeded(max) => SetError::invalid_properties() .with_description(format!( "Maximum number of shares per item exceeded (max: {max})" )), ShareValidationError::InvalidAccountId(id) => SetError::invalid_properties() .with_description(format!("Account id {id} is invalid.")), } } } ================================================ FILE: crates/jmap/src/api/auth.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::auth::AccessToken; use directory::Permission; use jmap_proto::request::{ CopyRequestMethod, GetRequestMethod, ParseRequestMethod, QueryChangesRequestMethod, QueryRequestMethod, RequestMethod, SetRequestMethod, method::MethodObject, }; use types::{collection::Collection, id::Id}; pub trait JmapAuthorization { fn assert_is_member(&self, account_id: Id) -> trc::Result<&Self>; fn assert_has_jmap_permission( &self, request: &RequestMethod, object: MethodObject, ) -> trc::Result<()>; fn assert_has_access(&self, to_account_id: Id, to_collection: Collection) -> trc::Result<&Self>; } impl JmapAuthorization for AccessToken { fn assert_is_member(&self, account_id: Id) -> trc::Result<&Self> { if self.is_member(account_id.document_id()) { Ok(self) } else { Err(trc::JmapEvent::Forbidden .into_err() .details(format!("You are not an owner of account {}", account_id))) } } fn assert_has_access( &self, to_account_id: Id, to_collection: Collection, ) -> trc::Result<&Self> { if self.has_access(to_account_id.document_id(), to_collection) { Ok(self) } else { Err(trc::JmapEvent::Forbidden.into_err().details(format!( "You do not have access to account {}", to_account_id ))) } } fn assert_has_jmap_permission( &self, request: &RequestMethod, object: MethodObject, ) -> trc::Result<()> { let permission = match request { RequestMethod::Get(m) => match &m { GetRequestMethod::Email(_) => Permission::JmapEmailGet, GetRequestMethod::Mailbox(_) => Permission::JmapMailboxGet, GetRequestMethod::Thread(_) => Permission::JmapThreadGet, GetRequestMethod::Identity(_) => Permission::JmapIdentityGet, GetRequestMethod::EmailSubmission(_) => Permission::JmapEmailSubmissionGet, GetRequestMethod::PushSubscription(_) => Permission::JmapPushSubscriptionGet, GetRequestMethod::Sieve(_) => Permission::JmapSieveScriptGet, GetRequestMethod::VacationResponse(_) => Permission::JmapVacationResponseGet, GetRequestMethod::Principal(_) => Permission::JmapPrincipalGet, GetRequestMethod::Quota(_) => Permission::JmapQuotaGet, GetRequestMethod::Blob(_) => Permission::JmapBlobGet, GetRequestMethod::AddressBook(_) => Permission::JmapAddressBookGet, GetRequestMethod::ContactCard(_) => Permission::JmapContactCardGet, GetRequestMethod::FileNode(_) => Permission::JmapFileNodeGet, GetRequestMethod::PrincipalAvailability(_) => { Permission::JmapPrincipalGetAvailability } GetRequestMethod::Calendar(_) => Permission::JmapCalendarGet, GetRequestMethod::CalendarEvent(_) => Permission::JmapCalendarEventGet, GetRequestMethod::CalendarEventNotification(_) => { Permission::JmapCalendarEventNotificationGet } GetRequestMethod::ParticipantIdentity(_) => Permission::JmapParticipantIdentityGet, GetRequestMethod::ShareNotification(_) => Permission::JmapShareNotificationGet, }, RequestMethod::Set(m) => match &m { SetRequestMethod::Email(_) => Permission::JmapEmailSet, SetRequestMethod::Mailbox(_) => Permission::JmapMailboxSet, SetRequestMethod::Identity(_) => Permission::JmapIdentitySet, SetRequestMethod::EmailSubmission(_) => Permission::JmapEmailSubmissionSet, SetRequestMethod::PushSubscription(_) => Permission::JmapPushSubscriptionSet, SetRequestMethod::Sieve(_) => Permission::JmapSieveScriptSet, SetRequestMethod::VacationResponse(_) => Permission::JmapVacationResponseSet, SetRequestMethod::AddressBook(_) => Permission::JmapAddressBookSet, SetRequestMethod::ContactCard(_) => Permission::JmapContactCardSet, SetRequestMethod::FileNode(_) => Permission::JmapFileNodeSet, SetRequestMethod::ShareNotification(_) => Permission::JmapShareNotificationSet, SetRequestMethod::Calendar(_) => Permission::JmapCalendarSet, SetRequestMethod::CalendarEvent(_) => Permission::JmapCalendarEventSet, SetRequestMethod::CalendarEventNotification(_) => { Permission::JmapCalendarEventNotificationSet } SetRequestMethod::ParticipantIdentity(_) => Permission::JmapParticipantIdentitySet, }, RequestMethod::Changes(_) => match object { MethodObject::Email => Permission::JmapEmailChanges, MethodObject::Mailbox => Permission::JmapMailboxChanges, MethodObject::Thread => Permission::JmapThreadChanges, MethodObject::Identity => Permission::JmapIdentityChanges, MethodObject::EmailSubmission => Permission::JmapEmailSubmissionChanges, MethodObject::Quota => Permission::JmapQuotaChanges, MethodObject::ContactCard => Permission::JmapContactCardChanges, MethodObject::FileNode => Permission::JmapFileNodeChanges, MethodObject::Calendar => Permission::JmapCalendarChanges, MethodObject::CalendarEvent => Permission::JmapCalendarEventChanges, MethodObject::CalendarEventNotification => { Permission::JmapCalendarEventNotificationChanges } MethodObject::ParticipantIdentity => Permission::JmapParticipantIdentityChanges, MethodObject::ShareNotification => Permission::JmapShareNotificationChanges, MethodObject::Principal => Permission::JmapPrincipalChanges, MethodObject::Core | MethodObject::Blob | MethodObject::PushSubscription | MethodObject::SearchSnippet | MethodObject::VacationResponse | MethodObject::SieveScript | MethodObject::AddressBook => Permission::JmapEmailChanges, }, RequestMethod::Copy(m) => match &m { CopyRequestMethod::Email(_) => Permission::JmapEmailCopy, CopyRequestMethod::Blob(_) => Permission::JmapBlobCopy, CopyRequestMethod::ContactCard(_) => Permission::JmapContactCardCopy, CopyRequestMethod::CalendarEvent(_) => Permission::JmapCalendarEventCopy, }, RequestMethod::ImportEmail(_) => Permission::JmapEmailImport, RequestMethod::Parse(m) => match &m { ParseRequestMethod::Email(_) => Permission::JmapEmailParse, ParseRequestMethod::ContactCard(_) => Permission::JmapContactCardParse, ParseRequestMethod::CalendarEvent(_) => Permission::JmapCalendarEventParse, }, RequestMethod::QueryChanges(m) => match m { QueryChangesRequestMethod::Email(_) => Permission::JmapEmailQueryChanges, QueryChangesRequestMethod::Mailbox(_) => Permission::JmapMailboxQueryChanges, QueryChangesRequestMethod::EmailSubmission(_) => { Permission::JmapEmailSubmissionQueryChanges } QueryChangesRequestMethod::Sieve(_) => Permission::JmapSieveScriptQueryChanges, QueryChangesRequestMethod::Principal(_) => Permission::JmapPrincipalQueryChanges, QueryChangesRequestMethod::Quota(_) => Permission::JmapQuotaQueryChanges, QueryChangesRequestMethod::ContactCard(_) => { Permission::JmapContactCardQueryChanges } QueryChangesRequestMethod::FileNode(_) => Permission::JmapFileNodeQueryChanges, QueryChangesRequestMethod::CalendarEvent(_) => { Permission::JmapCalendarEventQueryChanges } QueryChangesRequestMethod::CalendarEventNotification(_) => { Permission::JmapCalendarEventNotificationQueryChanges } QueryChangesRequestMethod::ShareNotification(_) => { Permission::JmapShareNotificationQueryChanges } }, RequestMethod::Query(m) => match m { QueryRequestMethod::Email(_) => Permission::JmapEmailQuery, QueryRequestMethod::Mailbox(_) => Permission::JmapMailboxQuery, QueryRequestMethod::EmailSubmission(_) => Permission::JmapEmailSubmissionQuery, QueryRequestMethod::Sieve(_) => Permission::JmapSieveScriptQuery, QueryRequestMethod::Principal(_) => Permission::JmapPrincipalQuery, QueryRequestMethod::Quota(_) => Permission::JmapQuotaQuery, QueryRequestMethod::ContactCard(_) => Permission::JmapContactCardQuery, QueryRequestMethod::FileNode(_) => Permission::JmapFileNodeQuery, QueryRequestMethod::CalendarEvent(_) => Permission::JmapCalendarEventQuery, QueryRequestMethod::CalendarEventNotification(_) => { Permission::JmapCalendarEventNotificationQuery } QueryRequestMethod::ShareNotification(_) => Permission::JmapShareNotificationQuery, }, RequestMethod::SearchSnippet(_) => Permission::JmapSearchSnippet, RequestMethod::ValidateScript(_) => Permission::JmapSieveScriptValidate, RequestMethod::LookupBlob(_) => Permission::JmapBlobLookup, RequestMethod::UploadBlob(_) => Permission::JmapBlobUpload, RequestMethod::Echo(_) => Permission::JmapEcho, RequestMethod::Error(_) => return Ok(()), }; if self.has_permission(permission) { Ok(()) } else { Err(trc::JmapEvent::Forbidden .into_err() .details("You are not authorized to perform this action")) } } } ================================================ FILE: crates/jmap/src/api/event_source.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::api::IntoPushObject; use common::{LONG_1D_SLUMBER, Server, auth::AccessToken, ipc::PushNotification}; use http_body_util::{StreamBody, combinators::BoxBody}; use http_proto::*; use hyper::{ StatusCode, body::{Bytes, Frame}, }; use jmap_proto::{response::status::PushObject, types::state::State}; use std::{future::Future, str::FromStr}; use std::{ sync::Arc, time::{Duration, Instant}, }; use types::{id::Id, type_state::DataType}; use utils::map::{bitmap::Bitmap, vec_map::VecMap}; struct Ping { interval: Duration, last_ping: Instant, payload: Bytes, } pub trait EventSourceHandler: Sync + Send { fn handle_event_source( &self, req: HttpRequest, access_token: Arc, ) -> impl Future> + Send; } impl EventSourceHandler for Server { async fn handle_event_source( &self, req: HttpRequest, access_token: Arc, ) -> trc::Result { // Parse query let mut ping = 0; let mut types = Bitmap::default(); let mut close_after_state = false; for (key, value) in http_proto::form_urlencoded::parse(req.uri().query().unwrap_or_default().as_bytes()) { hashify::fnc_map!(key.as_bytes(), "types" => { for type_state in value.split(',') { if type_state == "*" { types = Bitmap::all(); break; } else if let Ok(type_state) = DataType::from_str(type_state) { types.insert(type_state); } else { return Err(trc::ResourceEvent::BadParameters.into_err()); } } }, "closeafter" => match value.as_ref() { "state" => { close_after_state = true; } "no" => {} _ => return Err(trc::ResourceEvent::BadParameters.into_err()), }, "ping" => match value.parse::() { Ok(value) => { ping = value; } Err(_) => return Err(trc::ResourceEvent::BadParameters.into_err()), }, _ => {} ); } let mut ping = if ping > 0 { #[cfg(not(feature = "test_mode"))] let interval = std::cmp::max(ping, 30) * 1000; #[cfg(feature = "test_mode")] let interval = ping * 1000; Ping { interval: Duration::from_millis(interval as u64), last_ping: Instant::now() - Duration::from_millis(interval as u64), payload: Bytes::from(format!( "event: ping\ndata: {{\"interval\": {}}}\n\n", interval )), } .into() } else { None }; // Register with push manager let mut push_rx = self.subscribe_push_manager(&access_token, types).await?; let mut changed: VecMap> = VecMap::new(); let throttle = self.core.jmap.event_source_throttle; Ok(HttpResponse::new(StatusCode::OK) .with_content_type("text/event-stream") .with_cache_control("no-store") .with_stream_body(BoxBody::new(StreamBody::new(async_stream::stream! { let mut last_message = Instant::now() - throttle; let mut timeout = ping.as_ref().map(|p| p.interval).unwrap_or(LONG_1D_SLUMBER); loop { match tokio::time::timeout(timeout, push_rx.recv()).await { Ok(Some(notification)) => { match notification { PushNotification::StateChange(state_change) => { for type_state in state_change.types { changed .get_mut_or_insert(state_change.account_id.into()) .set(type_state, (state_change.change_id).into()); } } PushNotification::CalendarAlert(calendar_alert) => { yield Ok(Frame::data(Bytes::from(format!( "event: calendarAlert\ndata: {}\n\n", serde_json::to_string(&calendar_alert.into_push_object()).unwrap() )))); } PushNotification::EmailPush(email_push) => { let state_change = email_push.to_state_change(); for type_state in state_change.types { changed .get_mut_or_insert(state_change.account_id.into()) .set(type_state, state_change.change_id.into()); } } } } Ok(None) => { break; } Err(_) => (), } timeout = if !changed.is_empty() { let elapsed = last_message.elapsed(); if elapsed >= throttle { last_message = Instant::now(); let response = PushObject::StateChange { changed: std::mem::take(&mut changed) }; yield Ok(Frame::data(Bytes::from(format!( "event: state\ndata: {}\n\n", serde_json::to_string(&response).unwrap() )))); if close_after_state { break; } ping.as_ref().map(|p| p.interval).unwrap_or(LONG_1D_SLUMBER) } else { throttle - elapsed } } else if let Some(ping) = &mut ping { let elapsed = ping.last_ping.elapsed(); if elapsed >= ping.interval { ping.last_ping = Instant::now(); yield Ok(Frame::data(ping.payload.clone())); ping.interval } else { ping.interval - elapsed } } else { LONG_1D_SLUMBER }; } })))) } } ================================================ FILE: crates/jmap/src/api/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::blob::UploadResponse; use calcard::jscalendar::JSCalendarDateTime; use common::ipc::{CalendarAlert, PushNotification}; use http_proto::{HttpResponse, JsonResponse, ToHttpResponse}; use hyper::StatusCode; use jmap_proto::{ error::request::{RequestError, RequestLimitError}, request::capability::Session, response::{Response, status::PushObject}, types::state::State, }; use types::{id::Id, type_state::DataType}; use utils::map::vec_map::VecMap; pub mod acl; pub mod auth; pub mod event_source; pub mod query; pub mod request; pub mod session; impl ToHttpResponse for UploadResponse { fn into_http_response(self) -> HttpResponse { JsonResponse::new(self).into_http_response() } } pub trait ToJmapHttpResponse { fn into_http_response(self) -> HttpResponse; } impl ToJmapHttpResponse for Response<'_> { fn into_http_response(self) -> HttpResponse { JsonResponse::new(self).into_http_response() } } impl ToJmapHttpResponse for Session { fn into_http_response(self) -> HttpResponse { JsonResponse::new(self).into_http_response() } } impl ToJmapHttpResponse for RequestError<'_> { fn into_http_response(self) -> HttpResponse { HttpResponse::new(StatusCode::from_u16(self.status).unwrap_or(StatusCode::BAD_REQUEST)) .with_content_type("application/problem+json") .with_text_body(serde_json::to_string(&self).unwrap_or_default()) } } pub trait ToRequestError { fn to_request_error(&self) -> RequestError<'_>; } impl ToRequestError for trc::Error { fn to_request_error(&self) -> RequestError<'_> { let details_or_reason = self .value(trc::Key::Details) .or_else(|| self.value(trc::Key::Reason)) .and_then(|v| v.as_str()); let details = details_or_reason.unwrap_or_else(|| self.as_ref().message()); match self.as_ref() { trc::EventType::Jmap(cause) => match cause { trc::JmapEvent::UnknownCapability => RequestError::unknown_capability(details), trc::JmapEvent::NotJson => RequestError::not_json(details), trc::JmapEvent::NotRequest => RequestError::not_request(details), _ => RequestError::invalid_parameters(), }, trc::EventType::Limit(cause) => match cause { trc::LimitEvent::SizeRequest => RequestError::limit(RequestLimitError::SizeRequest), trc::LimitEvent::SizeUpload => RequestError::limit(RequestLimitError::SizeUpload), trc::LimitEvent::CallsIn => RequestError::limit(RequestLimitError::CallsIn), trc::LimitEvent::ConcurrentRequest | trc::LimitEvent::ConcurrentConnection => { RequestError::limit(RequestLimitError::ConcurrentRequest) } trc::LimitEvent::ConcurrentUpload => { RequestError::limit(RequestLimitError::ConcurrentUpload) } trc::LimitEvent::Quota => RequestError::over_quota(), trc::LimitEvent::TenantQuota => RequestError::tenant_over_quota(), trc::LimitEvent::BlobQuota => RequestError::over_blob_quota( self.value(trc::Key::Total) .and_then(|v| v.to_uint()) .unwrap_or_default() as usize, self.value(trc::Key::Size) .and_then(|v| v.to_uint()) .unwrap_or_default() as usize, ), trc::LimitEvent::TooManyRequests => RequestError::too_many_requests(), }, trc::EventType::Auth(cause) => match cause { trc::AuthEvent::MissingTotp => { RequestError::blank(402, "TOTP code required", cause.message()) } trc::AuthEvent::TooManyAttempts => RequestError::too_many_auth_attempts(), _ => RequestError::unauthorized(), }, trc::EventType::Security(cause) => match cause { trc::SecurityEvent::AuthenticationBan | trc::SecurityEvent::ScanBan | trc::SecurityEvent::AbuseBan | trc::SecurityEvent::LoiterBan | trc::SecurityEvent::IpBlocked => RequestError::too_many_auth_attempts(), trc::SecurityEvent::Unauthorized => RequestError::forbidden(), }, trc::EventType::Resource(cause) => match cause { trc::ResourceEvent::NotFound => RequestError::not_found(), trc::ResourceEvent::BadParameters => RequestError::blank( StatusCode::BAD_REQUEST.as_u16(), "Invalid parameters", details_or_reason.unwrap_or("One or multiple parameters could not be parsed."), ), trc::ResourceEvent::Error => RequestError::internal_server_error(), _ => RequestError::internal_server_error(), }, _ => RequestError::internal_server_error(), } } } pub(crate) trait IntoPushObject { fn into_push_object(self) -> PushObject; } impl IntoPushObject for Vec { fn into_push_object(self) -> PushObject { let mut changed: VecMap> = VecMap::new(); let mut objects = Vec::with_capacity(self.len()); for notification in self { match notification { PushNotification::StateChange(state_change) => { for type_state in state_change.types { changed .get_mut_or_insert(state_change.account_id.into()) .set(type_state, (state_change.change_id).into()); } } PushNotification::CalendarAlert(calendar_alert) => { objects.push(calendar_alert.into_push_object()); } PushNotification::EmailPush(email_push) => { let state_change = email_push.to_state_change(); for type_state in state_change.types { changed .get_mut_or_insert(state_change.account_id.into()) .set(type_state, state_change.change_id.into()); } } } } if !objects.is_empty() { if changed.is_empty() { objects.push(PushObject::StateChange { changed }); } if objects.len() > 1 { PushObject::Group { entries: objects } } else { objects.into_iter().next().unwrap() } } else { PushObject::StateChange { changed } } } } impl IntoPushObject for CalendarAlert { fn into_push_object(self) -> PushObject { PushObject::CalendarAlert { account_id: self.account_id.into(), calendar_event_id: self.event_id.into(), uid: self.uid, recurrence_id: self .recurrence_id .map(|timestamp| JSCalendarDateTime::new(timestamp, true).to_rfc3339()), alert_id: self.alert_id, } } } ================================================ FILE: crates/jmap/src/api/query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use jmap_proto::{ method::query::{QueryRequest, QueryResponse}, object::JmapObject, types::state::State, }; use types::id::Id; pub struct QueryResponseBuilder { requested_position: i32, position: i32, pub limit: usize, anchor: u32, anchor_offset: i32, has_anchor: bool, anchor_found: bool, pub response: QueryResponse, } impl QueryResponseBuilder { pub fn new( total_results: usize, max_results: usize, query_state: State, request: &QueryRequest, ) -> Self { let (limit_total, limit) = if let Some(limit) = request.limit { if limit > 0 { let limit = std::cmp::min(limit, max_results); (std::cmp::min(limit, total_results), limit) } else { (0, 0) } } else { (std::cmp::min(max_results, total_results), max_results) }; let (has_anchor, anchor) = request .anchor .map(|anchor| (true, anchor.document_id())) .unwrap_or((false, 0)); QueryResponseBuilder { requested_position: request.position.unwrap_or(0), position: request.position.unwrap_or(0), limit: limit_total, anchor, anchor_offset: request.anchor_offset.unwrap_or(0), has_anchor, anchor_found: false, response: QueryResponse { account_id: request.account_id, query_state, can_calculate_changes: true, position: 0, ids: vec![], total: if request.calculate_total.unwrap_or(false) { Some(total_results) } else { None }, limit: if total_results > limit { Some(limit) } else { None }, }, } } #[inline(always)] pub fn add(&mut self, prefix_id: u32, document_id: u32) -> bool { self.add_id(Id::from_parts(prefix_id, document_id)) } pub fn add_id(&mut self, id: Id) -> bool { let document_id = id.document_id(); // Pagination if !self.has_anchor { if self.position >= 0 { if self.position > 0 { self.position -= 1; } else { self.response.ids.push(id); if self.response.ids.len() == self.limit { return false; } } } else { self.response.ids.push(id); } } else if self.anchor_offset >= 0 { if !self.anchor_found { if document_id != self.anchor { return true; } self.anchor_found = true; } if self.anchor_offset > 0 { self.anchor_offset -= 1; } else { self.response.ids.push(id); if self.response.ids.len() == self.limit { return false; } } } else { self.anchor_found = document_id == self.anchor; self.response.ids.push(id); if self.anchor_found { self.position = self.anchor_offset; return false; } } true } pub fn is_full(&self) -> bool { self.response.ids.len() == self.limit } pub fn build(mut self) -> trc::Result { if !self.has_anchor || self.anchor_found { if !self.has_anchor && self.requested_position >= 0 { self.response.position = if self.position == 0 { self.requested_position } else { 0 }; } else if self.position >= 0 { self.response.position = self.position; } else { let position = self.position.unsigned_abs() as usize; let start_offset = if position < self.response.ids.len() { self.response.ids.len() - position } else { 0 }; self.response.position = start_offset as i32; let end_offset = if self.limit > 0 { std::cmp::min(start_offset + self.limit, self.response.ids.len()) } else { self.response.ids.len() }; self.response.ids = self.response.ids[start_offset..end_offset].to_vec() } Ok(self.response) } else { Err(trc::JmapEvent::AnchorNotFound.into_err()) } } } ================================================ FILE: crates/jmap/src/api/request.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ addressbook::{get::AddressBookGet, set::AddressBookSet}, api::auth::JmapAuthorization, blob::{copy::BlobCopy, get::BlobOperations, upload::BlobUpload}, calendar::{get::CalendarGet, set::CalendarSet}, calendar_event::{ copy::JmapCalendarEventCopy, get::CalendarEventGet, parse::CalendarEventParse, query::CalendarEventQuery, set::CalendarEventSet, }, calendar_event_notification::{ get::CalendarEventNotificationGet, query::CalendarEventNotificationQuery, set::CalendarEventNotificationSet, }, changes::{get::ChangesLookup, query::QueryChanges}, contact::{ copy::JmapContactCardCopy, get::ContactCardGet, parse::ContactCardParse, query::ContactCardQuery, set::ContactCardSet, }, email::{ copy::JmapEmailCopy, get::EmailGet, import::EmailImport, parse::EmailParse, query::EmailQuery, set::EmailSet, snippet::EmailSearchSnippet, }, file::{get::FileNodeGet, query::FileNodeQuery, set::FileNodeSet}, identity::{get::IdentityGet, set::IdentitySet}, mailbox::{get::MailboxGet, query::MailboxQuery, set::MailboxSet}, participant_identity::{get::ParticipantIdentityGet, set::ParticipantIdentitySet}, principal::{availability::PrincipalGetAvailability, get::PrincipalGet, query::PrincipalQuery}, push::{get::PushSubscriptionFetch, set::PushSubscriptionSet}, quota::{get::QuotaGet, query::QuotaQuery}, share_notification::{ get::ShareNotificationGet, query::ShareNotificationQuery, set::ShareNotificationSet, }, sieve::{ get::SieveScriptGet, query::SieveScriptQuery, set::SieveScriptSet, validate::SieveScriptValidate, }, submission::{get::EmailSubmissionGet, query::EmailSubmissionQuery, set::EmailSubmissionSet}, thread::get::ThreadGet, vacation::{get::VacationResponseGet, set::VacationResponseSet}, }; use common::{Server, auth::AccessToken}; use http_proto::HttpSessionData; use jmap_proto::{ request::{ Call, CopyRequestMethod, GetRequestMethod, ParseRequestMethod, QueryRequestMethod, Request, RequestMethod, SetRequestMethod, method::MethodName, }, response::{Response, ResponseMethod, SetResponseMethod}, }; use std::future::Future; use std::{sync::Arc, time::Instant}; use trc::JmapEvent; use types::{collection::Collection, id::Id}; pub trait RequestHandler: Sync + Send { fn handle_jmap_request<'x>( &self, request: Request<'x>, access_token: Arc, session: &HttpSessionData, ) -> impl Future> + Send; fn handle_method_call<'x>( &self, method: RequestMethod<'x>, method_name: MethodName, access_token: &AccessToken, next_call: &mut Option>>, session: &HttpSessionData, ) -> impl Future>> + Send; } impl RequestHandler for Server { #![allow(clippy::large_futures)] async fn handle_jmap_request<'x>( &self, request: Request<'x>, access_token: Arc, session: &HttpSessionData, ) -> Response<'x> { let add_created_ids = request.created_ids.is_some(); let mut response = Response::new( access_token.state(), request.created_ids.unwrap_or_default(), request.method_calls.len(), ); for mut call in request.method_calls { // Resolve result and id references if let Err(error) = response.resolve_references(&mut call.method) { let method_error = error.clone(); trc::error!(error.span_id(session.session_id)); response.push_response(call.id, MethodName::error(), method_error); continue; } loop { let mut next_call = None; // Add response let method_name = call.name.as_str(); match self .handle_method_call( call.method, call.name, &access_token, &mut next_call, session, ) .await { Ok(mut method_response) => { match &mut method_response { ResponseMethod::Set(set_response) => { // Add created ids match set_response { SetResponseMethod::Email(set_response) => { set_response.update_created_ids(&mut response); } SetResponseMethod::Mailbox(set_response) => { set_response.update_created_ids(&mut response); } SetResponseMethod::Identity(set_response) => { set_response.update_created_ids(&mut response); } SetResponseMethod::EmailSubmission(set_response) => { set_response.update_created_ids(&mut response); } SetResponseMethod::PushSubscription(set_response) => { set_response.update_created_ids(&mut response); } SetResponseMethod::Sieve(set_response) => { set_response.update_created_ids(&mut response); } SetResponseMethod::VacationResponse(set_response) => { set_response.update_created_ids(&mut response); } SetResponseMethod::AddressBook(set_response) => { set_response.update_created_ids(&mut response); } SetResponseMethod::ContactCard(set_response) => { set_response.update_created_ids(&mut response); } SetResponseMethod::FileNode(set_response) => { set_response.update_created_ids(&mut response); } SetResponseMethod::ShareNotification(set_response) => { set_response.update_created_ids(&mut response); } SetResponseMethod::Calendar(set_response) => { set_response.update_created_ids(&mut response); } SetResponseMethod::CalendarEvent(set_response) => { set_response.update_created_ids(&mut response); } SetResponseMethod::ParticipantIdentity(set_response) => { set_response.update_created_ids(&mut response); } SetResponseMethod::CalendarEventNotification(_) => {} } } ResponseMethod::ImportEmail(import_response) => { // Add created ids import_response.update_created_ids(&mut response); } ResponseMethod::UploadBlob(upload_response) => { // Add created blobIds upload_response.update_created_ids(&mut response); } _ => {} } response.push_response(call.id, call.name, method_response); } Err(error) => { let method_error = error.clone(); trc::error!( error .span_id(session.session_id) .ctx_unique(trc::Key::AccountId, access_token.primary_id()) .caused_by(method_name) ); response.push_error(call.id, method_error); } } // Process next call if let Some(next_call) = next_call { call = next_call; call.id .clone_from(&response.method_responses.last().unwrap().id); } else { break; } } } if !add_created_ids { response.created_ids.clear(); } response } async fn handle_method_call<'x>( &self, method: RequestMethod<'x>, method_name: MethodName, access_token: &AccessToken, next_call: &mut Option>>, session: &HttpSessionData, ) -> trc::Result> { let op_start = Instant::now(); // Check permissions access_token.assert_has_jmap_permission(&method, method_name.obj)?; // Handle method let response = match method { RequestMethod::Get(req) => match req { GetRequestMethod::Email(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::Email)?; self.email_get(req, access_token).await?.into() } GetRequestMethod::Mailbox(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::Mailbox)?; self.mailbox_get(req, access_token).await?.into() } GetRequestMethod::Thread(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::Email)?; self.thread_get(req).await?.into() } GetRequestMethod::Identity(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.identity_get(req).await?.into() } GetRequestMethod::EmailSubmission(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.email_submission_get(req).await?.into() } GetRequestMethod::PushSubscription(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); self.push_subscription_get(req, access_token).await?.into() } GetRequestMethod::Sieve(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.sieve_script_get(req).await?.into() } GetRequestMethod::VacationResponse(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.vacation_response_get(req).await?.into() } GetRequestMethod::Principal(req) => { self.principal_get(req, access_token).await?.into() } GetRequestMethod::Quota(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.quota_get(req, access_token).await?.into() } GetRequestMethod::Blob(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.blob_get(req, access_token).await?.into() } GetRequestMethod::AddressBook(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::AddressBook)?; self.address_book_get(req, access_token).await?.into() } GetRequestMethod::ContactCard(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::ContactCard)?; self.contact_card_get(req, access_token).await?.into() } GetRequestMethod::FileNode(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::FileNode)?; self.file_node_get(req, access_token).await?.into() } GetRequestMethod::PrincipalAvailability(req) => self .principal_get_availability(req, access_token) .await? .into(), GetRequestMethod::Calendar(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::Calendar)?; self.calendar_get(req, access_token).await?.into() } GetRequestMethod::CalendarEvent(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::CalendarEvent)?; self.calendar_event_get(req, access_token).await?.into() } GetRequestMethod::CalendarEventNotification(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.calendar_event_notification_get(req, access_token) .await? .into() } GetRequestMethod::ParticipantIdentity(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.participant_identity_get(req).await?.into() } GetRequestMethod::ShareNotification(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.share_notification_get(req).await?.into() } }, RequestMethod::Query(req) => match req { QueryRequestMethod::Email(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::Email)?; self.email_query(req, access_token).await?.into() } QueryRequestMethod::Mailbox(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::Mailbox)?; self.mailbox_query(req, access_token).await?.into() } QueryRequestMethod::EmailSubmission(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.email_submission_query(req).await?.into() } QueryRequestMethod::Sieve(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.sieve_script_query(req).await?.into() } QueryRequestMethod::Principal(req) => self .principal_query(req, access_token, session) .await? .into(), QueryRequestMethod::Quota(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.quota_query(req, access_token).await?.into() } QueryRequestMethod::ContactCard(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::ContactCard)?; self.contact_card_query(req, access_token).await?.into() } QueryRequestMethod::FileNode(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::FileNode)?; self.file_node_query(req, access_token).await?.into() } QueryRequestMethod::CalendarEvent(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::CalendarEvent)?; self.calendar_event_query(req, access_token).await?.into() } QueryRequestMethod::CalendarEventNotification(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.calendar_event_notification_query(req, access_token) .await? .into() } QueryRequestMethod::ShareNotification(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.share_notification_query(req).await?.into() } }, RequestMethod::Set(req) => match req { SetRequestMethod::Email(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::Email)?; self.email_set(req, access_token, session).await?.into() } SetRequestMethod::Mailbox(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::Mailbox)?; self.mailbox_set(req, access_token).await?.into() } SetRequestMethod::Identity(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.identity_set(req, access_token).await?.into() } SetRequestMethod::EmailSubmission(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.email_submission_set(req, &session.instance, next_call) .await? .into() } SetRequestMethod::PushSubscription(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); self.push_subscription_set(req, access_token).await?.into() } SetRequestMethod::Sieve(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.sieve_script_set(req, access_token, session) .await? .into() } SetRequestMethod::VacationResponse(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.vacation_response_set(req, access_token).await?.into() } SetRequestMethod::AddressBook(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::AddressBook)?; self.address_book_set(req, access_token, session) .await? .into() } SetRequestMethod::ContactCard(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::ContactCard)?; self.contact_card_set(req, access_token, session) .await? .into() } SetRequestMethod::FileNode(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::FileNode)?; self.file_node_set(req, access_token, session).await?.into() } SetRequestMethod::ShareNotification(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.share_notification_set(req).await?.into() } SetRequestMethod::Calendar(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::Calendar)?; self.calendar_set(req, access_token, session).await?.into() } SetRequestMethod::CalendarEvent(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::CalendarEvent)?; self.calendar_event_set(req, access_token, session) .await? .into() } SetRequestMethod::CalendarEventNotification(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.calendar_event_notification_set(req, access_token, session) .await? .into() } SetRequestMethod::ParticipantIdentity(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.participant_identity_set(req, access_token) .await? .into() } }, RequestMethod::Changes(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); self.changes(req, method_name.obj, access_token) .await? .into_method_response() } RequestMethod::Copy(req) => match req { CopyRequestMethod::Email(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); set_account_id_if_missing(&mut req.from_account_id, access_token); access_token .assert_has_access(req.account_id, Collection::Email)? .assert_has_access(req.from_account_id, Collection::Email)?; self.email_copy(req, access_token, next_call, session) .await? .into() } CopyRequestMethod::Blob(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.blob_copy(req, access_token).await?.into() } CopyRequestMethod::ContactCard(mut req) => { set_account_id_if_missing(&mut req.from_account_id, access_token); set_account_id_if_missing(&mut req.account_id, access_token); access_token .assert_has_access(req.account_id, Collection::ContactCard)? .assert_has_access(req.from_account_id, Collection::ContactCard)?; self.contact_card_copy(req, access_token, next_call, session) .await? .into() } CopyRequestMethod::CalendarEvent(mut req) => { set_account_id_if_missing(&mut req.from_account_id, access_token); set_account_id_if_missing(&mut req.account_id, access_token); access_token .assert_has_access(req.account_id, Collection::CalendarEvent)? .assert_has_access(req.from_account_id, Collection::CalendarEvent)?; self.calendar_event_copy(req, access_token, next_call, session) .await? .into() } }, RequestMethod::ImportEmail(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::Email)?; self.email_import(req, access_token, session).await?.into() } RequestMethod::Parse(req) => match req { ParseRequestMethod::Email(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::Email)?; self.email_parse(req, access_token).await?.into() } ParseRequestMethod::ContactCard(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::ContactCard)?; self.contact_card_parse(req, access_token).await?.into() } ParseRequestMethod::CalendarEvent(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::CalendarEvent)?; self.calendar_event_parse(req, access_token).await?.into() } }, RequestMethod::QueryChanges(req) => self.query_changes(req, access_token).await?.into(), RequestMethod::SearchSnippet(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_has_access(req.account_id, Collection::Email)?; self.email_search_snippet(req, access_token).await?.into() } RequestMethod::ValidateScript(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.sieve_script_validate(req, access_token).await?.into() } RequestMethod::LookupBlob(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.blob_lookup(req).await?.into() } RequestMethod::UploadBlob(mut req) => { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; self.blob_upload_many(req, access_token).await?.into() } RequestMethod::Echo(req) => req.into(), RequestMethod::Error(error) => return Err(error), }; trc::event!( Jmap(JmapEvent::MethodCall), Id = method_name.as_str(), SpanId = session.session_id, AccountId = access_token.primary_id(), Elapsed = op_start.elapsed(), ); Ok(response) } } #[inline] pub(crate) fn set_account_id_if_missing(account_id: &mut Id, access_token: &AccessToken) { if !account_id.is_valid() { *account_id = Id::from(access_token.primary_id()); } } ================================================ FILE: crates/jmap/src/api/session.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{Server, auth::AccessToken}; use directory::Permission; use jmap_proto::request::capability::{ Account, Capabilities, Capability, EmptyCapabilities, Session, }; use std::future::Future; use std::sync::Arc; use types::id::Id; use utils::map::vec_map::VecMap; pub trait SessionHandler: Sync + Send { fn handle_session_resource( &self, base_url: String, access_token: Arc, ) -> impl Future> + Send; } impl SessionHandler for Server { async fn handle_session_resource( &self, base_url: String, access_token: Arc, ) -> trc::Result { let mut session = Session::new(base_url, &self.core.jmap.capabilities); session.set_state(access_token.state()); let account_capabilities = &self.core.jmap.capabilities.account; // Set primary account session.username = access_token.name.to_string(); let account_id = Id::from(access_token.primary_id()); let mut account = Account { name: access_token.name.to_string(), is_personal: true, is_read_only: false, account_capabilities: VecMap::with_capacity(account_capabilities.len()), }; for capability in access_token.account_capabilities() { session.primary_accounts.append(capability, account_id); account.account_capabilities.append( capability, account_capabilities .get(&capability) .map(|v| v.to_account_capabilities(account_id.into(), true)) .unwrap_or_else(|| Capabilities::Empty(EmptyCapabilities::default())), ); } session.accounts.append(account_id, account); // Add secondary accounts for &account_id in access_token.secondary_ids() { let is_owner = access_token.is_member(account_id); let access_token = match self.get_access_token(account_id).await { Ok(token) => token, Err(err) => { if err.matches(trc::EventType::Auth(trc::AuthEvent::Error)) { continue; } else { return Err(err.caused_by(trc::location!())); } } }; let account_id = Id::from(account_id); let mut account = Account { name: access_token.name.to_string(), is_personal: false, is_read_only: false, account_capabilities: VecMap::with_capacity(account_capabilities.len()), }; for capability in access_token.account_capabilities() { account.account_capabilities.append( capability, account_capabilities .get(&capability) .map(|v| v.to_account_capabilities(account_id.into(), is_owner)) .unwrap_or_else(|| Capabilities::Empty(EmptyCapabilities::default())), ); } session.accounts.append(account_id, account); } Ok(session) } } trait AccountCapabilities { fn account_capabilities(&self) -> impl Iterator; } impl AccountCapabilities for AccessToken { fn account_capabilities(&self) -> impl Iterator { Capability::all_capabilities() .iter() .filter(move |capability| { let permission = match capability { Capability::Mail => Permission::JmapEmailGet, Capability::Submission => Permission::JmapEmailSubmissionSet, Capability::VacationResponse => Permission::JmapVacationResponseGet, Capability::Contacts => Permission::JmapContactCardGet, Capability::ContactsParse => Permission::JmapContactCardParse, Capability::Calendars => Permission::JmapCalendarEventGet, Capability::CalendarsParse => Permission::JmapCalendarEventParse, Capability::Sieve => Permission::JmapSieveScriptGet, Capability::Blob => Permission::JmapBlobGet, Capability::Quota => Permission::JmapQuotaGet, Capability::FileNode => Permission::JmapFileNodeGet, Capability::WebSocket | Capability::Principals | Capability::PrincipalsAvailability => return true, Capability::Core | Capability::PrincipalsOwner => return false, }; self.has_permission(permission) }) .copied() } } ================================================ FILE: crates/jmap/src/blob/copy.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::download::BlobDownload; use common::{Server, auth::AccessToken}; use directory::Permission; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::copy::{CopyBlobRequest, CopyBlobResponse}, request::IntoValid, }; use std::future::Future; use store::write::{BatchBuilder, BlobLink, BlobOp, now}; use trc::AddContext; use types::blob::{BlobClass, BlobId}; use utils::map::vec_map::VecMap; pub trait BlobCopy: Sync + Send { fn blob_copy( &self, request: CopyBlobRequest, access_token: &AccessToken, ) -> impl Future> + Send; } impl BlobCopy for Server { async fn blob_copy( &self, request: CopyBlobRequest, access_token: &AccessToken, ) -> trc::Result { let mut response = CopyBlobResponse { from_account_id: request.from_account_id, account_id: request.account_id, copied: VecMap::with_capacity(request.blob_ids.len()), not_copied: VecMap::new(), }; let account_id = request.account_id.document_id(); for blob_id in request.blob_ids.into_valid() { if self.has_access_blob(&blob_id, access_token).await? { // Enforce quota let used = self .core .storage .data .blob_quota(account_id) .await .caused_by(trc::location!())?; if ((self.core.jmap.upload_tmp_quota_size > 0 && used.bytes >= self.core.jmap.upload_tmp_quota_size) || (self.core.jmap.upload_tmp_quota_amount > 0 && used.count + 1 > self.core.jmap.upload_tmp_quota_amount)) && !access_token.has_permission(Permission::UnlimitedUploads) { response.not_copied.append( blob_id, SetError::over_quota().with_description(format!( "You have exceeded the blob quota of {} files or {} bytes.", self.core.jmap.upload_tmp_quota_amount, self.core.jmap.upload_tmp_quota_size )), ); continue; } let mut batch = BatchBuilder::new(); let until = now() + self.core.jmap.upload_tmp_ttl; batch.with_account_id(account_id).set( BlobOp::Link { hash: blob_id.hash.clone(), to: BlobLink::Temporary { until }, }, vec![], ); self.store() .write(batch.build_all()) .await .caused_by(trc::location!())?; let dest_blob_id = BlobId { hash: blob_id.hash.clone(), class: BlobClass::Reserved { account_id, expires: until, }, section: blob_id.section.clone(), }; response.copied.append(blob_id, dest_blob_id); } else { response.not_copied.append( blob_id, SetError::new(SetErrorType::BlobNotFound).with_description( "blobId does not exist or not enough permissions to access it.", ), ); } } Ok(response) } } ================================================ FILE: crates/jmap/src/blob/download.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{Server, auth::AccessToken}; use email::cache::MessageCacheFetch; use email::cache::email::MessageCacheAccess; use email::message::metadata::MessageMetadata; use groupware::cache::GroupwareCache; use store::ValueKey; use store::write::{AlignedBytes, Archive}; use std::future::Future; use trc::AddContext; use types::acl::Acl; use types::blob::{BlobClass, BlobId}; use types::collection::{Collection, SyncCollection}; use types::field::EmailField; use utils::chained_bytes::ChainedBytes; pub trait BlobDownload: Sync + Send { fn blob_download( &self, blob_id: &BlobId, access_token: &AccessToken, ) -> impl Future>>> + Send; fn has_access_blob( &self, blob_id: &BlobId, access_token: &AccessToken, ) -> impl Future> + Send; } impl BlobDownload for Server { #[allow(clippy::blocks_in_conditions)] async fn blob_download( &self, blob_id: &BlobId, access_token: &AccessToken, ) -> trc::Result>> { if self.has_access_blob(blob_id, access_token).await? { if let Some(section) = &blob_id.section { self.get_blob_section(&blob_id.hash, section) .await .caused_by(trc::location!()) } else { let blob = self .blob_store() .get_blob(blob_id.hash.as_slice(), 0..usize::MAX) .await .caused_by(trc::location!()); match (&blob_id.class, blob) { ( BlobClass::Linked { account_id, collection, document_id, }, Ok(Some(data)), ) if *collection == Collection::Email as u8 => { let Some(archive) = self .store() .get_value::>(ValueKey::property( *account_id, Collection::Email, *document_id, EmailField::Metadata, )) .await .caused_by(trc::location!())? else { return Ok(Some(data)); }; let metadata = archive .to_unarchived::() .caused_by(trc::location!())?; let body_offset = metadata.inner.blob_body_offset.to_native(); if metadata.inner.root_part().offset_body.to_native() != body_offset { let raw_message = ChainedBytes::new( metadata.inner.raw_headers.as_ref(), ) .with_last(data.get(body_offset as usize..).unwrap_or_default()); Ok(Some(raw_message.to_bytes())) } else { Ok(Some(data)) } } (_, blob) => blob, } } } else { Ok(None) } } async fn has_access_blob( &self, blob_id: &BlobId, access_token: &AccessToken, ) -> trc::Result { Ok(self .store() .blob_has_access(&blob_id.hash, &blob_id.class) .await .caused_by(trc::location!())? && match &blob_id.class { BlobClass::Linked { account_id, collection, document_id, } => { if access_token.is_member(*account_id) { true } else { match Collection::from(*collection) { Collection::Email => self .get_cached_messages(*account_id) .await .caused_by(trc::location!())? .shared_messages(access_token, Acl::ReadItems) .contains(*document_id), collection @ (Collection::FileNode | Collection::ContactCard | Collection::CalendarEvent) => self .fetch_dav_resources( access_token, *account_id, SyncCollection::from(collection), ) .await .caused_by(trc::location!())? .shared_items(access_token, [Acl::ReadItems], true) .contains(*document_id), _ => false, } } } BlobClass::Reserved { account_id, .. } => access_token.is_member(*account_id), }) } } ================================================ FILE: crates/jmap/src/blob/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::download::BlobDownload; use common::{Server, auth::AccessToken}; use email::message::metadata::MessageData; use jmap_proto::{ method::{ get::{GetRequest, GetResponse}, lookup::{BlobInfo, BlobLookupRequest, BlobLookupResponse}, }, object::blob::{Blob, BlobProperty, BlobValue, DataProperty, DigestProperty}, request::{IntoValid, MaybeInvalid}, }; use jmap_tools::{Map, Value}; use mail_builder::encoders::base64::base64_encode; use sha1::{Digest, Sha1}; use sha2::{Sha256, Sha512}; use store::{ValueKey, write::{AlignedBytes, Archive}}; use std::future::Future; use trc::AddContext; use types::{blob::BlobClass, collection::Collection, id::Id, type_state::DataType}; use utils::map::vec_map::VecMap; pub trait BlobOperations: Sync + Send { fn blob_get( &self, request: GetRequest, access_token: &AccessToken, ) -> impl Future>> + Send; fn blob_lookup( &self, request: BlobLookupRequest, ) -> impl Future> + Send; } impl BlobOperations for Server { async fn blob_get( &self, mut request: GetRequest, access_token: &AccessToken, ) -> trc::Result> { let ids = request .unwrap_ids(self.core.jmap.get_max_objects)? .unwrap_or_default(); let properties = request.unwrap_properties(&[ BlobProperty::Id, BlobProperty::Data(DataProperty::Default), BlobProperty::Size, ]); let mut response = GetResponse { account_id: request.account_id.into(), state: None, list: Vec::with_capacity(ids.len()), not_found: vec![], }; let range_from = request.arguments.offset.unwrap_or(0); let range_to = request .arguments .length .map(|length| range_from.saturating_add(length)) .unwrap_or(usize::MAX); for blob_id in ids { if let Some(bytes) = self.blob_download(&blob_id, access_token).await? { let mut blob = Map::with_capacity(properties.len()); let bytes_range = if range_from == 0 && range_to == usize::MAX { &bytes[..] } else { let range_to = if range_to != usize::MAX && range_to > bytes.len() { blob.insert_unchecked(BlobProperty::IsTruncated, true); bytes.len() } else { range_to }; bytes.get(range_from..range_to).unwrap_or_default() }; for property in &properties { let mut property = property.clone(); let value: Value<'static, BlobProperty, BlobValue> = match &property { BlobProperty::Id => Value::Element(BlobValue::BlobId(blob_id.clone())), BlobProperty::Size => bytes.len().into(), BlobProperty::Digest(digest) => match digest { DigestProperty::Sha => { let mut hasher = Sha1::new(); hasher.update(bytes_range); String::from_utf8( base64_encode(&hasher.finalize()[..]).unwrap_or_default(), ) .unwrap() } DigestProperty::Sha256 => { let mut hasher = Sha256::new(); hasher.update(bytes_range); String::from_utf8( base64_encode(&hasher.finalize()[..]).unwrap_or_default(), ) .unwrap() } DigestProperty::Sha512 => { let mut hasher = Sha512::new(); hasher.update(bytes_range); String::from_utf8( base64_encode(&hasher.finalize()[..]).unwrap_or_default(), ) .unwrap() } } .into(), BlobProperty::Data(data) => match data { DataProperty::AsText => match std::str::from_utf8(bytes_range) { Ok(text) => text.to_string().into(), Err(_) => { blob.insert_unchecked(BlobProperty::IsEncodingProblem, true); Value::Null } }, DataProperty::AsBase64 => { String::from_utf8(base64_encode(bytes_range).unwrap_or_default()) .unwrap() .into() } DataProperty::Default => match std::str::from_utf8(bytes_range) { Ok(text) => { property = BlobProperty::Data(DataProperty::AsText); text.to_string().into() } Err(_) => { property = BlobProperty::Data(DataProperty::AsBase64); blob.insert_unchecked(BlobProperty::IsEncodingProblem, true); String::from_utf8( base64_encode(bytes_range).unwrap_or_default(), ) .unwrap() .into() } }, }, _ => Value::Null, }; blob.insert_unchecked(property, value); } // Add result to response response.list.push(blob.into()); } else { response.not_found.push(blob_id); } } Ok(response) } async fn blob_lookup(&self, request: BlobLookupRequest) -> trc::Result { let mut include_email = false; let mut include_mailbox = false; let mut include_thread = false; let type_names = request .type_names .into_iter() .map(|tn| match tn { MaybeInvalid::Value(value) => { match &value { DataType::Email => { include_email = true; } DataType::Mailbox => { include_mailbox = true; } DataType::Thread => { include_thread = true; } _ => (), } Ok(value) } MaybeInvalid::Invalid(_) => Err(trc::JmapEvent::UnknownDataType.into_err()), }) .collect::, _>>()?; let req_account_id = request.account_id.document_id(); let mut response = BlobLookupResponse { account_id: request.account_id, list: Vec::with_capacity(request.ids.len()), not_found: vec![], }; for id in request.ids.into_valid() { let mut matched_ids = VecMap::new(); match &id.class { BlobClass::Linked { account_id, collection, document_id, } if *account_id == req_account_id => { let collection = Collection::from(*collection); if collection == Collection::Email { if let Some(data_) = self .store() .get_value::>(ValueKey::archive( req_account_id, Collection::Email, *document_id, )) .await? { let data = data_ .unarchive::() .caused_by(trc::location!())?; if include_email { matched_ids.append( DataType::Email, vec![Id::from_parts(u32::from(data.thread_id), *document_id)], ); } if include_thread { matched_ids.append( DataType::Thread, vec![Id::from(u32::from(data.thread_id))], ); } if include_mailbox { matched_ids.append( DataType::Mailbox, data.mailboxes .iter() .map(|m| { debug_assert!(m.uid != 0); Id::from(u32::from(m.mailbox_id)) }) .collect::>(), ); } } } else { match DataType::try_from(collection) { Ok(data_type) if type_names.contains(&data_type) => { matched_ids.append(data_type, vec![Id::from(*document_id)]); } _ => (), } } } BlobClass::Reserved { account_id, .. } if *account_id == req_account_id => {} _ => { response.not_found.push(id); continue; } } response.list.push(BlobInfo { id, matched_ids }); } Ok(response) } } ================================================ FILE: crates/jmap/src/blob/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use types::{blob::BlobId, id::Id}; pub mod copy; pub mod download; pub mod get; pub mod upload; #[derive(Debug, serde::Serialize)] pub struct UploadResponse { #[serde(rename(serialize = "accountId"))] account_id: Id, #[serde(rename(serialize = "blobId"))] blob_id: BlobId, #[serde(rename(serialize = "type"))] c_type: String, size: usize, } ================================================ FILE: crates/jmap/src/blob/upload.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::Arc; use super::{UploadResponse, download::BlobDownload}; use common::{Server, auth::AccessToken}; use directory::Permission; use jmap_proto::{ error::set::SetError, method::upload::{ BlobUploadRequest, BlobUploadResponse, BlobUploadResponseObject, DataSourceObject, }, request::reference::MaybeIdReference, }; use std::future::Future; use trc::AddContext; use types::id::Id; #[cfg(feature = "test_mode")] pub static DISABLE_UPLOAD_QUOTA: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(true); pub trait BlobUpload: Sync + Send { fn blob_upload_many( &self, request: BlobUploadRequest, access_token: &AccessToken, ) -> impl Future> + Send; fn blob_upload( &self, account_id: Id, content_type: &str, data: &[u8], access_token: Arc, ) -> impl Future> + Send; } impl BlobUpload for Server { async fn blob_upload_many( &self, request: BlobUploadRequest, access_token: &AccessToken, ) -> trc::Result { let mut response = BlobUploadResponse { account_id: request.account_id, created: Default::default(), not_created: Default::default(), }; let account_id = request.account_id.document_id(); if request.create.len() > self.core.jmap.set_max_objects { return Err(trc::JmapEvent::RequestTooLarge.into_err()); } 'outer: for (create_id, upload_object) in request.create { let mut data = Vec::new(); for data_source in upload_object.data { let bytes = match data_source { DataSourceObject::Id { id, length, offset } => { let id = match id { MaybeIdReference::Id(id) => id, MaybeIdReference::Reference(reference) => { if let Some(obj) = response.created.get(&reference) { obj.id.clone() } else { response.not_created.append( create_id, SetError::not_found().with_description(format!( "Id reference {reference:?} not found." )), ); continue 'outer; } } MaybeIdReference::Invalid(id) => { response.not_created.append( create_id, SetError::invalid_properties() .with_description(format!("Invalid blobId {id}.")), ); continue 'outer; } }; if !self.has_access_blob(&id, access_token).await? { response.not_created.append( create_id, SetError::forbidden().with_description(format!( "You do not have access to blobId {id}." )), ); continue 'outer; } let offset = offset.unwrap_or(0); let length = length .map(|length| length.saturating_add(offset)) .unwrap_or(usize::MAX); let bytes = if let Some(section) = &id.section { self.get_blob_section(&id.hash, section) .await? .map(|bytes| { if offset == 0 && length == usize::MAX { bytes } else { bytes .get(offset..std::cmp::min(length, bytes.len())) .unwrap_or_default() .to_vec() } }) } else { self.blob_store() .get_blob(id.hash.as_slice(), offset..length) .await? }; if let Some(bytes) = bytes { bytes } else { response.not_created.append( create_id, SetError::blob_not_found() .with_description(format!("BlobId {id} not found.")), ); continue 'outer; } } DataSourceObject::Value(bytes) => bytes, DataSourceObject::Null => { response.not_created.append( create_id, SetError::invalid_properties() .with_description("Invalid DataSourceObject."), ); continue 'outer; } }; if bytes.len() + data.len() < self.core.jmap.upload_max_size { data.extend(bytes); } else { response.not_created.append( create_id, SetError::too_large().with_description(format!( "Upload size exceeds maximum of {} bytes.", self.core.jmap.upload_max_size )), ); continue 'outer; } } if data.is_empty() { response.not_created.append( create_id, SetError::invalid_properties() .with_description("Must specify at least one valid DataSourceObject."), ); continue 'outer; } // Enforce quota let used = self .core .storage .data .blob_quota(account_id) .await .caused_by(trc::location!())?; if ((self.core.jmap.upload_tmp_quota_size > 0 && used.bytes + data.len() > self.core.jmap.upload_tmp_quota_size) || (self.core.jmap.upload_tmp_quota_amount > 0 && used.count + 1 > self.core.jmap.upload_tmp_quota_amount)) && !access_token.has_permission(Permission::UnlimitedUploads) { response.not_created.append( create_id, SetError::over_quota().with_description(format!( "You have exceeded the blob upload quota of {} files or {} bytes.", self.core.jmap.upload_tmp_quota_amount, self.core.jmap.upload_tmp_quota_size )), ); continue 'outer; } // Write blob response.created.insert( create_id, BlobUploadResponseObject { id: self.put_jmap_blob(account_id, &data).await?, type_: upload_object.type_, size: data.len(), }, ); } Ok(response) } async fn blob_upload( &self, account_id: Id, content_type: &str, data: &[u8], access_token: Arc, ) -> trc::Result { // Limit concurrent uploads let _in_flight = self .is_upload_allowed(&access_token) .caused_by(trc::location!())?; #[cfg(feature = "test_mode")] { // Used for concurrent upload tests if data == b"sleep" { tokio::time::sleep(std::time::Duration::from_secs(1)).await; } } // Enforce quota let used = self .core .storage .data .blob_quota(account_id.document_id()) .await .caused_by(trc::location!())?; if ((self.core.jmap.upload_tmp_quota_size > 0 && used.bytes + data.len() > self.core.jmap.upload_tmp_quota_size) || (self.core.jmap.upload_tmp_quota_amount > 0 && used.count + 1 > self.core.jmap.upload_tmp_quota_amount)) && !access_token.has_permission(Permission::UnlimitedUploads) { let err = Err(trc::LimitEvent::BlobQuota .into_err() .ctx(trc::Key::Size, self.core.jmap.upload_tmp_quota_size) .ctx(trc::Key::Total, self.core.jmap.upload_tmp_quota_amount)); #[cfg(feature = "test_mode")] if !DISABLE_UPLOAD_QUOTA.load(std::sync::atomic::Ordering::Relaxed) { return err; } #[cfg(not(feature = "test_mode"))] return err; } Ok(UploadResponse { account_id, blob_id: self .put_jmap_blob(account_id.document_id(), data) .await .caused_by(trc::location!())?, c_type: content_type.to_string(), size: data.len(), }) } } ================================================ FILE: crates/jmap/src/calendar/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{api::acl::JmapRights, calendar::Availability, changes::state::JmapCacheState}; use calcard::jscalendar::{JSCalendarAlertAction, JSCalendarRelativeTo, JSCalendarType}; use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; use groupware::{ cache::GroupwareCache, calendar::{ ALERT_EMAIL, ALERT_RELATIVE_TO_END, ArchivedDefaultAlert, CALENDAR_INVISIBLE, CALENDAR_SUBSCRIBED, Calendar, }, }; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::calendar::{self, CalendarProperty, CalendarValue, IncludeInAvailability}, }; use jmap_tools::{Key, Map, Value}; use store::{ValueKey, roaring::RoaringBitmap, write::{AlignedBytes, Archive, ValueClass}}; use trc::AddContext; use types::{ acl::{Acl, AclGrant}, collection::{Collection, SyncCollection}, field::PrincipalField, }; pub trait CalendarGet: Sync + Send { fn calendar_get( &self, request: GetRequest, access_token: &AccessToken, ) -> impl Future>> + Send; } impl CalendarGet for Server { async fn calendar_get( &self, mut request: GetRequest, access_token: &AccessToken, ) -> trc::Result> { let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ CalendarProperty::Id, CalendarProperty::Name, CalendarProperty::Description, CalendarProperty::Color, CalendarProperty::TimeZone, CalendarProperty::SortOrder, CalendarProperty::IsDefault, CalendarProperty::IsSubscribed, CalendarProperty::MyRights, ]); let account_id = request.account_id.document_id(); let cache = self .fetch_dav_resources(access_token, account_id, SyncCollection::Calendar) .await?; let is_owner = access_token.is_member(account_id); let calendar_ids = if is_owner { cache.document_ids(true).collect::() } else { cache.shared_containers(access_token, [Acl::Read, Acl::ReadItems], true) }; let default_calendar_id = self .store() .get_value::(ValueKey { account_id, collection: Collection::Principal.into(), document_id: 0, class: ValueClass::Property(PrincipalField::DefaultCalendarId.into()), }) .await .caused_by(trc::location!())? .or_else(|| { if calendar_ids.len() == 1 { calendar_ids.iter().next() } else { None } }); let ids = if let Some(ids) = ids { ids } else { calendar_ids .iter() .take(self.core.jmap.get_max_objects) .map(Into::into) .collect::>() }; let mut response = GetResponse { account_id: request.account_id.into(), state: cache.get_state(true).into(), list: Vec::with_capacity(ids.len()), not_found: vec![], }; for id in ids { // Obtain the calendar object let document_id = id.document_id(); if !calendar_ids.contains(document_id) { response.not_found.push(id); continue; } let _calendar = if let Some(calendar) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::Calendar, document_id, )) .await? { calendar } else { response.not_found.push(id); continue; }; let calendar = _calendar .unarchive::() .caused_by(trc::location!())?; let mut result = Map::with_capacity(properties.len()); for property in &properties { match property { CalendarProperty::Id => { result.insert_unchecked(CalendarProperty::Id, CalendarValue::Id(id)); } CalendarProperty::Name => { result.insert_unchecked( CalendarProperty::Name, calendar.preferences(access_token).name.to_string(), ); } CalendarProperty::Description => { result.insert_unchecked( CalendarProperty::Description, calendar .preferences(access_token) .description .as_ref() .map(|v| v.to_string()), ); } CalendarProperty::SortOrder => { result.insert_unchecked( CalendarProperty::SortOrder, calendar.preferences(access_token).sort_order.to_native(), ); } CalendarProperty::IsDefault => { result.insert_unchecked( CalendarProperty::IsDefault, default_calendar_id == Some(document_id), ); } CalendarProperty::IsSubscribed => { result.insert_unchecked( CalendarProperty::IsSubscribed, Value::Bool( calendar.preferences(access_token).flags & CALENDAR_SUBSCRIBED != 0, ), ); } CalendarProperty::Color => { result.insert_unchecked( CalendarProperty::Color, calendar .preferences(access_token) .color .as_ref() .map(|c| c.to_string()), ); } CalendarProperty::IsVisible => { result.insert_unchecked( CalendarProperty::IsVisible, Value::Bool( calendar.preferences(access_token).flags & CALENDAR_INVISIBLE == 0, ), ); } CalendarProperty::IncludeInAvailability => { result.insert_unchecked( CalendarProperty::IncludeInAvailability, Value::Element(CalendarValue::IncludeInAvailability( IncludeInAvailability::from_flags( calendar.preferences(access_token).flags.to_native(), ) .unwrap_or(if is_owner { IncludeInAvailability::All } else { IncludeInAvailability::None }), )), ); } CalendarProperty::DefaultAlertsWithTime => { result.insert_unchecked( CalendarProperty::DefaultAlertsWithTime, Value::Object(Map::from_iter( calendar .default_alerts(access_token, true) .map(default_alarm_to_value), )), ); } CalendarProperty::DefaultAlertsWithoutTime => { result.insert_unchecked( CalendarProperty::DefaultAlertsWithoutTime, Value::Object(Map::from_iter( calendar .default_alerts(access_token, false) .map(default_alarm_to_value), )), ); } CalendarProperty::TimeZone => { result.insert_unchecked( CalendarProperty::TimeZone, calendar .preferences(access_token) .time_zone .tz() .map(|tz| Value::Element(CalendarValue::Timezone(tz))) .unwrap_or(Value::Null), ); } CalendarProperty::ShareWith => { result.insert_unchecked( CalendarProperty::ShareWith, JmapRights::share_with::( account_id, access_token, &calendar.acls.iter().map(AclGrant::from).collect::>(), ), ); } CalendarProperty::MyRights => { result.insert_unchecked( CalendarProperty::MyRights, if access_token.is_shared(account_id) { JmapRights::rights::( calendar.acls.effective_acl(access_token), ) } else { JmapRights::all_rights::() }, ); } property => { result.insert_unchecked(property.clone(), Value::Null); } } } response.list.push(result.into()); } Ok(response) } } fn default_alarm_to_value( alarm: &ArchivedDefaultAlert, ) -> ( Key<'static, CalendarProperty>, Value<'static, CalendarProperty, CalendarValue>, ) { ( Key::Owned(alarm.id.to_string()), Value::Object(Map::from(vec![ ( Key::Property(CalendarProperty::Type), Value::Element(CalendarValue::Type(JSCalendarType::Alert)), ), ( Key::Property(CalendarProperty::Action), Value::Element(CalendarValue::Action(if alarm.flags & ALERT_EMAIL != 0 { JSCalendarAlertAction::Email } else { JSCalendarAlertAction::Display })), ), ( Key::Property(CalendarProperty::Trigger), Value::Object(Map::from(vec![ ( Key::Property(CalendarProperty::Type), Value::Element(CalendarValue::Type(JSCalendarType::OffsetTrigger)), ), ( Key::Property(CalendarProperty::Offset), Value::Element(CalendarValue::Duration(alarm.offset.to_native())), ), ( Key::Property(CalendarProperty::RelativeTo), Value::Element(CalendarValue::RelativeTo( if alarm.flags & ALERT_RELATIVE_TO_END != 0 { JSCalendarRelativeTo::End } else { JSCalendarRelativeTo::Start }, )), ), ])), ), ])), ) } ================================================ FILE: crates/jmap/src/calendar/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use groupware::calendar::{ CALENDAR_AVAILABILITY_ALL, CALENDAR_AVAILABILITY_ATTENDING, CALENDAR_AVAILABILITY_NONE, }; use jmap_proto::object::calendar::IncludeInAvailability; pub mod get; pub mod set; pub(crate) trait Availability: Sized { fn from_flags(flags: u16) -> Option; } impl Availability for IncludeInAvailability { fn from_flags(flags: u16) -> Option { if flags & CALENDAR_AVAILABILITY_ALL != 0 { Some(IncludeInAvailability::All) } else if flags & CALENDAR_AVAILABILITY_ATTENDING != 0 { Some(IncludeInAvailability::Attending) } else if flags & CALENDAR_AVAILABILITY_NONE != 0 { Some(IncludeInAvailability::None) } else { None } } } ================================================ FILE: crates/jmap/src/calendar/set.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::api::acl::{JmapAcl, JmapRights}; use calcard::jscalendar::{JSCalendarAlertAction, JSCalendarRelativeTo, JSCalendarType}; use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; use groupware::{ DestroyArchive, cache::GroupwareCache, calendar::{ ALERT_EMAIL, ALERT_RELATIVE_TO_END, ALERT_WITH_TIME, CALENDAR_AVAILABILITY_ALL, CALENDAR_AVAILABILITY_ATTENDING, CALENDAR_AVAILABILITY_NONE, CALENDAR_INVISIBLE, CALENDAR_SUBSCRIBED, Calendar, CalendarEvent, CalendarPreferences, DefaultAlert, Timezone, }, }; use http_proto::HttpSessionData; use jmap_proto::{ error::set::SetError, method::set::{SetRequest, SetResponse}, object::calendar::{self, CalendarProperty, CalendarValue, IncludeInAvailability}, request::{IntoValid, reference::MaybeIdReference}, types::state::State, }; use jmap_tools::{JsonPointerItem, Key, Map, Value}; use rand::{Rng, distr::Alphanumeric}; use store::{ SerializeInfallible, ValueKey, ahash::AHashSet, write::{AlignedBytes, Archive, BatchBuilder, ValueClass}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, field::PrincipalField, }; pub trait CalendarSet: Sync + Send { fn calendar_set( &self, request: SetRequest<'_, calendar::Calendar>, access_token: &AccessToken, session: &HttpSessionData, ) -> impl Future>> + Send; } impl CalendarSet for Server { async fn calendar_set( &self, mut request: SetRequest<'_, calendar::Calendar>, access_token: &AccessToken, _session: &HttpSessionData, ) -> trc::Result> { let account_id = request.account_id.document_id(); let cache = self .fetch_dav_resources(access_token, account_id, SyncCollection::Calendar) .await?; let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; let will_destroy = request.unwrap_destroy().into_valid().collect::>(); let is_shared = access_token.is_shared(account_id); let mut set_default = None; // Process creates let mut batch = BatchBuilder::new(); 'create: for (id, object) in request.unwrap_create() { if is_shared { response.not_created.append( id, SetError::forbidden() .with_description("Cannot create calendars in a shared account."), ); continue 'create; } let mut calendar = Calendar { name: rand::rng() .sample_iter(Alphanumeric) .take(10) .map(char::from) .collect::(), preferences: vec![CalendarPreferences { account_id, name: "".to_string(), ..Default::default() }], ..Default::default() }; // Process changes if let Err(err) = update_calendar(object, &mut calendar, access_token) { response.not_created.append(id, err); continue 'create; } // Validate ACLs if !calendar.acls.is_empty() { if let Err(err) = self.acl_validate(&calendar.acls).await { response.not_created.append(id, err.into()); continue 'create; } self.refresh_acls(&calendar.acls, None).await; } // Insert record let document_id = self .store() .assign_document_ids(account_id, Collection::Calendar, 1) .await .caused_by(trc::location!())?; calendar .insert(access_token, account_id, document_id, &mut batch) .caused_by(trc::location!())?; if let Some(MaybeIdReference::Reference(id_ref)) = &request.arguments.on_success_set_is_default && id_ref == &id { set_default = Some(document_id); } response.created(id, document_id); } // Process updates 'update: for (id, object) in request.unwrap_update().into_valid() { // Make sure id won't be destroyed if will_destroy.contains(&id) { response.not_updated.append(id, SetError::will_destroy()); continue 'update; } // Obtain calendar let document_id = id.document_id(); let calendar_ = if let Some(calendar_) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::Calendar, document_id, )) .await? { calendar_ } else { response.not_updated.append(id, SetError::not_found()); continue 'update; }; let calendar = calendar_ .to_unarchived::() .caused_by(trc::location!())?; let mut new_calendar = calendar .deserialize::() .caused_by(trc::location!())?; // Apply changes let has_acl_changes = match update_calendar(object, &mut new_calendar, access_token) { Ok(has_acl_changes_) => has_acl_changes_, Err(err) => { response.not_updated.append(id, err); continue 'update; } }; // Validate ACL if is_shared { let acl = calendar.inner.acls.effective_acl(access_token); if !acl.contains(Acl::Modify) || (has_acl_changes && !acl.contains(Acl::Share)) { response.not_updated.append( id, SetError::forbidden() .with_description("You are not allowed to modify this calendar."), ); continue 'update; } } if has_acl_changes { if let Err(err) = self.acl_validate(&new_calendar.acls).await { response.not_updated.append(id, err.into()); continue 'update; } self.refresh_archived_acls(&new_calendar.acls, calendar.inner.acls.as_slice()) .await; } // Update record new_calendar .update(access_token, calendar, account_id, document_id, &mut batch) .caused_by(trc::location!())?; response.updated.append(id, None); } // Process deletions let mut reset_default_calendar = false; if !will_destroy.is_empty() { let mut destroy_children = AHashSet::new(); let mut destroy_parents = AHashSet::new(); let default_calendar_id = self .store() .get_value::(ValueKey { account_id, collection: Collection::Principal.into(), document_id: 0, class: ValueClass::Property(PrincipalField::DefaultCalendarId.into()), }) .await .caused_by(trc::location!())?; let on_destroy_remove_events = request.arguments.on_destroy_remove_events.unwrap_or(false); for id in will_destroy { let document_id = id.document_id(); if !cache.has_container_id(&document_id) { response.not_destroyed.append(id, SetError::not_found()); continue; }; let Some(calendar_) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::Calendar, document_id, )) .await .caused_by(trc::location!())? else { response.not_destroyed.append(id, SetError::not_found()); continue; }; let calendar = calendar_ .to_unarchived::() .caused_by(trc::location!())?; // Validate ACLs if is_shared && !calendar .inner .acls .effective_acl(access_token) .contains_all([Acl::Delete, Acl::RemoveItems].into_iter()) { response.not_destroyed.append( id, SetError::forbidden() .with_description("You are not allowed to delete this calendar."), ); continue; } // Obtain children ids let children_ids = cache.children_ids(document_id).collect::>(); if !children_ids.is_empty() && !on_destroy_remove_events { response .not_destroyed .append(id, SetError::calendar_has_event()); continue; } destroy_children.extend(children_ids.iter().copied()); destroy_parents.insert(document_id); // Delete record DestroyArchive(calendar) .delete(access_token, account_id, document_id, None, &mut batch) .caused_by(trc::location!())?; if default_calendar_id == Some(document_id) { reset_default_calendar = true; } response.destroyed.push(id); } // Delete children if !destroy_children.is_empty() { for document_id in destroy_children { if let Some(event_) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, )) .await? { let event = event_ .to_unarchived::() .caused_by(trc::location!())?; if event .inner .names .iter() .all(|n| destroy_parents.contains(&n.parent_id.to_native())) { // Event only belongs to calendars being deleted, delete it DestroyArchive(event).delete_all( access_token, account_id, document_id, false, &mut batch, )?; } else { // Unlink calendar id from event let mut new_event = event .deserialize::() .caused_by(trc::location!())?; new_event .names .retain(|n| !destroy_parents.contains(&n.parent_id)); new_event.update( access_token, event, account_id, document_id, &mut batch, )?; } } } } } // Set default calendar if let Some(MaybeIdReference::Id(id)) = &request.arguments.on_success_set_is_default { set_default = Some(id.document_id()); } if let Some(default_calendar_id) = set_default { if response.not_created.is_empty() && response.not_updated.is_empty() && response.not_destroyed.is_empty() { batch .with_account_id(account_id) .with_collection(Collection::Principal) .with_document(0) .set( PrincipalField::DefaultCalendarId, default_calendar_id.serialize(), ); } } else if reset_default_calendar { batch .with_account_id(account_id) .with_collection(Collection::Principal) .with_document(0) .clear(PrincipalField::DefaultCalendarId); } // Write changes if !batch.is_empty() && let Ok(change_id) = self .commit_batch(batch) .await .caused_by(trc::location!())? .last_change_id(account_id) { self.notify_task_queue(); response.new_state = State::Exact(change_id).into(); } Ok(response) } } fn update_calendar( updates: Value<'_, CalendarProperty, CalendarValue>, calendar: &mut Calendar, access_token: &AccessToken, ) -> Result> { let mut has_acl_changes = false; for (property, value) in updates.into_expanded_object() { let Key::Property(property) = property else { return Err(SetError::invalid_properties() .with_property(property.to_owned()) .with_description("Invalid property.")); }; match (property, value) { (CalendarProperty::Name, Value::Str(value)) if (1..=255).contains(&value.len()) => { calendar.preferences_mut(access_token).name = value.into_owned(); } (CalendarProperty::Description, Value::Str(value)) if value.len() < 255 => { calendar.preferences_mut(access_token).description = value.into_owned().into(); } (CalendarProperty::Description, Value::Null) => { calendar.preferences_mut(access_token).description = None; } (CalendarProperty::Color, Value::Str(value)) if value.len() < 16 => { calendar.preferences_mut(access_token).color = value.into_owned().into(); } (CalendarProperty::Color, Value::Null) => { calendar.preferences_mut(access_token).color = None; } (CalendarProperty::TimeZone, Value::Element(CalendarValue::Timezone(tz))) => { calendar.preferences_mut(access_token).time_zone = Timezone::IANA(tz.as_id()); } (CalendarProperty::TimeZone, Value::Null) => { calendar.preferences_mut(access_token).time_zone = Timezone::Default; } (CalendarProperty::SortOrder, Value::Number(value)) => { calendar.preferences_mut(access_token).sort_order = value.cast_to_u64() as u32; } (CalendarProperty::IsSubscribed, Value::Bool(subscribe)) => { if subscribe { calendar.preferences_mut(access_token).flags |= CALENDAR_SUBSCRIBED; } else { calendar.preferences_mut(access_token).flags &= !CALENDAR_SUBSCRIBED; } } (CalendarProperty::IsVisible, Value::Bool(visible)) => { if visible { calendar.preferences_mut(access_token).flags &= !CALENDAR_INVISIBLE; } else { calendar.preferences_mut(access_token).flags |= CALENDAR_INVISIBLE; } } ( CalendarProperty::IncludeInAvailability, Value::Element(CalendarValue::IncludeInAvailability(availability)), ) => { let flags = &mut calendar.preferences_mut(access_token).flags; match availability { IncludeInAvailability::All => { *flags &= !(CALENDAR_AVAILABILITY_NONE | CALENDAR_AVAILABILITY_ATTENDING); *flags |= CALENDAR_AVAILABILITY_ALL; } IncludeInAvailability::Attending => { *flags &= !(CALENDAR_AVAILABILITY_NONE | CALENDAR_AVAILABILITY_ALL); *flags |= CALENDAR_AVAILABILITY_ATTENDING; } IncludeInAvailability::None => { *flags &= !(CALENDAR_AVAILABILITY_ATTENDING | CALENDAR_AVAILABILITY_ALL); *flags |= CALENDAR_AVAILABILITY_NONE; } } } ( property @ (CalendarProperty::DefaultAlertsWithTime | CalendarProperty::DefaultAlertsWithoutTime), Value::Object(value), ) => { let with_time = matches!(property, CalendarProperty::DefaultAlertsWithTime); let alerts = &mut calendar.preferences_mut(access_token).default_alerts; alerts.retain(|alert| (alert.flags & ALERT_WITH_TIME != 0) != with_time); for (key, value) in value.into_vec() { if let Value::Object(value) = value { alerts.push(value_to_default_alert( key.to_string().into_owned(), value, with_time, )?); } } } (CalendarProperty::ShareWith, value) => { calendar.acls = JmapRights::acl_set::(value)?; has_acl_changes = true; } (CalendarProperty::Pointer(pointer), value) => { let mut ptr_iter = pointer.iter(); match ptr_iter.next() { Some(JsonPointerItem::Key(Key::Property(CalendarProperty::ShareWith))) => { calendar.acls = JmapRights::acl_patch::( std::mem::take(&mut calendar.acls), ptr_iter, value, )?; has_acl_changes = true; } Some(JsonPointerItem::Key(Key::Property( property @ (CalendarProperty::DefaultAlertsWithTime | CalendarProperty::DefaultAlertsWithoutTime), ))) => match (ptr_iter.next(), ptr_iter.next()) { ( Some(key @ (JsonPointerItem::Key(_) | JsonPointerItem::Number(_))), None, ) => { let id = match key { JsonPointerItem::Key(key) => key.to_string().into_owned(), JsonPointerItem::Number(n) => n.to_string(), _ => unreachable!(), }; let with_time = matches!(property, CalendarProperty::DefaultAlertsWithTime); let alerts = &mut calendar.preferences_mut(access_token).default_alerts; alerts.retain(|alert| { (alert.flags & ALERT_WITH_TIME != 0) != with_time || alert.id != id }); if let Value::Object(value) = value { alerts.push(value_to_default_alert(id, value, with_time)?); } } _ => { return Err(SetError::invalid_properties() .with_property(CalendarProperty::Pointer(pointer)) .with_description("Field could not be patched.")); } }, _ => { return Err(SetError::invalid_properties() .with_property(CalendarProperty::Pointer(pointer)) .with_description("Field could not be patched.")); } } } (property, _) => { return Err(SetError::invalid_properties() .with_property(property) .with_description("Field could not be set.")); } } } // Validate name if calendar.preferences(access_token).name.is_empty() { return Err(SetError::invalid_properties() .with_property(CalendarProperty::Name) .with_description("Missing name.")); } Ok(has_acl_changes) } fn value_to_default_alert( id: String, value: Map<'_, CalendarProperty, CalendarValue>, with_time: bool, ) -> Result> { let mut alert = DefaultAlert { id, ..Default::default() }; let mut has_offset = false; for (key, value) in value.into_vec() { let Key::Property(key) = key else { continue; }; match (key, value) { (CalendarProperty::Type, Value::Element(CalendarValue::Type(value))) => { if value != JSCalendarType::Alert { return Err(SetError::invalid_properties() .with_property(CalendarProperty::Trigger) .with_description("Invalid alert object type.")); } } ( CalendarProperty::Action, Value::Element(CalendarValue::Action(JSCalendarAlertAction::Email)), ) => { alert.flags |= ALERT_EMAIL; } (CalendarProperty::Trigger, Value::Object(value)) => { for (key, value) in value.into_vec() { let Key::Property(key) = key else { continue; }; match (key, value) { ( CalendarProperty::RelativeTo, Value::Element(CalendarValue::RelativeTo(JSCalendarRelativeTo::End)), ) => { alert.flags |= ALERT_RELATIVE_TO_END; } ( CalendarProperty::Offset, Value::Element(CalendarValue::Duration(value)), ) => { alert.offset = value; has_offset = true; } (CalendarProperty::Offset, Value::Element(CalendarValue::Type(value))) => { if value != JSCalendarType::OffsetTrigger { return Err(SetError::invalid_properties() .with_property(CalendarProperty::Trigger) .with_description("Invalid alert trigger type.")); } } _ => {} } } } _ => {} } } if has_offset { if with_time { alert.flags |= ALERT_WITH_TIME; } Ok(alert) } else { Err(SetError::invalid_properties() .with_property(CalendarProperty::Trigger) .with_description("Missing alert offset.")) } } ================================================ FILE: crates/jmap/src/calendar_event/copy.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ calendar_event::{CalendarSyntheticId, set::CalendarEventSet}, changes::state::JmapCacheState, }; use calcard::jscalendar::JSCalendarProperty; use common::{Server, auth::AccessToken}; use groupware::{cache::GroupwareCache, calendar::CalendarEvent}; use http_proto::HttpSessionData; use jmap_proto::{ error::set::SetError, method::{ copy::{CopyRequest, CopyResponse}, set::SetRequest, }, object::calendar_event, request::{ Call, IntoValid, MaybeInvalid, RequestMethod, SetRequestMethod, method::{MethodFunction, MethodName, MethodObject}, reference::MaybeResultReference, }, types::state::State, }; use store::{ValueKey, roaring::RoaringBitmap, write::{AlignedBytes, Archive, BatchBuilder}}; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, }; use utils::map::vec_map::VecMap; pub trait JmapCalendarEventCopy: Sync + Send { fn calendar_event_copy<'x>( &self, request: CopyRequest<'x, calendar_event::CalendarEvent>, access_token: &AccessToken, next_call: &mut Option>>, session: &HttpSessionData, ) -> impl Future>> + Send; } impl JmapCalendarEventCopy for Server { async fn calendar_event_copy<'x>( &self, request: CopyRequest<'x, calendar_event::CalendarEvent>, access_token: &AccessToken, next_call: &mut Option>>, _session: &HttpSessionData, ) -> trc::Result> { let account_id = request.account_id.document_id(); let from_account_id = request.from_account_id.document_id(); if account_id == from_account_id { return Err(trc::JmapEvent::InvalidArguments .into_err() .details("From accountId is equal to fromAccountId")); } let cache = self .fetch_dav_resources(access_token, account_id, SyncCollection::Calendar) .await .caused_by(trc::location!())?; let old_state = cache.assert_state(false, &request.if_in_state)?; let mut response = CopyResponse { from_account_id: request.from_account_id, account_id: request.account_id, new_state: old_state.clone(), old_state, created: VecMap::with_capacity(request.create.len()), not_created: VecMap::new(), }; let from_cache = self .fetch_dav_resources(access_token, from_account_id, SyncCollection::Calendar) .await .caused_by(trc::location!())?; let from_calendar_event_ids = if access_token.is_member(from_account_id) { from_cache.document_ids(false).collect::() } else { from_cache.shared_items(access_token, [Acl::ReadItems], true) }; let can_add_calendars = if access_token.is_shared(account_id) { cache .shared_containers(access_token, [Acl::AddItems], true) .into() } else { None }; let on_success_delete = request.on_success_destroy_original.unwrap_or(false); let mut destroy_ids = Vec::new(); // Obtain quota let mut batch = BatchBuilder::new(); 'create: for (id, create) in request.create.into_valid() { let from_calendar_event_id = id.document_id(); if !from_calendar_event_ids.contains(from_calendar_event_id) { response.not_created.append( id, SetError::not_found().with_description(format!( "Item {} not found in account {}.", id, response.from_account_id )), ); continue; } if id.is_synthetic() { response.not_created.append( id, SetError::invalid_properties() .with_property(JSCalendarProperty::Id) .with_description(format!( "Item {} is a synthetic id and cannot be copied.", id )), ); continue; } let Some(_calendar_event) = self .store() .get_value::>(ValueKey::archive( from_account_id, Collection::CalendarEvent, from_calendar_event_id, )) .await? else { response.not_created.append( id, SetError::not_found().with_description(format!( "Item {} not found in account {}.", id, response.from_account_id )), ); continue; }; let calendar_event = _calendar_event .deserialize::() .caused_by(trc::location!())?; match self .create_calendar_event( &cache, &mut batch, access_token, account_id, false, &can_add_calendars, calendar_event.data.event.into_jscalendar(), create, ) .await? { Ok(document_id) => { response.created(id, document_id); // Add to destroy list if on_success_delete { destroy_ids.push(MaybeInvalid::Value(id)); } } Err(err) => { response.not_created.append(id, err); continue 'create; } } } // Write changes if !batch.is_empty() { let change_id = self .commit_batch(batch) .await .and_then(|ids| ids.last_change_id(account_id)) .caused_by(trc::location!())?; self.notify_task_queue(); response.new_state = State::Exact(change_id); } // Destroy ids if on_success_delete && !destroy_ids.is_empty() { *next_call = Call { id: String::new(), name: MethodName::new(MethodObject::CalendarEvent, MethodFunction::Set), method: RequestMethod::Set(SetRequestMethod::CalendarEvent(SetRequest { account_id: request.from_account_id, if_in_state: request.destroy_from_if_in_state, create: None, update: None, destroy: MaybeResultReference::Value(destroy_ids).into(), arguments: Default::default(), })), } .into(); } Ok(response) } } ================================================ FILE: crates/jmap/src/calendar_event/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{calendar_event::CalendarSyntheticId, changes::state::JmapCacheState}; use calcard::{ common::{PartialDateTime, timezone::Tz}, icalendar::{ ICalendar, ICalendarComponent, ICalendarComponentType, ICalendarEntry, ICalendarParameter, ICalendarParameterName, ICalendarParameterValue, ICalendarParticipationRole, ICalendarProperty, ICalendarValue, }, jscalendar::{ JSCalendarDateTime, JSCalendarProperty, JSCalendarValue, import::ConversionOptions, }, }; use chrono::DateTime; use common::{Server, auth::AccessToken}; use groupware::{ cache::GroupwareCache, calendar::{ CalendarEvent, EVENT_DRAFT, EVENT_HIDE_ATTENDEES, EVENT_INVITE_OTHERS, EVENT_INVITE_SELF, PREF_USE_DEFAULT_ALERTS, expand::CalendarEventExpansion, }, }; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::{JmapObjectId, calendar_event}, request::{IntoValid, reference::MaybeResultReference}, }; use jmap_tools::{Key, Map, Value}; use std::{str::FromStr, sync::Arc}; use store::{ ValueKey, ahash::{AHashMap, AHashSet}, roaring::RoaringBitmap, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{ acl::Acl, blob::BlobId, collection::{Collection, SyncCollection}, id::Id, }; pub trait CalendarEventGet: Sync + Send { fn calendar_event_get( &self, request: GetRequest, access_token: &AccessToken, ) -> impl Future>> + Send; } impl CalendarEventGet for Server { async fn calendar_event_get( &self, mut request: GetRequest, access_token: &AccessToken, ) -> trc::Result> { let return_all_properties = request .properties .as_ref() .is_none_or(|v| matches!(v, MaybeResultReference::Value(v) if v.is_empty())); let properties = request.unwrap_properties(&[]); let account_id = request.account_id.document_id(); let cache = self .fetch_dav_resources(access_token, account_id, SyncCollection::Calendar) .await?; let calendar_event_ids = if access_token.is_member(account_id) { cache.document_ids(false).collect::() } else { cache.shared_items(access_token, [Acl::ReadItems], true) }; let (mut ids, has_synthetic_ids) = if let Some(rr) = request.ids.take() { let rr = rr.unwrap(); if rr.len() > self.core.jmap.get_max_objects { return Err(trc::JmapEvent::RequestTooLarge.into_err()); } let mut ids = Vec::with_capacity(rr.len()); let mut has_synthetic_ids = false; for id in rr.into_valid() { has_synthetic_ids |= id.is_synthetic(); ids.push(id); } (ids, has_synthetic_ids) } else { ( calendar_event_ids .iter() .take(self.core.jmap.get_max_objects) .map(Into::into) .collect::>(), false, ) }; let mut response = GetResponse { account_id: request.account_id.into(), state: cache.get_state(false).into(), list: Vec::with_capacity(ids.len()), not_found: vec![], }; let mut return_converted_props = !return_all_properties; let mut return_is_origin = false; let mut return_utc_dates = false; let (jmap_properties, jscal_properties) = if !return_all_properties { let mut jmap_properties = Vec::with_capacity(4); let mut jscal_properties = Vec::with_capacity(properties.len()); for property in properties { match property { JSCalendarProperty::Id | JSCalendarProperty::BaseEventId | JSCalendarProperty::CalendarIds | JSCalendarProperty::IsDraft | JSCalendarProperty::UseDefaultAlerts | JSCalendarProperty::MayInviteSelf | JSCalendarProperty::MayInviteOthers | JSCalendarProperty::HideAttendees => { jmap_properties.push(property); } JSCalendarProperty::UtcStart | JSCalendarProperty::UtcEnd => { return_utc_dates = true; jmap_properties.push(property); } JSCalendarProperty::IsOrigin => { return_is_origin = true; } _ => { if matches!(property, JSCalendarProperty::ICalendar) { return_converted_props = true; } jscal_properties.push(property); } } } (jmap_properties, jscal_properties) } else { return_is_origin = true; ( vec![ JSCalendarProperty::Id, JSCalendarProperty::CalendarIds, JSCalendarProperty::IsDraft, JSCalendarProperty::IsOrigin, ], vec![], ) }; let return_is_origin = if return_is_origin { if access_token.primary_id() == account_id { OriginAddresses::Ref(access_token) } else { OriginAddresses::Owned(self.get_access_token(account_id).await?) } } else { OriginAddresses::None }; // Sort by baseId let mut original_order: Option> = None; if has_synthetic_ids { original_order = Some(ids.iter().enumerate().map(|(i, id)| (*id, i)).collect()); ids.sort_unstable_by_key(|id| id.document_id()); } let mut ids = ids.into_iter().peekable(); // Process arguments let override_range = if request.arguments.recurrence_overrides_after.is_some() || request.arguments.recurrence_overrides_before.is_some() { let after = request .arguments .recurrence_overrides_after .map(|v| v.timestamp) .unwrap_or(i64::MIN); let before = request .arguments .recurrence_overrides_before .map(|v| v.timestamp) .unwrap_or(i64::MAX); if after < before { Some(after..before) } else { None } } else { None }; let default_tz = request.arguments.time_zone.unwrap_or(Tz::UTC); let reduce_participants = request.arguments.reduce_participants.unwrap_or(false); 'outer: while let Some(id) = ids.next() { // Obtain the calendar_event object let document_id = id.document_id(); if !calendar_event_ids.contains(document_id) { response.not_found.push(id); continue; } let Some(_calendar_event) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, )) .await? else { response.not_found.push(id); continue; }; let mut calendar_event = _calendar_event .deserialize::() .caused_by(trc::location!())?; // Extract expansion ids from synthetic ids let mut expansion_ids = AHashSet::new(); let mut include_base_event = false; if let Some(expansion_id) = id.expansion_id() { expansion_ids.insert(expansion_id); } else { include_base_event = true; } while let Some(next_id) = ids.peek() { if next_id.document_id() == document_id { if let Some(expansion_id) = next_id.expansion_id() { expansion_ids.insert(expansion_id); } else { include_base_event = true; } ids.next(); } else { break; } } // Reduce participants if reduce_participants { for component in &mut calendar_event.data.event.components { if component.component_type.is_scheduling_object() { component.entries.retain(|entry| match &entry.name { ICalendarProperty::Attendee => { entry.parameters(&ICalendarParameterName::Role).any(|role| { matches!( role, ICalendarParameterValue::Role( ICalendarParticipationRole::Owner, ), ) }) || entry.calendar_address().is_some_and(|addr| { access_token .emails .iter() .any(|a| a.eq_ignore_ascii_case(addr)) }) } _ => true, }); } } } // Expand synthetic ids let mut results = Vec::with_capacity(expansion_ids.len() + 1); if !expansion_ids.is_empty() { let ical = &calendar_event.data.event; if let Some(expansions) = calendar_event .data .expand_from_ids(&mut expansion_ids, default_tz) { for expansion in expansions { if !expansion.is_valid() { response.not_found.push(::new( expansion.expansion_id, document_id, )); continue 'outer; } let component = &ical.components[expansion.comp_id as usize]; let is_recurrent = component.is_recurrent(); let is_recurrent_or_override = is_recurrent || component.is_recurrence_override(); let mut has_duration = false; let component_ids = &component.component_ids; let mut tz = None; let mut component = ICalendarComponent { component_type: component.component_type.clone(), component_ids: Vec::new(), entries: component .entries .iter() .filter(|entry| match &entry.name { ICalendarProperty::Dtstart | ICalendarProperty::Dtend | ICalendarProperty::Exdate | ICalendarProperty::Exrule | ICalendarProperty::Rdate | ICalendarProperty::Rrule | ICalendarProperty::RecurrenceId => { if let Some(new_tz) = entry .tz_id() .and_then(|id| Tz::from_str(id).ok()) .filter(|tz| *tz != Tz::UTC) { tz = Some(new_tz); } false } ICalendarProperty::Due | ICalendarProperty::Completed | ICalendarProperty::Created => is_recurrent, ICalendarProperty::Duration => { has_duration = true; true } _ => true, }) .cloned() .collect::>(), }; let tz = tz.unwrap_or(default_tz); let tz_name = tz.name().unwrap_or_default().to_string(); let start_timestamp = DateTime::from_timestamp(expansion.start, 0) .map(|dt| dt.with_timezone(&tz)) .map(|dt| dt.naive_local()) .map(|dt| dt.and_utc().timestamp()) .unwrap_or(expansion.start); let end_timestamp = DateTime::from_timestamp(expansion.end, 0) .map(|dt| dt.with_timezone(&tz)) .map(|dt| dt.naive_local()) .map(|dt| dt.and_utc().timestamp()) .unwrap_or(expansion.end); component.entries.push(ICalendarEntry { name: ICalendarProperty::Dtstart, params: vec![ICalendarParameter::tzid(tz_name.clone())], values: vec![ICalendarValue::PartialDateTime(Box::new( PartialDateTime::from_naive_timestamp(start_timestamp), ))], }); if is_recurrent_or_override { component.entries.push(ICalendarEntry { name: ICalendarProperty::RecurrenceId, params: vec![ICalendarParameter::tzid(tz_name.clone())], values: vec![ICalendarValue::PartialDateTime(Box::new( PartialDateTime::from_naive_timestamp(start_timestamp), ))], }); } if !has_duration { component.entries.push(ICalendarEntry { name: ICalendarProperty::Dtend, params: vec![ICalendarParameter::tzid(tz_name)], values: vec![ICalendarValue::PartialDateTime(Box::new( PartialDateTime::from_naive_timestamp(end_timestamp), ))], }); } let mut expanded_ical = ICalendar { components: vec![ ICalendarComponent { component_type: ICalendarComponentType::VCalendar, entries: vec![], component_ids: vec![1], }, component, ], }; if !component_ids.is_empty() { for component_id in component_ids { let mut sub_component = ical.components[*component_id as usize].clone(); sub_component.component_ids.clear(); let component_id = expanded_ical.components.len() as u32; expanded_ical.components.push(sub_component); expanded_ical.components[1].component_ids.push(component_id); } } results.push(( ::new(expansion.expansion_id, document_id), expanded_ical, expansion, )); } } else { response .not_found .extend(expansion_ids.into_iter().map(|expansion_id| { ::new(expansion_id, document_id) })); continue; } } if include_base_event { let mut event = std::mem::take(&mut calendar_event.data.event); // Obtain UTC start/end if requested let expansion = if return_utc_dates && let Some(expansion) = event .components .iter() .position(|c| { c.component_type.is_scheduling_object() && !c.is_recurrence_override() }) .and_then(|comp_id| { calendar_event .data .expand_single(comp_id as u32, default_tz) }) { expansion } else { CalendarEventExpansion::default() }; // Remove recurrence ids if let Some(range) = &override_range { let remove_ids = event .components .iter() .enumerate() .filter_map(|(comp_id, c)| { if c.is_recurrence_override() && let Some(timestamp) = c .property(&ICalendarProperty::RecurrenceId) .and_then(|p| p.values.first()) .and_then(|v| v.as_partial_date_time()) .and_then(|v| v.to_date_time()) .and_then(|v| v.to_date_time_with_tz(default_tz)) .map(|v| v.timestamp()) && !range.contains(×tamp) { Some(comp_id as u32) } else { None } }) .collect::>(); if !remove_ids.is_empty() { for component in &mut event.components { component .component_ids .retain(|id| !remove_ids.contains(id)); } } } results.push((Id::from(document_id), event, expansion)); } for (id, ical, expansion) in results { let is_origin = return_is_origin.addresses().is_some_and(|addresses| { ical.components .iter() .find(|c| c.component_type.is_scheduling_object()) .and_then(|c| c.property(&ICalendarProperty::Organizer)) .and_then(|v| v.calendar_address()) .is_none_or(|v| addresses.iter().any(|a| a.eq_ignore_ascii_case(v))) }); let jscal = ical .into_jscalendar_with_opt::( ConversionOptions::default() .include_ical_components(return_converted_props) .return_first(true), ) .into_inner(); let mut result = if return_all_properties { jscal.into_object().unwrap() } else { Map::from_iter(jscal.into_expanded_object().filter(|(k, _)| { k.as_property() .is_some_and(|p| jscal_properties.contains(p)) })) }; for property in &jmap_properties { match property { JSCalendarProperty::Id => { result.insert_unchecked( JSCalendarProperty::Id, Value::Element(JSCalendarValue::Id(id)), ); } JSCalendarProperty::BaseEventId => { result.insert_unchecked( JSCalendarProperty::BaseEventId, Value::Element(JSCalendarValue::Id(id.document_id().into())), ); } JSCalendarProperty::CalendarIds => { let mut obj = Map::with_capacity(calendar_event.names.len()); for id in calendar_event.names.iter() { obj.insert_unchecked( JSCalendarProperty::IdValue(Id::from(id.parent_id)), true, ); } result.insert_unchecked( JSCalendarProperty::CalendarIds, Value::Object(obj), ); } JSCalendarProperty::IsDraft => { result.insert_unchecked( JSCalendarProperty::IsDraft, Value::Bool(calendar_event.flags & EVENT_DRAFT != 0), ); } JSCalendarProperty::IsOrigin => { result.insert_unchecked( JSCalendarProperty::IsOrigin, Value::Bool(is_origin), ); } JSCalendarProperty::MayInviteSelf => { result.insert_unchecked( JSCalendarProperty::MayInviteSelf, Value::Bool(calendar_event.flags & EVENT_INVITE_SELF != 0), ); } JSCalendarProperty::MayInviteOthers => { result.insert_unchecked( JSCalendarProperty::MayInviteOthers, Value::Bool(calendar_event.flags & EVENT_INVITE_OTHERS != 0), ); } JSCalendarProperty::HideAttendees => { result.insert_unchecked( JSCalendarProperty::HideAttendees, Value::Bool(calendar_event.flags & EVENT_HIDE_ATTENDEES != 0), ); } JSCalendarProperty::UtcStart => { result.insert_unchecked( JSCalendarProperty::UtcStart, Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::new( expansion.start, false, ))), ); } JSCalendarProperty::UtcEnd => { result.insert_unchecked( JSCalendarProperty::UtcEnd, Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::new( expansion.end, false, ))), ); } JSCalendarProperty::UseDefaultAlerts => { result.insert_unchecked( JSCalendarProperty::UseDefaultAlerts, Value::Bool( calendar_event .preferences(access_token) .is_none_or(|v| v.flags & PREF_USE_DEFAULT_ALERTS != 0), ), ); } _ => {} } } response.list.push(result.into()); } } // Restore original order if let Some(original_order) = original_order { response.list.sort_by_key(|obj| { obj.as_object() .unwrap() .get(&Key::Property(JSCalendarProperty::::Id)) .and_then(|v| v.as_element()) .and_then(|v: &JSCalendarValue| v.as_id()) .and_then(|id| original_order.get(&id)) .cloned() .unwrap_or(usize::MAX) }); } Ok(response) } } enum OriginAddresses<'x> { Owned(Arc), Ref(&'x AccessToken), None, } impl<'x> OriginAddresses<'x> { fn addresses(&self) -> Option<&[String]> { match self { OriginAddresses::Owned(t) if !t.emails.is_empty() => Some(&t.emails), OriginAddresses::Ref(t) if !t.emails.is_empty() => Some(&t.emails), _ => None, } } } ================================================ FILE: crates/jmap/src/calendar_event/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use calcard::jscalendar::JSCalendarProperty; use common::Server; use jmap_proto::error::set::SetError; use trc::AddContext; use types::{collection::Collection, field::CalendarEventField, id::Id}; pub mod copy; pub mod get; pub mod parse; pub mod query; pub mod set; /* TODO: Not yet implemented: - CalendarEvent - Per-user properties (However, the database schema is ready to support this) - mayInviteSelf, mayInviteOthers and hideAttendees (stored but not enforced) - CalendarEvent/set - synthetic id update and removal - Principal/getAvailability - If there are overlapping BusyPeriod time ranges with different "busyStatus" properties the server MUST choose the value in the following order: confirmed > unavailable > tentative. - Return event properties */ pub trait CalendarSyntheticId { fn new(expansion_id: u32, document_id: u32) -> Self; fn is_synthetic(&self) -> bool; fn expansion_id(&self) -> Option; } impl CalendarSyntheticId for Id { fn new(expansion_id: u32, document_id: u32) -> Id { Id::from_parts(expansion_id + 1, document_id) } fn expansion_id(&self) -> Option { let prefix = self.prefix_id(); if prefix > 0 { Some(prefix - 1) } else { None } } fn is_synthetic(&self) -> bool { self.prefix_id() > 0 } } pub(super) async fn assert_is_unique_uid( server: &Server, account_id: u32, uid: Option<&str>, ) -> trc::Result>>> { if let Some(uid) = uid && server .document_exists( account_id, Collection::CalendarEvent, CalendarEventField::Uid, uid.as_bytes(), ) .await .caused_by(trc::location!())? { Ok(Err(SetError::invalid_properties() .with_property(JSCalendarProperty::Uid) .with_description(format!( "An event with UID {uid} already exists.", )))) } else { Ok(Ok(())) } } ================================================ FILE: crates/jmap/src/calendar_event/parse.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::blob::download::BlobDownload; use calcard::{ icalendar::ICalendar, jscalendar::{JSCalendarProperty, import::ConversionOptions}, }; use common::{Server, auth::AccessToken}; use jmap_proto::{ method::parse::{ParseRequest, ParseResponse}, object::calendar_event::CalendarEvent, request::IntoValid, }; use jmap_tools::{Key, Value}; use types::{blob::BlobId, id::Id}; use utils::map::vec_map::VecMap; pub trait CalendarEventParse: Sync + Send { fn calendar_event_parse( &self, request: ParseRequest, access_token: &AccessToken, ) -> impl Future>> + Send; } impl CalendarEventParse for Server { async fn calendar_event_parse( &self, request: ParseRequest, access_token: &AccessToken, ) -> trc::Result> { if request.blob_ids.len() > self.core.jmap.calendar_parse_max_items { return Err(trc::JmapEvent::RequestTooLarge.into_err()); } let return_all_properties = request.properties.is_none(); let properties = request .properties .map(|v| v.into_valid().collect::>()) .unwrap_or_default(); let mut response = ParseResponse { account_id: request.account_id, parsed: VecMap::with_capacity(request.blob_ids.len()), not_parsable: vec![], not_found: vec![], }; for blob_id in request.blob_ids.into_valid() { // Fetch raw message to parse let raw_vcard = match self.blob_download(&blob_id, access_token).await? { Some(raw_vcard) => raw_vcard, None => { response.not_found.push(blob_id); continue; } }; let Ok(vcard) = ICalendar::parse(std::str::from_utf8(&raw_vcard).unwrap_or_default()) else { response.not_parsable.push(blob_id); continue; }; let mut js_calendar_entries = vcard .into_jscalendar_with_opt::(ConversionOptions::default()) .into_inner() .into_object() .unwrap() .remove(&Key::Property(JSCalendarProperty::Entries)) .unwrap() .into_array() .unwrap(); if !return_all_properties { for entry in &mut js_calendar_entries { entry .as_object_mut() .unwrap() .as_mut_vec() .retain(|(k, _)| k.as_property().is_some_and(|k| properties.contains(k))); } } response .parsed .append(blob_id, Value::Array(js_calendar_entries)); } Ok(response) } } ================================================ FILE: crates/jmap/src/calendar_event/query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{api::query::QueryResponseBuilder, changes::state::JmapCacheState}; use calcard::{common::timezone::Tz, jscalendar::JSCalendarDateTime}; use chrono::offset::TimeZone; use common::{Server, auth::AccessToken}; use groupware::{cache::GroupwareCache, calendar::CalendarEvent}; use jmap_proto::{ method::query::{Filter, QueryRequest, QueryResponse}, object::calendar_event::{self, CalendarEventComparator, CalendarEventFilter}, request::MaybeInvalid, }; use nlp::language::Language; use std::{cmp::Ordering, sync::Arc}; use store::{ ValueKey, roaring::RoaringBitmap, search::{CalendarSearchField, SearchComparator, SearchFilter, SearchQuery}, write::{AlignedBytes, Archive, SearchIndex} }; use trc::AddContext; use types::{ TimeRange, acl::Acl, collection::{Collection, SyncCollection}, }; pub trait CalendarEventQuery: Sync + Send { fn calendar_event_query( &self, request: QueryRequest, access_token: &AccessToken, ) -> impl Future> + Send; } impl CalendarEventQuery for Server { async fn calendar_event_query( &self, mut request: QueryRequest, access_token: &AccessToken, ) -> trc::Result { let account_id = request.account_id.document_id(); let mut filters = Vec::with_capacity(request.filter.len()); let cache = self .fetch_dav_resources(access_token, account_id, SyncCollection::Calendar) .await?; let default_tz = request.arguments.time_zone.unwrap_or(Tz::UTC); let mut filter: Option = None; // Extract from/to arguments for cond in &request.filter { if let Filter::Property(CalendarEventFilter::After(date)) = cond { if let Some(after) = local_timestamp(date, default_tz) { filter.get_or_insert_default().start = after; } } else if let Filter::Property(CalendarEventFilter::Before(date)) = cond && let Some(before) = local_timestamp(date, default_tz) { filter.get_or_insert_default().end = before; } } for cond in std::mem::take(&mut request.filter) { match cond { Filter::Property(cond) => match cond { CalendarEventFilter::InCalendar(MaybeInvalid::Value(id)) => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.children_ids(id.document_id()), ))) } CalendarEventFilter::Uid(uid) => { filters.push(SearchFilter::eq(CalendarSearchField::Uid, uid)); } CalendarEventFilter::Text(value) => { let (text, language) = Language::detect(value, self.core.jmap.default_language); filters.push(SearchFilter::Or); filters.push(SearchFilter::has_text( CalendarSearchField::Title, text.clone(), language, )); filters.push(SearchFilter::has_text( CalendarSearchField::Description, text.clone(), language, )); filters.push(SearchFilter::has_text( CalendarSearchField::Location, text.clone(), language, )); filters.push(SearchFilter::has_text( CalendarSearchField::Owner, text.clone(), language, )); filters.push(SearchFilter::has_text( CalendarSearchField::Attendee, text, language, )); filters.push(SearchFilter::End); } CalendarEventFilter::Title(title) => { filters.push(SearchFilter::has_text_detect( CalendarSearchField::Title, title, self.core.jmap.default_language, )); } CalendarEventFilter::Description(description) => { filters.push(SearchFilter::has_text_detect( CalendarSearchField::Description, description, self.core.jmap.default_language, )); } CalendarEventFilter::Location(location) => { filters.push(SearchFilter::has_text_detect( CalendarSearchField::Location, location, self.core.jmap.default_language, )); } CalendarEventFilter::Owner(owner) => { filters.push(SearchFilter::has_text( CalendarSearchField::Owner, owner, Language::None, )); } CalendarEventFilter::Attendee(attendee) => { filters.push(SearchFilter::has_text( CalendarSearchField::Attendee, attendee, Language::None, )); } CalendarEventFilter::After(after) => { /* The end of the event, or any recurrence of the event, in the time zone given as the "timeZone" argument, must be after this date to match the condition. */ if let Some(after) = local_timestamp(&after, default_tz) { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.resources.iter().filter_map(|r| { r.event_time_range() .and_then(|(_, end)| (after < end).then_some(r.document_id)) }), ))); } } CalendarEventFilter::Before(before) => { /* The start of the event, or any recurrence of the event, in the time zone given as the "timeZone" argument, must be before this date to match the condition. */ if let Some(before) = local_timestamp(&before, default_tz) { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.resources.iter().filter_map(|r| { r.event_time_range().and_then(|(start, _)| { (before > start).then_some(r.document_id) }) }), ))); } } unsupported => { return Err(trc::JmapEvent::UnsupportedFilter .into_err() .details(unsupported.into_string())); } }, Filter::And => { filters.push(SearchFilter::And); } Filter::Or => { filters.push(SearchFilter::Or); } Filter::Not => { filters.push(SearchFilter::Not); } Filter::Close => { filters.push(SearchFilter::End); } } } let expand_recurrences = request.arguments.expand_recurrences.unwrap_or(false); let comparators = if !expand_recurrences { request .sort .take() .unwrap_or_default() .into_iter() .map(|comparator| match comparator.property { CalendarEventComparator::Start | CalendarEventComparator::RecurrenceId => { Ok(SearchComparator::field( CalendarSearchField::Start, comparator.is_ascending, )) } CalendarEventComparator::Uid => Ok(SearchComparator::field( CalendarSearchField::Uid, comparator.is_ascending, )), CalendarEventComparator::Created | CalendarEventComparator::Updated => { Err(trc::JmapEvent::UnsupportedSort .into_err() .details(comparator.property.into_string().into_owned())) } CalendarEventComparator::_T(other) => Err(trc::JmapEvent::UnsupportedSort .into_err() .details(other.to_string())), }) .collect::, _>>()? } else { vec![] }; let results = self .search_store() .query_account( SearchQuery::new(SearchIndex::Calendar) .with_filters(filters) .with_comparators(comparators) .with_account_id(account_id) .with_mask(if access_token.is_shared(account_id) { cache.shared_items(access_token, [Acl::ReadItems], true) } else { cache.document_ids(false).collect() }), ) .await?; // Extract comparators let comparators = request .sort .as_deref() .filter(|s| !s.is_empty()) .unwrap_or_default(); if expand_recurrences && !results.is_empty() { let Some(time_range) = filter.filter(|f| f.start != i64::MIN && f.end != i64::MAX) else { return Err(trc::JmapEvent::InvalidArguments.into_err().details( "Both 'after' and 'before' filters are required when expanding recurrences", )); }; let max_instances = self.core.groupware.max_ical_instances; let mut expanded_results = Vec::with_capacity(results.len() as usize); let has_uid_comparator = comparators .iter() .any(|c| matches!(c.property, CalendarEventComparator::Uid)); for document_id in results { let Some(_calendar_event) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, )) .await? else { continue; }; let calendar_event = _calendar_event .unarchive::() .caused_by(trc::location!())?; // Expand recurrences let uid = if has_uid_comparator { Arc::new( calendar_event .data .event .uids() .next() .unwrap_or_default() .to_string(), ) } else { Arc::new(String::new()) }; for expansion in calendar_event .data .expand(default_tz, time_range) .unwrap_or_default() { if expanded_results.len() < max_instances { expanded_results.push(SearchResult { created: calendar_event.created.to_native().to_be_bytes(), updated: calendar_event.modified.to_native().to_be_bytes(), start: expansion.start.to_be_bytes(), uid: uid.clone(), document_id, expansion_id: expansion.expansion_id.into(), }); } else { return Err(trc::JmapEvent::InvalidArguments.into_err().details( "The number of expanded recurrences exceeds the server limit", )); } } } let mut response = QueryResponseBuilder::new( expanded_results.len(), self.core.jmap.query_max_results, cache.get_state(false), &request, ); // Sort results if !expanded_results.is_empty() { expanded_results.sort_by(|a, b| { for comparator in comparators { let ordering = if comparator.is_ascending { a.get_property(&comparator.property) .cmp(b.get_property(&comparator.property)) } else { b.get_property(&comparator.property) .cmp(a.get_property(&comparator.property)) }; if ordering != Ordering::Equal { return ordering; } } Ordering::Equal }); // Add results for result in expanded_results { if !response.add(result.expansion_id.unwrap() + 1, result.document_id) { break; } } } response.build() } else { let mut response = QueryResponseBuilder::new( results.len(), self.core.jmap.query_max_results, cache.get_state(false), &request, ); for document_id in results { if !response.add(0, document_id) { break; } } response.build() } } } fn local_timestamp(dt: &JSCalendarDateTime, tz: Tz) -> Option { tz.from_local_datetime(&dt.to_naive_date_time()?) .single() .map(|dt| dt.timestamp()) } #[derive(Debug)] struct SearchResult { expansion_id: Option, document_id: u32, start: [u8; std::mem::size_of::()], created: [u8; std::mem::size_of::()], updated: [u8; std::mem::size_of::()], uid: Arc, } impl SearchResult { fn get_property(&self, comparator: &CalendarEventComparator) -> &[u8] { match comparator { CalendarEventComparator::Uid => self.uid.as_bytes(), CalendarEventComparator::Start | CalendarEventComparator::RecurrenceId => { self.start.as_ref() } CalendarEventComparator::Created => self.created.as_ref(), CalendarEventComparator::Updated => self.updated.as_ref(), CalendarEventComparator::_T(_) => &[], } } } ================================================ FILE: crates/jmap/src/calendar_event/set.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::calendar_event::{CalendarSyntheticId, assert_is_unique_uid}; use calcard::{ common::timezone::Tz, icalendar::{ ICalendarAction, ICalendarComponent, ICalendarComponentType, ICalendarDuration, ICalendarEntry, ICalendarParameter, ICalendarParameterValue, ICalendarProperty, ICalendarRelated, ICalendarValue, }, jscalendar::{JSCalendar, JSCalendarDateTime, JSCalendarProperty, JSCalendarValue}, }; use chrono::DateTime; use common::{DavName, DavResources, Server, auth::AccessToken}; use directory::Permission; use groupware::{ DestroyArchive, cache::GroupwareCache, calendar::{ ALERT_EMAIL, ALERT_RELATIVE_TO_END, ArchivedDefaultAlert, Calendar, CalendarEvent, CalendarEventData, EVENT_DRAFT, EVENT_HIDE_ATTENDEES, EVENT_INVITE_OTHERS, EVENT_INVITE_SELF, }, scheduling::{ItipMessages, event_create::itip_create, event_update::itip_update}, }; use http_proto::HttpSessionData; use jmap_proto::{ error::set::SetError, method::set::{SetRequest, SetResponse}, object::calendar_event, request::IntoValid, types::state::State, }; use jmap_tools::{JsonPointerHandler, JsonPointerItem, Key, Map, Value}; use std::{borrow::Cow, str::FromStr}; use store::{ ValueKey, ahash::AHashSet, roaring::RoaringBitmap, write::{AlignedBytes, Archive, BatchBuilder, now, serialize::rkyv_deserialize}, }; use trc::AddContext; use types::{ acl::Acl, blob::BlobId, collection::{Collection, SyncCollection}, id::Id, }; pub trait CalendarEventSet: Sync + Send { fn calendar_event_set( &self, request: SetRequest<'_, calendar_event::CalendarEvent>, access_token: &AccessToken, session: &HttpSessionData, ) -> impl Future>> + Send; #[allow(clippy::too_many_arguments)] fn create_calendar_event( &self, cache: &DavResources, batch: &mut BatchBuilder, access_token: &AccessToken, account_id: u32, send_scheduling_messages: bool, can_add_calendars: &Option, js_calendar_event: JSCalendar<'_, Id, BlobId>, updates: Value<'_, JSCalendarProperty, JSCalendarValue>, ) -> impl Future>>>>; } impl CalendarEventSet for Server { async fn calendar_event_set( &self, mut request: SetRequest<'_, calendar_event::CalendarEvent>, access_token: &AccessToken, _session: &HttpSessionData, ) -> trc::Result> { let account_id = request.account_id.document_id(); let cache = self .fetch_dav_resources(access_token, account_id, SyncCollection::Calendar) .await?; let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; let will_destroy = request.unwrap_destroy().into_valid().collect::>(); // Obtain calendarIds let (can_add_calendars, can_delete_calendars, can_modify_calendars) = if access_token.is_shared(account_id) { ( cache .shared_containers(access_token, [Acl::AddItems], true) .into(), cache .shared_containers(access_token, [Acl::RemoveItems], true) .into(), cache .shared_containers(access_token, [Acl::ModifyItems], true) .into(), ) } else { (None, None, None) }; // Process creates let mut batch = BatchBuilder::new(); let send_scheduling_messages = request.arguments.send_scheduling_messages.unwrap_or(false); 'create: for (id, object) in request.unwrap_create() { match self .create_calendar_event( &cache, &mut batch, access_token, account_id, send_scheduling_messages, &can_add_calendars, JSCalendar::default(), object, ) .await? { Ok(document_id) => { response.created(id, document_id); } Err(err) => { response.not_created.append(id, err); continue 'create; } } } // Process updates 'update: for (id, object) in request.unwrap_update().into_valid() { // Make sure id won't be destroyed if will_destroy.contains(&id) { response.not_updated.append(id, SetError::will_destroy()); continue 'update; } else if id.is_synthetic() { response.not_updated.append( id, SetError::invalid_properties() .with_property(JSCalendarProperty::Id) .with_description("Updating synthetic ids is not yet supported."), ); continue 'update; } // Obtain calendar_event card let document_id = id.document_id(); let calendar_event_ = if let Some(calendar_event_) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, )) .await? { calendar_event_ } else { response.not_updated.append(id, SetError::not_found()); continue 'update; }; let calendar_event = calendar_event_ .to_unarchived::() .caused_by(trc::location!())?; let mut new_calendar_event = calendar_event .deserialize::() .caused_by(trc::location!())?; let mut js_calendar_group = std::mem::take(&mut new_calendar_event.data.event).into_jscalendar::(); // Process changes if let Err(err) = update_calendar_event( access_token, object, &mut new_calendar_event, &mut js_calendar_group, ) { response.not_updated.append(id, err); continue 'update; } // Convert JSCalendar to iCalendar let Some(ical) = js_calendar_group.into_icalendar() else { response.not_updated.append( id, SetError::invalid_properties() .with_description("Failed to convert calendar event to iCalendar."), ); continue 'update; }; new_calendar_event.data.event = ical; // Validate UID match ( new_calendar_event.data.event.uids().next(), calendar_event.inner.data.event.uids().next(), ) { (Some(old_uid), Some(new_uid)) if old_uid == new_uid => {} (None, None) | (None, Some(_)) => {} _ => { response.not_updated.append( id, SetError::invalid_properties() .with_property(JSCalendarProperty::Uid) .with_description("You cannot change the UID of a calendar event."), ); continue 'update; } } // Validate new calendarIds for calendar_id in new_calendar_event.added_calendar_ids(calendar_event.inner) { if !cache.has_container_id(&calendar_id) { response.not_updated.append( id, SetError::invalid_properties() .with_property(JSCalendarProperty::CalendarIds) .with_description(format!( "calendarId {} does not exist.", Id::from(calendar_id) )), ); continue 'update; } else if can_add_calendars .as_ref() .is_some_and(|ids| !ids.contains(calendar_id)) { response.not_updated.append( id, SetError::forbidden().with_description(format!( "You are not allowed to add calendar events to calendar {}.", Id::from(calendar_id) )), ); continue 'update; } } // Validate deleted calendarIds if let Some(can_delete_calendars) = &can_delete_calendars { for calendar_id in new_calendar_event.removed_calendar_ids(calendar_event.inner) { if !can_delete_calendars.contains(calendar_id) { response.not_updated.append( id, SetError::forbidden().with_description(format!( "You are not allowed to remove calendar events from calendar {}.", Id::from(calendar_id) )), ); continue 'update; } } } // Validate changed calendarIds if let Some(can_modify_calendars) = &can_modify_calendars { for calendar_id in new_calendar_event.unchanged_calendar_ids(calendar_event.inner) { if !can_modify_calendars.contains(calendar_id) { response.not_updated.append( id, SetError::forbidden().with_description(format!( "You are not allowed to modify calendar {}.", Id::from(calendar_id) )), ); continue 'update; } } } // Check size and quota new_calendar_event.size = new_calendar_event.data.event.size() as u32; if new_calendar_event.size as usize > self.core.groupware.max_ical_size { response.not_updated.append( id, SetError::invalid_properties().with_description(format!( "Event size {} exceeds the maximum allowed size of {} bytes.", new_calendar_event.size, self.core.groupware.max_ical_size )), ); continue 'update; } // Obtain previous alarm let now = now() as i64; let prev_email_alarm = calendar_event.inner.data.next_alarm(now, Tz::Floating); // Build event let mut next_email_alarm = None; new_calendar_event.data = CalendarEventData::new( new_calendar_event.data.event, Tz::Floating, self.core.groupware.max_ical_instances, &mut next_email_alarm, ); // Scheduling let mut itip_messages = None; if send_scheduling_messages && self.core.groupware.itip_enabled && !access_token.emails.is_empty() && access_token.has_permission(Permission::CalendarSchedulingSend) && new_calendar_event.data.event_range_end() > now { let result = if new_calendar_event.schedule_tag.is_some() { let old_ical = rkyv_deserialize(&calendar_event.inner.data.event) .caused_by(trc::location!())?; itip_update( &mut new_calendar_event.data.event, &old_ical, access_token.emails.as_slice(), ) } else { itip_create( &mut new_calendar_event.data.event, access_token.emails.as_slice(), ) }; match result { Ok(messages) => { let mut is_organizer = false; if messages .iter() .map(|r| { is_organizer = r.from_organizer; r.to.len() }) .sum::() < self.core.groupware.itip_outbound_max_recipients { // Only update schedule tag if the user is the organizer if is_organizer { if let Some(schedule_tag) = &mut new_calendar_event.schedule_tag { *schedule_tag += 1; } else { new_calendar_event.schedule_tag = Some(1); } } itip_messages = Some(ItipMessages::new(messages)); } else { response.not_updated.append( id, SetError::invalid_properties() .with_property(JSCalendarProperty::Participants) .with_description(concat!( "The number of scheduling message recipients ", "exceeds the maximum allowed." )), ); continue 'update; } } Err(err) => { if err.is_jmap_error() { response.not_updated.append( id, SetError::invalid_properties() .with_property(JSCalendarProperty::Participants) .with_description(err.to_string()), ); continue 'update; } // Event changed, but there are no iTIP messages to send if let Some(schedule_tag) = &mut new_calendar_event.schedule_tag { *schedule_tag += 1; } } } } // Validate quota let extra_bytes = (new_calendar_event.size as u64) .saturating_sub(u32::from(calendar_event.inner.size) as u64); if extra_bytes > 0 { match self .has_available_quota( &self.get_resource_token(access_token, account_id).await?, extra_bytes, ) .await { Ok(_) => {} Err(err) if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) => { response.not_updated.append(id, SetError::over_quota()); continue 'update; } Err(err) => return Err(err.caused_by(trc::location!())), } } // Update record new_calendar_event .update( access_token, calendar_event, account_id, document_id, &mut batch, ) .caused_by(trc::location!())?; if prev_email_alarm != next_email_alarm { if let Some(prev_alarm) = prev_email_alarm { prev_alarm.delete_task(&mut batch); } if let Some(next_alarm) = next_email_alarm { next_alarm.write_task(&mut batch); } } if let Some(itip_messages) = itip_messages { itip_messages .queue(&mut batch) .caused_by(trc::location!())?; } response.updated.append(id, None); } // Process deletions 'destroy: for id in will_destroy { let document_id = id.document_id(); if !cache.has_item_id(&document_id) { response.not_destroyed.append(id, SetError::not_found()); continue; } else if id.is_synthetic() { response.not_destroyed.append( id, SetError::invalid_properties() .with_property(JSCalendarProperty::Id) .with_description("Deleting synthetic ids is not yet supported."), ); continue; } let Some(calendar_event_) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, )) .await .caused_by(trc::location!())? else { response.not_destroyed.append(id, SetError::not_found()); continue; }; let calendar_event = calendar_event_ .to_unarchived::() .caused_by(trc::location!())?; // Validate ACLs if let Some(can_delete_calendars) = &can_delete_calendars { for name in calendar_event.inner.names.iter() { let parent_id = name.parent_id.to_native(); if !can_delete_calendars.contains(parent_id) { response.not_destroyed.append( id, SetError::forbidden().with_description(format!( "You are not allowed to remove events from calendar {}.", Id::from(parent_id) )), ); continue 'destroy; } } } // Delete event DestroyArchive(calendar_event) .delete_all( access_token, account_id, document_id, send_scheduling_messages, &mut batch, ) .caused_by(trc::location!())?; response.destroyed.push(id); } // Write changes if !batch.is_empty() { let change_id = self .commit_batch(batch) .await .and_then(|ids| ids.last_change_id(account_id)) .caused_by(trc::location!())?; self.notify_task_queue(); response.new_state = State::Exact(change_id).into(); } Ok(response) } async fn create_calendar_event( &self, cache: &DavResources, batch: &mut BatchBuilder, access_token: &AccessToken, account_id: u32, send_scheduling_messages: bool, can_add_calendars: &Option, mut js_calendar_group: JSCalendar<'_, Id, BlobId>, updates: Value<'_, JSCalendarProperty, JSCalendarValue>, ) -> trc::Result>>> { // Process changes let mut event = CalendarEvent::default(); let use_default_alerts = match update_calendar_event( access_token, updates, &mut event, &mut js_calendar_group, ) { Ok(use_default_alerts) => use_default_alerts, Err(err) => { return Ok(Err(err)); } }; // Convert JSCalendar to iCalendar let Some(mut ical) = js_calendar_group.into_icalendar() else { return Ok(Err(SetError::invalid_properties().with_description( "Failed to convert calendar event to iCalendar.", ))); }; // Verify that the calendar ids valid let default_alert_comp_id = ical.components.len(); for name in &event.names { if !cache.has_container_id(&name.parent_id) { return Ok(Err(SetError::invalid_properties() .with_property(JSCalendarProperty::CalendarIds) .with_description(format!( "calendarId {} does not exist.", Id::from(name.parent_id) )))); } else if can_add_calendars .as_ref() .is_some_and(|ids| !ids.contains(name.parent_id)) { return Ok(Err(SetError::forbidden().with_description(format!( "You are not allowed to add calendar events to calendar {}.", Id::from(name.parent_id) )))); } else if let Some(show_without_time) = use_default_alerts && let Some(_calendar) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::Calendar, name.parent_id, )) .await? { ical.components.extend( _calendar .unarchive::() .caused_by(trc::location!())? .default_alerts(access_token, !show_without_time) .map(default_alert_to_ical), ); } } // Add default alarms if ical.components.len() > default_alert_comp_id { let component_ids = default_alert_comp_id as u32..ical.components.len() as u32; for component in &mut ical.components { if component.component_type.is_event_or_todo() && !component.is_recurrence_override() { component.component_ids.extend(component_ids.clone()); } } } // Validate UID if let Err(err) = assert_is_unique_uid(self, account_id, ical.uids().next()).await? { return Ok(Err(err)); } // Check size and quota let size = ical.size(); if size > self.core.groupware.max_ical_size { return Ok(Err(SetError::invalid_properties().with_description( format!( "Event size {} exceeds the maximum allowed size of {} bytes.", size, self.core.groupware.max_ical_size ), ))); } // Build event let mut next_email_alarm = None; event.data = CalendarEventData::new( ical, Tz::Floating, self.core.groupware.max_ical_instances, &mut next_email_alarm, ); event.size = size as u32; // Scheduling let mut itip_messages = None; if send_scheduling_messages && self.core.groupware.itip_enabled && !access_token.emails.is_empty() && access_token.has_permission(Permission::CalendarSchedulingSend) && event.data.event_range_end() > now() as i64 { match itip_create(&mut event.data.event, access_token.emails.as_slice()) { Ok(messages) => { if messages.iter().map(|r| r.to.len()).sum::() < self.core.groupware.itip_outbound_max_recipients { event.schedule_tag = Some(1); itip_messages = Some(ItipMessages::new(messages)); } else { return Ok(Err(SetError::invalid_properties() .with_property(JSCalendarProperty::Participants) .with_description(concat!( "The number of scheduling message recipients ", "exceeds the maximum allowed." )))); } } Err(err) => { if err.is_jmap_error() { return Ok(Err(SetError::invalid_properties() .with_property(JSCalendarProperty::Participants) .with_description(err.to_string()))); } } } } // Validate quota match self .has_available_quota( &self.get_resource_token(access_token, account_id).await?, size as u64, ) .await { Ok(_) => {} Err(err) if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) => { return Ok(Err(SetError::over_quota())); } Err(err) => return Err(err.caused_by(trc::location!())), } // Insert record let document_id = self .store() .assign_document_ids(account_id, Collection::CalendarEvent, 1) .await .caused_by(trc::location!())?; event .insert( access_token, account_id, document_id, next_email_alarm, batch, ) .caused_by(trc::location!())?; if let Some(itip_messages) = itip_messages { itip_messages.queue(batch).caused_by(trc::location!())?; } Ok(Ok(document_id)) } } fn update_calendar_event<'x>( _access_token: &AccessToken, updates: Value<'x, JSCalendarProperty, JSCalendarValue>, event: &mut CalendarEvent, js_calendar_group: &mut JSCalendar<'x, Id, BlobId>, ) -> Result, SetError>> { // Extract event let js_calendar_events = js_calendar_group .0 .as_object_mut() .unwrap() .get_mut(&Key::Property(JSCalendarProperty::Entries)) .unwrap() .as_array_mut() .unwrap(); let js_calendar_event = if let Some(js_calendar_event) = js_calendar_events.first_mut() { js_calendar_event } else { js_calendar_events.push(Value::Object(Map::new())); js_calendar_events.first_mut().unwrap() }; let mut utc_start = None; let mut utc_end = None; let mut use_default_alerts = false; let mut show_without_time = false; let mut entries = js_calendar_event.as_object_mut().unwrap(); for (property, value) in updates.into_expanded_object() { let Key::Property(property) = property else { return Err(SetError::invalid_properties() .with_property(property.to_owned()) .with_description("Invalid property.")); }; match (property, value) { (JSCalendarProperty::IsDraft, Value::Bool(set)) => { if set { event.flags |= EVENT_DRAFT; } else { event.flags &= !EVENT_DRAFT; } } (JSCalendarProperty::MayInviteSelf, Value::Bool(set)) => { if set { event.flags |= EVENT_INVITE_SELF; } else { event.flags &= !EVENT_INVITE_SELF; } } (JSCalendarProperty::MayInviteOthers, Value::Bool(set)) => { if set { event.flags |= EVENT_INVITE_OTHERS; } else { event.flags &= !EVENT_INVITE_OTHERS; } } (JSCalendarProperty::HideAttendees, Value::Bool(set)) => { if set { event.flags |= EVENT_HIDE_ATTENDEES; } else { event.flags &= !EVENT_HIDE_ATTENDEES; } } (JSCalendarProperty::UseDefaultAlerts, Value::Bool(set)) => { use_default_alerts = set; } (JSCalendarProperty::UtcStart, Value::Element(JSCalendarValue::DateTime(start))) => { utc_start = Some(start.timestamp); } (JSCalendarProperty::UtcEnd, Value::Element(JSCalendarValue::DateTime(end))) => { utc_end = Some(end.timestamp); } (JSCalendarProperty::CalendarIds, value) => { patch_parent_ids(&mut event.names, None, value)?; } (JSCalendarProperty::Pointer(pointer), value) => { if matches!( pointer.first(), Some(JsonPointerItem::Key(Key::Property( JSCalendarProperty::CalendarIds ))) ) { let mut pointer = pointer.iter(); pointer.next(); patch_parent_ids(&mut event.names, pointer.next(), value)?; } else if !js_calendar_event.patch_jptr(pointer.iter(), value) { return Err(SetError::invalid_properties() .with_property(JSCalendarProperty::Pointer(pointer)) .with_description("Patch operation failed.")); } entries = js_calendar_event.as_object_mut().unwrap(); } ( property @ (JSCalendarProperty::Id | JSCalendarProperty::BaseEventId | JSCalendarProperty::IsOrigin | JSCalendarProperty::Method), _, ) => { return Err(SetError::invalid_properties() .with_property(property) .with_description("This property is immutable.")); } ( property @ (JSCalendarProperty::IsDraft | JSCalendarProperty::MayInviteSelf | JSCalendarProperty::MayInviteOthers | JSCalendarProperty::HideAttendees | JSCalendarProperty::UseDefaultAlerts | JSCalendarProperty::UtcStart | JSCalendarProperty::UtcEnd), _, ) => { return Err(SetError::invalid_properties() .with_property(property) .with_description("Invalid value.")); } ( property @ (JSCalendarProperty::Locations | JSCalendarProperty::Participants), Value::Object(values), ) => { for (_, value) in values.iter() { if let Some(values) = value .as_object_and_get(&Key::Property(JSCalendarProperty::Links)) .and_then(|v| v.as_object()) { for (_, value) in values.iter() { if value.as_object().is_some_and(|v| { v.keys() .any(|k| matches!(k, Key::Property(JSCalendarProperty::BlobId))) }) { return Err(SetError::invalid_properties() .with_property(property) .with_description("blobIds in links is not supported.")); } } } } entries.insert(property, Value::Object(values)); } (property, value) => { if let (JSCalendarProperty::ShowWithoutTime, Value::Bool(set)) = (&property, &value) { show_without_time = *set; } entries.insert(property, value); } } } // Validate UTC start/end if let (Some(mut start), Some(mut end)) = (utc_start, utc_end) { if start >= end { return Err(SetError::invalid_properties() .with_properties([JSCalendarProperty::UtcStart, JSCalendarProperty::UtcEnd]) .with_description("utcStart must be before utcEnd.")); } if let Some(timezone) = entries .get(&Key::Property(JSCalendarProperty::TimeZone)) .and_then(|v| v.as_str()) .and_then(|tz| Tz::from_str(tz.as_ref()).ok()) { if let Some(dt_start) = DateTime::from_timestamp(start, 0).map(|dt| dt.with_timezone(&timezone)) { start = dt_start.naive_local().and_utc().timestamp(); } if let Some(dt_end) = DateTime::from_timestamp(end, 0).map(|dt| dt.with_timezone(&timezone)) { end = dt_end.naive_local().and_utc().timestamp(); } } else { entries.insert( Key::Property(JSCalendarProperty::TimeZone), Value::Str(Cow::Borrowed("Etc/UTC")), ); } entries.insert( Key::Property(JSCalendarProperty::Start), Value::Element(JSCalendarValue::DateTime(JSCalendarDateTime::new( start, true, ))), ); entries.insert( Key::Property(JSCalendarProperty::Duration), Value::Element(JSCalendarValue::Duration(ICalendarDuration::from_seconds( end - start, ))), ); } else if utc_start.is_some() || utc_end.is_some() { return Err(SetError::invalid_properties() .with_properties([JSCalendarProperty::UtcStart, JSCalendarProperty::UtcEnd]) .with_description("Both utcStart and utcEnd must be provided.")); } // Make sure the calendar_event belongs to at least one calendar if event.names.is_empty() { return Err(SetError::invalid_properties() .with_property(JSCalendarProperty::CalendarIds) .with_description("Event has to belong to at least one calendar.")); } Ok(use_default_alerts.then_some(show_without_time)) } fn patch_parent_ids( current: &mut Vec, patch: Option<&JsonPointerItem>>, update: Value<'_, JSCalendarProperty, JSCalendarValue>, ) -> Result<(), SetError>> { match (patch, update) { ( Some(JsonPointerItem::Key(Key::Property(JSCalendarProperty::IdValue(id)))), Value::Bool(false) | Value::Null, ) => { let id = id.document_id(); current.retain(|name| name.parent_id != id); Ok(()) } ( Some(JsonPointerItem::Key(Key::Property(JSCalendarProperty::IdValue(id)))), Value::Bool(true), ) => { let id = id.document_id(); if !current.iter().any(|name| name.parent_id == id) { current.push(DavName::new_with_rand_name(id)); } Ok(()) } (None, Value::Object(object)) => { let mut new_ids = object .into_expanded_boolean_set() .filter_map(|id| { if let Key::Property(JSCalendarProperty::IdValue(id)) = id { Some(id.document_id()) } else { None } }) .collect::>(); current.retain(|name| new_ids.remove(&name.parent_id)); for id in new_ids { current.push(DavName::new_with_rand_name(id)); } Ok(()) } _ => Err(SetError::invalid_properties() .with_property(JSCalendarProperty::CalendarIds) .with_description("Invalid patch operation for calendarIds.")), } } fn default_alert_to_ical(alert: &ArchivedDefaultAlert) -> ICalendarComponent { let flags = alert.flags.to_native(); ICalendarComponent { component_type: ICalendarComponentType::VAlarm, entries: vec![ ICalendarEntry::new(ICalendarProperty::Action).with_value( if flags & ALERT_EMAIL != 0 { ICalendarValue::Action(ICalendarAction::Email) } else { ICalendarValue::Action(ICalendarAction::Display) }, ), ICalendarEntry::new(ICalendarProperty::Trigger) .with_param_opt((flags & ALERT_RELATIVE_TO_END != 0).then_some( ICalendarParameter::related(ICalendarParameterValue::Related( ICalendarRelated::End, )), )) .with_value(ICalendarValue::Duration(alert.offset.to_native())), ], component_ids: vec![], } } ================================================ FILE: crates/jmap/src/calendar_event_notification/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::changes::state::JmapCacheState; use calcard::{ icalendar::{ArchivedICalendarProperty, ICalendar}, jscalendar::import::ConversionOptions, }; use common::{Server, auth::AccessToken}; use groupware::{ cache::GroupwareCache, calendar::{ ArchivedChangedBy, CalendarEventNotification, EVENT_NOTIFICATION_IS_CHANGE, EVENT_NOTIFICATION_IS_DRAFT, }, }; use jmap_proto::{ method::get::GetRequest, object::calendar_event_notification::{ self, CalendarEventNotificationGetResponse, CalendarEventNotificationObject, CalendarEventNotificationProperty, CalendarEventNotificationType, PersonObject, }, types::date::UTCDate, }; use store::{ValueKey, write::{AlignedBytes, Archive, serialize::rkyv_deserialize}}; use trc::AddContext; use types::{ blob::BlobId, collection::{Collection, SyncCollection}, id::Id, }; pub trait CalendarEventNotificationGet: Sync + Send { fn calendar_event_notification_get( &self, request: GetRequest, access_token: &AccessToken, ) -> impl Future> + Send; } impl CalendarEventNotificationGet for Server { async fn calendar_event_notification_get( &self, mut request: GetRequest, access_token: &AccessToken, ) -> trc::Result { let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ CalendarEventNotificationProperty::Id, CalendarEventNotificationProperty::Created, CalendarEventNotificationProperty::Type, CalendarEventNotificationProperty::ChangedBy, ]); let account_id = request.account_id.document_id(); let cache = self .fetch_dav_resources( access_token, account_id, SyncCollection::CalendarEventNotification, ) .await .caused_by(trc::location!())?; let ids = if let Some(ids) = ids { ids } else { cache .document_ids(false) .take(self.core.jmap.get_max_objects) .map(Into::into) .collect::>() }; let mut response = CalendarEventNotificationGetResponse { account_id: request.account_id.into(), state: cache.get_state(false).into(), list: Vec::with_capacity(ids.len()), not_found: vec![], }; for id in ids { // Obtain the event object let document_id = id.document_id(); let _event = if let Some(event) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEventNotification, document_id, )) .await? { event } else { response.not_found.push(id); continue; }; let event = _event .unarchive::() .caused_by(trc::location!())?; let mut result = CalendarEventNotificationObject { id, ..Default::default() }; for property in &properties { match property { CalendarEventNotificationProperty::Id => {} CalendarEventNotificationProperty::Created => { result.created = Some(UTCDate::from_timestamp(event.created.to_native())); } CalendarEventNotificationProperty::CalendarEventId => { result.calendar_event_id = event.event_id.as_ref().map(|id| id.to_native().into()); } CalendarEventNotificationProperty::ChangedBy => { let mut changed_by = PersonObject::default(); match &event.changed_by { ArchivedChangedBy::PrincipalId(id) => { if let Ok(token) = self.get_access_token(id.to_native()).await { changed_by.name = token.description.clone().unwrap_or_default(); changed_by.email = token.emails.first().cloned(); } changed_by.principal_id = Some(id.to_native().into()); } ArchivedChangedBy::CalendarAddress(email) => { changed_by.email = Some(email.to_string()); changed_by.calendar_address = Some(format!("mailto:{email}")); } } result.changed_by = Some(changed_by); } CalendarEventNotificationProperty::Comment => { result.comment = event .event .components .iter() .filter(|c| c.component_type.is_scheduling_object()) .flat_map(|c| c.entries.iter()) .find(|e| matches!(e.name, ArchivedICalendarProperty::Comment)) .and_then(|e| e.values.first().and_then(|v| v.as_text())) .map(|v| v.to_string()); } CalendarEventNotificationProperty::Type => { result.notification_type = Some(if event.flags & EVENT_NOTIFICATION_IS_CHANGE != 0 { CalendarEventNotificationType::Updated } else if !event.event.components.is_empty() { CalendarEventNotificationType::Created } else { CalendarEventNotificationType::Destroyed }); } CalendarEventNotificationProperty::IsDraft => { result.is_draft = Some(event.flags & EVENT_NOTIFICATION_IS_DRAFT != 0); } CalendarEventNotificationProperty::Event => { if event.flags & EVENT_NOTIFICATION_IS_CHANGE == 0 && result.event.is_none() { let js_event = rkyv_deserialize::<_, ICalendar>(&event.event) .caused_by(trc::location!())? .into_jscalendar_with_opt::( ConversionOptions::default() .include_ical_components(false) .return_first(true), ); result.event = js_event.into(); } } CalendarEventNotificationProperty::EventPatch => { if event.flags & EVENT_NOTIFICATION_IS_CHANGE != 0 && result.event_patch.is_none() { let js_event = rkyv_deserialize::<_, ICalendar>(&event.event) .caused_by(trc::location!())? .into_jscalendar_with_opt::( ConversionOptions::default() .include_ical_components(false) .return_first(true), ); result.event_patch = js_event.into(); } } } } response.list.push(result); } Ok(response) } } ================================================ FILE: crates/jmap/src/calendar_event_notification/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod get; pub mod query; pub mod set; ================================================ FILE: crates/jmap/src/calendar_event_notification/query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{api::query::QueryResponseBuilder, changes::state::JmapCacheState}; use common::{Server, auth::AccessToken}; use groupware::cache::GroupwareCache; use jmap_proto::{ method::query::{Filter, QueryRequest, QueryResponse}, object::calendar_event_notification::{ CalendarEventNotification, CalendarEventNotificationComparator, CalendarEventNotificationFilter, }, request::IntoValid, }; use store::{ IterateParams, U32_LEN, U64_LEN, ValueKey, ahash::AHashSet, roaring::RoaringBitmap, search::{SearchFilter, SearchQuery}, write::{IndexPropertyClass, SearchIndex, ValueClass, key::DeserializeBigEndian}, }; use trc::AddContext; use types::{ collection::{Collection, SyncCollection}, field::CalendarNotificationField, }; pub trait CalendarEventNotificationQuery: Sync + Send { fn calendar_event_notification_query( &self, request: QueryRequest, access_token: &AccessToken, ) -> impl Future> + Send; } struct Notification { document_id: u32, created: u64, event_id: u32, } impl CalendarEventNotificationQuery for Server { async fn calendar_event_notification_query( &self, mut request: QueryRequest, access_token: &AccessToken, ) -> trc::Result { let account_id = request.account_id.document_id(); let mut filters = Vec::with_capacity(request.filter.len()); let cache = self .fetch_dav_resources( access_token, account_id, SyncCollection::CalendarEventNotification, ) .await?; let mut notifications = Vec::with_capacity(16); let mut document_ids = RoaringBitmap::new(); self.store() .iterate( IterateParams::new( ValueKey { account_id, collection: Collection::CalendarEventNotification.into(), document_id: 0, class: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: CalendarNotificationField::CreatedToId.into(), value: 0, }), }, ValueKey { account_id, collection: Collection::CalendarEventNotification.into(), document_id: 0, class: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: CalendarNotificationField::CreatedToId.into(), value: u64::MAX, }), }, ) .ascending(), |key, value| { let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; notifications.push(Notification { document_id, created: key.deserialize_be_u64(key.len() - U32_LEN - U64_LEN)?, event_id: value.deserialize_be_u32(0)?, }); document_ids.insert(document_id); Ok(true) }, ) .await .caused_by(trc::location!())?; for cond in std::mem::take(&mut request.filter) { match cond { Filter::Property(cond) => match cond { CalendarEventNotificationFilter::Before(before) => { let before = before.timestamp() as u64; filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( notifications .iter() .filter_map(|n| (n.created < before).then_some(n.document_id)), ))) } CalendarEventNotificationFilter::After(after) => { let after = after.timestamp() as u64; filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( notifications .iter() .filter_map(|n| (n.created > after).then_some(n.document_id)), ))) } CalendarEventNotificationFilter::CalendarEventIds(ids) => { let ids = ids .into_valid() .map(|id| id.document_id()) .collect::>(); filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( notifications .iter() .filter_map(|n| ids.contains(&n.event_id).then_some(n.document_id)), ))) } unsupported => { return Err(trc::JmapEvent::UnsupportedFilter .into_err() .details(unsupported.into_string())); } }, Filter::And => { filters.push(SearchFilter::And); } Filter::Or => { filters.push(SearchFilter::Or); } Filter::Not => { filters.push(SearchFilter::Not); } Filter::Close => { filters.push(SearchFilter::End); } } } // Parse sort criteria let mut is_ascending = true; for comparator in request.sort.take().unwrap_or_default() { match comparator.property { CalendarEventNotificationComparator::Created => { is_ascending = comparator.is_ascending; } CalendarEventNotificationComparator::_T(unsupported) => { return Err(trc::JmapEvent::UnsupportedSort .into_err() .details(unsupported)); } }; } if !is_ascending { notifications.reverse(); } let results = SearchQuery::new(SearchIndex::InMemory) .with_filters(filters) .with_mask(document_ids) .filter() .into_bitmap(); let mut response = QueryResponseBuilder::new( results.len() as usize, self.core.jmap.query_max_results, cache.get_state(false), &request, ); if !results.is_empty() { let results = results.into_iter().collect::>(); for notification in notifications { if results.contains(¬ification.document_id) && !response.add(0, notification.document_id) { break; } } } response.build() } } ================================================ FILE: crates/jmap/src/calendar_event_notification/set.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{Server, auth::AccessToken}; use groupware::{DestroyArchive, cache::GroupwareCache, calendar::CalendarEventNotification}; use http_proto::HttpSessionData; use jmap_proto::{ error::set::SetError, method::set::{SetRequest, SetResponse}, object::calendar_event_notification, request::IntoValid, types::state::State, }; use store::{ValueKey, write::{AlignedBytes, Archive, BatchBuilder}}; use trc::AddContext; use types::collection::{Collection, SyncCollection}; pub trait CalendarEventNotificationSet: Sync + Send { fn calendar_event_notification_set( &self, request: SetRequest<'_, calendar_event_notification::CalendarEventNotification>, access_token: &AccessToken, session: &HttpSessionData, ) -> impl Future< Output = trc::Result>, > + Send; } impl CalendarEventNotificationSet for Server { async fn calendar_event_notification_set( &self, mut request: SetRequest<'_, calendar_event_notification::CalendarEventNotification>, access_token: &AccessToken, _session: &HttpSessionData, ) -> trc::Result> { let account_id = request.account_id.document_id(); let cache = self .fetch_dav_resources( access_token, account_id, SyncCollection::CalendarEventNotification, ) .await?; let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; let mut batch = BatchBuilder::new(); for (id, _) in request.unwrap_create() { response.not_created.append( id, SetError::forbidden().with_description("Cannot create event notifications."), ); } // Process updates for (id, _) in request.unwrap_update().into_valid() { response.not_updated.append( id, SetError::forbidden().with_description("Cannot update event notifications."), ); } // Process deletions for id in request.unwrap_destroy().into_valid() { let document_id = id.document_id(); if !cache.has_item_id(&document_id) { response.not_destroyed.append(id, SetError::not_found()); continue; }; let _event = if let Some(event) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEventNotification, document_id, )) .await? { event } else { response.not_destroyed.append(id, SetError::not_found()); continue; }; let event = _event .to_unarchived::() .caused_by(trc::location!())?; DestroyArchive(event) .delete(access_token, account_id, document_id, &mut batch) .caused_by(trc::location!())?; response.destroyed.push(id); } // Write changes if !batch.is_empty() { let change_id = self .commit_batch(batch) .await .and_then(|ids| ids.last_change_id(account_id)) .caused_by(trc::location!())?; response.new_state = State::Exact(change_id).into(); } Ok(response) } } ================================================ FILE: crates/jmap/src/changes/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{api::auth::JmapAuthorization, changes::state::JmapCacheState}; use common::{Server, auth::AccessToken}; use email::cache::MessageCacheFetch; use groupware::cache::GroupwareCache; use jmap_proto::{ method::changes::{ChangesRequest, ChangesResponse}, object::{JmapObject, NullObject, mailbox::MailboxProperty}, request::method::MethodObject, response::{ChangesResponseMethod, ResponseMethod}, types::state::State, }; use std::future::Future; use store::query::log::{Change, Query}; use trc::AddContext; use types::collection::{Collection, SyncCollection}; pub trait ChangesLookup: Sync + Send { fn changes( &self, request: ChangesRequest, object: MethodObject, access_token: &AccessToken, ) -> impl Future> + Send; } pub struct IntermediateChangesResponse { pub response: ChangesResponse, pub object: MethodObject, pub only_container_changes: bool, } impl ChangesLookup for Server { async fn changes( &self, request: ChangesRequest, object: MethodObject, access_token: &AccessToken, ) -> trc::Result { // Map collection and validate ACLs let (collection, is_container) = match object { MethodObject::Email => { access_token.assert_has_access(request.account_id, Collection::Email)?; (SyncCollection::Email, false) } MethodObject::Mailbox => { access_token.assert_has_access(request.account_id, Collection::Mailbox)?; (SyncCollection::Email, true) } MethodObject::Thread => { access_token.assert_has_access(request.account_id, Collection::Email)?; (SyncCollection::Thread, true) } MethodObject::Identity => { access_token.assert_is_member(request.account_id)?; (SyncCollection::Identity, false) } MethodObject::EmailSubmission => { access_token.assert_is_member(request.account_id)?; (SyncCollection::EmailSubmission, false) } MethodObject::AddressBook => { access_token.assert_has_access(request.account_id, Collection::AddressBook)?; (SyncCollection::AddressBook, true) } MethodObject::ContactCard => { access_token.assert_has_access(request.account_id, Collection::ContactCard)?; (SyncCollection::AddressBook, false) } MethodObject::FileNode => { access_token.assert_has_access(request.account_id, Collection::FileNode)?; (SyncCollection::FileNode, false) } MethodObject::Calendar => { access_token.assert_has_access(request.account_id, Collection::Calendar)?; (SyncCollection::Calendar, true) } MethodObject::CalendarEvent => { access_token.assert_has_access(request.account_id, Collection::CalendarEvent)?; (SyncCollection::Calendar, false) } MethodObject::CalendarEventNotification => { access_token.assert_is_member(request.account_id)?; (SyncCollection::CalendarEventNotification, false) } MethodObject::ShareNotification => { access_token.assert_is_member(request.account_id)?; (SyncCollection::ShareNotification, false) } _ => { return Err(trc::JmapEvent::CannotCalculateChanges.into_err()); } }; let max_changes = std::cmp::min( request .max_changes .filter(|n| *n != 0) .unwrap_or(usize::MAX), self.core.jmap.changes_max_results.unwrap_or(usize::MAX), ); let mut response: ChangesResponse = ChangesResponse { account_id: request.account_id, old_state: request.since_state.clone(), new_state: State::Initial, has_more_changes: false, created: vec![], updated: vec![], destroyed: vec![], updated_properties: None, }; let account_id = request.account_id.document_id(); let (items_sent, changelog) = match &request.since_state { State::Initial => { let changelog = self .store() .changes(account_id, collection.into(), Query::All) .await?; if changelog.changes.is_empty() && changelog.from_change_id == 0 { return Ok(IntermediateChangesResponse { response, object, only_container_changes: false, }); } (0, changelog) } State::Exact(change_id) => { let last_state = match collection { SyncCollection::Calendar | SyncCollection::AddressBook => self .fetch_dav_resources(access_token, account_id, collection) .await .caused_by(trc::location!())? .get_state(is_container) .into(), SyncCollection::Email => self .get_cached_messages(account_id) .await? .get_state(is_container) .into(), _ => None, }; if let Some(last_state) = last_state { response.new_state = last_state; if response.new_state == State::Exact(*change_id) { return Ok(IntermediateChangesResponse { response, object, only_container_changes: false, }); } } ( 0, self.store() .changes(account_id, collection.into(), Query::Since(*change_id)) .await?, ) } State::Intermediate(intermediate_state) => { let changelog = self .store() .changes( account_id, collection.into(), Query::RangeInclusive(intermediate_state.from_id, intermediate_state.to_id), ) .await?; if (is_container && intermediate_state.items_sent >= changelog.total_container_changes()) || (!is_container && intermediate_state.items_sent >= changelog.total_item_changes()) { ( 0, self.store() .changes( account_id, collection.into(), Query::Since(intermediate_state.to_id), ) .await?, ) } else { (intermediate_state.items_sent, changelog) } } }; if (changelog.is_truncated || changelog.from_change_id == 0) && request.since_state != State::Initial { return Err(trc::JmapEvent::CannotCalculateChanges.into_err().details( if changelog.is_truncated { "Change log is truncated" } else { "Since state is invalid" }, )); } let mut changes = changelog .changes .into_iter() .filter(|change| { (is_container && change.is_container_change()) || (!is_container && change.is_item_change()) }) .skip(items_sent) .peekable(); let mut items_changed = false; for change in (&mut changes).take(max_changes) { match change { Change::InsertContainer(item) | Change::InsertItem(item) => { response.created.push(item.into()); } Change::UpdateContainer(item) | Change::UpdateItem(item) => { response.updated.push(item.into()); items_changed = true; } Change::DeleteContainer(item) | Change::DeleteItem(item) => { response.destroyed.push(item.into()); } Change::UpdateContainerProperty(item) => { response.updated.push(item.into()); } }; } let change_id = (if is_container { changelog.container_change_id } else { changelog.item_change_id }) .unwrap_or(changelog.to_change_id); response.has_more_changes = changes.peek().is_some(); if response.has_more_changes { response.new_state = State::new_intermediate( changelog.from_change_id, change_id, items_sent + max_changes, ); } else if response.new_state == State::Initial { response.new_state = State::new_exact(change_id) } Ok(IntermediateChangesResponse { only_container_changes: is_container && !response.updated.is_empty() && !items_changed, response, object, }) } } impl IntermediateChangesResponse { pub fn into_method_response(self) -> ResponseMethod<'static> { ResponseMethod::Changes(match self.object { MethodObject::Email => ChangesResponseMethod::Email(transmute_response(self.response)), MethodObject::Mailbox => { let mut response = transmute_response(self.response); if self.only_container_changes { response.updated_properties = vec![ MailboxProperty::TotalEmails.into(), MailboxProperty::UnreadEmails.into(), MailboxProperty::TotalThreads.into(), MailboxProperty::UnreadThreads.into(), ] .into(); } ChangesResponseMethod::Mailbox(response) } MethodObject::Thread => { ChangesResponseMethod::Thread(transmute_response(self.response)) } MethodObject::Identity => { ChangesResponseMethod::Identity(transmute_response(self.response)) } MethodObject::EmailSubmission => { ChangesResponseMethod::EmailSubmission(transmute_response(self.response)) } MethodObject::AddressBook => { ChangesResponseMethod::AddressBook(transmute_response(self.response)) } MethodObject::ContactCard => { ChangesResponseMethod::ContactCard(transmute_response(self.response)) } MethodObject::FileNode => { ChangesResponseMethod::FileNode(transmute_response(self.response)) } MethodObject::Calendar => { ChangesResponseMethod::Calendar(transmute_response(self.response)) } MethodObject::CalendarEvent => { ChangesResponseMethod::CalendarEvent(transmute_response(self.response)) } MethodObject::CalendarEventNotification => { ChangesResponseMethod::CalendarEventNotification(transmute_response(self.response)) } MethodObject::ShareNotification => { ChangesResponseMethod::ShareNotification(transmute_response(self.response)) } MethodObject::ParticipantIdentity | MethodObject::Core | MethodObject::Blob | MethodObject::PushSubscription | MethodObject::SearchSnippet | MethodObject::VacationResponse | MethodObject::SieveScript | MethodObject::Principal | MethodObject::Quota => unreachable!(), }) } } fn transmute_response(response: ChangesResponse) -> ChangesResponse { ChangesResponse { account_id: response.account_id, old_state: response.old_state, new_state: response.new_state, has_more_changes: response.has_more_changes, created: response.created, updated: response.updated, destroyed: response.destroyed, updated_properties: None, } } ================================================ FILE: crates/jmap/src/changes/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod get; pub mod query; pub mod state; ================================================ FILE: crates/jmap/src/changes/query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::get::ChangesLookup; use crate::{ api::request::set_account_id_if_missing, calendar_event::query::CalendarEventQuery, calendar_event_notification::query::CalendarEventNotificationQuery, contact::query::ContactCardQuery, email::query::EmailQuery, file::query::FileNodeQuery, mailbox::query::MailboxQuery, share_notification::query::ShareNotificationQuery, sieve::query::SieveScriptQuery, submission::query::EmailSubmissionQuery, }; use common::{Server, auth::AccessToken}; use jmap_proto::{ method::{ changes::{ChangesRequest, ChangesResponse}, query_changes::{AddedItem, QueryChangesRequest, QueryChangesResponse}, }, object::{JmapObject, NullObject}, request::{QueryChangesRequestMethod, method::MethodObject}, }; use std::future::Future; pub trait QueryChanges: Sync + Send { fn query_changes( &self, request: QueryChangesRequestMethod, access_token: &AccessToken, ) -> impl Future> + Send; } impl QueryChanges for Server { async fn query_changes( &self, request: QueryChangesRequestMethod, access_token: &AccessToken, ) -> trc::Result { let mut response; let mut is_mutable = true; let results; let changes; let has_changes; let up_to_id; match request { QueryChangesRequestMethod::Email(mut request) => { // Query changes set_account_id_if_missing(&mut request.account_id, access_token); changes = self .changes( build_changes_request(&request), MethodObject::Email, access_token, ) .await? .response; let calculate_total = request.calculate_total.unwrap_or(false); has_changes = changes.has_changes(); response = build_query_changes_response(&request, &changes); if !has_changes && !calculate_total { return Ok(response); } up_to_id = request.up_to_id; is_mutable = request.filter.iter().any(|f| !f.is_immutable()) || request .sort .as_ref() .is_some_and(|sort| sort.iter().any(|s| !s.is_immutable())); results = self.email_query(request.into(), access_token).await?; } QueryChangesRequestMethod::Mailbox(mut request) => { // Query changes set_account_id_if_missing(&mut request.account_id, access_token); changes = self .changes( build_changes_request(&request), MethodObject::Mailbox, access_token, ) .await? .response; let calculate_total = request.calculate_total.unwrap_or(false); has_changes = changes.has_changes(); response = build_query_changes_response(&request, &changes); if !has_changes && !calculate_total { return Ok(response); } up_to_id = request.up_to_id; results = self.mailbox_query(request.into(), access_token).await?; } QueryChangesRequestMethod::EmailSubmission(mut request) => { // Query changes set_account_id_if_missing(&mut request.account_id, access_token); changes = self .changes( build_changes_request(&request), MethodObject::EmailSubmission, access_token, ) .await? .response; let calculate_total = request.calculate_total.unwrap_or(false); has_changes = changes.has_changes(); response = build_query_changes_response(&request, &changes); if !has_changes && !calculate_total { return Ok(response); } up_to_id = request.up_to_id; results = self.email_submission_query(request.into()).await?; } QueryChangesRequestMethod::Sieve(mut request) => { // Query changes set_account_id_if_missing(&mut request.account_id, access_token); changes = self .changes( build_changes_request(&request), MethodObject::SieveScript, access_token, ) .await? .response; let calculate_total = request.calculate_total.unwrap_or(false); has_changes = changes.has_changes(); response = build_query_changes_response(&request, &changes); if !has_changes && !calculate_total { return Ok(response); } up_to_id = request.up_to_id; results = self.sieve_script_query(request.into()).await?; } QueryChangesRequestMethod::ContactCard(mut request) => { // Query changes set_account_id_if_missing(&mut request.account_id, access_token); changes = self .changes( build_changes_request(&request), MethodObject::ContactCard, access_token, ) .await? .response; let calculate_total = request.calculate_total.unwrap_or(false); has_changes = changes.has_changes(); response = build_query_changes_response(&request, &changes); if !has_changes && !calculate_total { return Ok(response); } up_to_id = request.up_to_id; results = self .contact_card_query(request.into(), access_token) .await?; } QueryChangesRequestMethod::FileNode(mut request) => { // Query changes set_account_id_if_missing(&mut request.account_id, access_token); changes = self .changes( build_changes_request(&request), MethodObject::FileNode, access_token, ) .await? .response; let calculate_total = request.calculate_total.unwrap_or(false); has_changes = changes.has_changes(); response = build_query_changes_response(&request, &changes); if !has_changes && !calculate_total { return Ok(response); } up_to_id = request.up_to_id; results = self.file_node_query(request.into(), access_token).await?; } QueryChangesRequestMethod::CalendarEvent(mut request) => { // Query changes set_account_id_if_missing(&mut request.account_id, access_token); changes = self .changes( build_changes_request(&request), MethodObject::CalendarEvent, access_token, ) .await? .response; let calculate_total = request.calculate_total.unwrap_or(false); has_changes = changes.has_changes(); response = build_query_changes_response(&request, &changes); if !has_changes && !calculate_total { return Ok(response); } up_to_id = request.up_to_id; results = self .calendar_event_query(request.into(), access_token) .await?; } QueryChangesRequestMethod::CalendarEventNotification(mut request) => { // Query changes set_account_id_if_missing(&mut request.account_id, access_token); changes = self .changes( build_changes_request(&request), MethodObject::CalendarEventNotification, access_token, ) .await? .response; let calculate_total = request.calculate_total.unwrap_or(false); has_changes = changes.has_changes(); response = build_query_changes_response(&request, &changes); if !has_changes && !calculate_total { return Ok(response); } up_to_id = request.up_to_id; results = self .calendar_event_notification_query(request.into(), access_token) .await?; } QueryChangesRequestMethod::ShareNotification(mut request) => { // Query changes set_account_id_if_missing(&mut request.account_id, access_token); changes = self .changes( build_changes_request(&request), MethodObject::ShareNotification, access_token, ) .await? .response; let calculate_total = request.calculate_total.unwrap_or(false); has_changes = changes.has_changes(); response = build_query_changes_response(&request, &changes); if !has_changes && !calculate_total { return Ok(response); } up_to_id = request.up_to_id; results = self.share_notification_query(request.into()).await?; } QueryChangesRequestMethod::Principal(_) => { return Err(trc::JmapEvent::CannotCalculateChanges.into_err()); } QueryChangesRequestMethod::Quota(_) => { return Err(trc::JmapEvent::CannotCalculateChanges.into_err()); } } if has_changes { if is_mutable { for (index, id) in results.ids.into_iter().enumerate() { if changes.created.contains(&id) || changes.updated.contains(&id) { response.added.push(AddedItem::new(id, index)); } } response.removed = changes.updated; } else { for (index, id) in results.ids.into_iter().enumerate() { if changes.created.contains(&id) { response.added.push(AddedItem::new(id, index)); } if matches!(up_to_id, Some(up_to_id) if up_to_id == id) { break; } } } if !changes.destroyed.is_empty() { response.removed.extend(changes.destroyed); } } response.total = results.total; Ok(response) } } fn build_changes_request(req: &QueryChangesRequest) -> ChangesRequest { ChangesRequest { account_id: req.account_id, since_state: req.since_query_state.clone(), max_changes: req.max_changes, } } fn build_query_changes_response( req: &QueryChangesRequest, changes: &ChangesResponse, ) -> QueryChangesResponse { QueryChangesResponse { account_id: req.account_id, old_query_state: changes.old_state.clone(), new_query_state: changes.new_state.clone(), total: None, removed: vec![], added: vec![], } } ================================================ FILE: crates/jmap/src/changes/state.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{DavResources, MessageStoreCache, Server}; use jmap_proto::types::state::State; use std::future::Future; use trc::AddContext; use types::collection::SyncCollection; pub trait StateManager: Sync + Send { fn get_state( &self, account_id: u32, collection: SyncCollection, ) -> impl Future> + Send; fn assert_state( &self, account_id: u32, collection: SyncCollection, if_in_state: &Option, ) -> impl Future> + Send; } pub trait JmapCacheState: Sync + Send { fn get_state(&self, is_container: bool) -> State; fn assert_state(&self, is_container: bool, if_in_state: &Option) -> trc::Result { let old_state: State = self.get_state(is_container); if let Some(if_in_state) = if_in_state && &old_state != if_in_state { return Err(trc::JmapEvent::StateMismatch.into_err()); } Ok(old_state) } } impl StateManager for Server { async fn get_state(&self, account_id: u32, collection: SyncCollection) -> trc::Result { self.core .storage .data .get_last_change_id(account_id, collection.into()) .await .caused_by(trc::location!()) .map(State::from) } async fn assert_state( &self, account_id: u32, collection: SyncCollection, if_in_state: &Option, ) -> trc::Result { let old_state: State = self.get_state(account_id, collection).await?; if let Some(if_in_state) = if_in_state && &old_state != if_in_state { return Err(trc::JmapEvent::StateMismatch.into_err()); } Ok(old_state) } } impl JmapCacheState for MessageStoreCache { fn get_state(&self, is_container: bool) -> State { if is_container { State::from(self.mailboxes.change_id) } else { State::from(self.emails.change_id) } } } impl JmapCacheState for DavResources { fn get_state(&self, is_container: bool) -> State { if is_container { State::from(self.container_change_id) } else { State::from(self.item_change_id) } } } ================================================ FILE: crates/jmap/src/contact/copy.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{changes::state::JmapCacheState, contact::set::ContactCardSet}; use common::{Server, auth::AccessToken}; use groupware::{cache::GroupwareCache, contact::ContactCard}; use http_proto::HttpSessionData; use jmap_proto::{ error::set::SetError, method::{ copy::{CopyRequest, CopyResponse}, set::SetRequest, }, object::contact, request::{ Call, IntoValid, MaybeInvalid, RequestMethod, SetRequestMethod, method::{MethodFunction, MethodName, MethodObject}, reference::MaybeResultReference, }, types::state::State, }; use store::{ValueKey, roaring::RoaringBitmap, write::{AlignedBytes, Archive, BatchBuilder}}; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, }; use utils::map::vec_map::VecMap; pub trait JmapContactCardCopy: Sync + Send { fn contact_card_copy<'x>( &self, request: CopyRequest<'x, contact::ContactCard>, access_token: &AccessToken, next_call: &mut Option>>, session: &HttpSessionData, ) -> impl Future>> + Send; } impl JmapContactCardCopy for Server { async fn contact_card_copy<'x>( &self, request: CopyRequest<'x, contact::ContactCard>, access_token: &AccessToken, next_call: &mut Option>>, _session: &HttpSessionData, ) -> trc::Result> { let account_id = request.account_id.document_id(); let from_account_id = request.from_account_id.document_id(); if account_id == from_account_id { return Err(trc::JmapEvent::InvalidArguments .into_err() .details("From accountId is equal to fromAccountId")); } let cache = self .fetch_dav_resources(access_token, account_id, SyncCollection::AddressBook) .await .caused_by(trc::location!())?; let old_state = cache.assert_state(false, &request.if_in_state)?; let mut response = CopyResponse { from_account_id: request.from_account_id, account_id: request.account_id, new_state: old_state.clone(), old_state, created: VecMap::with_capacity(request.create.len()), not_created: VecMap::new(), }; let from_cache = self .fetch_dav_resources(access_token, from_account_id, SyncCollection::AddressBook) .await .caused_by(trc::location!())?; let from_contact_ids = if access_token.is_member(from_account_id) { from_cache.document_ids(false).collect::() } else { from_cache.shared_items(access_token, [Acl::ReadItems], true) }; let can_add_address_books = if access_token.is_shared(account_id) { cache .shared_containers(access_token, [Acl::AddItems], true) .into() } else { None }; let on_success_delete = request.on_success_destroy_original.unwrap_or(false); let mut destroy_ids = Vec::new(); // Obtain quota let mut batch = BatchBuilder::new(); 'create: for (id, create) in request.create.into_valid() { let from_contact_id = id.document_id(); if !from_contact_ids.contains(from_contact_id) { response.not_created.append( id, SetError::not_found().with_description(format!( "Item {} not found in account {}.", id, response.from_account_id )), ); continue; } let Some(_contact) = self .store() .get_value::>(ValueKey::archive( from_account_id, Collection::ContactCard, from_contact_id, )) .await? else { response.not_created.append( id, SetError::not_found().with_description(format!( "Item {} not found in account {}.", id, response.from_account_id )), ); continue; }; let contact = _contact .deserialize::() .caused_by(trc::location!())?; match self .create_contact_card( &cache, &mut batch, access_token, account_id, &can_add_address_books, contact.card.into_jscontact(), create, ) .await? { Ok(document_id) => { response.created(id, document_id); // Add to destroy list if on_success_delete { destroy_ids.push(MaybeInvalid::Value(id)); } } Err(err) => { response.not_created.append(id, err); continue 'create; } } } // Write changes if !batch.is_empty() { let change_id = self .commit_batch(batch) .await .and_then(|ids| ids.last_change_id(account_id)) .caused_by(trc::location!())?; response.new_state = State::Exact(change_id); } // Destroy ids if on_success_delete && !destroy_ids.is_empty() { *next_call = Call { id: String::new(), name: MethodName::new(MethodObject::ContactCard, MethodFunction::Set), method: RequestMethod::Set(SetRequestMethod::ContactCard(SetRequest { account_id: request.from_account_id, if_in_state: request.destroy_from_if_in_state, create: None, update: None, destroy: MaybeResultReference::Value(destroy_ids).into(), arguments: Default::default(), })), } .into(); } Ok(response) } } ================================================ FILE: crates/jmap/src/contact/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::changes::state::JmapCacheState; use calcard::jscontact::{JSContactProperty, JSContactValue, import::ConversionOptions}; use common::{Server, auth::AccessToken}; use groupware::{cache::GroupwareCache, contact::ContactCard}; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::contact, request::reference::MaybeResultReference, }; use jmap_tools::{Map, Value}; use store::{ValueKey, roaring::RoaringBitmap, write::{AlignedBytes, Archive}}; use trc::AddContext; use types::{ acl::Acl, blob::BlobId, collection::{Collection, SyncCollection}, id::Id, }; pub trait ContactCardGet: Sync + Send { fn contact_card_get( &self, request: GetRequest, access_token: &AccessToken, ) -> impl Future>> + Send; } impl ContactCardGet for Server { async fn contact_card_get( &self, mut request: GetRequest, access_token: &AccessToken, ) -> trc::Result> { let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let return_all_properties = request .properties .as_ref() .is_none_or(|v| matches!(v, MaybeResultReference::Value(v) if v.is_empty())); let properties = request.unwrap_properties(&[JSContactProperty::Id, JSContactProperty::AddressBookIds]); let account_id = request.account_id.document_id(); let cache = self .fetch_dav_resources(access_token, account_id, SyncCollection::AddressBook) .await?; let contact_ids = if access_token.is_member(account_id) { cache.document_ids(false).collect::() } else { cache.shared_items(access_token, [Acl::ReadItems], true) }; let ids = if let Some(ids) = ids { ids } else { contact_ids .iter() .take(self.core.jmap.get_max_objects) .map(Into::into) .collect::>() }; let mut response = GetResponse { account_id: request.account_id.into(), state: cache.get_state(false).into(), list: Vec::with_capacity(ids.len()), not_found: vec![], }; let mut return_id = return_all_properties; let mut return_address_book_ids = return_all_properties; let mut return_converted_props = !return_all_properties; if !return_all_properties { for property in &properties { match property { JSContactProperty::Id => { return_id = true; } JSContactProperty::AddressBookIds => { return_address_book_ids = true; } JSContactProperty::VCard => { return_converted_props = true; } _ => {} } } } for id in ids { // Obtain the contact object let document_id = id.document_id(); if !contact_ids.contains(document_id) { response.not_found.push(id); continue; } let _contact = if let Some(contact) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::ContactCard, document_id, )) .await? { contact } else { response.not_found.push(id); continue; }; let contact = _contact .deserialize::() .caused_by(trc::location!())?; let jscontact = contact .card .into_jscontact_with_options::( ConversionOptions::default().include_vcard_parameters(return_converted_props), ) .into_inner(); let mut result = if return_all_properties { jscontact.into_object().unwrap() } else { Map::from_iter( jscontact .into_expanded_object() .filter(|(k, _)| k.as_property().is_some_and(|p| properties.contains(p))), ) }; if return_id { result.insert_unchecked( JSContactProperty::Id, Value::Element(JSContactValue::Id(id)), ); } if return_address_book_ids { let mut obj = Map::with_capacity(contact.names.len()); for id in contact.names.iter() { obj.insert_unchecked(JSContactProperty::IdValue(Id::from(id.parent_id)), true); } result.insert_unchecked(JSContactProperty::AddressBookIds, Value::Object(obj)); } response.list.push(result.into()); } Ok(response) } } ================================================ FILE: crates/jmap/src/contact/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use calcard::jscontact::JSContactProperty; use common::{DavName, DavResources, Server}; use jmap_proto::error::set::SetError; use trc::AddContext; use types::{collection::Collection, field::ContactField, id::Id}; pub mod copy; pub mod get; pub mod parse; pub mod query; pub mod set; pub(super) async fn assert_is_unique_uid( server: &Server, resources: &DavResources, account_id: u32, addressbook_ids: &[DavName], uid: Option<&str>, ) -> trc::Result>>> { if let Some(uid) = uid { let hits = server .document_ids_matching( account_id, Collection::ContactCard, ContactField::Uid, uid.as_bytes(), ) .await .caused_by(trc::location!())?; if !hits.is_empty() { for document_id in resources .paths .iter() .filter(move |item| { item.parent_id .is_some_and(|id| addressbook_ids.iter().any(|ab| ab.parent_id == id)) }) .map(|path| resources.resources[path.resource_idx].document_id) { if hits.contains(document_id) { return Ok(Err(SetError::invalid_properties() .with_property(JSContactProperty::Uid) .with_description(format!( "Contact with UID {uid} already exists with id {}.", Id::from(document_id) )))); } } } } Ok(Ok(())) } ================================================ FILE: crates/jmap/src/contact/parse.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::blob::download::BlobDownload; use calcard::vcard::VCard; use common::{Server, auth::AccessToken}; use jmap_proto::{ method::parse::{ParseRequest, ParseResponse}, object::contact::ContactCard, request::IntoValid, }; use types::{blob::BlobId, id::Id}; use utils::map::vec_map::VecMap; pub trait ContactCardParse: Sync + Send { fn contact_card_parse( &self, request: ParseRequest, access_token: &AccessToken, ) -> impl Future>> + Send; } impl ContactCardParse for Server { async fn contact_card_parse( &self, request: ParseRequest, access_token: &AccessToken, ) -> trc::Result> { if request.blob_ids.len() > self.core.jmap.contact_parse_max_items { return Err(trc::JmapEvent::RequestTooLarge.into_err()); } let return_all_properties = request.properties.is_none(); let properties = request .properties .map(|v| v.into_valid().collect::>()) .unwrap_or_default(); let mut response = ParseResponse { account_id: request.account_id, parsed: VecMap::with_capacity(request.blob_ids.len()), not_parsable: vec![], not_found: vec![], }; for blob_id in request.blob_ids.into_valid() { // Fetch raw message to parse let raw_vcard = match self.blob_download(&blob_id, access_token).await? { Some(raw_vcard) => raw_vcard, None => { response.not_found.push(blob_id); continue; } }; let Ok(vcard) = VCard::parse(std::str::from_utf8(&raw_vcard).unwrap_or_default()) else { response.not_parsable.push(blob_id); continue; }; let mut js_contact = vcard.into_jscontact::(); if !return_all_properties { js_contact .0 .as_object_mut() .unwrap() .as_mut_vec() .retain(|(k, _)| k.as_property().is_some_and(|k| properties.contains(k))); } response.parsed.append(blob_id, js_contact.into_inner()); } Ok(response) } } ================================================ FILE: crates/jmap/src/contact/query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{api::query::QueryResponseBuilder, changes::state::JmapCacheState}; use common::{Server, auth::AccessToken}; use groupware::cache::GroupwareCache; use jmap_proto::{ method::query::{Filter, QueryRequest, QueryResponse}, object::contact::{ContactCard, ContactCardComparator, ContactCardFilter}, request::MaybeInvalid, }; use store::{ IterateParams, U32_LEN, U64_LEN, ValueKey, roaring::RoaringBitmap, search::{ContactSearchField, SearchComparator, SearchFilter, SearchQuery}, write::{IndexPropertyClass, SearchIndex, ValueClass, key::DeserializeBigEndian}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, field::ContactField, }; use utils::sanitize_email; pub trait ContactCardQuery: Sync + Send { fn contact_card_query( &self, request: QueryRequest, access_token: &AccessToken, ) -> impl Future> + Send; } #[derive(Clone)] struct CreatedUpdated { document_id: u32, created: u64, updated: u64, } impl ContactCardQuery for Server { async fn contact_card_query( &self, mut request: QueryRequest, access_token: &AccessToken, ) -> trc::Result { let account_id = request.account_id.document_id(); let mut filters = Vec::with_capacity(request.filter.len()); let cache = self .fetch_dav_resources(access_token, account_id, SyncCollection::AddressBook) .await?; let mut created_to_updated = Vec::new(); if request.filter.iter().any(|cond| { matches!( cond, Filter::Property( ContactCardFilter::CreatedBefore(_) | ContactCardFilter::CreatedAfter(_) | ContactCardFilter::UpdatedBefore(_) | ContactCardFilter::UpdatedAfter(_) ) ) }) || request.sort.as_ref().is_some_and(|v| { v.iter().any(|sort| { matches!( sort.property, ContactCardComparator::Created | ContactCardComparator::Updated ) }) }) { self.store() .iterate( IterateParams::new( ValueKey { account_id, collection: Collection::ContactCard.into(), document_id: 0, class: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: ContactField::CreatedToUpdated.into(), value: 0, }), }, ValueKey { account_id, collection: Collection::ContactCard.into(), document_id: 0, class: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: ContactField::CreatedToUpdated.into(), value: u64::MAX, }), }, ) .ascending(), |key, value| { created_to_updated.push(CreatedUpdated { document_id: key.deserialize_be_u32(key.len() - U32_LEN)?, created: key.deserialize_be_u64(key.len() - U32_LEN - U64_LEN)?, updated: value.deserialize_be_u64(0)?, }); Ok(true) }, ) .await .caused_by(trc::location!())?; } for cond in std::mem::take(&mut request.filter) { match cond { Filter::Property(cond) => match cond { ContactCardFilter::InAddressBook(MaybeInvalid::Value(id)) => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.children_ids(id.document_id()), ))) } ContactCardFilter::Name(value) | ContactCardFilter::NameGiven(value) | ContactCardFilter::NameSurname(value) | ContactCardFilter::NameSurname2(value) => { filters.push(SearchFilter::has_keyword(ContactSearchField::Name, value)); } ContactCardFilter::Nickname(value) => { filters.push(SearchFilter::has_keyword( ContactSearchField::Nickname, value, )); } ContactCardFilter::Organization(value) => { filters.push(SearchFilter::has_keyword( ContactSearchField::Organization, value, )); } ContactCardFilter::Phone(value) => { filters.push(SearchFilter::has_keyword(ContactSearchField::Phone, value)); } ContactCardFilter::OnlineService(value) => { filters.push(SearchFilter::has_keyword( ContactSearchField::OnlineService, value, )); } ContactCardFilter::Address(value) => { filters.push(SearchFilter::has_keyword( ContactSearchField::Address, value, )); } ContactCardFilter::Note(value) => { filters.push(SearchFilter::has_text_detect( ContactSearchField::Note, value, self.core.jmap.default_language, )); } ContactCardFilter::HasMember(value) => { filters.push(SearchFilter::has_keyword(ContactSearchField::Member, value)); } ContactCardFilter::Kind(value) => { filters.push(SearchFilter::eq(ContactSearchField::Kind, value)); } ContactCardFilter::Uid(value) => { filters.push(SearchFilter::eq(ContactSearchField::Uid, value)) } ContactCardFilter::Email(email) => filters.push(SearchFilter::has_keyword( ContactSearchField::Email, sanitize_email(&email).unwrap_or(email), )), ContactCardFilter::Text(value) => { filters.push(SearchFilter::Or); filters.push(SearchFilter::has_keyword( ContactSearchField::Name, value.clone(), )); filters.push(SearchFilter::has_keyword( ContactSearchField::Nickname, value.clone(), )); filters.push(SearchFilter::has_keyword( ContactSearchField::Organization, value.clone(), )); filters.push(SearchFilter::has_keyword( ContactSearchField::Email, value.clone(), )); filters.push(SearchFilter::has_keyword( ContactSearchField::Phone, value.clone(), )); filters.push(SearchFilter::has_keyword( ContactSearchField::OnlineService, value.clone(), )); filters.push(SearchFilter::has_keyword( ContactSearchField::Address, value.clone(), )); filters.push(SearchFilter::has_text_detect( ContactSearchField::Note, value, self.core.jmap.default_language, )); filters.push(SearchFilter::End); } ContactCardFilter::CreatedBefore(before) => { let before = before.timestamp() as u64; filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( created_to_updated .iter() .filter_map(|cu| (cu.created < before).then_some(cu.document_id)), ))); } ContactCardFilter::CreatedAfter(after) => { let after = after.timestamp() as u64; filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( created_to_updated .iter() .filter_map(|cu| (cu.created > after).then_some(cu.document_id)), ))); } ContactCardFilter::UpdatedBefore(before) => { let before = before.timestamp() as u64; filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( created_to_updated .iter() .filter_map(|cu| (cu.updated < before).then_some(cu.document_id)), ))); } ContactCardFilter::UpdatedAfter(after) => { let after = after.timestamp() as u64; filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( created_to_updated .iter() .filter_map(|cu| (cu.updated > after).then_some(cu.document_id)), ))); } unsupported => { return Err(trc::JmapEvent::UnsupportedFilter .into_err() .details(unsupported.into_string())); } }, Filter::And => { filters.push(SearchFilter::And); } Filter::Or => { filters.push(SearchFilter::Or); } Filter::Not => { filters.push(SearchFilter::Not); } Filter::Close => { filters.push(SearchFilter::End); } } } let comparators = request .sort .take() .unwrap_or_default() .into_iter() .map(|comparator| match comparator.property { ContactCardComparator::Created => Ok(SearchComparator::sorted_set( created_to_updated .iter() .enumerate() .map(|(idx, u)| (u.document_id, idx as u32)) .collect(), comparator.is_ascending, )), ContactCardComparator::Updated => { let mut updated = created_to_updated.clone(); updated.sort_by(|a, b| a.updated.cmp(&b.updated)); Ok(SearchComparator::sorted_set( updated .iter() .enumerate() .map(|(idx, u)| (u.document_id, idx as u32)) .collect(), comparator.is_ascending, )) } other => Err(trc::JmapEvent::UnsupportedSort .into_err() .details(other.into_string())), }) .collect::, _>>()?; let results = self .search_store() .query_account( SearchQuery::new(SearchIndex::Contacts) .with_filters(filters) .with_comparators(comparators) .with_account_id(account_id) .with_mask(if access_token.is_shared(account_id) { cache.shared_items(access_token, [Acl::ReadItems], true) } else { cache.document_ids(false).collect() }), ) .await?; let mut response = QueryResponseBuilder::new( results.len(), self.core.jmap.query_max_results, cache.get_state(false), &request, ); for document_id in results { if !response.add(0, document_id) { break; } } response.build() } } ================================================ FILE: crates/jmap/src/contact/set.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::contact::assert_is_unique_uid; use calcard::jscontact::{JSContact, JSContactProperty, JSContactValue}; use common::{DavName, DavResources, Server, auth::AccessToken}; use groupware::{DestroyArchive, cache::GroupwareCache, contact::ContactCard}; use http_proto::HttpSessionData; use jmap_proto::{ error::set::SetError, method::set::{SetRequest, SetResponse}, object::contact, request::IntoValid, types::state::State, }; use jmap_tools::{JsonPointerHandler, JsonPointerItem, Key, Value}; use store::{ ValueKey, ahash::AHashSet, roaring::RoaringBitmap, write::{AlignedBytes, Archive, BatchBuilder}, }; use trc::AddContext; use types::{ acl::Acl, blob::BlobId, collection::{Collection, SyncCollection}, id::Id, }; pub trait ContactCardSet: Sync + Send { fn contact_card_set( &self, request: SetRequest<'_, contact::ContactCard>, access_token: &AccessToken, session: &HttpSessionData, ) -> impl Future>> + Send; #[allow(clippy::too_many_arguments)] fn create_contact_card( &self, cache: &DavResources, batch: &mut BatchBuilder, access_token: &AccessToken, account_id: u32, can_add_address_books: &Option, js_contact: JSContact<'_, Id, BlobId>, updates: Value<'_, JSContactProperty, JSContactValue>, ) -> impl Future>>>>; } impl ContactCardSet for Server { async fn contact_card_set( &self, mut request: SetRequest<'_, contact::ContactCard>, access_token: &AccessToken, _session: &HttpSessionData, ) -> trc::Result> { let account_id = request.account_id.document_id(); let cache = self .fetch_dav_resources(access_token, account_id, SyncCollection::AddressBook) .await?; let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; let will_destroy = request.unwrap_destroy().into_valid().collect::>(); // Obtain addressBookIds let (can_add_address_books, can_delete_address_books, can_modify_address_books) = if access_token.is_shared(account_id) { ( cache .shared_containers(access_token, [Acl::AddItems], true) .into(), cache .shared_containers(access_token, [Acl::RemoveItems], true) .into(), cache .shared_containers(access_token, [Acl::ModifyItems], true) .into(), ) } else { (None, None, None) }; // Process creates let mut batch = BatchBuilder::new(); 'create: for (id, object) in request.unwrap_create() { match self .create_contact_card( &cache, &mut batch, access_token, account_id, &can_add_address_books, JSContact::default(), object, ) .await? { Ok(document_id) => { response.created(id, document_id); } Err(err) => { response.not_created.append(id, err); continue 'create; } } } // Process updates 'update: for (id, object) in request.unwrap_update().into_valid() { // Make sure id won't be destroyed if will_destroy.contains(&id) { response.not_updated.append(id, SetError::will_destroy()); continue 'update; } // Obtain contact card let document_id = id.document_id(); let contact_card_ = if let Some(contact_card_) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::ContactCard, document_id, )) .await? { contact_card_ } else { response.not_updated.append(id, SetError::not_found()); continue 'update; }; let contact_card = contact_card_ .to_unarchived::() .caused_by(trc::location!())?; let mut new_contact_card = contact_card .deserialize::() .caused_by(trc::location!())?; let mut js_contact = new_contact_card.card.into_jscontact(); // Process changes if let Err(err) = update_contact_card(object, &mut new_contact_card.names, &mut js_contact) { response.not_updated.append(id, err); continue 'update; } // Convert JSContact to vCard if let Some(vcard) = js_contact.into_vcard() { new_contact_card.size = vcard.size() as u32; new_contact_card.card = vcard; } else { response.not_updated.append( id, SetError::invalid_properties() .with_description("Failed to convert contact to vCard."), ); continue 'update; } // Validate UID match (new_contact_card.card.uid(), contact_card.inner.card.uid()) { (Some(old_uid), Some(new_uid)) if old_uid == new_uid => {} (None, None) | (None, Some(_)) => {} _ => { response.not_updated.append( id, SetError::invalid_properties() .with_property(JSContactProperty::Uid) .with_description("You cannot change the UID of a contact."), ); continue 'update; } } // Validate new addressBookIds for addressbook_id in new_contact_card.added_addressbook_ids(contact_card.inner) { if !cache.has_container_id(&addressbook_id) { response.not_updated.append( id, SetError::invalid_properties() .with_property(JSContactProperty::AddressBookIds) .with_description(format!( "addressBookId {} does not exist.", Id::from(addressbook_id) )), ); continue 'update; } else if can_add_address_books .as_ref() .is_some_and(|ids| !ids.contains(addressbook_id)) { response.not_updated.append( id, SetError::forbidden().with_description(format!( "You are not allowed to add contacts to address book {}.", Id::from(addressbook_id) )), ); continue 'update; } } // Validate deleted addressBookIds if let Some(can_delete_address_books) = &can_delete_address_books { for addressbook_id in new_contact_card.removed_addressbook_ids(contact_card.inner) { if !can_delete_address_books.contains(addressbook_id) { response.not_updated.append( id, SetError::forbidden().with_description(format!( "You are not allowed to remove contacts from address book {}.", Id::from(addressbook_id) )), ); continue 'update; } } } // Validate changed addressBookIds if let Some(can_modify_address_books) = &can_modify_address_books { for addressbook_id in new_contact_card.unchanged_addressbook_ids(contact_card.inner) { if !can_modify_address_books.contains(addressbook_id) { response.not_updated.append( id, SetError::forbidden().with_description(format!( "You are not allowed to modify address book {}.", Id::from(addressbook_id) )), ); continue 'update; } } } // Check size and quota if new_contact_card.size as usize > self.core.groupware.max_vcard_size { response.not_updated.append( id, SetError::invalid_properties().with_description(format!( "Contact size {} exceeds the maximum allowed size of {} bytes.", new_contact_card.size, self.core.groupware.max_vcard_size )), ); continue 'update; } let extra_bytes = (new_contact_card.size as u64) .saturating_sub(u32::from(contact_card.inner.size) as u64); if extra_bytes > 0 { match self .has_available_quota( &self.get_resource_token(access_token, account_id).await?, extra_bytes, ) .await { Ok(_) => {} Err(err) if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) => { response.not_updated.append(id, SetError::over_quota()); continue 'update; } Err(err) => return Err(err.caused_by(trc::location!())), } } // Update record new_contact_card .update( access_token, contact_card, account_id, document_id, &mut batch, ) .caused_by(trc::location!())?; response.updated.append(id, None); } // Process deletions 'destroy: for id in will_destroy { let document_id = id.document_id(); if !cache.has_item_id(&document_id) { response.not_destroyed.append(id, SetError::not_found()); continue; }; let Some(contact_card_) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::ContactCard, document_id, )) .await .caused_by(trc::location!())? else { response.not_destroyed.append(id, SetError::not_found()); continue; }; let contact_card = contact_card_ .to_unarchived::() .caused_by(trc::location!())?; // Validate ACLs if let Some(can_delete_address_books) = &can_delete_address_books { for name in contact_card.inner.names.iter() { let parent_id = name.parent_id.to_native(); if !can_delete_address_books.contains(parent_id) { response.not_destroyed.append( id, SetError::forbidden().with_description(format!( "You are not allowed to remove contacts from address book {}.", Id::from(parent_id) )), ); continue 'destroy; } } } // Delete record DestroyArchive(contact_card) .delete_all(access_token, account_id, document_id, &mut batch) .caused_by(trc::location!())?; response.destroyed.push(id); } // Write changes if !batch.is_empty() { let change_id = self .commit_batch(batch) .await .and_then(|ids| ids.last_change_id(account_id)) .caused_by(trc::location!())?; self.notify_task_queue(); response.new_state = State::Exact(change_id).into(); } Ok(response) } async fn create_contact_card( &self, cache: &DavResources, batch: &mut BatchBuilder, access_token: &AccessToken, account_id: u32, can_add_address_books: &Option, mut js_contact: JSContact<'_, Id, BlobId>, updates: Value<'_, JSContactProperty, JSContactValue>, ) -> trc::Result>>> { // Process changes let mut names = Vec::new(); if let Err(err) = update_contact_card(updates, &mut names, &mut js_contact) { return Ok(Err(err)); } // Verify that the address book ids valid for name in &names { if !cache.has_container_id(&name.parent_id) { return Ok(Err(SetError::invalid_properties() .with_property(JSContactProperty::AddressBookIds) .with_description(format!( "addressBookId {} does not exist.", Id::from(name.parent_id) )))); } else if can_add_address_books .as_ref() .is_some_and(|ids| !ids.contains(name.parent_id)) { return Ok(Err(SetError::forbidden().with_description(format!( "You are not allowed to add contacts to address book {}.", Id::from(name.parent_id) )))); } } // Convert JSContact to vCard let Some(card) = js_contact.into_vcard() else { return Ok(Err(SetError::invalid_properties() .with_description("Failed to convert contact to vCard."))); }; // Validate UID if let Err(err) = assert_is_unique_uid(self, cache, account_id, &names, card.uid()).await? { return Ok(Err(err)); } // Check size and quota let size = card.size(); if size > self.core.groupware.max_vcard_size { return Ok(Err(SetError::invalid_properties().with_description( format!( "Contact size {} exceeds the maximum allowed size of {} bytes.", size, self.core.groupware.max_vcard_size ), ))); } match self .has_available_quota( &self.get_resource_token(access_token, account_id).await?, size as u64, ) .await { Ok(_) => {} Err(err) if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) => { return Ok(Err(SetError::over_quota())); } Err(err) => return Err(err.caused_by(trc::location!())), } // Insert record let document_id = self .store() .assign_document_ids(account_id, Collection::ContactCard, 1) .await .caused_by(trc::location!())?; ContactCard { names, size: size as u32, card, ..Default::default() } .insert(access_token, account_id, document_id, batch) .caused_by(trc::location!()) .map(|_| Ok(document_id)) } } fn update_contact_card<'x>( updates: Value<'x, JSContactProperty, JSContactValue>, addressbooks: &mut Vec, js_contact: &mut JSContact<'x, Id, BlobId>, ) -> Result<(), SetError>> { let mut entries = js_contact.0.as_object_mut().unwrap(); for (property, value) in updates.into_expanded_object() { let Key::Property(property) = property else { return Err(SetError::invalid_properties() .with_property(property.to_owned()) .with_description("Invalid property.")); }; match (property, value) { (JSContactProperty::AddressBookIds, value) => { patch_parent_ids(addressbooks, None, value)?; } (JSContactProperty::Pointer(pointer), value) => { if matches!( pointer.first(), Some(JsonPointerItem::Key(Key::Property( JSContactProperty::AddressBookIds ))) ) { let mut pointer = pointer.iter(); pointer.next(); patch_parent_ids(addressbooks, pointer.next(), value)?; } else if !js_contact.0.patch_jptr(pointer.iter(), value) { return Err(SetError::invalid_properties() .with_property(JSContactProperty::Pointer(pointer)) .with_description("Patch operation failed.")); } entries = js_contact.0.as_object_mut().unwrap(); } (JSContactProperty::Media, Value::Object(media)) => { for (_, value) in media.iter() { if value.as_object().is_some_and(|v| { v.keys() .any(|k| matches!(k, Key::Property(JSContactProperty::BlobId))) }) { return Err(SetError::invalid_properties() .with_property(JSContactProperty::Media) .with_description("blobIds in media is not supported.")); } } entries.insert(JSContactProperty::Media, Value::Object(media)); } (property, value) => { entries.insert(property, value); } } } // Make sure the contact belongs to at least one address book if addressbooks.is_empty() { return Err(SetError::invalid_properties() .with_property(JSContactProperty::AddressBookIds) .with_description("Contact has to belong to at least one address book.")); } Ok(()) } fn patch_parent_ids( current: &mut Vec, patch: Option<&JsonPointerItem>>, update: Value<'_, JSContactProperty, JSContactValue>, ) -> Result<(), SetError>> { match (patch, update) { ( Some(JsonPointerItem::Key(Key::Property(JSContactProperty::IdValue(id)))), Value::Bool(false) | Value::Null, ) => { let id = id.document_id(); current.retain(|name| name.parent_id != id); Ok(()) } ( Some(JsonPointerItem::Key(Key::Property(JSContactProperty::IdValue(id)))), Value::Bool(true), ) => { let id = id.document_id(); if !current.iter().any(|name| name.parent_id == id) { current.push(DavName::new_with_rand_name(id)); } Ok(()) } (None, Value::Object(object)) => { let mut new_ids = object .into_expanded_boolean_set() .filter_map(|id| { if let Key::Property(JSContactProperty::IdValue(id)) = id { Some(id.document_id()) } else { None } }) .collect::>(); current.retain(|name| new_ids.remove(&name.parent_id)); for id in new_ids { current.push(DavName::new_with_rand_name(id)); } Ok(()) } _ => Err(SetError::invalid_properties() .with_property(JSContactProperty::AddressBookIds) .with_description("Invalid patch operation for addressBookIds.")), } } ================================================ FILE: crates/jmap/src/email/body.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use email::message::metadata::{ ArchivedMessageMetadataContents, ArchivedMetadataHeaderValue, ArchivedMetadataPartType, PART_ENCODING_BASE64, PART_ENCODING_QP, PART_SIZE_MASK, }; use jmap_proto::object::email::{EmailProperty, EmailValue}; use jmap_tools::{Map, Value}; use mail_parser::{HeaderValue, MessagePart, MimeHeaders, PartType}; use types::blob::BlobId; use utils::chained_bytes::ChainedBytes; use super::headers::HeaderToValue; pub trait ToBodyPart { fn to_body_part( &self, part_id: u32, properties: &[EmailProperty], raw_message: &ChainedBytes<'_>, blob_id: &BlobId, blob_body_offset: isize, ) -> Value<'static, EmailProperty, EmailValue>; } impl ToBodyPart for Vec> { fn to_body_part( &self, part_id: u32, properties: &[EmailProperty], raw_message: &ChainedBytes<'_>, blob_id: &BlobId, blob_body_offset: isize, ) -> Value<'static, EmailProperty, EmailValue> { let mut parts = vec![part_id].into_iter(); let mut parts_stack = Vec::new(); let mut subparts = Vec::with_capacity(1); loop { if let Some((part_id, part)) = parts .next() .map(|part_id| (part_id, &self[part_id as usize])) { let mut values = Map::with_capacity(properties.len()); let multipart = if let PartType::Multipart(parts) = &part.body { parts.into() } else { None }; for property in properties { let value = match property { EmailProperty::PartId if multipart.is_none() => part_id.to_string().into(), EmailProperty::BlobId if multipart.is_none() => { let base_offset = blob_id.start_offset() as isize + blob_body_offset; BlobId::new_section( blob_id.hash.clone(), blob_id.class.clone(), (part.offset_body as isize + base_offset) as usize, (part.offset_end as isize + base_offset) as usize, part.encoding as u8, ) .into() } EmailProperty::Size if multipart.is_none() => match &part.body { PartType::Text(text) | PartType::Html(text) => text.len(), PartType::Binary(bin) | PartType::InlineBinary(bin) => bin.len(), PartType::Message(message) => message.root_part().raw_len() as usize, PartType::Multipart(_) => 0, } .into(), EmailProperty::Name => part.attachment_name().map(|v| v.to_string()).into(), EmailProperty::Type => part .content_type() .map(|ct| { ct.subtype() .map(|st| format!("{}/{}", ct.ctype(), st)) .unwrap_or_else(|| ct.ctype().to_string()) }) .or_else(|| match &part.body { PartType::Text(_) => Some("text/plain".to_string()), PartType::Html(_) => Some("text/html".to_string()), PartType::Message(_) => Some("message/rfc822".to_string()), _ => None, }) .into(), EmailProperty::Charset => part .content_type() .and_then(|ct| ct.attribute("charset")) .or(match &part.body { PartType::Text(_) | PartType::Html(_) => Some("us-ascii"), _ => None, }) .map(|v| v.to_string()) .into(), EmailProperty::Disposition => part .content_disposition() .map(|cd| cd.ctype()) .map(|v| v.to_string()) .into(), EmailProperty::Cid => part.content_id().map(|v| v.to_string()).into(), EmailProperty::Language => match part.content_language() { HeaderValue::Text(text) => vec![text.to_string()].into(), HeaderValue::TextList(list) => list .iter() .map(|text| text.to_string().into()) .collect::>>() .into(), _ => Value::Null, }, EmailProperty::Location => { part.content_location().map(|v| v.to_string()).into() } EmailProperty::Header(_) => { part.headers.header_to_value(property, raw_message) } EmailProperty::Headers => part.headers.headers_to_value(raw_message), EmailProperty::SubParts => continue, _ => Value::Null, }; values.insert_unchecked(property.clone(), value); } subparts.push(values); if let Some(multipart) = multipart { if parts_stack.len() == 10_000 { debug_assert!(false, "Too much nesting in message metadata"); return Value::Null; } let multipart = multipart.clone(); parts_stack.push(( parts, std::mem::replace(&mut subparts, Vec::with_capacity(multipart.len())), )); parts = multipart.into_iter(); } } else if let Some((prev_parts, mut prev_subparts)) = parts_stack.pop() { prev_subparts .last_mut() .unwrap() .insert_unchecked(EmailProperty::SubParts, subparts); parts = prev_parts; subparts = prev_subparts; } else { return subparts.pop().map(Into::into).unwrap_or_default(); } } } } impl ToBodyPart for ArchivedMessageMetadataContents { fn to_body_part( &self, part_id: u32, properties: &[EmailProperty], raw_message: &ChainedBytes<'_>, blob_id: &BlobId, blob_body_offset: isize, ) -> Value<'static, EmailProperty, EmailValue> { let mut parts = vec![part_id].into_iter(); let mut parts_stack = Vec::new(); let mut subparts = Vec::with_capacity(1); loop { if let Some((part_id, part)) = parts .next() .map(|part_id| (part_id, &self.parts[part_id as usize])) { let mut values = Map::with_capacity(properties.len()); let multipart = if let ArchivedMetadataPartType::Multipart(parts) = &part.body { parts.into() } else { None }; for property in properties { let value = match property { EmailProperty::PartId if multipart.is_none() => part_id.to_string().into(), EmailProperty::BlobId if multipart.is_none() => { let base_offset = blob_id.start_offset() as isize + blob_body_offset; let flags = part.flags.to_native(); let encoding = if flags & PART_ENCODING_BASE64 != 0 { 2 } else if flags & PART_ENCODING_QP != 0 { 1 } else { 0 }; BlobId::new_section( blob_id.hash.clone(), blob_id.class.clone(), (u32::from(part.offset_body) as isize + base_offset) as usize, (u32::from(part.offset_end) as isize + base_offset) as usize, encoding, ) .into() } EmailProperty::Size if multipart.is_none() => { (part.flags.to_native() & PART_SIZE_MASK).into() } EmailProperty::Name => part.attachment_name().map(|v| v.to_string()).into(), EmailProperty::Type => part .content_type() .map(|ct| { ct.subtype() .map(|st| format!("{}/{}", ct.ctype(), st)) .unwrap_or_else(|| ct.ctype().to_string()) }) .or_else(|| match &part.body { ArchivedMetadataPartType::Text => Some("text/plain".to_string()), ArchivedMetadataPartType::Html => Some("text/html".to_string()), ArchivedMetadataPartType::Message(_) => { Some("message/rfc822".to_string()) } _ => None, }) .into(), EmailProperty::Charset => { part.content_type() .and_then(|ct| ct.attribute("charset")) .or(match &part.body { ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => Some("us-ascii"), _ => None, }) .map(|v| v.to_string()) .into() } EmailProperty::Disposition => part .content_disposition() .map(|cd| cd.ctype()) .map(|v| v.to_string()) .into(), EmailProperty::Cid => part.content_id().map(|v| v.to_string()).into(), EmailProperty::Language => match part.content_language() { ArchivedMetadataHeaderValue::Text(text) => { vec![text.to_string()].into() } ArchivedMetadataHeaderValue::TextList(list) => list .iter() .map(|text| text.to_string().into()) .collect::>>() .into(), _ => Value::Null, }, EmailProperty::Location => { part.content_location().map(|v| v.to_string()).into() } EmailProperty::Header(_) => part.header_to_value(property, raw_message), EmailProperty::Headers => part.headers_to_value(raw_message), EmailProperty::SubParts => continue, _ => Value::Null, }; values.insert_unchecked(property.clone(), value); } subparts.push(values); if let Some(multipart) = multipart { if parts_stack.len() == 10_000 { debug_assert!(false, "Too much nesting in message metadata"); return Value::Null; } let multipart = multipart .iter() .map(|id| u16::from(id) as u32) .collect::>(); parts_stack.push(( parts, std::mem::replace(&mut subparts, Vec::with_capacity(multipart.len())), )); parts = multipart.into_iter(); } } else if let Some((prev_parts, mut prev_subparts)) = parts_stack.pop() { prev_subparts .last_mut() .unwrap() .insert_unchecked(EmailProperty::SubParts, subparts); parts = prev_parts; subparts = prev_subparts; } else { return subparts.pop().map(Into::into).unwrap_or_default(); } } } } pub(super) trait TruncateBody { fn truncate(&self, max_len: usize) -> (bool, String); } impl TruncateBody for PartType<'_> { fn truncate(&self, max_len: usize) -> (bool, String) { match self { PartType::Text(text) => truncate_plain(text, max_len), PartType::Html(html) => truncate_html(html, max_len), PartType::Binary(bytes) | PartType::InlineBinary(bytes) => { PartType::Text(String::from_utf8_lossy(bytes)).truncate(max_len) } _ => (false, "".into()), } } } pub(crate) fn truncate_plain(text: &str, mut max_len: usize) -> (bool, String) { if max_len != 0 && text.len() > max_len { let add_dots = max_len > 6; if add_dots { max_len -= 3; } let mut result = String::with_capacity(max_len); for ch in text.chars() { if ch != '\r' { if ch.len_utf8() + result.len() > max_len { break; } result.push(ch); } } if add_dots { result.push_str("..."); } (true, result) } else { (false, text.replace('\r', "")) } } pub(crate) fn truncate_html(html: &str, mut max_len: usize) -> (bool, String) { if max_len != 0 && html.len() > max_len { let add_dots = max_len > 6; if add_dots { max_len -= 3; } let mut result = String::with_capacity(max_len); let mut in_tag = false; let mut in_comment = false; let mut last_tag_end_pos = 0; let mut cr_count = 0; for (pos, ch) in html.char_indices() { let mut set_last_tag = 0; match ch { '<' if !in_tag => { in_tag = true; if let Some("!--") = html.get(pos + 1..pos + 4) { in_comment = true; } set_last_tag = pos; } '>' if in_tag => { if in_comment { if let Some("--") = html.get(pos - 2..pos) { in_comment = false; in_tag = false; set_last_tag = pos + 1; } } else { in_tag = false; set_last_tag = pos + 1; } } '\r' => { cr_count += 1; continue; } _ => (), } if ch.len_utf8() + pos - cr_count > max_len { result.push_str( &html[0..if (in_tag || set_last_tag > 0) && last_tag_end_pos > 0 { last_tag_end_pos } else { pos }] .replace('\r', ""), ); if add_dots { result.push_str("..."); } break; } else if set_last_tag > 0 { last_tag_end_pos = set_last_tag; } } (true, result) } else { (false, html.replace('\r', "")) } } ================================================ FILE: crates/jmap/src/email/copy.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ changes::state::JmapCacheState, email::{PatchResult, handle_email_patch, ingested_into_object}, }; use common::{Server, auth::AccessToken}; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess, mailbox::MailboxCacheAccess}, message::copy::{CopyMessageError, EmailCopy}, }; use http_proto::HttpSessionData; use jmap_proto::{ error::set::SetError, method::{ copy::{CopyRequest, CopyResponse}, set::SetRequest, }, object::email::{Email, EmailProperty, EmailValue}, request::{ Call, IntoValid, MaybeInvalid, RequestMethod, SetRequestMethod, method::{MethodFunction, MethodName, MethodObject}, reference::MaybeResultReference, }, }; use jmap_tools::{Key, Value}; use std::future::Future; use trc::AddContext; use types::acl::Acl; use utils::map::vec_map::VecMap; pub trait JmapEmailCopy: Sync + Send { fn email_copy<'x>( &self, request: CopyRequest<'x, Email>, access_token: &AccessToken, next_call: &mut Option>>, session: &HttpSessionData, ) -> impl Future>> + Send; } impl JmapEmailCopy for Server { async fn email_copy<'x>( &self, request: CopyRequest<'x, Email>, access_token: &AccessToken, next_call: &mut Option>>, session: &HttpSessionData, ) -> trc::Result> { let account_id = request.account_id.document_id(); let from_account_id = request.from_account_id.document_id(); if account_id == from_account_id { return Err(trc::JmapEvent::InvalidArguments .into_err() .details("From accountId is equal to fromAccountId")); } let cache = self.get_cached_messages(account_id).await?; let old_state = cache.assert_state(false, &request.if_in_state)?; let mut response = CopyResponse { from_account_id: request.from_account_id, account_id: request.account_id, new_state: old_state.clone(), old_state, created: VecMap::with_capacity(request.create.len()), not_created: VecMap::new(), }; let from_cache = self .get_cached_messages(from_account_id) .await .caused_by(trc::location!())?; let from_message_ids = if access_token.is_member(from_account_id) { from_cache.email_document_ids() } else { from_cache.shared_messages(access_token, Acl::ReadItems) }; let can_add_mailbox_ids = if access_token.is_shared(account_id) { cache.shared_mailboxes(access_token, Acl::AddItems).into() } else { None }; let on_success_delete = request.on_success_destroy_original.unwrap_or(false); let mut destroy_ids = Vec::new(); // Obtain quota let resource_token = self.get_resource_token(access_token, account_id).await?; 'create: for (id, create) in request.create.into_valid() { let from_message_id = id.document_id(); if !from_message_ids.contains(from_message_id) { response.not_created.append( id, SetError::not_found().with_description(format!( "Item {} not found in account {}.", id, response.from_account_id )), ); continue; } let mut mailboxes = Vec::new(); let mut keywords = Vec::new(); let mut received_at = None; for (property, value) in create.into_expanded_object() { match (property, value) { (Key::Property(EmailProperty::MailboxIds), Value::Object(ids)) => { mailboxes = ids .into_expanded_boolean_set() .filter_map(|id| { id.try_into_property()?.try_into_id()?.document_id().into() }) .collect(); } (Key::Property(EmailProperty::Keywords), Value::Object(keywords_)) => { keywords = keywords_ .into_expanded_boolean_set() .filter_map(|id| id.try_into_property()?.try_into_keyword()) .collect(); } (Key::Property(EmailProperty::Pointer(pointer)), value) => { match handle_email_patch(&pointer, value) { PatchResult::SetKeyword(keyword) => { if !keywords.contains(keyword) { keywords.push(keyword.clone()); } } PatchResult::RemoveKeyword(keyword) => { keywords.retain(|k| k != keyword); } PatchResult::AddMailbox(id) => { if !mailboxes.contains(&id) { mailboxes.push(id); } } PatchResult::RemoveMailbox(id) => { mailboxes.retain(|mid| mid != &id); } PatchResult::Invalid(set_error) => { response.not_created.append(id, set_error); continue 'create; } } } ( Key::Property(EmailProperty::ReceivedAt), Value::Element(EmailValue::Date(value)), ) => { received_at = value.into(); } (property, _) => { response.not_created.append( id, SetError::invalid_properties() .with_property(property.into_owned()) .with_description("Invalid property or value.".to_string()), ); continue 'create; } } } // Make sure message belongs to at least one mailbox if mailboxes.is_empty() { response.not_created.append( id, SetError::invalid_properties() .with_property(EmailProperty::MailboxIds) .with_description("Message has to belong to at least one mailbox."), ); continue 'create; } // Verify that the mailboxIds are valid for mailbox_id in &mailboxes { if !cache.has_mailbox_id(mailbox_id) { response.not_created.append( id, SetError::invalid_properties() .with_property(EmailProperty::MailboxIds) .with_description(format!("mailboxId {mailbox_id} does not exist.")), ); continue 'create; } else if matches!(&can_add_mailbox_ids, Some(ids) if !ids.contains(*mailbox_id)) { response.not_created.append( id, SetError::forbidden().with_description(format!( "You are not allowed to add messages to mailbox {mailbox_id}." )), ); continue 'create; } } // Add response match self .copy_message( from_account_id, from_message_id, &resource_token, mailboxes, keywords, received_at.map(|dt| dt.timestamp() as u64), session.session_id, ) .await? { Ok(email) => { response .created .append(id, ingested_into_object(email).into()); } Err(err) => { response.not_created.append( id, match err { CopyMessageError::NotFound => SetError::not_found() .with_description("Message not found in account."), CopyMessageError::OverQuota => SetError::over_quota(), }, ); } } // Add to destroy list if on_success_delete { destroy_ids.push(MaybeInvalid::Value(id)); } } // Update state if !response.created.is_empty() { response.new_state = self.get_cached_messages(account_id).await?.get_state(false); } // Destroy ids if on_success_delete && !destroy_ids.is_empty() { *next_call = Call { id: String::new(), name: MethodName::new(MethodObject::Email, MethodFunction::Set), method: RequestMethod::Set(SetRequestMethod::Email(SetRequest { account_id: request.from_account_id, if_in_state: request.destroy_from_if_in_state, create: None, update: None, destroy: MaybeResultReference::Value(destroy_ids).into(), arguments: Default::default(), })), } .into(); } Ok(response) } } ================================================ FILE: crates/jmap/src/email/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ body::{ToBodyPart, truncate_html, truncate_plain}, headers::IntoForm, }; use crate::{changes::state::JmapCacheState, email::headers::HeaderToValue}; use common::{Server, auth::AccessToken}; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, message::metadata::{ ArchivedMetadataPartType, MESSAGE_HAS_ATTACHMENT, MESSAGE_RECEIVED_MASK, MessageMetadata, MetadataHeaderName, PART_ENCODING_PROBLEM, }, }; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::email::{Email, EmailProperty, EmailValue, HeaderForm}, request::IntoValid, types::date::UTCDate, }; use jmap_tools::{Key, Map, Value}; use mail_parser::HeaderValue; use std::future::Future; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::{AddContext, StoreEvent}; use types::{ acl::Acl, blob::{BlobClass, BlobId}, blob_hash::BlobHash, collection::Collection, field::EmailField, id::Id, }; use utils::chained_bytes::ChainedBytes; pub trait EmailGet: Sync + Send { fn email_get( &self, request: GetRequest, access_token: &AccessToken, ) -> impl Future>> + Send; } impl EmailGet for Server { async fn email_get( &self, mut request: GetRequest, access_token: &AccessToken, ) -> trc::Result> { let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ EmailProperty::Id, EmailProperty::BlobId, EmailProperty::ThreadId, EmailProperty::MailboxIds, EmailProperty::Keywords, EmailProperty::Size, EmailProperty::ReceivedAt, EmailProperty::MessageId, EmailProperty::InReplyTo, EmailProperty::References, EmailProperty::Sender, EmailProperty::From, EmailProperty::To, EmailProperty::Cc, EmailProperty::Bcc, EmailProperty::ReplyTo, EmailProperty::Subject, EmailProperty::SentAt, EmailProperty::HasAttachment, EmailProperty::Preview, EmailProperty::BodyValues, EmailProperty::TextBody, EmailProperty::HtmlBody, EmailProperty::Attachments, ]); let body_properties = request .arguments .body_properties .map(|v| v.into_valid().collect()) .unwrap_or_else(|| { vec![ EmailProperty::PartId, EmailProperty::BlobId, EmailProperty::Size, EmailProperty::Name, EmailProperty::Type, EmailProperty::Charset, EmailProperty::Disposition, EmailProperty::Cid, EmailProperty::Language, EmailProperty::Location, ] }); let fetch_text_body_values = request.arguments.fetch_text_body_values.unwrap_or(false); let fetch_html_body_values = request.arguments.fetch_html_body_values.unwrap_or(false); let fetch_all_body_values = request.arguments.fetch_all_body_values.unwrap_or(false); let max_body_value_bytes = request.arguments.max_body_value_bytes.unwrap_or(0); let account_id = request.account_id.document_id(); let cache = self .get_cached_messages(account_id) .await .caused_by(trc::location!())?; let message_ids = if access_token.is_member(account_id) { cache.email_document_ids() } else { cache.shared_messages(access_token, Acl::ReadItems) }; let ids = if let Some(ids) = ids { ids } else { cache .emails .items .iter() .take(self.core.jmap.get_max_objects) .map(|item| Id::from_parts(item.thread_id, item.document_id)) .collect() }; let mut response = GetResponse { account_id: request.account_id.into(), state: cache.get_state(false).into(), list: Vec::with_capacity(ids.len()), not_found: vec![], }; // Check if we need to fetch the raw headers or body let mut needs_body = false; for property in &properties { if matches!( property, EmailProperty::BodyValues | EmailProperty::TextBody | EmailProperty::HtmlBody | EmailProperty::Attachments | EmailProperty::BodyStructure ) { needs_body = true; break; } } for id in ids { // Obtain the email object if !message_ids.contains(id.document_id()) { response.not_found.push(id); continue; } let metadata_ = match self .store() .get_value::>(ValueKey::property( account_id, Collection::Email, id.document_id(), EmailField::Metadata, )) .await? { Some(metadata) => metadata, None => { response.not_found.push(id); continue; } }; let metadata = metadata_ .unarchive::() .caused_by(trc::location!())?; // Obtain message data let data = match cache.email_by_id(&id.document_id()) { Some(data) => data, None => { response.not_found.push(id); continue; } }; // Retrieve raw message if needed let blob_hash = BlobHash::from(&metadata.blob_hash); let raw_body; let mut raw_message = ChainedBytes::new(metadata.raw_headers.as_ref()); if needs_body { raw_body = self .blob_store() .get_blob(blob_hash.as_slice(), 0..usize::MAX) .await?; if let Some(raw_body) = &raw_body { raw_message.append( raw_body .get(metadata.blob_body_offset.to_native() as usize..) .unwrap_or_default(), ); } else { trc::event!( Store(StoreEvent::NotFound), AccountId = account_id, DocumentId = id.document_id(), Collection = Collection::Email, BlobId = blob_hash.to_hex(), Details = "Blob not found.", CausedBy = trc::location!(), ); response.not_found.push(id); continue; } } let blob_id = BlobId { hash: blob_hash, class: BlobClass::Linked { account_id, collection: Collection::Email.into(), document_id: id.document_id(), }, section: None, }; // Prepare response let mut email: Map<'_, EmailProperty, EmailValue> = Map::with_capacity(properties.len()); let contents = &metadata.contents[0]; let root_part = &contents.parts[0]; let blob_body_offset = metadata.blob_body_offset.to_native() as isize - root_part.offset_body.to_native() as isize; for property in &properties { match property { EmailProperty::Id => { email.insert_unchecked(EmailProperty::Id, Id::from(*id)); } EmailProperty::ThreadId => { email.insert_unchecked(EmailProperty::ThreadId, Id::from(id.prefix_id())); } EmailProperty::BlobId => { email.insert_unchecked(EmailProperty::BlobId, blob_id.clone()); } EmailProperty::MailboxIds => { let mut obj = Map::with_capacity(data.mailboxes.len()); for id in data.mailboxes.iter() { debug_assert!(id.uid != 0); obj.insert_unchecked( EmailProperty::IdValue(Id::from(id.mailbox_id)), true, ); } email.insert_unchecked(property.clone(), Value::Object(obj)); } EmailProperty::Keywords => { let mut obj = Map::with_capacity(2); for keyword in cache.expand_keywords(data) { obj.insert_unchecked(EmailProperty::Keyword(keyword), true); } email.insert_unchecked(property.clone(), Value::Object(obj)); } EmailProperty::Size => { email.insert_unchecked(EmailProperty::Size, data.size); } EmailProperty::ReceivedAt => { email.insert_unchecked( EmailProperty::ReceivedAt, EmailValue::Date(UTCDate::from_timestamp( (metadata.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK) as i64, )), ); } EmailProperty::Preview => { if !metadata.preview.is_empty() { email.insert_unchecked( EmailProperty::Preview, metadata.preview.to_string(), ); } } EmailProperty::HasAttachment => { email.insert_unchecked( EmailProperty::HasAttachment, (metadata.rcvd_attach.to_native() & MESSAGE_HAS_ATTACHMENT) != 0, ); } EmailProperty::Subject => { email.insert_unchecked( EmailProperty::Subject, root_part .header_value(&MetadataHeaderName::Subject) .map(|value| HeaderValue::from(value).into_form(&HeaderForm::Text)) .unwrap_or_default(), ); } EmailProperty::SentAt => { email.insert_unchecked( EmailProperty::SentAt, root_part .header_value(&MetadataHeaderName::Date) .map(|value| HeaderValue::from(value).into_form(&HeaderForm::Date)) .unwrap_or_default(), ); } EmailProperty::MessageId | EmailProperty::InReplyTo | EmailProperty::References => { email.insert_unchecked( property.clone(), root_part .header_value(&match property { EmailProperty::MessageId => MetadataHeaderName::MessageId, EmailProperty::InReplyTo => MetadataHeaderName::InReplyTo, EmailProperty::References => MetadataHeaderName::References, _ => unreachable!(), }) .map(|value| { HeaderValue::from(value).into_form(&HeaderForm::MessageIds) }) .unwrap_or_default(), ); } EmailProperty::Sender | EmailProperty::From | EmailProperty::To | EmailProperty::Cc | EmailProperty::Bcc | EmailProperty::ReplyTo => { email.insert_unchecked( property.clone(), root_part .header_value(&match property { EmailProperty::Sender => MetadataHeaderName::Sender, EmailProperty::From => MetadataHeaderName::From, EmailProperty::To => MetadataHeaderName::To, EmailProperty::Cc => MetadataHeaderName::Cc, EmailProperty::Bcc => MetadataHeaderName::Bcc, EmailProperty::ReplyTo => MetadataHeaderName::ReplyTo, _ => unreachable!(), }) .map(|value| { HeaderValue::from(value).into_form(&HeaderForm::Addresses) }) .unwrap_or_default(), ); } EmailProperty::Header(_) => { email.insert_unchecked( property.clone(), root_part.header_to_value(property, &raw_message), ); } EmailProperty::Headers => { email.insert_unchecked( EmailProperty::Headers, root_part.headers_to_value(&raw_message), ); } EmailProperty::TextBody | EmailProperty::HtmlBody | EmailProperty::Attachments => { let list = match property { EmailProperty::TextBody => &contents.text_body, EmailProperty::HtmlBody => &contents.html_body, EmailProperty::Attachments => &contents.attachments, _ => unreachable!(), } .iter(); email.insert_unchecked( property.clone(), list.map(|part_id| { contents.to_body_part( u16::from(part_id) as u32, &body_properties, &raw_message, &blob_id, blob_body_offset, ) }) .collect::>(), ); } EmailProperty::BodyStructure => { email.insert_unchecked( EmailProperty::BodyStructure, contents.to_body_part( 0, &body_properties, &raw_message, &blob_id, blob_body_offset, ), ); } EmailProperty::BodyValues => { let mut body_values = Map::with_capacity(contents.parts.len()); for (part_id, part) in contents.parts.iter().enumerate() { if ((contents.is_html_part(part_id as u16) && (fetch_all_body_values || fetch_html_body_values)) || (contents.is_text_part(part_id as u16) && (fetch_all_body_values || fetch_text_body_values))) && matches!( part.body, ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html ) { let contents = part.decode_contents(&raw_message); let (is_truncated, value) = match &part.body { ArchivedMetadataPartType::Text => { truncate_plain(contents.as_str(), max_body_value_bytes) } ArchivedMetadataPartType::Html => { truncate_html(contents.as_str(), max_body_value_bytes) } _ => unreachable!(), }; body_values.insert_unchecked( Key::Owned(part_id.to_string()), Map::with_capacity(3) .with_key_value( EmailProperty::IsEncodingProblem, (part.flags & PART_ENCODING_PROBLEM) != 0, ) .with_key_value(EmailProperty::IsTruncated, is_truncated) .with_key_value(EmailProperty::Value, value), ); } } email.insert_unchecked(EmailProperty::BodyValues, body_values); } _ => { return Err(trc::JmapEvent::InvalidArguments .into_err() .details(format!("Invalid property {property:?}"))); } } } response.list.push(email.into()); } Ok(response) } } ================================================ FILE: crates/jmap/src/email/headers.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use email::message::metadata::{ArchivedMessageMetadataPart, ArchivedMetadataHeaderValue}; use jmap_proto::{ object::email::{EmailProperty, EmailValue, HeaderForm, HeaderProperty}, types::date::UTCDate, }; use jmap_tools::{Key, Map, Value}; use mail_builder::{ MessageBuilder, headers::{ address::{Address, EmailAddress, GroupedAddresses}, date::Date, message_id::MessageId, raw::Raw, text::Text, url::URL, }, }; use mail_parser::{Addr, DateTime, Group, Header, HeaderName, HeaderValue, parsers::MessageStream}; use utils::chained_bytes::ChainedBytes; pub trait IntoForm { fn into_form(self, form: &HeaderForm) -> Value<'static, EmailProperty, EmailValue>; } pub trait HeaderToValue { fn header_to_value( &self, property: &EmailProperty, raw_message: &ChainedBytes<'_>, ) -> Value<'static, EmailProperty, EmailValue>; fn headers_to_value( &self, raw_message: &ChainedBytes<'_>, ) -> Value<'static, EmailProperty, EmailValue>; } pub trait ValueToHeader<'x> { fn try_into_grouped_addresses(self) -> Option>; fn try_into_address_list(self) -> Option>>; fn try_into_address(self) -> Option>; } pub trait BuildHeader<'x>: Sized { fn build_header( self, header: HeaderProperty, value: Value<'x, EmailProperty, EmailValue>, ) -> Result; } impl HeaderToValue for Vec> { fn header_to_value( &self, property: &EmailProperty, raw_message: &ChainedBytes<'_>, ) -> Value<'static, EmailProperty, EmailValue> { let (header_name, form, all) = match property { EmailProperty::Header(header) => ( HeaderName::parse(header.header.as_str()) .unwrap_or_else(|| HeaderName::Other(header.header.as_str().into())), header.form, header.all, ), EmailProperty::Sender => (HeaderName::Sender, HeaderForm::Addresses, false), EmailProperty::From => (HeaderName::From, HeaderForm::Addresses, false), EmailProperty::To => (HeaderName::To, HeaderForm::Addresses, false), EmailProperty::Cc => (HeaderName::Cc, HeaderForm::Addresses, false), EmailProperty::Bcc => (HeaderName::Bcc, HeaderForm::Addresses, false), EmailProperty::ReplyTo => (HeaderName::ReplyTo, HeaderForm::Addresses, false), EmailProperty::Subject => (HeaderName::Subject, HeaderForm::Text, false), EmailProperty::MessageId => (HeaderName::MessageId, HeaderForm::MessageIds, false), EmailProperty::InReplyTo => (HeaderName::InReplyTo, HeaderForm::MessageIds, false), EmailProperty::References => (HeaderName::References, HeaderForm::MessageIds, false), EmailProperty::SentAt => (HeaderName::Date, HeaderForm::Date, false), _ => return Value::Null, }; let is_raw = matches!(form, HeaderForm::Raw) || matches!(header_name, HeaderName::Other(_)); let mut headers = Vec::new(); let header_name = header_name.as_str(); for header in self.iter().rev() { if header.name.as_str().eq_ignore_ascii_case(header_name) { let raw_header; let header_value = if is_raw || matches!(header.value, HeaderValue::Empty) { raw_header = raw_message.get(header.offset_start as usize..header.offset_end as usize); if let Some(bytes) = &raw_header { let bytes = bytes.as_ref(); match form { HeaderForm::Raw => { HeaderValue::Text(String::from_utf8_lossy(bytes.trim_end())) } HeaderForm::Text => MessageStream::new(bytes).parse_unstructured(), HeaderForm::Addresses | HeaderForm::GroupedAddresses | HeaderForm::URLs => MessageStream::new(bytes).parse_address(), HeaderForm::MessageIds => MessageStream::new(bytes).parse_id(), HeaderForm::Date => MessageStream::new(bytes).parse_date(), } } else { HeaderValue::Empty } } else { header.value.clone() }; headers.push(header_value.into_form(&form)); if !all { break; } } } if !all { headers.pop().unwrap_or_default() } else { if headers.len() > 1 { headers.reverse(); } Value::Array(headers) } } fn headers_to_value( &self, raw_message: &ChainedBytes<'_>, ) -> Value<'static, EmailProperty, EmailValue> { let mut headers = Vec::with_capacity(self.len()); for header in self.iter() { headers.push(Value::Object( Map::with_capacity(2) .with_key_value(EmailProperty::Name, header.name().to_string()) .with_key_value( EmailProperty::Value, String::from_utf8_lossy( raw_message .get(header.offset_start as usize..header.offset_end as usize) .unwrap_or_default() .as_ref() .trim_end(), ) .into_owned(), ), )); } headers.into() } } impl IntoForm for HeaderValue<'_> { fn into_form(self, form: &HeaderForm) -> Value<'static, EmailProperty, EmailValue> { match (self, form) { (HeaderValue::Text(text), HeaderForm::Raw | HeaderForm::Text) => { text.into_owned().into() } (HeaderValue::TextList(texts), HeaderForm::Raw | HeaderForm::Text) => { texts.join(", ").into() } (HeaderValue::Text(text), HeaderForm::MessageIds) => { Value::Array(vec![text.into_owned().into()]) } (HeaderValue::TextList(texts), HeaderForm::MessageIds) => { Value::Array(texts.into_iter().map(|t| t.into_owned().into()).collect()) } (HeaderValue::DateTime(datetime), HeaderForm::Date) => from_mail_datetime(datetime), (HeaderValue::Address(mail_parser::Address::List(addrlist)), HeaderForm::URLs) => { Value::Array( addrlist .into_iter() .filter_map(|addr| match addr { Addr { address: Some(addr), .. } if addr.contains(':') => Some(addr.into_owned().into()), _ => None, }) .collect(), ) } (HeaderValue::Address(mail_parser::Address::List(addrlist)), HeaderForm::Addresses) => { from_mail_addrlist(addrlist) } ( HeaderValue::Address(mail_parser::Address::Group(grouplist)), HeaderForm::Addresses, ) => Value::Array( grouplist .into_iter() .flat_map(|group| group.addresses.into_iter().map(from_mail_addr)) .collect(), ), ( HeaderValue::Address(mail_parser::Address::List(addrlist)), HeaderForm::GroupedAddresses, ) => Value::Array(vec![ Map::with_capacity(2) .with_key_value(EmailProperty::Name, Value::Null) .with_key_value(EmailProperty::Addresses, from_mail_addrlist(addrlist)) .into(), ]), ( HeaderValue::Address(mail_parser::Address::Group(grouplist)), HeaderForm::GroupedAddresses, ) => Value::Array( grouplist .into_iter() .map(from_mail_group) .collect::>>(), ), _ => Value::Null, } } } impl<'x> ValueToHeader<'x> for Value<'x, EmailProperty, EmailValue> { fn try_into_grouped_addresses(self) -> Option> { let mut obj = self.into_object()?; Some(GroupedAddresses { name: obj .remove(&Key::Property(EmailProperty::Name)) .and_then(|n| n.into_string()), addresses: obj .remove(&Key::Property(EmailProperty::Addresses))? .try_into_address_list()?, }) } fn try_into_address_list(self) -> Option>> { let list = self.into_array()?; let mut addresses = Vec::with_capacity(list.len()); for value in list { addresses.push(Address::Address(value.try_into_address()?)); } Some(addresses) } fn try_into_address(self) -> Option> { let mut obj = self.into_object()?; Some(EmailAddress { name: obj .remove(&Key::Property(EmailProperty::Name)) .and_then(|n| n.into_string()), email: obj .remove(&Key::Property(EmailProperty::Email))? .into_string()?, }) } } impl<'x> BuildHeader<'x> for MessageBuilder<'x> { fn build_header( self, header: HeaderProperty, value: Value<'x, EmailProperty, EmailValue>, ) -> Result { Ok(match (&header.form, header.all, value) { (HeaderForm::Raw, false, Value::Str(value)) => { self.header(header.header, Raw::from(value)) } (HeaderForm::Raw, true, Value::Array(value)) => self.headers( header.header, value .into_iter() .filter_map(|v| Raw::from(v.into_string()?).into()), ), (HeaderForm::Date, false, Value::Element(EmailValue::Date(value))) => { self.header(header.header, Date::new(value.timestamp())) } (HeaderForm::Date, true, Value::Array(value)) => self.headers( header.header, value .into_iter() .filter_map(|v| Date::new(unwrap_date(v)?.timestamp()).into()), ), (HeaderForm::Text, false, Value::Str(value)) => { self.header(header.header, Text::from(value)) } (HeaderForm::Text, true, Value::Array(value)) => self.headers( header.header, value .into_iter() .filter_map(|v| Text::from(v.into_string()?).into()), ), (HeaderForm::URLs, false, Value::Array(value)) => self.header( header.header, URL { url: value .into_iter() .filter_map(|v| v.into_string()?.into()) .collect(), }, ), (HeaderForm::URLs, true, Value::Array(value)) => self.headers( header.header, value.into_iter().filter_map(|value| { URL { url: value .into_array()? .into_iter() .filter_map(|v| v.into_string()?.into()) .collect(), } .into() }), ), (HeaderForm::MessageIds, false, Value::Array(value)) => self.header( header.header, MessageId { id: value .into_iter() .filter_map(|v| v.into_string()?.into()) .collect(), }, ), (HeaderForm::MessageIds, true, Value::Array(value)) => self.headers( header.header, value.into_iter().filter_map(|value| { MessageId { id: value .into_array()? .into_iter() .filter_map(|v| v.into_string()?.into()) .collect(), } .into() }), ), (HeaderForm::Addresses, false, Value::Array(value)) => self.header( header.header, Address::new_list( value .into_iter() .filter_map(|v| Address::Address(v.try_into_address()?).into()) .collect(), ), ), (HeaderForm::Addresses, true, Value::Array(value)) => self.headers( header.header, value .into_iter() .filter_map(|v| Address::new_list(v.try_into_address_list()?).into()), ), (HeaderForm::GroupedAddresses, false, Value::Array(value)) => self.header( header.header, Address::new_list( value .into_iter() .filter_map(|v| Address::Group(v.try_into_grouped_addresses()?).into()) .collect(), ), ), (HeaderForm::GroupedAddresses, true, Value::Array(value)) => self.headers( header.header, value.into_iter().filter_map(|v| { Address::new_list( v.into_array()? .into_iter() .filter_map(|v| Address::Group(v.try_into_grouped_addresses()?).into()) .collect::>(), ) .into() }), ), _ => { return Err(header); } }) } } impl HeaderToValue for ArchivedMessageMetadataPart { fn header_to_value( &self, property: &EmailProperty, raw_message: &ChainedBytes<'_>, ) -> Value<'static, EmailProperty, EmailValue> { let (header_name, form, all) = match property { EmailProperty::Header(header) => ( HeaderName::parse(header.header.as_str()) .unwrap_or_else(|| HeaderName::Other(header.header.as_str().into())), header.form, header.all, ), EmailProperty::Sender => (HeaderName::Sender, HeaderForm::Addresses, false), EmailProperty::From => (HeaderName::From, HeaderForm::Addresses, false), EmailProperty::To => (HeaderName::To, HeaderForm::Addresses, false), EmailProperty::Cc => (HeaderName::Cc, HeaderForm::Addresses, false), EmailProperty::Bcc => (HeaderName::Bcc, HeaderForm::Addresses, false), EmailProperty::ReplyTo => (HeaderName::ReplyTo, HeaderForm::Addresses, false), EmailProperty::Subject => (HeaderName::Subject, HeaderForm::Text, false), EmailProperty::MessageId => (HeaderName::MessageId, HeaderForm::MessageIds, false), EmailProperty::InReplyTo => (HeaderName::InReplyTo, HeaderForm::MessageIds, false), EmailProperty::References => (HeaderName::References, HeaderForm::MessageIds, false), EmailProperty::SentAt => (HeaderName::Date, HeaderForm::Date, false), _ => return Value::Null, }; let is_raw = matches!(form, HeaderForm::Raw) || matches!(header_name, HeaderName::Other(_)); let mut headers = Vec::new(); let header_name = header_name.as_str(); for header in self.headers.iter().rev() { if header.name.as_str().eq_ignore_ascii_case(header_name) { let raw_header; let header_value = if is_raw || matches!(header.value, ArchivedMetadataHeaderValue::Empty) { raw_header = raw_message.get(header.value_range()); if let Some(bytes) = &raw_header { let bytes = bytes.as_ref(); match form { HeaderForm::Raw => { HeaderValue::Text(String::from_utf8_lossy(bytes.trim_end())) } HeaderForm::Text => MessageStream::new(bytes).parse_unstructured(), HeaderForm::Addresses | HeaderForm::GroupedAddresses | HeaderForm::URLs => MessageStream::new(bytes).parse_address(), HeaderForm::MessageIds => MessageStream::new(bytes).parse_id(), HeaderForm::Date => MessageStream::new(bytes).parse_date(), } } else { HeaderValue::Empty } } else { HeaderValue::from(&header.value) }; headers.push(header_value.into_form(&form)); if !all { break; } } } if !all { headers.pop().unwrap_or_default() } else { if headers.len() > 1 { headers.reverse(); } Value::Array(headers) } } fn headers_to_value( &self, raw_message: &ChainedBytes<'_>, ) -> Value<'static, EmailProperty, EmailValue> { let mut headers = Vec::with_capacity(self.headers.len()); for header in self.headers.iter() { headers.push(Value::Object( Map::with_capacity(2) .with_key_value(EmailProperty::Name, header.name.as_str().to_string()) .with_key_value( EmailProperty::Value, String::from_utf8_lossy( raw_message .get(header.value_range()) .unwrap_or_default() .as_ref() .trim_end(), ) .into_owned(), ), )); } headers.into() } } trait ByteTrim { fn trim_end(&self) -> Self; } impl ByteTrim for &[u8] { fn trim_end(&self) -> Self { let mut end = self.len(); while end > 0 && self[end - 1].is_ascii_whitespace() { end -= 1; } &self[..end] } } #[inline] pub(crate) fn unwrap_date(value: Value<'_, EmailProperty, EmailValue>) -> Option { match value { Value::Element(EmailValue::Date(date)) => Some(date), _ => None, } } fn from_mail_datetime(date: DateTime) -> Value<'static, EmailProperty, EmailValue> { Value::Element(EmailValue::Date(UTCDate { year: date.year, month: date.month, day: date.day, hour: date.hour, minute: date.minute, second: date.second, tz_before_gmt: date.tz_before_gmt, tz_hour: date.tz_hour, tz_minute: date.tz_minute, })) } fn from_mail_addr(value: Addr<'_>) -> Value<'static, EmailProperty, EmailValue> { Value::Object( Map::with_capacity(2) .with_key_value(EmailProperty::Name, value.name.map(|v| v.into_owned())) .with_key_value( EmailProperty::Email, value.address.unwrap_or_default().into_owned(), ), ) } fn from_mail_group(group: Group<'_>) -> Value<'static, EmailProperty, EmailValue> { Value::Object( Map::with_capacity(2) .with_key_value(EmailProperty::Name, group.name.map(|v| v.into_owned())) .with_key_value( EmailProperty::Addresses, from_mail_addrlist(group.addresses), ), ) } fn from_mail_addrlist(addrlist: Vec>) -> Value<'static, EmailProperty, EmailValue> { Value::Array( addrlist .into_iter() .map(from_mail_addr) .collect::>>(), ) } ================================================ FILE: crates/jmap/src/email/import.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ blob::download::BlobDownload, changes::state::JmapCacheState, email::ingested_into_object, }; use common::{Server, auth::AccessToken}; use email::{ cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}, mailbox::JUNK_ID, message::ingest::{EmailIngest, IngestEmail, IngestSource}, }; use http_proto::HttpSessionData; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::import::{ImportEmailRequest, ImportEmailResponse}, object::email::EmailProperty, request::MaybeInvalid, types::state::State, }; use mail_parser::MessageParser; use std::future::Future; use types::{acl::Acl, id::Id, keyword::Keyword}; use utils::map::vec_map::VecMap; pub trait EmailImport: Sync + Send { fn email_import( &self, request: ImportEmailRequest, access_token: &AccessToken, session: &HttpSessionData, ) -> impl Future> + Send; } impl EmailImport for Server { async fn email_import( &self, request: ImportEmailRequest, access_token: &AccessToken, session: &HttpSessionData, ) -> trc::Result { // Validate state let account_id = request.account_id.document_id(); let cache = self.get_cached_messages(account_id).await?; let old_state: State = cache.assert_state(false, &request.if_in_state)?; let can_add_mailbox_ids = if access_token.is_shared(account_id) { cache.shared_mailboxes(access_token, Acl::AddItems).into() } else { None }; // Obtain import access token let import_access_token = if account_id != access_token.primary_id() { #[cfg(feature = "test_mode")] { std::sync::Arc::new(AccessToken::from_id(account_id)).into() } #[cfg(not(feature = "test_mode"))] { use trc::AddContext; self.get_access_token(account_id) .await .caused_by(trc::location!())? .into() } } else { None }; let mut response = ImportEmailResponse { account_id: request.account_id, new_state: old_state.clone(), old_state: old_state.into(), created: VecMap::with_capacity(request.emails.len()), not_created: VecMap::new(), }; 'outer: for (id, email) in request.emails { // Validate mailboxIds let mailbox_ids = email .mailbox_ids .unwrap() .into_iter() .filter_map(|m| m.try_unwrap().map(|m| m.document_id())) .collect::>(); if mailbox_ids.is_empty() { response.not_created.append( id, SetError::invalid_properties() .with_property(EmailProperty::MailboxIds) .with_description("Message must belong to at least one mailbox."), ); continue; } for mailbox_id in &mailbox_ids { if !cache.has_mailbox_id(mailbox_id) { response.not_created.append( id, SetError::invalid_properties() .with_property(EmailProperty::MailboxIds) .with_description(format!( "Mailbox {} does not exist.", Id::from(*mailbox_id) )), ); continue 'outer; } else if matches!(&can_add_mailbox_ids, Some(ids) if !ids.contains(*mailbox_id)) { response.not_created.append( id, SetError::forbidden().with_description(format!( "You are not allowed to add messages to mailbox {}.", Id::from(*mailbox_id) )), ); continue 'outer; } } let MaybeInvalid::Value(blob_id) = email.blob_id else { response.not_created.append( id, SetError::invalid_properties() .with_property(EmailProperty::BlobId) .with_description("Invalid blob id."), ); continue; }; // Fetch raw message to import let raw_message = match self.blob_download(&blob_id, access_token).await? { Some(raw_message) => raw_message, None => { response.not_created.append( id, SetError::new(SetErrorType::BlobNotFound) .with_description(format!("BlobId {} not found.", blob_id)), ); continue; } }; // Import message match self .email_ingest(IngestEmail { raw_message: &raw_message, message: MessageParser::new().parse(&raw_message), blob_hash: Some(&blob_id.hash), access_token: import_access_token.as_deref().unwrap_or(access_token), source: IngestSource::Jmap { train_classifier: email .keywords .iter() .any(|k| matches!(k, Keyword::Junk | Keyword::NotJunk)) || mailbox_ids.contains(&JUNK_ID), }, mailbox_ids, keywords: email.keywords, received_at: email.received_at.map(|r| r.into()), session_id: session.session_id, }) .await { Ok(email) => { response .created .append(id, ingested_into_object(email).into()); } Err(mut err) => match err.as_ref() { trc::EventType::Limit(trc::LimitEvent::Quota) => { response.not_created.append( id, SetError::new(SetErrorType::OverQuota) .with_description("You have exceeded your disk quota."), ); } trc::EventType::MessageIngest(trc::MessageIngestEvent::Error) => { response.not_created.append( id, SetError::new(SetErrorType::InvalidEmail).with_description( err.take_value(trc::Key::Reason) .and_then(|v| v.into_string()) .unwrap(), ), ); } _ => { return Err(err); } }, } } // Update state if !response.created.is_empty() { response.new_state = self.get_cached_messages(account_id).await?.get_state(false); } Ok(response) } } ================================================ FILE: crates/jmap/src/email/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use email::message::ingest::IngestedEmail; use jmap_proto::{ error::set::SetError, object::email::{EmailProperty, EmailValue}, }; use jmap_tools::{JsonPointer, JsonPointerItem, Key, Map, Value}; use types::{id::Id, keyword::Keyword}; pub mod body; pub mod copy; pub mod get; pub mod headers; pub mod import; pub mod parse; pub mod query; pub mod set; pub mod snippet; fn ingested_into_object(email: IngestedEmail) -> Map<'static, EmailProperty, EmailValue> { Map::with_capacity(3) .with_key_value( EmailProperty::Id, Id::from_parts(email.thread_id, email.document_id), ) .with_key_value(EmailProperty::ThreadId, Id::from(email.thread_id)) .with_key_value(EmailProperty::BlobId, email.blob_id) .with_key_value(EmailProperty::Size, email.size) } pub(crate) enum PatchResult<'x> { SetKeyword(&'x Keyword), RemoveKeyword(&'x Keyword), AddMailbox(u32), RemoveMailbox(u32), Invalid(SetError), } pub(crate) fn handle_email_patch<'x>( pointer: &'x JsonPointer, value: Value<'_, EmailProperty, EmailValue>, ) -> PatchResult<'x> { let mut pointer_iter = pointer.iter(); match (pointer_iter.next(), pointer_iter.next()) { ( Some(JsonPointerItem::Key(Key::Property(EmailProperty::Keywords))), Some(JsonPointerItem::Key(Key::Property(EmailProperty::Keyword(keyword)))), ) => match value { Value::Bool(true) => return PatchResult::SetKeyword(keyword), Value::Bool(false) | Value::Null => return PatchResult::RemoveKeyword(keyword), _ => (), }, ( Some(JsonPointerItem::Key(Key::Property(EmailProperty::MailboxIds))), Some(JsonPointerItem::Key(Key::Property(EmailProperty::IdValue(id)))), ) => match value { Value::Bool(true) => return PatchResult::AddMailbox(id.document_id()), Value::Bool(false) | Value::Null => { return PatchResult::RemoveMailbox(id.document_id()); } _ => (), }, _ => (), } PatchResult::Invalid( SetError::invalid_properties() .with_property(EmailProperty::Pointer(pointer.clone())) .with_description("Invalid patch value".to_string()), ) } ================================================ FILE: crates/jmap/src/email/parse.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ body::{ToBodyPart, TruncateBody}, headers::HeaderToValue, }; use crate::blob::download::BlobDownload; use common::{Server, auth::AccessToken}; use email::message::index::PREVIEW_LENGTH; use jmap_proto::{ method::parse::{ParseRequest, ParseResponse}, object::email::{Email, EmailProperty}, request::IntoValid, }; use jmap_tools::{Key, Map, Value}; use mail_parser::{ MessageParser, PartType, decoders::html::html_to_text, parsers::preview::preview_text, }; use std::future::Future; use utils::{chained_bytes::ChainedBytes, map::vec_map::VecMap}; pub trait EmailParse: Sync + Send { fn email_parse( &self, request: ParseRequest, access_token: &AccessToken, ) -> impl Future>> + Send; } impl EmailParse for Server { async fn email_parse( &self, request: ParseRequest, access_token: &AccessToken, ) -> trc::Result> { if request.blob_ids.len() > self.core.jmap.mail_parse_max_items { return Err(trc::JmapEvent::RequestTooLarge.into_err()); } let properties = request .properties .map(|v| v.into_valid().collect()) .unwrap_or_else(|| { vec![ EmailProperty::BlobId, EmailProperty::Size, EmailProperty::ReceivedAt, EmailProperty::MessageId, EmailProperty::InReplyTo, EmailProperty::References, EmailProperty::Sender, EmailProperty::From, EmailProperty::To, EmailProperty::Cc, EmailProperty::Bcc, EmailProperty::ReplyTo, EmailProperty::Subject, EmailProperty::SentAt, EmailProperty::HasAttachment, EmailProperty::Preview, EmailProperty::BodyValues, EmailProperty::TextBody, EmailProperty::HtmlBody, EmailProperty::Attachments, ] }); let body_properties = request .arguments .body_properties .map(|v| v.into_valid().collect()) .unwrap_or_else(|| { vec![ EmailProperty::PartId, EmailProperty::BlobId, EmailProperty::Size, EmailProperty::Name, EmailProperty::Type, EmailProperty::Charset, EmailProperty::Disposition, EmailProperty::Cid, EmailProperty::Language, EmailProperty::Location, ] }); let fetch_text_body_values = request.arguments.fetch_text_body_values.unwrap_or(false); let fetch_html_body_values = request.arguments.fetch_html_body_values.unwrap_or(false); let fetch_all_body_values = request.arguments.fetch_all_body_values.unwrap_or(false); let max_body_value_bytes = request.arguments.max_body_value_bytes.unwrap_or(0); let mut response = ParseResponse { account_id: request.account_id, parsed: VecMap::with_capacity(request.blob_ids.len()), not_parsable: vec![], not_found: vec![], }; for blob_id in request.blob_ids.into_valid() { // Fetch raw message to parse let raw_message = match self.blob_download(&blob_id, access_token).await? { Some(raw_message) => raw_message, None => { response.not_found.push(blob_id); continue; } }; let message = if let Some(message) = MessageParser::new().parse(&raw_message) { message } else { response.not_parsable.push(blob_id); continue; }; let raw_message = ChainedBytes::new(&raw_message); // Prepare response let mut email = Map::with_capacity(properties.len()); for property in &properties { match property { EmailProperty::BlobId => { email.insert_unchecked(EmailProperty::BlobId, blob_id.clone()); } EmailProperty::Size => { email.insert_unchecked( EmailProperty::Size, Value::Number(raw_message.len().into()), ); } EmailProperty::HasAttachment => { email.insert_unchecked( EmailProperty::HasAttachment, Value::Bool(message.parts.iter().enumerate().any(|(part_id, part)| { let part_id = part_id as u32; match &part.body { PartType::Html(_) | PartType::Text(_) => { !message.text_body.contains(&part_id) && !message.html_body.contains(&part_id) } PartType::Binary(_) | PartType::Message(_) => true, _ => false, } })), ); } EmailProperty::Preview => { email.insert_unchecked( EmailProperty::Preview, match message .text_body .first() .or_else(|| message.html_body.first()) .and_then(|idx| message.parts.get(*idx as usize)) .map(|part| &part.body) { Some(PartType::Text(text)) => { preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH) .into() } Some(PartType::Html(html)) => preview_text( html_to_text(html).replace('\r', "").into(), PREVIEW_LENGTH, ) .into(), _ => Value::Null, }, ); } EmailProperty::MessageId | EmailProperty::InReplyTo | EmailProperty::References | EmailProperty::Sender | EmailProperty::From | EmailProperty::To | EmailProperty::Cc | EmailProperty::Bcc | EmailProperty::ReplyTo | EmailProperty::Subject | EmailProperty::SentAt | EmailProperty::Header(_) => { email.insert_unchecked( property.clone(), message.parts[0] .headers .header_to_value(property, &raw_message), ); } EmailProperty::Headers => { email.insert_unchecked( EmailProperty::Headers, message.parts[0].headers.headers_to_value(&raw_message), ); } EmailProperty::TextBody | EmailProperty::HtmlBody | EmailProperty::Attachments => { let list = match property { EmailProperty::TextBody => &message.text_body, EmailProperty::HtmlBody => &message.html_body, EmailProperty::Attachments => &message.attachments, _ => unreachable!(), } .iter(); email.insert_unchecked( property.clone(), list.map(|part_id| { message.parts.to_body_part( *part_id, &body_properties, &raw_message, &blob_id, 0, ) }) .collect::>(), ); } EmailProperty::BodyStructure => { email.insert_unchecked( EmailProperty::BodyStructure, message.parts.to_body_part( 0, &body_properties, &raw_message, &blob_id, 0, ), ); } EmailProperty::BodyValues => { let mut body_values = Map::with_capacity(message.parts.len()); for (part_id, part) in message.parts.iter().enumerate() { let part_id = part_id as u32; if ((message.html_body.contains(&part_id) && (fetch_all_body_values || fetch_html_body_values)) || (message.text_body.contains(&part_id) && (fetch_all_body_values || fetch_text_body_values))) && part.is_text() { let (is_truncated, value) = part.body.truncate(max_body_value_bytes); body_values.insert_unchecked( Key::Owned(part_id.to_string()), Map::with_capacity(3) .with_key_value( EmailProperty::IsEncodingProblem, part.is_encoding_problem, ) .with_key_value(EmailProperty::IsTruncated, is_truncated) .with_key_value(EmailProperty::Value, value), ); } } email.insert_unchecked(EmailProperty::BodyValues, body_values); } EmailProperty::Id | EmailProperty::ThreadId | EmailProperty::Keywords | EmailProperty::MailboxIds | EmailProperty::ReceivedAt => { email.insert_unchecked(property.clone(), Value::Null); } _ => { return Err(trc::JmapEvent::InvalidArguments .into_err() .details(format!("Invalid property {property:?}"))); } } } response.parsed.append(blob_id, email.into()); } Ok(response) } } ================================================ FILE: crates/jmap/src/email/query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{api::query::QueryResponseBuilder, changes::state::JmapCacheState}; use common::{MessageStoreCache, Server, auth::AccessToken}; use email::cache::{MessageCacheFetch, email::MessageCacheAccess}; use jmap_proto::{ method::query::{Filter, QueryRequest, QueryResponse}, object::email::{Email, EmailComparator, EmailFilter}, }; use mail_parser::HeaderName; use nlp::language::Language; use std::future::Future; use store::{ ahash::{AHashMap, AHashSet}, roaring::RoaringBitmap, search::{ EmailSearchField, SearchComparator, SearchFilter, SearchOperator, SearchQuery, SearchValue, }, write::SearchIndex, }; use trc::AddContext; use types::{acl::Acl, keyword::Keyword}; use utils::map::vec_map::VecMap; pub trait EmailQuery: Sync + Send { fn email_query( &self, request: QueryRequest, access_token: &AccessToken, ) -> impl Future> + Send; } impl EmailQuery for Server { async fn email_query( &self, mut request: QueryRequest, access_token: &AccessToken, ) -> trc::Result { let account_id = request.account_id.document_id(); let mut filters = Vec::with_capacity(request.filter.len()); let cached_messages = self .get_cached_messages(account_id) .await .caused_by(trc::location!())?; for filter in std::mem::take(&mut request.filter) { match filter { Filter::Property(cond) => match cond { EmailFilter::Text(text) => { let (text, language) = Language::detect(text, self.core.jmap.default_language); filters.push(SearchFilter::Or); filters.push(SearchFilter::has_text( EmailSearchField::From, &text, Language::None, )); filters.push(SearchFilter::has_text( EmailSearchField::To, &text, Language::None, )); filters.push(SearchFilter::has_text( EmailSearchField::Cc, &text, Language::None, )); filters.push(SearchFilter::has_text( EmailSearchField::Bcc, &text, Language::None, )); filters.push(SearchFilter::has_text( EmailSearchField::Subject, &text, language, )); filters.push(SearchFilter::has_text( EmailSearchField::Body, &text, language, )); filters.push(SearchFilter::has_text( EmailSearchField::Attachment, text, language, )); filters.push(SearchFilter::End); } EmailFilter::From(text) => filters.push(SearchFilter::has_text( EmailSearchField::From, text, Language::None, )), EmailFilter::To(text) => filters.push(SearchFilter::has_text( EmailSearchField::To, text, Language::None, )), EmailFilter::Cc(text) => filters.push(SearchFilter::has_text( EmailSearchField::Cc, text, Language::None, )), EmailFilter::Bcc(text) => filters.push(SearchFilter::has_text( EmailSearchField::Bcc, text, Language::None, )), EmailFilter::Subject(text) => filters.push(SearchFilter::has_text_detect( EmailSearchField::Subject, text, self.core.jmap.default_language, )), EmailFilter::Body(text) => filters.push(SearchFilter::has_text_detect( EmailSearchField::Body, text, self.core.jmap.default_language, )), EmailFilter::Header(header) => { let mut header = header.into_iter(); let header_name = header.next().ok_or_else(|| { trc::JmapEvent::InvalidArguments .into_err() .details("Header name is missing.".to_string()) })?; if let Some(header_name) = HeaderName::parse(header_name) { let value = header.next(); let op = if matches!( header_name, HeaderName::MessageId | HeaderName::InReplyTo | HeaderName::References | HeaderName::ResentMessageId ) || value.is_none() { SearchOperator::Equal } else { SearchOperator::Contains }; filters.push(SearchFilter::cond( EmailSearchField::Headers, op, SearchValue::KeyValues(VecMap::with_capacity(1).with_append( header_name.as_str().to_lowercase(), value.unwrap_or_default(), )), )); } } EmailFilter::InMailbox(mailbox) => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cached_messages .in_mailbox(mailbox.document_id()) .map(|item| item.document_id), ))) } EmailFilter::InMailboxOtherThan(mailboxes) => { let mailboxes = mailboxes .into_iter() .map(|m| m.document_id()) .collect::>(); filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cached_messages.emails.items.iter().filter_map(|item| { if item .mailboxes .iter() .any(|mb| mailboxes.contains(&mb.mailbox_id)) { None } else { Some(item.document_id) } }), ))); } EmailFilter::Before(date) => filters.push(SearchFilter::lt( EmailSearchField::ReceivedAt, date.timestamp(), )), EmailFilter::After(date) => filters.push(SearchFilter::gt( EmailSearchField::ReceivedAt, date.timestamp(), )), EmailFilter::MinSize(size) => { filters.push(SearchFilter::ge(EmailSearchField::Size, size)) } EmailFilter::MaxSize(size) => { filters.push(SearchFilter::lt(EmailSearchField::Size, size)) } EmailFilter::AllInThreadHaveKeyword(keyword) => filters.push( SearchFilter::is_in_set(thread_keywords(&cached_messages, keyword, true)), ), EmailFilter::SomeInThreadHaveKeyword(keyword) => filters.push( SearchFilter::is_in_set(thread_keywords(&cached_messages, keyword, false)), ), EmailFilter::NoneInThreadHaveKeyword(keyword) => { filters.push(SearchFilter::Not); filters.push(SearchFilter::is_in_set(thread_keywords( &cached_messages, keyword, false, ))); filters.push(SearchFilter::End); } EmailFilter::HasKeyword(keyword) => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cached_messages .with_keyword(&keyword) .map(|item| item.document_id), ))); } EmailFilter::NotKeyword(keyword) => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cached_messages .without_keyword(&keyword) .map(|item| item.document_id), ))); } EmailFilter::HasAttachment(has_attach) => { filters.push(SearchFilter::eq( EmailSearchField::HasAttachment, has_attach, )); } // Non-standard EmailFilter::Id(ids) => { let mut set = RoaringBitmap::new(); for id in ids { set.insert(id.document_id()); } filters.push(SearchFilter::is_in_set(set)); } EmailFilter::SentBefore(date) => { filters.push(SearchFilter::lt(EmailSearchField::SentAt, date.timestamp())) } EmailFilter::SentAfter(date) => { filters.push(SearchFilter::gt(EmailSearchField::SentAt, date.timestamp())) } EmailFilter::InThread(id) => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cached_messages .in_thread(id.document_id()) .map(|item| item.document_id), ))) } other => { return Err(trc::JmapEvent::UnsupportedFilter .into_err() .details(other.to_string())); } }, Filter::And => { filters.push(SearchFilter::And); } Filter::Or => { filters.push(SearchFilter::Or); } Filter::Not => { filters.push(SearchFilter::Not); } Filter::Close => { filters.push(SearchFilter::End); } } } // Parse sort criteria let mut comparators = Vec::with_capacity(request.sort.as_ref().map_or(1, |s| s.len())); for comparator in request .sort .take() .filter(|s| !s.is_empty()) .unwrap_or_default() { comparators.push(match comparator.property { EmailComparator::ReceivedAt => { SearchComparator::field(EmailSearchField::ReceivedAt, comparator.is_ascending) } EmailComparator::Size => { SearchComparator::field(EmailSearchField::Size, comparator.is_ascending) } EmailComparator::From => { SearchComparator::field(EmailSearchField::From, comparator.is_ascending) } EmailComparator::To => { SearchComparator::field(EmailSearchField::To, comparator.is_ascending) } EmailComparator::Subject => { SearchComparator::field(EmailSearchField::Subject, comparator.is_ascending) } EmailComparator::SentAt => { SearchComparator::field(EmailSearchField::SentAt, comparator.is_ascending) } EmailComparator::HasKeyword(keyword) => SearchComparator::set( RoaringBitmap::from_iter( cached_messages .with_keyword(&keyword) .map(|item| item.document_id), ), comparator.is_ascending, ), EmailComparator::AllInThreadHaveKeyword(keyword) => SearchComparator::set( thread_keywords(&cached_messages, keyword, true), comparator.is_ascending, ), EmailComparator::SomeInThreadHaveKeyword(keyword) => SearchComparator::set( thread_keywords(&cached_messages, keyword, false), comparator.is_ascending, ), // Non-standard EmailComparator::Cc => { SearchComparator::field(EmailSearchField::Cc, comparator.is_ascending) } other => { return Err(trc::JmapEvent::UnsupportedSort .into_err() .details(other.to_string())); } }); } let results = self .search_store() .query_account( SearchQuery::new(SearchIndex::Email) .with_filters(filters) .with_comparators(comparators) .with_account_id(account_id) .with_mask(if access_token.is_shared(account_id) { cached_messages.shared_messages(access_token, Acl::ReadItems) } else { cached_messages .emails .items .iter() .map(|item| item.document_id) .collect() }), ) .await?; let mut response = QueryResponseBuilder::new( results.len(), self.core.jmap.query_max_results, cached_messages.get_state(false), &request, ); if !results.is_empty() { let collapse_threads = request.arguments.collapse_threads.unwrap_or(false); let mut seen_thread_ids = AHashSet::new(); for document_id in results { let Some(thread_id) = cached_messages .email_by_id(&document_id) .map(|email| email.thread_id) else { continue; }; if collapse_threads && !seen_thread_ids.insert(thread_id) { continue; } if !response.add(thread_id, document_id) { break; } } } response.build() } } fn thread_keywords(cache: &MessageStoreCache, keyword: Keyword, match_all: bool) -> RoaringBitmap { let keyword_doc_ids = RoaringBitmap::from_iter(cache.with_keyword(&keyword).map(|item| item.document_id)); if keyword_doc_ids.is_empty() { return keyword_doc_ids; } let mut not_matched_ids = RoaringBitmap::new(); let mut matched_ids = RoaringBitmap::new(); let mut thread_map: AHashMap = AHashMap::new(); for item in &cache.emails.items { thread_map .entry(item.thread_id) .or_default() .insert(item.document_id); } for item in &cache.emails.items { let keyword_doc_id = item.document_id; if !keyword_doc_ids.contains(keyword_doc_id) || matched_ids.contains(keyword_doc_id) || not_matched_ids.contains(keyword_doc_id) { continue; } if let Some(thread_doc_ids) = thread_map.get(&item.thread_id) { let mut thread_tag_intersection = thread_doc_ids.clone(); thread_tag_intersection &= &keyword_doc_ids; if (match_all && &thread_tag_intersection == thread_doc_ids) || (!match_all && !thread_tag_intersection.is_empty()) { matched_ids |= thread_doc_ids; } else if !thread_tag_intersection.is_empty() { not_matched_ids |= &thread_tag_intersection; } } } matched_ids } ================================================ FILE: crates/jmap/src/email/set.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::headers::{BuildHeader, ValueToHeader}; use crate::{ blob::download::BlobDownload, changes::state::JmapCacheState, email::{PatchResult, handle_email_patch, ingested_into_object}, }; use common::{ Server, auth::AccessToken, ipc::PushNotification, storage::index::ObjectIndexBuilder, }; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess, mailbox::MailboxCacheAccess}, mailbox::{JUNK_ID, TRASH_ID, UidMailbox}, message::{ delete::EmailDeletion, ingest::{EmailIngest, IngestEmail, IngestSource}, metadata::MessageData, }, }; use http_proto::HttpSessionData; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{SetRequest, SetResponse}, object::email::{Email, EmailProperty, EmailValue}, references::resolve::ResolveCreatedReference, request::IntoValid, types::state::State, }; use jmap_tools::{Key, Value}; use mail_builder::{ MessageBuilder, headers::{ HeaderType, address::Address, content_type::ContentType, date::Date, message_id::MessageId, raw::Raw, text::Text, }, mime::{BodyPart, MimePart}, }; use mail_parser::MessageParser; use std::future::Future; use std::{borrow::Cow, collections::HashMap}; use store::{ ValueKey, ahash::AHashMap, roaring::RoaringBitmap, write::{AlignedBytes, Archive, BatchBuilder}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection, VanishedCollection}, id::Id, keyword::{ArchivedKeyword, Keyword}, type_state::{DataType, StateChange}, }; pub trait EmailSet: Sync + Send { fn email_set( &self, request: SetRequest<'_, Email>, access_token: &AccessToken, session: &HttpSessionData, ) -> impl Future>> + Send; } impl EmailSet for Server { async fn email_set( &self, mut request: SetRequest<'_, Email>, access_token: &AccessToken, session: &HttpSessionData, ) -> trc::Result> { // Prepare response let account_id = request.account_id.document_id(); let cache = self.get_cached_messages(account_id).await?; let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)? .with_state(cache.assert_state(false, &request.if_in_state)?); // Obtain mailboxIds let (can_add_mailbox_ids, can_delete_mailbox_ids, can_modify_mailbox_ids) = if access_token.is_shared(account_id) { ( cache.shared_mailboxes(access_token, Acl::AddItems).into(), cache .shared_mailboxes(access_token, Acl::RemoveItems) .into(), cache .shared_mailboxes(access_token, Acl::ModifyItems) .into(), ) } else { (None, None, None) }; // Obtain import access token let import_access_token = if account_id != access_token.primary_id() { #[cfg(feature = "test_mode")] { std::sync::Arc::new(AccessToken::from_id(account_id)).into() } #[cfg(not(feature = "test_mode"))] { self.get_access_token(account_id) .await .caused_by(trc::location!())? .into() } } else { None }; let mut last_change_id = None; let will_destroy = request.unwrap_destroy().into_valid().collect::>(); // Process creates 'create: for (id, object) in request.unwrap_create() { let Value::Object(mut object) = object else { continue; }; let has_body_structure = object.contains_key(&Key::Property(EmailProperty::BodyStructure)); let mut builder = MessageBuilder::new(); let mut mailboxes = Vec::new(); let mut keywords = Vec::new(); let mut received_at = None; // Parse body values let body_values = object .remove(&Key::Property(EmailProperty::BodyValues)) .and_then(|obj| obj.into_object()) .and_then(|obj| { let mut values = HashMap::with_capacity(obj.len()); for (key, value) in obj.into_vec() { let id = key.into_string(); if let Value::Object(mut bv) = value { values.insert( id, bv.remove(&Key::Property(EmailProperty::Value))? .into_string()?, ); } else { return None; } } Some(values) }); let mut size_attachments = 0; // Parse properties for (property, mut value) in object.into_vec() { if let Err(err) = response.resolve_self_references(&mut value) { response.not_created.append(id, err); continue 'create; }; let Key::Property(property) = property else { response.invalid_property_create(id, property.into_owned()); continue 'create; }; match (property, value) { (EmailProperty::MailboxIds, Value::Object(ids)) => { mailboxes = ids .into_expanded_boolean_set() .filter_map(|id| { id.try_into_property()?.try_into_id()?.document_id().into() }) .collect(); } (EmailProperty::Keywords, Value::Object(keywords_)) => { keywords = keywords_ .into_expanded_boolean_set() .filter_map(|id| id.try_into_property()?.try_into_keyword()) .collect(); } (EmailProperty::Pointer(pointer), value) => { match handle_email_patch(&pointer, value) { PatchResult::SetKeyword(keyword) => { if !keywords.contains(keyword) { keywords.push(keyword.clone()); } } PatchResult::RemoveKeyword(keyword) => { keywords.retain(|k| k != keyword); } PatchResult::AddMailbox(id) => { if !mailboxes.contains(&id) { mailboxes.push(id); } } PatchResult::RemoveMailbox(id) => { mailboxes.retain(|mid| mid != &id); } PatchResult::Invalid(set_error) => { response.not_created.append(id, set_error); continue 'create; } } } ( header @ (EmailProperty::MessageId | EmailProperty::InReplyTo | EmailProperty::References), Value::Array(values), ) => { builder = builder.header( header.as_rfc_header(), MessageId { id: values .into_iter() .filter_map(|value| value.into_string()) .collect(), }, ); } ( header @ (EmailProperty::Sender | EmailProperty::From | EmailProperty::To | EmailProperty::Cc | EmailProperty::Bcc | EmailProperty::ReplyTo), value, ) => { if let Some(addresses) = value.try_into_address_list() { builder = builder.header(header.as_rfc_header(), Address::List(addresses)); } else { response.invalid_property_create(id, header); continue 'create; } } (EmailProperty::Subject, Value::Str(value)) => { builder = builder.subject(value); } (EmailProperty::ReceivedAt, Value::Element(EmailValue::Date(value))) => { received_at = (value.timestamp() as u64).into(); } (EmailProperty::SentAt, Value::Element(EmailValue::Date(value))) => { builder = builder.date(Date::new(value.timestamp())); } ( property @ (EmailProperty::TextBody | EmailProperty::HtmlBody | EmailProperty::Attachments | EmailProperty::BodyStructure), value, ) => { // Validate request let (values, expected_content_type) = match property { EmailProperty::BodyStructure => (vec![value], None), EmailProperty::TextBody | EmailProperty::HtmlBody if !has_body_structure => { let values = value.into_array().unwrap_or_default(); if values.len() <= 1 { ( values, Some(match property { EmailProperty::TextBody => "text/plain", EmailProperty::HtmlBody => "text/html", _ => unreachable!(), }), ) } else { response.not_created.append( id, SetError::invalid_properties() .with_property(property) .with_description("Only one part is allowed."), ); continue 'create; } } EmailProperty::Attachments if !has_body_structure => { (value.into_array().unwrap_or_default(), None) } _ => { response.not_created.append( id, SetError::invalid_properties() .with_properties([property, EmailProperty::BodyStructure]) .with_description( "Cannot set both properties on a same request.", ), ); continue 'create; } }; // Iterate parts let mut values_stack = Vec::new(); let mut values = values.into_iter(); let mut parts = Vec::new(); loop { while let Some(value) = values.next() { let mut blob_id = None; let mut part_id = None; let mut content_type = None; let mut content_disposition = None; let mut name = None; let mut charset = None; let mut subparts = None; let mut has_size = false; let mut headers: Vec<(Cow, HeaderType)> = Vec::new(); if let Some(obj) = value.into_object() { for (body_property, value) in obj.into_vec() { let Key::Property(body_property) = body_property else { continue; }; match (body_property, value) { (EmailProperty::Type, Value::Str(value)) => { content_type = value.into_owned().into(); } (EmailProperty::PartId, Value::Str(value)) => { part_id = value.into_owned().into(); } ( EmailProperty::BlobId, Value::Element(EmailValue::BlobId(value)), ) => { blob_id = value.into(); } (EmailProperty::Disposition, Value::Str(value)) => { content_disposition = value.into_owned().into(); } (EmailProperty::Name, Value::Str(value)) => { name = value.into_owned().into(); } (EmailProperty::Charset, Value::Str(value)) => { charset = value.into_owned().into(); } (EmailProperty::Language, Value::Array(values)) => { headers.push(( "Content-Language".into(), Text::new( values .into_iter() .filter_map(|v| v.into_string()) .fold( String::with_capacity(64), |mut h, v| { if !h.is_empty() { h.push_str(", "); } h.push_str(&v); h }, ), ) .into(), )); } (EmailProperty::Cid, Value::Str(value)) => { headers.push(( "Content-ID".into(), MessageId::new(value).into(), )); } (EmailProperty::Location, Value::Str(value)) => { headers.push(( "Content-Location".into(), Text::new(value).into(), )); } (EmailProperty::Header(header), Value::Str(value)) if !header.header.eq_ignore_ascii_case( "content-transfer-encoding", ) => { headers.push(( header.header.into(), Raw::from(value).into(), )); } ( EmailProperty::Header(header), Value::Array(values), ) if !header.header.eq_ignore_ascii_case( "content-transfer-encoding", ) => { for value in values { if let Some(value) = value.into_string() { headers.push(( header.header.clone().into(), Raw::from(value).into(), )); } } } (EmailProperty::Headers, _) => { response.not_created.append( id, SetError::invalid_properties() .with_property(( property, EmailProperty::Headers, )) .with_description( "Headers have to be set individually.", ), ); continue 'create; } (EmailProperty::Size, _) => { has_size = true; } (EmailProperty::SubParts, Value::Array(values)) => { subparts = values.into(); } (body_property, value) if value != Value::Null => { response.not_created.append( id, SetError::invalid_properties() .with_property((property, body_property)) .with_description("Cannot set property."), ); continue 'create; } _ => {} } } } // Validate content-type let content_type = content_type.unwrap_or_else(|| "text/plain".to_string()); let is_multipart = content_type.starts_with("multipart/"); if is_multipart { if !matches!(property, EmailProperty::BodyStructure) { response.not_created.append( id, SetError::invalid_properties() .with_property((property, EmailProperty::Type)) .with_description("Multiparts can only be set with bodyStructure."), ); continue 'create; } } else if expected_content_type .as_ref() .is_some_and(|v| v != &content_type) { response.not_created.append( id, SetError::invalid_properties() .with_property((property, EmailProperty::Type)) .with_description(format!( "Expected one body part of type \"{}\"", expected_content_type.unwrap() )), ); continue 'create; } // Validate partId/blobId match (blob_id.is_some(), part_id.is_some()) { (true, true) if !is_multipart => { response.not_created.append( id, SetError::invalid_properties() .with_properties([(property.clone(), EmailProperty::BlobId), (property, EmailProperty::PartId)]) .with_description( "Cannot specify both \"partId\" and \"blobId\".", ), ); continue 'create; } (false, false) if !is_multipart => { response.not_created.append( id, SetError::invalid_properties() .with_description("Expected a \"partId\" or \"blobId\" field in body part."), ); continue 'create; } (false, true) if !is_multipart && has_size => { response.not_created.append( id, SetError::invalid_properties() .with_property((property, EmailProperty::Size)) .with_description( "Cannot specify \"size\" when providing a \"partId\".", ), ); continue 'create; } (true, _) | (_, true) if is_multipart => { response.not_created.append( id, SetError::invalid_properties() .with_properties([(property.clone(), EmailProperty::BlobId), (property, EmailProperty::PartId)]) .with_description( "Cannot specify \"partId\" or \"blobId\" in multipart body parts.", ), ); continue 'create; } _ => (), } // Set Content-Type and Content-Disposition let mut content_type = ContentType::new(content_type); if !is_multipart { if let Some(charset) = charset { if part_id.is_none() { content_type .attributes .push(("charset".into(), charset.into())); } else { response.not_created.append( id, SetError::invalid_properties() .with_property((property, EmailProperty::Charset)) .with_description( "Cannot specify a character set when providing a \"partId\".", ), ); continue 'create; } } else if part_id.is_some() { content_type .attributes .push(("charset".into(), "utf-8".into())); } match (content_disposition, name) { (Some(disposition), Some(filename)) => { headers.push(( "Content-Disposition".into(), ContentType::new(disposition) .attribute("filename", filename) .into(), )); } (Some(disposition), None) => { headers.push(( "Content-Disposition".into(), ContentType::new(disposition).into(), )); } (None, Some(filename)) => { content_type .attributes .push(("name".into(), filename.into())); } (None, None) => (), }; } headers.push(("Content-Type".into(), content_type.into())); // In test, sort headers to avoid randomness #[cfg(feature = "test_mode")] { headers.sort_unstable_by(|a, b| match a.0.cmp(&b.0) { std::cmp::Ordering::Equal => a.1.cmp(&b.1), ord => ord, }); } // Retrieve contents parts.push(MimePart { headers, contents: if !is_multipart { if let Some(blob_id) = blob_id { match self.blob_download(&blob_id, access_token).await? { Some(contents) => { BodyPart::Binary(contents.into()) } None => { response.not_created.append( id, SetError::new(SetErrorType::BlobNotFound).with_description( format!("blobId {blob_id} does not exist on this server.") ), ); continue 'create; } } } else if let Some(part_id) = part_id { if let Some(contents) = body_values.as_ref().and_then(|bv| bv.get(&part_id)) { BodyPart::Text(contents.as_ref().into()) } else { response.not_created.append( id, SetError::invalid_properties() .with_property((property, EmailProperty::PartId)) .with_description(format!( "Missing body value for partId {part_id:?}" )), ); continue 'create; } } else { unreachable!() } } else { BodyPart::Multipart(vec![]) }, }); // Check attachment sizes if !is_multipart { size_attachments += parts.last().unwrap().size(); if self.core.jmap.mail_attachments_max_size > 0 && size_attachments > self.core.jmap.mail_attachments_max_size { response.not_created.append( id, SetError::invalid_properties() .with_property(property) .with_description(format!( "Message exceeds maximum size of {} bytes.", self.core.jmap.mail_attachments_max_size )), ); continue 'create; } } else if let Some(subparts) = subparts { values_stack.push((values, parts)); parts = Vec::with_capacity(subparts.len()); values = subparts.into_iter(); continue; } } if let Some((prev_values, mut prev_parts)) = values_stack.pop() { values = prev_values; prev_parts.last_mut().unwrap().contents = BodyPart::Multipart(parts); parts = prev_parts; } else { break; } } match property { EmailProperty::TextBody => { builder.text_body = parts.pop(); } EmailProperty::HtmlBody => { builder.html_body = parts.pop(); } EmailProperty::Attachments => { builder.attachments = parts.into(); } _ => { builder.body = parts.pop(); } } } (EmailProperty::Header(header), value) => { match builder.build_header(header, value) { Ok(builder_) => { builder = builder_; } Err(header) => { response.invalid_property_create(id, EmailProperty::Header(header)); continue 'create; } } } (_, Value::Null) => (), (property, _) => { response.invalid_property_create(id, property); continue 'create; } } } // Make sure message belongs to at least one mailbox if mailboxes.is_empty() { response.not_created.append( id, SetError::invalid_properties() .with_property(EmailProperty::MailboxIds) .with_description("Message has to belong to at least one mailbox."), ); continue 'create; } // Verify that the mailboxIds are valid for mailbox_id in &mailboxes { if !cache.has_mailbox_id(mailbox_id) { response.not_created.append( id, SetError::invalid_properties() .with_property(EmailProperty::MailboxIds) .with_description(format!( "mailboxId {} does not exist.", Id::from(*mailbox_id) )), ); continue 'create; } else if can_add_mailbox_ids .as_ref() .is_some_and(|ids| !ids.contains(*mailbox_id)) { response.not_created.append( id, SetError::forbidden().with_description(format!( "You are not allowed to add messages to mailbox {}.", Id::from(*mailbox_id) )), ); continue 'create; } } // Make sure the message is not empty if builder.headers.is_empty() && builder.body.is_none() && builder.html_body.is_none() && builder.text_body.is_none() && builder.attachments.is_none() { response.not_created.append( id, SetError::invalid_properties() .with_description("Message has to have at least one header or body part."), ); continue 'create; } // In test, sort headers to avoid randomness #[cfg(feature = "test_mode")] { builder .headers .sort_unstable_by(|a, b| match a.0.cmp(&b.0) { std::cmp::Ordering::Equal => a.1.cmp(&b.1), ord => ord, }); } // Build message let mut raw_message = Vec::with_capacity((4 * size_attachments / 3) + 1024); builder.write_to(&mut raw_message).unwrap_or_default(); // Ingest message match self .email_ingest(IngestEmail { raw_message: &raw_message, message: MessageParser::new().parse(&raw_message), blob_hash: None, access_token: import_access_token.as_deref().unwrap_or(access_token), mailbox_ids: mailboxes, keywords, received_at, source: IngestSource::Jmap { train_classifier: true, }, session_id: session.session_id, }) .await { Ok(message) => { last_change_id = message.change_id.into(); response .created .insert(id, ingested_into_object(message).into()); } Err(err) if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) => { response.not_created.append( id, SetError::new(SetErrorType::OverQuota) .with_description("You have exceeded your disk quota."), ); } Err(err) => return Err(err), } } // Process updates let mut batch = BatchBuilder::new(); let mut changed_mailboxes: AHashMap> = AHashMap::new(); let mut will_update = Vec::with_capacity(request.update.as_ref().map_or(0, |u| u.len())); 'update: for (id, object) in request.unwrap_update().into_valid() { // Make sure id won't be destroyed if will_destroy.contains(&id) { response.not_updated.append(id, SetError::will_destroy()); continue 'update; } // Obtain message data let document_id = id.document_id(); let data_ = match self .store() .get_value::>(ValueKey::archive( account_id, Collection::Email, document_id, )) .await? { Some(data) => data, None => { response.not_updated.append(id, SetError::not_found()); continue 'update; } }; let data = data_ .to_unarchived::() .caused_by(trc::location!())?; let mut new_data = data.inner.to_builder(); for (property, mut value) in object.into_expanded_object() { if let Err(err) = response.resolve_self_references(&mut value) { response.not_updated.append(id, err); continue 'update; }; match (property, value) { (Key::Property(EmailProperty::MailboxIds), Value::Object(ids)) => { new_data.set_mailboxes( ids.into_expanded_boolean_set() .filter_map(|id| { UidMailbox::new_unassigned( id.try_into_property()?.try_into_id()?.document_id(), ) .into() }) .collect(), ); } (Key::Property(EmailProperty::Keywords), Value::Object(keywords_)) => { new_data.set_keywords( keywords_ .into_expanded_boolean_set() .filter_map(|keyword| { keyword.try_into_property()?.try_into_keyword() }) .collect(), ); } (Key::Property(EmailProperty::Pointer(pointer)), value) => { match handle_email_patch(&pointer, value) { PatchResult::SetKeyword(keyword) => { new_data.add_keyword(keyword.clone()); } PatchResult::RemoveKeyword(keyword) => { new_data.remove_keyword(keyword); } PatchResult::AddMailbox(id) => { new_data.add_mailbox(UidMailbox::new_unassigned(id)); } PatchResult::RemoveMailbox(id) => { new_data.remove_mailbox(id); } PatchResult::Invalid(set_error) => { response.not_updated.append(id, set_error); continue 'update; } } } (property, _) => { response.invalid_property_update(id, property.into_owned()); continue 'update; } } } let has_keyword_changes = new_data.has_keyword_changes(data.inner); let has_mailbox_changes = new_data.has_mailbox_changes(data.inner); if !has_keyword_changes && !has_mailbox_changes { response.updated.append(id, None); continue 'update; } // Process keywords let mut train_spam = None; if has_keyword_changes { // Verify permissions on shared accounts if can_modify_mailbox_ids.as_ref().is_some_and(|ids| { !new_data .mailboxes .iter() .any(|mb| ids.contains(mb.mailbox_id)) }) { response.not_updated.append( id, SetError::forbidden() .with_description("You are not allowed to modify keywords."), ); continue 'update; } // Process keyword changes let mut changed_seen = false; for keyword in new_data.added_keywords(data.inner) { match keyword { Keyword::Seen => { changed_seen = true; } Keyword::Junk => { train_spam = Some(true); } Keyword::NotJunk => { train_spam = Some(false); } _ => {} } } for keyword in new_data.removed_keywords(data.inner) { match keyword { ArchivedKeyword::Seen => { changed_seen = true; } ArchivedKeyword::Junk if train_spam.is_none() => { train_spam = Some(false); } _ => {} } } // Set all current mailboxes as changed if the Seen tag changed if changed_seen { for mailbox_id in new_data.mailboxes.iter() { changed_mailboxes.insert(mailbox_id.mailbox_id, Vec::new()); } } } // Process mailboxes if has_mailbox_changes { // Make sure the message is at least in one mailbox if new_data.mailboxes.is_empty() { response.not_updated.append( id, SetError::invalid_properties() .with_property(EmailProperty::MailboxIds) .with_description("Message has to belong to at least one mailbox."), ); continue 'update; } // Make sure all new mailboxIds are valid for mailbox_id in new_data.added_mailboxes(data.inner) { if cache.has_mailbox_id(&mailbox_id.mailbox_id) { // Verify permissions on shared accounts if can_add_mailbox_ids .as_ref() .is_none_or(|ids| ids.contains(mailbox_id.mailbox_id)) { if mailbox_id.mailbox_id == JUNK_ID { train_spam = Some(true); } changed_mailboxes.insert(mailbox_id.mailbox_id, Vec::new()); } else { response.not_updated.append( id, SetError::forbidden().with_description(format!( "You are not allowed to add messages to mailbox {}.", Id::from(mailbox_id.mailbox_id) )), ); continue 'update; } } else { response.not_updated.append( id, SetError::invalid_properties() .with_property(EmailProperty::MailboxIds) .with_description(format!( "mailboxId {} does not exist.", Id::from(mailbox_id.mailbox_id) )), ); continue 'update; } } // Add all removed mailboxes to change list for mailbox_id in new_data.removed_mailboxes(data.inner) { // Verify permissions on shared accounts if can_delete_mailbox_ids .as_ref() .is_none_or(|ids| ids.contains(u32::from(mailbox_id.mailbox_id))) { if mailbox_id.mailbox_id == JUNK_ID && !new_data .mailboxes .iter() .any(|mb| mb.mailbox_id == TRASH_ID) { train_spam = Some(false); } changed_mailboxes .entry(mailbox_id.mailbox_id.to_native()) .or_default() .push(mailbox_id.uid.to_native()); } else { response.not_updated.append( id, SetError::forbidden().with_description(format!( "You are not allowed to delete messages from mailbox {}.", mailbox_id.mailbox_id )), ); continue 'update; } } // Obtain IMAP UIDs for added mailboxes let ids = self .assign_email_ids( account_id, new_data .mailboxes .iter() .filter(|m| m.uid == 0) .map(|m| m.mailbox_id), false, ) .await .caused_by(trc::location!())?; for (uid_mailbox, uid) in new_data .mailboxes .iter_mut() .filter(|m| m.uid == 0) .zip(ids) { uid_mailbox.uid = uid; } } // Write changes batch .with_account_id(account_id) .with_collection(Collection::Email) .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_current(data) .with_changes(new_data.seal()), ) .caused_by(trc::location!())?; if let Some(train_spam) = train_spam { self.add_account_spam_sample( &mut batch, account_id, document_id, train_spam, session.session_id, ) .await .caused_by(trc::location!())?; } batch.commit_point(); will_update.push(id); } if !batch.is_empty() { // Log mailbox changes for (parent_id, deleted_uids) in changed_mailboxes { batch.log_container_property_change(SyncCollection::Email, parent_id); for deleted_uid in deleted_uids { batch.log_vanished_item(VanishedCollection::Email, (parent_id, deleted_uid)); } } match self .commit_batch(batch) .await .and_then(|ids| ids.last_change_id(account_id)) { Ok(change_id) => { last_change_id = change_id.into(); // Add to updated list for id in will_update { response.updated.append(id, None); } } Err(err) if err.is_assertion_failure() => { for id in will_update { response.not_updated.append( id, SetError::forbidden().with_description( "Another process modified this message, please try again.", ), ); } } Err(err) => { return Err(err.caused_by(trc::location!())); } } } // Process deletions if !will_destroy.is_empty() { let email_ids = cache.email_document_ids(); let can_destroy_message_ids = if access_token.is_shared(account_id) { cache.shared_messages(access_token, Acl::RemoveItems).into() } else { None }; let mut destroy_ids = RoaringBitmap::new(); for destroy_id in will_destroy { let document_id = destroy_id.document_id(); if email_ids.contains(document_id) { if !matches!(&can_destroy_message_ids, Some(ids) if !ids.contains(document_id)) { destroy_ids.insert(document_id); response.destroyed.push(destroy_id); } else { response.not_destroyed.append( destroy_id, SetError::forbidden() .with_description("You are not allowed to delete this message."), ); } } else { response .not_destroyed .append(destroy_id, SetError::not_found()); } } if !destroy_ids.is_empty() { // Batch delete messages let mut batch = BatchBuilder::new(); let not_destroyed = self .emails_delete( account_id, access_token.tenant_id(), &mut batch, destroy_ids, ) .await?; if !batch.is_empty() { last_change_id = self .commit_batch(batch) .await .and_then(|ids| ids.last_change_id(account_id)) .caused_by(trc::location!())? .into(); self.notify_task_queue(); } // Mark messages that were not found as not destroyed (this should not occur in practice) if !not_destroyed.is_empty() { let mut destroyed = Vec::with_capacity(response.destroyed.len()); for destroy_id in response.destroyed { if not_destroyed.contains(destroy_id.document_id()) { response .not_destroyed .append(destroy_id, SetError::not_found()); } else { destroyed.push(destroy_id); } } response.destroyed = destroyed; } } } // Update state if let Some(change_id) = last_change_id { if response.updated.is_empty() && response.destroyed.is_empty() { // Message ingest does not broadcast state changes self.broadcast_push_notification(PushNotification::StateChange( StateChange::new(account_id) .with_change_id(change_id) .with_change(DataType::Email) .with_change(DataType::Mailbox) .with_change(DataType::Thread), )) .await; } response.new_state = State::Exact(change_id).into(); } Ok(response) } } ================================================ FILE: crates/jmap/src/email/snippet.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{Server, auth::AccessToken}; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, message::metadata::{ ArchivedMetadataPartType, DecodedPartContent, MessageMetadata, MetadataHeaderName, }, }; use jmap_proto::{ method::{ query::Filter, search_snippet::{GetSearchSnippetRequest, GetSearchSnippetResponse, SearchSnippet}, }, object::email::EmailFilter, request::IntoValid, }; use mail_parser::decoders::html::html_to_text; use nlp::language::{Language, search_snippet::generate_snippet, stemmer::Stemmer}; use std::future::Future; use store::{ ValueKey, backend::MAX_TOKEN_LENGTH, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{acl::Acl, collection::Collection, field::EmailField}; use utils::chained_bytes::ChainedBytes; pub trait EmailSearchSnippet: Sync + Send { fn email_search_snippet( &self, request: GetSearchSnippetRequest, access_token: &AccessToken, ) -> impl Future> + Send; } impl EmailSearchSnippet for Server { async fn email_search_snippet( &self, request: GetSearchSnippetRequest, access_token: &AccessToken, ) -> trc::Result { let mut filter_stack = vec![]; let mut include_term = true; let mut terms = vec![]; let mut is_exact = false; let mut language = self.core.jmap.default_language; for cond in request.filter { match cond { Filter::Property(cond) => { if let EmailFilter::Text(text) | EmailFilter::Subject(text) | EmailFilter::Body(text) = cond && include_term { let (text, language_) = Language::detect(text, self.core.jmap.default_language); language = language_; if (text.starts_with('"') && text.ends_with('"')) || (text.starts_with('\'') && text.ends_with('\'')) { for token in language.tokenize_text(&text, MAX_TOKEN_LENGTH) { terms.push(token.word.into_owned()); } is_exact = true; } else { for token in Stemmer::new(&text, language, MAX_TOKEN_LENGTH) { terms.push(token.word.into_owned()); if let Some(stemmed_word) = token.stemmed_word { terms.push(stemmed_word.into_owned()); } } } } } Filter::And | Filter::Or => { filter_stack.push(cond); } Filter::Not => { filter_stack.push(cond); include_term = !include_term; } Filter::Close => { if matches!(filter_stack.pop(), Some(Filter::Not)) { include_term = !include_term; } } } } let account_id = request.account_id.document_id(); let cached_messages = self .get_cached_messages(account_id) .await .caused_by(trc::location!())?; let document_ids = if access_token.is_member(account_id) { cached_messages.email_document_ids() } else { cached_messages.shared_messages(access_token, Acl::ReadItems) }; let email_ids = request.email_ids.unwrap(); let mut response = GetSearchSnippetResponse { account_id: request.account_id, list: Vec::with_capacity(email_ids.len()), not_found: vec![], }; if email_ids.len() > self.core.jmap.snippet_max_results { return Err(trc::JmapEvent::RequestTooLarge.into_err()); } for email_id in email_ids.into_valid() { let document_id = email_id.document_id(); let mut snippet = SearchSnippet { email_id, subject: None, preview: None, }; if !document_ids.contains(document_id) { response.not_found.push(email_id); continue; } else if terms.is_empty() { response.list.push(snippet); continue; } let metadata_ = match self .store() .get_value::>(ValueKey::property( account_id, Collection::Email, document_id, EmailField::Metadata, )) .await? { Some(metadata) => metadata, None => { response.not_found.push(email_id); continue; } }; let metadata = metadata_ .unarchive::() .caused_by(trc::location!())?; // Add subject snippet let contents = &metadata.contents[0]; if let Some(subject) = contents .root_part() .header_value(&MetadataHeaderName::Subject) .and_then(|v| v.as_text()) .and_then(|v| generate_snippet(v, &terms, language, is_exact)) { snippet.subject = subject.into(); } // Download message let raw_body = if let Some(raw_body) = self .blob_store() .get_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX) .await? { raw_body } else { trc::event!( Store(trc::StoreEvent::NotFound), AccountId = account_id, DocumentId = email_id.document_id(), Collection = Collection::Email, BlobId = metadata.blob_hash.0.as_slice(), Details = "Blob not found.", CausedBy = trc::location!(), ); response.not_found.push(email_id); continue; }; let raw_message = ChainedBytes::new(metadata.raw_headers.as_ref()).with_last( raw_body .get(metadata.blob_body_offset.to_native() as usize..) .unwrap_or_default(), ); // Find a matching part 'outer: for part in contents.parts.iter() { match &part.body { ArchivedMetadataPartType::Text => { let text = match part.decode_contents(&raw_message) { DecodedPartContent::Text(text) => text, _ => unreachable!(), }; if let Some(body) = generate_snippet(&text, &terms, language, is_exact) { snippet.preview = body.into(); break; } } ArchivedMetadataPartType::Html => { let text = match part.decode_contents(&raw_message) { DecodedPartContent::Text(html) => html_to_text(&html), _ => unreachable!(), }; if let Some(body) = generate_snippet(&text, &terms, language, is_exact) { snippet.preview = body.into(); break; } } ArchivedMetadataPartType::Message(message) => { for part in metadata.contents[u16::from(message) as usize].parts.iter() { if let ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html = part.body { let text = match (part.decode_contents(&raw_message), &part.body) { ( DecodedPartContent::Text(text), ArchivedMetadataPartType::Text, ) => text, ( DecodedPartContent::Text(html), ArchivedMetadataPartType::Html, ) => html_to_text(&html).into(), _ => unreachable!(), }; if let Some(body) = generate_snippet(&text, &terms, language, is_exact) { snippet.preview = body.into(); break 'outer; } } } } _ => (), } } //} response.list.push(snippet); } Ok(response) } } ================================================ FILE: crates/jmap/src/file/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{api::acl::JmapRights, changes::state::JmapCacheState}; use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; use groupware::{cache::GroupwareCache, file::FileNode}; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::file_node::{self, FileNodeProperty, FileNodeValue}, types::date::UTCDate, }; use jmap_tools::{Map, Value}; use store::{ValueKey, roaring::RoaringBitmap, write::{AlignedBytes, Archive, now}}; use trc::AddContext; use types::{ acl::{Acl, AclGrant}, blob::{BlobClass, BlobId}, blob_hash::BlobHash, collection::{Collection, SyncCollection}, }; pub trait FileNodeGet: Sync + Send { fn file_node_get( &self, request: GetRequest, access_token: &AccessToken, ) -> impl Future>> + Send; } impl FileNodeGet for Server { async fn file_node_get( &self, mut request: GetRequest, access_token: &AccessToken, ) -> trc::Result> { let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ FileNodeProperty::Id, FileNodeProperty::Name, FileNodeProperty::ParentId, FileNodeProperty::Size, ]); let account_id = request.account_id.document_id(); let cache = self .fetch_dav_resources(access_token, account_id, SyncCollection::FileNode) .await?; let file_node_ids = if access_token.is_member(account_id) { cache .resources .iter() .map(|r| r.document_id) .collect::() } else { cache.shared_containers(access_token, [Acl::Read, Acl::ReadItems], true) }; let ids = if let Some(ids) = ids { ids } else { file_node_ids .iter() .take(self.core.jmap.get_max_objects) .map(Into::into) .collect::>() }; let mut response = GetResponse { account_id: request.account_id.into(), state: cache.get_state(true).into(), list: Vec::with_capacity(ids.len()), not_found: vec![], }; for id in ids { // Obtain the file_node object let document_id = id.document_id(); if !file_node_ids.contains(document_id) { response.not_found.push(id); continue; } let _file_node = if let Some(file_node) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::FileNode, document_id, )) .await? { file_node } else { response.not_found.push(id); continue; }; let file_node = _file_node .unarchive::() .caused_by(trc::location!())?; let mut result = Map::with_capacity(properties.len()); for property in &properties { match property { FileNodeProperty::Id => { result.insert_unchecked(FileNodeProperty::Id, FileNodeValue::Id(id)); } FileNodeProperty::Name => { result.insert_unchecked(FileNodeProperty::Name, file_node.name.to_string()); } FileNodeProperty::ShareWith => { result.insert_unchecked( FileNodeProperty::ShareWith, JmapRights::share_with::( account_id, access_token, &file_node .acls .iter() .map(AclGrant::from) .collect::>(), ), ); } FileNodeProperty::MyRights => { result.insert_unchecked( FileNodeProperty::MyRights, if access_token.is_shared(account_id) { JmapRights::rights::( file_node.acls.effective_acl(access_token), ) } else { JmapRights::all_rights::() }, ); } FileNodeProperty::ParentId => { let parent_id = file_node.parent_id.to_native(); result.insert_unchecked( FileNodeProperty::ParentId, if parent_id > 0 { Value::Element(FileNodeValue::Id((parent_id - 1).into())) } else { Value::Null }, ); } FileNodeProperty::BlobId => { result.insert_unchecked( FileNodeProperty::BlobId, if let Some(file) = file_node.file.as_ref() { Value::Element(FileNodeValue::BlobId(BlobId::new( BlobHash::from(&file.blob_hash), BlobClass::Linked { account_id, collection: Collection::FileNode.into(), document_id: id.document_id(), }, ))) } else { Value::Null }, ); } FileNodeProperty::Size => { result.insert_unchecked( FileNodeProperty::Size, if let Some(file) = file_node.file.as_ref() { Value::Number(file.size.to_native().into()) } else { Value::Null }, ); } FileNodeProperty::Type => { result.insert_unchecked( FileNodeProperty::Type, if let Some(file) = file_node.file.as_ref().and_then(|f| f.media_type.as_ref()) { Value::Str(file.to_string().into()) } else { Value::Null }, ); } FileNodeProperty::Executable => { result.insert_unchecked( FileNodeProperty::Executable, if let Some(file) = file_node.file.as_ref() { Value::Bool(file.executable) } else { Value::Null }, ); } FileNodeProperty::Created => { result.insert_unchecked( FileNodeProperty::Created, Value::Element(FileNodeValue::Date(UTCDate::from_timestamp( file_node.created.to_native(), ))), ); } FileNodeProperty::Modified => { result.insert_unchecked( FileNodeProperty::Modified, Value::Element(FileNodeValue::Date(UTCDate::from_timestamp( file_node.modified.to_native(), ))), ); } FileNodeProperty::Accessed => { result.insert_unchecked( FileNodeProperty::Accessed, Value::Element(FileNodeValue::Date(UTCDate::from_timestamp( now() as i64 ))), ); } FileNodeProperty::IsSubscribed => { result.insert_unchecked(FileNodeProperty::IsSubscribed, Value::Bool(true)); } property => { result.insert_unchecked(property.clone(), Value::Null); } } } response.list.push(result.into()); } Ok(response) } } ================================================ FILE: crates/jmap/src/file/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod get; pub mod query; pub mod set; ================================================ FILE: crates/jmap/src/file/query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{api::query::QueryResponseBuilder, changes::state::JmapCacheState}; use common::{Server, auth::AccessToken}; use groupware::cache::GroupwareCache; use jmap_proto::{ method::query::{Filter, QueryRequest, QueryResponse}, object::file_node::{FileNode, FileNodeFilter}, request::MaybeInvalid, }; use store::{ roaring::RoaringBitmap, search::{SearchFilter, SearchQuery}, write::SearchIndex, }; use types::{acl::Acl, collection::SyncCollection}; pub trait FileNodeQuery: Sync + Send { fn file_node_query( &self, request: QueryRequest, access_token: &AccessToken, ) -> impl Future> + Send; } impl FileNodeQuery for Server { async fn file_node_query( &self, mut request: QueryRequest, access_token: &AccessToken, ) -> trc::Result { let account_id = request.account_id.document_id(); let mut filters = Vec::with_capacity(request.filter.len()); let cache = self .fetch_dav_resources(access_token, account_id, SyncCollection::FileNode) .await?; for cond in std::mem::take(&mut request.filter) { match cond { Filter::Property(cond) => match cond { FileNodeFilter::AncestorId(MaybeInvalid::Value(id)) => { if let Some(resource) = cache.container_resource_path_by_id(id.document_id()) { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.subtree(resource.path()).map(|r| r.document_id()), ))) } else { filters.push(SearchFilter::is_in_set(RoaringBitmap::new())); } } FileNodeFilter::ParentId(MaybeInvalid::Value(id)) => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.children_ids(id.document_id()), ))); } FileNodeFilter::HasParentId(has_parent_id) => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.resources.iter().filter_map(|r| { if has_parent_id == r.parent_id().is_some() { Some(r.document_id) } else { None } }), ))); } FileNodeFilter::Name(name) => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.resources.iter().filter_map(|r| { if r.container_name().is_some_and(|n| n == name) { Some(r.document_id) } else { None } }), ))); } FileNodeFilter::NameMatch(name) => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.resources.iter().filter_map(|r| { if r.container_name().is_some_and(|n| name.matches(n)) { Some(r.document_id) } else { None } }), ))); } FileNodeFilter::MinSize(size) => { let size = size as u32; filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.resources.iter().filter_map(|r| { if r.size().is_some_and(|s| s >= size) { Some(r.document_id) } else { None } }), ))); } FileNodeFilter::MaxSize(size) => { let size = size as u32; filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.resources.iter().filter_map(|r| { if r.size().is_some_and(|s| s <= size) { Some(r.document_id) } else { None } }), ))); } unsupported => { return Err(trc::JmapEvent::UnsupportedFilter .into_err() .details(unsupported.into_string())); } }, Filter::And => { filters.push(SearchFilter::And); } Filter::Or => { filters.push(SearchFilter::Or); } Filter::Not => { filters.push(SearchFilter::Not); } Filter::Close => { filters.push(SearchFilter::End); } } } if request.sort.as_ref().is_some_and(|s| !s.is_empty()) { return Err(trc::JmapEvent::UnsupportedSort .into_err() .details("Sorting is not supported on FileNode")); } let results = SearchQuery::new(SearchIndex::InMemory) .with_filters(filters) .with_mask(if access_token.is_shared(account_id) { cache.shared_containers(access_token, [Acl::ReadItems], true) } else { cache.document_ids(false).collect() }) .filter() .into_bitmap(); let mut response = QueryResponseBuilder::new( results.len() as usize, self.core.jmap.query_max_results, cache.get_state(false), &request, ); for document_id in results { if !response.add(0, document_id) { break; } } response.build() } } ================================================ FILE: crates/jmap/src/file/set.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ api::acl::{JmapAcl, JmapRights}, blob::download::BlobDownload, }; use common::{DavResourceMetadata, DavResources, Server, auth::AccessToken, sharing::EffectiveAcl}; use groupware::{DestroyArchive, cache::GroupwareCache, file::FileNode}; use http_proto::HttpSessionData; use jmap_proto::{ error::set::SetError, method::set::{SetRequest, SetResponse}, object::file_node::{self, FileNodeProperty, FileNodeValue}, references::resolve::ResolveCreatedReference, request::IntoValid, types::state::State, }; use jmap_tools::{JsonPointerItem, Key, Value}; use store::{ ValueKey, ahash::{AHashMap, AHashSet}, write::{AlignedBytes, Archive, BatchBuilder}, }; use trc::AddContext; use types::{ acl::{Acl, AclGrant}, blob::BlobId, collection::{Collection, SyncCollection}, id::Id, }; pub trait FileNodeSet: Sync + Send { fn file_node_set( &self, request: SetRequest<'_, file_node::FileNode>, access_token: &AccessToken, session: &HttpSessionData, ) -> impl Future>> + Send; } impl FileNodeSet for Server { async fn file_node_set( &self, mut request: SetRequest<'_, file_node::FileNode>, access_token: &AccessToken, _session: &HttpSessionData, ) -> trc::Result> { let account_id = request.account_id.document_id(); let cache = self .fetch_dav_resources(access_token, account_id, SyncCollection::FileNode) .await?; let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; let will_destroy = request.unwrap_destroy().into_valid().collect::>(); let is_shared = access_token.is_shared(account_id); // Process creates let mut batch = BatchBuilder::new(); let mut created_folders = AHashMap::new(); 'create: for (id, object) in request.unwrap_create() { let mut file_node = FileNode::default(); // Process changes let has_acl_changes = match update_file_node(object, &mut file_node, &mut response) { Ok(result) => { if let Some(blob_id) = result.blob_id { let file_details = file_node.file.get_or_insert_default(); if !self.has_access_blob(&blob_id, access_token).await? { response.not_created.append( id, SetError::forbidden().with_description(format!( "You do not have access to blobId {blob_id}." )), ); continue; } else if let Some(blob_contents) = self .blob_store() .get_blob(blob_id.hash.as_slice(), 0..usize::MAX) .await? { file_details.size = blob_contents.len() as u32; } else { response.not_created.append( id, SetError::invalid_properties() .with_property(FileNodeProperty::BlobId) .with_description("Blob could not be found."), ); continue 'create; } file_details.blob_hash = blob_id.hash; } // Validate blob hash if file_node .file .as_ref() .is_some_and(|f| f.blob_hash.is_empty()) { response.not_created.append( id, SetError::invalid_properties() .with_property(FileNodeProperty::BlobId) .with_description("Missing blob id."), ); continue 'create; } result.has_acl_changes } Err(err) => { response.not_created.append(id, err); continue 'create; } }; // Validate hierarchy if let Err(err) = validate_file_node_hierarchy(None, &file_node, is_shared, &cache, &created_folders) { response.not_created.append(id, err); continue 'create; } // Inherit ACLs from parent if file_node.parent_id > 0 { let parent_id = file_node.parent_id - 1; let parent_acls = created_folders.get(&parent_id).cloned().or_else(|| { cache .container_resource_by_id(parent_id) .and_then(|r| r.acls()) .map(|a| a.to_vec()) }); if !has_acl_changes { if let Some(parent_acls) = parent_acls { file_node.acls = parent_acls; } } else if is_shared && parent_acls .is_none_or(|acls| !acls.effective_acl(access_token).contains(Acl::Share)) { response.not_created.append( id, SetError::forbidden() .with_description("You are not allowed to share this file node."), ); continue 'create; } } // Validate ACLs if !file_node.acls.is_empty() { if let Err(err) = self.acl_validate(&file_node.acls).await { response.not_created.append(id, err.into()); continue 'create; } self.refresh_acls(&file_node.acls, None).await; } // Insert record let document_id = self .store() .assign_document_ids(account_id, Collection::FileNode, 1) .await .caused_by(trc::location!())?; if file_node.file.is_none() { created_folders.insert(document_id, file_node.acls.clone()); } file_node .insert(access_token, account_id, document_id, &mut batch) .caused_by(trc::location!())?; response.created(id, document_id); } // Process updates 'update: for (id, object) in request.unwrap_update().into_valid() { // Make sure id won't be destroyed if will_destroy.contains(&id) { response.not_updated.append(id, SetError::will_destroy()); continue 'update; } // Obtain file node let document_id = id.document_id(); let file_node_ = if let Some(file_node_) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::FileNode, document_id, )) .await? { file_node_ } else { response.not_updated.append(id, SetError::not_found()); continue 'update; }; let file_node = file_node_ .to_unarchived::() .caused_by(trc::location!())?; let mut new_file_node = file_node .deserialize::() .caused_by(trc::location!())?; // Apply changes let has_acl_changes = match update_file_node(object, &mut new_file_node, &mut response) { Ok(result) => { if let Some(blob_id) = result.blob_id { let file_details = new_file_node.file.get_or_insert_default(); if !self.has_access_blob(&blob_id, access_token).await? { response.not_updated.append( id, SetError::forbidden().with_description(format!( "You do not have access to blobId {blob_id}." )), ); continue; } else if let Some(blob_contents) = self .blob_store() .get_blob(blob_id.hash.as_slice(), 0..usize::MAX) .await? { file_details.size = blob_contents.len() as u32; } else { response.not_updated.append( id, SetError::invalid_properties() .with_property(FileNodeProperty::BlobId) .with_description("Blob could not be found."), ); continue 'update; } file_details.blob_hash = blob_id.hash; } result.has_acl_changes } Err(err) => { response.not_updated.append(id, err); continue 'update; } }; // Validate hierarchy if let Err(err) = validate_file_node_hierarchy( Some(document_id), &new_file_node, is_shared, &cache, &created_folders, ) { response.not_updated.append(id, err); continue 'update; } // Validate ACL if is_shared { let acl = file_node.inner.acls.effective_acl(access_token); if !acl.contains(Acl::Modify) || (has_acl_changes && !acl.contains(Acl::Share)) { response.not_updated.append( id, SetError::forbidden() .with_description("You are not allowed to modify this file node."), ); continue 'update; } } if has_acl_changes { if let Err(err) = self.acl_validate(&new_file_node.acls).await { response.not_updated.append(id, err.into()); continue 'update; } self.refresh_acls( &new_file_node.acls, Some( file_node .inner .acls .iter() .map(AclGrant::from) .collect::>() .as_slice(), ), ) .await; } // Update record new_file_node .update(access_token, file_node, account_id, document_id, &mut batch) .caused_by(trc::location!())?; response.updated.append(id, None); } // Process deletions let on_destroy_remove_children = request .arguments .on_destroy_remove_children .unwrap_or(false); let mut destroy_ids = AHashSet::with_capacity(will_destroy.len()); 'destroy: for id in will_destroy { let document_id = id.document_id(); let Some(file_node) = cache.any_resource_path_by_id(document_id) else { response.not_destroyed.append(id, SetError::not_found()); continue 'destroy; }; // Find ids to delete let mut ids = cache.subtree(file_node.path()).collect::>(); if ids.is_empty() { debug_assert!(false, "Resource found in cache but not in subtree"); continue 'destroy; } // Sort ids descending from the deepest to the root ids.sort_unstable_by_key(|b| std::cmp::Reverse(b.hierarchy_seq())); let mut sorted_ids = Vec::with_capacity(ids.len()); sorted_ids.extend(ids.into_iter().map(|a| a.document_id())); // Validate not already deleted for child_id in &sorted_ids { if !destroy_ids.insert(*child_id) { response.not_destroyed.append( id, SetError::will_destroy().with_description( "File node or one of its children is already marked for deletion.", ), ); continue 'destroy; } } // Validate ACLs if !access_token.is_member(account_id) { let permissions = cache.shared_containers(access_token, [Acl::Delete], false); if permissions.len() < sorted_ids.len() as u64 || !sorted_ids.iter().all(|id| permissions.contains(*id)) { response.not_destroyed.append( id, SetError::forbidden() .with_description("You are not allowed to delete this file node."), ); continue 'destroy; } } // Obtain children ids if sorted_ids.len() > 1 && !on_destroy_remove_children { response .not_destroyed .append(id, SetError::node_has_children()); continue 'destroy; } // Delete record response .destroyed .extend(sorted_ids.iter().copied().map(Id::from)); DestroyArchive(sorted_ids) .delete_batch( self, access_token, account_id, cache.format_resource(file_node).into(), &mut batch, ) .await?; } // Write changes if !batch.is_empty() { let change_id = self .commit_batch(batch) .await .and_then(|ids| ids.last_change_id(account_id)) .caused_by(trc::location!())?; response.new_state = State::Exact(change_id).into(); } Ok(response) } } struct UpdateResult { has_acl_changes: bool, blob_id: Option, } fn update_file_node( updates: Value<'_, FileNodeProperty, FileNodeValue>, file_node: &mut FileNode, response: &mut SetResponse, ) -> Result> { let mut has_acl_changes = false; let mut blob_id = None; for (property, mut value) in updates.into_expanded_object() { let Key::Property(property) = property else { return Err(SetError::invalid_properties() .with_property(property.to_owned()) .with_description("Invalid property.")); }; response.resolve_self_references(&mut value)?; match (property, value) { (FileNodeProperty::Name, Value::Str(value)) if (1..=255).contains(&value.len()) && !value.contains('/') && ![".", ".."].contains(&value.as_ref()) => { file_node.name = value.into_owned(); } (FileNodeProperty::ParentId, Value::Element(FileNodeValue::Id(value))) => { file_node.parent_id = value.document_id() + 1; } (FileNodeProperty::ParentId, Value::Null) => { file_node.parent_id = 0; } (FileNodeProperty::BlobId, Value::Element(FileNodeValue::BlobId(value))) => { if file_node .file .as_ref() .is_none_or(|f| f.blob_hash != value.hash) { blob_id = Some(value); } } (FileNodeProperty::BlobId, Value::Null) => {} (FileNodeProperty::Size, Value::Number(value)) => { file_node.file.get_or_insert_default().size = value.cast_to_u64() as u32; } (FileNodeProperty::Type, Value::Str(value)) if (1..=30).contains(&value.len()) => { file_node.file.get_or_insert_default().media_type = value.into_owned().into(); } (FileNodeProperty::Type, Value::Null) => { file_node.file.get_or_insert_default().media_type = None; } (FileNodeProperty::Executable, Value::Bool(value)) => { file_node.file.get_or_insert_default().executable = value; } (FileNodeProperty::Executable, Value::Null) => { file_node.file.get_or_insert_default().executable = false; } (FileNodeProperty::Created, Value::Element(FileNodeValue::Date(value))) => { file_node.created = value.timestamp(); } (FileNodeProperty::Modified, Value::Element(FileNodeValue::Date(value))) => { file_node.modified = value.timestamp(); } (FileNodeProperty::ShareWith, value) => { file_node.acls = JmapRights::acl_set::(value)?; has_acl_changes = true; } (FileNodeProperty::Pointer(pointer), value) if matches!( pointer.first(), Some(JsonPointerItem::Key(Key::Property( FileNodeProperty::ShareWith ))) ) => { let mut pointer = pointer.iter(); pointer.next(); file_node.acls = JmapRights::acl_patch::( std::mem::take(&mut file_node.acls), pointer, value, )?; has_acl_changes = true; } (property, _) => { return Err(SetError::invalid_properties() .with_property(property.clone()) .with_description("Field could not be set.")); } } } // Validate name if file_node.name.is_empty() { return Err(SetError::invalid_properties() .with_property(FileNodeProperty::Name) .with_description("Missing name.")); } Ok(UpdateResult { has_acl_changes, blob_id, }) } fn validate_file_node_hierarchy( document_id: Option, node: &FileNode, is_shared: bool, cache: &DavResources, created_folders: &AHashMap>, ) -> Result<(), SetError> { let node_parent_id = if node.parent_id == 0 { if is_shared && document_id.is_none() { return Err(SetError::invalid_properties() .with_property(FileNodeProperty::ParentId) .with_description("Cannot create top-level folder in a shared account.")); } None } else { let parent_id = node.parent_id - 1; if let Some(document_id) = document_id { if document_id == parent_id { return Err(SetError::invalid_properties() .with_property(FileNodeProperty::ParentId) .with_description("A file node cannot be its own parent.")); } // Validate circular references if let Some(file) = cache.container_resource_path_by_id(document_id) && cache .subtree(file.path()) .any(|r| r.document_id() == parent_id) { return Err(SetError::invalid_properties() .with_property(FileNodeProperty::ParentId) .with_description("Circular reference in parent ids.")); } } // Make sure the parent is a container if !created_folders.contains_key(&parent_id) && cache.container_resource_by_id(parent_id).is_none() { return Err(SetError::invalid_properties() .with_property(FileNodeProperty::ParentId) .with_description("Parent ID does not exist or is not a folder.")); } Some(parent_id) }; // Validate name uniqueness for resource in &cache.resources { if let DavResourceMetadata::File { name, parent_id, .. } = &resource.data && document_id.is_none_or(|id| id != resource.document_id) && node_parent_id == *parent_id && node.name == *name { return Err(SetError::invalid_properties() .with_property(FileNodeProperty::Name) .with_description("A node with the same name already exists in this folder.")); } } Ok(()) } ================================================ FILE: crates/jmap/src/identity/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::changes::state::StateManager; use common::{Server, storage::index::ObjectIndexBuilder}; use directory::{PrincipalData, QueryParams}; use email::identity::{ArchivedEmailAddress, Identity}; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::identity::{self, IdentityProperty, IdentityValue}, }; use jmap_tools::{Map, Value}; use std::future::Future; use store::{ ValueKey, rkyv::{option::ArchivedOption, vec::ArchivedVec}, roaring::RoaringBitmap, write::{AlignedBytes, Archive, BatchBuilder} }; use trc::AddContext; use types::{ collection::{Collection, SyncCollection}, field::IdentityField, }; use utils::sanitize_email; pub trait IdentityGet: Sync + Send { fn identity_get( &self, request: GetRequest, ) -> impl Future>> + Send; fn identity_get_or_create( &self, account_id: u32, ) -> impl Future> + Send; } impl IdentityGet for Server { async fn identity_get( &self, mut request: GetRequest, ) -> trc::Result> { let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ IdentityProperty::Id, IdentityProperty::Name, IdentityProperty::Email, IdentityProperty::ReplyTo, IdentityProperty::Bcc, IdentityProperty::TextSignature, IdentityProperty::HtmlSignature, IdentityProperty::MayDelete, ]); let account_id = request.account_id.document_id(); let identity_ids = self.identity_get_or_create(account_id).await?; let ids = if let Some(ids) = ids { ids } else { identity_ids .iter() .take(self.core.jmap.get_max_objects) .map(Into::into) .collect::>() }; let mut response = GetResponse { account_id: request.account_id.into(), state: self .get_state(account_id, SyncCollection::Identity) .await? .into(), list: Vec::with_capacity(ids.len()), not_found: vec![], }; for id in ids { // Obtain the identity object let document_id = id.document_id(); if !identity_ids.contains(document_id) { response.not_found.push(id); continue; } let _identity = if let Some(identity) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::Identity, document_id, )) .await? { identity } else { response.not_found.push(id); continue; }; let identity = _identity .unarchive::() .caused_by(trc::location!())?; let mut result = Map::with_capacity(properties.len()); for property in &properties { match property { IdentityProperty::Id => { result.insert_unchecked(IdentityProperty::Id, IdentityValue::Id(id)); } IdentityProperty::MayDelete => { result.insert_unchecked(IdentityProperty::MayDelete, Value::Bool(true)); } IdentityProperty::Name => { result.insert_unchecked(IdentityProperty::Name, identity.name.to_string()); } IdentityProperty::Email => { result .insert_unchecked(IdentityProperty::Email, identity.email.to_string()); } IdentityProperty::TextSignature => { result.insert_unchecked( IdentityProperty::TextSignature, identity.text_signature.to_string(), ); } IdentityProperty::HtmlSignature => { result.insert_unchecked( IdentityProperty::HtmlSignature, identity.html_signature.to_string(), ); } IdentityProperty::Bcc => { result .insert_unchecked(IdentityProperty::Bcc, email_to_value(&identity.bcc)); } IdentityProperty::ReplyTo => { result.insert_unchecked( IdentityProperty::ReplyTo, email_to_value(&identity.reply_to), ); } property => { result.insert_unchecked(property.clone(), Value::Null); } } } response.list.push(result.into()); } Ok(response) } async fn identity_get_or_create(&self, account_id: u32) -> trc::Result { let mut identity_ids = self .document_ids(account_id, Collection::Identity, IdentityField::DocumentId) .await?; if !identity_ids.is_empty() { return Ok(identity_ids); } // Obtain principal let principal = if let Some(principal) = self .core .storage .directory .query(QueryParams::id(account_id).with_return_member_of(false)) .await .caused_by(trc::location!())? { principal } else { return Ok(identity_ids); }; let mut emails = Vec::new(); let mut description = None; for data in principal.data { match data { PrincipalData::PrimaryEmail(v) | PrincipalData::EmailAlias(v) => emails.push(v), PrincipalData::Description(v) => description = Some(v), _ => {} } } let num_emails = emails.len(); if num_emails == 0 { return Ok(identity_ids); } let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Identity); // Create identities let name = description.unwrap_or(principal.name); let mut next_document_id = self .store() .assign_document_ids(account_id, Collection::Identity, num_emails as u64) .await .caused_by(trc::location!())?; for email in &emails { let email = sanitize_email(email).unwrap_or_default(); if email.is_empty() || email.starts_with('@') { continue; } let name = if name.is_empty() { email.clone() } else { name.clone() }; let document_id = next_document_id; next_document_id -= 1; batch .with_document(document_id) .tag(IdentityField::DocumentId) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(Identity { name, email, ..Default::default() })) .caused_by(trc::location!())?; identity_ids.insert(document_id); } self.commit_batch(batch).await.caused_by(trc::location!())?; Ok(identity_ids) } } fn email_to_value( email: &ArchivedOption>, ) -> Value<'static, IdentityProperty, IdentityValue> { if let ArchivedOption::Some(email) = email { Value::Array( email .iter() .map(|email| { Value::Object( Map::with_capacity(2) .with_key_value(IdentityProperty::Name, &email.name) .with_key_value(IdentityProperty::Email, &email.email), ) }) .collect(), ) } else { Value::Null } } ================================================ FILE: crates/jmap/src/identity/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod get; pub mod set; ================================================ FILE: crates/jmap/src/identity/set.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; use directory::QueryParams; use email::identity::{EmailAddress, Identity}; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{SetRequest, SetResponse}, object::identity::{self, IdentityProperty, IdentityValue}, references::resolve::ResolveCreatedReference, request::IntoValid, types::state::State, }; use jmap_tools::{Key, Value}; use std::future::Future; use store::{ValueKey, write::{AlignedBytes, Archive, BatchBuilder}}; use trc::AddContext; use types::{ collection::{Collection, SyncCollection}, field::{Field, IdentityField}, }; use utils::sanitize_email; pub trait IdentitySet: Sync + Send { fn identity_set( &self, request: SetRequest<'_, identity::Identity>, access_token: &AccessToken, ) -> impl Future>> + Send; } impl IdentitySet for Server { async fn identity_set( &self, mut request: SetRequest<'_, identity::Identity>, access_token: &AccessToken, ) -> trc::Result> { let account_id = request.account_id.document_id(); let identity_ids = self .document_ids(account_id, Collection::Identity, IdentityField::DocumentId) .await?; let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; let will_destroy = request.unwrap_destroy().into_valid().collect::>(); // Process creates let mut batch = BatchBuilder::new(); 'create: for (id, object) in request.unwrap_create() { let mut identity = Identity::default(); for (property, mut value) in object.into_expanded_object() { if let Err(err) = response .resolve_self_references(&mut value) .and_then(|_| validate_identity_value(&property, value, &mut identity, true)) { response.not_created.append(id, err); continue 'create; } } // Validate email address if !identity.email.is_empty() { if self .directory() .query(QueryParams::id(account_id).with_return_member_of(false)) .await? .is_none_or(|p| !p.email_addresses().any(|e| e == identity.email)) { response.not_created.append( id, SetError::invalid_properties() .with_property(IdentityProperty::Email) .with_description( "E-mail address not configured for this account.".to_string(), ), ); continue 'create; } } else { response.not_created.append( id, SetError::invalid_properties() .with_property(IdentityProperty::Email) .with_description("Missing e-mail address."), ); continue 'create; } // Validate quota if identity_ids.len() >= access_token.object_quota(Collection::Identity) as u64 { response.not_created.append( id, SetError::new(SetErrorType::OverQuota).with_description(concat!( "There are too many identities, ", "please delete some before adding a new one." )), ); continue 'create; } // Insert record let document_id = self .store() .assign_document_ids(account_id, Collection::Identity, 1) .await .caused_by(trc::location!())?; batch .with_account_id(account_id) .with_collection(Collection::Identity) .with_document(document_id) .tag(IdentityField::DocumentId) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(identity)) .caused_by(trc::location!())? .commit_point(); response.created(id, document_id); } // Process updates 'update: for (id, object) in request.unwrap_update().into_valid() { // Make sure id won't be destroyed if will_destroy.contains(&id) { response.not_updated.append(id, SetError::will_destroy()); continue 'update; } // Obtain identity let document_id = id.document_id(); let identity_ = if let Some(identity_) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::Identity, document_id, )) .await? { identity_ } else { response.not_updated.append(id, SetError::not_found()); continue 'update; }; let identity = identity_ .to_unarchived::() .caused_by(trc::location!())?; let mut new_identity = identity .deserialize::() .caused_by(trc::location!())?; for (property, mut value) in object.into_expanded_object() { if let Err(err) = response.resolve_self_references(&mut value).and_then(|_| { validate_identity_value(&property, value, &mut new_identity, false) }) { response.not_updated.append(id, err); continue 'update; } } // Update record batch .with_account_id(account_id) .with_collection(Collection::Identity) .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_current(identity) .with_changes(new_identity), ) .caused_by(trc::location!())? .commit_point(); response.updated.append(id, None); } // Process deletions for id in will_destroy { let document_id = id.document_id(); if identity_ids.contains(document_id) { // Update record batch .with_account_id(account_id) .with_collection(Collection::Identity) .with_document(document_id) .untag(IdentityField::DocumentId) .clear(Field::ARCHIVE) .log_item_delete(SyncCollection::Identity, None) .commit_point(); response.destroyed.push(id); } else { response.not_destroyed.append(id, SetError::not_found()); } } // Write changes if !batch.is_empty() { let change_id = self .commit_batch(batch) .await .and_then(|ids| ids.last_change_id(account_id)) .caused_by(trc::location!())?; response.new_state = State::Exact(change_id).into(); } Ok(response) } } fn validate_identity_value( property: &Key<'_, IdentityProperty>, value: Value<'_, IdentityProperty, IdentityValue>, identity: &mut Identity, is_create: bool, ) -> Result<(), SetError> { let Key::Property(property) = property else { return Err(SetError::invalid_properties() .with_property(property.to_owned()) .with_description("Invalid property.")); }; match (property, value) { (IdentityProperty::Name, Value::Str(value)) if value.len() < 255 => { identity.name = value.into_owned(); } (IdentityProperty::Email, Value::Str(value)) if is_create && value.len() < 255 => { identity.email = sanitize_email(&value).ok_or_else(|| { SetError::invalid_properties() .with_property(IdentityProperty::Email) .with_description("Invalid e-mail address.") })?; } (IdentityProperty::TextSignature, Value::Str(value)) if value.len() < 2048 => { identity.text_signature = value.into_owned(); } (IdentityProperty::HtmlSignature, Value::Str(value)) if value.len() < 2048 => { identity.html_signature = value.into_owned(); } (IdentityProperty::ReplyTo | IdentityProperty::Bcc, Value::Array(value)) => { let mut addresses = Vec::with_capacity(value.len()); for addr in value { let mut address = EmailAddress { name: None, email: "".into(), }; let mut is_valid = false; if let Value::Object(obj) = addr { for (key, value) in obj.into_vec() { match (key, value) { (Key::Property(IdentityProperty::Email), Value::Str(value)) if value.len() < 255 => { is_valid = true; address.email = value.into_owned(); } (Key::Property(IdentityProperty::Name), Value::Str(value)) if value.len() < 255 => { address.name = Some(value.into_owned()); } (Key::Property(IdentityProperty::Name), Value::Null) => (), _ => { is_valid = false; break; } } } } if is_valid && !address.email.is_empty() { addresses.push(address); } else { return Err(SetError::invalid_properties() .with_property(property.clone()) .with_description("Invalid e-mail address object.")); } } match property { IdentityProperty::ReplyTo => { identity.reply_to = Some(addresses); } IdentityProperty::Bcc => { identity.bcc = Some(addresses); } _ => unreachable!(), } } (IdentityProperty::Name, Value::Null) => { identity.name.clear(); } (IdentityProperty::TextSignature, Value::Null) => { identity.text_signature.clear(); } (IdentityProperty::HtmlSignature, Value::Null) => { identity.html_signature.clear(); } (IdentityProperty::ReplyTo, Value::Null) => identity.reply_to = None, (IdentityProperty::Bcc, Value::Null) => identity.bcc = None, (property, _) => { return Err(SetError::invalid_properties() .with_property(property.clone()) .with_description("Field could not be set.")); } } Ok(()) } ================================================ FILE: crates/jmap/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ #![warn(clippy::large_futures)] pub mod addressbook; pub mod api; pub mod blob; pub mod calendar; pub mod calendar_event; pub mod calendar_event_notification; pub mod changes; pub mod contact; pub mod email; pub mod file; pub mod identity; pub mod mailbox; pub mod participant_identity; pub mod principal; pub mod push; pub mod quota; pub mod share_notification; pub mod sieve; pub mod submission; pub mod thread; pub mod vacation; pub mod websocket; ================================================ FILE: crates/jmap/src/mailbox/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; use email::cache::{MessageCacheFetch, email::MessageCacheAccess, mailbox::MailboxCacheAccess}; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::mailbox::{Mailbox, MailboxProperty, MailboxValue}, }; use jmap_tools::{Map, Value}; use std::future::Future; use store::ahash::AHashSet; use types::{acl::Acl, keyword::Keyword, special_use::SpecialUse}; use crate::api::acl::JmapRights; pub trait MailboxGet: Sync + Send { fn mailbox_get( &self, request: GetRequest, access_token: &AccessToken, ) -> impl Future>> + Send; } impl MailboxGet for Server { async fn mailbox_get( &self, mut request: GetRequest, access_token: &AccessToken, ) -> trc::Result> { let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ MailboxProperty::Id, MailboxProperty::Name, MailboxProperty::ParentId, MailboxProperty::Role, MailboxProperty::SortOrder, MailboxProperty::IsSubscribed, MailboxProperty::TotalEmails, MailboxProperty::UnreadEmails, MailboxProperty::TotalThreads, MailboxProperty::UnreadThreads, MailboxProperty::MyRights, ]); let account_id = request.account_id.document_id(); let cache = self.get_cached_messages(account_id).await?; let shared_ids = if access_token.is_shared(account_id) { cache.shared_mailboxes(access_token, Acl::Read).into() } else { None }; let ids = if let Some(ids) = ids { ids } else { cache .mailboxes .index .keys() .filter(|id| shared_ids.as_ref().is_none_or(|ids| ids.contains(**id))) .copied() .take(self.core.jmap.get_max_objects) .map(Into::into) .collect::>() }; let mut response = GetResponse { account_id: request.account_id.into(), state: Some(cache.mailboxes.change_id.into()), list: Vec::with_capacity(ids.len()), not_found: vec![], }; for id in ids { // Obtain the mailbox object let document_id = id.document_id(); let cached_mailbox = if let Some(mailbox) = cache.mailbox_by_id(&document_id).filter(|_| { shared_ids .as_ref() .is_none_or(|ids| ids.contains(document_id)) }) { mailbox } else { response.not_found.push(id); continue; }; let mut mailbox = Map::with_capacity(properties.len()); for property in &properties { let value = match property { MailboxProperty::Id => Value::Element(MailboxValue::Id(id)), MailboxProperty::Name => Value::Str(cached_mailbox.name.to_string().into()), MailboxProperty::Role => match cached_mailbox.role { SpecialUse::None => Value::Null, role => Value::Element(MailboxValue::Role(role)), }, MailboxProperty::SortOrder => { Value::Number(cached_mailbox.sort_order().unwrap_or_default().into()) } MailboxProperty::ParentId => { if let Some(parent_id) = cached_mailbox.parent_id() { Value::Element(MailboxValue::Id(parent_id.into())) } else { Value::Null } } MailboxProperty::TotalEmails => { Value::Number(cache.in_mailbox(document_id).count().into()) } MailboxProperty::UnreadEmails => Value::Number( cache .in_mailbox_without_keyword(document_id, &Keyword::Seen) .count() .into(), ), MailboxProperty::TotalThreads => Value::Number( cache .in_mailbox(document_id) .map(|m| m.thread_id) .collect::>() .len() .into(), ), MailboxProperty::UnreadThreads => Value::Number( cache .in_mailbox_without_keyword(document_id, &Keyword::Seen) .map(|m| m.thread_id) .collect::>() .len() .into(), ), MailboxProperty::MyRights => { if access_token.is_shared(account_id) { JmapRights::rights::( cached_mailbox.acls.as_slice().effective_acl(access_token), ) } else { JmapRights::all_rights::() } } MailboxProperty::IsSubscribed => Value::Bool( cached_mailbox .subscribers .contains(&access_token.primary_id()), ), MailboxProperty::ShareWith => JmapRights::share_with::( account_id, access_token, &cached_mailbox.acls, ), _ => Value::Null, }; mailbox.insert_unchecked(property.clone(), value); } // Add result to response response.list.push(mailbox.into()); } Ok(response) } } ================================================ FILE: crates/jmap/src/mailbox/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod get; pub mod query; pub mod set; ================================================ FILE: crates/jmap/src/mailbox/query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{api::query::QueryResponseBuilder, changes::state::JmapCacheState}; use common::{Server, auth::AccessToken}; use email::cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}; use jmap_proto::{ method::query::{Comparator, Filter, QueryRequest, QueryResponse}, object::mailbox::{Mailbox, MailboxComparator, MailboxFilter}, }; use std::{collections::BTreeMap, future::Future}; use store::{ ahash::AHashMap, roaring::RoaringBitmap, search::{SearchComparator, SearchFilter, SearchQuery}, write::SearchIndex, }; use types::{acl::Acl, special_use::SpecialUse}; pub trait MailboxQuery: Sync + Send { fn mailbox_query( &self, request: QueryRequest, access_token: &AccessToken, ) -> impl Future> + Send; } impl MailboxQuery for Server { async fn mailbox_query( &self, mut request: QueryRequest, access_token: &AccessToken, ) -> trc::Result { let account_id = request.account_id.document_id(); let sort_as_tree = request.arguments.sort_as_tree.unwrap_or(false); let filter_as_tree = request.arguments.filter_as_tree.unwrap_or(false); let mut filters = Vec::with_capacity(request.filter.len()); let mailboxes = self.get_cached_messages(account_id).await?; for cond in std::mem::take(&mut request.filter) { match cond { Filter::Property(cond) => { match cond { MailboxFilter::ParentId(parent_id) => { let parent_id = parent_id .and_then(|id| id.try_unwrap().map(|id| id.document_id())) .unwrap_or(u32::MAX); filters.push(SearchFilter::is_in_set( mailboxes .mailboxes .items .iter() .filter(|mailbox| mailbox.parent_id == parent_id) .map(|m| m.document_id) .collect::(), )); } MailboxFilter::Name(name) => { #[cfg(feature = "test_mode")] { // Used for concurrent requests tests if name == "__sleep" { tokio::time::sleep(std::time::Duration::from_secs(1)).await; } } let name = name.to_lowercase(); filters.push(SearchFilter::is_in_set( mailboxes .mailboxes .items .iter() .filter(|mailbox| mailbox.name.to_lowercase().contains(&name)) .map(|m| m.document_id) .collect::(), )); } MailboxFilter::Role(role) => { if let Some(role) = role { filters.push(SearchFilter::is_in_set( mailboxes .mailboxes .items .iter() .filter(|mailbox| mailbox.role == role) .map(|m| m.document_id) .collect::(), )); } else { filters.push(SearchFilter::is_in_set( mailboxes .mailboxes .items .iter() .filter(|mailbox| matches!(mailbox.role, SpecialUse::None)) .map(|m| m.document_id) .collect::(), )); } } MailboxFilter::HasAnyRole(has_role) => { filters.push(SearchFilter::is_in_set( mailboxes .mailboxes .items .iter() .filter(|mailbox| { matches!(mailbox.role, SpecialUse::None) != has_role }) .map(|m| m.document_id) .collect::(), )); } MailboxFilter::IsSubscribed(is_subscribed) => { filters.push(SearchFilter::is_in_set( mailboxes .mailboxes .items .iter() .filter(|mailbox| { mailbox.subscribers.contains(&access_token.primary_id) == is_subscribed }) .map(|m| m.document_id) .collect::(), )); } MailboxFilter::_T(other) => { return Err(trc::JmapEvent::UnsupportedFilter .into_err() .details(other)); } } } Filter::And => { filters.push(SearchFilter::And); } Filter::Or => { filters.push(SearchFilter::Or); } Filter::Not => { filters.push(SearchFilter::Not); } Filter::Close => { filters.push(SearchFilter::End); } } } let mut comparators = Vec::with_capacity(request.sort.as_ref().map_or(1, |s| s.len())); // Sort as tree if sort_as_tree { let sorted_set = mailboxes .mailboxes .items .iter() .map(|mailbox| (mailbox.path.as_str(), mailbox.document_id)) .collect::>(); comparators.push(SearchComparator::sorted_set( sorted_set .into_iter() .enumerate() .map(|(i, (_, v))| (v, i as u32)) .collect(), true, )); } // Parse sort criteria for comparator in request .sort .take() .filter(|s| !s.is_empty()) .unwrap_or_else(|| vec![Comparator::ascending(MailboxComparator::ParentId)]) { comparators.push(match comparator.property { MailboxComparator::Name => { let sorted_set = mailboxes .mailboxes .items .iter() .map(|mailbox| (mailbox.name.as_str(), mailbox.document_id)) .collect::>(); SearchComparator::sorted_set( sorted_set .into_iter() .enumerate() .map(|(i, (_, v))| (v, i as u32)) .collect(), comparator.is_ascending, ) } MailboxComparator::SortOrder => { let sorted_set = mailboxes .mailboxes .items .iter() .map(|mailbox| (mailbox.document_id, mailbox.sort_order)) .collect::>(); SearchComparator::sorted_set(sorted_set, comparator.is_ascending) } MailboxComparator::ParentId => { let sorted_set = mailboxes .mailboxes .items .iter() .map(|mailbox| { ( mailbox.document_id, mailbox.parent_id().map(|id| id + 1).unwrap_or_default(), ) }) .collect::>(); SearchComparator::sorted_set(sorted_set, comparator.is_ascending) } MailboxComparator::_T(other) => { return Err(trc::JmapEvent::UnsupportedSort.into_err().details(other)); } }); } let mut results = SearchQuery::new(SearchIndex::InMemory) .with_filters(filters) .with_comparators(comparators) .with_mask(if access_token.is_shared(account_id) { mailboxes.shared_mailboxes(access_token, Acl::Read) } else { mailboxes .mailboxes .items .iter() .map(|m| m.document_id) .collect() }) .filter(); // Filter as tree if filter_as_tree { let mut new_results = RoaringBitmap::new(); for document_id in results.results() { let mut check_id = document_id; for _ in 0..self.core.jmap.mailbox_max_depth { if let Some(mailbox) = mailboxes.mailbox_by_id(&check_id) { if let Some(parent_id) = mailbox.parent_id() { if results.results().contains(parent_id) { check_id = parent_id; } else { break; } } else { new_results.insert(document_id); } } } } results.update_results(new_results); } let mut response = QueryResponseBuilder::new( results.results().len() as usize, self.core.jmap.query_max_results, mailboxes.get_state(true), &request, ); for document_id in results.into_sorted() { if !response.add(0, document_id) { break; } } response.build() } } ================================================ FILE: crates/jmap/src/mailbox/set.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ api::acl::{JmapAcl, JmapRights}, changes::state::JmapCacheState, }; use common::{ Server, auth::AccessToken, sharing::EffectiveAcl, storage::index::ObjectIndexBuilder, }; #[allow(unused_imports)] use email::mailbox::{INBOX_ID, JUNK_ID, TRASH_ID, UidMailbox}; use email::{ cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}, mailbox::{ Mailbox, destroy::{MailboxDestroy, MailboxDestroyError}, }, }; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{SetRequest, SetResponse}, object::mailbox::{self, MailboxProperty, MailboxValue}, references::resolve::ResolveCreatedReference, request::IntoValid, types::state::State, }; use jmap_tools::{JsonPointerItem, Key, Map, Value}; use std::future::Future; use store::{ ValueKey, roaring::RoaringBitmap, write::{AlignedBytes, Archive, BatchBuilder, assert::AssertValue}, }; use trc::AddContext; use types::{ acl::Acl, collection::Collection, field::MailboxField, id::Id, special_use::SpecialUse, }; pub struct SetContext<'x> { account_id: u32, access_token: &'x AccessToken, is_shared: bool, response: SetResponse, mailbox_ids: RoaringBitmap, will_destroy: Vec, } pub trait MailboxSet: Sync + Send { fn mailbox_set( &self, request: SetRequest<'_, mailbox::Mailbox>, access_token: &AccessToken, ) -> impl Future>> + Send; fn mailbox_set_item( &self, changes_: Map<'_, MailboxProperty, MailboxValue>, update: Option<(u32, Archive)>, ctx: &SetContext, ) -> impl Future< Output = trc::Result< Result, SetError>, >, > + Send; } impl MailboxSet for Server { #[allow(clippy::blocks_in_conditions)] async fn mailbox_set( &self, mut request: SetRequest<'_, mailbox::Mailbox>, access_token: &AccessToken, ) -> trc::Result> { // Prepare response let account_id = request.account_id.document_id(); let on_destroy_remove_emails = request.arguments.on_destroy_remove_emails.unwrap_or(false); let cache = self.get_cached_messages(account_id).await?; let mut ctx = SetContext { account_id, is_shared: access_token.is_shared(account_id), access_token, response: SetResponse::from_request(&request, self.core.jmap.set_max_objects)? .with_state(cache.assert_state(true, &request.if_in_state)?), mailbox_ids: RoaringBitmap::from_iter(cache.mailboxes.index.keys()), will_destroy: request.unwrap_destroy().into_valid().collect(), }; let mut change_id = None; // Process creates let mut batch = BatchBuilder::new(); 'create: for (id, object) in request.unwrap_create() { let Some(object) = object.into_object() else { continue; }; // Validate quota if ctx.mailbox_ids.len() >= access_token.object_quota(Collection::Mailbox) as u64 { ctx.response.not_created.append( id, SetError::new(SetErrorType::OverQuota).with_description(concat!( "There are too many mailboxes, ", "please delete some before adding a new one." )), ); continue 'create; } match self.mailbox_set_item(object, None, &ctx).await? { Ok(builder) => { batch .with_account_id(account_id) .with_collection(Collection::Mailbox); let parent_id = builder.changes().unwrap().parent_id; if parent_id > 0 { batch .with_document(parent_id - 1) .assert_value(MailboxField::Archive, AssertValue::Some); } let document_id = self .store() .assign_document_ids(account_id, Collection::Mailbox, 1) .await .caused_by(trc::location!())?; batch .with_document(document_id) .custom(builder) .caused_by(trc::location!())? .commit_point(); ctx.mailbox_ids.insert(document_id); ctx.response.created(id, document_id); } Err(err) => { ctx.response.not_created.append(id, err); continue 'create; } } } if !batch.is_empty() { change_id = self .commit_batch(batch) .await .and_then(|ids| ids.last_change_id(account_id)) .caused_by(trc::location!())? .into(); } // Process updates let mut will_update = Vec::with_capacity(request.update.as_ref().map_or(0, |u| u.len())); let mut batch = BatchBuilder::new(); 'update: for (id, object) in request.unwrap_update().into_valid() { // Make sure id won't be destroyed if ctx.will_destroy.contains(&id) { ctx.response .not_updated .append(id, SetError::will_destroy()); continue 'update; } let Some(object) = object.into_object() else { continue 'update; }; // Obtain mailbox let document_id = id.document_id(); if let Some(mailbox) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::Mailbox, document_id, )) .await? { // Validate ACL let mailbox = mailbox .into_deserialized::() .caused_by(trc::location!())?; if ctx.is_shared { let acl = mailbox.inner.acls.effective_acl(access_token); if !acl.contains(Acl::Modify) { ctx.response.not_updated.append( id, SetError::forbidden() .with_description("You are not allowed to modify this mailbox."), ); continue 'update; } else if object.contains_key(&Key::Property(MailboxProperty::ShareWith)) && !acl.contains(Acl::Share) { ctx.response.not_updated.append( id, SetError::forbidden().with_description( "You are not allowed to change the permissions of this mailbox.", ), ); continue 'update; } } match self .mailbox_set_item(object, (document_id, mailbox).into(), &ctx) .await? { Ok(builder) => { batch .with_account_id(account_id) .with_collection(Collection::Mailbox); let parent_id = builder.changes().unwrap().parent_id; if parent_id > 0 { batch .with_document(parent_id - 1) .assert_value(MailboxField::Archive, AssertValue::Some); } batch .with_document(document_id) .custom(builder) .caused_by(trc::location!())? .commit_point(); will_update.push(id); } Err(err) => { ctx.response.not_updated.append(id, err); continue 'update; } } } else { ctx.response.not_updated.append(id, SetError::not_found()); } } if !batch.is_empty() { match self .commit_batch(batch) .await .and_then(|ids| ids.last_change_id(account_id)) { Ok(change_id_) => { change_id = Some(change_id_); for id in will_update { ctx.response.updated.append(id, None); } } Err(err) if err.is_assertion_failure() => { for id in will_update { ctx.response.not_updated.append( id, SetError::forbidden().with_description( "Another process modified this mailbox, please try again.", ), ); } } Err(err) => { return Err(err.caused_by(trc::location!())); } } } // Process deletions for id in ctx.will_destroy { match self .mailbox_destroy( account_id, id.document_id(), ctx.access_token, on_destroy_remove_emails, ) .await? { Ok(change_id_) => { if change_id_.is_some() { change_id = change_id_; } ctx.response.destroyed.push(id); } Err(err) => { ctx.response.not_destroyed.append( id, match err { MailboxDestroyError::CannotDestroy => SetError::forbidden() .with_description( "You are not allowed to delete Inbox, Junk or Trash folders.", ), MailboxDestroyError::Forbidden => SetError::forbidden() .with_description("You are not allowed to delete this mailbox."), MailboxDestroyError::HasChildren => { SetError::new(SetErrorType::MailboxHasChild) .with_description("Mailbox has at least one children.") } MailboxDestroyError::HasEmails => { SetError::new(SetErrorType::MailboxHasEmail) .with_description("Mailbox is not empty.") } MailboxDestroyError::NotFound => SetError::not_found(), MailboxDestroyError::AssertionFailed => SetError::forbidden() .with_description(concat!( "Another process modified a message in this mailbox ", "while deleting it, please try again." )), }, ); } } } // Write changes if let Some(change_id) = change_id { ctx.response.new_state = State::Exact(change_id).into(); } Ok(ctx.response) } #[allow(clippy::blocks_in_conditions)] async fn mailbox_set_item( &self, changes_: Map<'_, MailboxProperty, MailboxValue>, update: Option<(u32, Archive)>, ctx: &SetContext<'_>, ) -> trc::Result, SetError>> { // Parse properties let mut changes = update .as_ref() .map(|(_, obj)| obj.inner.clone()) .unwrap_or_else(|| Mailbox::new(String::new())); let mut has_acl_changes = false; for (property, mut value) in changes_.into_vec() { if let Err(err) = ctx.response.resolve_self_references(&mut value) { return Ok(Err(err)); }; match (&property, value) { (Key::Property(MailboxProperty::Name), Value::Str(value)) => { let value = value.trim(); if !value.is_empty() && value.len() < self.core.jmap.mailbox_name_max_len { changes.name = value.into(); } else { return Ok(Err(SetError::invalid_properties() .with_property(MailboxProperty::Name) .with_description( if !value.is_empty() { "Mailbox name is too long." } else { "Mailbox name cannot be empty." } .to_string(), ))); } } ( Key::Property(MailboxProperty::ParentId), Value::Element(MailboxValue::Id(value)), ) => { let parent_id = value.document_id(); if ctx.will_destroy.contains(&value) { return Ok(Err(SetError::will_destroy() .with_description("Parent ID will be destroyed."))); } else if !ctx.mailbox_ids.contains(parent_id) { return Ok(Err(SetError::invalid_properties() .with_description("Parent ID does not exist."))); } changes.parent_id = parent_id + 1; } (Key::Property(MailboxProperty::ParentId), Value::Null) => { changes.parent_id = 0; } (Key::Property(MailboxProperty::IsSubscribed), Value::Bool(subscribe)) => { let account_id = ctx.access_token.primary_id(); if subscribe { if !changes.subscribers.contains(&account_id) { changes.subscribers.push(account_id); } } else { changes.subscribers.retain(|id| *id != account_id); } } ( Key::Property(MailboxProperty::Role), Value::Element(MailboxValue::Role(role)), ) => { changes.role = role; } (Key::Property(MailboxProperty::Role), Value::Null) => { changes.role = SpecialUse::None; } (Key::Property(MailboxProperty::SortOrder), Value::Number(value)) => { changes.sort_order = Some(value.cast_to_u64() as u32); } (Key::Property(MailboxProperty::ShareWith), value) => { match JmapRights::acl_set::(value) { Ok(acls) => { has_acl_changes = true; changes.acls = acls; continue; } Err(err) => { return Ok(Err(err)); } } } (Key::Property(MailboxProperty::Pointer(pointer)), value) if matches!( pointer.first(), Some(JsonPointerItem::Key(Key::Property( MailboxProperty::ShareWith ))) ) => { let mut pointer = pointer.iter(); pointer.next(); match JmapRights::acl_patch::(changes.acls, pointer, value) { Ok(acls) => { has_acl_changes = true; changes.acls = acls; continue; } Err(err) => { return Ok(Err(err)); } } } _ => { return Ok(Err(SetError::invalid_properties() .with_property(property.into_owned()) .with_description("Invalid property or value.".to_string()))); } } } // Validate depth and circular parent-child relationship if update .as_ref() .is_none_or(|(_, m)| m.inner.parent_id != changes.parent_id) { let mut mailbox_parent_id = changes.parent_id; let current_mailbox_id = update .as_ref() .map_or(u32::MAX, |(mailbox_id, _)| *mailbox_id + 1); let mut success = false; for depth in 0..self.core.jmap.mailbox_max_depth { if mailbox_parent_id == current_mailbox_id { return Ok(Err(SetError::invalid_properties() .with_property(MailboxProperty::ParentId) .with_description("Mailbox cannot be a parent of itself."))); } else if mailbox_parent_id == 0 { if depth == 0 && ctx.is_shared { return Ok(Err(SetError::forbidden() .with_description("You are not allowed to create root folders."))); } success = true; break; } let parent_document_id = mailbox_parent_id - 1; if let Some(mailbox_) = self .store() .get_value::>(ValueKey::archive( ctx.account_id, Collection::Mailbox, parent_document_id, )) .await? { let mailbox = mailbox_ .unarchive::() .caused_by(trc::location!())?; if depth == 0 && ctx.is_shared && !mailbox .acls .effective_acl(ctx.access_token) .contains(Acl::CreateChild) { return Ok(Err(SetError::forbidden().with_description( "You are not allowed to create sub mailboxes under this mailbox.", ))); } mailbox_parent_id = mailbox.parent_id.into(); } else if ctx.mailbox_ids.contains(parent_document_id) { // Parent mailbox is probably created within the same request success = true; break; } else { return Ok(Err(SetError::invalid_properties() .with_property(MailboxProperty::ParentId) .with_description("Mailbox parent does not exist."))); } } if !success { return Ok(Err(SetError::invalid_properties() .with_property(MailboxProperty::ParentId) .with_description( "Mailbox parent-child relationship is too deep.", ))); } } let cached_mailboxes = self.get_cached_messages(ctx.account_id).await?; // Verify that the mailbox role is unique. if update .as_ref() .is_none_or(|(_, m)| m.inner.role != changes.role) { if !matches!(changes.role, SpecialUse::None) && cached_mailboxes.mailbox_by_role(&changes.role).is_some() { return Ok(Err(SetError::invalid_properties() .with_property(MailboxProperty::Role) .with_description(format!( "A mailbox with role '{}' already exists.", changes.role.as_str().unwrap_or_default() )))); } // Role of internal folders cannot be modified if update.as_ref().is_some_and(|(document_id, _)| { *document_id == INBOX_ID || *document_id == TRASH_ID || *document_id == JUNK_ID }) { return Ok(Err(SetError::invalid_properties() .with_property(MailboxProperty::Role) .with_description( "You are not allowed to change the role of Inbox, Junk or Trash folders.", ))); } } // Verify that the mailbox name is unique. if !changes.name.is_empty() { // Obtain parent mailbox id let lower_name = changes.name.to_lowercase(); if update .as_ref() .is_none_or(|(_, m)| m.inner.name != changes.name) && cached_mailboxes.mailboxes.items.iter().any(|m| { m.name.to_lowercase() == lower_name && m.parent_id().map_or(0, |id| id + 1) == changes.parent_id }) { return Ok(Err(SetError::invalid_properties() .with_property(MailboxProperty::Name) .with_description(format!( "A mailbox with name '{}' already exists.", changes.name )))); } } else { return Ok(Err(SetError::invalid_properties() .with_property(MailboxProperty::Name) .with_description("Mailbox name cannot be empty."))); } // Refresh ACLs let current = update.map(|(_, current)| current); if has_acl_changes { if !changes.acls.is_empty() && let Err(err) = self.acl_validate(&changes.acls).await { return Ok(Err(err.into())); } self.refresh_acls( &changes.acls, current.as_ref().map(|m| m.inner.acls.as_slice()), ) .await; } // Validate Ok(Ok(ObjectIndexBuilder::new() .with_changes(changes) .with_current_opt(current))) } } ================================================ FILE: crates/jmap/src/participant_identity/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::Server; use directory::{PrincipalData, QueryParams}; use groupware::calendar::{ParticipantIdentities, ParticipantIdentity}; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::participant_identity::{self, ParticipantIdentityProperty, ParticipantIdentityValue}, }; use jmap_tools::{Map, Value}; use store::{ Serialize, ValueKey, write::{AlignedBytes, Archive, Archiver, BatchBuilder}, }; use trc::AddContext; use types::{collection::Collection, field::PrincipalField, id::Id}; pub trait ParticipantIdentityGet: Sync + Send { fn participant_identity_get( &self, request: GetRequest, ) -> impl Future>> + Send; fn participant_identity_get_or_create( &self, account_id: u32, ) -> impl Future>>> + Send; } impl ParticipantIdentityGet for Server { async fn participant_identity_get( &self, mut request: GetRequest, ) -> trc::Result> { let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ ParticipantIdentityProperty::Id, ParticipantIdentityProperty::Name, ParticipantIdentityProperty::CalendarAddress, ParticipantIdentityProperty::IsDefault, ]); let account_id = request.account_id.document_id(); let identities = self.participant_identity_get_or_create(account_id).await?; let mut response = GetResponse { account_id: request.account_id.into(), state: None, list: Vec::new(), not_found: vec![], }; let Some(identities) = identities else { response.not_found = ids.unwrap_or_default(); return Ok(response); }; let identities = identities .unarchive::() .caused_by(trc::location!())?; let ids = if let Some(ids) = ids { ids } else { identities .identities .iter() .take(self.core.jmap.get_max_objects) .map(|i| Id::from(i.id.to_native())) .collect::>() }; for id in ids { // Obtain the identity object let document_id = id.document_id(); let Some(identity) = identities.identities.iter().find(|i| i.id == document_id) else { response.not_found.push(id); continue; }; let mut result = Map::with_capacity(properties.len()); for property in &properties { let value = match &property { ParticipantIdentityProperty::Id => { Value::Element(ParticipantIdentityValue::Id(id)) } ParticipantIdentityProperty::Name => Value::Str( identity .name .as_ref() .map(|n| n.as_str()) .unwrap_or(identities.default_name.as_str()) .to_string() .into(), ), ParticipantIdentityProperty::CalendarAddress => { Value::Str(identity.calendar_address.to_string().into()) } ParticipantIdentityProperty::IsDefault => { Value::Bool(identities.default == document_id) } }; result.insert_unchecked(property.clone(), value); } response.list.push(result.into()); } Ok(response) } async fn participant_identity_get_or_create( &self, account_id: u32, ) -> trc::Result>> { if let Some(identities) = self .store() .get_value::>(ValueKey::property( account_id, Collection::Principal, 0, PrincipalField::ParticipantIdentities, )) .await? { return Ok(Some(identities)); } // Obtain principal let principal = if let Some(principal) = self .core .storage .directory .query(QueryParams::id(account_id).with_return_member_of(false)) .await .caused_by(trc::location!())? { principal } else { return Ok(None); }; let mut emails = Vec::new(); let mut description = None; for data in principal.data { match data { PrincipalData::PrimaryEmail(v) | PrincipalData::EmailAlias(v) => emails.push(v), PrincipalData::Description(v) => description = Some(v), _ => {} } } let num_emails = emails.len(); if num_emails == 0 { return Ok(None); } // Build identities let identities = ParticipantIdentities { identities: emails .iter() .enumerate() .map(|(id, email)| ParticipantIdentity { id: id as u32, name: None, calendar_address: format!("mailto:{email}"), }) .collect(), default: 0, default_name: description.unwrap_or(principal.name), }; let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Principal) .with_document(0) .set( PrincipalField::ParticipantIdentities, Archiver::new(identities) .serialize() .caused_by(trc::location!())?, ); self.commit_batch(batch).await.caused_by(trc::location!())?; self.store() .get_value::>(ValueKey::property( account_id, Collection::Principal, 0, PrincipalField::ParticipantIdentities, )) .await } } ================================================ FILE: crates/jmap/src/participant_identity/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod get; pub mod set; ================================================ FILE: crates/jmap/src/participant_identity/set.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::participant_identity::get::ParticipantIdentityGet; use common::{Server, auth::AccessToken}; use directory::QueryParams; use groupware::calendar::{ParticipantIdentities, ParticipantIdentity}; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{SetRequest, SetResponse}, object::participant_identity::{self, ParticipantIdentityProperty, ParticipantIdentityValue}, request::{IntoValid, reference::MaybeIdReference}, }; use jmap_tools::{Key, Value}; use store::{ Serialize, ahash::AHashSet, write::{Archiver, BatchBuilder}, }; use trc::AddContext; use types::{collection::Collection, field::PrincipalField}; use utils::sanitize_email; pub trait ParticipantIdentitySet: Sync + Send { fn participant_identity_set( &self, request: SetRequest<'_, participant_identity::ParticipantIdentity>, access_token: &AccessToken, ) -> impl Future>> + Send; } impl ParticipantIdentitySet for Server { async fn participant_identity_set( &self, mut request: SetRequest<'_, participant_identity::ParticipantIdentity>, access_token: &AccessToken, ) -> trc::Result> { let account_id = request.account_id.document_id(); let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; let will_destroy = request.unwrap_destroy().into_valid().collect::>(); let (identity_archive, mut identities) = match self.participant_identity_get_or_create(account_id).await? { Some(archive) => { let identities = archive .deserialize::() .caused_by(trc::location!())?; (Some(archive), identities) } None => (None, ParticipantIdentities::default()), }; // Obtain allowed emails let allowed_emails = self .directory() .query(QueryParams::id(account_id).with_return_member_of(false)) .await? .map(|p| p.into_email_addresses().collect::>()) .unwrap_or_default(); // Process creates let mut has_changes = false; 'create: for (id, object) in request.unwrap_create() { let mut identity = ParticipantIdentity::default(); if let Err(err) = validate_identity_value(object, &mut identity, &allowed_emails) { response.not_created.append(id, err); continue 'create; } if identities .identities .iter() .any(|i| i.calendar_address == identity.calendar_address) { response.not_created.append( id, SetError::invalid_properties() .with_property(ParticipantIdentityProperty::CalendarAddress) .with_description("Calendar address already in use.".to_string()), ); continue 'create; } // Validate quota if identities.identities.len() >= access_token.object_quota(Collection::Identity) as usize { response.not_created.append( id, SetError::new(SetErrorType::OverQuota).with_description(concat!( "There are too many identities, ", "please delete some before adding a new one." )), ); continue 'create; } let document_id = identities .identities .iter() .map(|i| i.id) .max() .unwrap_or_default() + 1; identity.id = document_id; identities.identities.push(identity); if let Some(MaybeIdReference::Reference(id_ref)) = &request.arguments.on_success_set_is_default && id_ref == &id { identities.default = document_id; } has_changes = true; response.created(id, document_id); } // Process updates 'update: for (id, object) in request.unwrap_update().into_valid() { // Make sure id won't be destroyed if will_destroy.contains(&id) { response.not_updated.append(id, SetError::will_destroy()); continue 'update; } let Some(identity) = identities .identities .iter_mut() .find(|i| i.id == id.document_id()) else { response.not_updated.append(id, SetError::not_found()); continue 'update; }; if let Err(err) = validate_identity_value(object, identity, &allowed_emails) { response.not_updated.append(id, err); continue 'update; } has_changes = true; response.updated.append(id, None); } // Process deletions for id in &will_destroy { let document_id = id.document_id(); if identities.identities.iter().any(|i| i.id == document_id) { response.destroyed.push(*id); } else { response.not_destroyed.append(*id, SetError::not_found()); } } if !response.destroyed.is_empty() { has_changes = true; identities .identities .retain(|i| !response.destroyed.iter().any(|id| id.document_id() == i.id)); } if let Some(MaybeIdReference::Id(id)) = request.arguments.on_success_set_is_default { let id = id.document_id(); if identities.identities.iter().any(|i| i.id == id) { identities.default = id; has_changes = true; } } // Write changes if has_changes { let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Principal) .with_document(0); if let Some(archive) = identity_archive { batch.assert_value(PrincipalField::ParticipantIdentities, archive); } batch.set( PrincipalField::ParticipantIdentities, Archiver::new(identities) .serialize() .caused_by(trc::location!())?, ); self.commit_batch(batch).await.caused_by(trc::location!())?; } Ok(response) } } fn validate_identity_value( update: Value<'_, ParticipantIdentityProperty, ParticipantIdentityValue>, identity: &mut ParticipantIdentity, allowed_emails: &AHashSet, ) -> Result<(), SetError> { for (property, value) in update.into_expanded_object() { let Key::Property(property) = property else { return Err(SetError::invalid_properties() .with_property(property.to_owned()) .with_description("Invalid property.")); }; match (property, value) { (ParticipantIdentityProperty::Name, Value::Str(value)) if value.len() < 255 => { identity.name = value.into_owned().into(); } (ParticipantIdentityProperty::CalendarAddress, Value::Str(value)) => { if identity.calendar_address != value { let email = if let Some(email) = value.strip_prefix("mailto:") { sanitize_email(email) } else { sanitize_email(&value) }; if let Some(email) = email { if allowed_emails.iter().any(|e| e == &email) { identity.calendar_address = format!("mailto:{email}"); } else { return Err(SetError::invalid_properties() .with_property(ParticipantIdentityProperty::CalendarAddress) .with_description( "Calendar address not configured for this account.".to_string(), )); } } else { return Err(SetError::invalid_properties() .with_property(ParticipantIdentityProperty::CalendarAddress) .with_description("Invalid or missing calendar address.".to_string())); } } } (property, _) => { return Err(SetError::invalid_properties() .with_property(property.clone()) .with_description("Field could not be set.")); } } } // Validate email address if !identity.calendar_address.is_empty() { Ok(()) } else { Err(SetError::invalid_properties() .with_property(ParticipantIdentityProperty::CalendarAddress) .with_description("Missing calendar address.")) } } ================================================ FILE: crates/jmap/src/principal/availability.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{calendar::Availability, calendar_event::CalendarSyntheticId}; use calcard::{ common::timezone::Tz, icalendar::{ ArchivedICalendarClassification, ArchivedICalendarParameterValue, ArchivedICalendarParticipationStatus, ArchivedICalendarProperty, ArchivedICalendarStatus, ArchivedICalendarTransparency, ArchivedICalendarValue, ICalendarParameterName, }, jscalendar::{JSCalendar, JSCalendarProperty, JSCalendarValue}, }; use common::{Server, TinyCalendarPreferences, auth::AccessToken}; use directory::Permission; use groupware::{ cache::GroupwareCache, calendar::{CALENDAR_SUBSCRIBED, CalendarEvent}, }; use jmap_proto::{ method::availability::{ BusyPeriod, BusyStatus, GetAvailabilityRequest, GetAvailabilityResponse, }, object::calendar::IncludeInAvailability, request::IntoValid, types::date::UTCDate, }; use jmap_tools::{Key, Map, Value}; use std::{collections::hash_map::Entry, future::Future}; use store::{ValueKey, ahash::AHashMap, write::{AlignedBytes, Archive}}; use trc::AddContext; use types::{ TimeRange, acl::Acl, collection::{Collection, SyncCollection}, id::Id, }; use utils::sanitize_email; pub trait PrincipalGetAvailability: Sync + Send { fn principal_get_availability( &self, request: GetAvailabilityRequest, access_token: &AccessToken, ) -> impl Future> + Send; } impl PrincipalGetAvailability for Server { async fn principal_get_availability( &self, request: GetAvailabilityRequest, access_token: &AccessToken, ) -> trc::Result { if !self.core.groupware.allow_directory_query && !access_token.has_permission(Permission::IndividualList) { return Err(trc::JmapEvent::Forbidden .into_err() .details("The administrator has disabled directory queries.".to_string())); } // Process parameters if !request.id.is_valid() { return Err(trc::JmapEvent::InvalidArguments .into_err() .details("Missing principal id")); } let properties = request .event_properties .map(|props| props.into_valid().collect::>()) .unwrap_or_default(); if properties .iter() .any(|p| !matches!(p, JSCalendarProperty::Id | JSCalendarProperty::BaseEventId)) { return Err(trc::JmapEvent::InvalidArguments .into_err() .details("Only 'id' and 'baseEventId' properties are supported in results")); } let return_event_details = !properties.is_empty(); let max_instances = self.core.groupware.max_ical_instances; let filter = TimeRange { start: request.utc_start.timestamp(), end: request.utc_end.timestamp(), }; let principal_id = request.id.document_id(); let principal = self .get_access_token(principal_id) .await .caused_by(trc::location!())?; let mut periods = Vec::new(); for account_id in principal.all_ids_by_collection(Collection::Calendar) { let resources = self .fetch_dav_resources(access_token, account_id, SyncCollection::Calendar) .await .caused_by(trc::location!())?; // Obtain shared ids let is_account_owner = principal_id == account_id; let shared_ids = if !access_token.is_member(account_id) { // Condition: The user has the "mayReadFreeBusy" permission for the calendar. let shared_ids = resources.shared_items( access_token, [Acl::ReadItems, Acl::SchedulingReadFreeBusy], true, ); if shared_ids.is_empty() { continue; } shared_ids.into() } else { None }; // Condition: The event finishes after the "utcStart" argument and starts before the "utcEnd" argument. let mut preferences_cache: AHashMap> = AHashMap::default(); 'next_event: for resource in resources.resources.iter().filter(|r| { r.event_time_range().is_some_and(|(start, end)| { shared_ids .as_ref() .is_none_or(|ids| ids.contains(r.document_id)) && filter.is_in_range(false, start, end) }) }) { // Obtain calendar settings let mut include_in_availability = None; let mut default_tz = Tz::UTC; let mut is_subscribed = is_account_owner; for calendar_id in resource .child_names() .unwrap_or_default() .iter() .map(|n| n.parent_id) { match preferences_cache.entry(calendar_id) { Entry::Occupied(e) => { if let Some(prefs) = e.get() { default_tz = prefs.tz; is_subscribed |= prefs.flags & CALENDAR_SUBSCRIBED != 0; include_in_availability = IncludeInAvailability::from_flags(prefs.flags); } } Entry::Vacant(e) => { if let Some(prefs) = resources .container_resource_by_id(calendar_id) .and_then(|r| r.calendar_preferences(principal_id)) { default_tz = prefs.tz; is_subscribed |= prefs.flags & CALENDAR_SUBSCRIBED != 0; include_in_availability = IncludeInAvailability::from_flags(prefs.flags); e.insert(Some(prefs)); } else { e.insert(None); } } } } let include_in_availability = include_in_availability.unwrap_or({ if is_account_owner { IncludeInAvailability::All } else { IncludeInAvailability::None } }); if !is_subscribed || include_in_availability == IncludeInAvailability::None { continue 'next_event; } // Fetch event let document_id = resource.document_id; let Some(archive) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, )) .await .caused_by(trc::location!())? else { continue; }; let event = archive .unarchive::() .caused_by(trc::location!())?; // Find the component ids that match the criteria let mut matching_component_ids = AHashMap::new(); 'next_component: for (component_id, component) in event.data.event.components.iter().enumerate() { if !component.component_type.is_event_or_todo() { continue 'next_component; } let mut is_cancelled = false; let mut is_main_event = true; let mut busy_status = None; for entry in component.entries.iter() { match (&entry.name, entry.values.first()) { ( ArchivedICalendarProperty::Status, Some(ArchivedICalendarValue::Status( ArchivedICalendarStatus::Cancelled, )), ) => { // The "status" property of the event is not "cancelled". is_cancelled = true; } (ArchivedICalendarProperty::RecurrenceId, _) => { is_main_event = false; } ( ArchivedICalendarProperty::Class, Some(ArchivedICalendarValue::Classification( ArchivedICalendarClassification::Confidential, )), ) => { // Condition: The event's "privacy" property is not "secret". continue 'next_component; } ( ArchivedICalendarProperty::Transp, Some(ArchivedICalendarValue::Transparency( ArchivedICalendarTransparency::Transparent, )), ) => { // Condition: The "freeBusyStatus" property of the event is "busy" (or omitted, as this is the default). continue 'next_component; } (ArchivedICalendarProperty::Attendee, Some(value)) if include_in_availability == IncludeInAvailability::Attending => { if let Some(attendee) = value.as_text().and_then(|attendee| { sanitize_email( attendee.strip_prefix("mailto:").unwrap_or(attendee), ) }) { // Condition: the Principal is a participant of the event, and has a "participationStatus" of "accepted" or "tentative". if principal.emails.contains(&attendee) { busy_status = Some( entry .parameters(&ICalendarParameterName::Partstat) .next() .map(|v| { match v { ArchivedICalendarParameterValue::Partstat( ArchivedICalendarParticipationStatus::Accepted, ) => BusyStatus::Confirmed, ArchivedICalendarParameterValue::Partstat( ArchivedICalendarParticipationStatus::Tentative, ) => BusyStatus::Tentative, ArchivedICalendarParameterValue::Partstat( ArchivedICalendarParticipationStatus::Declined, ) => { is_cancelled = true; BusyStatus::Unavailable } _ => BusyStatus::Unavailable, } }) .unwrap_or(BusyStatus::Unavailable), ); } } } _ => (), } } if is_cancelled { if is_main_event { continue 'next_event; } else { continue 'next_component; } } let busy_status = if let Some(busy_status) = busy_status { busy_status } else if include_in_availability == IncludeInAvailability::All { BusyStatus::Confirmed } else { continue 'next_component; }; matching_component_ids.insert(component_id as u32, busy_status); } if matching_component_ids.is_empty() { // No events matched the criteria continue 'next_event; } for expansion in event.data.expand(default_tz, filter).unwrap_or_default() { let Some(busy_status) = matching_component_ids.get(&expansion.comp_id) else { continue; }; if periods.len() < max_instances { periods.push(FreeBusyResult { utc_start: expansion.start, utc_end: expansion.end, busy_status: *busy_status, expansion_id: expansion.comp_id, document_id, }); } else { return Err(trc::JmapEvent::RequestTooLarge .into_err() .details("The number of expanded instances exceeds the server limit")); } } } } let mut result = GetAvailabilityResponse { list: Vec::with_capacity(periods.len()), }; if periods.is_empty() { return Ok(result); } // Sort by busy status and start time periods.sort_unstable_by(|a, b| { a.busy_status .cmp(&b.busy_status) .then_with(|| a.utc_start.cmp(&b.utc_start)) }); if return_event_details { for period in periods { result.list.push(period.into()); } } else { // Merge intervals with same busy status let mut start_time = periods[0].utc_start; let mut end_time = periods[0].utc_end; let mut current_status = periods[0].busy_status; for curr in periods.iter().skip(1) { if curr.utc_start <= end_time && curr.busy_status == current_status { end_time = end_time.max(curr.utc_end); } else { result.list.push(BusyPeriod { utc_start: UTCDate::from_timestamp(start_time), utc_end: UTCDate::from_timestamp(end_time), busy_status: Some(current_status), event: None, }); start_time = curr.utc_start; end_time = curr.utc_end; current_status = curr.busy_status; } } result.list.push(BusyPeriod { utc_start: UTCDate::from_timestamp(start_time), utc_end: UTCDate::from_timestamp(end_time), busy_status: Some(current_status), event: None, }); } Ok(result) } } struct FreeBusyResult { utc_start: i64, utc_end: i64, busy_status: BusyStatus, expansion_id: u32, document_id: u32, } impl From for BusyPeriod { fn from(value: FreeBusyResult) -> Self { BusyPeriod { utc_start: UTCDate::from_timestamp(value.utc_start), utc_end: UTCDate::from_timestamp(value.utc_end), busy_status: Some(value.busy_status), event: JSCalendar(Value::Object(Map::from(vec![ ( Key::Property(JSCalendarProperty::Id), Value::Element(JSCalendarValue::Id(::new( value.expansion_id, value.document_id, ))), ), ( Key::Property(JSCalendarProperty::BaseEventId), Value::Element(JSCalendarValue::Id(Id::from(value.document_id))), ), ]))) .into(), } } } ================================================ FILE: crates/jmap/src/principal/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{Server, auth::AccessToken}; use directory::{Permission, QueryParams, Type, backend::internal::manage::ManageDirectory}; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::principal::{Principal, PrincipalProperty, PrincipalType, PrincipalValue}, request::capability::Capability, types::state::State, }; use jmap_tools::{Key, Map, Value}; use std::future::Future; use store::roaring::RoaringBitmap; use trc::AddContext; pub trait PrincipalGet: Sync + Send { fn principal_get( &self, request: GetRequest, access_token: &AccessToken, ) -> impl Future>> + Send; } impl PrincipalGet for Server { async fn principal_get( &self, mut request: GetRequest, access_token: &AccessToken, ) -> trc::Result> { if !self.core.groupware.allow_directory_query && !access_token.has_permission(Permission::IndividualList) { return Err(trc::JmapEvent::Forbidden .into_err() .details("The administrator has disabled directory queries.".to_string())); } let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ PrincipalProperty::Id, PrincipalProperty::Type, PrincipalProperty::Name, PrincipalProperty::Description, PrincipalProperty::Email, ]); // Return all principals let principal_ids = self .store() .list_principals( None, access_token.tenant_id(), &[ Type::Individual, Type::Group, Type::Resource, Type::Location, ], false, 0, 0, ) .await .caused_by(trc::location!())? .items .into_iter() .map(|p| p.id()) .collect::(); let ids = if let Some(ids) = ids { ids } else { principal_ids .iter() .take(self.core.jmap.get_max_objects) .map(Into::into) .collect::>() }; let mut response = GetResponse { account_id: request.account_id.into(), state: State::Initial.into(), list: Vec::with_capacity(ids.len()), not_found: vec![], }; for id in ids { // Obtain the principal let document_id = id.document_id(); let principal = if principal_ids.contains(document_id) && let Some(principal) = self .core .storage .directory .query(QueryParams::id(document_id).with_return_member_of(false)) .await? { principal } else { response.not_found.push(id); continue; }; let mut result = Map::with_capacity(properties.len()); for property in &properties { let value = match property { PrincipalProperty::Id => Value::Element(PrincipalValue::Id(id)), PrincipalProperty::Type => { Value::Element(PrincipalValue::Type(match principal.typ() { Type::Individual => PrincipalType::Individual, Type::Group => PrincipalType::Group, Type::Resource => PrincipalType::Resource, Type::Location => PrincipalType::Location, _ => PrincipalType::Other, })) } PrincipalProperty::Name => Value::Str(principal.name().to_string().into()), PrincipalProperty::Description => principal .description() .map(|v| Value::Str(v.to_string().into())) .unwrap_or(Value::Null), PrincipalProperty::Email => principal .primary_email() .map(|email| Value::Str(email.to_string().into())) .unwrap_or(Value::Null), PrincipalProperty::Accounts => Value::Object(Map::from(vec![( Key::Property(PrincipalProperty::IdValue(id)), Value::Object(Map::from_iter( [ Capability::Mail, Capability::Contacts, Capability::Calendars, Capability::FileNode, Capability::Principals, ] .iter() .map(|cap| { ( Key::Property(PrincipalProperty::Capability(*cap)), Value::Object(Map::new()), ) }) .chain([ ( Key::Property(PrincipalProperty::Capability( Capability::PrincipalsOwner, )), Value::Object(Map::from(vec![ ( Key::Borrowed("accountIdForPrincipal"), Value::Element(PrincipalValue::Id(id)), ), ( Key::Borrowed("principalId"), Value::Element(PrincipalValue::Id(id)), ), ])), ), ( Key::Property(PrincipalProperty::Capability( Capability::Calendars, )), Value::Object(Map::from(vec![ ( Key::Borrowed("accountId"), Value::Element(PrincipalValue::Id(id)), ), (Key::Borrowed("mayGetAvailability"), Value::Bool(true)), (Key::Borrowed("mayShareWith"), Value::Bool(true)), ( Key::Borrowed("calendarAddress"), Value::Str( principal .primary_email() .map(|email| format!("mailto:{}", email)) .unwrap_or_default() .into(), ), ), ])), ), ]), )), )])), PrincipalProperty::Capabilities => Value::Object(Map::from_iter( [ Capability::Mail, Capability::Contacts, Capability::Calendars, Capability::FileNode, Capability::Principals, ] .iter() .map(|cap| { ( Key::Property(PrincipalProperty::Capability(*cap)), Value::Object(Map::new()), ) }), )), _ => Value::Null, }; result.insert_unchecked(property.clone(), value); } response.list.push(result.into()); } Ok(response) } } ================================================ FILE: crates/jmap/src/principal/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod availability; pub mod get; pub mod query; ================================================ FILE: crates/jmap/src/principal/query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::api::query::QueryResponseBuilder; use common::{Server, auth::AccessToken}; use directory::{Permission, QueryParams, Type, backend::internal::manage::ManageDirectory}; use http_proto::HttpSessionData; use jmap_proto::{ method::query::{Filter, QueryRequest, QueryResponse}, object::principal::{Principal, PrincipalFilter, PrincipalType}, types::state::State, }; use std::future::Future; use store::{ roaring::RoaringBitmap, search::{SearchFilter, SearchQuery}, write::SearchIndex, }; use trc::AddContext; pub trait PrincipalQuery: Sync + Send { fn principal_query( &self, request: QueryRequest, access_token: &AccessToken, session: &HttpSessionData, ) -> impl Future> + Send; } impl PrincipalQuery for Server { async fn principal_query( &self, mut request: QueryRequest, access_token: &AccessToken, session: &HttpSessionData, ) -> trc::Result { if !self.core.groupware.allow_directory_query && !access_token.has_permission(Permission::IndividualList) { return Err(trc::JmapEvent::Forbidden .into_err() .details("The administrator has disabled directory queries.".to_string())); } let principal_ids = self .store() .list_principals( None, access_token.tenant_id(), &[ Type::Individual, Type::Group, Type::Resource, Type::Location, ], false, 0, 0, ) .await .caused_by(trc::location!())? .items .into_iter() .map(|p| p.id()) .collect::(); let mut filters = Vec::with_capacity(request.filter.len()); for cond in std::mem::take(&mut request.filter) { match cond { Filter::Property(cond) => match cond { PrincipalFilter::Name(name) => { if let Some(principal) = self .core .storage .directory .query(QueryParams::name(name.as_str()).with_return_member_of(false)) .await? { filters.push(SearchFilter::is_in_set( RoaringBitmap::from_sorted_iter([principal.id()]).unwrap(), )); } } PrincipalFilter::Email(email) => { if let Some(id) = self .email_to_id(self.directory(), &email, session.session_id) .await? { filters.push(SearchFilter::is_in_set( RoaringBitmap::from_sorted_iter([id]).unwrap(), )); } } PrincipalFilter::AccountIds(ids) => { filters.push(SearchFilter::is_in_set( ids.into_iter() .filter_map(|id| { let id = id.document_id(); if principal_ids.contains(id) { Some(id) } else { None } }) .collect::(), )); } PrincipalFilter::Text(text) => { filters.push(SearchFilter::is_in_set( self.store() .list_principals( Some(text.as_str()), access_token.tenant.map(|t| t.id), &[], false, 0, 0, ) .await? .items .into_iter() .map(|p| p.id()) .collect::(), )); } PrincipalFilter::Type(principal_type) => { let typ = match principal_type { PrincipalType::Individual => Type::Individual, PrincipalType::Group => Type::Group, PrincipalType::Resource => Type::Resource, PrincipalType::Location => Type::Location, PrincipalType::Other => Type::Other, }; filters.push(SearchFilter::is_in_set( self.store() .list_principals( None, access_token.tenant.map(|t| t.id), &[typ], false, 0, 0, ) .await? .items .into_iter() .map(|p| p.id()) .collect::(), )); } other => { return Err(trc::JmapEvent::UnsupportedFilter .into_err() .details(other.to_string())); } }, Filter::And => { filters.push(SearchFilter::And); } Filter::Or => { filters.push(SearchFilter::Or); } Filter::Not => { filters.push(SearchFilter::Not); } Filter::Close => { filters.push(SearchFilter::End); } } } let results = SearchQuery::new(SearchIndex::InMemory) .with_filters(filters) .with_mask(principal_ids) .filter() .into_bitmap(); let mut response = QueryResponseBuilder::new( results.len() as usize, self.core.jmap.query_max_results, State::Initial, &request, ); for document_id in results { if !response.add(0, document_id) { break; } } response.build() } } ================================================ FILE: crates/jmap/src/push/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{Server, auth::AccessToken, ipc::PushEvent}; use email::push::PushSubscriptions; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::push_subscription::{self, PushSubscriptionProperty, PushSubscriptionValue}, types::date::UTCDate, }; use jmap_tools::{Map, Value}; use std::future::Future; use store::{ Serialize, ValueKey, write::{AlignedBytes, Archive, Archiver, BatchBuilder, now}, }; use trc::{AddContext, ServerEvent}; use types::{collection::Collection, field::PrincipalField, id::Id}; use utils::map::bitmap::Bitmap; pub trait PushSubscriptionFetch: Sync + Send { fn push_subscription_get( &self, request: GetRequest, access_token: &AccessToken, ) -> impl Future>> + Send; } impl PushSubscriptionFetch for Server { async fn push_subscription_get( &self, mut request: GetRequest, access_token: &AccessToken, ) -> trc::Result> { let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ PushSubscriptionProperty::Id, PushSubscriptionProperty::DeviceClientId, PushSubscriptionProperty::VerificationCode, PushSubscriptionProperty::Expires, PushSubscriptionProperty::Types, ]); let account_id = access_token.primary_id(); let mut response = GetResponse { account_id: request.account_id.into(), state: None, list: Vec::new(), not_found: vec![], }; let Some(subscriptions_) = self .store() .get_value::>(ValueKey::property( account_id, Collection::Principal, 0, PrincipalField::PushSubscriptions, )) .await? else { for id in ids.unwrap_or_default() { response.not_found.push(id); } return Ok(response); }; let subscriptions = subscriptions_ .to_unarchived::() .caused_by(trc::location!())?; let ids = if let Some(ids) = ids { ids } else { subscriptions .inner .subscriptions .iter() .take(self.core.jmap.get_max_objects) .map(|s| Id::from(s.id.to_native())) .collect::>() }; for id in ids { // Obtain the push subscription object let document_id = id.document_id(); let Some(push) = subscriptions .inner .subscriptions .iter() .find(|p| p.id.to_native() == document_id) else { response.not_found.push(id); continue; }; let mut result = Map::with_capacity(properties.len()); for property in &properties { match property { PushSubscriptionProperty::Id => { result.insert_unchecked(PushSubscriptionProperty::Id, id); } PushSubscriptionProperty::Url | PushSubscriptionProperty::Keys => { return Err(trc::JmapEvent::Forbidden.into_err().details( "The 'url' and 'keys' properties are not readable".to_string(), )); } PushSubscriptionProperty::DeviceClientId => { result.insert_unchecked( PushSubscriptionProperty::DeviceClientId, &push.device_client_id, ); } PushSubscriptionProperty::Types => { let mut types = Vec::new(); for typ in Bitmap::from(&push.types).into_iter() { types.push(Value::Element(PushSubscriptionValue::Types(typ))); } result .insert_unchecked(PushSubscriptionProperty::Types, Value::Array(types)); } PushSubscriptionProperty::Expires => { if push.expires > 0 { result.insert_unchecked( PushSubscriptionProperty::Expires, Value::Element(PushSubscriptionValue::Date( UTCDate::from_timestamp(u64::from(push.expires) as i64), )), ); } else { result.insert_unchecked(PushSubscriptionProperty::Expires, Value::Null); } } property => { result.insert_unchecked(property.clone(), Value::Null); } } } response.list.push(result.into()); } // Purge old subscriptions let current_time = now(); if subscriptions .inner .subscriptions .iter() .any(|s| s.expires.to_native() < current_time) { let mut updated_subscriptions = subscriptions.deserialize::()?; updated_subscriptions .subscriptions .retain(|s| s.expires >= current_time); let mut batch = BatchBuilder::new(); if updated_subscriptions.subscriptions.is_empty() { batch .with_account_id(u32::MAX) .with_collection(Collection::Principal) .with_account_id(account_id) .tag(PrincipalField::PushSubscriptions); } batch .with_account_id(account_id) .with_collection(Collection::Principal) .with_document(0) .assert_value(PrincipalField::PushSubscriptions, subscriptions); if !updated_subscriptions.subscriptions.is_empty() { batch.set( PrincipalField::PushSubscriptions, Archiver::new(updated_subscriptions) .serialize() .caused_by(trc::location!())?, ); } else { batch.clear(PrincipalField::PushSubscriptions); } self.commit_batch(batch).await.caused_by(trc::location!())?; // Update push servers if self .inner .ipc .push_tx .clone() .send(PushEvent::PushServerUpdate { account_id, broadcast: true, }) .await .is_err() { trc::event!( Server(ServerEvent::ThreadError), Details = "Error sending push updates.", CausedBy = trc::location!() ); } } Ok(response) } } ================================================ FILE: crates/jmap/src/push/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod get; pub mod set; ================================================ FILE: crates/jmap/src/push/set.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use base64::{Engine, engine::general_purpose}; use common::{Server, auth::AccessToken, ipc::PushEvent}; use email::push::{Keys, PushSubscription, PushSubscriptions}; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{SetRequest, SetResponse}, object::push_subscription::{self, PushSubscriptionProperty, PushSubscriptionValue}, references::resolve::ResolveCreatedReference, request::IntoValid, types::date::UTCDate, }; use jmap_tools::{Key, Map, Value}; use rand::distr::Alphanumeric; use std::future::Future; use store::{ Serialize, ValueKey, rand::{Rng, rng}, write::{AlignedBytes, Archive, Archiver, BatchBuilder, now}, }; use trc::{AddContext, ServerEvent}; use types::{collection::Collection, field::PrincipalField}; use utils::map::bitmap::Bitmap; const EXPIRES_MAX: i64 = 7 * 24 * 3600; // 7 days const VERIFICATION_CODE_LEN: usize = 32; pub trait PushSubscriptionSet: Sync + Send { fn push_subscription_set( &self, request: SetRequest<'_, push_subscription::PushSubscription>, access_token: &AccessToken, ) -> impl Future>> + Send; } impl PushSubscriptionSet for Server { async fn push_subscription_set( &self, mut request: SetRequest<'_, push_subscription::PushSubscription>, access_token: &AccessToken, ) -> trc::Result> { // Load existing push subscriptions let account_id = access_token.primary_id(); let subscriptions_archive = self .store() .get_value::>(ValueKey::property( account_id, Collection::Principal, 0, PrincipalField::PushSubscriptions, )) .await?; let mut subscriptions = if let Some(subscriptions) = &subscriptions_archive { subscriptions .deserialize::() .caused_by(trc::location!())? } else { PushSubscriptions::default() }; let num_subscriptions = subscriptions.subscriptions.len(); let mut max_id = 0; let current_time = now(); subscriptions.subscriptions.retain(|s| { max_id = max_id.max(s.id); s.expires > current_time }); let mut has_changes = num_subscriptions != subscriptions.subscriptions.len(); // Prepare response let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; let will_destroy = request.unwrap_destroy().into_valid().collect::>(); // Process creates 'create: for (id, object) in request.unwrap_create() { let mut push = PushSubscription::default(); if subscriptions.subscriptions.len() >= access_token.object_quota(Collection::PushSubscription) as usize { response.not_created.append(id, SetError::new(SetErrorType::OverQuota).with_description( "There are too many subscriptions, please delete some before adding a new one.", )); continue 'create; } for (property, mut value) in object.into_expanded_object() { if let Err(err) = response .resolve_self_references(&mut value) .and_then(|_| validate_push_value(&property, value, &mut push, true)) { response.not_created.append(id, err); continue 'create; } } if push.device_client_id.is_empty() || push.url.is_empty() { response.not_created.append( id, SetError::invalid_properties() .with_properties([ PushSubscriptionProperty::DeviceClientId, PushSubscriptionProperty::Url, ]) .with_description("Missing required properties"), ); continue 'create; } // Add expiry time if missing if push.expires == 0 { push.expires = now() + EXPIRES_MAX as u64; } let expires = UTCDate::from_timestamp(push.expires as i64); // Generate random verification code push.verification_code = rng() .sample_iter(Alphanumeric) .take(VERIFICATION_CODE_LEN) .map(char::from) .collect::(); // Set id max_id += 1; let document_id = max_id; push.id = document_id; // Insert record subscriptions.subscriptions.push(push); response.created.insert( id, Map::with_capacity(1) .with_key_value( PushSubscriptionProperty::Id, PushSubscriptionValue::Id(document_id.into()), ) .with_key_value(PushSubscriptionProperty::Keys, Value::Null) .with_key_value( PushSubscriptionProperty::Expires, PushSubscriptionValue::Date(expires), ) .into(), ); has_changes = true; } // Process updates 'update: for (id, object) in request.unwrap_update().into_valid() { // Make sure id won't be destroyed if will_destroy.contains(&id) { response.not_updated.append(id, SetError::will_destroy()); continue 'update; } // Obtain push subscription let document_id = id.document_id(); let Some(push) = subscriptions .subscriptions .iter_mut() .find(|p| p.id == document_id) else { response.not_updated.append(id, SetError::not_found()); continue 'update; }; for (property, mut value) in object.into_expanded_object() { if let Err(err) = response .resolve_self_references(&mut value) .and_then(|_| validate_push_value(&property, value, push, false)) { response.not_updated.append(id, err); continue 'update; } } has_changes = true; response.updated.append(id, None); } // Process deletions for id in will_destroy { let document_id = id.document_id(); if let Some(idx) = subscriptions .subscriptions .iter() .position(|p| p.id == document_id) { subscriptions.subscriptions.swap_remove(idx); has_changes = true; response.destroyed.push(id); } else { response.not_destroyed.append(id, SetError::not_found()); } } // Update push subscriptions if has_changes { // Save changes let mut batch = BatchBuilder::new(); if subscriptions_archive.is_none() { batch .with_account_id(u32::MAX) .with_collection(Collection::Principal) .with_document(account_id) .tag(PrincipalField::PushSubscriptions); } else if subscriptions.subscriptions.is_empty() { batch .with_account_id(u32::MAX) .with_collection(Collection::Principal) .with_document(account_id) .untag(PrincipalField::PushSubscriptions); } batch .with_account_id(account_id) .with_collection(Collection::Principal) .with_document(0); if let Some(subscriptions_archive) = subscriptions_archive { batch.assert_value(PrincipalField::PushSubscriptions, subscriptions_archive); } if !subscriptions.subscriptions.is_empty() { batch.set( PrincipalField::PushSubscriptions, Archiver::new(subscriptions) .serialize() .caused_by(trc::location!())?, ); } else { batch.clear(PrincipalField::PushSubscriptions); } self.commit_batch(batch).await.caused_by(trc::location!())?; // Notify push manager if self .inner .ipc .push_tx .clone() .send(PushEvent::PushServerUpdate { account_id, broadcast: true, }) .await .is_err() { trc::event!( Server(ServerEvent::ThreadError), Details = "Error sending push updates.", CausedBy = trc::location!() ); } } Ok(response) } } fn validate_push_value( property: &Key, value: Value<'_, PushSubscriptionProperty, PushSubscriptionValue>, push: &mut PushSubscription, is_create: bool, ) -> Result<(), SetError> { let Key::Property(property) = property else { return Err(SetError::invalid_properties() .with_property(property.to_owned()) .with_description("Invalid property.")); }; match (property, value) { (PushSubscriptionProperty::DeviceClientId, Value::Str(value)) if is_create && value.len() < 255 => { push.device_client_id = value.into_owned(); } (PushSubscriptionProperty::Url, Value::Str(value)) if is_create && value.len() < 512 && value.starts_with("https://") => { push.url = value.into_owned(); } (PushSubscriptionProperty::Keys, Value::Object(value)) if is_create && value.len() == 2 => { if let (Some(auth), Some(p256dh)) = ( value .get(&Key::Property(PushSubscriptionProperty::Auth)) .and_then(|v| v.as_str()) .and_then(|v| general_purpose::URL_SAFE.decode(v.as_ref()).ok()), value .get(&Key::Property(PushSubscriptionProperty::P256dh)) .and_then(|v| v.as_str()) .and_then(|v| general_purpose::URL_SAFE.decode(v.as_ref()).ok()), ) { push.keys = Some(Keys { auth, p256dh }); } else { return Err(SetError::invalid_properties() .with_property(property.clone()) .with_description("Failed to decode keys.")); } } (PushSubscriptionProperty::Expires, Value::Element(PushSubscriptionValue::Date(value))) => { let current_time = now() as i64; let expires = value.timestamp(); push.expires = if expires > current_time && (expires - current_time) > EXPIRES_MAX { current_time + EXPIRES_MAX } else { expires } as u64; } (PushSubscriptionProperty::Expires, Value::Null) => { push.expires = now() + EXPIRES_MAX as u64; } (PushSubscriptionProperty::Types, Value::Array(value)) => { push.types.clear(); for item in value { if let Value::Element(PushSubscriptionValue::Types(dt)) = item { push.types.insert(dt); } else { return Err(SetError::invalid_properties() .with_property(property.clone()) .with_description("Invalid data type.")); } } } (PushSubscriptionProperty::VerificationCode, Value::Str(value)) if !is_create => { if push.verification_code == value { push.verified = true; } else { return Err(SetError::invalid_properties() .with_property(property.clone()) .with_description("Verification code does not match.".to_string())); } } (PushSubscriptionProperty::Keys, Value::Null) => { push.keys = None; } (PushSubscriptionProperty::Types, Value::Null) => { push.types = Bitmap::all(); } (PushSubscriptionProperty::VerificationCode, Value::Null) => {} (property, _) => { return Err(SetError::invalid_properties() .with_property(property.clone()) .with_description("Field could not be set.")); } } if is_create && push.types.is_empty() { push.types = Bitmap::all(); } Ok(()) } ================================================ FILE: crates/jmap/src/quota/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{Server, auth::AccessToken}; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::quota::{Quota, QuotaProperty, QuotaValue}, types::state::State, }; use jmap_tools::{Map, Value}; use std::{future::Future, sync::Arc}; use trc::AddContext; use types::{id::Id, type_state::DataType}; pub trait QuotaGet: Sync + Send { fn quota_get( &self, request: GetRequest, access_token: &AccessToken, ) -> impl Future>> + Send; } impl QuotaGet for Server { async fn quota_get( &self, mut request: GetRequest, access_token: &AccessToken, ) -> trc::Result> { let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ QuotaProperty::Id, QuotaProperty::ResourceType, QuotaProperty::Used, QuotaProperty::WarnLimit, QuotaProperty::SoftLimit, QuotaProperty::HardLimit, QuotaProperty::Scope, QuotaProperty::Name, QuotaProperty::Description, QuotaProperty::Types, ]); let account_id = request.account_id.document_id(); let quota_ids = if access_token.quota > 0 { vec![0u32] } else { vec![] }; let ids = if let Some(ids) = ids { ids } else { quota_ids.iter().map(|id| Id::from(*id)).collect() }; let mut response = GetResponse { account_id: request.account_id.into(), state: State::Initial.into(), list: Vec::with_capacity(ids.len()), not_found: vec![], }; let access_token = if account_id == access_token.primary_id() { AccessTokenRef::Borrowed(access_token) } else { AccessTokenRef::Owned( self.get_access_token(account_id) .await .caused_by(trc::location!())?, ) }; for id in ids { // Obtain the sieve script object let document_id = id.document_id(); if !quota_ids.contains(&document_id) { response.not_found.push(id); continue; } let mut result = Map::with_capacity(properties.len()); for property in &properties { let value = match property { QuotaProperty::Id => Value::Element(id.into()), QuotaProperty::ResourceType => "octets".to_string().into(), QuotaProperty::Used => (self.get_used_quota(account_id).await? as u64).into(), QuotaProperty::HardLimit => access_token.as_ref().quota.into(), QuotaProperty::Scope => "account".to_string().into(), QuotaProperty::Name => access_token.as_ref().name.to_string().into(), QuotaProperty::Description => access_token .as_ref() .description .as_ref() .map(|s| s.to_string()) .into(), QuotaProperty::Types => vec![ Value::Element(QuotaValue::Types(DataType::Email)), Value::Element(QuotaValue::Types(DataType::SieveScript)), Value::Element(QuotaValue::Types(DataType::FileNode)), Value::Element(QuotaValue::Types(DataType::CalendarEvent)), Value::Element(QuotaValue::Types(DataType::ContactCard)), ] .into(), _ => Value::Null, }; result.insert_unchecked(property.clone(), value); } response.list.push(result.into()); } Ok(response) } } enum AccessTokenRef<'x> { Owned(Arc), Borrowed(&'x AccessToken), } impl AccessTokenRef<'_> { fn as_ref(&self) -> &AccessToken { match self { AccessTokenRef::Owned(token) => token, AccessTokenRef::Borrowed(token) => token, } } } ================================================ FILE: crates/jmap/src/quota/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod get; pub mod query; ================================================ FILE: crates/jmap/src/quota/query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{Server, auth::AccessToken}; use jmap_proto::{ method::query::{QueryRequest, QueryResponse}, object::quota::Quota, types::state::State, }; use std::future::Future; use types::id::Id; pub trait QuotaQuery: Sync + Send { fn quota_query( &self, request: QueryRequest, access_token: &AccessToken, ) -> impl Future> + Send; } impl QuotaQuery for Server { async fn quota_query( &self, request: QueryRequest, access_token: &AccessToken, ) -> trc::Result { Ok(QueryResponse { account_id: request.account_id, query_state: State::Initial, can_calculate_changes: false, position: 0, ids: if access_token.quota > 0 { vec![Id::new(0)] } else { vec![] }, total: Some(1), limit: None, }) } } ================================================ FILE: crates/jmap/src/share_notification/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{Server, auth::AccessToken, sharing::notification::ShareNotification}; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::{ JmapRight, addressbook::AddressBookRight, calendar::CalendarRight, file_node::FileNodeRight, mailbox::MailboxRight, share_notification::{self, ShareNotificationProperty, ShareNotificationValue}, }, request::IntoValid, types::{date::UTCDate, state::State}, }; use jmap_tools::{Key, Map, Value}; use std::{sync::Arc, time::Duration}; use store::{ Deserialize, IterateParams, LogKey, U64_LEN, ahash::{AHashMap, AHashSet}, write::key::DeserializeBigEndian, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, SyncCollection}, id::Id, type_state::DataType, }; use utils::{map::bitmap::Bitmap, snowflake::SnowflakeIdGenerator}; pub trait ShareNotificationGet: Sync + Send { fn share_notification_get( &self, request: GetRequest, ) -> impl Future>> + Send; } impl ShareNotificationGet for Server { async fn share_notification_get( &self, mut request: GetRequest, ) -> trc::Result> { let properties = request.unwrap_properties(&[ ShareNotificationProperty::Id, ShareNotificationProperty::Name, ShareNotificationProperty::ChangedBy, ShareNotificationProperty::Created, ShareNotificationProperty::ObjectAccountId, ShareNotificationProperty::ObjectId, ShareNotificationProperty::ObjectType, ShareNotificationProperty::OldRights, ShareNotificationProperty::NewRights, ShareNotificationProperty::Name, ]); let account_id = request.account_id.document_id(); let mut min_id = u64::MAX; let mut max_id = 0u64; let mut token_cache: AHashMap> = AHashMap::new(); let mut ids = if let Some(ids) = request.ids.take() { let ids = ids.unwrap(); if ids.len() <= self.core.jmap.get_max_objects { ids.into_valid() .map(|id| { let id_num = *id.as_ref(); if id_num < min_id { min_id = id_num; } if id_num > max_id { max_id = id_num; } id_num }) .collect::>() } else { return Err(trc::JmapEvent::RequestTooLarge.into_err()); } } else { AHashSet::new() }; let has_ids = !ids.is_empty(); if min_id == u64::MAX { min_id = SnowflakeIdGenerator::from_duration( self.core .jmap .share_notification_max_history .unwrap_or(Duration::from_secs(30 * 86400)), ) .unwrap_or_default(); } if max_id == 0 { max_id = u64::MAX; } let mut response = GetResponse { account_id: request.account_id.into(), state: None, list: Vec::with_capacity(ids.len()), not_found: vec![], }; let mut notifications = Vec::new(); self.store() .iterate( IterateParams::new( LogKey { account_id, collection: SyncCollection::ShareNotification.into(), change_id: min_id, }, LogKey { account_id, collection: SyncCollection::ShareNotification.into(), change_id: max_id.saturating_add(1), }, ) .descending(), |key, value| { let change_id = key.deserialize_be_u64(key.len() - U64_LEN)?; if response.state.is_none() { response.state = Some(State::Exact(change_id)); } if !has_ids || ids.remove(&change_id) { notifications.push(( change_id, ShareNotification::deserialize(value).caused_by(trc::location!())?, )); } Ok((!has_ids || !ids.is_empty()) && notifications.len() < self.core.jmap.get_max_objects) }, ) .await .caused_by(trc::location!())?; for (change_id, notification) in notifications { let changed_by_token = if let Some(token) = token_cache.get(¬ification.changed_by) { token.clone() } else { let token = if let Ok(token) = self.get_access_token(notification.changed_by).await { token } else { Arc::new(AccessToken::from_id(notification.changed_by)) }; token_cache.insert(notification.changed_by, token.clone()); token }; response.list.push(build_share_notification( change_id, notification, &changed_by_token, &properties, )); } if response.state.is_none() { response.state = Some(State::Initial); } response .not_found .extend(ids.into_iter().map(Id::from).collect::>()); Ok(response) } } fn build_share_notification( id: u64, mut notification: ShareNotification, changed_by: &AccessToken, properties: &[ShareNotificationProperty], ) -> Value<'static, ShareNotificationProperty, ShareNotificationValue> { let mut result = Map::with_capacity(properties.len()); for property in properties { let value = match property { ShareNotificationProperty::Id => Value::Element(ShareNotificationValue::Id(id.into())), ShareNotificationProperty::Created => Value::Element(ShareNotificationValue::Date( UTCDate::from_timestamp(SnowflakeIdGenerator::to_timestamp(id) as i64), )), ShareNotificationProperty::ChangedBy => Value::Object(Map::from(vec![ ( Key::Property(ShareNotificationProperty::ChangedByPrincipalId), Value::Element(ShareNotificationValue::Id(notification.changed_by.into())), ), ( Key::Property(ShareNotificationProperty::ChangedByName), Value::Str( changed_by .description .as_deref() .unwrap_or(changed_by.name.as_str()) .to_string() .into(), ), ), ( Key::Property(ShareNotificationProperty::ChangedByEmail), changed_by .emails .first() .map_or(Value::Null, |email| Value::Str(email.to_string().into())), ), ])), ShareNotificationProperty::ObjectType => DataType::try_from(notification.object_type) .ok() .map(|typ| Value::Element(ShareNotificationValue::ObjectType(typ))) .unwrap_or(Value::Null), ShareNotificationProperty::ObjectAccountId => Value::Element( ShareNotificationValue::Id(notification.object_account_id.into()), ), ShareNotificationProperty::ObjectId => { Value::Element(ShareNotificationValue::Id(notification.object_id.into())) } ShareNotificationProperty::OldRights => { map_rights(notification.object_type, notification.old_rights) } ShareNotificationProperty::NewRights => { map_rights(notification.object_type, notification.new_rights) } ShareNotificationProperty::Name => { Value::Str(std::mem::take(&mut notification.name).into()) } _ => Value::Null, }; result.insert_unchecked(property.clone(), value); } Value::Object(result) } fn map_rights( object_type: Collection, rights: Bitmap, ) -> Value<'static, ShareNotificationProperty, ShareNotificationValue> { let mut obj = Map::with_capacity(3); match object_type { Collection::Calendar | Collection::CalendarEvent => { for right in CalendarRight::all_rights() { obj.insert_unchecked( Key::Borrowed(right.as_str()), Value::Bool(right.to_acl().iter().all(|acl| rights.contains(*acl))), ); } } Collection::AddressBook | Collection::ContactCard => { for right in AddressBookRight::all_rights() { obj.insert_unchecked( Key::Borrowed(right.as_str()), Value::Bool(right.to_acl().iter().all(|acl| rights.contains(*acl))), ); } } Collection::FileNode => { for right in FileNodeRight::all_rights() { obj.insert_unchecked( Key::Borrowed(right.as_str()), Value::Bool(right.to_acl().iter().all(|acl| rights.contains(*acl))), ); } } Collection::Mailbox | Collection::Email => { for right in MailboxRight::all_rights() { obj.insert_unchecked( Key::Borrowed(right.as_str()), Value::Bool(right.to_acl().iter().all(|acl| rights.contains(*acl))), ); } } _ => {} } Value::Object(obj) } ================================================ FILE: crates/jmap/src/share_notification/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod get; pub mod query; pub mod set; ================================================ FILE: crates/jmap/src/share_notification/query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::api::query::QueryResponseBuilder; use common::{Server, sharing::notification::ShareNotification}; use jmap_proto::{ method::query::{Filter, QueryRequest, QueryResponse}, object::share_notification::{self, ShareNotificationFilter}, types::state::State, }; use std::time::Duration; use store::{Deserialize, IterateParams, LogKey, U64_LEN, write::key::DeserializeBigEndian}; use trc::AddContext; use types::{ collection::{Collection, SyncCollection}, id::Id, }; use utils::snowflake::SnowflakeIdGenerator; pub trait ShareNotificationQuery: Sync + Send { fn share_notification_query( &self, request: QueryRequest, ) -> impl Future> + Send; } impl ShareNotificationQuery for Server { async fn share_notification_query( &self, mut request: QueryRequest, ) -> trc::Result { let account_id = request.account_id.document_id(); let mut from_change_id = SnowflakeIdGenerator::from_duration( self.core .jmap .share_notification_max_history .unwrap_or(Duration::from_secs(30 * 86400)), ) .unwrap_or_default(); let mut to_change_id = u64::MAX; let mut collection = None; let mut object_type = None; for cond in std::mem::take(&mut request.filter) { match cond { Filter::Property(cond) => match cond { ShareNotificationFilter::After(utcdate) => { from_change_id = SnowflakeIdGenerator::from_timestamp(utcdate.timestamp() as u64) .unwrap_or(0); } ShareNotificationFilter::Before(utcdate) => { to_change_id = SnowflakeIdGenerator::from_timestamp(utcdate.timestamp() as u64) .unwrap_or(u64::MAX); } ShareNotificationFilter::ObjectType(typ) => { collection = Collection::try_from(typ).ok(); } ShareNotificationFilter::ObjectAccountId(id) => { object_type = Some(id.document_id()); } ShareNotificationFilter::_T(other) => { return Err(trc::JmapEvent::UnsupportedFilter.into_err().details(other)); } }, Filter::And | Filter::Or | Filter::Not | Filter::Close => { return Err(trc::JmapEvent::UnsupportedFilter .into_err() .details("Logical operators are not supported")); } } } let mut results = Vec::new(); self.store() .iterate( IterateParams::new( LogKey { account_id, collection: SyncCollection::ShareNotification.into(), change_id: from_change_id, }, LogKey { account_id, collection: SyncCollection::ShareNotification.into(), change_id: to_change_id, }, ) .descending(), |key, value| { let change_id = key.deserialize_be_u64(key.len() - U64_LEN)?; if collection.is_some() || object_type.is_some() { let notification = ShareNotification::deserialize(value).caused_by(trc::location!())?; if collection.is_some_and(|c| c != notification.object_type) || object_type.is_some_and(|o| o != notification.object_account_id) { return Ok(true); } } results.push(Id::from(change_id)); Ok(true) }, ) .await .caused_by(trc::location!())?; let mut response = QueryResponseBuilder::new( results.len(), self.core.jmap.query_max_results, State::Initial, &request, ); for id in results { if !response.add_id(id) { break; } } response.build() } } ================================================ FILE: crates/jmap/src/share_notification/set.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::Server; use jmap_proto::{ error::set::SetError, method::set::{SetRequest, SetResponse}, object::share_notification::ShareNotification, request::IntoValid, }; use store::write::{BatchBuilder, ValueClass}; use trc::AddContext; pub trait ShareNotificationSet: Sync + Send { fn share_notification_set( &self, request: SetRequest<'_, ShareNotification>, ) -> impl Future>> + Send; } impl ShareNotificationSet for Server { async fn share_notification_set( &self, mut request: SetRequest<'_, ShareNotification>, ) -> trc::Result> { let account_id = request.account_id.document_id(); let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; for (id, _) in request.unwrap_create() { response.not_created.append( id, SetError::forbidden().with_description("Cannot create share notifications."), ); } // Process updates for (id, _) in request.unwrap_update().into_valid() { response.not_updated.append( id, SetError::forbidden().with_description("Cannot update share notifications."), ); } // Process deletions let mut batch = BatchBuilder::new(); batch.with_account_id(account_id); for id in request.unwrap_destroy().into_valid() { batch.clear(ValueClass::ShareNotification { notification_id: id.id(), notify_account_id: account_id, }); response.destroyed.push(id); } // Write changes if !batch.is_empty() { self.commit_batch(batch).await.caused_by(trc::location!())?; } Ok(response) } } ================================================ FILE: crates/jmap/src/sieve/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::changes::state::StateManager; use common::Server; use email::sieve::{SieveScript, ingest::SieveScriptIngest}; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::sieve::{Sieve, SieveProperty, SieveValue}, }; use jmap_tools::{Map, Value}; use store::{ValueKey, write::{AlignedBytes, Archive}}; use std::future::Future; use trc::AddContext; use types::{ blob::{BlobClass, BlobId, BlobSection}, collection::{Collection, SyncCollection}, field::SieveField, }; pub trait SieveScriptGet: Sync + Send { fn sieve_script_get( &self, request: GetRequest, ) -> impl Future>> + Send; } impl SieveScriptGet for Server { async fn sieve_script_get( &self, mut request: GetRequest, ) -> trc::Result> { let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ SieveProperty::Id, SieveProperty::Name, SieveProperty::BlobId, SieveProperty::IsActive, ]); let account_id = request.account_id.document_id(); let script_ids = self .document_ids(account_id, Collection::SieveScript, SieveField::Name) .await?; let ids = if let Some(ids) = ids { ids } else { script_ids .iter() .take(self.core.jmap.get_max_objects) .map(Into::into) .collect::>() }; let mut response = GetResponse { account_id: request.account_id.into(), state: self .get_state(account_id, SyncCollection::SieveScript) .await? .into(), list: Vec::with_capacity(ids.len()), not_found: vec![], }; let active_script_id = self.sieve_script_get_active_id(account_id).await?; for id in ids { // Obtain the sieve script object let document_id = id.document_id(); if !script_ids.contains(document_id) { response.not_found.push(id); continue; } let sieve_ = if let Some(sieve) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, document_id, )) .await? { sieve } else { response.not_found.push(id); continue; }; let sieve = sieve_ .unarchive::() .caused_by(trc::location!())?; let mut result = Map::with_capacity(properties.len()); for property in &properties { match property { SieveProperty::Id => { result.insert_unchecked(SieveProperty::Id, id); } SieveProperty::Name => { result.insert_unchecked(SieveProperty::Name, &sieve.name); } SieveProperty::IsActive => { result.insert_unchecked( SieveProperty::IsActive, active_script_id == Some(document_id), ); } SieveProperty::BlobId => { let blob_id = BlobId { hash: (&sieve.blob_hash).into(), class: BlobClass::Linked { account_id, collection: Collection::SieveScript.into(), document_id, }, section: BlobSection { size: u32::from(sieve.size) as usize, ..Default::default() } .into(), }; result.insert_unchecked( SieveProperty::BlobId, Value::Element(SieveValue::BlobId(blob_id)), ); } } } response.list.push(result.into()); } Ok(response) } } ================================================ FILE: crates/jmap/src/sieve/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod get; pub mod query; pub mod set; pub mod validate; ================================================ FILE: crates/jmap/src/sieve/query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{api::query::QueryResponseBuilder, changes::state::StateManager}; use common::Server; use email::sieve::ingest::SieveScriptIngest; use jmap_proto::{ method::query::{Filter, QueryRequest, QueryResponse}, object::sieve::{Sieve, SieveComparator, SieveFilter}, }; use std::future::Future; use store::{ IndexKeyPrefix, IterateParams, U32_LEN, roaring::RoaringBitmap, search::{SearchFilter, SearchQuery}, write::{SearchIndex, key::DeserializeBigEndian}, }; use trc::AddContext; use types::{ collection::{Collection, SyncCollection}, field::SieveField, }; pub trait SieveScriptQuery: Sync + Send { fn sieve_script_query( &self, request: QueryRequest, ) -> impl Future> + Send; } impl SieveScriptQuery for Server { async fn sieve_script_query( &self, mut request: QueryRequest, ) -> trc::Result { let account_id = request.account_id.document_id(); let mut filters = Vec::with_capacity(request.filter.len()); let active_script_id = if request .filter .iter() .any(|f| matches!(f, Filter::Property(SieveFilter::IsActive(_)))) || request.sort.as_ref().is_some_and(|s| { s.iter() .any(|c| matches!(c.property, SieveComparator::IsActive)) }) { self.sieve_script_get_active_id(account_id).await? } else { None }; let mut document_ids = RoaringBitmap::new(); let mut names = Vec::new(); self.store() .iterate( IterateParams::new( IndexKeyPrefix { account_id, collection: Collection::SieveScript.into(), field: SieveField::Name.into(), }, IndexKeyPrefix { account_id, collection: Collection::SieveScript.into(), field: u8::from(SieveField::Name) + 1, }, ) .no_values(), |key, _| { let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; names.push(( document_id, key.get(IndexKeyPrefix::len()..key.len() - U32_LEN) .and_then(|v| std::str::from_utf8(v).ok()) .unwrap_or_default() .to_string(), )); document_ids.insert(document_id); Ok(true) }, ) .await .caused_by(trc::location!())?; for cond in std::mem::take(&mut request.filter) { match cond { Filter::Property(cond) => match cond { SieveFilter::Name(name) => { let name = name.to_lowercase(); filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( names .iter() .filter_map(|(id, n)| (n.contains(&name)).then_some(*id)) .collect::>(), ))); } SieveFilter::IsActive(is_active) => { if is_active { if let Some(active_script_id) = active_script_id { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter([ active_script_id, ]))); } else { // No active script, so no results filters.push(SearchFilter::is_in_set(RoaringBitmap::new())); } } else { let mut inactive_set = document_ids.clone(); if let Some(active_script_id) = active_script_id { inactive_set.remove(active_script_id); } filters.push(SearchFilter::is_in_set(inactive_set)); } } SieveFilter::_T(other) => { return Err(trc::JmapEvent::UnsupportedFilter.into_err().details(other)); } }, Filter::And => { filters.push(SearchFilter::And); } Filter::Or => { filters.push(SearchFilter::Or); } Filter::Not => { filters.push(SearchFilter::Not); } Filter::Close => { filters.push(SearchFilter::End); } } } // Parse sort criteria let mut sort_by_active = None; for comparator in request .sort .take() .filter(|s| !s.is_empty()) .unwrap_or_default() { match comparator.property { SieveComparator::Name => { if !comparator.is_ascending { names.reverse(); } } SieveComparator::IsActive => { sort_by_active = Some(comparator.is_ascending); } SieveComparator::_T(other) => { return Err(trc::JmapEvent::UnsupportedSort.into_err().details(other)); } }; } let mut results = SearchQuery::new(SearchIndex::InMemory) .with_filters(filters) .with_mask(document_ids) .filter() .into_bitmap(); let mut response = QueryResponseBuilder::new( results.len() as usize, self.core.jmap.query_max_results, self.get_state(account_id, SyncCollection::SieveScript) .await?, &request, ); if !results.is_empty() { if matches!(sort_by_active, Some(true)) && results.remove(active_script_id.unwrap_or_default()) && !response.add(0, active_script_id.unwrap()) { return response.build(); } let mut last_id = None; for (document_id, _) in names { if results.contains(document_id) { if sort_by_active.is_some() && Some(document_id) == active_script_id { last_id = Some(document_id); } else if !response.add(0, document_id) { return response.build(); } } } if let Some(active_id) = last_id { response.add(0, active_id); } } response.build() } } ================================================ FILE: crates/jmap/src/sieve/set.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{blob::download::BlobDownload, changes::state::StateManager}; use common::{ Server, auth::{AccessToken, ResourceToken}, storage::index::ObjectIndexBuilder, }; use email::sieve::{ ArchivedSieveScript, SieveScript, delete::SieveScriptDelete, ingest::SieveScriptIngest, }; use http_proto::HttpSessionData; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{SetRequest, SetResponse}, object::sieve::{Sieve, SieveProperty, SieveValue}, references::resolve::ResolveCreatedReference, request::{IntoValid, reference::MaybeIdReference}, types::state::State, }; use jmap_tools::{Key, Map, Value}; use rand::distr::Alphanumeric; use sieve::compiler::ErrorType; use std::future::Future; use store::{ Serialize, SerializeInfallible, ValueKey, rand::{Rng, rng}, write::{AlignedBytes, Archive, Archiver, BatchBuilder} }; use trc::AddContext; use types::{ blob::{BlobClass, BlobId, BlobSection}, collection::{Collection, SyncCollection}, field::{PrincipalField, SieveField}, id::Id, }; pub struct SetContext<'x> { resource_token: ResourceToken, access_token: &'x AccessToken, response: SetResponse, } pub trait SieveScriptSet: Sync + Send { fn sieve_script_set( &self, request: SetRequest<'_, Sieve>, access_token: &AccessToken, session: &HttpSessionData, ) -> impl Future>> + Send; #[allow(clippy::type_complexity)] fn sieve_set_item<'x>( &self, changes_: Value<'_, SieveProperty, SieveValue>, update: Option<(u32, Archive<&'x ArchivedSieveScript>)>, ctx: &SetContext, session_id: u64, ) -> impl Future< Output = trc::Result< Result< ( ObjectIndexBuilder<&'x ArchivedSieveScript, SieveScript>, Option>, ), SetError, >, >, > + Send; } impl SieveScriptSet for Server { async fn sieve_script_set( &self, mut request: SetRequest<'_, Sieve>, access_token: &AccessToken, session: &HttpSessionData, ) -> trc::Result> { let account_id = request.account_id.document_id(); let sieve_ids = self .document_ids(account_id, Collection::SieveScript, SieveField::Name) .await?; let mut ctx = SetContext { resource_token: self.get_resource_token(access_token, account_id).await?, access_token, response: SetResponse::from_request(&request, self.core.jmap.set_max_objects)? .with_state( self.assert_state( account_id, SyncCollection::SieveScript, &request.if_in_state, ) .await?, ), }; let will_destroy = request.unwrap_destroy().into_valid().collect::>(); // Validate active script id if let Some(MaybeIdReference::Id(id)) = &request.arguments.on_success_activate_script && !sieve_ids.contains(id.document_id()) { request.arguments.on_success_activate_script = None; } // Process creates let mut batch = BatchBuilder::new(); for (id, object) in request.unwrap_create() { if sieve_ids.len() < access_token.object_quota(Collection::SieveScript) as u64 { match self .sieve_set_item(object, None, &ctx, session.session_id) .await? { Ok((mut builder, Some(blob))) => { // Store blob let sieve = &mut builder.changes_mut().unwrap(); let (blob_hash, blob_hold) = self.put_temporary_blob(account_id, &blob, 60).await?; sieve.blob_hash = blob_hash; let blob_size = sieve.size as usize; let blob_hash = sieve.blob_hash.clone(); // Write record let document_id = self .store() .assign_document_ids(account_id, Collection::SieveScript, 1) .await .caused_by(trc::location!())?; batch .with_account_id(account_id) .with_collection(Collection::SieveScript) .with_document(document_id) .custom(builder.with_access_token(ctx.access_token)) .caused_by(trc::location!())? .clear(blob_hold) .commit_point(); let mut result = Map::with_capacity(1) .with_key_value(SieveProperty::Id, SieveValue::Id(document_id.into())) .with_key_value( SieveProperty::BlobId, SieveValue::BlobId(BlobId { hash: blob_hash, class: BlobClass::Linked { account_id, collection: Collection::SieveScript.into(), document_id, }, section: BlobSection { size: blob_size, ..Default::default() } .into(), }), ); // Update active script if needed if let Some(MaybeIdReference::Reference(id_ref)) = &request.arguments.on_success_activate_script && id_ref == &id { request.arguments.on_success_activate_script = Some(MaybeIdReference::Id(Id::from(document_id))); result.insert_unchecked(SieveProperty::IsActive, true); } // Add result with updated blobId ctx.response.created.insert(id, result.into()); } Err(err) => { ctx.response.not_created.append(id, err); } _ => unreachable!(), } } else { ctx.response.not_created.append( id, SetError::new(SetErrorType::OverQuota).with_description(concat!( "There are too many sieve scripts, ", "please delete some before adding a new one." )), ); } } // Process updates 'update: for (id, object) in request.unwrap_update().into_valid() { // Make sure id won't be destroyed if will_destroy.contains(&id) { ctx.response .not_updated .append(id, SetError::will_destroy()); continue 'update; } // Obtain sieve script let document_id = id.document_id(); if let Some(sieve_) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, document_id, )) .await? { let sieve = sieve_ .to_unarchived::() .caused_by(trc::location!())?; match self .sieve_set_item( object, (document_id, sieve).into(), &ctx, session.session_id, ) .await? { Ok((mut builder, blob)) => { // Prepare write batch batch .with_account_id(account_id) .with_collection(Collection::SieveScript) .with_document(document_id); let blob_id = if let Some(blob) = blob { // Store blob let sieve = &mut builder.changes_mut().unwrap(); let (blob_hash, blob_hold) = self.put_temporary_blob(account_id, &blob, 60).await?; sieve.blob_hash = blob_hash; batch.clear(blob_hold); BlobId { hash: sieve.blob_hash.clone(), class: BlobClass::Linked { account_id, collection: Collection::SieveScript.into(), document_id, }, section: BlobSection { size: sieve.size as usize, ..Default::default() } .into(), } .into() } else { None }; // Write record batch .custom(builder.with_access_token(ctx.access_token)) .caused_by(trc::location!())? .commit_point(); // Update blobId property if needed let mut result = Map::with_capacity(1); if let Some(blob_id) = blob_id { result.insert_unchecked( SieveProperty::BlobId, SieveValue::BlobId(blob_id), ); } // Add active script property if needed if let Some(MaybeIdReference::Id(id)) = &request.arguments.on_success_activate_script && document_id == id.document_id() { result.insert_unchecked(SieveProperty::IsActive, true); } // Add result ctx.response.updated.append( id, if !result.is_empty() { Value::Object(result).into() } else { None }, ); } Err(err) => { ctx.response.not_updated.append(id, err); continue 'update; } } } else { ctx.response.not_updated.append(id, SetError::not_found()); } } // Process deletions let active_script_id = self.sieve_script_get_active_id(account_id).await?; for id in will_destroy { let document_id = id.document_id(); if sieve_ids.contains(document_id) { if active_script_id != Some(document_id) { if self .sieve_script_delete(account_id, document_id, ctx.access_token, &mut batch) .await? { ctx.response.destroyed.push(id); } else { ctx.response.not_destroyed.append(id, SetError::not_found()); } } else { ctx.response.not_destroyed.append( id, SetError::new(SetErrorType::ScriptIsActive) .with_description("Deactivate Sieve script before deletion."), ); } } else { ctx.response.not_destroyed.append(id, SetError::not_found()); } } // Activate / deactivate scripts let on_success_deactivate_script = request .arguments .on_success_deactivate_script .unwrap_or(false); if ctx.response.not_created.is_empty() && ctx.response.not_updated.is_empty() && ctx.response.not_destroyed.is_empty() && (request.arguments.on_success_activate_script.is_some() || on_success_deactivate_script) { if let Some(MaybeIdReference::Id(id)) = request.arguments.on_success_activate_script { batch .with_account_id(account_id) .with_collection(Collection::Principal) .with_document(0) .set(PrincipalField::ActiveScriptId, id.document_id().serialize()); } else if on_success_deactivate_script { batch .with_account_id(account_id) .with_collection(Collection::Principal) .with_document(0) .clear(PrincipalField::ActiveScriptId); } } // Write changes if !batch.is_empty() && let Ok(change_id) = self .commit_batch(batch) .await .caused_by(trc::location!())? .last_change_id(account_id) { ctx.response.new_state = State::Exact(change_id).into(); } Ok(ctx.response) } #[allow(clippy::blocks_in_conditions)] async fn sieve_set_item<'x>( &self, changes_: Value<'_, SieveProperty, SieveValue>, update: Option<(u32, Archive<&'x ArchivedSieveScript>)>, ctx: &SetContext<'_>, session_id: u64, ) -> trc::Result< Result< ( ObjectIndexBuilder<&'x ArchivedSieveScript, SieveScript>, Option>, ), SetError, >, > { // Vacation script cannot be modified if update .as_ref() .is_some_and(|(_, obj)| obj.inner.name.eq_ignore_ascii_case("vacation")) { return Ok(Err(SetError::forbidden().with_description(concat!( "The 'vacation' script cannot be modified, ", "use VacationResponse/set instead." )))); } // Parse properties let mut changes = update .as_ref() .map(|(_, obj)| obj.deserialize().unwrap_or_default()) .unwrap_or_default(); let mut blob_id = None; for (property, mut value) in changes_.into_expanded_object() { if let Err(err) = ctx.response.resolve_self_references(&mut value) { return Ok(Err(err)); }; match (&property, value) { (Key::Property(SieveProperty::Name), Value::Str(value)) => { if value.len() > self.core.jmap.sieve_max_script_name { return Ok(Err(SetError::invalid_properties() .with_property(property.into_owned()) .with_description("Script name is too long."))); } else if value.eq_ignore_ascii_case("vacation") { return Ok(Err(SetError::forbidden() .with_property(property.into_owned()) .with_description( "The 'vacation' name is reserved, please use a different name.", ))); } else if update .as_ref() .is_none_or(|(_, obj)| obj.inner.name != value.as_ref()) && let Some(id) = self .document_ids_matching( ctx.resource_token.account_id, Collection::SieveScript, SieveField::Name, value.as_bytes(), ) .await? .min() { return Ok(Err(SetError::already_exists() .with_existing_id(id.into()) .with_description(format!( "A sieve script with name '{}' already exists.", value )))); } changes.name = value.into_owned(); } ( Key::Property(SieveProperty::BlobId), Value::Element(SieveValue::BlobId(value)), ) => { blob_id = value.into(); continue; } (Key::Property(SieveProperty::Name), Value::Null) => { continue; } _ => { return Ok(Err(SetError::invalid_properties() .with_property(property.into_owned()) .with_description("Invalid property or value.".to_string()))); } } } if update.is_none() { // Add name if missing if changes.name.is_empty() { changes.name = rng() .sample_iter(Alphanumeric) .take(15) .map(char::from) .collect::(); } } let blob_update = if let Some(blob_id) = blob_id { if update.as_ref().is_none_or( |(document_id, _)| { !matches!(blob_id.class, BlobClass::Linked { account_id, collection, document_id: d } if account_id == ctx.resource_token.account_id && collection == u8::from(Collection::SieveScript) && *document_id == d) }) { // Check access if let Some(mut bytes) = self.blob_download(&blob_id, ctx.access_token).await? { // Check quota match self .has_available_quota(&ctx.resource_token, bytes.len() as u64) .await { Ok(_) => (), Err(err) => { if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) || err.matches(trc::EventType::Limit(trc::LimitEvent::TenantQuota)) { trc::error!(err.account_id(ctx.resource_token.account_id).span_id(session_id)); return Ok(Err(SetError::over_quota())); } else { return Err(err); } } } // Compile script match self.core.sieve.untrusted_compiler.compile(&bytes) { Ok(script) => { changes.size = bytes.len() as u32; bytes.extend(Archiver::new(script).untrusted().serialize().caused_by(trc::location!())?); bytes.into() } Err(err) => { return Ok(Err(SetError::new( if let ErrorType::ScriptTooLong = &err.error_type() { SetErrorType::TooLarge } else { SetErrorType::InvalidScript }, ) .with_description(err.to_string()))); } } } else { return Ok(Err(SetError::new(SetErrorType::BlobNotFound) .with_property(SieveProperty::BlobId) .with_description("Blob does not exist."))); } } else { None } } else if update.is_none() { return Ok(Err(SetError::invalid_properties() .with_property(SieveProperty::BlobId) .with_description("Missing blobId."))); } else { None }; // Validate Ok(Ok(( ObjectIndexBuilder::new() .with_changes(changes) .with_current_opt(update.map(|(_, current)| current)), blob_update, ))) } } ================================================ FILE: crates/jmap/src/sieve/validate.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::blob::download::BlobDownload; use common::{Server, auth::AccessToken}; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::validate::{ValidateSieveScriptRequest, ValidateSieveScriptResponse}, request::MaybeInvalid, }; use std::future::Future; pub trait SieveScriptValidate: Sync + Send { fn sieve_script_validate( &self, request: ValidateSieveScriptRequest, access_token: &AccessToken, ) -> impl Future> + Send; } impl SieveScriptValidate for Server { async fn sieve_script_validate( &self, request: ValidateSieveScriptRequest, access_token: &AccessToken, ) -> trc::Result { Ok(ValidateSieveScriptResponse { account_id: request.account_id, error: match request.blob_id { MaybeInvalid::Value(blob_id) => { match self .blob_download(&blob_id, access_token) .await? .map(|bytes| self.core.sieve.untrusted_compiler.compile(&bytes)) { Some(Ok(_)) => None, Some(Err(err)) => SetError::new(SetErrorType::InvalidScript) .with_description(err.to_string()) .into(), None => SetError::new(SetErrorType::BlobNotFound).into(), } } MaybeInvalid::Invalid(_) => SetError::new(SetErrorType::BlobNotFound).into(), }, }) } } ================================================ FILE: crates/jmap/src/submission/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::changes::state::StateManager; use common::Server; use email::submission::{ ArchivedAddress, ArchivedEnvelope, ArchivedUndoStatus, Delivered, DeliveryStatus, EmailSubmission, }; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::email_submission::{self, Displayed, EmailSubmissionProperty, EmailSubmissionValue}, types::date::UTCDate, }; use jmap_tools::{Key, Map, Value}; use smtp::queue::{ArchivedError, ArchivedErrorDetails, ArchivedStatus, Message, spool::SmtpSpool}; use smtp_proto::ArchivedResponse; use std::future::Future; use store::{ IterateParams, U32_LEN, ValueKey, rkyv::option::ArchivedOption, write::{ AlignedBytes, Archive, IndexPropertyClass, ValueClass, key::DeserializeBigEndian, now, }, }; use trc::AddContext; use types::{ collection::{Collection, SyncCollection}, field::EmailSubmissionField, id::Id, }; use utils::map::vec_map::VecMap; pub trait EmailSubmissionGet: Sync + Send { fn email_submission_get( &self, request: GetRequest, ) -> impl Future>> + Send; } impl EmailSubmissionGet for Server { async fn email_submission_get( &self, mut request: GetRequest, ) -> trc::Result> { let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ EmailSubmissionProperty::Id, EmailSubmissionProperty::EmailId, EmailSubmissionProperty::IdentityId, EmailSubmissionProperty::ThreadId, EmailSubmissionProperty::Envelope, EmailSubmissionProperty::SendAt, EmailSubmissionProperty::UndoStatus, EmailSubmissionProperty::DeliveryStatus, EmailSubmissionProperty::DsnBlobIds, EmailSubmissionProperty::MdnBlobIds, ]); let account_id = request.account_id.document_id(); let ids = if let Some(ids) = ids { ids } else { let mut ids = Vec::with_capacity(16); self.store() .iterate( IterateParams::new( ValueKey { account_id, collection: Collection::CalendarEventNotification.into(), document_id: 0, class: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: EmailSubmissionField::Metadata.into(), value: now() - (3 * 86400), }), }, ValueKey { account_id, collection: Collection::CalendarEventNotification.into(), document_id: 0, class: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: EmailSubmissionField::Metadata.into(), value: u64::MAX, }), }, ) .ascending() .no_values(), |key, _| { ids.push(Id::from(key.deserialize_be_u32(key.len() - U32_LEN)?)); Ok(ids.len() < self.core.jmap.get_max_objects) }, ) .await .caused_by(trc::location!())?; ids }; let mut response = GetResponse { account_id: request.account_id.into(), state: self .get_state(account_id, SyncCollection::EmailSubmission) .await? .into(), list: Vec::with_capacity(ids.len()), not_found: vec![], }; for id in ids { // Obtain the email_submission object let document_id = id.document_id(); let submission_ = if let Some(submission) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::EmailSubmission, document_id, )) .await? { submission } else { response.not_found.push(id); continue; }; let submission = submission_ .unarchive::() .caused_by(trc::location!())?; // Obtain queueId let mut delivery_status = submission .delivery_status .iter() .map(|(k, v)| (k.to_string(), DeliveryStatus::from(v))) .collect::>(); let mut is_pending = false; if let Some(queue_id) = submission.queue_id.as_ref().map(u64::from) && let Some(queued_message_) = self .read_message_archive(queue_id) .await .caused_by(trc::location!())? { let queued_message = queued_message_ .unarchive::() .caused_by(trc::location!())?; for rcpt in queued_message.recipients.iter() { *delivery_status.get_mut_or_insert(rcpt.address().to_string()) = DeliveryStatus { smtp_reply: match &rcpt.status { ArchivedStatus::Completed(reply) => { format_archived_response(&reply.response) } ArchivedStatus::TemporaryFailure(reply) | ArchivedStatus::PermanentFailure(reply) => { format_archived_error_details(reply) } ArchivedStatus::Scheduled => "250 2.1.5 Queued".to_string(), }, delivered: match &rcpt.status { ArchivedStatus::Scheduled | ArchivedStatus::TemporaryFailure(_) => { Delivered::Queued } ArchivedStatus::Completed(_) => Delivered::Yes, ArchivedStatus::PermanentFailure(_) => Delivered::No, }, displayed: false, }; } is_pending = true; } let mut result = Map::with_capacity(properties.len()); for property in &properties { let value = match property { EmailSubmissionProperty::Id => Value::Element(id.into()), EmailSubmissionProperty::DeliveryStatus => { let mut status = Map::with_capacity(delivery_status.len()); for (rcpt, delivery_status) in std::mem::take(&mut delivery_status) { status.insert_unchecked( Key::Owned(rcpt), Map::with_capacity(3) .with_key_value( EmailSubmissionProperty::Delivered, EmailSubmissionValue::Delivered( match delivery_status.delivered { Delivered::Queued => { email_submission::Delivered::Queued } Delivered::Yes => email_submission::Delivered::Yes, Delivered::No => email_submission::Delivered::No, Delivered::Unknown => { email_submission::Delivered::Unknown } }, ), ) .with_key_value( EmailSubmissionProperty::SmtpReply, delivery_status.smtp_reply, ) .with_key_value( EmailSubmissionProperty::Displayed, Value::Element(EmailSubmissionValue::Displayed( Displayed::Unknown, )), ), ); } Value::Object(status) } EmailSubmissionProperty::UndoStatus => { Value::Element(EmailSubmissionValue::UndoStatus(if is_pending { email_submission::UndoStatus::Pending } else { match submission.undo_status { ArchivedUndoStatus::Pending => { email_submission::UndoStatus::Pending } ArchivedUndoStatus::Final => email_submission::UndoStatus::Final, ArchivedUndoStatus::Canceled => { email_submission::UndoStatus::Canceled } } })) } EmailSubmissionProperty::EmailId => Value::Element( Id::from_parts( u32::from(submission.thread_id), u32::from(submission.email_id), ) .into(), ), EmailSubmissionProperty::IdentityId => { Value::Element(Id::from(u32::from(submission.identity_id)).into()) } EmailSubmissionProperty::ThreadId => { Value::Element(Id::from(u32::from(submission.thread_id)).into()) } EmailSubmissionProperty::Envelope => build_envelope(&submission.envelope), EmailSubmissionProperty::SendAt => Value::Element(EmailSubmissionValue::Date( UTCDate::from_timestamp(u64::from(submission.send_at) as i64), )), EmailSubmissionProperty::MdnBlobIds | EmailSubmissionProperty::DsnBlobIds => { Value::Array(vec![]) } _ => Value::Null, }; result.insert_unchecked(property.clone(), value); } response.list.push(result.into()); } Ok(response) } } fn build_envelope( envelope: &ArchivedEnvelope, ) -> Value<'static, EmailSubmissionProperty, EmailSubmissionValue> { Map::with_capacity(2) .with_key_value( EmailSubmissionProperty::MailFrom, build_address(&envelope.mail_from), ) .with_key_value( EmailSubmissionProperty::RcptTo, Value::Array(envelope.rcpt_to.iter().map(build_address).collect()), ) .into() } fn build_address( envelope: &ArchivedAddress, ) -> Value<'static, EmailSubmissionProperty, EmailSubmissionValue> { Map::with_capacity(2) .with_key_value( EmailSubmissionProperty::Email, Value::Str(envelope.email.to_string().into()), ) .with_key_value( EmailSubmissionProperty::Parameters, if let ArchivedOption::Some(params) = &envelope.parameters { Value::Object(Map::from_iter( params .iter() .map(|(k, v)| (Key::Owned(k.to_string()), v.into())), )) } else { Value::Null }, ) .into() } fn format_archived_response(response: &ArchivedResponse>) -> String { format!( "Code: {}, Enhanced code: {}.{}.{}, Message: {}", response.code, response.esc[0], response.esc[1], response.esc[2], response.message.replace('\n', " "), ) } fn format_archived_error_details(response: &ArchivedErrorDetails) -> String { match &response.details { ArchivedError::UnexpectedResponse(response) => format_archived_response(&response.response), ArchivedError::DnsError(details) | ArchivedError::Io(details) | ArchivedError::ConnectionError(details) | ArchivedError::TlsError(details) | ArchivedError::DaneError(details) | ArchivedError::MtaStsError(details) => details.to_string(), ArchivedError::RateLimited => "Rate limited".to_string(), ArchivedError::ConcurrencyLimited => "Concurrency limited".to_string(), } } ================================================ FILE: crates/jmap/src/submission/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod get; pub mod query; pub mod set; ================================================ FILE: crates/jmap/src/submission/query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{api::query::QueryResponseBuilder, changes::state::StateManager}; use common::Server; use email::submission::UndoStatus; use jmap_proto::{ method::query::{Filter, QueryRequest, QueryResponse}, object::email_submission::{self, EmailSubmissionComparator, EmailSubmissionFilter}, request::IntoValid, }; use std::future::Future; use store::{ IterateParams, U32_LEN, U64_LEN, ValueKey, ahash::AHashSet, roaring::RoaringBitmap, search::{SearchFilter, SearchQuery}, write::{IndexPropertyClass, SearchIndex, ValueClass, key::DeserializeBigEndian, now}, }; use trc::AddContext; use types::{ collection::{Collection, SyncCollection}, field::EmailSubmissionField, }; pub trait EmailSubmissionQuery: Sync + Send { fn email_submission_query( &self, request: QueryRequest, ) -> impl Future> + Send; } struct Submission { document_id: u32, send_at: u64, email_id: u32, thread_id: u32, identity_id: u32, undo_status: u8, } impl EmailSubmissionQuery for Server { async fn email_submission_query( &self, mut request: QueryRequest, ) -> trc::Result { let account_id = request.account_id.document_id(); let mut submissions = Vec::with_capacity(16); let mut document_ids = RoaringBitmap::new(); self.store() .iterate( IterateParams::new( ValueKey { account_id, collection: Collection::EmailSubmission.into(), document_id: 0, class: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: EmailSubmissionField::Metadata.into(), value: now() - (3 * 86400), }), }, ValueKey { account_id, collection: Collection::EmailSubmission.into(), document_id: 0, class: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: EmailSubmissionField::Metadata.into(), value: u64::MAX, }), }, ) .ascending(), |key, value| { let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; submissions.push(Submission { document_id, send_at: key.deserialize_be_u64(key.len() - U32_LEN - U64_LEN)?, email_id: value.deserialize_be_u32(0)?, thread_id: value.deserialize_be_u32(U32_LEN)?, identity_id: value.deserialize_be_u32(U32_LEN + U32_LEN)?, undo_status: value.last().copied().unwrap(), }); document_ids.insert(document_id); Ok(true) }, ) .await .caused_by(trc::location!())?; let mut filters = Vec::with_capacity(request.filter.len()); for cond in std::mem::take(&mut request.filter) { match cond { Filter::Property(cond) => match cond { EmailSubmissionFilter::IdentityIds(ids) => { let ids = ids .into_valid() .map(|id| id.document_id()) .collect::>(); filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( submissions .iter() .filter(|s| ids.contains(&s.identity_id)) .map(|s| s.document_id), ))); } EmailSubmissionFilter::EmailIds(ids) => { let ids = ids .into_valid() .map(|id| id.document_id()) .collect::>(); filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( submissions .iter() .filter(|s| ids.contains(&s.email_id)) .map(|s| s.document_id), ))); } EmailSubmissionFilter::ThreadIds(ids) => { let ids = ids .into_valid() .map(|id| id.document_id()) .collect::>(); filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( submissions .iter() .filter(|s| ids.contains(&s.thread_id)) .map(|s| s.document_id), ))); } EmailSubmissionFilter::UndoStatus(undo_status) => { let undo_status = match undo_status { email_submission::UndoStatus::Pending => UndoStatus::Pending, email_submission::UndoStatus::Final => UndoStatus::Final, email_submission::UndoStatus::Canceled => UndoStatus::Canceled, } .as_index(); filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( submissions .iter() .filter(|s| s.undo_status == undo_status) .map(|s| s.document_id), ))); } EmailSubmissionFilter::Before(before) => { let before = before.timestamp() as u64; filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( submissions .iter() .filter(|s| s.send_at < before) .map(|s| s.document_id), ))); } EmailSubmissionFilter::After(after) => { let after = after.timestamp() as u64; filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( submissions .iter() .filter(|s| s.send_at > after) .map(|s| s.document_id), ))); } EmailSubmissionFilter::_T(other) => { return Err(trc::JmapEvent::UnsupportedFilter.into_err().details(other)); } }, Filter::And => { filters.push(SearchFilter::And); } Filter::Or => { filters.push(SearchFilter::Or); } Filter::Not => { filters.push(SearchFilter::Not); } Filter::Close => { filters.push(SearchFilter::End); } } } let results = SearchQuery::new(SearchIndex::InMemory) .with_filters(filters) .with_mask(document_ids) .filter() .into_bitmap(); let mut response = QueryResponseBuilder::new( results.len() as usize, self.core.jmap.query_max_results, self.get_state(account_id, SyncCollection::EmailSubmission) .await?, &request, ); if !results.is_empty() { if let Some(comparator) = request.sort.take().unwrap_or_default().into_iter().next() { match comparator.property { EmailSubmissionComparator::EmailId => { if comparator.is_ascending { submissions.sort_by_key(|s| s.email_id); } else { submissions.sort_by_key(|s| u32::MAX - s.email_id); } } EmailSubmissionComparator::ThreadId => { if comparator.is_ascending { submissions.sort_by_key(|s| s.thread_id); } else { submissions.sort_by_key(|s| u32::MAX - s.thread_id); } } EmailSubmissionComparator::SentAt => { if !comparator.is_ascending { submissions.reverse(); } } EmailSubmissionComparator::_T(other) => { return Err(trc::JmapEvent::UnsupportedSort.into_err().details(other)); } } } for submission in submissions { if results.contains(submission.document_id) && !response.add(0, submission.document_id) { break; } } } response.build() } } ================================================ FILE: crates/jmap/src/submission/set.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{ Server, config::smtp::queue::QueueName, listener::{ServerInstance, stream::NullIo}, storage::index::ObjectIndexBuilder, }; use email::{ identity::Identity, message::metadata::{ArchivedMetadataHeaderName, ArchivedMetadataHeaderValue, MessageMetadata}, submission::{Address, Delivered, DeliveryStatus, EmailSubmission, UndoStatus}, }; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{SetRequest, SetResponse}, object::email_submission::{self, EmailSubmissionProperty, EmailSubmissionValue}, references::resolve::ResolveCreatedReference, request::{ Call, IntoValid, MaybeInvalid, RequestMethod, SetRequestMethod, method::{MethodFunction, MethodName, MethodObject}, reference::{MaybeIdReference, MaybeResultReference}, }, types::state::State, }; use jmap_tools::{Key, Value}; use smtp::{ core::{Session, SessionData}, queue::spool::SmtpSpool, }; use smtp_proto::{MailFrom, RcptTo, request::parser::Rfc5321Parser}; use std::{borrow::Cow, future::Future}; use std::{collections::HashMap, sync::Arc, time::Duration}; use store::{ ValueKey, write::{AlignedBytes, Archive, BatchBuilder, now}, }; use trc::AddContext; use types::{collection::Collection, field::EmailField, id::Id}; use utils::{map::vec_map::VecMap, sanitize_email}; pub trait EmailSubmissionSet: Sync + Send { fn email_submission_set<'x>( &self, request: SetRequest<'x, email_submission::EmailSubmission>, instance: &Arc, next_call: &mut Option>>, ) -> impl Future>> + Send; fn send_message( &self, account_id: u32, response: &SetResponse, instance: &Arc, object: Value<'_, EmailSubmissionProperty, EmailSubmissionValue>, ) -> impl Future< Output = trc::Result>>, > + Send; } impl EmailSubmissionSet for Server { async fn email_submission_set<'x>( &self, mut request: SetRequest<'x, email_submission::EmailSubmission>, instance: &Arc, next_call: &mut Option>>, ) -> trc::Result> { let account_id = request.account_id.document_id(); let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; let will_destroy = request.unwrap_destroy().into_valid().collect::>(); // Process creates let mut success_email_ids = HashMap::new(); let mut batch = BatchBuilder::new(); for (id, object) in request.unwrap_create() { match self .send_message(account_id, &response, instance, object) .await? { Ok(submission) => { // Add id mapping success_email_ids.insert( id.clone(), Id::from_parts(submission.thread_id, submission.email_id), ); // Insert record let document_id = self .store() .assign_document_ids(account_id, Collection::EmailSubmission, 1) .await .caused_by(trc::location!())?; batch .with_account_id(account_id) .with_collection(Collection::EmailSubmission) .with_document(document_id) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(submission)) .caused_by(trc::location!())? .commit_point(); response.created(id, document_id); } Err(err) => { response.not_created.append(id, err); } } } // Process updates 'update: for (id, object) in request.unwrap_update().into_valid() { // Make sure id won't be destroyed if will_destroy.contains(&id) { response.not_updated.append(id, SetError::will_destroy()); continue 'update; } // Obtain submission let document_id = id.document_id(); let submission = if let Some(submission) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::EmailSubmission, document_id, )) .await? { submission .into_deserialized::() .caused_by(trc::location!())? } else { response.not_updated.append(id, SetError::not_found()); continue 'update; }; let mut queue_id = u64::MAX; let mut undo_status = None; for (property, mut value) in object.into_expanded_object() { if let Err(err) = response.resolve_self_references(&mut value) { response.not_updated.append(id, err); continue 'update; }; if let ( Key::Property(EmailSubmissionProperty::UndoStatus), Value::Element(EmailSubmissionValue::UndoStatus(undo_status_)), Some(queue_id_), ) = (&property, value, submission.inner.queue_id) { undo_status = undo_status_.into(); queue_id = queue_id_; } else { response.not_updated.append( id, SetError::invalid_properties() .with_property(property.into_owned()) .with_description("Field could not be set."), ); continue 'update; } } match undo_status { Some(email_submission::UndoStatus::Canceled) => { if let Some(queue_message) = self.read_message(queue_id, QueueName::default()).await { // Delete message from queue queue_message.remove(self, None).await; // Update record let mut new_submission = submission.inner.clone(); new_submission.undo_status = UndoStatus::Canceled; batch .with_account_id(account_id) .with_collection(Collection::EmailSubmission) .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_current(submission) .with_changes(new_submission), ) .caused_by(trc::location!())? .commit_point(); response.updated.append(id, None); } else { response.not_updated.append( id, SetError::new(SetErrorType::CannotUnsend).with_description( "The requested message is no longer in the queue.", ), ); } } Some(_) => { response.not_updated.append( id, SetError::invalid_properties() .with_property(EmailSubmissionProperty::UndoStatus) .with_description("Email submissions can only be cancelled."), ); } None => { response.not_updated.append( id, SetError::invalid_properties() .with_description("No properties to set were found."), ); } } } // Process deletions for id in will_destroy { let document_id = id.document_id(); if let Some(submission) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::EmailSubmission, document_id, )) .await? { // Update record batch .with_account_id(account_id) .with_collection(Collection::EmailSubmission) .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new().with_current( submission .to_unarchived::() .caused_by(trc::location!())?, ), ) .caused_by(trc::location!())? .commit_point(); response.destroyed.push(id); } else { response.not_destroyed.append(id, SetError::not_found()); } } // Write changes if !batch.is_empty() { let change_id = self .commit_batch(batch) .await .and_then(|ids| ids.last_change_id(account_id)) .caused_by(trc::location!())?; response.new_state = State::Exact(change_id).into(); } // On success if (request .arguments .on_success_destroy_email .as_ref() .is_some_and(|p| !p.is_empty()) || request .arguments .on_success_update_email .as_ref() .is_some_and(|p| !p.is_empty())) && response.has_changes() { *next_call = Call { id: String::new(), name: MethodName::new(MethodObject::Email, MethodFunction::Set), method: RequestMethod::Set(SetRequestMethod::Email(SetRequest { account_id: request.account_id, if_in_state: None, create: None, update: request.arguments.on_success_update_email.map(|update| { update .into_iter() .filter_map(|(id, value)| { ( match id { MaybeIdReference::Id(id) => MaybeInvalid::Value(id), MaybeIdReference::Reference(id_ref) => { MaybeInvalid::Value(*(success_email_ids.get(&id_ref)?)) } MaybeIdReference::Invalid(id) => MaybeInvalid::Invalid(id), }, value, ) .into() }) .collect() }), destroy: request.arguments.on_success_destroy_email.map(|ids| { MaybeResultReference::Value( ids.into_iter() .filter_map(|id| match id { MaybeIdReference::Id(id) => Some(id), MaybeIdReference::Reference(id_ref) => { success_email_ids.get(&id_ref).copied() } MaybeIdReference::Invalid(_) => None, }) .map(MaybeInvalid::Value) .collect(), ) }), arguments: Default::default(), })), } .into(); } Ok(response) } async fn send_message( &self, account_id: u32, response: &SetResponse, instance: &Arc, object: Value<'_, EmailSubmissionProperty, EmailSubmissionValue>, ) -> trc::Result>> { let mut submission = EmailSubmission { email_id: u32::MAX, identity_id: u32::MAX, thread_id: u32::MAX, ..Default::default() }; let mut mail_from: Option>> = None; let mut rcpt_to: Vec>> = Vec::new(); for (property, mut value) in object.into_expanded_object() { if let Err(err) = response.resolve_self_references(&mut value) { return Ok(Err(err)); }; match (&property, value) { ( Key::Property(EmailSubmissionProperty::EmailId), Value::Element(EmailSubmissionValue::Id(value)), ) => { submission.email_id = value.document_id(); submission.thread_id = value.prefix_id(); } ( Key::Property(EmailSubmissionProperty::IdentityId), Value::Element(EmailSubmissionValue::Id(value)), ) => { submission.identity_id = value.document_id(); } (Key::Property(EmailSubmissionProperty::Envelope), Value::Object(value)) => { for (property, value) in value.into_vec() { match (&property, value) { (Key::Property(EmailSubmissionProperty::MailFrom), value) => { match parse_envelope_address(value) { Ok((addr, params, smtp_params)) => { match Rfc5321Parser::new( &mut smtp_params .as_ref() .map_or(&b"\n"[..], |p| p.as_bytes()) .iter(), ) .mail_from_parameters(addr.into()) { Ok(addr) => { submission.envelope.mail_from = Address { email: addr.address.as_ref().to_string(), parameters: params, }; mail_from = from_into_static(addr).into(); } Err(err) => { return Ok(Err(SetError::invalid_properties() .with_property(EmailSubmissionProperty::Envelope) .with_description(format!( "Failed to parse mailFrom parameters: {err}." )))); } } } Err(err) => { return Ok(Err(err)); } } } ( Key::Property(EmailSubmissionProperty::RcptTo), Value::Array(value), ) => { for addr in value { match parse_envelope_address(addr) { Ok((addr, params, smtp_params)) => { match Rfc5321Parser::new( &mut smtp_params .as_ref() .map_or(&b"\n"[..], |p| p.as_bytes()) .iter(), ) .rcpt_to_parameters(addr.into()) { Ok(addr) => { if !rcpt_to .iter() .any(|rcpt| rcpt.address == addr.address) { submission.envelope.rcpt_to.push(Address { email: addr .address .as_ref() .to_string(), parameters: params, }); rcpt_to.push(rcpt_into_static(addr)); } } Err(err) => { return Ok(Err(SetError::invalid_properties() .with_property(EmailSubmissionProperty::Envelope) .with_description(format!( "Failed to parse rcptTo parameters: {err}." )))); } } } Err(err) => { return Ok(Err(err)); } } } } _ => { return Ok(Err(SetError::invalid_properties() .with_property(EmailSubmissionProperty::Envelope) .with_description("Invalid object property."))); } } } } (Key::Property(EmailSubmissionProperty::Envelope), Value::Null) => { continue; } (Key::Property(EmailSubmissionProperty::UndoStatus), Value::Element(_)) => { continue; } _ => { return Ok(Err(SetError::invalid_properties() .with_property(property.into_owned()) .with_description("Field could not be set."))); } } } // Make sure we have all required fields. if submission.email_id == u32::MAX || submission.identity_id == u32::MAX { return Ok(Err(SetError::invalid_properties() .with_properties([ EmailSubmissionProperty::EmailId, EmailSubmissionProperty::IdentityId, ]) .with_description( "emailId and identityId properties are required.", ))); } // Fetch identity's mailFrom let identity_mail_from = if let Some(identity) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::Identity, submission.identity_id, )) .await? { identity .unarchive::() .caused_by(trc::location!())? .email .to_string() } else { return Ok(Err(SetError::invalid_properties() .with_property(EmailSubmissionProperty::IdentityId) .with_description("Identity not found."))); }; // Make sure the envelope address matches the identity email address let mail_from = if let Some(mail_from) = mail_from { if !mail_from.address.eq_ignore_ascii_case(&identity_mail_from) { return Ok(Err(SetError::new(SetErrorType::ForbiddenFrom) .with_description( "Envelope mailFrom does not match identity email address.", ))); } mail_from } else { submission.envelope.mail_from = Address { email: identity_mail_from.clone(), parameters: None, }; MailFrom { address: Cow::Owned(identity_mail_from), ..Default::default() } }; // Obtain message metadata let metadata_ = if let Some(metadata) = self .store() .get_value::>(ValueKey::property( account_id, Collection::Email, submission.email_id, EmailField::Metadata, )) .await? { metadata } else { return Ok(Err(SetError::invalid_properties() .with_property(EmailSubmissionProperty::EmailId) .with_description("Email not found."))); }; let metadata = metadata_ .unarchive::() .caused_by(trc::location!())?; // Add recipients to envelope if missing let mut bcc_header = None; if rcpt_to.is_empty() { for header in metadata.contents[0].parts[0].headers.iter() { if matches!( header.name, ArchivedMetadataHeaderName::To | ArchivedMetadataHeaderName::Cc | ArchivedMetadataHeaderName::Bcc ) { if matches!(header.name, ArchivedMetadataHeaderName::Bcc) { bcc_header = Some(header); } match &header.value { ArchivedMetadataHeaderValue::AddressList(addr) => { for address in addr.iter() { if let Some(address) = address .address .as_ref() .map(|v| v.as_ref()) .and_then(sanitize_email) && !rcpt_to.iter().any(|rcpt| rcpt.address == address) { submission.envelope.rcpt_to.push(Address { email: address.to_string(), parameters: None, }); rcpt_to.push(RcptTo { address: Cow::Owned(address), ..Default::default() }); } } } ArchivedMetadataHeaderValue::AddressGroup(groups) => { for group in groups.iter() { for address in group.addresses.iter() { if let Some(address) = address .address .as_ref() .map(|v| v.as_ref()) .and_then(sanitize_email) && !rcpt_to.iter().any(|rcpt| rcpt.address == address) { submission.envelope.rcpt_to.push(Address { email: address.to_string(), parameters: None, }); rcpt_to.push(RcptTo { address: Cow::Owned(address), ..Default::default() }); } } } } _ => {} } } } if rcpt_to.is_empty() { return Ok(Err(SetError::new(SetErrorType::NoRecipients) .with_description("No recipients found in email."))); } } else { bcc_header = metadata.contents[0].parts[0] .headers .iter() .find(|header| matches!(header.name, ArchivedMetadataHeaderName::Bcc)); } // Update sendAt submission.send_at = if mail_from.hold_until > 0 { mail_from.hold_until } else if mail_from.hold_for > 0 { mail_from.hold_for + now() } else { now() }; // Obtain raw message let mut message = if let Some(message) = self .blob_store() .get_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX) .await? { if message.len() > self.core.jmap.mail_max_size { return Ok(Err(SetError::new(SetErrorType::InvalidEmail) .with_description(format!( "Message exceeds maximum size of {} bytes.", self.core.jmap.mail_max_size )))); } message } else { return Ok(Err(SetError::invalid_properties() .with_property(EmailSubmissionProperty::EmailId) .with_description("Blob for email not found."))); }; // Remove BCC header if present if let Some(bcc_header) = bcc_header { let mut new_message = Vec::with_capacity(message.len()); let range = bcc_header.name_value_range(); new_message.extend_from_slice(&message[..range.start]); new_message.extend_from_slice(&message[range.end..]); message = new_message; } // Begin local SMTP session let mut session = Session::::local( self.clone(), instance.clone(), SessionData::local( self.get_access_token(account_id) .await .caused_by(trc::location!())?, None, vec![], vec![], 0, ), ); // Spawn SMTP session to avoid overflowing the stack let handle = tokio::spawn(async move { // MAIL FROM let _ = session.handle_mail_from(mail_from).await; if let Some(error) = session.has_failed() { return Err(SetError::new(SetErrorType::ForbiddenMailFrom) .with_description(format!("Server rejected MAIL-FROM: {}", error.trim()))); } // RCPT TO let mut responses = Vec::new(); let mut has_success = false; session.params.rcpt_errors_wait = Duration::from_secs(0); for rcpt in rcpt_to { let addr = rcpt.address.clone(); let _ = session.handle_rcpt_to(rcpt).await; let response = session.has_failed(); if response.is_none() { has_success = true; } responses.push((addr, response)); } // DATA if has_success { session.data.message = message; let response = session.queue_message().await; if let smtp::core::State::Accepted(queue_id) = session.state { Ok((true, responses, Some(queue_id))) } else { Err( SetError::new(SetErrorType::ForbiddenToSend).with_description(format!( "Server rejected DATA: {}", std::str::from_utf8(&response).unwrap().trim() )), ) } } else { Ok((false, responses, None)) } }); match handle.await { Ok(Ok((has_success, responses, queue_id))) => { // Set queue ID if let Some(queue_id) = queue_id { submission.queue_id = Some(queue_id); } // Set responses submission.undo_status = if has_success { UndoStatus::Final } else { UndoStatus::Pending }; submission.delivery_status = responses .into_iter() .map(|(addr, response)| { ( addr.to_string(), DeliveryStatus { delivered: if response.is_none() { Delivered::Unknown } else { Delivered::No }, smtp_reply: response .map(|s| s.to_string()) .unwrap_or_else(|| "250 2.1.5 Queued".to_string()), displayed: false, }, ) }) .collect(); Ok(Ok(submission)) } Ok(Err(err)) => Ok(Err(err)), Err(err) => Err(trc::EventType::Server(trc::ServerEvent::ThreadError) .reason(err) .caused_by(trc::location!()) .details("Join Error")), } } } #[allow(clippy::type_complexity)] fn parse_envelope_address( envelope: Value<'_, EmailSubmissionProperty, EmailSubmissionValue>, ) -> Result< ( String, Option>>, Option, ), SetError, > { if let Value::Object(mut envelope) = envelope { if let Some(Value::Str(addr)) = envelope.remove(&Key::Property(EmailSubmissionProperty::Email)) { if let Some(addr) = sanitize_email(&addr) { if let Some(Value::Object(params)) = envelope.remove(&Key::Property(EmailSubmissionProperty::Parameters)) { let mut params_text = String::new(); let mut params_list = VecMap::with_capacity(params.len()); for (k, v) in params.into_vec() { let k = k.into_string(); if !k.is_empty() { if !params_text.is_empty() { params_text.push(' '); } params_text.push_str(&k); if let Value::Str(v) = v { params_text.push('='); params_text.push_str(&v); params_list.append(k, Some(v.into_owned())); } else { params_list.append(k, None); } } } params_text.push('\n'); Ok((addr.to_string(), Some(params_list), Some(params_text))) } else { Ok((addr.to_string(), None, None)) } } else { Err(SetError::invalid_properties() .with_property(EmailSubmissionProperty::Envelope) .with_description(format!("Invalid e-mail address {addr:?}."))) } } else { Err(SetError::invalid_properties() .with_property(EmailSubmissionProperty::Envelope) .with_description("Missing e-mail address field.")) } } else { Err(SetError::invalid_properties() .with_property(EmailSubmissionProperty::Envelope) .with_description("Invalid envelope object.")) } } fn from_into_static(from: MailFrom>) -> MailFrom> { MailFrom { address: from.address.into_owned().into(), flags: from.flags, size: from.size, trans_id: from.trans_id.map(Cow::into_owned).map(Cow::Owned), by: from.by, env_id: from.env_id.map(Cow::into_owned).map(Cow::Owned), solicit: from.solicit.map(Cow::into_owned).map(Cow::Owned), mtrk: from .mtrk .map(smtp_proto::Mtrk::into_owned) .map(|v| smtp_proto::Mtrk { certifier: Cow::Owned(v.certifier), timeout: v.timeout, }), auth: from.auth.map(Cow::into_owned).map(Cow::Owned), hold_for: from.hold_for, hold_until: from.hold_until, mt_priority: from.mt_priority, } } fn rcpt_into_static(rcpt: RcptTo>) -> RcptTo> { RcptTo { address: rcpt.address.into_owned().into(), orcpt: rcpt.orcpt.map(Cow::into_owned).map(Cow::Owned), rrvs: rcpt.rrvs, flags: rcpt.flags, } } ================================================ FILE: crates/jmap/src/thread/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::changes::state::StateManager; use common::Server; use email::cache::MessageCacheFetch; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::thread::{Thread, ThreadProperty, ThreadValue}, request::MaybeInvalid, }; use jmap_tools::Map; use std::future::Future; use store::{ ahash::AHashMap, roaring::RoaringBitmap, search::{EmailSearchField, SearchComparator, SearchField, SearchQuery}, write::SearchIndex, }; use trc::AddContext; use types::{collection::SyncCollection, id::Id}; pub trait ThreadGet: Sync + Send { fn thread_get( &self, request: GetRequest, ) -> impl Future>> + Send; } impl ThreadGet for Server { async fn thread_get( &self, mut request: GetRequest, ) -> trc::Result> { let account_id = request.account_id.document_id(); let mut thread_map: AHashMap = AHashMap::with_capacity(32); let mut all_ids = RoaringBitmap::new(); for item in &self .get_cached_messages(account_id) .await .caused_by(trc::location!())? .emails .items { thread_map .entry(item.thread_id) .or_default() .insert(item.document_id); all_ids.insert(item.document_id); } let ids = if let Some(ids) = request.unwrap_ids(self.core.jmap.get_max_objects)? { ids } else { thread_map .keys() .copied() .take(self.core.jmap.get_max_objects) .map(Into::into) .collect() }; let add_email_ids = request.properties.is_none_or(|p| { p.unwrap() .contains(&MaybeInvalid::Value(ThreadProperty::EmailIds)) }); let mut response = GetResponse { account_id: request.account_id.into(), state: self .get_state(account_id, SyncCollection::Thread) .await? .into(), list: Vec::with_capacity(ids.len()), not_found: vec![], }; let ordered_ids = if add_email_ids && !all_ids.is_empty() { Some( self.search_store() .query_account( SearchQuery::new(SearchIndex::Email) .with_account_id(account_id) .with_mask(all_ids) .with_comparator(SearchComparator::Field { field: SearchField::Email(EmailSearchField::ReceivedAt), ascending: true, }), ) .await?, ) } else { None }; for id in ids { let thread_id = id.document_id(); if let Some(mut document_ids) = thread_map.remove(&thread_id) { let mut thread: Map<'_, ThreadProperty, ThreadValue> = Map::with_capacity(2).with_key_value(ThreadProperty::Id, id); if let Some(ordered_ids) = &ordered_ids { let mut ids = Vec::with_capacity(document_ids.len() as usize); for &id in ordered_ids.iter() { if document_ids.remove(id) { ids.push(Id::from_parts(thread_id, id)); } } for id in document_ids.iter() { ids.push(Id::from_parts(thread_id, id)); } thread.insert_unchecked(ThreadProperty::EmailIds, ids); } response.list.push(thread.into()); } else { response.not_found.push(id); } } Ok(response) } } ================================================ FILE: crates/jmap/src/thread/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod get; ================================================ FILE: crates/jmap/src/vacation/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::changes::state::StateManager; use common::Server; use email::sieve::{SieveScript, ingest::SieveScriptIngest}; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::vacation_response::{ VacationResponse, VacationResponseProperty, VacationResponseValue, }, request::reference::MaybeResultReference, types::date::UTCDate, }; use jmap_tools::{Map, Value}; use std::future::Future; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{ collection::{Collection, SyncCollection}, field::SieveField, id::Id, }; pub trait VacationResponseGet: Sync + Send { fn vacation_response_get( &self, request: GetRequest, ) -> impl Future>> + Send; fn get_vacation_sieve_script_id( &self, account_id: u32, ) -> impl Future>> + Send; } impl VacationResponseGet for Server { async fn vacation_response_get( &self, mut request: GetRequest, ) -> trc::Result> { let account_id = request.account_id.document_id(); let properties = request.unwrap_properties(&[ VacationResponseProperty::Id, VacationResponseProperty::IsEnabled, VacationResponseProperty::FromDate, VacationResponseProperty::ToDate, VacationResponseProperty::Subject, VacationResponseProperty::TextBody, VacationResponseProperty::HtmlBody, ]); let mut response = GetResponse { account_id: request.account_id.into(), state: self .get_state(account_id, SyncCollection::SieveScript) .await? .into(), list: Vec::with_capacity(1), not_found: vec![], }; let do_get = if let Some(MaybeResultReference::Value(ids)) = request.ids { let mut do_get = false; for id in ids { match id.try_unwrap() { Some(id) if id.is_singleton() => { do_get = true; } Some(id) => { response.not_found.push(id); } _ => {} } } do_get } else { true }; if do_get { if let Some(document_id) = self.get_vacation_sieve_script_id(account_id).await? { if let Some(sieve_) = self .store() .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, document_id, )) .await? { let active_script_id = self.sieve_script_get_active_id(account_id).await?; let sieve = sieve_ .unarchive::() .caused_by(trc::location!())?; let vacation = sieve.vacation_response.as_ref(); let mut result = Map::with_capacity(properties.len()); for property in &properties { match property { VacationResponseProperty::Id => { result.insert_unchecked( VacationResponseProperty::Id, Id::singleton(), ); } VacationResponseProperty::IsEnabled => { result.insert_unchecked( VacationResponseProperty::IsEnabled, active_script_id == Some(document_id), ); } VacationResponseProperty::FromDate => { result.insert_unchecked( VacationResponseProperty::FromDate, vacation.and_then(|r| { r.from_date .as_ref() .map(u64::from) .map(UTCDate::from) .map(|v| Value::Element(VacationResponseValue::Date(v))) }), ); } VacationResponseProperty::ToDate => { result.insert_unchecked( VacationResponseProperty::ToDate, vacation.and_then(|r| { r.to_date .as_ref() .map(u64::from) .map(UTCDate::from) .map(|v| Value::Element(VacationResponseValue::Date(v))) }), ); } VacationResponseProperty::Subject => { result.insert_unchecked( VacationResponseProperty::Subject, vacation.and_then(|r| r.subject.as_ref()), ); } VacationResponseProperty::TextBody => { result.insert_unchecked( VacationResponseProperty::TextBody, vacation.and_then(|r| r.text_body.as_ref()), ); } VacationResponseProperty::HtmlBody => { result.insert_unchecked( VacationResponseProperty::HtmlBody, vacation.and_then(|r| r.html_body.as_ref()), ); } } } response.list.push(result.into()); } else { response.not_found.push(Id::singleton()); } } else { response.not_found.push(Id::singleton()); } } Ok(response) } async fn get_vacation_sieve_script_id(&self, account_id: u32) -> trc::Result> { self.document_ids_matching( account_id, Collection::SieveScript, SieveField::Name, "vacation".as_bytes(), ) .await .map(|r| r.min()) } } ================================================ FILE: crates/jmap/src/vacation/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod get; pub mod set; ================================================ FILE: crates/jmap/src/vacation/set.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::get::VacationResponseGet; use crate::changes::state::StateManager; use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; use email::sieve::{ SieveScript, VacationResponse, delete::SieveScriptDelete, ingest::SieveScriptIngest, }; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{SetRequest, SetResponse}, object::vacation_response::{self, VacationResponseProperty, VacationResponseValue}, references::resolve::ResolveCreatedReference, request::IntoValid, types::date::UTCDate, }; use jmap_tools::{Key, Map, Value}; use mail_builder::MessageBuilder; use mail_parser::decoders::html::html_to_text; use std::borrow::Cow; use std::future::Future; use store::{ Serialize, SerializeInfallible, ValueKey, write::{AlignedBytes, Archive, Archiver, BatchBuilder}, }; use trc::AddContext; use types::{ collection::{Collection, SyncCollection}, field::PrincipalField, id::Id, }; pub trait VacationResponseSet: Sync + Send { fn vacation_response_set( &self, request: SetRequest<'_, vacation_response::VacationResponse>, access_token: &AccessToken, ) -> impl Future>> + Send; fn build_script(&self, obj: &mut SieveScript) -> trc::Result>; } impl VacationResponseSet for Server { async fn vacation_response_set( &self, mut request: SetRequest<'_, vacation_response::VacationResponse>, access_token: &AccessToken, ) -> trc::Result> { let account_id = request.account_id.document_id(); let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)? .with_state( self.assert_state( account_id, SyncCollection::SieveScript, &request.if_in_state, ) .await?, ); let will_destroy = request.unwrap_destroy().into_valid().collect::>(); // Process set or update requests let mut create_id = None; let mut changes = None; match (request.create, request.update) { (Some(create), Some(update)) if !create.is_empty() && !update.is_empty() => { return Err(trc::JmapEvent::InvalidArguments .into_err() .details("Creating and updating on the same request is not allowed.")); } (Some(create), _) if !create.is_empty() => { for (id, obj) in create { if will_destroy.contains(&Id::singleton()) { response.not_created.append( id, SetError::new(SetErrorType::WillDestroy) .with_description("ID will be destroyed."), ); } else if create_id.is_some() { response.not_created.append( id, SetError::forbidden() .with_description("Only one object can be created."), ); } else { create_id = Some(id); changes = Some(obj); } } } (_, Some(update)) if !update.is_empty() => { for (id, obj) in update.into_valid() { if id.is_singleton() { if !will_destroy.contains(&id) { changes = Some(obj); } else { response.not_updated.append( id, SetError::new(SetErrorType::WillDestroy) .with_description("ID will be destroyed."), ); } } else { response.not_updated.append( id, SetError::new(SetErrorType::NotFound).with_description("ID not found."), ); } } } _ => { if will_destroy.is_empty() { return Ok(response); } } } // Prepare write batch let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::SieveScript); // Process changes let active_script_id = self.sieve_script_get_active_id(account_id).await?; if let Some(changes) = changes { // Obtain current script let document_id = self.get_vacation_sieve_script_id(account_id).await?; let (mut sieve, prev_sieve) = if let Some(document_id) = document_id { let prev_sieve = self .store() .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, document_id, )) .await? .ok_or_else(|| { trc::StoreEvent::NotFound .into_err() .caused_by(trc::location!()) })? .into_deserialized::() .caused_by(trc::location!())?; let mut sieve = prev_sieve.inner.clone(); if sieve.vacation_response.is_none() { sieve.vacation_response = VacationResponse::default().into(); } (sieve, Some(prev_sieve)) } else { ( SieveScript { name: "vacation".into(), blob_hash: Default::default(), size: 0, vacation_response: VacationResponse::default().into(), }, None, ) }; // Parse properties let mut is_active = false; let mut build_script = create_id.is_some(); let vacation = sieve.vacation_response.as_mut().unwrap(); for (property, mut value) in changes.into_expanded_object() { if let Err(err) = response.resolve_self_references(&mut value) { return Ok(set_error(response, create_id, err)); }; match (&property, value) { (Key::Property(VacationResponseProperty::Subject), Value::Str(value)) if value.len() < 512 => { build_script = true; vacation.subject = Some(value.into_owned()); } (Key::Property(VacationResponseProperty::HtmlBody), Value::Str(value)) if value.len() < 2048 => { build_script = true; vacation.html_body = Some(value.into_owned()); } (Key::Property(VacationResponseProperty::TextBody), Value::Str(value)) if value.len() < 2048 => { build_script = true; vacation.text_body = Some(value.into_owned()); } ( Key::Property(VacationResponseProperty::FromDate), Value::Element(VacationResponseValue::Date(date)), ) => { vacation.from_date = Some(date.timestamp() as u64); build_script = true; } ( Key::Property(VacationResponseProperty::ToDate), Value::Element(VacationResponseValue::Date(date)), ) => { vacation.to_date = Some(date.timestamp() as u64); build_script = true; } (Key::Property(VacationResponseProperty::IsEnabled), Value::Bool(value)) => { is_active = value; } (Key::Property(VacationResponseProperty::IsEnabled), Value::Null) => { is_active = false; } ( Key::Property( VacationResponseProperty::Subject | VacationResponseProperty::HtmlBody | VacationResponseProperty::TextBody | VacationResponseProperty::ToDate | VacationResponseProperty::FromDate, ), Value::Null, ) => { if create_id.is_none() { build_script = true; match property { Key::Property(VacationResponseProperty::Subject) => { vacation.subject = None; } Key::Property(VacationResponseProperty::HtmlBody) => { vacation.html_body = None; } Key::Property(VacationResponseProperty::TextBody) => { vacation.text_body = None; } Key::Property(VacationResponseProperty::FromDate) => { vacation.from_date = None; } Key::Property(VacationResponseProperty::ToDate) => { vacation.to_date = None; } _ => unreachable!(), } } } _ => { return Ok(set_error( response, create_id, SetError::invalid_properties() .with_property(property.into_owned()) .with_description("Field could not be set."), )); } } } let mut obj = ObjectIndexBuilder::new() .with_current_opt(prev_sieve) .with_changes(sieve) .with_access_token(access_token); // Update id let document_id = if let Some(document_id) = document_id { batch.with_document(document_id); document_id } else { let document_id = self .store() .assign_document_ids(account_id, Collection::SieveScript, 1) .await .caused_by(trc::location!())?; batch.with_document(document_id); document_id }; // Create sieve script only if there are changes if build_script { // Upload new blob let (blob_hash, blob_hold) = self .put_temporary_blob( account_id, &self.build_script(obj.changes_mut().unwrap())?, 60, ) .await?; obj.changes_mut().unwrap().blob_hash = blob_hash; batch.clear(blob_hold); }; batch.custom(obj).caused_by(trc::location!())?; // Deactivate other sieve scripts let was_active = active_script_id == Some(document_id); if is_active { if !was_active { batch .with_collection(Collection::Principal) .with_document(0) .set(PrincipalField::ActiveScriptId, document_id.serialize()); } } else if was_active { batch .with_collection(Collection::Principal) .with_document(0) .clear(PrincipalField::ActiveScriptId); } // Write changes if !batch.is_empty() { response.new_state = Some( self.commit_batch(batch) .await .and_then(|ids| ids.last_change_id(account_id)) .caused_by(trc::location!())? .into(), ); } // Add result if let Some(create_id) = create_id { response.created.insert( create_id, Map::with_capacity(1) .with_key_value(VacationResponseProperty::Id, Id::singleton()) .into(), ); } else { response.updated.append(Id::singleton(), None); } } else if !will_destroy.is_empty() { for id in will_destroy { if id.is_singleton() && let Some(document_id) = self.get_vacation_sieve_script_id(account_id).await? { self.sieve_script_delete(account_id, document_id, access_token, &mut batch) .await?; if active_script_id == Some(document_id) { batch .with_collection(Collection::Principal) .with_document(0) .clear(PrincipalField::ActiveScriptId); } response.destroyed.push(id); break; } response.not_destroyed.append(id, SetError::not_found()); } // Write changes if !batch.is_empty() { response.new_state = Some( self.commit_batch(batch) .await .and_then(|ids| ids.last_change_id(account_id)) .caused_by(trc::location!())? .into(), ); } } Ok(response) } fn build_script(&self, obj: &mut SieveScript) -> trc::Result> { // Build Sieve script let mut script = Vec::with_capacity(1024); script.extend_from_slice(b"require [\"vacation\", \"relational\", \"date\"];\r\n\r\n"); let mut num_blocks = 0; // Add start date if let Some(value) = obj.vacation_response.as_ref().and_then(|v| v.from_date) { script.extend_from_slice(b"if currentdate :value \"ge\" \"iso8601\" \""); script.extend_from_slice(UTCDate::from(value).to_string().as_bytes()); script.extend_from_slice(b"\" {\r\n"); num_blocks += 1; } // Add end date if let Some(value) = obj.vacation_response.as_ref().and_then(|v| v.to_date) { script.extend_from_slice(b"if currentdate :value \"le\" \"iso8601\" \""); script.extend_from_slice(UTCDate::from(value).to_string().as_bytes()); script.extend_from_slice(b"\" {\r\n"); num_blocks += 1; } script.extend_from_slice(b"vacation :mime "); if let Some(value) = obj .vacation_response .as_ref() .and_then(|v| v.subject.as_ref()) { script.extend_from_slice(b":subject \""); for &ch in value.as_bytes().iter() { match ch { b'\\' | b'\"' => { script.push(b'\\'); } b'\r' | b'\n' => { continue; } _ => (), } script.push(ch); } script.extend_from_slice(b"\" "); } let mut text_body = if let Some(value) = obj .vacation_response .as_ref() .and_then(|v| v.text_body.as_ref()) { Cow::from(value.as_str()).into() } else { None }; let html_body = if let Some(value) = obj .vacation_response .as_ref() .and_then(|v| v.html_body.as_ref()) { Cow::from(value.as_str()).into() } else { None }; match (&html_body, &text_body) { (Some(html_body), None) => { text_body = Cow::from(html_to_text(html_body.as_ref())).into(); } (None, None) => { text_body = Cow::from("I am away.").into(); } _ => (), } let mut builder = MessageBuilder::new(); let mut body_len = 0; if let Some(html_body) = html_body { body_len = html_body.len(); builder = builder.html_body(html_body); } if let Some(text_body) = text_body { body_len += text_body.len(); builder = builder.text_body(text_body); } let mut message_body = Vec::with_capacity(body_len + 128); builder.write_body(&mut message_body).ok(); script.push(b'\"'); for ch in message_body { if [b'\\', b'\"'].contains(&ch) { script.push(b'\\'); } script.push(ch); } script.extend_from_slice(b"\";\r\n"); // Close blocks for _ in 0..num_blocks { script.extend_from_slice(b"}\r\n"); } match self.core.sieve.untrusted_compiler.compile(&script) { Ok(compiled_script) => { // Update blob length obj.size = script.len() as u32; // Serialize script script.extend( Archiver::new(compiled_script) .untrusted() .serialize() .caused_by(trc::location!())?, ); Ok(script) } Err(err) => Err(trc::StoreEvent::UnexpectedError .caused_by(trc::location!()) .reason(err) .details("Vacation Sieve Script failed to compile.")), } } } fn set_error( mut response: SetResponse, id: Option, err: SetError, ) -> SetResponse { if let Some(id) = id { response.not_created.append(id, err); } else { response.not_updated.append(Id::singleton(), err); } response } ================================================ FILE: crates/jmap/src/websocket/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod stream; pub mod upgrade; ================================================ FILE: crates/jmap/src/websocket/stream.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::api::{IntoPushObject, ToRequestError, request::RequestHandler}; use common::{Server, auth::AccessToken, ipc::PushNotification}; use futures_util::{SinkExt, StreamExt}; use http_proto::HttpSessionData; use hyper::upgrade::Upgraded; use hyper_util::rt::TokioIo; use jmap_proto::{ error::request::RequestError, request::websocket::{ WebSocketMessage, WebSocketPushObject, WebSocketRequestError, WebSocketResponse, }, }; use std::future::Future; use std::{sync::Arc, time::Instant}; use tokio_tungstenite::WebSocketStream; use trc::JmapEvent; use tungstenite::Message; use types::type_state::{DataType, StateChange}; use utils::map::bitmap::Bitmap; pub trait WebSocketHandler: Sync + Send { fn handle_websocket_stream( &self, stream: WebSocketStream>, access_token: Arc, session: HttpSessionData, ) -> impl Future + Send; } impl WebSocketHandler for Server { #![allow(clippy::large_futures)] async fn handle_websocket_stream( &self, mut stream: WebSocketStream>, access_token: Arc, session: HttpSessionData, ) { trc::event!( Jmap(JmapEvent::WebsocketStart), SpanId = session.session_id, AccountId = access_token.primary_id(), ); // Set timeouts let throttle = self.core.jmap.web_socket_throttle; let timeout = self.core.jmap.web_socket_timeout; let heartbeat = self.core.jmap.web_socket_heartbeat; let mut last_request = Instant::now(); let mut last_changes_sent = Instant::now() - throttle; let mut last_heartbeat = Instant::now() - heartbeat; let mut next_event = heartbeat; // Register with push manager let mut push_rx = match self .subscribe_push_manager(&access_token, Bitmap::all()) .await { Ok(push_rx) => push_rx, Err(err) => { trc::error!( err.details("Failed to subscribe to push manager") .span_id(session.session_id) ); let _ = stream .send(Message::Text( WebSocketRequestError::from(RequestError::internal_server_error()) .to_json() .into(), )) .await; return; } }; let mut notifications = Vec::new(); let mut change_types: Bitmap = Bitmap::new(); loop { tokio::select! { event = tokio::time::timeout(next_event, stream.next()) => { match event { Ok(Some(Ok(event))) => { match event { Message::Text(text) => { let response = match WebSocketMessage::parse( text.as_bytes(), self.core.jmap.request_max_calls, self.core.jmap.request_max_size, ) { Ok(WebSocketMessage::Request(request)) => { let response = self .handle_jmap_request( request.request, access_token.clone(), &session, ) .await; WebSocketResponse::from_response(response, request.id) .to_json() } Ok(WebSocketMessage::PushEnable(push_enable)) => { change_types = if !push_enable.data_types.is_empty() { push_enable.data_types.into() } else { Bitmap::all() }; continue; } Ok(WebSocketMessage::PushDisable) => { change_types = Bitmap::new(); continue; } Err(err) => { let response = WebSocketRequestError::from(err.to_request_error()).to_json(); trc::error!(err.details("Failed to parse WebSocket message").span_id(session.session_id)); response }, }; if let Err(err) = stream.send(Message::Text(response.into())).await { trc::event!(Jmap(JmapEvent::WebsocketError), Details = "Failed to send text message", SpanId = session.session_id, Reason = err.to_string() ); } } Message::Ping(bytes) => { if let Err(err) = stream.send(Message::Pong(bytes)).await { trc::event!(Jmap(JmapEvent::WebsocketError), Details = "Failed to send pong message", SpanId = session.session_id, Reason = err.to_string() ); } } Message::Close(frame) => { let _ = stream.close(frame).await; break; } _ => (), } last_request = Instant::now(); last_heartbeat = Instant::now(); } Ok(Some(Err(err))) => { trc::event!(Jmap(JmapEvent::WebsocketError), Details = "Websocket error", SpanId = session.session_id, Reason = err.to_string() ); break; } Ok(None) => break, Err(_) => { // Verify timeout if last_request.elapsed() > timeout { trc::event!( Jmap(JmapEvent::WebsocketStop), SpanId = session.session_id, Reason = "Idle client" ); break; } } } } push_notification = push_rx.recv() => { if let Some(push_notification) = push_notification { match push_notification { PushNotification::StateChange(state_change) => { let mut types = state_change.types; types.intersection(&change_types); if !types.is_empty() { notifications.push(PushNotification::StateChange( StateChange { account_id: state_change.account_id, types, change_id: state_change.change_id, } )); } }, PushNotification::EmailPush(email_push) => { let state_change = email_push.to_state_change(); let mut types = state_change.types; types.intersection(&change_types); if !types.is_empty() { notifications.push(PushNotification::StateChange( StateChange { account_id: state_change.account_id, types, change_id: state_change.change_id, } )); } }, PushNotification::CalendarAlert(calendar_alert) => { if change_types.contains(DataType::CalendarAlert) { notifications.push(PushNotification::CalendarAlert( calendar_alert )); } }, } } else { trc::event!( Jmap(JmapEvent::WebsocketStop), SpanId = session.session_id, Reason = "State manager channel closed" ); break; } } } if !notifications.is_empty() { // Send any queued changes let elapsed = last_changes_sent.elapsed(); if elapsed >= throttle { let payload = WebSocketPushObject { push: std::mem::take(&mut notifications).into_push_object(), push_state: None, }; if let Err(err) = stream.send(Message::Text(payload.to_json().into())).await { trc::event!( Jmap(JmapEvent::WebsocketError), Details = "Failed to send state change message.", SpanId = session.session_id, Reason = err.to_string() ); } last_changes_sent = Instant::now(); last_heartbeat = Instant::now(); next_event = heartbeat; } else { next_event = throttle - elapsed; } } else if last_heartbeat.elapsed() > heartbeat { if let Err(err) = stream.send(Message::Ping(Vec::::new().into())).await { trc::event!( Jmap(JmapEvent::WebsocketError), Details = "Failed to send ping message.", SpanId = session.session_id, Reason = err.to_string() ); break; } last_heartbeat = Instant::now(); next_event = heartbeat; } } } } ================================================ FILE: crates/jmap/src/websocket/upgrade.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::Arc; use common::{Server, auth::AccessToken}; use hyper::StatusCode; use hyper_util::rt::TokioIo; use tokio_tungstenite::WebSocketStream; use trc::JmapEvent; use tungstenite::{handshake::derive_accept_key, protocol::Role}; use http_proto::*; use std::future::Future; use super::stream::WebSocketHandler; pub trait WebSocketUpgrade: Sync + Send { fn upgrade_websocket_connection( &self, req: HttpRequest, access_token: Arc, session: HttpSessionData, ) -> impl Future> + Send; } impl WebSocketUpgrade for Server { async fn upgrade_websocket_connection( &self, req: HttpRequest, access_token: Arc, session: HttpSessionData, ) -> trc::Result { let headers = req.headers(); if headers .get(hyper::header::CONNECTION) .and_then(|h| h.to_str().ok()) != Some("Upgrade") || headers .get(hyper::header::UPGRADE) .and_then(|h| h.to_str().ok()) != Some("websocket") { return Err(trc::ResourceEvent::BadParameters .into_err() .details("WebSocket upgrade failed") .ctx( trc::Key::Reason, "Missing or Invalid Connection or Upgrade headers.", )); } let derived_key = match ( headers .get("Sec-WebSocket-Key") .and_then(|h| h.to_str().ok()), headers .get("Sec-WebSocket-Version") .and_then(|h| h.to_str().ok()), ) { (Some(key), Some("13")) => derive_accept_key(key.as_bytes()), _ => { return Err(trc::ResourceEvent::BadParameters .into_err() .details("WebSocket upgrade failed") .ctx( trc::Key::Reason, "Missing or Invalid Sec-WebSocket-Key headers.", )); } }; // Spawn WebSocket connection let jmap = self.clone(); tokio::spawn(async move { // Upgrade connection let session_id = session.session_id; match hyper::upgrade::on(req).await { Ok(upgraded) => { Box::pin( jmap.handle_websocket_stream( WebSocketStream::from_raw_socket( TokioIo::new(upgraded), Role::Server, None, ) .await, access_token, session, ), ) .await; } Err(err) => { trc::event!( Jmap(JmapEvent::WebsocketError), Details = "Websocket upgrade failed", SpanId = session_id, Reason = err.to_string() ); } } }); Ok(HttpResponse::new(StatusCode::SWITCHING_PROTOCOLS).with_websocket_upgrade(derived_key)) } } ================================================ FILE: crates/jmap-proto/Cargo.toml ================================================ [package] name = "jmap_proto" version = "0.15.5" edition = "2024" [dependencies] utils = { path = "../utils" } types = { path = "../types" } trc = { path = "../trc" } jmap-tools = { version = "0.1" } calcard = { version = "0.3" } mail-parser = { version = "0.11", features = ["full_encoding", "rkyv"] } serde = { version = "1.0", features = ["derive"]} ahash = { version = "0.8.2", features = ["serde"] } serde_json = { version = "1.0", features = ["raw_value"] } hashify = "0.2" rkyv = { version = "0.8.10", features = ["little_endian"] } compact_str = { version = "0.9.0", features = ["rkyv", "serde"] } [dev-dependencies] tokio = { version = "1.47", features = ["full"] } ================================================ FILE: crates/jmap-proto/src/error/method.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use serde::Serialize; use serde::ser::SerializeMap; use std::fmt::Display; #[derive(Debug)] pub enum MethodError { InvalidArguments(String), RequestTooLarge, StateMismatch, AnchorNotFound, UnsupportedFilter(String), UnsupportedSort(String), ServerFail(String), UnknownMethod(String), ServerUnavailable, ServerPartialFail, InvalidResultReference(String), Forbidden(String), AccountNotFound, AccountNotSupportedByMethod, AccountReadOnly, NotFound, CannotCalculateChanges, UnknownDataType, } #[derive(Debug)] pub struct MethodErrorWrapper(trc::Error); impl From for MethodErrorWrapper { fn from(value: trc::Error) -> Self { MethodErrorWrapper(value) } } impl Display for MethodError { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { MethodError::InvalidArguments(err) => write!(f, "Invalid arguments: {}", err), MethodError::RequestTooLarge => write!(f, "Request too large"), MethodError::StateMismatch => write!(f, "State mismatch"), MethodError::AnchorNotFound => write!(f, "Anchor not found"), MethodError::UnsupportedFilter(err) => write!(f, "Unsupported filter: {}", err), MethodError::UnsupportedSort(err) => write!(f, "Unsupported sort: {}", err), MethodError::ServerFail(err) => write!(f, "Server error: {}", err), MethodError::UnknownMethod(err) => write!(f, "Unknown method: {}", err), MethodError::ServerUnavailable => write!(f, "Server unavailable"), MethodError::ServerPartialFail => write!(f, "Server partial fail"), MethodError::InvalidResultReference(err) => { write!(f, "Invalid result reference: {}", err) } MethodError::Forbidden(err) => write!(f, "Forbidden: {}", err), MethodError::AccountNotFound => write!(f, "Account not found"), MethodError::AccountNotSupportedByMethod => { write!(f, "Account not supported by method") } MethodError::AccountReadOnly => write!(f, "Account read only"), MethodError::NotFound => write!(f, "Not found"), MethodError::UnknownDataType => write!(f, "Unknown data type"), MethodError::CannotCalculateChanges => write!(f, "Cannot calculate changes"), } } } impl Serialize for MethodErrorWrapper { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { let mut map = serializer.serialize_map(2.into())?; let description = self.0.value(trc::Key::Details).and_then(|v| v.as_str()); let (error_type, description) = match self.0.as_ref() { trc::EventType::Jmap(cause) => match cause { trc::JmapEvent::InvalidArguments => { ("invalidArguments", description.unwrap_or_default()) } trc::JmapEvent::RequestTooLarge => ( "requestTooLarge", concat!( "The number of ids requested by the client exceeds the maximum number ", "the server is willing to process in a single method call." ), ), trc::JmapEvent::StateMismatch => ( "stateMismatch", concat!( "An \"ifInState\" argument was supplied, but ", "it does not match the current state." ), ), trc::JmapEvent::AnchorNotFound => ( "anchorNotFound", concat!( "An anchor argument was supplied, but it ", "cannot be found in the results of the query." ), ), trc::JmapEvent::UnsupportedFilter => { ("unsupportedFilter", description.unwrap_or_default()) } trc::JmapEvent::UnsupportedSort => { ("unsupportedSort", description.unwrap_or_default()) } trc::JmapEvent::NotFound => ("serverPartialFail", { concat!( "One or more items are no longer available on the ", "server, please try again." ) }), trc::JmapEvent::UnknownMethod => ("unknownMethod", description.unwrap_or_default()), trc::JmapEvent::InvalidResultReference => { ("invalidResultReference", description.unwrap_or_default()) } trc::JmapEvent::Forbidden => ("forbidden", description.unwrap_or_default()), trc::JmapEvent::AccountNotFound => ( "accountNotFound", "The accountId does not correspond to a valid account", ), trc::JmapEvent::AccountNotSupportedByMethod => ( "accountNotSupportedByMethod", concat!( "The accountId given corresponds to a valid account, ", "but the account does not support this method or data type." ), ), trc::JmapEvent::AccountReadOnly => ( "accountReadOnly", "This method modifies state, but the account is read-only.", ), trc::JmapEvent::UnknownDataType => ( "unknownDataType", concat!( "The server does not recognise this data type, ", "or the capability to enable it is not present ", "in the current Request Object." ), ), trc::JmapEvent::CannotCalculateChanges => ( "cannotCalculateChanges", concat!( "The server cannot calculate the changes ", "between the old and new states." ), ), trc::JmapEvent::UnknownCapability | trc::JmapEvent::NotJson | trc::JmapEvent::NotRequest => ( "serverUnavailable", concat!( "This server is temporarily unavailable. ", "Attempting this same operation later may succeed." ), ), _ => ( "serverUnavailable", "This server is temporarily unavailable.", ), }, _ => ( "serverUnavailable", concat!( "This server is temporarily unavailable. ", "Attempting this same operation later may succeed." ), ), }; map.serialize_entry("type", error_type)?; if !description.is_empty() { map.serialize_entry("description", description)?; } map.end() } } ================================================ FILE: crates/jmap-proto/src/error/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod method; pub mod request; pub mod set; ================================================ FILE: crates/jmap-proto/src/error/request.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{borrow::Cow, fmt::Display}; #[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize)] pub enum RequestLimitError { #[serde(rename = "maxSizeRequest")] SizeRequest, #[serde(rename = "maxSizeUpload")] SizeUpload, #[serde(rename = "maxCallsInRequest")] CallsIn, #[serde(rename = "maxConcurrentRequests")] ConcurrentRequest, #[serde(rename = "maxConcurrentUpload")] ConcurrentUpload, } #[derive(Debug, serde::Serialize, serde::Deserialize)] pub enum RequestErrorType { #[serde(rename = "urn:ietf:params:jmap:error:unknownCapability")] UnknownCapability, #[serde(rename = "urn:ietf:params:jmap:error:notJSON")] NotJSON, #[serde(rename = "urn:ietf:params:jmap:error:notRequest")] NotRequest, #[serde(rename = "urn:ietf:params:jmap:error:limit")] Limit, #[serde(rename = "about:blank")] Other, } #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct RequestError<'x> { #[serde(rename = "type")] pub p_type: RequestErrorType, pub status: u16, #[serde(skip_serializing_if = "Option::is_none")] pub title: Option>, pub detail: Cow<'x, str>, #[serde(skip_serializing_if = "Option::is_none")] pub limit: Option, } impl<'x> RequestError<'x> { pub fn blank( status: u16, title: impl Into>, detail: impl Into>, ) -> Self { RequestError { p_type: RequestErrorType::Other, status, title: Some(title.into()), detail: detail.into(), limit: None, } } pub fn internal_server_error() -> Self { RequestError::blank( 500, "Internal Server Error", concat!( "There was a problem while processing your request. ", "Please contact the system administrator if this problem persists." ), ) } pub fn unavailable() -> Self { RequestError::blank( 503, "Temporarily Unavailable", concat!( "There was a temporary problem while processing your request. ", "Please try again in a few moments." ), ) } pub fn invalid_parameters() -> Self { RequestError::blank( 400, "Invalid Parameters", "One or multiple parameters could not be parsed.", ) } pub fn forbidden() -> Self { RequestError::blank( 403, "Forbidden", "You do not have enough permissions to access this resource.", ) } pub fn over_blob_quota(max_files: usize, max_bytes: usize) -> Self { RequestError::blank( 403, "Quota exceeded", format!( "You have exceeded the blob upload quota of {} files or {} bytes.", max_files, max_bytes ), ) } pub fn over_quota() -> Self { RequestError::blank( 403, "Quota exceeded", "You have exceeded your account quota.", ) } pub fn tenant_over_quota() -> Self { RequestError::blank( 403, "Tenant quota exceeded", "Your organization has exceeded its quota.", ) } pub fn too_many_requests() -> Self { RequestError::blank( 429, "Too Many Requests", "Your request has been rate limited. Please try again in a few seconds.", ) } pub fn too_many_auth_attempts() -> Self { RequestError::blank( 429, "Too Many Authentication Attempts", "Your request has been rate limited. Please try again in a few minutes.", ) } pub fn limit(limit_type: RequestLimitError) -> Self { RequestError { p_type: RequestErrorType::Limit, status: 400, title: None, detail: match limit_type { RequestLimitError::SizeRequest => concat!( "The request is larger than the server ", "is willing to process." ), RequestLimitError::SizeUpload => concat!( "The uploaded file is larger than the server ", "is willing to process." ), RequestLimitError::CallsIn => concat!( "The request exceeds the maximum number ", "of calls in a single request." ), RequestLimitError::ConcurrentRequest => concat!( "The request exceeds the maximum number ", "of concurrent requests." ), RequestLimitError::ConcurrentUpload => concat!( "The request exceeds the maximum number ", "of concurrent uploads." ), } .into(), limit: Some(limit_type), } } pub fn not_found() -> Self { RequestError::blank( 404, "Not Found", "The requested resource does not exist on this server.", ) } pub fn unauthorized() -> Self { RequestError::blank(401, "Unauthorized", "You have to authenticate first.") } pub fn unknown_capability(capability: &'_ str) -> RequestError<'_> { RequestError { p_type: RequestErrorType::UnknownCapability, limit: None, title: None, status: 400, detail: format!( concat!( "The Request object used capability ", "'{}', which is not supported", "by this server." ), capability ) .into(), } } pub fn not_json(detail: &'_ str) -> RequestError<'_> { RequestError { p_type: RequestErrorType::NotJSON, limit: None, title: None, status: 400, detail: format!("Failed to parse JSON: {detail}").into(), } } pub fn not_request(detail: impl Into>) -> RequestError<'x> { RequestError { p_type: RequestErrorType::NotRequest, limit: None, title: None, status: 400, detail: detail.into(), } } } impl Display for RequestError<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(&self.detail) } } ================================================ FILE: crates/jmap-proto/src/error/set.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use jmap_tools::{Key, Property}; use std::borrow::Cow; use types::id::Id; #[derive(Debug, Clone, serde::Serialize)] #[serde(bound(serialize = "InvalidProperty

(key: &Key<'_, Self::Property>, value: &str) -> Option { if let Key::Property(prop) = key { match prop.patch_or_prop() { AddressBookProperty::Id => match parse_ref(value) { MaybeReference::Value(v) => Some(AddressBookValue::Id(v)), MaybeReference::Reference(v) => Some(AddressBookValue::IdReference(v)), MaybeReference::ParseError => None, }, _ => None, } } else { None } } fn to_cow(&self) -> Cow<'static, str> { match self { AddressBookValue::Id(id) => id.to_string().into(), AddressBookValue::IdReference(r) => format!("#{r}").into(), AddressBookValue::Role(special_use) => special_use.as_str().unwrap_or_default().into(), } } } impl AddressBookProperty { fn parse(value: &str, allow_patch: bool) -> Option { hashify::tiny_map!(value.as_bytes(), b"id" => AddressBookProperty::Id, b"name" => AddressBookProperty::Name, b"description" => AddressBookProperty::Description, b"sortOrder" => AddressBookProperty::SortOrder, b"isDefault" => AddressBookProperty::IsDefault, b"isSubscribed" => AddressBookProperty::IsSubscribed, b"shareWith" => AddressBookProperty::ShareWith, b"myRights" => AddressBookProperty::MyRights, b"mayRead" => AddressBookProperty::Rights(AddressBookRight::MayRead), b"mayWrite" => AddressBookProperty::Rights(AddressBookRight::MayWrite), b"mayShare" => AddressBookProperty::Rights(AddressBookRight::MayShare), b"mayDelete" => AddressBookProperty::Rights(AddressBookRight::MayDelete) ) .or_else(|| { if allow_patch && value.contains('/') { AddressBookProperty::Pointer(JsonPointer::parse(value)).into() } else { None } }) } fn patch_or_prop(&self) -> &AddressBookProperty { if let AddressBookProperty::Pointer(ptr) = self && let Some(JsonPointerItem::Key(Key::Property(prop))) = ptr.last() { prop } else { self } } } #[derive(Debug, Clone, Default)] pub struct AddressBookSetArguments { pub on_destroy_remove_contents: Option, pub on_success_set_is_default: Option>, } impl<'de> DeserializeArguments<'de> for AddressBookSetArguments { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"onDestroyRemoveContents" => { self.on_destroy_remove_contents = map.next_value()?; }, b"onSuccessSetIsDefault" => { self.on_success_set_is_default = map.next_value()?; }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl FromStr for AddressBookProperty { type Err = (); fn from_str(s: &str) -> Result { AddressBookProperty::parse(s, false).ok_or(()) } } impl JmapObject for AddressBook { type Property = AddressBookProperty; type Element = AddressBookValue; type Id = Id; type Filter = (); type Comparator = (); type GetArguments = (); type SetArguments<'de> = AddressBookSetArguments; type QueryArguments = (); type CopyArguments = (); type ParseArguments = (); const ID_PROPERTY: Self::Property = AddressBookProperty::Id; } impl JmapSharedObject for AddressBook { type Right = AddressBookRight; const SHARE_WITH_PROPERTY: Self::Property = AddressBookProperty::ShareWith; } impl From for AddressBookProperty { fn from(id: Id) -> Self { AddressBookProperty::IdValue(id) } } impl TryFrom for Id { type Error = (); fn try_from(value: AddressBookProperty) -> Result { if let AddressBookProperty::IdValue(id) = value { Ok(id) } else { Err(()) } } } impl TryFrom for AddressBookRight { type Error = (); fn try_from(value: AddressBookProperty) -> Result { if let AddressBookProperty::Rights(right) = value { Ok(right) } else { Err(()) } } } impl From for AddressBookValue { fn from(id: Id) -> Self { AddressBookValue::Id(id) } } impl JmapObjectId for AddressBookValue { fn as_id(&self) -> Option { if let AddressBookValue::Id(id) = self { Some(*id) } else { None } } fn as_any_id(&self) -> Option { if let AddressBookValue::Id(id) = self { Some(AnyId::Id(*id)) } else { None } } fn as_id_ref(&self) -> Option<&str> { if let AddressBookValue::IdReference(r) = self { Some(r) } else { None } } fn try_set_id(&mut self, new_id: AnyId) -> bool { if let AnyId::Id(new_id) = new_id { *self = AddressBookValue::Id(new_id); return true; } false } } impl JmapRight for AddressBookRight { fn to_acl(&self) -> &'static [Acl] { match self { AddressBookRight::MayDelete => &[Acl::Delete, Acl::RemoveItems], AddressBookRight::MayShare => &[Acl::Share], AddressBookRight::MayRead => &[Acl::Read, Acl::ReadItems], AddressBookRight::MayWrite => &[Acl::Modify, Acl::AddItems, Acl::ModifyItems], } } fn all_rights() -> &'static [Self] { &[ AddressBookRight::MayRead, AddressBookRight::MayWrite, AddressBookRight::MayDelete, AddressBookRight::MayShare, ] } } impl From for AddressBookProperty { fn from(right: AddressBookRight) -> Self { AddressBookProperty::Rights(right) } } impl JmapObjectId for AddressBookProperty { fn as_id(&self) -> Option { if let AddressBookProperty::IdValue(id) = self { Some(*id) } else { None } } fn as_any_id(&self) -> Option { if let AddressBookProperty::IdValue(id) = self { Some(AnyId::Id(*id)) } else { None } } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, new_id: AnyId) -> bool { if let AnyId::Id(new_id) = new_id { *self = AddressBookProperty::IdValue(new_id); return true; } false } } impl std::fmt::Display for AddressBookProperty { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.to_cow()) } } ================================================ FILE: crates/jmap-proto/src/object/blob.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ object::{AnyId, JmapObject, JmapObjectId, MaybeReference, parse_ref}, request::deserialize::DeserializeArguments, }; use jmap_tools::{Element, Key, Property}; use std::{borrow::Cow, str::FromStr}; use types::{blob::BlobId, id::Id}; #[derive(Debug, Clone, Default)] pub struct Blob; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum BlobProperty { Id, BlobId, Type, Size, Digest(DigestProperty), Data(DataProperty), IsEncodingProblem, IsTruncated, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum DigestProperty { Sha, Sha256, Sha512, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum DataProperty { AsText, AsBase64, Default, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum BlobValue { BlobId(BlobId), IdReference(String), } impl Property for BlobProperty { fn try_parse(_: Option<&Key<'_, Self>>, value: &str) -> Option { BlobProperty::parse(value) } fn to_cow(&self) -> Cow<'static, str> { match self { BlobProperty::BlobId => "blobId", BlobProperty::Id => "id", BlobProperty::Size => "size", BlobProperty::Type => "type", BlobProperty::IsEncodingProblem => "isEncodingProblem", BlobProperty::IsTruncated => "isTruncated", BlobProperty::Data(data) => match data { DataProperty::AsText => "data:asText", DataProperty::AsBase64 => "data:asBase64", DataProperty::Default => "data", }, BlobProperty::Digest(digest) => match digest { DigestProperty::Sha => "digest:sha", DigestProperty::Sha256 => "digest:sha-256", DigestProperty::Sha512 => "digest:sha-512", }, } .into() } } impl Element for BlobValue { type Property = BlobProperty; fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option { if let Key::Property(prop) = key { match prop { BlobProperty::BlobId => match parse_ref(value) { MaybeReference::Value(v) => Some(BlobValue::BlobId(v)), MaybeReference::Reference(v) => Some(BlobValue::IdReference(v)), MaybeReference::ParseError => None, }, _ => None, } } else { None } } fn to_cow(&self) -> Cow<'static, str> { match self { BlobValue::BlobId(blob_id) => blob_id.to_string().into(), BlobValue::IdReference(r) => format!("#{r}").into(), } } } impl BlobProperty { fn parse(value: &str) -> Option { hashify::tiny_map!(value.as_bytes(), b"blobId" => BlobProperty::BlobId, b"id" => BlobProperty::Id, b"size" => BlobProperty::Size, b"type" => BlobProperty::Type, b"isEncodingProblem" => BlobProperty::IsEncodingProblem, b"isTruncated" => BlobProperty::IsTruncated, b"data:asText" => BlobProperty::Data(DataProperty::AsText), b"data:asBase64" => BlobProperty::Data(DataProperty::AsBase64), b"data" => BlobProperty::Data(DataProperty::Default), b"digest:sha" => BlobProperty::Digest(DigestProperty::Sha), b"digest:sha-256" => BlobProperty::Digest(DigestProperty::Sha256), b"digest:sha-512" => BlobProperty::Digest(DigestProperty::Sha512), ) } } impl FromStr for BlobProperty { type Err = (); fn from_str(s: &str) -> Result { BlobProperty::parse(s).ok_or(()) } } #[derive(Debug, Clone, Default)] pub struct BlobGetArguments { pub offset: Option, pub length: Option, } impl<'de> DeserializeArguments<'de> for BlobGetArguments { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"offset" => { self.offset = map.next_value()?; }, b"length" => { self.length = map.next_value()?; }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl JmapObject for Blob { type Property = BlobProperty; type Element = BlobValue; type Id = BlobId; type Filter = (); type Comparator = (); type GetArguments = BlobGetArguments; type SetArguments<'de> = (); type QueryArguments = (); type CopyArguments = (); type ParseArguments = (); const ID_PROPERTY: Self::Property = BlobProperty::Id; } impl From for BlobValue { fn from(id: BlobId) -> Self { BlobValue::BlobId(id) } } impl JmapObjectId for BlobValue { fn as_id(&self) -> Option { None } fn as_any_id(&self) -> Option { match self { BlobValue::BlobId(id) => Some(AnyId::BlobId(id.clone())), _ => None, } } fn as_id_ref(&self) -> Option<&str> { if let BlobValue::IdReference(r) = self { Some(r) } else { None } } fn try_set_id(&mut self, new_id: AnyId) -> bool { if let AnyId::BlobId(id) = new_id { *self = BlobValue::BlobId(id); return true; } false } } impl JmapObjectId for BlobProperty { fn as_id(&self) -> Option { None } fn as_any_id(&self) -> Option { None } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, _: AnyId) -> bool { false } } ================================================ FILE: crates/jmap-proto/src/object/calendar.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ object::{ AnyId, JmapObject, JmapObjectId, JmapRight, JmapSharedObject, MaybeReference, parse_ref, }, request::{deserialize::DeserializeArguments, reference::MaybeIdReference}, types::date::UTCDate, }; use calcard::{ common::{IanaParse, timezone::Tz}, icalendar::ICalendarDuration, jscalendar::{JSCalendarAlertAction, JSCalendarRelativeTo, JSCalendarType}, }; use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, Property}; use std::{borrow::Cow, fmt::Display, str::FromStr}; use types::{acl::Acl, id::Id}; #[derive(Debug, Clone, Default)] pub struct Calendar; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum CalendarProperty { Id, Name, Description, Color, SortOrder, IsSubscribed, IsVisible, IsDefault, IncludeInAvailability, DefaultAlertsWithTime, DefaultAlertsWithoutTime, TimeZone, ShareWith, MyRights, // Alert object properties When, Trigger, Offset, RelativeTo, Action, Type, // Other IdValue(Id), Rights(CalendarRight), Pointer(JsonPointer), } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum CalendarRight { MayReadFreeBusy, MayReadItems, MayWriteAll, MayWriteOwn, MayUpdatePrivate, MayRSVP, MayShare, MayDelete, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum CalendarValue { Id(Id), IdReference(String), IncludeInAvailability(IncludeInAvailability), Date(UTCDate), Timezone(Tz), Action(JSCalendarAlertAction), RelativeTo(JSCalendarRelativeTo), Type(JSCalendarType), Duration(ICalendarDuration), } #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum IncludeInAvailability { #[default] All, Attending, None, } impl Property for CalendarProperty { fn try_parse(key: Option<&Key<'_, Self>>, value: &str) -> Option { let allow_patch = key.is_none(); if let Some(Key::Property(key)) = key { match key.patch_or_prop() { CalendarProperty::ShareWith => { Id::from_str(value).ok().map(CalendarProperty::IdValue) } _ => CalendarProperty::parse(value, allow_patch), } } else { CalendarProperty::parse(value, allow_patch) } } fn to_cow(&self) -> Cow<'static, str> { match self { CalendarProperty::Id => "id", CalendarProperty::Name => "name", CalendarProperty::Description => "description", CalendarProperty::Color => "color", CalendarProperty::SortOrder => "sortOrder", CalendarProperty::IsSubscribed => "isSubscribed", CalendarProperty::IsVisible => "isVisible", CalendarProperty::IsDefault => "isDefault", CalendarProperty::IncludeInAvailability => "includeInAvailability", CalendarProperty::DefaultAlertsWithTime => "defaultAlertsWithTime", CalendarProperty::DefaultAlertsWithoutTime => "defaultAlertsWithoutTime", CalendarProperty::TimeZone => "timeZone", CalendarProperty::ShareWith => "shareWith", CalendarProperty::MyRights => "myRights", CalendarProperty::When => "when", CalendarProperty::Trigger => "trigger", CalendarProperty::Offset => "offset", CalendarProperty::RelativeTo => "relativeTo", CalendarProperty::Action => "action", CalendarProperty::Type => "@type", CalendarProperty::Rights(calendar_right) => calendar_right.as_str(), CalendarProperty::Pointer(json_pointer) => return json_pointer.to_string().into(), CalendarProperty::IdValue(id) => return id.to_string().into(), } .into() } } impl CalendarRight { pub fn as_str(&self) -> &'static str { match self { CalendarRight::MayReadFreeBusy => "mayReadFreeBusy", CalendarRight::MayReadItems => "mayReadItems", CalendarRight::MayWriteAll => "mayWriteAll", CalendarRight::MayWriteOwn => "mayWriteOwn", CalendarRight::MayUpdatePrivate => "mayUpdatePrivate", CalendarRight::MayRSVP => "mayRSVP", CalendarRight::MayShare => "mayShare", CalendarRight::MayDelete => "mayDelete", } } } impl IncludeInAvailability { fn parse(value: &str) -> Option { hashify::tiny_map!(value.as_bytes(), b"all" => IncludeInAvailability::All, b"attending" => IncludeInAvailability::Attending, b"none" => IncludeInAvailability::None, ) } pub fn as_str(&self) -> &'static str { match self { IncludeInAvailability::All => "all", IncludeInAvailability::Attending => "attending", IncludeInAvailability::None => "none", } } } impl Element for CalendarValue { type Property = CalendarProperty; fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option { if let Key::Property(prop) = key { match prop.patch_or_prop() { CalendarProperty::Id => match parse_ref(value) { MaybeReference::Value(v) => Some(CalendarValue::Id(v)), MaybeReference::Reference(v) => Some(CalendarValue::IdReference(v)), MaybeReference::ParseError => None, }, CalendarProperty::TimeZone => Tz::from_str(value).ok().map(CalendarValue::Timezone), CalendarProperty::IncludeInAvailability => { IncludeInAvailability::parse(value).map(CalendarValue::IncludeInAvailability) } CalendarProperty::Action => JSCalendarAlertAction::from_str(value) .ok() .map(CalendarValue::Action), CalendarProperty::RelativeTo => JSCalendarRelativeTo::from_str(value) .ok() .map(CalendarValue::RelativeTo), CalendarProperty::When => UTCDate::from_str(value).ok().map(CalendarValue::Date), CalendarProperty::Offset => { ICalendarDuration::parse(value.as_bytes()).map(CalendarValue::Duration) } _ => None, } } else { None } } fn to_cow(&self) -> Cow<'static, str> { match self { CalendarValue::Id(id) => id.to_string().into(), CalendarValue::IdReference(r) => format!("#{r}").into(), CalendarValue::IncludeInAvailability(include) => include.as_str().into(), CalendarValue::Date(date) => date.to_string().into(), CalendarValue::Action(action) => action.as_str().into(), CalendarValue::RelativeTo(relative) => relative.as_str().into(), CalendarValue::Type(typ) => typ.as_str().into(), CalendarValue::Duration(dur) => dur.to_string().into(), CalendarValue::Timezone(tz) => tz.name().unwrap_or_default(), } } } impl CalendarProperty { fn parse(value: &str, allow_patch: bool) -> Option { hashify::tiny_map!(value.as_bytes(), b"id" => CalendarProperty::Id, b"name" => CalendarProperty::Name, b"description" => CalendarProperty::Description, b"color" => CalendarProperty::Color, b"sortOrder" => CalendarProperty::SortOrder, b"isSubscribed" => CalendarProperty::IsSubscribed, b"isVisible" => CalendarProperty::IsVisible, b"isDefault" => CalendarProperty::IsDefault, b"includeInAvailability" => CalendarProperty::IncludeInAvailability, b"defaultAlertsWithTime" => CalendarProperty::DefaultAlertsWithTime, b"defaultAlertsWithoutTime" => CalendarProperty::DefaultAlertsWithoutTime, b"timeZone" => CalendarProperty::TimeZone, b"shareWith" => CalendarProperty::ShareWith, b"myRights" => CalendarProperty::MyRights, b"mayReadFreeBusy" => CalendarProperty::Rights(CalendarRight::MayReadFreeBusy), b"mayReadItems" => CalendarProperty::Rights(CalendarRight::MayReadItems), b"mayWriteAll" => CalendarProperty::Rights(CalendarRight::MayWriteAll), b"mayWriteOwn" => CalendarProperty::Rights(CalendarRight::MayWriteOwn), b"mayUpdatePrivate" => CalendarProperty::Rights(CalendarRight::MayUpdatePrivate), b"mayRSVP" => CalendarProperty::Rights(CalendarRight::MayRSVP), b"mayShare" => CalendarProperty::Rights(CalendarRight::MayShare), b"mayDelete" => CalendarProperty::Rights(CalendarRight::MayDelete), b"@type" => CalendarProperty::Type, b"when" => CalendarProperty::When, b"trigger" => CalendarProperty::Trigger, b"offset" => CalendarProperty::Offset, b"relativeTo" => CalendarProperty::RelativeTo, b"action" => CalendarProperty::Action, ) .or_else(|| { if allow_patch && value.contains('/') { CalendarProperty::Pointer(JsonPointer::parse(value)).into() } else { None } }) } fn patch_or_prop(&self) -> &CalendarProperty { if let CalendarProperty::Pointer(ptr) = self && let Some(JsonPointerItem::Key(Key::Property(prop))) = ptr.last() { prop } else { self } } } #[derive(Debug, Clone, Default)] pub struct CalendarSetArguments { pub on_destroy_remove_events: Option, pub on_success_set_is_default: Option>, } impl<'de> DeserializeArguments<'de> for CalendarSetArguments { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"onDestroyRemoveEvents" => { self.on_destroy_remove_events = map.next_value()?; }, b"onSuccessSetIsDefault" => { self.on_success_set_is_default = map.next_value()?; }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl FromStr for CalendarProperty { type Err = (); fn from_str(s: &str) -> Result { CalendarProperty::parse(s, false).ok_or(()) } } impl JmapObject for Calendar { type Property = CalendarProperty; type Element = CalendarValue; type Id = Id; type Filter = (); type Comparator = (); type GetArguments = (); type SetArguments<'de> = CalendarSetArguments; type QueryArguments = (); type CopyArguments = (); type ParseArguments = (); const ID_PROPERTY: Self::Property = CalendarProperty::Id; } impl JmapSharedObject for Calendar { type Right = CalendarRight; const SHARE_WITH_PROPERTY: Self::Property = CalendarProperty::ShareWith; } impl From for CalendarProperty { fn from(id: Id) -> Self { CalendarProperty::IdValue(id) } } impl TryFrom for Id { type Error = (); fn try_from(value: CalendarProperty) -> Result { if let CalendarProperty::IdValue(id) = value { Ok(id) } else { Err(()) } } } impl TryFrom for CalendarRight { type Error = (); fn try_from(value: CalendarProperty) -> Result { if let CalendarProperty::Rights(right) = value { Ok(right) } else { Err(()) } } } impl From for CalendarValue { fn from(id: Id) -> Self { CalendarValue::Id(id) } } impl JmapObjectId for CalendarValue { fn as_id(&self) -> Option { if let CalendarValue::Id(id) = self { Some(*id) } else { None } } fn as_any_id(&self) -> Option { if let CalendarValue::Id(id) = self { Some(AnyId::Id(*id)) } else { None } } fn as_id_ref(&self) -> Option<&str> { if let CalendarValue::IdReference(r) = self { Some(r) } else { None } } fn try_set_id(&mut self, new_id: AnyId) -> bool { if let AnyId::Id(new_id) = new_id { *self = CalendarValue::Id(new_id); return true; } false } } impl JmapRight for CalendarRight { fn to_acl(&self) -> &'static [Acl] { match self { CalendarRight::MayReadFreeBusy => &[Acl::SchedulingReadFreeBusy], CalendarRight::MayReadItems => &[Acl::Read, Acl::ReadItems], CalendarRight::MayWriteAll => &[ Acl::Modify, Acl::AddItems, Acl::ModifyItems, Acl::RemoveItems, ], CalendarRight::MayWriteOwn => &[Acl::ModifyItemsOwn], CalendarRight::MayUpdatePrivate => &[Acl::ModifyPrivateProperties], CalendarRight::MayRSVP => &[Acl::ModifyRSVP], CalendarRight::MayShare => &[Acl::Share], CalendarRight::MayDelete => &[Acl::Delete, Acl::RemoveItems], } } fn all_rights() -> &'static [Self] { &[ CalendarRight::MayReadFreeBusy, CalendarRight::MayReadItems, CalendarRight::MayWriteAll, CalendarRight::MayWriteOwn, CalendarRight::MayUpdatePrivate, CalendarRight::MayRSVP, CalendarRight::MayShare, CalendarRight::MayDelete, ] } } impl From for CalendarProperty { fn from(right: CalendarRight) -> Self { CalendarProperty::Rights(right) } } impl JmapObjectId for CalendarProperty { fn as_id(&self) -> Option { if let CalendarProperty::IdValue(id) = self { Some(*id) } else { None } } fn as_any_id(&self) -> Option { if let CalendarProperty::IdValue(id) = self { Some(AnyId::Id(*id)) } else { None } } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, new_id: AnyId) -> bool { if let AnyId::Id(new_id) = new_id { *self = CalendarProperty::IdValue(new_id); return true; } false } } impl Display for CalendarProperty { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.to_cow()) } } ================================================ FILE: crates/jmap-proto/src/object/calendar_event.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ object::{AnyId, JmapObject, JmapObjectId}, request::{MaybeInvalid, deserialize::DeserializeArguments}, }; use calcard::{ common::timezone::Tz, jscalendar::{JSCalendarDateTime, JSCalendarProperty, JSCalendarValue}, }; use jmap_tools::{JsonPointerItem, Key}; use mail_parser::DateTime; use std::{borrow::Cow, str::FromStr}; use types::{blob::BlobId, id::Id}; #[derive(Debug, Clone, Default)] pub struct CalendarEvent; impl JmapObject for CalendarEvent { type Property = JSCalendarProperty; type Element = JSCalendarValue; type Id = Id; type Filter = CalendarEventFilter; type Comparator = CalendarEventComparator; type GetArguments = CalendarEventGetArguments; type SetArguments<'de> = CalendarEventSetArguments; type QueryArguments = CalendarEventQueryArguments; type CopyArguments = (); type ParseArguments = (); const ID_PROPERTY: Self::Property = JSCalendarProperty::Id; } impl JmapObjectId for JSCalendarValue { fn as_id(&self) -> Option { if let JSCalendarValue::Id(id) = self { Some(*id) } else { None } } fn as_any_id(&self) -> Option { match self { JSCalendarValue::Id(id) => Some(AnyId::Id(*id)), JSCalendarValue::BlobId(blob_id) => Some(AnyId::BlobId(blob_id.clone())), _ => None, } } fn as_id_ref(&self) -> Option<&str> { match self { JSCalendarValue::IdReference(r) => Some(r), _ => None, } } fn try_set_id(&mut self, new_id: AnyId) -> bool { if let AnyId::Id(id) = new_id { *self = JSCalendarValue::Id(id); true } else { false } } } #[derive(Debug, Clone, PartialEq, Eq)] pub enum CalendarEventFilter { InCalendar(MaybeInvalid), After(JSCalendarDateTime), Before(JSCalendarDateTime), Text(String), Title(String), Description(String), Location(String), Owner(String), Attendee(String), Uid(String), _T(String), } #[derive(Debug, Clone, PartialEq, Eq)] pub enum CalendarEventComparator { Start, Uid, RecurrenceId, Created, Updated, _T(String), } #[derive(Debug, Clone, Default)] pub struct CalendarEventGetArguments { pub recurrence_overrides_before: Option, pub recurrence_overrides_after: Option, pub reduce_participants: Option, pub time_zone: Option, } #[derive(Debug, Clone, Default)] pub struct CalendarEventSetArguments { pub send_scheduling_messages: Option, } #[derive(Debug, Clone, Default)] pub struct CalendarEventQueryArguments { pub expand_recurrences: Option, pub time_zone: Option, } impl<'de> DeserializeArguments<'de> for CalendarEventFilter { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"inCalendar" => { *self = CalendarEventFilter::InCalendar(map.next_value()?); }, b"after" => { *self = CalendarEventFilter::After(map.next_value::()?.0); }, b"before" => { *self = CalendarEventFilter::Before(map.next_value::()?.0); }, b"text" => { *self = CalendarEventFilter::Text(map.next_value::>()?.to_lowercase()); }, b"title" => { *self = CalendarEventFilter::Title(map.next_value::>()?.to_lowercase()); }, b"description" => { *self = CalendarEventFilter::Description(map.next_value::>()?.to_lowercase()); }, b"location" => { *self = CalendarEventFilter::Location(map.next_value::>()?.to_lowercase()); }, b"owner" => { *self = CalendarEventFilter::Owner(map.next_value::>()?.to_lowercase()); }, b"attendee" => { *self = CalendarEventFilter::Attendee(map.next_value::>()?.to_lowercase()); }, b"uid" => { *self = CalendarEventFilter::Uid(map.next_value()?); }, _ => { *self = CalendarEventFilter::_T(key.to_string()); let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> DeserializeArguments<'de> for CalendarEventComparator { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { if key == "property" { let value = map.next_value::>()?; hashify::fnc_map!(value.as_bytes(), b"start" => { *self = CalendarEventComparator::Start; }, b"uid" => { *self = CalendarEventComparator::Uid; }, b"recurrenceId" => { *self = CalendarEventComparator::RecurrenceId; }, b"created" => { *self = CalendarEventComparator::Created; }, b"updated" => { *self = CalendarEventComparator::Updated; }, _ => { *self = CalendarEventComparator::_T(value.to_string()); } ); } else { let _ = map.next_value::()?; } Ok(()) } } impl<'de> DeserializeArguments<'de> for CalendarEventGetArguments { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"recurrenceOverridesBefore" => { self.recurrence_overrides_before = map.next_value::>()?.map(|lt| lt.0) }, b"recurrenceOverridesAfter" => { self.recurrence_overrides_after = map.next_value::>()?.map(|lt| lt.0); }, b"reduceParticipants" => { self.reduce_participants = map.next_value()?; }, b"timeZone" => { self.time_zone = map.next_value::>()?.and_then(|s| Tz::from_str(s).ok()); }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> DeserializeArguments<'de> for CalendarEventSetArguments { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"sendSchedulingMessages" => { self.send_scheduling_messages = map.next_value()?; }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> DeserializeArguments<'de> for CalendarEventQueryArguments { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"expandRecurrences" => { self.expand_recurrences = map.next_value()?; }, b"timeZone" => { self.time_zone = map.next_value::>()?.and_then(|s| Tz::from_str(s).ok()); }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl CalendarEventFilter { pub fn into_string(self) -> Cow<'static, str> { match self { CalendarEventFilter::InCalendar(_) => "inCalendar", CalendarEventFilter::After(_) => "after", CalendarEventFilter::Before(_) => "before", CalendarEventFilter::Text(_) => "text", CalendarEventFilter::Title(_) => "title", CalendarEventFilter::Description(_) => "description", CalendarEventFilter::Location(_) => "location", CalendarEventFilter::Owner(_) => "owner", CalendarEventFilter::Attendee(_) => "attendee", CalendarEventFilter::Uid(_) => "uid", CalendarEventFilter::_T(s) => return Cow::Owned(s), } .into() } } impl CalendarEventComparator { pub fn into_string(self) -> Cow<'static, str> { match self { CalendarEventComparator::Start => "start", CalendarEventComparator::Uid => "uid", CalendarEventComparator::RecurrenceId => "recurrenceId", CalendarEventComparator::Created => "created", CalendarEventComparator::Updated => "updated", CalendarEventComparator::_T(s) => return Cow::Owned(s), } .into() } } impl Default for CalendarEventFilter { fn default() -> Self { CalendarEventFilter::_T(String::new()) } } impl Default for CalendarEventComparator { fn default() -> Self { CalendarEventComparator::_T(String::new()) } } struct LocalTime(JSCalendarDateTime); impl<'de> serde::Deserialize<'de> for LocalTime { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { let value = <&str>::deserialize(deserializer)?; if let Some(dt) = DateTime::parse_rfc3339(value) { Ok(LocalTime(JSCalendarDateTime { timestamp: dt.to_timestamp_local(), is_local: true, })) } else { Err(serde::de::Error::custom(format!( "Invalid datetime: {}", value ))) } } } impl JmapObjectId for JSCalendarProperty { fn as_id(&self) -> Option { if let JSCalendarProperty::IdValue(id) = self { Some(*id) } else { None } } fn as_any_id(&self) -> Option { if let JSCalendarProperty::IdValue(id) = self { Some(AnyId::Id(*id)) } else { None } } fn as_id_ref(&self) -> Option<&str> { match self { JSCalendarProperty::IdReference(r) => Some(r), JSCalendarProperty::Pointer(value) => { let value = value.as_slice(); match (value.first(), value.get(1)) { ( Some(JsonPointerItem::Key(Key::Property(JSCalendarProperty::CalendarIds))), Some(JsonPointerItem::Key(Key::Property(JSCalendarProperty::IdReference( r, )))), ) => Some(r), _ => None, } } _ => None, } } fn try_set_id(&mut self, new_id: AnyId) -> bool { if let AnyId::Id(id) = new_id { if let JSCalendarProperty::Pointer(value) = self { let value = value.as_mut_slice(); if let Some(value) = value.get_mut(1) { *value = JsonPointerItem::Key(Key::Property(JSCalendarProperty::IdValue(id))); return true; } } else { *self = JSCalendarProperty::IdValue(id); return true; } } false } } ================================================ FILE: crates/jmap-proto/src/object/calendar_event_notification.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ object::{AnyId, JmapObject, JmapObjectId}, request::{MaybeInvalid, deserialize::DeserializeArguments}, types::{date::UTCDate, state::State}, }; use calcard::jscalendar::JSCalendar; use jmap_tools::{Element, Key, Property}; use serde::Serialize; use std::{borrow::Cow, fmt::Display, str::FromStr}; use types::{blob::BlobId, id::Id}; #[derive(Debug, Clone, Default)] pub struct CalendarEventNotification; #[derive(Debug, Serialize, Clone, Default)] #[serde(rename_all = "camelCase")] pub struct CalendarEventNotificationObject { pub id: Id, #[serde(skip_serializing_if = "Option::is_none")] pub created: Option, #[serde(skip_serializing_if = "Option::is_none")] pub changed_by: Option, #[serde(skip_serializing_if = "Option::is_none")] pub comment: Option, #[serde(skip_serializing_if = "Option::is_none")] #[serde(rename = "type")] pub notification_type: Option, #[serde(skip_serializing_if = "Option::is_none")] pub calendar_event_id: Option, #[serde(skip_serializing_if = "Option::is_none")] pub is_draft: Option, #[serde(skip_serializing_if = "Option::is_none")] pub event: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub event_patch: Option>, } #[derive(Debug, Serialize, Clone, Default)] #[serde(rename_all = "camelCase")] pub struct PersonObject { pub name: String, #[serde(skip_serializing_if = "Option::is_none")] pub email: Option, #[serde(skip_serializing_if = "Option::is_none")] pub principal_id: Option, #[serde(skip_serializing_if = "Option::is_none")] pub calendar_address: Option, } #[derive(Debug, Clone, serde::Serialize)] pub struct CalendarEventNotificationGetResponse { #[serde(rename = "accountId")] #[serde(skip_serializing_if = "Option::is_none")] pub account_id: Option, #[serde(skip_serializing_if = "Option::is_none")] pub state: Option, pub list: Vec, #[serde(rename = "notFound")] pub not_found: Vec, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum CalendarEventNotificationProperty { Id, Created, ChangedBy, Comment, Type, CalendarEventId, IsDraft, Event, EventPatch, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum CalendarEventNotificationValue { Id(Id), Date(UTCDate), Type(CalendarEventNotificationType), } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum CalendarEventNotificationType { Created, Updated, Destroyed, } impl Property for CalendarEventNotificationProperty { fn try_parse(_: Option<&Key<'_, Self>>, value: &str) -> Option { CalendarEventNotificationProperty::parse(value) } fn to_cow(&self) -> Cow<'static, str> { match self { CalendarEventNotificationProperty::Id => "id", CalendarEventNotificationProperty::Created => "created", CalendarEventNotificationProperty::ChangedBy => "changedBy", CalendarEventNotificationProperty::Comment => "comment", CalendarEventNotificationProperty::Type => "type", CalendarEventNotificationProperty::CalendarEventId => "calendarEventId", CalendarEventNotificationProperty::IsDraft => "isDraft", CalendarEventNotificationProperty::Event => "event", CalendarEventNotificationProperty::EventPatch => "eventPatch", } .into() } } impl Element for CalendarEventNotificationValue { type Property = CalendarEventNotificationProperty; fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option { if let Key::Property(prop) = key { match prop { CalendarEventNotificationProperty::Id | CalendarEventNotificationProperty::CalendarEventId => Id::from_str(value) .ok() .map(CalendarEventNotificationValue::Id), CalendarEventNotificationProperty::Created => UTCDate::from_str(value) .ok() .map(CalendarEventNotificationValue::Date), CalendarEventNotificationProperty::Type => { CalendarEventNotificationType::parse(value) .map(CalendarEventNotificationValue::Type) } _ => None, } } else { None } } fn to_cow(&self) -> Cow<'static, str> { match self { CalendarEventNotificationValue::Id(id) => id.to_string().into(), CalendarEventNotificationValue::Date(date) => date.to_string().into(), CalendarEventNotificationValue::Type(t) => t.as_str().into(), } } } impl CalendarEventNotificationType { fn parse(value: &str) -> Option { hashify::tiny_map!(value.as_bytes(), b"created" => CalendarEventNotificationType::Created, b"updated" => CalendarEventNotificationType::Updated, b"destroyed" => CalendarEventNotificationType::Destroyed, ) } pub fn as_str(&self) -> &'static str { match self { CalendarEventNotificationType::Created => "created", CalendarEventNotificationType::Updated => "updated", CalendarEventNotificationType::Destroyed => "destroyed", } } } impl CalendarEventNotificationProperty { fn parse(value: &str) -> Option { hashify::tiny_map!(value.as_bytes(), b"id" => CalendarEventNotificationProperty::Id, b"created" => CalendarEventNotificationProperty::Created, b"changedBy" => CalendarEventNotificationProperty::ChangedBy, b"comment" => CalendarEventNotificationProperty::Comment, b"type" => CalendarEventNotificationProperty::Type, b"calendarEventId" => CalendarEventNotificationProperty::CalendarEventId, b"isDraft" => CalendarEventNotificationProperty::IsDraft, b"event" => CalendarEventNotificationProperty::Event, b"eventPatch" => CalendarEventNotificationProperty::EventPatch ) } } impl FromStr for CalendarEventNotificationProperty { type Err = (); fn from_str(s: &str) -> Result { CalendarEventNotificationProperty::parse(s).ok_or(()) } } impl JmapObject for CalendarEventNotification { type Property = CalendarEventNotificationProperty; type Element = CalendarEventNotificationValue; type Id = Id; type Filter = CalendarEventNotificationFilter; type Comparator = CalendarEventNotificationComparator; type GetArguments = (); type SetArguments<'de> = (); type QueryArguments = (); type CopyArguments = (); type ParseArguments = (); const ID_PROPERTY: Self::Property = CalendarEventNotificationProperty::Id; } #[derive(Debug, Clone, PartialEq, Eq)] pub enum CalendarEventNotificationFilter { After(UTCDate), Before(UTCDate), Type(CalendarEventNotificationType), CalendarEventIds(Vec>), _T(String), } #[derive(Debug, Clone, PartialEq, Eq)] pub enum CalendarEventNotificationComparator { Created, _T(String), } impl<'de> DeserializeArguments<'de> for CalendarEventNotificationFilter { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"after" => { *self = CalendarEventNotificationFilter::After(map.next_value()?); }, b"before" => { *self = CalendarEventNotificationFilter::Before(map.next_value()?); }, b"type" => { *self = CalendarEventNotificationFilter::Type(map.next_value()?); }, b"calendarEventIds" => { *self = CalendarEventNotificationFilter::CalendarEventIds(map.next_value()?); }, _ => { *self = CalendarEventNotificationFilter::_T(key.to_string()); let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> DeserializeArguments<'de> for CalendarEventNotificationComparator { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { if key == "property" { let value = map.next_value::>()?; hashify::fnc_map!(value.as_bytes(), b"created" => { *self = CalendarEventNotificationComparator::Created; }, _ => { *self = CalendarEventNotificationComparator::_T(value.to_string()); } ); } else { let _ = map.next_value::()?; } Ok(()) } } impl<'de> serde::Deserialize<'de> for CalendarEventNotificationType { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { CalendarEventNotificationType::parse(<&str>::deserialize(deserializer)?) .ok_or_else(|| serde::de::Error::custom("invalid CalendarEventNotificationType")) } } impl CalendarEventNotificationFilter { pub fn into_string(self) -> Cow<'static, str> { match self { CalendarEventNotificationFilter::After(_) => "after", CalendarEventNotificationFilter::Before(_) => "before", CalendarEventNotificationFilter::Type(_) => "type", CalendarEventNotificationFilter::CalendarEventIds(_) => "calendarEventIds", CalendarEventNotificationFilter::_T(s) => return Cow::Owned(s), } .into() } } impl CalendarEventNotificationComparator { pub fn into_string(self) -> Cow<'static, str> { match self { CalendarEventNotificationComparator::Created => "created", CalendarEventNotificationComparator::_T(s) => return Cow::Owned(s), } .into() } } impl Default for CalendarEventNotificationFilter { fn default() -> Self { CalendarEventNotificationFilter::_T(String::new()) } } impl Default for CalendarEventNotificationComparator { fn default() -> Self { CalendarEventNotificationComparator::_T(String::new()) } } impl TryFrom for Id { type Error = (); fn try_from(_: CalendarEventNotificationProperty) -> Result { Err(()) } } impl From for CalendarEventNotificationValue { fn from(id: Id) -> Self { CalendarEventNotificationValue::Id(id) } } impl JmapObjectId for CalendarEventNotificationValue { fn as_id(&self) -> Option { if let CalendarEventNotificationValue::Id(id) = self { Some(*id) } else { None } } fn as_any_id(&self) -> Option { if let CalendarEventNotificationValue::Id(id) = self { Some(AnyId::Id(*id)) } else { None } } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, _: AnyId) -> bool { false } } impl JmapObjectId for CalendarEventNotificationProperty { fn as_id(&self) -> Option { None } fn as_any_id(&self) -> Option { None } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, _: AnyId) -> bool { false } } impl serde::Serialize for CalendarEventNotificationType { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { serializer.serialize_str(self.as_str()) } } impl Display for CalendarEventNotificationProperty { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.to_cow()) } } ================================================ FILE: crates/jmap-proto/src/object/contact.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ object::{AnyId, JmapObject, JmapObjectId}, request::{MaybeInvalid, deserialize::DeserializeArguments}, types::date::UTCDate, }; use calcard::jscontact::{JSContactProperty, JSContactValue}; use jmap_tools::{JsonPointerItem, Key}; use std::borrow::Cow; use types::{blob::BlobId, id::Id}; #[derive(Debug, Clone, Default)] pub struct ContactCard; impl JmapObject for ContactCard { type Property = JSContactProperty; type Element = JSContactValue; type Id = Id; type Filter = ContactCardFilter; type Comparator = ContactCardComparator; type GetArguments = (); type SetArguments<'de> = (); type QueryArguments = (); type CopyArguments = (); type ParseArguments = (); const ID_PROPERTY: Self::Property = JSContactProperty::Id; } impl JmapObjectId for JSContactValue { fn as_id(&self) -> Option { if let JSContactValue::Id(id) = self { Some(*id) } else { None } } fn as_any_id(&self) -> Option { match self { JSContactValue::Id(id) => Some(AnyId::Id(*id)), JSContactValue::BlobId(id) => Some(AnyId::BlobId(id.clone())), _ => None, } } fn as_id_ref(&self) -> Option<&str> { match self { JSContactValue::IdReference(r) => Some(r), _ => None, } } fn try_set_id(&mut self, new_id: AnyId) -> bool { match new_id { AnyId::Id(id) => { *self = JSContactValue::Id(id); } AnyId::BlobId(id) => { *self = JSContactValue::BlobId(id); } } true } } #[derive(Debug, Clone, PartialEq, Eq)] pub enum ContactCardFilter { InAddressBook(MaybeInvalid), Uid(String), HasMember(String), Kind(String), CreatedBefore(UTCDate), CreatedAfter(UTCDate), UpdatedBefore(UTCDate), UpdatedAfter(UTCDate), Text(String), Name(String), NameGiven(String), NameSurname(String), NameSurname2(String), Nickname(String), Organization(String), Email(String), Phone(String), OnlineService(String), Address(String), Note(String), _T(String), } #[derive(Debug, Clone, PartialEq, Eq)] pub enum ContactCardComparator { Created, Updated, NameGiven, NameSurname, NameSurname2, _T(String), } impl<'de> DeserializeArguments<'de> for ContactCardFilter { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"inAddressBook" => { *self = ContactCardFilter::InAddressBook(map.next_value()?); }, b"uid" => { *self = ContactCardFilter::Uid(map.next_value()?); }, b"hasMember" => { *self = ContactCardFilter::HasMember(map.next_value()?); }, b"kind" => { *self = ContactCardFilter::Kind(map.next_value()?); }, b"createdBefore" => { *self = ContactCardFilter::CreatedBefore(map.next_value()?); }, b"createdAfter" => { *self = ContactCardFilter::CreatedAfter(map.next_value()?); }, b"updatedBefore" => { *self = ContactCardFilter::UpdatedBefore(map.next_value()?); }, b"updatedAfter" => { *self = ContactCardFilter::UpdatedAfter(map.next_value()?); }, b"text" => { *self = ContactCardFilter::Text(map.next_value::>()?.to_lowercase()); }, b"name" => { *self = ContactCardFilter::Name(map.next_value::>()?.to_lowercase()); }, b"name/given" => { *self = ContactCardFilter::NameGiven(map.next_value::>()?.to_lowercase()); }, b"name/surname" => { *self = ContactCardFilter::NameSurname(map.next_value::>()?.to_lowercase()); }, b"name/surname2" => { *self = ContactCardFilter::NameSurname2(map.next_value::>()?.to_lowercase()); }, b"nickname" => { *self = ContactCardFilter::Nickname(map.next_value::>()?.to_lowercase()); }, b"organization" => { *self = ContactCardFilter::Organization(map.next_value::>()?.to_lowercase()); }, b"email" => { *self = ContactCardFilter::Email(map.next_value()?); }, b"phone" => { *self = ContactCardFilter::Phone(map.next_value::>()?.to_lowercase()); }, b"onlineService" => { *self = ContactCardFilter::OnlineService(map.next_value::>()?.to_lowercase()); }, b"address" => { *self = ContactCardFilter::Address(map.next_value::>()?.to_lowercase()); }, b"note" => { *self = ContactCardFilter::Note(map.next_value::>()?.to_lowercase()); }, _ => { *self = ContactCardFilter::_T(key.to_string()); let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> DeserializeArguments<'de> for ContactCardComparator { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { if key == "property" { let value = map.next_value::>()?; hashify::fnc_map!(value.as_bytes(), b"created" => { *self = ContactCardComparator::Created; }, b"updated" => { *self = ContactCardComparator::Updated; }, b"name/given" => { *self = ContactCardComparator::NameGiven; }, b"name/surname" => { *self = ContactCardComparator::NameSurname; }, b"name/surname2" => { *self = ContactCardComparator::NameSurname2; }, _ => { *self = ContactCardComparator::_T(value.to_string()); } ); } else { let _ = map.next_value::()?; } Ok(()) } } impl ContactCardFilter { pub fn into_string(self) -> Cow<'static, str> { match self { ContactCardFilter::InAddressBook(_) => "inAddressBook", ContactCardFilter::Uid(_) => "uid", ContactCardFilter::HasMember(_) => "hasMember", ContactCardFilter::Kind(_) => "kind", ContactCardFilter::CreatedBefore(_) => "createdBefore", ContactCardFilter::CreatedAfter(_) => "createdAfter", ContactCardFilter::UpdatedBefore(_) => "updatedBefore", ContactCardFilter::UpdatedAfter(_) => "updatedAfter", ContactCardFilter::Text(_) => "text", ContactCardFilter::Name(_) => "name", ContactCardFilter::NameGiven(_) => "name/given", ContactCardFilter::NameSurname(_) => "name/surname", ContactCardFilter::NameSurname2(_) => "name/surname2", ContactCardFilter::Nickname(_) => "nickname", ContactCardFilter::Organization(_) => "organization", ContactCardFilter::Email(_) => "email", ContactCardFilter::Phone(_) => "phone", ContactCardFilter::OnlineService(_) => "onlineService", ContactCardFilter::Address(_) => "address", ContactCardFilter::Note(_) => "note", ContactCardFilter::_T(s) => return Cow::Owned(s), } .into() } } impl ContactCardComparator { pub fn into_string(self) -> Cow<'static, str> { match self { ContactCardComparator::Created => "created", ContactCardComparator::Updated => "updated", ContactCardComparator::NameGiven => "name/given", ContactCardComparator::NameSurname => "name/surname", ContactCardComparator::NameSurname2 => "name/surname2", ContactCardComparator::_T(s) => return Cow::Owned(s), } .into() } } impl Default for ContactCardFilter { fn default() -> Self { ContactCardFilter::_T(String::new()) } } impl Default for ContactCardComparator { fn default() -> Self { ContactCardComparator::_T(String::new()) } } impl JmapObjectId for JSContactProperty { fn as_id(&self) -> Option { if let JSContactProperty::IdValue(id) = self { Some(*id) } else { None } } fn as_any_id(&self) -> Option { if let JSContactProperty::IdValue(id) = self { Some(AnyId::Id(*id)) } else { None } } fn as_id_ref(&self) -> Option<&str> { match self { JSContactProperty::IdReference(r) => Some(r), JSContactProperty::Pointer(value) => { let value = value.as_slice(); match (value.first(), value.get(1)) { ( Some(JsonPointerItem::Key(Key::Property( JSContactProperty::AddressBookIds, ))), Some(JsonPointerItem::Key(Key::Property(JSContactProperty::IdReference( r, )))), ) => Some(r), _ => None, } } _ => None, } } fn try_set_id(&mut self, new_id: AnyId) -> bool { if let AnyId::Id(id) = new_id { if let JSContactProperty::Pointer(value) = self { let value = value.as_mut_slice(); if let Some(value) = value.get_mut(1) { *value = JsonPointerItem::Key(Key::Property(JSContactProperty::IdValue(id))); return true; } } else { *self = JSContactProperty::IdValue(id); return true; } } false } } ================================================ FILE: crates/jmap-proto/src/object/email.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ method::query::{Comparator, Filter}, object::{AnyId, JmapObject, JmapObjectId, MaybeReference, parse_ref}, request::{MaybeInvalid, deserialize::DeserializeArguments}, types::date::UTCDate, }; use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, Property}; use mail_parser::HeaderName; use serde::Serialize; use std::{borrow::Cow, fmt::Display, str::FromStr}; use types::{blob::BlobId, id::Id, keyword::Keyword}; #[derive(Debug, Clone, Default)] pub struct Email; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum EmailProperty { // Metadata Id, BlobId, ThreadId, MailboxIds, Keywords, Size, ReceivedAt, // Address Name, Email, // GroupedAddresses Addresses, // Header Fields Properties Value, Header(HeaderProperty), // Convenience properties MessageId, InReplyTo, References, Sender, From, To, Cc, Bcc, ReplyTo, Subject, SentAt, // Body Parts TextBody, HtmlBody, Attachments, PartId, Headers, Type, Charset, Disposition, Cid, Language, Location, SubParts, BodyStructure, BodyValues, IsEncodingProblem, IsTruncated, HasAttachment, Preview, // Other Keyword(Keyword), IdValue(Id), IdReference(String), Pointer(JsonPointer), } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct HeaderProperty { pub form: HeaderForm, pub header: String, pub all: bool, } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum HeaderForm { Raw, Text, Addresses, GroupedAddresses, MessageIds, Date, URLs, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum EmailValue { Id(Id), Date(UTCDate), BlobId(BlobId), IdReference(String), } impl Property for EmailProperty { fn try_parse(key: Option<&Key<'_, Self>>, value: &str) -> Option { let allow_patch = key.is_none(); if let Some(Key::Property(key)) = key { match key.patch_or_prop() { EmailProperty::Keywords => EmailProperty::Keyword(Keyword::parse(value)).into(), EmailProperty::MailboxIds => match parse_ref(value) { MaybeReference::Value(v) => Some(EmailProperty::IdValue(v)), MaybeReference::Reference(v) => Some(EmailProperty::IdReference(v)), MaybeReference::ParseError => None, }, _ => EmailProperty::parse(value, allow_patch), } } else { EmailProperty::parse(value, allow_patch) } } fn to_cow(&self) -> Cow<'static, str> { match self { EmailProperty::Attachments => "attachments", EmailProperty::Bcc => "bcc", EmailProperty::BlobId => "blobId", EmailProperty::BodyStructure => "bodyStructure", EmailProperty::BodyValues => "bodyValues", EmailProperty::Cc => "cc", EmailProperty::Charset => "charset", EmailProperty::Cid => "cid", EmailProperty::Disposition => "disposition", EmailProperty::Email => "email", EmailProperty::From => "from", EmailProperty::HasAttachment => "hasAttachment", EmailProperty::Headers => "headers", EmailProperty::HtmlBody => "htmlBody", EmailProperty::Id => "id", EmailProperty::InReplyTo => "inReplyTo", EmailProperty::Keywords => "keywords", EmailProperty::Language => "language", EmailProperty::Location => "location", EmailProperty::MailboxIds => "mailboxIds", EmailProperty::MessageId => "messageId", EmailProperty::Name => "name", EmailProperty::PartId => "partId", EmailProperty::Preview => "preview", EmailProperty::ReceivedAt => "receivedAt", EmailProperty::References => "references", EmailProperty::ReplyTo => "replyTo", EmailProperty::Sender => "sender", EmailProperty::SentAt => "sentAt", EmailProperty::Size => "size", EmailProperty::Subject => "subject", EmailProperty::SubParts => "subParts", EmailProperty::TextBody => "textBody", EmailProperty::ThreadId => "threadId", EmailProperty::To => "to", EmailProperty::Type => "type", EmailProperty::Addresses => "addresses", EmailProperty::Value => "value", EmailProperty::IsEncodingProblem => "isEncodingProblem", EmailProperty::IsTruncated => "isTruncated", EmailProperty::Header(header) => return header.to_string().into(), EmailProperty::Keyword(keyword) => return keyword.to_string().into(), EmailProperty::IdValue(id) => return id.to_string().into(), EmailProperty::Pointer(json_pointer) => return json_pointer.to_string().into(), EmailProperty::IdReference(r) => return format!("#{r}").into(), } .into() } } impl Element for EmailValue { type Property = EmailProperty; fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option { if let Key::Property(prop) = key { match prop.patch_or_prop() { EmailProperty::Id | EmailProperty::ThreadId | EmailProperty::MailboxIds => { match parse_ref(value) { MaybeReference::Value(v) => Some(EmailValue::Id(v)), MaybeReference::Reference(v) => Some(EmailValue::IdReference(v)), MaybeReference::ParseError => None, } } EmailProperty::BlobId => match parse_ref(value) { MaybeReference::Value(v) => Some(EmailValue::BlobId(v)), MaybeReference::Reference(v) => Some(EmailValue::IdReference(v)), MaybeReference::ParseError => None, }, EmailProperty::Header(HeaderProperty { form: HeaderForm::Date, .. }) | EmailProperty::ReceivedAt | EmailProperty::SentAt => UTCDate::from_str(value).ok().map(EmailValue::Date), _ => None, } } else { None } } fn to_cow(&self) -> Cow<'static, str> { match self { EmailValue::Id(id) => id.to_string().into(), EmailValue::Date(utcdate) => utcdate.to_string().into(), EmailValue::BlobId(blob_id) => blob_id.to_string().into(), EmailValue::IdReference(r) => format!("#{r}").into(), } } } impl EmailProperty { fn parse(value: &str, allow_patch: bool) -> Option { hashify::tiny_map!(value.as_bytes(), "id" => EmailProperty::Id, "blobId" => EmailProperty::BlobId, "threadId" => EmailProperty::ThreadId, "mailboxIds" => EmailProperty::MailboxIds, "keywords" => EmailProperty::Keywords, "size" => EmailProperty::Size, "receivedAt" => EmailProperty::ReceivedAt, "name" => EmailProperty::Name, "email" => EmailProperty::Email, "addresses" => EmailProperty::Addresses, "value" => EmailProperty::Value, "messageId" => EmailProperty::MessageId, "inReplyTo" => EmailProperty::InReplyTo, "references" => EmailProperty::References, "sender" => EmailProperty::Sender, "from" => EmailProperty::From, "to" => EmailProperty::To, "cc" => EmailProperty::Cc, "bcc" => EmailProperty::Bcc, "replyTo" => EmailProperty::ReplyTo, "subject" => EmailProperty::Subject, "sentAt" => EmailProperty::SentAt, "textBody" => EmailProperty::TextBody, "htmlBody" => EmailProperty::HtmlBody, "attachments" => EmailProperty::Attachments, "partId" => EmailProperty::PartId, "headers" => EmailProperty::Headers, "type" => EmailProperty::Type, "charset" => EmailProperty::Charset, "disposition" => EmailProperty::Disposition, "cid" => EmailProperty::Cid, "language" => EmailProperty::Language, "location" => EmailProperty::Location, "subParts" => EmailProperty::SubParts, "bodyStructure" => EmailProperty::BodyStructure, "bodyValues" => EmailProperty::BodyValues, "isEncodingProblem" => EmailProperty::IsEncodingProblem, "isTruncated" => EmailProperty::IsTruncated, "hasAttachment" => EmailProperty::HasAttachment, "preview" => EmailProperty::Preview ) .or_else(|| { if let Some(header) = value.strip_prefix("header:") { HeaderProperty::parse(header).map(EmailProperty::Header) } else if allow_patch && value.contains('/') { EmailProperty::Pointer(JsonPointer::parse(value)).into() } else { None } }) } fn patch_or_prop(&self) -> &EmailProperty { if let EmailProperty::Pointer(ptr) = self && let Some(JsonPointerItem::Key(Key::Property(prop))) = ptr.last() { prop } else { self } } pub fn as_rfc_header(&self) -> HeaderName<'static> { match self { EmailProperty::MessageId => HeaderName::MessageId, EmailProperty::InReplyTo => HeaderName::InReplyTo, EmailProperty::References => HeaderName::References, EmailProperty::Sender => HeaderName::Sender, EmailProperty::From => HeaderName::From, EmailProperty::To => HeaderName::To, EmailProperty::Cc => HeaderName::Cc, EmailProperty::Bcc => HeaderName::Bcc, EmailProperty::ReplyTo => HeaderName::ReplyTo, EmailProperty::Subject => HeaderName::Subject, EmailProperty::SentAt => HeaderName::Date, _ => unreachable!(), } } pub fn try_into_id(self) -> Option { match self { EmailProperty::IdValue(id) => Some(id), _ => None, } } pub fn try_into_keyword(self) -> Option { match self { EmailProperty::Keyword(keyword) => Some(keyword), _ => None, } } } impl HeaderProperty { fn parse(value: &str) -> Option { let mut result = HeaderProperty { form: HeaderForm::Raw, header: String::new(), all: false, }; for (pos, value) in value.split(':').enumerate() { match pos { 0 => { result.header = value.to_string(); } 1 => { hashify::fnc_map!(value.as_bytes(), b"asText" => { result.form = HeaderForm::Text;}, b"asAddresses" => { result.form = HeaderForm::Addresses;}, b"asGroupedAddresses" => { result.form = HeaderForm::GroupedAddresses;}, b"asMessageIds" => { result.form = HeaderForm::MessageIds;}, b"asDate" => { result.form = HeaderForm::Date;}, b"asURLs" => { result.form = HeaderForm::URLs;}, b"asRaw" => { result.form = HeaderForm::Raw; }, b"all" => { result.all = true; }, _ => { return None; } ); } 2 if value == "all" && !result.all => { result.all = true; } _ => return None, } } if !result.header.is_empty() { Some(result) } else { None } } } impl Display for HeaderProperty { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "header:{}", self.header)?; self.form.fmt(f)?; if self.all { write!(f, ":all") } else { Ok(()) } } } impl Display for HeaderForm { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { HeaderForm::Raw => Ok(()), HeaderForm::Text => write!(f, ":asText"), HeaderForm::Addresses => write!(f, ":asAddresses"), HeaderForm::GroupedAddresses => write!(f, ":asGroupedAddresses"), HeaderForm::MessageIds => write!(f, ":asMessageIds"), HeaderForm::Date => write!(f, ":asDate"), HeaderForm::URLs => write!(f, ":asURLs"), } } } impl FromStr for EmailProperty { type Err = (); fn from_str(s: &str) -> Result { EmailProperty::parse(s, false).ok_or(()) } } #[derive(Debug, Clone, Default)] pub struct EmailGetArguments { pub body_properties: Option>>, pub fetch_text_body_values: Option, pub fetch_html_body_values: Option, pub fetch_all_body_values: Option, pub max_body_value_bytes: Option, } #[derive(Debug, Clone, Default)] pub struct EmailQueryArguments { pub collapse_threads: Option, } #[derive(Debug, Clone, Default)] pub struct EmailParseArguments { pub body_properties: Option>>, pub fetch_text_body_values: Option, pub fetch_html_body_values: Option, pub fetch_all_body_values: Option, pub max_body_value_bytes: Option, } impl<'de> DeserializeArguments<'de> for EmailGetArguments { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"bodyProperties" => { self.body_properties = map.next_value()?; }, b"fetchTextBodyValues" => { self.fetch_text_body_values = map.next_value()?; }, b"fetchHTMLBodyValues" => { self.fetch_html_body_values = map.next_value()?; }, b"fetchAllBodyValues" => { self.fetch_all_body_values = map.next_value()?; }, b"maxBodyValueBytes" => { self.max_body_value_bytes = map.next_value()?; }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> DeserializeArguments<'de> for EmailQueryArguments { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { if key == "collapseThreads" { self.collapse_threads = map.next_value()?; } else { let _ = map.next_value::()?; } Ok(()) } } impl<'de> DeserializeArguments<'de> for EmailParseArguments { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"bodyProperties" => { self.body_properties = map.next_value()?; }, b"fetchTextBodyValues" => { self.fetch_text_body_values = map.next_value()?; }, b"fetchHTMLBodyValues" => { self.fetch_html_body_values = map.next_value()?; }, b"fetchAllBodyValues" => { self.fetch_all_body_values = map.next_value()?; }, b"maxBodyValueBytes" => { self.max_body_value_bytes = map.next_value()?; }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl JmapObject for Email { type Property = EmailProperty; type Element = EmailValue; type Id = Id; type Filter = EmailFilter; type Comparator = EmailComparator; type GetArguments = EmailGetArguments; type SetArguments<'de> = (); type QueryArguments = EmailQueryArguments; type CopyArguments = (); type ParseArguments = EmailParseArguments; const ID_PROPERTY: Self::Property = EmailProperty::Id; } #[derive(Debug, Clone, PartialEq, Eq)] pub enum EmailFilter { InMailbox(Id), InMailboxOtherThan(Vec), Before(UTCDate), After(UTCDate), MinSize(u32), MaxSize(u32), AllInThreadHaveKeyword(Keyword), SomeInThreadHaveKeyword(Keyword), NoneInThreadHaveKeyword(Keyword), HasKeyword(Keyword), NotKeyword(Keyword), HasAttachment(bool), From(String), To(String), Cc(String), Bcc(String), Subject(String), Body(String), Header(Vec), Text(String), SentBefore(UTCDate), SentAfter(UTCDate), InThread(Id), Id(Vec), _T(String), } #[derive(Debug, Clone, PartialEq, Eq)] pub enum EmailComparator { ReceivedAt, Size, From, To, Subject, Cc, SentAt, ThreadId, HasKeyword(Keyword), AllInThreadHaveKeyword(Keyword), SomeInThreadHaveKeyword(Keyword), _T(String), } impl<'de> DeserializeArguments<'de> for EmailFilter { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"inMailbox" => { *self = EmailFilter::InMailbox(map.next_value()?); }, b"inMailboxOtherThan" => { *self = EmailFilter::InMailboxOtherThan(map.next_value()?); }, b"before" => { *self = EmailFilter::Before(map.next_value()?); }, b"after" => { *self = EmailFilter::After(map.next_value()?); }, b"minSize" => { *self = EmailFilter::MinSize(map.next_value()?); }, b"maxSize" => { *self = EmailFilter::MaxSize(map.next_value()?); }, b"allInThreadHaveKeyword" => { *self = EmailFilter::AllInThreadHaveKeyword(map.next_value()?); }, b"someInThreadHaveKeyword" => { *self = EmailFilter::SomeInThreadHaveKeyword(map.next_value()?); }, b"noneInThreadHaveKeyword" => { *self = EmailFilter::NoneInThreadHaveKeyword(map.next_value()?); }, b"hasKeyword" => { *self = EmailFilter::HasKeyword(map.next_value()?); }, b"notKeyword" => { *self = EmailFilter::NotKeyword(map.next_value()?); }, b"hasAttachment" => { *self = EmailFilter::HasAttachment(map.next_value()?); }, b"from" => { *self = EmailFilter::From(map.next_value()?); }, b"to" => { *self = EmailFilter::To(map.next_value()?); }, b"cc" => { *self = EmailFilter::Cc(map.next_value()?); }, b"bcc" => { *self = EmailFilter::Bcc(map.next_value()?); }, b"subject" => { *self = EmailFilter::Subject(map.next_value()?); }, b"body" => { *self = EmailFilter::Body(map.next_value()?); }, b"header" => { *self = EmailFilter::Header(map.next_value()?); }, b"text" => { *self = EmailFilter::Text(map.next_value()?); }, b"sentBefore" => { *self = EmailFilter::SentBefore(map.next_value()?); }, b"sentAfter" => { *self = EmailFilter::SentAfter(map.next_value()?); }, b"inThread" => { *self = EmailFilter::InThread(map.next_value()?); }, b"id" => { *self = EmailFilter::Id(map.next_value()?); }, _ => { *self = EmailFilter::_T(key.to_string()); let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> DeserializeArguments<'de> for EmailComparator { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { if key == "property" { let value = map.next_value::>()?; hashify::fnc_map!(value.as_bytes(), b"receivedAt" => { *self = EmailComparator::ReceivedAt; }, b"size" => { *self = EmailComparator::Size; }, b"from" => { *self = EmailComparator::From; }, b"to" => { *self = EmailComparator::To; }, b"cc" => { *self = EmailComparator::Cc; }, b"subject" => { *self = EmailComparator::Subject; }, b"sentAt" => { *self = EmailComparator::SentAt; }, b"threadId" => { *self = EmailComparator::ThreadId; }, b"hasKeyword" => { *self = EmailComparator::HasKeyword(self.take_keyword()); }, b"allInThreadHaveKeyword" => { *self = EmailComparator::AllInThreadHaveKeyword(self.take_keyword()); }, b"someInThreadHaveKeyword" => { *self = EmailComparator::SomeInThreadHaveKeyword(self.take_keyword()); }, _ => { *self = EmailComparator::_T(key.to_string()); } ); } else if key == "keyword" { let keyword: Keyword = map.next_value()?; match self { EmailComparator::HasKeyword(_) => *self = EmailComparator::HasKeyword(keyword), EmailComparator::AllInThreadHaveKeyword(_) => { *self = EmailComparator::AllInThreadHaveKeyword(keyword) } EmailComparator::SomeInThreadHaveKeyword(_) => { *self = EmailComparator::SomeInThreadHaveKeyword(keyword) } _ => { *self = EmailComparator::HasKeyword(keyword); } } } else { let _ = map.next_value::()?; } Ok(()) } } impl Default for EmailFilter { fn default() -> Self { EmailFilter::_T("".to_string()) } } impl Default for EmailComparator { fn default() -> Self { EmailComparator::_T("".to_string()) } } impl EmailComparator { fn take_keyword(&mut self) -> Keyword { match self { EmailComparator::HasKeyword(k) => { std::mem::replace(k, Keyword::Other(Default::default())) } EmailComparator::AllInThreadHaveKeyword(k) => { std::mem::replace(k, Keyword::Other(Default::default())) } EmailComparator::SomeInThreadHaveKeyword(k) => { std::mem::replace(k, Keyword::Other(Default::default())) } _ => Keyword::Other(Default::default()), } } } impl Display for EmailFilter { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(match self { EmailFilter::InMailbox(_) => "inMailbox", EmailFilter::InMailboxOtherThan(_) => "inMailboxOtherThan", EmailFilter::Before(_) => "before", EmailFilter::After(_) => "after", EmailFilter::MinSize(_) => "minSize", EmailFilter::MaxSize(_) => "maxSize", EmailFilter::AllInThreadHaveKeyword(_) => "allInThreadHaveKeyword", EmailFilter::SomeInThreadHaveKeyword(_) => "someInThreadHaveKeyword", EmailFilter::NoneInThreadHaveKeyword(_) => "noneInThreadHaveKeyword", EmailFilter::HasKeyword(_) => "hasKeyword", EmailFilter::NotKeyword(_) => "notKeyword", EmailFilter::HasAttachment(_) => "hasAttachment", EmailFilter::From(_) => "from", EmailFilter::To(_) => "to", EmailFilter::Cc(_) => "cc", EmailFilter::Bcc(_) => "bcc", EmailFilter::Subject(_) => "subject", EmailFilter::Body(_) => "body", EmailFilter::Header(_) => "header", EmailFilter::Text(_) => "text", EmailFilter::SentBefore(_) => "sentBefore", EmailFilter::SentAfter(_) => "sentAfter", EmailFilter::InThread(_) => "inThread", EmailFilter::Id(_) => "id", EmailFilter::_T(v) => v.as_str(), }) } } impl Display for EmailComparator { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(self.as_str()) } } impl EmailComparator { pub fn as_str(&self) -> &str { match self { EmailComparator::ReceivedAt => "receivedAt", EmailComparator::Size => "size", EmailComparator::From => "from", EmailComparator::To => "to", EmailComparator::Subject => "subject", EmailComparator::Cc => "cc", EmailComparator::SentAt => "sentAt", EmailComparator::ThreadId => "threadId", EmailComparator::HasKeyword(_) => "hasKeyword", EmailComparator::AllInThreadHaveKeyword(_) => "allInThreadHaveKeyword", EmailComparator::SomeInThreadHaveKeyword(_) => "someInThreadHaveKeyword", EmailComparator::_T(v) => v.as_str(), } } } impl Serialize for EmailComparator { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { serializer.serialize_str(self.as_str()) } } impl Filter { pub fn is_immutable(&self) -> bool { match self { Filter::Property(f) => f.is_immutable(), Filter::And | Filter::Or | Filter::Not | Filter::Close => true, } } } impl EmailFilter { pub fn is_immutable(&self) -> bool { matches!( self, EmailFilter::Before(_) | EmailFilter::After(_) | EmailFilter::MinSize(_) | EmailFilter::MaxSize(_) | EmailFilter::HasAttachment(_) | EmailFilter::From(_) | EmailFilter::To(_) | EmailFilter::Cc(_) | EmailFilter::Bcc(_) | EmailFilter::Subject(_) | EmailFilter::Body(_) | EmailFilter::Header(_) | EmailFilter::Text(_) | EmailFilter::Id(_) | EmailFilter::SentBefore(_) | EmailFilter::SentAfter(_) ) } } impl Comparator { pub fn is_immutable(&self) -> bool { self.property.is_immutable() } } impl EmailComparator { pub fn is_immutable(&self) -> bool { matches!( self, EmailComparator::ReceivedAt | EmailComparator::Size | EmailComparator::From | EmailComparator::To | EmailComparator::Subject | EmailComparator::Cc | EmailComparator::SentAt ) } } impl JmapObjectId for EmailValue { fn as_id(&self) -> Option { if let EmailValue::Id(id) = self { Some(*id) } else { None } } fn as_any_id(&self) -> Option { match self { EmailValue::Id(id) => Some(AnyId::Id(*id)), EmailValue::BlobId(id) => Some(AnyId::BlobId(id.clone())), _ => None, } } fn as_id_ref(&self) -> Option<&str> { if let EmailValue::IdReference(r) = self { Some(r) } else { None } } fn try_set_id(&mut self, new_id: AnyId) -> bool { match new_id { AnyId::Id(id) => { *self = EmailValue::Id(id); } AnyId::BlobId(id) => { *self = EmailValue::BlobId(id); } } true } } impl From for EmailValue { fn from(id: Id) -> Self { EmailValue::Id(id) } } impl From for EmailValue { fn from(id: BlobId) -> Self { EmailValue::BlobId(id) } } impl From for EmailValue { fn from(date: UTCDate) -> Self { EmailValue::Date(date) } } impl JmapObjectId for EmailProperty { fn as_id(&self) -> Option { if let EmailProperty::IdValue(id) = self { Some(*id) } else { None } } fn as_any_id(&self) -> Option { if let EmailProperty::IdValue(id) = self { Some(AnyId::Id(*id)) } else { None } } fn as_id_ref(&self) -> Option<&str> { match self { EmailProperty::IdReference(r) => Some(r), EmailProperty::Pointer(value) => { let value = value.as_slice(); match (value.first(), value.get(1)) { ( Some(JsonPointerItem::Key(Key::Property(EmailProperty::MailboxIds))), Some(JsonPointerItem::Key(Key::Property(EmailProperty::IdReference(r)))), ) => Some(r), _ => None, } } _ => None, } } fn try_set_id(&mut self, new_id: AnyId) -> bool { if let AnyId::Id(id) = new_id { if let EmailProperty::Pointer(value) = self { let value = value.as_mut_slice(); if let Some(value) = value.get_mut(1) { *value = JsonPointerItem::Key(Key::Property(EmailProperty::IdValue(id))); return true; } } else { *self = EmailProperty::IdValue(id); return true; } } false } } ================================================ FILE: crates/jmap-proto/src/object/email_submission.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ object::{ AnyId, JmapObject, JmapObjectId, MaybeReference, email::{EmailProperty, EmailValue}, parse_ref, }, request::{MaybeInvalid, deserialize::DeserializeArguments, reference::MaybeIdReference}, types::date::UTCDate, }; use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, Property, Value}; use std::{borrow::Cow, str::FromStr}; use types::{blob::BlobId, id::Id}; use utils::map::vec_map::VecMap; #[derive(Debug, Clone, Default)] pub struct EmailSubmission; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum EmailSubmissionProperty { Id, IdentityId, ThreadId, EmailId, Envelope, MailFrom, RcptTo, Email, Parameters, SendAt, UndoStatus, DeliveryStatus, SmtpReply, Delivered, Displayed, DsnBlobIds, MdnBlobIds, Pointer(JsonPointer), } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum EmailSubmissionValue { Id(Id), Date(UTCDate), BlobId(BlobId), UndoStatus(UndoStatus), Delivered(Delivered), Displayed(Displayed), IdReference(String), } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum UndoStatus { Pending, Final, Canceled, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Delivered { Queued, Yes, No, Unknown, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Displayed { Yes, Unknown, } impl Property for EmailSubmissionProperty { fn try_parse(key: Option<&Key<'_, Self>>, value: &str) -> Option { EmailSubmissionProperty::parse(value, key.is_none()) } fn to_cow(&self) -> Cow<'static, str> { match self { EmailSubmissionProperty::DeliveryStatus => "deliveryStatus", EmailSubmissionProperty::DsnBlobIds => "dsnBlobIds", EmailSubmissionProperty::Email => "email", EmailSubmissionProperty::Envelope => "envelope", EmailSubmissionProperty::Id => "id", EmailSubmissionProperty::IdentityId => "identityId", EmailSubmissionProperty::MdnBlobIds => "mdnBlobIds", EmailSubmissionProperty::SendAt => "sendAt", EmailSubmissionProperty::ThreadId => "threadId", EmailSubmissionProperty::UndoStatus => "undoStatus", EmailSubmissionProperty::Parameters => "parameters", EmailSubmissionProperty::SmtpReply => "smtpReply", EmailSubmissionProperty::Delivered => "delivered", EmailSubmissionProperty::Displayed => "displayed", EmailSubmissionProperty::MailFrom => "mailFrom", EmailSubmissionProperty::RcptTo => "rcptTo", EmailSubmissionProperty::EmailId => "emailId", EmailSubmissionProperty::Pointer(json_pointer) => { return json_pointer.to_string().into(); } } .into() } } impl Element for EmailSubmissionValue { type Property = EmailSubmissionProperty; fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option { if let Key::Property(prop) = key { match prop.patch_or_prop() { EmailSubmissionProperty::Id | EmailSubmissionProperty::ThreadId | EmailSubmissionProperty::IdentityId | EmailSubmissionProperty::EmailId => match parse_ref(value) { MaybeReference::Value(v) => Some(EmailSubmissionValue::Id(v)), MaybeReference::Reference(v) => Some(EmailSubmissionValue::IdReference(v)), MaybeReference::ParseError => None, }, EmailSubmissionProperty::MdnBlobIds | EmailSubmissionProperty::DsnBlobIds => { match parse_ref(value) { MaybeReference::Value(v) => Some(EmailSubmissionValue::BlobId(v)), MaybeReference::Reference(v) => Some(EmailSubmissionValue::IdReference(v)), MaybeReference::ParseError => None, } } EmailSubmissionProperty::SendAt => UTCDate::from_str(value) .ok() .map(EmailSubmissionValue::Date), EmailSubmissionProperty::UndoStatus => { UndoStatus::parse(value).map(EmailSubmissionValue::UndoStatus) } EmailSubmissionProperty::Delivered => { Delivered::parse(value).map(EmailSubmissionValue::Delivered) } EmailSubmissionProperty::Displayed => { Displayed::parse(value).map(EmailSubmissionValue::Displayed) } _ => None, } } else { None } } fn to_cow(&self) -> Cow<'static, str> { match self { EmailSubmissionValue::Id(id) => id.to_string().into(), EmailSubmissionValue::Date(utcdate) => utcdate.to_string().into(), EmailSubmissionValue::BlobId(blob_id) => blob_id.to_string().into(), EmailSubmissionValue::IdReference(r) => format!("#{r}").into(), EmailSubmissionValue::UndoStatus(undo_status) => undo_status.as_str().into(), EmailSubmissionValue::Delivered(delivered) => delivered.as_str().into(), EmailSubmissionValue::Displayed(displayed) => displayed.as_str().into(), } } } impl EmailSubmissionProperty { fn parse(value: &str, allow_patch: bool) -> Option { hashify::tiny_map!(value.as_bytes(), "id" => EmailSubmissionProperty::Id, "identityId" => EmailSubmissionProperty::IdentityId, "threadId" => EmailSubmissionProperty::ThreadId, "emailId" => EmailSubmissionProperty::EmailId, "envelope" => EmailSubmissionProperty::Envelope, "mailFrom" => EmailSubmissionProperty::MailFrom, "rcptTo" => EmailSubmissionProperty::RcptTo, "email" => EmailSubmissionProperty::Email, "parameters" => EmailSubmissionProperty::Parameters, "sendAt" => EmailSubmissionProperty::SendAt, "undoStatus" => EmailSubmissionProperty::UndoStatus, "deliveryStatus" => EmailSubmissionProperty::DeliveryStatus, "smtpReply" => EmailSubmissionProperty::SmtpReply, "delivered" => EmailSubmissionProperty::Delivered, "displayed" => EmailSubmissionProperty::Displayed, "dsnBlobIds" => EmailSubmissionProperty::DsnBlobIds, "mdnBlobIds" => EmailSubmissionProperty::MdnBlobIds, ) .or_else(|| { if allow_patch && value.contains('/') { EmailSubmissionProperty::Pointer(JsonPointer::parse(value)).into() } else { None } }) } fn patch_or_prop(&self) -> &EmailSubmissionProperty { if let EmailSubmissionProperty::Pointer(ptr) = self && let Some(JsonPointerItem::Key(Key::Property(prop))) = ptr.last() { prop } else { self } } } impl UndoStatus { fn parse(value: &str) -> Option { hashify::tiny_map!(value.as_bytes(), b"pending" => UndoStatus::Pending, b"final" => UndoStatus::Final, b"canceled" => UndoStatus::Canceled, ) } fn as_str(&self) -> &'static str { match self { UndoStatus::Pending => "pending", UndoStatus::Final => "final", UndoStatus::Canceled => "canceled", } } } impl Delivered { fn parse(value: &str) -> Option { hashify::tiny_map!(value.as_bytes(), b"queued" => Delivered::Queued, b"yes" => Delivered::Yes, b"no" => Delivered::No, b"unknown" => Delivered::Unknown, ) } fn as_str(&self) -> &'static str { match self { Delivered::Queued => "queued", Delivered::Yes => "yes", Delivered::No => "no", Delivered::Unknown => "unknown", } } } impl Displayed { fn parse(value: &str) -> Option { hashify::tiny_map!(value.as_bytes(), b"yes" => Displayed::Yes, b"unknown" => Displayed::Unknown, ) } fn as_str(&self) -> &'static str { match self { Displayed::Yes => "yes", Displayed::Unknown => "unknown", } } } #[derive(Debug, Clone, Default)] pub struct EmailSubmissionSetArguments<'x> { pub on_success_update_email: Option, Value<'x, EmailProperty, EmailValue>>>, pub on_success_destroy_email: Option>>, } impl<'x> DeserializeArguments<'x> for EmailSubmissionSetArguments<'x> { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'x>, { hashify::fnc_map!(key.as_bytes(), b"onSuccessUpdateEmail" => { self.on_success_update_email = map.next_value()?; }, b"onSuccessDestroyEmail" => { self.on_success_destroy_email = map.next_value()?; }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl FromStr for EmailSubmissionProperty { type Err = (); fn from_str(s: &str) -> Result { EmailSubmissionProperty::parse(s, false).ok_or(()) } } impl<'de> serde::Deserialize<'de> for UndoStatus { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { UndoStatus::parse(<&str>::deserialize(deserializer)?) .ok_or_else(|| serde::de::Error::custom("invalid JMAP UndoStatus")) } } impl JmapObject for EmailSubmission { type Property = EmailSubmissionProperty; type Element = EmailSubmissionValue; type Id = Id; type Filter = EmailSubmissionFilter; type Comparator = EmailSubmissionComparator; type GetArguments = (); type SetArguments<'de> = EmailSubmissionSetArguments<'de>; type QueryArguments = (); type CopyArguments = (); type ParseArguments = (); const ID_PROPERTY: Self::Property = EmailSubmissionProperty::Id; } #[derive(Debug, Clone, PartialEq, Eq)] pub enum EmailSubmissionFilter { IdentityIds(Vec>), EmailIds(Vec>), ThreadIds(Vec>), Before(UTCDate), After(UTCDate), UndoStatus(UndoStatus), _T(String), } #[derive(Debug, Clone, PartialEq, Eq)] pub enum EmailSubmissionComparator { EmailId, ThreadId, SentAt, _T(String), } impl<'de> DeserializeArguments<'de> for EmailSubmissionFilter { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"identityIds" => { *self = EmailSubmissionFilter::IdentityIds(map.next_value()?); }, b"emailIds" => { *self = EmailSubmissionFilter::EmailIds(map.next_value()?); }, b"threadIds" => { *self = EmailSubmissionFilter::ThreadIds(map.next_value()?); }, b"before" => { *self = EmailSubmissionFilter::Before(map.next_value()?); }, b"after" => { *self = EmailSubmissionFilter::After(map.next_value()?); }, b"undoStatus" => { *self = EmailSubmissionFilter::UndoStatus(map.next_value()?); }, _ => { *self = EmailSubmissionFilter::_T(key.to_string()); let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> DeserializeArguments<'de> for EmailSubmissionComparator { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { if key == "property" { let value = map.next_value::>()?; hashify::fnc_map!(value.as_bytes(), b"emailId" => { *self = EmailSubmissionComparator::EmailId; }, b"threadId" => { *self = EmailSubmissionComparator::ThreadId; }, b"sentAt" => { *self = EmailSubmissionComparator::SentAt; }, _ => { *self = EmailSubmissionComparator::_T(key.to_string()); } ); } else { let _ = map.next_value::()?; } Ok(()) } } impl Default for EmailSubmissionFilter { fn default() -> Self { EmailSubmissionFilter::_T("".to_string()) } } impl Default for EmailSubmissionComparator { fn default() -> Self { EmailSubmissionComparator::_T("".to_string()) } } impl From for EmailSubmissionValue { fn from(id: Id) -> Self { EmailSubmissionValue::Id(id) } } impl JmapObjectId for EmailSubmissionValue { fn as_id(&self) -> Option { match self { EmailSubmissionValue::Id(id) => Some(*id), _ => None, } } fn as_any_id(&self) -> Option { match self { EmailSubmissionValue::Id(id) => Some(AnyId::Id(*id)), EmailSubmissionValue::BlobId(blob_id) => Some(AnyId::BlobId(blob_id.clone())), _ => None, } } fn as_id_ref(&self) -> Option<&str> { if let EmailSubmissionValue::IdReference(r) = self { Some(r) } else { None } } fn try_set_id(&mut self, new_id: AnyId) -> bool { match new_id { AnyId::Id(id) => { *self = EmailSubmissionValue::Id(id); } AnyId::BlobId(blob_id) => { *self = EmailSubmissionValue::BlobId(blob_id); } } true } } impl JmapObjectId for EmailSubmissionProperty { fn as_id(&self) -> Option { None } fn as_any_id(&self) -> Option { None } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, _: AnyId) -> bool { false } } ================================================ FILE: crates/jmap-proto/src/object/file_node.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ object::{ AnyId, JmapObject, JmapObjectId, JmapRight, JmapSharedObject, MaybeReference, parse_ref, }, request::{MaybeInvalid, deserialize::DeserializeArguments}, types::date::UTCDate, }; use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, Property}; use std::{borrow::Cow, fmt::Display, str::FromStr}; use types::{acl::Acl, blob::BlobId, id::Id}; use utils::glob::GlobPattern; #[derive(Debug, Clone, Default)] pub struct FileNode; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum FileNodeProperty { Id, ParentId, BlobId, Size, Name, Type, Created, Modified, Accessed, Executable, MyRights, ShareWith, IsSubscribed, // Other IdValue(Id), Rights(FileNodeRight), Pointer(JsonPointer), } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum FileNodeRight { MayRead, MayWrite, MayShare, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum FileNodeValue { Id(Id), Date(UTCDate), BlobId(BlobId), IdReference(String), } impl Property for FileNodeProperty { fn try_parse(key: Option<&Key<'_, Self>>, value: &str) -> Option { let allow_patch = key.is_none(); if let Some(Key::Property(key)) = key { match key.patch_or_prop() { FileNodeProperty::ShareWith => { Id::from_str(value).ok().map(FileNodeProperty::IdValue) } _ => FileNodeProperty::parse(value, allow_patch), } } else { FileNodeProperty::parse(value, allow_patch) } } fn to_cow(&self) -> Cow<'static, str> { match self { FileNodeProperty::Id => "id", FileNodeProperty::ParentId => "parentId", FileNodeProperty::BlobId => "blobId", FileNodeProperty::Size => "size", FileNodeProperty::Name => "name", FileNodeProperty::Type => "type", FileNodeProperty::Created => "created", FileNodeProperty::Modified => "modified", FileNodeProperty::Accessed => "accessed", FileNodeProperty::Executable => "executable", FileNodeProperty::MyRights => "myRights", FileNodeProperty::ShareWith => "shareWith", FileNodeProperty::IsSubscribed => "isSubscribed", FileNodeProperty::Rights(file_right) => file_right.as_str(), FileNodeProperty::Pointer(json_pointer) => return json_pointer.to_string().into(), FileNodeProperty::IdValue(id) => return id.to_string().into(), } .into() } } impl FileNodeRight { pub fn as_str(&self) -> &'static str { match self { FileNodeRight::MayRead => "mayRead", FileNodeRight::MayWrite => "mayWrite", FileNodeRight::MayShare => "mayShare", } } } impl Element for FileNodeValue { type Property = FileNodeProperty; fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option { if let Key::Property(prop) = key { match prop.patch_or_prop() { FileNodeProperty::Id | FileNodeProperty::ParentId => match parse_ref(value) { MaybeReference::Value(v) => Some(FileNodeValue::Id(v)), MaybeReference::Reference(v) => Some(FileNodeValue::IdReference(v)), MaybeReference::ParseError => None, }, FileNodeProperty::BlobId => match parse_ref(value) { MaybeReference::Value(v) => Some(FileNodeValue::BlobId(v)), MaybeReference::Reference(v) => Some(FileNodeValue::IdReference(v)), MaybeReference::ParseError => None, }, FileNodeProperty::Created | FileNodeProperty::Modified | FileNodeProperty::Accessed => { UTCDate::from_str(value).ok().map(FileNodeValue::Date) } _ => None, } } else { None } } fn to_cow(&self) -> Cow<'static, str> { match self { FileNodeValue::Id(id) => id.to_string().into(), FileNodeValue::Date(utcdate) => utcdate.to_string().into(), FileNodeValue::BlobId(blob_id) => blob_id.to_string().into(), FileNodeValue::IdReference(r) => format!("#{r}").into(), } } } impl FileNodeProperty { fn parse(value: &str, allow_patch: bool) -> Option { hashify::tiny_map!(value.as_bytes(), b"id" => FileNodeProperty::Id, b"parentId" => FileNodeProperty::ParentId, b"blobId" => FileNodeProperty::BlobId, b"size" => FileNodeProperty::Size, b"name" => FileNodeProperty::Name, b"type" => FileNodeProperty::Type, b"created" => FileNodeProperty::Created, b"modified" => FileNodeProperty::Modified, b"accessed" => FileNodeProperty::Accessed, b"executable" => FileNodeProperty::Executable, b"myRights" => FileNodeProperty::MyRights, b"shareWith" => FileNodeProperty::ShareWith, b"isSubscribed" => FileNodeProperty::IsSubscribed, b"mayRead" => FileNodeProperty::Rights(FileNodeRight::MayRead), b"mayWrite" => FileNodeProperty::Rights(FileNodeRight::MayWrite), b"mayShare" => FileNodeProperty::Rights(FileNodeRight::MayShare), ) .or_else(|| { if allow_patch && value.contains('/') { FileNodeProperty::Pointer(JsonPointer::parse(value)).into() } else { None } }) } fn patch_or_prop(&self) -> &FileNodeProperty { if let FileNodeProperty::Pointer(ptr) = self && let Some(JsonPointerItem::Key(Key::Property(prop))) = ptr.last() { prop } else { self } } } #[derive(Debug, Clone, Default)] pub struct FileNodeSetArguments { pub on_destroy_remove_children: Option, } impl<'x> DeserializeArguments<'x> for FileNodeSetArguments { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'x>, { if key == "onDestroyRemoveChildren" { self.on_destroy_remove_children = map.next_value()?; } else { let _ = map.next_value::()?; } Ok(()) } } #[derive(Debug, Clone, Default)] pub struct FileNodeQueryArguments { pub depth: Option, } impl<'x> DeserializeArguments<'x> for FileNodeQueryArguments { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'x>, { if key == "depth" { self.depth = map.next_value()?; } else { let _ = map.next_value::()?; } Ok(()) } } impl FromStr for FileNodeProperty { type Err = (); fn from_str(s: &str) -> Result { FileNodeProperty::parse(s, false).ok_or(()) } } impl JmapObject for FileNode { type Property = FileNodeProperty; type Element = FileNodeValue; type Id = Id; type Filter = FileNodeFilter; type Comparator = FileNodeComparator; type GetArguments = (); type SetArguments<'de> = FileNodeSetArguments; type QueryArguments = FileNodeQueryArguments; type CopyArguments = (); type ParseArguments = (); const ID_PROPERTY: Self::Property = FileNodeProperty::Id; } impl JmapSharedObject for FileNode { type Right = FileNodeRight; const SHARE_WITH_PROPERTY: Self::Property = FileNodeProperty::ShareWith; } impl From for FileNodeProperty { fn from(id: Id) -> Self { FileNodeProperty::IdValue(id) } } impl JmapRight for FileNodeRight { fn to_acl(&self) -> &'static [Acl] { match self { FileNodeRight::MayShare => &[Acl::Share], FileNodeRight::MayRead => &[Acl::Read, Acl::ReadItems], FileNodeRight::MayWrite => &[ Acl::Modify, Acl::AddItems, Acl::ModifyItems, Acl::Delete, Acl::RemoveItems, ], } } fn all_rights() -> &'static [Self] { &[ FileNodeRight::MayRead, FileNodeRight::MayWrite, FileNodeRight::MayShare, ] } } impl From for FileNodeProperty { fn from(right: FileNodeRight) -> Self { FileNodeProperty::Rights(right) } } #[derive(Debug, Clone, PartialEq, Eq)] pub enum FileNodeFilter { HasParentId(bool), ParentId(MaybeInvalid), AncestorId(MaybeInvalid), HasType(bool), BlobId(MaybeInvalid), IsExecutable(bool), CreatedBefore(UTCDate), CreatedAfter(UTCDate), ModifiedBefore(UTCDate), ModifiedAfter(UTCDate), AccessedBefore(UTCDate), AccessedAfter(UTCDate), MinSize(u64), MaxSize(u64), Name(String), NameMatch(GlobPattern), Type(String), TypeMatch(GlobPattern), Text(String), Body(String), _T(String), } #[derive(Debug, Clone, PartialEq, Eq)] pub enum FileNodeComparator { Name, Size, Created, Modified, Type, _T(String), } impl<'de> DeserializeArguments<'de> for FileNodeFilter { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"hasParentId" => { *self = FileNodeFilter::HasParentId(map.next_value()?); }, b"parentId" => { *self = FileNodeFilter::ParentId(map.next_value()?); }, b"ancestorId" => { *self = FileNodeFilter::AncestorId(map.next_value()?); }, b"hasType" => { *self = FileNodeFilter::HasType(map.next_value()?); }, b"blobId" => { *self = FileNodeFilter::BlobId(map.next_value()?); }, b"isExecutable" => { *self = FileNodeFilter::IsExecutable(map.next_value()?); }, b"createdBefore" => { *self = FileNodeFilter::CreatedBefore(map.next_value()?); }, b"createdAfter" => { *self = FileNodeFilter::CreatedAfter(map.next_value()?); }, b"modifiedBefore" => { *self = FileNodeFilter::ModifiedBefore(map.next_value()?); }, b"modifiedAfter" => { *self = FileNodeFilter::ModifiedAfter(map.next_value()?); }, b"accessedBefore" => { *self = FileNodeFilter::AccessedBefore(map.next_value()?); }, b"accessedAfter" => { *self = FileNodeFilter::AccessedAfter(map.next_value()?); }, b"minSize" => { *self = FileNodeFilter::MinSize(map.next_value()?); }, b"maxSize" => { *self = FileNodeFilter::MaxSize(map.next_value()?); }, b"name" => { *self = FileNodeFilter::Name(map.next_value()?); }, b"nameMatch" => { *self = FileNodeFilter::NameMatch(map.next_value()?); }, b"type" => { *self = FileNodeFilter::Type(map.next_value()?); }, b"typeMatch" => { *self = FileNodeFilter::TypeMatch(map.next_value()?); }, b"body" => { *self = FileNodeFilter::Body(map.next_value()?); }, b"text" => { *self = FileNodeFilter::Text(map.next_value()?); }, _ => { *self = FileNodeFilter::_T(key.to_string()); let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> DeserializeArguments<'de> for FileNodeComparator { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { if key == "property" { let value = map.next_value::>()?; hashify::fnc_map!(value.as_bytes(), b"name" => { *self = FileNodeComparator::Name; }, b"size" => { *self = FileNodeComparator::Size; }, b"created" => { *self = FileNodeComparator::Created; }, b"modified" => { *self = FileNodeComparator::Modified; }, b"type" => { *self = FileNodeComparator::Type; }, _ => { *self = FileNodeComparator::_T(key.to_string()); } ); } else { let _ = map.next_value::()?; } Ok(()) } } impl Default for FileNodeFilter { fn default() -> Self { FileNodeFilter::_T("".to_string()) } } impl Default for FileNodeComparator { fn default() -> Self { FileNodeComparator::_T("".to_string()) } } impl From for FileNodeValue { fn from(id: Id) -> Self { FileNodeValue::Id(id) } } impl JmapObjectId for FileNodeValue { fn as_id(&self) -> Option { match self { FileNodeValue::Id(id) => Some(*id), _ => None, } } fn as_any_id(&self) -> Option { match self { FileNodeValue::Id(id) => Some(AnyId::Id(*id)), FileNodeValue::BlobId(blob_id) => Some(AnyId::BlobId(blob_id.clone())), _ => None, } } fn as_id_ref(&self) -> Option<&str> { if let FileNodeValue::IdReference(r) = self { Some(r) } else { None } } fn try_set_id(&mut self, new_id: AnyId) -> bool { match new_id { AnyId::Id(id) => { *self = FileNodeValue::Id(id); } AnyId::BlobId(blob_id) => { *self = FileNodeValue::BlobId(blob_id); } } true } } impl FileNodeFilter { pub fn into_string(self) -> Cow<'static, str> { match self { FileNodeFilter::HasParentId(_) => "hasParentId", FileNodeFilter::ParentId(_) => "parentId", FileNodeFilter::AncestorId(_) => "ancestorId", FileNodeFilter::HasType(_) => "hasType", FileNodeFilter::BlobId(_) => "blobId", FileNodeFilter::IsExecutable(_) => "isExecutable", FileNodeFilter::CreatedBefore(_) => "createdBefore", FileNodeFilter::CreatedAfter(_) => "createdAfter", FileNodeFilter::ModifiedBefore(_) => "modifiedBefore", FileNodeFilter::ModifiedAfter(_) => "modifiedAfter", FileNodeFilter::AccessedBefore(_) => "accessedBefore", FileNodeFilter::AccessedAfter(_) => "accessedAfter", FileNodeFilter::MinSize(_) => "minSize", FileNodeFilter::MaxSize(_) => "maxSize", FileNodeFilter::Name(_) => "name", FileNodeFilter::NameMatch(_) => "nameMatch", FileNodeFilter::Type(_) => "type", FileNodeFilter::TypeMatch(_) => "typeMatch", FileNodeFilter::Text(_) => "text", FileNodeFilter::Body(_) => "body", FileNodeFilter::_T(s) => return s.into(), } .into() } } impl FileNodeComparator { pub fn as_str(&self) -> &str { match self { FileNodeComparator::Name => "name", FileNodeComparator::Size => "size", FileNodeComparator::Created => "created", FileNodeComparator::Modified => "modified", FileNodeComparator::Type => "type", FileNodeComparator::_T(s) => s.as_ref(), } } pub fn into_string(self) -> Cow<'static, str> { match self { FileNodeComparator::Name => "name", FileNodeComparator::Size => "size", FileNodeComparator::Created => "created", FileNodeComparator::Modified => "modified", FileNodeComparator::Type => "type", FileNodeComparator::_T(s) => return s.into(), } .into() } } impl serde::Serialize for FileNodeComparator { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { serializer.serialize_str(self.as_str()) } } impl TryFrom for Id { type Error = (); fn try_from(value: FileNodeProperty) -> Result { if let FileNodeProperty::IdValue(id) = value { Ok(id) } else { Err(()) } } } impl TryFrom for FileNodeRight { type Error = (); fn try_from(value: FileNodeProperty) -> Result { if let FileNodeProperty::Rights(right) = value { Ok(right) } else { Err(()) } } } impl JmapObjectId for FileNodeProperty { fn as_id(&self) -> Option { if let FileNodeProperty::IdValue(id) = self { Some(*id) } else { None } } fn as_any_id(&self) -> Option { if let FileNodeProperty::IdValue(id) = self { Some(AnyId::Id(*id)) } else { None } } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, new_id: AnyId) -> bool { if let AnyId::Id(id) = new_id { *self = FileNodeProperty::IdValue(id); true } else { false } } } impl Display for FileNodeProperty { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.to_cow()) } } ================================================ FILE: crates/jmap-proto/src/object/identity.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::object::{AnyId, JmapObject, JmapObjectId}; use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, Property}; use std::{borrow::Cow, str::FromStr}; use types::id::Id; #[derive(Debug, Clone, Default)] pub struct Identity; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum IdentityProperty { Id, Name, Email, ReplyTo, Bcc, TextSignature, HtmlSignature, MayDelete, // Other Pointer(JsonPointer), } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum IdentityValue { Id(Id), } impl Property for IdentityProperty { fn try_parse(key: Option<&Key<'_, Self>>, value: &str) -> Option { IdentityProperty::parse(value, key.is_none()) } fn to_cow(&self) -> Cow<'static, str> { match self { IdentityProperty::Bcc => "bcc", IdentityProperty::Email => "email", IdentityProperty::HtmlSignature => "htmlSignature", IdentityProperty::Id => "id", IdentityProperty::MayDelete => "mayDelete", IdentityProperty::Name => "name", IdentityProperty::ReplyTo => "replyTo", IdentityProperty::TextSignature => "textSignature", IdentityProperty::Pointer(json_pointer) => return json_pointer.to_string().into(), } .into() } } impl Element for IdentityValue { type Property = IdentityProperty; fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option { if let Key::Property(prop) = key { match prop.patch_or_prop() { IdentityProperty::Id => Id::from_str(value).ok().map(IdentityValue::Id), _ => None, } } else { None } } fn to_cow(&self) -> Cow<'static, str> { match self { IdentityValue::Id(id) => id.to_string().into(), } } } impl IdentityProperty { fn parse(value: &str, allow_patch: bool) -> Option { hashify::tiny_map!(value.as_bytes(), b"id" => IdentityProperty::Id, b"name" => IdentityProperty::Name, b"email" => IdentityProperty::Email, b"replyTo" => IdentityProperty::ReplyTo, b"bcc" => IdentityProperty::Bcc, b"textSignature" => IdentityProperty::TextSignature, b"htmlSignature" => IdentityProperty::HtmlSignature, b"mayDelete" => IdentityProperty::MayDelete, ) .or_else(|| { if allow_patch && value.contains('/') { IdentityProperty::Pointer(JsonPointer::parse(value)).into() } else { None } }) } fn patch_or_prop(&self) -> &IdentityProperty { if let IdentityProperty::Pointer(ptr) = self && let Some(JsonPointerItem::Key(Key::Property(prop))) = ptr.last() { prop } else { self } } } impl FromStr for IdentityProperty { type Err = (); fn from_str(s: &str) -> Result { IdentityProperty::parse(s, false).ok_or(()) } } impl JmapObject for Identity { type Property = IdentityProperty; type Element = IdentityValue; type Id = Id; type Filter = (); type Comparator = (); type GetArguments = (); type SetArguments<'de> = (); type QueryArguments = (); type CopyArguments = (); type ParseArguments = (); const ID_PROPERTY: Self::Property = IdentityProperty::Id; } impl From for IdentityValue { fn from(id: Id) -> Self { IdentityValue::Id(id) } } impl JmapObjectId for IdentityValue { fn as_id(&self) -> Option { match self { IdentityValue::Id(id) => Some(*id), } } fn as_any_id(&self) -> Option { match self { IdentityValue::Id(id) => Some(AnyId::Id(*id)), } } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, new_id: AnyId) -> bool { if let AnyId::Id(id) = new_id { *self = IdentityValue::Id(id); true } else { false } } } impl JmapObjectId for IdentityProperty { fn as_id(&self) -> Option { None } fn as_any_id(&self) -> Option { None } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, _: AnyId) -> bool { false } } ================================================ FILE: crates/jmap-proto/src/object/mailbox.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ object::{ AnyId, JmapObject, JmapObjectId, JmapRight, JmapSharedObject, MaybeReference, parse_ref, }, request::{deserialize::DeserializeArguments, reference::MaybeIdReference}, }; use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, Property}; use std::{borrow::Cow, str::FromStr}; use types::{acl::Acl, id::Id, special_use::SpecialUse}; #[derive(Debug, Clone, Default)] pub struct Mailbox; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum MailboxProperty { Id, Name, ParentId, Role, SortOrder, TotalEmails, UnreadEmails, TotalThreads, UnreadThreads, ShareWith, MyRights, IsSubscribed, // Other IdValue(Id), Rights(MailboxRight), Pointer(JsonPointer), } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum MailboxRight { MayReadItems, MayAddItems, MayRemoveItems, MaySetSeen, MaySetKeywords, MayCreateChild, MayRename, MaySubmit, MayDelete, MayShare, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum MailboxValue { Id(Id), IdReference(String), Role(SpecialUse), } impl Property for MailboxProperty { fn try_parse(key: Option<&Key<'_, Self>>, value: &str) -> Option { let allow_patch = key.is_none(); if let Some(Key::Property(key)) = key { match key.patch_or_prop() { MailboxProperty::ShareWith => { Id::from_str(value).ok().map(MailboxProperty::IdValue) } _ => MailboxProperty::parse(value, allow_patch), } } else { MailboxProperty::parse(value, allow_patch) } } fn to_cow(&self) -> Cow<'static, str> { match self { MailboxProperty::Id => "id", MailboxProperty::IsSubscribed => "isSubscribed", MailboxProperty::MyRights => "myRights", MailboxProperty::Name => "name", MailboxProperty::ParentId => "parentId", MailboxProperty::Role => "role", MailboxProperty::SortOrder => "sortOrder", MailboxProperty::TotalEmails => "totalEmails", MailboxProperty::TotalThreads => "totalThreads", MailboxProperty::UnreadEmails => "unreadEmails", MailboxProperty::UnreadThreads => "unreadThreads", MailboxProperty::ShareWith => "shareWith", MailboxProperty::Rights(mailbox_right) => mailbox_right.as_str(), MailboxProperty::Pointer(json_pointer) => return json_pointer.to_string().into(), MailboxProperty::IdValue(id) => return id.to_string().into(), } .into() } } impl MailboxRight { pub fn as_str(&self) -> &'static str { match self { MailboxRight::MayReadItems => "mayReadItems", MailboxRight::MayAddItems => "mayAddItems", MailboxRight::MayRemoveItems => "mayRemoveItems", MailboxRight::MaySetSeen => "maySetSeen", MailboxRight::MaySetKeywords => "maySetKeywords", MailboxRight::MayCreateChild => "mayCreateChild", MailboxRight::MayRename => "mayRename", MailboxRight::MaySubmit => "maySubmit", MailboxRight::MayDelete => "mayDelete", MailboxRight::MayShare => "mayShare", } } } impl Element for MailboxValue { type Property = MailboxProperty; fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option { if let Key::Property(prop) = key { match prop.patch_or_prop() { MailboxProperty::Id | MailboxProperty::ParentId => match parse_ref(value) { MaybeReference::Value(v) => Some(MailboxValue::Id(v)), MaybeReference::Reference(v) => Some(MailboxValue::IdReference(v)), MaybeReference::ParseError => None, }, MailboxProperty::Role => SpecialUse::parse(value).map(MailboxValue::Role), _ => None, } } else { None } } fn to_cow(&self) -> Cow<'static, str> { match self { MailboxValue::Id(id) => id.to_string().into(), MailboxValue::IdReference(r) => format!("#{r}").into(), MailboxValue::Role(special_use) => special_use.as_str().unwrap_or_default().into(), } } } impl MailboxProperty { fn parse(value: &str, allow_patch: bool) -> Option { hashify::tiny_map!(value.as_bytes(), b"id" => MailboxProperty::Id, b"name" => MailboxProperty::Name, b"parentId" => MailboxProperty::ParentId, b"role" => MailboxProperty::Role, b"sortOrder" => MailboxProperty::SortOrder, b"totalEmails" => MailboxProperty::TotalEmails, b"unreadEmails" => MailboxProperty::UnreadEmails, b"totalThreads" => MailboxProperty::TotalThreads, b"unreadThreads" => MailboxProperty::UnreadThreads, b"shareWith" => MailboxProperty::ShareWith, b"myRights" => MailboxProperty::MyRights, b"mayReadItems" => MailboxProperty::Rights(MailboxRight::MayReadItems), b"mayAddItems" => MailboxProperty::Rights(MailboxRight::MayAddItems), b"mayRemoveItems" => MailboxProperty::Rights(MailboxRight::MayRemoveItems), b"maySetSeen" => MailboxProperty::Rights(MailboxRight::MaySetSeen), b"maySetKeywords" => MailboxProperty::Rights(MailboxRight::MaySetKeywords), b"mayCreateChild" => MailboxProperty::Rights(MailboxRight::MayCreateChild), b"mayRename" => MailboxProperty::Rights(MailboxRight::MayRename), b"maySubmit" => MailboxProperty::Rights(MailboxRight::MaySubmit), b"mayDelete" => MailboxProperty::Rights(MailboxRight::MayDelete), b"mayShare" => MailboxProperty::Rights(MailboxRight::MayShare), b"isSubscribed" => MailboxProperty::IsSubscribed, ) .or_else(|| { if allow_patch && value.contains('/') { MailboxProperty::Pointer(JsonPointer::parse(value)).into() } else { None } }) } fn patch_or_prop(&self) -> &MailboxProperty { if let MailboxProperty::Pointer(ptr) = self && let Some(JsonPointerItem::Key(Key::Property(prop))) = ptr.last() { prop } else { self } } } #[derive(Debug, Clone, Default)] pub struct MailboxSetArguments { pub on_destroy_remove_emails: Option, } #[derive(Debug, Clone, Default)] pub struct MailboxQueryArguments { pub sort_as_tree: Option, pub filter_as_tree: Option, } impl<'de> DeserializeArguments<'de> for MailboxSetArguments { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { if key == "onDestroyRemoveEmails" { self.on_destroy_remove_emails = map.next_value()?; } else { let _ = map.next_value::()?; } Ok(()) } } impl<'de> DeserializeArguments<'de> for MailboxQueryArguments { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"sortAsTree" => { self.sort_as_tree = map.next_value()?; }, b"filterAsTree" => { self.filter_as_tree = map.next_value()?; }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl FromStr for MailboxProperty { type Err = (); fn from_str(s: &str) -> Result { MailboxProperty::parse(s, false).ok_or(()) } } impl JmapObject for Mailbox { type Property = MailboxProperty; type Element = MailboxValue; type Id = Id; type Filter = MailboxFilter; type Comparator = MailboxComparator; type GetArguments = (); type SetArguments<'de> = MailboxSetArguments; type QueryArguments = MailboxQueryArguments; type CopyArguments = (); type ParseArguments = (); const ID_PROPERTY: Self::Property = MailboxProperty::Id; } impl JmapSharedObject for Mailbox { type Right = MailboxRight; const SHARE_WITH_PROPERTY: Self::Property = MailboxProperty::ShareWith; } impl From for MailboxProperty { fn from(id: Id) -> Self { MailboxProperty::IdValue(id) } } impl TryFrom for Id { type Error = (); fn try_from(value: MailboxProperty) -> Result { if let MailboxProperty::IdValue(id) = value { Ok(id) } else { Err(()) } } } impl TryFrom for MailboxRight { type Error = (); fn try_from(value: MailboxProperty) -> Result { if let MailboxProperty::Rights(right) = value { Ok(right) } else { Err(()) } } } #[derive(Debug, Clone, PartialEq, Eq)] pub enum MailboxFilter { Name(String), ParentId(Option>), Role(Option), HasAnyRole(bool), IsSubscribed(bool), _T(String), } #[derive(Debug, Clone, PartialEq, Eq)] pub enum MailboxComparator { SortOrder, Name, ParentId, _T(String), } impl<'de> DeserializeArguments<'de> for MailboxFilter { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"name" => { *self = MailboxFilter::Name(map.next_value()?); }, b"parentId" => { *self = MailboxFilter::ParentId(map.next_value()?); }, b"role" => { *self = MailboxFilter::Role(map.next_value::>()?.map(|r| r.0)); }, b"hasAnyRole" => { *self = MailboxFilter::HasAnyRole(map.next_value()?); }, b"isSubscribed" => { *self = MailboxFilter::IsSubscribed(map.next_value()?); }, _ => { *self = MailboxFilter::_T(key.to_string()); let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> DeserializeArguments<'de> for MailboxComparator { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { if key == "property" { let value = map.next_value::>()?; hashify::fnc_map!(value.as_bytes(), b"sortOrder" => { *self = MailboxComparator::SortOrder; }, b"name" => { *self = MailboxComparator::Name; }, b"parentId" => { *self = MailboxComparator::ParentId; }, _ => { *self = MailboxComparator::_T(key.to_string()); } ); } else { let _ = map.next_value::()?; } Ok(()) } } impl Default for MailboxFilter { fn default() -> Self { MailboxFilter::_T("".to_string()) } } impl Default for MailboxComparator { fn default() -> Self { MailboxComparator::_T("".to_string()) } } struct RoleWrapper(SpecialUse); impl<'de> serde::Deserialize<'de> for RoleWrapper { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { SpecialUse::parse(<&str>::deserialize(deserializer)?) .map(RoleWrapper) .ok_or_else(|| serde::de::Error::custom("invalid JMAP role")) } } impl From for MailboxValue { fn from(id: Id) -> Self { MailboxValue::Id(id) } } impl JmapObjectId for MailboxValue { fn as_id(&self) -> Option { if let MailboxValue::Id(id) = self { Some(*id) } else { None } } fn as_any_id(&self) -> Option { if let MailboxValue::Id(id) = self { Some(AnyId::Id(*id)) } else { None } } fn as_id_ref(&self) -> Option<&str> { if let MailboxValue::IdReference(r) = self { Some(r) } else { None } } fn try_set_id(&mut self, new_id: AnyId) -> bool { if let AnyId::Id(id) = new_id { *self = MailboxValue::Id(id); true } else { false } } } impl JmapRight for MailboxRight { fn to_acl(&self) -> &'static [Acl] { match self { MailboxRight::MayReadItems => &[Acl::Read, Acl::ReadItems], MailboxRight::MayAddItems => &[Acl::AddItems], MailboxRight::MayRemoveItems => &[Acl::RemoveItems], MailboxRight::MaySetSeen => &[Acl::ModifyItems], MailboxRight::MaySetKeywords => &[Acl::ModifyItems], MailboxRight::MayCreateChild => &[Acl::CreateChild], MailboxRight::MayRename => &[Acl::Modify], MailboxRight::MaySubmit => &[Acl::Submit], MailboxRight::MayDelete => &[Acl::Delete], MailboxRight::MayShare => &[Acl::Share], } } fn all_rights() -> &'static [Self] { &[ MailboxRight::MayReadItems, MailboxRight::MayAddItems, MailboxRight::MayRemoveItems, MailboxRight::MaySetSeen, MailboxRight::MaySetKeywords, MailboxRight::MayCreateChild, MailboxRight::MayRename, MailboxRight::MaySubmit, MailboxRight::MayDelete, MailboxRight::MayShare, ] } } impl From for MailboxProperty { fn from(right: MailboxRight) -> Self { MailboxProperty::Rights(right) } } impl JmapObjectId for MailboxProperty { fn as_id(&self) -> Option { if let MailboxProperty::IdValue(id) = self { Some(*id) } else { None } } fn as_any_id(&self) -> Option { if let MailboxProperty::IdValue(id) = self { Some(AnyId::Id(*id)) } else { None } } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, new_id: AnyId) -> bool { if let AnyId::Id(id) = new_id { *self = MailboxProperty::IdValue(id); true } else { false } } } ================================================ FILE: crates/jmap-proto/src/object/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::request::deserialize::DeserializeArguments; use jmap_tools::{Element, Null, Property}; use serde::Serialize; use std::{fmt::Debug, str::FromStr}; use types::{acl::Acl, blob::BlobId, id::Id}; pub mod addressbook; pub mod blob; pub mod calendar; pub mod calendar_event; pub mod calendar_event_notification; pub mod contact; pub mod email; pub mod email_submission; pub mod file_node; pub mod identity; pub mod mailbox; pub mod participant_identity; pub mod principal; pub mod push_subscription; pub mod quota; pub mod search_snippet; pub mod share_notification; pub mod sieve; pub mod thread; pub mod vacation_response; pub trait JmapObject: std::fmt::Debug { type Property: Property + JmapObjectId + FromStr + Debug + Sync + Send; type Element: Element + JmapObjectId + Debug + Sync + Send; type Id: FromStr + TryFrom + Into + Serialize + Debug + Sync + Send; type Filter: Default + for<'de> DeserializeArguments<'de> + Debug + Sync + Send; type Comparator: Default + for<'de> DeserializeArguments<'de> + Debug + Sync + Send; type GetArguments: Default + for<'de> DeserializeArguments<'de> + Debug + Sync + Send; type SetArguments<'de>: Default + DeserializeArguments<'de> + Debug + Sync + Send; type QueryArguments: Default + for<'de> DeserializeArguments<'de> + Debug + Sync + Send; type CopyArguments: Default + for<'de> DeserializeArguments<'de> + Debug + Sync + Send; type ParseArguments: Default + for<'de> DeserializeArguments<'de> + Debug + Sync + Send; const ID_PROPERTY: Self::Property; } pub trait JmapSharedObject: JmapObject { type Right: JmapRight + Into + Debug + Clone + Copy + Sync + Send; const SHARE_WITH_PROPERTY: Self::Property; } pub trait JmapRight: Clone + Copy + Sized + 'static { fn all_rights() -> &'static [Self]; fn to_acl(&self) -> &'static [Acl]; } #[derive(Debug, Clone, PartialEq, Eq, Serialize)] #[serde(untagged)] pub enum AnyId { Id(Id), BlobId(BlobId), } pub trait JmapObjectId { fn as_id(&self) -> Option; fn as_any_id(&self) -> Option; fn as_id_ref(&self) -> Option<&str>; fn try_set_id(&mut self, new_id: AnyId) -> bool; } #[derive(Debug, Clone, PartialEq, Eq)] enum MaybeReference { Value(T), Reference(String), ParseError, } fn parse_ref(value: &str) -> MaybeReference { if let Some(reference) = value.strip_prefix('#') { MaybeReference::Reference(reference.to_string()) } else { T::from_str(value) .map(MaybeReference::Value) .unwrap_or(MaybeReference::ParseError) } } impl From for AnyId { fn from(value: Id) -> Self { AnyId::Id(value) } } impl From for AnyId { fn from(value: BlobId) -> Self { AnyId::BlobId(value) } } impl TryFrom for Id { type Error = (); fn try_from(value: AnyId) -> Result { if let AnyId::Id(id) = value { Ok(id) } else { Err(()) } } } impl TryFrom for BlobId { type Error = (); fn try_from(value: AnyId) -> Result { if let AnyId::BlobId(id) = value { Ok(id) } else { Err(()) } } } impl<'de> serde::Deserialize<'de> for AnyId { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { let value = <&str>::deserialize(deserializer)?; if let Some(blob_id) = BlobId::from_base32(value) { Ok(AnyId::BlobId(blob_id)) } else if let Ok(id) = Id::from_str(value) { Ok(AnyId::Id(id)) } else { Err(serde::de::Error::custom(format!( "Invalid AnyId: {}", value ))) } } } #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)] pub struct NullObject; impl JmapObject for NullObject { type Property = Null; type Element = Null; type Id = Null; type Filter = (); type Comparator = (); type GetArguments = (); type SetArguments<'de> = (); type QueryArguments = (); type CopyArguments = (); type ParseArguments = (); const ID_PROPERTY: Self::Property = Null; } impl JmapRight for Null { fn all_rights() -> &'static [Self] { unreachable!() } fn to_acl(&self) -> &'static [Acl] { unreachable!() } } impl FromStr for NullObject { type Err = (); fn from_str(_: &str) -> Result { unreachable!() } } impl JmapObjectId for Null { fn as_id(&self) -> Option { unreachable!() } fn as_any_id(&self) -> Option { unreachable!() } fn as_id_ref(&self) -> Option<&str> { unreachable!() } fn try_set_id(&mut self, _: AnyId) -> bool { unreachable!() } } impl TryFrom for Null { type Error = (); fn try_from(_: AnyId) -> Result { unreachable!() } } ================================================ FILE: crates/jmap-proto/src/object/participant_identity.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ object::{AnyId, JmapObject, JmapObjectId}, request::{deserialize::DeserializeArguments, reference::MaybeIdReference}, }; use jmap_tools::{Element, Key, Property}; use std::{borrow::Cow, fmt::Display, str::FromStr}; use types::id::Id; #[derive(Debug, Clone, Default)] pub struct ParticipantIdentity; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum ParticipantIdentityProperty { Id, Name, CalendarAddress, IsDefault, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum ParticipantIdentityValue { Id(Id), } impl Property for ParticipantIdentityProperty { fn try_parse(_: Option<&Key<'_, Self>>, value: &str) -> Option { ParticipantIdentityProperty::parse(value) } fn to_cow(&self) -> Cow<'static, str> { match self { ParticipantIdentityProperty::Id => "id", ParticipantIdentityProperty::Name => "name", ParticipantIdentityProperty::CalendarAddress => "calendarAddress", ParticipantIdentityProperty::IsDefault => "isDefault", } .into() } } impl Element for ParticipantIdentityValue { type Property = ParticipantIdentityProperty; fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option { if let Key::Property(prop) = key { match prop { ParticipantIdentityProperty::Id => { Id::from_str(value).ok().map(ParticipantIdentityValue::Id) } _ => None, } } else { None } } fn to_cow(&self) -> Cow<'static, str> { match self { ParticipantIdentityValue::Id(id) => id.to_string().into(), } } } impl ParticipantIdentityProperty { fn parse(value: &str) -> Option { hashify::tiny_map!(value.as_bytes(), b"id" => ParticipantIdentityProperty::Id, b"name" => ParticipantIdentityProperty::Name, b"calendarAddress" => ParticipantIdentityProperty::CalendarAddress, b"isDefault" => ParticipantIdentityProperty::IsDefault ) } fn as_str(&self) -> &'static str { match self { ParticipantIdentityProperty::Id => "id", ParticipantIdentityProperty::Name => "name", ParticipantIdentityProperty::CalendarAddress => "calendarAddress", ParticipantIdentityProperty::IsDefault => "isDefault", } } } #[derive(Debug, Clone, Default)] pub struct ParticipantIdentitySetArguments { pub on_success_set_is_default: Option>, } impl<'de> DeserializeArguments<'de> for ParticipantIdentitySetArguments { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"onSuccessSetIsDefault" => { self.on_success_set_is_default = map.next_value()?; }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl FromStr for ParticipantIdentityProperty { type Err = (); fn from_str(s: &str) -> Result { ParticipantIdentityProperty::parse(s).ok_or(()) } } impl JmapObject for ParticipantIdentity { type Property = ParticipantIdentityProperty; type Element = ParticipantIdentityValue; type Id = Id; type Filter = (); type Comparator = (); type GetArguments = (); type SetArguments<'de> = ParticipantIdentitySetArguments; type QueryArguments = (); type CopyArguments = (); type ParseArguments = (); const ID_PROPERTY: Self::Property = ParticipantIdentityProperty::Id; } impl TryFrom for Id { type Error = (); fn try_from(_: ParticipantIdentityProperty) -> Result { Err(()) } } impl From for ParticipantIdentityValue { fn from(id: Id) -> Self { ParticipantIdentityValue::Id(id) } } impl JmapObjectId for ParticipantIdentityValue { fn as_id(&self) -> Option { let ParticipantIdentityValue::Id(id) = self; Some(*id) } fn as_any_id(&self) -> Option { let ParticipantIdentityValue::Id(id) = self; Some(AnyId::Id(*id)) } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, new_id: AnyId) -> bool { if let AnyId::Id(new_id) = new_id { *self = ParticipantIdentityValue::Id(new_id); return true; } false } } impl JmapObjectId for ParticipantIdentityProperty { fn as_id(&self) -> Option { None } fn as_any_id(&self) -> Option { None } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, _: AnyId) -> bool { false } } impl Display for ParticipantIdentityProperty { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { self.as_str().fmt(f) } } ================================================ FILE: crates/jmap-proto/src/object/principal.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use jmap_tools::{Element, Key, Property}; use std::{borrow::Cow, fmt::Display, str::FromStr}; use types::id::Id; use crate::{ object::{AnyId, JmapObject, JmapObjectId}, request::{capability::Capability, deserialize::DeserializeArguments}, }; #[derive(Debug, Clone, Default)] pub struct Principal; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum PrincipalProperty { Id, Type, Name, Description, Email, Timezone, Capabilities, Accounts, IdValue(Id), Capability(Capability), } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum PrincipalValue { Id(Id), Type(PrincipalType), } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum PrincipalType { Individual, Group, Resource, Location, Other, } impl Property for PrincipalProperty { fn try_parse(_: Option<&Key<'_, Self>>, value: &str) -> Option { PrincipalProperty::parse(value) } fn to_cow(&self) -> Cow<'static, str> { match self { PrincipalProperty::Capabilities => "capabilities", PrincipalProperty::Description => "description", PrincipalProperty::Email => "email", PrincipalProperty::Id => "id", PrincipalProperty::Name => "name", PrincipalProperty::Timezone => "timezone", PrincipalProperty::Type => "type", PrincipalProperty::Accounts => "accounts", PrincipalProperty::Capability(cap) => cap.as_str(), PrincipalProperty::IdValue(id) => return id.to_string().into(), } .into() } } impl Element for PrincipalValue { type Property = PrincipalProperty; fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option { if let Key::Property(prop) = key { match prop { PrincipalProperty::Id => Id::from_str(value).ok().map(PrincipalValue::Id), PrincipalProperty::Type => PrincipalType::parse(value).map(PrincipalValue::Type), _ => None, } } else { None } } fn to_cow(&self) -> Cow<'static, str> { match self { PrincipalValue::Id(id) => id.to_string().into(), PrincipalValue::Type(t) => t.as_str().into(), } } } impl PrincipalProperty { pub fn parse(value: &str) -> Option { hashify::tiny_map!(value.as_bytes(), b"id" => PrincipalProperty::Id, b"type" => PrincipalProperty::Type, b"name" => PrincipalProperty::Name, b"description" => PrincipalProperty::Description, b"email" => PrincipalProperty::Email, b"timeZone" => PrincipalProperty::Timezone, b"capabilities" => PrincipalProperty::Capabilities, b"accounts" => PrincipalProperty::Accounts, ) } pub fn as_str(&self) -> &'static str { match self { PrincipalProperty::Id => "id", PrincipalProperty::Type => "type", PrincipalProperty::Name => "name", PrincipalProperty::Description => "description", PrincipalProperty::Email => "email", PrincipalProperty::Timezone => "timeZone", PrincipalProperty::Capabilities => "capabilities", PrincipalProperty::Accounts => "accounts", PrincipalProperty::Capability(cap) => cap.as_str(), PrincipalProperty::IdValue(_) => "", } } } impl PrincipalType { pub fn parse(s: &str) -> Option { hashify::tiny_map!(s.as_bytes(), b"individual" => PrincipalType::Individual, b"group" => PrincipalType::Group, b"resource" => PrincipalType::Resource, b"location" => PrincipalType::Location, b"other" => PrincipalType::Other, ) } pub fn as_str(&self) -> &'static str { match self { PrincipalType::Individual => "individual", PrincipalType::Group => "group", PrincipalType::Resource => "resource", PrincipalType::Location => "location", PrincipalType::Other => "other", } } } impl FromStr for PrincipalProperty { type Err = (); fn from_str(s: &str) -> Result { PrincipalProperty::parse(s).ok_or(()) } } impl JmapObject for Principal { type Property = PrincipalProperty; type Element = PrincipalValue; type Id = Id; type Filter = PrincipalFilter; type Comparator = PrincipalComparator; type GetArguments = (); type SetArguments<'de> = (); type QueryArguments = (); type CopyArguments = (); type ParseArguments = (); const ID_PROPERTY: Self::Property = PrincipalProperty::Id; } #[derive(Debug, Clone, PartialEq, Eq)] pub enum PrincipalFilter { AccountIds(Vec), Email(String), Name(String), Text(String), Type(PrincipalType), Timezone(String), _T(String), } #[derive(Debug, Clone, PartialEq, Eq)] pub enum PrincipalComparator { Name, Email, Type, _T(String), } impl<'de> DeserializeArguments<'de> for PrincipalFilter { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"accountIds" => { *self = PrincipalFilter::AccountIds(map.next_value()?); }, b"email" => { *self = PrincipalFilter::Email(map.next_value()?); }, b"name" => { *self = PrincipalFilter::Name(map.next_value()?); }, b"text" => { *self = PrincipalFilter::Text(map.next_value()?); }, b"type" => { *self = PrincipalFilter::Type(map.next_value()?); }, b"timeZone" => { *self = PrincipalFilter::Timezone(map.next_value()?); }, _ => { *self = PrincipalFilter::_T(key.to_string()); let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> DeserializeArguments<'de> for PrincipalComparator { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { if key == "property" { let value = map.next_value::>()?; hashify::fnc_map!(value.as_bytes(), b"name" => { *self = PrincipalComparator::Name; }, b"email" => { *self = PrincipalComparator::Email; }, b"type" => { *self = PrincipalComparator::Type; }, _ => { *self = PrincipalComparator::_T(key.to_string()); } ); } else { let _ = map.next_value::()?; } Ok(()) } } impl Default for PrincipalFilter { fn default() -> Self { PrincipalFilter::_T("".to_string()) } } impl Default for PrincipalComparator { fn default() -> Self { PrincipalComparator::_T("".to_string()) } } impl<'de> serde::Deserialize<'de> for PrincipalType { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { PrincipalType::parse(<&str>::deserialize(deserializer)?) .ok_or_else(|| serde::de::Error::custom("invalid JMAP PrincipalType")) } } impl From for PrincipalValue { fn from(id: Id) -> Self { PrincipalValue::Id(id) } } impl JmapObjectId for PrincipalValue { fn as_id(&self) -> Option { if let PrincipalValue::Id(id) = self { Some(*id) } else { None } } fn as_any_id(&self) -> Option { if let PrincipalValue::Id(id) = self { Some(AnyId::Id(*id)) } else { None } } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, new_id: AnyId) -> bool { if let AnyId::Id(id) = new_id { *self = PrincipalValue::Id(id); true } else { false } } } impl Display for PrincipalFilter { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(match self { PrincipalFilter::AccountIds(_) => "accountIds", PrincipalFilter::Email(_) => "email", PrincipalFilter::Name(_) => "name", PrincipalFilter::Text(_) => "text", PrincipalFilter::Type(_) => "type", PrincipalFilter::Timezone(_) => "timezone", PrincipalFilter::_T(other) => other, }) } } impl JmapObjectId for PrincipalProperty { fn as_id(&self) -> Option { None } fn as_any_id(&self) -> Option { None } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, _: AnyId) -> bool { false } } impl Display for PrincipalProperty { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(self.as_str()) } } ================================================ FILE: crates/jmap-proto/src/object/push_subscription.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::object::{AnyId, JmapObject, JmapObjectId}; use crate::types::date::UTCDate; use jmap_tools::{Element, JsonPointer, JsonPointerItem}; use jmap_tools::{Key, Property}; use std::borrow::Cow; use std::str::FromStr; use types::{id::Id, type_state::DataType}; #[derive(Debug, Clone, Default)] pub struct PushSubscription; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum PushSubscriptionProperty { Id, DeviceClientId, Url, Keys, P256dh, Auth, VerificationCode, Expires, Types, // Other Pointer(JsonPointer), } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum PushSubscriptionValue { Id(Id), Date(UTCDate), Types(DataType), } impl Property for PushSubscriptionProperty { fn try_parse(key: Option<&Key<'_, Self>>, value: &str) -> Option { PushSubscriptionProperty::parse(value, key.is_none()) } fn to_cow(&self) -> Cow<'static, str> { match self { PushSubscriptionProperty::DeviceClientId => "deviceClientId", PushSubscriptionProperty::Expires => "expires", PushSubscriptionProperty::Id => "id", PushSubscriptionProperty::Keys => "keys", PushSubscriptionProperty::Types => "types", PushSubscriptionProperty::Url => "url", PushSubscriptionProperty::VerificationCode => "verificationCode", PushSubscriptionProperty::P256dh => "p256dh", PushSubscriptionProperty::Auth => "auth", PushSubscriptionProperty::Pointer(json_pointer) => { return json_pointer.to_string().into(); } } .into() } } impl PushSubscriptionProperty { fn parse(value: &str, allow_patch: bool) -> Option { hashify::tiny_map!(value.as_bytes(), b"id" => PushSubscriptionProperty::Id, b"deviceClientId" => PushSubscriptionProperty::DeviceClientId, b"url" => PushSubscriptionProperty::Url, b"keys" => PushSubscriptionProperty::Keys, b"p256dh" => PushSubscriptionProperty::P256dh, b"auth" => PushSubscriptionProperty::Auth, b"verificationCode" => PushSubscriptionProperty::VerificationCode, b"expires" => PushSubscriptionProperty::Expires, b"types" => PushSubscriptionProperty::Types, ) .or_else(|| { if allow_patch && value.contains('/') { PushSubscriptionProperty::Pointer(JsonPointer::parse(value)).into() } else { None } }) } fn patch_or_prop(&self) -> &PushSubscriptionProperty { if let PushSubscriptionProperty::Pointer(ptr) = self && let Some(JsonPointerItem::Key(Key::Property(prop))) = ptr.last() { prop } else { self } } } impl Element for PushSubscriptionValue { type Property = PushSubscriptionProperty; fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option { if let Key::Property(prop) = key { match prop.patch_or_prop() { PushSubscriptionProperty::Id => { Id::from_str(value).ok().map(PushSubscriptionValue::Id) } PushSubscriptionProperty::Types => { DataType::parse(value).map(PushSubscriptionValue::Types) } PushSubscriptionProperty::Expires => UTCDate::from_str(value) .ok() .map(PushSubscriptionValue::Date), _ => None, } } else { None } } fn to_cow(&self) -> Cow<'static, str> { match self { PushSubscriptionValue::Id(id) => id.to_string().into(), PushSubscriptionValue::Date(utcdate) => utcdate.to_string().into(), PushSubscriptionValue::Types(data_type) => data_type.as_str().into(), } } } impl FromStr for PushSubscriptionProperty { type Err = (); fn from_str(s: &str) -> Result { PushSubscriptionProperty::parse(s, false).ok_or(()) } } impl JmapObject for PushSubscription { type Property = PushSubscriptionProperty; type Element = PushSubscriptionValue; type Id = Id; type Filter = (); type Comparator = (); type GetArguments = (); type SetArguments<'de> = (); type QueryArguments = (); type CopyArguments = (); type ParseArguments = (); const ID_PROPERTY: Self::Property = PushSubscriptionProperty::Id; } impl From for PushSubscriptionValue { fn from(id: Id) -> Self { PushSubscriptionValue::Id(id) } } impl JmapObjectId for PushSubscriptionValue { fn as_id(&self) -> Option { match self { PushSubscriptionValue::Id(id) => Some(*id), _ => None, } } fn as_any_id(&self) -> Option { match self { PushSubscriptionValue::Id(id) => Some(AnyId::Id(*id)), _ => None, } } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, new_id: AnyId) -> bool { if let AnyId::Id(id) = new_id { *self = PushSubscriptionValue::Id(id); true } else { false } } } impl JmapObjectId for PushSubscriptionProperty { fn as_id(&self) -> Option { None } fn as_any_id(&self) -> Option { None } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, _: AnyId) -> bool { false } } ================================================ FILE: crates/jmap-proto/src/object/quota.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ object::{AnyId, JmapObject, JmapObjectId}, request::deserialize::DeserializeArguments, }; use jmap_tools::{Element, Key, Property}; use std::{borrow::Cow, str::FromStr}; use types::{id::Id, type_state::DataType}; #[derive(Debug, Clone, Default)] pub struct Quota; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum QuotaProperty { Id, ResourceType, Used, Name, Scope, Types, HardLimit, WarnLimit, SoftLimit, Description, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum QuotaValue { Id(Id), Types(DataType), } impl Property for QuotaProperty { fn try_parse(_: Option<&Key<'_, Self>>, value: &str) -> Option { QuotaProperty::parse(value) } fn to_cow(&self) -> Cow<'static, str> { match self { QuotaProperty::Description => "description", QuotaProperty::Id => "id", QuotaProperty::Name => "name", QuotaProperty::Types => "types", QuotaProperty::ResourceType => "resourceType", QuotaProperty::Used => "used", QuotaProperty::HardLimit => "hardLimit", QuotaProperty::Scope => "scope", QuotaProperty::WarnLimit => "warnLimit", QuotaProperty::SoftLimit => "softLimit", } .into() } } impl QuotaProperty { fn parse(value: &str) -> Option { hashify::tiny_map!(value.as_bytes(), b"id" => QuotaProperty::Id, b"resourceType" => QuotaProperty::ResourceType, b"used" => QuotaProperty::Used, b"name" => QuotaProperty::Name, b"scope" => QuotaProperty::Scope, b"types" => QuotaProperty::Types, b"hardLimit" => QuotaProperty::HardLimit, b"warnLimit" => QuotaProperty::WarnLimit, b"softLimit" => QuotaProperty::SoftLimit, b"description" => QuotaProperty::Description, ) } } impl Element for QuotaValue { type Property = QuotaProperty; fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option { if let Key::Property(prop) = key { match prop { QuotaProperty::Id => Id::from_str(value).ok().map(QuotaValue::Id), QuotaProperty::Types => DataType::parse(value).map(QuotaValue::Types), _ => None, } } else { None } } fn to_cow(&self) -> Cow<'static, str> { match self { QuotaValue::Id(id) => id.to_string().into(), QuotaValue::Types(data_type) => data_type.as_str().into(), } } } impl FromStr for QuotaProperty { type Err = (); fn from_str(s: &str) -> Result { QuotaProperty::parse(s).ok_or(()) } } impl JmapObject for Quota { type Property = QuotaProperty; type Element = QuotaValue; type Id = Id; type Filter = QuotaFilter; type Comparator = QuotaComparator; type GetArguments = (); type SetArguments<'de> = (); type QueryArguments = (); type CopyArguments = (); type ParseArguments = (); const ID_PROPERTY: Self::Property = QuotaProperty::Id; } #[derive(Debug, Clone, PartialEq, Eq)] pub enum QuotaFilter { Name(String), Type(String), Scope(String), ResourceType(String), _T(String), } #[derive(Debug, Clone, PartialEq, Eq)] pub enum QuotaComparator { Name, Type, Used, _T(String), } impl<'de> DeserializeArguments<'de> for QuotaFilter { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"name" => { *self = QuotaFilter::Name(map.next_value()?); }, b"type" => { *self = QuotaFilter::Type(map.next_value()?); }, b"scope" => { *self = QuotaFilter::Scope(map.next_value()?); }, b"resourceType" => { *self = QuotaFilter::ResourceType(map.next_value()?); }, _ => { *self = QuotaFilter::_T(key.to_string()); let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> DeserializeArguments<'de> for QuotaComparator { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { if key == "property" { let value = map.next_value::>()?; hashify::fnc_map!(value.as_bytes(), b"name" => { *self = QuotaComparator::Name; }, b"type" => { *self = QuotaComparator::Type; }, b"used" => { *self = QuotaComparator::Used; }, _ => { *self = QuotaComparator::_T(key.to_string()); } ); } else { let _ = map.next_value::()?; } Ok(()) } } impl Default for QuotaFilter { fn default() -> Self { QuotaFilter::_T("".to_string()) } } impl Default for QuotaComparator { fn default() -> Self { QuotaComparator::_T("".to_string()) } } impl From for QuotaValue { fn from(id: Id) -> Self { QuotaValue::Id(id) } } impl JmapObjectId for QuotaValue { fn as_id(&self) -> Option { if let QuotaValue::Id(id) = self { Some(*id) } else { None } } fn as_any_id(&self) -> Option { self.as_id().map(AnyId::Id) } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, new_id: AnyId) -> bool { if let AnyId::Id(id) = new_id { *self = QuotaValue::Id(id); true } else { false } } } impl JmapObjectId for QuotaProperty { fn as_id(&self) -> Option { None } fn as_any_id(&self) -> Option { None } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, _: AnyId) -> bool { false } } ================================================ FILE: crates/jmap-proto/src/object/search_snippet.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use jmap_tools::{Element, Key, Property}; use std::{borrow::Cow, str::FromStr}; use types::id::Id; #[derive(Debug, Clone, Default)] pub struct SearchSnippet; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum SearchSnippetProperty { EmailId, Subject, Preview, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum SearchSnippetValue { Id(Id), } impl Property for SearchSnippetProperty { fn try_parse(_: Option<&Key<'_, Self>>, value: &str) -> Option { SearchSnippetProperty::parse(value) } fn to_cow(&self) -> Cow<'static, str> { match self { SearchSnippetProperty::Preview => "preview", SearchSnippetProperty::Subject => "subject", SearchSnippetProperty::EmailId => "emailId", } .into() } } impl Element for SearchSnippetValue { type Property = SearchSnippetProperty; fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option { if let Key::Property(prop) = key { match prop { SearchSnippetProperty::EmailId => { Id::from_str(value).ok().map(SearchSnippetValue::Id) } _ => None, } } else { None } } fn to_cow(&self) -> Cow<'static, str> { match self { SearchSnippetValue::Id(id) => id.to_string().into(), } } } impl SearchSnippetProperty { fn parse(value: &str) -> Option { hashify::tiny_map!(value.as_bytes(), b"emailId" => SearchSnippetProperty::EmailId, b"subject" => SearchSnippetProperty::Subject, b"preview" => SearchSnippetProperty::Preview, ) } } ================================================ FILE: crates/jmap-proto/src/object/share_notification.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ object::{AnyId, JmapObject, JmapObjectId}, request::deserialize::DeserializeArguments, types::date::UTCDate, }; use jmap_tools::{Element, Key, Property}; use std::{borrow::Cow, fmt::Display, str::FromStr}; use types::{id::Id, type_state::DataType}; #[derive(Debug, Clone, Default)] pub struct ShareNotification; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum ShareNotificationProperty { Id, Created, ChangedBy, ChangedByName, ChangedByEmail, ChangedByPrincipalId, ObjectType, ObjectAccountId, ObjectId, OldRights, NewRights, Name, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum ShareNotificationValue { Id(Id), Date(UTCDate), ObjectType(DataType), } impl Property for ShareNotificationProperty { fn try_parse(_: Option<&Key<'_, Self>>, value: &str) -> Option { ShareNotificationProperty::parse(value) } fn to_cow(&self) -> Cow<'static, str> { match self { ShareNotificationProperty::Id => "id", ShareNotificationProperty::Created => "created", ShareNotificationProperty::ChangedBy => "changedBy", ShareNotificationProperty::ChangedByName => "name", ShareNotificationProperty::ChangedByEmail => "email", ShareNotificationProperty::ChangedByPrincipalId => "principalId", ShareNotificationProperty::ObjectType => "objectType", ShareNotificationProperty::ObjectAccountId => "objectAccountId", ShareNotificationProperty::ObjectId => "objectId", ShareNotificationProperty::OldRights => "oldRights", ShareNotificationProperty::NewRights => "newRights", ShareNotificationProperty::Name => "name", } .into() } } impl Element for ShareNotificationValue { type Property = ShareNotificationProperty; fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option { if let Key::Property(prop) = key { match prop { ShareNotificationProperty::Id | ShareNotificationProperty::ChangedByPrincipalId | ShareNotificationProperty::ObjectAccountId | ShareNotificationProperty::ObjectId => { Id::from_str(value).ok().map(ShareNotificationValue::Id) } ShareNotificationProperty::Created => UTCDate::from_str(value) .ok() .map(ShareNotificationValue::Date), _ => None, } } else { None } } fn to_cow(&self) -> Cow<'static, str> { match self { ShareNotificationValue::Id(id) => id.to_string().into(), ShareNotificationValue::Date(date) => date.to_string().into(), ShareNotificationValue::ObjectType(ty) => ty.as_str().into(), } } } impl ShareNotificationProperty { fn parse(value: &str) -> Option { hashify::tiny_map!(value.as_bytes(), b"id" => ShareNotificationProperty::Id, b"created" => ShareNotificationProperty::Created, b"changedBy" => ShareNotificationProperty::ChangedBy, b"name" => ShareNotificationProperty::ChangedByName, b"email" => ShareNotificationProperty::ChangedByEmail, b"principalId" => ShareNotificationProperty::ChangedByPrincipalId, b"objectType" => ShareNotificationProperty::ObjectType, b"objectAccountId" => ShareNotificationProperty::ObjectAccountId, b"objectId" => ShareNotificationProperty::ObjectId, b"oldRights" => ShareNotificationProperty::OldRights, b"newRights" => ShareNotificationProperty::NewRights ) } } impl FromStr for ShareNotificationProperty { type Err = (); fn from_str(s: &str) -> Result { ShareNotificationProperty::parse(s).ok_or(()) } } impl JmapObject for ShareNotification { type Property = ShareNotificationProperty; type Element = ShareNotificationValue; type Id = Id; type Filter = ShareNotificationFilter; type Comparator = ShareNotificationComparator; type GetArguments = (); type SetArguments<'de> = (); type QueryArguments = (); type CopyArguments = (); type ParseArguments = (); const ID_PROPERTY: Self::Property = ShareNotificationProperty::Id; } #[derive(Debug, Clone, PartialEq, Eq)] pub enum ShareNotificationFilter { After(UTCDate), Before(UTCDate), ObjectType(DataType), ObjectAccountId(Id), _T(String), } #[derive(Debug, Clone, PartialEq, Eq)] pub enum ShareNotificationComparator { Created, _T(String), } impl<'de> DeserializeArguments<'de> for ShareNotificationFilter { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"after" => { *self = ShareNotificationFilter::After(map.next_value()?); }, b"before" => { *self = ShareNotificationFilter::Before(map.next_value()?); }, b"objectType" => { *self = ShareNotificationFilter::ObjectType(map.next_value()?); }, b"objectAccountId" => { *self = ShareNotificationFilter::ObjectAccountId(map.next_value()?); }, _ => { *self = ShareNotificationFilter::_T(key.to_string()); let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> DeserializeArguments<'de> for ShareNotificationComparator { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { if key == "property" { let value = map.next_value::>()?; hashify::fnc_map!(value.as_bytes(), b"created" => { *self = ShareNotificationComparator::Created; }, _ => { *self = ShareNotificationComparator::_T(value.to_string()); } ); } else { let _ = map.next_value::()?; } Ok(()) } } impl ShareNotificationFilter { pub fn into_string(self) -> Cow<'static, str> { match self { ShareNotificationFilter::After(_) => "after", ShareNotificationFilter::Before(_) => "before", ShareNotificationFilter::ObjectType(_) => "objectType", ShareNotificationFilter::ObjectAccountId(_) => "objectAccountId", ShareNotificationFilter::_T(s) => return Cow::Owned(s), } .into() } } impl ShareNotificationComparator { pub fn into_string(self) -> Cow<'static, str> { match self { ShareNotificationComparator::Created => "created", ShareNotificationComparator::_T(s) => return Cow::Owned(s), } .into() } } impl Default for ShareNotificationFilter { fn default() -> Self { ShareNotificationFilter::_T(String::new()) } } impl Default for ShareNotificationComparator { fn default() -> Self { ShareNotificationComparator::_T(String::new()) } } impl TryFrom for Id { type Error = (); fn try_from(_: ShareNotificationProperty) -> Result { Err(()) } } impl From for ShareNotificationValue { fn from(id: Id) -> Self { ShareNotificationValue::Id(id) } } impl JmapObjectId for ShareNotificationValue { fn as_id(&self) -> Option { if let ShareNotificationValue::Id(id) = self { Some(*id) } else { None } } fn as_any_id(&self) -> Option { if let ShareNotificationValue::Id(id) = self { Some(AnyId::Id(*id)) } else { None } } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, _: AnyId) -> bool { false } } impl JmapObjectId for ShareNotificationProperty { fn as_id(&self) -> Option { None } fn as_any_id(&self) -> Option { None } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, _: AnyId) -> bool { false } } impl Display for ShareNotificationProperty { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.to_cow()) } } ================================================ FILE: crates/jmap-proto/src/object/sieve.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ object::{AnyId, DeserializeArguments, JmapObject, JmapObjectId, MaybeReference, parse_ref}, request::reference::MaybeIdReference, }; use jmap_tools::{Element, Key, Property}; use std::{borrow::Cow, str::FromStr}; use types::{blob::BlobId, id::Id}; #[derive(Debug, Clone, Default)] pub struct Sieve; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum SieveProperty { Id, Name, BlobId, IsActive, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum SieveValue { Id(Id), BlobId(BlobId), IdReference(String), } impl Property for SieveProperty { fn try_parse(_: Option<&Key<'_, Self>>, value: &str) -> Option { SieveProperty::parse(value) } fn to_cow(&self) -> Cow<'static, str> { match self { SieveProperty::BlobId => "blobId", SieveProperty::Id => "id", SieveProperty::Name => "name", SieveProperty::IsActive => "isActive", } .into() } } impl Element for SieveValue { type Property = SieveProperty; fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option { if let Key::Property(prop) = key { match prop { SieveProperty::Id => match parse_ref(value) { MaybeReference::Value(v) => Some(SieveValue::Id(v)), MaybeReference::Reference(v) => Some(SieveValue::IdReference(v)), MaybeReference::ParseError => None, }, SieveProperty::BlobId => match parse_ref(value) { MaybeReference::Value(v) => Some(SieveValue::BlobId(v)), MaybeReference::Reference(v) => Some(SieveValue::IdReference(v)), MaybeReference::ParseError => None, }, _ => None, } } else { None } } fn to_cow(&self) -> Cow<'static, str> { match self { SieveValue::Id(id) => id.to_string().into(), SieveValue::BlobId(blob_id) => blob_id.to_string().into(), SieveValue::IdReference(r) => format!("#{r}").into(), } } } impl SieveProperty { fn parse(value: &str) -> Option { hashify::tiny_map!(value.as_bytes(), b"id" => SieveProperty::Id, b"name" => SieveProperty::Name, b"blobId" => SieveProperty::BlobId, b"isActive" => SieveProperty::IsActive, ) } } #[derive(Debug, Clone, Default)] pub struct SieveSetArguments { pub on_success_activate_script: Option>, pub on_success_deactivate_script: Option, } impl<'de> DeserializeArguments<'de> for SieveSetArguments { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"onSuccessActivateScript" => { self.on_success_activate_script = map.next_value()?; }, b"onSuccessDeactivateScript" => { self.on_success_deactivate_script = map.next_value()?; }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl FromStr for SieveProperty { type Err = (); fn from_str(s: &str) -> Result { SieveProperty::parse(s).ok_or(()) } } impl JmapObject for Sieve { type Property = SieveProperty; type Element = SieveValue; type Id = Id; type Filter = SieveFilter; type Comparator = SieveComparator; type GetArguments = (); type SetArguments<'de> = SieveSetArguments; type QueryArguments = (); type CopyArguments = (); type ParseArguments = (); const ID_PROPERTY: Self::Property = SieveProperty::Id; } #[derive(Debug, Clone, PartialEq, Eq)] pub enum SieveFilter { Name(String), IsActive(bool), _T(String), } #[derive(Debug, Clone, PartialEq, Eq)] pub enum SieveComparator { Name, IsActive, _T(String), } impl<'de> DeserializeArguments<'de> for SieveFilter { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"name" => { *self = SieveFilter::Name(map.next_value()?); }, b"isActive" => { *self = SieveFilter::IsActive(map.next_value()?); }, _ => { *self = SieveFilter::_T(key.to_string()); let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> DeserializeArguments<'de> for SieveComparator { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { if key == "property" { let value = map.next_value::>()?; hashify::fnc_map!(value.as_bytes(), b"name" => { *self = SieveComparator::Name; }, b"isActive" => { *self = SieveComparator::IsActive; }, _ => { *self = SieveComparator::_T(key.to_string()); } ); } else { let _ = map.next_value::()?; } Ok(()) } } impl Default for SieveFilter { fn default() -> Self { SieveFilter::_T("".to_string()) } } impl Default for SieveComparator { fn default() -> Self { SieveComparator::_T("".to_string()) } } impl From for SieveValue { fn from(id: Id) -> Self { SieveValue::Id(id) } } impl JmapObjectId for SieveValue { fn as_id(&self) -> Option { match self { SieveValue::Id(id) => Some(*id), _ => None, } } fn as_any_id(&self) -> Option { match self { SieveValue::Id(id) => Some(AnyId::Id(*id)), SieveValue::BlobId(id) => Some(AnyId::BlobId(id.clone())), SieveValue::IdReference(_) => None, } } fn as_id_ref(&self) -> Option<&str> { if let SieveValue::IdReference(r) = self { Some(r) } else { None } } fn try_set_id(&mut self, new_id: AnyId) -> bool { match new_id { AnyId::Id(id) => { *self = SieveValue::Id(id); } AnyId::BlobId(id) => { *self = SieveValue::BlobId(id); } } true } } impl JmapObjectId for SieveProperty { fn as_id(&self) -> Option { None } fn as_any_id(&self) -> Option { None } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, _: AnyId) -> bool { false } } ================================================ FILE: crates/jmap-proto/src/object/thread.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use jmap_tools::{Element, Key, Property}; use std::{borrow::Cow, str::FromStr}; use types::id::Id; use crate::object::{AnyId, JmapObject, JmapObjectId}; #[derive(Debug, Clone, Default)] pub struct Thread; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum ThreadProperty { Id, EmailIds, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum ThreadValue { Id(Id), } impl Property for ThreadProperty { fn try_parse(_: Option<&Key<'_, Self>>, value: &str) -> Option { ThreadProperty::parse(value) } fn to_cow(&self) -> Cow<'static, str> { match self { ThreadProperty::Id => "id", ThreadProperty::EmailIds => "emailIds", } .into() } } impl Element for ThreadValue { type Property = ThreadProperty; fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option { if let Key::Property(_) = key { Id::from_str(value).ok().map(ThreadValue::Id) } else { None } } fn to_cow(&self) -> Cow<'static, str> { match self { ThreadValue::Id(id) => id.to_string().into(), } } } impl ThreadProperty { fn parse(value: &str) -> Option { hashify::tiny_map!(value.as_bytes(), b"id" => ThreadProperty::Id, b"emailIds" => ThreadProperty::EmailIds, ) } } impl FromStr for ThreadProperty { type Err = (); fn from_str(s: &str) -> Result { ThreadProperty::parse(s).ok_or(()) } } impl JmapObject for Thread { type Property = ThreadProperty; type Element = ThreadValue; type Id = Id; type Filter = (); type Comparator = (); type GetArguments = (); type SetArguments<'de> = (); type QueryArguments = (); type CopyArguments = (); type ParseArguments = (); const ID_PROPERTY: Self::Property = ThreadProperty::Id; } impl From for ThreadValue { fn from(id: Id) -> Self { ThreadValue::Id(id) } } impl JmapObjectId for ThreadValue { fn as_id(&self) -> Option { match self { ThreadValue::Id(id) => Some(*id), } } fn as_any_id(&self) -> Option { self.as_id().map(AnyId::Id) } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, new_id: AnyId) -> bool { if let AnyId::Id(id) = new_id { *self = ThreadValue::Id(id); true } else { false } } } impl JmapObjectId for ThreadProperty { fn as_id(&self) -> Option { None } fn as_any_id(&self) -> Option { None } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, _: AnyId) -> bool { false } } ================================================ FILE: crates/jmap-proto/src/object/vacation_response.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ object::{AnyId, JmapObject, JmapObjectId}, types::date::UTCDate, }; use jmap_tools::{Element, Key, Property}; use std::{borrow::Cow, str::FromStr}; use types::id::Id; #[derive(Debug, Clone, Default)] pub struct VacationResponse; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum VacationResponseProperty { Id, IsEnabled, FromDate, ToDate, Subject, TextBody, HtmlBody, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum VacationResponseValue { Id(Id), Date(UTCDate), } impl Property for VacationResponseProperty { fn try_parse(_: Option<&Key<'_, Self>>, value: &str) -> Option { VacationResponseProperty::parse(value) } fn to_cow(&self) -> Cow<'static, str> { match self { VacationResponseProperty::HtmlBody => "htmlBody", VacationResponseProperty::Id => "id", VacationResponseProperty::TextBody => "textBody", VacationResponseProperty::FromDate => "fromDate", VacationResponseProperty::IsEnabled => "isEnabled", VacationResponseProperty::ToDate => "toDate", VacationResponseProperty::Subject => "subject", } .into() } } impl Element for VacationResponseValue { type Property = VacationResponseProperty; fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option { if let Key::Property(prop) = key { match prop { VacationResponseProperty::Id => { Id::from_str(value).ok().map(VacationResponseValue::Id) } VacationResponseProperty::FromDate | VacationResponseProperty::ToDate => { UTCDate::from_str(value) .ok() .map(VacationResponseValue::Date) } _ => None, } } else { None } } fn to_cow(&self) -> Cow<'static, str> { match self { VacationResponseValue::Id(id) => id.to_string().into(), VacationResponseValue::Date(utcdate) => utcdate.to_string().into(), } } } impl VacationResponseProperty { fn parse(value: &str) -> Option { hashify::tiny_map!(value.as_bytes(), b"id" => VacationResponseProperty::Id, b"isEnabled" => VacationResponseProperty::IsEnabled, b"fromDate" => VacationResponseProperty::FromDate, b"toDate" => VacationResponseProperty::ToDate, b"textBody" => VacationResponseProperty::TextBody, b"htmlBody" => VacationResponseProperty::HtmlBody, b"subject" => VacationResponseProperty::Subject, ) } } impl FromStr for VacationResponseProperty { type Err = (); fn from_str(s: &str) -> Result { VacationResponseProperty::parse(s).ok_or(()) } } impl JmapObject for VacationResponse { type Property = VacationResponseProperty; type Element = VacationResponseValue; type Id = Id; type Filter = (); type Comparator = (); type GetArguments = (); type SetArguments<'de> = (); type QueryArguments = (); type CopyArguments = (); type ParseArguments = (); const ID_PROPERTY: Self::Property = VacationResponseProperty::Id; } impl From for VacationResponseValue { fn from(id: Id) -> Self { VacationResponseValue::Id(id) } } impl JmapObjectId for VacationResponseValue { fn as_id(&self) -> Option { match self { VacationResponseValue::Id(id) => Some(*id), _ => None, } } fn as_any_id(&self) -> Option { match self { VacationResponseValue::Id(id) => Some(AnyId::Id(*id)), _ => None, } } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, new_id: AnyId) -> bool { if let AnyId::Id(id) = new_id { *self = VacationResponseValue::Id(id); true } else { false } } } impl JmapObjectId for VacationResponseProperty { fn as_id(&self) -> Option { None } fn as_any_id(&self) -> Option { None } fn as_id_ref(&self) -> Option<&str> { None } fn try_set_id(&mut self, _: AnyId) -> bool { false } } ================================================ FILE: crates/jmap-proto/src/references/eval.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ object::{AnyId, JmapObjectId}, references::{ Graph, jsptr::{EvalResults, ResponsePtr}, }, request::reference::ResultReference, response::{ChangesResponseMethod, GetResponseMethod, Response, ResponseMethod}, }; use compact_str::format_compact; use jmap_tools::{Element, Key, Property, Value}; use types::{blob::BlobId, id::Id}; impl Response<'_> { pub(crate) fn eval_result_references(&self, rr: &ResultReference) -> trc::Result { let mut results = EvalResults::default(); for response in &self.method_responses { if response.id == rr.result_of && response.name == rr.name { let path = rr.path.iter(); let success = match &response.method { ResponseMethod::Get(response) => match response { GetResponseMethod::Email(response) => { response.eval_jptr(path, &mut results) } GetResponseMethod::Mailbox(response) => { response.eval_jptr(path, &mut results) } GetResponseMethod::Thread(response) => { response.eval_jptr(path, &mut results) } GetResponseMethod::Identity(response) => { response.eval_jptr(path, &mut results) } GetResponseMethod::EmailSubmission(response) => { response.eval_jptr(path, &mut results) } GetResponseMethod::PushSubscription(response) => { response.eval_jptr(path, &mut results) } GetResponseMethod::Sieve(response) => { response.eval_jptr(path, &mut results) } GetResponseMethod::VacationResponse(response) => { response.eval_jptr(path, &mut results) } GetResponseMethod::Principal(response) => { response.eval_jptr(path, &mut results) } GetResponseMethod::Quota(response) => { response.eval_jptr(path, &mut results) } GetResponseMethod::Blob(response) => response.eval_jptr(path, &mut results), GetResponseMethod::AddressBook(response) => { response.eval_jptr(path, &mut results) } GetResponseMethod::ContactCard(response) => { response.eval_jptr(path, &mut results) } GetResponseMethod::FileNode(response) => { response.eval_jptr(path, &mut results) } GetResponseMethod::Calendar(response) => { response.eval_jptr(path, &mut results) } GetResponseMethod::CalendarEvent(response) => { response.eval_jptr(path, &mut results) } GetResponseMethod::CalendarEventNotification(response) => { response.eval_jptr(path, &mut results) } GetResponseMethod::ParticipantIdentity(response) => { response.eval_jptr(path, &mut results) } GetResponseMethod::ShareNotification(response) => { response.eval_jptr(path, &mut results) } GetResponseMethod::PrincipalAvailability(response) => { response.eval_jptr(path, &mut results) } }, ResponseMethod::Changes(response) => match response { ChangesResponseMethod::Email(response) => { response.eval_jptr(path, &mut results) } ChangesResponseMethod::Mailbox(response) => { response.eval_jptr(path, &mut results) } ChangesResponseMethod::Thread(response) => { response.eval_jptr(path, &mut results) } ChangesResponseMethod::Identity(response) => { response.eval_jptr(path, &mut results) } ChangesResponseMethod::EmailSubmission(response) => { response.eval_jptr(path, &mut results) } ChangesResponseMethod::Quota(response) => { response.eval_jptr(path, &mut results) } ChangesResponseMethod::AddressBook(response) => { response.eval_jptr(path, &mut results) } ChangesResponseMethod::ContactCard(response) => { response.eval_jptr(path, &mut results) } ChangesResponseMethod::FileNode(response) => { response.eval_jptr(path, &mut results) } ChangesResponseMethod::Calendar(response) => { response.eval_jptr(path, &mut results) } ChangesResponseMethod::CalendarEvent(response) => { response.eval_jptr(path, &mut results) } ChangesResponseMethod::CalendarEventNotification(response) => { response.eval_jptr(path, &mut results) } ChangesResponseMethod::ShareNotification(response) => { response.eval_jptr(path, &mut results) } }, ResponseMethod::Query(response) => response.eval_jptr(path, &mut results), ResponseMethod::QueryChanges(response) => { response.eval_jptr(path, &mut results) } _ => false, }; if success { return Ok(results); } } } Err(trc::JmapEvent::InvalidResultReference .into_err() .details(format_compact!( "Result reference to {}#{} not found.", rr.result_of, rr.name ))) } pub(crate) fn eval_id_reference(&self, ir: &str) -> trc::Result { if let Some(AnyId::Id(id)) = self.created_ids.get(ir) { Ok(*id) } else { Err(trc::JmapEvent::InvalidResultReference .into_err() .details(format_compact!("Id reference {ir:?} not found."))) } } pub(crate) fn eval_blob_id_reference(&self, ir: &str) -> trc::Result { if let Some(AnyId::BlobId(id)) = self.created_ids.get(ir) { Ok(id.clone()) } else { Err(trc::JmapEvent::InvalidResultReference .into_err() .details(format_compact!("blobId reference {ir:?} not found."))) } } } pub(crate) trait EvalObjectReferences { fn eval_object_references( &mut self, response: &Response<'_>, graph: &mut Graph<'_>, depth: usize, ) -> trc::Result<()>; } impl<'x, P, E> EvalObjectReferences for Value<'x, P, E> where P: Property + JmapObjectId, E: Element + JmapObjectId, { fn eval_object_references( &mut self, response: &Response<'_>, graph: &mut Graph<'_>, depth: usize, ) -> trc::Result<()> { let Value::Object(obj) = self else { return Ok(()); }; for (key, value) in obj.as_mut_vec() { // Resolve patch with references (e.g. mailboxIds/#idRef) if depth == 0 && let Key::Property(property) = key && let Some(id_ref) = property.as_id_ref() { if let Some(id) = response.created_ids.get(id_ref) { if !property.try_set_id(id.clone()) { return Err(trc::JmapEvent::InvalidResultReference .into_err() .details("Id reference points to invalid type.")); } } else { return Err(trc::JmapEvent::InvalidResultReference .into_err() .details(format_compact!("Id reference {id_ref:?} not found."))); } } match value { Value::Element(element) => { if let Some(id_ref) = element.as_id_ref() { if let Some(id) = response.created_ids.get(id_ref) { if !element.try_set_id(id.clone()) { return Err(trc::JmapEvent::InvalidResultReference .into_err() .details("Id reference points to invalid type.")); } } else if let Graph::Some { child_id, graph } = graph { graph .entry(child_id.to_string()) .or_insert_with(Vec::new) .push(id_ref.to_string()); } else { return Err(trc::JmapEvent::InvalidResultReference .into_err() .details(format_compact!("Id reference {id_ref:?} not found."))); } } } Value::Array(items) if depth == 0 => { // Resolve references in arrays (e.g. emailIds: [#idRef1, #idRef2]) for item in items { item.eval_object_references(response, graph, depth + 1)?; } } Value::Object(items) if depth == 0 => { // Resolve references in JMAP sets (e.g. mailboxIds: { "#idRef1": true, "#idRef2": true }) for (key, _) in items.as_mut_vec() { if let Key::Property(property) = key && let Some(id_ref) = property.as_id_ref() { if let Some(id) = response.created_ids.get(id_ref) { if !property.try_set_id(id.clone()) { return Err(trc::JmapEvent::InvalidResultReference .into_err() .details("Id reference points to invalid type.")); } } else { return Err(trc::JmapEvent::InvalidResultReference .into_err() .details(format_compact!( "Id reference {id_ref:?} not found." ))); } } } } _ => {} } } Ok(()) } } ================================================ FILE: crates/jmap-proto/src/references/jsptr.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ method::{ PropertyWrapper, availability::{BusyPeriod, GetAvailabilityResponse}, changes::ChangesResponse, get::GetResponse, query::QueryResponse, query_changes::{AddedItem, QueryChangesResponse}, }, object::{ AnyId, JmapObject, JmapObjectId, calendar_event_notification::{ CalendarEventNotificationGetResponse, CalendarEventNotificationObject, }, }, request::reference::ResultReference, }; use compact_str::format_compact; use jmap_tools::{Element, JsonPointerItem, JsonPointerIter, Key, Null, Property, Value}; use std::{borrow::Cow, str::FromStr}; use types::{blob::BlobId, id::Id}; pub(crate) trait ResponsePtr { fn eval_jptr(&self, pointer: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool; } #[derive(Debug, Default)] #[repr(transparent)] pub(crate) struct EvalResults(Vec); #[derive(Debug)] pub(crate) enum EvalResult { Id(AnyId), Property(Cow<'static, str>), } impl ResponsePtr for Vec where T: ResponsePtr, { fn eval_jptr(&self, mut pointer: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool { match pointer.next() { Some(JsonPointerItem::Number(n)) => { if let Some(v) = self.get(*n as usize) { v.eval_jptr(pointer, results); } } Some(JsonPointerItem::Wildcard | JsonPointerItem::Root) | None => { for v in self { v.eval_jptr(pointer.clone(), results); } } _ => (), } true } } impl<'ctx, P, E> ResponsePtr for Value<'ctx, P, E> where P: Property, E: Element + JmapObjectId, { fn eval_jptr(&self, mut pointer: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool { match pointer.next() { Some(JsonPointerItem::Key(key)) => { if let Some(key) = key.as_string_key() && let Value::Object(map) = self && let Some(v) = map.get(&Key::Borrowed(key)) { v.eval_jptr(pointer, results); } } Some(JsonPointerItem::Number(n)) => match self { Value::Array(values) => { if let Some(v) = values.get(*n as usize) { v.eval_jptr(pointer, results); } } Value::Object(map) => { let n = Key::Owned(n.to_string()); if let Some(v) = map.get(&n) { v.eval_jptr(pointer, results); } } _ => {} }, Some(JsonPointerItem::Wildcard) => match self { Value::Array(values) => { for v in values { v.eval_jptr(pointer.clone(), results); } } Value::Object(map) => { for v in map.values() { v.eval_jptr(pointer.clone(), results); } } _ => {} }, Some(JsonPointerItem::Root) | None => match self { Value::Element(e) => { if let Some(id) = e.as_any_id() { results.0.push(EvalResult::Id(id)); } } Value::Array(list) => { for item in list { if let Value::Element(e) = item && let Some(id) = e.as_any_id() { results.0.push(EvalResult::Id(id)); } } } _ => (), }, } true } } impl ResponsePtr for Id { fn eval_jptr(&self, _pointer: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool { results.0.push(EvalResult::Id(AnyId::Id(*self))); true } } impl ResponsePtr for BlobId { fn eval_jptr(&self, _pointer: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool { results.0.push(EvalResult::Id(AnyId::BlobId(self.clone()))); true } } impl ResponsePtr for PropertyWrapper { fn eval_jptr(&self, _: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool { results.0.push(EvalResult::Property(self.0.to_cow())); true } } impl ResponsePtr for GetResponse { fn eval_jptr(&self, mut pointer: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool { match pointer.next().and_then(|item| item.as_string_key()) { Some("list") => { self.list.eval_jptr(pointer, results); true } _ => false, } } } impl ResponsePtr for ChangesResponse { fn eval_jptr(&self, mut pointer: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool { if let Some(property) = pointer.next().and_then(|item| item.as_string_key()) { hashify::fnc_map!(property.as_bytes(), "created" => { self.created.eval_jptr(pointer, results); }, "updated" => { self.updated.eval_jptr(pointer, results); }, "updatedProperties" => { if let Some(props) = &self.updated_properties { props.eval_jptr(pointer, results); } }, _ => { return false; } ); true } else { false } } } impl ResponsePtr for QueryResponse { fn eval_jptr(&self, mut pointer: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool { match pointer.next().and_then(|item| item.as_string_key()) { Some("ids") => { self.ids.eval_jptr(pointer, results); true } _ => false, } } } impl ResponsePtr for QueryChangesResponse { fn eval_jptr(&self, mut pointer: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool { match pointer.next().and_then(|item| item.as_string_key()) { Some("added") => { self.added.eval_jptr(pointer, results); true } _ => false, } } } impl ResponsePtr for AddedItem { fn eval_jptr(&self, mut pointer: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool { match pointer.next().and_then(|item| item.as_string_key()) { Some("id") => { results.0.push(EvalResult::Id(AnyId::Id(self.id))); true } _ => false, } } } impl ResponsePtr for CalendarEventNotificationGetResponse { fn eval_jptr(&self, mut pointer: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool { match pointer.next().and_then(|item| item.as_string_key()) { Some("list") => { self.list.eval_jptr(pointer, results); true } _ => false, } } } impl ResponsePtr for CalendarEventNotificationObject { fn eval_jptr(&self, mut pointer: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool { match pointer.next().and_then(|item| item.as_string_key()) { Some("id") => { results.0.push(EvalResult::Id(AnyId::Id(self.id))); true } Some("calendarEventId") => { if let Some(id) = &self.calendar_event_id { results.0.push(EvalResult::Id(AnyId::Id(*id))); } true } Some("event") => { if let Some(event) = &self.event { event.0.eval_jptr(pointer, results); } true } _ => false, } } } impl ResponsePtr for GetAvailabilityResponse { fn eval_jptr(&self, mut pointer: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool { match pointer.next().and_then(|item| item.as_string_key()) { Some("list") => { self.list.eval_jptr(pointer, results); true } _ => false, } } } impl ResponsePtr for BusyPeriod { fn eval_jptr(&self, mut pointer: JsonPointerIter<'_, Null>, results: &mut EvalResults) -> bool { match pointer.next().and_then(|item| item.as_string_key()) { Some("event") => { if let Some(event) = &self.event { event.0.eval_jptr(pointer, results); } true } _ => false, } } } impl EvalResults { pub fn into_ids>( self, rr: &ResultReference, ) -> impl Iterator> { self.0.into_iter().map(move |id| { if let EvalResult::Id(any_id) = id { T::try_from(any_id).map_err(|_| { trc::JmapEvent::InvalidResultReference .into_err() .details(format_compact!( "Failed to evaluate {rr} result reference: Invalid Id type." )) }) } else { Err(trc::JmapEvent::InvalidResultReference .into_err() .details(format_compact!( "Failed to evaluate {rr} result reference: Invalid Id type." ))) } }) } pub fn into_properties( self, rr: &ResultReference, ) -> impl Iterator> { self.0.into_iter().map(move |prop| { if let EvalResult::Property(prop) = prop { T::from_str(&prop).map_err(|_| { trc::JmapEvent::InvalidResultReference .into_err() .details(format_compact!( "Failed to evaluate {rr} result reference: Invalid property." )) }) } else { Err(trc::JmapEvent::InvalidResultReference .into_err() .details(format_compact!( "Failed to evaluate {rr} result reference: Invalid property." ))) } }) } } ================================================ FILE: crates/jmap-proto/src/references/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use compact_str::format_compact; use std::collections::HashMap; use utils::map::vec_map::VecMap; pub mod eval; pub mod jsptr; pub mod resolve; pub(crate) enum Graph<'x> { Some { child_id: &'x str, graph: &'x mut HashMap>, }, None, } fn topological_sort( create: &mut VecMap, graph: HashMap>, ) -> trc::Result> { // Make sure all references exist for (from_id, to_ids) in graph.iter() { for to_id in to_ids { if !create.contains_key(to_id) { return Err(trc::JmapEvent::InvalidResultReference.into_err().details( format_compact!( "Invalid reference to non-existing object {to_id:?} from {from_id:?}" ), )); } } } let mut sorted_create = VecMap::with_capacity(create.len()); let mut it_stack = Vec::new(); let keys = graph.keys().cloned().collect::>(); let mut it = keys.iter(); 'main: loop { while let Some(from_id) = it.next() { if let Some(to_ids) = graph.get(from_id) { it_stack.push((it, from_id)); if it_stack.len() > 1000 { return Err(trc::JmapEvent::InvalidArguments .into_err() .details("Cyclical references are not allowed.")); } it = to_ids.iter(); continue; } else if let Some((id, value)) = create.remove_entry(from_id) { sorted_create.append(id, value); if create.is_empty() { break 'main; } } } if let Some((prev_it, from_id)) = it_stack.pop() { it = prev_it; if let Some((id, value)) = create.remove_entry(from_id) { sorted_create.append(id, value); if create.is_empty() { break 'main; } } } else { break; } } // Add remaining items if !create.is_empty() { for (id, value) in std::mem::take(create) { sorted_create.append(id, value); } } Ok(sorted_create) } #[cfg(test)] mod tests { use crate::{ method::{changes::ChangesResponse, get::GetResponse, query::QueryResponse}, object::{ email::{EmailProperty, EmailValue}, mailbox::{MailboxProperty, MailboxValue}, thread::{ThreadProperty, ThreadValue}, }, request::{ Call, GetRequestMethod, Request, RequestMethod, SetRequestMethod, reference::{MaybeIdReference, MaybeResultReference}, }, response::{ChangesResponseMethod, GetResponseMethod, Response, ResponseMethod}, }; use jmap_tools::{Key, Map, Value}; use std::collections::HashMap; use types::id::Id; #[test] fn eval_value_references() { let request = Request::parse( br##"{ "using":["urn:ietf:params:jmap:mail"], "methodCalls": [[ "Email/query", { "accountId": "a", "filter": { "inMailbox": "a" }, "sort": [{ "property": "receivedAt", "isAscending": false }], "collapseThreads": true, "position": 0, "limit": 10, "calculateTotal": true }, "t0" ], [ "Email/get", { "accountId": "a", "#ids": { "resultOf": "t0", "name": "Email/query", "path": "/ids" }, "properties": [ "threadId" ] }, "t1" ], [ "Thread/get", { "accountId": "a", "#ids": { "resultOf": "t1", "name": "Email/get", "path": "/list/*/threadId" } }, "t2" ], [ "Email/get", { "accountId": "a", "#ids": { "resultOf": "t2", "name": "Thread/get", "path": "/list/*/emailIds" }, "properties": [ "from", "receivedAt", "subject" ] }, "t3" ]] }"##, 100, 1024 * 1024, ) .unwrap(); let mut response = Response::new( 1234, request.created_ids.unwrap_or_default(), request.method_calls.len(), ); assert_eq!(request.method_calls.len(), 4); for (test_num, mut call) in request.method_calls.into_iter().enumerate() { match test_num { 0 => { response.method_responses.push(Call { id: call.id, name: call.name, method: ResponseMethod::Query(QueryResponse { account_id: Id::new(1), query_state: Default::default(), can_calculate_changes: Default::default(), position: Default::default(), ids: vec![Id::new(4), Id::new(5)], total: Default::default(), limit: Default::default(), }), }); } 1 => { response.resolve_references(&mut call.method).unwrap(); match call.method { RequestMethod::Get(GetRequestMethod::Email(req)) => { assert_eq!( req.ids, Some(MaybeResultReference::Value(vec![ MaybeIdReference::Id(Id::new(4)), MaybeIdReference::Id(Id::new(5)) ])) ); } _ => panic!("Expected Email Get Request"), } response.method_responses.push(Call { id: call.id, name: call.name, method: ResponseMethod::Get(GetResponseMethod::Email(GetResponse { account_id: Id::new(1).into(), state: Default::default(), list: vec![ Value::Object(Map::from(vec![( Key::Property(EmailProperty::ThreadId), Value::Element(EmailValue::Id(Id::new(9))), )])), Value::Object(Map::from(vec![( Key::Property(EmailProperty::ThreadId), Value::Element(EmailValue::Id(Id::new(10))), )])), ], not_found: Default::default(), })), }); } 2 => { response.resolve_references(&mut call.method).unwrap(); match call.method { RequestMethod::Get(GetRequestMethod::Thread(req)) => { assert_eq!( req.ids, Some(MaybeResultReference::Value(vec![ MaybeIdReference::Id(Id::new(9)), MaybeIdReference::Id(Id::new(10)) ])) ); } _ => panic!("Expected Thread Get Request"), } response.method_responses.push(Call { id: call.id, name: call.name, method: ResponseMethod::Get(GetResponseMethod::Thread(GetResponse { account_id: Id::new(1).into(), state: Default::default(), list: vec![ Value::Object(Map::from(vec![( Key::Property(ThreadProperty::EmailIds), Value::Array(vec![ Value::Element(ThreadValue::Id(Id::new(100))), Value::Element(ThreadValue::Id(Id::new(101))), ]), )])), Value::Object(Map::from(vec![( Key::Property(ThreadProperty::EmailIds), Value::Array(vec![ Value::Element(ThreadValue::Id(Id::new(102))), Value::Element(ThreadValue::Id(Id::new(103))), ]), )])), ], not_found: Default::default(), })), }); } 3 => { response.resolve_references(&mut call.method).unwrap(); match call.method { RequestMethod::Get(GetRequestMethod::Email(req)) => { assert_eq!( req.ids, Some(MaybeResultReference::Value(vec![ MaybeIdReference::Id(Id::new(100)), MaybeIdReference::Id(Id::new(101)), MaybeIdReference::Id(Id::new(102)), MaybeIdReference::Id(Id::new(103)), ])) ); } _ => panic!("Expected Mailbox Get Request"), } } _ => panic!("Unexpected invocation {}", test_num), } } } #[test] fn eval_property_references() { let request = Request::parse( br##"{ "using":["urn:ietf:params:jmap:mail"], "methodCalls": [ ["Mailbox/changes",{ "accountId":"s", "sinceState":"srxqk071myhgkyay" },"0"], ["Mailbox/get",{ "accountId":"s", "#ids":{"name":"Mailbox/changes","path":"/created","resultOf":"0"} },"1"], ["Mailbox/get",{ "accountId":"s", "#ids":{"name":"Mailbox/changes","path":"/updated","resultOf":"0"}, "#properties":{"name":"Mailbox/changes","path":"/updatedProperties","resultOf":"0"} },"2"] ] }"##, 100, 1024 * 1024, ) .unwrap(); let mut response = Response::new( 1234, request.created_ids.unwrap_or_default(), request.method_calls.len(), ); assert_eq!(request.method_calls.len(), 3); for (test_num, mut call) in request.method_calls.into_iter().enumerate() { match test_num { 0 => { response.method_responses.push(Call { id: call.id, name: call.name, method: ResponseMethod::Changes(ChangesResponseMethod::Mailbox( ChangesResponse { account_id: Id::new(1), old_state: Default::default(), new_state: Default::default(), has_more_changes: Default::default(), created: Default::default(), updated: vec![Id::new(2), Id::new(3)], destroyed: Default::default(), updated_properties: Some(vec![ MailboxProperty::Name.into(), MailboxProperty::ParentId.into(), ]), }, )), }); } 1 => { response.resolve_references(&mut call.method).unwrap(); match call.method { RequestMethod::Get(GetRequestMethod::Mailbox(req)) => { assert_eq!(req.ids, Some(MaybeResultReference::Value(vec![]))); } _ => panic!("Expected Mailbox Get Request"), } } 2 => { response.resolve_references(&mut call.method).unwrap(); match call.method { RequestMethod::Get(GetRequestMethod::Mailbox(req)) => { assert_eq!( req.ids, Some(MaybeResultReference::Value(vec![ MaybeIdReference::Id(Id::new(2)), MaybeIdReference::Id(Id::new(3)) ])) ); } _ => panic!("Expected Mailbox Get Request"), } } _ => panic!("Unexpected invocation {}", test_num), } } } #[test] fn eval_create_references() { let request = Request::parse( br##"{ "using": [ "urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail" ], "methodCalls": [ [ "Mailbox/set", { "accountId": "b", "create": { "a": { "name": "Folder a", "parentId": "#b" }, "b": { "name": "Folder b", "parentId": "#c" }, "c": { "name": "Folder c", "parentId": "#d" }, "d": { "name": "Folder d", "parentId": "#e" }, "e": { "name": "Folder e", "parentId": "#f" }, "f": { "name": "Folder f", "parentId": "#g" }, "g": { "name": "Folder g", "parentId": null } } }, "fulltree" ], [ "Mailbox/set", { "accountId": "b", "create": { "a1": { "name": "Folder a1", "parentId": null }, "b2": { "name": "Folder b2", "parentId": "#a1" }, "c3": { "name": "Folder c3", "parentId": "#a1" }, "d4": { "name": "Folder d4", "parentId": "#b2" }, "e5": { "name": "Folder e5", "parentId": "#b2" }, "f6": { "name": "Folder f6", "parentId": "#d4" }, "g7": { "name": "Folder g7", "parentId": "#e5" } } }, "fulltree2" ], [ "Mailbox/set", { "accountId": "b", "create": { "z": { "name": "Folder Z", "parentId": "#x" }, "y": { "name": null }, "x": { "name": "Folder X" } } }, "xyz" ], [ "Mailbox/set", { "accountId": "b", "create": { "a": { "name": "Folder a", "parentId": "#b" }, "b": { "name": "Folder b", "parentId": "#c" }, "c": { "name": "Folder c", "parentId": "#d" }, "d": { "name": "Folder d", "parentId": "#a" } } }, "circular" ] ] }"##, 100, 1024 * 1024, ) .unwrap(); let response = Response::new( 1234, request.created_ids.unwrap_or_default(), request.method_calls.len(), ); for (test_num, mut call) in request.method_calls.into_iter().enumerate() { match response.resolve_references(&mut call.method) { Ok(_) => assert!( (0..3).contains(&test_num), "Unexpected invocation {}", test_num ), Err(err) => { assert_eq!(test_num, 3); assert!( err.matches(trc::EventType::Jmap(trc::JmapEvent::InvalidArguments)), "{:?}", err ); continue; } } if let RequestMethod::Set(SetRequestMethod::Mailbox(request)) = call.method { if test_num == 0 { assert_eq!( request .create .unwrap() .into_iter() .map(|b| b.0) .collect::>(), ["g", "f", "e", "d", "c", "b", "a"] .iter() .map(|i| i.to_string()) .collect::>() ); } else if test_num == 1 { let mut pending_ids = vec!["a1", "b2", "d4", "e5", "f6", "c3", "g7"]; for (id, _) in request.create.as_ref().unwrap() { match id.as_str() { "a1" => (), "b2" | "c3" => assert!(!pending_ids.contains(&"a1")), "d4" | "e5" => assert!(!pending_ids.contains(&"b2")), "f6" => assert!(!pending_ids.contains(&"d4")), "g7" => assert!(!pending_ids.contains(&"e5")), _ => panic!("Unexpected ID"), } pending_ids.retain(|i| i != id); } if !pending_ids.is_empty() { panic!( "Unexpected order: {:?}", request .create .as_ref() .unwrap() .iter() .map(|b| b.0.to_string()) .collect::>() ); } } else if test_num == 2 { assert_eq!( request .create .unwrap() .into_iter() .map(|b| b.0) .collect::>(), ["x", "z", "y"] .iter() .map(|i| i.to_string()) .collect::>() ); } } else { panic!("Expected Set Mailbox Request"); } } let request = Request::parse( br##"{ "using": [ "urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail" ], "methodCalls": [ [ "Mailbox/set", { "accountId": "b", "create": { "a": { "name": "a", "parentId": "#x" }, "b": { "name": "b", "parentId": "#y" }, "c": { "name": "c", "parentId": "#z" } } }, "ref1" ], [ "Mailbox/set", { "accountId": "b", "create": { "a1": { "name": "a1", "parentId": "#a" }, "b2": { "name": "b2", "parentId": "#b" }, "c3": { "name": "c3", "parentId": "#c" } } }, "red2" ] ], "createdIds": { "x": "b", "y": "c", "z": "d" } }"##, 1024, 1024 * 1024, ) .unwrap(); let mut response = Response::new( 1234, request.created_ids.unwrap_or_default(), request.method_calls.len(), ); let mut invocations = request.method_calls.into_iter(); let mut call = invocations.next().unwrap(); response.resolve_references(&mut call.method).unwrap(); if let RequestMethod::Set(SetRequestMethod::Mailbox(request)) = call.method { let create = request .create .as_ref() .unwrap() .iter() .map(|(p, v)| { ( p.as_str(), v.as_object() .unwrap() .get(&Key::Property(MailboxProperty::ParentId)) .unwrap(), ) }) .collect::>(); assert_eq!( *create.get("a").unwrap(), &Value::Element(MailboxValue::Id(Id::new(1))) ); assert_eq!( *create.get("b").unwrap(), &Value::Element(MailboxValue::Id(Id::new(2))) ); assert_eq!( *create.get("c").unwrap(), &Value::Element(MailboxValue::Id(Id::new(3))) ); } else { panic!("Expected Mailbox Set Request"); } response .created_ids .insert("a".to_string(), Id::new(5).into()); response .created_ids .insert("b".to_string(), Id::new(6).into()); response .created_ids .insert("c".to_string(), Id::new(7).into()); let mut call = invocations.next().unwrap(); response.resolve_references(&mut call.method).unwrap(); if let RequestMethod::Set(SetRequestMethod::Mailbox(request)) = call.method { let create = request .create .as_ref() .unwrap() .iter() .map(|(p, v)| { ( p.as_str(), v.as_object() .unwrap() .get(&Key::Property(MailboxProperty::ParentId)) .unwrap(), ) }) .collect::>(); assert_eq!( *create.get("a1").unwrap(), &Value::Element(MailboxValue::Id(Id::new(5))) ); assert_eq!( *create.get("b2").unwrap(), &Value::Element(MailboxValue::Id(Id::new(6))) ); assert_eq!( *create.get("c3").unwrap(), &Value::Element(MailboxValue::Id(Id::new(7))) ); } else { panic!("Expected Mailbox Set Request"); } } } ================================================ FILE: crates/jmap-proto/src/references/resolve.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ error::set::SetError, method::{ copy::CopyRequest, get::GetRequest, import::ImportEmailRequest, parse::ParseRequest, search_snippet::GetSearchSnippetRequest, set::{SetRequest, SetResponse}, upload::{BlobUploadRequest, DataSourceObject}, }, object::{AnyId, JmapObject, JmapObjectId}, references::{Graph, eval::EvalObjectReferences, topological_sort}, request::{ CopyRequestMethod, GetRequestMethod, MaybeInvalid, ParseRequestMethod, RequestMethod, SetRequestMethod, reference::{MaybeIdReference, MaybeResultReference}, }, response::Response, }; use compact_str::format_compact; use jmap_tools::{Element, Key, Property, Value}; use std::collections::HashMap; use types::id::Id; impl Response<'_> { pub fn resolve_references(&self, request: &mut RequestMethod) -> trc::Result<()> { match request { RequestMethod::Get(request) => match request { GetRequestMethod::Email(request) => request.resolve_references(self)?, GetRequestMethod::Mailbox(request) => request.resolve_references(self)?, GetRequestMethod::Thread(request) => request.resolve_references(self)?, GetRequestMethod::Identity(request) => request.resolve_references(self)?, GetRequestMethod::EmailSubmission(request) => request.resolve_references(self)?, GetRequestMethod::PushSubscription(request) => request.resolve_references(self)?, GetRequestMethod::Sieve(request) => request.resolve_references(self)?, GetRequestMethod::VacationResponse(request) => request.resolve_references(self)?, GetRequestMethod::Principal(request) => request.resolve_references(self)?, GetRequestMethod::Quota(request) => request.resolve_references(self)?, GetRequestMethod::Blob(request) => request.resolve_references(self)?, GetRequestMethod::AddressBook(request) => request.resolve_references(self)?, GetRequestMethod::ContactCard(request) => request.resolve_references(self)?, GetRequestMethod::FileNode(request) => request.resolve_references(self)?, GetRequestMethod::ShareNotification(request) => request.resolve_references(self)?, GetRequestMethod::Calendar(request) => request.resolve_references(self)?, GetRequestMethod::CalendarEvent(request) => request.resolve_references(self)?, GetRequestMethod::CalendarEventNotification(request) => { request.resolve_references(self)? } GetRequestMethod::ParticipantIdentity(request) => { request.resolve_references(self)? } GetRequestMethod::PrincipalAvailability(_) => (), }, RequestMethod::Set(request) => match request { SetRequestMethod::Email(request) => request.resolve_references(self)?, SetRequestMethod::Mailbox(request) => request.resolve_references(self)?, SetRequestMethod::Identity(request) => request.resolve_references(self)?, SetRequestMethod::EmailSubmission(request) => request.resolve_references(self)?, SetRequestMethod::PushSubscription(request) => request.resolve_references(self)?, SetRequestMethod::Sieve(request) => request.resolve_references(self)?, SetRequestMethod::VacationResponse(request) => request.resolve_references(self)?, SetRequestMethod::AddressBook(request) => request.resolve_references(self)?, SetRequestMethod::ContactCard(request) => request.resolve_references(self)?, SetRequestMethod::FileNode(request) => request.resolve_references(self)?, SetRequestMethod::ShareNotification(request) => request.resolve_references(self)?, SetRequestMethod::Calendar(request) => request.resolve_references(self)?, SetRequestMethod::CalendarEvent(request) => request.resolve_references(self)?, SetRequestMethod::CalendarEventNotification(request) => { request.resolve_references(self)? } SetRequestMethod::ParticipantIdentity(request) => { request.resolve_references(self)? } }, RequestMethod::Copy(request) => match request { CopyRequestMethod::Email(request) => request.resolve_references(self)?, CopyRequestMethod::CalendarEvent(request) => request.resolve_references(self)?, CopyRequestMethod::ContactCard(request) => request.resolve_references(self)?, CopyRequestMethod::Blob(_) => (), }, RequestMethod::ImportEmail(request) => request.resolve_references(self)?, RequestMethod::SearchSnippet(request) => request.resolve_references(self)?, RequestMethod::UploadBlob(request) => request.resolve_references(self)?, RequestMethod::Parse(request) => match request { ParseRequestMethod::Email(request) => request.resolve_references(self)?, ParseRequestMethod::ContactCard(request) => request.resolve_references(self)?, ParseRequestMethod::CalendarEvent(request) => request.resolve_references(self)?, }, _ => {} } Ok(()) } } pub trait ResolveCreatedReference where P: Property, E: Element + JmapObjectId, { fn get_created_id(&self, id_ref: &str) -> Option; fn resolve_self_references(&self, value: &mut Value<'_, P, E>) -> Result<(), SetError

> { match value { Value::Element(element) => { if let Some(id_ref) = element.as_id_ref() { if let Some(id) = self.get_created_id(id_ref) { if !element.try_set_id(id) { return Err(SetError::invalid_properties() .with_description("Id reference points to invalid type.")); } } else { return Err(SetError::not_found() .with_description(format!("Id reference {id_ref:?} not found."))); } } } Value::Array(items) => { for item in items { self.resolve_self_references(item)?; } } _ => {} } Ok(()) } } pub(crate) trait ResolveReference { fn resolve_references(&mut self, response: &Response<'_>) -> trc::Result<()>; } impl ResolveReference for GetRequest { fn resolve_references(&mut self, response: &Response<'_>) -> trc::Result<()> { // Resolve id references match &mut self.ids { Some(MaybeResultReference::Reference(reference)) => { self.ids = Some(MaybeResultReference::Value( response .eval_result_references(reference)? .into_ids::(reference) .map(|f| f.map(MaybeIdReference::Id)) .collect::>()?, )); } Some(MaybeResultReference::Value(ids)) => { for id in ids { if let MaybeIdReference::Reference(reference) = id { if let Some(resolved_id) = response .created_ids .get(reference) .cloned() .and_then(|v| T::Id::try_from(v).ok()) { *id = MaybeIdReference::Id(resolved_id); } else { return Err(trc::JmapEvent::InvalidResultReference.into_err().details( format_compact!( "Id reference {reference:?} does not exist or is invalid." ), )); } } } } _ => (), } // Resolve properties references if let Some(MaybeResultReference::Reference(reference)) = &self.properties { self.properties = Some(MaybeResultReference::Value( response .eval_result_references(reference)? .into_properties::(reference) .map(|f| f.map(MaybeInvalid::Value)) .collect::>()?, )); } Ok(()) } } impl<'x, T: JmapObject> ResolveReference for SetRequest<'x, T> { fn resolve_references(&mut self, response: &Response<'_>) -> trc::Result<()> { // Resolve create references if let Some(create) = &mut self.create { let mut graph = HashMap::with_capacity(create.len()); for (id, obj) in create.iter_mut() { obj.eval_object_references( response, &mut Graph::Some { child_id: &*id, graph: &mut graph, }, 0, )?; } // Perform topological sort if !graph.is_empty() { self.create = topological_sort(create, graph)?.into(); } } // Resolve update references if let Some(update) = &mut self.update { for obj in update.values_mut() { obj.eval_object_references(response, &mut Graph::None, 0)?; } } // Resolve destroy references if let Some(MaybeResultReference::Reference(reference)) = &self.destroy { self.destroy = Some(MaybeResultReference::Value( response .eval_result_references(reference)? .into_ids::(reference) .map(|f| f.map(MaybeInvalid::Value)) .collect::>()?, )); } Ok(()) } } impl<'x, T: JmapObject> ResolveReference for CopyRequest<'x, T> { fn resolve_references(&mut self, response: &Response<'_>) -> trc::Result<()> { // Resolve create references for (id, obj) in self.create.iter_mut() { obj.eval_object_references(response, &mut Graph::None, 0)?; if let MaybeIdReference::Reference(ir) = id { *id = MaybeIdReference::Id(response.eval_id_reference(ir)?); } } Ok(()) } } impl ResolveReference for ParseRequest { fn resolve_references(&mut self, response: &Response<'_>) -> trc::Result<()> { // Resolve blobId references for id in self.blob_ids.iter_mut() { if let MaybeIdReference::Reference(ir) = id { *id = MaybeIdReference::Id(response.eval_blob_id_reference(ir)?); } } Ok(()) } } impl ResolveReference for ImportEmailRequest { fn resolve_references(&mut self, response: &Response<'_>) -> trc::Result<()> { // Resolve email mailbox references for email in self.emails.values_mut() { match &mut email.mailbox_ids { MaybeResultReference::Reference(reference) => { email.mailbox_ids = MaybeResultReference::Value( response .eval_result_references(reference)? .into_ids::(reference) .map(|f| f.map(MaybeIdReference::Id)) .collect::>()?, ); } MaybeResultReference::Value(values) => { for value in values { if let MaybeIdReference::Reference(ir) = value { *value = MaybeIdReference::Id(response.eval_id_reference(ir)?); } } } } } Ok(()) } } impl ResolveReference for GetSearchSnippetRequest { fn resolve_references(&mut self, response: &Response<'_>) -> trc::Result<()> { // Resolve emailIds references if let MaybeResultReference::Reference(reference) = &self.email_ids { self.email_ids = MaybeResultReference::Value( response .eval_result_references(reference)? .into_ids::(reference) .map(|f| f.map(MaybeInvalid::Value)) .collect::>()?, ); } Ok(()) } } impl ResolveReference for BlobUploadRequest { fn resolve_references(&mut self, response: &Response<'_>) -> trc::Result<()> { let mut graph = HashMap::with_capacity(self.create.len()); for (create_id, object) in self.create.iter_mut() { for data in &mut object.data { if let DataSourceObject::Id { id, .. } = data && let MaybeIdReference::Reference(parent_id) = id { match response.created_ids.get(parent_id) { Some(AnyId::BlobId(blob_id)) => { *id = MaybeIdReference::Id(blob_id.clone()); } Some(_) => { return Err(trc::JmapEvent::InvalidResultReference.into_err().details( format_compact!( "Id reference {parent_id:?} points to invalid type." ), )); } None => { graph .entry(create_id.to_string()) .or_insert_with(Vec::new) .push(parent_id.to_string()); } } } } } // Perform topological sort if !graph.is_empty() { self.create = topological_sort(&mut self.create, graph)?; } Ok(()) } } impl ResolveCreatedReference for SetResponse where T: JmapObject, { fn get_created_id(&self, id_ref: &str) -> Option { self.created .get(id_ref) .and_then(|v| v.as_object()) .and_then(|v| v.get(&Key::Property(T::ID_PROPERTY))) .and_then(|v| v.as_element()) .and_then(|v| v.as_any_id()) } } ================================================ FILE: crates/jmap-proto/src/request/capability.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::fmt; use crate::{ object::{email::EmailComparator, file_node::FileNodeComparator}, response::serialize::serialize_hex, types::date::UTCDate, }; use ahash::AHashMap; use serde::{Deserialize, Deserializer}; use types::{id::Id, type_state::DataType}; use utils::map::vec_map::VecMap; #[derive(Debug, Clone, serde::Serialize)] pub struct Session { #[serde(rename(serialize = "capabilities"))] pub capabilities: VecMap, #[serde(rename(serialize = "accounts"))] pub accounts: VecMap, #[serde(rename(serialize = "primaryAccounts"))] pub primary_accounts: VecMap, #[serde(rename(serialize = "username"))] pub username: String, #[serde(rename(serialize = "apiUrl"))] pub api_url: String, #[serde(rename(serialize = "downloadUrl"))] pub download_url: String, #[serde(rename(serialize = "uploadUrl"))] pub upload_url: String, #[serde(rename(serialize = "eventSourceUrl"))] pub event_source_url: String, #[serde(rename(serialize = "state"))] #[serde(serialize_with = "serialize_hex")] pub state: u32, #[serde(skip)] pub base_url: String, } #[derive(Debug, Clone, serde::Serialize)] pub struct Account { #[serde(rename(serialize = "name"))] pub name: String, #[serde(rename(serialize = "isPersonal"))] pub is_personal: bool, #[serde(rename(serialize = "isReadOnly"))] pub is_read_only: bool, #[serde(rename(serialize = "accountCapabilities"))] pub account_capabilities: VecMap, } #[derive(Debug, Clone, Copy, serde::Serialize, Hash, PartialEq, Eq, PartialOrd, Ord)] pub enum Capability { #[serde(rename(serialize = "urn:ietf:params:jmap:core"))] Core = 1 << 0, #[serde(rename(serialize = "urn:ietf:params:jmap:mail"))] Mail = 1 << 1, #[serde(rename(serialize = "urn:ietf:params:jmap:submission"))] Submission = 1 << 2, #[serde(rename(serialize = "urn:ietf:params:jmap:vacationresponse"))] VacationResponse = 1 << 3, #[serde(rename(serialize = "urn:ietf:params:jmap:contacts"))] Contacts = 1 << 4, #[serde(rename(serialize = "urn:ietf:params:jmap:contacts:parse"))] ContactsParse = 1 << 5, #[serde(rename(serialize = "urn:ietf:params:jmap:calendars"))] Calendars = 1 << 6, #[serde(rename(serialize = "urn:ietf:params:jmap:calendars:parse"))] CalendarsParse = 1 << 7, #[serde(rename(serialize = "urn:ietf:params:jmap:websocket"))] WebSocket = 1 << 8, #[serde(rename(serialize = "urn:ietf:params:jmap:sieve"))] Sieve = 1 << 9, #[serde(rename(serialize = "urn:ietf:params:jmap:blob"))] Blob = 1 << 10, #[serde(rename(serialize = "urn:ietf:params:jmap:quota"))] Quota = 1 << 11, #[serde(rename(serialize = "urn:ietf:params:jmap:principals"))] Principals = 1 << 12, #[serde(rename(serialize = "urn:ietf:params:jmap:principals:owner"))] PrincipalsOwner = 1 << 13, #[serde(rename(serialize = "urn:ietf:params:jmap:principals:availability"))] PrincipalsAvailability = 1 << 14, #[serde(rename(serialize = "urn:ietf:params:jmap:filenode"))] FileNode = 1 << 15, } #[derive(Debug, Clone, Copy, Default)] #[repr(transparent)] pub struct CapabilityIds(pub u32); #[derive(Debug, Clone, serde::Serialize)] #[serde(untagged)] #[allow(dead_code)] pub enum Capabilities { Core(CoreCapabilities), Mail(MailCapabilities), Submission(SubmissionCapabilities), WebSocket(WebSocketCapabilities), SieveAccount(SieveAccountCapabilities), SieveSession(SieveSessionCapabilities), Blob(BlobCapabilities), Contacts(ContactsCapabilities), Principals(PrincipalCapabilities), PrincipalsAvailability(PrincipalAvailabilityCapabilities), Calendar(CalendarCapabilities), FileNode(FileNodeCapabilities), Empty(EmptyCapabilities), } #[derive(Debug, Clone, serde::Serialize)] pub struct CoreCapabilities { #[serde(rename(serialize = "maxSizeUpload"))] pub max_size_upload: usize, #[serde(rename(serialize = "maxConcurrentUpload"))] pub max_concurrent_upload: usize, #[serde(rename(serialize = "maxSizeRequest"))] pub max_size_request: usize, #[serde(rename(serialize = "maxConcurrentRequests"))] pub max_concurrent_requests: usize, #[serde(rename(serialize = "maxCallsInRequest"))] pub max_calls_in_request: usize, #[serde(rename(serialize = "maxObjectsInGet"))] pub max_objects_in_get: usize, #[serde(rename(serialize = "maxObjectsInSet"))] pub max_objects_in_set: usize, #[serde(rename(serialize = "collationAlgorithms"))] pub collation_algorithms: Vec, } #[derive(Debug, Clone, serde::Serialize)] pub struct WebSocketCapabilities { #[serde(rename(serialize = "url"))] pub url: String, #[serde(rename(serialize = "supportsPush"))] pub supports_push: bool, } #[derive(Debug, Clone, serde::Serialize)] pub struct SieveSessionCapabilities { #[serde(rename(serialize = "implementation"))] pub implementation: &'static str, } #[derive(Debug, Clone, serde::Serialize)] pub struct SieveAccountCapabilities { #[serde(rename(serialize = "maxSizeScriptName"))] pub max_script_name: usize, #[serde(rename(serialize = "maxSizeScript"))] pub max_script_size: usize, #[serde(rename(serialize = "maxNumberScripts"))] pub max_scripts: usize, #[serde(rename(serialize = "maxNumberRedirects"))] pub max_redirects: usize, #[serde(rename(serialize = "sieveExtensions"))] pub extensions: Vec, #[serde(rename(serialize = "notificationMethods"))] pub notification_methods: Option>, #[serde(rename(serialize = "externalLists"))] pub ext_lists: Option>, } #[derive(Debug, Clone, serde::Serialize)] pub struct MailCapabilities { #[serde(rename(serialize = "maxMailboxesPerEmail"))] pub max_mailboxes_per_email: Option, #[serde(rename(serialize = "maxMailboxDepth"))] pub max_mailbox_depth: usize, #[serde(rename(serialize = "maxSizeMailboxName"))] pub max_size_mailbox_name: usize, #[serde(rename(serialize = "maxSizeAttachmentsPerEmail"))] pub max_size_attachments_per_email: usize, #[serde(rename(serialize = "emailQuerySortOptions"))] pub email_query_sort_options: Vec, #[serde(rename(serialize = "mayCreateTopLevelMailbox"))] pub may_create_top_level_mailbox: bool, } #[derive(Debug, Clone, serde::Serialize)] pub struct SubmissionCapabilities { #[serde(rename(serialize = "maxDelayedSend"))] pub max_delayed_send: usize, #[serde(rename(serialize = "submissionExtensions"))] pub submission_extensions: VecMap>, } #[derive(Debug, Clone, serde::Serialize)] pub struct BlobCapabilities { #[serde(rename(serialize = "maxSizeBlobSet"))] pub max_size_blob_set: usize, #[serde(rename(serialize = "maxDataSources"))] pub max_data_sources: usize, #[serde(rename(serialize = "supportedTypeNames"))] pub supported_type_names: Vec, #[serde(rename(serialize = "supportedDigestAlgorithms"))] pub supported_digest_algorithms: Vec<&'static str>, } #[derive(Debug, Clone, serde::Serialize)] pub struct CalendarCapabilities { #[serde(rename(serialize = "maxCalendarsPerEvent"))] pub max_calendars_per_event: Option, #[serde(rename(serialize = "minDateTime"))] pub min_date_time: UTCDate, #[serde(rename(serialize = "maxDateTime"))] pub max_date_time: UTCDate, #[serde(rename(serialize = "maxExpandedQueryDuration"))] pub max_expanded_query_duration: String, #[serde(rename(serialize = "maxParticipantsPerEvent"))] pub max_participants_per_event: Option, #[serde(rename(serialize = "mayCreateCalendar"))] pub may_create_calendar: bool, } #[derive(Debug, Clone, serde::Serialize)] pub struct ContactsCapabilities { #[serde(rename(serialize = "maxAddressBooksPerCard"))] pub max_address_books_per_card: Option, #[serde(rename(serialize = "mayCreateAddressBook"))] pub may_create_address_book: bool, } #[derive(Debug, Clone, serde::Serialize)] pub struct PrincipalAvailabilityCapabilities { #[serde(rename(serialize = "maxAvailabilityDuration"))] pub max_availability_duration: String, } #[derive(Debug, Clone, serde::Serialize)] pub struct PrincipalCapabilities { #[serde(rename(serialize = "currentUserPrincipalId"))] pub current_user_principal_id: Option, } /*#[derive(Debug, Clone, serde::Serialize)] pub struct PrincipalOwnerCapabilities { #[serde(rename(serialize = "accountIdForPrincipal"))] pub account_id_for_principal: Id, #[serde(rename(serialize = "principalId"))] pub principal_id: Id, } #[derive(Debug, Clone, serde::Serialize)] pub struct PrincipalCalendarCapabilities { #[serde(rename(serialize = "accountIdForPrincipal"))] pub account_id_for_principal: Option, #[serde(rename(serialize = "mayGetAvailability"))] pub may_get_availability: bool, #[serde(rename(serialize = "mayShareWith"))] pub may_share_with: bool, #[serde(rename(serialize = "calendarAddress"))] pub calendar_address: String, }*/ #[derive(Debug, Clone, serde::Serialize)] pub struct FileNodeCapabilities { #[serde(rename(serialize = "maxFileNodeDepth"))] pub max_file_node_depth: Option, #[serde(rename(serialize = "maxSizeFileNodeName"))] pub max_size_file_node_name: usize, #[serde(rename(serialize = "fileNodeQuerySortOptions"))] pub file_node_query_sort_options: Vec, #[serde(rename(serialize = "mayCreateTopLevelFileNode"))] pub may_create_top_level_file_node: bool, } #[derive(Debug, Clone, Default, serde::Serialize)] pub struct EmptyCapabilities {} #[derive(Default, Clone)] pub struct BaseCapabilities { pub session: VecMap, pub account: AHashMap, } impl Capability { pub fn as_str(&self) -> &'static str { match self { Capability::Core => "urn:ietf:params:jmap:core", Capability::Mail => "urn:ietf:params:jmap:mail", Capability::Submission => "urn:ietf:params:jmap:submission", Capability::VacationResponse => "urn:ietf:params:jmap:vacationresponse", Capability::Contacts => "urn:ietf:params:jmap:contacts", Capability::ContactsParse => "urn:ietf:params:jmap:contacts:parse", Capability::Calendars => "urn:ietf:params:jmap:calendars", Capability::CalendarsParse => "urn:ietf:params:jmap:calendars:parse", Capability::WebSocket => "urn:ietf:params:jmap:websocket", Capability::Sieve => "urn:ietf:params:jmap:sieve", Capability::Blob => "urn:ietf:params:jmap:blob", Capability::Quota => "urn:ietf:params:jmap:quota", Capability::Principals => "urn:ietf:params:jmap:principals", Capability::PrincipalsOwner => "urn:ietf:params:jmap:principals:owner", Capability::PrincipalsAvailability => "urn:ietf:params:jmap:principals:availability", Capability::FileNode => "urn:ietf:params:jmap:filenode", } } pub fn all_capabilities() -> &'static [Capability] { &[ Capability::Core, Capability::Mail, Capability::Submission, Capability::VacationResponse, Capability::Contacts, Capability::ContactsParse, Capability::Calendars, Capability::CalendarsParse, Capability::WebSocket, Capability::Sieve, Capability::Blob, Capability::Quota, Capability::Principals, Capability::PrincipalsAvailability, Capability::FileNode, ] } } impl Session { pub fn new(base_url: impl Into, base_capabilities: &BaseCapabilities) -> Session { let base_url = base_url.into(); let mut capabilities = base_capabilities.session.clone(); capabilities.append( Capability::WebSocket, Capabilities::WebSocket(WebSocketCapabilities::new(&base_url)), ); Session { capabilities, accounts: VecMap::new(), primary_accounts: VecMap::new(), username: "".to_string(), api_url: format!("{}/jmap/", base_url), download_url: format!( "{}/jmap/download/{{accountId}}/{{blobId}}/{{name}}?accept={{type}}", base_url ), upload_url: format!("{}/jmap/upload/{{accountId}}/", base_url), event_source_url: format!( "{}/jmap/eventsource/?types={{types}}&closeafter={{closeafter}}&ping={{ping}}", base_url ), base_url, state: 0, } } pub fn set_state(&mut self, state: u32) { self.state = state; } pub fn api_url(&self) -> &str { &self.api_url } pub fn base_url(&self) -> &str { &self.base_url } } impl Default for SieveSessionCapabilities { fn default() -> Self { Self { implementation: "Stalwart v1.0.0", } } } impl WebSocketCapabilities { pub fn new(base_url: &str) -> Self { WebSocketCapabilities { url: format!( "ws{}/jmap/ws", base_url.strip_prefix("http").unwrap_or_default() ), supports_push: true, } } } impl Capabilities { pub fn to_account_capabilities( &self, current_user_principal_id: Option, may_create: bool, ) -> Capabilities { match self { Capabilities::Contacts(contacts_capabilities) => { Capabilities::Contacts(ContactsCapabilities { may_create_address_book: may_create, ..contacts_capabilities.clone() }) } Capabilities::Principals(_) => Capabilities::Principals(PrincipalCapabilities { current_user_principal_id, }), Capabilities::Calendar(calendar_capabilities) => { Capabilities::Calendar(CalendarCapabilities { may_create_calendar: may_create, ..calendar_capabilities.clone() }) } Capabilities::FileNode(file_node_capabilities) => { Capabilities::FileNode(FileNodeCapabilities { may_create_top_level_file_node: may_create, ..file_node_capabilities.clone() }) } _ => self.clone(), } } } impl Capability { pub fn parse(s: &str) -> Option { hashify::tiny_map!(s.as_bytes(), "urn:ietf:params:jmap:core" => Capability::Core, "urn:ietf:params:jmap:mail" => Capability::Mail, "urn:ietf:params:jmap:submission" => Capability::Submission, "urn:ietf:params:jmap:vacationresponse" => Capability::VacationResponse, "urn:ietf:params:jmap:contacts" => Capability::Contacts, "urn:ietf:params:jmap:calendars" => Capability::Calendars, "urn:ietf:params:jmap:websocket" => Capability::WebSocket, "urn:ietf:params:jmap:sieve" => Capability::Sieve, "urn:ietf:params:jmap:blob" => Capability::Blob, "urn:ietf:params:jmap:quota" => Capability::Quota, "urn:ietf:params:jmap:principals" => Capability::Principals, "urn:ietf:params:jmap:principals:owner" => Capability::PrincipalsOwner, "urn:ietf:params:jmap:filenode" => Capability::FileNode, "urn:ietf:params:jmap:principals:availability" => Capability::PrincipalsAvailability, "urn:ietf:params:jmap:contacts:parse" => Capability::ContactsParse, "urn:ietf:params:jmap:calendars:parse" => Capability::CalendarsParse, ) } } impl<'de> Deserialize<'de> for CapabilityIds { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { struct CapabilityIdsVisitor; impl<'de> serde::de::Visitor<'de> for CapabilityIdsVisitor { type Value = CapabilityIds; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("an array of capability strings") } fn visit_seq(self, mut seq: A) -> Result where A: serde::de::SeqAccess<'de>, { let mut capability_flags = 0u32; while let Some(capability_str) = seq.next_element::<&str>()? { let capability = Capability::parse(capability_str).ok_or_else(|| { serde::de::Error::custom(format!("Unknown capability: {capability_str:?}")) })?; capability_flags |= capability as u32; } Ok(CapabilityIds(capability_flags)) } } deserializer.deserialize_seq(CapabilityIdsVisitor) } } ================================================ FILE: crates/jmap-proto/src/request/deserialize.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{fmt, marker::PhantomData}; use serde::{ Deserializer, de::{self, MapAccess, Visitor}, }; pub trait DeserializeArguments<'de> { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: MapAccess<'de>; } impl<'de> DeserializeArguments<'de> for () { fn deserialize_argument(&mut self, _key: &str, map: &mut A) -> Result<(), A::Error> where A: MapAccess<'de>, { let _: de::IgnoredAny = map.next_value()?; Ok(()) } } pub(crate) fn deserialize_request<'de, T, D>(deserializer: D) -> Result where T: DeserializeArguments<'de> + Default, D: Deserializer<'de>, { struct DirectArgumentsVisitor { _phantom: PhantomData, } impl DirectArgumentsVisitor { fn new() -> Self { Self { _phantom: PhantomData, } } } impl<'de, T> Visitor<'de> for DirectArgumentsVisitor where T: DeserializeArguments<'de> + Default, { type Value = T; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("a JMAP request object") } fn visit_map(self, mut map: A) -> Result where A: MapAccess<'de>, { let mut target = T::default(); while let Some(key) = map.next_key::<&str>()? { target .deserialize_argument(key, &mut map) .map_err(de::Error::custom)?; } Ok(target) } } deserializer.deserialize_map(DirectArgumentsVisitor::::new()) } ================================================ FILE: crates/jmap-proto/src/request/method.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::fmt::Display; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct MethodName { pub obj: MethodObject, pub fnc: MethodFunction, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MethodObject { Email, Mailbox, Core, Blob, PushSubscription, Thread, SearchSnippet, Identity, EmailSubmission, VacationResponse, SieveScript, Principal, Quota, Calendar, CalendarEvent, CalendarEventNotification, AddressBook, ContactCard, FileNode, ParticipantIdentity, ShareNotification, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MethodFunction { Get, Set, Changes, Query, QueryChanges, Copy, Import, Parse, Validate, Lookup, Upload, Echo, GetAvailability, } impl Display for MethodName { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(self.as_str()) } } impl MethodName { pub fn new(obj: MethodObject, fnc: MethodFunction) -> Self { Self { obj, fnc } } pub fn error() -> Self { Self { obj: MethodObject::Thread, fnc: MethodFunction::Echo, } } pub fn as_str(&self) -> &'static str { match (self.fnc, self.obj) { (MethodFunction::Get, MethodObject::PushSubscription) => "PushSubscription/get", (MethodFunction::Set, MethodObject::PushSubscription) => "PushSubscription/set", (MethodFunction::Get, MethodObject::Mailbox) => "Mailbox/get", (MethodFunction::Changes, MethodObject::Mailbox) => "Mailbox/changes", (MethodFunction::Query, MethodObject::Mailbox) => "Mailbox/query", (MethodFunction::QueryChanges, MethodObject::Mailbox) => "Mailbox/queryChanges", (MethodFunction::Set, MethodObject::Mailbox) => "Mailbox/set", (MethodFunction::Get, MethodObject::Thread) => "Thread/get", (MethodFunction::Changes, MethodObject::Thread) => "Thread/changes", (MethodFunction::Get, MethodObject::Email) => "Email/get", (MethodFunction::Changes, MethodObject::Email) => "Email/changes", (MethodFunction::Query, MethodObject::Email) => "Email/query", (MethodFunction::QueryChanges, MethodObject::Email) => "Email/queryChanges", (MethodFunction::Set, MethodObject::Email) => "Email/set", (MethodFunction::Copy, MethodObject::Email) => "Email/copy", (MethodFunction::Import, MethodObject::Email) => "Email/import", (MethodFunction::Parse, MethodObject::Email) => "Email/parse", (MethodFunction::Get, MethodObject::SearchSnippet) => "SearchSnippet/get", (MethodFunction::Get, MethodObject::Identity) => "Identity/get", (MethodFunction::Changes, MethodObject::Identity) => "Identity/changes", (MethodFunction::Set, MethodObject::Identity) => "Identity/set", (MethodFunction::Get, MethodObject::EmailSubmission) => "EmailSubmission/get", (MethodFunction::Changes, MethodObject::EmailSubmission) => "EmailSubmission/changes", (MethodFunction::Query, MethodObject::EmailSubmission) => "EmailSubmission/query", (MethodFunction::QueryChanges, MethodObject::EmailSubmission) => { "EmailSubmission/queryChanges" } (MethodFunction::Set, MethodObject::EmailSubmission) => "EmailSubmission/set", (MethodFunction::Get, MethodObject::VacationResponse) => "VacationResponse/get", (MethodFunction::Set, MethodObject::VacationResponse) => "VacationResponse/set", (MethodFunction::Get, MethodObject::SieveScript) => "SieveScript/get", (MethodFunction::Set, MethodObject::SieveScript) => "SieveScript/set", (MethodFunction::Query, MethodObject::SieveScript) => "SieveScript/query", (MethodFunction::Validate, MethodObject::SieveScript) => "SieveScript/validate", (MethodFunction::Get, MethodObject::Principal) => "Principal/get", (MethodFunction::Set, MethodObject::Principal) => "Principal/set", (MethodFunction::Query, MethodObject::Principal) => "Principal/query", (MethodFunction::Changes, MethodObject::Principal) => "Principal/changes", (MethodFunction::QueryChanges, MethodObject::Principal) => "Principal/queryChanges", (MethodFunction::GetAvailability, MethodObject::Principal) => "Principal/getAvailability", (MethodFunction::Get, MethodObject::Quota) => "Quota/get", (MethodFunction::Changes, MethodObject::Quota) => "Quota/changes", (MethodFunction::Query, MethodObject::Quota) => "Quota/query", (MethodFunction::QueryChanges, MethodObject::Quota) => "Quota/queryChanges", (MethodFunction::Get, MethodObject::Blob) => "Blob/get", (MethodFunction::Copy, MethodObject::Blob) => "Blob/copy", (MethodFunction::Lookup, MethodObject::Blob) => "Blob/lookup", (MethodFunction::Upload, MethodObject::Blob) => "Blob/upload", (MethodFunction::Get, MethodObject::AddressBook) => "AddressBook/get", (MethodFunction::Changes, MethodObject::AddressBook) => "AddressBook/changes", (MethodFunction::Set, MethodObject::AddressBook) => "AddressBook/set", (MethodFunction::Get, MethodObject::ContactCard) => "ContactCard/get", (MethodFunction::Changes, MethodObject::ContactCard) => "ContactCard/changes", (MethodFunction::Query, MethodObject::ContactCard) => "ContactCard/query", (MethodFunction::QueryChanges, MethodObject::ContactCard) => "ContactCard/queryChanges", (MethodFunction::Set, MethodObject::ContactCard) => "ContactCard/set", (MethodFunction::Copy, MethodObject::ContactCard) => "ContactCard/copy", (MethodFunction::Parse, MethodObject::ContactCard) => "ContactCard/parse", (MethodFunction::Get, MethodObject::FileNode) => "FileNode/get", (MethodFunction::Changes, MethodObject::FileNode) => "FileNode/changes", (MethodFunction::Query, MethodObject::FileNode) => "FileNode/query", (MethodFunction::QueryChanges, MethodObject::FileNode) => "FileNode/queryChanges", (MethodFunction::Set, MethodObject::FileNode) => "FileNode/set", (MethodFunction::Get, MethodObject::ShareNotification) => "ShareNotification/get", (MethodFunction::Changes, MethodObject::ShareNotification) => "ShareNotification/changes", (MethodFunction::Query, MethodObject::ShareNotification) => "ShareNotification/query", (MethodFunction::QueryChanges, MethodObject::ShareNotification) => "ShareNotification/queryChanges", (MethodFunction::Set, MethodObject::ShareNotification) => "ShareNotification/set", (MethodFunction::Get, MethodObject::Calendar) => "Calendar/get", (MethodFunction::Changes, MethodObject::Calendar) => "Calendar/changes", (MethodFunction::Set, MethodObject::Calendar) => "Calendar/set", (MethodFunction::Get, MethodObject::CalendarEvent) => "CalendarEvent/get", (MethodFunction::Changes, MethodObject::CalendarEvent) => "CalendarEvent/changes", (MethodFunction::Query, MethodObject::CalendarEvent) => "CalendarEvent/query", (MethodFunction::QueryChanges, MethodObject::CalendarEvent) => "CalendarEvent/queryChanges", (MethodFunction::Set, MethodObject::CalendarEvent) => "CalendarEvent/set", (MethodFunction::Copy, MethodObject::CalendarEvent) => "CalendarEvent/copy", (MethodFunction::Parse, MethodObject::CalendarEvent) => "CalendarEvent/parse", (MethodFunction::Get, MethodObject::CalendarEventNotification) => "CalendarEventNotification/get", (MethodFunction::Changes, MethodObject::CalendarEventNotification) => "CalendarEventNotification/changes", (MethodFunction::Query, MethodObject::CalendarEventNotification) => "CalendarEventNotification/query", (MethodFunction::QueryChanges, MethodObject::CalendarEventNotification) => "CalendarEventNotification/queryChanges", (MethodFunction::Set, MethodObject::CalendarEventNotification) => "CalendarEventNotification/set", (MethodFunction::Get, MethodObject::ParticipantIdentity) => "ParticipantIdentity/get", (MethodFunction::Changes, MethodObject::ParticipantIdentity) => "ParticipantIdentity/changes", (MethodFunction::Set, MethodObject::ParticipantIdentity) => "ParticipantIdentity/set", (MethodFunction::Echo, MethodObject::Core) => "Core/echo", _ => "error", } } pub fn parse(s: &str) -> Option { hashify::tiny_map!(s.as_bytes(), "PushSubscription/get" => (MethodObject::PushSubscription, MethodFunction::Get), "PushSubscription/set" => (MethodObject::PushSubscription, MethodFunction::Set), "Mailbox/get" => (MethodObject::Mailbox, MethodFunction::Get), "Mailbox/changes" => (MethodObject::Mailbox, MethodFunction::Changes), "Mailbox/query" => (MethodObject::Mailbox, MethodFunction::Query), "Mailbox/queryChanges" => (MethodObject::Mailbox, MethodFunction::QueryChanges), "Mailbox/set" => (MethodObject::Mailbox, MethodFunction::Set), "Thread/get" => (MethodObject::Thread, MethodFunction::Get), "Thread/changes" => (MethodObject::Thread, MethodFunction::Changes), "Email/get" => (MethodObject::Email, MethodFunction::Get), "Email/changes" => (MethodObject::Email, MethodFunction::Changes), "Email/query" => (MethodObject::Email, MethodFunction::Query), "Email/queryChanges" => (MethodObject::Email, MethodFunction::QueryChanges), "Email/set" => (MethodObject::Email, MethodFunction::Set), "Email/copy" => (MethodObject::Email, MethodFunction::Copy), "Email/import" => (MethodObject::Email, MethodFunction::Import), "Email/parse" => (MethodObject::Email, MethodFunction::Parse), "SearchSnippet/get" => (MethodObject::SearchSnippet, MethodFunction::Get), "Identity/get" => (MethodObject::Identity, MethodFunction::Get), "Identity/changes" => (MethodObject::Identity, MethodFunction::Changes), "Identity/set" => (MethodObject::Identity, MethodFunction::Set), "EmailSubmission/get" => (MethodObject::EmailSubmission, MethodFunction::Get), "EmailSubmission/changes" => (MethodObject::EmailSubmission, MethodFunction::Changes), "EmailSubmission/query" => (MethodObject::EmailSubmission, MethodFunction::Query), "EmailSubmission/queryChanges" => (MethodObject::EmailSubmission, MethodFunction::QueryChanges), "EmailSubmission/set" => (MethodObject::EmailSubmission, MethodFunction::Set), "VacationResponse/get" => (MethodObject::VacationResponse, MethodFunction::Get), "VacationResponse/set" => (MethodObject::VacationResponse, MethodFunction::Set), "SieveScript/get" => (MethodObject::SieveScript, MethodFunction::Get), "SieveScript/set" => (MethodObject::SieveScript, MethodFunction::Set), "SieveScript/query" => (MethodObject::SieveScript, MethodFunction::Query), "SieveScript/validate" => (MethodObject::SieveScript, MethodFunction::Validate), "Principal/get" => (MethodObject::Principal, MethodFunction::Get), "Principal/set" => (MethodObject::Principal, MethodFunction::Set), "Principal/query" => (MethodObject::Principal, MethodFunction::Query), "Principal/changes" => (MethodObject::Principal, MethodFunction::Changes), "Principal/queryChanges" => (MethodObject::Principal, MethodFunction::QueryChanges), "Principal/getAvailability" => (MethodObject::Principal, MethodFunction::GetAvailability), "Quota/get" => (MethodObject::Quota, MethodFunction::Get), "Quota/changes" => (MethodObject::Quota, MethodFunction::Changes), "Quota/query" => (MethodObject::Quota, MethodFunction::Query), "Quota/queryChanges" => (MethodObject::Quota, MethodFunction::QueryChanges), "Blob/get" => (MethodObject::Blob, MethodFunction::Get), "Blob/copy" => (MethodObject::Blob, MethodFunction::Copy), "Blob/lookup" => (MethodObject::Blob, MethodFunction::Lookup), "Blob/upload" => (MethodObject::Blob, MethodFunction::Upload), "AddressBook/get" => (MethodObject::AddressBook, MethodFunction::Get), "AddressBook/changes" => (MethodObject::AddressBook, MethodFunction::Changes), "AddressBook/set" => (MethodObject::AddressBook, MethodFunction::Set), "ContactCard/get" => (MethodObject::ContactCard, MethodFunction::Get), "ContactCard/changes" => (MethodObject::ContactCard, MethodFunction::Changes), "ContactCard/query" => (MethodObject::ContactCard, MethodFunction::Query), "ContactCard/queryChanges" => (MethodObject::ContactCard, MethodFunction::QueryChanges), "ContactCard/set" => (MethodObject::ContactCard, MethodFunction::Set), "ContactCard/copy" => (MethodObject::ContactCard, MethodFunction::Copy), "ContactCard/parse" => (MethodObject::ContactCard, MethodFunction::Parse), "FileNode/get" => (MethodObject::FileNode, MethodFunction::Get), "FileNode/changes" => (MethodObject::FileNode, MethodFunction::Changes), "FileNode/query" => (MethodObject::FileNode, MethodFunction::Query), "FileNode/queryChanges" => (MethodObject::FileNode, MethodFunction::QueryChanges), "FileNode/set" => (MethodObject::FileNode, MethodFunction::Set), "ShareNotification/get" => (MethodObject::ShareNotification, MethodFunction::Get), "ShareNotification/changes" => (MethodObject::ShareNotification, MethodFunction::Changes), "ShareNotification/set" => (MethodObject::ShareNotification, MethodFunction::Set), "ShareNotification/query" => (MethodObject::ShareNotification, MethodFunction::Query), "ShareNotification/queryChanges" => (MethodObject::ShareNotification, MethodFunction::QueryChanges), "Calendar/get" => (MethodObject::Calendar, MethodFunction::Get), "Calendar/changes" => (MethodObject::Calendar, MethodFunction::Changes), "Calendar/set" => (MethodObject::Calendar, MethodFunction::Set), "CalendarEvent/get" => (MethodObject::CalendarEvent, MethodFunction::Get), "CalendarEvent/changes" => (MethodObject::CalendarEvent, MethodFunction::Changes), "CalendarEvent/query" => (MethodObject::CalendarEvent, MethodFunction::Query), "CalendarEvent/queryChanges" => (MethodObject::CalendarEvent, MethodFunction::QueryChanges), "CalendarEvent/set" => (MethodObject::CalendarEvent, MethodFunction::Set), "CalendarEvent/copy" => (MethodObject::CalendarEvent, MethodFunction::Copy), "CalendarEvent/parse" => (MethodObject::CalendarEvent, MethodFunction::Parse), "CalendarEventNotification/get" => (MethodObject::CalendarEventNotification, MethodFunction::Get), "CalendarEventNotification/changes" => (MethodObject::CalendarEventNotification, MethodFunction::Changes), "CalendarEventNotification/set" => (MethodObject::CalendarEventNotification, MethodFunction::Set), "CalendarEventNotification/query" => (MethodObject::CalendarEventNotification, MethodFunction::Query), "CalendarEventNotification/queryChanges" => (MethodObject::CalendarEventNotification, MethodFunction::QueryChanges), "ParticipantIdentity/get" => (MethodObject::ParticipantIdentity, MethodFunction::Get), "ParticipantIdentity/changes" => (MethodObject::ParticipantIdentity, MethodFunction::Changes), "ParticipantIdentity/set" => (MethodObject::ParticipantIdentity, MethodFunction::Set), "Core/echo" => (MethodObject::Core, MethodFunction::Echo), ).map(|(obj, fnc)| MethodName { obj, fnc }) } } impl Display for MethodObject { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(match self { MethodObject::Blob => "Blob", MethodObject::EmailSubmission => "EmailSubmission", MethodObject::SearchSnippet => "SearchSnippet", MethodObject::Identity => "Identity", MethodObject::VacationResponse => "VacationResponse", MethodObject::PushSubscription => "PushSubscription", MethodObject::SieveScript => "SieveScript", MethodObject::Principal => "Principal", MethodObject::Core => "Core", MethodObject::Mailbox => "Mailbox", MethodObject::Thread => "Thread", MethodObject::Email => "Email", MethodObject::Quota => "Quota", MethodObject::AddressBook => "AddressBook", MethodObject::ContactCard => "ContactCard", MethodObject::FileNode => "FileNode", MethodObject::ParticipantIdentity => "ParticipantIdentity", MethodObject::Calendar => "Calendar", MethodObject::CalendarEvent => "CalendarEvent", MethodObject::CalendarEventNotification => "CalendarEventNotification", MethodObject::ShareNotification => "ShareNotification", }) } } impl<'de> serde::Deserialize<'de> for MethodName { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { let value = <&str>::deserialize(deserializer)?; MethodName::parse(value).ok_or_else(|| { serde::de::Error::custom(format!("Invalid method name: {:?}", value)) }) } } impl serde::Serialize for MethodName { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { serializer.serialize_str(self.as_str()) } } ================================================ FILE: crates/jmap-proto/src/request/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod capability; pub mod deserialize; pub mod method; pub mod parser; pub mod reference; pub mod websocket; use self::method::MethodName; use crate::{ method::{ availability::GetAvailabilityRequest, changes::ChangesRequest, copy::{CopyBlobRequest, CopyRequest}, get::GetRequest, import::ImportEmailRequest, lookup::BlobLookupRequest, parse::ParseRequest, query::QueryRequest, query_changes::QueryChangesRequest, search_snippet::GetSearchSnippetRequest, set::SetRequest, upload::BlobUploadRequest, validate::ValidateSieveScriptRequest, }, object::{ AnyId, addressbook::AddressBook, blob::Blob, calendar::Calendar, calendar_event::CalendarEvent, calendar_event_notification::CalendarEventNotification, contact::ContactCard, email::Email, email_submission::EmailSubmission, file_node::FileNode, identity::Identity, mailbox::Mailbox, participant_identity::ParticipantIdentity, principal::Principal, push_subscription::PushSubscription, quota::Quota, share_notification::ShareNotification, sieve::Sieve, thread::Thread, vacation_response::VacationResponse, }, request::{capability::CapabilityIds, reference::MaybeIdReference}, }; use jmap_tools::{Null, Value}; use std::{collections::HashMap, fmt::Debug, str::FromStr}; use utils::map::vec_map::VecMap; #[derive(Debug)] pub struct Request<'x> { pub using: CapabilityIds, pub method_calls: Vec>>, pub created_ids: Option>, } #[derive(Debug)] pub struct Call { pub id: String, pub name: MethodName, pub method: T, } #[derive(Debug)] pub enum RequestMethod<'x> { Get(GetRequestMethod), Set(SetRequestMethod<'x>), Changes(ChangesRequest), Copy(CopyRequestMethod<'x>), ImportEmail(ImportEmailRequest), Parse(ParseRequestMethod), Query(QueryRequestMethod), QueryChanges(QueryChangesRequestMethod), SearchSnippet(GetSearchSnippetRequest), ValidateScript(ValidateSieveScriptRequest), LookupBlob(BlobLookupRequest), UploadBlob(BlobUploadRequest), Echo(Value<'x, Null, Null>), Error(trc::Error), } #[derive(Debug)] pub enum GetRequestMethod { Email(GetRequest), Mailbox(GetRequest), Thread(GetRequest), Identity(GetRequest), EmailSubmission(GetRequest), PushSubscription(GetRequest), Sieve(GetRequest), VacationResponse(GetRequest), Principal(GetRequest), PrincipalAvailability(GetAvailabilityRequest), Quota(GetRequest), Blob(GetRequest), AddressBook(GetRequest), ContactCard(GetRequest), FileNode(GetRequest), Calendar(GetRequest), CalendarEvent(GetRequest), CalendarEventNotification(GetRequest), ParticipantIdentity(GetRequest), ShareNotification(GetRequest), } #[derive(Debug)] pub enum SetRequestMethod<'x> { Email(SetRequest<'x, Email>), Mailbox(SetRequest<'x, Mailbox>), Identity(SetRequest<'x, Identity>), EmailSubmission(SetRequest<'x, EmailSubmission>), PushSubscription(SetRequest<'x, PushSubscription>), Sieve(SetRequest<'x, Sieve>), VacationResponse(SetRequest<'x, VacationResponse>), AddressBook(SetRequest<'x, AddressBook>), ContactCard(SetRequest<'x, ContactCard>), FileNode(SetRequest<'x, FileNode>), ShareNotification(SetRequest<'x, ShareNotification>), Calendar(SetRequest<'x, Calendar>), CalendarEvent(SetRequest<'x, CalendarEvent>), CalendarEventNotification(SetRequest<'x, CalendarEventNotification>), ParticipantIdentity(SetRequest<'x, ParticipantIdentity>), } #[derive(Debug)] pub enum CopyRequestMethod<'x> { Email(CopyRequest<'x, Email>), ContactCard(CopyRequest<'x, ContactCard>), CalendarEvent(CopyRequest<'x, CalendarEvent>), Blob(CopyBlobRequest), } #[derive(Debug)] pub enum QueryRequestMethod { Email(QueryRequest), Mailbox(QueryRequest), EmailSubmission(QueryRequest), Sieve(QueryRequest), Principal(QueryRequest), Quota(QueryRequest), ContactCard(QueryRequest), FileNode(QueryRequest), CalendarEvent(QueryRequest), CalendarEventNotification(QueryRequest), ShareNotification(QueryRequest), } #[derive(Debug)] pub enum QueryChangesRequestMethod { Email(QueryChangesRequest), Mailbox(QueryChangesRequest), EmailSubmission(QueryChangesRequest), Sieve(QueryChangesRequest), Principal(QueryChangesRequest), Quota(QueryChangesRequest), ContactCard(QueryChangesRequest), FileNode(QueryChangesRequest), CalendarEvent(QueryChangesRequest), CalendarEventNotification(QueryChangesRequest), ShareNotification(QueryChangesRequest), } #[derive(Debug)] pub enum ParseRequestMethod { Email(ParseRequest), ContactCard(ParseRequest), CalendarEvent(ParseRequest), } #[derive(Debug, Clone, PartialEq, Eq)] pub enum MaybeInvalid { Value(V), Invalid(String), } impl<'de, V: FromStr> serde::Deserialize<'de> for MaybeInvalid { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { let value = <&str>::deserialize(deserializer)?; if let Ok(id) = V::from_str(value) { Ok(MaybeInvalid::Value(id)) } else { Ok(MaybeInvalid::Invalid(value.to_string())) } } } impl Default for MaybeInvalid { fn default() -> Self { MaybeInvalid::Invalid("".to_string()) } } #[allow(clippy::derivable_impls)] impl Default for Request<'_> { fn default() -> Self { Request { using: CapabilityIds::default(), method_calls: Vec::new(), created_ids: None, } } } impl MaybeInvalid where T: FromStr, { pub fn try_unwrap(self) -> Option { match self { MaybeInvalid::Value(id) => Some(id), MaybeInvalid::Invalid(_) => None, } } } pub trait IntoValid { type Item; fn into_valid(self) -> impl Iterator; } impl IntoValid for Vec> { type Item = T; fn into_valid(self) -> impl Iterator { self.into_iter().filter_map(|v| v.try_unwrap()) } } impl IntoValid for Vec> { type Item = T; fn into_valid(self) -> impl Iterator { self.into_iter().filter_map(|v| v.try_unwrap()) } } impl IntoValid for VecMap, V> { type Item = (T, V); fn into_valid(self) -> impl Iterator { self.into_iter() .filter_map(|(k, v)| k.try_unwrap().map(|k| (k, v))) } } impl IntoValid for VecMap, V> { type Item = (T, V); fn into_valid(self) -> impl Iterator { self.into_iter() .filter_map(|(k, v)| k.try_unwrap().map(|k| (k, v))) } } ================================================ FILE: crates/jmap-proto/src/request/parser.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ Call, Request, RequestMethod, method::{MethodFunction, MethodName, MethodObject}, }; use crate::request::{ CopyRequestMethod, GetRequestMethod, ParseRequestMethod, QueryChangesRequestMethod, QueryRequestMethod, SetRequestMethod, deserialize::{DeserializeArguments, deserialize_request}, }; use serde::{ Deserialize, Deserializer, de::{self, SeqAccess, Visitor}, }; use std::fmt::{self, Display}; impl<'x> Request<'x> { pub fn parse(json: &'x [u8], max_calls: usize, max_size: usize) -> trc::Result { if json.len() <= max_size { match serde_json::from_slice::(json) { Ok(request) => { if request.method_calls.len() <= max_calls { Ok(request) } else { Err(trc::LimitEvent::CallsIn.into_err()) } } Err(err) => Err(trc::JmapEvent::NotRequest .into_err() .details(err.to_string())), } } else { Err(trc::LimitEvent::SizeRequest.into_err()) } } } impl<'de> DeserializeArguments<'de> for Request<'de> { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"using" => { self.using = map.next_value()?; }, b"methodCalls" => { self.method_calls = map.next_value()?; }, b"createdIds" => { self.created_ids = map.next_value()?; }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } struct CallVisitor; impl<'de> Visitor<'de> for CallVisitor { type Value = Call>; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("an array with 3 elements") } fn visit_seq(self, mut seq: V) -> Result>, V::Error> where V: SeqAccess<'de>, { let method_name = seq .next_element::<&str>()? .ok_or_else(|| de::Error::invalid_length(0, &self))?; let name = match MethodName::parse(method_name) { Some(name) => name, None => { // Ignore the rest of the call let _ = seq .next_element::()? .ok_or_else(|| de::Error::invalid_length(1, &self))?; let id = seq .next_element::()? .ok_or_else(|| de::Error::invalid_length(2, &self))?; return Ok(Call { id, method: RequestMethod::Error( trc::JmapEvent::UnknownMethod .into_err() .details(method_name.to_string()), ), name: MethodName::error(), }); } }; let method = match (&name.fnc, &name.obj) { (MethodFunction::Get, MethodObject::Email) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Email(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Get, MethodObject::Mailbox) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Mailbox(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Get, MethodObject::Thread) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Thread(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Get, MethodObject::Identity) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Identity(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Get, MethodObject::EmailSubmission) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::EmailSubmission(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Get, MethodObject::PushSubscription) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::PushSubscription(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Get, MethodObject::VacationResponse) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::VacationResponse(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Get, MethodObject::SieveScript) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Sieve(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Get, MethodObject::Principal) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Principal(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Get, MethodObject::Quota) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Quota(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Get, MethodObject::Blob) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Blob(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Get, MethodObject::Calendar) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Calendar(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Get, MethodObject::CalendarEvent) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::CalendarEvent(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Get, MethodObject::CalendarEventNotification) => { match seq.next_element() { Ok(Some(value)) => { RequestMethod::Get(GetRequestMethod::CalendarEventNotification(value)) } Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } } } (MethodFunction::Get, MethodObject::ParticipantIdentity) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::ParticipantIdentity(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Get, MethodObject::AddressBook) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::AddressBook(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Get, MethodObject::ContactCard) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::ContactCard(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Get, MethodObject::FileNode) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::FileNode(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Get, MethodObject::ShareNotification) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::ShareNotification(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Get, MethodObject::SearchSnippet) => match seq.next_element() { Ok(Some(value)) => RequestMethod::SearchSnippet(value), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Set, MethodObject::Email) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::Email(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Set, MethodObject::Mailbox) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::Mailbox(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Set, MethodObject::Identity) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::Identity(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Set, MethodObject::EmailSubmission) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::EmailSubmission(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Set, MethodObject::PushSubscription) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::PushSubscription(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Set, MethodObject::VacationResponse) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::VacationResponse(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Set, MethodObject::SieveScript) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::Sieve(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Set, MethodObject::Calendar) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::Calendar(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Set, MethodObject::CalendarEvent) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::CalendarEvent(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Set, MethodObject::CalendarEventNotification) => { match seq.next_element() { Ok(Some(value)) => { RequestMethod::Set(SetRequestMethod::CalendarEventNotification(value)) } Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } } } (MethodFunction::Set, MethodObject::ParticipantIdentity) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::ParticipantIdentity(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Set, MethodObject::AddressBook) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::AddressBook(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Set, MethodObject::ContactCard) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::ContactCard(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Set, MethodObject::FileNode) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::FileNode(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Set, MethodObject::ShareNotification) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::ShareNotification(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Query, MethodObject::Email) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::Email(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Query, MethodObject::Mailbox) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::Mailbox(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Query, MethodObject::EmailSubmission) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::EmailSubmission(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Query, MethodObject::SieveScript) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::Sieve(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Query, MethodObject::Principal) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::Principal(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Query, MethodObject::Quota) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::Quota(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Query, MethodObject::CalendarEvent) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::CalendarEvent(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Query, MethodObject::CalendarEventNotification) => { match seq.next_element() { Ok(Some(value)) => { RequestMethod::Query(QueryRequestMethod::CalendarEventNotification(value)) } Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } } } (MethodFunction::Query, MethodObject::ContactCard) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::ContactCard(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Query, MethodObject::FileNode) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Query(QueryRequestMethod::FileNode(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Query, MethodObject::ShareNotification) => match seq.next_element() { Ok(Some(value)) => { RequestMethod::Query(QueryRequestMethod::ShareNotification(value)) } Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::QueryChanges, MethodObject::Email) => match seq.next_element() { Ok(Some(value)) => { RequestMethod::QueryChanges(QueryChangesRequestMethod::Email(value)) } Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::QueryChanges, MethodObject::Mailbox) => match seq.next_element() { Ok(Some(value)) => { RequestMethod::QueryChanges(QueryChangesRequestMethod::Mailbox(value)) } Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::QueryChanges, MethodObject::EmailSubmission) => { match seq.next_element() { Ok(Some(value)) => RequestMethod::QueryChanges( QueryChangesRequestMethod::EmailSubmission(value), ), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } } } (MethodFunction::QueryChanges, MethodObject::SieveScript) => match seq.next_element() { Ok(Some(value)) => { RequestMethod::QueryChanges(QueryChangesRequestMethod::Sieve(value)) } Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::QueryChanges, MethodObject::Principal) => match seq.next_element() { Ok(Some(value)) => { RequestMethod::QueryChanges(QueryChangesRequestMethod::Principal(value)) } Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::QueryChanges, MethodObject::Quota) => match seq.next_element() { Ok(Some(value)) => { RequestMethod::QueryChanges(QueryChangesRequestMethod::Quota(value)) } Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::QueryChanges, MethodObject::CalendarEvent) => match seq.next_element() { Ok(Some(value)) => { RequestMethod::QueryChanges(QueryChangesRequestMethod::CalendarEvent(value)) } Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::QueryChanges, MethodObject::CalendarEventNotification) => { match seq.next_element() { Ok(Some(value)) => RequestMethod::QueryChanges( QueryChangesRequestMethod::CalendarEventNotification(value), ), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } } } (MethodFunction::QueryChanges, MethodObject::ContactCard) => match seq.next_element() { Ok(Some(value)) => { RequestMethod::QueryChanges(QueryChangesRequestMethod::ContactCard(value)) } Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::QueryChanges, MethodObject::FileNode) => match seq.next_element() { Ok(Some(value)) => { RequestMethod::QueryChanges(QueryChangesRequestMethod::FileNode(value)) } Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::QueryChanges, MethodObject::ShareNotification) => { match seq.next_element() { Ok(Some(value)) => RequestMethod::QueryChanges( QueryChangesRequestMethod::ShareNotification(value), ), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } } } (MethodFunction::Changes, _) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Changes(value), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Copy, MethodObject::Email) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Copy(CopyRequestMethod::Email(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Copy, MethodObject::Blob) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Copy(CopyRequestMethod::Blob(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Copy, MethodObject::CalendarEvent) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Copy(CopyRequestMethod::CalendarEvent(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Copy, MethodObject::ContactCard) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Copy(CopyRequestMethod::ContactCard(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Lookup, MethodObject::Blob) => match seq.next_element() { Ok(Some(value)) => RequestMethod::LookupBlob(value), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Upload, MethodObject::Blob) => match seq.next_element() { Ok(Some(value)) => RequestMethod::UploadBlob(value), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Import, MethodObject::Email) => match seq.next_element() { Ok(Some(value)) => RequestMethod::ImportEmail(value), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Parse, MethodObject::Email) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Parse(ParseRequestMethod::Email(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Parse, MethodObject::CalendarEvent) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Parse(ParseRequestMethod::CalendarEvent(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Parse, MethodObject::ContactCard) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Parse(ParseRequestMethod::ContactCard(value)), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::GetAvailability, MethodObject::Principal) => { match seq.next_element() { Ok(Some(value)) => { RequestMethod::Get(GetRequestMethod::PrincipalAvailability(value)) } Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } } } (MethodFunction::Validate, MethodObject::SieveScript) => match seq.next_element() { Ok(Some(value)) => RequestMethod::ValidateScript(value), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, (MethodFunction::Echo, MethodObject::Core) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Echo(value), Err(err) => RequestMethod::invalid(err), Ok(None) => { return Err(de::Error::invalid_length(1, &self)); } }, _ => { return Err(de::Error::custom(format!( "Invalid method function/object combination: {}", method_name ))); } }; let id = seq .next_element::()? .ok_or_else(|| de::Error::invalid_length(2, &self))?; Ok(Call { id, method, name }) } } impl RequestMethod<'_> { fn invalid(err: impl Display) -> Self { RequestMethod::Error( trc::JmapEvent::InvalidArguments .into_err() .details(err.to_string()), ) } } impl<'de> Deserialize<'de> for Request<'de> { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserialize_request(deserializer) } } impl<'de> Deserialize<'de> for Call> { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserializer.deserialize_seq(CallVisitor) } } #[cfg(test)] mod tests { use crate::request::Request; const TEST: &str = r#" { "using": [ "urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail" ], "methodCalls": [ [ "method1", { "arg1": "arg1data", "arg2": "arg2data" }, "c1" ], [ "Core/echo", { "hello": true, "high": 5 }, "c2" ], [ "method3", {"hello": [{"a": {"b": true}}]}, "c3" ] ], "createdIds": { "c1": "m1", "c2": "m2" } } "#; const TEST1: &str = r#" { "using": [ "urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail" ], "methodCalls": [ [ "Email/query", { "accountId": "0", "filter": { "conditions": [ { "hasKeyword": "music", "maxSize": 455 }, { "hasKeyword": "video" }, { "operator": "AND", "conditions": [ { "subject": "test" }, { "minSize": 100 } ] } ], "operator": "OR" }, "sort": [ { "property": "subject", "isAscending": true }, { "property": "allInThreadHaveKeyword", "isAscending": false, "keyword": "$seen" }, { "keyword": "$junk", "property": "someInThreadHaveKeyword", "collation": "i;octet", "isAscending": false } ], "position": 0, "limit": 10 }, "c1" ] ], "createdIds": {} } "#; const TEST2: &str = r##" { "using": [ "urn:ietf:params:jmap:submission", "urn:ietf:params:jmap:mail", "urn:ietf:params:jmap:core" ], "methodCalls": [ [ "Email/set", { "accountId": "c", "create": { "c37ee58b-e224-4799-88e6-1d7484e3b782": { "mailboxIds": { "9": true }, "subject": "test", "from": [ { "name": "Foo", "email": "foo@bar.com" } ], "to": [ { "name": null, "email": "bar@foo.com" } ], "cc": [], "bcc": [], "replyTo": [ { "name": null, "email": "foo@bar.com" } ], "htmlBody": [ { "partId": "c37ee58b-e224-4799-88e6-1d7484e3b782", "type": "text/html" } ], "bodyValues": { "c37ee58b-e224-4799-88e6-1d7484e3b782": { "value": "

test email

", "isEncodingProblem": false, "isTruncated": false } }, "header:User-Agent:asText": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/113.0" } } }, "c0" ], [ "EmailSubmission/set", { "accountId": "c", "create": { "c37ee58b-e224-4799-88e6-1d7484e3b782": { "identityId": "a", "emailId": "#c37ee58b-e224-4799-88e6-1d7484e3b782", "envelope": { "mailFrom": { "email": "foo@bar.com" }, "rcptTo": [ { "email": "bar@foo.com" } ] } } }, "onSuccessUpdateEmail": { "#c37ee58b-e224-4799-88e6-1d7484e3b782": { "mailboxIds/d": true, "mailboxIds/9": null, "keywords/$seen": true, "keywords/$draft": null } } }, "c1" ] ] } "##; #[test] fn parse_request() { println!("{:#?}", Request::parse(TEST.as_bytes(), 10, 10240)); println!("{:#?}", Request::parse(TEST1.as_bytes(), 10, 10240)); println!("{:#?}", Request::parse(TEST2.as_bytes(), 10, 10240)); } } ================================================ FILE: crates/jmap-proto/src/request/reference.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::method::MethodName; use jmap_tools::{JsonPointer, Null}; use std::{borrow::Cow, fmt::Display, str::FromStr}; #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct ResultReference { #[serde(rename = "resultOf")] pub result_of: String, pub name: MethodName, pub path: JsonPointer, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum MaybeIdReference { Id(V), Reference(String), Invalid(String), } #[derive(Debug, Clone, PartialEq, Eq)] pub enum MaybeResultReference { Value(V), Reference(ResultReference), } impl Display for ResultReference { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( f, "{{ resultOf: {}, name: {}, path: {} }}", self.result_of, self.name, self.path ) } } impl Display for MaybeIdReference { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { MaybeIdReference::Id(id) => write!(f, "{}", id), MaybeIdReference::Reference(str) => write!(f, "#{}", str), MaybeIdReference::Invalid(str) => write!(f, "{}", str), } } } impl<'de, V: FromStr> serde::Deserialize<'de> for MaybeIdReference { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { let value = >::deserialize(deserializer)?; if let Some(reference) = value.strip_prefix('#') { if reference.is_empty() { return Ok(MaybeIdReference::Invalid(value.into_owned())); } Ok(MaybeIdReference::Reference(reference.to_string())) } else if let Ok(id) = V::from_str(value.as_ref()) { Ok(MaybeIdReference::Id(id)) } else { Ok(MaybeIdReference::Invalid(value.into_owned())) } } } impl FromStr for MaybeIdReference { type Err = V::Err; fn from_str(s: &str) -> Result { if let Some(reference) = s.strip_prefix('#') { if reference.is_empty() { return Ok(MaybeIdReference::Invalid(s.to_string())); } Ok(MaybeIdReference::Reference(reference.to_string())) } else if let Ok(id) = V::from_str(s) { Ok(MaybeIdReference::Id(id)) } else { Ok(MaybeIdReference::Invalid(s.to_string())) } } } impl serde::Serialize for MaybeIdReference { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { match self { MaybeIdReference::Id(id) => serializer.serialize_str(&id.to_string()), MaybeIdReference::Reference(str) => serializer.serialize_str(&format!("#{}", str)), MaybeIdReference::Invalid(str) => serializer.serialize_str(str), } } } impl Default for MaybeResultReference { fn default() -> Self { MaybeResultReference::Value(V::default()) } } impl MaybeResultReference { pub fn unwrap(self) -> T { match self { MaybeResultReference::Value(v) => v, MaybeResultReference::Reference(_) => T::default(), } } } impl MaybeIdReference { pub fn try_unwrap(self) -> Option { match self { MaybeIdReference::Id(id) => Some(id), _ => None, } } } ================================================ FILE: crates/jmap-proto/src/request/websocket.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::Request; use crate::{ error::request::{RequestError, RequestErrorType, RequestLimitError}, object::AnyId, request::{Call, deserialize::DeserializeArguments}, response::{Response, ResponseMethod, serialize::serialize_hex, status::PushObject}, }; use serde::{ Deserialize, Deserializer, de::{self, MapAccess, Visitor}, }; use std::{borrow::Cow, collections::HashMap, fmt}; use types::type_state::DataType; #[derive(Debug)] pub struct WebSocketRequest<'x> { pub id: Option, pub request: Request<'x>, } #[derive(Debug, serde::Serialize)] pub struct WebSocketResponse<'x> { #[serde(rename = "@type")] _type: WebSocketResponseType, #[serde(rename = "methodResponses")] method_responses: Vec>>, #[serde(rename = "sessionState")] #[serde(serialize_with = "serialize_hex")] session_state: u32, #[serde(rename(deserialize = "createdIds"))] #[serde(skip_serializing_if = "HashMap::is_empty")] created_ids: HashMap, #[serde(rename = "requestId")] #[serde(skip_serializing_if = "Option::is_none")] request_id: Option, } #[derive(Debug, PartialEq, Eq, serde::Serialize)] pub enum WebSocketResponseType { Response, } #[derive(Debug, Default, PartialEq, Eq)] pub struct WebSocketPushEnable { pub data_types: Vec, pub push_state: Option, } #[derive(Debug)] pub enum WebSocketMessage<'x> { Request(WebSocketRequest<'x>), PushEnable(WebSocketPushEnable), PushDisable, } #[derive(serde::Serialize, Debug)] pub struct WebSocketPushObject { #[serde(flatten)] pub push: PushObject, #[serde(rename = "pushState")] #[serde(skip_serializing_if = "Option::is_none")] pub push_state: Option, } #[derive(Debug, serde::Serialize)] pub struct WebSocketRequestError<'x> { #[serde(rename = "@type")] pub type_: WebSocketRequestErrorType, #[serde(rename = "type")] p_type: RequestErrorType, #[serde(skip_serializing_if = "Option::is_none")] limit: Option, status: u16, detail: Cow<'x, str>, #[serde(rename = "requestId")] #[serde(skip_serializing_if = "Option::is_none")] pub request_id: Option, } #[derive(serde::Serialize, Debug)] pub enum WebSocketRequestErrorType { RequestError, } enum MessageType { Request, PushEnable, PushDisable, None, } impl<'x> WebSocketMessage<'x> { pub fn parse(json: &'x [u8], max_calls: usize, max_size: usize) -> trc::Result { if json.len() <= max_size { match serde_json::from_slice::(json) { Ok(WebSocketMessage::Request(req)) if req.request.method_calls.len() > max_calls => { Err(trc::LimitEvent::CallsIn.into_err()) } Ok(msg) => Ok(msg), Err(err) => Err(trc::JmapEvent::NotRequest .into_err() .details(format!("Invalid WebSocket JMAP request {err}"))), } } else { Err(trc::LimitEvent::SizeRequest.into_err()) } } } impl<'de: 'x, 'x: 'de> Deserialize<'de> for WebSocketMessage<'x> { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserializer.deserialize_map(WebSocketMessageVisitor) } } struct WebSocketMessageVisitor; impl<'de> Visitor<'de> for WebSocketMessageVisitor { type Value = WebSocketMessage<'de>; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("a WebSocketMessage as a map") } fn visit_map(self, mut map: V) -> Result, V::Error> where V: MapAccess<'de>, { let mut message_type = MessageType::None; let mut request = WebSocketRequest { id: None, request: Request::default(), }; let mut push_enable = WebSocketPushEnable::default(); let mut found_request_keys = false; let mut found_push_keys = false; while let Some(key) = map.next_key::<&str>()? { hashify::fnc_map!(key.as_bytes(), b"@type" => { message_type = MessageType::parse(map.next_value()?); }, b"dataTypes" => { push_enable.data_types = map.next_value::>>()?.unwrap_or_default(); found_push_keys = true; }, b"pushState" => { push_enable.push_state = map.next_value()?; found_push_keys = true; }, b"id" => { request.id = map.next_value()?; }, _ => { request.request.deserialize_argument(key, &mut map)?; found_request_keys = true; } ); } match message_type { MessageType::Request if found_request_keys => Ok(WebSocketMessage::Request(request)), MessageType::PushEnable if found_push_keys => { Ok(WebSocketMessage::PushEnable(push_enable)) } MessageType::PushDisable if !found_request_keys && !found_push_keys => { Ok(WebSocketMessage::PushDisable) } _ => Err(de::Error::custom("Invalid WebSocket JMAP request")), } } } impl MessageType { fn parse(s: &str) -> Self { hashify::tiny_map!(s.as_bytes(), b"Request" => MessageType::Request, b"WebSocketPushEnable" => MessageType::PushEnable, b"WebSocketPushDisable" => MessageType::PushDisable, ) .unwrap_or(MessageType::None) } } impl<'x> WebSocketRequestError<'x> { pub fn from_error(error: RequestError<'x>, request_id: Option) -> Self { Self { type_: WebSocketRequestErrorType::RequestError, p_type: error.p_type, limit: error.limit, status: error.status, detail: error.detail, request_id, } } pub fn to_json(&self) -> String { serde_json::to_string(self).unwrap() } } impl<'x> From> for WebSocketRequestError<'x> { fn from(value: RequestError<'x>) -> Self { Self::from_error(value, None) } } impl<'x> WebSocketResponse<'x> { pub fn from_response(response: Response<'x>, request_id: Option) -> Self { Self { _type: WebSocketResponseType::Response, method_responses: response.method_responses, session_state: response.session_state, created_ids: response.created_ids, request_id, } } pub fn to_json(&self) -> String { serde_json::to_string(self).unwrap() } } impl WebSocketPushObject { pub fn to_json(&self) -> String { serde_json::to_string(self).unwrap() } } ================================================ FILE: crates/jmap-proto/src/response/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod serialize; pub mod status; use self::serialize::serialize_hex; use crate::{ error::method::MethodErrorWrapper, method::{ availability::GetAvailabilityResponse, changes::ChangesResponse, copy::{CopyBlobResponse, CopyResponse}, get::GetResponse, import::ImportEmailResponse, lookup::BlobLookupResponse, parse::ParseResponse, query::QueryResponse, query_changes::QueryChangesResponse, search_snippet::GetSearchSnippetResponse, set::SetResponse, upload::BlobUploadResponse, validate::ValidateSieveScriptResponse, }, object::{ AnyId, addressbook::AddressBook, blob::Blob, calendar::Calendar, calendar_event::CalendarEvent, calendar_event_notification::{ CalendarEventNotification, CalendarEventNotificationGetResponse, }, contact::ContactCard, email::Email, email_submission::EmailSubmission, file_node::FileNode, identity::Identity, mailbox::Mailbox, participant_identity::ParticipantIdentity, principal::Principal, push_subscription::PushSubscription, quota::Quota, share_notification::ShareNotification, sieve::Sieve, thread::Thread, vacation_response::VacationResponse, }, request::{Call, method::MethodName}, }; use jmap_tools::{Null, Value}; use std::collections::HashMap; #[derive(Debug, serde::Serialize)] #[serde(untagged)] pub enum ResponseMethod<'x> { Get(GetResponseMethod), Set(SetResponseMethod), Changes(ChangesResponseMethod), Copy(CopyResponseMethod), ImportEmail(ImportEmailResponse), Parse(ParseResponseMethod), QueryChanges(QueryChangesResponse), Query(QueryResponse), SearchSnippet(GetSearchSnippetResponse), ValidateScript(ValidateSieveScriptResponse), LookupBlob(BlobLookupResponse), UploadBlob(BlobUploadResponse), Echo(Value<'x, Null, Null>), Error(MethodErrorWrapper), } #[derive(Debug, serde::Serialize)] #[serde(untagged)] pub enum GetResponseMethod { Email(GetResponse), Mailbox(GetResponse), Thread(GetResponse), Identity(GetResponse), EmailSubmission(GetResponse), PushSubscription(GetResponse), Sieve(GetResponse), VacationResponse(GetResponse), Principal(GetResponse), PrincipalAvailability(GetAvailabilityResponse), Quota(GetResponse), Blob(GetResponse), AddressBook(GetResponse), ContactCard(GetResponse), FileNode(GetResponse), Calendar(GetResponse), CalendarEvent(GetResponse), CalendarEventNotification(CalendarEventNotificationGetResponse), ParticipantIdentity(GetResponse), ShareNotification(GetResponse), } #[derive(Debug, serde::Serialize)] #[serde(untagged)] pub enum SetResponseMethod { Email(SetResponse), Mailbox(SetResponse), Identity(SetResponse), EmailSubmission(SetResponse), PushSubscription(SetResponse), Sieve(SetResponse), VacationResponse(SetResponse), AddressBook(SetResponse), ContactCard(SetResponse), FileNode(SetResponse), ShareNotification(SetResponse), Calendar(SetResponse), CalendarEvent(SetResponse), CalendarEventNotification(SetResponse), ParticipantIdentity(SetResponse), } #[derive(Debug, serde::Serialize)] #[serde(untagged)] pub enum ChangesResponseMethod { Email(ChangesResponse), Mailbox(ChangesResponse), Thread(ChangesResponse), Identity(ChangesResponse), EmailSubmission(ChangesResponse), Quota(ChangesResponse), AddressBook(ChangesResponse), ContactCard(ChangesResponse), FileNode(ChangesResponse), Calendar(ChangesResponse), CalendarEvent(ChangesResponse), CalendarEventNotification(ChangesResponse), ShareNotification(ChangesResponse), } #[derive(Debug, serde::Serialize)] #[serde(untagged)] pub enum CopyResponseMethod { Email(CopyResponse), ContactCard(CopyResponse), CalendarEvent(CopyResponse), Blob(CopyBlobResponse), } #[derive(Debug, serde::Serialize)] #[serde(untagged)] pub enum ParseResponseMethod { Email(ParseResponse), ContactCard(ParseResponse), CalendarEvent(ParseResponse), } #[derive(Debug, serde::Serialize)] pub struct Response<'x> { #[serde(rename = "methodResponses")] pub method_responses: Vec>>, #[serde(rename = "sessionState")] #[serde(serialize_with = "serialize_hex")] pub session_state: u32, #[serde(rename = "createdIds")] #[serde(skip_serializing_if = "HashMap::is_empty")] pub created_ids: HashMap, } impl<'x> Response<'x> { pub fn new(session_state: u32, created_ids: HashMap, capacity: usize) -> Self { Response { session_state, created_ids, method_responses: Vec::with_capacity(capacity), } } pub fn push_response( &mut self, id: String, name: MethodName, method: impl Into>, ) { self.method_responses.push(Call { id, method: method.into(), name, }); } pub fn push_error(&mut self, id: String, err: impl Into) { self.method_responses.push(Call { id, method: ResponseMethod::Error(err.into()), name: MethodName::error(), }); } pub fn push_created_id(&mut self, create_id: String, id: impl Into) { self.created_ids.insert(create_id, id.into()); } } impl From for ResponseMethod<'_> { fn from(error: trc::Error) -> Self { ResponseMethod::Error(error.into()) } } impl<'x, T: Into>> From> for ResponseMethod<'x> { fn from(result: trc::Result) -> Self { match result { Ok(value) => value.into(), Err(error) => error.into(), } } } impl<'x> From> for ResponseMethod<'x> { fn from(value: GetResponse) -> Self { ResponseMethod::Get(GetResponseMethod::Email(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: GetResponse) -> Self { ResponseMethod::Get(GetResponseMethod::Mailbox(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: GetResponse) -> Self { ResponseMethod::Get(GetResponseMethod::Thread(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: GetResponse) -> Self { ResponseMethod::Get(GetResponseMethod::Identity(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: GetResponse) -> Self { ResponseMethod::Get(GetResponseMethod::EmailSubmission(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: GetResponse) -> Self { ResponseMethod::Get(GetResponseMethod::PushSubscription(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: GetResponse) -> Self { ResponseMethod::Get(GetResponseMethod::Sieve(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: GetResponse) -> Self { ResponseMethod::Get(GetResponseMethod::VacationResponse(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: GetResponse) -> Self { ResponseMethod::Get(GetResponseMethod::Principal(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: GetResponse) -> Self { ResponseMethod::Get(GetResponseMethod::Quota(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: GetResponse) -> Self { ResponseMethod::Get(GetResponseMethod::Blob(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: GetResponse) -> Self { ResponseMethod::Get(GetResponseMethod::ContactCard(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: GetResponse) -> Self { ResponseMethod::Get(GetResponseMethod::AddressBook(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: SetResponse) -> Self { ResponseMethod::Set(SetResponseMethod::Email(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: SetResponse) -> Self { ResponseMethod::Set(SetResponseMethod::Mailbox(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: SetResponse) -> Self { ResponseMethod::Set(SetResponseMethod::Identity(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: SetResponse) -> Self { ResponseMethod::Set(SetResponseMethod::EmailSubmission(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: SetResponse) -> Self { ResponseMethod::Set(SetResponseMethod::PushSubscription(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: SetResponse) -> Self { ResponseMethod::Set(SetResponseMethod::Sieve(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: SetResponse) -> Self { ResponseMethod::Set(SetResponseMethod::VacationResponse(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: SetResponse) -> Self { ResponseMethod::Set(SetResponseMethod::AddressBook(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: SetResponse) -> Self { ResponseMethod::Set(SetResponseMethod::ContactCard(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: ChangesResponse) -> Self { ResponseMethod::Changes(ChangesResponseMethod::Email(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: ChangesResponse) -> Self { ResponseMethod::Changes(ChangesResponseMethod::Mailbox(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: ChangesResponse) -> Self { ResponseMethod::Changes(ChangesResponseMethod::Thread(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: ChangesResponse) -> Self { ResponseMethod::Changes(ChangesResponseMethod::Identity(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: ChangesResponse) -> Self { ResponseMethod::Changes(ChangesResponseMethod::EmailSubmission(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: ChangesResponse) -> Self { ResponseMethod::Changes(ChangesResponseMethod::Quota(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: ChangesResponse) -> Self { ResponseMethod::Changes(ChangesResponseMethod::AddressBook(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: CopyResponse) -> Self { ResponseMethod::Copy(CopyResponseMethod::Email(value)) } } impl<'x> From for ResponseMethod<'x> { fn from(value: CopyBlobResponse) -> Self { ResponseMethod::Copy(CopyResponseMethod::Blob(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: CopyResponse) -> Self { ResponseMethod::Copy(CopyResponseMethod::ContactCard(value)) } } impl<'x> From for ResponseMethod<'x> { fn from(value: ImportEmailResponse) -> Self { ResponseMethod::ImportEmail(value) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: ParseResponse) -> Self { ResponseMethod::Parse(ParseResponseMethod::Email(value)) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: ParseResponse) -> Self { ResponseMethod::Parse(ParseResponseMethod::ContactCard(value)) } } impl<'x> From for ResponseMethod<'x> { fn from(value: QueryChangesResponse) -> Self { ResponseMethod::QueryChanges(value) } } impl<'x> From for ResponseMethod<'x> { fn from(value: QueryResponse) -> Self { ResponseMethod::Query(value) } } impl<'x> From for ResponseMethod<'x> { fn from(value: GetSearchSnippetResponse) -> Self { ResponseMethod::SearchSnippet(value) } } impl<'x> From for ResponseMethod<'x> { fn from(value: ValidateSieveScriptResponse) -> Self { ResponseMethod::ValidateScript(value) } } impl<'x> From for ResponseMethod<'x> { fn from(value: BlobLookupResponse) -> Self { ResponseMethod::LookupBlob(value) } } impl<'x> From for ResponseMethod<'x> { fn from(value: BlobUploadResponse) -> Self { ResponseMethod::UploadBlob(value) } } impl<'x> From> for ResponseMethod<'x> { fn from(value: Value<'x, Null, Null>) -> Self { ResponseMethod::Echo(value) } } impl<'x> From for ResponseMethod<'x> { fn from(value: MethodErrorWrapper) -> Self { ResponseMethod::Error(value) } } impl From> for ResponseMethod<'_> { fn from(response: GetResponse) -> Self { ResponseMethod::Get(GetResponseMethod::FileNode(response)) } } impl From> for ResponseMethod<'_> { fn from(response: SetResponse) -> Self { ResponseMethod::Set(SetResponseMethod::FileNode(response)) } } impl From> for ResponseMethod<'_> { fn from(response: ChangesResponse) -> Self { ResponseMethod::Changes(ChangesResponseMethod::FileNode(response)) } } impl From for ResponseMethod<'_> { fn from(response: GetAvailabilityResponse) -> Self { ResponseMethod::Get(GetResponseMethod::PrincipalAvailability(response)) } } impl From> for ResponseMethod<'_> { fn from(response: GetResponse) -> Self { ResponseMethod::Get(GetResponseMethod::Calendar(response)) } } impl From> for ResponseMethod<'_> { fn from(response: SetResponse) -> Self { ResponseMethod::Set(SetResponseMethod::Calendar(response)) } } impl From> for ResponseMethod<'_> { fn from(response: ChangesResponse) -> Self { ResponseMethod::Changes(ChangesResponseMethod::CalendarEvent(response)) } } impl From> for ResponseMethod<'_> { fn from(response: ChangesResponse) -> Self { ResponseMethod::Changes(ChangesResponseMethod::CalendarEventNotification(response)) } } impl From> for ResponseMethod<'_> { fn from(response: SetResponse) -> Self { ResponseMethod::Set(SetResponseMethod::CalendarEvent(response)) } } impl From> for ResponseMethod<'_> { fn from(response: SetResponse) -> Self { ResponseMethod::Set(SetResponseMethod::ParticipantIdentity(response)) } } impl From> for ResponseMethod<'_> { fn from(response: GetResponse) -> Self { ResponseMethod::Get(GetResponseMethod::ParticipantIdentity(response)) } } impl From> for ResponseMethod<'_> { fn from(response: ChangesResponse) -> Self { ResponseMethod::Changes(ChangesResponseMethod::ShareNotification(response)) } } impl From> for ResponseMethod<'_> { fn from(response: SetResponse) -> Self { ResponseMethod::Set(SetResponseMethod::ShareNotification(response)) } } impl From> for ResponseMethod<'_> { fn from(response: GetResponse) -> Self { ResponseMethod::Get(GetResponseMethod::ShareNotification(response)) } } impl From> for ResponseMethod<'_> { fn from(response: GetResponse) -> Self { ResponseMethod::Get(GetResponseMethod::CalendarEvent(response)) } } impl From> for ResponseMethod<'_> { fn from(value: ParseResponse) -> Self { ResponseMethod::Parse(ParseResponseMethod::CalendarEvent(value)) } } impl From> for ResponseMethod<'_> { fn from(value: CopyResponse) -> Self { ResponseMethod::Copy(CopyResponseMethod::CalendarEvent(value)) } } impl From for ResponseMethod<'_> { fn from(value: CalendarEventNotificationGetResponse) -> Self { ResponseMethod::Get(GetResponseMethod::CalendarEventNotification(value)) } } impl From> for ResponseMethod<'_> { fn from(value: SetResponse) -> Self { ResponseMethod::Set(SetResponseMethod::CalendarEventNotification(value)) } } ================================================ FILE: crates/jmap-proto/src/response/serialize.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::ResponseMethod; use crate::request::Call; use serde::{Serialize, ser::SerializeSeq}; impl Serialize for Call> { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { let mut seq = serializer.serialize_seq(3.into())?; seq.serialize_element(&self.name.to_string())?; seq.serialize_element(&self.method)?; seq.serialize_element(&self.id)?; seq.end() } } pub fn serialize_hex(value: &u32, serializer: S) -> Result where S: serde::Serializer, { format!("{:x}", value).serialize(serializer) } ================================================ FILE: crates/jmap-proto/src/response/status.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::types::state::State; use types::{id::Id, type_state::DataType}; use utils::map::vec_map::VecMap; #[derive(serde::Serialize, serde::Deserialize, Debug)] #[serde(tag = "@type")] pub enum PushObject { StateChange { changed: VecMap>, }, EmailPush { #[serde(rename = "accountId")] account_id: Id, email: EmailPushObject, }, CalendarAlert { #[serde(rename = "accountId")] account_id: Id, #[serde(rename = "calendarEventId")] calendar_event_id: Id, uid: String, #[serde(rename = "recurrenceId")] recurrence_id: Option, #[serde(rename = "alertId")] alert_id: String, }, Group { entries: Vec, }, } #[derive(serde::Serialize, serde::Deserialize, Debug)] pub struct EmailPushObject { pub subject: String, } ================================================ FILE: crates/jmap-proto/src/types/date.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{fmt::Display, str::FromStr}; #[derive(Debug, Default, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct UTCDate { pub year: u16, pub month: u8, pub day: u8, pub hour: u8, pub minute: u8, pub second: u8, pub tz_before_gmt: bool, pub tz_hour: u8, pub tz_minute: u8, } impl FromStr for UTCDate { type Err = (); fn from_str(s: &str) -> Result { // 2004 - 06 - 28 T 23 : 43 : 45 . 000 Z // 1969 - 02 - 13 T 23 : 32 : 00 - 03 : 30 // 0 1 2 3 4 5 6 7 let mut pos = 0; let mut parts = [0u32; 8]; let mut parts_sizes = [ 4u32, // Year (0) 2u32, // Month (1) 2u32, // Day (2) 2u32, // Hour (3) 2u32, // Minute (4) 2u32, // Second (5) 2u32, // TZ Hour (6) 2u32, // TZ Minute (7) ]; let mut skip_digits = false; let mut is_plus = true; for ch in s.as_bytes() { match ch { b'0'..=b'9' => { if !skip_digits { if parts_sizes[pos] > 0 { parts_sizes[pos] -= 1; parts[pos] += (ch - b'0') as u32 * u32::pow(10, parts_sizes[pos]); } else { break; } } } b'-' => { if pos <= 1 { pos += 1; } else if pos == 5 { pos += 1; is_plus = false; skip_digits = false; } else { break; } } b'T' => { if pos == 2 { pos += 1; } else { break; } } b':' => { if [3, 4, 6].contains(&pos) { pos += 1; } else { break; } } b'+' => { if pos == 5 { pos += 1; skip_digits = false; } else { break; } } b'.' => { if pos == 5 { skip_digits = true; } else { break; } } b'Z' | b'z' => (), _ => { break; } } } if pos >= 5 { Ok(UTCDate { year: parts[0] as u16, month: parts[1] as u8, day: parts[2] as u8, hour: parts[3] as u8, minute: parts[4] as u8, second: parts[5] as u8, tz_hour: parts[6] as u8, tz_minute: parts[7] as u8, tz_before_gmt: !is_plus, }) } else { Err(()) } } } impl UTCDate { pub fn from_timestamp(timestamp: i64) -> Self { // Ported from http://howardhinnant.github.io/date_algorithms.html#civil_from_days let (z, seconds) = ((timestamp / 86400) + 719468, timestamp % 86400); let era: i64 = (if z >= 0 { z } else { z - 146096 }) / 146097; let doe: u64 = (z - era * 146097) as u64; // [0, 146096] let yoe: u64 = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; // [0, 399] let y: i64 = (yoe as i64) + era * 400; let doy: u64 = doe - (365 * yoe + yoe / 4 - yoe / 100); // [0, 365] let mp = (5 * doy + 2) / 153; // [0, 11] let d: u64 = doy - (153 * mp + 2) / 5 + 1; // [1, 31] let m: u64 = if mp < 10 { mp + 3 } else { mp - 9 }; // [1, 12] let (h, mn, s) = (seconds / 3600, (seconds / 60) % 60, seconds % 60); UTCDate { year: (y + i64::from(m <= 2)) as u16, month: m as u8, day: d as u8, hour: h as u8, minute: mn as u8, second: s as u8, tz_before_gmt: false, tz_hour: 0, tz_minute: 0, } } pub fn is_valid(&self) -> bool { (0..=23).contains(&self.tz_hour) && (1970..=3000).contains(&self.year) && (0..=59).contains(&self.tz_minute) && (1..=12).contains(&self.month) && (1..=31).contains(&self.day) && (0..=23).contains(&self.hour) && (0..=59).contains(&self.minute) && (0..=59).contains(&self.second) } pub fn timestamp(&self) -> i64 { // Ported from https://github.com/protocolbuffers/upb/blob/22182e6e/upb/json_decode.c#L982-L992 let month = self.month as u32; let year_base = 4800; /* Before min year, multiple of 400. */ let m_adj = month.wrapping_sub(3); /* March-based month. */ let carry = i64::from(m_adj > month); let adjust = if carry > 0 { 12 } else { 0 }; let y_adj = self.year as i64 + year_base - carry; let month_days = ((m_adj.wrapping_add(adjust)) * 62719 + 769) / 2048; let leap_days = y_adj / 4 - y_adj / 100 + y_adj / 400; (y_adj * 365 + leap_days + month_days as i64 + (self.day as i64 - 1) - 2472632) * 86400 + self.hour as i64 * 3600 + self.minute as i64 * 60 + self.second as i64 + ((self.tz_hour as i64 * 3600 + self.tz_minute as i64 * 60) * if self.tz_before_gmt { 1 } else { -1 }) } } impl Display for UTCDate { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { if self.tz_hour != 0 || self.tz_minute != 0 { write!( f, "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}{}{:02}:{:02}", self.year, self.month, self.day, self.hour, self.minute, self.second, if self.tz_before_gmt && (self.tz_hour > 0 || self.tz_minute > 0) { "-" } else { "+" }, self.tz_hour, self.tz_minute, ) } else { write!( f, "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z", self.year, self.month, self.day, self.hour, self.minute, self.second, ) } } } impl serde::Serialize for UTCDate { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { serializer.serialize_str(self.to_string().as_str()) } } impl<'de> serde::Deserialize<'de> for UTCDate { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { UTCDate::from_str(<&str>::deserialize(deserializer)?) .map_err(|_| serde::de::Error::custom("invalid JMAP UTCDate")) } } impl From for u64 { fn from(value: UTCDate) -> Self { value.timestamp() as u64 } } impl From for UTCDate { fn from(value: u64) -> Self { UTCDate::from_timestamp(value as i64) } } #[cfg(test)] mod tests { use crate::types::date::UTCDate; use std::str::FromStr; #[test] fn parse_jmap_date() { for (input, expected_result) in [ ("1997-11-21T09:55:06-06:00", "1997-11-21T09:55:06-06:00"), ("1997-11-21T09:55:06+00:00", "1997-11-21T09:55:06Z"), ("2021-01-01T09:55:06+02:00", "2021-01-01T09:55:06+02:00"), ("2004-06-28T23:43:45.000Z", "2004-06-28T23:43:45Z"), ("1997-11-21T09:55:06.123+00:00", "1997-11-21T09:55:06Z"), ( "2021-01-01T09:55:06.4567+02:00", "2021-01-01T09:55:06+02:00", ), ] { let date = UTCDate::from_str(input).unwrap(); assert_eq!(date.to_string(), expected_result); let timestamp = date.timestamp(); assert_eq!(UTCDate::from_timestamp(timestamp).timestamp(), timestamp); } } } ================================================ FILE: crates/jmap-proto/src/types/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod date; pub mod state; ================================================ FILE: crates/jmap-proto/src/types/state.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use types::ChangeId; use utils::codec::{ base32_custom::{Base32Reader, Base32Writer}, leb128::{Leb128Iterator, Leb128Writer}, }; #[derive(Debug, Clone, PartialEq, Eq)] pub struct JMAPIntermediateState { pub from_id: ChangeId, pub to_id: ChangeId, pub items_sent: usize, } #[derive(Debug, Clone, PartialEq, Eq, Default)] pub enum State { #[default] Initial, Exact(ChangeId), Intermediate(JMAPIntermediateState), } impl From for State { fn from(change_id: ChangeId) -> Self { State::Exact(change_id) } } impl From> for State { fn from(change_id: Option) -> Self { match change_id { Some(change_id) => State::Exact(change_id), None => State::Initial, } } } impl State { pub fn parse(value: &str) -> Option { let mut it = value.as_bytes().iter(); match it.next()? { b'n' => Some(State::Initial), b's' => { let mut reader = Base32Reader::from_iter(it); reader.next_leb128::().map(State::Exact) } b'r' => { let mut it = Base32Reader::from_iter(it); if let (Some(from_id), Some(to_id), Some(items_sent)) = ( it.next_leb128::(), it.next_leb128::(), it.next_leb128::(), ) { if items_sent > 0 { Some(State::Intermediate(JMAPIntermediateState { from_id, to_id: from_id.saturating_add(to_id), items_sent, })) } else { None } } else { None } } _ => None, } } pub fn new_initial() -> Self { State::Initial } pub fn new_exact(id: ChangeId) -> Self { State::Exact(id) } pub fn new_intermediate(from_id: ChangeId, to_id: ChangeId, items_sent: usize) -> Self { State::Intermediate(JMAPIntermediateState { from_id, to_id, items_sent, }) } pub fn get_change_id(&self) -> ChangeId { match self { State::Exact(id) => *id, State::Intermediate(intermediate) => intermediate.to_id, State::Initial => ChangeId::MAX, } } } impl serde::Serialize for State { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { serializer.serialize_str(self.to_string().as_str()) } } impl<'de> serde::Deserialize<'de> for State { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { State::parse(<&str>::deserialize(deserializer)?) .ok_or_else(|| serde::de::Error::custom("invalid JMAP State")) } } impl std::fmt::Display for State { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let mut writer = Base32Writer::with_capacity(10); match self { State::Initial => { writer.push_char('n'); } State::Exact(id) => { writer.push_char('s'); writer.write_leb128(*id).unwrap(); } State::Intermediate(intermediate) => { writer.push_char('r'); writer.write_leb128(intermediate.from_id).unwrap(); writer .write_leb128(intermediate.to_id - intermediate.from_id) .unwrap(); writer.write_leb128(intermediate.items_sent).unwrap(); } } f.write_str(&writer.finalize()) } } #[cfg(test)] mod tests { use super::State; use types::ChangeId; #[test] fn test_state_id() { for id in [ State::new_initial(), State::new_exact(0), State::new_exact(12345678), State::new_exact(ChangeId::MAX), State::new_intermediate(0, 0, 1), State::new_intermediate(1024, 2048, 100), State::new_intermediate(12345678, 87654321, 1), State::new_intermediate(0, 0, 12345678), State::new_intermediate(0, 87654321, 12345678), State::new_intermediate(12345678, 87654321, 1), State::new_intermediate(12345678, 87654321, 12345678), State::new_intermediate(ChangeId::MAX, ChangeId::MAX, ChangeId::MAX as usize), ] { assert_eq!(State::parse(&id.to_string()).unwrap(), id); } } } ================================================ FILE: crates/main/Cargo.toml ================================================ [package] name = "stalwart" description = "Stalwart Mail and Collaboration Server" authors = [ "Stalwart Labs LLC "] repository = "https://github.com/stalwartlabs/stalwart" homepage = "https://stalw.art" keywords = ["imap", "jmap", "smtp", "email", "mail", "webdav", "server"] categories = ["email"] license = "AGPL-3.0-only OR LicenseRef-SEL" version = "0.15.5" edition = "2024" [[bin]] name = "stalwart" path = "src/main.rs" [dependencies] store = { path = "../store" } jmap = { path = "../jmap" } types = { path = "../types" } smtp = { path = "../smtp" } imap = { path = "../imap" } pop3 = { path = "../pop3" } spam-filter = { path = "../spam-filter" } managesieve = { path = "../managesieve" } common = { path = "../common" } email = { path = "../email" } directory = { path = "../directory" } http = { path = "../http" } dav = { path = "../dav" } groupware = { path = "../groupware" } services = { path = "../services" } trc = { path = "../trc" } utils = { path = "../utils" } migration = { path = "../migration" } tokio = { version = "1.47", features = ["full"] } [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.0" [features] #default = ["sqlite", "postgres", "mysql", "rocks", "s3", "redis", "azure", "nats", "enterprise"] default = ["rocks", "enterprise"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation", "common/foundation"] postgres = ["store/postgres"] mysql = ["store/mysql"] rocks = ["store/rocks"] s3 = ["store/s3"] redis = ["store/redis"] nats = ["store/nats"] azure = ["store/azure"] zenoh = ["store/zenoh"] kafka = ["store/kafka"] enterprise = [ "jmap/enterprise", "smtp/enterprise", "common/enterprise", "store/enterprise", "managesieve/enterprise", "directory/enterprise", "email/enterprise", "spam-filter/enterprise", "http/enterprise", "dav/enterprise", "groupware/enterprise", "trc/enterprise", "services/enterprise", "migration/enterprise" ] ================================================ FILE: crates/main/src/main.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ #![warn(clippy::large_futures)] #![warn(clippy::cast_possible_truncation)] #![warn(clippy::cast_possible_wrap)] #![warn(clippy::cast_sign_loss)] use common::{config::server::ServerProtocol, core::BuildServer, manager::boot::BootManager}; use http::HttpSessionManager; use imap::core::ImapSessionManager; use managesieve::core::ManageSieveSessionManager; use pop3::Pop3SessionManager; use services::{StartServices, broadcast::subscriber::spawn_broadcast_subscriber}; use smtp::{StartQueueManager, core::SmtpSessionManager}; use std::time::Duration; use trc::Collector; use utils::wait_for_shutdown; #[cfg(not(target_env = "msvc"))] use jemallocator::Jemalloc; #[cfg(not(target_env = "msvc"))] #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; #[tokio::main] async fn main() -> std::io::Result<()> { // Load config and apply macros let mut init = Box::pin(BootManager::init()).await; // Migrate database if let Err(err) = migration::try_migrate(&init.inner.build_server()).await { trc::event!( Server(trc::ServerEvent::StartupError), Details = "Failed to migrate database, aborting startup.", Reason = err, ); return Ok(()); } // Init services init.start_services().await; init.start_queue_manager(); // Log configuration errors init.config.log_errors(); init.config.log_warnings(); // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL // Log licensing information #[cfg(feature = "enterprise")] init.inner.build_server().log_license_details(); // SPDX-SnippetEnd // Spawn servers let (shutdown_tx, shutdown_rx) = init.servers.spawn(|server, acceptor, shutdown_rx| { match &server.protocol { ServerProtocol::Smtp | ServerProtocol::Lmtp => server.spawn( SmtpSessionManager::new(init.inner.clone()), init.inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Http => server.spawn( HttpSessionManager::new(init.inner.clone()), init.inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Imap => server.spawn( ImapSessionManager::new(init.inner.clone()), init.inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Pop3 => server.spawn( Pop3SessionManager::new(init.inner.clone()), init.inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::ManageSieve => server.spawn( ManageSieveSessionManager::new(init.inner.clone()), init.inner.clone(), acceptor, shutdown_rx, ), }; }); // Start broadcast subscriber spawn_broadcast_subscriber(init.inner, shutdown_rx); // Wait for shutdown signal wait_for_shutdown().await; // Shutdown collector Collector::shutdown(); // Stop services let _ = shutdown_tx.send(true); // Wait for services to finish tokio::time::sleep(Duration::from_secs(1)).await; Ok(()) } ================================================ FILE: crates/managesieve/Cargo.toml ================================================ [package] name = "managesieve" version = "0.15.5" edition = "2024" [dependencies] imap_proto = { path = "../imap-proto" } imap = { path = "../imap" } types = { path = "../types" } jmap_proto = { path = "../jmap-proto" } directory = { path = "../directory" } common = { path = "../common" } store = { path = "../store" } utils = { path = "../utils" } email = { path = "../email" } trc = { path = "../trc" } mail-parser = { version = "0.11", features = ["full_encoding"] } mail-send = { version = "0.5", default-features = false, features = ["cram-md5", "ring", "tls12"] } sieve-rs = { version = "0.7", features = ["rkyv"] } rustls = { version = "0.23.5", default-features = false, features = ["std", "ring", "tls12"] } rustls-pemfile = "2.0" tokio = { version = "1.47", features = ["full"] } tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } parking_lot = "0.12" ahash = { version = "0.8" } md5 = "0.8.0" compact_str = "0.9.0" rkyv = { version = "0.8.10", features = ["little_endian"] } [features] test_mode = [] enterprise = [] ================================================ FILE: crates/managesieve/src/core/client.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{Command, ResponseCode, SerializeResponse, Session, State}; use common::{ KV_RATE_LIMIT_IMAP, listener::{SessionResult, SessionStream}, }; use imap_proto::receiver::{self, Request}; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use trc::{AddContext, SecurityEvent}; use types::{collection::Collection, field::SieveField}; impl Session { pub async fn ingest(&mut self, bytes: &[u8]) -> SessionResult { let mut bytes = bytes.iter(); let mut requests = Vec::with_capacity(2); let mut needs_literal = None; loop { match self.receiver.parse(&mut bytes) { Ok(request) => match self.validate_request(request).await { Ok(request) => { requests.push(request); } Err(err) => { let mut disconnect = err.must_disconnect(); if let Err(err) = self.write_error(err).await { trc::error!(err.span_id(self.session_id)); disconnect = true; } if disconnect { return SessionResult::Close; } } }, Err(receiver::Error::NeedsMoreData) => { break; } Err(receiver::Error::NeedsLiteral { size }) => { needs_literal = size.into(); break; } Err(receiver::Error::Error { response }) => { // Check for port scanners if matches!( (&self.state, response.key(trc::Key::Code)), ( State::NotAuthenticated { .. }, Some(trc::Value::String(v)) ) if v == "PARSE" ) { match self.server.is_scanner_fail2banned(self.remote_addr).await { Ok(true) => { trc::event!( Security(SecurityEvent::ScanBan), SpanId = self.session_id, RemoteIp = self.remote_addr, Reason = "Invalid ManageSieve command", ); return SessionResult::Close; } Ok(false) => {} Err(err) => { trc::error!( err.span_id(self.session_id) .details("Failed to check for fail2ban") ); } } } if let Err(err) = self.write_error(response).await { trc::error!(err.span_id(self.session_id)); return SessionResult::Close; } break; } } } for request in requests { let command = request.command; match match command { Command::ListScripts => self.handle_listscripts().await, Command::PutScript => self.handle_putscript(request).await, Command::SetActive => self.handle_setactive(request).await, Command::GetScript => self.handle_getscript(request).await, Command::DeleteScript => self.handle_deletescript(request).await, Command::RenameScript => self.handle_renamescript(request).await, Command::CheckScript => self.handle_checkscript(request).await, Command::HaveSpace => self.handle_havespace(request).await, Command::Capability => self.handle_capability("").await, Command::Authenticate => self.handle_authenticate(request).await, Command::StartTls => self.handle_start_tls().await, Command::Logout => self.handle_logout().await, Command::Noop => self.handle_noop(request).await, Command::Unauthenticate => self.handle_unauthenticate().await, } { Ok(response) => { if let Err(err) = self.write(&response).await { trc::error!(err.span_id(self.session_id)); return SessionResult::Close; } match command { Command::Logout => return SessionResult::Close, Command::StartTls => return SessionResult::UpgradeTls, _ => (), } } Err(err) => { let mut disconnect = err.must_disconnect(); if let Err(err) = self.write_error(err).await { trc::error!(err.span_id(self.session_id)); disconnect = true; } if disconnect { return SessionResult::Close; } } } } if let Some(needs_literal) = needs_literal && let Err(err) = self .write(format!("OK Ready for {} bytes.\r\n", needs_literal).as_bytes()) .await { trc::error!(err.span_id(self.session_id)); return SessionResult::Close; } SessionResult::Continue } async fn validate_request(&self, command: Request) -> trc::Result> { match &command.command { Command::Capability | Command::Logout | Command::Noop => Ok(command), Command::Authenticate => { if let State::NotAuthenticated { .. } = &self.state { if self.stream.is_tls() || self.server.core.imap.allow_plain_auth { Ok(command) } else { Err(trc::ManageSieveEvent::Error .into_err() .code(ResponseCode::EncryptNeeded) .details("Cannot authenticate over plain-text.")) } } else { Err(trc::ManageSieveEvent::Error .into_err() .details("Already authenticated.")) } } Command::StartTls => { if !self.stream.is_tls() { Ok(command) } else { Err(trc::ManageSieveEvent::Error .into_err() .details("Already in TLS mode.")) } } Command::HaveSpace | Command::PutScript | Command::ListScripts | Command::SetActive | Command::GetScript | Command::DeleteScript | Command::RenameScript | Command::CheckScript | Command::Unauthenticate => { if let State::Authenticated { access_token, .. } = &self.state { if let Some(rate) = &self.server.core.imap.rate_requests { if self .server .core .storage .lookup .is_rate_allowed( KV_RATE_LIMIT_IMAP, &access_token.primary_id().to_be_bytes(), rate, true, ) .await .caused_by(trc::location!())? .is_none() { Ok(command) } else { Err(trc::LimitEvent::TooManyRequests .into_err() .code(ResponseCode::TryLater)) } } else { Ok(command) } } else { Err(trc::ManageSieveEvent::Error .into_err() .details("Not authenticated.")) } } } } } impl Session { #[inline(always)] pub async fn write(&mut self, bytes: &[u8]) -> trc::Result<()> { trc::event!( ManageSieve(trc::ManageSieveEvent::RawOutput), SpanId = self.session_id, Size = bytes.len(), Contents = trc::Value::from_maybe_string(bytes), ); self.stream.write_all(bytes).await.map_err(|err| { trc::NetworkEvent::WriteError .into_err() .reason(err) .caused_by(trc::location!()) })?; self.stream.flush().await.map_err(|err| { trc::NetworkEvent::FlushError .into_err() .reason(err) .caused_by(trc::location!()) })?; Ok(()) } pub async fn write_error(&mut self, error: trc::Error) -> trc::Result<()> { let bytes = error.serialize(); trc::error!(error.span_id(self.session_id)); self.write(&bytes).await } #[inline(always)] pub async fn read(&mut self, bytes: &mut [u8]) -> trc::Result { let len = self.stream.read(bytes).await.map_err(|err| { trc::NetworkEvent::ReadError .into_err() .reason(err) .caused_by(trc::location!()) })?; trc::event!( ManageSieve(trc::ManageSieveEvent::RawInput), SpanId = self.session_id, Size = len, Contents = trc::Value::from_maybe_string(bytes.get(0..len).unwrap_or_default()), ); Ok(len) } } impl Session { pub async fn get_script_id(&self, account_id: u32, name: &str) -> trc::Result { self.server .document_ids_matching( account_id, Collection::SieveScript, SieveField::Name, name.to_lowercase().as_bytes(), ) .await .caused_by(trc::location!()) .and_then(|results| { results.min().ok_or_else(|| { trc::ManageSieveEvent::Error .into_err() .code(ResponseCode::NonExistent) .reason("There is no script by that name") }) }) } } ================================================ FILE: crates/managesieve/src/core/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod client; pub mod session; use std::{borrow::Cow, net::IpAddr, sync::Arc}; use common::{ Inner, Server, auth::AccessToken, listener::{ServerInstance, limiter::InFlight}, }; use compact_str::CompactString; use imap_proto::receiver::{CommandParser, Receiver}; use tokio::io::{AsyncRead, AsyncWrite}; pub struct Session { pub server: Server, pub instance: Arc, pub receiver: Receiver, pub state: State, pub remote_addr: IpAddr, pub stream: T, pub session_id: u64, pub in_flight: InFlight, } pub enum State { NotAuthenticated { auth_failures: u32, }, Authenticated { access_token: Arc, in_flight: Option, }, } impl State { pub fn access_token(&self) -> &AccessToken { match self { State::Authenticated { access_token, .. } => access_token, State::NotAuthenticated { .. } => unreachable!("Not authenticated"), } } } #[derive(Clone)] pub struct ManageSieveSessionManager { pub inner: Arc, } impl ManageSieveSessionManager { pub fn new(inner: Arc) -> Self { Self { inner } } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum Command { Authenticate, StartTls, Logout, Capability, HaveSpace, PutScript, ListScripts, SetActive, GetScript, DeleteScript, RenameScript, CheckScript, #[default] Noop, Unauthenticate, } impl CommandParser for Command { fn parse(value: &[u8], _is_uid: bool) -> Option { match value { b"AUTHENTICATE" => Some(Command::Authenticate), b"STARTTLS" => Some(Command::StartTls), b"LOGOUT" => Some(Command::Logout), b"CAPABILITY" => Some(Command::Capability), b"HAVESPACE" => Some(Command::HaveSpace), b"PUTSCRIPT" => Some(Command::PutScript), b"LISTSCRIPTS" => Some(Command::ListScripts), b"SETACTIVE" => Some(Command::SetActive), b"GETSCRIPT" => Some(Command::GetScript), b"DELETESCRIPT" => Some(Command::DeleteScript), b"RENAMESCRIPT" => Some(Command::RenameScript), b"CHECKSCRIPT" => Some(Command::CheckScript), b"NOOP" => Some(Command::Noop), b"UNAUTHENTICATE" => Some(Command::Unauthenticate), _ => None, } } fn tokenize_brackets(&self) -> bool { false } } #[derive(Debug, Clone, PartialEq, Eq)] pub struct StatusResponse { pub code: Option, pub message: Cow<'static, str>, pub rtype: ResponseType, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum ResponseType { Ok, No, Bye, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum ResponseCode { AuthTooWeak, EncryptNeeded, Quota, QuotaMaxScripts, QuotaMaxSize, Referral, Sasl, TransitionNeeded, TryLater, Active, NonExistent, AlreadyExists, Tag(String), Warnings, } impl ResponseCode { pub fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(match self { ResponseCode::AuthTooWeak => b"AUTH-TOO-WEAK", ResponseCode::EncryptNeeded => b"ENCRYPT-NEEDED", ResponseCode::Quota => b"QUOTA", ResponseCode::QuotaMaxScripts => b"QUOTA/MAXSCRIPTS", ResponseCode::QuotaMaxSize => b"QUOTA/MAXSIZE", ResponseCode::Referral => b"REFERRAL", ResponseCode::Sasl => b"SASL", ResponseCode::TransitionNeeded => b"TRANSITION-NEEDED", ResponseCode::TryLater => b"TRYLATER", ResponseCode::Active => b"ACTIVE", ResponseCode::NonExistent => b"NONEXISTENT", ResponseCode::AlreadyExists => b"ALREADYEXISTS", ResponseCode::Tag(tag) => { buf.extend_from_slice(b"TAG {"); buf.extend_from_slice(tag.len().to_string().as_bytes()); buf.extend_from_slice(b"}\r\n"); buf.extend_from_slice(tag.as_bytes()); return; } ResponseCode::Warnings => b"WARNINGS", }); } pub fn as_str(&self) -> &'static str { match self { ResponseCode::AuthTooWeak => "AUTH-TOO-WEAK", ResponseCode::EncryptNeeded => "ENCRYPT-NEEDED", ResponseCode::Quota => "QUOTA", ResponseCode::QuotaMaxScripts => "QUOTA/MAXSCRIPTS", ResponseCode::QuotaMaxSize => "QUOTA/MAXSIZE", ResponseCode::Referral => "REFERRAL", ResponseCode::Sasl => "SASL", ResponseCode::TransitionNeeded => "TRANSITION-NEEDED", ResponseCode::TryLater => "TRYLATER", ResponseCode::Active => "ACTIVE", ResponseCode::NonExistent => "NONEXISTENT", ResponseCode::AlreadyExists => "ALREADYEXISTS", ResponseCode::Tag(_) => "TAG", ResponseCode::Warnings => "WARNINGS", } } } impl ResponseType { pub fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(self.as_str().as_bytes()); } pub fn as_str(&self) -> &'static str { match self { ResponseType::Ok => "OK", ResponseType::No => "NO", ResponseType::Bye => "BYE", } } } impl StatusResponse { pub fn serialize(self, mut buf: Vec) -> Vec { self.rtype.serialize(&mut buf); if let Some(code) = &self.code { buf.extend_from_slice(b" ("); code.serialize(&mut buf); buf.push(b')'); } if !self.message.is_empty() { buf.extend_from_slice(b" \""); for ch in self.message.as_bytes() { if [b'\"', b'\\'].contains(ch) { buf.push(b'\\'); } buf.push(*ch); } buf.push(b'\"'); } buf.extend_from_slice(b"\r\n"); buf } pub fn into_bytes(self) -> Vec { self.serialize(Vec::with_capacity(16)) } pub fn with_code(mut self, code: ResponseCode) -> Self { self.code = Some(code); self } pub fn no(message: impl Into>) -> Self { StatusResponse { code: None, message: message.into(), rtype: ResponseType::No, } } pub fn ok(message: impl Into>) -> Self { StatusResponse { code: None, message: message.into(), rtype: ResponseType::Ok, } } pub fn bye(message: impl Into>) -> Self { StatusResponse { code: None, message: message.into(), rtype: ResponseType::Bye, } } pub fn database_failure() -> Self { StatusResponse { code: Some(ResponseCode::TryLater), message: Cow::Borrowed("Database failure"), rtype: ResponseType::No, } } } pub trait SerializeResponse { fn serialize(&self) -> Vec; } impl SerializeResponse for trc::Error { fn serialize(&self) -> Vec { let mut buf = Vec::with_capacity(64); buf.extend_from_slice(self.value_as_str(trc::Key::Type).unwrap_or("NO").as_bytes()); if let Some(code) = self .value_as_str(trc::Key::Code) .or_else(|| match self.as_ref() { trc::EventType::Store(trc::StoreEvent::NotFound) => { Some(ResponseCode::NonExistent.as_str()) } trc::EventType::Store(_) => Some(ResponseCode::TryLater.as_str()), trc::EventType::Limit(trc::LimitEvent::Quota) => Some(ResponseCode::Quota.as_str()), trc::EventType::Limit(_) => Some(ResponseCode::TryLater.as_str()), _ => None, }) { buf.extend_from_slice(b" ("); buf.extend_from_slice(code.as_bytes()); buf.push(b')'); } let message = self .value_as_str(trc::Key::Details) .unwrap_or_else(|| self.as_ref().message()); buf.extend_from_slice(b" \""); for ch in message.as_bytes() { if [b'\"', b'\\'].contains(ch) { buf.push(b'\\'); } buf.push(*ch); } buf.push(b'\"'); buf.extend_from_slice(b"\r\n"); buf } } impl From for trc::Value { fn from(value: ResponseCode) -> Self { trc::Value::String(CompactString::const_new(value.as_str())) } } impl From for trc::Value { fn from(value: ResponseType) -> Self { trc::Value::String(CompactString::const_new(value.as_str())) } } ================================================ FILE: crates/managesieve/src/core/session.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{ core::BuildServer, listener::{SessionData, SessionManager, SessionResult, SessionStream}, }; use imap_proto::receiver::{self, Receiver}; use tokio_rustls::server::TlsStream; use crate::SERVER_GREETING; use super::{ManageSieveSessionManager, Session, State}; impl SessionManager for ManageSieveSessionManager { #[allow(clippy::manual_async_fn)] fn handle( self, session: SessionData, ) -> impl std::future::Future + Send { async move { // Create session let server = self.inner.build_server(); let mut session = Session { receiver: Receiver::with_max_request_size(server.core.imap.max_request_size) .with_start_state(receiver::State::Command { is_uid: false }), server, instance: session.instance, state: State::NotAuthenticated { auth_failures: 0 }, session_id: session.session_id, stream: session.stream, in_flight: session.in_flight, remote_addr: session.remote_ip, }; if session .write(&session.handle_capability(SERVER_GREETING).await.unwrap()) .await .is_ok() && session.handle_conn().await && session.instance.acceptor.is_tls() && let Ok(mut session) = session.into_tls().await { let _ = session .write(&session.handle_capability(SERVER_GREETING).await.unwrap()) .await; session.handle_conn().await; } } } #[allow(clippy::manual_async_fn)] fn shutdown(&self) -> impl std::future::Future + Send { async {} } } impl Session { pub async fn handle_conn(&mut self) -> bool { let mut buf = vec![0; 8192]; let mut shutdown_rx = self.instance.shutdown_rx.clone(); loop { tokio::select! { result = tokio::time::timeout( if !matches!(self.state, State::NotAuthenticated {..}) { self.server.core.imap.timeout_auth } else { self.server.core.imap.timeout_unauth }, self.read(&mut buf)) => { match result { Ok(Ok(bytes_read)) => { if bytes_read > 0 { match self.ingest(&buf[..bytes_read]).await { SessionResult::Continue => (), SessionResult::UpgradeTls => { return true; } SessionResult::Close => { break; } } } else { trc::event!( Network(trc::NetworkEvent::Closed), SpanId = self.session_id, CausedBy = trc::location!() ); break; } } Ok(Err(err)) => { trc::event!( Network(trc::NetworkEvent::ReadError), SpanId = self.session_id, Reason = err, CausedBy = trc::location!() ); break; } Err(_) => { trc::event!( Network(trc::NetworkEvent::Timeout), SpanId = self.session_id, CausedBy = trc::location!() ); self .write(b"BYE \"Connection timed out.\"\r\n") .await .ok(); break; } } }, _ = shutdown_rx.changed() => { trc::event!( Network(trc::NetworkEvent::Closed), SpanId = self.session_id, Reason = "Server shutting down", CausedBy = trc::location!() ); self.write(b"BYE \"Server shutting down.\"\r\n").await.ok(); break; } }; } false } pub async fn into_tls(self) -> Result>, ()> { Ok(Session { stream: self .instance .tls_accept(self.stream, self.session_id) .await?, state: self.state, instance: self.instance, in_flight: self.in_flight, session_id: self.session_id, server: self.server, receiver: self.receiver, remote_addr: self.remote_addr, }) } } ================================================ FILE: crates/managesieve/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod core; pub mod op; static SERVER_GREETING: &str = "Stalwart ManageSieve at your service."; #[cfg(test)] mod tests { use imap_proto::receiver::{Error, Receiver, Request, State, Token}; use crate::core::Command; #[test] fn receiver_parse_managesieve() { let mut receiver = Receiver::new().with_start_state(State::Command { is_uid: false }); for (frames, expected_requests) in [ ( vec!["Authenticate \"DIGEST-MD5\"\r\n"], vec![Request { tag: "".into(), command: Command::Authenticate, tokens: vec![Token::Argument(b"DIGEST-MD5".to_vec())], }], ), ( vec![ " AUTHENTICATE \"GSSAPI\" {56+}\r\n", "cnNwYXV0aD1lYTQwZjYwMzM1YzQyN2I1NTI3Yjg0ZGJhYmNkZmZmZA==\r\n", ], vec![Request { tag: "".into(), command: Command::Authenticate, tokens: vec![ Token::Argument(b"GSSAPI".to_vec()), Token::Argument( b"cnNwYXV0aD1lYTQwZjYwMzM1YzQyN2I1NTI3Yjg0ZGJhYmNkZmZmZA==".to_vec(), ), ], }], ), ( vec!["Authenticate \"PLAIN\" \"QJIrweAPyo6Q1T9xu\"\r\n"], vec![Request { tag: "".into(), command: Command::Authenticate, tokens: vec![ Token::Argument(b"PLAIN".to_vec()), Token::Argument(b"QJIrweAPyo6Q1T9xu".to_vec()), ], }], ), ( vec!["StartTls\r\n"], vec![Request { tag: "".into(), command: Command::StartTls, tokens: vec![], }], ), ( vec!["HAVESPACE \"myscript\" 999999\r\n"], vec![Request { tag: "".into(), command: Command::HaveSpace, tokens: vec![ Token::Argument(b"myscript".to_vec()), Token::Argument(b"999999".to_vec()), ], }], ), ( vec![ "Putscript \"foo\" {31+}\r\n", "#comment\r\n", "InvalidSieveCommand\r\n\r\n", ], vec![Request { tag: "".into(), command: Command::PutScript, tokens: vec![ Token::Argument(b"foo".to_vec()), Token::Argument(b"#comment\r\nInvalidSieveCommand\r\n".to_vec()), ], }], ), ( vec!["Listscripts\r\n"], vec![Request { tag: "".into(), command: Command::ListScripts, tokens: vec![], }], ), ( vec!["Setactive \"baz\"\r\n"], vec![Request { tag: "".into(), command: Command::SetActive, tokens: vec![Token::Argument(b"baz".to_vec())], }], ), ( vec!["Renamescript \"foo\" \"bar\"\r\n"], vec![Request { tag: "".into(), command: Command::RenameScript, tokens: vec![ Token::Argument(b"foo".to_vec()), Token::Argument(b"bar".to_vec()), ], }], ), ( vec!["NOOP \"STARTTLS-SYNC-42\"\r\n"], vec![Request { tag: "".into(), command: Command::Noop, tokens: vec![Token::Argument(b"STARTTLS-SYNC-42".to_vec())], }], ), ] { let mut requests = Vec::new(); for frame in &frames { let mut bytes = frame.as_bytes().iter(); loop { match receiver.parse(&mut bytes) { Ok(request) => requests.push(request), Err(Error::NeedsMoreData | Error::NeedsLiteral { .. }) => break, Err(err) => panic!("{:?} for frames {:#?}", err, frames), } } } assert_eq!(requests, expected_requests, "{:#?}", frames); } } } ================================================ FILE: crates/managesieve/src/op/authenticate.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{ auth::{ AuthRequest, sasl::{sasl_decode_challenge_oauth, sasl_decode_challenge_plain}, }, listener::{SessionStream, limiter::LimiterResult}, }; use directory::Permission; use imap_proto::{ protocol::authenticate::Mechanism, receiver::{self, Request}, }; use mail_parser::decoders::base64::base64_decode; use crate::core::{Command, Session, State, StatusResponse}; impl Session { pub async fn handle_authenticate(&mut self, request: Request) -> trc::Result> { if request.tokens.is_empty() { return Err(trc::AuthEvent::Error .into_err() .details("Authentication mechanism missing.")); } let mut tokens = request.tokens.into_iter(); let mechanism = Mechanism::parse(&tokens.next().unwrap().unwrap_bytes()) .map_err(|err| trc::AuthEvent::Error.into_err().details(err))?; let mut params: Vec = tokens .filter_map(|token| token.unwrap_string().ok()) .collect(); let credentials = match mechanism { Mechanism::Plain | Mechanism::OAuthBearer | Mechanism::XOauth2 => { if !params.is_empty() { base64_decode(params.pop().unwrap().as_bytes()) .and_then(|challenge| { if mechanism == Mechanism::Plain { sasl_decode_challenge_plain(&challenge) } else { sasl_decode_challenge_oauth(&challenge) } }) .ok_or_else(|| { trc::AuthEvent::Error .into_err() .details("Failed to decode challenge.") })? } else { self.receiver.request = receiver::Request { tag: "".into(), command: Command::Authenticate, tokens: vec![receiver::Token::Argument(mechanism.into_bytes())], }; self.receiver.state = receiver::State::Argument { last_ch: b' ' }; return Ok(b"{0}\r\n".to_vec()); } } _ => { return Err(trc::AuthEvent::Error .into_err() .details("Authentication mechanism not supported.")); } }; // Authenticate let access_token = self .server .authenticate(&AuthRequest::from_credentials( credentials, self.session_id, self.remote_addr, )) .await .map_err(|err| { if err.matches(trc::EventType::Auth(trc::AuthEvent::Failed)) { match &self.state { State::NotAuthenticated { auth_failures } if *auth_failures < self.server.core.imap.max_auth_failures => { self.state = State::NotAuthenticated { auth_failures: auth_failures + 1, }; } _ => { return trc::AuthEvent::TooManyAttempts.into_err().caused_by(err); } } } err }) .and_then(|token| { token .assert_has_permission(Permission::SieveAuthenticate) .map(|_| token) })?; // Enforce concurrency limits let in_flight = match access_token.is_imap_request_allowed() { LimiterResult::Allowed(in_flight) => Some(in_flight), LimiterResult::Forbidden => { return Err(trc::LimitEvent::ConcurrentRequest.into_err()); } LimiterResult::Disabled => None, }; // Create session self.state = State::Authenticated { access_token, in_flight, }; Ok(StatusResponse::ok("Authentication successful").into_bytes()) } pub async fn handle_unauthenticate(&mut self) -> trc::Result> { self.state = State::NotAuthenticated { auth_failures: 0 }; trc::event!( ManageSieve(trc::ManageSieveEvent::Unauthenticate), SpanId = self.session_id, Elapsed = trc::Value::Duration(0) ); Ok(StatusResponse::ok("Unauthenticate successful.").into_bytes()) } } ================================================ FILE: crates/managesieve/src/op/capability.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::core::{Session, StatusResponse}; use common::listener::SessionStream; use jmap_proto::request::capability::Capabilities; use std::time::Instant; impl Session { pub async fn handle_capability(&self, message: &'static str) -> trc::Result> { let op_start = Instant::now(); let mut response = Vec::with_capacity(128); response.extend_from_slice(b"\"IMPLEMENTATION\" \"Stalwart ManageSieve\"\r\n"); response.extend_from_slice(b"\"VERSION\" \"1.0\"\r\n"); if !self.stream.is_tls() { response.extend_from_slice(b"\"STARTTLS\"\r\n"); } if self.stream.is_tls() || self.server.core.imap.allow_plain_auth { response.extend_from_slice(b"\"SASL\" \"PLAIN OAUTHBEARER XOAUTH2\"\r\n"); } else { response.extend_from_slice(b"\"SASL\" \"OAUTHBEARER XOAUTH2\"\r\n"); }; if let Some(sieve) = self.server .core .jmap .capabilities .account .iter() .find_map(|(_, item)| { if let Capabilities::SieveAccount(sieve) = item { Some(sieve) } else { None } }) { response.extend_from_slice(b"\"SIEVE\" \""); response.extend_from_slice(sieve.extensions.join(" ").as_bytes()); response.extend_from_slice(b"\"\r\n"); if let Some(notification_methods) = &sieve.notification_methods { response.extend_from_slice(b"\"NOTIFY\" \""); response.extend_from_slice(notification_methods.join(" ").as_bytes()); response.extend_from_slice(b"\"\r\n"); } if sieve.max_redirects > 0 { response.extend_from_slice(b"\"MAXREDIRECTS\" \""); response.extend_from_slice(sieve.max_redirects.to_string().as_bytes()); response.extend_from_slice(b"\"\r\n"); } } else { response.extend_from_slice(b"\"SIEVE\" \"\"\r\n"); } trc::event!( ManageSieve(trc::ManageSieveEvent::Capabilities), SpanId = self.session_id, Tls = self.stream.is_tls(), Strict = !self.server.core.imap.allow_plain_auth, Elapsed = op_start.elapsed() ); Ok(StatusResponse::ok(message).serialize(response)) } } ================================================ FILE: crates/managesieve/src/op/checkscript.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; use imap_proto::receiver::Request; use crate::core::{Command, Session, StatusResponse}; impl Session { pub async fn handle_checkscript(&mut self, request: Request) -> trc::Result> { // Validate access self.assert_has_permission(Permission::SieveCheckScript)?; let op_start = Instant::now(); if request.tokens.is_empty() { return Err(trc::ManageSieveEvent::Error .into_err() .details("Expected script as a parameter.")); } let script = request.tokens.into_iter().next().unwrap().unwrap_bytes(); self.server .core .sieve .untrusted_compiler .compile(&script) .map(|_| { trc::event!( ManageSieve(trc::ManageSieveEvent::CheckScript), SpanId = self.session_id, Size = script.len(), Elapsed = op_start.elapsed() ); StatusResponse::ok("Script is valid.").into_bytes() }) .map_err(|err| { trc::ManageSieveEvent::Error .into_err() .details(err.to_string()) }) } } ================================================ FILE: crates/managesieve/src/op/deletescript.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::core::{Command, ResponseCode, Session, StatusResponse}; use common::listener::SessionStream; use directory::Permission; use email::sieve::{delete::SieveScriptDelete, ingest::SieveScriptIngest}; use imap_proto::receiver::Request; use std::time::Instant; use store::write::BatchBuilder; use trc::AddContext; impl Session { pub async fn handle_deletescript(&mut self, request: Request) -> trc::Result> { // Validate access self.assert_has_permission(Permission::SieveDeleteScript)?; let op_start = Instant::now(); let name = request .tokens .into_iter() .next() .and_then(|s| s.unwrap_string().ok()) .ok_or_else(|| { trc::ManageSieveEvent::Error .into_err() .details("Expected script name as a parameter.") })?; let access_token = self.state.access_token(); let account_id = access_token.primary_id(); let document_id = self.get_script_id(account_id, &name).await?; let mut batch = BatchBuilder::new(); let active_script_id = self.server.sieve_script_get_active_id(account_id).await?; if active_script_id != Some(document_id) { if self .server .sieve_script_delete(account_id, document_id, access_token, &mut batch) .await .caused_by(trc::location!())? { if !batch.is_empty() { self.server .commit_batch(batch) .await .caused_by(trc::location!())?; } trc::event!( ManageSieve(trc::ManageSieveEvent::DeleteScript), SpanId = self.session_id, Id = name, DocumentId = document_id, Elapsed = op_start.elapsed() ); Ok(StatusResponse::ok("Deleted.").into_bytes()) } else { Err(trc::ManageSieveEvent::Error .into_err() .details("Script not found")) } } else { Err(trc::ManageSieveEvent::Error .into_err() .details("You may not delete an active script") .code(ResponseCode::Active)) } } } ================================================ FILE: crates/managesieve/src/op/getscript.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::core::{Command, ResponseCode, Session, StatusResponse}; use common::listener::SessionStream; use directory::Permission; use email::sieve::SieveScript; use imap_proto::receiver::Request; use std::time::Instant; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{blob::BlobSection, blob_hash::BlobHash, collection::Collection}; impl Session { pub async fn handle_getscript(&mut self, request: Request) -> trc::Result> { // Validate access self.assert_has_permission(Permission::SieveGetScript)?; let op_start = Instant::now(); let name = request .tokens .into_iter() .next() .and_then(|s| s.unwrap_string().ok()) .ok_or_else(|| { trc::ManageSieveEvent::Error .into_err() .details("Expected script name as a parameter.") })?; let account_id = self.state.access_token().primary_id(); let document_id = self.get_script_id(account_id, &name).await?; let sieve_ = self .server .store() .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, document_id, )) .await .caused_by(trc::location!())? .ok_or_else(|| { trc::ManageSieveEvent::Error .into_err() .details("Script not found") .code(ResponseCode::NonExistent) })?; let sieve = sieve_ .unarchive::() .caused_by(trc::location!())?; let blob_size = u32::from(sieve.size) as usize; let script = self .server .get_blob_section( &BlobHash::from(&sieve.blob_hash), &BlobSection { size: blob_size, ..Default::default() }, ) .await .caused_by(trc::location!())? .ok_or_else(|| { trc::ManageSieveEvent::Error .into_err() .details("Script blob not found") .code(ResponseCode::NonExistent) })?; debug_assert_eq!(script.len(), blob_size); let mut response = Vec::with_capacity(script.len() + 32); response.push(b'{'); response.extend_from_slice(blob_size.to_string().as_bytes()); response.extend_from_slice(b"}\r\n"); response.extend(script); response.extend_from_slice(b"\r\n"); trc::event!( ManageSieve(trc::ManageSieveEvent::GetScript), SpanId = self.session_id, Id = name, DocumentId = document_id, Elapsed = op_start.elapsed() ); Ok(StatusResponse::ok("").serialize(response)) } } ================================================ FILE: crates/managesieve/src/op/havespace.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; use imap_proto::receiver::Request; use trc::AddContext; use crate::core::{Command, ResponseCode, Session, StatusResponse}; impl Session { pub async fn handle_havespace(&mut self, request: Request) -> trc::Result> { // Validate access self.assert_has_permission(Permission::SieveHaveSpace)?; let op_start = Instant::now(); let mut tokens = request.tokens.into_iter(); let name = tokens .next() .and_then(|s| s.unwrap_string().ok()) .ok_or_else(|| { trc::ManageSieveEvent::Error .into_err() .details("Expected script name as a parameter.") })?; let size: usize = tokens .next() .and_then(|s| s.unwrap_string().ok()) .ok_or_else(|| { trc::ManageSieveEvent::Error .into_err() .details("Expected script size as a parameter.") })? .parse::() .map_err(|_| { trc::ManageSieveEvent::Error .into_err() .details("Invalid size parameter.") })?; // Validate name let access_token = self.state.access_token(); let account_id = access_token.primary_id(); self.validate_name(account_id, &name).await?; // Validate quota if access_token.quota == 0 || size as i64 + self .server .get_used_quota(account_id) .await .caused_by(trc::location!())? <= access_token.quota as i64 { trc::event!( ManageSieve(trc::ManageSieveEvent::HaveSpace), SpanId = self.session_id, Size = size, Elapsed = op_start.elapsed() ); Ok(StatusResponse::ok("").into_bytes()) } else { Err(trc::ManageSieveEvent::Error .into_err() .details("Quota exceeded.") .code(ResponseCode::QuotaMaxSize)) } } } ================================================ FILE: crates/managesieve/src/op/listscripts.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::core::{Session, StatusResponse}; use common::listener::SessionStream; use directory::Permission; use email::sieve::{SieveScript, ingest::SieveScriptIngest}; use std::time::Instant; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{collection::Collection, field::SieveField}; impl Session { pub async fn handle_listscripts(&mut self) -> trc::Result> { // Validate access self.assert_has_permission(Permission::SieveListScripts)?; let op_start = Instant::now(); let account_id = self.state.access_token().primary_id(); let document_ids = self .server .document_ids(account_id, Collection::SieveScript, SieveField::Name) .await .caused_by(trc::location!())?; if document_ids.is_empty() { return Ok(StatusResponse::ok("").into_bytes()); } let mut response = Vec::with_capacity(128); let count = document_ids.len(); let active_script_id = self.server.sieve_script_get_active_id(account_id).await?; for document_id in document_ids { if let Some(script_) = self .server .store() .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, document_id, )) .await .caused_by(trc::location!())? { let script = script_ .unarchive::() .caused_by(trc::location!())?; response.push(b'\"'); for ch in script.name.as_bytes() { if [b'\\', b'\"'].contains(ch) { response.push(b'\\'); } response.push(*ch); } if active_script_id == Some(document_id) { response.extend_from_slice(b"\" ACTIVE\r\n"); } else { response.extend_from_slice(b"\"\r\n"); } } } trc::event!( ManageSieve(trc::ManageSieveEvent::ListScripts), SpanId = self.session_id, Total = count, Elapsed = op_start.elapsed() ); Ok(StatusResponse::ok("").serialize(response)) } } ================================================ FILE: crates/managesieve/src/op/logout.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use tokio::io::{AsyncRead, AsyncWrite}; use crate::core::{Session, StatusResponse}; impl Session { pub async fn handle_logout(&mut self) -> trc::Result> { trc::event!( ManageSieve(trc::ManageSieveEvent::Logout), SpanId = self.session_id, Elapsed = trc::Value::Duration(0) ); Ok(StatusResponse::ok("Stalwart ManageSieve bids you farewell.").into_bytes()) } } ================================================ FILE: crates/managesieve/src/op/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::core::{Session, State, StatusResponse}; use common::listener::SessionStream; use directory::Permission; pub mod authenticate; pub mod capability; pub mod checkscript; pub mod deletescript; pub mod getscript; pub mod havespace; pub mod listscripts; pub mod logout; pub mod noop; pub mod putscript; pub mod renamescript; pub mod setactive; impl Session { pub async fn handle_start_tls(&self) -> trc::Result> { trc::event!( ManageSieve(trc::ManageSieveEvent::StartTls), SpanId = self.session_id, Elapsed = trc::Value::Duration(0) ); Ok(StatusResponse::ok("Begin TLS negotiation now").into_bytes()) } pub fn assert_has_permission(&self, permission: Permission) -> trc::Result { match &self.state { State::Authenticated { access_token, .. } => { access_token.assert_has_permission(permission) } State::NotAuthenticated { .. } => Ok(false), } } } ================================================ FILE: crates/managesieve/src/op/noop.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use imap_proto::receiver::Request; use tokio::io::{AsyncRead, AsyncWrite}; use crate::core::{Command, ResponseCode, Session, StatusResponse}; impl Session { pub async fn handle_noop(&mut self, request: Request) -> trc::Result> { trc::event!( ManageSieve(trc::ManageSieveEvent::Noop), SpanId = self.session_id, Elapsed = trc::Value::Duration(0) ); Ok(if let Some(tag) = request .tokens .into_iter() .next() .and_then(|t| t.unwrap_string().ok()) { StatusResponse::ok("Done").with_code(ResponseCode::Tag(tag)) } else { StatusResponse::ok("Done") } .into_bytes()) } } ================================================ FILE: crates/managesieve/src/op/putscript.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::core::{Command, ResponseCode, Session, StatusResponse}; use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; use email::sieve::SieveScript; use imap_proto::receiver::Request; use sieve::compiler::ErrorType; use std::time::Instant; use store::{ Serialize, ValueKey, write::{AlignedBytes, Archive, Archiver, BatchBuilder}, }; use trc::AddContext; use types::{collection::Collection, field::SieveField}; impl Session { pub async fn handle_putscript(&mut self, request: Request) -> trc::Result> { // Validate access self.assert_has_permission(Permission::SievePutScript)?; let op_start = Instant::now(); let mut tokens = request.tokens.into_iter(); let name = tokens .next() .and_then(|s| s.unwrap_string().ok()) .ok_or_else(|| { trc::ManageSieveEvent::Error .into_err() .details("Expected script name as a parameter.") })? .trim() .to_string(); let mut script_bytes = tokens .next() .ok_or_else(|| { trc::ManageSieveEvent::Error .into_err() .details("Expected script as a parameter.") })? .unwrap_bytes(); let script_size = script_bytes.len() as i64; // Check quota let access_token = self.state.access_token(); let account_id = access_token.primary_id(); self.server .has_available_quota(&access_token.as_resource_token(), script_bytes.len() as u64) .await .caused_by(trc::location!())?; if self .server .document_ids(account_id, Collection::SieveScript, SieveField::Name) .await .caused_by(trc::location!())? .len() > access_token.object_quota(Collection::SieveScript) as u64 { return Err(trc::ManageSieveEvent::Error .into_err() .details("Too many scripts.") .code(ResponseCode::QuotaMaxScripts)); } // Compile script match self .server .core .sieve .untrusted_compiler .compile(&script_bytes) { Ok(compiled_script) => { script_bytes.extend( Archiver::new(compiled_script) .untrusted() .serialize() .caused_by(trc::location!())?, ); } Err(err) => { return Err(if let ErrorType::ScriptTooLong = &err.error_type() { trc::ManageSieveEvent::Error .into_err() .details(err.to_string()) .code(ResponseCode::QuotaMaxSize) } else { trc::ManageSieveEvent::Error .into_err() .details(err.to_string()) }); } } // Validate name if let Some(document_id) = self.validate_name(account_id, &name).await? { // Obtain script values let script_ = self .server .store() .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, document_id, )) .await .caused_by(trc::location!())? .ok_or_else(|| { trc::ManageSieveEvent::Error .into_err() .details("Script not found") .code(ResponseCode::NonExistent) })?; let script = script_ .to_unarchived::() .caused_by(trc::location!())?; // Write script blob let (blob_hash, blob_hold) = self .server .put_temporary_blob(account_id, &script_bytes, 60) .await?; // Write record let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::SieveScript) .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_changes( script .deserialize() .caused_by(trc::location!())? .with_size(script_size as u32) .with_blob_hash(blob_hash.clone()), ) .with_current(script) .with_access_token(access_token), ) .caused_by(trc::location!())? .clear(blob_hold); self.server .commit_batch(batch) .await .caused_by(trc::location!())?; trc::event!( ManageSieve(trc::ManageSieveEvent::UpdateScript), SpanId = self.session_id, Id = name.to_string(), DocumentId = document_id, Size = script_size, Elapsed = op_start.elapsed(), ); } else { // Write script blob let (blob_hash, blob_hold) = self .server .put_temporary_blob(account_id, &script_bytes, 60) .await?; // Write record let mut batch = BatchBuilder::new(); let document_id = self .server .store() .assign_document_ids(account_id, Collection::SieveScript, 1) .await .caused_by(trc::location!())?; batch .with_account_id(account_id) .with_collection(Collection::SieveScript) .with_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes( SieveScript::new(name.clone(), blob_hash.clone()) .with_size(script_size as u32), ) .with_access_token(access_token), ) .caused_by(trc::location!())? .clear(blob_hold); self.server .commit_batch(batch) .await .caused_by(trc::location!())?; trc::event!( ManageSieve(trc::ManageSieveEvent::CreateScript), SpanId = self.session_id, Id = name, DocumentId = document_id, Elapsed = op_start.elapsed() ); } Ok(StatusResponse::ok("Success.").into_bytes()) } pub async fn validate_name(&self, account_id: u32, name: &str) -> trc::Result> { if name.is_empty() { Err(trc::ManageSieveEvent::Error .into_err() .details("Script name cannot be empty.")) } else if name.len() > self.server.core.jmap.sieve_max_script_name { Err(trc::ManageSieveEvent::Error .into_err() .details("Script name is too long.")) } else if name.eq_ignore_ascii_case("vacation") { Err(trc::ManageSieveEvent::Error .into_err() .details("The 'vacation' name is reserved, please use a different name.")) } else { Ok(self .server .document_ids_matching( account_id, Collection::SieveScript, SieveField::Name, name.to_lowercase().as_bytes(), ) .await .caused_by(trc::location!())? .min()) } } } ================================================ FILE: crates/managesieve/src/op/renamescript.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::core::{Command, ResponseCode, Session, StatusResponse}; use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; use email::sieve::SieveScript; use imap_proto::receiver::Request; use std::time::Instant; use store::{ ValueKey, write::{AlignedBytes, Archive, BatchBuilder}, }; use trc::AddContext; use types::collection::Collection; impl Session { pub async fn handle_renamescript(&mut self, request: Request) -> trc::Result> { // Validate access self.assert_has_permission(Permission::SieveRenameScript)?; let op_start = Instant::now(); let mut tokens = request.tokens.into_iter(); let name = tokens .next() .and_then(|s| s.unwrap_string().ok()) .ok_or_else(|| { trc::ManageSieveEvent::Error .into_err() .details("Expected old script name as a parameter.") })? .trim() .to_string(); let new_name = tokens .next() .and_then(|s| s.unwrap_string().ok()) .ok_or_else(|| { trc::ManageSieveEvent::Error .into_err() .details("Expected new script name as a parameter.") })? .trim() .to_string(); // Validate name if name == new_name { return Ok(StatusResponse::ok("Old and new script names are the same.").into_bytes()); } let account_id = self.state.access_token().primary_id(); let document_id = self.get_script_id(account_id, &name).await?; if self.validate_name(account_id, &new_name).await?.is_some() { return Err(trc::ManageSieveEvent::Error .into_err() .details(format!("A sieve script with name '{name}' already exists.",)) .code(ResponseCode::AlreadyExists)); } // Obtain script values let script = self .server .store() .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, document_id, )) .await .caused_by(trc::location!())? .ok_or_else(|| { trc::ManageSieveEvent::Error .into_err() .details("Script not found") .code(ResponseCode::NonExistent) })? .into_deserialized::() .caused_by(trc::location!())?; // Write record let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::SieveScript) .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_changes(script.inner.clone().with_name(new_name.clone())) .with_current(script), ) .caused_by(trc::location!())?; if !batch.is_empty() { self.server .commit_batch(batch) .await .caused_by(trc::location!())?; } trc::event!( ManageSieve(trc::ManageSieveEvent::RenameScript), SpanId = self.session_id, Id = new_name, DocumentId = document_id, Elapsed = op_start.elapsed() ); Ok(StatusResponse::ok("Success.").into_bytes()) } } ================================================ FILE: crates/managesieve/src/op/setactive.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; use imap_proto::receiver::Request; use store::{SerializeInfallible, write::BatchBuilder}; use trc::AddContext; use types::{collection::Collection, field::PrincipalField}; use crate::core::{Command, Session, StatusResponse}; impl Session { pub async fn handle_setactive(&mut self, request: Request) -> trc::Result> { // Validate access self.assert_has_permission(Permission::SieveSetActive)?; let op_start = Instant::now(); let name = request .tokens .into_iter() .next() .and_then(|s| s.unwrap_string().ok()) .ok_or_else(|| { trc::ManageSieveEvent::Error .into_err() .details("Expected script name as a parameter.") })?; // De/activate script let account_id = self.state.access_token().primary_id(); let mut batch = BatchBuilder::new(); if !name.is_empty() { let document_id = self.get_script_id(account_id, &name).await?; batch .with_account_id(account_id) .with_collection(Collection::Principal) .with_document(0) .set(PrincipalField::ActiveScriptId, document_id.serialize()); } else { batch .with_account_id(account_id) .with_collection(Collection::Principal) .with_document(0) .clear(PrincipalField::ActiveScriptId); } self.server .commit_batch(batch) .await .caused_by(trc::location!())?; trc::event!( ManageSieve(trc::ManageSieveEvent::SetActive), SpanId = self.session_id, Id = name, Elapsed = op_start.elapsed() ); Ok(StatusResponse::ok("Success").into_bytes()) } } ================================================ FILE: crates/migration/Cargo.toml ================================================ [package] name = "migration" version = "0.15.5" edition = "2024" [dependencies] utils = { path = "../utils" } nlp = { path = "../nlp" } store = { path = "../store" } trc = { path = "../trc" } types = { path = "../types" } common = { path = "../common" } email = { path = "../email" } directory = { path = "../directory" } smtp = { path = "../smtp" } groupware = { path = "../groupware" } dav-proto = { path = "../dav-proto" } proc_macros = { path = "../utils/proc-macros" } mail-parser = { version = "0.11", features = ["full_encoding"] } mail-auth = { version = "0.7.1", features = ["rkyv"] } smtp-proto = { version = "0.2", features = ["rkyv", "serde"] } sieve-rs = { version = "0.7", features = ["rkyv"] } calcard_latest = { package = "calcard", version = "0.3", features = ["rkyv"] } calcard_v01 = { package = "calcard", version = "0.1", features = ["rkyv"] } tokio = { version = "1.47", features = ["net", "macros"] } serde = { version = "1.0", features = ["derive"]} serde_json = "1.0" rkyv = { version = "0.8.10", features = ["little_endian"] } compact_str = "0.9.0" bincode = "1.3.3" lz4_flex = { version = "0.12", default-features = false } base64 = "0.22" futures = "0.3" num_cpus = "1.13.1" [features] test_mode = [] enterprise = [] [dev-dependencies] tokio = { version = "1.47", features = ["full"] } ================================================ FILE: crates/migration/src/addressbook_v2.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::Server; use groupware::contact::{AddressBook, AddressBookPreferences}; use store::{ Serialize, ValueKey, write::{AlignedBytes, Archive, Archiver, BatchBuilder, serialize::rkyv_deserialize}, }; use trc::AddContext; use types::{acl::AclGrant, collection::Collection, dead_property::DeadProperty, field::Field}; use crate::get_document_ids; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] #[rkyv(derive(Debug))] pub struct AddressBookV2 { pub name: String, pub display_name: Option, pub description: Option, pub sort_order: u32, pub is_default: bool, pub subscribers: Vec, pub dead_properties: DeadProperty, pub acls: Vec, pub created: i64, pub modified: i64, } pub(crate) async fn migrate_addressbook_v013(server: &Server, account_id: u32) -> trc::Result { let document_ids = get_document_ids(server, account_id, Collection::AddressBook) .await .caused_by(trc::location!())? .unwrap_or_default(); if document_ids.is_empty() { return Ok(0); } let mut num_migrated = 0; for document_id in document_ids.iter() { let Some(archive) = server .store() .get_value::>(ValueKey::archive( account_id, Collection::AddressBook, document_id, )) .await .caused_by(trc::location!())? else { continue; }; match archive.unarchive_untrusted::() { Ok(book) => { let book = rkyv_deserialize::<_, AddressBookV2>(book).unwrap(); let new_book = AddressBook { name: book.name, preferences: vec![AddressBookPreferences { account_id, name: book .display_name .unwrap_or_else(|| "Address Book".to_string()), description: book.description, sort_order: book.sort_order, }], subscribers: book.subscribers, dead_properties: book.dead_properties, acls: book.acls, created: book.created, modified: book.modified, }; let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::AddressBook) .with_document(document_id) .set( Field::ARCHIVE, Archiver::new(new_book) .serialize() .caused_by(trc::location!())?, ); server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; num_migrated += 1; } Err(err) => { if let Err(err_) = archive.unarchive_untrusted::() { trc::error!(err_.caused_by(trc::location!())); return Err(err.caused_by(trc::location!())); } } } } Ok(num_migrated) } ================================================ FILE: crates/migration/src/blob.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::Server; use store::{ IterateParams, SUBSPACE_BLOB_LINK, Serialize, SerializeInfallible, U32_LEN, U64_LEN, ValueKey, write::{ AnyClass, Archiver, BatchBuilder, BlobLink, BlobOp, ValueClass, key::DeserializeBigEndian, now, }, }; use trc::AddContext; use types::blob_hash::{BLOB_HASH_LEN, BlobHash}; const SUBSPACE_BLOB_RESERVE: u8 = b'j'; pub(crate) async fn migrate_blobs_v014(server: &Server) -> trc::Result<()> { let mut num_blobs = 0; for byte in 0..=u8::MAX { // Validate linked blobs let mut from_hash = BlobHash::default(); let mut to_hash = BlobHash::new_max(); from_hash.0[0] = byte; to_hash.0[0] = byte; let from_key = ValueKey { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Blob(BlobOp::Commit { hash: from_hash }), }; let to_key = ValueKey { account_id: u32::MAX, collection: u8::MAX, document_id: u32::MAX, class: ValueClass::Blob(BlobOp::Link { hash: to_hash, to: BlobLink::Document, }), }; let mut keys = Vec::new(); server .store() .iterate( IterateParams::new(from_key, to_key).ascending().no_values(), |key, value| { if key.len() == BLOB_HASH_LEN + U64_LEN + 1 { let hash = BlobHash::try_from_hash_slice(key.get(0..BLOB_HASH_LEN).ok_or_else( || trc::Error::corrupted_key(key, value.into(), trc::location!()), )?) .unwrap(); let account_id = key.deserialize_be_u32(BLOB_HASH_LEN)?; let document_id = key.deserialize_be_u32(BLOB_HASH_LEN + U32_LEN + 1)?; let collection = key[BLOB_HASH_LEN + U32_LEN]; if account_id == u32::MAX && document_id == u32::MAX && collection == 0 { keys.push((key.to_vec(), BlobOp::Commit { hash })); } else if collection == u8::MAX { keys.push(( key.to_vec(), BlobOp::Link { hash, to: BlobLink::Id { id: ((account_id as u64) << 32) | document_id as u64, }, }, )); } } Ok(true) }, ) .await .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); num_blobs += keys.len(); for (key, op) in keys { batch .clear(ValueClass::Any(AnyClass { subspace: SUBSPACE_BLOB_LINK, key, })) .set(op, vec![]); if batch.is_large_batch() { server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); } } if !batch.is_empty() { server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } } trc::event!( Server(trc::ServerEvent::Startup), Details = format!("Migrated {num_blobs} blob links") ); enum OldType { Quota { size: u32 }, Undelete { deleted_at: u64, size: u32 }, Temp, None, } struct OldBlobEntry { account_id: u32, until: u64, hash: BlobHash, blob_type: OldType, old_key: Vec, } let mut entries = Vec::new(); let now = now(); server .store() .iterate( IterateParams::new( ValueKey::from(ValueClass::Any(AnyClass { subspace: SUBSPACE_BLOB_RESERVE, key: vec![0u8], })), ValueKey::from(ValueClass::Any(AnyClass { subspace: SUBSPACE_BLOB_RESERVE, key: vec![u8::MAX; 32], })), ) .ascending(), |key, value| { if key.len() == BLOB_HASH_LEN + U64_LEN + U32_LEN { let account_id = key.deserialize_be_u32(0)?; let hash = BlobHash::try_from_hash_slice( key.get(U32_LEN..BLOB_HASH_LEN + U32_LEN).ok_or_else(|| { trc::Error::corrupted_key(key, value.into(), trc::location!()) })?, ) .unwrap(); let until = key.deserialize_be_u64(BLOB_HASH_LEN + U32_LEN)?; let blob_type = if until > now { if value.len() == U32_LEN { let size = value.deserialize_be_u32(0)?; if size != 0 { OldType::Quota { size } } else { OldType::Temp } } else if value.len() == U64_LEN + U32_LEN + 1 { let size = value.deserialize_be_u32(0)?; let deleted_at = value.deserialize_be_u64(U32_LEN)?; OldType::Undelete { deleted_at, size } } else { OldType::Temp } } else { OldType::None }; entries.push(OldBlobEntry { account_id, until, hash, blob_type, old_key: key.to_vec(), }); } Ok(true) }, ) .await .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); let num_entries = entries.len(); for entry in entries { batch .clear(ValueClass::Any(AnyClass { subspace: SUBSPACE_BLOB_RESERVE, key: entry.old_key, })) .with_account_id(entry.account_id); match entry.blob_type { OldType::Quota { size } => { batch .set( BlobOp::Link { hash: entry.hash.clone(), to: BlobLink::Temporary { until: entry.until }, }, vec![BlobLink::QUOTA_LINK], ) .set( BlobOp::Quota { hash: entry.hash, until: entry.until, }, size.serialize(), ); } OldType::Undelete { deleted_at, size } => { // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] { batch .set( BlobOp::Link { hash: entry.hash.clone(), to: BlobLink::Temporary { until: entry.until }, }, vec![BlobLink::UNDELETE_LINK], ) .set( BlobOp::Undelete { hash: entry.hash, until: entry.until, }, Archiver::new(common::enterprise::undelete::DeletedItem { typ: common::enterprise::undelete::DeletedItemType::Email { from: "unknown".into(), subject: "unknown".into(), received_at: deleted_at, }, size, deleted_at, }) .serialize() .caused_by(trc::location!())?, ); } // SPDX-SnippetEnd } OldType::Temp => { batch.set( BlobOp::Link { hash: entry.hash, to: BlobLink::Temporary { until: entry.until }, }, vec![], ); } OldType::None => (), } if batch.is_large_batch() { server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); } } trc::event!( Server(trc::ServerEvent::Startup), Details = format!("Migrated {num_entries} temporary blob links") ); if !batch.is_empty() { server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } Ok(()) } ================================================ FILE: crates/migration/src/calendar_v2.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::Server; use groupware::calendar::{Calendar, CalendarPreferences, Timezone}; use store::{ Serialize, ValueKey, write::{AlignedBytes, Archive, Archiver, BatchBuilder, serialize::rkyv_deserialize}, }; use trc::AddContext; use types::{acl::AclGrant, collection::Collection, dead_property::DeadProperty, field::Field}; use crate::{event_v2::migrate_icalendar_v02, get_document_ids}; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct CalendarV2 { pub name: String, pub preferences: Vec, pub default_alerts: Vec, pub acls: Vec, pub dead_properties: DeadProperty, pub created: i64, pub modified: i64, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct CalendarPreferencesV2 { pub account_id: u32, pub name: String, pub description: Option, pub sort_order: u32, pub color: Option, pub flags: u16, pub time_zone: TimezoneV2, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub enum TimezoneV2 { IANA(u16), Custom(calcard_v01::icalendar::ICalendar), #[default] Default, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct DefaultAlertV2 { pub account_id: u32, pub id: String, pub alert: calcard_v01::icalendar::ICalendar, pub with_time: bool, } pub(crate) async fn migrate_calendar_v013(server: &Server, account_id: u32) -> trc::Result { let document_ids = get_document_ids(server, account_id, Collection::Calendar) .await .caused_by(trc::location!())? .unwrap_or_default(); if document_ids.is_empty() { return Ok(0); } let mut num_migrated = 0; for document_id in document_ids.iter() { let Some(archive) = server .store() .get_value::>(ValueKey::archive( account_id, Collection::Calendar, document_id, )) .await .caused_by(trc::location!())? else { continue; }; match archive.unarchive_untrusted::() { Ok(calendar) => { let calendar = rkyv_deserialize::<_, CalendarV2>(calendar).unwrap(); let new_calendar = Calendar { name: calendar.name, preferences: calendar .preferences .into_iter() .map(|pref| CalendarPreferences { account_id: pref.account_id, name: pref.name, description: pref.description, sort_order: pref.sort_order, color: pref.color, flags: 0, time_zone: match pref.time_zone { TimezoneV2::IANA(tzid) => Timezone::IANA(tzid), TimezoneV2::Custom(tz) => { Timezone::Custom(migrate_icalendar_v02(tz)) } TimezoneV2::Default => Timezone::Default, }, default_alerts: Vec::new(), }) .collect(), acls: calendar.acls, supported_components: 0, dead_properties: calendar.dead_properties, created: calendar.created, modified: calendar.modified, }; let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Calendar) .with_document(document_id) .set( Field::ARCHIVE, Archiver::new(new_calendar) .serialize() .caused_by(trc::location!())?, ); server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; num_migrated += 1; } Err(err) => { if let Err(err_) = archive.unarchive_untrusted::() { trc::error!(err_.caused_by(trc::location!())); return Err(err.caused_by(trc::location!())); } } } } Ok(num_migrated) } ================================================ FILE: crates/migration/src/changelog.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::Server; use store::{ SUBSPACE_LOGS, U64_LEN, write::{AnyKey, key::KeySerializer}, }; use trc::AddContext; pub(crate) async fn reset_changelog(server: &Server) -> trc::Result<()> { // Delete changes server .store() .delete_range( AnyKey { subspace: SUBSPACE_LOGS, key: KeySerializer::new(U64_LEN).write(0u8).finalize(), }, AnyKey { subspace: SUBSPACE_LOGS, key: KeySerializer::new(U64_LEN) .write(&[u8::MAX; 16][..]) .finalize(), }, ) .await .caused_by(trc::location!()) } ================================================ FILE: crates/migration/src/contact_v2.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{DavName, Server}; use groupware::contact::ContactCard; use store::{ Serialize, ValueKey, write::{AlignedBytes, Archive, Archiver, BatchBuilder, serialize::rkyv_deserialize}, }; use trc::AddContext; use types::{collection::Collection, dead_property::DeadProperty, field::Field}; use crate::get_document_ids; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct ContactCardV2 { pub names: Vec, pub display_name: Option, pub card: calcard_v01::vcard::VCard, pub dead_properties: DeadProperty, pub created: i64, pub modified: i64, pub size: u32, } pub(crate) async fn migrate_contacts_v013(server: &Server, account_id: u32) -> trc::Result { let document_ids = get_document_ids(server, account_id, Collection::ContactCard) .await .caused_by(trc::location!())? .unwrap_or_default(); let mut num_migrated = 0; for document_id in document_ids.iter() { let Some(archive) = server .store() .get_value::>(ValueKey::archive( account_id, Collection::ContactCard, document_id, )) .await .caused_by(trc::location!())? else { continue; }; match archive.unarchive_untrusted::() { Ok(contact) => { let contact = rkyv_deserialize::<_, ContactCardV2>(contact).unwrap(); let new_contact = ContactCard { names: contact.names, display_name: contact.display_name, dead_properties: contact.dead_properties, size: contact.size, created: contact.created, modified: contact.modified, card: calcard_latest::vcard::VCard::parse(contact.card.to_string()) .unwrap_or_default(), }; let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::ContactCard) .with_document(document_id) .set( Field::ARCHIVE, Archiver::new(new_contact) .serialize() .caused_by(trc::location!())?, ); server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; num_migrated += 1; } Err(err) => { if let Err(err_) = archive.unarchive_untrusted::() { trc::error!(err_.caused_by(trc::location!())); return Err(err.caused_by(trc::location!())); } } } } Ok(num_migrated) } ================================================ FILE: crates/migration/src/email_v1.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{LegacyBincode, get_properties}; use crate::{email_v2::LegacyKeyword, get_bitmap, get_document_ids, v014::SUBSPACE_BITMAP_TAG}; use common::Server; use email::{ mailbox::*, message::{ index::extractors::VisitTextArchived, ingest::ThreadInfo, metadata::{ MESSAGE_HAS_ATTACHMENT, MESSAGE_RECEIVED_MASK, MessageDataBuilder, MessageMetadata, MessageMetadataContents, MessageMetadataPart, MetadataHeader, MetadataHeaderName, MetadataHeaderValue, MetadataPartType, PART_ENCODING_BASE64, PART_ENCODING_PROBLEM, PART_ENCODING_QP, PART_SIZE_MASK, }, }, }; use mail_parser::{ Address, Attribute, ContentType, DateTime, Encoding, HeaderName, HeaderValue, Received, parsers::fields::thread::thread_name, }; use std::{borrow::Cow, collections::VecDeque}; use store::{ Deserialize, SUBSPACE_INDEXES, SUBSPACE_PROPERTY, Serialize, SerializeInfallible, U32_LEN, U64_LEN, ValueKey, ahash::AHashMap, write::{ AlignedBytes, AnyKey, Archive, Archiver, BatchBuilder, IndexPropertyClass, ValueClass, key::KeySerializer, }, }; use trc::AddContext; use types::{ blob_hash::BlobHash, collection::Collection, field::{EmailField, Field}, keyword::*, }; use utils::{cheeky_hash::CheekyHash, codec::leb128::Leb128Iterator}; const FIELD_KEYWORDS: u8 = 4; const FIELD_THREAD_ID: u8 = 33; const FIELD_CID: u8 = 76; pub(crate) const FIELD_MAILBOX_IDS: u8 = 7; const BM_MARKER: u8 = 1 << 7; pub(crate) async fn migrate_emails_v011(server: &Server, account_id: u32) -> trc::Result { // Obtain email ids let mut document_ids = get_document_ids(server, account_id, Collection::Email) .await .caused_by(trc::location!())? .unwrap_or_default(); let num_emails = document_ids.len(); if num_emails == 0 { return Ok(0); } let tombstoned_ids = get_bitmap( server, AnyKey { subspace: SUBSPACE_BITMAP_TAG, key: KeySerializer::new(U64_LEN + U32_LEN + 1) .write(account_id) .write(u8::from(Collection::Email)) .write(FIELD_MAILBOX_IDS) .write_leb128(u32::MAX - 1) .finalize(), }, AnyKey { subspace: SUBSPACE_BITMAP_TAG, key: KeySerializer::new(U64_LEN + U32_LEN + 1) .write(account_id) .write(u8::from(Collection::Email)) .write(FIELD_MAILBOX_IDS) .write_leb128(u32::MAX - 1) .finalize(), }, ) .await .caused_by(trc::location!())? .unwrap_or_default(); let mut message_data: AHashMap = AHashMap::with_capacity(num_emails as usize); let mut did_migrate = false; // Obtain mailboxes for (message_id, uid_mailbox) in get_properties::( server, account_id, Collection::Email, &(), FIELD_MAILBOX_IDS, ) .await .caused_by(trc::location!())? { message_data.entry(message_id).or_default().mailboxes = uid_mailbox.0; } // Obtain keywords for (message_id, keywords) in get_properties::(server, account_id, Collection::Email, &(), FIELD_KEYWORDS) .await .caused_by(trc::location!())? { message_data.entry(message_id).or_default().keywords = keywords.0.into_iter().map(Into::into).collect(); } // Obtain threadIds for (message_id, thread_id) in get_properties::(server, account_id, Collection::Email, &(), FIELD_THREAD_ID) .await .caused_by(trc::location!())? { message_data.entry(message_id).or_default().thread_id = thread_id; } // Write message data for (message_id, mut data) in message_data { if !tombstoned_ids.contains(message_id) { let (size, metadata) = match server .store() .get_value::>(ValueKey { account_id, collection: Collection::Email.into(), document_id: message_id, class: ValueClass::Property(EmailField::Metadata.into()), }) .await { Ok(Some(legacy_metadata)) => ( legacy_metadata.inner.size as u32, MessageMetadata::from_legacy(legacy_metadata.inner), ), Ok(None) => { continue; } Err(err) => { match server .store() .get_value::>(ValueKey { account_id, collection: Collection::Email.into(), document_id: message_id, class: ValueClass::Property(EmailField::Metadata.into()), }) .await { Ok(Some(archive)) => { let metadata: MessageMetadata = archive .deserialize_untrusted() .caused_by(trc::location!())?; (metadata.root_part().offset_end, metadata) } _ => { return Err(err .account_id(account_id) .document_id(message_id) .caused_by(trc::location!())); } } } }; did_migrate = true; document_ids.insert(message_id); let mut message_ids = Vec::new(); let mut subject = ""; for header in &metadata.contents[0].parts[0].headers { match &header.name { MetadataHeaderName::MessageId => { header.value.visit_text(|id| { if !id.is_empty() { message_ids.push(CheekyHash::new(id.as_bytes())); } }); } MetadataHeaderName::InReplyTo | MetadataHeaderName::References | MetadataHeaderName::ResentMessageId => { header.value.visit_text(|id| { if !id.is_empty() { message_ids.push(CheekyHash::new(id.as_bytes())); } }); } MetadataHeaderName::Subject if subject.is_empty() => { subject = thread_name(match &header.value { MetadataHeaderValue::Text(text) => text.as_ref(), MetadataHeaderValue::TextList(list) if !list.is_empty() => { list.first().unwrap().as_ref() } _ => "", }); } _ => (), } } let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Email) .with_document(message_id); if data .mailboxes .iter() .any(|mailbox| mailbox.mailbox_id == TRASH_ID || mailbox.mailbox_id == JUNK_ID) { batch.set( ValueClass::Property(EmailField::DeletedAt.into()), (metadata.rcvd_attach & MESSAGE_RECEIVED_MASK).serialize(), ); } data.size = size; batch .set( ValueClass::IndexProperty(IndexPropertyClass::Hash { property: EmailField::Threading.into(), hash: CheekyHash::new(if !subject.is_empty() { subject } else { "!" }), }), ThreadInfo::serialize(data.thread_id, &message_ids), ) .set( Field::ARCHIVE, Archiver::new(data.seal()) .serialize() .caused_by(trc::location!())?, ) .set( EmailField::Metadata, Archiver::new(metadata) .serialize() .caused_by(trc::location!())?, ); server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } } // Delete keyword bitmaps for field in [FIELD_KEYWORDS, FIELD_KEYWORDS | BM_MARKER] { server .store() .delete_range( AnyKey { subspace: SUBSPACE_BITMAP_TAG, key: KeySerializer::new(U64_LEN) .write(account_id) .write(u8::from(Collection::Email)) .write(field) .finalize(), }, AnyKey { subspace: SUBSPACE_BITMAP_TAG, key: KeySerializer::new(U64_LEN) .write(account_id) .write(u8::from(Collection::Email)) .write(field) .write(&[u8::MAX; 8][..]) .finalize(), }, ) .await .caused_by(trc::location!())?; } // Delete messageId index, now in References const MESSAGE_ID_FIELD: u8 = 11; server .store() .delete_range( AnyKey { subspace: SUBSPACE_INDEXES, key: KeySerializer::new(U64_LEN) .write(account_id) .write(u8::from(Collection::Email)) .write(MESSAGE_ID_FIELD) .finalize(), }, AnyKey { subspace: SUBSPACE_INDEXES, key: KeySerializer::new(U64_LEN) .write(account_id) .write(u8::from(Collection::Email)) .write(MESSAGE_ID_FIELD) .write(&[u8::MAX; 8][..]) .finalize(), }, ) .await .caused_by(trc::location!())?; // Delete values for property in [ FIELD_MAILBOX_IDS, FIELD_KEYWORDS, FIELD_THREAD_ID, FIELD_CID, ] { server .store() .delete_range( AnyKey { subspace: SUBSPACE_PROPERTY, key: KeySerializer::new(U64_LEN) .write(account_id) .write(u8::from(Collection::Email)) .write(property) .finalize(), }, AnyKey { subspace: SUBSPACE_PROPERTY, key: KeySerializer::new(U64_LEN) .write(account_id) .write(u8::from(Collection::Email)) .write(property) .write(&[u8::MAX; 8][..]) .finalize(), }, ) .await .caused_by(trc::location!())?; } // Increment document id counter if did_migrate { server .store() .assign_document_ids( account_id, Collection::Email, document_ids.max().map(|id| id as u64).unwrap_or(num_emails) + 1, ) .await .caused_by(trc::location!())?; Ok(num_emails) } else { Ok(0) } } pub trait FromLegacy { fn from_legacy(legacy: LegacyMessageMetadata<'_>) -> Self; } impl FromLegacy for MessageMetadata { fn from_legacy(legacy: LegacyMessageMetadata<'_>) -> Self { let mut contents = Vec::new(); let mut messages = VecDeque::from([legacy.contents]); let mut message_id = 0; while let Some(message) = messages.pop_front() { let mut parts = Vec::new(); for part in message.parts { let body = match part.body { LegacyMetadataPartType::Text => MetadataPartType::Text, LegacyMetadataPartType::Html => MetadataPartType::Html, LegacyMetadataPartType::Binary => MetadataPartType::Binary, LegacyMetadataPartType::InlineBinary => MetadataPartType::InlineBinary, LegacyMetadataPartType::Message(message) => { messages.push_back(message); message_id += 1; MetadataPartType::Message(message_id) } LegacyMetadataPartType::Multipart(parts) => { MetadataPartType::Multipart(parts.into_iter().map(|p| p as u16).collect()) } }; let flags = match part.encoding { Encoding::None => 0, Encoding::QuotedPrintable => PART_ENCODING_QP, Encoding::Base64 => PART_ENCODING_BASE64, } | (if part.is_encoding_problem { PART_ENCODING_PROBLEM } else { 0 }) | (part.size as u32 & PART_SIZE_MASK); parts.push(MessageMetadataPart { headers: part .headers .into_iter() .map(|hdr| MetadataHeader { value: if matches!( &hdr.name, HeaderName::Subject | HeaderName::From | HeaderName::To | HeaderName::Cc | HeaderName::Date | HeaderName::Bcc | HeaderName::ReplyTo | HeaderName::Sender | HeaderName::Comments | HeaderName::InReplyTo | HeaderName::Keywords | HeaderName::MessageId | HeaderName::References | HeaderName::ResentMessageId | HeaderName::ContentDescription | HeaderName::ContentId | HeaderName::ContentLanguage | HeaderName::ContentLocation | HeaderName::ContentTransferEncoding | HeaderName::ContentType | HeaderName::ContentDisposition | HeaderName::ListId ) { HeaderValue::from(hdr.value) } else { HeaderValue::Empty } .into(), name: hdr.name.into(), base_offset: hdr.offset_field as u32, start: (hdr.offset_start - hdr.offset_field) as u16, end: (hdr.offset_end - hdr.offset_field) as u16, }) .collect(), flags, body, offset_header: part.offset_header as u32, offset_body: part.offset_body as u32, offset_end: part.offset_end as u32, }); } contents.push(MessageMetadataContents { html_body: message.html_body.into_iter().map(|c| c as u16).collect(), text_body: message.text_body.into_iter().map(|c| c as u16).collect(), attachments: message.attachments.into_iter().map(|c| c as u16).collect(), parts: parts.into_boxed_slice(), }); } MessageMetadata { blob_body_offset: contents.first().unwrap().root_part().offset_body, contents: contents.into_boxed_slice(), blob_hash: legacy.blob_hash, preview: legacy.preview.into_boxed_str(), raw_headers: legacy.raw_headers.into_boxed_slice(), rcvd_attach: (if legacy.has_attachments { MESSAGE_HAS_ATTACHMENT } else { 0 }) | (legacy.received_at & MESSAGE_RECEIVED_MASK), } } } pub struct Mailboxes(Vec); pub struct Keywords(Vec); impl Deserialize for Mailboxes { fn deserialize(bytes: &[u8]) -> trc::Result { let mut bytes = bytes.iter(); let len: usize = bytes .next_leb128() .ok_or_else(|| trc::StoreEvent::DataCorruption.caused_by(trc::location!()))?; let mut list = Vec::with_capacity(len); for _ in 0..len { list.push(UidMailbox { mailbox_id: bytes .next_leb128() .ok_or_else(|| trc::StoreEvent::DataCorruption.caused_by(trc::location!()))?, uid: bytes .next_leb128() .ok_or_else(|| trc::StoreEvent::DataCorruption.caused_by(trc::location!()))?, }); } Ok(Mailboxes(list)) } } impl Deserialize for Keywords { fn deserialize(bytes: &[u8]) -> trc::Result { let mut bytes = bytes.iter(); let len: usize = bytes .next_leb128() .ok_or_else(|| trc::StoreEvent::DataCorruption.caused_by(trc::location!()))?; let mut list = Vec::with_capacity(len); for _ in 0..len { list.push( deserialize_keyword(&mut bytes) .ok_or_else(|| trc::StoreEvent::DataCorruption.caused_by(trc::location!()))?, ); } Ok(Keywords(list)) } } fn deserialize_keyword(bytes: &mut std::slice::Iter<'_, u8>) -> Option { match bytes.next_leb128::()? { SEEN => Some(LegacyKeyword::Seen), DRAFT => Some(LegacyKeyword::Draft), FLAGGED => Some(LegacyKeyword::Flagged), ANSWERED => Some(LegacyKeyword::Answered), RECENT => Some(LegacyKeyword::Recent), IMPORTANT => Some(LegacyKeyword::Important), PHISHING => Some(LegacyKeyword::Phishing), JUNK => Some(LegacyKeyword::Junk), NOTJUNK => Some(LegacyKeyword::NotJunk), DELETED => Some(LegacyKeyword::Deleted), FORWARDED => Some(LegacyKeyword::Forwarded), MDN_SENT => Some(LegacyKeyword::MdnSent), other => { let len = other - 12; let mut keyword = Vec::with_capacity(len); for _ in 0..len { keyword.push(*bytes.next()?); } Some(LegacyKeyword::Other(String::from_utf8(keyword).ok()?)) } } } pub type LegacyMessagePartId = usize; #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct LegacyMessageMetadata<'x> { pub contents: LegacyMessageMetadataContents<'x>, pub blob_hash: BlobHash, pub size: usize, pub received_at: u64, pub preview: String, pub has_attachments: bool, pub raw_headers: Vec, } #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct LegacyMessageMetadataContents<'x> { pub html_body: Vec, pub text_body: Vec, pub attachments: Vec, pub parts: Vec>, } #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct LegacyMessageMetadataPart<'x> { pub headers: Vec>, pub is_encoding_problem: bool, pub body: LegacyMetadataPartType<'x>, pub encoding: Encoding, pub size: usize, pub offset_header: usize, pub offset_body: usize, pub offset_end: usize, } #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct LegacyHeader<'x> { pub name: HeaderName<'x>, pub value: LegacyHeaderValue<'x>, pub offset_field: usize, pub offset_start: usize, pub offset_end: usize, } #[derive(Debug, serde::Serialize, serde::Deserialize, Default)] pub enum LegacyHeaderValue<'x> { /// Address list or group Address(Address<'x>), /// String Text(Cow<'x, str>), /// List of strings TextList(Vec>), /// Datetime DateTime(DateTime), /// Content-Type or Content-Disposition header ContentType(LegacyContentType<'x>), /// Received header Received(Box>), #[default] Empty, } #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct LegacyContentType<'x> { pub c_type: Cow<'x, str>, pub c_subtype: Option>, pub attributes: Option, Cow<'x, str>)>>, } #[derive(Debug, serde::Serialize, serde::Deserialize)] pub enum LegacyMetadataPartType<'x> { Text, Html, Binary, InlineBinary, Message(LegacyMessageMetadataContents<'x>), Multipart(Vec), } impl From> for HeaderValue<'static> { fn from(value: LegacyHeaderValue<'_>) -> Self { match value { LegacyHeaderValue::Address(address) => HeaderValue::Address(address.into_owned()), LegacyHeaderValue::Text(cow) => HeaderValue::Text(cow.into_owned().into()), LegacyHeaderValue::TextList(cows) => HeaderValue::TextList( cows.into_iter() .map(|cow| cow.into_owned().into()) .collect(), ), LegacyHeaderValue::DateTime(date_time) => HeaderValue::DateTime(date_time), LegacyHeaderValue::ContentType(legacy_content_type) => { HeaderValue::ContentType(ContentType { c_type: legacy_content_type.c_type.into_owned().into(), c_subtype: legacy_content_type.c_subtype.map(|s| s.into_owned().into()), attributes: legacy_content_type.attributes.map(|attrs| { attrs .into_iter() .map(|(k, v)| Attribute { name: k.into_owned().into(), value: v.into_owned().into(), }) .collect() }), }) } LegacyHeaderValue::Received(received) => { HeaderValue::Received(Box::new(received.into_owned())) } LegacyHeaderValue::Empty => HeaderValue::Empty, } } } /*pub(crate) fn encode_message_id(message_id: &str) -> Vec { let mut msg_id = Vec::with_capacity(message_id.len() + 1); msg_id.extend_from_slice(message_id.as_bytes()); msg_id.push(0); msg_id }*/ ================================================ FILE: crates/migration/src/email_v2.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{email_v1::FIELD_MAILBOX_IDS, get_bitmap, v014::SUBSPACE_BITMAP_TAG}; use common::Server; use email::{ mailbox::{JUNK_ID, TRASH_ID, UidMailbox}, message::{ index::extractors::VisitTextArchived, ingest::ThreadInfo, metadata::{ MESSAGE_HAS_ATTACHMENT, MESSAGE_RECEIVED_MASK, MessageData, MessageMetadata, MessageMetadataContents, MessageMetadataPart, MetadataHeader, MetadataHeaderName, MetadataHeaderValue, MetadataPartType, PART_ENCODING_BASE64, PART_ENCODING_PROBLEM, PART_ENCODING_QP, PART_SIZE_MASK, }, }, }; use mail_parser::{Encoding, Header, parsers::fields::thread::thread_name}; use store::{ Serialize, SerializeInfallible, U32_LEN, U64_LEN, ValueKey, rand::{self, seq::SliceRandom}, write::{ AlignedBytes, AnyKey, Archive, Archiver, BatchBuilder, IndexPropertyClass, ValueClass, key::KeySerializer, }, }; use trc::AddContext; use types::{blob_hash::BlobHash, collection::Collection, field::EmailField, keyword::*}; use utils::cheeky_hash::CheekyHash; pub(crate) async fn migrate_emails_v014(server: &Server, account_id: u32) -> trc::Result { let tombstoned_ids = get_bitmap( server, AnyKey { subspace: SUBSPACE_BITMAP_TAG, key: KeySerializer::new(U64_LEN + U32_LEN + 1) .write(account_id) .write(u8::from(Collection::Email)) .write(FIELD_MAILBOX_IDS) .write_leb128(u32::MAX - 1) .finalize(), }, AnyKey { subspace: SUBSPACE_BITMAP_TAG, key: KeySerializer::new(U64_LEN + U32_LEN + 1) .write(account_id) .write(u8::from(Collection::Email)) .write(FIELD_MAILBOX_IDS) .write_leb128(u32::MAX - 1) .finalize(), }, ) .await .caused_by(trc::location!())? .unwrap_or_default(); let mut migrate = Vec::new(); server .archives( account_id, Collection::Email, &(), |document_id, archive| { match archive.deserialize_untrusted::() { Ok(metadata) => { migrate.push((document_id, metadata)); } Err(err) => { if archive.deserialize_untrusted::().is_err() { return Err(err .account_id(account_id) .document_id(document_id) .caused_by(trc::location!())); } } } Ok(true) }, ) .await .caused_by(trc::location!())?; migrate.shuffle(&mut rand::rng()); let num_emails = migrate.len(); for (document_id, legacy_data) in migrate { let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Email) .with_document(document_id); if !tombstoned_ids.contains(document_id) { let (size, metadata) = match server .store() .get_value::>(ValueKey::property( account_id, Collection::Email, document_id, EmailField::Metadata, )) .await? { Some(metadata) => match metadata.deserialize_untrusted::() { Ok(legacy) => (legacy.size, MessageMetadata::from(legacy)), Err(err) => match metadata.deserialize_untrusted::() { Ok(metadata) => (metadata.root_part().offset_end, metadata), Err(_) => { return Err(err .account_id(account_id) .document_id(document_id) .caused_by(trc::location!())); } }, }, None => { batch.clear(EmailField::Archive).clear(EmailField::Metadata); continue; } }; let data = MessageData { mailboxes: legacy_data.mailboxes.into_boxed_slice(), keywords: legacy_data.keywords.into_iter().map(Into::into).collect(), thread_id: legacy_data.thread_id, size, }; let mut message_ids = Vec::new(); let mut subject = ""; for header in &metadata.contents[0].parts[0].headers { match &header.name { MetadataHeaderName::MessageId => { header.value.visit_text(|id| { if !id.is_empty() { message_ids.push(CheekyHash::new(id.as_bytes())); } }); } MetadataHeaderName::InReplyTo | MetadataHeaderName::References | MetadataHeaderName::ResentMessageId => { header.value.visit_text(|id| { if !id.is_empty() { message_ids.push(CheekyHash::new(id.as_bytes())); } }); } MetadataHeaderName::Subject if subject.is_empty() => { subject = thread_name(match &header.value { MetadataHeaderValue::Text(text) => text.as_ref(), MetadataHeaderValue::TextList(list) if !list.is_empty() => { list.first().unwrap().as_ref() } _ => "", }); } _ => (), } } if data .mailboxes .iter() .any(|mailbox| mailbox.mailbox_id == TRASH_ID || mailbox.mailbox_id == JUNK_ID) { batch.set( ValueClass::Property(EmailField::DeletedAt.into()), (metadata.rcvd_attach & MESSAGE_RECEIVED_MASK).serialize(), ); } batch .set( ValueClass::IndexProperty(IndexPropertyClass::Hash { property: EmailField::Threading.into(), hash: CheekyHash::new(if !subject.is_empty() { subject } else { "!" }), }), ThreadInfo::serialize(data.thread_id, &message_ids), ) .set( EmailField::Archive, Archiver::new(data) .serialize() .caused_by(trc::location!())?, ) .set( EmailField::Metadata, Archiver::new(metadata) .serialize() .caused_by(trc::location!())?, ); } else { batch.clear(EmailField::Archive).clear(EmailField::Metadata); } server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } Ok(num_emails as u64) } #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Default)] pub struct LegacyMessageData { pub mailboxes: Vec, pub keywords: Vec, pub thread_id: u32, } #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] pub struct LegacyMessageMetadata<'x> { pub contents: Vec>, pub blob_hash: BlobHash, pub size: u32, pub received_at: u64, pub preview: String, pub has_attachments: bool, pub raw_headers: Vec, } impl<'x> From> for MessageMetadata { fn from(legacy: LegacyMessageMetadata<'x>) -> Self { MessageMetadata { blob_body_offset: legacy .contents .first() .unwrap() .parts .first() .unwrap() .offset_body, contents: legacy.contents.into_iter().map(Into::into).collect(), blob_hash: legacy.blob_hash, preview: legacy.preview.into_boxed_str(), raw_headers: legacy.raw_headers.into_boxed_slice(), rcvd_attach: (if legacy.has_attachments { MESSAGE_HAS_ATTACHMENT } else { 0 }) | (legacy.received_at & MESSAGE_RECEIVED_MASK), } } } #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] pub struct LegacyMessageMetadataContents<'x> { pub html_body: Vec, pub text_body: Vec, pub attachments: Vec, pub parts: Vec>, } impl<'x> From> for MessageMetadataContents { fn from(contents: LegacyMessageMetadataContents) -> Self { MessageMetadataContents { html_body: contents.html_body.into_boxed_slice(), text_body: contents.text_body.into_boxed_slice(), attachments: contents.attachments.into_boxed_slice(), parts: contents.parts.into_iter().map(Into::into).collect(), } } } #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] pub struct LegacyMessageMetadataPart<'x> { pub headers: Vec>, pub is_encoding_problem: bool, pub body: LegacyMetadataPartType, pub encoding: Encoding, pub size: u32, pub offset_header: u32, pub offset_body: u32, pub offset_end: u32, } impl<'x> From> for MessageMetadataPart { fn from(part: LegacyMessageMetadataPart<'x>) -> Self { let flags = match part.encoding { Encoding::None => 0, Encoding::QuotedPrintable => PART_ENCODING_QP, Encoding::Base64 => PART_ENCODING_BASE64, } | (if part.is_encoding_problem { PART_ENCODING_PROBLEM } else { 0 }) | (part.size & PART_SIZE_MASK); MessageMetadataPart { headers: part .headers .into_iter() .map(|hdr| MetadataHeader { value: hdr.value.into(), name: hdr.name.into(), base_offset: hdr.offset_field, start: (hdr.offset_start - hdr.offset_field) as u16, end: (hdr.offset_end - hdr.offset_field) as u16, }) .collect(), flags, body: part.body.into(), offset_header: part.offset_header, offset_body: part.offset_body, offset_end: part.offset_end, } } } #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] pub enum LegacyMetadataPartType { Text, Html, Binary, InlineBinary, Message(u16), Multipart(Vec), } impl From for MetadataPartType { fn from(value: LegacyMetadataPartType) -> Self { match value { LegacyMetadataPartType::Text => MetadataPartType::Text, LegacyMetadataPartType::Html => MetadataPartType::Html, LegacyMetadataPartType::Binary => MetadataPartType::Binary, LegacyMetadataPartType::InlineBinary => MetadataPartType::InlineBinary, LegacyMetadataPartType::Message(id) => MetadataPartType::Message(id), LegacyMetadataPartType::Multipart(children) => { MetadataPartType::Multipart(children.into_boxed_slice()) } } } } #[derive( rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Clone, PartialEq, Eq, Hash, Default, PartialOrd, Ord, serde::Serialize, )] #[serde(untagged)] #[rkyv(derive(PartialEq), compare(PartialEq))] pub enum LegacyKeyword { #[serde(rename(serialize = "$seen"))] Seen, #[serde(rename(serialize = "$draft"))] Draft, #[serde(rename(serialize = "$flagged"))] Flagged, #[serde(rename(serialize = "$answered"))] Answered, #[default] #[serde(rename(serialize = "$recent"))] Recent, #[serde(rename(serialize = "$important"))] Important, #[serde(rename(serialize = "$phishing"))] Phishing, #[serde(rename(serialize = "$junk"))] Junk, #[serde(rename(serialize = "$notjunk"))] NotJunk, #[serde(rename(serialize = "$deleted"))] Deleted, #[serde(rename(serialize = "$forwarded"))] Forwarded, #[serde(rename(serialize = "$mdnsent"))] MdnSent, Other(String), } impl From for Keyword { fn from(kw: LegacyKeyword) -> Self { match kw { LegacyKeyword::Seen => Keyword::Seen, LegacyKeyword::Draft => Keyword::Draft, LegacyKeyword::Flagged => Keyword::Flagged, LegacyKeyword::Answered => Keyword::Answered, LegacyKeyword::Recent => Keyword::Recent, LegacyKeyword::Important => Keyword::Important, LegacyKeyword::Phishing => Keyword::Phishing, LegacyKeyword::Junk => Keyword::Junk, LegacyKeyword::NotJunk => Keyword::NotJunk, LegacyKeyword::Deleted => Keyword::Deleted, LegacyKeyword::Forwarded => Keyword::Forwarded, LegacyKeyword::MdnSent => Keyword::MdnSent, LegacyKeyword::Other(s) => Keyword::Other(s.into_boxed_str()), } } } ================================================ FILE: crates/migration/src/encryption_v1.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::Server; use email::message::crypto::EncryptionParams; use store::{ Deserialize, Serialize, ValueKey, write::{AlignedBytes, Archive, Archiver, BatchBuilder, ValueClass}, }; use trc::AddContext; use types::{collection::Collection, field::PrincipalField}; use crate::encryption_v2::LegacyEncryptionParams; pub(crate) async fn migrate_encryption_params_v011( server: &Server, account_id: u32, ) -> trc::Result { match server .store() .get_value::(ValueKey { account_id, collection: Collection::Principal.into(), document_id: 0, class: ValueClass::from(PrincipalField::EncryptionKeys), }) .await { Ok(Some(legacy)) => { let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Principal) .with_document(0) .set( PrincipalField::EncryptionKeys, Archiver::new(EncryptionParams::from(legacy.0)) .serialize() .caused_by(trc::location!())?, ); server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; return Ok(1); } Ok(None) => (), Err(err) => { if server .store() .get_value::>(ValueKey { account_id, collection: Collection::Principal.into(), document_id: 0, class: ValueClass::from(PrincipalField::EncryptionKeys), }) .await .is_err() { return Err(err.account_id(account_id).caused_by(trc::location!())); } } } Ok(0) } struct VeryOldLegacyEncryptionParams(LegacyEncryptionParams); impl Deserialize for VeryOldLegacyEncryptionParams { fn deserialize(bytes: &[u8]) -> trc::Result { let version = *bytes .first() .ok_or_else(|| trc::StoreEvent::DataCorruption.caused_by(trc::location!()))?; match version { 1 if bytes.len() > 1 => bincode::deserialize(&bytes[1..]) .map(VeryOldLegacyEncryptionParams) .map_err(|err| { trc::EventType::Store(trc::StoreEvent::DeserializeError) .reason(err) .caused_by(trc::location!()) }), _ => Err(trc::StoreEvent::DeserializeError .into_err() .caused_by(trc::location!()) .ctx(trc::Key::Value, version as u64)), } } } ================================================ FILE: crates/migration/src/encryption_v2.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::Server; use email::message::crypto::{ ENCRYPT_ALGO_AES128, ENCRYPT_ALGO_AES256, ENCRYPT_METHOD_PGP, ENCRYPT_METHOD_SMIME, EncryptionParams, }; use store::{ Serialize, ValueKey, write::{AlignedBytes, Archive, Archiver, BatchBuilder, ValueClass}, }; use trc::AddContext; use types::{collection::Collection, field::PrincipalField}; #[derive( rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, )] pub enum EncryptionMethod { PGP, SMIME, } #[derive( rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Clone, Copy, serde::Serialize, serde::Deserialize, )] #[rkyv(derive(Clone, Copy))] pub enum Algorithm { Aes128, Aes256, } #[derive( Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, serde::Serialize, serde::Deserialize, )] pub struct LegacyEncryptionParams { pub method: EncryptionMethod, pub algo: Algorithm, pub certs: Vec>, } pub(crate) async fn migrate_encryption_params_v014( server: &Server, account_id: u32, ) -> trc::Result { let Some(params) = server .store() .get_value::>(ValueKey { account_id, collection: Collection::Principal.into(), document_id: 0, class: ValueClass::from(PrincipalField::EncryptionKeys), }) .await .caused_by(trc::location!())? else { return Ok(0); }; match params.deserialize_untrusted::() { Ok(legacy) => { let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Principal) .with_document(0) .set( PrincipalField::EncryptionKeys, Archiver::new(EncryptionParams::from(legacy)) .serialize() .caused_by(trc::location!())?, ); server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; Ok(1) } Err(err) => { if params.deserialize_untrusted::().is_err() { return Err(err.account_id(account_id).caused_by(trc::location!())); } Ok(0) } } } impl From for EncryptionParams { fn from(legacy: LegacyEncryptionParams) -> Self { EncryptionParams { flags: match legacy.method { EncryptionMethod::PGP => ENCRYPT_METHOD_PGP, EncryptionMethod::SMIME => ENCRYPT_METHOD_SMIME, } | match legacy.algo { Algorithm::Aes128 => ENCRYPT_ALGO_AES128, Algorithm::Aes256 => ENCRYPT_ALGO_AES256, }, certs: legacy .certs .into_iter() .map(|c| c.into_boxed_slice()) .collect(), } } } ================================================ FILE: crates/migration/src/event_v1.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{DavName, Server}; use groupware::calendar::{AlarmDelta, CalendarEvent, CalendarEventData, ComponentTimeRange}; use store::{ Serialize, ValueKey, rand::{self, seq::SliceRandom}, write::{AlignedBytes, Archive, Archiver, BatchBuilder, serialize::rkyv_deserialize}, }; use trc::AddContext; use types::{collection::Collection, dead_property::DeadProperty, field::Field}; use crate::{event_v2::migrate_icalendar_v02, get_document_ids}; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct CalendarEventV1 { pub names: Vec, pub display_name: Option, pub data: CalendarEventDataV1, pub user_properties: Vec, pub flags: u16, pub dead_properties: DeadProperty, pub size: u32, pub created: i64, pub modified: i64, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct UserProperties { pub account_id: u32, pub properties: calcard_v01::icalendar::ICalendar, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct CalendarEventDataV1 { pub event: calcard_v01::icalendar::ICalendar, pub time_ranges: Box<[ComponentTimeRange]>, pub alarms: Box<[AlarmV1]>, pub base_offset: i64, pub base_time_utc: u32, pub duration: u32, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] #[rkyv(compare(PartialEq), derive(Debug))] pub struct AlarmV1 { pub comp_id: u16, pub alarms: Box<[AlarmDelta]>, } pub(crate) async fn migrate_calendar_events_v012(server: &Server) -> trc::Result<()> { // Obtain email ids let account_ids = get_document_ids(server, u32::MAX, Collection::Principal) .await .caused_by(trc::location!())? .unwrap_or_default(); let num_accounts = account_ids.len(); if num_accounts == 0 { return Ok(()); } let mut account_ids = account_ids.into_iter().collect::>(); account_ids.shuffle(&mut rand::rng()); for account_id in account_ids { let document_ids = get_document_ids(server, account_id, Collection::CalendarEvent) .await .caused_by(trc::location!())? .unwrap_or_default(); if document_ids.is_empty() { continue; } let mut num_migrated = 0; for document_id in document_ids.iter() { let Some(archive) = server .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, )) .await .caused_by(trc::location!())? else { continue; }; match archive.unarchive_untrusted::() { Ok(event) => { let event = rkyv_deserialize::<_, CalendarEventV1>(event).unwrap(); let mut next_email_alarm = None; let new_event = CalendarEvent { names: event.names, display_name: event.display_name, data: CalendarEventData::new( migrate_icalendar_v02(event.data.event), calcard_latest::common::timezone::Tz::Floating, server.core.groupware.max_ical_instances, &mut next_email_alarm, ), preferences: Default::default(), flags: event.flags, dead_properties: event.dead_properties, size: event.size, created: event.created, modified: event.modified, schedule_tag: None, }; let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::CalendarEvent) .with_document(document_id) .set( Field::ARCHIVE, Archiver::new(new_event) .serialize() .caused_by(trc::location!())?, ); if let Some(next_email_alarm) = next_email_alarm { next_email_alarm.write_task(&mut batch); } server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; num_migrated += 1; } Err(err) => { if let Err(err_) = archive.unarchive_untrusted::() { trc::error!(err_.caused_by(trc::location!())); return Err(err.caused_by(trc::location!())); } } } } if num_migrated > 0 { trc::event!( Server(trc::ServerEvent::Startup), Details = format!("Migrated {num_migrated} Calendar Events for account {account_id}") ); } } Ok(()) } ================================================ FILE: crates/migration/src/event_v2.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{DavName, Server}; use groupware::calendar::{ Alarm, CalendarEvent, CalendarEventData, CalendarEventNotification, ComponentTimeRange, }; use store::{ Serialize, ValueKey, write::{AlignedBytes, Archive, Archiver, BatchBuilder, serialize::rkyv_deserialize}, }; use trc::AddContext; use types::{collection::Collection, dead_property::DeadProperty, field::Field}; use crate::get_document_ids; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct CalendarEventV2 { pub names: Vec, pub display_name: Option, pub data: CalendarEventDataV2, pub user_properties: Vec, pub flags: u16, pub dead_properties: DeadProperty, pub size: u32, pub created: i64, pub modified: i64, pub schedule_tag: Option, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct UserPropertiesV2 { pub account_id: u32, pub properties: calcard_v01::icalendar::ICalendar, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct CalendarEventDataV2 { pub event: calcard_v01::icalendar::ICalendar, pub time_ranges: Box<[ComponentTimeRange]>, pub alarms: Box<[Alarm]>, pub base_offset: i64, pub base_time_utc: u32, pub duration: u32, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] pub struct CalendarEventNotificationV2 { pub itip: calcard_v01::icalendar::ICalendar, pub event_id: Option, pub flags: u16, pub size: u32, pub created: i64, pub modified: i64, } pub(crate) async fn migrate_calendar_events_v013( server: &Server, account_id: u32, ) -> trc::Result { let document_ids = get_document_ids(server, account_id, Collection::CalendarEvent) .await .caused_by(trc::location!())? .unwrap_or_default(); let mut num_migrated = 0; for document_id in document_ids.iter() { let Some(archive) = server .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, )) .await .caused_by(trc::location!())? else { continue; }; match archive.unarchive_untrusted::() { Ok(event) => { let event = rkyv_deserialize::<_, CalendarEventV2>(event).unwrap(); let new_event = CalendarEvent { names: event.names, display_name: event.display_name, data: CalendarEventData { event: migrate_icalendar_v02(event.data.event), time_ranges: event.data.time_ranges, alarms: event.data.alarms, base_offset: event.data.base_offset, base_time_utc: event.data.base_time_utc, duration: event.data.duration, }, preferences: Default::default(), flags: event.flags, dead_properties: event.dead_properties, size: event.size, created: event.created, modified: event.modified, schedule_tag: None, }; let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::CalendarEvent) .with_document(document_id) .set( Field::ARCHIVE, Archiver::new(new_event) .serialize() .caused_by(trc::location!())?, ); server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; num_migrated += 1; } Err(err) => { if let Err(err_) = archive.unarchive_untrusted::() { trc::error!(err_.caused_by(trc::location!())); return Err(err.caused_by(trc::location!())); } } } } Ok(num_migrated) } pub(crate) async fn migrate_calendar_scheduling_v013( server: &Server, account_id: u32, ) -> trc::Result { let document_ids = get_document_ids(server, account_id, Collection::CalendarEventNotification) .await .caused_by(trc::location!())? .unwrap_or_default(); let mut num_migrated = 0; for document_id in document_ids.iter() { let Some(archive) = server .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEventNotification, document_id, )) .await .caused_by(trc::location!())? else { continue; }; match archive.unarchive_untrusted::() { Ok(event) => { let event = rkyv_deserialize::<_, CalendarEventNotificationV2>(event).unwrap(); let new_event = CalendarEventNotification { event: migrate_icalendar_v02(event.itip), event_id: event.event_id, changed_by: Default::default(), flags: 0, size: event.size, created: event.created, modified: event.modified, }; let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::CalendarEventNotification) .with_document(document_id) .set( Field::ARCHIVE, Archiver::new(new_event) .serialize() .caused_by(trc::location!())?, ); server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; num_migrated += 1; } Err(err) => { if let Err(err_) = archive.unarchive_untrusted::() { trc::error!(err_.caused_by(trc::location!())); return Err(err.caused_by(trc::location!())); } } } } Ok(num_migrated) } pub(crate) fn migrate_icalendar_v02( ical: calcard_v01::icalendar::ICalendar, ) -> calcard_latest::icalendar::ICalendar { calcard_latest::icalendar::ICalendar::parse(ical.to_string()).unwrap_or_default() } ================================================ FILE: crates/migration/src/identity_v1.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::object::Object; use crate::{ get_document_ids, object::{FromLegacy, Property, Value}, }; use common::Server; use email::identity::{EmailAddress, Identity}; use store::{ Serialize, ValueKey, write::{AlignedBytes, Archive, Archiver, BatchBuilder, ValueClass}, }; use trc::AddContext; use types::{collection::Collection, field::Field}; pub(crate) async fn migrate_identities_v011(server: &Server, account_id: u32) -> trc::Result { // Obtain identity ids let identity_ids = get_document_ids(server, account_id, Collection::Identity) .await .caused_by(trc::location!())? .unwrap_or_default(); let num_identities = identity_ids.len(); if num_identities == 0 { return Ok(0); } let mut did_migrate = false; for identity_id in &identity_ids { match server .store() .get_value::>(ValueKey { account_id, collection: Collection::Identity.into(), document_id: identity_id, class: ValueClass::Property(Field::ARCHIVE.into()), }) .await { Ok(Some(legacy)) => { let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Identity) .with_document(identity_id) .set( Field::ARCHIVE, Archiver::new(Identity::from_legacy(legacy)) .serialize() .caused_by(trc::location!())?, ); did_migrate = true; server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } Ok(None) => (), Err(err) => { if server .store() .get_value::>(ValueKey { account_id, collection: Collection::Identity.into(), document_id: identity_id, class: ValueClass::Property(Field::ARCHIVE.into()), }) .await .is_err() { return Err(err .account_id(account_id) .document_id(identity_id) .caused_by(trc::location!())); } } } } // Increment document id counter if did_migrate { server .store() .assign_document_ids( account_id, Collection::Identity, identity_ids .max() .map(|id| id as u64) .unwrap_or(num_identities) + 1, ) .await .caused_by(trc::location!())?; Ok(num_identities) } else { Ok(0) } } impl FromLegacy for Identity { fn from_legacy(legacy: Object) -> Self { Identity { name: legacy .get(&Property::Name) .as_string() .unwrap_or_default() .to_string(), email: legacy .get(&Property::Email) .as_string() .unwrap_or_default() .to_string(), reply_to: convert_email_addresses(legacy.get(&Property::ReplyTo)), bcc: convert_email_addresses(legacy.get(&Property::Bcc)), text_signature: legacy .get(&Property::TextSignature) .as_string() .unwrap_or_default() .to_string(), html_signature: legacy .get(&Property::HtmlSignature) .as_string() .unwrap_or_default() .to_string(), } } } fn convert_email_addresses(value: &Value) -> Option> { if let Value::List(value) = value { let mut addrs = Vec::with_capacity(value.len()); for addr in value { if let Value::Object(obj) = addr { let mut addr = EmailAddress { name: None, email: String::new(), }; for (key, value) in &obj.properties { match (key, value) { (Property::Email, Value::Text(value)) => { addr.email = value.to_string(); } (Property::Name, Value::Text(value)) => { addr.name = Some(value.to_string()); } _ => { break; } } } if !addr.email.is_empty() { addrs.push(addr); } } } if !addrs.is_empty() { Some(addrs) } else { None } } else { None } } ================================================ FILE: crates/migration/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ blob::migrate_blobs_v014, queue_v1::{migrate_queue_v011, migrate_queue_v012}, queue_v2::migrate_queue_v014, v011::migrate_v0_11, v012::migrate_v0_12, v013::migrate_v0_13, v014::{SUBSPACE_BITMAP_ID, migrate_principal_v0_14, migrate_v0_14}, }; use common::{DATABASE_SCHEMA_VERSION, Server, manager::boot::DEFAULT_SETTINGS}; use std::time::Duration; use store::{ Deserialize, IterateParams, SUBSPACE_PROPERTY, SUBSPACE_QUEUE_MESSAGE, SUBSPACE_REPORT_IN, SUBSPACE_REPORT_OUT, SUBSPACE_SETTINGS, SerializeInfallible, U32_LEN, Value, ValueKey, dispatch::DocumentSet, roaring::RoaringBitmap, write::{ AnyClass, AnyKey, BatchBuilder, ValueClass, key::{DeserializeBigEndian, KeySerializer}, }, }; use trc::AddContext; use types::collection::Collection; pub mod addressbook_v2; pub mod blob; pub mod calendar_v2; pub mod changelog; pub mod contact_v2; pub mod email_v1; pub mod email_v2; pub mod encryption_v1; pub mod encryption_v2; pub mod event_v1; pub mod event_v2; pub mod identity_v1; pub mod mailbox; pub mod object; pub mod principal_v1; pub mod principal_v2; pub mod push_v1; pub mod push_v2; pub mod queue_v1; pub mod queue_v2; pub mod report; pub mod sieve_v1; pub mod sieve_v2; pub mod submission; pub mod tasks_v1; pub mod tasks_v2; pub mod threads; pub mod v011; pub mod v012; pub mod v013; pub mod v014; const LOCK_WAIT_TIME_ACCOUNT: u64 = 3 * 60; const LOCK_WAIT_TIME_CORE: u64 = 5 * 60; const LOCK_RETRY_TIME: Duration = Duration::from_secs(30); pub async fn try_migrate(server: &Server) -> trc::Result<()> { for var in [ "FORCE_MIGRATE_QUEUE", "FORCE_MIGRATE_BLOBS", "FORCE_MIGRATE_ACCOUNT", "FORCE_MIGRATE", ] { let Some(version) = std::env::var(var).ok().and_then(|s| s.parse::().ok()) else { continue; }; match var { "FORCE_MIGRATE_QUEUE" => match version { 1 => { migrate_queue_v011(server) .await .caused_by(trc::location!())?; } 2 => { migrate_queue_v012(server) .await .caused_by(trc::location!())?; } 4 => { migrate_queue_v014(server) .await .caused_by(trc::location!())?; } _ => { panic!("Unknown migration queue version: {version}"); } }, "FORCE_MIGRATE_BLOBS" => { migrate_blobs_v014(server) .await .caused_by(trc::location!())?; } "FORCE_MIGRATE" => match version { 1 => { migrate_v0_12(server, true) .await .caused_by(trc::location!())?; migrate_v0_13(server).await.caused_by(trc::location!())?; migrate_v0_14(server).await.caused_by(trc::location!())?; } 2 => { migrate_v0_12(server, false) .await .caused_by(trc::location!())?; migrate_v0_13(server).await.caused_by(trc::location!())?; migrate_v0_14(server).await.caused_by(trc::location!())?; } 3 => { migrate_v0_13(server).await.caused_by(trc::location!())?; migrate_v0_14(server).await.caused_by(trc::location!())?; } 4 => { migrate_v0_14(server).await.caused_by(trc::location!())?; } _ => { panic!("Unknown migration version: {version}"); } }, "FORCE_MIGRATE_ACCOUNT" => { migrate_principal_v0_14(server, version) .await .caused_by(trc::location!())?; } _ => unreachable!(), } return Ok(()); } let add_v013_config = match server .store() .get_value::(AnyKey { subspace: SUBSPACE_PROPERTY, key: vec![0u8], }) .await .caused_by(trc::location!())? { Some(DATABASE_SCHEMA_VERSION) => { return Ok(()); } Some(1) => { migrate_v0_12(server, true) .await .caused_by(trc::location!())?; migrate_v0_13(server).await.caused_by(trc::location!())?; migrate_v0_14(server).await.caused_by(trc::location!())?; true } Some(2) => { migrate_v0_12(server, false) .await .caused_by(trc::location!())?; migrate_v0_13(server).await.caused_by(trc::location!())?; migrate_v0_14(server).await.caused_by(trc::location!())?; true } Some(3) => { migrate_v0_13(server).await.caused_by(trc::location!())?; migrate_v0_14(server).await.caused_by(trc::location!())?; false } Some(4) => { migrate_v0_14(server).await.caused_by(trc::location!())?; false } Some(version) => { panic!( "Unknown database schema version, expected {} or below, found {}", DATABASE_SCHEMA_VERSION, version ); } _ => { if !is_new_install(server).await.caused_by(trc::location!())? { migrate_v0_11(server).await.caused_by(trc::location!())?; true } else { false } } }; let mut batch = BatchBuilder::new(); batch.set( ValueClass::Any(AnyClass { subspace: SUBSPACE_PROPERTY, key: vec![0u8], }), DATABASE_SCHEMA_VERSION.serialize(), ); if add_v013_config { for (key, value) in DEFAULT_SETTINGS { if key .strip_prefix("queue.") .is_some_and(|s| !s.starts_with("limiter.") && !s.starts_with("quota.")) { batch.set( ValueClass::Any(AnyClass { subspace: SUBSPACE_SETTINGS, key: key.as_bytes().to_vec(), }), value.as_bytes().to_vec(), ); } } } server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; Ok(()) } async fn is_new_install(server: &Server) -> trc::Result { for subspace in [ SUBSPACE_QUEUE_MESSAGE, SUBSPACE_REPORT_IN, SUBSPACE_REPORT_OUT, SUBSPACE_PROPERTY, ] { let mut has_data = false; server .store() .iterate( IterateParams::new( AnyKey { subspace, key: vec![0u8], }, AnyKey { subspace, key: vec![u8::MAX; 16], }, ) .no_values(), |_, _| { has_data = true; Ok(false) }, ) .await .caused_by(trc::location!())?; if has_data { return Ok(false); } } Ok(true) } async fn get_properties( server: &Server, account_id: u32, collection: Collection, iterate: &I, property: u8, ) -> trc::Result> where I: DocumentSet + Send + Sync, U: Deserialize + 'static, { let collection: u8 = collection.into(); let expected_results = iterate.len(); let mut results = Vec::with_capacity(expected_results); server .core .storage .data .iterate( IterateParams::new( ValueKey { account_id, collection, document_id: iterate.min(), class: ValueClass::Property(property), }, ValueKey { account_id, collection, document_id: iterate.max(), class: ValueClass::Property(property), }, ), |key, value| { let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; if iterate.contains(document_id) { results.push((document_id, U::deserialize(value)?)); Ok(expected_results == 0 || results.len() < expected_results) } else { Ok(true) } }, ) .await .add_context(|err| { err.caused_by(trc::location!()) .account_id(account_id) .collection(collection) .id(property.to_string()) }) .map(|_| results) } pub async fn get_document_ids( server: &Server, account_id: u32, collection: Collection, ) -> trc::Result> { let collection: u8 = collection.into(); get_bitmap( server, AnyKey { subspace: SUBSPACE_BITMAP_ID, key: KeySerializer::new(U32_LEN + 1) .write(account_id) .write(collection) .write(0u32) .finalize(), }, AnyKey { subspace: SUBSPACE_BITMAP_ID, key: KeySerializer::new(U32_LEN + 1) .write(account_id) .write(collection) .write(u32::MAX) .finalize(), }, ) .await } pub async fn get_bitmap( server: &Server, from_key: AnyKey>, to_key: AnyKey>, ) -> trc::Result> { let mut results = RoaringBitmap::new(); server .core .storage .data .iterate( IterateParams::new(from_key, to_key).no_values(), |key, _| { results.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); Ok(true) }, ) .await .caused_by(trc::location!()) .map(|_| { if !results.is_empty() { Some(results) } else { None } }) } pub struct LegacyBincode { pub inner: T, } impl LegacyBincode { pub fn new(inner: T) -> Self { Self { inner } } } impl From> for LegacyBincode { fn from(_: Value<'static>) -> Self { unreachable!("From Value called on LegacyBincode") } } impl Deserialize for LegacyBincode { fn deserialize(bytes: &[u8]) -> trc::Result { lz4_flex::decompress_size_prepended(bytes) .map_err(|err| { trc::StoreEvent::DecompressError .ctx(trc::Key::Value, bytes) .caused_by(trc::location!()) .reason(err) }) .and_then(|result| { bincode::deserialize(&result).map_err(|err| { trc::StoreEvent::DataCorruption .ctx(trc::Key::Value, bytes) .caused_by(trc::location!()) .reason(err) }) }) .map(|inner| Self { inner }) } } ================================================ FILE: crates/migration/src/mailbox.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::object::Object; use crate::{ get_document_ids, object::{FromLegacy, Property, Value}, v014::{SUBSPACE_BITMAP_TAG, SUBSPACE_BITMAP_TEXT}, }; use common::Server; use email::mailbox::Mailbox; use store::{ SUBSPACE_INDEXES, Serialize, U64_LEN, ValueKey, rand, write::{ AlignedBytes, AnyKey, Archive, Archiver, BatchBuilder, ValueClass, key::KeySerializer, }, }; use trc::AddContext; use types::{collection::Collection, field::Field, special_use::SpecialUse}; use utils::config::utils::ParseValue; pub(crate) async fn migrate_mailboxes(server: &Server, account_id: u32) -> trc::Result { // Obtain email ids let mailbox_ids = get_document_ids(server, account_id, Collection::Mailbox) .await .caused_by(trc::location!())? .unwrap_or_default(); let num_mailboxes = mailbox_ids.len(); if num_mailboxes == 0 { return Ok(0); } let mut did_migrate = false; for mailbox_id in &mailbox_ids { match server .store() .get_value::>(ValueKey { account_id, collection: Collection::Mailbox.into(), document_id: mailbox_id, class: ValueClass::Property(Field::ARCHIVE.into()), }) .await { Ok(Some(legacy)) => { let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Mailbox) .with_document(mailbox_id) .set( Field::ARCHIVE, Archiver::new(Mailbox::from_legacy(legacy)) .serialize() .caused_by(trc::location!())?, ); did_migrate = true; server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } Ok(None) => (), Err(err) => { if server .store() .get_value::>(ValueKey { account_id, collection: Collection::Mailbox.into(), document_id: mailbox_id, class: ValueClass::Property(Field::ARCHIVE.into()), }) .await .is_err() { return Err(err .account_id(account_id) .document_id(mailbox_id) .caused_by(trc::location!())); } } } } // Delete indexes for subspace in [SUBSPACE_INDEXES, SUBSPACE_BITMAP_TAG, SUBSPACE_BITMAP_TEXT] { server .store() .delete_range( AnyKey { subspace, key: KeySerializer::new(U64_LEN) .write(account_id) .write(u8::from(Collection::Mailbox)) .finalize(), }, AnyKey { subspace, key: KeySerializer::new(U64_LEN) .write(account_id) .write(u8::from(Collection::Mailbox)) .write(&[u8::MAX; 16][..]) .finalize(), }, ) .await .caused_by(trc::location!())?; } // Increment document id counter if did_migrate { server .store() .assign_document_ids( account_id, Collection::Mailbox, mailbox_ids .max() .map(|id| id as u64) .unwrap_or(num_mailboxes) + 1, ) .await .caused_by(trc::location!())?; Ok(num_mailboxes) } else { Ok(0) } } impl FromLegacy for Mailbox { fn from_legacy(legacy: Object) -> Self { Mailbox { name: legacy .get(&Property::Name) .as_string() .unwrap_or_default() .to_string(), role: legacy .get(&Property::Role) .as_string() .and_then(|r| SpecialUse::parse_value(r).ok()) .unwrap_or(SpecialUse::None), parent_id: legacy .get(&Property::ParentId) .as_uint() .unwrap_or_default() as u32, sort_order: legacy.get(&Property::SortOrder).as_uint().map(|s| s as u32), uid_validity: rand::random(), subscribers: legacy .get(&Property::IsSubscribed) .as_list() .map(|s| s.as_slice()) .unwrap_or_default() .iter() .filter_map(|s| s.as_uint()) .map(|s| s as u32) .collect(), acls: legacy .get(&Property::Acl) .as_acl() .cloned() .unwrap_or_default(), } } } ================================================ FILE: crates/migration/src/object.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::slice::Iter; use store::{Deserialize, U64_LEN}; use types::{acl::AclGrant, blob::BlobId, id::Id, keyword::*}; use utils::{ codec::leb128::Leb128Iterator, map::{bitmap::Bitmap, vec_map::VecMap}, }; #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct Object { pub properties: VecMap, } #[derive(Debug, PartialEq, Eq, Hash, Clone)] pub enum Property { Acl, Aliases, Attachments, Bcc, BlobId, BodyStructure, BodyValues, Capabilities, Cc, Charset, Cid, DeliveryStatus, Description, DeviceClientId, Disposition, DsnBlobIds, Email, EmailId, EmailIds, Envelope, Expires, From, FromDate, HasAttachment, Headers, HtmlBody, HtmlSignature, Id, IdentityId, InReplyTo, IsActive, IsEnabled, IsSubscribed, Keys, Keywords, Language, Location, MailboxIds, MayDelete, MdnBlobIds, Members, MessageId, MyRights, Name, ParentId, PartId, Picture, Preview, Quota, ReceivedAt, References, ReplyTo, Role, Secret, SendAt, Sender, SentAt, Size, SortOrder, Subject, SubParts, TextBody, TextSignature, ThreadId, Timezone, To, ToDate, TotalEmails, TotalThreads, Type, Types, UndoStatus, UnreadEmails, UnreadThreads, Url, VerificationCode, Addresses, P256dh, Auth, Value, SmtpReply, Delivered, Displayed, MailFrom, RcptTo, Parameters, IsEncodingProblem, IsTruncated, MayReadItems, MayAddItems, MayRemoveItems, MaySetSeen, MaySetKeywords, MayCreateChild, MayRename, MaySubmit, ResourceType, Used, HardLimit, WarnLimit, SoftLimit, Scope, _T(String), } impl Object { pub fn with_capacity(capacity: usize) -> Self { Self { properties: VecMap::with_capacity(capacity), } } pub fn set(&mut self, property: Property, value: impl Into) -> bool { self.properties.set(property, value.into()) } pub fn append(&mut self, property: Property, value: impl Into) { self.properties.append(property, value.into()); } pub fn with_property(mut self, property: Property, value: impl Into) -> Self { self.properties.append(property, value.into()); self } pub fn remove(&mut self, property: &Property) -> Value { self.properties.remove(property).unwrap_or(Value::Null) } pub fn get(&self, property: &Property) -> &Value { self.properties.get(property).unwrap_or(&Value::Null) } } #[derive(Debug, Default, Clone, PartialEq, Eq)] pub enum Value { Text(String), UnsignedInt(u64), Bool(bool), Id(Id), Date(UTCDate), BlobId(BlobId), Keyword(Keyword), List(Vec), Object(Object), Acl(Vec), Blob(Vec), #[default] Null, } #[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] pub struct UTCDate { pub year: u16, pub month: u8, pub day: u8, pub hour: u8, pub minute: u8, pub second: u8, pub tz_before_gmt: bool, pub tz_hour: u8, pub tz_minute: u8, } const TEXT: u8 = 0; const UNSIGNED_INT: u8 = 1; const BOOL_TRUE: u8 = 2; const BOOL_FALSE: u8 = 3; const ID: u8 = 4; const DATE: u8 = 5; const BLOB_ID: u8 = 6; const BLOB: u8 = 7; const KEYWORD: u8 = 8; const LIST: u8 = 9; const OBJECT: u8 = 10; const ACL: u8 = 11; const NULL: u8 = 12; pub trait DeserializeFrom: Sized { fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option; } impl Deserialize for Object { fn deserialize(bytes: &[u8]) -> trc::Result { Object::deserialize_from(&mut bytes.iter()).ok_or_else(|| { trc::StoreEvent::DataCorruption .caused_by(trc::location!()) .ctx(trc::Key::Value, bytes) }) } } impl DeserializeFrom for AclGrant { fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { let account_id = bytes.next_leb128()?; let mut grants = [0u8; U64_LEN]; for byte in grants.iter_mut() { *byte = *bytes.next()?; } Some(Self { account_id, grants: Bitmap::from(u64::from_be_bytes(grants)), }) } } impl DeserializeFrom for Object { fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option> { let len = bytes.next_leb128()?; let mut properties = VecMap::with_capacity(len); for _ in 0..len { let key = Property::deserialize_from(bytes)?; let value = Value::deserialize_from(bytes)?; properties.append(key, value); } Some(Object { properties }) } } impl DeserializeFrom for Value { fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { match *bytes.next()? { TEXT => Some(Value::Text(String::deserialize_from(bytes)?)), UNSIGNED_INT => Some(Value::UnsignedInt(bytes.next_leb128()?)), BOOL_TRUE => Some(Value::Bool(true)), BOOL_FALSE => Some(Value::Bool(false)), ID => Some(Value::Id(Id::new(bytes.next_leb128()?))), DATE => Some(Value::Date(UTCDate::from_timestamp( bytes.next_leb128::()? as i64, ))), BLOB_ID => Some(Value::BlobId(BlobId::deserialize_from(bytes)?)), KEYWORD => Some(Value::Keyword(Keyword::deserialize_from(bytes)?)), LIST => { let len = bytes.next_leb128()?; let mut items = Vec::with_capacity(len); for _ in 0..len { items.push(Value::deserialize_from(bytes)?); } Some(Value::List(items)) } OBJECT => Some(Value::Object(Object::deserialize_from(bytes)?)), BLOB => Some(Value::Blob(Vec::deserialize_from(bytes)?)), ACL => { let len = bytes.next_leb128()?; let mut items = Vec::with_capacity(len); for _ in 0..len { items.push(AclGrant::deserialize_from(bytes)?); } Some(Value::Acl(items)) } NULL => Some(Value::Null), _ => None, } } } impl DeserializeFrom for u32 { fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { bytes.next_leb128() } } impl DeserializeFrom for u64 { fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { bytes.next_leb128() } } impl DeserializeFrom for String { fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { >::deserialize_from(bytes).and_then(|s| String::from_utf8(s).ok()) } } impl DeserializeFrom for Vec { fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { let len: usize = bytes.next_leb128()?; let mut buf = Vec::with_capacity(len); for _ in 0..len { buf.push(*bytes.next()?); } buf.into() } } impl DeserializeFrom for BlobId { fn deserialize_from(bytes: &mut std::slice::Iter<'_, u8>) -> Option { BlobId::from_iter(bytes) } } impl DeserializeFrom for Keyword { fn deserialize_from(bytes: &mut std::slice::Iter<'_, u8>) -> Option { match bytes.next_leb128::()? { SEEN => Some(Keyword::Seen), DRAFT => Some(Keyword::Draft), FLAGGED => Some(Keyword::Flagged), ANSWERED => Some(Keyword::Answered), RECENT => Some(Keyword::Recent), IMPORTANT => Some(Keyword::Important), PHISHING => Some(Keyword::Phishing), JUNK => Some(Keyword::Junk), NOTJUNK => Some(Keyword::NotJunk), DELETED => Some(Keyword::Deleted), FORWARDED => Some(Keyword::Forwarded), MDN_SENT => Some(Keyword::MdnSent), other => { let len = other - 12; let mut keyword = Vec::with_capacity(len); for _ in 0..len { keyword.push(*bytes.next()?); } Some(Keyword::Other( String::from_utf8(keyword).ok()?.into_boxed_str(), )) } } } } impl DeserializeFrom for Property { fn deserialize_from(bytes: &mut std::slice::Iter<'_, u8>) -> Option { match *bytes.next()? { 0 => Some(Property::IsActive), 1 => Some(Property::IsEnabled), 2 => Some(Property::IsSubscribed), 3 => Some(Property::Keys), 4 => Some(Property::Keywords), 5 => Some(Property::Language), 6 => Some(Property::Location), 7 => Some(Property::MailboxIds), 8 => Some(Property::MayDelete), 9 => Some(Property::MdnBlobIds), 10 => Some(Property::Members), 11 => Some(Property::MessageId), 12 => Some(Property::MyRights), 13 => Some(Property::Name), 14 => Some(Property::ParentId), 15 => Some(Property::PartId), 16 => Some(Property::Picture), 17 => Some(Property::Preview), 18 => Some(Property::Quota), 19 => Some(Property::ReceivedAt), 20 => Some(Property::References), 21 => Some(Property::ReplyTo), 22 => Some(Property::Role), 23 => Some(Property::Secret), 24 => Some(Property::SendAt), 25 => Some(Property::Sender), 26 => Some(Property::SentAt), 27 => Some(Property::Size), 28 => Some(Property::SortOrder), 29 => Some(Property::Subject), 30 => Some(Property::SubParts), 31 => Some(Property::TextBody), 32 => Some(Property::TextSignature), 33 => Some(Property::ThreadId), 34 => Some(Property::Timezone), 35 => Some(Property::To), 36 => Some(Property::ToDate), 37 => Some(Property::TotalEmails), 38 => Some(Property::TotalThreads), 39 => Some(Property::Type), 40 => Some(Property::Types), 41 => Some(Property::UndoStatus), 42 => Some(Property::UnreadEmails), 43 => Some(Property::UnreadThreads), 44 => Some(Property::Url), 45 => Some(Property::VerificationCode), 46 => Some(Property::Parameters), 47 => Some(Property::Addresses), 48 => Some(Property::P256dh), 49 => Some(Property::Auth), 50 => Some(Property::Value), 51 => Some(Property::SmtpReply), 52 => Some(Property::Delivered), 53 => Some(Property::Displayed), 54 => Some(Property::MailFrom), 55 => Some(Property::RcptTo), 56 => Some(Property::IsEncodingProblem), 57 => Some(Property::IsTruncated), 58 => Some(Property::MayReadItems), 59 => Some(Property::MayAddItems), 60 => Some(Property::MayRemoveItems), 61 => Some(Property::MaySetSeen), 62 => Some(Property::MaySetKeywords), 63 => Some(Property::MayCreateChild), 64 => Some(Property::MayRename), 65 => Some(Property::MaySubmit), 66 => Some(Property::Acl), 67 => Some(Property::Aliases), 68 => Some(Property::Attachments), 69 => Some(Property::Bcc), 70 => Some(Property::BlobId), 71 => Some(Property::BodyStructure), 72 => Some(Property::BodyValues), 73 => Some(Property::Capabilities), 74 => Some(Property::Cc), 75 => Some(Property::Charset), 76 => Some(Property::Cid), 77 => Some(Property::DeliveryStatus), 78 => Some(Property::Description), 79 => Some(Property::DeviceClientId), 80 => Some(Property::Disposition), 81 => Some(Property::DsnBlobIds), 82 => Some(Property::Email), 83 => Some(Property::EmailId), 84 => Some(Property::EmailIds), 85 => Some(Property::Envelope), 86 => Some(Property::Expires), 87 => Some(Property::From), 88 => Some(Property::FromDate), 89 => Some(Property::HasAttachment), 91 => Some(Property::Headers), 92 => Some(Property::HtmlBody), 93 => Some(Property::HtmlSignature), 94 => Some(Property::Id), 95 => Some(Property::IdentityId), 96 => Some(Property::InReplyTo), 97 => String::deserialize_from(bytes).map(Property::_T), 98 => Some(Property::ResourceType), 99 => Some(Property::Used), 100 => Some(Property::HardLimit), 101 => Some(Property::WarnLimit), 102 => Some(Property::SoftLimit), 103 => Some(Property::Scope), _ => None, } } } pub trait FromLegacy { fn from_legacy(legacy: Object) -> Self; } pub trait TryFromLegacy: Sized { fn try_from_legacy(legacy: Object) -> Option; } impl Value { pub fn try_unwrap_id(self) -> Option { match self { Value::Id(id) => id.into(), _ => None, } } pub fn try_unwrap_bool(self) -> Option { match self { Value::Bool(b) => b.into(), _ => None, } } pub fn try_unwrap_keyword(self) -> Option { match self { Value::Keyword(k) => k.into(), _ => None, } } pub fn try_unwrap_string(self) -> Option { match self { Value::Text(s) => Some(s), _ => None, } } pub fn try_unwrap_object(self) -> Option> { match self { Value::Object(o) => Some(o), _ => None, } } pub fn try_unwrap_list(self) -> Option> { match self { Value::List(l) => Some(l), _ => None, } } pub fn try_unwrap_date(self) -> Option { match self { Value::Date(d) => Some(d), _ => None, } } pub fn try_unwrap_blob_id(self) -> Option { match self { Value::BlobId(b) => Some(b), _ => None, } } pub fn try_unwrap_uint(self) -> Option { match self { Value::UnsignedInt(u) => Some(u), _ => None, } } pub fn as_string(&self) -> Option<&str> { match self { Value::Text(s) => Some(s), _ => None, } } pub fn as_id(&self) -> Option<&Id> { match self { Value::Id(id) => Some(id), _ => None, } } pub fn as_blob_id(&self) -> Option<&BlobId> { match self { Value::BlobId(id) => Some(id), _ => None, } } pub fn as_list(&self) -> Option<&Vec> { match self { Value::List(l) => Some(l), _ => None, } } pub fn as_acl(&self) -> Option<&Vec> { match self { Value::Acl(l) => Some(l), _ => None, } } pub fn as_uint(&self) -> Option { match self { Value::UnsignedInt(u) => Some(*u), Value::Id(id) => Some(*id.as_ref()), _ => None, } } pub fn as_bool(&self) -> Option { match self { Value::Bool(b) => Some(*b), _ => None, } } pub fn as_date(&self) -> Option<&UTCDate> { match self { Value::Date(d) => Some(d), _ => None, } } pub fn as_obj(&self) -> Option<&Object> { match self { Value::Object(o) => Some(o), _ => None, } } pub fn as_obj_mut(&mut self) -> Option<&mut Object> { match self { Value::Object(o) => Some(o), _ => None, } } pub fn try_cast_uint(&self) -> Option { match self { Value::UnsignedInt(u) => Some(*u), Value::Id(id) => Some(id.id()), Value::Bool(b) => Some(*b as u64), _ => None, } } } impl UTCDate { pub fn from_timestamp(timestamp: i64) -> Self { // Ported from http://howardhinnant.github.io/date_algorithms.html#civil_from_days let (z, seconds) = ((timestamp / 86400) + 719468, timestamp % 86400); let era: i64 = (if z >= 0 { z } else { z - 146096 }) / 146097; let doe: u64 = (z - era * 146097) as u64; // [0, 146096] let yoe: u64 = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; // [0, 399] let y: i64 = (yoe as i64) + era * 400; let doy: u64 = doe - (365 * yoe + yoe / 4 - yoe / 100); // [0, 365] let mp = (5 * doy + 2) / 153; // [0, 11] let d: u64 = doy - (153 * mp + 2) / 5 + 1; // [1, 31] let m: u64 = if mp < 10 { mp + 3 } else { mp - 9 }; // [1, 12] let (h, mn, s) = (seconds / 3600, (seconds / 60) % 60, seconds % 60); UTCDate { year: (y + i64::from(m <= 2)) as u16, month: m as u8, day: d as u8, hour: h as u8, minute: mn as u8, second: s as u8, tz_before_gmt: false, tz_hour: 0, tz_minute: 0, } } pub fn timestamp(&self) -> i64 { // Ported from https://github.com/protocolbuffers/upb/blob/22182e6e/upb/json_decode.c#L982-L992 let month = self.month as u32; let year_base = 4800; /* Before min year, multiple of 400. */ let m_adj = month.wrapping_sub(3); /* March-based month. */ let carry = i64::from(m_adj > month); let adjust = if carry > 0 { 12 } else { 0 }; let y_adj = self.year as i64 + year_base - carry; let month_days = ((m_adj.wrapping_add(adjust)) * 62719 + 769) / 2048; let leap_days = y_adj / 4 - y_adj / 100 + y_adj / 400; (y_adj * 365 + leap_days + month_days as i64 + (self.day as i64 - 1) - 2472632) * 86400 + self.hour as i64 * 3600 + self.minute as i64 * 60 + self.second as i64 + ((self.tz_hour as i64 * 3600 + self.tz_minute as i64 * 60) * if self.tz_before_gmt { 1 } else { -1 }) } } ================================================ FILE: crates/migration/src/principal_v1.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ email_v1::migrate_emails_v011, encryption_v1::migrate_encryption_params_v011, get_document_ids, identity_v1::migrate_identities_v011, mailbox::migrate_mailboxes, push_v1::migrate_push_subscriptions_v011, sieve_v1::migrate_sieve_v011, submission::migrate_email_submissions, threads::migrate_threads, }; use common::Server; use directory::{ Permission, Principal, PrincipalData, ROLE_ADMIN, ROLE_USER, Type, backend::internal::{PrincipalField, PrincipalSet, SpecialSecrets}, }; use nlp::tokenizers::word::WordTokenizer; use std::{slice::Iter, time::Instant}; use store::{ Deserialize, Serialize, ValueKey, ahash::{AHashMap, AHashSet}, backend::MAX_TOKEN_LENGTH, roaring::RoaringBitmap, write::{AlignedBytes, Archive, Archiver, BatchBuilder, DirectoryClass, ValueClass}, }; use trc::AddContext; use types::collection::Collection; use utils::codec::leb128::Leb128Iterator; pub(crate) async fn migrate_principals_v0_11(server: &Server) -> trc::Result { // Obtain email ids let principal_ids = get_document_ids(server, u32::MAX, Collection::Principal) .await .caused_by(trc::location!())? .unwrap_or_default(); let num_principals = principal_ids.len(); if num_principals == 0 { return Ok(principal_ids); } let mut num_migrated = 0; for principal_id in principal_ids.iter() { match server .store() .get_value::(ValueKey { account_id: u32::MAX, collection: Collection::Principal.into(), document_id: principal_id, class: ValueClass::Directory(DirectoryClass::Principal(principal_id)), }) .await { Ok(Some(legacy)) => { let mut principal = Principal::from_legacy(legacy); principal.sort(); let mut batch = BatchBuilder::new(); batch .with_account_id(u32::MAX) .with_collection(Collection::Principal) .with_document(principal_id); build_search_index(&mut batch, principal_id, &principal); batch.set( ValueClass::Directory(DirectoryClass::Principal(principal_id)), Archiver::new(principal) .serialize() .caused_by(trc::location!())?, ); num_migrated += 1; server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } Ok(None) => (), Err(err) => { if server .store() .get_value::>(ValueKey { account_id: u32::MAX, collection: Collection::Principal.into(), document_id: principal_id, class: ValueClass::Directory(DirectoryClass::Principal(principal_id)), }) .await .is_err() { return Err(err.account_id(principal_id).caused_by(trc::location!())); } } } } // Increment document id counter if num_migrated > 0 { server .store() .assign_document_ids( u32::MAX, Collection::Principal, principal_ids .max() .map(|id| id as u64) .unwrap_or(num_principals) + 1, ) .await .caused_by(trc::location!())?; trc::event!( Server(trc::ServerEvent::Startup), Details = format!("Migrated {num_migrated} principals",) ); } Ok(principal_ids) } pub(crate) async fn migrate_principal_v0_11(server: &Server, account_id: u32) -> trc::Result<()> { let start_time = Instant::now(); let num_emails = migrate_emails_v011(server, account_id) .await .caused_by(trc::location!())?; let num_mailboxes = migrate_mailboxes(server, account_id) .await .caused_by(trc::location!())?; let num_params = migrate_encryption_params_v011(server, account_id) .await .caused_by(trc::location!())?; let num_subscriptions = migrate_push_subscriptions_v011(server, account_id) .await .caused_by(trc::location!())?; let num_sieve = migrate_sieve_v011(server, account_id) .await .caused_by(trc::location!())?; let num_submissions = migrate_email_submissions(server, account_id) .await .caused_by(trc::location!())?; let num_threads = migrate_threads(server, account_id) .await .caused_by(trc::location!())?; let num_identities = migrate_identities_v011(server, account_id) .await .caused_by(trc::location!())?; if num_emails > 0 || num_mailboxes > 0 || num_params > 0 || num_subscriptions > 0 || num_sieve > 0 || num_submissions > 0 || num_threads > 0 || num_identities > 0 { trc::event!( Server(trc::ServerEvent::Startup), Details = format!( "Migrated accountId {account_id} with {num_emails} emails, {num_mailboxes} mailboxes, {num_params} encryption params, {num_submissions} email submissions, {num_sieve} sieve scripts, {num_subscriptions} push subscriptions, {num_threads} threads, and {num_identities} identities" ), Elapsed = start_time.elapsed() ); } Ok(()) } trait FromLegacy { fn from_legacy(legacy: LegacyPrincipal) -> Self; } impl FromLegacy for Principal { fn from_legacy(legacy: LegacyPrincipal) -> Self { let mut legacy = legacy.0; let mut principal = Principal { id: legacy.id, typ: legacy.typ, name: legacy.name().to_string(), data: Default::default(), }; // Map fields let mut has_secret = false; for secret in legacy .take_str_array(PrincipalField::Secrets) .unwrap_or_default() { if secret.is_otp_secret() { principal.data.push(PrincipalData::OtpAuth(secret)); } else if secret.is_app_secret() { principal.data.push(PrincipalData::AppPassword(secret)); } else if !has_secret { principal.data.push(PrincipalData::Password(secret)); has_secret = true; } } for (idx, email) in legacy .take_str_array(PrincipalField::Emails) .unwrap_or_default() .into_iter() .enumerate() { if idx == 0 { principal .data .push(PrincipalData::PrimaryEmail(email.clone())); } else { principal .data .push(PrincipalData::EmailAlias(email.clone())); } } if let Some(picture) = legacy.take_str(PrincipalField::Picture) { principal.data.push(PrincipalData::Picture(picture)); } for url in legacy .take_str_array(PrincipalField::Urls) .unwrap_or_default() { principal.data.push(PrincipalData::Url(url)); } for member in legacy .take_str_array(PrincipalField::ExternalMembers) .unwrap_or_default() { principal.data.push(PrincipalData::ExternalMember(member)); } if let Some(quotas) = legacy.take_int_array(PrincipalField::Quota) { for (idx, quota) in quotas.into_iter().take(Type::MAX_ID + 2).enumerate() { if quota != 0 { if idx != 0 { principal.data.push(PrincipalData::DirectoryQuota { quota: quota as u32, typ: Type::from_u8((idx - 1) as u8), }); } else { principal.data.push(PrincipalData::DiskQuota(quota)); } } } } // Map permissions let mut permissions = AHashMap::new(); for field in [ PrincipalField::EnabledPermissions, PrincipalField::DisabledPermissions, ] { let is_disabled = field == PrincipalField::DisabledPermissions; if let Some(ids) = legacy.take_int_array(field) { for id in ids { if Permission::from_id(id as u32).is_some() { permissions.insert(id as u32, is_disabled); } } } } if !permissions.is_empty() { for (k, v) in permissions { principal.data.push(PrincipalData::Permission { permission_id: k, grant: !v, }); } } principal } } #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct LegacyPrincipal(PrincipalSet); impl Deserialize for LegacyPrincipal { fn deserialize(bytes: &[u8]) -> trc::Result { deserialize(bytes).ok_or_else(|| { trc::StoreEvent::DataCorruption .caused_by(trc::location!()) .ctx(trc::Key::Value, bytes) }) } } const INT_MARKER: u8 = 1 << 7; fn deserialize(bytes: &[u8]) -> Option { let mut bytes = bytes.iter(); match *bytes.next()? { 1 => { // Version 1 (legacy) let id = bytes.next_leb128()?; let type_id = *bytes.next()?; let mut principal = PrincipalSet { id, typ: Type::from_u8(type_id), ..Default::default() }; principal.set(PrincipalField::Quota, bytes.next_leb128::()?); principal.set(PrincipalField::Name, deserialize_string(&mut bytes)?); if let Some(description) = deserialize_string(&mut bytes).filter(|s| !s.is_empty()) { principal.set(PrincipalField::Description, description); } for key in [PrincipalField::Secrets, PrincipalField::Emails] { for _ in 0..bytes.next_leb128::()? { principal.append_str(key, deserialize_string(&mut bytes)?); } } LegacyPrincipal(principal.with_field( PrincipalField::Roles, if type_id != 4 { ROLE_USER } else { ROLE_ADMIN }, )) .into() } 2 => { // Version 2 let typ = Type::from_u8(*bytes.next()?); let num_fields = bytes.next_leb128::()?; let mut principal = PrincipalSet { id: u32::MAX, typ, fields: AHashMap::with_capacity(num_fields), }; for _ in 0..num_fields { let id = *bytes.next()?; let num_values = bytes.next_leb128::()?; if (id & INT_MARKER) == 0 { let field = PrincipalField::from_id(id)?; if num_values == 1 { principal.set(field, deserialize_string(&mut bytes)?); } else { let mut values = Vec::with_capacity(num_values); for _ in 0..num_values { values.push(deserialize_string(&mut bytes)?); } principal.set(field, values); } } else { let field = PrincipalField::from_id(id & !INT_MARKER)?; if num_values == 1 { principal.set(field, bytes.next_leb128::()?); } else { let mut values = Vec::with_capacity(num_values); for _ in 0..num_values { values.push(bytes.next_leb128::()?); } principal.set(field, values); } } } LegacyPrincipal(principal).into() } _ => None, } } fn deserialize_string(bytes: &mut Iter<'_, u8>) -> Option { let len = bytes.next_leb128()?; let mut string = Vec::with_capacity(len); for _ in 0..len { string.push(*bytes.next()?); } String::from_utf8(string).ok() } pub(crate) fn build_search_index(batch: &mut BatchBuilder, principal_id: u32, new: &Principal) { let mut new_words = AHashSet::new(); for word in [Some(new.name.as_str()), new.description()] .into_iter() .chain(new.email_addresses().map(Some)) .flatten() { new_words.extend(WordTokenizer::new(word, MAX_TOKEN_LENGTH).map(|t| t.word)); } for word in new_words { batch.set( DirectoryClass::Index { word: word.as_bytes().to_vec(), principal_id, }, vec![], ); } } ================================================ FILE: crates/migration/src/principal_v2.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ addressbook_v2::migrate_addressbook_v013, calendar_v2::migrate_calendar_v013, contact_v2::migrate_contacts_v013, event_v2::{migrate_calendar_events_v013, migrate_calendar_scheduling_v013}, get_document_ids, push_v2::migrate_push_subscriptions_v013, sieve_v2::migrate_sieve_v013, }; use common::Server; use directory::{Principal, PrincipalData, Type, backend::internal::SpecialSecrets}; use proc_macros::EnumMethods; use std::time::Instant; use store::{ Serialize, ValueKey, roaring::RoaringBitmap, write::{AlignedBytes, Archive, Archiver, BatchBuilder, DirectoryClass, ValueClass}, }; use trc::AddContext; use types::collection::Collection; pub(crate) async fn migrate_principals_v0_13(server: &Server) -> trc::Result { // Obtain email ids let principal_ids = get_document_ids(server, u32::MAX, Collection::Principal) .await .caused_by(trc::location!())? .unwrap_or_default(); let num_principals = principal_ids.len(); if num_principals == 0 { return Ok(principal_ids); } let mut num_migrated = 0; for principal_id in principal_ids.iter() { match server .store() .get_value::>(ValueKey { account_id: u32::MAX, collection: Collection::Principal.into(), document_id: principal_id, class: ValueClass::Directory(DirectoryClass::Principal(principal_id)), }) .await { Ok(Some(legacy)) => match legacy.deserialize_untrusted::() { Ok(old_principal) => { let mut principal = Principal { id: principal_id, typ: old_principal.typ, name: old_principal.name, data: Vec::new(), }; let mut has_secret = false; for secret in old_principal.secrets { if secret.is_otp_secret() { principal.data.push(PrincipalData::OtpAuth(secret)); } else if secret.is_app_secret() { principal.data.push(PrincipalData::AppPassword(secret)); } else if !has_secret { principal.data.push(PrincipalData::Password(secret)); has_secret = true; } } for (idx, email) in old_principal.emails.into_iter().enumerate() { if idx == 0 { principal.data.push(PrincipalData::PrimaryEmail(email)); } else { principal.data.push(PrincipalData::EmailAlias(email)); } } if let Some(description) = old_principal.description { principal.data.push(PrincipalData::Description(description)); } if let Some(quota) = old_principal.quota && quota > 0 { principal.data.push(PrincipalData::DiskQuota(quota)); } if let Some(tenant) = old_principal.tenant { principal.data.push(PrincipalData::Tenant(tenant)); } for item in old_principal.data { match item { PrincipalDataV2::MemberOf(items) => { for item in items { principal.data.push(PrincipalData::MemberOf(item)); } } PrincipalDataV2::Roles(items) => { for item in items { principal.data.push(PrincipalData::Role(item)); } } PrincipalDataV2::Lists(items) => { for item in items { principal.data.push(PrincipalData::List(item)); } } PrincipalDataV2::Permissions(items) => { for item in items { principal.data.push(PrincipalData::Permission { permission_id: item.permission.id(), grant: item.grant, }); } } PrincipalDataV2::Picture(item) => { principal.data.push(PrincipalData::Picture(item)); } PrincipalDataV2::ExternalMembers(items) => { for item in items { principal.data.push(PrincipalData::ExternalMember(item)); } } PrincipalDataV2::Urls(items) => { for item in items { principal.data.push(PrincipalData::Url(item)); } } PrincipalDataV2::PrincipalQuota(items) => { for item in items { principal.data.push(PrincipalData::DirectoryQuota { quota: item.quota as u32, typ: item.typ, }); } } PrincipalDataV2::Locale(item) => { principal.data.push(PrincipalData::Locale(item)); } } } principal.sort(); let mut batch = BatchBuilder::new(); batch .with_account_id(u32::MAX) .with_collection(Collection::Principal) .with_document(principal_id); batch.set( ValueClass::Directory(DirectoryClass::Principal(principal_id)), Archiver::new(principal) .serialize() .caused_by(trc::location!())?, ); num_migrated += 1; server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } Err(_) => { if let Err(err) = legacy.deserialize_untrusted::() { return Err(err.account_id(principal_id).caused_by(trc::location!())); } } }, Ok(None) => (), Err(err) => { return Err(err.account_id(principal_id).caused_by(trc::location!())); } } } if num_migrated > 0 { trc::event!( Server(trc::ServerEvent::Startup), Details = format!("Migrated {num_migrated} principals",) ); } Ok(principal_ids) } pub(crate) async fn migrate_principal_v0_13(server: &Server, account_id: u32) -> trc::Result<()> { let start_time = Instant::now(); let num_push = migrate_push_subscriptions_v013(server, account_id) .await .caused_by(trc::location!())?; let num_sieve = migrate_sieve_v013(server, account_id) .await .caused_by(trc::location!())?; let num_calendars = migrate_calendar_v013(server, account_id) .await .caused_by(trc::location!())?; let num_events = migrate_calendar_events_v013(server, account_id) .await .caused_by(trc::location!())?; let num_event_scheduling = migrate_calendar_scheduling_v013(server, account_id) .await .caused_by(trc::location!())?; let num_books = migrate_addressbook_v013(server, account_id) .await .caused_by(trc::location!())?; let num_contacts = migrate_contacts_v013(server, account_id) .await .caused_by(trc::location!())?; if num_sieve > 0 || num_books > 0 || num_contacts > 0 || num_calendars > 0 || num_events > 0 || num_push > 0 || num_event_scheduling > 0 { trc::event!( Server(trc::ServerEvent::Startup), Details = format!( "Migrated accountId {account_id} with {num_sieve} sieve scripts, {num_push} push subscriptions, {num_calendars} calendars, {num_events} calendar events, {num_event_scheduling} event scheduling, {num_books} address books and {num_contacts} contacts" ), Elapsed = start_time.elapsed() ); } Ok(()) } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] pub struct PrincipalV2 { pub id: u32, pub typ: Type, pub name: String, pub description: Option, pub secrets: Vec, pub emails: Vec, pub quota: Option, pub tenant: Option, pub data: Vec, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] pub enum PrincipalDataV2 { MemberOf(Vec), Roles(Vec), Lists(Vec), Permissions(Vec), Picture(String), ExternalMembers(Vec), Urls(Vec), PrincipalQuota(Vec), Locale(String), } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] pub struct PrincipalQuotaV2 { pub quota: u64, pub typ: Type, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] pub struct PermissionGrantV2 { pub permission: PermissionV2, pub grant: bool, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, EnumMethods, )] #[serde(rename_all = "kebab-case")] pub enum PermissionV2 { // WARNING: add new ids at the end (TODO: use static ids) // Admin Impersonate, UnlimitedRequests, UnlimitedUploads, DeleteSystemFolders, MessageQueueList, MessageQueueGet, MessageQueueUpdate, MessageQueueDelete, OutgoingReportList, OutgoingReportGet, OutgoingReportDelete, IncomingReportList, IncomingReportGet, IncomingReportDelete, SettingsList, SettingsUpdate, SettingsDelete, SettingsReload, IndividualList, IndividualGet, IndividualUpdate, IndividualDelete, IndividualCreate, GroupList, GroupGet, GroupUpdate, GroupDelete, GroupCreate, DomainList, DomainGet, DomainCreate, DomainUpdate, DomainDelete, TenantList, TenantGet, TenantCreate, TenantUpdate, TenantDelete, MailingListList, MailingListGet, MailingListCreate, MailingListUpdate, MailingListDelete, RoleList, RoleGet, RoleCreate, RoleUpdate, RoleDelete, PrincipalList, PrincipalGet, PrincipalCreate, PrincipalUpdate, PrincipalDelete, BlobFetch, PurgeBlobStore, PurgeDataStore, PurgeInMemoryStore, PurgeAccount, FtsReindex, Undelete, DkimSignatureCreate, DkimSignatureGet, SpamFilterUpdate, WebadminUpdate, LogsView, SpamFilterTrain, Restart, TracingList, TracingGet, TracingLive, MetricsList, MetricsLive, // Generic Authenticate, AuthenticateOauth, EmailSend, EmailReceive, // Account Management ManageEncryption, ManagePasswords, // JMAP JmapEmailGet, JmapMailboxGet, JmapThreadGet, JmapIdentityGet, JmapEmailSubmissionGet, JmapPushSubscriptionGet, JmapSieveScriptGet, JmapVacationResponseGet, JmapPrincipalGet, JmapQuotaGet, JmapBlobGet, JmapEmailSet, JmapMailboxSet, JmapIdentitySet, JmapEmailSubmissionSet, JmapPushSubscriptionSet, JmapSieveScriptSet, JmapVacationResponseSet, JmapEmailChanges, JmapMailboxChanges, JmapThreadChanges, JmapIdentityChanges, JmapEmailSubmissionChanges, JmapQuotaChanges, JmapEmailCopy, JmapBlobCopy, JmapEmailImport, JmapEmailParse, JmapEmailQueryChanges, JmapMailboxQueryChanges, JmapEmailSubmissionQueryChanges, JmapSieveScriptQueryChanges, JmapPrincipalQueryChanges, JmapQuotaQueryChanges, JmapEmailQuery, JmapMailboxQuery, JmapEmailSubmissionQuery, JmapSieveScriptQuery, JmapPrincipalQuery, JmapQuotaQuery, JmapSearchSnippet, JmapSieveScriptValidate, JmapBlobLookup, JmapBlobUpload, JmapEcho, // IMAP ImapAuthenticate, ImapAclGet, ImapAclSet, ImapMyRights, ImapListRights, ImapAppend, ImapCapability, ImapId, ImapCopy, ImapMove, ImapCreate, ImapDelete, ImapEnable, ImapExpunge, ImapFetch, ImapIdle, ImapList, ImapLsub, ImapNamespace, ImapRename, ImapSearch, ImapSort, ImapSelect, ImapExamine, ImapStatus, ImapStore, ImapSubscribe, ImapThread, // POP3 Pop3Authenticate, Pop3List, Pop3Uidl, Pop3Stat, Pop3Retr, Pop3Dele, // ManageSieve SieveAuthenticate, SieveListScripts, SieveSetActive, SieveGetScript, SievePutScript, SieveDeleteScript, SieveRenameScript, SieveCheckScript, SieveHaveSpace, // API keys ApiKeyList, ApiKeyGet, ApiKeyCreate, ApiKeyUpdate, ApiKeyDelete, // OAuth clients OauthClientList, OauthClientGet, OauthClientCreate, OauthClientUpdate, OauthClientDelete, // OAuth client registration OauthClientRegistration, OauthClientOverride, AiModelInteract, Troubleshoot, SpamFilterClassify, // WebDAV permissions DavSyncCollection, DavExpandProperty, DavPrincipalAcl, DavPrincipalList, DavPrincipalMatch, DavPrincipalSearch, DavPrincipalSearchPropSet, DavFilePropFind, DavFilePropPatch, DavFileGet, DavFileMkCol, DavFileDelete, DavFilePut, DavFileCopy, DavFileMove, DavFileLock, DavFileAcl, DavCardPropFind, DavCardPropPatch, DavCardGet, DavCardMkCol, DavCardDelete, DavCardPut, DavCardCopy, DavCardMove, DavCardLock, DavCardAcl, DavCardQuery, DavCardMultiGet, DavCalPropFind, DavCalPropPatch, DavCalGet, DavCalMkCol, DavCalDelete, DavCalPut, DavCalCopy, DavCalMove, DavCalLock, DavCalAcl, DavCalQuery, DavCalMultiGet, DavCalFreeBusyQuery, CalendarAlarms, CalendarSchedulingSend, CalendarSchedulingReceive, // WARNING: add new ids at the end (TODO: use static ids) } ================================================ FILE: crates/migration/src/push_v1.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::object::Object; use crate::{ get_document_ids, object::{FromLegacy, Property, Value}, }; use base64::{Engine, engine::general_purpose}; use common::Server; use email::push::{Keys, PushSubscription, PushSubscriptions}; use store::{ Serialize, ValueKey, write::{Archiver, BatchBuilder, ValueClass}, }; use trc::AddContext; use types::{ collection::Collection, field::{Field, PrincipalField}, type_state::DataType, }; pub(crate) async fn migrate_push_subscriptions_v011( server: &Server, account_id: u32, ) -> trc::Result { // Obtain email ids let push_subscription_ids = get_document_ids(server, account_id, Collection::PushSubscription) .await .caused_by(trc::location!())? .unwrap_or_default(); let num_push_subscriptions = push_subscription_ids.len(); if num_push_subscriptions == 0 { return Ok(0); } let mut subscriptions = Vec::with_capacity(num_push_subscriptions as usize); for push_subscription_id in &push_subscription_ids { match server .store() .get_value::>(ValueKey { account_id, collection: Collection::PushSubscription.into(), document_id: push_subscription_id, class: ValueClass::Property(Field::ARCHIVE.into()), }) .await { Ok(Some(legacy)) => { let mut subscription = PushSubscription::from_legacy(legacy); subscription.id = push_subscription_id; subscriptions.push(subscription); } Ok(None) => (), Err(err) => { return Err(err .account_id(account_id) .document_id(push_subscription_id) .caused_by(trc::location!())); } } } if !subscriptions.is_empty() { // Save changes let num_push_subscriptions = subscriptions.len() as u64; let mut batch = BatchBuilder::new(); batch .with_account_id(u32::MAX) .with_collection(Collection::Principal) .with_document(account_id) .tag(PrincipalField::PushSubscriptions) .with_account_id(account_id) .with_collection(Collection::PushSubscription); for subscription in &subscriptions { batch.with_document(subscription.id).clear(Field::ARCHIVE); } batch .with_collection(Collection::Principal) .with_document(0) .set( PrincipalField::PushSubscriptions, Archiver::new(PushSubscriptions { subscriptions }) .serialize() .caused_by(trc::location!())?, ); server .commit_batch(batch) .await .caused_by(trc::location!())?; Ok(num_push_subscriptions) } else { Ok(0) } } impl FromLegacy for PushSubscription { fn from_legacy(legacy: Object) -> Self { let (verification_code, verified) = legacy .get(&Property::VerificationCode) .as_string() .map(|c| (c.to_string(), true)) .or_else(|| { legacy .get(&Property::Value) .as_string() .map(|c| (c.to_string(), false)) }) .unwrap_or_default(); PushSubscription { id: 0, url: legacy .get(&Property::Url) .as_string() .unwrap_or_default() .to_string(), device_client_id: legacy .get(&Property::DeviceClientId) .as_string() .unwrap_or_default() .to_string(), expires: legacy .get(&Property::Expires) .as_date() .map(|s| s.timestamp() as u64) .unwrap_or_default(), verification_code, verified, types: legacy .get(&Property::Types) .as_list() .map(|l| l.as_slice()) .unwrap_or_default() .iter() .filter_map(|v| v.as_string().and_then(DataType::parse)) .collect(), keys: convert_keys(legacy.get(&Property::Keys)), email_push: vec![], } } } fn convert_keys(value: &Value) -> Option { let mut addr = Keys { p256dh: Default::default(), auth: Default::default(), }; if let Value::Object(obj) = value { for (key, value) in &obj.properties { match (key, value) { (Property::Auth, Value::Text(value)) => { addr.auth = general_purpose::URL_SAFE.decode(value).unwrap_or_default(); } (Property::P256dh, Value::Text(value)) => { addr.p256dh = general_purpose::URL_SAFE.decode(value).unwrap_or_default(); } _ => {} } } } if !addr.p256dh.is_empty() && !addr.auth.is_empty() { Some(addr) } else { None } } ================================================ FILE: crates/migration/src/push_v2.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::Server; use email::push::{Keys, PushSubscription, PushSubscriptions}; use store::{ Serialize, ValueKey, write::{AlignedBytes, Archive, Archiver, BatchBuilder, now}, }; use trc::AddContext; use types::{ collection::Collection, field::{Field, PrincipalField}, type_state::DataType, }; use utils::map::bitmap::Bitmap; use crate::get_document_ids; pub(crate) async fn migrate_push_subscriptions_v013( server: &Server, account_id: u32, ) -> trc::Result { // Obtain email ids let push_ids = get_document_ids(server, account_id, Collection::PushSubscription) .await .caused_by(trc::location!())? .unwrap_or_default(); let num_pushes = push_ids.len(); if num_pushes == 0 { return Ok(0); } let mut subscriptions = Vec::with_capacity(num_pushes as usize); for push_id in &push_ids { match server .store() .get_value::>(ValueKey::archive( account_id, Collection::PushSubscription, push_id, )) .await { Ok(Some(legacy)) => match legacy.deserialize_untrusted::() { Ok(old_push) => { subscriptions.push(PushSubscription { id: push_id, url: old_push.url, device_client_id: old_push.device_client_id, expires: old_push.expires, verification_code: old_push.verification_code, verified: old_push.verified, types: old_push.types, keys: old_push.keys, email_push: Vec::new(), }); } Err(err) => { return Err(err.account_id(push_id).caused_by(trc::location!())); } }, Ok(None) => (), Err(err) => { return Err(err.account_id(push_id).caused_by(trc::location!())); } } } if !subscriptions.is_empty() { // Save changes let num_push_subscriptions = subscriptions.len() as u64; let now = now(); let mut batch = BatchBuilder::new(); // Delete archived and document ids batch .with_account_id(account_id) .with_collection(Collection::PushSubscription); for subscription in &subscriptions { batch.with_document(subscription.id).clear(Field::ARCHIVE); } subscriptions.retain(|s| s.verified && s.expires > now); if !subscriptions.is_empty() { batch .with_account_id(u32::MAX) .with_collection(Collection::Principal) .with_document(account_id) .tag(PrincipalField::PushSubscriptions) .with_account_id(account_id) .with_collection(Collection::Principal) .with_document(0) .set( PrincipalField::PushSubscriptions, Archiver::new(PushSubscriptions { subscriptions }) .serialize() .caused_by(trc::location!())?, ); } server .commit_batch(batch) .await .caused_by(trc::location!())?; Ok(num_push_subscriptions) } else { Ok(0) } } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Default, Debug, Clone, PartialEq, Eq, )] pub struct PushSubscriptionV2 { pub url: String, pub device_client_id: String, pub expires: u64, pub verification_code: String, pub verified: bool, pub types: Bitmap, pub keys: Option, } ================================================ FILE: crates/migration/src/queue_v1.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ LegacyBincode, queue_v2::{LegacyHostResponse, LegacyQuotaKey}, }; use common::{ Server, config::smtp::queue::{DEFAULT_QUEUE_NAME, QueueExpiry, QueueName}, }; use smtp::queue::{ Error, ErrorDetails, HostResponse, Message, QueueId, Recipient, Schedule, Status, UnexpectedResponse, }; use smtp_proto::Response; use std::net::{IpAddr, Ipv4Addr}; use store::{ IterateParams, SUBSPACE_QUEUE_EVENT, Serialize, U64_LEN, ValueKey, ahash::AHashMap, write::{ AlignedBytes, AnyClass, Archive, Archiver, BatchBuilder, QueueClass, ValueClass, key::{DeserializeBigEndian, KeySerializer}, now, }, }; use trc::AddContext; use types::blob_hash::BlobHash; pub(crate) async fn migrate_queue_v011(server: &Server) -> trc::Result<()> { let mut count = 0; let now = now(); for (queue_id, due) in get_queue_events(server).await? { match server .store() .get_value::>(ValueKey::from(ValueClass::Queue( QueueClass::Message(queue_id), ))) .await { Ok(Some(bincoded)) => { let mut batch = BatchBuilder::new(); let message = Message::from(bincoded.inner); if let Some(due) = due { batch.clear(ValueClass::Any(AnyClass { subspace: SUBSPACE_QUEUE_EVENT, key: KeySerializer::new(16).write(due).write(queue_id).finalize(), })); } batch .set( ValueClass::Queue(QueueClass::MessageEvent(store::write::QueueEvent { due: due.unwrap_or(now), queue_id, queue_name: DEFAULT_QUEUE_NAME.into_inner(), })), vec![], ) .set( ValueClass::Queue(QueueClass::Message(queue_id)), Archiver::new(message) .serialize() .caused_by(trc::location!())?, ); count += 1; server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } Ok(None) => { if let Some(due) = due { let mut batch = BatchBuilder::new(); batch.clear(ValueClass::Any(AnyClass { subspace: SUBSPACE_QUEUE_EVENT, key: KeySerializer::new(16).write(due).write(queue_id).finalize(), })); server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } } Err(err) => { if server .store() .get_value::>(ValueKey::from(ValueClass::Queue( QueueClass::Message(queue_id), ))) .await .is_err() { return Err(err .ctx(trc::Key::QueueId, queue_id) .caused_by(trc::location!())); } } } } if count > 0 { trc::event!( Server(trc::ServerEvent::Startup), Details = format!("Migrated {count} queued messages",) ); } Ok(()) } pub(crate) async fn migrate_queue_v012(server: &Server) -> trc::Result<()> { let mut count = 0; let now = now(); for (queue_id, due) in get_queue_events(server).await? { match server .store() .get_value::>(ValueKey::from(ValueClass::Queue( QueueClass::Message(queue_id), ))) .await .and_then(|archive| { if let Some(archive) = archive { archive.deserialize_untrusted::().map(Some) } else { Ok(None) } }) { Ok(Some(archive)) => { let message = Message::from(archive); let mut batch = BatchBuilder::new(); if let Some(due) = due { batch.clear(ValueClass::Any(AnyClass { subspace: SUBSPACE_QUEUE_EVENT, key: KeySerializer::new(16).write(due).write(queue_id).finalize(), })); } batch .set( ValueClass::Queue(QueueClass::MessageEvent(store::write::QueueEvent { due: due.unwrap_or(now), queue_id, queue_name: DEFAULT_QUEUE_NAME.into_inner(), })), vec![], ) .set( ValueClass::Queue(QueueClass::Message(queue_id)), Archiver::new(message) .serialize() .caused_by(trc::location!())?, ); count += 1; server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } Ok(None) => { if let Some(due) = due { let mut batch = BatchBuilder::new(); batch.clear(ValueClass::Any(AnyClass { subspace: SUBSPACE_QUEUE_EVENT, key: KeySerializer::new(16).write(due).write(queue_id).finalize(), })); server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } } Err(err) => { if server .store() .get_value::>(ValueKey::from(ValueClass::Queue( QueueClass::Message(queue_id), ))) .await .and_then(|archive| { if let Some(archive) = archive { archive.deserialize_untrusted::().map(Some) } else { Ok(None) } }) .is_err() { return Err(err .ctx(trc::Key::QueueId, queue_id) .caused_by(trc::location!())); } } } } if count > 0 { trc::event!( Server(trc::ServerEvent::Startup), Details = format!("Migrated {count} queued messages",) ); } Ok(()) } async fn get_queue_events(server: &Server) -> trc::Result>> { let from_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( store::write::QueueEvent { due: 0, queue_id: 0, queue_name: [0; 8], }, ))); let to_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( store::write::QueueEvent { due: u64::MAX, queue_id: u64::MAX, queue_name: [u8::MAX; 8], }, ))); let mut queue_ids: AHashMap> = AHashMap::new(); server .store() .iterate( IterateParams::new(from_key, to_key).ascending().no_values(), |key, _| { queue_ids.insert( key.deserialize_be_u64(U64_LEN)?, Some(key.deserialize_be_u64(0)?), ); Ok(true) }, ) .await .caused_by(trc::location!())?; let from_key = ValueKey::from(ValueClass::Queue(QueueClass::Message(0))); let to_key = ValueKey::from(ValueClass::Queue(QueueClass::Message(u64::MAX))); server .store() .iterate( IterateParams::new(from_key, to_key).ascending().no_values(), |key, _| { let queue_id = key.deserialize_be_u64(0)?; if !queue_ids.contains_key(&queue_id) { queue_ids.insert(queue_id, None); } Ok(true) }, ) .await .caused_by(trc::location!())?; Ok(queue_ids) } impl From> for Message where SIZE: AsU64, IDX: AsU64, { fn from(message: LegacyMessage) -> Self { let domains = message.domains; Message { created: message.created, blob_hash: message.blob_hash, return_path: message.return_path_lcase.into_boxed_str(), recipients: message .recipients .into_iter() .map(|r| { let domain = &domains[r.domain_idx.as_u64() as usize]; let mut rcpt = Recipient::new(r.address); rcpt.status = match r.status { Status::Scheduled => match &domain.status { Status::Scheduled | Status::Completed(_) => Status::Scheduled, Status::TemporaryFailure(err) => { Status::TemporaryFailure(migrate_legacy_error(&domain.domain, err)) } Status::PermanentFailure(err) => { Status::PermanentFailure(migrate_legacy_error(&domain.domain, err)) } }, Status::Completed(details) => Status::Completed(HostResponse { hostname: details.hostname.into_boxed_str(), response: Response { code: details.response.code, esc: details.response.esc, message: details.response.message.into_boxed_str(), }, }), Status::TemporaryFailure(err) => { Status::TemporaryFailure(migrate_host_response(err)) } Status::PermanentFailure(err) => { Status::PermanentFailure(migrate_host_response(err)) } }; rcpt.flags = r.flags; rcpt.orcpt = r.orcpt.map(|o| o.into_boxed_str()); rcpt.retry = domain.retry.clone(); rcpt.notify = domain.notify.clone(); rcpt.queue = QueueName::default(); rcpt.expires = QueueExpiry::Ttl(domain.expires.saturating_sub(now())); rcpt }) .collect(), flags: message.flags, env_id: message.env_id.map(|e| e.into_boxed_str()), priority: message.priority, size: message.size.as_u64(), quota_keys: message.quota_keys.into_iter().map(Into::into).collect(), received_from_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), received_via_port: 0, } } } trait AsU64 { fn as_u64(&self) -> u64; } impl AsU64 for usize { fn as_u64(&self) -> u64 { *self as u64 } } impl AsU64 for u32 { fn as_u64(&self) -> u64 { *self as u64 } } impl AsU64 for u64 { fn as_u64(&self) -> u64 { *self } } fn migrate_legacy_error(domain: &str, err: &LegacyError) -> ErrorDetails { match err { LegacyError::DnsError(err) => ErrorDetails { entity: domain.into(), details: Error::DnsError(err.as_str().into()), }, LegacyError::UnexpectedResponse(err) => ErrorDetails { entity: err.hostname.entity.as_str().into(), details: Error::UnexpectedResponse(UnexpectedResponse { command: err.hostname.details.as_str().into(), response: Response { code: err.response.code, esc: err.response.esc, message: err.response.message.as_str().into(), }, }), }, LegacyError::ConnectionError(err) => ErrorDetails { entity: err.entity.as_str().into(), details: Error::ConnectionError(err.details.as_str().into()), }, LegacyError::TlsError(err) => ErrorDetails { entity: err.entity.as_str().into(), details: Error::TlsError(err.details.as_str().into()), }, LegacyError::DaneError(err) => ErrorDetails { entity: err.entity.as_str().into(), details: Error::DaneError(err.details.as_str().into()), }, LegacyError::MtaStsError(err) => ErrorDetails { entity: domain.into(), details: Error::MtaStsError(err.as_str().into()), }, LegacyError::RateLimited => ErrorDetails { entity: domain.into(), details: Error::RateLimited, }, LegacyError::ConcurrencyLimited => ErrorDetails { entity: domain.into(), details: Error::ConcurrencyLimited, }, LegacyError::Io(err) => ErrorDetails { entity: domain.into(), details: Error::Io(err.as_str().into()), }, } } fn migrate_host_response(response: LegacyHostResponse) -> ErrorDetails { ErrorDetails { entity: response.hostname.entity.into_boxed_str(), details: Error::UnexpectedResponse(UnexpectedResponse { command: response.hostname.details.into_boxed_str(), response: Response { code: response.response.code, esc: response.response.esc, message: response.response.message.into_boxed_str(), }, }), } } pub type MessageV011 = LegacyMessage; pub type MessageV012 = LegacyMessage; #[derive( Debug, Clone, PartialEq, Eq, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, serde::Deserialize, )] pub struct LegacyMessage { pub queue_id: QueueId, pub created: u64, pub blob_hash: BlobHash, pub return_path: String, pub return_path_lcase: String, pub return_path_domain: String, pub recipients: Vec>, pub domains: Vec, pub flags: u64, pub env_id: Option, pub priority: i16, pub size: SIZE, pub quota_keys: Vec, #[serde(skip)] #[rkyv(with = rkyv::with::Skip)] pub span_id: u64, } #[derive( Debug, Clone, PartialEq, Eq, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, serde::Deserialize, )] pub struct LegacyRecipient { pub domain_idx: IDX, pub address: String, pub address_lcase: String, pub status: Status, LegacyHostResponse>, pub flags: u64, pub orcpt: Option, } #[derive( Debug, Clone, PartialEq, Eq, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, serde::Deserialize, )] pub struct LegacyDomain { pub domain: String, pub retry: Schedule, pub notify: Schedule, pub expires: u64, pub status: Status<(), LegacyError>, } #[derive( Debug, Clone, PartialEq, Eq, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, serde::Deserialize, )] pub enum LegacyError { DnsError(String), UnexpectedResponse(LegacyHostResponse), ConnectionError(LegacyErrorDetails), TlsError(LegacyErrorDetails), DaneError(LegacyErrorDetails), MtaStsError(String), RateLimited, ConcurrencyLimited, Io(String), } #[derive( Debug, Clone, PartialEq, Eq, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, serde::Deserialize, )] pub struct LegacyErrorDetails { pub entity: String, pub details: String, } ================================================ FILE: crates/migration/src/queue_v2.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{ Server, config::smtp::queue::{QueueExpiry, QueueName}, }; use smtp::queue::{ Error, ErrorDetails, HostResponse, Message, QuotaKey, Recipient, Schedule, Status, UnexpectedResponse, }; use smtp_proto::Response; use std::net::IpAddr; use store::{ Deserialize, IterateParams, Serialize, ValueKey, write::{ AlignedBytes, Archive, Archiver, BatchBuilder, QueueClass, ValueClass, key::DeserializeBigEndian, }, }; use trc::AddContext; use types::blob_hash::BlobHash; #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Clone, PartialEq, Eq)] pub struct LegacyMessage { pub created: u64, pub blob_hash: BlobHash, pub return_path: String, pub recipients: Vec, pub received_from_ip: IpAddr, pub received_via_port: u16, pub flags: u64, pub env_id: Option, pub priority: i16, pub size: u64, pub quota_keys: Vec, } #[derive( rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Clone, PartialEq, Eq, serde::Deserialize, )] pub struct LegacyRecipient { pub address: String, pub retry: Schedule, pub notify: Schedule, pub expires: QueueExpiry, pub queue: QueueName, pub status: Status, LegacyErrorDetails>, pub flags: u64, pub orcpt: Option, } #[derive( Debug, Clone, PartialEq, Eq, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, serde::Deserialize, )] pub struct LegacyHostResponse { pub hostname: T, pub response: Response, } #[derive( Debug, Clone, PartialEq, Eq, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, serde::Deserialize, )] pub struct LegacyUnexpectedResponse { pub command: String, pub response: Response, } #[derive( Debug, Clone, PartialEq, Eq, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Default, serde::Deserialize, )] pub struct LegacyErrorDetails { pub entity: String, pub details: LegacyError, } #[derive( Debug, Clone, PartialEq, Eq, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, serde::Deserialize, Default, )] pub enum LegacyError { DnsError(String), UnexpectedResponse(LegacyUnexpectedResponse), ConnectionError(String), TlsError(String), DaneError(String), MtaStsError(String), RateLimited, #[default] ConcurrencyLimited, Io(String), } #[derive( rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Clone, PartialEq, Eq, serde::Deserialize, )] pub enum LegacyQuotaKey { Size { key: Vec, id: u64 }, Count { key: Vec, id: u64 }, } pub(crate) async fn migrate_queue_v014(server: &Server) -> trc::Result<()> { let mut messages = Vec::new(); server .store() .iterate( IterateParams::new( ValueKey::from(ValueClass::Queue(QueueClass::Message(0))), ValueKey::from(ValueClass::Queue(QueueClass::Message(u64::MAX))), ), |key, value| { let archive = as Deserialize>::deserialize(value) .caused_by(trc::location!())?; match archive.deserialize_untrusted::() { Ok(message) => { messages.push((key.deserialize_be_u64(0)?, Message::from(message))); } Err(err) => { if archive.deserialize_untrusted::().is_err() { return Err(err.caused_by(trc::location!())); } } } Ok(true) }, ) .await .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); let count = messages.len(); for (queue_id, message) in messages { batch.set( ValueClass::Queue(QueueClass::Message(queue_id)), Archiver::new(message) .serialize() .caused_by(trc::location!())?, ); if batch.is_large_batch() { server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); } } if !batch.is_empty() { server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } trc::event!( Server(trc::ServerEvent::Startup), Details = format!("Migrated {count} queued messages",) ); Ok(()) } impl From for Message { fn from(legacy: LegacyMessage) -> Self { Message { created: legacy.created, blob_hash: legacy.blob_hash, return_path: legacy.return_path.into_boxed_str(), recipients: legacy.recipients.into_iter().map(|r| r.into()).collect(), received_from_ip: legacy.received_from_ip, received_via_port: legacy.received_via_port, flags: legacy.flags, env_id: legacy.env_id.map(|s| s.into_boxed_str()), priority: legacy.priority, size: legacy.size, quota_keys: legacy.quota_keys.into_iter().map(|qk| qk.into()).collect(), } } } impl From for Recipient { fn from(legacy: LegacyRecipient) -> Self { Recipient { address: legacy.address.into_boxed_str(), retry: legacy.retry, notify: legacy.notify, expires: legacy.expires, queue: legacy.queue, status: match legacy.status { Status::Scheduled => Status::Scheduled, Status::Completed(status) => Status::Completed(status.into()), Status::TemporaryFailure(status) => Status::TemporaryFailure(status.into()), Status::PermanentFailure(status) => Status::PermanentFailure(status.into()), }, flags: legacy.flags, orcpt: legacy.orcpt.map(|s| s.into_boxed_str()), } } } impl From for ErrorDetails { fn from(legacy: LegacyErrorDetails) -> Self { ErrorDetails { entity: legacy.entity.into_boxed_str(), details: legacy.details.into(), } } } impl From for QuotaKey { fn from(legacy: LegacyQuotaKey) -> Self { match legacy { LegacyQuotaKey::Size { key, id } => QuotaKey::Size { key: key.into(), id, }, LegacyQuotaKey::Count { key, id } => QuotaKey::Count { key: key.into(), id, }, } } } impl From for Error { fn from(legacy: LegacyError) -> Self { match legacy { LegacyError::DnsError(s) => Error::DnsError(s.into_boxed_str()), LegacyError::UnexpectedResponse(ur) => Error::UnexpectedResponse(ur.into()), LegacyError::ConnectionError(s) => Error::ConnectionError(s.into_boxed_str()), LegacyError::TlsError(s) => Error::TlsError(s.into_boxed_str()), LegacyError::DaneError(s) => Error::DaneError(s.into_boxed_str()), LegacyError::MtaStsError(s) => Error::MtaStsError(s.into_boxed_str()), LegacyError::RateLimited => Error::RateLimited, LegacyError::ConcurrencyLimited => Error::ConcurrencyLimited, LegacyError::Io(s) => Error::Io(s.into_boxed_str()), } } } impl From for UnexpectedResponse { fn from(legacy: LegacyUnexpectedResponse) -> Self { UnexpectedResponse { command: legacy.command.into_boxed_str(), response: Response { code: legacy.response.code, esc: legacy.response.esc, message: legacy.response.message.into_boxed_str(), }, } } } impl From> for HostResponse> { fn from(legacy: LegacyHostResponse) -> Self { HostResponse { hostname: legacy.hostname.into_boxed_str(), response: Response { code: legacy.response.code, esc: legacy.response.esc, message: legacy.response.message.into_boxed_str(), }, } } } ================================================ FILE: crates/migration/src/report.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::LegacyBincode; use common::Server; use mail_auth::report::{Feedback, Report, tlsrpt::TlsReport}; use smtp::reporting::analysis::IncomingReport; use store::{ IterateParams, SUBSPACE_REPORT_OUT, Serialize, U64_LEN, ValueKey, ahash::AHashSet, write::{ AlignedBytes, AnyKey, Archive, Archiver, BatchBuilder, ReportClass, ValueClass, key::{DeserializeBigEndian, KeySerializer}, }, }; use trc::AddContext; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] enum ReportType { Dmarc, Tls, Arf, } pub(crate) async fn migrate_reports(server: &Server) -> trc::Result<()> { let mut num_dmarc = 0; let mut num_tls = 0; let mut num_arf = 0; for report in [ReportType::Dmarc, ReportType::Tls, ReportType::Arf] { let (from_key, to_key) = match report { ReportType::Dmarc => ( ValueKey::from(ValueClass::Report(ReportClass::Dmarc { id: 0, expires: 0 })), ValueKey::from(ValueClass::Report(ReportClass::Dmarc { id: u64::MAX, expires: u64::MAX, })), ), ReportType::Tls => ( ValueKey::from(ValueClass::Report(ReportClass::Tls { id: 0, expires: 0 })), ValueKey::from(ValueClass::Report(ReportClass::Tls { id: u64::MAX, expires: u64::MAX, })), ), ReportType::Arf => ( ValueKey::from(ValueClass::Report(ReportClass::Arf { id: 0, expires: 0 })), ValueKey::from(ValueClass::Report(ReportClass::Arf { id: u64::MAX, expires: u64::MAX, })), ), }; let mut results = AHashSet::new(); server .core .storage .data .iterate( IterateParams::new(from_key, to_key).no_values(), |key, _| { results.insert(( report, key.deserialize_be_u64(U64_LEN + 1)?, key.deserialize_be_u64(1)?, )); Ok(true) }, ) .await .caused_by(trc::location!())?; for (report, id, expires) in results { match report { ReportType::Dmarc => { match server .store() .get_value::>>(ValueKey::from( ValueClass::Report(ReportClass::Dmarc { id, expires }), )) .await { Ok(Some(bincoded)) => { let mut batch = BatchBuilder::new(); batch.set( ValueClass::Report(ReportClass::Dmarc { id, expires }), Archiver::new(bincoded.inner) .serialize() .caused_by(trc::location!())?, ); num_dmarc += 1; server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } Ok(None) => (), Err(err) => { if server .store() .get_value::>(ValueKey::from( ValueClass::Report(ReportClass::Dmarc { id, expires }), )) .await .is_err() { return Err(err.ctx(trc::Key::Id, id).caused_by(trc::location!())); } } } } ReportType::Tls => { match server .store() .get_value::>>(ValueKey::from( ValueClass::Report(ReportClass::Tls { id, expires }), )) .await { Ok(Some(bincoded)) => { let mut batch = BatchBuilder::new(); batch.set( ValueClass::Report(ReportClass::Tls { id, expires }), Archiver::new(bincoded.inner) .serialize() .caused_by(trc::location!())?, ); num_tls += 1; server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } Ok(None) => (), Err(err) => { if server .store() .get_value::>(ValueKey::from( ValueClass::Report(ReportClass::Tls { id, expires }), )) .await .is_err() { return Err(err.ctx(trc::Key::Id, id).caused_by(trc::location!())); } } } } ReportType::Arf => { match server .store() .get_value::>>(ValueKey::from( ValueClass::Report(ReportClass::Arf { id, expires }), )) .await { Ok(Some(bincoded)) => { let mut batch = BatchBuilder::new(); batch.set( ValueClass::Report(ReportClass::Arf { id, expires }), Archiver::new(bincoded.inner) .serialize() .caused_by(trc::location!())?, ); num_arf += 1; server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } Ok(None) => (), Err(err) => { if server .store() .get_value::>(ValueKey::from( ValueClass::Report(ReportClass::Arf { id, expires }), )) .await .is_err() { return Err(err.ctx(trc::Key::Id, id).caused_by(trc::location!())); } } } } } } } // Delete outgoing reports server .store() .delete_range( AnyKey { subspace: SUBSPACE_REPORT_OUT, key: KeySerializer::new(U64_LEN).write(0u8).finalize(), }, AnyKey { subspace: SUBSPACE_REPORT_OUT, key: KeySerializer::new(U64_LEN) .write(&[u8::MAX; 16][..]) .finalize(), }, ) .await .caused_by(trc::location!())?; if num_dmarc > 0 || num_tls > 0 || num_arf > 0 { trc::event!( Server(trc::ServerEvent::Startup), Details = format!("Migrated {num_dmarc} DMARC, {num_tls} TLS, and {num_arf} ARF reports") ); } Ok(()) } ================================================ FILE: crates/migration/src/sieve_v1.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::object::Object; use crate::{ get_document_ids, object::{Property, TryFromLegacy, Value}, v014::SUBSPACE_BITMAP_TEXT, }; use common::Server; use email::sieve::{SieveScript, VacationResponse}; use store::{ SUBSPACE_INDEXES, SUBSPACE_PROPERTY, Serialize, SerializeInfallible, U64_LEN, ValueKey, write::{ AlignedBytes, AnyKey, Archive, Archiver, BatchBuilder, ValueClass, key::KeySerializer, }, }; use trc::{AddContext, StoreEvent}; use types::{ collection::Collection, field::{Field, PrincipalField, SieveField}, }; pub(crate) async fn migrate_sieve_v011(server: &Server, account_id: u32) -> trc::Result { // Obtain email ids let script_ids = get_document_ids(server, account_id, Collection::SieveScript) .await .caused_by(trc::location!())? .unwrap_or_default(); let num_scripts = script_ids.len(); if num_scripts == 0 { return Ok(0); } let mut did_migrate = false; // Delete indexes for subspace in [SUBSPACE_INDEXES, SUBSPACE_BITMAP_TEXT] { server .store() .delete_range( AnyKey { subspace, key: KeySerializer::new(U64_LEN) .write(account_id) .write(u8::from(Collection::SieveScript)) .finalize(), }, AnyKey { subspace, key: KeySerializer::new(U64_LEN) .write(account_id) .write(u8::from(Collection::SieveScript)) .write(&[u8::MAX; 16][..]) .finalize(), }, ) .await .caused_by(trc::location!())?; } for script_id in &script_ids { match server .store() .get_value::>(ValueKey { account_id, collection: Collection::SieveScript.into(), document_id: script_id, class: ValueClass::Property(Field::ARCHIVE.into()), }) .await { Ok(Some(legacy)) => { let is_active = legacy .get(&Property::IsActive) .as_bool() .unwrap_or_default(); if let Some(script) = SieveScript::try_from_legacy(legacy) { let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::SieveScript) .with_document(script_id) .index(SieveField::Name, script.name.to_lowercase()) .set( Field::ARCHIVE, Archiver::new(script) .serialize() .caused_by(trc::location!())?, ); if is_active { batch .with_collection(Collection::Principal) .with_document(0) .set(PrincipalField::ActiveScriptId, script_id.serialize()); } did_migrate = true; server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } else { trc::event!( Store(StoreEvent::DataCorruption), Details = "Failed to migrate SieveScript", AccountId = account_id, ) } } Ok(None) => (), Err(err) => { if server .store() .get_value::>(ValueKey { account_id, collection: Collection::SieveScript.into(), document_id: script_id, class: ValueClass::Property(Field::ARCHIVE.into()), }) .await .is_err() { return Err(err .account_id(account_id) .document_id(script_id) .caused_by(trc::location!())); } } } } // Delete emailIds property server .store() .delete_range( AnyKey { subspace: SUBSPACE_PROPERTY, key: KeySerializer::new(U64_LEN) .write(account_id) .write(u8::from(Collection::SieveScript)) .write(u8::from(SieveField::Ids)) .finalize(), }, AnyKey { subspace: SUBSPACE_PROPERTY, key: KeySerializer::new(U64_LEN) .write(account_id) .write(u8::from(Collection::SieveScript)) .write(u8::from(SieveField::Ids)) .write(&[u8::MAX; 8][..]) .finalize(), }, ) .await .caused_by(trc::location!())?; // Increment document id counter if did_migrate { server .store() .assign_document_ids( account_id, Collection::SieveScript, script_ids.max().map(|id| id as u64).unwrap_or(num_scripts) + 1, ) .await .caused_by(trc::location!())?; Ok(num_scripts) } else { Ok(0) } } impl TryFromLegacy for SieveScript { fn try_from_legacy(legacy: Object) -> Option { let blob_id = legacy.get(&Property::BlobId).as_blob_id()?; Some(SieveScript { name: legacy .get(&Property::Name) .as_string() .unwrap_or_default() .to_string(), blob_hash: blob_id.hash.clone(), size: blob_id.section.as_ref()?.size as u32, vacation_response: VacationResponse::try_from_legacy(legacy), }) } } impl TryFromLegacy for VacationResponse { fn try_from_legacy(legacy: Object) -> Option { let vacation = VacationResponse { from_date: legacy .get(&Property::FromDate) .as_date() .map(|s| s.timestamp() as u64), to_date: legacy .get(&Property::ToDate) .as_date() .map(|s| s.timestamp() as u64), subject: legacy .get(&Property::Name) .as_string() .map(|s| s.to_string()), text_body: legacy .get(&Property::TextBody) .as_string() .map(|s| s.to_string()), html_body: legacy .get(&Property::HtmlBody) .as_string() .map(|s| s.to_string()), }; if vacation.from_date.is_some() || vacation.to_date.is_some() || vacation.subject.is_some() || vacation.text_body.is_some() || vacation.html_body.is_some() { Some(vacation) } else { None } } } ================================================ FILE: crates/migration/src/sieve_v2.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::Server; use email::sieve::{SieveScript, VacationResponse}; use store::{ Serialize, SerializeInfallible, ValueKey, write::{AlignedBytes, Archive, Archiver, BatchBuilder}, }; use trc::AddContext; use types::{ blob_hash::BlobHash, collection::Collection, field::{Field, PrincipalField}, }; use crate::get_document_ids; pub(crate) async fn migrate_sieve_v013(server: &Server, account_id: u32) -> trc::Result { // Obtain email ids let script_ids = get_document_ids(server, account_id, Collection::SieveScript) .await .caused_by(trc::location!())? .unwrap_or_default(); let num_scripts = script_ids.len(); if num_scripts == 0 { return Ok(0); } let mut num_migrated = 0; for script_id in &script_ids { match server .store() .get_value::>(ValueKey::archive( account_id, Collection::SieveScript, script_id, )) .await { Ok(Some(legacy)) => match legacy.deserialize_untrusted::() { Ok(old_sieve) => { let script = SieveScript { name: old_sieve.name, blob_hash: old_sieve.blob_hash, size: old_sieve.size, vacation_response: old_sieve.vacation_response, }; let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::SieveScript) .with_document(script_id) .unindex(Field::new(0u8), vec![u8::from(old_sieve.is_active)]) .set( Field::ARCHIVE, Archiver::new(script) .serialize() .caused_by(trc::location!())?, ); if old_sieve.is_active { batch .with_account_id(account_id) .with_collection(Collection::Principal) .with_document(0) .set(PrincipalField::ActiveScriptId, script_id.serialize()); } num_migrated += 1; server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } Err(_) => { if let Err(err) = legacy.deserialize_untrusted::() { return Err(err.account_id(script_id).caused_by(trc::location!())); } } }, Ok(None) => (), Err(err) => { return Err(err.account_id(script_id).caused_by(trc::location!())); } } } Ok(num_migrated) } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] #[rkyv(derive(Debug))] pub struct SieveScriptV2 { pub name: String, pub is_active: bool, pub blob_hash: BlobHash, pub size: u32, pub vacation_response: Option, } ================================================ FILE: crates/migration/src/submission.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::object::Object; use crate::{ get_document_ids, object::{FromLegacy, Property, Value}, v014::{SUBSPACE_BITMAP_TAG, SUBSPACE_BITMAP_TEXT}, }; use common::Server; use email::submission::{ Address, Delivered, DeliveryStatus, EmailSubmission, Envelope, UndoStatus, }; use store::{ SUBSPACE_INDEXES, Serialize, U32_LEN, U64_LEN, ValueKey, write::{ AlignedBytes, AnyKey, Archive, Archiver, BatchBuilder, IndexPropertyClass, ValueClass, key::KeySerializer, }, }; use trc::AddContext; use types::{ collection::Collection, field::{EmailSubmissionField, Field}, }; use utils::map::vec_map::VecMap; pub(crate) async fn migrate_email_submissions( server: &Server, account_id: u32, ) -> trc::Result { // Obtain email ids let email_submission_ids = get_document_ids(server, account_id, Collection::EmailSubmission) .await .caused_by(trc::location!())? .unwrap_or_default(); let num_email_submissions = email_submission_ids.len(); if num_email_submissions == 0 { return Ok(0); } let mut did_migrate = false; // Delete indexes for subspace in [SUBSPACE_INDEXES, SUBSPACE_BITMAP_TAG, SUBSPACE_BITMAP_TEXT] { server .store() .delete_range( AnyKey { subspace, key: KeySerializer::new(U64_LEN) .write(account_id) .write(u8::from(Collection::EmailSubmission)) .finalize(), }, AnyKey { subspace, key: KeySerializer::new(U64_LEN) .write(account_id) .write(u8::from(Collection::EmailSubmission)) .write(&[u8::MAX; 16][..]) .finalize(), }, ) .await .caused_by(trc::location!())?; } for email_submission_id in &email_submission_ids { match server .store() .get_value::>(ValueKey { account_id, collection: Collection::EmailSubmission.into(), document_id: email_submission_id, class: ValueClass::Property(Field::ARCHIVE.into()), }) .await { Ok(Some(legacy)) => { let es = EmailSubmission::from_legacy(legacy); let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::EmailSubmission) .with_document(email_submission_id) .set( ValueClass::IndexProperty(IndexPropertyClass::Integer { property: EmailSubmissionField::Metadata.into(), value: es.send_at, }), KeySerializer::new(U32_LEN * 3 + 1) .write(es.email_id) .write(es.thread_id) .write(es.identity_id) .write(es.undo_status.as_index()) .finalize(), ) .set( Field::ARCHIVE, Archiver::new(es).serialize().caused_by(trc::location!())?, ); did_migrate = true; server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } Ok(None) => (), Err(err) => { if server .store() .get_value::>(ValueKey { account_id, collection: Collection::EmailSubmission.into(), document_id: email_submission_id, class: ValueClass::Property(Field::ARCHIVE.into()), }) .await .is_err() { return Err(err .account_id(account_id) .document_id(email_submission_id) .caused_by(trc::location!())); } } } } // Increment document id counter if did_migrate { server .store() .assign_document_ids( account_id, Collection::EmailSubmission, email_submission_ids .max() .map(|id| id as u64) .unwrap_or(num_email_submissions) + 1, ) .await .caused_by(trc::location!())?; Ok(num_email_submissions) } else { Ok(0) } } impl FromLegacy for EmailSubmission { fn from_legacy(legacy: Object) -> Self { EmailSubmission { email_id: legacy.get(&Property::EmailId).as_uint().unwrap_or_default() as u32, thread_id: legacy .get(&Property::ThreadId) .as_uint() .unwrap_or_default() as u32, identity_id: legacy .get(&Property::IdentityId) .as_uint() .unwrap_or_default() as u32, send_at: legacy .get(&Property::SentAt) .as_date() .map(|s| s.timestamp() as u64) .unwrap_or_default(), queue_id: legacy.get(&Property::MessageId).as_uint(), undo_status: legacy .get(&Property::UndoStatus) .as_string() .and_then(UndoStatus::parse) .unwrap_or(UndoStatus::Final), envelope: convert_envelope(legacy.get(&Property::Envelope)), delivery_status: convert_delivery_status(legacy.get(&Property::DeliveryStatus)), } } } fn convert_delivery_status(value: &Value) -> VecMap { let mut status = VecMap::new(); if let Value::List(list) = value { for value in list { if let Value::Object(obj) = value { for (k, v) in obj.properties.iter() { if let (Property::_T(k), Value::Object(v)) = (k, v) { let mut delivery_status = DeliveryStatus { smtp_reply: String::new(), delivered: Delivered::Unknown, displayed: false, }; for (property, value) in &v.properties { match (property, value) { (Property::Delivered, Value::Text(v)) => match v.as_str() { "queued" => delivery_status.delivered = Delivered::Queued, "yes" => delivery_status.delivered = Delivered::Yes, "unknown" => delivery_status.delivered = Delivered::Unknown, "no" => delivery_status.delivered = Delivered::No, _ => {} }, (Property::SmtpReply, Value::Text(v)) => { delivery_status.smtp_reply = v.to_string(); } _ => {} } } status.append(k.to_string(), delivery_status); } } } } } status } fn convert_envelope(value: &Value) -> Envelope { let mut envelope = Envelope { mail_from: Default::default(), rcpt_to: vec![], }; if let Value::Object(obj) = value { for (property, value) in &obj.properties { match (property, value) { (Property::MailFrom, _) => { envelope.mail_from = convert_envelope_address(value).unwrap_or_default(); } (Property::RcptTo, Value::List(value)) => { for addr in value { if let Some(addr) = convert_envelope_address(addr) { envelope.rcpt_to.push(addr); } } } _ => {} } } } envelope } fn convert_envelope_address(envelope: &Value) -> Option
{ if let Value::Object(envelope) = envelope && let (Value::Text(email), Value::Object(params)) = ( envelope.get(&Property::Email), envelope.get(&Property::Parameters), ) { let mut addr = Address { email: email.to_string(), parameters: None, }; for (k, v) in params.properties.iter() { if let Property::_T(k) = &k && !k.is_empty() { let k = k.to_string(); let v = v.as_string().map(|s| s.to_string()); addr.parameters.get_or_insert_default().append(k, v); } } return Some(addr); } None } ================================================ FILE: crates/migration/src/tasks_v1.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::Server; use store::{ IterateParams, SUBSPACE_TASK_QUEUE, U64_LEN, ValueKey, write::{ AnyClass, BatchBuilder, ValueClass, key::{DeserializeBigEndian, KeySerializer}, now, }, }; use trc::AddContext; pub(crate) async fn migrate_tasks_v011(server: &Server) -> trc::Result<()> { let from_key = ValueKey:: { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Any(AnyClass { subspace: SUBSPACE_TASK_QUEUE, key: KeySerializer::new(U64_LEN).write(0u64).finalize(), }), }; let to_key = ValueKey:: { account_id: u32::MAX, collection: u8::MAX, document_id: u32::MAX, class: ValueClass::Any(AnyClass { subspace: SUBSPACE_TASK_QUEUE, key: KeySerializer::new(U64_LEN).write(u64::MAX).finalize(), }), }; let now = now(); let mut migrate_tasks = Vec::new(); server .core .storage .data .iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { let due = key.deserialize_be_u64(0)?; if due > now { migrate_tasks.push((key.to_vec(), value.to_vec())); } Ok(true) }, ) .await .caused_by(trc::location!())?; if !migrate_tasks.is_empty() { let num_migrated = migrate_tasks.len(); let mut batch = BatchBuilder::new(); for (key, value) in migrate_tasks { let mut new_key = key.clone(); new_key[0..8].copy_from_slice(&now.to_be_bytes()); batch .clear(ValueClass::Any(AnyClass { subspace: SUBSPACE_TASK_QUEUE, key, })) .set( ValueClass::Any(AnyClass { subspace: SUBSPACE_TASK_QUEUE, key: new_key, }), value, ); } server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; trc::event!( Server(trc::ServerEvent::Startup), Details = format!("Migrated {num_migrated} tasks") ); } Ok(()) } ================================================ FILE: crates/migration/src/tasks_v2.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::Server; use store::{ IterateParams, SUBSPACE_TASK_QUEUE, U32_LEN, U64_LEN, ValueKey, write::{ AnyClass, BatchBuilder, TaskEpoch, ValueClass, key::{DeserializeBigEndian, KeySerializer}, }, }; use trc::AddContext; pub(crate) async fn migrate_tasks_v014(server: &Server) -> trc::Result<()> { let from_key = ValueKey:: { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Any(AnyClass { subspace: SUBSPACE_TASK_QUEUE, key: KeySerializer::new(U64_LEN).write(0u64).finalize(), }), }; let to_key = ValueKey:: { account_id: u32::MAX, collection: u8::MAX, document_id: u32::MAX, class: ValueClass::Any(AnyClass { subspace: SUBSPACE_TASK_QUEUE, key: KeySerializer::new(U64_LEN).write(u64::MAX).finalize(), }), }; let mut delete_tasks = Vec::new(); let mut insert_tasks = Vec::new(); server .core .storage .data .iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { match key.get(U64_LEN + U32_LEN) { Some(0..=2) => { delete_tasks.push(key.to_vec()); } None => { return Err(trc::Error::corrupted_key(key, None, trc::location!())); } _ => { let due = key.deserialize_be_u64(0)?; let maybe_epoch = TaskEpoch::from_inner(due); if maybe_epoch.attempt() != 0 { delete_tasks.push(key.to_vec()); let epoch = TaskEpoch::new(due).inner(); let mut new_key = Vec::with_capacity(key.len()); new_key.extend_from_slice(&epoch.to_be_bytes()); new_key.extend_from_slice(&key[U64_LEN..]); insert_tasks.push((new_key, value.to_vec())); } } }; Ok(true) }, ) .await .caused_by(trc::location!())?; let num_migrated = delete_tasks.len() + insert_tasks.len(); if num_migrated != 0 { let mut batch = BatchBuilder::new(); let mut batch_len = 0; for (key, value) in insert_tasks { batch_len += key.len() + value.len(); batch.set( ValueClass::Any(AnyClass { subspace: SUBSPACE_TASK_QUEUE, key, }), value, ); if batch_len > 4 * 1024 * 1024 { server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); batch_len = 0; } } for key in delete_tasks { batch_len += key.len(); batch.clear(ValueClass::Any(AnyClass { subspace: SUBSPACE_TASK_QUEUE, key, })); if batch_len > 4 * 1024 * 1024 { server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); batch_len = 0; } } server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } trc::event!( Server(trc::ServerEvent::Startup), Details = format!("Migrated {num_migrated} tasks") ); Ok(()) } ================================================ FILE: crates/migration/src/threads.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::Server; use store::{ U64_LEN, write::{AnyKey, key::KeySerializer}, }; use trc::AddContext; use types::collection::Collection; use crate::{get_document_ids, v014::SUBSPACE_BITMAP_ID}; pub(crate) async fn migrate_threads(server: &Server, account_id: u32) -> trc::Result { // Obtain email ids let thread_ids = get_document_ids(server, account_id, Collection::Thread) .await .caused_by(trc::location!())? .unwrap_or_default(); let num_threads = thread_ids.len(); if num_threads == 0 { return Ok(0); } // Delete threads server .store() .delete_range( AnyKey { subspace: SUBSPACE_BITMAP_ID, key: KeySerializer::new(U64_LEN) .write(account_id) .write(u8::from(Collection::Thread)) .finalize(), }, AnyKey { subspace: SUBSPACE_BITMAP_ID, key: KeySerializer::new(U64_LEN) .write(account_id) .write(u8::from(Collection::Thread)) .write(&[u8::MAX; 16][..]) .finalize(), }, ) .await .caused_by(trc::location!())?; // Increment document id counter server .store() .assign_document_ids( account_id, Collection::Thread, thread_ids.max().map(|id| id as u64).unwrap_or(num_threads) + 1, ) .await .caused_by(trc::location!())?; Ok(num_threads) } ================================================ FILE: crates/migration/src/v011.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ LOCK_RETRY_TIME, LOCK_WAIT_TIME_ACCOUNT, LOCK_WAIT_TIME_CORE, changelog::reset_changelog, get_document_ids, principal_v1::{migrate_principal_v0_11, migrate_principals_v0_11}, queue_v1::migrate_queue_v011, report::migrate_reports, }; use common::{KV_LOCK_HOUSEKEEPER, Server}; use store::{ dispatch::lookup::KeyValue, rand::{self, seq::SliceRandom}, }; use trc::AddContext; use types::collection::Collection; pub(crate) async fn migrate_v0_11(server: &Server) -> trc::Result<()> { let force_lock = std::env::var("FORCE_LOCK").is_ok(); let in_memory = server.in_memory_store(); let principal_ids; loop { if force_lock || in_memory .try_lock( KV_LOCK_HOUSEKEEPER, b"migrate_core_lock", LOCK_WAIT_TIME_CORE, ) .await .caused_by(trc::location!())? { if in_memory .key_get::<()>(KeyValue::<()>::build_key( KV_LOCK_HOUSEKEEPER, b"migrate_core_done", )) .await .caused_by(trc::location!())? .is_none() { migrate_queue_v011(server) .await .caused_by(trc::location!())?; migrate_reports(server).await.caused_by(trc::location!())?; reset_changelog(server).await.caused_by(trc::location!())?; principal_ids = migrate_principals_v0_11(server) .await .caused_by(trc::location!())?; in_memory .key_set( KeyValue::new( KeyValue::<()>::build_key(KV_LOCK_HOUSEKEEPER, b"migrate_core_done"), b"1".to_vec(), ) .expires(86400), ) .await .caused_by(trc::location!())?; } else { principal_ids = get_document_ids(server, u32::MAX, Collection::Principal) .await .caused_by(trc::location!())? .unwrap_or_default(); trc::event!( Server(trc::ServerEvent::Startup), Details = format!("Migration completed by another node.",) ); } in_memory .remove_lock(KV_LOCK_HOUSEKEEPER, b"migrate_core_lock") .await .caused_by(trc::location!())?; break; } else { trc::event!( Server(trc::ServerEvent::Startup), Details = format!("Migration lock busy, waiting 30 seconds.",) ); tokio::time::sleep(LOCK_RETRY_TIME).await; } } if !principal_ids.is_empty() { let mut principal_ids = principal_ids.into_iter().collect::>(); principal_ids.shuffle(&mut rand::rng()); loop { let mut skipped_principal_ids = Vec::new(); let mut num_migrated = 0; for principal_id in principal_ids { let lock_key = format!("migrate_{principal_id}_lock"); let done_key = format!("migrate_{principal_id}_done"); if force_lock || in_memory .try_lock( KV_LOCK_HOUSEKEEPER, lock_key.as_bytes(), LOCK_WAIT_TIME_ACCOUNT, ) .await .caused_by(trc::location!())? { if in_memory .key_get::<()>(KeyValue::<()>::build_key( KV_LOCK_HOUSEKEEPER, done_key.as_bytes(), )) .await .caused_by(trc::location!())? .is_none() { migrate_principal_v0_11(server, principal_id) .await .caused_by(trc::location!())?; num_migrated += 1; in_memory .key_set( KeyValue::new( KeyValue::<()>::build_key( KV_LOCK_HOUSEKEEPER, done_key.as_bytes(), ), b"1".to_vec(), ) .expires(86400), ) .await .caused_by(trc::location!())?; } in_memory .remove_lock(KV_LOCK_HOUSEKEEPER, lock_key.as_bytes()) .await .caused_by(trc::location!())?; } else { skipped_principal_ids.push(principal_id); } } if !skipped_principal_ids.is_empty() { trc::event!( Server(trc::ServerEvent::Startup), Details = format!( "Migrated {num_migrated} accounts and {} are locked by another node, waiting 30 seconds.", skipped_principal_ids.len() ) ); tokio::time::sleep(LOCK_RETRY_TIME).await; principal_ids = skipped_principal_ids; } else { trc::event!( Server(trc::ServerEvent::Startup), Details = format!("Account migration completed.",) ); break; } } } Ok(()) } ================================================ FILE: crates/migration/src/v012.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ LOCK_RETRY_TIME, LOCK_WAIT_TIME_CORE, event_v1::migrate_calendar_events_v012, queue_v1::migrate_queue_v012, tasks_v1::migrate_tasks_v011, }; use common::{KV_LOCK_HOUSEKEEPER, Server}; use trc::AddContext; pub(crate) async fn migrate_v0_12(server: &Server, migrate_tasks: bool) -> trc::Result<()> { let force_lock = std::env::var("FORCE_LOCK").is_ok(); let in_memory = server.in_memory_store(); loop { if force_lock || in_memory .try_lock( KV_LOCK_HOUSEKEEPER, b"migrate_core_lock", LOCK_WAIT_TIME_CORE, ) .await .caused_by(trc::location!())? { migrate_queue_v012(server) .await .caused_by(trc::location!())?; if migrate_tasks { migrate_tasks_v011(server) .await .caused_by(trc::location!())?; } in_memory .remove_lock(KV_LOCK_HOUSEKEEPER, b"migrate_core_lock") .await .caused_by(trc::location!())?; break; } else { trc::event!( Server(trc::ServerEvent::Startup), Details = format!("Migration lock busy, waiting 30 seconds.",) ); tokio::time::sleep(LOCK_RETRY_TIME).await; } } if migrate_tasks { migrate_calendar_events_v012(server) .await .caused_by(trc::location!()) } else { Ok(()) } } ================================================ FILE: crates/migration/src/v013.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ LOCK_RETRY_TIME, LOCK_WAIT_TIME_ACCOUNT, LOCK_WAIT_TIME_CORE, get_document_ids, principal_v2::{migrate_principal_v0_13, migrate_principals_v0_13}, }; use common::{KV_LOCK_HOUSEKEEPER, Server}; use store::{ dispatch::lookup::KeyValue, rand::{self, seq::SliceRandom}, }; use trc::AddContext; use types::collection::Collection; pub(crate) async fn migrate_v0_13(server: &Server) -> trc::Result<()> { let force_lock = std::env::var("FORCE_LOCK").is_ok(); let in_memory = server.in_memory_store(); let principal_ids; loop { if force_lock || in_memory .try_lock( KV_LOCK_HOUSEKEEPER, b"migrate_core_lock", LOCK_WAIT_TIME_CORE, ) .await .caused_by(trc::location!())? { if in_memory .key_get::<()>(KeyValue::<()>::build_key( KV_LOCK_HOUSEKEEPER, b"migrate_core_done", )) .await .caused_by(trc::location!())? .is_none() { principal_ids = migrate_principals_v0_13(server) .await .caused_by(trc::location!())?; in_memory .key_set( KeyValue::new( KeyValue::<()>::build_key(KV_LOCK_HOUSEKEEPER, b"migrate_core_done"), b"1".to_vec(), ) .expires(86400), ) .await .caused_by(trc::location!())?; } else { principal_ids = get_document_ids(server, u32::MAX, Collection::Principal) .await .caused_by(trc::location!())? .unwrap_or_default(); trc::event!( Server(trc::ServerEvent::Startup), Details = format!("Migration completed by another node.",) ); } in_memory .remove_lock(KV_LOCK_HOUSEKEEPER, b"migrate_core_lock") .await .caused_by(trc::location!())?; break; } else { trc::event!( Server(trc::ServerEvent::Startup), Details = format!("Migration lock busy, waiting 30 seconds.",) ); tokio::time::sleep(LOCK_RETRY_TIME).await; } } if !principal_ids.is_empty() { let mut principal_ids = principal_ids.into_iter().collect::>(); principal_ids.shuffle(&mut rand::rng()); loop { let mut skipped_principal_ids = Vec::new(); let mut num_migrated = 0; for principal_id in principal_ids { let lock_key = format!("migrate_{principal_id}_lock"); let done_key = format!("migrate_{principal_id}_done"); if force_lock || in_memory .try_lock( KV_LOCK_HOUSEKEEPER, lock_key.as_bytes(), LOCK_WAIT_TIME_ACCOUNT, ) .await .caused_by(trc::location!())? { if in_memory .key_get::<()>(KeyValue::<()>::build_key( KV_LOCK_HOUSEKEEPER, done_key.as_bytes(), )) .await .caused_by(trc::location!())? .is_none() { migrate_principal_v0_13(server, principal_id) .await .caused_by(trc::location!())?; num_migrated += 1; in_memory .key_set( KeyValue::new( KeyValue::<()>::build_key( KV_LOCK_HOUSEKEEPER, done_key.as_bytes(), ), b"1".to_vec(), ) .expires(86400), ) .await .caused_by(trc::location!())?; } in_memory .remove_lock(KV_LOCK_HOUSEKEEPER, lock_key.as_bytes()) .await .caused_by(trc::location!())?; } else { skipped_principal_ids.push(principal_id); } } if !skipped_principal_ids.is_empty() { trc::event!( Server(trc::ServerEvent::Startup), Details = format!( "Migrated {num_migrated} accounts and {} are locked by another node, waiting 30 seconds.", skipped_principal_ids.len() ) ); tokio::time::sleep(LOCK_RETRY_TIME).await; principal_ids = skipped_principal_ids; } else { trc::event!( Server(trc::ServerEvent::Startup), Details = format!("Account migration completed.",) ); break; } } } Ok(()) } ================================================ FILE: crates/migration/src/v014.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ blob::migrate_blobs_v014, email_v2::migrate_emails_v014, encryption_v2::migrate_encryption_params_v014, queue_v2::migrate_queue_v014, tasks_v2::migrate_tasks_v014, }; use common::Server; use directory::backend::internal::manage::ManageDirectory; use email::submission::EmailSubmission; use groupware::{calendar::CalendarEventNotification, contact::ContactCard}; use std::sync::Arc; use store::{ SUBSPACE_INDEXES, SerializeInfallible, U32_LEN, U64_LEN, rand::{self, seq::SliceRandom}, write::{ AnyKey, BatchBuilder, IndexPropertyClass, Operation, ValueClass, ValueOp, key::KeySerializer, }, }; use tokio::sync::Semaphore; use trc::AddContext; use types::{ collection::Collection, field::{CalendarNotificationField, ContactField, EmailSubmissionField, IdentityField}, }; pub const SUBSPACE_BITMAP_ID: u8 = b'b'; pub const SUBSPACE_BITMAP_TAG: u8 = b'c'; pub const SUBSPACE_BITMAP_TEXT: u8 = b'v'; pub const SUBSPACE_FTS_INDEX: u8 = b'g'; pub const SUBSPACE_TELEMETRY_INDEX: u8 = b'w'; pub async fn migrate_v0_14(server: &Server) -> trc::Result<()> { // Migrate global data let mut tasks = Vec::new(); let _server = server.clone(); tasks.push(tokio::spawn( async move { migrate_queue_v014(&_server).await }, )); let _server = server.clone(); tasks.push(tokio::spawn( async move { migrate_blobs_v014(&_server).await }, )); let _server = server.clone(); tasks.push(tokio::spawn( async move { migrate_tasks_v014(&_server).await }, )); futures::future::join_all(tasks) .await .into_iter() .collect::, _>>() .map_err(|err| { trc::EventType::Server(trc::ServerEvent::ThreadError) .reason(err) .caused_by(trc::location!()) .details("Join Error") })??; // Migrate account data let mut principal_ids = server .store() .principal_ids(None, None) .await .unwrap_or_default() .into_iter() .collect::>(); principal_ids.shuffle(&mut rand::rng()); let semaphore = Arc::new(Semaphore::new( std::env::var("NUM_THREADS") .ok() .and_then(|s| s.parse::().ok()) .unwrap_or_else(|| num_cpus::get().min(2) * 2), )); let mut tasks = Vec::with_capacity(principal_ids.len()); let num_principals = principal_ids.len(); for principal_id in principal_ids { let permit = semaphore.clone().acquire_owned().await.unwrap(); let _server = server.clone(); tasks.push(tokio::spawn(async move { let result = migrate_principal_v0_14(&_server, principal_id).await; drop(permit); result })); } futures::future::join_all(tasks) .await .into_iter() .collect::, _>>() .map_err(|err| { trc::EventType::Server(trc::ServerEvent::ThreadError) .reason(err) .caused_by(trc::location!()) .details("Join Error") })??; trc::event!( Server(trc::ServerEvent::Startup), Details = format!("Migrated {num_principals} accounts") ); // Delete old subspaces for subspace in [ SUBSPACE_BITMAP_ID, SUBSPACE_BITMAP_TAG, SUBSPACE_BITMAP_TEXT, SUBSPACE_FTS_INDEX, SUBSPACE_TELEMETRY_INDEX, ] { server .store() .delete_range( AnyKey { subspace, key: vec![0u8], }, AnyKey { subspace, key: vec![u8::MAX; 32], }, ) .await .caused_by(trc::location!())?; } trc::event!( Server(trc::ServerEvent::Startup), Details = format!("Migration to v0.15 completed") ); Ok(()) } pub(crate) async fn migrate_principal_v0_14(server: &Server, account_id: u32) -> trc::Result<()> { let emails = migrate_emails_v014(server, account_id).await?; let params = migrate_encryption_params_v014(server, account_id).await?; let (num_contacts, num_calendars, num_email_submissions, num_identities) = migrate_indexes(server, account_id).await?; trc::event!( Server(trc::ServerEvent::Startup), Details = format!( "Migrated account {account_id}: {emails} emails, {params} encryption params, {num_contacts} contacts, {num_calendars} calendars, {num_email_submissions} submissions, and {num_identities} identities" ) ); Ok(()) } pub(crate) async fn migrate_indexes( server: &Server, account_id: u32, ) -> trc::Result<(usize, usize, usize, usize)> { /* EmailSubmissionField::UndoStatus => 41, EmailSubmissionField::EmailId => 83, EmailSubmissionField::ThreadId => 33, EmailSubmissionField::IdentityId => 95, EmailSubmissionField::SendAt => 24, */ /* ContactField::Created => 2, ContactField::Updated => 3, ContactField::Text => 4, */ /* CalendarField::Text => 1, CalendarField::Created => 2, CalendarField::Updated => 3, CalendarField::Start => 4, CalendarField::EventId => 5, */ /* EmailField::From => 87, EmailField::To => 35, EmailField::Cc => 74, EmailField::Bcc => 69, EmailField::Subject => 29, EmailField::Size => 27, EmailField::References => 20, EmailField::MailboxIds => 7, EmailField::ReceivedAt => 19, EmailField::SentAt => 26, EmailField::HasAttachment => 89, */ for (collection, fields) in [ ( Collection::Email, &[87u8, 35, 74, 69, 29, 27, 20, 7, 19, 26, 89][..], ), (Collection::EmailSubmission, &[41, 83, 33, 95, 24][..]), (Collection::ContactCard, &[1, 2, 3, 4][..]), (Collection::CalendarEvent, &[1, 2, 3, 4][..]), (Collection::CalendarEventNotification, &[2, 5][..]), ] { for index in fields { server .store() .delete_range( AnyKey { subspace: SUBSPACE_INDEXES, key: KeySerializer::new(U64_LEN * 3) .write(account_id) .write(u8::from(collection)) .write(*index) .finalize(), }, AnyKey { subspace: SUBSPACE_INDEXES, key: KeySerializer::new(U64_LEN * 4) .write(account_id) .write(u8::from(collection)) .write(*index) .write(&[u8::MAX; 8][..]) .finalize(), }, ) .await .caused_by(trc::location!())?; } } let mut indexes = Vec::new(); let mut num_contacts = 0; let mut num_calendars = 0; let mut num_email_submissions = 0; let mut num_identities = 0; for collection in [ Collection::ContactCard, Collection::CalendarEventNotification, Collection::EmailSubmission, Collection::Identity, ] { server .archives(account_id, collection, &(), |document_id, archive| { match collection { Collection::ContactCard => { let data = archive .unarchive_untrusted::() .caused_by(trc::location!())?; if let Some(email) = data.emails().next() { indexes.push(( collection, document_id, Operation::Index { field: ContactField::Email.into(), key: email.into_bytes(), set: true, }, )); } num_contacts += 1; indexes.push(( collection, document_id, Operation::Value { class: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: ContactField::CreatedToUpdated.into(), value: data.created.to_native() as u64, }), op: ValueOp::Set((data.modified.to_native() as u64).serialize()), }, )); } Collection::CalendarEventNotification => { let data = archive .unarchive_untrusted::() .caused_by(trc::location!())?; num_calendars += 1; indexes.push(( collection, document_id, Operation::Value { class: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: CalendarNotificationField::CreatedToId.into(), value: data.created.to_native() as u64, }), op: ValueOp::Set( data.event_id .as_ref() .map(|v| v.to_native()) .unwrap_or(u32::MAX) .serialize(), ), }, )); } Collection::EmailSubmission => { let data = archive .unarchive_untrusted::() .caused_by(trc::location!())?; num_email_submissions += 1; indexes.push(( collection, document_id, Operation::Value { class: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: EmailSubmissionField::Metadata.into(), value: data.send_at.to_native(), }), op: ValueOp::Set( KeySerializer::new(U32_LEN * 3 + 1) .write(data.email_id.to_native()) .write(data.thread_id.to_native()) .write(data.identity_id.to_native()) .write(data.undo_status.as_index()) .finalize(), ), }, )); } Collection::Identity => { num_identities += 1; indexes.push(( collection, document_id, Operation::Index { field: IdentityField::DocumentId.into(), key: vec![], set: true, }, )); } _ => unreachable!(), } Ok(true) }) .await .caused_by(trc::location!())?; } let mut batch = BatchBuilder::new(); for (collection, document_id, op) in indexes { batch .with_account_id(account_id) .with_collection(collection) .with_document(document_id) .any_op(op); if batch.is_large_batch() || batch.len() == 255 { server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); } } if !batch.is_empty() { server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } Ok(( num_contacts, num_calendars, num_email_submissions, num_identities, )) } ================================================ FILE: crates/nlp/Cargo.toml ================================================ [package] name = "nlp" version = "0.15.5" edition = "2024" [dependencies] utils = { path = "../utils" } xxhash-rust = { version = "0.8.5", features = ["xxh3"] } serde = { version = "1.0", features = ["derive"]} nohash = "0.2.0" ahash = { version = "0.8.3", features = ["serde"] } whatlang = "0.18" # Language detection rust-stemmers = "1.2" # Stemmers jieba-rs = "0.8" # Chinese stemmer lru-cache = "0.1.2" parking_lot = "0.12.1" psl = "2" maplit = "1.0.2" hashify = "0.2.1" rand = "0.9.2" rkyv = { version = "0.8.10", features = ["little_endian"] } [features] test_mode = [] [dev-dependencies] tokio = { version = "1.47", features = ["full"] } bincode = "1.3.3" ================================================ FILE: crates/nlp/src/classifier/adam.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::classifier::{Optimizer, model::FhClassifier}; pub struct Adam { parameters: Vec, bias: f32, learning_rate: f32, beta1: f32, beta2: f32, epsilon: f32, t: f32, m0: Vec, v0: Vec, m_bias: f32, v_bias: f32, // Step info bias2_sqrt: f32, alpha_t: f32, } impl Adam { pub fn new(n_parameters: usize, learning_rate: f32) -> Self { Adam { parameters: vec![0.0; n_parameters], learning_rate, beta1: 0.9, beta2: 0.999, epsilon: 1e-8, t: 0.0, m0: vec![0.0; n_parameters], v0: vec![0.0; n_parameters], m_bias: 0.0, v_bias: 0.0, bias: 0.0, bias2_sqrt: 0.0, alpha_t: 0.0, } } pub fn with_hyperparams(mut self, beta1: f32, beta2: f32, epsilon: f32) -> Self { self.beta1 = beta1; self.beta2 = beta2; self.epsilon = epsilon; self } pub fn with_initial_weights(self, value: f32) -> Self { Adam { parameters: vec![value; self.parameters.len()], ..self } } } impl Optimizer for Adam { #[inline(always)] fn step(&mut self) { self.t += 1.0; let bias1 = 1.0 - self.beta1.powf(self.t); self.bias2_sqrt = (1.0 - self.beta2.powf(self.t)).sqrt(); self.alpha_t = self.learning_rate / bias1; } #[inline(always)] fn update_param(&mut self, i: usize, g: f32) { self.m0[i] = self.beta1 * self.m0[i] + (1.0 - self.beta1) * g; self.v0[i] = self.beta2 * self.v0[i] + (1.0 - self.beta2) * g * g; self.parameters[i] -= self.alpha_t * self.m0[i] / (self.v0[i].sqrt() / self.bias2_sqrt + self.epsilon); } #[inline(always)] fn update_bias(&mut self, g: f32) { self.m_bias = self.beta1 * self.m_bias + (1.0 - self.beta1) * g; self.v_bias = self.beta2 * self.v_bias + (1.0 - self.beta2) * g * g; self.bias -= self.alpha_t * self.m_bias / (self.v_bias.sqrt() / self.bias2_sqrt + self.epsilon); } #[inline(always)] fn get_param(&self, idx: usize) -> f32 { self.parameters[idx] } #[inline(always)] fn get_bias(&self) -> f32 { self.bias } #[inline(always)] fn get_param_mut(&mut self, idx: usize) -> &mut f32 { &mut self.parameters[idx] } fn build_classifier(&self) -> FhClassifier { FhClassifier { parameters: self.parameters.clone(), bias: self.bias, } } fn num_parameters(&self) -> usize { self.parameters.len() } } ================================================ FILE: crates/nlp/src/classifier/feature.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::collections::HashMap; use xxhash_rust::xxh3::xxh3_64_with_seed; #[derive(Debug)] pub struct Sample { pub features: Vec, pub class: f32, } pub struct FhFeatureBuilder { pub(super) weight_mask: u64, } #[derive(Debug)] pub struct FhFeature { pub idx: usize, pub weight: f32, } #[derive(Debug)] pub struct CcfhFeature { pub idx_w1: usize, pub idx_w2: usize, pub idx_i: usize, pub weight: f32, } pub struct CcfhFeatureBuilder { pub(super) weight_mask: u64, pub(super) indicator_mask: u64, } pub trait FeatureWeight { fn idx(&self) -> usize; fn weight(&self) -> f32; fn weight_mut(&mut self) -> &mut f32; } pub trait UnprocessedFeature { fn prefix(&self) -> u16; fn value(&self) -> &[u8]; } impl FeatureWeight for FhFeature { fn weight(&self) -> f32 { self.weight } fn weight_mut(&mut self) -> &mut f32 { &mut self.weight } fn idx(&self) -> usize { self.idx } } impl FeatureWeight for CcfhFeature { fn weight(&self) -> f32 { self.weight } fn weight_mut(&mut self) -> &mut f32 { &mut self.weight } fn idx(&self) -> usize { self.idx_w1 } } impl FeatureBuilder for FhFeatureBuilder { type Feature = FhFeature; fn build_feature(&self, bytes: &[u8], weight: f32) -> FhFeature { let hash1 = xxh3_64_with_seed(bytes, 0); let sign = if hash1 & (1 << 63) == 0 { 1.0 } else { -1.0 }; FhFeature { idx: (hash1 & self.weight_mask) as usize, weight: sign * weight, } } } impl FeatureBuilder for CcfhFeatureBuilder { type Feature = CcfhFeature; fn build_feature(&self, bytes: &[u8], weight: f32) -> CcfhFeature { let hash1 = xxh3_64_with_seed(bytes, 0); let hash2 = xxh3_64_with_seed(bytes, 0x9E3779B97F4A7C15); let hash3 = xxh3_64_with_seed(bytes, 0x517CC1B727220A95); let sign = if hash3 & (1 << 63) == 0 { 1.0 } else { -1.0 }; CcfhFeature { idx_w1: (hash1 & self.weight_mask) as usize, idx_w2: (hash2 & self.weight_mask) as usize, idx_i: (hash3 & self.indicator_mask) as usize, weight: sign * weight, } } } pub trait FeatureBuilder { // Feature type associated type type Feature: FeatureWeight; fn build_feature(&self, bytes: &[u8], weight: f32) -> Self::Feature; fn scale(&self, features: &mut HashMap) { // Log frequency scaling for x in features.values_mut() { *x = x.ln_1p(); } } fn build( &self, features_in: &HashMap, account_id: Option, l2_normalize: bool, ) -> Vec { let mut features_out = Vec::with_capacity(features_in.len()); let mut buf = Vec::with_capacity(2 + 4 + 63); for (feature, count) in features_in { buf.extend_from_slice(&feature.prefix().to_be_bytes()); buf.extend_from_slice(feature.value()); features_out.push(self.build_feature(&buf, *count)); if let Some(account_id) = account_id { buf.extend_from_slice(&account_id.to_be_bytes()); features_out.push(self.build_feature(&buf, *count)); } buf.clear(); } // L2 normalization if l2_normalize { let sum_of_squares = features_out .iter() .map(|f| f.weight() as f64 * f.weight() as f64) .sum::(); if sum_of_squares > 0.0 { let norm = sum_of_squares.sqrt() as f32; for feature in &mut features_out { *feature.weight_mut() /= norm; } } } features_out } } impl Sample { pub fn new(features: Vec, class: bool) -> Self { Self { features, class: if class { 1.0 } else { 0.0 }, } } } impl AsRef> for Sample { fn as_ref(&self) -> &Sample { self } } ================================================ FILE: crates/nlp/src/classifier/ftrl.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::classifier::{Optimizer, model::FhClassifier}; #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug)] pub struct Ftrl { alpha: f64, beta: f64, l1_ratio: f64, l2_ratio: f64, zn: Vec, zn_bias: Zn, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Clone, Copy, Debug, Default)] pub struct Zn { z: f32, n: f64, } impl Ftrl { pub fn new(n_features: usize) -> Self { Ftrl { alpha: 2.0, beta: 1.0, l1_ratio: 0.001, l2_ratio: 0.0001, zn: vec![Zn::default(); n_features], zn_bias: Zn::default(), } } pub fn with_hyperparams(mut self, alpha: f64, beta: f64, l1_ratio: f64, l2_ratio: f64) -> Self { self.alpha = alpha; self.beta = beta; self.l1_ratio = l1_ratio; self.l2_ratio = l2_ratio; self } pub fn set_hyperparams(&mut self, alpha: f64, beta: f64, l1_ratio: f64, l2_ratio: f64) { self.alpha = alpha; self.beta = beta; self.l1_ratio = l1_ratio; self.l2_ratio = l2_ratio; } pub fn with_initial_weights(self, value: f32) -> Self { Ftrl { zn: vec![Zn { z: value, n: 0.0 }; self.zn.len()], ..self } } } impl Optimizer for Ftrl { #[inline(always)] fn update_param(&mut self, idx: usize, grad: f32) { let zn = &mut self.zn[idx]; let current_w = if zn.z.abs() as f64 <= self.l1_ratio { 0.0 } else { -(zn.z - zn.z.signum() * self.l1_ratio as f32) / (self.l2_ratio + (self.beta + zn.n.sqrt()) / self.alpha) as f32 }; let grad = grad as f64; let grad_sq = grad * grad; let sigma = ((zn.n + grad_sq).sqrt() - zn.n.sqrt()) / self.alpha; zn.z += (grad - sigma * current_w as f64) as f32; zn.n += grad_sq; } #[inline(always)] fn update_bias(&mut self, grad: f32) { let current_bias = -self.zn_bias.z / ((self.zn_bias.n.sqrt() + self.beta) / self.alpha + self.l2_ratio) as f32; let grad = grad as f64; let grad_sq = grad * grad; let sigma = ((self.zn_bias.n + grad_sq).sqrt() - self.zn_bias.n.sqrt()) / self.alpha; self.zn_bias.z += (grad - sigma * current_bias as f64) as f32; self.zn_bias.n += grad_sq; } #[inline(always)] fn get_param(&self, idx: usize) -> f32 { let zn = self.zn[idx]; if zn.z.abs() as f64 <= self.l1_ratio { 0.0 } else { -(zn.z - zn.z.signum() * self.l1_ratio as f32) / (self.l2_ratio + (self.beta + zn.n.sqrt()) / self.alpha) as f32 } } #[inline(always)] fn get_bias(&self) -> f32 { -self.zn_bias.z / ((self.zn_bias.n.sqrt() + self.beta) / self.alpha + self.l2_ratio) as f32 } fn step(&mut self) {} #[inline(always)] fn get_param_mut(&mut self, idx: usize) -> &mut f32 { &mut self.zn[idx].z } fn build_classifier(&self) -> FhClassifier { FhClassifier { parameters: self .zn .iter() .map(|zn| { if zn.z.abs() as f64 <= self.l1_ratio { 0.0 } else { -(zn.z - zn.z.signum() * self.l1_ratio as f32) / (self.l2_ratio + (self.beta + zn.n.sqrt()) / self.alpha) as f32 } }) .collect(), bias: self.get_bias(), } } fn num_parameters(&self) -> usize { self.zn.len() } } ================================================ FILE: crates/nlp/src/classifier/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::classifier::model::FhClassifier; pub mod adam; pub mod feature; pub mod ftrl; pub mod model; pub mod reservoir; pub mod sgd; pub mod train; const MAX_DLOSS: f32 = 1e4; pub trait Optimizer { fn step(&mut self); fn update_param(&mut self, i: usize, g: f32); fn update_bias(&mut self, g: f32); fn get_param(&self, idx: usize) -> f32; fn get_param_mut(&mut self, idx: usize) -> &mut f32; fn get_bias(&self) -> f32; fn build_classifier(&self) -> FhClassifier; fn num_parameters(&self) -> usize; } #[inline(always)] fn sigmoid(z: f32) -> f32 { let z = z.clamp(-35.0, 35.0); if z >= 0.0 { 1.0 / (1.0 + (-z).exp()) } else { let exp_z = z.exp(); exp_z / (1.0 + exp_z) } } #[inline(always)] fn gradient(y: f32, p: f32) -> f32 { if p > -16.0 { let exp_tmp = (-p).exp(); ((1.0 - y) - y * exp_tmp) / (1.0 + exp_tmp) } else { p.exp() - y } } ================================================ FILE: crates/nlp/src/classifier/model.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::classifier::{ feature::{CcfhFeature, CcfhFeatureBuilder, FhFeature, FhFeatureBuilder}, sigmoid, }; #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default)] pub struct FhClassifier { pub(crate) parameters: Vec, pub(crate) bias: f32, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default)] pub struct CcfhClassifier { pub(crate) parameters: Vec, pub(crate) indicators: Vec, pub(crate) bias: f32, } impl FhClassifier { pub fn predict_proba_sample(&self, features: &[FhFeature]) -> f32 { let mut z: f32 = 0.0; for f in features { z += self.parameters[f.idx] * f.weight; } sigmoid(z + self.bias) } pub fn predict(&self, features: &[FhFeature]) -> f32 { if self.predict_proba_sample(features) > 0.7 { 1.0 } else { 0.0 } } pub fn predict_batch(&self, test: I) -> Vec where I: IntoIterator, I::Item: AsRef>, { test.into_iter() .map(|features| self.predict(features.as_ref())) .collect() } pub fn feature_builder(&self) -> FhFeatureBuilder { FhFeatureBuilder { weight_mask: (self.parameters.len() - 1) as u64, } } pub fn parameters(&self) -> &[f32] { &self.parameters } pub fn bias(&self) -> f32 { self.bias } } impl CcfhClassifier { pub fn predict_proba_sample(&self, features: &[CcfhFeature]) -> f32 { let mut z: f32 = 0.0; for f in features { let q = self.indicators[f.idx_i]; let v1 = self.parameters[f.idx_w1]; let v2 = self.parameters[f.idx_w2]; z += (q * v1 + (1.0 - q) * v2) * f.weight; } sigmoid(z + self.bias) } pub fn predict(&self, features: &[CcfhFeature]) -> f32 { if self.predict_proba_sample(features) >= 0.5 { 1.0 } else { 0.0 } } pub fn predict_batch(&self, test: I) -> Vec where I: IntoIterator, I::Item: AsRef>, { test.into_iter() .map(|features| self.predict(features.as_ref())) .collect() } pub fn feature_builder(&self) -> CcfhFeatureBuilder { CcfhFeatureBuilder { weight_mask: (self.parameters.len() - 1) as u64, indicator_mask: (self.indicators.len() - 1) as u64, } } pub fn is_active(&self) -> bool { !self.parameters.is_empty() } } ================================================ FILE: crates/nlp/src/classifier/reservoir.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use rand::{Rng, seq::IndexedRandom}; #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug)] pub struct SampleReservoir { pub spam: SampleReservoirClass, pub ham: SampleReservoirClass, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug)] pub struct SampleReservoirClass { pub buffer: Vec, pub total_seen: u64, } impl SampleReservoir { pub fn update_reservoir(&mut self, item: &T, is_spam: bool, capacity: usize) { let class = if is_spam { &mut self.spam } else { &mut self.ham }; class.total_seen += 1; if class.buffer.len() < capacity { class.buffer.push(item.clone()); } else if let Some(buf) = class .buffer .get_mut(rand::rng().random_range(0..class.total_seen as usize)) { *buf = item.clone(); } } pub fn update_counts(&mut self, is_spam: bool) { let class = if is_spam { &mut self.spam } else { &mut self.ham }; class.total_seen += 1; } pub fn replay_samples( &mut self, count_needed: usize, is_spam: bool, ) -> impl Iterator { (if is_spam { &mut self.spam } else { &mut self.ham }) .buffer .choose_multiple(&mut rand::rng(), count_needed) } pub fn remove_sample(&mut self, item: &T, is_spam: bool) { let class = if is_spam { &mut self.spam } else { &mut self.ham }; if let Some(pos) = class.buffer.iter().position(|x| x == item) { class.buffer.swap_remove(pos); } } } impl Default for SampleReservoir { fn default() -> Self { SampleReservoir { spam: SampleReservoirClass { buffer: Vec::new(), total_seen: 0, }, ham: SampleReservoirClass { buffer: Vec::new(), total_seen: 0, }, } } } ================================================ FILE: crates/nlp/src/classifier/sgd.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::classifier::{Optimizer, gradient, model::FhClassifier}; #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default)] pub struct Sgd { parameters: Vec, bias: f32, alpha: f64, l1_ratio: f64, l2_ratio: f64, t: f64, w_scale: f32, optimal_init: f64, eta: f32, u: f32, q: Vec, } impl Sgd { pub fn new(n_features: usize, alpha: f64, l1_ratio: f64, l2_ratio: f64) -> Self { let typw = (1.0 / alpha.sqrt()).sqrt(); let initial_eta0 = typw / 1.0_f64.max(gradient(1.0, -typw as f32) as f64); let optimal_init = 1.0 / (initial_eta0 * alpha); Sgd { parameters: vec![0.0; n_features], bias: 0.0, alpha, l1_ratio, l2_ratio, t: 0.0, w_scale: 1.0, optimal_init, eta: initial_eta0 as f32, u: 0.0, q: vec![0.0; n_features], } } pub fn with_initial_parameters(self, value: f32) -> Self { Sgd { parameters: vec![value; self.parameters.len()], ..self } } fn maybe_rescale(&mut self) { if !(1e-6..=1e6).contains(&self.w_scale) { for w in &mut self.parameters { *w *= self.w_scale; } self.w_scale = 1.0; } } #[inline(always)] fn apply_l1_penalty(&mut self) { if self.l1_ratio > 0.0 { for (z, q) in self.parameters.iter_mut().zip(self.q.iter_mut()) { let z_orig = *z; let scaled_z = *z * self.w_scale; if scaled_z > 0.0 { *z = (*z - (self.u + *q) / self.w_scale).max(0.0); } else if scaled_z < 0.0 { *z = (*z + (self.u - *q) / self.w_scale).min(0.0); } *q += self.w_scale * (z_orig - *z); } } } } impl Optimizer for Sgd { fn step(&mut self) { self.t += 1.0; self.eta = (1.0 / ((self.alpha) * (self.optimal_init + self.t - 1.0))) as f32; self.w_scale *= 1.0 - ((1.0 - self.l1_ratio) as f32 * self.eta * self.l2_ratio as f32); self.u += self.eta * self.l1_ratio as f32 * self.alpha as f32; } fn update_param(&mut self, i: usize, g: f32) { self.parameters[i] += (-self.eta * g) / self.w_scale; } fn update_bias(&mut self, g: f32) { self.bias += -self.eta * g; self.maybe_rescale(); self.apply_l1_penalty(); } #[inline(always)] fn get_param(&self, idx: usize) -> f32 { self.parameters[idx] * self.w_scale } #[inline(always)] fn get_bias(&self) -> f32 { self.bias } #[inline(always)] fn get_param_mut(&mut self, idx: usize) -> &mut f32 { &mut self.parameters[idx] } fn build_classifier(&self) -> FhClassifier { FhClassifier { parameters: self.parameters.iter().map(|w| w * self.w_scale).collect(), bias: self.bias, } } fn num_parameters(&self) -> usize { self.parameters.len() } } #[cfg(test)] pub mod tests { use crate::classifier::{ Optimizer, adam::Adam, feature::{ CcfhFeature, CcfhFeatureBuilder, FeatureBuilder, FhFeature, FhFeatureBuilder, Sample, UnprocessedFeature, }, ftrl::Ftrl, train::{CcfhTrainer, FhTrainer}, }; use rand::{SeedableRng, rngs::StdRng, seq::SliceRandom}; use std::{ collections::HashMap, fs::File, io::{BufRead, BufReader}, time::Instant, }; #[ignore] #[test] fn text_classifier() { let reader = BufReader::new( File::open("/Users/me/code/playground/phishing_email.csv") .expect("Could not open file"), ); let mut samples = Vec::with_capacity(1024); let time = Instant::now(); for line in reader.lines().skip(1) { let line = line.unwrap(); let (text, class) = line.trim().rsplit_once(',').unwrap(); //let (class, text) = line.trim().split_once(',').unwrap(); let text = text.trim_start_matches('"').trim_end_matches('"'); samples.push((text.to_string(), class == "1")); } println!("Loaded {} samples in {:?}", samples.len(), time.elapsed()); samples.shuffle(&mut StdRng::seed_from_u64(42)); let (train_samples, test_samples) = train_test_split(&samples, 0.2); println!( "Training samples: {}, Testing samples: {}", train_samples.len(), test_samples.len() ); const FH_SIZE: usize = 16; const CCFH_SIZE: usize = FH_SIZE - 2; let mut rng = StdRng::seed_from_u64(42); let fh_builder = FhFeatureBuilder { weight_mask: (1 << FH_SIZE) - 1, }; let mut fh_train_samples = build_fh_samples(train_samples.as_slice(), &fh_builder); fh_train_samples.shuffle(&mut rng); let fh_test_samples = build_fh_samples(test_samples.as_slice(), &fh_builder); let ccfh_builder = CcfhFeatureBuilder { weight_mask: (1 << FH_SIZE) - 1, indicator_mask: (1 << CCFH_SIZE) - 1, }; let mut ccfh_train_samples = build_ccfh_samples(train_samples.as_slice(), &ccfh_builder); ccfh_train_samples.shuffle(&mut rng); let ccfh_test_samples = build_ccfh_samples(test_samples.as_slice(), &ccfh_builder); fh_model_stats( "FTRL", FhTrainer::new(Ftrl::new(1 << FH_SIZE)), &fh_train_samples, &fh_test_samples, ); ccfh_model_stats( "FTRL + FTRL", CcfhTrainer::new( Ftrl::new(1 << FH_SIZE), Ftrl::new(1 << CCFH_SIZE).with_initial_weights(0.5), ), &ccfh_train_samples, &ccfh_test_samples, ); fh_model_stats( "Adam", FhTrainer::new(Adam::new(1 << FH_SIZE, 0.01)), &fh_train_samples, &fh_test_samples, ); ccfh_model_stats( "Adam + Adam", CcfhTrainer::new( Adam::new(1 << FH_SIZE, 0.01), Adam::new(1 << CCFH_SIZE, 0.01).with_initial_weights(0.5), ), &ccfh_train_samples, &ccfh_test_samples, ); /*fh_model_stats( "SGD", FhTrainer::new(Sgd::new(1 << FH_SIZE, 0.0001, 0.0, 0.0001)), &fh_train_samples, &fh_test_samples, ); ccfh_model_stats( "FTRL + SGD", CcfhTrainer::new( Ftrl::new(1 << FH_SIZE), Sgd::new(1 << CCFH_SIZE, 0.0001, 0.0, 0.0001).with_initial_parameters(0.5), ), &ccfh_train_samples, &ccfh_test_samples, );*/ } fn fh_model_stats( name: &str, mut model: FhTrainer, train_samples: &[Sample], test_samples: &[Sample], ) { print!("⏳ Training {}... ", name); let time = Instant::now(); let mut batch = Vec::new(); for sample in train_samples { batch.push(sample); if batch.len() == 128 { model.fit(&mut batch, 5); batch.clear(); } } if !batch.is_empty() { model.fit(&mut batch, 5); } println!(" trained in {:?}", time.elapsed()); let y_pred = model .build_classifier() .predict_batch(test_samples.iter().map(|s| &s.features)); let y_train: Vec = test_samples.iter().map(|s| s.class).collect(); println!("Accuracy: {:.4}", accuracy_score(&y_train, &y_pred)); println!("Precision: {:.4}", precision_score(&y_train, &y_pred, 1.0)); println!("Recall: {:.4}", recall_score(&y_train, &y_pred, 1.0)); println!("F1 Score: {:.4}", f1_score(&y_train, &y_pred, 1.0)); } fn ccfh_model_stats( name: &str, mut model: CcfhTrainer, train_samples: &[Sample], test_samples: &[Sample], ) { print!("⏳ Training {}... ", name); let time = Instant::now(); let mut batch = Vec::new(); for sample in train_samples { batch.push(sample); if batch.len() == 128 { model.fit(&mut batch, 5); batch.clear(); } } if !batch.is_empty() { model.fit(&mut batch, 5); } println!(" trained in {:?}", time.elapsed()); let y_pred = model .build_classifier() .predict_batch(test_samples.iter().map(|s| &s.features)); let y_train: Vec = test_samples.iter().map(|s| s.class).collect(); println!("Accuracy: {:.4}", accuracy_score(&y_train, &y_pred)); println!("Precision: {:.4}", precision_score(&y_train, &y_pred, 1.0)); println!("Recall: {:.4}", recall_score(&y_train, &y_pred, 1.0)); println!("F1 Score: {:.4}", f1_score(&y_train, &y_pred, 1.0)); } fn accuracy_score(y_true: &[f32], y_pred: &[f32]) -> f32 { y_true .iter() .zip(y_pred.iter()) .filter(|(true_val, pred_val)| **true_val == **pred_val) .count() as f32 / y_true.len() as f32 } fn precision_score(y_true: &[f32], y_pred: &[f32], positive_class: f32) -> f32 { let true_positives = y_true .iter() .zip(y_pred.iter()) .filter(|(true_val, pred_val)| { **pred_val == positive_class && **true_val == positive_class }) .count() as f32; let predicted_positives = y_pred .iter() .filter(|pred_val| **pred_val == positive_class) .count() as f32; if predicted_positives == 0.0 { 0.0 } else { true_positives / predicted_positives } } fn recall_score(y_true: &[f32], y_pred: &[f32], positive_class: f32) -> f32 { let true_positives = y_true .iter() .zip(y_pred.iter()) .filter(|(true_val, pred_val)| { **pred_val == positive_class && **true_val == positive_class }) .count() as f32; let actual_positives = y_true .iter() .filter(|true_val| **true_val == positive_class) .count() as f32; if actual_positives == 0.0 { 0.0 } else { true_positives / actual_positives } } fn f1_score(y_true: &[f32], y_pred: &[f32], positive_class: f32) -> f32 { let precision = precision_score(y_true, y_pred, positive_class); let recall = recall_score(y_true, y_pred, positive_class); if precision + recall == 0.0 { 0.0 } else { 2.0 * (precision * recall) / (precision + recall) } } #[allow(clippy::type_complexity)] pub fn train_test_split( data: &[(String, bool)], test_size: f32, ) -> (Vec<(&String, bool)>, Vec<(&String, bool)>) { let mut class_0: Vec<(&String, bool)> = Vec::new(); let mut class_1: Vec<(&String, bool)> = Vec::new(); for (sample, class) in data { if !*class { class_0.push((sample, *class)); } else { class_1.push((sample, *class)); } } let test_count_0 = (class_0.len() as f32 * test_size).round() as usize; let test_count_1 = (class_1.len() as f32 * test_size).round() as usize; let (test_0, train_0) = class_0.split_at(test_count_0); let (test_1, train_1) = class_1.split_at(test_count_1); let mut train = Vec::new(); let mut test = Vec::new(); train.extend_from_slice(train_0); train.extend_from_slice(train_1); test.extend_from_slice(test_0); test.extend_from_slice(test_1); (train, test) } pub fn build_fh_samples( data: &[(&String, bool)], builder: &FhFeatureBuilder, ) -> Vec> { let mut samples = Vec::with_capacity(data.len()); for (text, class) in data { let mut sample: HashMap = HashMap::new(); for word in text.split_whitespace() { *sample.entry(word.to_string()).or_default() += 1.0; } builder.scale(&mut sample); samples.push(Sample { features: builder.build(&sample, 12345.into(), true), class: if *class { 1.0 } else { 0.0 }, }); } samples } pub fn build_ccfh_samples( data: &[(&String, bool)], builder: &CcfhFeatureBuilder, ) -> Vec> { let mut samples = Vec::with_capacity(data.len()); for (text, class) in data { let mut sample: HashMap = HashMap::new(); for word in text.split_whitespace() { *sample.entry(word.to_string()).or_default() += 1.0; } builder.scale(&mut sample); samples.push(Sample { features: builder.build(&sample, 12345.into(), true), class: if *class { 1.0 } else { 0.0 }, }); } samples } impl UnprocessedFeature for String { fn prefix(&self) -> u16 { 0 } fn value(&self) -> &[u8] { self.as_bytes() } } } ================================================ FILE: crates/nlp/src/classifier/train.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::classifier::{ MAX_DLOSS, Optimizer, feature::{CcfhFeature, CcfhFeatureBuilder, FhFeature, FhFeatureBuilder, Sample}, gradient, model::{CcfhClassifier, FhClassifier}, }; use rand::{SeedableRng, rngs::StdRng, seq::SliceRandom}; #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default)] pub struct FhTrainer { pub optimizer: T, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default)] pub struct CcfhTrainer { pub w_optimizer: W, pub i_optimizer: I, } impl FhTrainer { pub fn new(optimizer: T) -> Self { FhTrainer { optimizer } } pub fn fit(&mut self, samples: &mut [impl AsRef>], num_epochs: usize) { for _ in 0..num_epochs { samples.shuffle(&mut StdRng::seed_from_u64(42)); for sample in samples.iter() { let sample = sample.as_ref(); let mut dot: f32 = 0.0; for f in &sample.features { dot += self.optimizer.get_param(f.idx) * f.weight; } let p = dot + self.optimizer.get_bias(); let dloss = gradient(sample.class, p).clamp(-MAX_DLOSS, MAX_DLOSS); self.optimizer.step(); for f in &sample.features { self.optimizer.update_param(f.idx, dloss * f.weight); } self.optimizer.update_bias(dloss); } } } pub fn feature_builder(&self) -> FhFeatureBuilder { FhFeatureBuilder { weight_mask: (self.optimizer.num_parameters() - 1) as u64, } } pub fn build_classifier(&self) -> FhClassifier { self.optimizer.build_classifier() } pub fn optimizer(&self) -> &T { &self.optimizer } pub fn optimizer_mut(&mut self) -> &mut T { &mut self.optimizer } } impl CcfhTrainer { pub fn new(w_optimizer: W, i_optimizer: I) -> Self { CcfhTrainer { w_optimizer, i_optimizer, } } pub fn fit(&mut self, samples: &mut [impl AsRef>], num_epochs: usize) { for _ in 0..num_epochs { samples.shuffle(&mut StdRng::seed_from_u64(42)); for sample in samples.iter() { let sample = sample.as_ref(); let mut dot: f32 = 0.0; for f in &sample.features { let q = self.i_optimizer.get_param(f.idx_i); let v1 = self.w_optimizer.get_param(f.idx_w1); let v2 = self.w_optimizer.get_param(f.idx_w2); dot += (q * v1 + (1.0 - q) * v2) * f.weight; } let p = dot + self.w_optimizer.get_bias(); let dloss = gradient(sample.class, p).clamp(-MAX_DLOSS, MAX_DLOSS); self.w_optimizer.step(); self.i_optimizer.step(); for f in &sample.features { let q = self.i_optimizer.get_param(f.idx_i); let v1 = self.w_optimizer.get_param(f.idx_w1); let v2 = self.w_optimizer.get_param(f.idx_w2); // Update weights let d_v1 = f.weight * q; let d_v2 = f.weight * (1.0 - q); self.w_optimizer.update_param(f.idx_w1, dloss * d_v1); self.w_optimizer.update_param(f.idx_w2, dloss * d_v2); // Update indicator let d_q = (v1 - v2) * f.weight; self.i_optimizer.update_param(f.idx_i, dloss * d_q); let fi = self.i_optimizer.get_param_mut(f.idx_i); *fi = fi.clamp(0.0, 1.0); } self.w_optimizer.update_bias(dloss); } } } pub fn feature_builder(&self) -> CcfhFeatureBuilder { CcfhFeatureBuilder { weight_mask: (self.w_optimizer.num_parameters() - 1) as u64, indicator_mask: (self.i_optimizer.num_parameters() - 1) as u64, } } pub fn build_classifier(&self) -> CcfhClassifier { let w_classifier = self.w_optimizer.build_classifier(); let i_classifier = self.i_optimizer.build_classifier(); CcfhClassifier { parameters: w_classifier.parameters, indicators: i_classifier.parameters, bias: w_classifier.bias, } } pub fn w_optimizer(&self) -> &W { &self.w_optimizer } pub fn w_optimizer_mut(&mut self) -> &mut W { &mut self.w_optimizer } pub fn i_optimizer(&self) -> &I { &self.i_optimizer } pub fn i_optimizer_mut(&mut self) -> &mut I { &mut self.i_optimizer } } ================================================ FILE: crates/nlp/src/language/detect.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::Language; use ahash::AHashMap; use whatlang::{Lang, detect}; pub const MIN_LANGUAGE_SCORE: f64 = 0.6; #[derive(Debug)] struct WeightedAverage { weight: usize, occurrences: usize, confidence: f64, } #[derive(Debug)] pub struct LanguageDetector { lang_detected: AHashMap, } impl Default for LanguageDetector { fn default() -> Self { Self::new() } } impl LanguageDetector { pub fn new() -> LanguageDetector { LanguageDetector { lang_detected: AHashMap::default(), } } pub fn detect(&mut self, text: &str, min_score: f64) -> Language { if let Some((language, confidence)) = LanguageDetector::detect_single(text) { let w = self .lang_detected .entry(language) .or_insert_with(|| WeightedAverage { weight: 0, confidence: 0.0, occurrences: 0, }); w.occurrences += 1; w.weight += text.len(); w.confidence += confidence * text.len() as f64; if confidence < min_score { Language::Unknown } else { language } } else { Language::Unknown } } pub fn most_frequent_language(&self) -> Option { self.lang_detected .iter() .filter(|(l, _)| !matches!(l, Language::None)) .max_by(|(_, a), (_, b)| { ((a.confidence / a.weight as f64) * a.occurrences as f64) .partial_cmp(&((b.confidence / b.weight as f64) * b.occurrences as f64)) .unwrap_or(std::cmp::Ordering::Less) }) .map(|(l, _)| *l) } pub fn detect_single(text: &str) -> Option<(Language, f64)> { detect(text).map(|info| { ( match info.lang() { Lang::Epo => Language::Esperanto, Lang::Eng => Language::English, Lang::Rus => Language::Russian, Lang::Cmn => Language::Mandarin, Lang::Spa => Language::Spanish, Lang::Por => Language::Portuguese, Lang::Ita => Language::Italian, Lang::Ben => Language::Bengali, Lang::Fra => Language::French, Lang::Deu => Language::German, Lang::Ukr => Language::Ukrainian, Lang::Kat => Language::Georgian, Lang::Ara => Language::Arabic, Lang::Hin => Language::Hindi, Lang::Jpn => Language::Japanese, Lang::Heb => Language::Hebrew, Lang::Yid => Language::Yiddish, Lang::Pol => Language::Polish, Lang::Amh => Language::Amharic, Lang::Jav => Language::Javanese, Lang::Kor => Language::Korean, Lang::Nob => Language::Bokmal, Lang::Dan => Language::Danish, Lang::Swe => Language::Swedish, Lang::Fin => Language::Finnish, Lang::Tur => Language::Turkish, Lang::Nld => Language::Dutch, Lang::Hun => Language::Hungarian, Lang::Ces => Language::Czech, Lang::Ell => Language::Greek, Lang::Bul => Language::Bulgarian, Lang::Bel => Language::Belarusian, Lang::Mar => Language::Marathi, Lang::Kan => Language::Kannada, Lang::Ron => Language::Romanian, Lang::Slv => Language::Slovene, Lang::Hrv => Language::Croatian, Lang::Srp => Language::Serbian, Lang::Mkd => Language::Macedonian, Lang::Lit => Language::Lithuanian, Lang::Lav => Language::Latvian, Lang::Est => Language::Estonian, Lang::Tam => Language::Tamil, Lang::Vie => Language::Vietnamese, Lang::Urd => Language::Urdu, Lang::Tha => Language::Thai, Lang::Guj => Language::Gujarati, Lang::Uzb => Language::Uzbek, Lang::Pan => Language::Punjabi, Lang::Aze => Language::Azerbaijani, Lang::Ind => Language::Indonesian, Lang::Tel => Language::Telugu, Lang::Pes => Language::Persian, Lang::Mal => Language::Malayalam, Lang::Ori => Language::Oriya, Lang::Mya => Language::Burmese, Lang::Nep => Language::Nepali, Lang::Sin => Language::Sinhalese, Lang::Khm => Language::Khmer, Lang::Tuk => Language::Turkmen, Lang::Aka => Language::Akan, Lang::Zul => Language::Zulu, Lang::Sna => Language::Shona, Lang::Afr => Language::Afrikaans, Lang::Lat => Language::Latin, Lang::Slk => Language::Slovak, Lang::Cat => Language::Catalan, Lang::Tgl => Language::Tagalog, Lang::Hye => Language::Armenian, _ => Language::Unknown, }, info.confidence(), ) }) } } #[cfg(test)] mod tests { use super::*; #[test] fn detect_languages() { let inputs = [ ( "The quick brown fox jumps over the lazy dog", Language::English, ), ( "Jovencillo emponzoñado de whisky: ¡qué figurota exhibe!", Language::Spanish, ), ( "Ma la volpe col suo balzo ha raggiunto il quieto Fido", Language::Italian, ), ( "Jaz em prisão bota que vexa dez cegonhas felizes", Language::Portuguese, ), ( "Zwölf Boxkämpfer jagten Victor quer über den großen Sylter Deich", Language::German, ), ("עטלף אבק נס דרך מזגן שהתפוצץ כי חם", Language::Hebrew), ( "Съешь ещё этих мягких французских булок, да выпей же чаю", Language::Russian, ), ( "Чуєш їх, доцю, га? Кумедна ж ти, прощайся без ґольфів!", Language::Ukrainian, ), ( "Љубазни фењерџија чађавог лица хоће да ми покаже штос", Language::Serbian, ), ( "Pijamalı hasta yağız şoföre çabucak güvendi", Language::Turkish, ), ("己所不欲,勿施于人。", Language::Mandarin), ("井の中の蛙大海を知らず", Language::Japanese), ("시작이 반이다", Language::Korean), ]; let mut detector = LanguageDetector::new(); for input in inputs.iter() { assert_eq!(detector.detect(input.0, 0.0), input.1); } } #[test] fn weighted_language() { let mut detector = LanguageDetector::new(); for lang in [ (Language::Spanish, 0.5, 70), (Language::Japanese, 0.2, 100), (Language::Japanese, 0.3, 100), (Language::Japanese, 0.4, 200), (Language::English, 0.7, 50), ] .iter() { let w = detector .lang_detected .entry(lang.0) .or_insert_with(|| WeightedAverage { weight: 0, confidence: 0.0, occurrences: 0, }); w.occurrences += 1; w.weight += lang.2; w.confidence += lang.1 * lang.2 as f64; } assert_eq!(detector.most_frequent_language(), Some(Language::Japanese)); } } ================================================ FILE: crates/nlp/src/language/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod detect; pub mod search_snippet; pub mod stemmer; pub mod stopwords; use self::detect::LanguageDetector; use crate::tokenizers::{ Token, chinese::ChineseTokenizer, japanese::JapaneseTokenizer, space::SpaceTokenizer, word::WordTokenizer, }; use std::borrow::Cow; use utils::config::utils::ParseValue; pub type LanguageTokenizer<'x> = Box>> + 'x + Sync + Send>; impl Language { pub fn tokenize_text<'x>( &self, text: &'x str, max_token_length: usize, ) -> LanguageTokenizer<'x> { match self { Language::Japanese => Box::new( JapaneseTokenizer::new(WordTokenizer::new(text, usize::MAX)) .filter(move |t| t.word.len() <= max_token_length), ), Language::Mandarin => Box::new( ChineseTokenizer::new(WordTokenizer::new(text, usize::MAX)) .filter(move |t| t.word.len() <= max_token_length), ), Language::None => { Box::new( SpaceTokenizer::new(text, max_token_length).map(|word| Token { word: word.into(), from: 0, to: 0, }), ) } _ => Box::new(WordTokenizer::new(text, max_token_length)), } } } #[derive( Debug, PartialEq, Clone, Copy, Hash, Eq, serde::Serialize, serde::Deserialize, Default, )] pub enum Language { Esperanto = 0, #[default] English = 1, Russian = 2, Mandarin = 3, Spanish = 4, Portuguese = 5, Italian = 6, Bengali = 7, French = 8, German = 9, Ukrainian = 10, Georgian = 11, Arabic = 12, Hindi = 13, Japanese = 14, Hebrew = 15, Yiddish = 16, Polish = 17, Amharic = 18, Javanese = 19, Korean = 20, Bokmal = 21, Danish = 22, Swedish = 23, Finnish = 24, Turkish = 25, Dutch = 26, Hungarian = 27, Czech = 28, Greek = 29, Bulgarian = 30, Belarusian = 31, Marathi = 32, Kannada = 33, Romanian = 34, Slovene = 35, Croatian = 36, Serbian = 37, Macedonian = 38, Lithuanian = 39, Latvian = 40, Estonian = 41, Tamil = 42, Vietnamese = 43, Urdu = 44, Thai = 45, Gujarati = 46, Uzbek = 47, Punjabi = 48, Azerbaijani = 49, Indonesian = 50, Telugu = 51, Persian = 52, Malayalam = 53, Oriya = 54, Burmese = 55, Nepali = 56, Sinhalese = 57, Khmer = 58, Turkmen = 59, Akan = 60, Zulu = 61, Shona = 62, Afrikaans = 63, Latin = 64, Slovak = 65, Catalan = 66, Tagalog = 67, Armenian = 68, Unknown = 69, None = 70, } impl Language { pub fn is_unknown(&self) -> bool { matches!(self, Language::Unknown) } pub fn from_iso_639(code: &str) -> Option { hashify::map!( code.split_once('-').map(|c| c.0).unwrap_or(code).as_bytes(), Language, "en" => Language::English, "es" => Language::Spanish, "pt" => Language::Portuguese, "it" => Language::Italian, "fr" => Language::French, "de" => Language::German, "da" => Language::Danish, "ru" => Language::Russian, "zh" => Language::Mandarin, "ja" => Language::Japanese, "ar" => Language::Arabic, "hi" => Language::Hindi, "ko" => Language::Korean, "bn" => Language::Bengali, "he" => Language::Hebrew, "ur" => Language::Urdu, "fa" => Language::Persian, "ml" => Language::Malayalam, "or" => Language::Oriya, "my" => Language::Burmese, "ne" => Language::Nepali, "si" => Language::Sinhalese, "km" => Language::Khmer, "tk" => Language::Turkmen, "am" => Language::Amharic, "az" => Language::Azerbaijani, "id" => Language::Indonesian, "te" => Language::Telugu, "ta" => Language::Tamil, "vi" => Language::Vietnamese, "gu" => Language::Gujarati, "pa" => Language::Punjabi, "uz" => Language::Uzbek, "hy" => Language::Armenian, "ka" => Language::Georgian, "la" => Language::Latin, "sl" => Language::Slovene, "hr" => Language::Croatian, "sr" => Language::Serbian, "mk" => Language::Macedonian, "lt" => Language::Lithuanian, "lv" => Language::Latvian, "et" => Language::Estonian, "tl" => Language::Tagalog, "af" => Language::Afrikaans, "zu" => Language::Zulu, "sn" => Language::Shona, "ak" => Language::Akan, "ca" => Language::Catalan, "el" => Language::Greek, "sv" => Language::Swedish, "pl" => Language::Polish ) .copied() } } impl Language { pub fn detect(text: String, default: Language) -> (String, Language) { if let Some((l, t)) = text .split_once(':') .and_then(|(l, t)| (Language::from_iso_639(l)?, t).into()) { (t.to_string(), l) } else { let l = LanguageDetector::detect_single(&text) .and_then(|(l, c)| if c > 0.3 { Some(l) } else { None }) .unwrap_or(default); (text, l) } } } impl ParseValue for Language { fn parse_value(value: &str) -> utils::config::Result { Language::from_iso_639(value).ok_or_else(|| format!("Invalid language code: {}", value)) } } ================================================ FILE: crates/nlp/src/language/search_snippet.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::Language; fn escape_char(c: char, string: &mut String) { match c { '&' => string.push_str("&"), '<' => string.push_str("<"), '>' => string.push_str(">"), '"' => string.push_str("""), '\n' | '\r' => string.push(' '), _ => string.push(c), } } fn escape_char_len(c: char) -> usize { match c { '&' => "&".len(), '<' => "<".len(), '>' => ">".len(), '"' => """.len(), '\r' | '\n' => 1, _ => c.len_utf8(), } } pub struct Term { offset: usize, len: usize, } pub fn generate_snippet( text: &str, needles: &[impl AsRef], language: Language, is_exact: bool, ) -> Option { let mut terms = Vec::new(); if is_exact { let tokens = language.tokenize_text(text, 200).collect::>(); for tokens in tokens.windows(needles.len()) { if needles .iter() .zip(tokens) .all(|(needle, token)| needle.as_ref() == token.word.as_ref()) { for token in tokens { terms.push(Term { offset: token.from, len: token.to - token.from, }); } } } } else { for token in language.tokenize_text(text, 200) { if needles.iter().any(|needle| { let needle = needle.as_ref(); needle == token.word.as_ref() || needle.len() > 2 && token.word.contains(needle) }) { terms.push(Term { offset: token.from, len: token.to - token.from, }); } } } if terms.is_empty() { return None; } let mut snippet = String::with_capacity(text.len()); let start_offset = terms.first()?.offset; if start_offset > 0 { let mut word_count = 0; let mut from_offset = 0; let mut last_is_space = false; if text.len() > 240 { for (pos, char) in text.get(0..start_offset)?.char_indices().rev() { // Add up to 2 words or 40 characters of context if char.is_whitespace() { if !last_is_space { word_count += 1; if word_count == 3 { break; } last_is_space = true; } } else { last_is_space = false; } from_offset = pos; if start_offset - from_offset >= 40 { break; } } } last_is_space = false; for char in text.get(from_offset..start_offset)?.chars() { if !char.is_whitespace() { last_is_space = false; } else { if last_is_space { continue; } last_is_space = true; } escape_char(char, &mut snippet); } } let mut terms = terms.iter().peekable(); 'outer: while let Some(term) = terms.next() { if snippet.len() + ("".len() * 2) + term.len + 1 > 255 { break; } snippet.push_str(""); snippet.push_str(text.get(term.offset..term.offset + term.len)?); snippet.push_str(""); let next_offset = if let Some(next_term) = terms.peek() { next_term.offset } else { text.len() }; let mut last_is_space = false; for char in text.get(term.offset + term.len..next_offset)?.chars() { if !char.is_whitespace() { last_is_space = false; } else { if last_is_space { continue; } last_is_space = true; } if snippet.len() + escape_char_len(char) <= 255 { escape_char(char, &mut snippet); } else { break 'outer; } } } Some(snippet) } #[cfg(test)] mod tests { use crate::language::{Language, search_snippet::generate_snippet}; #[test] fn search_snippets() { let inputs = [ ( vec![ "Help a friend from Abidjan Côte d'Ivoire", concat!( "When my mother died when she was given birth to me, my father took me so ", "special because I am motherless. Before the death of my late father on 22nd June ", "2013 in a private hospital here in Abidjan Côte d'Ivoire. He secretly called me on his ", "bedside and told me that he has a sum of $7.5M (Seven Million five Hundred ", "Thousand Dollars) left in a suspense account in a local bank here in Abidjan Côte ", "d'Ivoire, that he used my name as his only daughter for the next of kin in deposit of ", "the fund. ", "I am 24year old. Dear I am honorably seeking your assistance in the following ways. ", "1) To provide any bank account where this money would be transferred into. ", "2) To serve as the guardian of this fund. ", "3) To make arrangement for me to come over to your country to further my ", "education and to secure a residential permit for me in your country. ", "Moreover, I am willing to offer you 30 percent of the total sum as compensation for ", "your effort input after the successful transfer of this fund to your nominated ", "account overseas." ), ], vec![ ( vec!["côte"], vec![ "Help a friend from Abidjan Côte d'Ivoire", concat!( "in Abidjan Côte d'Ivoire. He secretly called me on his bedside ", "and told me that he has a sum of $7.5M (Seven Million five Hundred Thousand ", "Dollars) left in a suspense account in a local bank here in Abidjan ", "Côte d'Ivoire, that " ), ], ), ( vec!["your", "country"], vec![concat!( "honorably seeking your assistance in the following ways. ", "1) To provide any bank account where this money would be transferred into. 2) ", "To serve as the guardian of this fund. 3) To make arrangement for me to come ", "over to your " )], ), ( vec!["overseas"], vec!["nominated account overseas."], ), ], ), ( vec![ "孫子兵法", concat!( "<\"孫子兵法：\">", "孫子曰：兵者，國之大事，死生之地，存亡之道，不可不察也。", "孫子曰：凡用兵之法，馳車千駟，革車千乘，帶甲十萬；千里饋糧，則內外之費賓客之用，膠漆之材，", "車甲之奉，日費千金，然後十萬之師舉矣。", "孫子曰：凡用兵之法，全國為上，破國次之；全旅為上，破旅次之；全卒為上，破卒次之；全伍為上，破伍次之。", "是故百戰百勝，非善之善者也；不戰而屈人之兵，善之善者也。", "孫子曰：昔之善戰者，先為不可勝，以待敵之可勝，不可勝在己，可勝在敵。故善戰者，能為不可勝，不能使敵必可勝。", "故曰：勝可知，而不可為。", "兵者，詭道也。故能而示之不能，用而示之不用，近而示之遠，遠而示之近。利而誘之，亂而取之，實而備之，強而避之，", "怒而撓之，卑而驕之，佚而勞之，親而離之。攻其無備，出其不意，此兵家之勝，不可先傳也。", "夫未戰而廟算勝者，得算多也；未戰而廟算不勝者，得算少也；多算勝，少算不勝，而況於無算乎？吾以此觀之，勝負見矣。", "孫子曰：凡治眾如治寡，分數是也。鬥眾如鬥寡，形名是也。三軍之眾，可使必受敵而無敗者，奇正是也。兵之所加，", "如以碬投卵者，虛實是也。", ), ], vec![ ( vec!["孫子兵法"], vec![ "孫子兵法", concat!( "<"孫子兵法：">孫子曰：兵者，國之大事，死生之地，存亡之道，", "不可不察也。孫子曰：凡用兵之法，馳車千駟，革車千乘，帶甲十萬；千里饋糧，則內外之費賓客之用，膠" ), ], ), ( vec!["孫子曰"], vec![concat!( "<"孫子兵法：">孫子曰：兵者，國之大事，死生之地，存亡之道，", "不可不察也。孫子曰：凡用兵之法，馳車千駟，革車千乘，帶甲十萬；千里饋糧，則內外之費賓", )], ), ], ), ]; for (parts, tests) in inputs { for (needles, snippets) in tests { let mut results = Vec::new(); for part in &parts { if let Some(matched) = generate_snippet(part, &needles, Language::English, false) { results.push(matched); } } assert_eq!(snippets, results); } } } } ================================================ FILE: crates/nlp/src/language/stemmer.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::borrow::Cow; use rust_stemmers::Algorithm; use super::{Language, LanguageTokenizer}; #[derive(Debug, PartialEq, Eq)] pub struct StemmedToken<'x> { pub word: Cow<'x, str>, pub stemmed_word: Option>, pub from: usize, // Word offset in the text part pub to: usize, // Word length } pub struct Stemmer<'x> { stemmer: Option, tokenizer: LanguageTokenizer<'x>, } impl<'x> Stemmer<'x> { pub fn new(text: &'x str, language: Language, max_token_length: usize) -> Stemmer<'x> { Stemmer { tokenizer: language.tokenize_text(text, max_token_length), stemmer: STEMMER_MAP[language as usize].map(rust_stemmers::Stemmer::create), } } } impl<'x> Iterator for Stemmer<'x> { type Item = StemmedToken<'x>; fn next(&mut self) -> Option { let token = self.tokenizer.next()?; Some(StemmedToken { stemmed_word: self.stemmer.as_ref().and_then(|stemmer| { match stemmer.stem(&token.word) { Cow::Owned(text) if text.len() != token.word.len() || text != token.word => { Some(text.into()) } _ => None, } }), word: token.word, from: token.from, to: token.to, }) } } pub static STEMMER_MAP: &[Option] = &[ None, // Esperanto = 0, Some(Algorithm::English), // English = 1, Some(Algorithm::Russian), // Russian = 2, None, // Mandarin = 3, Some(Algorithm::Spanish), // Spanish = 4, Some(Algorithm::Portuguese), // Portuguese = 5, Some(Algorithm::Italian), // Italian = 6, None, // Bengali = 7, Some(Algorithm::French), // French = 8, Some(Algorithm::German), // German = 9, None, // Ukrainian = 10, None, // Georgian = 11, Some(Algorithm::Arabic), // Arabic = 12, None, // Hindi = 13, None, // Japanese = 14, None, // Hebrew = 15, None, // Yiddish = 16, None, // Polish = 17, None, // Amharic = 18, None, // Javanese = 19, None, // Korean = 20, Some(Algorithm::Norwegian), // Bokmal = 21, Some(Algorithm::Danish), // Danish = 22, Some(Algorithm::Swedish), // Swedish = 23, Some(Algorithm::Finnish), // Finnish = 24, Some(Algorithm::Turkish), // Turkish = 25, Some(Algorithm::Dutch), // Dutch = 26, Some(Algorithm::Hungarian), // Hungarian = 27, None, // Czech = 28, Some(Algorithm::Greek), // Greek = 29, None, // Bulgarian = 30, None, // Belarusian = 31, None, // Marathi = 32, None, // Kannada = 33, Some(Algorithm::Romanian), // Romanian = 34, None, // Slovene = 35, None, // Croatian = 36, None, // Serbian = 37, None, // Macedonian = 38, None, // Lithuanian = 39, None, // Latvian = 40, None, // Estonian = 41, Some(Algorithm::Tamil), // Tamil = 42, None, // Vietnamese = 43, None, // Urdu = 44, None, // Thai = 45, None, // Gujarati = 46, None, // Uzbek = 47, None, // Punjabi = 48, None, // Azerbaijani = 49, None, // Indonesian = 50, None, // Telugu = 51, None, // Persian = 52, None, // Malayalam = 53, None, // Oriya = 54, None, // Burmese = 55, None, // Nepali = 56, None, // Sinhalese = 57, None, // Khmer = 58, None, // Turkmen = 59, None, // Akan = 60, None, // Zulu = 61, None, // Shona = 62, None, // Afrikaans = 63, None, // Latin = 64, None, // Slovak = 65, None, // Catalan = 66, None, // Tagalog = 67, None, // Armenian = 68, None, // Unknown = 69, None, // None = 70, ]; #[cfg(test)] mod tests { use super::*; #[test] fn stemmer() { let inputs = [ ( "love loving lovingly loved lovely", Language::English, "love", ), ("querer queremos quer", Language::Spanish, "quer"), ]; for (input, language, result) in inputs { for token in Stemmer::new(input, language, 40) { assert_eq!(token.stemmed_word.unwrap_or(token.word), result); } } } } ================================================ FILE: crates/nlp/src/language/stopwords.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub type StopwordFnc = fn(&str) -> bool; pub static STOP_WORDS: &[Option] = &[ None, // Esperanto = 0, Some(english), // English = 1, Some(russian), // Russian = 2, None, // Mandarin = 3, Some(spanish), // Spanish = 4, Some(portuguese), // Portuguese = 5, Some(italian), // Italian = 6, None, // Bengali = 7, Some(french), // French = 8, Some(german), // German = 9, None, // Ukrainian = 10, None, // Georgian = 11, Some(arabic), // Arabic = 12, None, // Hindi = 13, None, // Japanese = 14, None, // Hebrew = 15, None, // Yiddish = 16, None, // Polish = 17, None, // Amharic = 18, None, // Javanese = 19, None, // Korean = 20, Some(norwegian), // Bokmal = 21, Some(danish), // Danish = 22, Some(swedish), // Swedish = 23, Some(finnish), // Finnish = 24, Some(turkish), // Turkish = 25, Some(dutch), // Dutch = 26, Some(hungarian), // Hungarian = 27, None, // Czech = 28, Some(greek), // Greek = 29, None, // Bulgarian = 30, None, // Belarusian = 31, None, // Marathi = 32, None, // Kannada = 33, Some(romanian), // Romanian = 34, None, // Slovene = 35, None, // Croatian = 36, None, // Serbian = 37, None, // Macedonian = 38, None, // Lithuanian = 39, None, // Latvian = 40, None, // Estonian = 41, None, // Tamil = 42, None, // Vietnamese = 43, None, // Urdu = 44, None, // Thai = 45, None, // Gujarati = 46, None, // Uzbek = 47, None, // Punjabi = 48, Some(azarbaijani), // Azerbaijani = 49, None, // Indonesian = 50, None, // Telugu = 51, None, // Persian = 52, None, // Malayalam = 53, None, // Oriya = 54, None, // Burmese = 55, Some(nepali), // Nepali = 56, None, // Sinhalese = 57, None, // Khmer = 58, None, // Turkmen = 59, None, // Akan = 60, None, // Zulu = 61, None, // Shona = 62, None, // Afrikaans = 63, None, // Latin = 64, None, // Slovak = 65, None, // Catalan = 66, None, // Tagalog = 67, None, // Armenian = 68, None, // Unknown = 69, None, // None = 70, ]; fn arabic(input: &str) -> bool { hashify::set!( input.as_bytes(), "آه", "آي", "أف", "أم", "أن", "أو", "أي", "إذ", "إن", "إي", "بخ", "بس", "بك", "بل", "به", "بي", "ته", "تي", "ثم", "ذا", "ذه", "ذو", "ذي", "عل", "عن", "في", "قد", "كل", "كم", "كي", "لا", "لك", "لم", "لن", "له", "لو", "لي", "ما", "مذ", "مع", "من", "مه", "ها", "هل", "هم", "هن", "هو", "هي", "يا", "آها", "أقل", "ألا", "أما", "أنا", "أنت", "أنى", "أوه", "أين", "إذا", "إذن", "إلا", "إلى", "إما", "إنا", "إنه", "إيه", "بعد", "بعض", "بكم", "بكن", "بلى", "بما", "بمن", "بنا", "بها", "بهم", "بهن", "بيد", "بين", "تلك", "تين", "ثمة", "حتى", "حيث", "حين", "خلا", "دون", "ذات", "ذاك", "ذان", "ذلك", "ذوا", "ذين", "ريث", "سوف", "سوى", "عدا", "عسى", "على", "عما", "عند", "غير", "فإن", "فلا", "فمن", "فيم", "فيه", "كأن", "كأي", "كذا", "كلا", "كما", "كيت", "كيف", "لئن", "لدى", "لست", "لسن", "لعل", "لكم", "لكن", "لكي", "لما", "لنا", "لها", "لهم", "لهن", "ليت", "ليس", "متى", "مما", "ممن", "منذ", "منه", "نحن", "نحو", "نعم", "هاك", "هذا", "هذه", "هذي", "هلا", "هما", "هنا", "هيا", "هيت", "وإذ", "وإن", "ولا", "ولو", "وما", "ومن", "وهو", "أكثر", "أنتم", "أنتن", "أيها", "إذما", "إليك", "إنما", "التي", "الذي", "بكما", "بهما", "تلكم", "تينك", "حاشا", "حبذا", "ذانك", "ذلكم", "ذلكن", "ذينك", "شتان", "عليك", "عليه", "فإذا", "فيما", "فيها", "كأين", "كذلك", "كلتا", "كلما", "لستم", "لستن", "لسنا", "لكما", "لهما", "لولا", "لوما", "ليسا", "ليست", "ماذا", "منها", "مهما", "هاته", "هاتي", "هذان", "هذين", "هكذا", "هناك", "وإذا", "ولكن", "أنتما", "أولئك", "أولاء", "أينما", "إليكم", "إليكن", "الذين", "بماذا", "تلكما", "حيثما", "ذلكما", "ذواتا", "ذواتي", "كأنما", "كيفما", "لستما", "لكنما", "لكيلا", "ليستا", "ليسوا", "هؤلاء", "هاتان", "هاتين", "هاهنا", "هنالك", "هيهات", "والذي", "إليكما", "اللائي", "اللاتي", "اللتان", "اللتيا", "اللتين", "اللذان", "اللذين", "كلاهما", "كليكما", "كليهما", "لاسيما", "والذين", "اللواتي", ) } fn azarbaijani(input: &str) -> bool { hashify::set!( input.as_bytes(), "a", "ad", "altmış", "altı", "amma", "arasında", "artıq", "ay", "az", "bax", "belə", "beş", "bilər", "bir", "biraz", "biri", "birşey", "biz", "bizim", "bizlər", "bu", "buna", "bundan", "bunların", "bunu", "bunun", "buradan", "bütün", "bəli", "bəlkə", "bəy", "bəzi", "bəzən", "ci", "çox", "cu", "cü", "çünki", "cı", "da", "daha", "dedi", "deyil", "dir", "doqquz", "doqsan", "dörd", "düz", "də", "dək", "dən", "dəqiqə", "edir", "edən", "elə", "et", "etdi", "etmə", "etmək", "faiz", "gilə", "görə", "ha", "haqqında", "harada", "heç", "hə", "həm", "həmin", "həmişə", "hər", "idi", "iki", "il", "ildə", "ilk", "ilə", "in", "indi", "istifadə", "isə", "iyirmi", "ki", "kim", "kimi", "kimə", "lakin", "lap", "mirşey", "məhz", "mən", "mənə", "niyə", "nə", "nəhayət", "o", "obirisi", "of", "olan", "olar", "olaraq", "oldu", "olduğu", "olmadı", "olmaz", "olmuşdur", "olsun", "olur", "on", "ona", "ondan", "onlar", "onlardan", "onların", "onsuzda", "onu", "onun", "oradan", "otuz", "öz", "özü", "qarşı", "qədər", "qırx", "saat", "sadəcə", "saniyə", "siz", "sizin", "sizlər", "sonra", "səhv", "səkkiz", "səksən", "sən", "sənin", "sənə", "təəssüf", "ü", "üç", "üçün", "var", "və", "xan", "xanım", "xeyr", "ya", "yalnız", "yaxşı", "yeddi", "yenə", "yetmiş", "yox", "yoxdur", "yoxsa", "yüz", "yəni", "zaman", "ı", "ə", "əgər", "əlbəttə", "əlli", "ən", "əslində", ) } fn danish(input: &str) -> bool { hashify::set!( input.as_bytes(), "ad", "af", "alle", "alt", "anden", "at", "blev", "blive", "bliver", "da", "de", "dem", "den", "denne", "der", "deres", "det", "dette", "dig", "din", "disse", "dog", "du", "efter", "eller", "en", "end", "er", "et", "for", "fra", "ham", "han", "hans", "har", "havde", "have", "hende", "hendes", "her", "hos", "hun", "hvad", "hvis", "hvor", "i", "ikke", "ind", "jeg", "jer", "jo", "kunne", "man", "mange", "med", "meget", "men", "mig", "min", "mine", "mit", "mod", "når", "ned", "noget", "nogle", "nu", "og", "også", "om", "op", "os", "over", "på", "sådan", "selv", "sig", "sin", "sine", "sit", "skal", "skulle", "som", "thi", "til", "ud", "under", "var", "være", "været", "vi", "vil", "ville", "vor", ) } fn dutch(input: &str) -> bool { hashify::set!( input.as_bytes(), "aan", "al", "alles", "als", "altijd", "andere", "ben", "bij", "daar", "dan", "dat", "de", "der", "deze", "die", "dit", "doch", "doen", "door", "dus", "een", "eens", "en", "er", "ge", "geen", "geweest", "haar", "had", "heb", "hebben", "heeft", "hem", "het", "hier", "hij", "hoe", "hun", "iemand", "iets", "ik", "in", "is", "ja", "je", "kan", "kon", "kunnen", "maar", "me", "meer", "men", "met", "mij", "mijn", "moet", "na", "naar", "niet", "niets", "nog", "nu", "of", "om", "omdat", "onder", "ons", "ook", "op", "over", "reeds", "te", "tegen", "toch", "toen", "tot", "u", "uit", "uw", "van", "veel", "voor", "want", "waren", "was", "wat", "werd", "wezen", "wie", "wil", "worden", "wordt", "zal", "ze", "zelf", "zich", "zij", "zijn", "zo", "zonder", "zou", ) } fn english(input: &str) -> bool { hashify::set!( input.as_bytes(), "a", "about", "above", "after", "again", "against", "ain", "all", "am", "an", "and", "any", "are", "aren", "aren't", "as", "at", "be", "because", "been", "before", "being", "below", "between", "both", "but", "by", "can", "couldn", "couldn't", "d", "did", "didn", "didn't", "do", "does", "doesn", "doesn't", "doing", "don", "don't", "down", "during", "each", "few", "for", "from", "further", "had", "hadn", "hadn't", "has", "hasn", "hasn't", "have", "haven", "haven't", "having", "he", "her", "here", "hers", "herself", "him", "himself", "his", "how", "i", "if", "in", "into", "is", "isn", "isn't", "it", "it's", "its", "itself", "just", "ll", "m", "ma", "me", "mightn", "mightn't", "more", "most", "mustn", "mustn't", "my", "myself", "needn", "needn't", "no", "nor", "not", "now", "o", "of", "off", "on", "once", "only", "or", "other", "our", "ours", "ourselves", "out", "over", "own", "re", "s", "same", "shan", "shan't", "she", "she's", "should", "should've", "shouldn", "shouldn't", "so", "some", "such", "t", "than", "that", "that'll", "the", "their", "theirs", "them", "themselves", "then", "there", "these", "they", "this", "those", "through", "to", "too", "under", "until", "up", "ve", "very", "was", "wasn", "wasn't", "we", "were", "weren", "weren't", "what", "when", "where", "which", "while", "who", "whom", "why", "will", "with", "won", "won't", "wouldn", "wouldn't", "y", "you", "you'd", "you'll", "you're", "you've", "your", "yours", "yourself", "yourselves", ) } fn finnish(input: &str) -> bool { hashify::set!( input.as_bytes(), "ei", "eivät", "emme", "en", "et", "että", "ette", "hän", "häneen", "hänellä", "hänelle", "häneltä", "hänen", "hänessä", "hänestä", "hänet", "häntä", "he", "heidän", "heidät", "heihin", "heillä", "heille", "heiltä", "heissä", "heistä", "heitä", "itse", "ja", "johon", "joiden", "joihin", "joiksi", "joilla", "joille", "joilta", "joina", "joissa", "joista", "joita", "joka", "joksi", "jolla", "jolle", "jolta", "jona", "jonka", "jos", "jossa", "josta", "jota", "jotka", "kanssa", "keiden", "keihin", "keiksi", "keillä", "keille", "keiltä", "keinä", "keissä", "keistä", "keitä", "keneen", "keneksi", "kenellä", "kenelle", "keneltä", "kenen", "kenenä", "kenessä", "kenestä", "kenet", "ketä", "ketkä", "koska", "kuin", "kuka", "kun", "me", "meidän", "meidät", "meihin", "meillä", "meille", "meiltä", "meissä", "meistä", "meitä", "mihin", "mikä", "miksi", "millä", "mille", "miltä", "minä", "minkä", "minua", "minulla", "minulle", "minulta", "minun", "minussa", "minusta", "minut", "minuun", "missä", "mistä", "mitä", "mitkä", "mukaan", "mutta", "näiden", "näihin", "näiksi", "näillä", "näille", "näiltä", "näinä", "näissä", "näistä", "näitä", "nämä", "ne", "niiden", "niihin", "niiksi", "niillä", "niille", "niiltä", "niin", "niinä", "niissä", "niistä", "niitä", "noiden", "noihin", "noiksi", "noilla", "noille", "noilta", "noin", "noina", "noissa", "noista", "noita", "nuo", "nyt", "ole", "olemme", "olen", "olet", "olette", "oli", "olimme", "olin", "olisi", "olisimme", "olisin", "olisit", "olisitte", "olisivat", "olit", "olitte", "olivat", "olla", "olleet", "ollut", "on", "ovat", "poikki", "se", "sekä", "sen", "siihen", "siinä", "siitä", "siksi", "sillä", "sille", "siltä", "sinä", "sinua", "sinulla", "sinulle", "sinulta", "sinun", "sinussa", "sinusta", "sinut", "sinuun", "sitä", "tähän", "tai", "täksi", "tallä", "tälle", "tältä", "tämä", "tämän", "tänä", "tässä", "tästä", "tätä", "te", "teidän", "teidät", "teihin", "teillä", "teille", "teiltä", "teissä", "teistä", "teitä", "tuo", "tuohon", "tuoksi", "tuolla", "tuolle", "tuolta", "tuon", "tuona", "tuossa", "tuosta", "tuotä", "vaan", "vai", "vaikka", "yli", ) } fn french(input: &str) -> bool { hashify::set!( input.as_bytes(), "à", "ai", "aie", "aient", "aies", "ait", "as", "au", "aura", "aurai", "auraient", "aurais", "aurait", "auras", "aurez", "auriez", "aurions", "aurons", "auront", "aux", "avaient", "avais", "avait", "avec", "avez", "aviez", "avions", "avons", "ayant", "ayante", "ayantes", "ayants", "ayez", "ayons", "c", "ce", "ces", "d", "dans", "de", "des", "du", "elle", "en", "es", "est", "et", "étaient", "étais", "était", "étant", "étante", "étantes", "étants", "été", "étée", "étées", "étés", "êtes", "étiez", "étions", "eu", "eue", "eues", "eûmes", "eurent", "eus", "eusse", "eussent", "eusses", "eussiez", "eussions", "eut", "eût", "eûtes", "eux", "fûmes", "furent", "fus", "fusse", "fussent", "fusses", "fussiez", "fussions", "fut", "fût", "fûtes", "il", "j", "je", "l", "la", "le", "leur", "lui", "m", "ma", "mais", "me", "même", "mes", "moi", "mon", "n", "ne", "nos", "notre", "nous", "on", "ont", "ou", "par", "pas", "pour", "qu", "que", "qui", "s", "sa", "se", "sera", "serai", "seraient", "serais", "serait", "seras", "serez", "seriez", "serions", "serons", "seront", "ses", "soient", "sois", "soit", "sommes", "son", "sont", "soyez", "soyons", "suis", "sur", "t", "ta", "te", "tes", "toi", "ton", "tu", "un", "une", "vos", "votre", "vous", "y", ) } fn german(input: &str) -> bool { hashify::set!( input.as_bytes(), "aber", "alle", "allem", "allen", "aller", "alles", "als", "also", "am", "an", "ander", "andere", "anderem", "anderen", "anderer", "anderes", "anderm", "andern", "anderr", "anders", "auch", "auf", "aus", "bei", "bin", "bis", "bist", "da", "damit", "dann", "das", "dasselbe", "dazu", "daß", "dein", "deine", "deinem", "deinen", "deiner", "deines", "dem", "demselben", "den", "denn", "denselben", "der", "derer", "derselbe", "derselben", "des", "desselben", "dessen", "dich", "die", "dies", "diese", "dieselbe", "dieselben", "diesem", "diesen", "dieser", "dieses", "dir", "doch", "dort", "du", "durch", "ein", "eine", "einem", "einen", "einer", "eines", "einig", "einige", "einigem", "einigen", "einiger", "einiges", "einmal", "er", "es", "etwas", "euch", "euer", "eure", "eurem", "euren", "eurer", "eures", "für", "gegen", "gewesen", "hab", "habe", "haben", "hat", "hatte", "hatten", "hier", "hin", "hinter", "ich", "ihm", "ihn", "ihnen", "ihr", "ihre", "ihrem", "ihren", "ihrer", "ihres", "im", "in", "indem", "ins", "ist", "jede", "jedem", "jeden", "jeder", "jedes", "jene", "jenem", "jenen", "jener", "jenes", "jetzt", "kann", "kein", "keine", "keinem", "keinen", "keiner", "keines", "können", "könnte", "machen", "man", "manche", "manchem", "manchen", "mancher", "manches", "mein", "meine", "meinem", "meinen", "meiner", "meines", "mich", "mir", "mit", "muss", "musste", "nach", "nicht", "nichts", "noch", "nun", "nur", "ob", "oder", "ohne", "sehr", "sein", "seine", "seinem", "seinen", "seiner", "seines", "selbst", "sich", "sie", "sind", "so", "solche", "solchem", "solchen", "solcher", "solches", "soll", "sollte", "sondern", "sonst", "über", "um", "und", "uns", "unser", "unsere", "unserem", "unseren", "unseres", "unter", "viel", "vom", "von", "vor", "während", "war", "waren", "warst", "was", "weg", "weil", "weiter", "welche", "welchem", "welchen", "welcher", "welches", "wenn", "werde", "werden", "wie", "wieder", "will", "wir", "wird", "wirst", "wo", "wollen", "wollte", "würde", "würden", "zu", "zum", "zur", "zwar", "zwischen", ) } fn greek(input: &str) -> bool { hashify::set!( input.as_bytes(), "η", "κ", "ο", "ἃ", "ἡ", "ἢ", "ἣ", "ἤ", "ἥ", "ὁ", "ὃ", "ὅ", "ὦ", "ᾧ", "δ'", "αν", "αἱ", "αἳ", "αἵ", "αὖ", "γα", "γε", "δέ", "δή", "δε", "δὲ", "δὴ", "δ’", "επ", "εἰ", "εἴ", "θα", "κι", "μή", "μα", "με", "μη", "μὴ", "να", "οι", "οἱ", "οἳ", "οὐ", "οὗ", "σε", "σύ", "σὺ", "τά", "τί", "τα", "τε", "τι", "το", "τό", "τὰ", "τὸ", "τῇ", "τῷ", "ωσ", "ἀπ", "ἀφ", "ἂν", "ἄν", "ἐκ", "ἐν", "ἐξ", "ἐφ", "ἧς", "ὃν", "ὃς", "ὅς", "ὅσ", "ὑπ", "ὡς", "ὡσ", "ὥς", "δι'", "γα^", "απο", "γάρ", "για", "γὰρ", "δαί", "δαὶ", "δεν", "διά", "διὰ", "εαν", "ενω", "επι", "εἰς", "εἰσ", "καί", "καθ", "και", "κατ", "καὶ", "κἀν", "κἂν", "μέν", "μεθ", "μετ", "μην", "μἐν", "μὲν", "μὴν", "οσο", "οτι", "οἷς", "οὐδ", "οὐκ", "οὐχ", "οὓς", "οὖν", "παρ", "που", "ποῦ", "προ", "πρὸ", "πως", "πωσ", "στη", "στο", "σόσ", "σύν", "σὸς", "σὺν", "τήν", "τίς", "τίσ", "την", "τησ", "τις", "τισ", "τοί", "τοι", "τον", "του", "τοῦ", "των", "τόν", "τὰς", "τὴν", "τὸν", "τῆς", "τῆσ", "τῶν", "ἀπό", "ἀπὸ", "ἄρα", "ἅμα", "ἐάν", "ἐγώ", "ἐγὼ", "ἐπί", "ἐπὶ", "ἐὰν", "ἔτι", "ἵνα", "ὅδε", "ὅτε", "ὅτι", "ὑπό", "ὑπὸ", "ἀλλ'", "αλλα", "αντι", "αυτα", "αυτη", "αυτο", "γοῦν", "δαίσ", "δαὶς", "εἰμί", "εἰμὶ", "εἴμι", "εἴτε", "ισωσ", "κατά", "κατα", "κατὰ", "μήτε", "μετά", "μετα", "μετὰ", "ομωσ", "οπωσ", "οὐδέ", "οὐδὲ", "οὐχὶ", "οὔτε", "οὕτω", "παρά", "παρα", "παρὰ", "περί", "περὶ", "ποια", "ποιο", "ποτε", "προσ", "πρόσ", "πρὸς", "στην", "στον", "ταῖς", "τινα", "τοτε", "τούσ", "τοὺς", "τοῖς", "τότε", "ἀλλά", "ἀλλὰ", "ἀλλ’", "ἐμόσ", "ἐμὸς", "ἐπεὶ", "ἐστι", "ὅθεν", "ὅπερ", "ὑμόσ", "ὑπέρ", "ὑπὲρ", "ὥστε", "αυτεσ", "αυτοι", "αυτοσ", "αυτων", "αὐτόσ", "αὐτὸς", "ειμαι", "ειναι", "εισαι", "ειστε", "οὐδὲν", "οὕτως", "οὕτωσ", "οὗτος", "οὗτοσ", "ποιεσ", "ποιοι", "ποιοσ", "ποιων", "ἄλλος", "ἄλλοσ", "ὅστις", "ὅστισ", "αυτουσ", "εκεινα", "εκεινη", "εκεινο", "καίτοι", "οὐδείσ", "οὐδεὶς", "ποιουσ", "ἑαυτοῦ", "ειμαστε", "εκεινεσ", "εκεινοι", "εκεινοσ", "εκεινων", "εκεινουσ", "τοιοῦτος", "τοιοῦτοσ", ) } fn hungarian(input: &str) -> bool { hashify::set!( input.as_bytes(), "a", "abban", "ahhoz", "ahogy", "ahol", "aki", "akik", "akkor", "alatt", "által", "általában", "amely", "amelyek", "amelyekben", "amelyeket", "amelyet", "amelynek", "ami", "amíg", "amikor", "amit", "amolyan", "annak", "arra", "arról", "át", "az", "azért", "azok", "azon", "azonban", "azt", "aztán", "azután", "azzal", "bár", "be", "belül", "benne", "cikk", "cikkek", "cikkeket", "csak", "de", "e", "ebben", "eddig", "egész", "egy", "egyéb", "egyes", "egyetlen", "egyik", "egyre", "ehhez", "ekkor", "el", "elég", "ellen", "elõ", "elõször", "elõtt", "elsõ", "emilyen", "én", "ennek", "éppen", "erre", "és", "ez", "ezek", "ezen", "ezért", "ezt", "ezzel", "fel", "felé", "hanem", "hiszen", "hogy", "hogyan", "igen", "így", "ill", "ill.", "illetve", "ilyen", "ilyenkor", "ismét", "ison", "itt", "jó", "jobban", "jól", "kell", "kellett", "keressünk", "keresztül", "ki", "kívül", "között", "közül", "legalább", "legyen", "lehet", "lehetett", "lenne", "lenni", "lesz", "lett", "maga", "magát", "majd", "már", "más", "másik", "meg", "még", "mellett", "mely", "melyek", "mert", "mi", "miért", "míg", "mikor", "milyen", "minden", "mindenki", "mindent", "mindig", "mint", "mintha", "mit", "mivel", "most", "nagy", "nagyobb", "nagyon", "ne", "néha", "néhány", "nekem", "neki", "nélkül", "nem", "nincs", "õ", "õk", "õket", "olyan", "össze", "ott", "pedig", "persze", "rá", "s", "saját", "sem", "semmi", "sok", "sokat", "sokkal", "számára", "szemben", "szerint", "szinte", "talán", "tehát", "teljes", "több", "tovább", "továbbá", "úgy", "ugyanis", "új", "újabb", "újra", "után", "utána", "utolsó", "vagy", "vagyis", "vagyok", "valaki", "valami", "valamint", "való", "van", "vannak", "vele", "vissza", "viszont", "volna", "volt", "voltak", "voltam", "voltunk", ) } fn italian(input: &str) -> bool { hashify::set!( input.as_bytes(), "a", "abbia", "abbiamo", "abbiano", "abbiate", "ad", "agl", "agli", "ai", "al", "all", "alla", "alle", "allo", "anche", "avemmo", "avendo", "avesse", "avessero", "avessi", "avessimo", "aveste", "avesti", "avete", "aveva", "avevamo", "avevano", "avevate", "avevi", "avevo", "avrà", "avrai", "avranno", "avrebbe", "avrebbero", "avrei", "avremmo", "avremo", "avreste", "avresti", "avrete", "avrò", "avuta", "avute", "avuti", "avuto", "c", "che", "chi", "ci", "coi", "col", "come", "con", "contro", "cui", "da", "dagl", "dagli", "dai", "dal", "dall", "dalla", "dalle", "dallo", "degl", "degli", "dei", "del", "dell", "della", "delle", "dello", "di", "dov", "dove", "e", "è", "ebbe", "ebbero", "ebbi", "ed", "era", "erano", "eravamo", "eravate", "eri", "ero", "essendo", "faccia", "facciamo", "facciano", "facciate", "faccio", "facemmo", "facendo", "facesse", "facessero", "facessi", "facessimo", "faceste", "facesti", "faceva", "facevamo", "facevano", "facevate", "facevi", "facevo", "fai", "fanno", "farà", "farai", "faranno", "farebbe", "farebbero", "farei", "faremmo", "faremo", "fareste", "faresti", "farete", "farò", "fece", "fecero", "feci", "fosse", "fossero", "fossi", "fossimo", "foste", "fosti", "fu", "fui", "fummo", "furono", "gli", "ha", "hai", "hanno", "ho", "i", "il", "in", "io", "l", "la", "le", "lei", "li", "lo", "loro", "lui", "ma", "mi", "mia", "mie", "miei", "mio", "ne", "negl", "negli", "nei", "nel", "nell", "nella", "nelle", "nello", "noi", "non", "nostra", "nostre", "nostri", "nostro", "o", "per", "perché", "più", "quale", "quanta", "quante", "quanti", "quanto", "quella", "quelle", "quelli", "quello", "questa", "queste", "questi", "questo", "sarà", "sarai", "saranno", "sarebbe", "sarebbero", "sarei", "saremmo", "saremo", "sareste", "saresti", "sarete", "sarò", "se", "sei", "si", "sia", "siamo", "siano", "siate", "siete", "sono", "sta", "stai", "stando", "stanno", "starà", "starai", "staranno", "starebbe", "starebbero", "starei", "staremmo", "staremo", "stareste", "staresti", "starete", "starò", "stava", "stavamo", "stavano", "stavate", "stavi", "stavo", "stemmo", "stesse", "stessero", "stessi", "stessimo", "steste", "stesti", "stette", "stettero", "stetti", "stia", "stiamo", "stiano", "stiate", "sto", "su", "sua", "sue", "sugl", "sugli", "sui", "sul", "sull", "sulla", "sulle", "sullo", "suo", "suoi", "ti", "tra", "tu", "tua", "tue", "tuo", "tuoi", "tutti", "tutto", "un", "una", "uno", "vi", "voi", "vostra", "vostre", "vostri", "vostro", ) } fn norwegian(input: &str) -> bool { hashify::set!( input.as_bytes(), "å", "alle", "at", "av", "både", "båe", "bare", "begge", "ble", "blei", "bli", "blir", "blitt", "da", "då", "de", "deg", "dei", "deim", "deira", "deires", "dem", "den", "denne", "der", "dere", "deres", "det", "dette", "di", "din", "disse", "ditt", "du", "dykk", "dykkar", "eg", "ein", "eit", "eitt", "eller", "elles", "en", "enn", "er", "et", "ett", "etter", "for", "før", "fordi", "fra", "ha", "hadde", "han", "hans", "har", "hennar", "henne", "hennes", "her", "hjå", "ho", "hoe", "honom", "hoss", "hossen", "hun", "hva", "hvem", "hver", "hvilke", "hvilken", "hvis", "hvor", "hvordan", "hvorfor", "i", "ikke", "ikkje", "ingen", "ingi", "inkje", "inn", "inni", "ja", "jeg", "kan", "kom", "korleis", "korso", "kun", "kunne", "kva", "kvar", "kvarhelst", "kven", "kvi", "kvifor", "man", "mange", "me", "med", "medan", "meg", "meget", "mellom", "men", "mi", "min", "mine", "mitt", "mot", "mykje", "nå", "når", "ned", "no", "noe", "noen", "noka", "noko", "nokon", "nokor", "nokre", "og", "også", "om", "opp", "oss", "over", "på", "så", "samme", "sånn", "seg", "selv", "si", "sia", "sidan", "siden", "sin", "sine", "sitt", "sjøl", "skal", "skulle", "slik", "so", "som", "somme", "somt", "til", "um", "upp", "ut", "uten", "var", "vår", "være", "vart", "vært", "varte", "ved", "vere", "verte", "vi", "vil", "ville", "vore", "vors", "vort", ) } fn nepali(input: &str) -> bool { hashify::set!( input.as_bytes(), "छ", "त", "न", "म", "र", "अब", "आए", "उप", "एक", "ओठ", "औं", "का", "कि", "के", "को", "गए", "छु", "छू", "जब", "जे", "जो", "तर", "तल", "ती", "नि", "नै", "नौ", "भए", "भन", "भर", "मा", "यस", "या", "यी", "यो", "ले", "सो", "हो", "कम से कम", "अझै", "अरु", "अलग", "आदि", "आफू", "आयो", "कतै", "कसै", "किन", "गयौ", "गरि", "गरी", "गैर", "चार", "छन्", "छैन", "छौं", "जान", "जुन", "ठीक", "तथा", "तिर", "तीन", "थिए", "दिए", "दुई", "पछि", "पटक", "पनि", "बने", "बरु", "बीच", "भने", "भन्", "यति", "यदि", "यसो", "रही", "रूप", "लाई", "संग", "सधै", "सबै", "समय", "सही", "सात", "साथ", "हरे", "हुन", "अन्य", "आजको", "आत्म", "उनको", "उनले", "एउटै", "एकदम", "कसरी", "कुनै", "कुरा", "केही", "कोही", "गरेर", "गरौं", "गर्छ", "गर्न", "चाले", "जबकि", "जसको", "जसमा", "जसले", "जहाँ", "तपाई", "तिनी", "तिमी", "त्यो", "थिएन", "थियो", "देखि", "देखे", "धेरै", "नत्र", "नयाँ", "पर्छ", "पाँच", "प्लस", "फेरी", "बारे", "भएको", "मलाई", "माथि", "मेरो", "यसको", "यसरी", "यहाँ", "राखे", "लगभग", "लागि", "शायद", "संगै", "सक्छ", "सम्म", "साथै", "सायद", "सारा", "सोही", "हरेक", "हुने", "हुन्", "अक्सर", "अगाडी", "अर्को", "आफ्नै", "आफ्नो", "कसैले", "कृपया", "गरेका", "गरेको", "गर्छु", "गर्दै", "गर्नु", "गर्ने", "चाहिए", "जसबाट", "जसलाई", "जस्तै", "जस्तो", "जाहिर", "तापनी", "देखेर", "नजिकै", "निम्न", "पक्का", "पक्कै", "पहिले", "पहिलो", "पूर्व", "प्रति", "बाहिर", "बाहेक", "बिशेष", "बीचमा", "भन्छु", "भन्दा", "भन्ने", "भित्र", "मात्र", "मुख्य", "यसपछि", "यस्तो", "रहेका", "रहेको", "राख्छ", "सट्टा", "सम्भव", "हुन्छ", "अनुसार", "अन्यथा", "अरुलाई", "अर्थात", "आफूलाई", "उदाहरण", "उहालाई", "किनभने", "क्रमशः", "जताततै", "तत्काल", "तपाईको", "तेस्रो", "त्यहाँ", "त्सपछि", "त्सैले", "देखियो", "देखेको", "दोस्रो", "निम्ति", "पाँचौं", "प्रतेक", "भन्छन्", "भित्री", "यथोचित", "यद्यपि", "राम्रो", "वरीपरी", "सबैलाई", "स्पष्ट", "अन्यत्र", "अर्थात्", "कहाँबाट", "चाहन्छु", "तदनुसार", "तिनीहरू", "देखिन्छ", "पछिल्लो", "पर्थ्यो", "पहिल्यै", "बिरुद्ध", "यसबाहेक", "साँच्चै", "अन्तर्गत", "तुरुन्तै", "तेस्कारण", "दिनुभएको", "पर्याप्त", "भन्नुभयो", "यहाँसम्म", "वास्तवमा", "गर्नुपर्छ", "जस्तोसुकै", "तिनीहरुको", "दिनुहुन्छ", "निर्दिष्ट", "कहिलेकाहीं", "चाहनुहुन्छ", "तिनिहरुलाई", "निम्नानुसार", ) } fn portuguese(input: &str) -> bool { hashify::set!( input.as_bytes(), "a", "à", "ao", "aos", "aquela", "aquelas", "aquele", "aqueles", "aquilo", "as", "às", "até", "com", "como", "da", "das", "de", "dela", "delas", "dele", "deles", "depois", "do", "dos", "e", "ela", "elas", "ele", "eles", "em", "entre", "era", "eram", "éramos", "essa", "essas", "esse", "esses", "esta", "está", "estamos", "estão", "estas", "estava", "estavam", "estávamos", "este", "esteja", "estejam", "estejamos", "estes", "esteve", "estive", "estivemos", "estiver", "estivera", "estiveram", "estivéramos", "estiverem", "estivermos", "estivesse", "estivessem", "estivéssemos", "estou", "eu", "foi", "fomos", "for", "fora", "foram", "fôramos", "forem", "formos", "fosse", "fossem", "fôssemos", "fui", "há", "haja", "hajam", "hajamos", "hão", "havemos", "hei", "houve", "houvemos", "houver", "houvera", "houverá", "houveram", "houvéramos", "houverão", "houverei", "houverem", "houveremos", "houveria", "houveriam", "houveríamos", "houvermos", "houvesse", "houvessem", "houvéssemos", "isso", "isto", "já", "lhe", "lhes", "mais", "mas", "me", "mesmo", "meu", "meus", "minha", "minhas", "muito", "na", "não", "nas", "nem", "no", "nos", "nós", "nossa", "nossas", "nosso", "nossos", "num", "numa", "o", "os", "ou", "para", "pela", "pelas", "pelo", "pelos", "por", "qual", "quando", "que", "quem", "são", "se", "seja", "sejam", "sejamos", "sem", "será", "serão", "serei", "seremos", "seria", "seriam", "seríamos", "seu", "seus", "só", "somos", "sou", "sua", "suas", "também", "te", "tem", "tém", "temos", "tenha", "tenham", "tenhamos", "tenho", "terá", "terão", "terei", "teremos", "teria", "teriam", "teríamos", "teu", "teus", "teve", "tinha", "tinham", "tínhamos", "tive", "tivemos", "tiver", "tivera", "tiveram", "tivéramos", "tiverem", "tivermos", "tivesse", "tivessem", "tivéssemos", "tu", "tua", "tuas", "um", "uma", "você", "vocês", "vos", ) } fn romanian(input: &str) -> bool { hashify::set!( input.as_bytes(), "a", "abia", "acea", "aceasta", "această", "aceea", "aceeasi", "acei", "aceia", "acel", "acela", "acelasi", "acele", "acelea", "acest", "acesta", "aceste", "acestea", "acestei", "acestia", "acestui", "aceşti", "aceştia", "adica", "ai", "aia", "aibă", "aici", "al", "ala", "ale", "alea", "alt", "alta", "altceva", "altcineva", "alte", "altfel", "alti", "altii", "altul", "am", "anume", "apoi", "ar", "are", "as", "asa", "asta", "astea", "astfel", "asupra", "atare", "atat", "atata", "atatea", "atatia", "ati", "atit", "atita", "atitea", "atitia", "atunci", "au", "avea", "avem", "aveţi", "avut", "aş", "aţi", "ba", "ca", "cam", "cand", "care", "careia", "carora", "caruia", "cat", "cât", "câte", "catre", "câtva", "câţi", "ce", "cea", "ceea", "cei", "ceilalti", "cel", "cele", "celor", "ceva", "chiar", "ci", "cind", "cînd", "cine", "cineva", "cit", "cît", "cita", "cite", "cîte", "citeva", "citi", "citiva", "cîtva", "cîţi", "cu", "cui", "cum", "cumva", "că", "căci", "cărei", "căror", "cărui", "către", "da", "daca", "dacă", "dar", "dat", "dată", "dau", "de", "deasupra", "deci", "decit", "deja", "desi", "despre", "deşi", "din", "dintr", "dintr-", "dintre", "doar", "doi", "doilea", "două", "drept", "dupa", "după", "dă", "e", "ea", "ei", "el", "ele", "era", "eram", "este", "eu", "eşti", "face", "fara", "fata", "fel", "fi", "fie", "fiecare", "fii", "fim", "fiu", "fiţi", "foarte", "fost", "fără", "i", "ia", "iar", "ii", "îi", "il", "îl", "imi", "îmi", "in", "în", "inainte", "inapoi", "inca", "incit", "insa", "intr", "intre", "isi", "iti", "îţi", "la", "lângă", "le", "li", "lîngă", "lor", "lui", "m", "ma", "mai", "mâine", "mea", "mei", "mele", "mereu", "meu", "mi", "mie", "mîine", "mine", "mod", "mult", "multa", "multe", "multi", "multă", "mulţi", "mă", "ne", "ni", "nici", "nimeni", "nimic", "niste", "nişte", "noastre", "noastră", "noi", "nostri", "nostru", "nou", "noua", "nouă", "noştri", "nu", "numai", "o", "or", "ori", "oricând", "oricare", "oricât", "orice", "oricînd", "oricine", "oricît", "oricum", "oriunde", "pai", "până", "parca", "patra", "patru", "pe", "pentru", "peste", "pic", "pina", "pînă", "poate", "pot", "prea", "prima", "primul", "prin", "printr-", "putini", "puţin", "puţina", "puţină", "sa", "sa-mi", "sa-ti", "sai", "sale", "sau", "se", "si", "sint", "sintem", "spate", "spre", "sub", "sunt", "suntem", "sunteţi", "sus", "să", "săi", "său", "t", "ta", "tale", "te", "ti", "tine", "toata", "toate", "toată", "tocmai", "tot", "toti", "totul", "totusi", "totuşi", "toţi", "trei", "treia", "treilea", "tu", "tuturor", "tăi", "tău", "u", "ul", "ului", "un", "una", "unde", "undeva", "unei", "uneia", "unele", "uneori", "unii", "unor", "unora", "unu", "unui", "unuia", "unul", "v", "va", "vi", "voastre", "voastră", "voi", "vom", "vor", "vostru", "vouă", "voştri", "vreo", "vreun", "vă", "zi", "zice", "şi", "ţi", "ţie", "ăla", "ălea", "ăsta", "ăstea", "ăştia", ) } fn russian(input: &str) -> bool { hashify::set!( input.as_bytes(), "а", "в", "ж", "и", "к", "о", "с", "у", "я", "бы", "во", "вы", "да", "до", "ее", "ей", "же", "за", "из", "им", "их", "ли", "мы", "на", "не", "ни", "но", "ну", "об", "он", "от", "по", "со", "то", "ты", "уж", "без", "был", "вам", "вас", "вот", "все", "всю", "где", "два", "для", "его", "ему", "еще", "или", "как", "кто", "мне", "мой", "моя", "над", "нас", "нее", "ней", "нет", "ним", "них", "она", "они", "под", "при", "про", "раз", "сам", "так", "там", "тем", "том", "тот", "три", "тут", "уже", "чем", "что", "эти", "эту", "была", "были", "было", "быть", "ведь", "всех", "даже", "если", "есть", "куда", "меня", "надо", "него", "один", "свою", "себе", "себя", "тебя", "того", "тоже", "хоть", "чего", "чтоб", "чуть", "этой", "этом", "этот", "более", "будет", "будто", "вдруг", "всего", "зачем", "здесь", "какая", "какой", "когда", "лучше", "между", "много", "может", "можно", "опять", "перед", "после", "потом", "почти", "разве", "такой", "тогда", "через", "чтобы", "этого", "больше", "всегда", "другой", "иногда", "нельзя", "нибудь", "ничего", "потому", "сейчас", "совсем", "теперь", "только", "хорошо", "впрочем", "конечно", "наконец", "никогда", ) } fn spanish(input: &str) -> bool { hashify::set!( input.as_bytes(), "a", "al", "algo", "algunas", "algunos", "ante", "antes", "como", "con", "contra", "cual", "cuando", "de", "del", "desde", "donde", "durante", "e", "el", "él", "ella", "ellas", "ellos", "en", "entre", "era", "erais", "éramos", "eran", "eras", "eres", "es", "esa", "esas", "ese", "eso", "esos", "esta", "está", "estaba", "estabais", "estábamos", "estaban", "estabas", "estad", "estada", "estadas", "estado", "estados", "estáis", "estamos", "están", "estando", "estar", "estará", "estarán", "estarás", "estaré", "estaréis", "estaremos", "estaría", "estaríais", "estaríamos", "estarían", "estarías", "estas", "estás", "este", "esté", "estéis", "estemos", "estén", "estés", "esto", "estos", "estoy", "estuve", "estuviera", "estuvierais", "estuviéramos", "estuvieran", "estuvieras", "estuvieron", "estuviese", "estuvieseis", "estuviésemos", "estuviesen", "estuvieses", "estuvimos", "estuviste", "estuvisteis", "estuvo", "fue", "fuera", "fuerais", "fuéramos", "fueran", "fueras", "fueron", "fuese", "fueseis", "fuésemos", "fuesen", "fueses", "fui", "fuimos", "fuiste", "fuisteis", "ha", "habéis", "había", "habíais", "habíamos", "habían", "habías", "habida", "habidas", "habido", "habidos", "habiendo", "habrá", "habrán", "habrás", "habré", "habréis", "habremos", "habría", "habríais", "habríamos", "habrían", "habrías", "han", "has", "hasta", "hay", "haya", "hayáis", "hayamos", "hayan", "hayas", "he", "hemos", "hube", "hubiera", "hubierais", "hubiéramos", "hubieran", "hubieras", "hubieron", "hubiese", "hubieseis", "hubiésemos", "hubiesen", "hubieses", "hubimos", "hubiste", "hubisteis", "hubo", "la", "las", "le", "les", "lo", "los", "más", "me", "mi", "mí", "mía", "mías", "mío", "míos", "mis", "mucho", "muchos", "muy", "nada", "ni", "no", "nos", "nosotras", "nosotros", "nuestra", "nuestras", "nuestro", "nuestros", "o", "os", "otra", "otras", "otro", "otros", "para", "pero", "poco", "por", "porque", "que", "qué", "quien", "quienes", "se", "sea", "seáis", "seamos", "sean", "seas", "sentid", "sentida", "sentidas", "sentido", "sentidos", "será", "serán", "serás", "seré", "seréis", "seremos", "sería", "seríais", "seríamos", "serían", "serías", "sí", "siente", "sin", "sintiendo", "sobre", "sois", "somos", "son", "soy", "su", "sus", "suya", "suyas", "suyo", "suyos", "también", "tanto", "te", "tendrá", "tendrán", "tendrás", "tendré", "tendréis", "tendremos", "tendría", "tendríais", "tendríamos", "tendrían", "tendrías", "tened", "tenéis", "tenemos", "tenga", "tengáis", "tengamos", "tengan", "tengas", "tengo", "tenía", "teníais", "teníamos", "tenían", "tenías", "tenida", "tenidas", "tenido", "tenidos", "teniendo", "ti", "tiene", "tienen", "tienes", "todo", "todos", "tu", "tú", "tus", "tuve", "tuviera", "tuvierais", "tuviéramos", "tuvieran", "tuvieras", "tuvieron", "tuviese", "tuvieseis", "tuviésemos", "tuviesen", "tuvieses", "tuvimos", "tuviste", "tuvisteis", "tuvo", "tuya", "tuyas", "tuyo", "tuyos", "un", "una", "uno", "unos", "vosostras", "vosostros", "vuestra", "vuestras", "vuestro", "vuestros", "y", "ya", "yo", ) } fn swedish(input: &str) -> bool { hashify::set!( input.as_bytes(), "alla", "allt", "än", "är", "åt", "att", "av", "blev", "bli", "blir", "blivit", "då", "där", "de", "dem", "den", "denna", "deras", "dess", "dessa", "det", "detta", "dig", "din", "dina", "ditt", "du", "efter", "ej", "eller", "en", "er", "era", "ert", "ett", "för", "från", "ha", "hade", "han", "hans", "har", "här", "henne", "hennes", "hon", "honom", "hur", "i", "icke", "ingen", "inom", "inte", "jag", "ju", "kan", "kunde", "man", "med", "mellan", "men", "mig", "min", "mina", "mitt", "mot", "mycket", "någon", "något", "några", "när", "ni", "nu", "och", "om", "oss", "över", "på", "så", "sådan", "sådana", "sådant", "samma", "sedan", "sig", "sin", "sina", "sitta", "själv", "skulle", "som", "till", "under", "upp", "ut", "utan", "vad", "var", "vår", "vara", "våra", "varför", "varit", "varje", "vars", "vart", "vårt", "vem", "vi", "vid", "vilka", "vilkas", "vilken", "vilket", ) } fn turkish(input: &str) -> bool { hashify::set!( input.as_bytes(), "acaba", "ama", "aslında", "az", "bazı", "belki", "biri", "birkaç", "birşey", "biz", "bu", "çok", "çünkü", "da", "daha", "de", "defa", "diye", "en", "eğer", "gibi", "hem", "hep", "hepsi", "her", "hiç", "için", "ile", "ise", "kez", "ki", "kim", "mu", "mü", "mı", "nasıl", "ne", "neden", "nerde", "nerede", "nereye", "niçin", "niye", "o", "sanki", "siz", "tüm", "ve", "veya", "ya", "yani", "şey", "şu", ) } /* Not yet available for auto-detection static KAZAKH: Set<&'static str> = phf_set! { "", "е", "о", "я", "ә", "ай", "ал", "ау", "ах", "ей", "еш", "ие", "кә", "ой", "ол", "ох", "па", "уа", "эй", "эх", "әй", "өз", "өй", "ана", "арс", "аһа", "бар", "беу", "біз", "бұл", "жоқ", "кәһ", "мен", "моһ", "осы", "оһо", "пай", "сен", "сол", "соң", "сіз", "тек", "тәк", "уай", "уау", "ура", "шек", "ырс", "ырқ", "ыңқ", "ірк", "қап", "құр", "үйт", "әні", "өзі", "арс-ұрс", "пай-пай", "паһ-паһ", "қош-қош", "анау", "барқ", "бері", "бойы", "болп", "борт", "былп", "бүйт", "бәрі", "гүрс", "гөрі", "дүрс", "дүңк", "емес", "жалп", "желп", "жуық", "кірт", "күрт", "күңк", "кәне", "кәні", "маңқ", "морт", "мына", "мышы", "мыңқ", "міне", "одан", "олар", "онда", "оның", "оған", "пфша", "пырс", "пішә", "сарт", "саңқ", "сона", "сыңқ", "тарс", "таяу", "тағы", "таңқ", "тырс", "тыңқ", "түге", "шаңқ", "шырт", "шіңк", "шәйт", "ғана", "қана", "қолп", "қорс", "қоса", "қыңқ", "үшін", "әйда", "әрне", "өзге", "өзім", "өзің", "жалт-жалт", "жалт-жұлт", "сарт-сұрт", "тарс-тұрс", "шаңқ-шаңқ", "шаңқ-шұңқ", "қалт-қалт", "қалт-құлт", "қаңқ-қаңқ", "қаңқ-құңқ", "барша", "бетер", "бізге", "бірақ", "бірге", "біреу", "бүкіл", "бұрын", "дейін", "ешбір", "ешкім", "кейін", "күллі", "күшім", "маған", "менде", "менен", "менің", "мынау", "пішту", "сайын", "салым", "саған", "сенде", "сенен", "сенің", "солай", "сонау", "сорап", "сізге", "таман", "тарта", "түгел", "шақты", "шейін", "ғұрлы", "қарай", "қатар", "құрау", "әрбір", "әрине", "әркім", "әттең", "әукім", "өзіме", "өзіне", "сенен онан", "арбаң-арбаң", "батыр-бұтыр", "далаң-далаң", "митың-митың", "салаң-сұлаң", "құрау-құрау", "ыржың-тыржың", "алайда", "алатау", "алақай", "арнайы", "арқылы", "барлық", "бізбен", "бізден", "біздер", "біздің", "бұндай", "дәнеңе", "ештеме", "кейбір", "кәнеки", "мұндай", "оларға", "онымен", "осылай", "осынау", "себебі", "сияқты", "сондай", "сізбен", "сізден", "сіздер", "сіздің", "тағыда", "туралы", "шамалы", "шіркін", "ғұрлым", "қаралы", "әлдене", "өзінің", "бүгжең-бүгжең", "тарбаң-тарбаң", "қайқаң-құйқаң", "қаңғыр-күңгір", "бойымен", "бірдеме", "бірнеше", "ешқайсы", "ешқашан", "менімен", "олардан", "олардың", "олармен", "осындай", "сенімен", "сонымен", "япырмай", "әйтпесе", "әлдекім", "әншейін", "әрқайсы", "әрқалай", "өзімнің", "өйткені", "әттеген-ай", "арсалаң-арсалаң", "ербелең-ербелең", "қызараң-қызараң", "айтпақшы", "біздерге", "дегенмен", "ешқандай", "кейбіреу", "масқарай", "мәссаған", "ойпырмай", "сіздерге", "қайсыбір", "әлденеше", "алдақашан", "біздерден", "біздердің", "біздермен", "бәрекелді", "сондықтан", "сіздерден", "сіздердің", "сіздермен", "әйткенмен", "әлдеқалай", "әлдеқашан", "әттегенай", "әлдеқайдан", "астапыралла", "жаракімалла", }; */ ================================================ FILE: crates/nlp/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod classifier; pub mod language; pub mod tokenizers; ================================================ FILE: crates/nlp/src/tokenizers/chinese.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{borrow::Cow, sync::LazyLock, vec::IntoIter}; use jieba_rs::Jieba; use super::{InnerToken, Token}; pub(crate) static JIEBA: LazyLock = LazyLock::new(Jieba::new); pub struct ChineseTokenizer<'x, T, I> where T: Iterator>, I: InnerToken<'x>, { tokenizer: T, tokens: IntoIter>, phantom: std::marker::PhantomData<&'x str>, } impl<'x, T, I> ChineseTokenizer<'x, T, I> where T: Iterator>, I: InnerToken<'x>, { pub fn new(tokenizer: T) -> Self { ChineseTokenizer { tokenizer, tokens: Vec::new().into_iter(), phantom: std::marker::PhantomData, } } } impl<'x, T, I> Iterator for ChineseTokenizer<'x, T, I> where T: Iterator>, I: InnerToken<'x>, { type Item = Token; fn next(&mut self) -> Option { loop { if let Some(token) = self.tokens.next() { return Some(token); } else { let token = self.tokenizer.next()?; if token.word.is_alphabetic_8bit() { let mut token_to = token.from; match token.word.unwrap_alphabetic() { Cow::Borrowed(word) => { self.tokens = JIEBA .cut(word, false) .into_iter() .map(|word| { let token_from = token_to; token_to += word.len(); Token { word: I::new_alphabetic(word), from: token_from, to: token_to, } }) .collect::>() .into_iter(); } Cow::Owned(word) => { self.tokens = JIEBA .cut(&word, false) .into_iter() .map(|word| { let token_from = token_to; token_to += word.len(); Token { word: I::new_alphabetic(word.to_string()), from: token_from, to: token_to, } }) .collect::>() .into_iter(); } } } else { return token.into(); } } } } } #[cfg(test)] mod tests { use crate::tokenizers::{Token, chinese::ChineseTokenizer, word::WordTokenizer}; #[test] fn chinese_tokenizer() { assert_eq!( ChineseTokenizer::new(WordTokenizer::new( "孫子曰：兵者，國之大事，死生之地，存亡之道，不可不察也。", 40 ),) .collect::>(), vec![ Token { word: "孫".into(), from: 0, to: 3 }, Token { word: "子".into(), from: 3, to: 6 }, Token { word: "曰".into(), from: 6, to: 9 }, Token { word: "兵".into(), from: 12, to: 15 }, Token { word: "者".into(), from: 15, to: 18 }, Token { word: "國".into(), from: 21, to: 24 }, Token { word: "之".into(), from: 24, to: 27 }, Token { word: "大事".into(), from: 27, to: 33 }, Token { word: "死".into(), from: 36, to: 39 }, Token { word: "生".into(), from: 39, to: 42 }, Token { word: "之".into(), from: 42, to: 45 }, Token { word: "地".into(), from: 45, to: 48 }, Token { word: "存亡".into(), from: 51, to: 57 }, Token { word: "之".into(), from: 57, to: 60 }, Token { word: "道".into(), from: 60, to: 63 }, Token { word: "不可不".into(), from: 66, to: 75 }, Token { word: "察".into(), from: 75, to: 78 }, Token { word: "也".into(), from: 78, to: 81 } ] ); } } ================================================ FILE: crates/nlp/src/tokenizers/japanese.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{InnerToken, Token}; use maplit::hashmap; use std::collections::HashMap; use std::vec::IntoIter; use std::{hash::Hash, sync::LazyLock}; pub struct JapaneseTokenizer<'x, T, I> where T: Iterator>, I: InnerToken<'x>, { tokenizer: T, tokens: IntoIter>, phantom: std::marker::PhantomData<&'x str>, } impl<'x, T, I> JapaneseTokenizer<'x, T, I> where T: Iterator>, I: InnerToken<'x>, { pub fn new(tokenizer: T) -> Self { JapaneseTokenizer { tokenizer, tokens: Vec::new().into_iter(), phantom: std::marker::PhantomData, } } } impl<'x, T, I> Iterator for JapaneseTokenizer<'x, T, I> where T: Iterator>, I: InnerToken<'x>, { type Item = Token; fn next(&mut self) -> Option { loop { if let Some(token) = self.tokens.next() { return Some(token); } else { let token = self.tokenizer.next()?; if token.word.is_alphabetic_8bit() { let mut token_to = token.from; self.tokens = tokenize(token.word.unwrap_alphabetic().as_ref()) .into_iter() .map(|word| { let token_from = token_to; token_to += word.len(); Token { word: I::new_alphabetic(word.to_string()), from: token_from, to: token_to, } }) .collect::>() .into_iter(); } else { return token.into(); } } } } } // Ported from https://github.com/woxtu/rust-tinysegmenter, MIT license const BIAS: i32 = -332; fn get_score(d: &HashMap, s: &T) -> i32 { d.get(s).cloned().unwrap_or(0) } fn get_ctype(c: char) -> char { match c as u32 { 0x4E00 | 0x4E8C | 0x4E09 | 0x56DB | 0x4E94 | 0x516D | 0x4E03 | 0x516B | 0x4E5D | 0x5341 => { 'M' } 0x767E | 0x5343 | 0x4E07 | 0x5104 | 0x5146 => 'M', 0x4E00..=0x9FA0 | 0x3005 | 0x3006 | 0x30F5 | 0x30F6 => 'H', 0x3041..=0x3093 => 'I', 0x30A1..=0x30F4 | 0x30FC | 0xFF71..=0xFF9D | 0xFF9E | 0xFF70 => 'K', 0x61..=0x7A | 0x41..=0x5A | 0xFF41..=0xFF5A | 0xFF21..=0xFF3A => 'A', 0x30..=0x3a | 0xFF10..=0xFF19 => 'N', _ => 'O', } } pub fn tokenize(s: &str) -> Vec { if s.is_empty() { return Vec::new(); } let mut result = Vec::with_capacity(s.chars().count()); let segments = [B3, B2, B1] .into_iter() .chain(s.chars()) .chain([E1, E2, E3]) .collect::>(); let ctypes = ['O'; 3] .into_iter() .chain(s.chars().map(get_ctype)) .chain(['O'; 3]) .collect::>(); let mut word = segments[3].to_string(); let mut p = vec!['U'; 3]; for index in 4..segments.len() - 3 { let mut score = BIAS; let w = &segments[index - 3..index + 3]; let c = &ctypes[index - 3..index + 3]; score += get_score(&*UP1, &p[0]); score += get_score(&*UP2, &p[1]); score += get_score(&*UP3, &p[2]); score += get_score(&*BP1, &(p[0], p[1])); score += get_score(&*BP2, &(p[1], p[2])); score += get_score(&*UW1, &w[0]); score += get_score(&*UW2, &w[1]); score += get_score(&*UW3, &w[2]); score += get_score(&*UW4, &w[3]); score += get_score(&*UW5, &w[4]); score += get_score(&*UW6, &w[5]); score += get_score(&*BW1, &(w[1], w[2])); score += get_score(&*BW2, &(w[2], w[3])); score += get_score(&*BW3, &(w[3], w[4])); score += get_score(&*TW1, &(w[0], w[1], w[2])); score += get_score(&*TW2, &(w[1], w[2], w[3])); score += get_score(&*TW3, &(w[2], w[3], w[4])); score += get_score(&*TW4, &(w[3], w[4], w[5])); score += get_score(&*UC1, &c[0]); score += get_score(&*UC2, &c[1]); score += get_score(&*UC3, &c[2]); score += get_score(&*UC4, &c[3]); score += get_score(&*UC5, &c[4]); score += get_score(&*UC6, &c[5]); score += get_score(&*BC1, &(c[1], c[2])); score += get_score(&*BC2, &(c[2], c[3])); score += get_score(&*BC3, &(c[3], c[4])); score += get_score(&*TC1, &(c[0], c[1], c[2])); score += get_score(&*TC2, &(c[1], c[2], c[3])); score += get_score(&*TC3, &(c[2], c[3], c[4])); score += get_score(&*TC4, &(c[3], c[4], c[5])); score += get_score(&*UQ1, &(p[0], c[0])); score += get_score(&*UQ2, &(p[1], c[1])); score += get_score(&*UQ3, &(p[2], c[2])); score += get_score(&*BQ1, &(p[1], c[1], c[2])); score += get_score(&*BQ2, &(p[1], c[2], c[3])); score += get_score(&*BQ3, &(p[2], c[1], c[2])); score += get_score(&*BQ4, &(p[2], c[2], c[3])); score += get_score(&*TQ1, &(p[1], c[0], c[1], c[2])); score += get_score(&*TQ2, &(p[1], c[1], c[2], c[3])); score += get_score(&*TQ3, &(p[2], c[0], c[1], c[2])); score += get_score(&*TQ4, &(p[2], c[1], c[2], c[3])); p.remove(0); p.push(if score < 0 { 'O' } else { 'B' }); if 0 < score { result.push(word.clone()); word.clear(); } word.push(segments[index]); } result.push(word.clone()); result } const B1: char = '\u{F0000}'; const B2: char = '\u{F0001}'; const B3: char = '\u{F0002}'; const E1: char = '\u{F0003}'; const E2: char = '\u{F0004}'; const E3: char = '\u{F0005}'; static BC1: LazyLock> = LazyLock::new(|| { hashmap! { ('H', 'H') => 6, ('I', 'I') => 2461, ('K', 'H') => 406, ('O', 'H') => -1378, } }); static BC2: LazyLock> = LazyLock::new(|| { hashmap! { ('A', 'A') => -3267, ('A', 'I') => 2744, ('A', 'N') => -878, ('H', 'H') => -4070, ('H', 'M') => -1711, ('H', 'N') => 4012, ('H', 'O') => 3761, ('I', 'A') => 1327, ('I', 'H') => -1184, ('I', 'I') => -1332, ('I', 'K') => 1721, ('I', 'O') => 5492, ('K', 'I') => 3831, ('K', 'K') => -8741, ('M', 'H') => -3132, ('M', 'K') => 3334, ('O', 'O') => -2920, } }); static BC3: LazyLock> = LazyLock::new(|| { hashmap! { ('H', 'H') => 996, ('H', 'I') => 626, ('H', 'K') => -721, ('H', 'N') => -1307, ('H', 'O') => -836, ('I', 'H') => -301, ('K', 'K') => 2762, ('M', 'K') => 1079, ('M', 'M') => 4034, ('O', 'A') => -1652, ('O', 'H') => 266, } }); static BP1: LazyLock> = LazyLock::new(|| { hashmap! { ('B', 'B') => 295, ('O', 'B') => 304, ('O', 'O') => -125, ('U', 'B') => 352, } }); static BP2: LazyLock> = LazyLock::new(|| { hashmap! { ('B', 'O') => 60, ('O', 'O') => -1762, } }); static BQ1: LazyLock> = LazyLock::new(|| { hashmap! { ('B', 'H', 'H') => 1150, ('B', 'H', 'M') => 1521, ('B', 'I', 'I') => -1158, ('B', 'I', 'M') => 886, ('B', 'M', 'H') => 1208, ('B', 'N', 'H') => 449, ('B', 'O', 'H') => -91, ('B', 'O', 'O') => -2597, ('O', 'H', 'I') => 451, ('O', 'I', 'H') => -296, ('O', 'K', 'A') => 1851, ('O', 'K', 'H') => -1020, ('O', 'K', 'K') => 904, ('O', 'O', 'O') => 2965, } }); static BQ2: LazyLock> = LazyLock::new(|| { hashmap! { ('B', 'H', 'H') => 118, ('B', 'H', 'I') => -1159, ('B', 'H', 'M') => 466, ('B', 'I', 'H') => -919, ('B', 'K', 'K') => -1720, ('B', 'K', 'O') => 864, ('O', 'H', 'H') => -1139, ('O', 'H', 'M') => -181, ('O', 'I', 'H') => 153, ('U', 'H', 'I') => -1146, } }); static BQ3: LazyLock> = LazyLock::new(|| { hashmap! { ('B', 'H', 'H') => -792, ('B', 'H', 'I') => 2664, ('B', 'I', 'I') => -299, ('B', 'K', 'I') => 419, ('B', 'M', 'H') => 937, ('B', 'M', 'M') => 8335, ('B', 'N', 'N') => 998, ('B', 'O', 'H') => 775, ('O', 'H', 'H') => 2174, ('O', 'H', 'M') => 439, ('O', 'I', 'I') => 280, ('O', 'K', 'H') => 1798, ('O', 'K', 'I') => -793, ('O', 'K', 'O') => -2242, ('O', 'M', 'H') => -2402, ('O', 'O', 'O') => 11699, } }); static BQ4: LazyLock> = LazyLock::new(|| { hashmap! { ('B', 'H', 'H') => -3895, ('B', 'I', 'H') => 3761, ('B', 'I', 'I') => -4654, ('B', 'I', 'K') => 1348, ('B', 'K', 'K') => -1806, ('B', 'M', 'I') => -3385, ('B', 'O', 'O') => -12396, ('O', 'A', 'H') => 926, ('O', 'H', 'H') => 266, ('O', 'H', 'K') => -2036, ('O', 'N', 'N') => -973, } }); static BW1: LazyLock> = LazyLock::new(|| { hashmap! { (',', 'と') => 660, (',', '同') => 727, (B1, 'あ') => 1404, (B1, '同') => 542, ('、', 'と') => 660, ('、', '同') => 727, ('」', 'と') => 1682, ('あ', 'っ') => 1505, ('い', 'う') => 1743, ('い', 'っ') => -2055, ('い', 'る') => 672, ('う', 'し') => -4817, ('う', 'ん') => 665, ('か', 'ら') => 3472, ('が', 'ら') => 600, ('こ', 'う') => -790, ('こ', 'と') => 2083, ('こ', 'ん') => -1262, ('さ', 'ら') => -4143, ('さ', 'ん') => 4573, ('し', 'た') => 2641, ('し', 'て') => 1104, ('す', 'で') => -3399, ('そ', 'こ') => 1977, ('そ', 'れ') => -871, ('た', 'ち') => 1122, ('た', 'め') => 601, ('っ', 'た') => 3463, ('つ', 'い') => -802, ('て', 'い') => 805, ('て', 'き') => 1249, ('で', 'き') => 1127, ('で', 'す') => 3445, ('で', 'は') => 844, ('と', 'い') => -4915, ('と', 'み') => 1922, ('ど', 'こ') => 3887, ('な', 'い') => 5713, ('な', 'っ') => 3015, ('な', 'ど') => 7379, ('な', 'ん') => -1113, ('に', 'し') => 2468, ('に', 'は') => 1498, ('に', 'も') => 1671, ('に', '対') => -912, ('の', '一') => -501, ('の', '中') => 741, ('ま', 'せ') => 2448, ('ま', 'で') => 1711, ('ま', 'ま') => 2600, ('ま', 'る') => -2155, ('や', 'む') => -1947, ('よ', 'っ') => -2565, ('れ', 'た') => 2369, ('れ', 'で') => -913, ('を', 'し') => 1860, ('を', '見') => 731, ('亡', 'く') => -1886, ('京', '都') => 2558, ('取', 'り') => -2784, ('大', 'き') => -2604, ('大', '阪') => 1497, ('平', '方') => -2314, ('引', 'き') => -1336, ('日', '本') => -195, ('本', '当') => -2423, ('毎', '日') => -2113, ('目', '指') => -724, ('｣', 'と') => 1682, } }); static BW2: LazyLock> = LazyLock::new(|| { hashmap! { ('.', '.') => -11822, ('1', '1') => -669, ('―', '―') => -5730, ('−', '−') => -13175, ('い', 'う') => -1609, ('う', 'か') => 2490, ('か', 'し') => -1350, ('か', 'も') => -602, ('か', 'ら') => -7194, ('か', 'れ') => 4612, ('が', 'い') => 853, ('が', 'ら') => -3198, ('き', 'た') => 1941, ('く', 'な') => -1597, ('こ', 'と') => -8392, ('こ', 'の') => -4193, ('さ', 'せ') => 4533, ('さ', 'れ') => 13168, ('さ', 'ん') => -3977, ('し', 'い') => -1819, ('し', 'か') => -545, ('し', 'た') => 5078, ('し', 'て') => 972, ('し', 'な') => 939, ('そ', 'の') => -3744, ('た', 'い') => -1253, ('た', 'た') => -662, ('た', 'だ') => -3857, ('た', 'ち') => -786, ('た', 'と') => 1224, ('た', 'は') => -939, ('っ', 'た') => 4589, ('っ', 'て') => 1647, ('っ', 'と') => -2094, ('て', 'い') => 6144, ('て', 'き') => 3640, ('て', 'く') => 2551, ('て', 'は') => -3110, ('て', 'も') => -3065, ('で', 'い') => 2666, ('で', 'き') => -1528, ('で', 'し') => -3828, ('で', 'す') => -4761, ('で', 'も') => -4203, ('と', 'い') => 1890, ('と', 'こ') => -1746, ('と', 'と') => -2279, ('と', 'の') => 720, ('と', 'み') => 5168, ('と', 'も') => -3941, ('な', 'い') => -2488, ('な', 'が') => -1313, ('な', 'ど') => -6509, ('な', 'の') => 2614, ('な', 'ん') => 3099, ('に', 'お') => -1615, ('に', 'し') => 2748, ('に', 'な') => 2454, ('に', 'よ') => -7236, ('に', '対') => -14943, ('に', '従') => -4688, ('に', '関') => -11388, ('の', 'か') => 2093, ('の', 'で') => -7059, ('の', 'に') => -6041, ('の', 'の') => -6125, ('は', 'い') => 1073, ('は', 'が') => -1033, ('は', 'ず') => -2532, ('ば', 'れ') => 1813, ('ま', 'し') => -1316, ('ま', 'で') => -6621, ('ま', 'れ') => 5409, ('め', 'て') => -3153, ('も', 'い') => 2230, ('も', 'の') => -10713, ('ら', 'か') => -944, ('ら', 'し') => -1611, ('ら', 'に') => -1897, ('り', 'し') => 651, ('り', 'ま') => 1620, ('れ', 'た') => 4270, ('れ', 'て') => 849, ('れ', 'ば') => 4114, ('ろ', 'う') => 6067, ('わ', 'れ') => 7901, ('を', '通') => -11877, ('ん', 'だ') => 728, ('ん', 'な') => -4115, ('一', '人') => 602, ('一', '方') => -1375, ('一', '日') => 970, ('一', '部') => -1051, ('上', 'が') => -4479, ('会', '社') => -1116, ('出', 'て') => 2163, ('分', 'の') => -7758, ('同', '党') => 970, ('同', '日') => -913, ('大', '阪') => -2471, ('委', '員') => -1250, ('少', 'な') => -1050, ('年', '度') => -8669, ('年', '間') => -1626, ('府', '県') => -2363, ('手', '権') => -1982, ('新', '聞') => -4066, ('日', '新') => -722, ('日', '本') => -7068, ('日', '米') => 3372, ('曜', '日') => -601, ('朝', '鮮') => -2355, ('本', '人') => -2697, ('東', '京') => -1543, ('然', 'と') => -1384, ('社', '会') => -1276, ('立', 'て') => -990, ('第', 'に') => -1612, ('米', '国') => -4268, ('１', '１') => -669, ('ｸ', 'ﾞ') => 1319,} }); static BW3: LazyLock> = LazyLock::new(|| { hashmap! { ('あ', 'た') => -2194, ('あ', 'り') => 719, ('あ', 'る') => 3846, ('い', '.') => -1185, ('い', '。') => -1185, ('い', 'い') => 5308, ('い', 'え') => 2079, ('い', 'く') => 3029, ('い', 'た') => 2056, ('い', 'っ') => 1883, ('い', 'る') => 5600, ('い', 'わ') => 1527, ('う', 'ち') => 1117, ('う', 'と') => 4798, ('え', 'と') => 1454, ('か', '.') => 2857, ('か', '。') => 2857, ('か', 'け') => -743, ('か', 'っ') => -4098, ('か', 'に') => -669, ('か', 'ら') => 6520, ('か', 'り') => -2670, ('が', ',') => 1816, ('が', '、') => 1816, ('が', 'き') => -4855, ('が', 'け') => -1127, ('が', 'っ') => -913, ('が', 'ら') => -4977, ('が', 'り') => -2064, ('き', 'た') => 1645, ('け', 'ど') => 1374, ('こ', 'と') => 7397, ('こ', 'の') => 1542, ('こ', 'ろ') => -2757, ('さ', 'い') => -714, ('さ', 'を') => 976, ('し', ',') => 1557, ('し', '、') => 1557, ('し', 'い') => -3714, ('し', 'た') => 3562, ('し', 'て') => 1449, ('し', 'な') => 2608, ('し', 'ま') => 1200, ('す', '.') => -1310, ('す', '。') => -1310, ('す', 'る') => 6521, ('ず', ',') => 3426, ('ず', '、') => 3426, ('ず', 'に') => 841, ('そ', 'う') => 428, ('た', '.') => 8875, ('た', '。') => 8875, ('た', 'い') => -594, ('た', 'の') => 812, ('た', 'り') => -1183, ('た', 'る') => -853, ('だ', '.') => 4098, ('だ', '。') => 4098, ('だ', 'っ') => 1004, ('っ', 'た') => -4748, ('っ', 'て') => 300, ('て', 'い') => 6240, ('て', 'お') => 855, ('て', 'も') => 302, ('で', 'す') => 1437, ('で', 'に') => -1482, ('で', 'は') => 2295, ('と', 'う') => -1387, ('と', 'し') => 2266, ('と', 'の') => 541, ('と', 'も') => -3543, ('ど', 'う') => 4664, ('な', 'い') => 1796, ('な', 'く') => -903, ('な', 'ど') => 2135, ('に', ',') => -1021, ('に', '、') => -1021, ('に', 'し') => 1771, ('に', 'な') => 1906, ('に', 'は') => 2644, ('の', ',') => -724, ('の', '、') => -724, ('の', '子') => -1000, ('は', ',') => 1337, ('は', '、') => 1337, ('べ', 'き') => 2181, ('ま', 'し') => 1113, ('ま', 'す') => 6943, ('ま', 'っ') => -1549, ('ま', 'で') => 6154, ('ま', 'れ') => -793, ('ら', 'し') => 1479, ('ら', 'れ') => 6820, ('る', 'る') => 3818, ('れ', ',') => 854, ('れ', '、') => 854, ('れ', 'た') => 1850, ('れ', 'て') => 1375, ('れ', 'ば') => -3246, ('れ', 'る') => 1091, ('わ', 'れ') => -605, ('ん', 'だ') => 606, ('ん', 'で') => 798, ('カ', '月') => 990, ('会', '議') => 860, ('入', 'り') => 1232, ('大', '会') => 2217, ('始', 'め') => 1681, ('市', ' ') => 965, ('新', '聞') => -5055, ('日', ',') => 974, ('日', '、') => 974, ('社', '会') => 2024, ('ｶ', '月') => 990, } }); static TC1: LazyLock> = LazyLock::new(|| { hashmap! { ('A', 'A', 'A') => 1093, ('H', 'H', 'H') => 1029, ('H', 'H', 'M') => 580, ('H', 'I', 'I') => 998, ('H', 'O', 'H') => -390, ('H', 'O', 'M') => -331, ('I', 'H', 'I') => 1169, ('I', 'O', 'H') => -142, ('I', 'O', 'I') => -1015, ('I', 'O', 'M') => 467, ('M', 'M', 'H') => 187, ('O', 'O', 'I') => -1832, } }); static TC2: LazyLock> = LazyLock::new(|| { hashmap! { ('H', 'H', 'O') => 2088, ('H', 'I', 'I') => -1023, ('H', 'M', 'M') => -1154, ('I', 'H', 'I') => -1965, ('K', 'K', 'H') => 703, ('O', 'I', 'I') => -2649, } }); static TC3: LazyLock> = LazyLock::new(|| { hashmap! { ('A', 'A', 'A') => -294, ('H', 'H', 'H') => 346, ('H', 'H', 'I') => -341, ('H', 'I', 'I') => -1088, ('H', 'I', 'K') => 731, ('H', 'O', 'H') => -1486, ('I', 'H', 'H') => 128, ('I', 'H', 'I') => -3041, ('I', 'H', 'O') => -1935, ('I', 'I', 'H') => -825, ('I', 'I', 'M') => -1035, ('I', 'O', 'I') => -542, ('K', 'H', 'H') => -1216, ('K', 'K', 'A') => 491, ('K', 'K', 'H') => -1217, ('K', 'O', 'K') => -1009, ('M', 'H', 'H') => -2694, ('M', 'H', 'M') => -457, ('M', 'H', 'O') => 123, ('M', 'M', 'H') => -471, ('N', 'N', 'H') => -1689, ('N', 'N', 'O') => 662, ('O', 'H', 'O') => -3393, } }); static TC4: LazyLock> = LazyLock::new(|| { hashmap! { ('H', 'H', 'H') => -203, ('H', 'H', 'I') => 1344, ('H', 'H', 'K') => 365, ('H', 'H', 'M') => -122, ('H', 'H', 'N') => 182, ('H', 'H', 'O') => 669, ('H', 'I', 'H') => 804, ('H', 'I', 'I') => 679, ('H', 'O', 'H') => 446, ('I', 'H', 'H') => 695, ('I', 'H', 'O') => -2324, ('I', 'I', 'H') => 321, ('I', 'I', 'I') => 1497, ('I', 'I', 'O') => 656, ('I', 'O', 'O') => 54, ('K', 'A', 'K') => 4845, ('K', 'K', 'A') => 3386, ('K', 'K', 'K') => 3065, ('M', 'H', 'H') => -405, ('M', 'H', 'I') => 201, ('M', 'M', 'H') => -241, ('M', 'M', 'M') => 661, ('M', 'O', 'M') => 841, } }); static TQ1: LazyLock> = LazyLock::new(|| { hashmap! { ('B', 'H', 'H', 'H') => -227, ('B', 'H', 'H', 'I') => 316, ('B', 'H', 'I', 'H') => -132, ('B', 'I', 'H', 'H') => 60, ('B', 'I', 'I', 'I') => 1595, ('B', 'N', 'H', 'H') => -744, ('B', 'O', 'H', 'H') => 225, ('B', 'O', 'O', 'O') => -908, ('O', 'A', 'K', 'K') => 482, ('O', 'H', 'H', 'H') => 281, ('O', 'H', 'I', 'H') => 249, ('O', 'I', 'H', 'I') => 200, ('O', 'I', 'I', 'H') => -68, } }); static TQ2: LazyLock> = LazyLock::new(|| { hashmap! { ('B', 'I', 'H', 'H') => -1401, ('B', 'I', 'I', 'I') => -1033, ('B', 'K', 'A', 'K') => -543, ('B', 'O', 'O', 'O') => -5591, } }); static TQ3: LazyLock> = LazyLock::new(|| { hashmap! { ('B', 'H', 'H', 'H') => 478, ('B', 'H', 'H', 'M') => -1073, ('B', 'H', 'I', 'H') => 222, ('B', 'H', 'I', 'I') => -504, ('B', 'I', 'I', 'H') => -116, ('B', 'I', 'I', 'I') => -105, ('B', 'M', 'H', 'I') => -863, ('B', 'M', 'H', 'M') => -464, ('B', 'O', 'M', 'H') => 620, ('O', 'H', 'H', 'H') => 346, ('O', 'H', 'H', 'I') => 1729, ('O', 'H', 'I', 'I') => 997, ('O', 'H', 'M', 'H') => 481, ('O', 'I', 'H', 'H') => 623, ('O', 'I', 'I', 'H') => 1344, ('O', 'K', 'A', 'K') => 2792, ('O', 'K', 'H', 'H') => 587, ('O', 'K', 'K', 'A') => 679, ('O', 'O', 'H', 'H') => 110, ('O', 'O', 'I', 'I') => -685, } }); static TQ4: LazyLock> = LazyLock::new(|| { hashmap! { ('B', 'H', 'H', 'H') => -721, ('B', 'H', 'H', 'M') => -3604, ('B', 'H', 'I', 'I') => -966, ('B', 'I', 'I', 'H') => -607, ('B', 'I', 'I', 'I') => -2181, ('O', 'A', 'A', 'A') => -2763, ('O', 'A', 'K', 'K') => 180, ('O', 'H', 'H', 'H') => -294, ('O', 'H', 'H', 'I') => 2446, ('O', 'H', 'H', 'O') => 480, ('O', 'H', 'I', 'H') => -1573, ('O', 'I', 'H', 'H') => 1935, ('O', 'I', 'H', 'I') => -493, ('O', 'I', 'I', 'H') => 626, ('O', 'I', 'I', 'I') => -4007, ('O', 'K', 'A', 'K') => -8156, } }); static TW1: LazyLock> = LazyLock::new(|| { hashmap! { ('に', 'つ', 'い') => -4681, ('東', '京', '都') => 2026, } }); static TW2: LazyLock> = LazyLock::new(|| { hashmap! { ('あ', 'る', '程') => -2049, ('い', 'っ', 'た') => -1256, ('こ', 'ろ', 'が') => -2434, ('し', 'ょ', 'う') => 3873, ('そ', 'の', '後') => -4430, ('だ', 'っ', 'て') => -1049, ('て', 'い', 'た') => 1833, ('と', 'し', 'て') => -4657, ('と', 'も', 'に') => -4517, ('も', 'の', 'で') => 1882, ('一', '気', 'に') => -792, ('初', 'め', 'て') => -1512, ('同', '時', 'に') => -8097, ('大', 'き', 'な') => -1255, ('対', 'し', 'て') => -2721, ('社', '会', '党') => -3216, } }); static TW3: LazyLock> = LazyLock::new(|| { hashmap! { ('い', 'た', 'だ') => -1734, ('し', 'て', 'い') => 1314, ('と', 'し', 'て') => -4314, ('に', 'つ', 'い') => -5483, ('に', 'と', 'っ') => -5989, ('に', '当', 'た') => -6247, ('の', 'で', ',') => -727, ('の', 'で', '、') => -727, ('の', 'も', 'の') => -600, ('れ', 'か', 'ら') => -3752, ('十', '二', '月') => -2287, } }); static TW4: LazyLock> = LazyLock::new(|| { hashmap! { ('い', 'う', '.') => 8576, ('い', 'う', '。') => 8576, ('か', 'ら', 'な') => -2348, ('し', 'て', 'い') => 2958, ('た', 'が', ',') => 1516, ('た', 'が', '、') => 1516, ('て', 'い', 'る') => 1538, ('と', 'い', 'う') => 1349, ('ま', 'し', 'た') => 5543, ('ま', 'せ', 'ん') => 1097, ('よ', 'う', 'と') => -4258, ('よ', 'る', 'と') => 5865, } }); static UC1: LazyLock> = LazyLock::new(|| { hashmap! { 'A' => 484, 'K' => 93, 'M' => 645, 'O' => -505, } }); static UC2: LazyLock> = LazyLock::new(|| { hashmap! { 'A' => 819, 'H' => 1059, 'I' => 409, 'M' => 3987, 'N' => 5775, 'O' => 646, } }); static UC3: LazyLock> = LazyLock::new(|| { hashmap! { 'A' => -1370, 'I' => 2311, } }); static UC4: LazyLock> = LazyLock::new(|| { hashmap! { 'A' => -2643, 'H' => 1809, 'I' => -1032, 'K' => -3450, 'M' => 3565, 'N' => 3876, 'O' => 6646, } }); static UC5: LazyLock> = LazyLock::new(|| { hashmap! { 'H' => 313, 'I' => -1238, 'K' => -799, 'M' => 539, 'O' => -831, } }); static UC6: LazyLock> = LazyLock::new(|| { hashmap! { 'H' => -506, 'I' => -253, 'K' => 87, 'M' => 247, 'O' => -387, } }); static UP1: LazyLock> = LazyLock::new(|| { hashmap! { 'O' => -214, } }); static UP2: LazyLock> = LazyLock::new(|| { hashmap! { 'B' => 69, 'O' => 935, } }); static UP3: LazyLock> = LazyLock::new(|| { hashmap! { 'B' => 189, } }); static UQ1: LazyLock> = LazyLock::new(|| { hashmap! { ('B', 'H') => 21, ('B', 'I') => -12, ('B', 'K') => -99, ('B', 'N') => 142, ('B', 'O') => -56, ('O', 'H') => -95, ('O', 'I') => 477, ('O', 'K') => 410, ('O', 'O') => -2422, } }); static UQ2: LazyLock> = LazyLock::new(|| { hashmap! { ('B', 'H') => 216, ('B', 'I') => 113, ('O', 'K') => 1759, } }); static UQ3: LazyLock> = LazyLock::new(|| { hashmap! { ('B', 'A') => -479, ('B', 'H') => 42, ('B', 'I') => 1913, ('B', 'K') => -7198, ('B', 'M') => 3160, ('B', 'N') => 6427, ('B', 'O') => 14761, ('O', 'I') => -827, ('O', 'N') => -3212, } }); static UW1: LazyLock> = LazyLock::new(|| { hashmap! { ',' => 156, '、' => 156, '「' => -463, 'あ' => -941, 'う' => -127, 'が' => -553, 'き' => 121, 'こ' => 505, 'で' => -201, 'と' => -547, 'ど' => -123, 'に' => -789, 'の' => -185, 'は' => -847, 'も' => -466, 'や' => -470, 'よ' => 182, 'ら' => -292, 'り' => 208, 'れ' => 169, 'を' => -446, 'ん' => -137, '・' => -135, '主' => -402, '京' => -268, '区' => -912, '午' => 871, '国' => -460, '大' => 561, '委' => 729, '市' => -411, '日' => -141, '理' => 361, '生' => -408, '県' => -386, '都' => -718, '｢' => -463, '･' => -135, } }); static UW2: LazyLock> = LazyLock::new(|| { hashmap! { ',' => -829, '、' => -829, '〇' => 892, '「' => -645, '」' => 3145, 'あ' => -538, 'い' => 505, 'う' => 134, 'お' => -502, 'か' => 1454, 'が' => -856, 'く' => -412, 'こ' => 1141, 'さ' => 878, 'ざ' => 540, 'し' => 1529, 'す' => -675, 'せ' => 300, 'そ' => -1011, 'た' => 188, 'だ' => 1837, 'つ' => -949, 'て' => -291, 'で' => -268, 'と' => -981, 'ど' => 1273, 'な' => 1063, 'に' => -1764, 'の' => 130, 'は' => -409, 'ひ' => -1273, 'べ' => 1261, 'ま' => 600, 'も' => -1263, 'や' => -402, 'よ' => 1639, 'り' => -579, 'る' => -694, 'れ' => 571, 'を' => -2516, 'ん' => 2095, 'ア' => -587, 'カ' => 306, 'キ' => 568, 'ッ' => 831, '三' => -758, '不' => -2150, '世' => -302, '中' => -968, '主' => -861, '事' => 492, '人' => -123, '会' => 978, '保' => 362, '入' => 548, '初' => -3025, '副' => -1566, '北' => -3414, '区' => -422, '大' => -1769, '天' => -865, '太' => -483, '子' => -1519, '学' => 760, '実' => 1023, '小' => -2009, '市' => -813, '年' => -1060, '強' => 1067, '手' => -1519, '揺' => -1033, '政' => 1522, '文' => -1355, '新' => -1682, '日' => -1815, '明' => -1462, '最' => -630, '朝' => -1843, '本' => -1650, '東' => -931, '果' => -665, '次' => -2378, '民' => -180, '気' => -1740, '理' => 752, '発' => 529, '目' => -1584, '相' => -242, '県' => -1165, '立' => -763, '第' => 810, '米' => 509, '自' => -1353, '行' => 838, '西' => -744, '見' => -3874, '調' => 1010, '議' => 1198, '込' => 3041, '開' => 1758, '間' => -1257, '｢' => -645, '｣' => 3145, 'ｯ' => 831, 'ｱ' => -587, 'ｶ' => 306, 'ｷ' => 568, } }); static UW3: LazyLock> = LazyLock::new(|| { hashmap! { ',' => 4889, '1' => -800, '−' => -1723, '、' => 4889, '々' => -2311, '〇' => 5827, '」' => 2670, '〓' => -3573, 'あ' => -2696, 'い' => 1006, 'う' => 2342, 'え' => 1983, 'お' => -4864, 'か' => -1163, 'が' => 3271, 'く' => 1004, 'け' => 388, 'げ' => 401, 'こ' => -3552, 'ご' => -3116, 'さ' => -1058, 'し' => -395, 'す' => 584, 'せ' => 3685, 'そ' => -5228, 'た' => 842, 'ち' => -521, 'っ' => -1444, 'つ' => -1081, 'て' => 6167, 'で' => 2318, 'と' => 1691, 'ど' => -899, 'な' => -2788, 'に' => 2745, 'の' => 4056, 'は' => 4555, 'ひ' => -2171, 'ふ' => -1798, 'へ' => 1199, 'ほ' => -5516, 'ま' => -4384, 'み' => -120, 'め' => 1205, 'も' => 2323, 'や' => -788, 'よ' => -202, 'ら' => 727, 'り' => 649, 'る' => 5905, 'れ' => 2773, 'わ' => -1207, 'を' => 6620, 'ん' => -518, 'ア' => 551, 'グ' => 1319, 'ス' => 874, 'ッ' => -1350, 'ト' => 521, 'ム' => 1109, 'ル' => 1591, 'ロ' => 2201, 'ン' => 278, '・' => -3794, '一' => -1619, '下' => -1759, '世' => -2087, '両' => 3815, '中' => 653, '主' => -758, '予' => -1193, '二' => 974, '人' => 2742, '今' => 792, '他' => 1889, '以' => -1368, '低' => 811, '何' => 4265, '作' => -361, '保' => -2439, '元' => 4858, '党' => 3593, '全' => 1574, '公' => -3030, '六' => 755, '共' => -1880, '円' => 5807, '再' => 3095, '分' => 457, '初' => 2475, '別' => 1129, '前' => 2286, '副' => 4437, '力' => 365, '動' => -949, '務' => -1872, '化' => 1327, '北' => -1038, '区' => 4646, '千' => -2309, '午' => -783, '協' => -1006, '口' => 483, '右' => 1233, '各' => 3588, '合' => -241, '同' => 3906, '和' => -837, '員' => 4513, '国' => 642, '型' => 1389, '場' => 1219, '外' => -241, '妻' => 2016, '学' => -1356, '安' => -423, '実' => -1008, '家' => 1078, '小' => -513, '少' => -3102, '州' => 1155, '市' => 3197, '平' => -1804, '年' => 2416, '広' => -1030, '府' => 1605, '度' => 1452, '建' => -2352, '当' => -3885, '得' => 1905, '思' => -1291, '性' => 1822, '戸' => -488, '指' => -3973, '政' => -2013, '教' => -1479, '数' => 3222, '文' => -1489, '新' => 1764, '日' => 2099, '旧' => 5792, '昨' => -661, '時' => -1248, '曜' => -951, '最' => -937, '月' => 4125, '期' => 360, '李' => 3094, '村' => 364, '東' => -805, '核' => 5156, '森' => 2438, '業' => 484, '氏' => 2613, '民' => -1694, '決' => -1073, '法' => 1868, '海' => -495, '無' => 979, '物' => 461, '特' => -3850, '生' => -273, '用' => 914, '町' => 1215, '的' => 7313, '直' => -1835, '省' => 792, '県' => 6293, '知' => -1528, '私' => 4231, '税' => 401, '立' => -960, '第' => 1201, '米' => 7767, '系' => 3066, '約' => 3663, '級' => 1384, '統' => -4229, '総' => 1163, '線' => 1255, '者' => 6457, '能' => 725, '自' => -2869, '英' => 785, '見' => 1044, '調' => -562, '財' => -733, '費' => 1777, '車' => 1835, '軍' => 1375, '込' => -1504, '通' => -1136, '選' => -681, '郎' => 1026, '郡' => 4404, '部' => 1200, '金' => 2163, '長' => 421, '開' => -1432, '間' => 1302, '関' => -1282, '雨' => 2009, '電' => -1045, '非' => 2066, '駅' => 1620, '１' => -800, '｣' => 2670, '･' => -3794, 'ｯ' => -1350, 'ｱ' => 551, 'ｽ' => 874, 'ﾄ' => 521, 'ﾑ' => 1109, 'ﾙ' => 1591, 'ﾛ' => 2201, 'ﾝ' => 278, } }); static UW4: LazyLock> = LazyLock::new(|| { hashmap! { ',' => 3930, '.' => 3508, '―' => -4841, '、' => 3930, '。' => 3508, '〇' => 4999, '「' => 1895, '」' => 3798, '〓' => -5156, 'あ' => 4752, 'い' => -3435, 'う' => -640, 'え' => -2514, 'お' => 2405, 'か' => 530, 'が' => 6006, 'き' => -4482, 'ぎ' => -3821, 'く' => -3788, 'け' => -4376, 'げ' => -4734, 'こ' => 2255, 'ご' => 1979, 'さ' => 2864, 'し' => -843, 'じ' => -2506, 'す' => -731, 'ず' => 1251, 'せ' => 181, 'そ' => 4091, 'た' => 5034, 'だ' => 5408, 'ち' => -3654, 'っ' => -5882, 'つ' => -1659, 'て' => 3994, 'で' => 7410, 'と' => 4547, 'な' => 5433, 'に' => 6499, 'ぬ' => 1853, 'ね' => 1413, 'の' => 7396, 'は' => 8578, 'ば' => 1940, 'ひ' => 4249, 'び' => -4134, 'ふ' => 1345, 'へ' => 6665, 'べ' => -744, 'ほ' => 1464, 'ま' => 1051, 'み' => -2082, 'む' => -882, 'め' => -5046, 'も' => 4169, 'ゃ' => -2666, 'や' => 2795, 'ょ' => -1544, 'よ' => 3351, 'ら' => -2922, 'り' => -9726, 'る' => -14896, 'れ' => -2613, 'ろ' => -4570, 'わ' => -1783, 'を' => 13150, 'ん' => -2352, 'カ' => 2145, 'コ' => 1789, 'セ' => 1287, 'ッ' => -724, 'ト' => -403, 'メ' => -1635, 'ラ' => -881, 'リ' => -541, 'ル' => -856, 'ン' => -3637, '・' => -4371, 'ー' => -11870, '一' => -2069, '中' => 2210, '予' => 782, '事' => -190, '井' => -1768, '人' => 1036, '以' => 544, '会' => 950, '体' => -1286, '作' => 530, '側' => 4292, '先' => 601, '党' => -2006, '共' => -1212, '内' => 584, '円' => 788, '初' => 1347, '前' => 1623, '副' => 3879, '力' => -302, '動' => -740, '務' => -2715, '化' => 776, '区' => 4517, '協' => 1013, '参' => 1555, '合' => -1834, '和' => -681, '員' => -910, '器' => -851, '回' => 1500, '国' => -619, '園' => -1200, '地' => 866, '場' => -1410, '塁' => -2094, '士' => -1413, '多' => 1067, '大' => 571, '子' => -4802, '学' => -1397, '定' => -1057, '寺' => -809, '小' => 1910, '屋' => -1328, '山' => -1500, '島' => -2056, '川' => -2667, '市' => 2771, '年' => 374, '庁' => -4556, '後' => 456, '性' => 553, '感' => 916, '所' => -1566, '支' => 856, '改' => 787, '政' => 2182, '教' => 704, '文' => 522, '方' => -856, '日' => 1798, '時' => 1829, '最' => 845, '月' => -9066, '木' => -485, '来' => -442, '校' => -360, '業' => -1043, '氏' => 5388, '民' => -2716, '気' => -910, '沢' => -939, '済' => -543, '物' => -735, '率' => 672, '球' => -1267, '生' => -1286, '産' => -1101, '田' => -2900, '町' => 1826, '的' => 2586, '目' => 922, '省' => -3485, '県' => 2997, '空' => -867, '立' => -2112, '第' => 788, '米' => 2937, '系' => 786, '約' => 2171, '経' => 1146, '統' => -1169, '総' => 940, '線' => -994, '署' => 749, '者' => 2145, '能' => -730, '般' => -852, '行' => -792, '規' => 792, '警' => -1184, '議' => -244, '谷' => -1000, '賞' => 730, '車' => -1481, '軍' => 1158, '輪' => -1433, '込' => -3370, '近' => 929, '道' => -1291, '選' => 2596, '郎' => -4866, '都' => 1192, '野' => -1100, '銀' => -2213, '長' => 357, '間' => -2344, '院' => -2297, '際' => -2604, '電' => -878, '領' => -1659, '題' => -792, '館' => -1984, '首' => 1749, '高' => 2120, '｢' => 1895, '｣' => 3798, '･' => -4371, 'ｯ' => -724, 'ｰ' => -11870, 'ｶ' => 2145, 'ｺ' => 1789, 'ｾ' => 1287, 'ﾄ' => -403, 'ﾒ' => -1635, 'ﾗ' => -881, 'ﾘ' => -541, 'ﾙ' => -856, 'ﾝ' => -3637, } }); static UW5: LazyLock> = LazyLock::new(|| { hashmap! { ',' => 465, '.' => -299, '1' => -514, E2 => -32768, ']' => -2762, '、' => 465, '。' => -299, '「' => 363, 'あ' => 1655, 'い' => 331, 'う' => -503, 'え' => 1199, 'お' => 527, 'か' => 647, 'が' => -421, 'き' => 1624, 'ぎ' => 1971, 'く' => 312, 'げ' => -983, 'さ' => -1537, 'し' => -1371, 'す' => -852, 'だ' => -1186, 'ち' => 1093, 'っ' => 52, 'つ' => 921, 'て' => -18, 'で' => -850, 'と' => -127, 'ど' => 1682, 'な' => -787, 'に' => -1224, 'の' => -635, 'は' => -578, 'べ' => 1001, 'み' => 502, 'め' => 865, 'ゃ' => 3350, 'ょ' => 854, 'り' => -208, 'る' => 429, 'れ' => 504, 'わ' => 419, 'を' => -1264, 'ん' => 327, 'イ' => 241, 'ル' => 451, 'ン' => -343, '中' => -871, '京' => 722, '会' => -1153, '党' => -654, '務' => 3519, '区' => -901, '告' => 848, '員' => 2104, '大' => -1296, '学' => -548, '定' => 1785, '嵐' => -1304, '市' => -2991, '席' => 921, '年' => 1763, '思' => 872, '所' => -814, '挙' => 1618, '新' => -1682, '日' => 218, '月' => -4353, '査' => 932, '格' => 1356, '機' => -1508, '氏' => -1347, '田' => 240, '町' => -3912, '的' => -3149, '相' => 1319, '省' => -1052, '県' => -4003, '研' => -997, '社' => -278, '空' => -813, '統' => 1955, '者' => -2233, '表' => 663, '語' => -1073, '議' => 1219, '選' => -1018, '郎' => -368, '長' => 786, '間' => 1191, '題' => 2368, '館' => -689, '１' => -514, '｢' => 363, 'ｲ' => 241, 'ﾙ' => 451, 'ﾝ' => -343, } }); static UW6: LazyLock> = LazyLock::new(|| { hashmap! { ',' => 227, '.' => 808, '1' => -270, E1 => 306, '、' => 227, '。' => 808, 'あ' => -307, 'う' => 189, 'か' => 241, 'が' => -73, 'く' => -121, 'こ' => -200, 'じ' => 1782, 'す' => 383, 'た' => -428, 'っ' => 573, 'て' => -1014, 'で' => 101, 'と' => -105, 'な' => -253, 'に' => -149, 'の' => -417, 'は' => -236, 'も' => -206, 'り' => 187, 'る' => -135, 'を' => 195, 'ル' => -673, 'ン' => -496, '一' => -277, '中' => 201, '件' => -800, '会' => 624, '前' => 302, '区' => 1792, '員' => -1212, '委' => 798, '学' => -960, '市' => 887, '広' => -695, '後' => 535, '業' => -697, '相' => 753, '社' => -507, '福' => 974, '空' => -822, '者' => 1811, '連' => 463, '郎' => 1082, '１' => -270, 'ﾙ' => -673, 'ﾝ' => -496, } }); #[cfg(test)] mod tests { use crate::tokenizers::{Token, japanese::JapaneseTokenizer, word::WordTokenizer}; #[test] fn japanese_tokenizer() { assert_eq!( JapaneseTokenizer::new(WordTokenizer::new( "お先に失礼します あなたの名前は何ですか 123 abc-872", 40 )) .collect::>(), vec![ Token { word: "お先".into(), from: 0, to: 6 }, Token { word: "に".into(), from: 6, to: 9 }, Token { word: "失礼".into(), from: 9, to: 15 }, Token { word: "し".into(), from: 15, to: 18 }, Token { word: "ます".into(), from: 18, to: 24 }, Token { word: "あなた".into(), from: 25, to: 34 }, Token { word: "の".into(), from: 34, to: 37 }, Token { word: "名前".into(), from: 37, to: 43 }, Token { word: "は".into(), from: 43, to: 46 }, Token { word: "何".into(), from: 46, to: 49 }, Token { word: "です".into(), from: 49, to: 55 }, Token { word: "か".into(), from: 55, to: 58 }, Token { word: "123".into(), from: 59, to: 62 }, Token { word: "abc".into(), from: 63, to: 66 }, Token { word: "872".into(), from: 67, to: 70 } ] ); } } ================================================ FILE: crates/nlp/src/tokenizers/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod chinese; pub mod japanese; pub mod space; pub mod stream; pub mod types; pub mod word; use std::borrow::Cow; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Token { pub word: T, pub from: usize, pub to: usize, } pub trait InnerToken<'x>: Sized { fn new_alphabetic(value: impl Into>) -> Self; fn unwrap_alphabetic(self) -> Cow<'x, str>; fn is_alphabetic(&self) -> bool; fn is_alphabetic_8bit(&self) -> bool; } impl<'x> InnerToken<'x> for Cow<'x, str> { fn new_alphabetic(value: impl Into>) -> Self { value.into() } fn is_alphabetic(&self) -> bool { true } fn is_alphabetic_8bit(&self) -> bool { !self.is_ascii() } fn unwrap_alphabetic(self) -> Cow<'x, str> { self } } impl Token { pub fn new(offset: usize, len: usize, word: T) -> Token { debug_assert!(offset <= u32::MAX as usize); debug_assert!(len <= u8::MAX as usize); Token { from: offset, to: offset + len, word, } } } ================================================ FILE: crates/nlp/src/tokenizers/space.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::str::Chars; pub struct SpaceTokenizer<'x> { iterator: Chars<'x>, token: String, max_token_length: usize, } impl SpaceTokenizer<'_> { pub fn new(text: &'_ str, max_token_length: usize) -> SpaceTokenizer<'_> { SpaceTokenizer { iterator: text.chars(), token: String::new(), max_token_length, } } } impl Iterator for SpaceTokenizer<'_> { type Item = String; fn next(&mut self) -> Option { for ch in self.iterator.by_ref() { if ch.is_alphanumeric() { if ch.is_uppercase() { for ch in ch.to_lowercase() { self.token.push(ch); } } else { self.token.push(ch); } } else if !self.token.is_empty() { if self.token.len() < self.max_token_length { return Some(std::mem::take(&mut self.token)); } else { self.token.clear(); } } } if !self.token.is_empty() { if self.token.len() < self.max_token_length { return Some(std::mem::take(&mut self.token)); } else { self.token.clear(); } } None } } ================================================ FILE: crates/nlp/src/tokenizers/stream.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ language::{ Language, detect::{LanguageDetector, MIN_LANGUAGE_SCORE}, stemmer::STEMMER_MAP, stopwords::{STOP_WORDS, StopwordFnc}, }, tokenizers::{chinese::JIEBA, japanese}, }; use std::borrow::Cow; pub struct WordStemTokenizer { stemmer: Stemmer, stop_words: Option, } enum Stemmer { IndoEuropean(rust_stemmers::Stemmer), Mandarin, Japanese, None, } impl WordStemTokenizer { pub fn new(text: &str) -> Self { // Detect language let (mut language, score) = LanguageDetector::detect_single(text).unwrap_or((Language::English, 1.0)); if score < MIN_LANGUAGE_SCORE { language = Language::English; } Self { stemmer: match language { Language::Mandarin => Stemmer::Mandarin, Language::Japanese => Stemmer::Japanese, _ => STEMMER_MAP[language as usize] .map(|algo| Stemmer::IndoEuropean(rust_stemmers::Stemmer::create(algo))) .unwrap_or(Stemmer::None), }, stop_words: STOP_WORDS[language as usize], } } pub fn tokenize<'x>(&self, word: &'x str, mut cb: impl FnMut(Cow<'x, str>)) { if self.stop_words.is_some_and(|sw| sw(word)) { return; } match &self.stemmer { Stemmer::IndoEuropean(stemmer) => { cb(stemmer.stem(word)); } Stemmer::Mandarin => { for word in JIEBA.cut(word, false) { cb(Cow::from(word)); } } Stemmer::Japanese => { for word in japanese::tokenize(word) { cb(Cow::from(word)); } } Stemmer::None => { cb(Cow::from(word)); } } } } #[cfg(test)] pub mod tests { use crate::tokenizers::{ stream::WordStemTokenizer, types::{TokenType, TypesTokenizer}, }; #[test] fn stream_tokenizer() { let inputs = [ ( "The quick brown fox jumps over the lazy dog", vec!["quick", "brown", "fox", "jump", "lazi", "dog"], ), ( "Jovencillo emponzoñado de whisky: ¡qué figurota exhibe!", vec!["jovencill", "emponzoñ", "whisky", "figurot", "exhib"], ), ( "Ma la volpe col suo balzo ha raggiunto il quieto Fido", vec!["volp", "balz", "raggiunt", "quiet", "fid"], ), ( "Jaz em prisão bota que vexa dez cegonhas felizes", vec!["jaz", "prisã", "bot", "vex", "dez", "cegonh", "feliz"], ), ( "Zwölf Boxkämpfer jagten Victor quer über den großen Sylter Deich", vec![ "zwolf", "boxkampf", "jagt", "victor", "quer", "gross", "sylt", "deich", ], ), ( "עטלף אבק נס דרך מזגן שהתפוצץ כי חם", vec!["עטלף", "אבק", "נס", "דרך", "מזגן", "שהתפוצץ", "כי", "חם"], ), ( "Съешь ещё этих мягких французских булок, да выпей же чаю", vec![ "съеш", "ещё", "эт", "мягк", "французск", "булок", "вып", "ча", ], ), ( "Чуєш їх, доцю, га? Кумедна ж ти, прощайся без ґольфів!", vec![ "чуєш", "їх", "доцю", "га", "кумедна", "ж", "ти", "прощайся", "без", "ґольфів", ], ), ( "Љубазни фењерџија чађавог лица хоће да ми покаже штос", vec![ "љубазни", "фењерџија", "чађавог", "лица", "хоће", "да", "ми", "покаже", "штос", ], ), ( "Pijamalı hasta yağız şoföre çabucak güvendi", vec!["pijamalı", "hasta", "yağız", "şoför", "çabucak", "güvendi"], ), ("己所不欲,勿施于人。", vec!["己所不欲", "勿施于人"]), ( "井の中の蛙大海を知らず", vec!["井", "の", "中", "の", "蛙大", "海", "を", "知ら", "ず"], ), ("시작이 반이다", vec!["시작이", "반이다"]), ]; for (input, expect) in inputs.iter() { let tokenizer = WordStemTokenizer::new(input); let mut result = Vec::new(); for token in TypesTokenizer::new(&input.to_lowercase()) { if let TokenType::Alphabetic(word) = token.word { tokenizer.tokenize(word, |t| { result.push(t.into_owned()); }); } } assert_eq!(&result, expect,); } } } ================================================ FILE: crates/nlp/src/tokenizers/types.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::str::CharIndices; use super::Token; #[derive(Debug)] pub struct TypesTokenizer<'x> { text: &'x str, iter: CharIndices<'x>, tokens: Vec>>, peek_pos: usize, last_ch_is_space: bool, last_token_is_dot: bool, eof: bool, tokenize_urls: bool, tokenize_urls_without_scheme: bool, tokenize_emails: bool, tokenize_numbers: bool, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum TokenType { Alphabetic(T), Alphanumeric(T), Integer(T), Other(char), Punctuation(char), Space, // Detected types Url(U), UrlNoScheme(U), UrlNoHost(T), IpAddr(I), Email(E), Float(T), } impl Copy for Token> {} impl<'x> Iterator for TypesTokenizer<'x> { type Item = Token>; fn next(&mut self) -> Option { let token = self.peek()?; let last_is_dot = self.last_token_is_dot; self.last_token_is_dot = matches!(token.word, TokenType::Punctuation('.')); // Try parsing URL with scheme if self.tokenize_urls && matches!( token.word, TokenType::Alphabetic(t) | TokenType::Alphanumeric(t) if t.len() <= 8 && t.is_ascii()) && self.try_skip_url_scheme() { if let Some(url) = self.try_parse_url(token.into()) { self.peek_advance(); return Some(url); } else { self.peek_rewind(); } } // Try parsing email if self.tokenize_emails && token.word.is_email_atom() { self.peek_rewind(); if let Some(email) = self.try_parse_email() { self.peek_advance(); return Some(email); } self.peek_rewind(); } // Try parsing URL without scheme if self.tokenize_urls_without_scheme && token.word.is_domain_atom(true) { self.peek_rewind(); if let Some(url) = self.try_parse_url(None) { self.peek_advance(); return Some(url); } self.peek_rewind(); } // Try parsing currencies and floating point numbers if self.tokenize_numbers && !last_is_dot && let Some(num) = self.try_parse_number() { self.peek_advance(); return Some(num); } self.peek_rewind(); self.next_() } } impl<'x> TypesTokenizer<'x> { pub fn new(text: &'x str) -> Self { Self { text, iter: text.char_indices(), tokens: Vec::new(), eof: false, peek_pos: 0, last_ch_is_space: false, last_token_is_dot: false, tokenize_urls: true, tokenize_urls_without_scheme: true, tokenize_emails: true, tokenize_numbers: true, } } pub fn tokenize_urls(mut self, tokenize: bool) -> Self { self.tokenize_urls = tokenize; self } pub fn tokenize_urls_without_scheme(mut self, tokenize: bool) -> Self { self.tokenize_urls_without_scheme = tokenize; self } pub fn tokenize_emails(mut self, tokenize: bool) -> Self { self.tokenize_emails = tokenize; self } pub fn tokenize_numbers(mut self, tokenize: bool) -> Self { self.tokenize_numbers = tokenize; self } fn consume(&mut self) -> bool { let mut has_alpha = false; let mut has_number = false; let mut start_pos = usize::MAX; let mut end_pos = usize::MAX; let mut stop_char = None; for (pos, ch) in self.iter.by_ref() { if ch.is_alphabetic() { has_alpha = true; } else if ch.is_ascii_digit() { has_number = true; } else { let last_was_space = self.last_ch_is_space; self.last_ch_is_space = ch.is_whitespace(); stop_char = Token { word: if self.last_ch_is_space { if last_was_space { continue; } else { TokenType::Space } } else if ch.is_ascii() { TokenType::Punctuation(ch) } else { TokenType::Other(ch) }, from: pos, to: pos + ch.len_utf8(), } .into(); break; } self.last_ch_is_space = false; if start_pos == usize::MAX { start_pos = pos; } end_pos = pos + ch.len_utf8(); } if start_pos != usize::MAX { let text = &self.text[start_pos..end_pos]; self.tokens.push(Token { word: if has_alpha && has_number { TokenType::Alphanumeric(text) } else if has_alpha { TokenType::Alphabetic(text) } else { TokenType::Integer(text) }, from: start_pos, to: end_pos, }); if let Some(stop_char) = stop_char { self.tokens.push(stop_char); } true } else if let Some(stop_char) = stop_char { self.tokens.push(stop_char); true } else { self.eof = true; false } } fn next_(&mut self) -> Option>> { if self.tokens.is_empty() && !self.eof { self.consume(); } if !self.tokens.is_empty() { Some(self.tokens.remove(0)) } else { None } } fn peek(&mut self) -> Option>> { while self.tokens.len() <= self.peek_pos && !self.eof { self.consume(); } self.tokens.get(self.peek_pos).map(|t| { self.peek_pos += 1; *t }) } fn peek_advance(&mut self) { if self.peek_pos > 0 { self.tokens.drain(..self.peek_pos); self.peek_pos = 0; } } #[inline(always)] fn peek_rewind(&mut self) { self.peek_pos = 0; } fn try_parse_url( &mut self, scheme_token: Option>>, ) -> Option>> { let (has_scheme, allow_blank_host) = scheme_token.as_ref().map_or((false, false), |t| { ( true, matches!(t.word, TokenType::Alphabetic(s) if s.eq_ignore_ascii_case("file")), ) }); if has_scheme { let restore_pos = self.peek_pos; let mut has_user_info = false; while let Some(token) = self.peek() { match token.word { TokenType::Punctuation('@') => { has_user_info = true; break; } TokenType::Alphabetic(_) | TokenType::Alphanumeric(_) | TokenType::Integer(_) | TokenType::Punctuation( '-' | '.' | '_' | '~' | '!' | '$' | '&' | '\'' | '(' | ')' | '*' | '+' | ',' | ';' | '=' | ':', ) => (), _ => break, } } if !has_user_info { self.peek_pos = restore_pos; } } // Try parsing hostname let mut is_valid_host = true; let (host_start_pos, mut end_pos, is_ip) = if has_scheme { let mut start_pos = usize::MAX; let mut end_pos = usize::MAX; let mut restore_pos = self.peek_pos; let mut text_count = 0; let mut int_count = 0; let mut dot_count = 0; let mut is_ipv6 = false; let mut last_label_is_tld = false; while let Some(token) = self.peek() { match token.word { TokenType::Alphabetic(text) | TokenType::Alphanumeric(text) => { last_label_is_tld = text.len() >= 2 && psl::Psl::find( &psl::List, [text.to_ascii_lowercase().as_bytes()].into_iter(), ) .typ .is_some(); text_count += 1; } TokenType::Integer(text) => { if text.len() <= 3 { int_count += 1; } } TokenType::Punctuation('.') => { dot_count += 1; continue; } TokenType::Punctuation('[') if start_pos == usize::MAX => { let (_, to) = self.try_parse_ipv6(token.from)?; start_pos = token.from; end_pos = to; restore_pos = self.peek_pos; is_ipv6 = true; break; } TokenType::Punctuation( '-' | '_' | '~' | '!' | '$' | '&' | '\'' | '(' | ')' | '*' | '+' | ',' | ';' | '=' | ':' | '%', ) => { continue; } TokenType::Punctuation('/') if allow_blank_host => { // Allow file://../ urls end_pos = token.from; restore_pos = self.peek_pos - 1; break; } _ => break, } if start_pos == usize::MAX { start_pos = token.from; } end_pos = token.to; restore_pos = self.peek_pos; } self.peek_pos = restore_pos; let is_ip = is_ipv6 || (int_count == 4 && dot_count == 3 && text_count == 0); if end_pos != usize::MAX { is_valid_host = (last_label_is_tld && dot_count >= 1 && (text_count + int_count) >= 2) || is_ip; (start_pos, end_pos, is_ip) } else { return None; } } else { // Strict hostname parsing self.try_parse_hostname()? }; // Try parsing port let start_pos = scheme_token.map(|t| t.from).unwrap_or(host_start_pos); let mut restore_pos = self.peek_pos; let mut has_port = false; let mut last_is_colon = false; let mut found_query_start = false; while let Some(token) = self.peek() { match token.word { TokenType::Punctuation(':') if !last_is_colon && !has_port => { last_is_colon = true; } TokenType::Integer(_) if last_is_colon => { has_port = true; last_is_colon = false; restore_pos = self.peek_pos; end_pos = token.to; } TokenType::Punctuation('/' | '?') if !last_is_colon => { found_query_start = true; end_pos = token.to; break; } _ => { self.peek_pos = restore_pos; break; } } } // Try parsing query if found_query_start { restore_pos = self.peek_pos; let mut p_count = 0; let mut b_count = 0; let mut c_count = 0; let mut seen_quote = false; while let Some(token) = self.peek() { match token.word { TokenType::Alphabetic(_) | TokenType::Alphanumeric(_) | TokenType::Integer(_) | TokenType::Other(_) => {} TokenType::Punctuation('(') => { p_count += 1; continue; } TokenType::Punctuation('[') => { b_count += 1; continue; } TokenType::Punctuation('{') => { c_count += 1; continue; } TokenType::Punctuation(')') if p_count > 0 => { p_count -= 1; } TokenType::Punctuation(']') if b_count > 0 => { b_count -= 1; } TokenType::Punctuation('}') if c_count > 0 => { c_count -= 1; } TokenType::Punctuation('\'') => { if !seen_quote { seen_quote = true; continue; } else { seen_quote = false; } } TokenType::Punctuation('/') => {} TokenType::Punctuation( '-' | '_' | '~' | '!' | '$' | '&' | '*' | '+' | ',' | ';' | '=' | ':' | '%' | '?' | '.' | '@', ) => { continue; } _ => break, } end_pos = token.to; restore_pos = self.peek_pos; } self.peek_pos = restore_pos; } let word = &self.text[start_pos..end_pos]; Token { word: if has_scheme { if is_valid_host { TokenType::Url(word) } else { TokenType::UrlNoHost(word) } } else if is_ip && !found_query_start { TokenType::IpAddr(word) } else { TokenType::UrlNoScheme(word) }, from: start_pos, to: end_pos, } .into() } fn try_parse_email(&mut self) -> Option>> { // Start token is a valid local part atom let start_token = self.peek()?; let mut last_is_dot = false; // Find local part loop { let token = self.peek()?; if token.to - start_token.from > 255 { return None; } match token.word { word if word.is_email_atom() => { last_is_dot = false; } TokenType::Punctuation('@') if !last_is_dot => { break; } TokenType::Punctuation('.') if !last_is_dot => { last_is_dot = true; } _ => { return None; } } } // Obtain domain part let (_, end_pos, _) = self.try_parse_hostname()?; Token { word: TokenType::Email(&self.text[start_token.from..end_pos]), from: start_token.from, to: end_pos, } .into() } fn try_parse_hostname(&mut self) -> Option<(usize, usize, bool)> { let mut last_ch = u8::MAX; let mut has_int = false; let mut has_alpha = false; let mut last_label_is_tld = false; let mut dot_count = 0; let mut start_pos = usize::MAX; let mut end_pos = usize::MAX; let mut restore_pos = self.peek_pos; while let Some(token) = self.peek() { match token.word { TokenType::Punctuation('.') if last_ch == 0 && start_pos != usize::MAX => { last_ch = b'.'; dot_count += 1; continue; } TokenType::Punctuation('-') if last_ch == 0 || last_ch == b'-' => { last_ch = b'-'; continue; } TokenType::Punctuation('[') if start_pos == usize::MAX => { return self .try_parse_ipv6(token.from) .map(|(from, to)| (from, to, true)); } TokenType::Alphabetic(text) | TokenType::Alphanumeric(text) if text.len() <= 63 => { last_label_is_tld = text.len() >= 2 && psl::Psl::find( &psl::List, [text.to_ascii_lowercase().as_bytes()].into_iter(), ) .typ .is_some(); has_alpha = true; last_ch = 0; } TokenType::Other(_) => { has_alpha = true; last_label_is_tld = false; last_ch = 0; } TokenType::Integer(text) => { if text.len() <= 3 { has_int = true; } last_label_is_tld = false; last_ch = 0; } _ => { break; } } if start_pos == usize::MAX { start_pos = token.from; } end_pos = token.to; restore_pos = self.peek_pos; if end_pos - start_pos > 255 { return None; } } self.peek_pos = restore_pos; if last_ch == b'.' { dot_count -= 1; } let is_ipv4 = has_int && !has_alpha && dot_count == 3; if end_pos != usize::MAX && dot_count >= 1 && (last_label_is_tld || is_ipv4) { (start_pos, end_pos, is_ipv4).into() } else { None } } fn try_parse_ipv6(&mut self, start_pos: usize) -> Option<(usize, usize)> { let mut found_colon = false; let mut last_ch = u8::MAX; while let Some(token) = self.peek() { match token.word { TokenType::Integer(_) | TokenType::Alphanumeric(_) => { last_ch = 0; } TokenType::Punctuation(':') if last_ch != b'.' => { found_colon = true; last_ch = b':'; } TokenType::Punctuation('.') if last_ch == 0 => { last_ch = b'.'; } TokenType::Punctuation(']') if found_colon && last_ch == 0 => { return (start_pos, token.to).into(); } _ => return None, } } None } fn try_parse_number(&mut self) -> Option>> { self.peek_rewind(); let mut start_pos = usize::MAX; let mut end_pos = usize::MAX; let mut restore_pos = self.peek_pos; let mut seen_integer = 0; let mut seen_dot = false; while let Some(token) = self.peek() { match token.word { TokenType::Punctuation('-') if start_pos == usize::MAX => {} TokenType::Integer(_) if seen_integer == 0 || seen_dot => { seen_integer += 1; } TokenType::Punctuation('.') if seen_integer != 0 => { if !seen_dot { seen_dot = true; continue; } else { // Avoid parsing num.num.num as floats return None; } } _ => break, } if start_pos == usize::MAX { start_pos = token.from; } end_pos = token.to; restore_pos = self.peek_pos; } self.peek_pos = restore_pos; if seen_integer > 0 { let text = &self.text[start_pos..end_pos]; Token { word: if seen_integer == 2 { TokenType::Float(text) } else { TokenType::Integer(text) }, from: start_pos, to: end_pos, } .into() } else { None } } fn try_skip_url_scheme(&mut self) -> bool { enum State { None, PlusAlpha, Colon, Slash1, Slash2, } let mut state = State::None; while let Some(token) = self.peek() { state = match (token.word, state) { (TokenType::Punctuation(':'), State::None | State::Colon) => State::Slash1, (TokenType::Punctuation('/'), State::Slash1) => State::Slash2, (TokenType::Punctuation('/'), State::Slash2) => return true, (TokenType::Punctuation('+'), State::None) => State::PlusAlpha, (TokenType::Alphabetic(t) | TokenType::Alphanumeric(t), State::PlusAlpha) if t.is_ascii() => { State::Colon } _ => break, }; } self.peek_rewind(); false } } impl TokenType { fn is_email_atom(&self) -> bool { matches!( self, TokenType::Alphabetic(_) | TokenType::Integer(_) | TokenType::Alphanumeric(_) | TokenType::Other(_) | TokenType::Punctuation( '!' | '#' | '$' | '%' | '&' | '\'' | '*' | '+' | '-' | '/' | '=' | '?' | '^' | '_' | '`' | '{' | '|' | '}' | '~', ) ) } fn is_domain_atom(&self, is_start: bool) -> bool { matches!( self, TokenType::Alphabetic(_) | TokenType::Integer(_) | TokenType::Alphanumeric(_) | TokenType::Other(_) ) || (!is_start && matches!(self, TokenType::Punctuation('-'))) } } #[cfg(test)] mod test { use super::{TokenType, TypesTokenizer}; #[test] fn type_tokenizer() { // Credits: test suite from linkify crate for (text, expected) in [ ("", vec![]), ("foo", vec![TokenType::Alphabetic("foo")]), (":", vec![TokenType::Punctuation(':')]), ( "://", vec![ TokenType::Punctuation(':'), TokenType::Punctuation('/'), TokenType::Punctuation('/'), ], ), ( ":::", vec![ TokenType::Punctuation(':'), TokenType::Punctuation(':'), TokenType::Punctuation(':'), ], ), ( "://foo", vec![ TokenType::Punctuation(':'), TokenType::Punctuation('/'), TokenType::Punctuation('/'), TokenType::Alphabetic("foo"), ], ), ( "1://foo", vec![ TokenType::Integer("1"), TokenType::Punctuation(':'), TokenType::Punctuation('/'), TokenType::Punctuation('/'), TokenType::Alphabetic("foo"), ], ), ( "123://foo", vec![ TokenType::Integer("123"), TokenType::Punctuation(':'), TokenType::Punctuation('/'), TokenType::Punctuation('/'), TokenType::Alphabetic("foo"), ], ), ( "+://foo", vec![ TokenType::Punctuation('+'), TokenType::Punctuation(':'), TokenType::Punctuation('/'), TokenType::Punctuation('/'), TokenType::Alphabetic("foo"), ], ), ( "-://foo", vec![ TokenType::Punctuation('-'), TokenType::Punctuation(':'), TokenType::Punctuation('/'), TokenType::Punctuation('/'), TokenType::Alphabetic("foo"), ], ), ( ".://foo", vec![ TokenType::Punctuation('.'), TokenType::Punctuation(':'), TokenType::Punctuation('/'), TokenType::Punctuation('/'), TokenType::Alphabetic("foo"), ], ), ("1abc://foo", vec![TokenType::UrlNoHost("1abc://foo")]), ("a://foo", vec![TokenType::UrlNoHost("a://foo")]), ("a123://foo", vec![TokenType::UrlNoHost("a123://foo")]), ("a123b://foo", vec![TokenType::UrlNoHost("a123b://foo")]), ("a+b://foo", vec![TokenType::UrlNoHost("a+b://foo")]), ( "a-b://foo", vec![ TokenType::Alphabetic("a"), TokenType::Punctuation('-'), TokenType::UrlNoHost("b://foo"), ], ), ( "a.b://foo", vec![ TokenType::Alphabetic("a"), TokenType::Punctuation('.'), TokenType::UrlNoHost("b://foo"), ], ), ("ABC://foo", vec![TokenType::UrlNoHost("ABC://foo")]), ( ".http://example.org/", vec![ TokenType::Punctuation('.'), TokenType::Url("http://example.org/"), ], ), ( "1.http://example.org/", vec![ TokenType::Integer("1"), TokenType::Punctuation('.'), TokenType::Url("http://example.org/"), ], ), ( "ab://", vec![ TokenType::Alphabetic("ab"), TokenType::Punctuation(':'), TokenType::Punctuation('/'), TokenType::Punctuation('/'), ], ), ( "file://", vec![ TokenType::Alphabetic("file"), TokenType::Punctuation(':'), TokenType::Punctuation('/'), TokenType::Punctuation('/'), ], ), ( "file:// ", vec![ TokenType::Alphabetic("file"), TokenType::Punctuation(':'), TokenType::Punctuation('/'), TokenType::Punctuation('/'), TokenType::Space, ], ), ( "\"file://\"", vec![ TokenType::Punctuation('"'), TokenType::Alphabetic("file"), TokenType::Punctuation(':'), TokenType::Punctuation('/'), TokenType::Punctuation('/'), TokenType::Punctuation('"'), ], ), ( "\"file://...\", ", vec![ TokenType::Punctuation('"'), TokenType::Alphabetic("file"), TokenType::Punctuation(':'), TokenType::Punctuation('/'), TokenType::Punctuation('/'), TokenType::Punctuation('.'), TokenType::Punctuation('.'), TokenType::Punctuation('.'), TokenType::Punctuation('"'), TokenType::Punctuation(','), TokenType::Space, ], ), ( "file://somefile", vec![TokenType::UrlNoHost("file://somefile")], ), ( "file://../relative", vec![TokenType::UrlNoHost("file://../relative")], ), ( "http://a.", vec![ TokenType::UrlNoHost("http://a"), TokenType::Punctuation('.'), ], ), ("http://127.0.0.1", vec![TokenType::Url("http://127.0.0.1")]), ( "http://127.0.0.1/", vec![TokenType::Url("http://127.0.0.1/")], ), ("ab://c", vec![TokenType::UrlNoHost("ab://c")]), ( "http://example.org/", vec![TokenType::Url("http://example.org/")], ), ( "http://example.org/123", vec![TokenType::Url("http://example.org/123")], ), ( "http://example.org/?foo=test&bar=123", vec![TokenType::Url("http://example.org/?foo=test&bar=123")], ), ( "http://example.org/?foo=%20", vec![TokenType::Url("http://example.org/?foo=%20")], ), ( "http://example.org/%3C", vec![TokenType::Url("http://example.org/%3C")], ), ("example.org/", vec![TokenType::UrlNoScheme("example.org/")]), ( "example.org/123", vec![TokenType::UrlNoScheme("example.org/123")], ), ( "example.org/?foo=test&bar=123", vec![TokenType::UrlNoScheme("example.org/?foo=test&bar=123")], ), ( "example.org/?foo=%20", vec![TokenType::UrlNoScheme("example.org/?foo=%20")], ), ( "example.org/%3C", vec![TokenType::UrlNoScheme("example.org/%3C")], ), ( "foo http://example.org/", vec![ TokenType::Alphabetic("foo"), TokenType::Space, TokenType::Url("http://example.org/"), ], ), ( "http://example.org/ bar", vec![ TokenType::Url("http://example.org/"), TokenType::Space, TokenType::Alphabetic("bar"), ], ), ( "http://example.org/\tbar", vec![ TokenType::Url("http://example.org/"), TokenType::Space, TokenType::Alphabetic("bar"), ], ), ( "http://example.org/\nbar", vec![ TokenType::Url("http://example.org/"), TokenType::Space, TokenType::Alphabetic("bar"), ], ), ( "http://example.org/\u{b}bar", vec![ TokenType::Url("http://example.org/"), TokenType::Space, TokenType::Alphabetic("bar"), ], ), ( "http://example.org/\u{c}bar", vec![ TokenType::Url("http://example.org/"), TokenType::Space, TokenType::Alphabetic("bar"), ], ), ( "http://example.org/\rbar", vec![ TokenType::Url("http://example.org/"), TokenType::Space, TokenType::Alphabetic("bar"), ], ), ( "foo example.org/", vec![ TokenType::Alphabetic("foo"), TokenType::Space, TokenType::UrlNoScheme("example.org/"), ], ), ( "example.org/ bar", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Space, TokenType::Alphabetic("bar"), ], ), ( "example.org/\tbar", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Space, TokenType::Alphabetic("bar"), ], ), ( "example.org/\nbar", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Space, TokenType::Alphabetic("bar"), ], ), ( "example.org/\u{b}bar", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Space, TokenType::Alphabetic("bar"), ], ), ( "example.org/\u{c}bar", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Space, TokenType::Alphabetic("bar"), ], ), ( "example.org/\rbar", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Space, TokenType::Alphabetic("bar"), ], ), ( "http://example.org/<", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('<'), ], ), ( "http://example.org/>", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('>'), ], ), ( "http://example.org/<>", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('<'), TokenType::Punctuation('>'), ], ), ( "http://example.org/\0", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('\0'), ], ), ( "http://example.org/\u{e}", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('\u{e}'), ], ), ( "http://example.org/\u{7f}", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('\u{7f}'), ], ), ( "http://example.org/\u{9f}", vec![TokenType::Url("http://example.org/\u{9f}")], ), ( "http://example.org/foo|bar", vec![ TokenType::Url("http://example.org/foo"), TokenType::Punctuation('|'), TokenType::Alphabetic("bar"), ], ), ( "example.org/<", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('<'), ], ), ( "example.org/>", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('>'), ], ), ( "example.org/<>", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('<'), TokenType::Punctuation('>'), ], ), ( "example.org/\0", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('\0'), ], ), ( "example.org/\u{e}", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('\u{e}'), ], ), ( "example.org/\u{7f}", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('\u{7f}'), ], ), ( "example.org/\u{9f}", vec![TokenType::UrlNoScheme("example.org/\u{9f}")], ), ( "http://example.org/.", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('.'), ], ), ( "http://example.org/..", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('.'), TokenType::Punctuation('.'), ], ), ( "http://example.org/,", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation(','), ], ), ( "http://example.org/:", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation(':'), ], ), ( "http://example.org/?", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('?'), ], ), ( "http://example.org/!", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('!'), ], ), ( "http://example.org/;", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation(';'), ], ), ( "example.org/.", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('.'), ], ), ( "example.org/..", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('.'), TokenType::Punctuation('.'), ], ), ( "example.org/,", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation(','), ], ), ( "example.org/:", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation(':'), ], ), ( "example.org/?", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('?'), ], ), ( "example.org/!", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('!'), ], ), ( "example.org/;", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation(';'), ], ), ( "http://example.org/a(b)", vec![TokenType::Url("http://example.org/a(b)")], ), ( "http://example.org/a[b]", vec![TokenType::Url("http://example.org/a[b]")], ), ( "http://example.org/a{b}", vec![TokenType::Url("http://example.org/a{b}")], ), ( "http://example.org/a'b'", vec![TokenType::Url("http://example.org/a'b'")], ), ( "(http://example.org/)", vec![ TokenType::Punctuation('('), TokenType::Url("http://example.org/"), TokenType::Punctuation(')'), ], ), ( "[http://example.org/]", vec![ TokenType::Punctuation('['), TokenType::Url("http://example.org/"), TokenType::Punctuation(']'), ], ), ( "{http://example.org/}", vec![ TokenType::Punctuation('{'), TokenType::Url("http://example.org/"), TokenType::Punctuation('}'), ], ), ( "\"http://example.org/\"", vec![ TokenType::Punctuation('"'), TokenType::Url("http://example.org/"), TokenType::Punctuation('"'), ], ), ( "'http://example.org/'", vec![ TokenType::Punctuation('\''), TokenType::Url("http://example.org/"), TokenType::Punctuation('\''), ], ), ( "example.org/a(b)", vec![TokenType::UrlNoScheme("example.org/a(b)")], ), ( "example.org/a[b]", vec![TokenType::UrlNoScheme("example.org/a[b]")], ), ( "example.org/a{b}", vec![TokenType::UrlNoScheme("example.org/a{b}")], ), ( "example.org/a'b'", vec![TokenType::UrlNoScheme("example.org/a'b'")], ), ( "(example.org/)", vec![ TokenType::Punctuation('('), TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation(')'), ], ), ( "[example.org/]", vec![ TokenType::Punctuation('['), TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation(']'), ], ), ( "{example.org/}", vec![ TokenType::Punctuation('{'), TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('}'), ], ), ( "\"example.org/\"", vec![ TokenType::Punctuation('"'), TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('"'), ], ), ( "'example.org/'", vec![ TokenType::Punctuation('\''), TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('\''), ], ), ( "((http://example.org/))", vec![ TokenType::Punctuation('('), TokenType::Punctuation('('), TokenType::Url("http://example.org/"), TokenType::Punctuation(')'), TokenType::Punctuation(')'), ], ), ( "((http://example.org/a(b)))", vec![ TokenType::Punctuation('('), TokenType::Punctuation('('), TokenType::Url("http://example.org/a(b)"), TokenType::Punctuation(')'), TokenType::Punctuation(')'), ], ), ( "[(http://example.org/)]", vec![ TokenType::Punctuation('['), TokenType::Punctuation('('), TokenType::Url("http://example.org/"), TokenType::Punctuation(')'), TokenType::Punctuation(']'), ], ), ( "(http://example.org/).", vec![ TokenType::Punctuation('('), TokenType::Url("http://example.org/"), TokenType::Punctuation(')'), TokenType::Punctuation('.'), ], ), ( "(http://example.org/.)", vec![ TokenType::Punctuation('('), TokenType::Url("http://example.org/"), TokenType::Punctuation('.'), TokenType::Punctuation(')'), ], ), ( "http://example.org/>", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('>'), ], ), ( "http://example.org/(", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('('), ], ), ( "http://example.org/(.", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('('), TokenType::Punctuation('.'), ], ), ( "http://example.org/]()", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation(']'), TokenType::Punctuation('('), TokenType::Punctuation(')'), ], ), ( "((example.org/))", vec![ TokenType::Punctuation('('), TokenType::Punctuation('('), TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation(')'), TokenType::Punctuation(')'), ], ), ( "((example.org/a(b)))", vec![ TokenType::Punctuation('('), TokenType::Punctuation('('), TokenType::UrlNoScheme("example.org/a(b)"), TokenType::Punctuation(')'), TokenType::Punctuation(')'), ], ), ( "[(example.org/)]", vec![ TokenType::Punctuation('['), TokenType::Punctuation('('), TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation(')'), TokenType::Punctuation(']'), ], ), ( "(example.org/).", vec![ TokenType::Punctuation('('), TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation(')'), TokenType::Punctuation('.'), ], ), ( "(example.org/.)", vec![ TokenType::Punctuation('('), TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('.'), TokenType::Punctuation(')'), ], ), ( "example.org/>", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('>'), ], ), ( "example.org/(", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('('), ], ), ( "example.org/(.", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('('), TokenType::Punctuation('.'), ], ), ( "example.org/]()", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation(']'), TokenType::Punctuation('('), TokenType::Punctuation(')'), ], ), ( "'https://example.org'", vec![ TokenType::Punctuation('\''), TokenType::Url("https://example.org"), TokenType::Punctuation('\''), ], ), ( "\"https://example.org\"", vec![ TokenType::Punctuation('"'), TokenType::Url("https://example.org"), TokenType::Punctuation('"'), ], ), ( "''https://example.org''", vec![ TokenType::Punctuation('\''), TokenType::Punctuation('\''), TokenType::Url("https://example.org"), TokenType::Punctuation('\''), TokenType::Punctuation('\''), ], ), ( "'https://example.org''", vec![ TokenType::Punctuation('\''), TokenType::Url("https://example.org"), TokenType::Punctuation('\''), TokenType::Punctuation('\''), ], ), ( "'https://example.org", vec![ TokenType::Punctuation('\''), TokenType::Url("https://example.org"), ], ), ( "http://example.org/'_(foo)", vec![TokenType::Url("http://example.org/'_(foo)")], ), ( "http://example.org/'_(foo)'", vec![TokenType::Url("http://example.org/'_(foo)'")], ), ( "http://example.org/''", vec![TokenType::Url("http://example.org/''")], ), ( "http://example.org/'''", vec![ TokenType::Url("http://example.org/''"), TokenType::Punctuation('\''), ], ), ( "http://example.org/'.", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('\''), TokenType::Punctuation('.'), ], ), ( "http://example.org/'a", vec![TokenType::Url("http://example.org/'a")], ), ( "http://example.org/it's", vec![TokenType::Url("http://example.org/it's")], ), ( "example.org/'_(foo)", vec![TokenType::UrlNoScheme("example.org/'_(foo)")], ), ( "example.org/'_(foo)'", vec![TokenType::UrlNoScheme("example.org/'_(foo)'")], ), ( "example.org/''", vec![TokenType::UrlNoScheme("example.org/''")], ), ( "example.org/'''", vec![ TokenType::UrlNoScheme("example.org/''"), TokenType::Punctuation('\''), ], ), ( "example.org/'.", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('\''), TokenType::Punctuation('.'), ], ), ( "example.org/'a", vec![TokenType::UrlNoScheme("example.org/'a")], ), ( "example.org/it's", vec![TokenType::UrlNoScheme("example.org/it's")], ), ( "http://example.org/\"a", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('"'), TokenType::Alphabetic("a"), ], ), ( "http://example.org/\"a\"", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('"'), TokenType::Alphabetic("a"), TokenType::Punctuation('"'), ], ), ( "http://example.org/`a", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('`'), TokenType::Alphabetic("a"), ], ), ( "http://example.org/`a`", vec![ TokenType::Url("http://example.org/"), TokenType::Punctuation('`'), TokenType::Alphabetic("a"), TokenType::Punctuation('`'), ], ), ( "https://example.org*", vec![ TokenType::Url("https://example.org"), TokenType::Punctuation('*'), ], ), ( "https://example.org/*", vec![ TokenType::Url("https://example.org/"), TokenType::Punctuation('*'), ], ), ( "https://example.org/**", vec![ TokenType::Url("https://example.org/"), TokenType::Punctuation('*'), TokenType::Punctuation('*'), ], ), ( "https://example.org/*/a", vec![TokenType::Url("https://example.org/*/a")], ), ( "example.org/`a", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('`'), TokenType::Alphabetic("a"), ], ), ( "example.org/`a`", vec![ TokenType::UrlNoScheme("example.org/"), TokenType::Punctuation('`'), TokenType::Alphabetic("a"), TokenType::Punctuation('`'), ], ), ( "http://example.org\">", vec![ TokenType::Url("http://example.org"), TokenType::Punctuation('"'), TokenType::Punctuation('>'), ], ), ( "http://example.org'>", vec![ TokenType::Url("http://example.org"), TokenType::Punctuation('\''), TokenType::Punctuation('>'), ], ), ( "http://example.org\"/>", vec![ TokenType::Url("http://example.org"), TokenType::Punctuation('"'), TokenType::Punctuation('/'), TokenType::Punctuation('>'), ], ), ( "http://example.org'/>", vec![ TokenType::Url("http://example.org"), TokenType::Punctuation('\''), TokenType::Punctuation('/'), TokenType::Punctuation('>'), ], ), ( "http://example.org

", vec![ TokenType::Url("http://example.org"), TokenType::Punctuation('<'), TokenType::Alphabetic("p"), TokenType::Punctuation('>'), ], ), ( "http://example.org

", vec![ TokenType::Url("http://example.org"), TokenType::Punctuation('<'), TokenType::Punctuation('/'), TokenType::Alphabetic("p"), TokenType::Punctuation('>'), ], ), ( "example.org\">", vec![ TokenType::UrlNoScheme("example.org"), TokenType::Punctuation('"'), TokenType::Punctuation('>'), ], ), ( "example.org'>", vec![ TokenType::UrlNoScheme("example.org"), TokenType::Punctuation('\''), TokenType::Punctuation('>'), ], ), ( "example.org\"/>", vec![ TokenType::UrlNoScheme("example.org"), TokenType::Punctuation('"'), TokenType::Punctuation('/'), TokenType::Punctuation('>'), ], ), ( "example.org'/>", vec![ TokenType::UrlNoScheme("example.org"), TokenType::Punctuation('\''), TokenType::Punctuation('/'), TokenType::Punctuation('>'), ], ), ( "example.org

", vec![ TokenType::UrlNoScheme("example.org"), TokenType::Punctuation('<'), TokenType::Alphabetic("p"), TokenType::Punctuation('>'), ], ), ( "example.org

", vec![ TokenType::UrlNoScheme("example.org"), TokenType::Punctuation('<'), TokenType::Punctuation('/'), TokenType::Alphabetic("p"), TokenType::Punctuation('>'), ], ), ( "http://example.org\");", vec![ TokenType::Url("http://example.org"), TokenType::Punctuation('"'), TokenType::Punctuation(')'), TokenType::Punctuation(';'), ], ), ( "http://example.org');", vec![ TokenType::Url("http://example.org"), TokenType::Punctuation('\''), TokenType::Punctuation(')'), TokenType::Punctuation(';'), ], ), ( "", vec![ TokenType::Punctuation('<'), TokenType::Alphabetic("img"), TokenType::Space, TokenType::Alphabetic("src"), TokenType::Punctuation('='), TokenType::Punctuation('"'), TokenType::Url("http://example.org/test.svg"), TokenType::Punctuation('"'), TokenType::Punctuation('>'), ], ), ( "", vec![ TokenType::Punctuation('<'), TokenType::Alphabetic("div"), TokenType::Punctuation('>'), TokenType::Punctuation('<'), TokenType::Alphabetic("a"), TokenType::Space, TokenType::Alphabetic("href"), TokenType::Punctuation('='), TokenType::Punctuation('"'), TokenType::Url("http://example.org"), TokenType::Punctuation('"'), TokenType::Punctuation('>'), TokenType::Punctuation('<'), TokenType::Punctuation('/'), TokenType::Alphabetic("a"), TokenType::Punctuation('>'), TokenType::Punctuation('<'), TokenType::Punctuation('/'), TokenType::Alphabetic("div"), TokenType::Punctuation('>'), ], ), ( "
", vec![ TokenType::Punctuation('<'), TokenType::Alphabetic("div"), TokenType::Punctuation('>'), TokenType::Punctuation('<'), TokenType::Alphabetic("a"), TokenType::Space, TokenType::Alphabetic("href"), TokenType::Punctuation('='), TokenType::Punctuation('"'), TokenType::Url("http://example.org"), TokenType::Punctuation('"'), TokenType::Space, TokenType::Punctuation('>'), TokenType::Punctuation('<'), TokenType::Punctuation('/'), TokenType::Alphabetic("a"), TokenType::Punctuation('>'), TokenType::Punctuation('<'), TokenType::Punctuation('/'), TokenType::Alphabetic("div"), TokenType::Punctuation('>'), ], ), ( "
\n \n
", vec![ TokenType::Punctuation('<'), TokenType::Alphabetic("div"), TokenType::Punctuation('>'), TokenType::Space, TokenType::Punctuation('<'), TokenType::Alphabetic("img"), TokenType::Space, TokenType::Alphabetic("src"), TokenType::Punctuation('='), TokenType::Punctuation('"'), TokenType::Url("http://example.org/test3.jpg"), TokenType::Punctuation('"'), TokenType::Space, TokenType::Punctuation('/'), TokenType::Punctuation('>'), TokenType::Space, TokenType::Punctuation('<'), TokenType::Punctuation('/'), TokenType::Alphabetic("div"), TokenType::Punctuation('>'), ], ), ( "example.org\");", vec![ TokenType::UrlNoScheme("example.org"), TokenType::Punctuation('"'), TokenType::Punctuation(')'), TokenType::Punctuation(';'), ], ), ( "example.org');", vec![ TokenType::UrlNoScheme("example.org"), TokenType::Punctuation('\''), TokenType::Punctuation(')'), TokenType::Punctuation(';'), ], ), ( "http://example.org/", vec![TokenType::Url("http://example.org/")], ), ( "http://example.org/a/", vec![TokenType::Url("http://example.org/a/")], ), ( "http://example.org//", vec![TokenType::Url("http://example.org//")], ), ("example.org/", vec![TokenType::UrlNoScheme("example.org/")]), ( "example.org/a/", vec![TokenType::UrlNoScheme("example.org/a/")], ), ( "example.org//", vec![TokenType::UrlNoScheme("example.org//")], ), ( "http://one.org/ http://two.org/", vec![ TokenType::Url("http://one.org/"), TokenType::Space, TokenType::Url("http://two.org/"), ], ), ( "http://one.org/ : http://two.org/", vec![ TokenType::Url("http://one.org/"), TokenType::Space, TokenType::Punctuation(':'), TokenType::Space, TokenType::Url("http://two.org/"), ], ), ( "(http://one.org/)(http://two.org/)", vec![ TokenType::Punctuation('('), TokenType::Url("http://one.org/"), TokenType::Punctuation(')'), TokenType::Punctuation('('), TokenType::Url("http://two.org/"), TokenType::Punctuation(')'), ], ), ( "one.org/ two.org/", vec![ TokenType::UrlNoScheme("one.org/"), TokenType::Space, TokenType::UrlNoScheme("two.org/"), ], ), ( "one.org/ : two.org/", vec![ TokenType::UrlNoScheme("one.org/"), TokenType::Space, TokenType::Punctuation(':'), TokenType::Space, TokenType::UrlNoScheme("two.org/"), ], ), ( "(one.org/)(two.org/)", vec![ TokenType::Punctuation('('), TokenType::UrlNoScheme("one.org/"), TokenType::Punctuation(')'), TokenType::Punctuation('('), TokenType::UrlNoScheme("two.org/"), TokenType::Punctuation(')'), ], ), ( "http://one.org/ two.org/", vec![ TokenType::Url("http://one.org/"), TokenType::Space, TokenType::UrlNoScheme("two.org/"), ], ), ( "one.org/ : http://two.org/", vec![ TokenType::UrlNoScheme("one.org/"), TokenType::Space, TokenType::Punctuation(':'), TokenType::Space, TokenType::Url("http://two.org/"), ], ), ( "(http://one.org/)(two.org/)", vec![ TokenType::Punctuation('('), TokenType::Url("http://one.org/"), TokenType::Punctuation(')'), TokenType::Punctuation('('), TokenType::UrlNoScheme("two.org/"), TokenType::Punctuation(')'), ], ), ( "http://üñîçøðé.com", vec![TokenType::Url("http://üñîçøðé.com")], ), ( "http://üñîçøðé.com/ä", vec![TokenType::Url("http://üñîçøðé.com/ä")], ), ( "http://example.org/¡", vec![TokenType::Url("http://example.org/¡")], ), ( "http://example.org/¢", vec![TokenType::Url("http://example.org/¢")], ), ( "http://example.org/😀", vec![TokenType::Url("http://example.org/😀")], ), ( "http://example.org/¢/", vec![TokenType::Url("http://example.org/¢/")], ), ( "http://xn--c1h.example.com/", vec![TokenType::Url("http://xn--c1h.example.com/")], ), ("üñîçøðé.com", vec![TokenType::UrlNoScheme("üñîçøðé.com")]), ( "üñîçøðé.com/ä", vec![TokenType::UrlNoScheme("üñîçøðé.com/ä")], ), ( "example.org/¡", vec![TokenType::UrlNoScheme("example.org/¡")], ), ( "example.org/¢", vec![TokenType::UrlNoScheme("example.org/¢")], ), ( "example.org/😀", vec![TokenType::UrlNoScheme("example.org/😀")], ), ( "example.org/¢/", vec![TokenType::UrlNoScheme("example.org/¢/")], ), ( "xn--c1h.example.com/", vec![TokenType::UrlNoScheme("xn--c1h.example.com/")], ), ( "example.", vec![ TokenType::Alphabetic("example"), TokenType::Punctuation('.'), ], ), ( "example./", vec![ TokenType::Alphabetic("example"), TokenType::Punctuation('.'), TokenType::Punctuation('/'), ], ), ( "foo.com.", vec![ TokenType::UrlNoScheme("foo.com"), TokenType::Punctuation('.'), ], ), ( "example.c", vec![ TokenType::Alphabetic("example"), TokenType::Punctuation('.'), TokenType::Alphabetic("c"), ], ), ("example.co", vec![TokenType::UrlNoScheme("example.co")]), ("example.com", vec![TokenType::UrlNoScheme("example.com")]), ("e.com", vec![TokenType::UrlNoScheme("e.com")]), ( "exampl.e.c", vec![ TokenType::Alphabetic("exampl"), TokenType::Punctuation('.'), TokenType::Alphabetic("e"), TokenType::Punctuation('.'), TokenType::Alphabetic("c"), ], ), ("exampl.e.co", vec![TokenType::UrlNoScheme("exampl.e.co")]), ( "e.xample.c", vec![ TokenType::Alphabetic("e"), TokenType::Punctuation('.'), TokenType::Alphabetic("xample"), TokenType::Punctuation('.'), TokenType::Alphabetic("c"), ], ), ("e.xample.co", vec![TokenType::UrlNoScheme("e.xample.co")]), ( "v1.1.1", vec![ TokenType::Alphanumeric("v1"), TokenType::Punctuation('.'), TokenType::Integer("1"), TokenType::Punctuation('.'), TokenType::Integer("1"), ], ), ( "foo.bar@example.org", vec![TokenType::Email("foo.bar@example.org")], ), ( "example.com@example.com", vec![TokenType::Email("example.com@example.com")], ), ( "Look, no scheme: example.org/foo email@foo.com", vec![ TokenType::Alphabetic("Look"), TokenType::Punctuation(','), TokenType::Space, TokenType::Alphabetic("no"), TokenType::Space, TokenType::Alphabetic("scheme"), TokenType::Punctuation(':'), TokenType::Space, TokenType::UrlNoScheme("example.org/foo"), TokenType::Space, TokenType::Email("email@foo.com"), ], ), ( "Web:\nwww.foobar.co\nE-Mail:\n bar@foobar.co (bla bla bla)", vec![ TokenType::Alphabetic("Web"), TokenType::Punctuation(':'), TokenType::Space, TokenType::UrlNoScheme("www.foobar.co"), TokenType::Space, TokenType::Alphabetic("E"), TokenType::Punctuation('-'), TokenType::Alphabetic("Mail"), TokenType::Punctuation(':'), TokenType::Space, TokenType::Email("bar@foobar.co"), TokenType::Space, TokenType::Punctuation('('), TokenType::Alphabetic("bla"), TokenType::Space, TokenType::Alphabetic("bla"), TokenType::Space, TokenType::Alphabetic("bla"), TokenType::Punctuation(')'), ], ), ( "upi://pay?pa=XXXXXXX&pn=XXXXX", vec![TokenType::UrlNoHost("upi://pay?pa=XXXXXXX&pn=XXXXX")], ), ( "https://example.org?pa=XXXXXXX&pn=XXXXX", vec![TokenType::Url("https://example.org?pa=XXXXXXX&pn=XXXXX")], ), ( "website https://domain.com", vec![ TokenType::Alphabetic("website"), TokenType::Space, TokenType::Url("https://domain.com"), ], ), ("a12.b-c.com", vec![TokenType::UrlNoScheme("a12.b-c.com")]), ( "v1.2.3", vec![ TokenType::Alphanumeric("v1"), TokenType::Punctuation('.'), TokenType::Integer("2"), TokenType::Punctuation('.'), TokenType::Integer("3"), ], ), ( "https://12-7.0.0.1/", vec![TokenType::UrlNoHost("https://12-7.0.0.1/")], ), ( "https://user:pass@example.com/", vec![TokenType::Url("https://user:pass@example.com/")], ), ( "https://user:-.!$@example.com/", vec![TokenType::Url("https://user:-.!$@example.com/")], ), ( "https://user:!$&'()*+,;=@example.com/", vec![TokenType::Url("https://user:!$&'()*+,;=@example.com/")], ), ( "https://user:pass@ex@mple.com/", vec![ TokenType::UrlNoHost("https://user:pass@ex"), TokenType::Punctuation('@'), TokenType::UrlNoScheme("mple.com/"), ], ), ( "https://localhost:8080!", vec![ TokenType::UrlNoHost("https://localhost:8080"), TokenType::Punctuation('!'), ], ), ( "https://localhost:8080/", vec![TokenType::UrlNoHost("https://localhost:8080/")], ), ( "https://user:pass@example.com:8080/hi", vec![TokenType::Url("https://user:pass@example.com:8080/hi")], ), ( "https://127.0.0.1/", vec![TokenType::Url("https://127.0.0.1/")], ), ("1.0.0.0", vec![TokenType::IpAddr("1.0.0.0")]), ( "1.0.0.0/foo/bar", vec![TokenType::UrlNoScheme("1.0.0.0/foo/bar")], ), ("1.0 ", vec![TokenType::Float("1.0"), TokenType::Space]), ( "1.0.0", vec![ TokenType::Integer("1"), TokenType::Punctuation('.'), TokenType::Integer("0"), TokenType::Punctuation('.'), TokenType::Integer("0"), ], ), ( "1.0.0.0.0", vec![ TokenType::Integer("1"), TokenType::Punctuation('.'), TokenType::IpAddr("0.0.0.0"), ], ), ( "1.0.0.", vec![ TokenType::Integer("1"), TokenType::Punctuation('.'), TokenType::Integer("0"), TokenType::Punctuation('.'), TokenType::Integer("0"), TokenType::Punctuation('.'), ], ), ( "https://example.com.:8080/test", vec![TokenType::Url("https://example.com.:8080/test")], ), ( "https://example.org'", vec![ TokenType::Url("https://example.org"), TokenType::Punctuation('\''), ], ), ( "https://example.org'a@example.com", vec![TokenType::Url("https://example.org'a@example.com")], ), ( "https://a.com'https://b.com", vec![ TokenType::UrlNoHost("https://a.com'https"), TokenType::Punctuation(':'), TokenType::Punctuation('/'), TokenType::Punctuation('/'), TokenType::UrlNoScheme("b.com"), ], ), ( "https://example.com...", vec![ TokenType::Url("https://example.com"), TokenType::Punctuation('.'), TokenType::Punctuation('.'), TokenType::Punctuation('.'), ], ), ( "www.example..com", vec![ TokenType::Alphabetic("www"), TokenType::Punctuation('.'), TokenType::Alphabetic("example"), TokenType::Punctuation('.'), TokenType::Punctuation('.'), TokenType::Alphabetic("com"), ], ), ( "https://.www.example.com", vec![TokenType::Url("https://.www.example.com")], ), ( "-a.com", vec![TokenType::Punctuation('-'), TokenType::UrlNoScheme("a.com")], ), ("https://a.-b.com", vec![TokenType::Url("https://a.-b.com")]), ( "a-.com", vec![ TokenType::Alphabetic("a"), TokenType::Punctuation('-'), TokenType::Punctuation('.'), TokenType::Alphabetic("com"), ], ), ( "a.b-.com", vec![ TokenType::Alphabetic("a"), TokenType::Punctuation('.'), TokenType::Alphabetic("b"), TokenType::Punctuation('-'), TokenType::Punctuation('.'), TokenType::Alphabetic("com"), ], ), ("https://a.b-.com", vec![TokenType::Url("https://a.b-.com")]), ( "https://example.com-/", vec![ TokenType::Url("https://example.com"), TokenType::Punctuation('-'), TokenType::Punctuation('/'), ], ), ( "https://example.org-", vec![ TokenType::Url("https://example.org"), TokenType::Punctuation('-'), ], ), ( "example.com@about", vec![ TokenType::UrlNoScheme("example.com"), TokenType::Punctuation('@'), TokenType::Alphabetic("about"), ], ), ( "example.com/@about", vec![TokenType::UrlNoScheme("example.com/@about")], ), ( "https://example.com/@about", vec![TokenType::Url("https://example.com/@about")], ), ( "info@v1.1.1", vec![ TokenType::Alphabetic("info"), TokenType::Punctuation('@'), TokenType::Alphanumeric("v1"), TokenType::Punctuation('.'), TokenType::Integer("1"), TokenType::Punctuation('.'), TokenType::Integer("1"), ], ), ("file:///", vec![TokenType::UrlNoHost("file:///")]), ( "file:///home/foo", vec![TokenType::UrlNoHost("file:///home/foo")], ), ( "file://localhost/home/foo", vec![TokenType::UrlNoHost("file://localhost/home/foo")], ), ( "facetime://+19995551234", vec![TokenType::UrlNoHost("facetime://+19995551234")], ), ( "test://123'456!!!", vec![ TokenType::UrlNoHost("test://123'456"), TokenType::Punctuation('!'), TokenType::Punctuation('!'), TokenType::Punctuation('!'), ], ), ( "test://123'456...", vec![ TokenType::UrlNoHost("test://123'456"), TokenType::Punctuation('.'), TokenType::Punctuation('.'), TokenType::Punctuation('.'), ], ), ( "test://123'456!!!/", vec![ TokenType::UrlNoHost("test://123'456"), TokenType::Punctuation('!'), TokenType::Punctuation('!'), TokenType::Punctuation('!'), TokenType::Punctuation('/'), ], ), ( "test://123'456.../", vec![ TokenType::UrlNoHost("test://123'456"), TokenType::Punctuation('.'), TokenType::Punctuation('.'), TokenType::Punctuation('.'), TokenType::Punctuation('/'), ], ), ( "1abc://example.com", vec![TokenType::Url("1abc://example.com")], ), ( "¡¢example.com", vec![TokenType::UrlNoScheme("¡¢example.com")], ), ("foo", vec![TokenType::Alphabetic("foo")]), ("@", vec![TokenType::Punctuation('@')]), ( "a@", vec![TokenType::Alphabetic("a"), TokenType::Punctuation('@')], ), ( "@a", vec![TokenType::Punctuation('@'), TokenType::Alphabetic("a")], ), ( "@@@", vec![ TokenType::Punctuation('@'), TokenType::Punctuation('@'), TokenType::Punctuation('@'), ], ), ("foo@example.com", vec![TokenType::Email("foo@example.com")]), ( "foo.bar@example.com", vec![TokenType::Email("foo.bar@example.com")], ), ( "#!$%&'*+-/=?^_`{}|~@example.org", vec![TokenType::Email("#!$%&'*+-/=?^_`{}|~@example.org")], ), ( "foo a@b.com", vec![ TokenType::Alphabetic("foo"), TokenType::Space, TokenType::Email("a@b.com"), ], ), ( "a@b.com foo", vec![ TokenType::Email("a@b.com"), TokenType::Space, TokenType::Alphabetic("foo"), ], ), ( "\na@b.com", vec![TokenType::Space, TokenType::Email("a@b.com")], ), ( "a@b.com\n", vec![TokenType::Email("a@b.com"), TokenType::Space], ), ( "(a@example.com)", vec![ TokenType::Punctuation('('), TokenType::Email("a@example.com"), TokenType::Punctuation(')'), ], ), ( "\"a@example.com\"", vec![ TokenType::Punctuation('"'), TokenType::Email("a@example.com"), TokenType::Punctuation('"'), ], ), ( "\"a@example.com\"", vec![ TokenType::Punctuation('"'), TokenType::Email("a@example.com"), TokenType::Punctuation('"'), ], ), ( ",a@example.com,", vec![ TokenType::Punctuation(','), TokenType::Email("a@example.com"), TokenType::Punctuation(','), ], ), ( ":a@example.com:", vec![ TokenType::Punctuation(':'), TokenType::Email("a@example.com"), TokenType::Punctuation(':'), ], ), ( ";a@example.com;", vec![ TokenType::Punctuation(';'), TokenType::Email("a@example.com"), TokenType::Punctuation(';'), ], ), ( ".@example.com", vec![ TokenType::Punctuation('.'), TokenType::Punctuation('@'), TokenType::UrlNoScheme("example.com"), ], ), ( "foo.@example.com", vec![ TokenType::Alphabetic("foo"), TokenType::Punctuation('.'), TokenType::Punctuation('@'), TokenType::UrlNoScheme("example.com"), ], ), ( ".foo@example.com", vec![ TokenType::Punctuation('.'), TokenType::Email("foo@example.com"), ], ), ( ".foo@example.com", vec![ TokenType::Punctuation('.'), TokenType::Email("foo@example.com"), ], ), ( "a..b@example.com", vec![ TokenType::Alphabetic("a"), TokenType::Punctuation('.'), TokenType::Punctuation('.'), TokenType::Email("b@example.com"), ], ), ( "a@example.com.", vec![ TokenType::Email("a@example.com"), TokenType::Punctuation('.'), ], ), ( "a@b", vec![ TokenType::Alphabetic("a"), TokenType::Punctuation('@'), TokenType::Alphabetic("b"), ], ), ( "a@b.", vec![ TokenType::Alphabetic("a"), TokenType::Punctuation('@'), TokenType::Alphabetic("b"), TokenType::Punctuation('.'), ], ), ( "a@b.com.", vec![TokenType::Email("a@b.com"), TokenType::Punctuation('.')], ), ( "a@example.com-", vec![ TokenType::Email("a@example.com"), TokenType::Punctuation('-'), ], ), ("a@foo-bar.com", vec![TokenType::Email("a@foo-bar.com")]), ( "a@-foo.com", vec![ TokenType::Alphabetic("a"), TokenType::Punctuation('@'), TokenType::Punctuation('-'), TokenType::UrlNoScheme("foo.com"), ], ), ( "a@b-.", vec![ TokenType::Alphabetic("a"), TokenType::Punctuation('@'), TokenType::Alphabetic("b"), TokenType::Punctuation('-'), TokenType::Punctuation('.'), ], ), ( "a@b", vec![ TokenType::Alphabetic("a"), TokenType::Punctuation('@'), TokenType::Alphabetic("b"), ], ), ( "a@b.", vec![ TokenType::Alphabetic("a"), TokenType::Punctuation('@'), TokenType::Alphabetic("b"), TokenType::Punctuation('.'), ], ), ( "a@example.com b@example.com", vec![ TokenType::Email("a@example.com"), TokenType::Space, TokenType::Email("b@example.com"), ], ), ( "a@example.com @ b@example.com", vec![ TokenType::Email("a@example.com"), TokenType::Space, TokenType::Punctuation('@'), TokenType::Space, TokenType::Email("b@example.com"), ], ), ( "a@xy.com;b@xy.com,c@xy.com", vec![ TokenType::Email("a@xy.com"), TokenType::Punctuation(';'), TokenType::Email("b@xy.com"), TokenType::Punctuation(','), TokenType::Email("c@xy.com"), ], ), ( "üñîçøðé@example.com", vec![TokenType::Email("üñîçøðé@example.com")], ), ( "üñîçøðé@üñîçøðé.com", vec![TokenType::Email("üñîçøðé@üñîçøðé.com")], ), ("www@example.com", vec![TokenType::Email("www@example.com")]), ( "a@a.xyϸ", vec![ TokenType::Alphabetic("a"), TokenType::Punctuation('@'), TokenType::Alphabetic("a"), TokenType::Punctuation('.'), TokenType::Alphabetic("xyϸ"), ], ), ( "100 -100 100.00 -100.00 $100 $100.00", vec![ TokenType::Integer("100"), TokenType::Space, TokenType::Integer("-100"), TokenType::Space, TokenType::Float("100.00"), TokenType::Space, TokenType::Float("-100.00"), TokenType::Space, TokenType::Punctuation('$'), TokenType::Integer("100"), TokenType::Space, TokenType::Punctuation('$'), TokenType::Float("100.00"), ], ), ( " - 100 100 . 00", vec![ TokenType::Space, TokenType::Punctuation('-'), TokenType::Space, TokenType::Integer("100"), TokenType::Space, TokenType::Integer("100"), TokenType::Space, TokenType::Punctuation('.'), TokenType::Space, TokenType::Integer("00"), ], ), ( "send $100.00 to user@domain.com or visit domain.com/pay-me!", vec![ TokenType::Alphabetic("send"), TokenType::Space, TokenType::Punctuation('$'), TokenType::Float("100.00"), TokenType::Space, TokenType::Alphabetic("to"), TokenType::Space, TokenType::Email("user@domain.com"), TokenType::Space, TokenType::Alphabetic("or"), TokenType::Space, TokenType::Alphabetic("visit"), TokenType::Space, TokenType::UrlNoScheme("domain.com/pay-me"), TokenType::Punctuation('!'), ], ), ( "vＥⓡ𝔂 𝔽𝕌Ňℕｙ ţ乇𝕏𝓣 wWiIiIIttHh l133t5p3/-\\|<", vec![ TokenType::Alphabetic("vＥⓡ𝔂"), TokenType::Space, TokenType::Alphabetic("𝔽𝕌Ňℕｙ"), TokenType::Space, TokenType::Alphabetic("ţ乇𝕏𝓣"), TokenType::Space, TokenType::Alphabetic("wWiIiIIttHh"), TokenType::Space, TokenType::Alphanumeric("l133t5p3"), TokenType::Punctuation('/'), TokenType::Punctuation('-'), TokenType::Punctuation('\\'), TokenType::Punctuation('|'), TokenType::Punctuation('<'), ], ), ] { let result = TypesTokenizer::new(text) .map(|t| t.word) .collect::>(); assert_eq!(result, expected, "text: {:?}", text); /*print!("({text:?}, "); print!("vec!["); for (pos, item) in result.into_iter().enumerate() { if pos > 0 { print!(", "); } print!("TokenType::{:?}", item); } println!("]),");*/ } } } ================================================ FILE: crates/nlp/src/tokenizers/word.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{borrow::Cow, str::CharIndices}; use super::Token; pub struct WordTokenizer<'x> { max_token_length: usize, text: &'x str, iterator: CharIndices<'x>, } impl WordTokenizer<'_> { pub fn new(text: &'_ str, max_token_length: usize) -> WordTokenizer<'_> { WordTokenizer { max_token_length, text, iterator: text.char_indices(), } } } /// Parses indo-european text into lowercase tokens. impl<'x> Iterator for WordTokenizer<'x> { type Item = Token>; fn next(&mut self) -> Option { while let Some((token_start, ch)) = self.iterator.next() { if ch.is_alphanumeric() { let mut is_uppercase = ch.is_uppercase(); let token_end = (&mut self.iterator) .filter_map(|(pos, ch)| { if ch.is_alphanumeric() { if !is_uppercase && ch.is_uppercase() { is_uppercase = true; } None } else { pos.into() } }) .next() .unwrap_or(self.text.len()); let token_len = token_end - token_start; if token_end > token_start && token_len <= self.max_token_length { return Token::new( token_start, token_len, if is_uppercase { self.text[token_start..token_end].to_lowercase().into() } else { self.text[token_start..token_end].into() }, ) .into(); } } } None } } #[cfg(test)] mod tests { use super::*; #[test] fn indo_european_tokenizer() { let inputs = [ ( "The quick brown fox jumps over the lazy dog", vec![ Token::new(0, 3, "the".into()), Token::new(4, 5, "quick".into()), Token::new(10, 5, "brown".into()), Token::new(16, 3, "fox".into()), Token::new(20, 5, "jumps".into()), Token::new(26, 4, "over".into()), Token::new(31, 3, "the".into()), Token::new(35, 4, "lazy".into()), Token::new(40, 3, "dog".into()), ], ), ( "Jovencillo EMPONZOÑADO de whisky: ¡qué figurota exhibe!", vec![ Token::new(0, 10, "jovencillo".into()), Token::new(11, 12, "emponzoñado".into()), Token::new(24, 2, "de".into()), Token::new(27, 6, "whisky".into()), Token::new(37, 4, "qué".into()), Token::new(42, 8, "figurota".into()), Token::new(51, 6, "exhibe".into()), ], ), ( "ZWÖLF Boxkämpfer jagten Victor quer über den großen Sylter Deich", vec![ Token::new(0, 6, "zwölf".into()), Token::new(7, 11, "boxkämpfer".into()), Token::new(19, 6, "jagten".into()), Token::new(26, 6, "victor".into()), Token::new(33, 4, "quer".into()), Token::new(38, 5, "über".into()), Token::new(44, 3, "den".into()), Token::new(48, 7, "großen".into()), Token::new(56, 6, "sylter".into()), Token::new(63, 5, "deich".into()), ], ), ( "Съешь ещё этих мягких французских булок, да выпей же чаю", vec![ Token::new(0, 10, "съешь".into()), Token::new(11, 6, "ещё".into()), Token::new(18, 8, "этих".into()), Token::new(27, 12, "мягких".into()), Token::new(40, 22, "французских".into()), Token::new(63, 10, "булок".into()), Token::new(75, 4, "да".into()), Token::new(80, 10, "выпей".into()), Token::new(91, 4, "же".into()), Token::new(96, 6, "чаю".into()), ], ), ( "Pijamalı hasta yağız şoföre çabucak güvendi", vec![ Token::new(0, 9, "pijamalı".into()), Token::new(10, 5, "hasta".into()), Token::new(16, 7, "yağız".into()), Token::new(24, 8, "şoföre".into()), Token::new(33, 8, "çabucak".into()), Token::new(42, 8, "güvendi".into()), ], ), ]; for (input, tokens) in inputs.iter() { for (pos, token) in WordTokenizer::new(input, 40).enumerate() { assert_eq!(token, tokens[pos]); } } } } ================================================ FILE: crates/pop3/Cargo.toml ================================================ [package] name = "pop3" version = "0.15.5" edition = "2024" [dependencies] store = { path = "../store" } common = { path = "../common" } directory = { path = "../directory" } imap = { path = "../imap" } utils = { path = "../utils" } trc = { path = "../trc" } types = { path = "../types" } email = { path = "../email" } mail-parser = { version = "0.11", features = ["full_encoding"] } mail-send = { version = "0.5", default-features = false, features = ["cram-md5", "ring", "tls12"] } rustls = { version = "0.23.5", default-features = false, features = ["std", "ring", "tls12"] } tokio = { version = "1.47", features = ["full"] } tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } [features] test_mode = [] ================================================ FILE: crates/pop3/src/client.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{ KV_RATE_LIMIT_IMAP, listener::{SessionResult, SessionStream}, }; use mail_send::Credentials; use trc::{AddContext, SecurityEvent}; use crate::{ Session, State, protocol::{Command, Mechanism, request::Error}, }; impl Session { pub async fn ingest(&mut self, bytes: &[u8]) -> SessionResult { trc::event!( Pop3(trc::Pop3Event::RawInput), SpanId = self.session_id, Size = bytes.len(), Contents = trc::Value::from_maybe_string(bytes), ); let mut bytes = bytes.iter(); let mut requests = Vec::with_capacity(2); loop { match self.receiver.parse(&mut bytes) { Ok(request) => { // Group delete requests when possible match (request, requests.last_mut()) { (Command::Dele { msg }, Some(Ok(Command::DeleMany { msgs }))) => { msgs.push(msg); } (Command::Dele { msg }, Some(Ok(Command::Dele { msg: other_msg }))) => { let request = Ok(Command::DeleMany { msgs: vec![*other_msg, msg], }); requests.pop(); requests.push(request); } (request, _) => { requests.push(Ok(request)); } } } Err(Error::NeedsMoreData) => { break; } Err(Error::Parse(err)) => { // Check for port scanners if matches!(&self.state, State::NotAuthenticated { .. },) { match self.server.is_scanner_fail2banned(self.remote_addr).await { Ok(true) => { trc::event!( Security(SecurityEvent::ScanBan), SpanId = self.session_id, RemoteIp = self.remote_addr, Reason = "Invalid POP3 command", ); return SessionResult::Close; } Ok(false) => {} Err(err) => { trc::error!( err.span_id(self.session_id) .details("Failed to check for fail2ban") ); } } } requests.push(Err(trc::Pop3Event::Error.into_err().details(err))); } } } for request in requests { let result = match request { Ok(command) => match self.validate_request(command).await { Ok(command) => match command { Command::User { name } => { if let State::NotAuthenticated { username, .. } = &mut self.state { let response = format!("{name} is a valid mailbox"); *username = Some(name); self.write_ok(response) .await .map(|_| SessionResult::Continue) } else { unreachable!(); } } Command::Pass { string } => { let username = if let State::NotAuthenticated { username, .. } = &mut self.state { username.take().unwrap() } else { unreachable!() }; self.handle_auth(Credentials::Plain { username, secret: string, }) .await .map(|_| SessionResult::Continue) } Command::Quit => self.handle_quit().await.map(|_| SessionResult::Close), Command::Stat => self.handle_stat().await.map(|_| SessionResult::Continue), Command::List { msg } => { self.handle_list(msg).await.map(|_| SessionResult::Continue) } Command::Retr { msg } => self .handle_fetch(msg, None) .await .map(|_| SessionResult::Continue), Command::Dele { msg } => self .handle_dele(vec![msg]) .await .map(|_| SessionResult::Continue), Command::DeleMany { msgs } => self .handle_dele(msgs) .await .map(|_| SessionResult::Continue), Command::Top { msg, n } => self .handle_fetch(msg, n.into()) .await .map(|_| SessionResult::Continue), Command::Uidl { msg } => { self.handle_uidl(msg).await.map(|_| SessionResult::Continue) } Command::Noop => { trc::event!( Pop3(trc::Pop3Event::Noop), SpanId = self.session_id, Elapsed = trc::Value::Duration(0) ); self.write_ok("NOOP").await.map(|_| SessionResult::Continue) } Command::Rset => self.handle_rset().await.map(|_| SessionResult::Continue), Command::Capa => self.handle_capa().await.map(|_| SessionResult::Continue), Command::Stls => { self.handle_stls().await.map(|_| SessionResult::UpgradeTls) } Command::Utf8 => self.handle_utf8().await.map(|_| SessionResult::Continue), Command::Auth { mechanism, params } => self .handle_sasl(mechanism, params) .await .map(|_| SessionResult::Continue), Command::Apop { .. } => Err(trc::Pop3Event::Error .into_err() .details("APOP not supported.")), }, Err(err) => Err(err), }, Err(err) => Err(err), }; match result { Ok(SessionResult::Continue) => (), Ok(result) => return result, Err(err) => { if !self.write_err(err).await { return SessionResult::Close; } } } } SessionResult::Continue } async fn validate_request( &self, command: Command, ) -> trc::Result> { match &command { Command::Capa | Command::Quit | Command::Noop => Ok(command), Command::Auth { mechanism: Mechanism::Plain, .. } | Command::User { .. } | Command::Pass { .. } | Command::Apop { .. } => { if let State::NotAuthenticated { username, .. } = &self.state { if self.stream.is_tls() || self.server.core.imap.allow_plain_auth { if !matches!(command, Command::Pass { .. }) || username.is_some() { Ok(command) } else { Err(trc::Pop3Event::Error .into_err() .details("Username was not provided.")) } } else { Err(trc::Pop3Event::Error .into_err() .details("Cannot authenticate over plain-text.")) } } else { Err(trc::Pop3Event::Error .into_err() .details("Already authenticated.")) } } Command::Auth { .. } => { if let State::NotAuthenticated { .. } = &self.state { Ok(command) } else { Err(trc::Pop3Event::Error .into_err() .details("Already authenticated.")) } } Command::Stls => { if !self.stream.is_tls() { Ok(command) } else { Err(trc::Pop3Event::Error .into_err() .details("Already in TLS mode.")) } } Command::List { .. } | Command::Retr { .. } | Command::Dele { .. } | Command::DeleMany { .. } | Command::Top { .. } | Command::Uidl { .. } | Command::Utf8 | Command::Stat | Command::Rset => { if let State::Authenticated { mailbox, .. } = &self.state { if let Some(rate) = &self.server.core.imap.rate_requests { if self .server .core .storage .lookup .is_rate_allowed( KV_RATE_LIMIT_IMAP, &mailbox.account_id.to_be_bytes(), rate, true, ) .await .caused_by(trc::location!())? .is_none() { Ok(command) } else { Err(trc::LimitEvent::TooManyRequests.into_err()) } } else { Ok(command) } } else { Err(trc::Pop3Event::Error .into_err() .details("Not authenticated.")) } } } } } ================================================ FILE: crates/pop3/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{net::IpAddr, sync::Arc}; use common::{ Inner, Server, auth::AccessToken, listener::{ServerInstance, SessionStream, limiter::InFlight}, }; use mailbox::Mailbox; use protocol::request::Parser; pub mod client; pub mod mailbox; pub mod op; pub mod protocol; pub mod session; static SERVER_GREETING: &str = "+OK Stalwart POP3 at your service.\r\n"; #[derive(Clone)] pub struct Pop3SessionManager { pub inner: Arc, } impl Pop3SessionManager { pub fn new(inner: Arc) -> Self { Self { inner } } } pub struct Session { pub server: Server, pub instance: Arc, pub receiver: Parser, pub state: State, pub stream: T, pub in_flight: InFlight, pub remote_addr: IpAddr, pub session_id: u64, } pub enum State { NotAuthenticated { auth_failures: u32, username: Option, }, Authenticated { mailbox: Mailbox, in_flight: Option, access_token: Arc, }, } impl State { pub fn mailbox(&self) -> &Mailbox { match self { State::Authenticated { mailbox, .. } => mailbox, _ => unreachable!(), } } pub fn mailbox_mut(&mut self) -> &mut Mailbox { match self { State::Authenticated { mailbox, .. } => mailbox, _ => unreachable!(), } } pub fn access_token(&self) -> &Arc { match self { State::Authenticated { access_token, .. } => access_token, _ => unreachable!(), } } } ================================================ FILE: crates/pop3/src/mailbox.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::Session; use common::listener::SessionStream; use email::{ cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}, mailbox::INBOX_ID, }; use std::collections::BTreeMap; use trc::AddContext; use types::special_use::SpecialUse; #[derive(Default)] pub struct Mailbox { pub messages: Vec, pub account_id: u32, pub uid_validity: u32, pub total: u32, pub size: u32, } pub struct Message { pub id: u32, pub uid: u32, pub size: u32, pub deleted: bool, } impl Session { pub async fn fetch_mailbox(&self, account_id: u32) -> trc::Result { // Obtain UID validity let cache = self .server .get_cached_messages(account_id) .await .caused_by(trc::location!())?; if cache.emails.items.is_empty() { return Ok(Mailbox::default()); } let uid_validity = cache .mailbox_by_role(&SpecialUse::Inbox) .map(|x| x.uid_validity) .unwrap_or_default(); // Sort by UID let message_map = cache .emails .items .iter() .filter_map(|message| { message .mailboxes .iter() .find(|m| m.mailbox_id == INBOX_ID) .map(|m| (m.uid, (message.document_id, message.size))) }) .collect::>(); // Create mailbox let mut mailbox = Mailbox { messages: Vec::with_capacity(message_map.len()), uid_validity, account_id, ..Default::default() }; for (uid, (id, size)) in message_map { mailbox.messages.push(Message { id, uid, size, deleted: false, }); mailbox.total += 1; mailbox.size += size; } Ok(mailbox) } } ================================================ FILE: crates/pop3/src/op/authenticate.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{ auth::{ AuthRequest, sasl::{sasl_decode_challenge_oauth, sasl_decode_challenge_plain}, }, listener::{SessionStream, limiter::LimiterResult}, }; use directory::Permission; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; use crate::{ Session, State, protocol::{Command, Mechanism, request}, }; impl Session { pub async fn handle_sasl( &mut self, mechanism: Mechanism, mut params: Vec, ) -> trc::Result<()> { match mechanism { Mechanism::Plain | Mechanism::OAuthBearer | Mechanism::XOauth2 => { if !params.is_empty() { let credentials = base64_decode(params.pop().unwrap().as_bytes()) .and_then(|challenge| { if mechanism == Mechanism::Plain { sasl_decode_challenge_plain(&challenge) } else { sasl_decode_challenge_oauth(&challenge) } }) .ok_or_else(|| { trc::AuthEvent::Error .into_err() .details("Invalid SASL challenge") })?; self.handle_auth(credentials).await } else { // TODO: This hack is temporary until the SASL library is developed self.receiver.state = request::State::Argument { request: Command::Auth { mechanism: mechanism.as_str().as_bytes().to_vec(), params: vec![], }, num: 1, last_is_space: true, }; self.write_bytes("+\r\n").await } } _ => Err(trc::AuthEvent::Error .into_err() .details("Authentication mechanism not supported.")), } } pub async fn handle_auth(&mut self, credentials: Credentials) -> trc::Result<()> { // Authenticate let access_token = self .server .authenticate(&AuthRequest::from_credentials( credentials, self.session_id, self.remote_addr, )) .await .map_err(|err| { if err.matches(trc::EventType::Auth(trc::AuthEvent::Failed)) { match &self.state { State::NotAuthenticated { auth_failures, username, } if *auth_failures < self.server.core.imap.max_auth_failures => { self.state = State::NotAuthenticated { auth_failures: auth_failures + 1, username: username.clone(), }; } _ => { return trc::AuthEvent::TooManyAttempts.into_err().caused_by(err); } } } err }) .and_then(|token| { token .assert_has_permission(Permission::Pop3Authenticate) .map(|_| token) })?; // Enforce concurrency limits let in_flight = match access_token.is_imap_request_allowed() { LimiterResult::Allowed(in_flight) => Some(in_flight), LimiterResult::Forbidden => { return Err(trc::LimitEvent::ConcurrentRequest.into_err()); } LimiterResult::Disabled => None, }; // Fetch mailbox let mailbox = self.fetch_mailbox(access_token.primary_id()).await?; // Create session self.state = State::Authenticated { in_flight, mailbox, access_token, }; self.write_ok("Authentication successful").await } } ================================================ FILE: crates/pop3/src/op/delete.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; use email::message::delete::EmailDeletion; use store::{roaring::RoaringBitmap, write::BatchBuilder}; use trc::AddContext; use crate::{Session, State, protocol::response::Response}; impl Session { pub async fn handle_dele(&mut self, msgs: Vec) -> trc::Result<()> { // Validate access self.state .access_token() .assert_has_permission(Permission::Pop3Dele)?; let op_start = Instant::now(); let mailbox = self.state.mailbox_mut(); let mut response = Vec::new(); for msg in &msgs { if let Some(message) = mailbox.messages.get_mut(msg.saturating_sub(1) as usize) { if !message.deleted { response.extend_from_slice(format!("+OK message {msg} deleted\r\n").as_bytes()); message.deleted = true; } else { response.extend_from_slice( format!("-ERR message {msg} already deleted\r\n").as_bytes(), ); } } else { response.extend_from_slice("-ERR no such message\r\n".as_bytes()); } } trc::event!( Pop3(trc::Pop3Event::Delete), SpanId = self.session_id, Total = msgs.len(), Elapsed = op_start.elapsed() ); self.write_bytes(response).await } pub async fn handle_rset(&mut self) -> trc::Result<()> { let op_start = Instant::now(); let mut count = 0; let mailbox = self.state.mailbox_mut(); for message in &mut mailbox.messages { if message.deleted { count += 1; message.deleted = false; } } trc::event!( Pop3(trc::Pop3Event::Reset), SpanId = self.session_id, Total = count as u64, Elapsed = op_start.elapsed() ); self.write_ok(format!("{count} messages undeleted")).await } pub async fn handle_quit(&mut self) -> trc::Result<()> { let op_start = Instant::now(); let mut deleted_docs = Vec::new(); if let State::Authenticated { mailbox, .. } = &self.state { let mut deleted = RoaringBitmap::new(); for message in &mailbox.messages { if message.deleted { deleted.insert(message.id); deleted_docs.push(trc::Value::from(message.id)); } } if !deleted.is_empty() { let num_deleted = deleted.len(); let mut batch = BatchBuilder::new(); let not_deleted = self .server .emails_delete( mailbox.account_id, self.state.access_token().tenant_id(), &mut batch, deleted, ) .await .caused_by(trc::location!())?; if !batch.is_empty() { self.server .commit_batch(batch) .await .caused_by(trc::location!())?; self.server.notify_task_queue(); } if not_deleted.is_empty() { self.write_ok(format!( "Stalwart POP3 bids you farewell ({num_deleted} messages deleted)." )) .await?; } else { self.write_bytes( Response::Err::("Some messages could not be deleted".into()) .serialize(), ) .await?; } } else { self.write_ok("Stalwart POP3 bids you farewell (no messages deleted).") .await?; } } else { self.write_ok("Stalwart POP3 bids you farewell.").await?; } trc::event!( Pop3(trc::Pop3Event::Quit), SpanId = self.session_id, DocumentId = deleted_docs, Elapsed = op_start.elapsed() ); Ok(()) } } ================================================ FILE: crates/pop3/src/op/fetch.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{Session, protocol::response::Response}; use common::listener::SessionStream; use directory::Permission; use email::message::metadata::MessageMetadata; use std::time::Instant; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{collection::Collection, field::EmailField}; use utils::chained_bytes::ChainedBytes; impl Session { pub async fn handle_fetch(&mut self, msg: u32, lines: Option) -> trc::Result<()> { // Validate access self.state .access_token() .assert_has_permission(Permission::Pop3Retr)?; let op_start = Instant::now(); let mailbox = self.state.mailbox(); if let Some(message) = mailbox.messages.get(msg.saturating_sub(1) as usize) { if let Some(metadata_) = self .server .store() .get_value::>(ValueKey::property( mailbox.account_id, Collection::Email, message.id, EmailField::Metadata, )) .await .caused_by(trc::location!())? { let metadata = metadata_ .unarchive::() .caused_by(trc::location!())?; if let Some(bytes) = self .server .blob_store() .get_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX) .await .caused_by(trc::location!())? { trc::event!( Pop3(trc::Pop3Event::Fetch), SpanId = self.session_id, DocumentId = message.id, Elapsed = op_start.elapsed() ); let bytes = ChainedBytes::new(metadata.raw_headers.as_ref()) .with_last( bytes .get(metadata.blob_body_offset.to_native() as usize..) .unwrap_or_default(), ) .get_full_range(); self.write_bytes( Response::Message:: { bytes, lines: lines.unwrap_or(0), } .serialize(), ) .await } else { Err(trc::Pop3Event::Error .into_err() .details("Failed to fetch message. Perhaps another session deleted it?") .caused_by(trc::location!())) } } else { Err(trc::Pop3Event::Error .into_err() .details("Failed to fetch message. Perhaps another session deleted it?") .caused_by(trc::location!())) } } else { Err(trc::Pop3Event::Error.into_err().details("No such message.")) } } } ================================================ FILE: crates/pop3/src/op/list.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; use crate::{Session, protocol::response::Response}; impl Session { pub async fn handle_list(&mut self, msg: Option) -> trc::Result<()> { // Validate access self.state .access_token() .assert_has_permission(Permission::Pop3List)?; let op_start = Instant::now(); let mailbox = self.state.mailbox(); if let Some(msg) = msg { if let Some(message) = mailbox.messages.get(msg.saturating_sub(1) as usize) { trc::event!( Pop3(trc::Pop3Event::ListMessage), SpanId = self.session_id, DocumentId = message.id, Size = message.size, Elapsed = op_start.elapsed() ); self.write_ok(format!("{} {}", msg, message.size)).await } else { Err(trc::Pop3Event::Error .into_err() .details("No such message.") .caused_by(trc::location!())) } } else { trc::event!( Pop3(trc::Pop3Event::List), SpanId = self.session_id, Total = mailbox.messages.len(), Elapsed = op_start.elapsed() ); self.write_bytes( Response::List(mailbox.messages.iter().map(|m| m.size).collect::>()) .serialize(), ) .await } } pub async fn handle_uidl(&mut self, msg: Option) -> trc::Result<()> { // Validate access self.state .access_token() .assert_has_permission(Permission::Pop3Uidl)?; let op_start = Instant::now(); let mailbox = self.state.mailbox(); if let Some(msg) = msg { if let Some(message) = mailbox.messages.get(msg.saturating_sub(1) as usize) { trc::event!( Pop3(trc::Pop3Event::UidlMessage), SpanId = self.session_id, DocumentId = message.id, Uid = message.uid, UidValidity = mailbox.uid_validity, Elapsed = op_start.elapsed() ); self.write_ok(format!("{} {}{}", msg, mailbox.uid_validity, message.uid)) .await } else { Err(trc::Pop3Event::Error .into_err() .details("No such message.") .caused_by(trc::location!())) } } else { trc::event!( Pop3(trc::Pop3Event::Uidl), SpanId = self.session_id, Total = mailbox.messages.len(), Elapsed = op_start.elapsed() ); self.write_bytes( Response::List( mailbox .messages .iter() .map(|m| format!("{}{}", mailbox.uid_validity, m.uid)) .collect::>(), ) .serialize(), ) .await } } pub async fn handle_stat(&mut self) -> trc::Result<()> { // Validate access self.state .access_token() .assert_has_permission(Permission::Pop3Stat)?; let op_start = Instant::now(); let mailbox = self.state.mailbox(); trc::event!( Pop3(trc::Pop3Event::Stat), SpanId = self.session_id, Total = mailbox.total, Size = mailbox.size, Elapsed = op_start.elapsed() ); self.write_ok(format!("{} {}", mailbox.total, mailbox.size)) .await } } ================================================ FILE: crates/pop3/src/op/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::listener::SessionStream; use crate::{ Session, protocol::{Mechanism, response::Response}, }; pub mod authenticate; pub mod delete; pub mod fetch; pub mod list; impl Session { pub async fn handle_capa(&mut self) -> trc::Result<()> { let mechanisms = if self.stream.is_tls() || self.server.core.imap.allow_plain_auth { vec![Mechanism::Plain, Mechanism::OAuthBearer, Mechanism::XOauth2] } else { vec![Mechanism::OAuthBearer, Mechanism::XOauth2] }; trc::event!( Pop3(trc::Pop3Event::Capabilities), SpanId = self.session_id, Tls = self.stream.is_tls(), Strict = !self.server.core.imap.allow_plain_auth, Elapsed = trc::Value::Duration(0) ); self.write_bytes( Response::Capability:: { mechanisms, stls: !self.stream.is_tls(), } .serialize(), ) .await } pub async fn handle_stls(&mut self) -> trc::Result<()> { trc::event!( Pop3(trc::Pop3Event::StartTls), SpanId = self.session_id, Elapsed = trc::Value::Duration(0) ); self.write_ok("Begin TLS negotiation now").await } pub async fn handle_utf8(&mut self) -> trc::Result<()> { trc::event!( Pop3(trc::Pop3Event::Utf8), SpanId = self.session_id, Elapsed = trc::Value::Duration(0) ); self.write_ok("UTF8 enabled").await } } ================================================ FILE: crates/pop3/src/protocol/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod request; pub mod response; #[derive(Debug, Clone, PartialEq, Eq, Default)] pub enum Command { // Authorization state User { name: T, }, Pass { string: T, }, Apop { name: T, digest: T, }, Quit, // Transaction state Stat, List { msg: Option, }, Retr { msg: u32, }, Dele { msg: u32, }, DeleMany { msgs: Vec, }, #[default] Noop, Rset, Top { msg: u32, n: u32, }, Uidl { msg: Option, }, // Extensions Capa, Stls, Utf8, Auth { mechanism: M, params: Vec, }, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum Mechanism { Plain, CramMd5, DigestMd5, ScramSha1, ScramSha256, Apop, Ntlm, Gssapi, Anonymous, External, OAuthBearer, XOauth2, } ================================================ FILE: crates/pop3/src/protocol/request.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::borrow::Cow; use super::{Command, Mechanism}; #[derive(Debug, Clone, PartialEq, Eq)] pub enum Error { NeedsMoreData, Parse(Cow<'static, str>), } #[derive(Default, Debug)] pub enum State { #[default] Init, Command { buf: [u8; 4], len: usize, }, Argument { request: Command, Vec>, num: usize, last_is_space: bool, }, Error { reason: Cow<'static, str>, }, } #[derive(Default)] pub struct Parser { pub state: State, } const MAX_ARG_LEN: usize = 256; impl Parser { pub fn parse( &mut self, bytes: &mut std::slice::Iter<'_, u8>, ) -> Result, Error> { for &byte in bytes { match &mut self.state { State::Init => match byte { b' ' | b'\t' | b'\r' | b'\n' => {} b'a'..=b'z' => { self.state = State::Command { buf: [byte, 0, 0, 0], len: 1, }; } b'A'..=b'Z' => { self.state = State::Command { buf: [byte | 0x20, 0, 0, 0], len: 1, }; } _ => { self.state = State::Error { reason: "Invalid command".into(), }; } }, State::Command { buf, len } => match byte { b'a'..=b'z' | b'8' if *len < 4 => { buf[*len] = byte; *len += 1; } b'A'..=b'Z' if *len < 4 => { buf[*len] = byte | 0x20; *len += 1; } b' ' | b'\t' if *len == 4 || *len == 3 => match Command::parse(buf) { Ok(request) => { self.state = State::Argument { request, num: 0, last_is_space: true, }; } Err(err) => { self.state = State::Error { reason: err }; } }, b'\r' => {} b'\n' if *len == 4 || *len == 3 => match Command::parse(buf) { Ok(request) => { self.state = State::Init; return request.finalize(0); } Err(err) => { self.state = State::Init; return Err(Error::Parse(err)); } }, _ => { self.state = State::Error { reason: "Invalid command".into(), }; } }, State::Argument { request, num, last_is_space, } => match byte { b' ' | b'\t' => { *last_is_space = true; } b'\r' => {} b'\n' => { let request = std::mem::take(request).finalize(*num); self.state = State::Init; return request; } _ => { if *last_is_space { *num += 1; } match request.update_argument(*num, byte) { Ok(_) => { *last_is_space = false; } Err(err) => { self.state = State::Error { reason: err }; } } } }, State::Error { reason } => { if byte == b'\n' { let reason = std::mem::take(reason); self.state = State::Init; return Err(Error::Parse(reason)); } } } } Err(Error::NeedsMoreData) } } impl Command, Vec> { pub fn parse(bytes: &[u8; 4]) -> Result> { match (bytes[0], bytes[1], bytes[2], bytes[3]) { (b'u', b's', b'e', b'r') => Ok(Self::User { name: Vec::new() }), (b'u', b'i', b'd', b'l') => Ok(Self::Uidl { msg: None }), (b'u', b't', b'f', b'8') => Ok(Self::Utf8), (b'p', b'a', b's', b's') => Ok(Self::Pass { string: Vec::new() }), (b'a', b'p', b'o', b'p') => Ok(Self::Apop { name: Vec::new(), digest: Vec::new(), }), (b'a', b'u', b't', b'h') => Ok(Self::Auth { mechanism: Vec::new(), params: Vec::new(), }), (b'q', b'u', b'i', b't') => Ok(Self::Quit), (b'l', b'i', b's', b't') => Ok(Self::List { msg: None }), (b'r', b'e', b't', b'r') => Ok(Self::Retr { msg: 0 }), (b'r', b's', b'e', b't') => Ok(Self::Rset), (b'd', b'e', b'l', b'e') => Ok(Self::Dele { msg: 0 }), (b'n', b'o', b'o', b'p') => Ok(Self::Noop), (b't', b'o', b'p', 0) => Ok(Self::Top { msg: 0, n: 0 }), (b'c', b'a', b'p', b'a') => Ok(Self::Capa), (b's', b't', b'l', b's') => Ok(Self::Stls), (b's', b't', b'a', b't') => Ok(Self::Stat), _ => Err("Invalid command".into()), } } pub fn update_argument(&mut self, arg_num: usize, byte: u8) -> Result<(), Cow<'static, str>> { match self { Command::User { name } if arg_num == 1 && name.len() < MAX_ARG_LEN => { name.push(byte); Ok(()) } Command::Pass { string } if arg_num == 1 && string.len() < MAX_ARG_LEN => { string.push(byte); Ok(()) } Command::Apop { name, digest } if arg_num <= 2 && name.len() < MAX_ARG_LEN && digest.len() < MAX_ARG_LEN => { if arg_num == 1 { name.push(byte); } else { digest.push(byte); } Ok(()) } Command::List { msg } if arg_num == 1 => add_digit(msg.get_or_insert(0), byte), Command::Retr { msg } if arg_num == 1 => add_digit(msg, byte), Command::Dele { msg } if arg_num == 1 => add_digit(msg, byte), Command::Top { msg, n } if arg_num <= 2 => { if arg_num == 1 { add_digit(msg, byte) } else { add_digit(n, byte) } } Command::Uidl { msg } if arg_num == 1 => add_digit(msg.get_or_insert(0), byte), Command::Auth { mechanism, params } if arg_num <= 4 && mechanism.len() < 64 && params.iter().map(|p| p.len()).sum::() < (MAX_ARG_LEN * 4) => { if arg_num == 1 { mechanism.push(byte); } else { if params.len() < arg_num - 1 { params.push(Vec::new()); } params.last_mut().unwrap().push(byte); } Ok(()) } _ => Err("Too many arguments".into()), } } pub fn finalize(self, num_args: usize) -> Result, Error> { match self { Command::User { name } if num_args == 1 => { into_string(name).map(|name| Command::User { name }) } Command::Pass { string } if num_args == 1 => { into_string(string).map(|string| Command::Pass { string }) } Command::Apop { name, digest } if num_args == 2 => { let name = into_string(name)?; let digest = into_string(digest)?; Ok(Command::Apop { name, digest }) } Command::Quit => Ok(Command::Quit), Command::Stat => Ok(Command::Stat), Command::List { msg } => Ok(Command::List { msg }), Command::Retr { msg } if num_args == 1 => Ok(Command::Retr { msg }), Command::Dele { msg } if num_args == 1 => Ok(Command::Dele { msg }), Command::Noop => Ok(Command::Noop), Command::Rset => Ok(Command::Rset), Command::Top { msg, n } if num_args == 2 => Ok(Command::Top { msg, n }), Command::Uidl { msg } => Ok(Command::Uidl { msg }), Command::Capa => Ok(Command::Capa), Command::Stls => Ok(Command::Stls), Command::Utf8 => Ok(Command::Utf8), Command::Auth { mechanism, params } if num_args >= 1 => { let mechanism = Mechanism::parse(&mechanism)?; let params = params .into_iter() .map(into_string) .collect::>()?; Ok(Command::Auth { mechanism, params }) } _ => Err(Error::Parse("Missing arguments".into())), } } } #[inline(always)] fn into_string(bytes: Vec) -> Result { String::from_utf8(bytes).map_err(|_| Error::Parse("Invalid UTF-8".into())) } #[inline(always)] fn add_digit(num: &mut u32, byte: u8) -> Result<(), Cow<'static, str>> { if byte.is_ascii_digit() { *num = num .checked_mul(10) .and_then(|n| n.checked_add((byte - b'0') as u32)) .ok_or("Numeric argument out of range")?; Ok(()) } else { Err("Invalid digit".into()) } } impl Mechanism { pub fn parse(value: &[u8]) -> Result { if value.eq_ignore_ascii_case(b"PLAIN") { Ok(Self::Plain) } else if value.eq_ignore_ascii_case(b"CRAM-MD5") { Ok(Self::CramMd5) } else if value.eq_ignore_ascii_case(b"DIGEST-MD5") { Ok(Self::DigestMd5) } else if value.eq_ignore_ascii_case(b"SCRAM-SHA-1") { Ok(Self::ScramSha1) } else if value.eq_ignore_ascii_case(b"SCRAM-SHA-256") { Ok(Self::ScramSha256) } else if value.eq_ignore_ascii_case(b"APOP") { Ok(Self::Apop) } else if value.eq_ignore_ascii_case(b"NTLM") { Ok(Self::Ntlm) } else if value.eq_ignore_ascii_case(b"GSSAPI") { Ok(Self::Gssapi) } else if value.eq_ignore_ascii_case(b"ANONYMOUS") { Ok(Self::Anonymous) } else if value.eq_ignore_ascii_case(b"EXTERNAL") { Ok(Self::External) } else if value.eq_ignore_ascii_case(b"OAUTHBEARER") { Ok(Self::OAuthBearer) } else if value.eq_ignore_ascii_case(b"XOAUTH2") { Ok(Self::XOauth2) } else { Err(Error::Parse( format!( "Unsupported mechanism '{}'.", String::from_utf8_lossy(value) ) .into(), )) } } } #[cfg(test)] mod tests { use crate::protocol::{Command, Mechanism, request::Error}; use super::Parser; #[test] fn parse_command() { let mut parser = Parser::default(); let mut chunked = String::new(); let mut chunked_expected = Vec::new(); for (cmd, request) in [ ("QuiT", Command::Quit), (" \r\n NOOP ", Command::Noop), ("STAT ", Command::Stat), ("LIST ", Command::List { msg: None }), (" list 100 ", Command::List { msg: 100.into() }), ("retr 55", Command::Retr { msg: 55 }), ("DELE 99", Command::Dele { msg: 99 }), (" rset ", Command::Rset), ("top 8000 1234", Command::Top { msg: 8000, n: 1234 }), ("uidl", Command::Uidl { msg: None }), ("uidl 000099999", Command::Uidl { msg: 99999.into() }), ( "USER test", Command::User { name: "test".to_string(), }, ), ( "PASS secret", Command::Pass { string: "secret".to_string(), }, ), ( "APOP mrose c4c9334bac560ecc979e58001b3e22fb", Command::Apop { name: "mrose".to_string(), digest: "c4c9334bac560ecc979e58001b3e22fb".to_string(), }, ), ("utf8", Command::Utf8), ("capa", Command::Capa), ( "AUTH GSSAPI", Command::Auth { mechanism: Mechanism::Gssapi, params: vec![], }, ), ( "AUTH PLAIN dGVzdAB0ZXN0AHRlc3Q=", Command::Auth { mechanism: Mechanism::Plain, params: vec!["dGVzdAB0ZXN0AHRlc3Q=".to_string()], }, ), ] { assert_eq!( parser.parse(&mut cmd.as_bytes().iter()), Err(Error::NeedsMoreData) ); assert_eq!( parser.parse(&mut b"\r\n".iter()), Ok(request.clone()), "{:?}", cmd ); chunked.push_str(cmd); chunked.push_str("\r\n"); chunked_expected.push(request); } for chunk_size in [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] { let mut parser = Parser::default(); let mut requests = Vec::new(); for chunk in chunked.as_bytes().chunks(chunk_size) { let mut chunk = chunk.iter(); loop { match parser.parse(&mut chunk) { Ok(request) => { requests.push(request); } Err(Error::NeedsMoreData) => break, Err(err) => { panic!("Unexpected error on chunk size {chunk_size}: {err:?}"); } } } } assert_eq!(requests, chunked_expected, "Chunk size: {}", chunk_size); } for cmd in [ "user", "pass", "user a b", "pass c d", "apop", "apop a", "apop a b c", "quit 1", "stat 1", "list 1 2", "retr", "retr 1 2", "dele", "dele 1 2", "noop 1", "rset 1", "top", "top 1 2 3", "uidl 1 2 3", "capa 1", "stls 1", "utf8 1", "auth", "auth unknown", ] { assert_eq!( parser.parse(&mut cmd.as_bytes().iter()), Err(Error::NeedsMoreData) ); let result = parser.parse(&mut b"\r\n".iter()); assert!(result.is_err(), "{:?}", result); } } } ================================================ FILE: crates/pop3/src/protocol/response.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::Mechanism; use std::{borrow::Cow, fmt::Display}; use utils::chained_bytes::SliceRange; pub enum Response<'x, T> { Ok(Cow<'static, str>), Err(Cow<'static, str>), List(Vec), Message { bytes: SliceRange<'x>, lines: u32, }, Capability { mechanisms: Vec, stls: bool, }, } impl<'x, T: Display> Response<'x, T> { pub fn serialize(&self) -> Vec { match self { Response::Ok(message) => { let mut buf = Vec::with_capacity(message.len() + 6); buf.extend_from_slice(b"+OK "); buf.extend_from_slice(message.as_bytes()); buf.extend_from_slice(b"\r\n"); buf } Response::Err(message) => { let mut buf = Vec::with_capacity(message.len() + 6); buf.extend_from_slice(b"-ERR "); buf.extend_from_slice(message.as_bytes()); buf.extend_from_slice(b"\r\n"); buf } Response::List(octets) => { let mut buf = Vec::with_capacity(octets.len() * 8 + 10); buf.extend_from_slice(format!("+OK {} messages\r\n", octets.len()).as_bytes()); for (num, octet) in octets.iter().enumerate() { buf.extend_from_slice((num + 1).to_string().as_bytes()); buf.extend_from_slice(b" "); buf.extend_from_slice(octet.to_string().as_bytes()); buf.extend_from_slice(b"\r\n"); } buf.extend_from_slice(b".\r\n"); buf } Response::Message { bytes, lines } => { let mut buf = Vec::with_capacity(bytes.len() + 10); buf.extend_from_slice(b"+OK "); buf.extend_from_slice(bytes.len().to_string().as_bytes()); buf.extend_from_slice(b" octets\r\n"); let mut line_count = 0; let mut last_byte = 0; // Transparency procedure for &byte in bytes.into_iter() { // POP3 requires that lines end with CRLF, do this check to ensure that if byte == b'\n' && last_byte != b'\r' { buf.push(b'\r'); } if byte == b'.' && last_byte == b'\n' { buf.push(b'.'); } buf.push(byte); last_byte = byte; if *lines > 0 && byte == b'\n' { line_count += 1; if line_count == *lines { break; } } } if last_byte != b'\n' { buf.extend_from_slice(b"\r\n"); } buf.extend_from_slice(b".\r\n"); buf } Response::Capability { mechanisms, stls } => { let mut buf = Vec::with_capacity(256); buf.extend_from_slice(b"+OK Capability list follows\r\n"); if !mechanisms.is_empty() { if mechanisms.contains(&Mechanism::Plain) { buf.extend_from_slice(b"USER\r\n"); } buf.extend_from_slice(b"SASL"); for mechanism in mechanisms { buf.extend_from_slice(b" "); buf.extend_from_slice(mechanism.as_str().as_bytes()); } buf.extend_from_slice(b"\r\n"); } if *stls { buf.extend_from_slice(b"STLS\r\n"); } for capa in [ "TOP", "RESP-CODES", "PIPELINING", "EXPIRE NEVER", "UIDL", "UTF8", "IMPLEMENTATION Stalwart Server", ] { buf.extend_from_slice(capa.as_bytes()); buf.extend_from_slice(b"\r\n"); } buf.extend_from_slice(b".\r\n"); buf } } } } impl Mechanism { pub fn as_str(&self) -> &'static str { match self { Mechanism::Plain => "PLAIN", Mechanism::CramMd5 => "CRAM-MD5", Mechanism::DigestMd5 => "DIGEST-MD5", Mechanism::ScramSha1 => "SCRAM-SHA-1", Mechanism::ScramSha256 => "SCRAM-SHA-256", Mechanism::Apop => "APOP", Mechanism::Ntlm => "NTLM", Mechanism::Gssapi => "GSSAPI", Mechanism::Anonymous => "ANONYMOUS", Mechanism::External => "EXTERNAL", Mechanism::OAuthBearer => "OAUTHBEARER", Mechanism::XOauth2 => "XOAUTH2", } } } pub trait SerializeResponse { fn serialize(&self) -> Vec; } impl SerializeResponse for trc::Error { fn serialize(&self) -> Vec { let message = self .value_as_str(trc::Key::Details) .unwrap_or_else(|| self.as_ref().message()); let mut buf = Vec::with_capacity(message.len() + 6); buf.extend_from_slice(b"-ERR "); buf.extend_from_slice(message.as_bytes()); buf.extend_from_slice(b"\r\n"); buf } } #[cfg(test)] mod tests { use super::Response; use crate::protocol::Mechanism; use utils::chained_bytes::SliceRange; #[test] fn serialize_response() { for (cmd, expected) in [ ( Response::Ok("message 1 deleted".into()), "+OK message 1 deleted\r\n", ), ( Response::Err("permission denied".into()), "-ERR permission denied\r\n", ), ( Response::List(vec![100, 200, 300]), "+OK 3 messages\r\n1 100\r\n2 200\r\n3 300\r\n.\r\n", ), ( Response::Capability { mechanisms: vec![Mechanism::Plain, Mechanism::CramMd5], stls: true, }, concat!( "+OK Capability list follows\r\n", "USER\r\n", "SASL PLAIN CRAM-MD5\r\n", "STLS\r\n", "TOP\r\n", "RESP-CODES\r\n", "PIPELINING\r\n", "EXPIRE NEVER\r\n", "UIDL\r\n", "UTF8\r\n", "IMPLEMENTATION Stalwart Server\r\n.\r\n" ), ), ( Response::Message { bytes: SliceRange::Split(b"Subject: test\r\n\r\n.\r\n", b"test.\r\n.test\r\na"), lines: 0, }, "+OK 35 octets\r\nSubject: test\r\n\r\n..\r\ntest.\r\n..test\r\na\r\n.\r\n", ), ] { assert_eq!(expected, String::from_utf8(cmd.serialize()).unwrap()); } } } ================================================ FILE: crates/pop3/src/session.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::borrow::Cow; use common::{ core::BuildServer, listener::{SessionData, SessionManager, SessionResult, SessionStream}, }; use tokio_rustls::server::TlsStream; use crate::{ Pop3SessionManager, SERVER_GREETING, Session, State, protocol::{ request::Parser, response::{Response, SerializeResponse}, }, }; use tokio::io::{AsyncReadExt, AsyncWriteExt}; impl SessionManager for Pop3SessionManager { #[allow(clippy::manual_async_fn)] fn handle( self, session: SessionData, ) -> impl std::future::Future + Send { async move { let mut session = Session { server: self.inner.build_server(), instance: session.instance, receiver: Parser::default(), state: State::NotAuthenticated { auth_failures: 0, username: None, }, stream: session.stream, in_flight: session.in_flight, remote_addr: session.remote_ip, session_id: session.session_id, }; if session .write_bytes(SERVER_GREETING.as_bytes()) .await .is_ok() && session.handle_conn().await && session.instance.acceptor.is_tls() && let Ok(mut session) = session.into_tls().await { session.handle_conn().await; } } } #[allow(clippy::manual_async_fn)] fn shutdown(&self) -> impl std::future::Future + Send { async {} } } impl Session { pub async fn handle_conn(&mut self) -> bool { let mut buf = vec![0; 8192]; let mut shutdown_rx = self.instance.shutdown_rx.clone(); loop { tokio::select! { result = tokio::time::timeout( if !matches!(self.state, State::NotAuthenticated {..}) { self.server.core.imap.timeout_auth } else { self.server.core.imap.timeout_unauth }, self.stream.read(&mut buf)) => { match result { Ok(Ok(bytes_read)) => { if bytes_read > 0 { match self.ingest(&buf[..bytes_read]).await { SessionResult::Continue => (), SessionResult::UpgradeTls => { return true; } SessionResult::Close => { break; } } } else { trc::event!( Network(trc::NetworkEvent::Closed), SpanId = self.session_id, CausedBy = trc::location!() ); break; } }, Ok(Err(err)) => { trc::event!( Network(trc::NetworkEvent::ReadError), SpanId = self.session_id, Reason = err.to_string() , CausedBy = trc::location!() ); break; }, Err(_) => { trc::event!( Network(trc::NetworkEvent::Timeout), SpanId = self.session_id, CausedBy = trc::location!() ); self.write_bytes(&b"-ERR Connection timed out.\r\n"[..]).await.ok(); break; } } }, _ = shutdown_rx.changed() => { trc::event!( Network(trc::NetworkEvent::Closed), SpanId = self.session_id, Reason = "Server shutting down", CausedBy = trc::location!() ); self.write_bytes(&b"* BYE Server shutting down.\r\n"[..]).await.ok(); break; } }; } false } pub async fn into_tls(self) -> Result>, ()> { Ok(Session { stream: self .instance .tls_accept(self.stream, self.session_id) .await?, server: self.server, instance: self.instance, receiver: self.receiver, state: self.state, session_id: self.session_id, in_flight: self.in_flight, remote_addr: self.remote_addr, }) } } impl Session { pub async fn write_bytes(&mut self, bytes: impl AsRef<[u8]>) -> trc::Result<()> { let bytes = bytes.as_ref(); trc::event!( Pop3(trc::Pop3Event::RawOutput), SpanId = self.session_id, Size = bytes.len(), Contents = trc::Value::from_maybe_string(bytes), ); self.stream.write_all(bytes.as_ref()).await.map_err(|err| { trc::NetworkEvent::WriteError .into_err() .reason(err) .caused_by(trc::location!()) })?; self.stream.flush().await.map_err(|err| { trc::NetworkEvent::WriteError .into_err() .reason(err) .caused_by(trc::location!()) }) } pub async fn write_ok(&mut self, message: impl Into>) -> trc::Result<()> { self.write_bytes(Response::Ok::(message.into()).serialize()) .await } pub async fn write_err(&mut self, err: trc::Error) -> bool { let disconnect = err.must_disconnect(); let response = err.serialize(); let write_err = err.should_write_err(); trc::error!(err.span_id(self.session_id)); if write_err && let Err(err) = self.write_bytes(response).await { trc::error!(err.span_id(self.session_id)); return false; } !disconnect } } ================================================ FILE: crates/services/Cargo.toml ================================================ [package] name = "services" version = "0.15.5" edition = "2024" [dependencies] store = { path = "../store" } common = { path = "../common" } utils = { path = "../utils" } trc = { path = "../trc" } email = { path = "../email" } smtp = { path = "../smtp" } groupware = { path = "../groupware" } spam-filter = { path = "../spam-filter" } types = { path = "../types" } jmap_proto = { path = "../jmap-proto" } directory = { path = "../directory" } smtp-proto = { version = "0.2", features = ["rkyv", "serde"] } tokio = { version = "1.47", features = ["rt"] } mail-parser = { version = "0.11", features = ["full_encoding", "rkyv"] } mail-builder = { version = "0.4" } calcard = { version = "0.3", features = ["rkyv"] } chrono = { version = "0.4", features = ["unstable-locales"] } serde = { version = "1.0", features = ["derive"]} serde_json = "1.0" memory-stats = "1.2.0" aes-gcm = "0.10.1" aes-gcm-siv = "0.11.1" rsa = "0.9.2" p256 = { version = "0.13", features = ["ecdh"] } hkdf = "0.12.3" sha2 = "0.10" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-webpki-roots", "http2"]} base64 = "0.22" compact_str = "0.9.0" [dev-dependencies] [features] test_mode = [] enterprise = [] ================================================ FILE: crates/services/src/broadcast/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::ipc::{BroadcastEvent, CalendarAlert, EmailPush, PushNotification}; use std::{borrow::Borrow, io::Write}; use types::type_state::StateChange; use utils::{ codec::leb128::{Leb128Iterator, Leb128Writer}, map::bitmap::Bitmap, }; pub mod publisher; pub mod subscriber; #[derive(Debug)] pub(crate) struct BroadcastBatch { messages: T, } const MAX_BATCH_SIZE: usize = 100; pub(crate) const BROADCAST_TOPIC: &str = "stwt.agora"; impl BroadcastBatch> { pub fn init() -> Self { Self { messages: Vec::with_capacity(MAX_BATCH_SIZE), } } pub fn insert(&mut self, message: BroadcastEvent) -> bool { self.messages.push(message); self.messages.len() < MAX_BATCH_SIZE } pub fn serialize(&self, node_id: u16) -> Vec { let mut serialized = Vec::with_capacity((self.messages.len() * 10) + std::mem::size_of::()); let _ = serialized.write_leb128(node_id); for message in &self.messages { match message { BroadcastEvent::PushNotification(notification) => match notification { PushNotification::StateChange(state_change) => { serialized.push(0u8); let _ = serialized.write_leb128(state_change.change_id); let _ = serialized.write_leb128(*state_change.types.as_ref()); let _ = serialized.write_leb128(state_change.account_id); } PushNotification::CalendarAlert(calendar_alert) => { serialized.push(1u8); let _ = serialized.write_leb128(calendar_alert.account_id); let _ = serialized.write_leb128(calendar_alert.event_id); let _ = serialized .write_leb128(calendar_alert.recurrence_id.unwrap_or_default() as u64); let _ = serialized.write_leb128(calendar_alert.uid.len()); let _ = serialized.write(calendar_alert.uid.as_bytes()); let _ = serialized.write_leb128(calendar_alert.alert_id.len()); let _ = serialized.write(calendar_alert.alert_id.as_bytes()); } PushNotification::EmailPush(email_push) => { serialized.push(2u8); let _ = serialized.write_leb128(email_push.account_id); let _ = serialized.write_leb128(email_push.email_id); let _ = serialized.write_leb128(email_push.change_id); } }, BroadcastEvent::InvalidateAccessTokens(items) => { serialized.push(3u8); let _ = serialized.write_leb128(items.len()); for item in items { let _ = serialized.write_leb128(*item); } } BroadcastEvent::InvalidateGroupwareCache(items) => { serialized.push(4u8); let _ = serialized.write_leb128(items.len()); for item in items { let _ = serialized.write_leb128(*item); } } BroadcastEvent::ReloadSettings => { serialized.push(5u8); } BroadcastEvent::ReloadBlockedIps => { serialized.push(6u8); } BroadcastEvent::ReloadPushServers(account_id) => { serialized.push(7u8); let _ = serialized.write_leb128(*account_id); } BroadcastEvent::ReloadSpamFilter => { serialized.push(8u8); } } } serialized } pub fn clear(&mut self) { self.messages.clear(); } } impl BroadcastBatch where T: Iterator + Leb128Iterator, I: Borrow, { pub fn node_id(&mut self) -> Option { self.messages.next_leb128::() } pub fn next_event(&mut self) -> Result, ()> { if let Some(id) = self.messages.next() { match id.borrow() { 0 => Ok(Some(BroadcastEvent::PushNotification( PushNotification::StateChange(StateChange { change_id: self.messages.next_leb128().ok_or(())?, types: Bitmap::from(self.messages.next_leb128::().ok_or(())?), account_id: self.messages.next_leb128().ok_or(())?, }), ))), 1 => { let account_id = self.messages.next_leb128().ok_or(())?; let event_id = self.messages.next_leb128().ok_or(())?; let recurrence_id = self.messages.next_leb128::().ok_or(())? as i64; let uid_len = self.messages.next_leb128::().ok_or(())?; let mut uid_bytes = vec![0u8; uid_len]; for byte in uid_bytes.iter_mut() { *byte = self.messages.next().ok_or(())?.borrow().to_owned(); } let uid = String::from_utf8(uid_bytes).map_err(|_| ())?; let alert_id_len = self.messages.next_leb128::().ok_or(())?; let mut alert_id_bytes = vec![0u8; alert_id_len]; for byte in alert_id_bytes.iter_mut() { *byte = self.messages.next().ok_or(())?.borrow().to_owned(); } let alert_id = String::from_utf8(alert_id_bytes).map_err(|_| ())?; Ok(Some(BroadcastEvent::PushNotification( PushNotification::CalendarAlert(CalendarAlert { account_id, event_id, recurrence_id: if recurrence_id == 0 { None } else { Some(recurrence_id) }, uid, alert_id, }), ))) } 2 => Ok(Some(BroadcastEvent::PushNotification( PushNotification::EmailPush(EmailPush { account_id: self.messages.next_leb128().ok_or(())?, email_id: self.messages.next_leb128().ok_or(())?, change_id: self.messages.next_leb128().ok_or(())?, }), ))), 3 => { let count = self.messages.next_leb128::().ok_or(())?; let mut items = Vec::with_capacity(count); for _ in 0..count { items.push(self.messages.next_leb128().ok_or(())?); } Ok(Some(BroadcastEvent::InvalidateAccessTokens(items))) } 4 => { let count = self.messages.next_leb128::().ok_or(())?; let mut items = Vec::with_capacity(count); for _ in 0..count { items.push(self.messages.next_leb128().ok_or(())?); } Ok(Some(BroadcastEvent::InvalidateGroupwareCache(items))) } 5 => Ok(Some(BroadcastEvent::ReloadSettings)), 6 => Ok(Some(BroadcastEvent::ReloadBlockedIps)), 7 => { let account_id = self.messages.next_leb128().ok_or(())?; Ok(Some(BroadcastEvent::ReloadPushServers(account_id))) } 8 => Ok(Some(BroadcastEvent::ReloadSpamFilter)), _ => Err(()), } } else { Ok(None) } } } impl BroadcastBatch { pub fn new(messages: T) -> Self { Self { messages } } } ================================================ FILE: crates/services/src/broadcast/publisher.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::Arc; use common::{Inner, ipc::BroadcastEvent}; use tokio::sync::mpsc; use trc::ClusterEvent; use super::{BROADCAST_TOPIC, BroadcastBatch}; pub fn spawn_broadcast_publisher(inner: Arc, mut event_rx: mpsc::Receiver) { let (pubsub, this_node_id) = { let _core = inner.shared_core.load(); let pubsub = inner.shared_core.load().storage.pubsub.clone(); if pubsub.is_none() { return; } (pubsub, _core.network.node_id as u16) }; tokio::spawn(async move { let mut batch = BroadcastBatch::init(); trc::event!(Cluster(ClusterEvent::PublisherStart)); while let Some(event) = event_rx.recv().await { batch.insert(event); while let Ok(event) = event_rx.try_recv() { if !batch.insert(event) { break; } } match pubsub .publish(BROADCAST_TOPIC, batch.serialize(this_node_id)) .await { Ok(_) => { batch.clear(); } Err(err) => { batch.clear(); trc::event!(Cluster(ClusterEvent::PublisherError), CausedBy = err); } } } trc::event!(Cluster(ClusterEvent::PublisherStop)); }); } ================================================ FILE: crates/services/src/broadcast/subscriber.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::broadcast::{BROADCAST_TOPIC, BroadcastBatch}; use common::{ Inner, core::BuildServer, ipc::{BroadcastEvent, HousekeeperEvent, PushEvent, PushNotification}, }; use compact_str::CompactString; use std::{sync::Arc, time::Duration}; use tokio::sync::watch; use trc::{ClusterEvent, ServerEvent}; pub fn spawn_broadcast_subscriber(inner: Arc, mut shutdown_rx: watch::Receiver) { let this_node_id = { let _core = inner.shared_core.load(); if _core.storage.pubsub.is_none() { return; } _core.network.node_id as u16 }; tokio::spawn(async move { let mut retry_count = 0; trc::event!(Cluster(ClusterEvent::SubscriberStart)); loop { let pubsub = inner.shared_core.load().storage.pubsub.clone(); if pubsub.is_none() { trc::event!( Cluster(ClusterEvent::SubscriberError), Details = "PubSub is no longer configured" ); break; } let mut stream = match pubsub.subscribe(BROADCAST_TOPIC).await { Ok(stream) => { retry_count = 0; stream } Err(err) => { trc::event!( Cluster(ClusterEvent::SubscriberError), CausedBy = err, Details = "Failed to subscribe to channel" ); match tokio::time::timeout( Duration::from_secs(1 << retry_count.max(6)), shutdown_rx.changed(), ) .await { Ok(_) => { break; } Err(_) => { retry_count += 1; continue; } } } }; tokio::select! { message = stream.next() => { match message { Some(message) => { let mut batch = BroadcastBatch::new(message.payload().iter()); let node_id = match batch.node_id() { Some(node_id) => { if node_id != this_node_id { node_id } else { trc::event!( Cluster(ClusterEvent::MessageSkipped), Details = message.payload() ); continue; } } None => { trc::event!( Cluster(ClusterEvent::MessageInvalid), Details = message.payload() ); continue; } }; loop { match batch.next_event() { Ok(Some(event)) => { trc::event!( Cluster(ClusterEvent::MessageReceived), From = node_id, To = this_node_id, Details = log_event(&event), ); match event { BroadcastEvent::PushNotification(notification) => { if inner .ipc .push_tx .send(PushEvent::Publish { notification, broadcast: false, }) .await .is_err() { trc::event!( Server(ServerEvent::ThreadError), Details = "Error sending push notification.", CausedBy = trc::location!() ); } } BroadcastEvent::ReloadPushServers(account_id) => { if inner .ipc .push_tx .send(PushEvent::PushServerUpdate { account_id, broadcast: false }) .await .is_err() { trc::event!( Server(ServerEvent::ThreadError), Details = "Error sending reload request.", CausedBy = trc::location!() ); } } BroadcastEvent::InvalidateAccessTokens(ids) => { for id in &ids { inner.cache.permissions.remove(id); inner.cache.access_tokens.remove(id); } } BroadcastEvent::InvalidateGroupwareCache(ids) => { for id in &ids { inner.cache.files.remove(id); inner.cache.contacts.remove(id); inner.cache.events.remove(id); inner.cache.scheduling.remove(id); } } BroadcastEvent::ReloadSettings => { match inner.build_server().reload().await { Ok(result) => { if let Some(new_core) = result.new_core { // Update core inner.shared_core.store(new_core.into()); if inner .ipc .housekeeper_tx .send(HousekeeperEvent::ReloadSettings) .await .is_err() { trc::event!( Server(trc::ServerEvent::ThreadError), Details = "Failed to send setting reload event to housekeeper", CausedBy = trc::location!(), ); } } } Err(err) => { trc::error!( err.details("Failed to reload settings") .caused_by(trc::location!()) ); } } } BroadcastEvent::ReloadBlockedIps => { if let Err(err) = inner.build_server().reload_blocked_ips().await { trc::error!( err.details("Failed to reload settings") .caused_by(trc::location!()) ); } } BroadcastEvent::ReloadSpamFilter => { if let Err(err) = inner.build_server().spam_model_reload().await { trc::error!( err.details("Failed to reload spam filter model") .caused_by(trc::location!()) ); } } } } Ok(None) => break, Err(_) => { trc::event!( Cluster(ClusterEvent::MessageInvalid), Details = message.payload() ); break; } } } } None => { trc::event!( Cluster(ClusterEvent::SubscriberDisconnected), ); } } }, _ = shutdown_rx.changed() => { break; } }; } trc::event!(Cluster(ClusterEvent::SubscriberStop)); }); } fn log_event(event: &BroadcastEvent) -> trc::Value { match event { BroadcastEvent::PushNotification(notification) => match notification { PushNotification::StateChange(state_change) => trc::Value::Array(vec![ "StateChange".into(), state_change.account_id.into(), state_change.change_id.into(), (*state_change.types.as_ref()).into(), ]), PushNotification::CalendarAlert(calendar_alert) => trc::Value::Array(vec![ "CalendarAlert".into(), calendar_alert.account_id.into(), calendar_alert.event_id.into(), calendar_alert.recurrence_id.into(), calendar_alert.uid.clone().into(), calendar_alert.alert_id.clone().into(), ]), PushNotification::EmailPush(email_push) => trc::Value::Array(vec![ "EmailPush".into(), email_push.account_id.into(), email_push.email_id.into(), email_push.change_id.into(), ]), }, BroadcastEvent::ReloadSettings => CompactString::const_new("ReloadSettings").into(), BroadcastEvent::ReloadBlockedIps => CompactString::const_new("ReloadBlockedIps").into(), BroadcastEvent::InvalidateAccessTokens(items) => { let mut array = Vec::with_capacity(items.len() + 1); array.push("InvalidateAccessTokens".into()); for item in items { array.push((*item).into()); } trc::Value::Array(array) } BroadcastEvent::InvalidateGroupwareCache(items) => { let mut array = Vec::with_capacity(items.len() + 1); array.push("InvalidateGroupwareCache".into()); for item in items { array.push((*item).into()); } trc::Value::Array(array) } BroadcastEvent::ReloadPushServers(account_id) => { trc::Value::Array(vec!["ReloadPushServers".into(), (*account_id).into()]) } BroadcastEvent::ReloadSpamFilter => CompactString::const_new("ReloadSpamFilter").into(), } } ================================================ FILE: crates/services/src/housekeeper/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{ Inner, KV_LOCK_HOUSEKEEPER, LONG_1D_SLUMBER, Server, config::{spamfilter, telemetry::OtelMetrics}, core::BuildServer, ipc::{BroadcastEvent, HousekeeperEvent, PurgeType}, }; use email::message::delete::EmailDeletion; use smtp::reporting::SmtpReporting; use spam_filter::modules::classifier::SpamClassifier; use std::{ collections::BinaryHeap, future::Future, sync::Arc, time::{Duration, Instant, SystemTime}, }; use store::{PurgeStore, write::now}; use tokio::sync::mpsc; use trc::{Collector, MetricType, PurgeEvent}; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] use common::telemetry::{ metrics::store::{MetricsStore, SharedMetricHistory}, tracers::store::TracingStore, }; // SPDX-SnippetEnd #[derive(PartialEq, Eq)] struct Action { due: Instant, event: ActionClass, } #[derive(PartialEq, Eq, Debug)] enum ActionClass { Account, Store(usize), Acme(String), OtelMetrics, CalculateMetrics, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] InternalMetrics, #[cfg(feature = "enterprise")] AlertMetrics, #[cfg(feature = "enterprise")] RenewLicense, // SPDX-SnippetEnd TrainSpamClassifier, } #[derive(Default)] struct Queue { heap: BinaryHeap, } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] const METRIC_ALERTS_INTERVAL: Duration = Duration::from_secs(5 * 60); // SPDX-SnippetEnd pub fn spawn_housekeeper(inner: Arc, mut rx: mpsc::Receiver) { tokio::spawn(async move { trc::event!(Housekeeper(trc::HousekeeperEvent::Start)); let start_time = SystemTime::now(); // Add all events to queue let mut queue = Queue::default(); { let server = inner.build_server(); let roles = &server.core.network.roles; // Account purge if roles.purge_accounts.is_enabled_or_sharded() { queue.schedule( Instant::now() + server.core.jmap.account_purge_frequency.time_to_next(), ActionClass::Account, ); } // Store purges if roles.purge_stores.is_enabled_or_sharded() { for (idx, schedule) in server.core.storage.purge_schedules.iter().enumerate() { queue.schedule( Instant::now() + schedule.cron.time_to_next(), ActionClass::Store(idx), ); } } // Spam classifier training if roles.spam_training.is_enabled_or_sharded() && let Some(train_frequency) = server .core .spam .classifier .as_ref() .and_then(|c| c.train_frequency) { let next_train = match server.inner.data.spam_classifier.load().as_ref() { spamfilter::SpamClassifier::FhClassifier { last_trained_at, .. } | spamfilter::SpamClassifier::CcfhClassifier { last_trained_at, .. } => now().saturating_sub(*last_trained_at).min(train_frequency), spamfilter::SpamClassifier::Disabled => train_frequency, }; queue.schedule( Instant::now() + Duration::from_secs(next_train), ActionClass::TrainSpamClassifier, ); } // OTEL Push Metrics if roles.push_metrics.is_enabled_or_sharded() && let Some(otel) = &server.core.metrics.otel { OtelMetrics::enable_errors(); queue.schedule(Instant::now() + otel.interval, ActionClass::OtelMetrics); } // Calculate expensive metrics queue.schedule(Instant::now(), ActionClass::CalculateMetrics); // Add all ACME renewals to heap for provider in server.core.acme.providers.values() { if roles.renew_acme.is_enabled_for_hash(&provider.id) { match server.init_acme(provider).await { Ok(renew_at) => { queue.schedule( Instant::now() + renew_at, ActionClass::Acme(provider.id.clone()), ); } Err(err) => { trc::error!( err.details("Failed to initialize ACME certificate manager.") ); } }; } } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL // Enterprise Edition license management #[cfg(feature = "enterprise")] if let Some(enterprise) = &server.core.enterprise { queue.schedule( Instant::now() + enterprise.license.renew_in(), ActionClass::RenewLicense, ); if let Some(metrics_store) = enterprise.metrics_store.as_ref() { queue.schedule( Instant::now() + metrics_store.interval.time_to_next(), ActionClass::InternalMetrics, ); } if !enterprise.metrics_alerts.is_empty() { queue.schedule( Instant::now() + METRIC_ALERTS_INTERVAL, ActionClass::AlertMetrics, ); } } // SPDX-SnippetEnd } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL // Metrics history #[cfg(feature = "enterprise")] let metrics_history = SharedMetricHistory::default(); // SPDX-SnippetEnd let mut next_metric_update = Instant::now(); loop { match tokio::time::timeout(queue.wake_up_time(), rx.recv()).await { Ok(Some(event)) => { match event { HousekeeperEvent::ReloadSettings => { let server = inner.build_server(); // Reload OTEL push metrics match &server.core.metrics.otel { Some(otel) if !queue.has_action(&ActionClass::OtelMetrics) => { OtelMetrics::enable_errors(); queue.schedule( Instant::now() + otel.interval, ActionClass::OtelMetrics, ); } _ => {} } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] if let Some(enterprise) = &server.core.enterprise { if !queue.has_action(&ActionClass::RenewLicense) { queue.schedule( Instant::now() + enterprise.license.renew_in(), ActionClass::RenewLicense, ); } if let Some(metrics_store) = enterprise.metrics_store.as_ref() && !queue.has_action(&ActionClass::InternalMetrics) { queue.schedule( Instant::now() + metrics_store.interval.time_to_next(), ActionClass::InternalMetrics, ); } if !enterprise.metrics_alerts.is_empty() && !queue.has_action(&ActionClass::AlertMetrics) { queue.schedule(Instant::now(), ActionClass::AlertMetrics); } } // SPDX-SnippetEnd // Reload queue settings server .inner .ipc .queue_tx .send(common::ipc::QueueEvent::ReloadSettings) .await .ok(); // Reload ACME certificates tokio::spawn(async move { for provider in server.core.acme.providers.values() { match server.init_acme(provider).await { Ok(renew_at) => { server .inner .ipc .housekeeper_tx .send(HousekeeperEvent::AcmeReschedule { provider_id: provider.id.clone(), renew_at: Instant::now() + renew_at, }) .await .ok(); } Err(err) => { trc::error!(err.details( "Failed to reload ACME certificate manager." )); } }; } }); } HousekeeperEvent::AcmeReschedule { provider_id, renew_at, } => { let action = ActionClass::Acme(provider_id); queue.remove_action(&action); queue.schedule(renew_at, action); } HousekeeperEvent::Purge(purge) => { let server = inner.build_server(); tokio::spawn(async move { server.purge(purge, 0).await; }); } HousekeeperEvent::Exit => { trc::event!( Housekeeper(trc::HousekeeperEvent::Stop), Reason = "Shutdown" ); return; } } } Ok(None) => { trc::event!( Housekeeper(trc::HousekeeperEvent::Stop), Reason = "Channel closed" ); return; } Err(_) => { let server = inner.build_server(); while let Some(event) = queue.pop() { match event.event { ActionClass::Acme(provider_id) => { trc::event!(Housekeeper(trc::HousekeeperEvent::Run), Type = "acme"); let server = server.clone(); tokio::spawn(async move { if let Some(provider) = server.core.acme.providers.get(&provider_id) { trc::event!( Acme(trc::AcmeEvent::OrderStart), Hostname = provider.domains.as_slice() ); let renew_at = match server.renew(provider).await { Ok(renew_at) => { trc::event!( Acme(trc::AcmeEvent::OrderCompleted), Domain = provider.domains.as_slice(), Expires = trc::Value::Timestamp( now() + renew_at.as_secs() ) ); renew_at } Err(err) => { trc::error!( err.details("Failed to renew certificates.") ); Duration::from_secs(3600) } }; server .cluster_broadcast(BroadcastEvent::ReloadSettings) .await; server .inner .ipc .housekeeper_tx .send(HousekeeperEvent::AcmeReschedule { provider_id: provider_id.clone(), renew_at: Instant::now() + renew_at, }) .await .ok(); } }); } ActionClass::Account => { trc::event!( Housekeeper(trc::HousekeeperEvent::Run), Type = "purge_account" ); let server = server.clone(); queue.schedule( Instant::now() + server.core.jmap.account_purge_frequency.time_to_next(), ActionClass::Account, ); tokio::spawn(async move { server .purge( PurgeType::Account { account_id: None, use_roles: true, }, 0, ) .await; }); } ActionClass::Store(idx) => { if let Some(schedule) = server.core.storage.purge_schedules.get(idx).cloned() { trc::event!( Housekeeper(trc::HousekeeperEvent::Run), Type = "purge_store", Id = idx ); queue.schedule( Instant::now() + schedule.cron.time_to_next(), ActionClass::Store(idx), ); let server = server.clone(); tokio::spawn(async move { server .purge( match schedule.store { PurgeStore::Data(store) => { PurgeType::Data(store) } PurgeStore::Blobs { store, blob_store } => { PurgeType::Blobs { store, blob_store } } PurgeStore::Lookup(in_memory_store) => { PurgeType::Lookup { store: in_memory_store, prefix: None, } } }, idx as u32, ) .await; }); } } ActionClass::OtelMetrics => { if let Some(otel) = &server.core.metrics.otel { trc::event!( Housekeeper(trc::HousekeeperEvent::Run), Type = "metrics_report" ); queue.schedule( Instant::now() + otel.interval, ActionClass::OtelMetrics, ); let otel = otel.clone(); // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] let is_enterprise = server.is_enterprise_edition(); // SPDX-SnippetEnd #[cfg(not(feature = "enterprise"))] let is_enterprise = false; tokio::spawn(async move { otel.push_metrics(is_enterprise, start_time).await; }); } } ActionClass::CalculateMetrics => { trc::event!( Housekeeper(trc::HousekeeperEvent::Run), Type = "metrics_calculate" ); // Calculate expensive metrics every 5 minutes queue.schedule( Instant::now() + Duration::from_secs(5 * 60), ActionClass::CalculateMetrics, ); let update_other_metrics = if Instant::now() >= next_metric_update { next_metric_update = Instant::now() + Duration::from_secs(86400); true } else { false }; let server = server.clone(); tokio::spawn(async move { if server .core .network .roles .calculate_metrics .is_enabled_or_sharded() { // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] if server.is_enterprise_edition() { // Obtain queue size match server.total_queued_messages().await { Ok(total) => { Collector::update_gauge( MetricType::QueueCount, total, ); } Err(err) => { trc::error!( err.details("Failed to obtain queue size") ); } } } // SPDX-SnippetEnd if update_other_metrics { match server.total_accounts().await { Ok(total) => { Collector::update_gauge( MetricType::UserCount, total, ); } Err(err) => { trc::error!( err.details( "Failed to obtain account count" ) ); } } match server.total_domains().await { Ok(total) => { Collector::update_gauge( MetricType::DomainCount, total, ); } Err(err) => { trc::error!( err.details( "Failed to obtain domain count" ) ); } } } } match tokio::task::spawn_blocking(memory_stats::memory_stats) .await { Ok(Some(stats)) => { Collector::update_gauge( MetricType::ServerMemory, stats.physical_mem as u64, ); } Ok(None) => {} Err(err) => { trc::error!( trc::EventType::Server( trc::ServerEvent::ThreadError, ) .reason(err) .caused_by(trc::location!()) .details("Join Error") ); } } }); } ActionClass::TrainSpamClassifier => { if server .core .network .roles .spam_training .is_enabled_or_sharded() && let Some(train_frequency) = server .core .spam .classifier .as_ref() .and_then(|c| c.train_frequency) { trc::event!( Housekeeper(trc::HousekeeperEvent::Run), Type = "spam_classifier_train" ); // Schedule next training queue.schedule( Instant::now() + Duration::from_secs(train_frequency), ActionClass::TrainSpamClassifier, ); let server = server.clone(); tokio::spawn(async move { if let Err(err) = server.spam_train(false).await { trc::error!( err.details("Failed to train spam classifier") ); } }); } } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] ActionClass::InternalMetrics => { if let Some(metrics_store) = &server .core .enterprise .as_ref() .and_then(|e| e.metrics_store.as_ref()) { trc::event!( Housekeeper(trc::HousekeeperEvent::Run), Type = "metrics_internal" ); queue.schedule( Instant::now() + metrics_store.interval.time_to_next(), ActionClass::InternalMetrics, ); let metrics_store = metrics_store.store.clone(); let metrics_history = metrics_history.clone(); let core = server.core.clone(); tokio::spawn(async move { if let Err(err) = metrics_store .write_metrics(core, now(), metrics_history) .await { trc::error!(err.details("Failed to write metrics")); } }); } } #[cfg(feature = "enterprise")] ActionClass::AlertMetrics => { trc::event!( Housekeeper(trc::HousekeeperEvent::Run), Type = "metrics_alert" ); let server = server.clone(); tokio::spawn(async move { if let Some(messages) = server.process_alerts().await { for message in messages { server .send_autogenerated( message.from, message.to.into_iter(), message.body, None, 0, ) .await; } } }); } #[cfg(feature = "enterprise")] ActionClass::RenewLicense => { trc::event!( Housekeeper(trc::HousekeeperEvent::Run), Type = "renew_license" ); match server.reload().await { Ok(result) => { if let Some(new_core) = result.new_core { if let Some(enterprise) = &new_core.enterprise { let renew_in = if enterprise.license.is_near_expiration() { // Something went wrong during renewal, try again in 1 day or 1 hour, // depending on the time left on the license if enterprise.license.expires_in() < Duration::from_secs(86400) { Duration::from_secs(3600) } else { Duration::from_secs(86400) } } else { enterprise.license.renew_in() }; queue.schedule( Instant::now() + renew_in, ActionClass::RenewLicense, ); } // Update core server.inner.shared_core.store(new_core.into()); server .cluster_broadcast(BroadcastEvent::ReloadSettings) .await; } } Err(err) => { trc::error!(err.details("Failed to reload configuration.")); } } } // SPDX-SnippetEnd } } } } } }); } pub trait Purge: Sync + Send { fn purge(&self, purge: PurgeType, store_idx: u32) -> impl Future + Send; } impl Purge for Server { async fn purge(&self, purge: PurgeType, store_idx: u32) { // Lock task let (lock_type, lock_name) = match &purge { PurgeType::Data(_) => ( "data", [0u8] .into_iter() .chain(store_idx.to_be_bytes().into_iter()) .collect::>() .into(), ), PurgeType::Blobs { .. } => ( "blob", [1u8] .into_iter() .chain(store_idx.to_be_bytes().into_iter()) .collect::>() .into(), ), PurgeType::Lookup { prefix: None, .. } => ( "in-memory", [2u8] .into_iter() .chain(store_idx.to_be_bytes().into_iter()) .collect::>() .into(), ), PurgeType::Lookup { .. } => ("in-memory-prefix", None), PurgeType::Account { .. } => ("account", None), }; if let Some(lock_name) = &lock_name { match self .core .storage .lookup .try_lock(KV_LOCK_HOUSEKEEPER, lock_name, 3600) .await { Ok(true) => (), Ok(false) => { trc::event!(Purge(PurgeEvent::InProgress), Details = lock_type); return; } Err(err) => { trc::error!(err.details("Failed to lock task.").details(lock_type)); return; } } } trc::event!(Purge(PurgeEvent::Started), Type = lock_type, Id = store_idx); let time = Instant::now(); match purge { PurgeType::Data(store) => { // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] let trace_retention = self .core .enterprise .as_ref() .and_then(|e| e.trace_store.as_ref()) .and_then(|t| t.retention); #[cfg(feature = "enterprise")] let metrics_retention = self .core .enterprise .as_ref() .and_then(|e| e.metrics_store.as_ref()) .and_then(|m| m.retention); // SPDX-SnippetEnd if let Err(err) = store.purge_store().await { trc::error!(err.details("Failed to purge data store")); } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] if let Some(trace_retention) = trace_retention && let Some(trace_store) = self .core .enterprise .as_ref() .and_then(|e| e.trace_store.as_ref()) && let Err(err) = trace_store .store .purge_spans(trace_retention, self.search_store().into()) .await { trc::error!(err.details("Failed to purge tracing spans")); } #[cfg(feature = "enterprise")] if let Some(metrics_retention) = metrics_retention && let Some(metrics_store) = self .core .enterprise .as_ref() .and_then(|e| e.metrics_store.as_ref()) && let Err(err) = metrics_store.store.purge_metrics(metrics_retention).await { trc::error!(err.details("Failed to purge metrics")); } // SPDX-SnippetEnd } PurgeType::Blobs { store, blob_store } => { if let Err(err) = store.purge_blobs(blob_store).await { trc::error!(err.details("Failed to purge blob store")); } } PurgeType::Lookup { store, prefix } => { if let Some(prefix) = prefix { if let Err(err) = store.key_delete_prefix(&prefix).await { trc::error!( err.details("Failed to delete key prefix") .ctx(trc::Key::Key, prefix) ); } } else if let Err(err) = store.purge_in_memory_store().await { trc::error!(err.details("Failed to purge in-memory store")); } } PurgeType::Account { account_id, use_roles, } => { if let Some(account_id) = account_id { self.purge_account(account_id).await; } else { self.purge_accounts(use_roles).await; } } } trc::event!( Purge(PurgeEvent::Finished), Type = lock_type, Id = store_idx, Elapsed = time.elapsed() ); // Remove lock if let Some(lock_name) = &lock_name && let Err(err) = self .in_memory_store() .remove_lock(KV_LOCK_HOUSEKEEPER, lock_name) .await { trc::error!( err.details("Failed to delete task lock.") .details(lock_type) ); } } } impl Queue { pub fn schedule(&mut self, due: Instant, event: ActionClass) { trc::event!( Housekeeper(trc::HousekeeperEvent::Schedule), Due = trc::Value::Timestamp( now() + due.saturating_duration_since(Instant::now()).as_secs() ), Id = format!("{:?}", event) ); self.heap.push(Action { due, event }); } pub fn remove_action(&mut self, event: &ActionClass) { self.heap.retain(|e| &e.event != event); } pub fn wake_up_time(&self) -> Duration { self.heap .peek() .map(|e| e.due.saturating_duration_since(Instant::now())) .unwrap_or(LONG_1D_SLUMBER) } pub fn pop(&mut self) -> Option { if self.heap.peek()?.due <= Instant::now() { self.heap.pop() } else { None } } pub fn has_action(&self, event: &ActionClass) -> bool { self.heap.iter().any(|e| &e.event == event) } } impl Ord for Action { fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.due.cmp(&other.due).reverse() } } impl PartialOrd for Action { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } ================================================ FILE: crates/services/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use broadcast::publisher::spawn_broadcast_publisher; use common::{ Inner, manager::boot::{BootManager, IpcReceivers}, }; use housekeeper::spawn_housekeeper; use state_manager::manager::spawn_push_router; use std::sync::Arc; use task_manager::spawn_task_manager; pub mod broadcast; pub mod housekeeper; pub mod state_manager; pub mod task_manager; pub trait StartServices: Sync + Send { fn start_services(&mut self) -> impl Future + Send; } pub trait SpawnServices { fn spawn_services(&mut self, inner: Arc); } impl StartServices for BootManager { async fn start_services(&mut self) { // Unpack webadmin if let Err(err) = self .inner .data .webadmin .unpack(&self.inner.shared_core.load().storage.blob) .await { trc::event!( Resource(trc::ResourceEvent::Error), Reason = err, Details = "Failed to unpack webadmin bundle" ); } self.ipc_rxs.spawn_services(self.inner.clone()); } } impl SpawnServices for IpcReceivers { fn spawn_services(&mut self, inner: Arc) { // Spawn push manager spawn_push_router(inner.clone(), self.push_rx.take().unwrap()); // Spawn housekeeper spawn_housekeeper(inner.clone(), self.housekeeper_rx.take().unwrap()); // Spawn broadcast publisher if let Some(event_rx) = self.broadcast_rx.take() { // Spawn broadcast publisher spawn_broadcast_publisher(inner.clone(), event_rx); } // Spawn task manager spawn_task_manager(inner); } } ================================================ FILE: crates/services/src/state_manager/ece.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use aes_gcm::{Aes128Gcm, Nonce, aead::Aead}; use hkdf::Hkdf; use p256::{ PublicKey, ecdh::EphemeralSecret, elliptic_curve::{rand_core::OsRng, sec1::ToEncodedPoint}, }; use sha2::Sha256; use store::rand::Rng; /* From https://github.com/mozilla/rust-ece (MPL-2.0 license) Adapted to use 'aes-gcm' and 'p256' crates instead of 'openssl'. */ const ECE_WEBPUSH_AES128GCM_IKM_INFO_PREFIX: &str = "WebPush: info\0"; const ECE_WEBPUSH_AES128GCM_IKM_INFO_LENGTH: usize = 144; const ECE_WEBPUSH_IKM_LENGTH: usize = 32; const ECE_WEBPUSH_PUBLIC_KEY_LENGTH: usize = 65; const ECE_WEBPUSH_DEFAULT_RS: u32 = 4096; const ECE_WEBPUSH_DEFAULT_PADDING_BLOCK_SIZE: usize = 128; const ECE_AES128GCM_PAD_SIZE: usize = 1; const ECE_AES128GCM_KEY_INFO: &str = "Content-Encoding: aes128gcm\0"; const ECE_AES128GCM_NONCE_INFO: &str = "Content-Encoding: nonce\0"; const ECE_AES128GCM_HEADER_LENGTH: usize = 21; const ECE_AES_KEY_LENGTH: usize = 16; const ECE_NONCE_LENGTH: usize = 12; const ECE_TAG_LENGTH: usize = 16; pub fn ece_encrypt( p256dh: &[u8], client_auth_secret: &[u8], mut data: &[u8], ) -> Result, String> { let salt = store::rand::rng().random::<[u8; 16]>(); let server_secret = EphemeralSecret::random(&mut OsRng); let server_public_key = server_secret.public_key(); let server_public_key_bytes = server_public_key.to_encoded_point(false); let client_public_key = PublicKey::from_sec1_bytes(p256dh).map_err(|e| e.to_string())?; let shared_secret = server_secret.diffie_hellman(&client_public_key); let ikm_info = generate_info(p256dh, server_public_key_bytes.as_bytes()); let ikm = hkdf_sha256( client_auth_secret, &shared_secret.raw_secret_bytes()[..], &ikm_info, ECE_WEBPUSH_IKM_LENGTH, )?; let key = hkdf_sha256( &salt, &ikm, ECE_AES128GCM_KEY_INFO.as_bytes(), ECE_AES_KEY_LENGTH, )?; let nonce = hkdf_sha256( &salt, &ikm, ECE_AES128GCM_NONCE_INFO.as_bytes(), ECE_NONCE_LENGTH, )?; // Calculate pad length let mut pad_length = ECE_WEBPUSH_DEFAULT_PADDING_BLOCK_SIZE - (data.len() % ECE_WEBPUSH_DEFAULT_PADDING_BLOCK_SIZE); if pad_length < ECE_AES128GCM_PAD_SIZE { pad_length += ECE_WEBPUSH_DEFAULT_PADDING_BLOCK_SIZE; } // Split into records let rs = ECE_WEBPUSH_DEFAULT_RS as usize - ECE_TAG_LENGTH; let mut min_num_records = data.len() / (rs - 1); if !data.len().is_multiple_of(rs - 1) { min_num_records += 1; } let mut pad_length = std::cmp::max(pad_length, min_num_records); let total_size = data.len() + pad_length; let mut num_records = total_size / rs; let size_of_final_record = total_size % rs; if size_of_final_record > 0 { num_records += 1; } let data_per_record = data.len() / num_records; let mut extra_data = data.len() % num_records; if size_of_final_record > 0 && data_per_record > size_of_final_record - 1 { extra_data += data_per_record - (size_of_final_record - 1) } let mut sequence_number = 0; let mut plain_text = Vec::with_capacity(data_per_record + ECE_WEBPUSH_DEFAULT_PADDING_BLOCK_SIZE); // Write header let key_id = server_public_key_bytes.as_bytes(); debug_assert_eq!(key_id.len(), ECE_WEBPUSH_PUBLIC_KEY_LENGTH); let mut output = Vec::with_capacity( ECE_AES128GCM_HEADER_LENGTH + key_id.len() + total_size + num_records * ECE_TAG_LENGTH, ); output.extend_from_slice(&salt); output.extend_from_slice(&ECE_WEBPUSH_DEFAULT_RS.to_be_bytes()); output.push(key_id.len() as u8); output.extend_from_slice(key_id); loop { let records_remaining = num_records - sequence_number; if records_remaining == 0 { break; } let mut data_share = data_per_record; if data_share > data.len() { data_share = data.len(); } else if extra_data > 0 { let mut extra_share = extra_data / (records_remaining - 1); if !extra_data.is_multiple_of(records_remaining - 1) { extra_share += 1; } data_share += extra_share; extra_data -= extra_share; } let cur_data = &data[0..data_share]; data = &data[data_share..]; let padding = std::cmp::min(pad_length, rs - data_share); pad_length -= padding; let cur_sequence_number = sequence_number; sequence_number += 1; let padded_plaintext_len = cur_data.len() + padding; plain_text.extend_from_slice(cur_data); plain_text.push(if sequence_number == num_records { 2 } else { 1 }); plain_text.resize(padded_plaintext_len, 0); output.extend_from_slice(&aes_gcm_128_encrypt( &key, &generate_iv(&nonce, cur_sequence_number), &plain_text, )?); plain_text.clear(); } Ok(output) } fn hkdf_sha256(salt: &[u8], secret: &[u8], info: &[u8], len: usize) -> Result, String> { let (_, hk) = Hkdf::::extract(Some(salt), secret); let mut okm = vec![0u8; len]; hk.expand(info, &mut okm).map_err(|e| e.to_string())?; Ok(okm) } // TODO: Remove allow deprecated when aes-gcm 0.10 is updated #[allow(deprecated)] fn aes_gcm_128_encrypt(key: &[u8], nonce: &[u8], data: &[u8]) -> Result, String> { ::new( &sha2::digest::generic_array::GenericArray::clone_from_slice(key), ) .encrypt(Nonce::from_slice(nonce), data) .map_err(|e| e.to_string()) } fn generate_info( client_public_key: &[u8], server_public_key: &[u8], ) -> [u8; ECE_WEBPUSH_AES128GCM_IKM_INFO_LENGTH] { let mut info = [0u8; ECE_WEBPUSH_AES128GCM_IKM_INFO_LENGTH]; let prefix = ECE_WEBPUSH_AES128GCM_IKM_INFO_PREFIX.as_bytes(); let mut offset = prefix.len(); info[0..offset].copy_from_slice(prefix); info[offset..offset + ECE_WEBPUSH_PUBLIC_KEY_LENGTH].copy_from_slice(client_public_key); offset += ECE_WEBPUSH_PUBLIC_KEY_LENGTH; info[offset..].copy_from_slice(server_public_key); info } pub fn generate_iv(nonce: &[u8], counter: usize) -> [u8; ECE_NONCE_LENGTH] { let mut iv = [0u8; ECE_NONCE_LENGTH]; let offset = ECE_NONCE_LENGTH - 8; iv[0..offset].copy_from_slice(&nonce[0..offset]); let mask = u64::from_be_bytes((&nonce[offset..]).try_into().unwrap()); iv[offset..].copy_from_slice(&(mask ^ (counter as u64)).to_be_bytes()); iv } ================================================ FILE: crates/services/src/state_manager/http.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{Event, ece::ece_encrypt}; use crate::state_manager::PushRegistration; use base64::Engine; use calcard::jscalendar::JSCalendarDateTime; use common::ipc::PushNotification; use email::push::PushSubscription; use jmap_proto::{ response::status::{EmailPushObject, PushObject}, types::state::State, }; use reqwest::header::{CONTENT_ENCODING, CONTENT_TYPE}; use std::time::{Duration, Instant}; use tokio::sync::mpsc; use trc::PushSubscriptionEvent; use types::{id::Id, type_state::DataType}; use utils::map::vec_map::VecMap; impl PushRegistration { pub fn send(&mut self, id: Id, push_tx: mpsc::Sender, push_timeout: Duration) { let server = self.server.clone(); let notifications = std::mem::take(&mut self.notifications); self.in_flight = true; self.last_request = Instant::now(); tokio::spawn(async move { let mut changed: VecMap> = VecMap::new(); let mut objects = Vec::with_capacity(notifications.len()); for notification in ¬ifications { match notification { PushNotification::StateChange(state_change) => { for type_state in state_change.types { changed .get_mut_or_insert(state_change.account_id.into()) .set(type_state, (state_change.change_id).into()); } } PushNotification::CalendarAlert(calendar_alert) => { objects.push(PushObject::CalendarAlert { account_id: calendar_alert.account_id.into(), calendar_event_id: calendar_alert.event_id.into(), uid: calendar_alert.uid.clone(), recurrence_id: calendar_alert.recurrence_id.map(|timestamp| { JSCalendarDateTime::new(timestamp, true).to_rfc3339() }), alert_id: calendar_alert.alert_id.clone(), }); } PushNotification::EmailPush(email_push) => { objects.push(PushObject::EmailPush { account_id: email_push.account_id.into(), email: EmailPushObject { subject: Default::default(), }, }); } } } let response = if !objects.is_empty() { if changed.is_empty() { objects.push(PushObject::StateChange { changed }); } if objects.len() > 1 { PushObject::Group { entries: objects } } else { objects.into_iter().next().unwrap() } } else { PushObject::StateChange { changed } }; push_tx .send( if http_request( &server, serde_json::to_string(&response).unwrap(), push_timeout, ) .await { Event::DeliverySuccess { id } } else { Event::DeliveryFailure { id, notifications } }, ) .await .ok(); }); } } pub(crate) async fn http_request( details: &PushSubscription, mut body: String, push_timeout: Duration, ) -> bool { let client_builder = reqwest::Client::builder().timeout(push_timeout); #[cfg(feature = "test_mode")] let client_builder = client_builder.danger_accept_invalid_certs(true); let mut client = client_builder .build() .unwrap_or_default() .post(details.url.as_str()) .header(CONTENT_TYPE, "application/json") .header("TTL", "86400"); if let Some(keys) = &details.keys { match ece_encrypt(&keys.p256dh, &keys.auth, body.as_bytes()) .map(|b| base64::engine::general_purpose::URL_SAFE.encode(b)) { Ok(body_) => { body = body_; client = client.header(CONTENT_ENCODING, "aes128gcm"); } Err(err) => { // Do not reattempt if encryption fails. trc::event!( PushSubscription(PushSubscriptionEvent::Error), Details = "Failed to encrypt push subscription", Url = details.url.to_string(), Reason = err ); return true; } } } match client.body(body).send().await { Ok(response) => { if response.status().is_success() { trc::event!( PushSubscription(PushSubscriptionEvent::Success), Url = details.url.to_string() ); true } else { trc::event!( PushSubscription(PushSubscriptionEvent::Error), Details = "HTTP POST failed", Url = details.url.to_string(), Code = response.status().as_u16(), ); false } } Err(err) => { trc::event!( PushSubscription(PushSubscriptionEvent::Error), Details = "HTTP POST failed", Url = details.url.to_string(), Reason = err.to_string() ); false } } } ================================================ FILE: crates/services/src/state_manager/manager.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{Event, PURGE_EVERY, SEND_TIMEOUT, push::spawn_push_manager}; use crate::state_manager::IpcSubscriber; use common::{ Inner, ipc::{BroadcastEvent, PushEvent}, }; use std::{sync::Arc, time::Instant}; use store::ahash::AHashMap; use tokio::sync::mpsc; use trc::ServerEvent; #[derive(Default)] struct Subscriber { ipc: Vec, is_push: bool, } #[allow(clippy::unwrap_or_default)] pub fn spawn_push_router(inner: Arc, mut change_rx: mpsc::Receiver) { let push_tx = spawn_push_manager(inner.clone()); tokio::spawn(async move { let mut subscribers: AHashMap = AHashMap::default(); let mut last_purge = Instant::now(); while let Some(event) = change_rx.recv().await { let mut purge_needed = last_purge.elapsed() >= PURGE_EVERY; match event { PushEvent::Stop => { if push_tx.send(Event::Reset).await.is_err() { trc::event!( Server(ServerEvent::ThreadError), Details = "Error sending push reset.", CausedBy = trc::location!() ); } break; } PushEvent::Subscribe { account_ids, types, tx, } => { for account_id in account_ids { subscribers .entry(account_id) .or_default() .ipc .push(IpcSubscriber { types, tx: tx.clone(), }); } } PushEvent::PushServerRegister { activate, expired } => { for account_id in activate { subscribers.entry(account_id).or_default().is_push = true; } for account_id in expired { let mut remove_account = false; if let Some(subscriber_list) = subscribers.get_mut(&account_id) { subscriber_list.is_push = false; remove_account = subscriber_list.ipc.is_empty(); } if remove_account { subscribers.remove(&account_id); } } } PushEvent::Publish { notification, broadcast, } => { // Publish event to cluster if broadcast && let Some(broadcast_tx) = &inner.ipc.broadcast_tx.clone() && broadcast_tx .send(BroadcastEvent::PushNotification(notification.clone())) .await .is_err() { trc::event!( Server(trc::ServerEvent::ThreadError), Details = "Error sending broadcast event.", CausedBy = trc::location!() ); } let account_id = notification.account_id(); if let Some(subscribers) = subscribers.get(&account_id) { for subscriber in &subscribers.ipc { if let Some(notification) = notification.filter_types(&subscriber.types) { if subscriber.is_valid() { let subscriber_tx = subscriber.tx.clone(); tokio::spawn(async move { // Timeout after 500ms in case there is a blocked client if subscriber_tx .send_timeout(notification, SEND_TIMEOUT) .await .is_err() { trc::event!( Server(ServerEvent::ThreadError), Details = "Error sending state change to subscriber.", CausedBy = trc::location!() ); } }); } else { purge_needed = true; } } } if subscribers.is_push && push_tx.send(Event::Push { notification }).await.is_err() { trc::event!( Server(ServerEvent::ThreadError), Details = "Error sending push updates.", CausedBy = trc::location!() ); } } } PushEvent::PushServerUpdate { account_id, broadcast, } => { // Publish event to cluster if broadcast && let Some(broadcast_tx) = &inner.ipc.broadcast_tx.clone() && broadcast_tx .send(BroadcastEvent::ReloadPushServers(account_id)) .await .is_err() { trc::event!( Server(trc::ServerEvent::ThreadError), Details = "Error sending broadcast event.", CausedBy = trc::location!() ); } // Notify push manager if push_tx.send(Event::Update { account_id }).await.is_err() { trc::event!( Server(ServerEvent::ThreadError), Details = "Error sending push updates.", CausedBy = trc::location!() ); } } } if purge_needed { let mut remove_account_ids = Vec::new(); for (account_id, subscribers) in &mut subscribers { subscribers.ipc.retain(|subscriber| subscriber.is_valid()); if subscribers.ipc.is_empty() && !subscribers.is_push { remove_account_ids.push(*account_id); } } for remove_account_id in remove_account_ids { subscribers.remove(&remove_account_id); } last_purge = Instant::now(); } } }); } ================================================ FILE: crates/services/src/state_manager/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod ece; pub mod http; pub mod manager; pub mod push; use common::ipc::PushNotification; use email::push::PushSubscription; use std::{ sync::Arc, time::{Duration, Instant}, }; use tokio::sync::mpsc; use types::{id::Id, type_state::DataType}; use utils::map::bitmap::Bitmap; const PURGE_EVERY: Duration = Duration::from_secs(3600); const SEND_TIMEOUT: Duration = Duration::from_millis(500); #[derive(Debug)] struct IpcSubscriber { types: Bitmap, tx: mpsc::Sender, } #[derive(Debug)] pub struct PushRegistration { server: Arc, member_account_ids: Vec, num_attempts: u32, last_request: Instant, notifications: Vec, in_flight: bool, } #[derive(Debug)] pub enum Event { Push { notification: PushNotification, }, Update { account_id: u32, }, DeliverySuccess { id: Id, }, DeliveryFailure { id: Id, notifications: Vec, }, Reset, } impl IpcSubscriber { fn is_valid(&self) -> bool { !self.tx.is_closed() } } ================================================ FILE: crates/services/src/state_manager/push.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{Event, http::http_request}; use crate::state_manager::PushRegistration; use common::{ IPC_CHANNEL_BUFFER, Inner, LONG_1Y_SLUMBER, Server, core::BuildServer, ipc::{PushEvent, PushNotification}, }; use email::push::PushSubscriptions; use std::{ collections::hash_map::Entry, sync::Arc, time::{Duration, Instant}, }; use store::{ ValueKey, ahash::{AHashMap, AHashSet}, write::{AlignedBytes, Archive, now}, }; use tokio::sync::mpsc; use trc::{AddContext, PushSubscriptionEvent, ServerEvent}; use types::{collection::Collection, field::PrincipalField, id::Id}; pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { let (push_tx_, mut push_rx) = mpsc::channel::(IPC_CHANNEL_BUFFER); let push_tx = push_tx_.clone(); tokio::spawn(async move {}); tokio::spawn(async move { let mut push_servers: AHashMap = AHashMap::default(); let mut account_push_ids: AHashMap> = AHashMap::default(); let mut last_verify: AHashMap = AHashMap::default(); let mut last_retry = Instant::now(); let mut retry_timeout = LONG_1Y_SLUMBER; let mut retry_ids = AHashSet::default(); // Load active subscriptions on startup { let server = inner.build_server(); match server .document_ids( u32::MAX, Collection::Principal, PrincipalField::PushSubscriptions, ) .await { Ok(account_ids) => { for account_id in account_ids { if server .core .network .roles .push_notifications .is_enabled_for_integer(account_id) { // Load push subscriptions for account let (subscriptions, member_account_ids) = match load_push_subscriptions(&server, account_id).await { Ok(subscriptions) => subscriptions, Err(err) => { trc::error!(err.caused_by(trc::location!())); continue; } }; let current_time = now(); for subscription in subscriptions .subscriptions .into_iter() .filter(|s| s.verified && s.expires > current_time) { let id = Id::from_parts(subscription.id, account_id); let subscription = Arc::new(subscription); for account_id in &member_account_ids { account_push_ids.entry(*account_id).or_default().insert(id); } push_servers.insert( id, PushRegistration { member_account_ids: member_account_ids.clone(), num_attempts: 0, last_request: Instant::now() - (server.core.jmap.push_throttle + Duration::from_millis(1)), notifications: Vec::new(), server: subscription.clone(), in_flight: false, }, ); } } } } Err(err) => { trc::error!(err.caused_by(trc::location!())); } } // Subscribe to push events if !account_push_ids.is_empty() && server .inner .ipc .push_tx .clone() .send(PushEvent::PushServerRegister { activate: account_push_ids.keys().copied().collect(), expired: vec![], }) .await .is_err() { trc::event!( Server(ServerEvent::ThreadError), Details = "Error sending state change.", CausedBy = trc::location!() ); } } loop { // Wait for the next event or timeout let event_or_timeout = tokio::time::timeout(retry_timeout, push_rx.recv()).await; // Load settings let server = inner.build_server(); let push_attempt_interval = server.core.jmap.push_attempt_interval; let push_attempts_max = server.core.jmap.push_attempts_max; let push_retry_interval = server.core.jmap.push_retry_interval; let push_timeout = server.core.jmap.push_timeout; let push_verify_timeout = server.core.jmap.push_verify_timeout; let push_throttle = server.core.jmap.push_throttle; match event_or_timeout { Ok(Some(event)) => match event { Event::Update { account_id } => { if !server .core .network .roles .push_notifications .is_enabled_for_integer(account_id) { continue; } // Load push subscriptions for account let (subscriptions, member_account_ids) = match load_push_subscriptions(&server, account_id).await { Ok(subscriptions) => subscriptions, Err(err) => { trc::error!(err.caused_by(trc::location!())); continue; } }; let old_account_push_ids = account_push_ids .remove(&account_id) .filter(|v| !v.is_empty()); // Process subscriptions let current_time = now(); for subscription in subscriptions .subscriptions .into_iter() .filter(|s| s.expires > current_time) { let id = Id::from_parts(subscription.id, account_id); let subscription = Arc::new(subscription); if subscription.verified { for account_id in &member_account_ids { account_push_ids.entry(*account_id).or_default().insert(id); } match push_servers.entry(id) { Entry::Occupied(mut entry) => { // Update existing subscription let entry = entry.get_mut(); entry.server = subscription.clone(); entry.member_account_ids = member_account_ids.clone(); } Entry::Vacant(entry) => { entry.insert(PushRegistration { member_account_ids: member_account_ids.clone(), num_attempts: 0, last_request: Instant::now() - (push_throttle + Duration::from_millis(1)), notifications: Vec::new(), server: subscription.clone(), in_flight: false, }); } } } else { let current_time = Instant::now(); #[cfg(feature = "test_mode")] if subscription.url.contains("skip_checks") { last_verify.insert( account_id, current_time - (push_verify_timeout + Duration::from_millis(1)), ); } if last_verify .get(&account_id) .map(|last_verify| { current_time - *last_verify > push_verify_timeout }) .unwrap_or(true) { tokio::spawn(async move { http_request( &subscription, format!( concat!( "{{\"@type\":\"PushVerification\",", "\"pushSubscriptionId\":\"{}\",", "\"verificationCode\":\"{}\"}}" ), Id::from(subscription.id), subscription.verification_code ), push_timeout, ) .await; }); last_verify.insert(account_id, current_time); } else { trc::event!( PushSubscription(PushSubscriptionEvent::Error), Details = "Failed to verify push subscription", Url = subscription.url.clone(), AccountId = account_id, Reason = "Too many requests" ); continue; } } } // Update subscriptions let mut remove_push_ids = AHashSet::new(); let mut active_account_ids = Vec::new(); let mut inactive_account_ids = Vec::new(); match (old_account_push_ids, account_push_ids.get(&account_id)) { (Some(old), Some(current)) if &old != current => { for id in old.difference(current) { remove_push_ids.insert(*id); } active_account_ids = member_account_ids; } (Some(old), None) => { remove_push_ids = old; } (None, Some(_)) => { active_account_ids = member_account_ids; } _ => {} } // Update push server registrations if !remove_push_ids.is_empty() { for id in remove_push_ids { if let Some(subscription) = push_servers.remove(&id) { for account_id in &subscription.member_account_ids { if let Some(ids) = account_push_ids.get_mut(account_id) { ids.remove(&id); if ids.is_empty() { account_push_ids.remove(account_id); inactive_account_ids.push(*account_id); } } } } } } if (!active_account_ids.is_empty() || !inactive_account_ids.is_empty()) && server .inner .ipc .push_tx .clone() .send(PushEvent::PushServerRegister { activate: active_account_ids, expired: inactive_account_ids, }) .await .is_err() { trc::event!( Server(ServerEvent::ThreadError), Details = "Error sending state change.", CausedBy = trc::location!() ); } } Event::Push { notification } => { let account_id = notification.account_id(); if let Some(ids) = account_push_ids.get_mut(&account_id) { let current_time = now(); let mut remove_ids = Vec::new(); for id in ids.iter() { if let Some(subscription) = push_servers.get_mut(id) { if subscription.server.expires > current_time { if let Some(mut notification) = notification.filter_types(&subscription.server.types) { // Build email push notification if let PushNotification::EmailPush(email_push) = ¬ification { if let Some(_email_push) = subscription .server .email_push .iter() .find(|ep| ep.account_id == account_id) { // TODO: Apply filters once RFC is finalized } else { notification = PushNotification::StateChange( email_push.to_state_change(), ); } } subscription.notifications.push(notification); let last_request = subscription.last_request.elapsed(); if !subscription.in_flight && ((subscription.num_attempts == 0 && last_request > push_throttle) || ((1..push_attempts_max) .contains(&subscription.num_attempts) && last_request > push_attempt_interval)) { subscription.send( *id, push_tx.clone(), push_timeout, ); retry_ids.remove(id); } else { retry_ids.insert(*id); } } } else { push_servers.remove(id); } } else { remove_ids.push(*id); } } if !remove_ids.is_empty() { for remove_id in remove_ids { ids.remove(&remove_id); } if ids.is_empty() { account_push_ids.remove(&account_id); if server .inner .ipc .push_tx .clone() .send(PushEvent::PushServerRegister { activate: vec![], expired: vec![account_id], }) .await .is_err() { trc::event!( Server(ServerEvent::ThreadError), Details = "Error sending state change.", CausedBy = trc::location!() ); } } } } } Event::Reset => { push_servers.clear(); account_push_ids.clear(); } Event::DeliverySuccess { id } => { if let Some(subscription) = push_servers.get_mut(&id) { subscription.num_attempts = 0; subscription.in_flight = false; retry_ids.remove(&id); } } Event::DeliveryFailure { id, notifications } => { if let Some(subscription) = push_servers.get_mut(&id) { subscription.last_request = Instant::now(); subscription.num_attempts += 1; subscription.notifications.extend(notifications); subscription.in_flight = false; retry_ids.insert(id); } } }, Ok(None) => { break; } Err(_) => (), } retry_timeout = if !retry_ids.is_empty() { let last_retry_elapsed = last_retry.elapsed(); if last_retry_elapsed >= push_retry_interval { let mut remove_ids = Vec::with_capacity(retry_ids.len()); for retry_id in &retry_ids { if let Some(subscription) = push_servers.get_mut(retry_id) { let last_request = subscription.last_request.elapsed(); if !subscription.in_flight && ((subscription.num_attempts == 0 && last_request >= push_throttle) || (subscription.num_attempts > 0 && last_request >= push_attempt_interval)) { if subscription.num_attempts < push_attempts_max { subscription.send(*retry_id, push_tx.clone(), push_timeout); } else { trc::event!( PushSubscription(PushSubscriptionEvent::Error), Details = "Failed to deliver push subscription", Url = subscription.server.url.clone(), Reason = "Too many failed attempts" ); subscription.notifications.clear(); subscription.num_attempts = 0; } remove_ids.push(*retry_id); } } else { remove_ids.push(*retry_id); } } if remove_ids.len() < retry_ids.len() { for remove_id in remove_ids { retry_ids.remove(&remove_id); } last_retry = Instant::now(); push_retry_interval } else { retry_ids.clear(); LONG_1Y_SLUMBER } } else { push_retry_interval - last_retry_elapsed } } else { LONG_1Y_SLUMBER }; } }); push_tx_ } async fn load_push_subscriptions( server: &Server, account_id: u32, ) -> trc::Result<(PushSubscriptions, Vec)> { let member_of = server .get_access_token(account_id) .await .caused_by(trc::location!())? .member_ids() .collect::>(); if let Some(push_subscriptions) = server .store() .get_value::>(ValueKey::property( account_id, Collection::Principal, 0, PrincipalField::PushSubscriptions, )) .await? { push_subscriptions .deserialize::() .map(|push_subscriptions| (push_subscriptions, member_of)) .caused_by(trc::location!()) } else { Ok((PushSubscriptions::default(), member_of)) } } ================================================ FILE: crates/services/src/task_manager/alarm.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use calcard::{ common::timezone::Tz, icalendar::{ArchivedICalendarParameterName, ArchivedICalendarProperty, ICalendarProperty}, }; use chrono::{DateTime, Locale}; use common::{ DEFAULT_LOGO_BASE64, Server, auth::AccessToken, config::groupware::CalendarTemplateVariable, i18n, ipc::{CalendarAlert, PushNotification}, listener::{ServerInstance, stream::NullIo}, }; use directory::Permission; use groupware::calendar::{ ArchivedCalendarEvent, CalendarEvent, alarm::{CalendarAlarm, CalendarAlarmType}, }; use mail_builder::{ MessageBuilder, headers::{HeaderType, content_type::ContentType}, mime::{BodyPart, MimePart}, }; use mail_parser::decoders::html::html_to_text; use smtp::core::{Session, SessionData}; use smtp_proto::{MailFrom, RcptTo}; use std::{str::FromStr, sync::Arc, time::Duration}; use store::{ ValueKey, write::{AlignedBytes, Archive, BatchBuilder, now}, }; use trc::{AddContext, TaskQueueEvent}; use types::collection::Collection; use utils::{sanitize_email, template::Variables}; pub trait SendAlarmTask: Sync + Send { fn send_alarm( &self, account_id: u32, document_id: u32, alarm: &CalendarAlarm, server_instance: Arc, ) -> impl Future + Send; } impl SendAlarmTask for Server { async fn send_alarm( &self, account_id: u32, document_id: u32, alarm: &CalendarAlarm, server_instance: Arc, ) -> bool { match &alarm.typ { CalendarAlarmType::Display { .. } => { match send_display_alarm(self, account_id, document_id, alarm).await { Ok(result) => result, Err(err) => { trc::error!( err.account_id(account_id) .document_id(document_id) .caused_by(trc::location!()) .details("Failed to process e-mail alarm") ); false } } } CalendarAlarmType::Email { .. } => { match send_email_alarm(self, account_id, document_id, alarm, server_instance).await { Ok(result) => result, Err(err) => { trc::error!( err.account_id(account_id) .document_id(document_id) .caused_by(trc::location!()) .details("Failed to process e-mail alarm") ); false } } } } } } async fn send_email_alarm( server: &Server, account_id: u32, document_id: u32, alarm: &CalendarAlarm, server_instance: Arc, ) -> trc::Result { // Obtain access token let access_token = server .get_access_token(account_id) .await .caused_by(trc::location!())?; if !access_token.has_permission(Permission::CalendarAlarms) { trc::event!( Calendar(trc::CalendarEvent::AlarmSkipped), Reason = "Account does not have permission to send calendar alarms", AccountId = account_id, DocumentId = document_id, ); return Ok(true); } else if access_token.emails.is_empty() { trc::event!( Calendar(trc::CalendarEvent::AlarmFailed), Reason = "Account does not have any email addresses", AccountId = account_id, DocumentId = document_id, ); return Ok(true); } // Fetch event let Some(event_) = server .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, )) .await .caused_by(trc::location!())? else { trc::event!( TaskQueue(TaskQueueEvent::MetadataNotFound), Details = "Calendar Event metadata not found", AccountId = account_id, DocumentId = document_id, ); return Ok(true); }; // Unarchive event let event = event_ .unarchive::() .caused_by(trc::location!())?; // Build message body let account_main_email = access_token.emails.first().unwrap(); let account_main_domain = account_main_email.rsplit('@').next().unwrap_or("localhost"); let logo_cid = format!("logo.{}@{account_main_domain}", now()); let Some(tpl) = build_template( server, &access_token, account_id, document_id, alarm, event, &logo_cid, ) .await? else { return Ok(true); }; let txt_body = html_to_text(&tpl.body); // Obtain logo image let logo = match server.logo_resource(account_main_domain).await { Ok(logo) => logo, Err(err) => { trc::error!( err.caused_by(trc::location!()) .details("Failed to fetch logo image") ); None } }; let logo = if let Some(logo) = &logo { MimePart::new( ContentType::new(logo.content_type.as_ref()), BodyPart::Binary(logo.contents.as_slice().into()), ) } else { MimePart::new( ContentType::new("image/png"), BodyPart::Binary(DEFAULT_LOGO_BASE64.as_bytes().into()), ) .transfer_encoding("base64") } .inline() .cid(&logo_cid); // Build message let mail_from = if let Some(from_email) = &server.core.groupware.alarms_from_email { from_email.to_string() } else { format!("calendar-notification@{account_main_domain}") }; let message = MessageBuilder::new() .from(( server.core.groupware.alarms_from_name.as_str(), mail_from.as_str(), )) .header("To", HeaderType::Text(tpl.to.as_str().into())) .header("Auto-Submitted", HeaderType::Text("auto-generated".into())) .header("Reply-To", HeaderType::Text(account_main_email.into())) .subject(tpl.subject) .body(MimePart::new( ContentType::new("multipart/related"), BodyPart::Multipart(vec![ MimePart::new( ContentType::new("multipart/alternative"), BodyPart::Multipart(vec![ MimePart::new( ContentType::new("text/plain"), BodyPart::Text(txt_body.into()), ), MimePart::new( ContentType::new("text/html"), BodyPart::Text(tpl.body.into()), ), ]), ), logo, ]), )) .write_to_vec() .unwrap_or_default(); // Send message let server_ = server.clone(); let mail_from = account_main_email.to_string(); let to = tpl.to; let result = tokio::spawn(async move { let mut session = Session::::local( server_, server_instance, SessionData::local(access_token, None, vec![], vec![], 0), ); // MAIL FROM let _ = session .handle_mail_from(MailFrom { address: mail_from.into(), ..Default::default() }) .await; if let Some(error) = session.has_failed() { return Err(format!("Server rejected MAIL-FROM: {}", error.trim())); } // RCPT TO session.params.rcpt_errors_wait = Duration::from_secs(0); let _ = session .handle_rcpt_to(RcptTo { address: to.into(), ..Default::default() }) .await; if let Some(error) = session.has_failed() { return Err(format!("Server rejected RCPT-TO: {}", error.trim())); } // DATA session.data.message = message; let response = session.queue_message().await; if let smtp::core::State::Accepted(queue_id) = session.state { Ok(queue_id) } else { Err(format!( "Server rejected DATA: {}", std::str::from_utf8(&response).unwrap().trim() )) } }) .await; match result { Ok(Ok(queue_id)) => { trc::event!( Calendar(trc::CalendarEvent::AlarmSent), AccountId = account_id, DocumentId = document_id, QueueId = queue_id, ); } Ok(Err(err)) => { trc::event!( Calendar(trc::CalendarEvent::AlarmFailed), AccountId = account_id, DocumentId = document_id, Reason = err, ); } Err(_) => { trc::event!( Server(trc::ServerEvent::ThreadError), Details = "Join Error", AccountId = account_id, DocumentId = document_id, CausedBy = trc::location!(), ); return Ok(false); } } write_next_alarm(server, account_id, document_id, event).await } async fn send_display_alarm( server: &Server, account_id: u32, document_id: u32, alarm: &CalendarAlarm, ) -> trc::Result { // Fetch event let Some(event_) = server .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, )) .await .caused_by(trc::location!())? else { trc::event!( TaskQueue(TaskQueueEvent::MetadataNotFound), Details = "Calendar Event metadata not found", AccountId = account_id, DocumentId = document_id, ); return Ok(true); }; // Unarchive event let event = event_ .unarchive::() .caused_by(trc::location!())?; let recurrence_id = match &alarm.typ { CalendarAlarmType::Display { recurrence_id } => *recurrence_id, _ => None, }; let ical = &event.data.event; server .broadcast_push_notification(PushNotification::CalendarAlert(CalendarAlert { account_id, event_id: document_id, recurrence_id, uid: ical.uids().next().unwrap_or_default().to_string(), alert_id: ical .components .get(alarm.alarm_id as usize) .and_then(|c| c.property(&ICalendarProperty::Jsid)) .and_then(|v| v.values.first()) .and_then(|v| v.as_text()) .map(|v| v.to_string()) .unwrap_or_else(|| { format!( "k{}", ical.components .get(alarm.event_id as usize) .and_then(|c| c .component_ids .iter() .position(|id| id.to_native() == alarm.alarm_id as u32)) .unwrap_or_default() + 1 ) }), })) .await; write_next_alarm(server, account_id, document_id, event).await } async fn write_next_alarm( server: &Server, account_id: u32, document_id: u32, event: &ArchivedCalendarEvent, ) -> trc::Result { // Find next alarm time and write to task queue let now = now() as i64; if let Some(next_alarm) = event .data .next_alarm(now, Default::default()) .and_then(|next_alarm| { // Verify minimum interval let max_next_alarm = now + server.core.groupware.alarms_minimum_interval; if next_alarm.alarm_time < max_next_alarm { trc::event!( Calendar(trc::CalendarEvent::AlarmSkipped), Reason = "Next alarm skipped due to minimum interval", Details = next_alarm.alarm_time - now, AccountId = account_id, DocumentId = document_id, ); event.data.next_alarm(max_next_alarm, Default::default()) } else { Some(next_alarm) } }) { let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::CalendarEvent) .with_document(document_id); next_alarm.write_task(&mut batch); server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; server.notify_task_queue(); } Ok(true) } struct Details { to: String, subject: String, body: String, } async fn build_template( server: &Server, access_token: &AccessToken, account_id: u32, document_id: u32, alarm: &CalendarAlarm, event: &ArchivedCalendarEvent, logo_cid: &str, ) -> trc::Result> { let (Some(event_component), Some(alarm_component)) = ( event.data.event.components.get(alarm.event_id as usize), event.data.event.components.get(alarm.alarm_id as usize), ) else { trc::event!( TaskQueue(TaskQueueEvent::MetadataNotFound), Details = "Calendar Alarm component not found", AccountId = account_id, DocumentId = document_id, ); return Ok(None); }; // Build webcal URI let webcal_uri = match event.webcal_uri(server, access_token).await { Ok(uri) => uri, Err(err) => { trc::error!( err.account_id(account_id) .document_id(document_id) .caused_by(trc::location!()) .details("Failed to generate webcal URI") ); String::from("#") } }; // Obtain alarm details let mut summary = None; let mut description = None; let mut rcpt_to = None; let mut location = None; let mut organizer = None; let mut guests = vec![]; for entry in alarm_component.entries.iter() { match &entry.name { ArchivedICalendarProperty::Summary => { summary = entry.values.first().and_then(|v| v.as_text()); } ArchivedICalendarProperty::Description => { description = entry.values.first().and_then(|v| v.as_text()); } ArchivedICalendarProperty::Attendee => { rcpt_to = entry .values .first() .and_then(|v| v.as_text()) .map(|v| v.strip_prefix("mailto:").unwrap_or(v)) .and_then(sanitize_email); } _ => {} } } for entry in event_component.entries.iter() { match &entry.name { ArchivedICalendarProperty::Summary if summary.is_none() => { summary = entry.values.first().and_then(|v| v.as_text()); } ArchivedICalendarProperty::Description if description.is_none() => { description = entry.values.first().and_then(|v| v.as_text()); } ArchivedICalendarProperty::Location => { location = entry.values.first().and_then(|v| v.as_text()); } ArchivedICalendarProperty::Organizer | ArchivedICalendarProperty::Attendee => { let email = entry .values .first() .and_then(|v| v.as_text()) .map(|v| v.strip_prefix("mailto:").unwrap_or(v)); let name = entry.params.iter().find_map(|param| { if let ArchivedICalendarParameterName::Cn = param.name { param.value.as_text() } else { None } }); if email.is_some() || name.is_some() { if matches!(entry.name, ArchivedICalendarProperty::Organizer) { organizer = Some((email, name)); } else { guests.push((email, name)); } } } _ => {} } } // Validate recipient let rcpt_to = if let Some(rcpt_to) = rcpt_to { if server.core.groupware.alarms_allow_external_recipients || access_token.emails.iter().any(|email| email == &rcpt_to) { rcpt_to } else { trc::event!( Calendar(trc::CalendarEvent::AlarmRecipientOverride), Reason = "External recipient not allowed for calendar alarms", Details = rcpt_to, AccountId = account_id, DocumentId = document_id, ); access_token.emails.first().unwrap().to_string() } } else { access_token.emails.first().unwrap().to_string() }; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] let template = server .core .enterprise .as_ref() .and_then(|e| e.template_calendar_alarm.as_ref()) .unwrap_or(&server.core.groupware.alarms_template); // SPDX-SnippetEnd #[cfg(not(feature = "enterprise"))] let template = &server.core.groupware.alarms_template; let locale = i18n::locale_or_default(access_token.locale.as_deref().unwrap_or("en")); let chrono_locale = access_token .locale .as_deref() .and_then(|locale| Locale::from_str(locale).ok()) .unwrap_or(Locale::en_US); let (event_start, event_start_tz, event_end, event_end_tz) = match alarm.typ { CalendarAlarmType::Email { event_start, event_start_tz, event_end, event_end_tz, } => (event_start, event_start_tz, event_end, event_end_tz), CalendarAlarmType::Display { .. } => unreachable!(), }; let start = format!( "{} ({})", DateTime::from_timestamp(event_start, 0) .unwrap_or_default() .format_localized(locale.calendar_date_template, chrono_locale), Tz::from_id(event_start_tz) .unwrap_or(Tz::UTC) .name() .unwrap_or_default() ); let end = format!( "{} ({})", DateTime::from_timestamp(event_end, 0) .unwrap_or_default() .format_localized(locale.calendar_date_template, chrono_locale), Tz::from_id(event_end_tz) .unwrap_or(Tz::UTC) .name() .unwrap_or_default() ); let subject = format!( "{}: {} @ {}", locale.calendar_alarm_subject_prefix, summary.or(description).unwrap_or("No Subject"), start ); let organizer = organizer .map(|(email, name)| match (email, name) { (Some(email), Some(name)) => format!("{} <{}>", name, email), (Some(email), None) => email.to_string(), (None, Some(name)) => name.to_string(), _ => unreachable!(), }) .unwrap_or_else(|| access_token.name.clone()); let mut variables = Variables::new(); variables.insert_single(CalendarTemplateVariable::PageTitle, subject.as_str()); variables.insert_single( CalendarTemplateVariable::Header, locale.calendar_alarm_header, ); variables.insert_single( CalendarTemplateVariable::Footer, locale.calendar_alarm_footer, ); variables.insert_single( CalendarTemplateVariable::ActionName, locale.calendar_alarm_open, ); variables.insert_single(CalendarTemplateVariable::ActionUrl, webcal_uri.as_str()); variables.insert_single( CalendarTemplateVariable::AttendeesTitle, locale.calendar_attendees, ); variables.insert_single( CalendarTemplateVariable::EventTitle, summary.unwrap_or_default(), ); variables.insert_single(CalendarTemplateVariable::LogoCid, logo_cid); if let Some(description) = description { variables.insert_single(CalendarTemplateVariable::EventDescription, description); } variables.insert_block( CalendarTemplateVariable::EventDetails, [ Some([ (CalendarTemplateVariable::Key, locale.calendar_start), (CalendarTemplateVariable::Value, start.as_str()), ]), Some([ (CalendarTemplateVariable::Key, locale.calendar_end), (CalendarTemplateVariable::Value, end.as_str()), ]), location.map(|location| { [ (CalendarTemplateVariable::Key, locale.calendar_location), (CalendarTemplateVariable::Value, location), ] }), Some([ (CalendarTemplateVariable::Key, locale.calendar_organizer), (CalendarTemplateVariable::Value, organizer.as_str()), ]), ] .into_iter() .flatten(), ); if !guests.is_empty() { variables.insert_block( CalendarTemplateVariable::Attendees, guests.into_iter().map(|(email, name)| { [ (CalendarTemplateVariable::Key, name.unwrap_or_default()), (CalendarTemplateVariable::Value, email.unwrap_or_default()), ] }), ); } Ok(Some(Details { to: rcpt_to, body: template.eval(&variables), subject, })) } ================================================ FILE: crates/services/src/task_manager/imip.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use calcard::{ common::timezone::Tz, icalendar::{ ArchivedICalendarDay, ArchivedICalendarFrequency, ArchivedICalendarMonth, ArchivedICalendarParticipationStatus, ArchivedICalendarRecurrenceRule, ArchivedICalendarWeekday, ICalendarParticipationStatus, ICalendarProperty, }, }; use chrono::{DateTime, Locale}; use common::{ DEFAULT_LOGO_BASE64, Server, auth::AccessToken, config::groupware::CalendarTemplateVariable, i18n, listener::{ServerInstance, stream::NullIo}, }; use groupware::{ calendar::itip::ItipIngest, scheduling::{ArchivedItipSummary, ArchivedItipValue, ItipMessages}, }; use mail_builder::{ MessageBuilder, headers::{HeaderType, content_type::ContentType}, mime::{BodyPart, MimePart}, }; use mail_parser::decoders::html::html_to_text; use smtp::core::{Session, SessionData}; use smtp_proto::{MailFrom, RcptTo}; use std::{str::FromStr, sync::Arc, time::Duration}; use store::{ ValueKey, ahash::AHashMap, rkyv::rend::{i16_le, i32_le}, write::{AlignedBytes, Archive, TaskEpoch, TaskQueueClass, ValueClass, now}, }; use trc::AddContext; use utils::template::{Variable, Variables}; pub trait SendImipTask: Sync + Send { fn send_imip( &self, account_id: u32, document_id: u32, due: TaskEpoch, server_instance: Arc, ) -> impl Future + Send; } impl SendImipTask for Server { async fn send_imip( &self, account_id: u32, document_id: u32, due: TaskEpoch, server_instance: Arc, ) -> bool { match send_imip(self, account_id, document_id, due, server_instance).await { Ok(result) => result, Err(err) => { trc::error!( err.account_id(account_id) .document_id(document_id) .caused_by(trc::location!()) .details("Failed to process alarm") ); false } } } } async fn send_imip( server: &Server, account_id: u32, document_id: u32, due: TaskEpoch, server_instance: Arc, ) -> trc::Result { // Obtain access token let access_token = server .get_access_token(account_id) .await .caused_by(trc::location!())?; // Obtain iMIP payload let Some(archive) = server .store() .get_value::>(ValueKey { account_id, collection: 0, document_id, class: ValueClass::TaskQueue(TaskQueueClass::SendImip { due, is_payload: true, }), }) .await .caused_by(trc::location!())? else { trc::event!( Calendar(trc::CalendarEvent::ItipMessageError), AccountId = account_id, DocumentId = document_id, Reason = "Missing iMIP payload", ); return Ok(true); }; let imip = archive .unarchive::() .caused_by(trc::location!())?; let sender_domain = imip .messages .first() .and_then(|msg| msg.from.rsplit('@').next()) .unwrap_or("localhost"); // Obtain logo image let logo = match server.logo_resource(sender_domain).await { Ok(logo) => logo, Err(err) => { trc::error!( err.caused_by(trc::location!()) .details("Failed to fetch logo image") ); None } }; let logo_cid = format!("logo.{}@{sender_domain}", now()); let logo = if let Some(logo) = &logo { MimePart::new( ContentType::new(logo.content_type.as_ref()), BodyPart::Binary(logo.contents.as_slice().into()), ) } else { MimePart::new( ContentType::new("image/png"), BodyPart::Binary(DEFAULT_LOGO_BASE64.as_bytes().into()), ) .transfer_encoding("base64") } .inline() .cid(&logo_cid); for itip_message in imip.messages.iter() { for recipient in itip_message.to.iter() { // Build template let tpl = build_itip_template( server, &access_token, account_id, document_id, itip_message.from.as_str(), recipient.as_str(), &itip_message.summary, &logo_cid, ) .await; let txt_body = html_to_text(&tpl.body); // Build message let message = MessageBuilder::new() .from(( access_token .description .as_deref() .unwrap_or(access_token.name.as_str()), itip_message.from.as_str(), )) .to(recipient.as_str()) .header("Auto-Submitted", HeaderType::Text("auto-generated".into())) .header( "Reply-To", HeaderType::Text(itip_message.from.as_str().into()), ) .subject(&tpl.subject) .body(MimePart::new( ContentType::new("multipart/mixed"), BodyPart::Multipart(vec![ MimePart::new( ContentType::new("multipart/related"), BodyPart::Multipart(vec![ MimePart::new( ContentType::new("multipart/alternative"), BodyPart::Multipart(vec![ MimePart::new( ContentType::new("text/plain"), BodyPart::Text(txt_body.into()), ), MimePart::new( ContentType::new("text/html"), BodyPart::Text(tpl.body.as_str().into()), ), ]), ), logo.clone(), ]), ), MimePart::new( ContentType::new("text/calendar") .attribute("method", itip_message.summary.method()) .attribute("charset", "utf-8"), BodyPart::Text(itip_message.message.as_str().into()), ) .attachment("event.ics"), ]), )) .write_to_vec() .unwrap_or_default(); // Send message let server_ = server.clone(); let server_instance = server_instance.clone(); let access_token = access_token.clone(); let from = itip_message.from.to_string(); let to = recipient.to_string(); tokio::spawn(async move { let mut session = Session::::local( server_, server_instance, SessionData::local(access_token, None, vec![], vec![], 0), ); // MAIL FROM let _ = session .handle_mail_from(MailFrom { address: from.as_str().into(), ..Default::default() }) .await; if let Some(error) = session.has_failed() { trc::event!( Calendar(trc::CalendarEvent::ItipMessageError), AccountId = account_id, DocumentId = document_id, From = from, To = to, Reason = format!("Server rejected MAIL-FROM: {}", error.trim()), ); return; } // RCPT TO session.params.rcpt_errors_wait = Duration::from_secs(0); let _ = session .handle_rcpt_to(RcptTo { address: to.as_str().into(), ..Default::default() }) .await; if let Some(error) = session.has_failed() { trc::event!( Calendar(trc::CalendarEvent::ItipMessageError), AccountId = account_id, DocumentId = document_id, From = from, To = to, Reason = format!("Server rejected RCPT-TO: {}", error.trim()), ); return; } // DATA session.data.message = message; let response = session.queue_message().await; if let smtp::core::State::Accepted(queue_id) = session.state { trc::event!( Calendar(trc::CalendarEvent::ItipMessageSent), From = from, To = to, AccountId = account_id, DocumentId = document_id, QueueId = queue_id, ); } else { trc::event!( Calendar(trc::CalendarEvent::ItipMessageError), From = from, To = to, AccountId = account_id, DocumentId = document_id, Reason = format!( "Server rejected DATA: {}", std::str::from_utf8(&response).unwrap().trim() ), ); } }) .await .map_err(|_| { trc::Error::new(trc::EventType::Server(trc::ServerEvent::ThreadError)) .caused_by(trc::location!()) })?; } } Ok(true) } pub struct Details { pub subject: String, pub body: String, } #[allow(clippy::too_many_arguments)] pub async fn build_itip_template( server: &Server, access_token: &AccessToken, account_id: u32, document_id: u32, from: &str, to: &str, summary: &ArchivedItipSummary, logo_cid: &str, ) -> Details { // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] let template = server .core .enterprise .as_ref() .and_then(|e| e.template_scheduling_email.as_ref()) .unwrap_or(&server.core.groupware.itip_template); // SPDX-SnippetEnd #[cfg(not(feature = "enterprise"))] let template = &server.core.groupware.itip_template; let locale = i18n::locale_or_default(access_token.locale.as_deref().unwrap_or("en")); let chrono_locale = access_token .locale .as_deref() .and_then(|locale| Locale::from_str(locale).ok()) .unwrap_or(Locale::en_US); let mut variables = Variables::new(); let mut subject; let (fields, old_fields) = match summary { ArchivedItipSummary::Invite(fields) => { subject = format!("{}: ", locale.calendar_invitation); (fields, None) } ArchivedItipSummary::Update { current, previous, .. } => { subject = format!("{}: ", locale.calendar_updated_invitation); variables.insert_single( CalendarTemplateVariable::Header, locale.calendar_event_updated.to_string(), ); variables.insert_single(CalendarTemplateVariable::Color, "info".to_string()); (current, Some(previous)) } ArchivedItipSummary::Cancel(fields) => { subject = format!("{}: ", locale.calendar_cancelled); variables.insert_single( CalendarTemplateVariable::Header, locale.calendar_event_cancelled.to_string(), ); variables.insert_single(CalendarTemplateVariable::Color, "danger".to_string()); (fields, None) } ArchivedItipSummary::Rsvp { part_stat, current } => { let (color, value) = match part_stat { ArchivedICalendarParticipationStatus::Accepted => { subject = format!("{}: ", locale.calendar_accepted); ( "info", locale.calendar_participant_accepted.replace("$name", from), ) } ArchivedICalendarParticipationStatus::Declined => { subject = format!("{}: ", locale.calendar_declined); ( "danger", locale.calendar_participant_declined.replace("$name", from), ) } ArchivedICalendarParticipationStatus::Tentative => { subject = format!("{}: ", locale.calendar_tentative); ( "warning", locale.calendar_participant_tentative.replace("$name", from), ) } ArchivedICalendarParticipationStatus::Delegated => { subject = format!("{}: ", locale.calendar_delegated); ( "warning", locale.calendar_participant_delegated.replace("$name", from), ) } _ => { subject = format!("{}: ", locale.calendar_reply); ( "info", locale.calendar_participant_reply.replace("$name", from), ) } }; variables.insert_single(CalendarTemplateVariable::Header, value); variables.insert_single(CalendarTemplateVariable::Color, color.to_string()); (current, None) } }; let mut has_rrule = false; let mut details = Vec::with_capacity(4); for field in [ ICalendarProperty::Summary, ICalendarProperty::Description, ICalendarProperty::Rrule, ICalendarProperty::Dtstart, ICalendarProperty::Location, ] { let Some(entry) = fields.iter().find(|e| e.name == field) else { continue; }; let field_name = match &field { ICalendarProperty::Summary => locale.calendar_summary, ICalendarProperty::Description => locale.calendar_description, ICalendarProperty::Rrule => { has_rrule = true; locale.calendar_when } ICalendarProperty::Dtstart if !has_rrule => locale.calendar_when, ICalendarProperty::Location => locale.calendar_location, _ => continue, }; let value = format_field( &entry.value, locale.calendar_date_template_long, chrono_locale, ); match &field { ICalendarProperty::Summary => { subject.push_str(&value); } ICalendarProperty::Dtstart | ICalendarProperty::Rrule => { subject.push_str(" @ "); subject.push_str(&value); } _ => (), } let mut fields = AHashMap::with_capacity(3); fields.insert(CalendarTemplateVariable::Key, field_name.to_string()); fields.insert(CalendarTemplateVariable::Value, value); if let Some(old_entry) = old_fields.and_then(|fields| fields.iter().find(|e| e.name == field)) { fields.insert( CalendarTemplateVariable::Changed, locale.calendar_changed.to_string(), ); fields.insert( CalendarTemplateVariable::OldValue, format_field( &old_entry.value, locale.calendar_date_template, chrono_locale, ), ); } details.push(fields); } variables.items.insert( CalendarTemplateVariable::EventDetails, Variable::Block(details), ); variables.insert_single(CalendarTemplateVariable::PageTitle, subject.clone()); variables.insert_single(CalendarTemplateVariable::LogoCid, format!("cid:{logo_cid}")); if let Some(guests) = fields .iter() .find(|e| e.name == ICalendarProperty::Attendee) && let ArchivedItipValue::Participants(guests) = &guests.value { variables.insert_single( CalendarTemplateVariable::AttendeesTitle, locale.calendar_attendees.to_string(), ); variables.insert_block( CalendarTemplateVariable::Attendees, guests.iter().map(|guest| { [ ( CalendarTemplateVariable::Key, if guest.is_organizer { if let Some(name) = guest.name.as_ref() { format!("{name} - {}", locale.calendar_organizer) } else { locale.calendar_organizer.to_string() } } else { guest .name .as_ref() .map(|n| n.as_str()) .unwrap_or_default() .to_string() }, ), (CalendarTemplateVariable::Value, guest.email.to_string()), ] }), ); } // Add RSVP buttons if matches!( summary, ArchivedItipSummary::Invite(_) | ArchivedItipSummary::Update { .. } ) && let Some(rsvp_url) = server.http_rsvp_url(account_id, document_id, to).await { variables.insert_single( CalendarTemplateVariable::Rsvp, locale.calendar_reply_as.replace("$name", to), ); variables.insert_block( CalendarTemplateVariable::Actions, [ ( ICalendarParticipationStatus::Accepted, locale.calendar_yes.to_string(), "info", ), ( ICalendarParticipationStatus::Declined, locale.calendar_no.to_string(), "danger", ), ( ICalendarParticipationStatus::Tentative, locale.calendar_maybe.to_string(), "warning", ), ] .into_iter() .map(|(status, title, color)| { [ (CalendarTemplateVariable::ActionName, title.to_string()), (CalendarTemplateVariable::ActionUrl, rsvp_url.url(&status)), (CalendarTemplateVariable::Color, color.to_string()), ] }), ); } // Add footer variables.insert_block( CalendarTemplateVariable::Footer, [ [( CalendarTemplateVariable::Key, locale.calendar_imip_footer_1.to_string(), )], [( CalendarTemplateVariable::Key, locale.calendar_imip_footer_2.to_string(), )], ] .into_iter(), ); Details { subject, body: template.eval(&variables), } } fn format_field(value: &ArchivedItipValue, template: &str, chrono_locale: Locale) -> String { match value { ArchivedItipValue::Text(text) => text.to_string(), ArchivedItipValue::Time(time) => { use chrono::TimeZone; let tz = Tz::from_id(time.tz_id.to_native()).unwrap_or(Tz::UTC); format!( "{} ({})", tz.from_utc_datetime( &DateTime::from_timestamp(time.start.to_native(), 0) .unwrap_or_default() .naive_local() ) .format_localized(template, chrono_locale), tz.name().unwrap_or_default() ) } ArchivedItipValue::Rrule(rrule) => RecurrenceFormatter.format(rrule), ArchivedItipValue::Participants(_) => String::new(), // Handled separately } } #[derive(Default)] pub struct RecurrenceFormatter; impl RecurrenceFormatter { pub fn format(&self, rule: &ArchivedICalendarRecurrenceRule) -> String { let mut parts = Vec::new(); // Format frequency and interval let freq_part = self.format_frequency( &rule.freq, rule.interval.as_ref().map(|i| i.to_native()).unwrap_or(1), ); parts.push(freq_part); // Format day constraints if !rule.byday.is_empty() { parts.push(self.format_by_day(&rule.byday)); } // Format time constraints if !rule.byhour.is_empty() || !rule.byminute.is_empty() { parts.push(self.format_time_constraints(&rule.byhour, &rule.byminute)); } // Format month day constraints if !rule.bymonthday.is_empty() { parts.push(self.format_month_days(&rule.bymonthday)); } // Format month constraints if !rule.bymonth.is_empty() { parts.push(self.format_months(&rule.bymonth)); } // Format year day constraints if !rule.byyearday.is_empty() { parts.push(self.format_year_days(&rule.byyearday)); } // Format week number constraints if !rule.byweekno.is_empty() { parts.push(self.format_week_numbers(&rule.byweekno)); } // Format set position constraints if !rule.bysetpos.is_empty() { parts.push(self.format_set_positions(&rule.bysetpos)); } // Format termination (until/count) /*if let Some(until) = &rule.until { parts.push(format!("until {}", self.format_datetime(until))); } else*/ if let Some(count) = rule.count.as_ref() { let times = if *count == 1 { "time" } else { "times" }; parts.push(format!("for {} {}", count, times)); } parts.join(" ") } fn format_frequency(&self, freq: &ArchivedICalendarFrequency, interval: u16) -> String { let (singular, plural) = match freq { ArchivedICalendarFrequency::Daily => ("day", "days"), ArchivedICalendarFrequency::Weekly => ("week", "weeks"), ArchivedICalendarFrequency::Monthly => ("month", "months"), ArchivedICalendarFrequency::Yearly => ("year", "years"), ArchivedICalendarFrequency::Hourly => ("hour", "hours"), ArchivedICalendarFrequency::Minutely => ("minute", "minutes"), ArchivedICalendarFrequency::Secondly => ("second", "seconds"), }; if interval == 1 { format!("Every {}", singular) } else { format!("Every {} {}", interval, plural) } } fn format_by_day(&self, days: &[ArchivedICalendarDay]) -> String { let day_names: Vec = days.iter().map(|day| self.format_day(day)).collect(); format!("on {}", self.format_list(&day_names)) } fn format_day(&self, day: &ArchivedICalendarDay) -> String { let day_name = match day.weekday { ArchivedICalendarWeekday::Monday => "Monday", ArchivedICalendarWeekday::Tuesday => "Tuesday", ArchivedICalendarWeekday::Wednesday => "Wednesday", ArchivedICalendarWeekday::Thursday => "Thursday", ArchivedICalendarWeekday::Friday => "Friday", ArchivedICalendarWeekday::Saturday => "Saturday", ArchivedICalendarWeekday::Sunday => "Sunday", }; if let Some(occurrence) = day.ordwk.as_ref().map(|o| o.to_native()) { if occurrence > 0 { format!("the {} {}", self.ordinal(occurrence as u32), day_name) } else { format!( "the {} {} from the end", self.ordinal((-occurrence) as u32), day_name ) } } else { day_name.to_string() } } fn format_time_constraints(&self, hours: &[u8], minutes: &[u8]) -> String { let mut time_parts = Vec::new(); if !hours.is_empty() && !minutes.is_empty() { // Combine hours and minutes for &hour in hours { for &minute in minutes { time_parts.push(format!("{}:{:02}", self.format_hour(hour), minute)); } } } else if !hours.is_empty() { for &hour in hours { time_parts.push(self.format_hour(hour)); } } else if !minutes.is_empty() { for &minute in minutes { time_parts.push(format!(":{:02}", minute)); } } if !time_parts.is_empty() { format!("at {}", self.format_list(&time_parts)) } else { String::new() } } fn format_hour(&self, hour: u8) -> String { match hour { 0 => "12:00 AM".to_string(), 1..=11 => format!("{}:00 AM", hour), 12 => "12:00 PM".to_string(), 13..=23 => format!("{}:00 PM", hour - 12), _ => format!("{:02}:00", hour), } } fn format_month_days(&self, days: &[i8]) -> String { let day_strings: Vec = days .iter() .map(|&day| { if day > 0 { self.ordinal(day as u32) } else { format!("{} from the end", self.ordinal((-day) as u32)) } }) .collect(); format!("on the {}", self.format_list(&day_strings)) } fn format_months(&self, months: &[ArchivedICalendarMonth]) -> String { let month_names: Vec = months .iter() .map(|month| self.month_name(month.month())) .collect(); format!("in {}", self.format_list(&month_names)) } fn format_year_days(&self, days: &[i16_le]) -> String { let day_strings: Vec = days .iter() .map(|&day| { if day > 0 { format!("day {} of the year", day) } else { format!("day {} from the end of the year", -day) } }) .collect(); format!("on {}", self.format_list(&day_strings)) } fn format_week_numbers(&self, weeks: &[i8]) -> String { let week_strings: Vec = weeks .iter() .map(|&week| { if week > 0 { format!("week {}", week) } else { format!("week {} from the end", -week) } }) .collect(); format!("in {}", self.format_list(&week_strings)) } fn format_set_positions(&self, positions: &[i32_le]) -> String { let pos_strings: Vec = positions .iter() .map(|&pos| { if pos > 0 { self.ordinal(pos.to_native() as u32) } else { format!("{} from the end", self.ordinal((-pos) as u32)) } }) .collect(); format!( "limited to the {} occurrence", self.format_list(&pos_strings) ) } fn format_list(&self, items: &[String]) -> String { match items.len() { 0 => String::new(), 1 => items[0].clone(), 2 => format!("{} and {}", items[0], items[1]), _ => { let rest = &items[..items.len() - 1]; format!("{}, and {}", rest.join(", "), items.last().unwrap()) } } } fn ordinal(&self, n: u32) -> String { let suffix = match n % 100 { 11..=13 => "th", _ => match n % 10 { 1 => "st", 2 => "nd", 3 => "rd", _ => "th", }, }; format!("{}{}", n, suffix) } fn month_name(&self, month: u8) -> String { match month { 1 => "January", 2 => "February", 3 => "March", 4 => "April", 5 => "May", 6 => "June", 7 => "July", 8 => "August", 9 => "September", 10 => "October", 11 => "November", 12 => "December", _ => "Unknown", } .to_string() } /*fn format_datetime(&self, dt: &PartialDateTime) -> String { format!("{:?}", dt) }*/ } ================================================ FILE: crates/services/src/task_manager/index.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::task_manager::{IndexAction, Task}; use common::{Server, auth::AccessToken}; use directory::{Type, backend::internal::manage::ManageDirectory}; use email::{cache::MessageCacheFetch, message::metadata::MessageMetadata}; use groupware::{cache::GroupwareCache, calendar::CalendarEvent, contact::ContactCard}; use std::cmp::Ordering; use store::{ IterateParams, SerializeInfallible, ValueKey, ahash::AHashMap, roaring::RoaringBitmap, search::{IndexDocument, SearchField, SearchFilter, SearchQuery}, write::{ AlignedBytes, Archive, BatchBuilder, SearchIndex, TaskEpoch, TaskQueueClass, TelemetryClass, ValueClass, key::DeserializeBigEndian, }, }; use trc::{AddContext, TaskQueueEvent}; use types::{ blob_hash::BlobHash, collection::{Collection, SyncCollection}, field::EmailField, }; pub(crate) trait SearchIndexTask: Sync + Send { fn index( &self, tasks: &[Task], ) -> impl Future> + Send; } pub trait ReindexIndexTask: Sync + Send { fn reindex( &self, index: SearchIndex, account_id: Option, tenant_id: Option, ) -> impl Future> + Send; } const NUM_INDEXES: usize = 5; #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum TaskType { Insert, Delete, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum TaskStatus { Success, Failed, Ignored, } #[derive(Debug)] pub(crate) struct IndexTaskResult { index: SearchIndex, task_type: TaskType, status: TaskStatus, } impl SearchIndexTask for Server { async fn index(&self, tasks: &[Task]) -> Vec { let mut results: Vec = Vec::with_capacity(tasks.len()); let mut batch = BatchBuilder::new(); let mut document_insertions = Vec::new(); let mut document_deletions: [AHashMap>; NUM_INDEXES] = std::array::from_fn(|_| AHashMap::new()); for task in tasks { if task.action.is_insert { let document = match task.action.index { SearchIndex::Email => { build_email_document(self, task.account_id, task.document_id).await } SearchIndex::Calendar => { build_calendar_document(self, task.account_id, task.document_id).await } SearchIndex::Contacts => { build_contact_document(self, task.account_id, task.document_id).await } SearchIndex::File => { // File indexing not implemented yet continue; } SearchIndex::Tracing => { build_tracing_span_document(self, task.account_id, task.document_id).await } SearchIndex::InMemory => unreachable!(), }; let result = match document { Ok(Some(doc)) if !doc.is_empty() => { document_insertions.push(doc); TaskStatus::Success } Err(err) => { trc::error!( err.account_id(task.account_id) .document_id(task.document_id) .caused_by(trc::location!()) .ctx(trc::Key::Collection, task.action.index.name()) .details("Failed to build document for indexing") ); TaskStatus::Failed } _ => { trc::event!( TaskQueue(TaskQueueEvent::TaskIgnored), Collection = task.action.index.name(), Reason = "Nothing to index", AccountId = task.account_id, DocumentId = task.document_id, ); TaskStatus::Ignored } }; results.push(IndexTaskResult { task_type: TaskType::Insert, index: task.action.index, status: result, }); } else { let idx = match task.action.index { SearchIndex::Email => { if let Err(err) = delete_email_metadata( self, &mut batch, task.account_id, task.document_id, ) .await { trc::error!( err.account_id(task.account_id) .document_id(task.document_id) .caused_by(trc::location!()) .details("Failed to delete email metadata from index") ); results.push(IndexTaskResult { task_type: TaskType::Delete, index: task.action.index, status: TaskStatus::Failed, }); continue; } 0 } SearchIndex::Calendar => 1, SearchIndex::Contacts => 2, SearchIndex::File => 3, SearchIndex::Tracing | SearchIndex::InMemory => unreachable!(), }; document_deletions[idx] .entry(task.account_id) .or_default() .push(task.document_id); results.push(IndexTaskResult { task_type: TaskType::Delete, index: task.action.index, status: TaskStatus::Success, }); } } // Commit deletion batch to data store if !batch.is_empty() && let Err(err) = self.store().write(batch.build_all()).await { trc::error!( err.caused_by(trc::location!()) .details("Failed to commit index deletions to data store") ); for r in results.iter_mut() { if r.task_type == TaskType::Delete && r.status == TaskStatus::Success && r.index == SearchIndex::Email { r.status = TaskStatus::Failed; } } return results; } // Index documents if !document_insertions.is_empty() && let Err(err) = self.search_store().index(document_insertions).await { trc::error!( err.caused_by(trc::location!()) .details("Failed to index documents") ); for r in results.iter_mut() { if r.task_type == TaskType::Insert && r.status == TaskStatus::Success { r.status = TaskStatus::Failed; } } return results; } // Delete documents for (accounts, index) in document_deletions.into_iter().zip([ SearchIndex::Email, SearchIndex::Calendar, SearchIndex::Contacts, ]) { let multi_account = match accounts.len().cmp(&1) { Ordering::Greater => true, Ordering::Equal => false, Ordering::Less => continue, }; let mut query = SearchQuery::new(index); if multi_account { query.add_filter(SearchFilter::Or); } for (account_id, document_ids) in accounts { let multi_document = document_ids.len() > 1; query .add_filter(SearchFilter::And) .add_filter(SearchFilter::eq(SearchField::AccountId, account_id)); if multi_document { query.add_filter(SearchFilter::Or); } for document_id in document_ids { query.add_filter(SearchFilter::eq(SearchField::DocumentId, document_id)); } if multi_document { query.add_filter(SearchFilter::End); } query.add_filter(SearchFilter::End); } if multi_account { query.add_filter(SearchFilter::End); } if let Err(err) = self.search_store().unindex(query).await { trc::error!( err.caused_by(trc::location!()) .details("Failed to delete documents from index") .ctx(trc::Key::Collection, index.name()) ); for r in results.iter_mut() { if r.task_type == TaskType::Delete && r.status == TaskStatus::Success { r.status = TaskStatus::Failed; } } return results; } } results } } impl ReindexIndexTask for Server { async fn reindex( &self, index: SearchIndex, account_id: Option, tenant_id: Option, ) -> trc::Result<()> { let accounts = if let Some(account_id) = account_id { RoaringBitmap::from_sorted_iter([account_id]).unwrap() } else { let mut accounts = RoaringBitmap::new(); for principal in self .core .storage .data .list_principals( None, tenant_id, &[Type::Individual, Type::Group], false, 0, 0, ) .await .caused_by(trc::location!())? .items { accounts.insert(principal.id()); } accounts }; let due = TaskEpoch::now(); match index { SearchIndex::Email => { for account_id in accounts { let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Email); for document_id in self .get_cached_messages(account_id) .await .caused_by(trc::location!())? .emails .items .iter() .map(|v| v.document_id) { batch.with_document(document_id).set( ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { due, index: SearchIndex::Email, is_insert: true, }), 0u64.serialize(), ); if batch.len() >= 2000 { self.core.storage.data.write(batch.build_all()).await?; batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Email); } } if !batch.is_empty() { self.core.storage.data.write(batch.build_all()).await?; } } } SearchIndex::Calendar | SearchIndex::Contacts => { for account_id in accounts { let cache = self .fetch_dav_resources( &AccessToken::from_id(account_id).with_tenant_id(tenant_id), account_id, if index == SearchIndex::Calendar { SyncCollection::Calendar } else { SyncCollection::AddressBook }, ) .await .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); batch.with_account_id(account_id); for document_id in cache.document_ids(false) { batch.with_document(document_id).set( ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { due, index, is_insert: true, }), 0u64.serialize(), ); if batch.len() >= 2000 { self.core.storage.data.write(batch.build_all()).await?; batch = BatchBuilder::new(); batch.with_account_id(account_id); } } if !batch.is_empty() { self.core.storage.data.write(batch.build_all()).await?; } } } SearchIndex::Tracing => { // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] if let Some(store) = self .core .enterprise .as_ref() .and_then(|e| e.trace_store.as_ref()) { let mut spans = Vec::new(); store .store .iterate( IterateParams::new( ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span { span_id: 0, })), ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span { span_id: u64::MAX, })), ) .no_values(), |key, _| { spans.push(key.deserialize_be_u64(0)?); Ok(true) }, ) .await .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); for span_id in spans { batch .with_account_id((span_id >> 32) as u32) // TODO: This is hacky, improve .with_document(span_id as u32) .set( ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { due: TaskEpoch::now(), index: SearchIndex::Tracing, is_insert: true, }), vec![], ); if batch.len() >= 2000 { self.core.storage.data.write(batch.build_all()).await?; batch = BatchBuilder::new(); } } if !batch.is_empty() { self.core.storage.data.write(batch.build_all()).await?; } } // SPDX-SnippetEnd } SearchIndex::File | SearchIndex::InMemory => (), } // Request indexing self.notify_task_queue(); Ok(()) } } async fn build_email_document( server: &Server, account_id: u32, document_id: u32, ) -> trc::Result> { let Some(index_fields) = server.core.jmap.index_fields.get(&SearchIndex::Email) else { return Ok(None); }; match server .store() .get_value::>(ValueKey::property( account_id, Collection::Email, document_id, EmailField::Metadata, )) .await? { Some(metadata_) => { let metadata = metadata_ .unarchive::() .caused_by(trc::location!())?; let raw_message = server .blob_store() .get_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX) .await .caused_by(trc::location!())? .ok_or_else(|| { trc::StoreEvent::NotFound .into_err() .details("Blob not found") })?; Ok(Some(metadata.index_document( account_id, document_id, &raw_message, index_fields, server.core.jmap.default_language, ))) } None => Ok(None), } } async fn build_calendar_document( server: &Server, account_id: u32, document_id: u32, ) -> trc::Result> { let Some(index_fields) = server.core.jmap.index_fields.get(&SearchIndex::Calendar) else { return Ok(None); }; match server .store() .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, document_id, )) .await? { Some(metadata_) => Ok(Some( metadata_ .unarchive::() .caused_by(trc::location!())? .index_document( account_id, document_id, index_fields, server.core.jmap.default_language, ), )), None => Ok(None), } } async fn build_contact_document( server: &Server, account_id: u32, document_id: u32, ) -> trc::Result> { let Some(index_fields) = server.core.jmap.index_fields.get(&SearchIndex::Contacts) else { return Ok(None); }; match server .store() .get_value::>(ValueKey::archive( account_id, Collection::ContactCard, document_id, )) .await? { Some(metadata_) => Ok(Some( metadata_ .unarchive::() .caused_by(trc::location!())? .index_document( account_id, document_id, index_fields, server.core.jmap.default_language, ), )), None => Ok(None), } } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] async fn build_tracing_span_document( server: &Server, account_id: u32, document_id: u32, ) -> trc::Result> { use common::telemetry::tracers::store::{TracingStore, build_span_document}; let Some(index_fields) = server.core.jmap.index_fields.get(&SearchIndex::Tracing) else { return Ok(None); }; let Some(store) = server .core .enterprise .as_ref() .and_then(|e| e.trace_store.as_ref()) else { return Ok(None); }; let span_id = ((account_id as u64) << 32) | document_id as u64; let span = store.store.get_span(span_id).await?; if !span.is_empty() { Ok(Some(build_span_document(span_id, span, index_fields))) } else { Ok(None) } } // SPDX-SnippetEnd #[cfg(not(feature = "enterprise"))] async fn build_tracing_span_document( _: &Server, _: u32, _: u32, ) -> trc::Result> { Ok(None) } async fn delete_email_metadata( server: &Server, batch: &mut BatchBuilder, account_id: u32, document_id: u32, ) -> trc::Result<()> { match server .store() .get_value::>(ValueKey::property( account_id, Collection::Email, document_id, EmailField::Metadata, )) .await? { Some(metadata_) => { batch .with_account_id(account_id) .with_collection(Collection::Email) .with_document(document_id); let metadata = metadata_ .unarchive::() .caused_by(trc::location!())?; metadata.unindex(batch); // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL // Hold blob for undeletion #[cfg(feature = "enterprise")] { use common::enterprise::undelete::DeletedItemType; use email::message::metadata::ArchivedMetadataHeaderName; if let Some(undelete) = server .core .enterprise .as_ref() .and_then(|e| e.undelete.as_ref()) { use common::enterprise::undelete::DeletedItem; use email::message::metadata::MESSAGE_RECEIVED_MASK; use store::{ Serialize, write::{Archiver, BlobLink, BlobOp, now}, }; let root_part = metadata.root_part(); let from: Option> = root_part.headers.iter().find_map(|h| { if let ArchivedMetadataHeaderName::From = &h.name { h.value.as_single_address().and_then(|addr| { match (addr.address.as_ref(), addr.name.as_ref()) { (Some(address), Some(name)) => { Some(format!("{} <{}>", name, address).into_boxed_str()) } (Some(address), None) => Some(address.as_ref().into()), (None, Some(name)) => Some(name.as_ref().into()), (None, None) => None, } }) } else { None } }); let subject: Option> = root_part.headers.iter().rev().find_map(|h| { if let ArchivedMetadataHeaderName::Subject = &h.name { h.value.as_text().map(Into::into) } else { None } }); let now = now(); let until = now + undelete.retention.as_secs(); let blob_hash = BlobHash::from(&metadata.blob_hash); batch .set( BlobOp::Link { hash: blob_hash.clone(), to: BlobLink::Temporary { until }, }, vec![BlobLink::UNDELETE_LINK], ) .set( BlobOp::Undelete { hash: blob_hash, until, }, Archiver::new(DeletedItem { typ: DeletedItemType::Email { from: from.unwrap_or_default(), subject: subject.unwrap_or_default(), received_at: metadata.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK, }, size: root_part.offset_end.to_native(), deleted_at: now, }) .serialize() .caused_by(trc::location!())?, ); } } // SPDX-SnippetEnd } None => { trc::event!( TaskQueue(TaskQueueEvent::MetadataNotFound), Details = "E-mail metadata not found", AccountId = account_id, DocumentId = document_id, ); } } Ok(()) } impl IndexTaskResult { pub fn is_done(&self) -> bool { self.status != TaskStatus::Failed } } ================================================ FILE: crates/services/src/task_manager/lock.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::task_manager::*; pub(crate) trait TaskLockManager: Sync + Send { fn try_lock_task( &self, account_id: u32, document_id: u32, lock_key: Vec, lock_expiry: u64, ) -> impl Future + Send; fn remove_index_lock(&self, lock_key: Vec) -> impl Future + Send; } impl TaskLockManager for Server { async fn try_lock_task( &self, account_id: u32, document_id: u32, lock_key: Vec, lock_expiry: u64, ) -> bool { match self .in_memory_store() .try_lock(KV_LOCK_TASK, &lock_key, lock_expiry) .await { Ok(result) => { if !result { trc::event!( TaskQueue(TaskQueueEvent::TaskLocked), AccountId = account_id, DocumentId = document_id, Expires = trc::Value::Timestamp(now() + lock_expiry), ); } result } Err(err) => { trc::error!( err.account_id(account_id) .document_id(document_id) .details("Failed to lock task") ); false } } } async fn remove_index_lock(&self, lock_key: Vec) { if let Err(err) = self .in_memory_store() .remove_lock(KV_LOCK_TASK, &lock_key) .await { trc::error!( err.details("Failed to unlock task") .ctx(trc::Key::Key, lock_key) .caused_by(trc::location!()) ); } } } pub(crate) trait TaskLock { fn account_id(&self) -> u32; fn document_id(&self) -> u32; fn lock_key(&self) -> Vec; fn lock_expiry(&self) -> u64; fn value_classes(&self) -> impl Iterator; } impl TaskLock for Task { fn account_id(&self) -> u32 { self.account_id } fn document_id(&self) -> u32 { self.document_id } fn lock_key(&self) -> Vec { KeySerializer::new((U32_LEN * 2) + U64_LEN + 2) .write(0u8) .write(self.due.inner()) .write_leb128(self.account_id) .write_leb128(self.document_id) .write(self.action.index.to_u8()) .finalize() } fn lock_expiry(&self) -> u64 { INDEX_EXPIRY } fn value_classes(&self) -> impl Iterator { std::iter::once(ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { due: self.due, index: self.action.index, is_insert: self.action.is_insert, })) } } impl TaskLock for Task { fn account_id(&self) -> u32 { self.account_id } fn document_id(&self) -> u32 { self.document_id } fn lock_key(&self) -> Vec { KeySerializer::new((U32_LEN * 2) + U64_LEN + 1) .write(2u8) .write(self.due.inner()) .write_leb128(self.account_id) .write_leb128(self.document_id) .finalize() } fn lock_expiry(&self) -> u64 { ALARM_EXPIRY } fn value_classes(&self) -> impl Iterator { std::iter::once(ValueClass::TaskQueue(TaskQueueClass::SendAlarm { event_id: self.action.event_id, alarm_id: self.action.alarm_id, due: self.due, is_email_alert: matches!(self.action.typ, CalendarAlarmType::Email { .. }), })) } } impl TaskLock for Task { fn account_id(&self) -> u32 { self.account_id } fn document_id(&self) -> u32 { self.document_id } fn lock_key(&self) -> Vec { KeySerializer::new((U32_LEN * 2) + U64_LEN + 1) .write(3u8) .write(self.due.inner()) .write_leb128(self.account_id) .write_leb128(self.document_id) .finalize() } fn lock_expiry(&self) -> u64 { ALARM_EXPIRY } fn value_classes(&self) -> impl Iterator { [ ValueClass::TaskQueue(TaskQueueClass::SendImip { due: self.due, is_payload: false, }), ValueClass::TaskQueue(TaskQueueClass::SendImip { due: self.due, is_payload: true, }), ] .into_iter() } } impl TaskLock for Task>> { fn account_id(&self) -> u32 { self.account_id } fn document_id(&self) -> u32 { self.document_id } fn lock_key(&self) -> Vec { KeySerializer::new((U32_LEN * 2) + U64_LEN + 1) .write(4u8) .write(self.due.inner()) .write_leb128(self.account_id) .write_leb128(self.document_id) .finalize() } fn lock_expiry(&self) -> u64 { ALARM_EXPIRY } fn value_classes(&self) -> impl Iterator { std::iter::once(ValueClass::TaskQueue(TaskQueueClass::MergeThreads { due: self.due, })) } } impl Task { pub(crate) fn lock_expiry(&self) -> u64 { match &self.action { TaskAction::UpdateIndex(_) => INDEX_EXPIRY, TaskAction::SendAlarm(_) => ALARM_EXPIRY, _ => ALARM_EXPIRY, } } pub fn deserialize(key: &[u8], value: &[u8]) -> trc::Result { let document_id = key.deserialize_be_u32(U64_LEN + U32_LEN + 1)?; Ok(Task { due: TaskEpoch::from_inner(key.deserialize_be_u64(0)?), account_id: key.deserialize_be_u32(U64_LEN)?, document_id, action: match key.get(U64_LEN + U32_LEN) { Some(v @ (7 | 8)) => TaskAction::UpdateIndex(IndexAction { index: key .last() .copied() .and_then(SearchIndex::try_from_u8) .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?, is_insert: *v == 7, }), Some(3) => TaskAction::SendAlarm(CalendarAlarm { event_id: key.deserialize_be_u16(U64_LEN + U32_LEN + U32_LEN + 1)?, alarm_id: key.deserialize_be_u16(U64_LEN + U32_LEN + U32_LEN + U16_LEN + 1)?, alarm_time: 0, typ: CalendarAlarmType::Email { event_start: value.deserialize_be_u64(0)? as i64, event_end: value.deserialize_be_u64(U64_LEN)? as i64, event_start_tz: value.deserialize_be_u16(U64_LEN * 2)?, event_end_tz: value.deserialize_be_u16((U64_LEN * 2) + U16_LEN)?, }, }), Some(6) => { let recurrence_id = value.deserialize_be_u64(0)? as i64; TaskAction::SendAlarm(CalendarAlarm { event_id: key.deserialize_be_u16(U64_LEN + U32_LEN + U32_LEN + 1)?, alarm_id: key .deserialize_be_u16(U64_LEN + U32_LEN + U32_LEN + U16_LEN + 1)?, alarm_time: 0, typ: CalendarAlarmType::Display { recurrence_id: if recurrence_id != 0 { Some(recurrence_id) } else { None }, }, }) } Some(4) => TaskAction::SendImip, Some(9) => { TaskAction::MergeThreads(MergeThreadIds::deserialize(value).ok_or_else( || trc::Error::corrupted_key(key, value.into(), trc::location!()), )?) } _ => return Err(trc::Error::corrupted_key(key, None, trc::location!())), }, }) } } ================================================ FILE: crates/services/src/task_manager/merge_threads.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{Server, storage::index::ObjectIndexBuilder}; use email::message::{ ingest::{MergeThreadIds, ThreadMerge}, metadata::MessageData, }; use std::time::Duration; use store::{ IndexKeyPrefix, IterateParams, U32_LEN, ValueKey, ahash::{AHashMap, AHashSet}, rand::Rng, write::{ AlignedBytes, Archive, BatchBuilder, IndexPropertyClass, ValueClass, key::DeserializeBigEndian, }, }; use trc::AddContext; use types::{ collection::{Collection, SyncCollection}, field::EmailField, }; const MAX_RETRIES: usize = 5; pub trait MergeThreadsTask: Sync + Send { fn merge_threads( &self, account_id: u32, threads: &MergeThreadIds>, ) -> impl Future + Send; } impl MergeThreadsTask for Server { async fn merge_threads( &self, account_id: u32, threads: &MergeThreadIds>, ) -> bool { match merge_threads(self, account_id, threads).await { Ok(_) => true, Err(err) => { trc::error!( err.account_id(account_id) .details("Failed to merge threads") ); false } } } } async fn merge_threads( server: &Server, account_id: u32, merge_threads: &MergeThreadIds>, ) -> trc::Result<()> { let key_len = IndexKeyPrefix::len() + merge_threads.thread_hash.len() + U32_LEN; let document_id_pos = key_len - U32_LEN; let mut thread_merge = ThreadMerge::new(); let mut thread_index = AHashMap::new(); let mut try_count = 0; 'retry: loop { // Find thread ids server .store() .iterate( IterateParams::new( ValueKey { account_id, collection: Collection::Email.into(), document_id: 0, class: ValueClass::IndexProperty(IndexPropertyClass::Hash { property: EmailField::Threading.into(), hash: merge_threads.thread_hash, }), }, ValueKey { account_id, collection: Collection::Email.into(), document_id: u32::MAX, class: ValueClass::IndexProperty(IndexPropertyClass::Hash { property: EmailField::Threading.into(), hash: merge_threads.thread_hash, }), }, ) .ascending(), |key, value| { if key.len() == key_len { let thread_id = value.deserialize_be_u32(0)?; if merge_threads.merge_ids.contains(&thread_id) { let document_id = key.deserialize_be_u32(document_id_pos)?; thread_merge.add(thread_id, document_id); thread_index.insert(document_id, value.to_vec()); } } Ok(true) }, ) .await .caused_by(trc::location!())?; if thread_merge.num_thread_ids() < 2 { // Another process merged the threads already? return Ok(()); } let thread_id = thread_merge.merge_thread_id(); // Delete all but the most common threadId let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Thread); for &delete_thread_id in thread_merge.thread_ids() { if delete_thread_id != thread_id { batch .with_document(delete_thread_id) .log_container_delete(SyncCollection::Thread); } } // Move messages to the new threadId batch.with_collection(Collection::Email); for (&group_thread_id, document_ids) in thread_merge.thread_groups() { if thread_id != group_thread_id { for &document_id in document_ids { if let Some(data_) = server .store() .get_value::>(ValueKey::archive( account_id, Collection::Email, document_id, )) .await .caused_by(trc::location!())? { let data = data_ .to_unarchived::() .caused_by(trc::location!())?; if data.inner.thread_id != group_thread_id { try_count += 1; continue 'retry; } // Update thread id let mut new_data = data .deserialize::() .caused_by(trc::location!())?; new_data.thread_id = thread_id; batch .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_current(data) .with_changes(new_data), ) .caused_by(trc::location!())?; // Update thread index property let mut thread_index = thread_index.remove(&document_id).unwrap(); thread_index[0..U32_LEN].copy_from_slice(&thread_id.to_be_bytes()); batch.set( ValueClass::IndexProperty(IndexPropertyClass::Hash { property: EmailField::Threading.into(), hash: merge_threads.thread_hash, }), thread_index, ); } } } } match server.commit_batch(batch).await { Ok(_) => return Ok(()), Err(err) if err.is_assertion_failure() && try_count < MAX_RETRIES => { let backoff = store::rand::rng().random_range(50..=300); tokio::time::sleep(Duration::from_millis(backoff)).await; try_count += 1; } Err(err) => { return Err(err.caused_by(trc::location!())); } } } } ================================================ FILE: crates/services/src/task_manager/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::task_manager::imip::SendImipTask; use crate::task_manager::index::SearchIndexTask; use crate::task_manager::lock::{TaskLock, TaskLockManager}; use crate::task_manager::merge_threads::MergeThreadsTask; use alarm::SendAlarmTask; use common::IPC_CHANNEL_BUFFER; use common::config::server::ServerProtocol; use common::listener::limiter::ConcurrencyLimiter; use common::listener::{ServerInstance, TcpAcceptor}; use common::{Inner, KV_LOCK_TASK, Server, core::BuildServer}; use email::message::ingest::MergeThreadIds; use groupware::calendar::alarm::{CalendarAlarm, CalendarAlarmType}; use std::collections::hash_map::Entry; use std::future::Future; use std::time::Duration; use std::{sync::Arc, time::Instant}; use store::ahash::AHashSet; use store::rand; use store::rand::seq::SliceRandom; use store::write::{SearchIndex, TaskEpoch}; use store::{ IterateParams, U16_LEN, U32_LEN, U64_LEN, ValueKey, ahash::AHashMap, write::{ BatchBuilder, TaskQueueClass, ValueClass, key::{DeserializeBigEndian, KeySerializer}, now, }, }; use tokio::sync::{mpsc, watch}; use trc::TaskQueueEvent; use utils::snowflake::SnowflakeIdGenerator; pub mod alarm; pub mod imip; pub mod index; pub mod lock; pub mod merge_threads; #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct Task { pub account_id: u32, pub document_id: u32, pub due: TaskEpoch, pub action: T, } #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub enum TaskAction { UpdateIndex(IndexAction), SendAlarm(CalendarAlarm), SendImip, MergeThreads(MergeThreadIds>), } #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct IndexAction { pub index: SearchIndex, pub is_insert: bool, } #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub(crate) struct ImipAction; const INDEX_EXPIRY: u64 = 60 * 5; // 5 minutes const ALARM_EXPIRY: u64 = 60 * 2; // 2 minutes const QUEUE_REFRESH_INTERVAL: u64 = 60 * 5; // 5 minutes pub(crate) struct TaskManagerIpc { tx_fts: mpsc::Sender>, tx_alarm: mpsc::Sender>, tx_imip: mpsc::Sender>, tx_threads: mpsc::Sender>>>, locked: AHashMap, Locked>, revision: u64, } struct Locked { expires: Instant, revision: u64, } pub fn spawn_task_manager(inner: Arc) { // Create mpsc channels for the different task types let (tx_index_1, mut rx_index_1) = mpsc::channel::>(IPC_CHANNEL_BUFFER); let (tx_index_2, mut rx_index_2) = mpsc::channel::>(IPC_CHANNEL_BUFFER); let (tx_index_3, mut rx_index_3) = mpsc::channel::>(IPC_CHANNEL_BUFFER); let (tx_index_4, mut rx_index_4) = mpsc::channel::>>>(IPC_CHANNEL_BUFFER); // Create dummy server instance for alarms let server_instance = Arc::new(ServerInstance { id: "_local".to_string(), protocol: ServerProtocol::Smtp, acceptor: TcpAcceptor::Plain, limiter: ConcurrencyLimiter::new(100), shutdown_rx: watch::channel(false).1, proxy_networks: vec![], span_id_gen: Arc::new(SnowflakeIdGenerator::new()), }); // Indexing worker { let inner = inner.clone(); tokio::spawn(async move { while let Some(task) = rx_index_1.recv().await { let server = inner.build_server(); let batch_size = server.core.jmap.index_batch_size; let mut batch = Vec::with_capacity(batch_size); batch.push(task); while batch.len() < batch_size { match rx_index_1.try_recv() { Ok(task) => batch.push(task), Err(_) => break, } } if batch.len() > 1 { batch.shuffle(&mut rand::rng()); } // Lock tasks let mut locked_batch = Vec::with_capacity(batch.len()); for task in batch { if server .try_lock_task( task.account_id, task.document_id, task.lock_key(), task.lock_expiry(), ) .await { locked_batch.push(task); } } // Dispatch if !locked_batch.is_empty() { let success = server.index(&locked_batch).await; if success.iter().all(|t| t.is_done()) { delete_tasks(&server, &locked_batch).await; } else { trc::event!( TaskQueue(TaskQueueEvent::TaskFailed), Total = locked_batch.len(), Details = "Indexing task failed", ); // Remove successful entries from queue let mut to_delete = Vec::with_capacity(locked_batch.len()); for (task, result) in locked_batch.into_iter().zip(success.into_iter()) { if result.is_done() { to_delete.push(task); } } if !to_delete.is_empty() { delete_tasks(&server, &to_delete).await; } } } } }); } // Send alarm worker { let inner = inner.clone(); let server_instance = server_instance.clone(); tokio::spawn(async move { while let Some(task) = rx_index_2.recv().await { let server = inner.build_server(); // Lock task if server.core.groupware.alarms_enabled && server .try_lock_task( task.account_id, task.document_id, task.lock_key(), task.lock_expiry(), ) .await { let success = server .send_alarm( task.account_id, task.document_id, &task.action, server_instance.clone(), ) .await; // Remove entry from queue if success { delete_tasks(&server, &[task]).await; } else { trc::event!( TaskQueue(TaskQueueEvent::TaskFailed), AccountId = task.account_id, DocumentId = task.document_id, Details = "Sending alarm task failed", ); } } } }); } // Send iMIP worker { let inner = inner.clone(); let server_instance = server_instance.clone(); tokio::spawn(async move { while let Some(task) = rx_index_3.recv().await { let server = inner.build_server(); // Lock task if server.core.groupware.itip_enabled && server .try_lock_task( task.account_id, task.document_id, task.lock_key(), task.lock_expiry(), ) .await { let success = server .send_imip( task.account_id, task.document_id, task.due, server_instance.clone(), ) .await; // Remove entry from queue if success { delete_tasks(&server, &[task]).await; } else { trc::event!( TaskQueue(TaskQueueEvent::TaskFailed), AccountId = task.account_id, DocumentId = task.document_id, Details = "Sending iMIP task failed", ); } } } }); } // Merge threads worker { let inner = inner.clone(); tokio::spawn(async move { while let Some(task) = rx_index_4.recv().await { let server = inner.build_server(); // Lock task if server .try_lock_task( task.account_id, task.document_id, task.lock_key(), task.lock_expiry(), ) .await { let success = server.merge_threads(task.account_id, &task.action).await; // Remove entry from queue if success { delete_tasks(&server, &[task]).await; } else { trc::event!( TaskQueue(TaskQueueEvent::TaskFailed), AccountId = task.account_id, DocumentId = task.document_id, Details = "Merging threads task failed", ); } } } }); } tokio::spawn(async move { let mut ipc = TaskManagerIpc { tx_fts: tx_index_1, tx_alarm: tx_index_2, tx_imip: tx_index_3, tx_threads: tx_index_4, locked: Default::default(), revision: 0, }; let rx = inner.ipc.task_tx.clone(); loop { // Index any queued tasks let sleep_for = inner.build_server().process_tasks(&mut ipc).await; // Wait for a signal or sleep until the next task is due let _ = tokio::time::timeout(sleep_for, rx.notified()).await; } }); } pub(crate) trait TaskQueueManager: Sync + Send { fn process_tasks(&self, ipc: &mut TaskManagerIpc) -> impl Future + Send; } impl TaskQueueManager for Server { async fn process_tasks(&self, ipc: &mut TaskManagerIpc) -> Duration { let now_timestamp = now(); let from_key = ValueKey:: { account_id: 0, collection: 0, document_id: 0, class: ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { due: TaskEpoch::from_inner(0), index: SearchIndex::Email, is_insert: true, }), }; let to_key = ValueKey:: { account_id: u32::MAX, collection: u8::MAX, document_id: u32::MAX, class: ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { due: TaskEpoch::new(now_timestamp + QUEUE_REFRESH_INTERVAL) .with_attempt(u16::MAX) .with_sequence_id(u16::MAX), index: SearchIndex::Email, is_insert: true, }), }; // Retrieve tasks pending to be processed let mut tasks = Vec::new(); let now = Instant::now(); let mut next_event = None; ipc.revision += 1; let _ = self .store() .iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { let task = Task::deserialize(key, value)?; let task_due = task.due.due(); if task_due <= now_timestamp { match ipc.locked.entry(key.to_vec()) { Entry::Occupied(mut entry) => { let locked = entry.get_mut(); if locked.expires <= now { locked.expires = Instant::now() + std::time::Duration::from_secs(task.lock_expiry() + 1); tasks.push(task); } locked.revision = ipc.revision; } Entry::Vacant(entry) => { entry.insert(Locked { expires: Instant::now() + std::time::Duration::from_secs(task.lock_expiry() + 1), revision: ipc.revision, }); tasks.push(task); } } Ok(true) } else { next_event = Some(task_due); Ok(false) } }, ) .await .map_err(|err| { trc::error!( err.caused_by(trc::location!()) .details("Failed to iterate over task queue.") ); }); if !tasks.is_empty() || !ipc.locked.is_empty() { trc::event!( TaskQueue(TaskQueueEvent::TaskAcquired), Total = tasks.len(), Details = ipc.locked.len(), ); } // Shuffle tasks if tasks.len() > 1 { tasks.shuffle(&mut rand::rng()); } // Dispatch tasks let roles = &self.core.network.roles; for event in tasks { match event.action { TaskAction::UpdateIndex(index) if roles.fts_indexing.is_enabled_for_hash(&event) => { if ipc .tx_fts .send(Task { account_id: event.account_id, document_id: event.document_id, due: event.due, action: index, }) .await .is_err() { trc::event!( Server(trc::ServerEvent::ThreadError), Details = "Error sending task.", CausedBy = trc::location!() ); } } TaskAction::SendAlarm(alarm) if roles.calendar_alerts.is_enabled_for_hash(&event) => { if ipc .tx_alarm .send(Task { account_id: event.account_id, document_id: event.document_id, due: event.due, action: alarm, }) .await .is_err() { trc::event!( Server(trc::ServerEvent::ThreadError), Details = "Error sending task.", CausedBy = trc::location!() ); } } TaskAction::SendImip if roles.imip_processing.is_enabled_for_hash(&event) => { if ipc .tx_imip .send(Task { account_id: event.account_id, document_id: event.document_id, due: event.due, action: ImipAction, }) .await .is_err() { trc::event!( Server(trc::ServerEvent::ThreadError), Details = "Error sending task.", CausedBy = trc::location!() ); } } TaskAction::MergeThreads(info) if roles.merge_threads.is_enabled_for_hash(&event) => { if ipc .tx_threads .send(Task { account_id: event.account_id, document_id: event.document_id, due: event.due, action: info, }) .await .is_err() { trc::event!( Server(trc::ServerEvent::ThreadError), Details = "Error sending task.", CausedBy = trc::location!() ); } } _ => { trc::event!( TaskQueue(TaskQueueEvent::TaskIgnored), Details = event.action.name(), AccountId = event.account_id, DocumentId = event.document_id, ); continue; } } } // Delete expired locks let now = Instant::now(); ipc.locked .retain(|_, locked| locked.expires > now && locked.revision == ipc.revision); Duration::from_secs(next_event.map_or(QUEUE_REFRESH_INTERVAL, |timestamp| { timestamp.saturating_sub(store::write::now()) })) } } async fn delete_tasks(server: &Server, tasks: &[T]) { let mut batch = BatchBuilder::new(); for task in tasks { batch .with_account_id(task.account_id()) .with_document(task.document_id()); for value in task.value_classes() { batch.clear(value); } } if let Err(err) = server.store().write(batch.build_all()).await { trc::error!(err.details("Failed to remove task(s) from queue.")); } for task in tasks { server.remove_index_lock(task.lock_key()).await; } } impl TaskAction { pub fn name(&self) -> &'static str { match self { TaskAction::UpdateIndex(_) => "UpdateIndex", TaskAction::SendAlarm(_) => "SendAlarm", TaskAction::SendImip => "SendImip", TaskAction::MergeThreads(_) => "MergeThreads", } } } ================================================ FILE: crates/smtp/Cargo.toml ================================================ [package] name = "smtp" description = "Stalwart SMTP Server" authors = [ "Stalwart Labs LLC "] repository = "https://github.com/stalwartlabs/smtp-server" homepage = "https://stalw.art/smtp" keywords = ["smtp", "email", "mail", "server"] categories = ["email"] license = "AGPL-3.0-only OR LicenseRef-SEL" version = "0.15.5" edition = "2024" [dependencies] store = { path = "../store" } types = { path = "../types" } utils = { path = "../utils" } nlp = { path = "../nlp" } directory = { path = "../directory" } common = { path = "../common" } email = { path = "../email" } spam-filter = { path = "../spam-filter" } trc = { path = "../trc" } mail-auth = { version = "0.7.1", features = ["rkyv"] } mail-send = { version = "0.5", default-features = false, features = ["cram-md5", "ring", "tls12"] } mail-parser = { version = "0.11", features = ["full_encoding"] } mail-builder = { version = "0.4" } smtp-proto = { version = "0.2", features = ["rkyv", "serde"] } sieve-rs = { version = "0.7", features = ["rkyv"] } ahash = { version = "0.8" } rustls = { version = "0.23.5", default-features = false, features = ["std", "ring", "tls12"] } rustls-pemfile = "2.0" rustls-pki-types = { version = "1" } tokio = { version = "1.47", features = ["full"] } tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } webpki-roots = { version = "1.0"} hyper = { version = "1.0.1", features = ["server", "http1", "http2"] } hyper-util = { version = "0.1.1", features = ["tokio"] } http-body-util = "0.1.0" form_urlencoded = "1.1.0" sha1 = "0.10" sha2 = "0.10.6" md5 = "0.8.0" rayon = "1.5" parking_lot = "0.12" regex = "1.7.0" blake3 = "1.3" lru-cache = "0.1.2" rand = "0.9.0" x509-parser = "0.18" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-webpki-roots", "http2"] } serde = { version = "1.0", features = ["derive", "rc"] } serde_json = "1.0" num_cpus = "1.15.0" chrono = "0.4" rkyv = { version = "0.8.10", features = ["little_endian"] } compact_str = "0.9.0" [features] test_mode = [] enterprise = [] #[[bench]] #name = "hash" #harness = false ================================================ FILE: crates/smtp/src/core/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{inbound::auth::SaslToken, queue::QueueId}; use common::{ Inner, Server, auth::AccessToken, config::smtp::auth::VerifyStrategy, listener::{ServerInstance, asn::AsnGeoLookupResult}, }; use directory::Directory; use mail_auth::{IprevOutput, SpfOutput}; use smtp_proto::request::receiver::{ BdatReceiver, DataReceiver, DummyDataReceiver, DummyLineReceiver, LineReceiver, RequestReceiver, }; use std::{ hash::Hash, net::IpAddr, sync::Arc, time::{Duration, Instant}, }; use tokio::io::{AsyncRead, AsyncWrite}; use utils::DomainPart; pub mod params; pub mod throttle; #[derive(Clone)] pub struct SmtpSessionManager { pub inner: Arc, } impl SmtpSessionManager { pub fn new(inner: Arc) -> Self { Self { inner } } } pub enum State { Request(RequestReceiver), Bdat(BdatReceiver), Data(DataReceiver), Sasl(LineReceiver), DataTooLarge(DummyDataReceiver), RequestTooLarge(DummyLineReceiver), Accepted(QueueId), None, } pub struct Session { pub hostname: String, pub state: State, pub instance: Arc, pub server: Server, pub stream: T, pub data: SessionData, pub params: SessionParameters, } pub struct SessionData { pub session_id: u64, pub local_ip: IpAddr, pub local_ip_str: String, pub local_port: u16, pub remote_ip: IpAddr, pub remote_ip_str: String, pub remote_port: u16, pub asn_geo_data: AsnGeoLookupResult, pub helo_domain: String, pub mail_from: Option, pub rcpt_to: Vec, pub rcpt_errors: usize, pub rcpt_oks: usize, pub message: Vec, pub authenticated_as: Option>, pub auth_errors: usize, pub priority: i16, pub delivery_by: i64, pub future_release: u64, pub valid_until: Instant, pub bytes_left: usize, pub messages_sent: usize, pub iprev: Option, pub spf_ehlo: Option, pub spf_mail_from: Option, pub dnsbl_error: Option>, } #[derive(Clone, Debug)] pub struct SessionAddress { pub address: String, pub address_lcase: String, pub domain: String, pub flags: u64, pub dsn_info: Option, } #[derive(Debug, Default)] pub struct SessionParameters { // Global parameters pub timeout: Duration, // Ehlo parameters pub ehlo_require: bool, pub ehlo_reject_non_fqdn: bool, // Auth parameters pub auth_directory: Option>, pub auth_require: bool, pub auth_errors_max: usize, pub auth_errors_wait: Duration, // Rcpt parameters pub rcpt_errors_max: usize, pub rcpt_errors_wait: Duration, pub rcpt_max: usize, pub rcpt_dsn: bool, pub can_expn: bool, pub can_vrfy: bool, pub max_message_size: usize, // Mail authentication parameters pub iprev: VerifyStrategy, pub spf_ehlo: VerifyStrategy, pub spf_mail_from: VerifyStrategy, } impl SessionData { pub fn new( local_ip: IpAddr, local_port: u16, remote_ip: IpAddr, remote_port: u16, asn_geo_data: AsnGeoLookupResult, session_id: u64, ) -> Self { SessionData { session_id, local_ip, local_port, remote_ip, local_ip_str: local_ip.to_string(), remote_ip_str: remote_ip.to_string(), remote_port, asn_geo_data, helo_domain: String::new(), mail_from: None, rcpt_to: Vec::new(), authenticated_as: None, priority: 0, valid_until: Instant::now(), rcpt_errors: 0, rcpt_oks: 0, message: Vec::with_capacity(0), auth_errors: 0, messages_sent: 0, bytes_left: 0, delivery_by: 0, future_release: 0, iprev: None, spf_ehlo: None, spf_mail_from: None, dnsbl_error: None, } } } impl Default for State { fn default() -> Self { State::Request(RequestReceiver::default()) } } impl PartialEq for SessionAddress { fn eq(&self, other: &Self) -> bool { self.address_lcase == other.address_lcase } } impl Eq for SessionAddress {} impl Hash for SessionAddress { fn hash(&self, state: &mut H) { self.address_lcase.hash(state); } } impl Ord for SessionAddress { fn cmp(&self, other: &Self) -> std::cmp::Ordering { match self.domain.cmp(&other.domain) { std::cmp::Ordering::Equal => self.address_lcase.cmp(&other.address_lcase), order => order, } } } impl PartialOrd for SessionAddress { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } impl Session { pub fn local( server: Server, instance: std::sync::Arc, data: SessionData, ) -> Self { Session { hostname: "localhost".into(), state: State::None, instance, server, stream: common::listener::stream::NullIo::default(), data, params: SessionParameters { timeout: Default::default(), ehlo_require: Default::default(), ehlo_reject_non_fqdn: Default::default(), auth_directory: Default::default(), auth_require: Default::default(), auth_errors_max: Default::default(), auth_errors_wait: Default::default(), rcpt_errors_max: Default::default(), rcpt_errors_wait: Default::default(), rcpt_max: Default::default(), rcpt_dsn: Default::default(), max_message_size: Default::default(), iprev: VerifyStrategy::Disable, spf_ehlo: VerifyStrategy::Disable, spf_mail_from: VerifyStrategy::Disable, can_expn: false, can_vrfy: false, }, } } pub fn has_failed(&mut self) -> Option { if self.stream.tx_buf.first().is_none_or(|&c| c == b'2') { self.stream.tx_buf.clear(); None } else { let response = std::str::from_utf8(&self.stream.tx_buf) .unwrap() .trim() .into(); self.stream.tx_buf.clear(); Some(response) } } } impl SessionData { pub fn local( authenticated_as: Arc, mail_from: Option, rcpt_to: Vec, message: Vec, session_id: u64, ) -> Self { SessionData { local_ip: IpAddr::V4(std::net::Ipv4Addr::new(127, 0, 0, 1)), remote_ip: IpAddr::V4(std::net::Ipv4Addr::new(127, 0, 0, 1)), local_ip_str: "127.0.0.1".into(), remote_ip_str: "127.0.0.1".into(), remote_port: 0, local_port: 0, session_id, asn_geo_data: AsnGeoLookupResult::default(), helo_domain: "localhost".into(), mail_from, rcpt_to, rcpt_errors: 0, rcpt_oks: 0, message, authenticated_as: Some(authenticated_as), auth_errors: 0, priority: 0, delivery_by: 0, future_release: 0, valid_until: Instant::now(), bytes_left: 0, messages_sent: 0, iprev: None, spf_ehlo: None, spf_mail_from: None, dnsbl_error: None, } } } impl Default for SessionData { fn default() -> Self { Self::local(Arc::new(AccessToken::from_id(0)), None, vec![], vec![], 0) } } impl SessionAddress { pub fn new(address: String) -> Self { let address_lcase = address.to_lowercase(); SessionAddress { domain: address_lcase.domain_part().into(), address_lcase, address, flags: 0, dsn_info: None, } } } ================================================ FILE: crates/smtp/src/core/params.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use common::{config::smtp::auth::VerifyStrategy, listener::SessionStream}; use super::Session; impl Session { pub async fn eval_session_params(&mut self) { let c = &self.server.core.smtp.session; self.data.bytes_left = self .server .eval_if(&c.transfer_limit, self, self.data.session_id) .await .unwrap_or(250 * 1024 * 1024); self.data.valid_until += self .server .eval_if(&c.duration, self, self.data.session_id) .await .unwrap_or_else(|| Duration::from_secs(15 * 60)); self.params.timeout = self .server .eval_if(&c.timeout, self, self.data.session_id) .await .unwrap_or_else(|| Duration::from_secs(5 * 60)); self.params.spf_ehlo = self .server .eval_if( &self.server.core.smtp.mail_auth.spf.verify_ehlo, self, self.data.session_id, ) .await .unwrap_or(VerifyStrategy::Relaxed); self.params.spf_mail_from = self .server .eval_if( &self.server.core.smtp.mail_auth.spf.verify_mail_from, self, self.data.session_id, ) .await .unwrap_or(VerifyStrategy::Relaxed); self.params.iprev = self .server .eval_if( &self.server.core.smtp.mail_auth.iprev.verify, self, self.data.session_id, ) .await .unwrap_or(VerifyStrategy::Relaxed); // Ehlo parameters let ec = &self.server.core.smtp.session.ehlo; self.params.ehlo_require = self .server .eval_if(&ec.require, self, self.data.session_id) .await .unwrap_or(true); self.params.ehlo_reject_non_fqdn = self .server .eval_if(&ec.reject_non_fqdn, self, self.data.session_id) .await .unwrap_or(true); // Auth parameters let ac = &self.server.core.smtp.session.auth; self.params.auth_directory = self .server .eval_if::(&ac.directory, self, self.data.session_id) .await .and_then(|name| self.server.get_directory(&name)) .cloned(); self.params.auth_require = self .server .eval_if(&ac.require, self, self.data.session_id) .await .unwrap_or(false); self.params.auth_errors_max = self .server .eval_if(&ac.errors_max, self, self.data.session_id) .await .unwrap_or(3); self.params.auth_errors_wait = self .server .eval_if(&ac.errors_wait, self, self.data.session_id) .await .unwrap_or_else(|| Duration::from_secs(30)); // VRFY/EXPN parameters let ec = &self.server.core.smtp.session.extensions; self.params.can_expn = self .server .eval_if(&ec.expn, self, self.data.session_id) .await .unwrap_or(false); self.params.can_vrfy = self .server .eval_if(&ec.vrfy, self, self.data.session_id) .await .unwrap_or(false); } pub async fn eval_post_auth_params(&mut self) { // Refresh VRFY/EXPN parameters let ec = &self.server.core.smtp.session.extensions; self.params.can_expn = self .server .eval_if(&ec.expn, self, self.data.session_id) .await .unwrap_or(false); self.params.can_vrfy = self .server .eval_if(&ec.vrfy, self, self.data.session_id) .await .unwrap_or(false); } pub async fn eval_rcpt_params(&mut self) { let rc = &self.server.core.smtp.session.rcpt; self.params.rcpt_errors_max = self .server .eval_if(&rc.errors_max, self, self.data.session_id) .await .unwrap_or(10); self.params.rcpt_errors_wait = self .server .eval_if(&rc.errors_wait, self, self.data.session_id) .await .unwrap_or_else(|| Duration::from_secs(30)); self.params.rcpt_max = self .server .eval_if(&rc.max_recipients, self, self.data.session_id) .await .unwrap_or(100); self.params.rcpt_dsn = self .server .eval_if( &self.server.core.smtp.session.extensions.dsn, self, self.data.session_id, ) .await .unwrap_or(true); self.params.max_message_size = self .server .eval_if( &self.server.core.smtp.session.data.max_message_size, self, self.data.session_id, ) .await .unwrap_or(25 * 1024 * 1024); } } ================================================ FILE: crates/smtp/src/core/throttle.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{ KV_RATE_LIMIT_SMTP, ThrottleKey, config::smtp::*, expr::{functions::ResolveVariable, *}, listener::SessionStream, }; use queue::QueueQuota; use trc::SmtpEvent; use utils::config::Rate; use super::Session; pub trait NewKey: Sized { fn new_key(&self, e: &impl ResolveVariable, context: &str) -> ThrottleKey; } impl NewKey for QueueQuota { fn new_key(&self, e: &impl ResolveVariable, _: &str) -> ThrottleKey { let mut hasher = blake3::Hasher::new(); if (self.keys & THROTTLE_RCPT) != 0 { hasher.update(e.resolve_variable(V_RECIPIENT).to_string().as_bytes()); } if (self.keys & THROTTLE_RCPT_DOMAIN) != 0 { hasher.update( e.resolve_variable(V_RECIPIENT_DOMAIN) .to_string() .as_bytes(), ); } if (self.keys & THROTTLE_SENDER) != 0 { let sender = e.resolve_variable(V_SENDER).into_string(); hasher.update( if !sender.is_empty() { sender.as_ref() } else { "<>" } .as_bytes(), ); } if (self.keys & THROTTLE_SENDER_DOMAIN) != 0 { let sender_domain = e.resolve_variable(V_SENDER_DOMAIN).into_string(); hasher.update( if !sender_domain.is_empty() { sender_domain.as_ref() } else { "<>" } .as_bytes(), ); } if let Some(messages) = &self.messages { hasher.update(&messages.to_ne_bytes()[..]); } if let Some(size) = &self.size { hasher.update(&size.to_ne_bytes()[..]); } ThrottleKey { hash: hasher.finalize().into(), } } } impl NewKey for QueueRateLimiter { fn new_key(&self, e: &impl ResolveVariable, context: &str) -> ThrottleKey { let mut hasher = blake3::Hasher::new(); if (self.keys & THROTTLE_RCPT) != 0 { hasher.update(e.resolve_variable(V_RECIPIENT).to_string().as_bytes()); } if (self.keys & THROTTLE_RCPT_DOMAIN) != 0 { hasher.update( e.resolve_variable(V_RECIPIENT_DOMAIN) .to_string() .as_bytes(), ); } if (self.keys & THROTTLE_SENDER) != 0 { let sender = e.resolve_variable(V_SENDER).into_string(); hasher.update( if !sender.is_empty() { sender.as_ref() } else { "<>" } .as_bytes(), ); } if (self.keys & THROTTLE_SENDER_DOMAIN) != 0 { let sender_domain = e.resolve_variable(V_SENDER_DOMAIN).into_string(); hasher.update( if !sender_domain.is_empty() { sender_domain.as_ref() } else { "<>" } .as_bytes(), ); } if (self.keys & THROTTLE_HELO_DOMAIN) != 0 { hasher.update(e.resolve_variable(V_HELO_DOMAIN).to_string().as_bytes()); } if (self.keys & THROTTLE_AUTH_AS) != 0 { hasher.update( e.resolve_variable(V_AUTHENTICATED_AS) .to_string() .as_bytes(), ); } if (self.keys & THROTTLE_LISTENER) != 0 { hasher.update(e.resolve_variable(V_LISTENER).to_string().as_bytes()); } if (self.keys & THROTTLE_MX) != 0 { hasher.update(e.resolve_variable(V_MX).to_string().as_bytes()); } if (self.keys & THROTTLE_REMOTE_IP) != 0 { hasher.update(e.resolve_variable(V_REMOTE_IP).to_string().as_bytes()); } if (self.keys & THROTTLE_LOCAL_IP) != 0 { hasher.update(e.resolve_variable(V_LOCAL_IP).to_string().as_bytes()); } hasher.update(&self.rate.period.as_secs().to_be_bytes()[..]); hasher.update(&self.rate.requests.to_be_bytes()[..]); hasher.update(context.as_bytes()); ThrottleKey { hash: hasher.finalize().into(), } } } impl Session { pub async fn is_allowed(&mut self) -> bool { let throttles = if !self.data.rcpt_to.is_empty() { &self.server.core.smtp.queue.inbound_limiters.rcpt } else if self.data.mail_from.is_some() { &self.server.core.smtp.queue.inbound_limiters.sender } else { &self.server.core.smtp.queue.inbound_limiters.remote }; for t in throttles { if t.expr.is_empty() || self .server .eval_expr(&t.expr, self, "throttle", self.data.session_id) .await .unwrap_or(false) { if (t.keys & THROTTLE_RCPT_DOMAIN) != 0 { let d = self .data .rcpt_to .last() .map(|r| r.domain.as_str()) .unwrap_or_default(); if self.data.rcpt_to.iter().filter(|p| p.domain == d).count() > 1 { continue; } } // Build throttle key let key = t.new_key(self, "inbound"); // Check rate match self .server .core .storage .lookup .is_rate_allowed(KV_RATE_LIMIT_SMTP, key.hash.as_slice(), &t.rate, false) .await { Ok(Some(_)) => { trc::event!( Smtp(SmtpEvent::RateLimitExceeded), SpanId = self.data.session_id, Id = t.id.clone(), Limit = vec![ trc::Value::from(t.rate.requests), trc::Value::from(t.rate.period) ], ); return false; } Err(err) => { trc::error!( err.span_id(self.data.session_id) .caused_by(trc::location!()) ); } _ => (), } } } true } pub async fn throttle_rcpt(&self, rcpt: &str, rate: &Rate, ctx: &str) -> bool { let mut hasher = blake3::Hasher::new(); hasher.update(rcpt.as_bytes()); hasher.update(ctx.as_bytes()); hasher.update(&rate.period.as_secs().to_ne_bytes()[..]); hasher.update(&rate.requests.to_ne_bytes()[..]); match self .server .core .storage .lookup .is_rate_allowed( KV_RATE_LIMIT_SMTP, hasher.finalize().as_bytes(), rate, false, ) .await { Ok(None) => true, Ok(Some(_)) => false, Err(err) => { trc::error!( err.span_id(self.data.session_id) .caused_by(trc::location!()) ); true } } } } ================================================ FILE: crates/smtp/src/inbound/auth.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{ auth::{ AuthRequest, sasl::{sasl_decode_challenge_oauth, sasl_decode_challenge_plain}, }, listener::SessionStream, }; use directory::Permission; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; use smtp_proto::{AUTH_LOGIN, AUTH_OAUTHBEARER, AUTH_PLAIN, AUTH_XOAUTH2, IntoString}; use trc::{AuthEvent, SmtpEvent}; use crate::core::Session; pub struct SaslToken { mechanism: u64, credentials: Credentials, } impl SaslToken { pub fn from_mechanism(mechanism: u64) -> Option { match mechanism { AUTH_PLAIN | AUTH_LOGIN => SaslToken { mechanism, credentials: Credentials::Plain { username: String::new(), secret: String::new(), }, } .into(), AUTH_OAUTHBEARER | AUTH_XOAUTH2 => SaslToken { mechanism, credentials: Credentials::OAuthBearer { token: String::new(), }, } .into(), _ => None, } } } impl Session { pub async fn handle_sasl_response( &mut self, token: &mut SaslToken, response: &[u8], ) -> Result { if response.is_empty() { match (token.mechanism, &token.credentials) { (AUTH_PLAIN | AUTH_XOAUTH2 | AUTH_OAUTHBEARER, _) => { self.write(b"334 Go ahead.\r\n").await?; return Ok(true); } (AUTH_LOGIN, Credentials::Plain { username, secret }) => { if username.is_empty() && secret.is_empty() { self.write(b"334 VXNlcm5hbWU6\r\n").await?; return Ok(true); } } _ => (), } } else if let Some(response) = base64_decode(response) { match (token.mechanism, &mut token.credentials) { (AUTH_PLAIN, _) => { if let Some(credentials) = sasl_decode_challenge_plain(&response) { return self.authenticate(credentials).await; } } (AUTH_LOGIN, Credentials::Plain { username, secret }) => { return if username.is_empty() { *username = response.into_string(); self.write(b"334 UGFzc3dvcmQ6\r\n").await?; Ok(true) } else { *secret = response.into_string(); self.authenticate(std::mem::take(&mut token.credentials)) .await }; } (AUTH_OAUTHBEARER | AUTH_XOAUTH2, _) => { if let Some(credentials) = sasl_decode_challenge_oauth(&response) { return self.authenticate(credentials).await; } } _ => (), } } self.auth_error(b"500 5.5.6 Invalid challenge.\r\n").await } pub async fn authenticate(&mut self, credentials: Credentials) -> Result { if let Some(directory) = &self.params.auth_directory { // Authenticate let result = self .server .authenticate( &AuthRequest::from_credentials( credentials, self.data.session_id, self.data.remote_ip, ) .with_directory(directory), ) .await .and_then(|access_token| { access_token .assert_has_permission(Permission::EmailSend) .map(|_| access_token) }); match result { Ok(access_token) => { self.data.authenticated_as = access_token.into(); self.eval_post_auth_params().await; self.write(b"235 2.7.0 Authentication succeeded.\r\n") .await?; return Ok(false); } Err(err) => { let reason = *err.as_ref(); trc::error!(err.span_id(self.data.session_id)); match reason { trc::EventType::Auth(trc::AuthEvent::Failed) => { return self .auth_error(b"535 5.7.8 Authentication credentials invalid.\r\n") .await; } trc::EventType::Auth(trc::AuthEvent::TokenExpired) => { return self.auth_error(b"535 5.7.8 OAuth token expired.\r\n").await; } trc::EventType::Auth(trc::AuthEvent::MissingTotp) => { return self .auth_error( b"334 5.7.8 Missing TOTP token, try with 'secret$totp_code'.\r\n", ) .await; } trc::EventType::Security(trc::SecurityEvent::Unauthorized) => { self.write( concat!( "550 5.7.1 Your account is not authorized ", "to use this service.\r\n" ) .as_bytes(), ) .await?; return Ok(false); } trc::EventType::Security(_) => { return Err(()); } _ => (), } } } } else { trc::event!( Smtp(SmtpEvent::MissingAuthDirectory), SpanId = self.data.session_id, ); } self.write(b"454 4.7.0 Temporary authentication failure\r\n") .await?; Ok(false) } pub async fn auth_error(&mut self, response: &[u8]) -> Result { tokio::time::sleep(self.params.auth_errors_wait).await; self.data.auth_errors += 1; self.write(response).await?; if self.data.auth_errors < self.params.auth_errors_max { Ok(false) } else { trc::event!( Auth(AuthEvent::TooManyAttempts), SpanId = self.data.session_id, ); self.write(b"455 4.3.0 Too many authentication errors, disconnecting.\r\n") .await?; Err(()) } } pub fn authenticated_as(&self) -> Option<&str> { self.data.authenticated_as.as_ref().map(|token| { if !token.name.is_empty() { token.name.as_str() } else { "unavailable" } }) } pub fn is_authenticated(&self) -> bool { self.data.authenticated_as.is_some() } pub fn authenticated_emails(&self) -> &[String] { self.data .authenticated_as .as_ref() .map(|token| token.emails.as_slice()) .unwrap_or_default() } } ================================================ FILE: crates/smtp/src/inbound/data.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ArcSeal, AuthResult, DkimSign}; use crate::{ core::{Session, SessionAddress, State}, inbound::milter::Modification, queue::{ self, Message, MessageSource, MessageWrapper, QueueEnvelope, RCPT_SPAM_PAYLOAD, quota::HasQueueQuota, }, reporting::analysis::AnalyzeReport, scripts::ScriptResult, }; use common::{ config::{ smtp::{ auth::VerifyStrategy, queue::{QueueExpiry, QueueName}, session::Stage, }, spamfilter::SpamFilterAction, }, listener::SessionStream, psl, scripts::ScriptModification, }; use mail_auth::{ AuthenticatedMessage, AuthenticationResults, DkimResult, DmarcResult, ReceivedSpf, common::{headers::HeaderWriter, verify::VerifySignature}, dmarc::{self, verify::DmarcParameters}, }; use mail_builder::headers::{date::Date, message_id::generate_message_id_header}; use mail_parser::MessageParser; use sieve::runtime::Variable; use smtp_proto::{ MAIL_BY_RETURN, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, }; use std::{ borrow::Cow, time::{Instant, SystemTime}, }; use trc::SmtpEvent; use utils::{DomainPart, config::Rate}; impl Session { pub async fn queue_message(&mut self) -> Cow<'static, [u8]> { // Parse message let raw_message = std::mem::take(&mut self.data.message); let parsed_message = match MessageParser::new() .parse(&raw_message) .filter(|p| p.headers().iter().any(|h| !h.name.is_other())) { Some(parsed_message) => parsed_message, None => { trc::event!( Smtp(SmtpEvent::MessageParseFailed), SpanId = self.data.session_id, ); return (&b"550 5.7.7 Failed to parse message.\r\n"[..]).into(); } }; // Authenticate message let auth_message = AuthenticatedMessage::from_parsed( &parsed_message, self.server.core.smtp.mail_auth.dkim.strict, ); let has_date_header = auth_message.has_date_header(); let has_message_id_header = auth_message.has_message_id_header(); // Loop detection let dc = &self.server.core.smtp.session.data; let ac = &self.server.core.smtp.mail_auth; let rc = &self.server.core.smtp.report; if auth_message.received_headers_count() > self .server .eval_if(&dc.max_received_headers, self, self.data.session_id) .await .unwrap_or(50) { trc::event!( Smtp(SmtpEvent::LoopDetected), SpanId = self.data.session_id, Total = auth_message.received_headers_count(), ); return (&b"450 4.4.6 Too many Received headers. Possible loop detected.\r\n"[..]) .into(); } // Verify DKIM let dkim = self .server .eval_if(&ac.dkim.verify, self, self.data.session_id) .await .unwrap_or(VerifyStrategy::Relaxed); let dmarc = self .server .eval_if(&ac.dmarc.verify, self, self.data.session_id) .await .unwrap_or(VerifyStrategy::Relaxed); let dkim_output = if dkim.verify() || dmarc.verify() { let time = Instant::now(); let dkim_output = self .server .core .smtp .resolvers .dns .verify_dkim(self.server.inner.cache.build_auth_parameters(&auth_message)) .await; let pass = dkim_output .iter() .any(|d| matches!(d.result(), DkimResult::Pass)); let strict = dkim.is_strict(); let rejected = strict && !pass; // Send reports for failed signatures if let Some(rate) = self .server .eval_if::(&rc.dkim.send, self, self.data.session_id) .await { for output in &dkim_output { if let Some(rcpt) = output.failure_report_addr() { self.send_dkim_report(rcpt, &auth_message, &rate, rejected, output) .await; } } } trc::event!( Smtp(if pass { SmtpEvent::DkimPass } else { SmtpEvent::DkimFail }), SpanId = self.data.session_id, Strict = strict, Result = dkim_output.iter().map(trc::Error::from).collect::>(), Elapsed = time.elapsed(), ); if rejected { // 'Strict' mode violates the advice of Section 6.1 of RFC6376 return if dkim_output .iter() .any(|d| matches!(d.result(), DkimResult::TempError(_))) { (&b"451 4.7.20 No passing DKIM signatures found.\r\n"[..]).into() } else { (&b"550 5.7.20 No passing DKIM signatures found.\r\n"[..]).into() }; } dkim_output } else { vec![] }; // Verify ARC let arc = self .server .eval_if(&ac.arc.verify, self, self.data.session_id) .await .unwrap_or(VerifyStrategy::Relaxed); let arc_sealer = self .server .eval_if::(&ac.arc.seal, self, self.data.session_id) .await .and_then(|name| self.server.get_arc_sealer(&name, self.data.session_id)); let arc_output = if arc.verify() || arc_sealer.is_some() { let time = Instant::now(); let arc_output = self .server .core .smtp .resolvers .dns .verify_arc(self.server.inner.cache.build_auth_parameters(&auth_message)) .await; let strict = arc.is_strict(); let pass = matches!(arc_output.result(), DkimResult::Pass | DkimResult::None); trc::event!( Smtp(if pass { SmtpEvent::ArcPass } else { SmtpEvent::ArcFail }), SpanId = self.data.session_id, Strict = strict, Result = trc::Error::from(arc_output.result()), Elapsed = time.elapsed(), ); if strict && !pass { return if matches!(arc_output.result(), DkimResult::TempError(_)) { (&b"451 4.7.29 ARC validation failed.\r\n"[..]).into() } else { (&b"550 5.7.29 ARC validation failed.\r\n"[..]).into() }; } arc_output.into() } else { None }; // Build authentication results header let mail_from = self.data.mail_from.as_ref().unwrap(); let mut auth_results = AuthenticationResults::new(&self.hostname); if !dkim_output.is_empty() { auth_results = auth_results.with_dkim_results(&dkim_output, auth_message.from()) } if let Some(spf_ehlo) = &self.data.spf_ehlo { auth_results = auth_results.with_spf_ehlo_result( spf_ehlo, self.data.remote_ip, &self.data.helo_domain, ); } if let Some(spf_mail_from) = &self.data.spf_mail_from { auth_results = auth_results.with_spf_mailfrom_result( spf_mail_from, self.data.remote_ip, &mail_from.address, &self.data.helo_domain, ); } if let Some(iprev) = &self.data.iprev { auth_results = auth_results.with_iprev_result(iprev, self.data.remote_ip); } // Verify DMARC let is_report = self.is_report(); let (dmarc_result, dmarc_policy) = match &self.data.spf_mail_from { Some(spf_output) if dmarc.verify() => { let time = Instant::now(); let dmarc_output = self.server .core .smtp .resolvers .dns .verify_dmarc(self.server.inner.cache.build_auth_parameters( DmarcParameters { message: &auth_message, dkim_output: &dkim_output, rfc5321_mail_from_domain: if !mail_from.domain.is_empty() { &mail_from.domain } else { &self.data.helo_domain }, spf_output, domain_suffix_fn: |domain| { psl::domain_str(domain).unwrap_or(domain) }, }, )) .await; let pass = matches!(dmarc_output.spf_result(), DmarcResult::Pass) || matches!(dmarc_output.dkim_result(), DmarcResult::Pass); let strict = dmarc.is_strict(); let rejected = strict && dmarc_output.policy() == dmarc::Policy::Reject && !pass; let is_temp_fail = rejected && matches!(dmarc_output.spf_result(), DmarcResult::TempError(_)) || matches!(dmarc_output.dkim_result(), DmarcResult::TempError(_)); // Add to DMARC output to the Authentication-Results header auth_results = auth_results.with_dmarc_result(&dmarc_output); let dmarc_result = if pass { DmarcResult::Pass } else if dmarc_output.spf_result() != &DmarcResult::None { dmarc_output.spf_result().clone() } else if dmarc_output.dkim_result() != &DmarcResult::None { dmarc_output.dkim_result().clone() } else { DmarcResult::None }; let dmarc_policy = dmarc_output.policy(); trc::event!( Smtp(if pass { SmtpEvent::DmarcPass } else { SmtpEvent::DmarcFail }), SpanId = self.data.session_id, Strict = strict, Domain = dmarc_output.domain().to_string(), Policy = dmarc_policy.to_string(), Result = trc::Error::from(&dmarc_result), Elapsed = time.elapsed(), ); // Send DMARC report if dmarc_output.requested_reports() && !is_report { self.send_dmarc_report( &auth_message, &auth_results, rejected, dmarc_output, &dkim_output, &arc_output, ) .await; } if rejected { return if is_temp_fail { (&b"451 4.7.1 Email temporarily rejected per DMARC policy.\r\n"[..]).into() } else { (&b"550 5.7.1 Email rejected per DMARC policy.\r\n"[..]).into() }; } (dmarc_result.into(), dmarc_policy.into()) } _ => (None, None), }; // Analyze reports if is_report { if !rc.analysis.forward { self.server.analyze_report( mail_parser::Message { html_body: parsed_message.html_body, text_body: parsed_message.text_body, attachments: parsed_message.attachments, parts: parsed_message .parts .into_iter() .map(|p| p.into_owned()) .collect(), raw_message: b"".into(), }, self.data.session_id, ); self.data.messages_sent += 1; return (b"250 2.0.0 Message queued for delivery.\r\n"[..]).into(); } else { self.server.analyze_report( mail_parser::Message { html_body: parsed_message.html_body.clone(), text_body: parsed_message.text_body.clone(), attachments: parsed_message.attachments.clone(), parts: parsed_message .parts .iter() .map(|p| p.clone().into_owned()) .collect(), raw_message: b"".into(), }, self.data.session_id, ); } } // Add Received header let message_id = self.server.inner.data.queue_id_gen.generate(); let mut headers = Vec::with_capacity(64); if self .server .eval_if(&dc.add_received, self, self.data.session_id) .await .unwrap_or(true) { self.write_received(&mut headers, message_id) } // Add authentication results header if self .server .eval_if(&dc.add_auth_results, self, self.data.session_id) .await .unwrap_or(true) { auth_results.write_header(&mut headers); } // Add Received-SPF header if let Some(spf_output) = &self.data.spf_mail_from && self .server .eval_if(&dc.add_received_spf, self, self.data.session_id) .await .unwrap_or(true) { ReceivedSpf::new( spf_output, self.data.remote_ip, &self.data.helo_domain, &mail_from.address_lcase, &self.hostname, ) .write_header(&mut headers); } // ARC Seal if let (Some(arc_sealer), Some(arc_output)) = (arc_sealer, &arc_output) && !dkim_output.is_empty() && arc_output.can_be_sealed() { match arc_sealer.seal(&auth_message, &auth_results, arc_output) { Ok(set) => { set.write_header(&mut headers); } Err(err) => { trc::error!( trc::Error::from(err) .span_id(self.data.session_id) .details("Failed to ARC seal message") ); } } } // Run SPAM filter let mut train_spam = None; if self.server.core.spam.enabled && self .server .eval_if(&dc.spam_filter, self, self.data.session_id) .await .unwrap_or(true) { match self .spam_classify( &parsed_message, &dkim_output, (&arc_output).into(), dmarc_result.as_ref(), dmarc_policy.as_ref(), ) .await { SpamFilterAction::Allow(score) => { // Add headers headers.extend_from_slice(score.headers.as_bytes()); train_spam = score.train_spam; // Add scores for local recipients for (is_spam, recipient) in score.results.into_iter().zip(self.data.rcpt_to.iter_mut()) { if is_spam { recipient.flags |= RCPT_SPAM_PAYLOAD; } } } SpamFilterAction::Discard => { self.data.messages_sent += 1; return (b"250 2.0.0 Message queued for delivery.\r\n"[..]).into(); } SpamFilterAction::Reject => { self.data.messages_sent += 1; return (b"550 5.7.1 Message rejected due to excessive spam score.\r\n"[..]) .into(); } SpamFilterAction::Disabled => {} } } // Run Milter filters let mut modifications = Vec::new(); match self.run_milters(Stage::Data, (&auth_message).into()).await { Ok(modifications_) => { if !modifications_.is_empty() { modifications = modifications_; } } Err(response) => { return response.into_bytes(); } }; // Run MTA Hooks match self .run_mta_hooks(Stage::Data, (&auth_message).into(), message_id.into()) .await { Ok(modifications_) => { if !modifications_.is_empty() { modifications.retain(|m| !matches!(m, Modification::ReplaceBody { .. })); modifications.extend(modifications_); } } Err(response) => { return response.into_bytes(); } }; // Apply modifications let mut edited_message = if !modifications.is_empty() { self.data .apply_milter_modifications(modifications, &auth_message) } else { None }; // Sieve filtering if let Some((script, script_id)) = self .server .eval_if::(&dc.script, self, self.data.session_id) .await .and_then(|name| { self.server .get_trusted_sieve_script(&name, self.data.session_id) .map(|s| (s, name)) }) { let params = self .build_script_parameters("data") .with_auth_headers(&headers) .set_variable( "arc.result", arc_output .as_ref() .map(|a| a.result().as_str()) .unwrap_or_default(), ) .set_variable( "dkim.result", dkim_output .iter() .find(|r| matches!(r.result(), DkimResult::Pass)) .or_else(|| dkim_output.first()) .map(|r| r.result().as_str()) .unwrap_or_default(), ) .set_variable( "dkim.domains", dkim_output .iter() .filter_map(|r| { if matches!(r.result(), DkimResult::Pass) { r.signature() .map(|s| Variable::from(s.domain().to_lowercase())) } else { None } }) .collect::>(), ) .set_variable( "dmarc.result", dmarc_result .as_ref() .map(|a| a.as_str()) .unwrap_or_default(), ) .set_variable( "dmarc.policy", dmarc_policy .as_ref() .map(|a| a.as_str()) .unwrap_or_default(), ) .with_message(parsed_message); let modifications = match self.run_script(script_id, script.clone(), params).await { ScriptResult::Accept { modifications } => modifications, ScriptResult::Replace { message, modifications, } => { edited_message = message.into(); modifications } ScriptResult::Reject(message) => { return message.as_bytes().to_vec().into(); } ScriptResult::Discard => { return (b"250 2.0.0 Message queued for delivery.\r\n"[..]).into(); } }; // Apply modifications for modification in modifications { match modification { ScriptModification::AddHeader { name, value } => { headers.extend_from_slice(name.as_bytes()); headers.extend_from_slice(b": "); headers.extend_from_slice(value.as_bytes()); if !value.ends_with('\n') { headers.extend_from_slice(b"\r\n"); } } ScriptModification::SetEnvelope { name, value } => { self.data.apply_envelope_modification(name, value); } } } } // Build message let mail_from = self.data.mail_from.clone().unwrap(); let rcpt_to = std::mem::take(&mut self.data.rcpt_to); let mut message = self .build_message(mail_from, rcpt_to, message_id, self.data.session_id) .await; // Add Return-Path if self .server .eval_if(&dc.add_return_path, self, self.data.session_id) .await .unwrap_or(true) { headers.extend_from_slice(b"Return-Path: <"); headers.extend_from_slice(message.message.return_path.as_bytes()); headers.extend_from_slice(b">\r\n"); } // Add any missing headers if !has_date_header && self .server .eval_if(&dc.add_date, self, self.data.session_id) .await .unwrap_or(true) { headers.extend_from_slice(b"Date: "); headers.extend_from_slice(Date::now().to_rfc822().as_bytes()); headers.extend_from_slice(b"\r\n"); } if !has_message_id_header && self .server .eval_if(&dc.add_message_id, self, self.data.session_id) .await .unwrap_or(true) { headers.extend_from_slice(b"Message-ID: "); let _ = generate_message_id_header(&mut headers, &self.hostname); headers.extend_from_slice(b"\r\n"); } // DKIM sign let raw_message = edited_message.as_deref().unwrap_or(raw_message.as_slice()); for signer in self .server .eval_if::, _>(&ac.dkim.sign, self, self.data.session_id) .await .unwrap_or_default() { if let Some(signer) = self.server.get_dkim_signer(&signer, self.data.session_id) { match signer.sign_chained(&[headers.as_ref(), raw_message]) { Ok(signature) => { signature.write_header(&mut headers); } Err(err) => { trc::error!( trc::Error::from(err) .span_id(self.data.session_id) .details("Failed to DKIM sign message") ); } } } } // Update size message.message.size = (raw_message.len() + headers.len()) as u64; // Verify queue quota if self.server.has_quota(&mut message).await { // Prepare webhook event let queue_id = message.queue_id; // Queue message let source = if !self.is_authenticated() { let dmarc_pass = dmarc_result.is_some_and(|result| result == DmarcResult::Pass); #[cfg(feature = "test_mode")] { MessageSource::Unauthenticated { dmarc_pass: dmarc_pass || message.message.return_path.starts_with("dmarc-"), train_spam, } } #[cfg(not(feature = "test_mode"))] { MessageSource::Unauthenticated { dmarc_pass, train_spam, } } } else { MessageSource::Authenticated }; if message .queue( Some(&headers), raw_message, self.data.session_id, &self.server, source, ) .await { self.state = State::Accepted(queue_id); self.data.messages_sent += 1; format!("250 2.0.0 Message queued with id {queue_id:x}.\r\n") .into_bytes() .into() } else { (b"451 4.3.5 Unable to accept message at this time.\r\n"[..]).into() } } else { (b"452 4.3.1 Mail system full, try again later.\r\n"[..]).into() } } pub async fn build_message( &self, mail_from: SessionAddress, mut rcpt_to: Vec, queue_id: u64, span_id: u64, ) -> MessageWrapper { // Build message let created = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()); let mut message = Message { created, return_path: mail_from.address.to_lowercase_domain().into_boxed_str(), recipients: Vec::with_capacity(rcpt_to.len()), flags: mail_from.flags, priority: self.data.priority, size: 0, env_id: mail_from.dsn_info.map(|i| i.into_boxed_str()), blob_hash: Default::default(), quota_keys: Default::default(), received_from_ip: self.data.remote_ip, received_via_port: self.data.local_port, }; // Add recipients let future_release = self.data.future_release; rcpt_to.sort_unstable(); for rcpt in rcpt_to { message.recipients.push( queue::Recipient::new(rcpt.address) .with_flags( if rcpt.flags & (RCPT_NOTIFY_DELAY | RCPT_NOTIFY_FAILURE | RCPT_NOTIFY_SUCCESS | RCPT_NOTIFY_NEVER) != 0 { rcpt.flags } else { rcpt.flags | RCPT_NOTIFY_DELAY | RCPT_NOTIFY_FAILURE }, ) .with_orcpt(rcpt.dsn_info.map(|v| v.into_boxed_str())), ); let envelope = QueueEnvelope::new(&message, message.recipients.last().unwrap()); // Set next retry time let retry = if self.data.future_release == 0 { queue::Schedule::now() } else { queue::Schedule::later(future_release) }; // Resolve queue let queue = self.server.get_queue_or_default( &self .server .eval_if::( &self.server.core.smtp.queue.queue, &envelope, self.data.session_id, ) .await .unwrap_or_else(|| "default".to_string()), self.data.session_id, ); // Set expiration and notification times let num_intervals = std::cmp::max(queue.notify.len(), 1); let next_notify = queue.notify.first().copied().unwrap_or(86400); let (notify, expires) = if self.data.delivery_by == 0 { ( queue::Schedule::later(future_release + next_notify), match queue.expiry { QueueExpiry::Ttl(time) => QueueExpiry::Ttl(future_release + time), QueueExpiry::Attempts(count) => QueueExpiry::Attempts(count), }, ) } else if (message.flags & MAIL_BY_RETURN) != 0 { ( queue::Schedule::later(future_release + next_notify), QueueExpiry::Ttl(self.data.delivery_by as u64), ) } else { let (notify, expires) = match queue.expiry { QueueExpiry::Ttl(expire_secs) => ( (if self.data.delivery_by.is_positive() { let notify_at = self.data.delivery_by as u64; if expire_secs > notify_at { notify_at } else { next_notify } } else { let notify_at = -self.data.delivery_by as u64; if expire_secs > notify_at { expire_secs - notify_at } else { next_notify } }), QueueExpiry::Ttl(expire_secs), ), QueueExpiry::Attempts(_) => ( next_notify, QueueExpiry::Ttl(self.data.delivery_by.unsigned_abs()), ), }; let mut notify = queue::Schedule::later(future_release + notify); notify.inner = (num_intervals - 1) as u32; // Disable further notification attempts (notify, expires) }; // Update recipient let recipient = message.recipients.last_mut().unwrap(); recipient.retry = retry; recipient.notify = notify; recipient.expires = expires; recipient.queue = queue.virtual_queue; } MessageWrapper { queue_id, queue_name: QueueName::default(), is_multi_queue: false, span_id, message, } } pub async fn can_send_data(&mut self) -> Result { if !self.data.rcpt_to.is_empty() { if self.data.messages_sent < self .server .eval_if( &self.server.core.smtp.session.data.max_messages, self, self.data.session_id, ) .await .unwrap_or(10) { Ok(true) } else { trc::event!( Smtp(SmtpEvent::TooManyMessages), SpanId = self.data.session_id, Limit = self.data.messages_sent ); self.write(b"452 4.4.5 Maximum number of messages per session exceeded.\r\n") .await?; Ok(false) } } else { trc::event!( Smtp(SmtpEvent::RcptToMissing), SpanId = self.data.session_id, ); self.write(b"503 5.5.1 RCPT is required first.\r\n").await?; Ok(false) } } fn write_received(&self, headers: &mut Vec, id: u64) { headers.extend_from_slice(b"Received: from "); headers.extend_from_slice(self.data.helo_domain.as_bytes()); headers.extend_from_slice(b" ("); headers.extend_from_slice( self.data .iprev .as_ref() .and_then(|ir| ir.ptr.as_ref()) .and_then(|ptr| ptr.first().map(|s| s.strip_suffix('.').unwrap_or(s))) .unwrap_or("unknown") .as_bytes(), ); headers.extend_from_slice(b" ["); headers.extend_from_slice(self.data.remote_ip.to_string().as_bytes()); headers.extend_from_slice(b"]"); if self.data.asn_geo_data.asn.is_some() || self.data.asn_geo_data.country.is_some() { headers.extend_from_slice(b" ("); if let Some(asn) = &self.data.asn_geo_data.asn { headers.extend_from_slice(b"AS"); headers.extend_from_slice(asn.id.to_string().as_bytes()); if let Some(name) = &asn.name { headers.extend_from_slice(b" "); headers.extend_from_slice(name.as_bytes()); } } if let Some(country) = &self.data.asn_geo_data.country { if self.data.asn_geo_data.asn.is_some() { headers.extend_from_slice(b", "); } headers.extend_from_slice(country.as_bytes()); } headers.extend_from_slice(b")"); } headers.extend_from_slice(b")\r\n\t"); if self.stream.is_tls() { let (version, cipher) = self.stream.tls_version_and_cipher(); headers.extend_from_slice(b"(using "); headers.extend_from_slice(version.as_bytes()); headers.extend_from_slice(b" with cipher "); headers.extend_from_slice(cipher.as_bytes()); headers.extend_from_slice(b")\r\n\t"); } headers.extend_from_slice(b"by "); headers.extend_from_slice(self.hostname.as_bytes()); headers.extend_from_slice(b" (Stalwart SMTP) with "); headers.extend_from_slice(match (self.stream.is_tls(), !self.is_authenticated()) { (true, true) => b"ESMTPS", (true, false) => b"ESMTPSA", (false, true) => b"ESMTP", (false, false) => b"ESMTPA", }); headers.extend_from_slice(b" id "); headers.extend_from_slice(format!("{id:X}").as_bytes()); headers.extend_from_slice(b";\r\n\t"); headers.extend_from_slice(Date::now().to_rfc822().as_bytes()); headers.extend_from_slice(b"\r\n"); } } ================================================ FILE: crates/smtp/src/inbound/ehlo.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{core::Session, scripts::ScriptResult}; use common::{ config::smtp::session::{Mechanism, Stage}, listener::SessionStream, }; use mail_auth::{ SpfResult, spf::verify::{HasValidLabels, SpfParameters}, }; use smtp_proto::*; use std::{ borrow::Cow, time::{Duration, Instant, SystemTime}, }; use trc::SmtpEvent; impl Session { pub async fn handle_ehlo(&mut self, domain: Cow<'_, str>, is_extended: bool) -> Result<(), ()> { // Set EHLO domain if domain != self.data.helo_domain { // Reject non-FQDN EHLO domains - simply checks that the hostname has at least one dot if self.params.ehlo_reject_non_fqdn && !domain.as_ref().has_valid_labels() { trc::event!( Smtp(SmtpEvent::InvalidEhlo), SpanId = self.data.session_id, Domain = domain.as_ref().to_string(), ); return self.write(b"550 5.5.0 Invalid EHLO domain.\r\n").await; } trc::event!( Smtp(SmtpEvent::Ehlo), SpanId = self.data.session_id, Domain = domain.as_ref().to_string(), ); // SPF check let prev_helo_domain = std::mem::replace(&mut self.data.helo_domain, domain.into_owned()); if self.params.spf_ehlo.verify() { let time = Instant::now(); let spf_output = self .server .core .smtp .resolvers .dns .verify_spf(self.server.inner.cache.build_auth_parameters( SpfParameters::verify_ehlo( self.data.remote_ip, &self.data.helo_domain, &self.hostname, ), )) .await; trc::event!( Smtp(if matches!(spf_output.result(), SpfResult::Pass) { SmtpEvent::SpfEhloPass } else { SmtpEvent::SpfEhloFail }), SpanId = self.data.session_id, Domain = self.data.helo_domain.clone(), Result = trc::Error::from(&spf_output), Elapsed = time.elapsed(), ); if self .handle_spf(&spf_output, self.params.spf_ehlo.is_strict()) .await? { self.data.spf_ehlo = spf_output.into(); } else { self.data.mail_from = None; self.data.helo_domain = prev_helo_domain; return Ok(()); } } // Sieve filtering if let Some((script, script_id)) = self .server .eval_if::( &self.server.core.smtp.session.ehlo.script, self, self.data.session_id, ) .await .and_then(|name| { self.server .get_trusted_sieve_script(&name, self.data.session_id) .map(|s| (s, name)) }) && let ScriptResult::Reject(message) = self .run_script( script_id, script.clone(), self.build_script_parameters("ehlo"), ) .await { self.data.mail_from = None; self.data.helo_domain = prev_helo_domain; self.data.spf_ehlo = None; return self.write(message.as_bytes()).await; } // Milter filtering if let Err(message) = self.run_milters(Stage::Ehlo, None).await { self.data.mail_from = None; self.data.helo_domain = prev_helo_domain; self.data.spf_ehlo = None; return self.write(message.message.as_bytes()).await; } // MTAHook filtering if let Err(message) = self.run_mta_hooks(Stage::Ehlo, None, None).await { self.data.mail_from = None; self.data.helo_domain = prev_helo_domain; self.data.spf_ehlo = None; return self.write(message.message.as_bytes()).await; } } // Reset if self.data.mail_from.is_some() { self.reset(); } if !is_extended { return self .write(format!("250 {} you had me at HELO\r\n", self.hostname).as_bytes()) .await; } let mut response = EhloResponse::new(self.hostname.as_str()); response.capabilities = EXT_ENHANCED_STATUS_CODES | EXT_8BIT_MIME | EXT_BINARY_MIME | EXT_SMTP_UTF8; if !self.stream.is_tls() && self.instance.acceptor.is_tls() { response.capabilities |= EXT_START_TLS; } let ec = &self.server.core.smtp.session.extensions; let ac = &self.server.core.smtp.session.auth; let dc = &self.server.core.smtp.session.data; // Pipelining if self .server .eval_if(&ec.pipelining, self, self.data.session_id) .await .unwrap_or(true) { response.capabilities |= EXT_PIPELINING; } // Chunking if self .server .eval_if(&ec.chunking, self, self.data.session_id) .await .unwrap_or(true) { response.capabilities |= EXT_CHUNKING; } // Address Expansion if self .server .eval_if(&ec.expn, self, self.data.session_id) .await .unwrap_or(false) { response.capabilities |= EXT_EXPN; } // Recipient Verification if self .server .eval_if(&ec.vrfy, self, self.data.session_id) .await .unwrap_or(false) { response.capabilities |= EXT_VRFY; } // Require TLS if self .server .eval_if(&ec.requiretls, self, self.data.session_id) .await .unwrap_or(true) { response.capabilities |= EXT_REQUIRE_TLS; } // DSN if self .server .eval_if(&ec.dsn, self, self.data.session_id) .await .unwrap_or(false) { response.capabilities |= EXT_DSN; } // Authentication if !self.is_authenticated() { response.auth_mechanisms = self .server .eval_if::(&ac.mechanisms, self, self.data.session_id) .await .unwrap_or_default() .into(); if response.auth_mechanisms != 0 { response.capabilities |= EXT_AUTH; } } // Future release if let Some(value) = self .server .eval_if::(&ec.future_release, self, self.data.session_id) .await { response.capabilities |= EXT_FUTURE_RELEASE; response.future_release_interval = value.as_secs(); response.future_release_datetime = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map(|d| d.as_secs()) .unwrap_or(0) + value.as_secs(); } // Deliver By if let Some(value) = self .server .eval_if::(&ec.deliver_by, self, self.data.session_id) .await { response.capabilities |= EXT_DELIVER_BY; response.deliver_by = value.as_secs(); } // Priority if let Some(value) = self .server .eval_if::(&ec.mt_priority, self, self.data.session_id) .await { response.capabilities |= EXT_MT_PRIORITY; response.mt_priority = value; } // Size response.size = self .server .eval_if(&dc.max_message_size, self, self.data.session_id) .await .unwrap_or(25 * 1024 * 1024); if response.size > 0 { response.capabilities |= EXT_SIZE; } // No soliciting if let Some(value) = self .server .eval_if::(&ec.no_soliciting, self, self.data.session_id) .await { response.capabilities |= EXT_NO_SOLICITING; response.no_soliciting = if !value.is_empty() { value.to_string().into() } else { None }; } // Generate response let mut buf = Vec::with_capacity(64); response.write(&mut buf).ok(); self.write(&buf).await } } ================================================ FILE: crates/smtp/src/inbound/hooks/client.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::config::smtp::session::MTAHook; use utils::HttpLimitResponse; use super::{Request, Response}; pub(super) async fn send_mta_hook_request( mta_hook: &MTAHook, request: Request, ) -> Result { let response = reqwest::Client::builder() .timeout(mta_hook.timeout) .danger_accept_invalid_certs(mta_hook.tls_allow_invalid_certs) .build() .map_err(|err| format!("Failed to create HTTP client: {}", err))? .post(&mta_hook.url) .headers(mta_hook.headers.clone()) .body( serde_json::to_string(&request) .map_err(|err| format!("Failed to serialize Hook request: {}", err))?, ) .send() .await .map_err(|err| format!("Hook request failed: {err}"))?; if response.status().is_success() { serde_json::from_slice( response .bytes_with_limit(mta_hook.max_response_size) .await .map_err(|err| format!("Failed to parse Hook response: {}", err))? .ok_or_else(|| "Hook response too large".to_string())? .as_ref(), ) .map_err(|err| format!("Failed to parse Hook response: {}", err)) } else { Err(format!( "Hook request failed with code {}: {}", response.status().as_u16(), response.status().canonical_reason().unwrap_or("Unknown") )) } } ================================================ FILE: crates/smtp/src/inbound/hooks/message.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Instant; use ahash::AHashMap; use common::{ DAEMON_NAME, config::smtp::session::{MTAHook, Stage}, listener::SessionStream, }; use mail_auth::AuthenticatedMessage; use trc::MtaHookEvent; use crate::{ core::Session, inbound::{ FilterResponse, hooks::{ Address, Client, Context, Envelope, Message, Protocol, Request, Sasl, Server, Tls, }, milter::Modification, }, queue::QueueId, }; use super::{Action, Queue, Response, client::send_mta_hook_request}; impl Session { pub async fn run_mta_hooks( &self, stage: Stage, message: Option<&AuthenticatedMessage<'_>>, queue_id: Option, ) -> Result, FilterResponse> { let mta_hooks = &self.server.core.smtp.session.hooks; if mta_hooks.is_empty() { return Ok(Vec::new()); } let mut modifications = Vec::new(); for mta_hook in mta_hooks { if !mta_hook.run_on_stage.contains(&stage) || !self .server .eval_if(&mta_hook.enable, self, self.data.session_id) .await .unwrap_or(false) { continue; } let time = Instant::now(); match self.run_mta_hook(stage, mta_hook, message, queue_id).await { Ok(response) => { trc::event!( MtaHook(match response.action { Action::Accept => MtaHookEvent::ActionAccept, Action::Discard => MtaHookEvent::ActionDiscard, Action::Reject => MtaHookEvent::ActionReject, Action::Quarantine => MtaHookEvent::ActionQuarantine, }), SpanId = self.data.session_id, Id = mta_hook.id.clone(), Elapsed = time.elapsed(), ); let mut new_modifications = Vec::with_capacity(response.modifications.len()); for modification in response.modifications { new_modifications.push(match modification { super::Modification::ChangeFrom { value, parameters } => { Modification::ChangeFrom { sender: value, args: flatten_parameters(parameters), } } super::Modification::AddRecipient { value, parameters } => { Modification::AddRcpt { recipient: value, args: flatten_parameters(parameters), } } super::Modification::DeleteRecipient { value } => { Modification::DeleteRcpt { recipient: value } } super::Modification::ReplaceContents { value } => { Modification::ReplaceBody { value: value.as_bytes().to_vec(), } } super::Modification::AddHeader { name, value } => { Modification::AddHeader { name, value } } super::Modification::InsertHeader { index, name, value } => { Modification::InsertHeader { index, name, value } } super::Modification::ChangeHeader { index, name, value } => { Modification::ChangeHeader { index, name, value } } super::Modification::DeleteHeader { index, name } => { Modification::ChangeHeader { index, name, value: String::new(), } } }); } if !modifications.is_empty() { // The message body can only be replaced once, so we need to remove // any previous replacements. if new_modifications .iter() .any(|m| matches!(m, Modification::ReplaceBody { .. })) { modifications .retain(|m| !matches!(m, Modification::ReplaceBody { .. })); } modifications.extend(new_modifications); } else { modifications = new_modifications; } let mut message = match response.action { Action::Accept => continue, Action::Discard => FilterResponse::accept(), Action::Reject => FilterResponse::reject(), Action::Quarantine => { modifications.push(Modification::AddHeader { name: "X-Quarantine".into(), value: "true".into(), }); FilterResponse::accept() } }; if let Some(response) = response.response { if let (Some(status), Some(text)) = (response.status, response.message) { if let Some(enhanced) = response.enhanced_status { message.message = format!("{status} {enhanced} {text}\r\n").into(); } else { message.message = format!("{status} {text}\r\n").into(); } } message.disconnect = response.disconnect; } return Err(message); } Err(err) => { trc::event!( MtaHook(MtaHookEvent::Error), SpanId = self.data.session_id, Id = mta_hook.id.clone(), Reason = err, Elapsed = time.elapsed(), ); if mta_hook.tempfail_on_error { return Err(FilterResponse::server_failure()); } } } } Ok(modifications) } pub async fn run_mta_hook( &self, stage: Stage, mta_hook: &MTAHook, message: Option<&AuthenticatedMessage<'_>>, queue_id: Option, ) -> Result { // Build request let (tls_version, tls_cipher) = self.stream.tls_version_and_cipher(); let request = Request { context: Context { stage: stage.into(), client: Client { ip: self.data.remote_ip.to_string(), port: self.data.remote_port, ptr: self .data .iprev .as_ref() .and_then(|ip_rev| ip_rev.ptr.as_ref()) .and_then(|ptrs| ptrs.first()) .map(Into::into), helo: (!self.data.helo_domain.is_empty()) .then(|| self.data.helo_domain.clone()), active_connections: 1, }, sasl: self.authenticated_as().map(|name| Sasl { login: name.into(), method: None, }), tls: (!tls_version.is_empty()).then(|| Tls { version: tls_version.as_ref().into(), cipher: tls_cipher.as_ref().into(), bits: None, issuer: None, subject: None, }), server: Server { name: Some(DAEMON_NAME.into()), port: self.data.local_port, ip: self.data.local_ip.to_string().into(), }, queue: queue_id.map(|id| Queue { id: format!("{:x}", id), }), protocol: Protocol { version: 1 }, }, envelope: self.data.mail_from.as_ref().map(|from| Envelope { from: Address { address: from.address_lcase.clone(), parameters: None, }, to: self .data .rcpt_to .iter() .map(|to| Address { address: to.address_lcase.clone(), parameters: None, }) .collect(), }), message: message.map(|message| Message { headers: message .raw_parsed_headers() .iter() .map(|(k, v)| { ( String::from_utf8_lossy(k).into_owned(), String::from_utf8_lossy(v).into_owned(), ) }) .collect(), server_headers: vec![], contents: String::from_utf8_lossy(message.raw_body()).into_owned(), size: message.raw_message().len(), }), }; send_mta_hook_request(mta_hook, request).await } } fn flatten_parameters(parameters: AHashMap>) -> String { let mut arguments = String::new(); for (key, value) in parameters { if !arguments.is_empty() { arguments.push(' '); } arguments.push_str(key.as_str()); if let Some(value) = value { arguments.push('='); arguments.push_str(value.as_str()); } } arguments } ================================================ FILE: crates/smtp/src/inbound/hooks/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod client; pub mod message; use ahash::AHashMap; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] pub struct Request { pub context: Context, #[serde(skip_serializing_if = "Option::is_none")] pub envelope: Option, #[serde(skip_serializing_if = "Option::is_none")] pub message: Option, } #[derive(Serialize, Deserialize)] pub struct Context { pub stage: Stage, pub client: Client, #[serde(skip_serializing_if = "Option::is_none")] pub sasl: Option, #[serde(skip_serializing_if = "Option::is_none")] pub tls: Option, pub server: Server, #[serde(skip_serializing_if = "Option::is_none")] pub queue: Option, pub protocol: Protocol, } #[derive(Serialize, Deserialize)] pub struct Sasl { pub login: String, #[serde(skip_serializing_if = "Option::is_none")] pub method: Option, } #[derive(Serialize, Deserialize)] pub struct Client { pub ip: String, pub port: u16, pub ptr: Option, pub helo: Option, #[serde(rename = "activeConnections")] pub active_connections: u32, } #[derive(Serialize, Deserialize)] pub struct Tls { pub version: String, pub cipher: String, #[serde(rename = "cipherBits")] #[serde(skip_serializing_if = "Option::is_none")] pub bits: Option, #[serde(rename = "certIssuer")] #[serde(skip_serializing_if = "Option::is_none")] pub issuer: Option, #[serde(rename = "certSubject")] #[serde(skip_serializing_if = "Option::is_none")] pub subject: Option, } #[derive(Serialize, Deserialize)] pub struct Server { pub name: Option, pub port: u16, pub ip: Option, } #[derive(Serialize, Deserialize)] pub struct Queue { pub id: String, } #[derive(Serialize, Deserialize)] pub struct Protocol { pub version: u32, } #[derive(Serialize, Deserialize)] pub enum Stage { #[serde(rename = "connect")] Connect, #[serde(rename = "ehlo")] Ehlo, #[serde(rename = "auth")] Auth, #[serde(rename = "mail")] Mail, #[serde(rename = "rcpt")] Rcpt, #[serde(rename = "data")] Data, } #[derive(Serialize, Deserialize)] pub struct Address { pub address: String, #[serde(skip_serializing_if = "Option::is_none")] pub parameters: Option>, } #[derive(Serialize, Deserialize)] pub struct Envelope { pub from: Address, pub to: Vec
, } #[derive(Serialize, Deserialize)] pub struct Message { pub headers: Vec<(String, String)>, #[serde(skip_serializing_if = "Vec::is_empty")] #[serde(rename = "serverHeaders")] #[serde(default)] pub server_headers: Vec<(String, String)>, pub contents: String, pub size: usize, } #[derive(Serialize, Deserialize)] pub struct Response { pub action: Action, #[serde(default)] pub response: Option, #[serde(default)] pub modifications: Vec, } #[derive(Serialize, Deserialize)] pub enum Action { #[serde(rename = "accept")] Accept, #[serde(rename = "discard")] Discard, #[serde(rename = "reject")] Reject, #[serde(rename = "quarantine")] Quarantine, } #[derive(Serialize, Deserialize, Default)] pub struct SmtpResponse { #[serde(default)] pub status: Option, #[serde(default)] pub enhanced_status: Option, #[serde(default)] pub message: Option, #[serde(default)] pub disconnect: bool, } #[derive(Serialize, Deserialize, Debug)] #[serde(tag = "type")] pub enum Modification { #[serde(rename = "changeFrom")] ChangeFrom { value: String, #[serde(default)] parameters: AHashMap>, }, #[serde(rename = "addRecipient")] AddRecipient { value: String, #[serde(default)] parameters: AHashMap>, }, #[serde(rename = "deleteRecipient")] DeleteRecipient { value: String }, #[serde(rename = "replaceContents")] ReplaceContents { value: String }, #[serde(rename = "addHeader")] AddHeader { name: String, value: String }, #[serde(rename = "insertHeader")] InsertHeader { index: u32, name: String, value: String, }, #[serde(rename = "changeHeader")] ChangeHeader { index: u32, name: String, value: String, }, #[serde(rename = "deleteHeader")] DeleteHeader { index: u32, name: String }, } impl From for Stage { fn from(value: common::config::smtp::session::Stage) -> Self { match value { common::config::smtp::session::Stage::Connect => Stage::Connect, common::config::smtp::session::Stage::Ehlo => Stage::Ehlo, common::config::smtp::session::Stage::Auth => Stage::Auth, common::config::smtp::session::Stage::Mail => Stage::Mail, common::config::smtp::session::Stage::Rcpt => Stage::Rcpt, common::config::smtp::session::Stage::Data => Stage::Data, } } } ================================================ FILE: crates/smtp/src/inbound/mail.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ core::{Session, SessionAddress}, scripts::ScriptResult, }; use common::{config::smtp::session::Stage, listener::SessionStream, scripts::ScriptModification}; use mail_auth::{IprevOutput, IprevResult, SpfOutput, SpfResult, spf::verify::SpfParameters}; use smtp_proto::{MAIL_BY_NOTIFY, MAIL_BY_RETURN, MAIL_REQUIRETLS, MailFrom, MtPriority}; use std::{ borrow::Cow, time::{Duration, Instant, SystemTime}, }; use trc::SmtpEvent; use utils::{DomainPart, config::Rate}; impl Session { pub async fn handle_mail_from(&mut self, from: MailFrom>) -> Result<(), ()> { if self.data.helo_domain.is_empty() && (self.params.ehlo_require || self.params.spf_ehlo.verify() || self.params.spf_mail_from.verify()) { trc::event!( Smtp(SmtpEvent::DidNotSayEhlo), SpanId = self.data.session_id, ); return self .write(b"503 5.5.1 Polite people say EHLO first.\r\n") .await; } else if self.data.mail_from.is_some() { trc::event!( Smtp(SmtpEvent::MultipleMailFrom), SpanId = self.data.session_id, ); return self .write(b"503 5.5.1 Multiple MAIL commands not allowed.\r\n") .await; } else if self.params.auth_require && !self.is_authenticated() { trc::event!( Smtp(SmtpEvent::MailFromUnauthenticated), SpanId = self.data.session_id, ); return self .write(b"503 5.5.1 You must authenticate first.\r\n") .await; } else if self.data.iprev.is_none() && self.params.iprev.verify() { let time = Instant::now(); let iprev = self .server .core .smtp .resolvers .dns .verify_iprev( self.server .inner .cache .build_auth_parameters(self.data.remote_ip), ) .await; trc::event!( Smtp(if matches!(iprev.result(), IprevResult::Pass) { SmtpEvent::IprevPass } else { SmtpEvent::IprevFail }), SpanId = self.data.session_id, Domain = self.data.helo_domain.clone(), Result = trc::Error::from(&iprev), Elapsed = time.elapsed(), ); self.data.iprev = iprev.into(); } // In strict mode reject messages from hosts that fail the reverse DNS lookup check if self.params.iprev.is_strict() && !matches!( &self.data.iprev, Some(IprevOutput { result: IprevResult::Pass, .. }) ) { let message = if matches!( &self.data.iprev, Some(IprevOutput { result: IprevResult::TempError(_), .. }) ) { &b"451 4.7.25 Temporary error validating reverse DNS.\r\n"[..] } else { &b"550 5.7.25 Reverse DNS validation failed.\r\n"[..] }; return self.write(message).await; } let (address, address_lcase, domain) = if !from.address.is_empty() { let address_lcase = from.address.to_lowercase(); let domain = address_lcase.domain_part().into(); (from.address.into_owned(), address_lcase, domain) } else { (String::new(), String::new(), String::new()) }; let has_dsn = from.env_id.is_some(); self.data.mail_from = SessionAddress { address, address_lcase, domain, flags: from.flags, dsn_info: from.env_id.map(|e| e.into_owned()), } .into(); // Check whether the address is allowed if !self .server .eval_if::( &self.server.core.smtp.session.mail.is_allowed, self, self.data.session_id, ) .await .unwrap_or(true) { let mail_from = self.data.mail_from.take().unwrap(); trc::event!( Smtp(SmtpEvent::MailFromNotAllowed), From = mail_from.address_lcase, SpanId = self.data.session_id, ); return self .write(b"550 5.7.1 Sender address not allowed.\r\n") .await; } // Sieve filtering if let Some((script, script_id)) = self .server .eval_if::( &self.server.core.smtp.session.mail.script, self, self.data.session_id, ) .await .and_then(|name| { self.server .get_trusted_sieve_script(&name, self.data.session_id) .map(|s| (s, name)) }) { match self .run_script( script_id, script.clone(), self.build_script_parameters("mail"), ) .await { ScriptResult::Accept { modifications } => { if !modifications.is_empty() { for modification in modifications { if let ScriptModification::SetEnvelope { name, value } = modification { self.data.apply_envelope_modification(name, value); } } } } ScriptResult::Reject(message) => { self.data.mail_from = None; return self.write(message.as_bytes()).await; } _ => (), } } // Milter filtering if let Err(message) = self.run_milters(Stage::Mail, None).await { self.data.mail_from = None; return self.write(message.message.as_bytes()).await; } // MTAHook filtering if let Err(message) = self.run_mta_hooks(Stage::Mail, None, None).await { self.data.mail_from = None; return self.write(message.message.as_bytes()).await; } // Address rewriting if let Some(new_address) = self .server .eval_if::( &self.server.core.smtp.session.mail.rewrite, self, self.data.session_id, ) .await { let mail_from = self.data.mail_from.as_mut().unwrap(); trc::event!( Smtp(SmtpEvent::MailFromRewritten), SpanId = self.data.session_id, Details = mail_from.address_lcase.clone(), From = new_address.clone(), ); if new_address.contains('@') { mail_from.address_lcase = new_address.to_lowercase(); mail_from.domain = mail_from.address_lcase.domain_part().into(); mail_from.address = new_address; } else if new_address.is_empty() { mail_from.address_lcase.clear(); mail_from.domain.clear(); mail_from.address.clear(); } } // Make sure that the authenticated user is allowed to send from this address match self.authenticated_as() { Some(authenticated_as) if self .server .eval_if( &self.server.core.smtp.session.auth.must_match_sender, self, self.data.session_id, ) .await .unwrap_or(true) => { let address_lcase = self.data.mail_from.as_ref().unwrap().address_lcase.as_str(); if authenticated_as != address_lcase && !self.authenticated_emails().iter().any(|e| { e == address_lcase || (e.starts_with('@') && address_lcase.ends_with(e.as_str())) }) { trc::event!( Smtp(SmtpEvent::MailFromUnauthorized), SpanId = self.data.session_id, From = address_lcase.to_string(), Details = [trc::Value::String(authenticated_as.into())] .into_iter() .chain( self.authenticated_emails() .iter() .map(|e| trc::Value::String(e.as_str().into())) ) .collect::>() ); self.data.mail_from = None; return self .write(b"501 5.5.4 You are not allowed to send from this address.\r\n") .await; } } _ => (), } // Validate parameters let config = &self.server.core.smtp.session.extensions; let config_data = &self.server.core.smtp.session.data; if (from.flags & MAIL_REQUIRETLS) != 0 && !self .server .eval_if(&config.requiretls, self, self.data.session_id) .await .unwrap_or(false) { trc::event!( Smtp(SmtpEvent::RequireTlsDisabled), SpanId = self.data.session_id, ); self.data.mail_from = None; return self .write(b"501 5.5.4 REQUIRETLS has been disabled.\r\n") .await; } if (from.flags & (MAIL_BY_NOTIFY | MAIL_BY_RETURN)) != 0 { if let Some(duration) = self .server .eval_if::(&config.deliver_by, self, self.data.session_id) .await { if from.by.checked_abs().unwrap_or(0) as u64 <= duration.as_secs() && (from.by.is_positive() || (from.flags & MAIL_BY_NOTIFY) != 0) { self.data.delivery_by = from.by; } else { self.data.mail_from = None; trc::event!( Smtp(SmtpEvent::DeliverByInvalid), SpanId = self.data.session_id, Details = from.by, ); return self .write( format!( "501 5.5.4 BY parameter exceeds maximum of {} seconds.\r\n", duration.as_secs() ) .as_bytes(), ) .await; } } else { trc::event!( Smtp(SmtpEvent::DeliverByDisabled), SpanId = self.data.session_id, ); self.data.mail_from = None; return self .write(b"501 5.5.4 DELIVERBY extension has been disabled.\r\n") .await; } } if from.mt_priority != 0 { if self .server .eval_if::(&config.mt_priority, self, self.data.session_id) .await .is_some() { if (-6..6).contains(&from.mt_priority) { self.data.priority = from.mt_priority as i16; } else { trc::event!( Smtp(SmtpEvent::MtPriorityInvalid), SpanId = self.data.session_id, Details = from.mt_priority, ); self.data.mail_from = None; return self.write(b"501 5.5.4 Invalid priority value.\r\n").await; } } else { trc::event!( Smtp(SmtpEvent::MtPriorityDisabled), SpanId = self.data.session_id, ); self.data.mail_from = None; return self .write(b"501 5.5.4 MT-PRIORITY extension has been disabled.\r\n") .await; } } if from.size > 0 && from.size > self .server .eval_if(&config_data.max_message_size, self, self.data.session_id) .await .unwrap_or(25 * 1024 * 1024) { trc::event!( Smtp(SmtpEvent::MessageTooLarge), SpanId = self.data.session_id, Size = from.size, ); self.data.mail_from = None; return self .write(b"552 5.3.4 Message too big for system.\r\n") .await; } if from.hold_for != 0 || from.hold_until != 0 { if let Some(max_hold) = self .server .eval_if::(&config.future_release, self, self.data.session_id) .await { let max_hold = max_hold.as_secs(); let hold_for = if from.hold_for != 0 { from.hold_for } else { let now = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()); from.hold_until.saturating_sub(now) }; if hold_for <= max_hold { self.data.future_release = hold_for; } else { trc::event!( Smtp(SmtpEvent::FutureReleaseInvalid), SpanId = self.data.session_id, Details = hold_for, ); self.data.mail_from = None; return self .write( format!( "501 5.5.4 Requested hold time exceeds maximum of {max_hold} seconds.\r\n" ) .as_bytes(), ) .await; } } else { trc::event!( Smtp(SmtpEvent::FutureReleaseDisabled), SpanId = self.data.session_id, ); self.data.mail_from = None; return self .write(b"501 5.5.4 FUTURERELEASE extension has been disabled.\r\n") .await; } } if has_dsn && !self .server .eval_if(&config.dsn, self, self.data.session_id) .await .unwrap_or(false) { trc::event!(Smtp(SmtpEvent::DsnDisabled), SpanId = self.data.session_id,); self.data.mail_from = None; return self .write(b"501 5.5.4 DSN extension has been disabled.\r\n") .await; } if self.is_allowed().await { // Verify SPF if self.params.spf_mail_from.verify() { let time = Instant::now(); let mail_from = self.data.mail_from.as_ref().unwrap(); let spf_output = if !mail_from.address.is_empty() { self.server .core .smtp .resolvers .dns .check_host(self.server.inner.cache.build_auth_parameters( SpfParameters::new( self.data.remote_ip, &mail_from.domain, &self.data.helo_domain, &self.hostname, &mail_from.address_lcase, ), )) .await } else { self.server .core .smtp .resolvers .dns .check_host(self.server.inner.cache.build_auth_parameters( SpfParameters::new( self.data.remote_ip, &self.data.helo_domain, &self.data.helo_domain, &self.hostname, &format!("postmaster@{}", self.data.helo_domain), ), )) .await }; trc::event!( Smtp(if matches!(spf_output.result(), SpfResult::Pass) { SmtpEvent::SpfFromPass } else { SmtpEvent::SpfFromFail }), SpanId = self.data.session_id, Domain = self.data.helo_domain.clone(), From = if !mail_from.address.is_empty() { mail_from.address.as_str() } else { "<>" } .to_string(), Result = trc::Error::from(&spf_output), Elapsed = time.elapsed(), ); if self .handle_spf(&spf_output, self.params.spf_mail_from.is_strict()) .await? { self.data.spf_mail_from = spf_output.into(); } else { self.data.mail_from = None; return Ok(()); } } trc::event!( Smtp(SmtpEvent::MailFrom), SpanId = self.data.session_id, From = self.data.mail_from.as_ref().unwrap().address_lcase.clone(), ); self.eval_rcpt_params().await; self.write(b"250 2.1.0 OK\r\n").await } else { trc::event!( Smtp(SmtpEvent::RateLimitExceeded), SpanId = self.data.session_id, From = self.data.mail_from.as_ref().unwrap().address_lcase.clone(), ); self.data.mail_from = None; self.write(b"452 4.4.5 Rate limit exceeded, try again later.\r\n") .await } } pub async fn handle_spf(&mut self, spf_output: &SpfOutput, strict: bool) -> Result { let result = match spf_output.result() { SpfResult::Pass => true, SpfResult::TempError if strict => { self.write(b"451 4.7.24 Temporary SPF validation error.\r\n") .await?; false } result => { if strict { self.write( format!("550 5.7.23 SPF validation failed, status: {result}.\r\n") .as_bytes(), ) .await?; false } else { true } } }; // Send report if let (Some(recipient), Some(rate)) = ( spf_output.report_address(), self.server .eval_if::( &self.server.core.smtp.report.spf.send, self, self.data.session_id, ) .await, ) { // Do not send SPF auth failures to local domains, as they are likely relay attempts (which are blocked later on) match self .server .core .storage .directory .is_local_domain(recipient.domain_part()) .await { Ok(true) => return Ok(result), Ok(false) => (), Err(err) => { trc::error!( err.caused_by(trc::location!()) .span_id(self.data.session_id) .details("Failed to lookup local domain") ); } } self.send_spf_report(recipient, &rate, !result, spf_output) .await; } Ok(result) } } ================================================ FILE: crates/smtp/src/inbound/milter/client.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::config::smtp::session::Milter; use rustls_pki_types::ServerName; use tokio::{ io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}, net::TcpStream, }; use tokio_rustls::{TlsConnector, client::TlsStream}; use trc::MilterEvent; use super::{ protocol::{SMFIC_CONNECT, SMFIC_HELO, SMFIC_MAIL, SMFIC_RCPT}, receiver::{FrameResult, Receiver}, *, }; const MILTER_CHUNK_SIZE: usize = 65535; impl MilterClient { pub async fn connect(config: &Milter, session_id: u64) -> Result { tokio::time::timeout(config.timeout_command, async { let mut last_err = Error::Disconnected; for addr in &config.addrs { match TcpStream::connect(addr).await { Ok(stream) => { return Ok(MilterClient { stream, timeout_cmd: config.timeout_command, timeout_data: config.timeout_data, buf: vec![0u8; 8192], bytes_read: 0, receiver: Receiver::with_max_frame_len(config.max_frame_len), options: 0, version: config.protocol_version, session_id, flags_actions: config.flags_actions.unwrap_or( SMFIF_ADDHDRS | SMFIF_CHGBODY | SMFIF_ADDRCPT | SMFIF_DELRCPT | SMFIF_CHGHDRS | SMFIF_QUARANTINE | SMFIF_CHGFROM | SMFIF_ADDRCPT_PAR, ), flags_protocol: config.flags_protocol.unwrap_or(0x42), id: config.id.clone(), }); } Err(err) => { last_err = Error::Io(err); } } } Err(last_err) }) .await .map_err(|_| Error::Timeout)? } pub async fn into_tls( self, tls_connector: &TlsConnector, tls_hostname: &str, ) -> Result>> { tokio::time::timeout(self.timeout_cmd, async { Ok(MilterClient { stream: tls_connector .connect( ServerName::try_from(tls_hostname) .map_err(|_| Error::TLSInvalidName)? .to_owned(), self.stream, ) .await?, buf: self.buf, timeout_cmd: self.timeout_cmd, timeout_data: self.timeout_data, receiver: self.receiver, bytes_read: self.bytes_read, options: self.options, version: self.version, session_id: self.session_id, flags_actions: self.flags_actions, flags_protocol: self.flags_protocol, id: self.id, }) }) .await .map_err(|_| Error::Timeout)? } } impl MilterClient { pub async fn init(&mut self) -> super::Result { self.write(Command::OptionNegotiation(Options { version: match self.version { MilterVersion::V2 => 2, MilterVersion::V6 => 6, }, actions: self.flags_actions, protocol: self.flags_protocol, })) .await?; match self.read().await? { Response::OptionNegotiation(options) => { self.options = options.protocol; Ok(options) } response => Err(Error::Unexpected(response)), } } pub async fn connection( &mut self, hostname: impl AsRef<[u8]>, remote_ip: IpAddr, remote_port: u16, macros: Macros<'_>, ) -> super::Result { if !self.has_option(SMFIP_NOCONNECT) { self.write(Command::Macro { macros: macros.with_cmd_code(SMFIC_CONNECT), }) .await?; self.write(Command::Connect { hostname: hostname.as_ref(), port: remote_port, address: remote_ip, }) .await?; if !self.has_option(SMFIP_NR_CONN) { return self.read().await?.into_action(); } } Ok(Action::Accept) } pub async fn helo( &mut self, hostname: impl AsRef<[u8]>, macros: Macros<'_>, ) -> super::Result { if !self.has_option(SMFIP_NOHELO) { self.write(Command::Macro { macros: macros.with_cmd_code(SMFIC_HELO), }) .await?; self.write(Command::Helo { hostname: hostname.as_ref(), }) .await?; if !self.has_option(SMFIP_NR_HELO) { return self.read().await?.into_action(); } } Ok(Action::Accept) } pub async fn mail_from( &mut self, addr: A, params: Option<&[V]>, macros: Macros<'_>, ) -> super::Result where A: AsRef<[u8]>, V: AsRef<[u8]>, { if !self.has_option(SMFIP_NOMAIL) { self.write(Command::Macro { macros: macros.with_cmd_code(SMFIC_MAIL), }) .await?; self.write(Command::MailFrom { sender: addr.as_ref(), args: params.map(|params| params.iter().map(|value| value.as_ref()).collect()), }) .await?; if !self.has_option(SMFIP_NR_MAIL) { return self.read().await?.into_action(); } } Ok(Action::Accept) } pub async fn rcpt_to( &mut self, addr: A, params: Option<&[V]>, macros: Macros<'_>, ) -> super::Result where A: AsRef<[u8]>, V: AsRef<[u8]>, { if !self.has_option(SMFIP_NORCPT) { self.write(Command::Macro { macros: macros.with_cmd_code(SMFIC_RCPT), }) .await?; self.write(Command::Rcpt { recipient: addr.as_ref(), args: params.map(|params| params.iter().map(|value| value.as_ref()).collect()), }) .await?; if !self.has_option(SMFIP_NR_RCPT) { return self.read().await?.into_action(); } } Ok(Action::Accept) } pub async fn headers(&mut self, headers: I) -> super::Result where I: Iterator, H: AsRef, V: AsRef, { if !self.has_option(SMFIP_NOHDRS) { for (name, value) in headers { self.write(Command::Header { name: name.as_ref().trim().as_bytes(), value: value.as_ref().trim().as_bytes(), }) .await?; if !self.has_option(SMFIP_NR_HDR) { match self.read().await? { Response::Action(Action::Accept | Action::Continue) => (), Response::Action(action) => return Ok(action), response => return Err(Error::Unexpected(response)), } } } // Write EndOfHeaders self.write(Command::EndOfHeader).await?; if !self.has_option(SMFIP_NR_EOH) { return self.read().await?.into_action(); } } Ok(Action::Accept) } pub async fn data(&mut self) -> super::Result { if matches!(self.version, MilterVersion::V6) && !self.has_option(SMFIP_NODATA) { self.write(Command::Data).await?; if !self.has_option(SMFIP_NR_DATA) { return self.read().await?.into_action(); } } Ok(Action::Accept) } pub async fn body(&mut self, body: &[u8]) -> super::Result<(Action, Vec)> { if !self.has_option(SMFIP_NOBODY) { // Write body chunks for value in body.chunks(MILTER_CHUNK_SIZE) { self.write(Command::Body { value }).await?; if !self.has_option(SMFIP_NR_BODY) { match self.read().await? { Response::Action(Action::Accept | Action::Continue) | Response::Progress => (), Response::Skip => break, Response::Action(reject) => { return Ok((reject, Vec::new())); } response => return Err(Error::Unexpected(response)), } } } // Write EndOfBody self.write(Command::EndOfBody).await?; // Collect responses let mut modifications = Vec::new(); loop { match self.read().await? { Response::Action(action) => { return Ok((action, modifications)); } Response::Modification(modification) => { modifications.push(modification); } Response::Progress => (), unexpected => { return Err(Error::Unexpected(unexpected)); } } } } else { Ok((Action::Accept, vec![])) } } pub async fn abort(&mut self) -> super::Result<()> { self.write(Command::Abort).await } pub async fn quit(&mut self) -> super::Result<()> { self.write(Command::Quit).await } async fn write(&mut self, action: Command<'_>) -> super::Result<()> { trc::event!( Milter(MilterEvent::Write), SpanId = self.session_id, Id = self.id.to_string(), Contents = action.to_string(), ); tokio::time::timeout(self.timeout_cmd, async { self.stream.write_all(action.serialize().as_ref()).await?; self.stream.flush().await.map_err(Error::Io) }) .await .map_err(|_| Error::Timeout)? } async fn read(&mut self) -> super::Result { loop { match self.receiver.read_frame(&self.buf[..self.bytes_read]) { FrameResult::Frame(frame) => { if let Some(response) = Response::deserialize(&frame) { trc::event!( Milter(MilterEvent::Read), SpanId = self.session_id, Id = self.id.to_string(), Contents = response.to_string(), ); return Ok(response); } else { return Err(Error::FrameInvalid(frame.into_owned())); } } FrameResult::Incomplete => { self.bytes_read = tokio::time::timeout(self.timeout_data, async { self.stream.read(&mut self.buf).await.map_err(Error::Io) }) .await .map_err(|_| Error::Timeout)??; if self.bytes_read == 0 { return Err(Error::Disconnected); } } FrameResult::TooLarge(size) => return Err(Error::FrameTooLarge(size)), } } } #[inline(always)] fn has_option(&self, opt: u32) -> bool { self.options & opt == opt } pub fn with_version(mut self, version: MilterVersion) -> Self { self.version = version; self } } ================================================ FILE: crates/smtp/src/inbound/milter/macros.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{borrow::Cow, net::IpAddr}; use super::{Macro, Macros}; pub trait IntoMacroValue<'x> { fn into_macro_value(self) -> Cow<'x, [u8]>; } impl<'x> Macros<'x> { pub fn new() -> Self { Macros::default() } pub fn with_cmd_code(mut self, cmd_code: u8) -> Self { self.cmdcode = cmd_code; self } pub fn with_macro(mut self, name: &'static [u8], value: impl IntoMacroValue<'x>) -> Self { self.macros.push(Macro { name, value: value.into_macro_value(), }); self } pub fn with_queue_id(self, queue_id: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"i", queue_id) } pub fn with_local_hostname(self, my_hostname: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"j", my_hostname) } pub fn with_validated_client_name(self, client_name: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"_", client_name) } pub fn with_sasl_login_name(self, sasl_login_name: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{auth_authen}", sasl_login_name) } pub fn with_sasl_sender(self, sasl_sender: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{auth_author}", sasl_sender) } pub fn with_sasl_method(self, sasl_method: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{auth_type}", sasl_method) } pub fn with_client_address(self, client_address: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{client_addr}", client_address) } pub fn with_client_connections(self, client_connections: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{client_connections}", client_connections) } pub fn with_client_name(self, client_name: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{client_name}", client_name) } pub fn with_client_port(self, client_port: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{client_port}", client_port) } pub fn with_client_ptr(self, client_ptr: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{client_ptr}", client_ptr) } pub fn with_cert_issuer(self, cert_issuer: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{cert_issuer}", cert_issuer) } pub fn with_cert_subject(self, cert_subject: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{cert_subject}", cert_subject) } pub fn with_cipher_bits(self, cipher_bits: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{cipher_bits}", cipher_bits) } pub fn with_cipher(self, cipher: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{cipher}", cipher) } pub fn with_daemon_address(self, daemon_address: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{daemon_addr}", daemon_address) } pub fn with_daemon_name(self, daemon_name: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{daemon_name}", daemon_name) } pub fn with_daemon_port(self, daemon_port: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{daemon_port}", daemon_port) } pub fn with_mail_address(self, mail_address: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{mail_addr}", mail_address) } pub fn with_mail_host(self, mail_host_address: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{mail_host}", mail_host_address) } pub fn with_mail_mailer(self, mail_mailer: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{mail_mailer}", mail_mailer) } pub fn with_rcpt_address(self, rcpt_address: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{rcpt_addr}", rcpt_address) } pub fn with_rcpt_host(self, rcpt_host: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{rcpt_host}", rcpt_host) } pub fn with_rcpt_mailer(self, rcpt_mailer: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{rcpt_mailer}", rcpt_mailer) } pub fn with_tls_version(self, tls_version: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{tls_version}", tls_version) } pub fn with_version(self, version: impl IntoMacroValue<'x>) -> Self { self.with_macro(b"{v}", version) } } impl<'x> IntoMacroValue<'x> for IpAddr { fn into_macro_value(self) -> Cow<'x, [u8]> { Cow::Owned(self.to_string().into_bytes()) } } impl<'x> IntoMacroValue<'x> for u16 { fn into_macro_value(self) -> Cow<'x, [u8]> { Cow::Owned(self.to_string().into_bytes()) } } impl<'x> IntoMacroValue<'x> for &'x [u8] { fn into_macro_value(self) -> Cow<'x, [u8]> { Cow::Borrowed(self) } } impl<'x> IntoMacroValue<'x> for &'x str { fn into_macro_value(self) -> Cow<'x, [u8]> { Cow::Borrowed(self.as_bytes()) } } impl<'x> IntoMacroValue<'x> for &'x String { fn into_macro_value(self) -> Cow<'x, [u8]> { Cow::Borrowed(self.as_bytes()) } } impl<'x> IntoMacroValue<'x> for String { fn into_macro_value(self) -> Cow<'x, [u8]> { Cow::Owned(self.into_bytes()) } } impl<'x> IntoMacroValue<'x> for Vec { fn into_macro_value(self) -> Cow<'x, [u8]> { Cow::Owned(self) } } impl<'x> IntoMacroValue<'x> for &'x Vec { fn into_macro_value(self) -> Cow<'x, [u8]> { Cow::Borrowed(self) } } ================================================ FILE: crates/smtp/src/inbound/milter/message.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{Action, Error, Macros, Modification}; use crate::{ core::{Session, SessionAddress, SessionData}, inbound::{FilterResponse, milter::MilterClient}, }; use common::{ DAEMON_NAME, config::smtp::session::{Milter, Stage}, listener::SessionStream, }; use mail_auth::AuthenticatedMessage; use smtp_proto::{IntoString, request::parser::Rfc5321Parser}; use std::{borrow::Cow, time::Instant}; use tokio::io::{AsyncRead, AsyncWrite}; use trc::MilterEvent; use utils::DomainPart; enum Rejection { Action(Action), Error(Error), } impl Session { pub async fn run_milters( &self, stage: Stage, message: Option<&AuthenticatedMessage<'_>>, ) -> Result, FilterResponse> { let milters = &self.server.core.smtp.session.milters; if milters.is_empty() { return Ok(Vec::new()); } let mut modifications = Vec::new(); for milter in milters { if !milter.run_on_stage.contains(&stage) || !self .server .eval_if(&milter.enable, self, self.data.session_id) .await .unwrap_or(false) { continue; } let time = Instant::now(); match self.connect_and_run(milter, message).await { Ok(new_modifications) => { trc::event!( Milter(MilterEvent::ActionAccept), SpanId = self.data.session_id, Id = milter.id.to_string(), Elapsed = time.elapsed(), ); if !modifications.is_empty() { // The message body can only be replaced once, so we need to remove // any previous replacements. if new_modifications .iter() .any(|m| matches!(m, Modification::ReplaceBody { .. })) { modifications .retain(|m| !matches!(m, Modification::ReplaceBody { .. })); } modifications.extend(new_modifications); } else { modifications = new_modifications; } } Err(Rejection::Action(action)) => { trc::event!( Milter(match &action { Action::Discard => MilterEvent::ActionDiscard, Action::Reject => MilterEvent::ActionReject, Action::TempFail => MilterEvent::ActionTempFail, Action::ReplyCode { .. } => { MilterEvent::ActionReplyCode } Action::Shutdown => MilterEvent::ActionShutdown, Action::ConnectionFailure => MilterEvent::ActionConnectionFailure, Action::Accept | Action::Continue => unreachable!(), }), SpanId = self.data.session_id, Id = milter.id.to_string(), Elapsed = time.elapsed(), ); return Err(match action { Action::Discard => FilterResponse::accept(), Action::Reject => FilterResponse::reject(), Action::TempFail => FilterResponse::temp_fail(), Action::ReplyCode { code, text } => { let mut response = Vec::with_capacity(text.len() + 6); response.extend_from_slice(code.as_slice()); response.push(b' '); response.extend_from_slice(text.as_bytes()); if !text.ends_with('\n') { response.extend_from_slice(b"\r\n"); } FilterResponse { message: response.into_string().into(), disconnect: false, } } Action::Shutdown => FilterResponse::shutdown(), Action::ConnectionFailure => FilterResponse::default().disconnect(), Action::Accept | Action::Continue => unreachable!(), }); } Err(Rejection::Error(err)) => { let (code, details) = match err { Error::Io(details) => { (MilterEvent::IoError, trc::Value::from(details.to_string())) } Error::FrameTooLarge(size) => { (MilterEvent::FrameTooLarge, trc::Value::from(size)) } Error::FrameInvalid(bytes) => { (MilterEvent::FrameInvalid, trc::Value::from(bytes)) } Error::Unexpected(response) => ( MilterEvent::UnexpectedResponse, trc::Value::from(response.to_string()), ), Error::Timeout => (MilterEvent::Timeout, trc::Value::None), Error::TLSInvalidName => (MilterEvent::TlsInvalidName, trc::Value::None), Error::Disconnected => (MilterEvent::Disconnected, trc::Value::None), }; trc::event!( Milter(code), SpanId = self.data.session_id, Id = milter.id.to_string(), Details = details, Elapsed = time.elapsed(), ); if milter.tempfail_on_error { return Err(FilterResponse::server_failure()); } } } } Ok(modifications) } async fn connect_and_run( &self, milter: &Milter, message: Option<&AuthenticatedMessage<'_>>, ) -> Result, Rejection> { // Build client let client = MilterClient::connect(milter, self.data.session_id).await?; if !milter.tls { self.run(client, message).await } else { self.run( client .into_tls( if !milter.tls_allow_invalid_certs { &self.server.inner.data.smtp_connectors.pki_verify } else { &self.server.inner.data.smtp_connectors.dummy_verify }, &milter.hostname, ) .await?, message, ) .await } } async fn run( &self, mut client: MilterClient, message: Option<&AuthenticatedMessage<'_>>, ) -> Result, Rejection> { // Option negotiation client.init().await?; // Connect stage let client_ptr = self .data .iprev .as_ref() .and_then(|ip_rev| ip_rev.ptr.as_ref()) .and_then(|ptrs| ptrs.first()) .map(|s| s.as_str()); client .connection( client_ptr.unwrap_or(self.data.helo_domain.as_str()), self.data.remote_ip, self.data.remote_port, Macros::new() .with_daemon_name(DAEMON_NAME) .with_local_hostname(&self.hostname) .with_client_address(self.data.remote_ip) .with_client_port(self.data.remote_port) .with_client_ptr(client_ptr.unwrap_or("unknown")), ) .await? .assert_continue()?; // EHLO/HELO let (tls_version, tls_cipher) = self.stream.tls_version_and_cipher(); client .helo( &self.data.helo_domain, Macros::new() .with_cipher(tls_cipher.as_ref()) .with_tls_version(tls_version.as_ref()), ) .await? .assert_continue()?; // Mail from if let Some(mail_from) = &self.data.mail_from { let addr = &mail_from.address_lcase; client .mail_from( &format!("<{addr}>"), None::<&[&str]>, if let Some(name) = self.authenticated_as() { Macros::new() .with_mail_address(addr) .with_sasl_login_name(name) } else { Macros::new().with_mail_address(addr) }, ) .await? .assert_continue()?; // Rcpt to for rcpt in &self.data.rcpt_to { client .rcpt_to( &format!("<{}>", rcpt.address_lcase), None::<&[&str]>, Macros::new().with_rcpt_address(&rcpt.address_lcase), ) .await? .assert_continue()?; } } if let Some(message) = message { // Data client.data().await?.assert_continue()?; // Headers client .headers(message.raw_parsed_headers().iter().map(|(k, v)| { ( std::str::from_utf8(k).unwrap_or_default(), std::str::from_utf8(v).unwrap_or_default(), ) })) .await? .assert_continue()?; // Message body let (action, modifications) = client.body(message.raw_message()).await?; action.assert_continue()?; // Quit let _ = client.quit().await; // Return modifications Ok(modifications) } else { // Quit let _ = client.quit().await; Ok(Vec::new()) } } } impl SessionData { pub fn apply_milter_modifications( &mut self, modifications: Vec, message: &AuthenticatedMessage<'_>, ) -> Option> { let mut body = Vec::new(); let mut header_changes = Vec::new(); let mut needs_rewrite = false; for modification in modifications { match modification { Modification::ChangeFrom { sender, mut args } => { // Change sender let sender = strip_brackets(&sender); let address_lcase = sender.to_lowercase(); let mut mail_from = SessionAddress { domain: address_lcase.domain_part().into(), address_lcase, address: sender, flags: 0, dsn_info: None, }; if !args.is_empty() { args.push('\n'); match Rfc5321Parser::new(&mut args.as_bytes().iter()) .mail_from_parameters(Cow::Borrowed("")) { Ok(addr) => { mail_from.flags = addr.flags; mail_from.dsn_info = addr.env_id.map(|e| e.into_owned()); } Err(err) => { trc::event!( Milter(MilterEvent::ParseError), SpanId = self.session_id, Details = "Failed to parse milter mailFrom parameters", Reason = err.to_string(), ); } } } self.mail_from = Some(mail_from); } Modification::AddRcpt { recipient, mut args, } => { // Add recipient let recipient = strip_brackets(&recipient); if recipient.contains('@') { let address_lcase = recipient.to_lowercase(); let mut rcpt = SessionAddress { domain: address_lcase.domain_part().into(), address_lcase, address: recipient, flags: 0, dsn_info: None, }; if !args.is_empty() { args.push('\n'); match Rfc5321Parser::new(&mut args.as_bytes().iter()) .rcpt_to_parameters(Cow::Borrowed("")) { Ok(addr) => { rcpt.flags = addr.flags; rcpt.dsn_info = addr.orcpt.map(|e| e.into_owned()); } Err(err) => { trc::event!( Milter(MilterEvent::ParseError), SpanId = self.session_id, Details = "Failed to parse milter rcptTo parameters", Reason = err.to_string(), ); } } } if !self.rcpt_to.contains(&rcpt) { self.rcpt_to.push(rcpt); } } } Modification::DeleteRcpt { recipient } => { let recipient = strip_brackets(&recipient); self.rcpt_to.retain(|r| r.address_lcase != recipient); } Modification::ReplaceBody { value } => { body.extend(value); } Modification::AddHeader { name, value } => { header_changes.push((0, name, value, false)); } Modification::InsertHeader { index, name, value } => { header_changes.push((index, name, value, false)); needs_rewrite = true; } Modification::ChangeHeader { index, name, value } => { if value.is_empty() || message .raw_parsed_headers() .iter() .any(|(n, _)| n.eq_ignore_ascii_case(name.as_bytes())) { header_changes.push((index, name, value, true)); needs_rewrite = true; } else { header_changes.push((0, name, value, false)); } } Modification::Quarantine { reason } => { header_changes.push((0, "X-Quarantine".into(), reason, false)); } } } // If there are no header changes return if header_changes.is_empty() { return if !body.is_empty() { let mut new_message = Vec::with_capacity(body.len() + message.raw_headers().len()); new_message.extend_from_slice(message.raw_headers()); new_message.extend(body); Some(new_message) } else { None }; } let new_body = if !body.is_empty() { &body[..] } else { message.raw_body() }; if needs_rewrite { let mut headers = message .raw_parsed_headers() .iter() .map(|(h, v)| (Cow::from(*h), Cow::from(*v))) .collect::>(); // Perform changes for (index, header_name, header_value, is_change) in header_changes { if is_change { let mut header_count = 0; for (pos, (name, value)) in headers.iter_mut().enumerate() { if name.eq_ignore_ascii_case(header_name.as_bytes()) { header_count += 1; if header_count == index { if !header_value.is_empty() { *value = Cow::from(header_value.as_bytes().to_vec()); } else { headers.remove(pos); } break; } } } } else { let mut header_pos = 0; if index > 0 { let mut header_count = 0; for (pos, (name, _)) in headers.iter().enumerate() { if name.eq_ignore_ascii_case(header_name.as_bytes()) { header_pos = pos; header_count += 1; if header_count == index { break; } } } } headers.insert( header_pos, ( Cow::from(header_name.as_bytes().to_vec()), Cow::from(header_value.as_bytes().to_vec()), ), ); } } // Write new headers let mut new_message = Vec::with_capacity( new_body.len() + message.raw_headers().len() + headers .iter() .map(|(h, v)| h.len() + v.len() + 4) .sum::(), ); for (header, value) in headers { new_message.extend_from_slice(header.as_ref()); if value.first().is_some_and(|c| c.is_ascii_whitespace()) { new_message.extend_from_slice(b":"); } else { new_message.extend_from_slice(b": "); } new_message.extend_from_slice(value.as_ref()); if value.last().is_none_or(|c| *c != b'\n') { new_message.extend_from_slice(b"\r\n"); } } new_message.extend_from_slice(b"\r\n"); new_message.extend(new_body); Some(new_message) } else { let mut new_message = Vec::with_capacity( new_body.len() + message.raw_headers().len() + header_changes .iter() .map(|(_, h, v, _)| h.len() + v.len() + 4) .sum::(), ); for (_, header, value, _) in header_changes { new_message.extend_from_slice(header.as_bytes()); new_message.extend_from_slice(b": "); new_message.extend_from_slice(value.as_bytes()); if !value.ends_with('\n') { new_message.extend_from_slice(b"\r\n"); } } new_message.extend_from_slice(message.raw_headers()); new_message.extend(new_body); Some(new_message) } } } impl Action { fn assert_continue(self) -> Result<(), Rejection> { match self { Action::Continue | Action::Accept => Ok(()), action => Err(Rejection::Action(action)), } } } impl From for Rejection { fn from(err: Error) -> Self { Rejection::Error(err) } } fn strip_brackets(addr: &str) -> String { let addr = addr.trim(); if let Some(addr) = addr.strip_prefix('<') { if let Some((addr, _)) = addr.rsplit_once('>') { addr.trim().into() } else { addr.trim().into() } } else { addr.into() } } ================================================ FILE: crates/smtp/src/inbound/milter/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{borrow::Cow, fmt::Display, net::IpAddr, sync::Arc, time::Duration}; use common::config::smtp::session::MilterVersion; use serde::{Deserialize, Serialize}; use tokio::io::{AsyncRead, AsyncWrite}; use self::receiver::Receiver; pub mod client; pub mod macros; pub mod message; pub mod protocol; pub mod receiver; pub struct MilterClient { stream: T, buf: Vec, bytes_read: usize, timeout_cmd: Duration, timeout_data: Duration, receiver: Receiver, version: MilterVersion, options: u32, flags_actions: u32, flags_protocol: u32, id: Arc, session_id: u64, } #[derive(Debug)] pub enum Error { Io(std::io::Error), FrameTooLarge(usize), FrameInvalid(Vec), Unexpected(Response), Timeout, TLSInvalidName, Disconnected, } impl From for Error { fn from(err: std::io::Error) -> Self { Error::Io(err) } } pub enum Command<'x> { Abort, Body { value: &'x [u8], }, EndOfBody, Data, Connect { hostname: &'x [u8], port: u16, address: IpAddr, }, Macro { macros: Macros<'x>, }, Header { name: &'x [u8], value: &'x [u8], }, EndOfHeader, Helo { hostname: &'x [u8], }, MailFrom { sender: &'x [u8], args: Option>, }, Rcpt { recipient: &'x [u8], args: Option>, }, OptionNegotiation(Options), Quit, QuitNewConnection, } #[derive(Debug)] pub enum Response { Action(Action), Modification(Modification), Progress, Skip, SetSymbols, OptionNegotiation(Options), } #[derive(Debug)] pub enum Action { Accept, Continue, Discard, Reject, TempFail, ReplyCode { code: [u8; 3], text: String }, Shutdown, ConnectionFailure, } #[derive(Debug, Clone, Serialize, Deserialize)] pub enum Modification { ChangeFrom { sender: String, args: String, }, AddRcpt { recipient: String, args: String, }, DeleteRcpt { recipient: String, }, ReplaceBody { value: Vec, }, AddHeader { name: String, value: String, }, InsertHeader { index: u32, name: String, value: String, }, ChangeHeader { index: u32, name: String, value: String, }, Quarantine { reason: String, }, } #[derive(Debug)] pub struct Options { pub version: u32, pub actions: u32, pub protocol: u32, } #[derive(Default)] pub struct Macros<'x> { cmdcode: u8, macros: Vec>, } pub struct Macro<'x> { name: &'x [u8], value: Cow<'x, [u8]>, } pub const SMFIF_NONE: u32 = 0x00000000; /* no flags */ pub const SMFIF_ADDHDRS: u32 = 0x00000001; /* filter may add headers */ pub const SMFIF_CHGBODY: u32 = 0x00000002; /* filter may replace body */ pub const SMFIF_MODBODY: u32 = SMFIF_CHGBODY; /* backwards compatible */ pub const SMFIF_ADDRCPT: u32 = 0x00000004; /* filter may add recipients */ pub const SMFIF_DELRCPT: u32 = 0x00000008; /* filter may delete recipients */ pub const SMFIF_CHGHDRS: u32 = 0x00000010; /* filter may change/delete headers */ pub const SMFIF_QUARANTINE: u32 = 0x00000020; /* filter may quarantine envelope */ pub const SMFIF_CHGFROM: u32 = 0x00000040; /* filter may change "from" (envelope sender) */ pub const SMFIF_ADDRCPT_PAR: u32 = 0x00000080; /* add recipients incl. args */ pub const SMFIF_SETSYMLIST: u32 = 0x00000100; /* filter can send set of symbols (macros) that it wants */ pub const SMFIP_NOCONNECT: u32 = 0x00000001; /* MTA should not send connect info */ pub const SMFIP_NOHELO: u32 = 0x00000002; /* MTA should not send HELO info */ pub const SMFIP_NOMAIL: u32 = 0x00000004; /* MTA should not send MAIL info */ pub const SMFIP_NORCPT: u32 = 0x00000008; /* MTA should not send RCPT info */ pub const SMFIP_NOBODY: u32 = 0x00000010; /* MTA should not send body */ pub const SMFIP_NOHDRS: u32 = 0x00000020; /* MTA should not send headers */ pub const SMFIP_NOEOH: u32 = 0x00000040; /* MTA should not send EOH */ pub const SMFIP_NR_HDR: u32 = 0x00000080; /* No reply for headers */ pub const SMFIP_NOHREPL: u32 = SMFIP_NR_HDR; /* No reply for headers */ pub const SMFIP_NOUNKNOWN: u32 = 0x00000100; /* MTA should not send unknown commands */ pub const SMFIP_NODATA: u32 = 0x00000200; /* MTA should not send DATA */ pub const SMFIP_SKIP: u32 = 0x00000400; /* MTA understands SMFIS_SKIP */ pub const SMFIP_RCPT_REJ: u32 = 0x00000800; /* MTA should also send rejected RCPTs */ pub const SMFIP_NR_CONN: u32 = 0x00001000; /* No reply for connect */ pub const SMFIP_NR_HELO: u32 = 0x00002000; /* No reply for HELO */ pub const SMFIP_NR_MAIL: u32 = 0x00004000; /* No reply for MAIL */ pub const SMFIP_NR_RCPT: u32 = 0x00008000; /* No reply for RCPT */ pub const SMFIP_NR_DATA: u32 = 0x00010000; /* No reply for DATA */ pub const SMFIP_NR_UNKN: u32 = 0x00020000; /* No reply for UNKN */ pub const SMFIP_NR_EOH: u32 = 0x00040000; /* No reply for eoh */ pub const SMFIP_NR_BODY: u32 = 0x00080000; /* No reply for body chunk */ pub const SMFIP_HDR_LEADSPC: u32 = 0x00100000; /* header value leading space */ pub const SMFIP_MDS_256K: u32 = 0x10000000; /* MILTER_MAX_DATA_SIZE=256K */ pub const SMFIP_MDS_1M: u32 = 0x20000000; /* MILTER_MAX_DATA_SIZE=1M */ pub type Result = std::result::Result; impl Display for Command<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Command::Abort => write!(f, "ABORT"), Command::Body { value } => write!(f, "BODY [{} bytes]", value.len()), Command::EndOfBody => write!(f, "EOB"), Command::Connect { hostname, port, address, } => write!( f, "CONNECT (host: {:?}, port: {}, address: {})", std::str::from_utf8(hostname).unwrap_or_default(), port, address ), Command::Macro { macros } => { write!(f, "MACRO (code: {}, params: ", macros.cmdcode)?; for macro_ in ¯os.macros { write!( f, "({:?}, {:?})", std::str::from_utf8(macro_.name).unwrap_or_default(), std::str::from_utf8(macro_.value.as_ref()).unwrap_or_default() )?; } write!(f, ")") } Command::Header { name, value } => { write!( f, "HEADER ({}: {:?})", std::str::from_utf8(name).unwrap_or_default(), std::str::from_utf8(value).unwrap_or_default() ) } Command::EndOfHeader => write!(f, "EOH"), Command::Helo { hostname } => write!( f, "HELO {:?}", std::str::from_utf8(hostname).unwrap_or_default() ), Command::MailFrom { sender, args } => { write!( f, "MAIL (from: {}, params: ", std::str::from_utf8(sender).unwrap_or_default() )?; if let Some(args) = args { for arg in args { write!(f, " {}", std::str::from_utf8(arg).unwrap_or_default())?; } } write!(f, ")") } Command::Rcpt { recipient, args } => { write!( f, "RCPT (to: {}, params: ", std::str::from_utf8(recipient).unwrap_or_default() )?; if let Some(args) = args { for arg in args { write!(f, " {}", std::str::from_utf8(arg).unwrap_or_default())?; } } write!(f, ")") } Command::OptionNegotiation(opt) => write!(f, "OPTNEG ({})", opt), Command::Quit => write!(f, "QUIT"), Command::Data => write!(f, "DATA"), Command::QuitNewConnection => write!(f, "QUIT_NC"), } } } impl Display for Response { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Response::Action(action) => write!(f, "ACTION ({})", action), Response::Modification(modification) => write!(f, "MODIFICATION ({})", modification), Response::Progress => write!(f, "PROGRESS"), Response::OptionNegotiation(opt) => write!(f, "OPTNEG ({})", opt), Response::Skip => write!(f, "SKIP"), Response::SetSymbols => write!(f, "SET_SYMBOLS"), } } } impl Display for Action { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Action::Accept => write!(f, "ACCEPT"), Action::Continue => write!(f, "CONTINUE"), Action::Discard => write!(f, "DISCARD"), Action::Reject => write!(f, "REJECT"), Action::TempFail => write!(f, "TEMPFAIL"), Action::ReplyCode { code, text } => { write!(f, "REPLYCODE (code: {:?}, text: {})", code, text) } Action::Shutdown => write!(f, "SHUTDOWN"), Action::ConnectionFailure => write!(f, "CONN_FAIL"), } } } impl Display for Modification { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Modification::AddRcpt { recipient, args } => { write!(f, "ADD_RCPT (recipient: {}, args: {})", recipient, args) } Modification::DeleteRcpt { recipient } => { write!(f, "DEL_RCPT (recipient: {})", recipient) } Modification::ReplaceBody { value } => { write!(f, "REPLACE_BODY ({} bytes)", value.len()) } Modification::AddHeader { name, value } => { write!(f, "ADD_HEADER ({}: {})", name, value) } Modification::ChangeHeader { index, name, value } => { write!(f, "CHANGE_HEADER (index: {}, {}: {})", index, name, value) } Modification::Quarantine { reason } => write!(f, "QUARANTINE ({})", reason), Modification::ChangeFrom { sender, args } => { write!(f, "CHANGE_FROM (<{}> {})", sender, args) } Modification::InsertHeader { index, name, value } => { write!(f, "INSERT_HEADER (index: {}, {}: {})", index, name, value) } } } } impl Display for Options { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "version: {}, actions: [", self.version,)?; if self.actions & SMFIF_ADDHDRS != 0 { write!(f, "ADDHDRS ")?; } if self.actions & SMFIF_CHGBODY != 0 { write!(f, "CHGBODY ")?; } if self.actions & SMFIF_CHGHDRS != 0 { write!(f, "CHGHDRS ")?; } if self.actions & SMFIF_ADDRCPT != 0 { write!(f, "ADDRCPT ")?; } if self.actions & SMFIF_DELRCPT != 0 { write!(f, "DELRCPT ")?; } if self.actions & SMFIF_CHGFROM != 0 { write!(f, "CHGFROM ")?; } if self.actions & SMFIF_QUARANTINE != 0 { write!(f, "QUARANTINE ")?; } if self.actions & SMFIF_CHGFROM != 0 { write!(f, "CHGFROM ")?; } if self.actions & SMFIF_ADDRCPT_PAR != 0 { write!(f, "ADDRCPT_PAR ")?; } if self.actions & SMFIF_SETSYMLIST != 0 { write!(f, "SETSYMLIST ")?; } write!(f, "], options: [",)?; if self.protocol & SMFIP_NOCONNECT != 0 { write!(f, "NOCONNECT ")?; } if self.protocol & SMFIP_NOHELO != 0 { write!(f, "NOHELO ")?; } if self.protocol & SMFIP_NOMAIL != 0 { write!(f, "NOMAIL ")?; } if self.protocol & SMFIP_NORCPT != 0 { write!(f, "NORCPT ")?; } if self.protocol & SMFIP_NOBODY != 0 { write!(f, "NOBODY ")?; } if self.protocol & SMFIP_NOHDRS != 0 { write!(f, "NOHDRS ")?; } if self.protocol & SMFIP_NOEOH != 0 { write!(f, "NOEOH ")?; } if self.protocol & SMFIP_NR_HDR != 0 { write!(f, "NR_HDR ")?; } if self.protocol & SMFIP_NOUNKNOWN != 0 { write!(f, "NOUNKNOWN ")?; } if self.protocol & SMFIP_NODATA != 0 { write!(f, "NODATA ")?; } if self.protocol & SMFIP_SKIP != 0 { write!(f, "SKIP ")?; } if self.protocol & SMFIP_RCPT_REJ != 0 { write!(f, "RCPT_REJ ")?; } if self.protocol & SMFIP_NR_CONN != 0 { write!(f, "NR_CONN ")?; } if self.protocol & SMFIP_NR_HELO != 0 { write!(f, "NR_HELO ")?; } if self.protocol & SMFIP_NR_MAIL != 0 { write!(f, "NR_MAIL ")?; } if self.protocol & SMFIP_NR_RCPT != 0 { write!(f, "NR_RCPT ")?; } if self.protocol & SMFIP_NR_DATA != 0 { write!(f, "NR_DATA ")?; } if self.protocol & SMFIP_NR_UNKN != 0 { write!(f, "NR_UNKN ")?; } if self.protocol & SMFIP_NR_EOH != 0 { write!(f, "NR_EOH ")?; } if self.protocol & SMFIP_NR_BODY != 0 { write!(f, "NR_BODY ")?; } if self.protocol & SMFIP_HDR_LEADSPC != 0 { write!(f, "HDR_LEADSPC ")?; } if self.protocol & SMFIP_MDS_256K != 0 { write!(f, "MDS_256K ")?; } if self.protocol & SMFIP_MDS_1M != 0 { write!(f, "MDS_1M ")?; } write!(f, "]") } } impl Display for Error { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Error::Io(err) => write!(f, "IO error: {}", err), Error::FrameTooLarge(size) => { write!(f, "Milter response of {} bytes is too large.", size) } Error::FrameInvalid(frame) => write!( f, "Invalid milter response: {:?}", frame.get(0..100).unwrap_or(frame.as_ref()) ), Error::Unexpected(response) => write!(f, "Unexpected response: {}", response), Error::Timeout => write!(f, "Connection timed out"), Error::TLSInvalidName => write!(f, "Invalid TLS name"), Error::Disconnected => write!(f, "Disconnected unexpectedly"), } } } ================================================ FILE: crates/smtp/src/inbound/milter/protocol.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::net::IpAddr; use crate::inbound::milter::Action; use super::{Command, Error, Modification, Options, Response}; pub const SMFIR_ADDRCPT: u8 = b'+'; /* add recipient */ pub const SMFIR_DELRCPT: u8 = b'-'; /* remove recipient */ pub const SMFIR_ADDRCPT_PAR: u8 = b'2'; /* add recipient (incl. ESMTP args) */ pub const SMFIR_SHUTDOWN: u8 = b'4'; /* 421: shutdown (internal to MTA) */ pub const SMFIR_ACCEPT: u8 = b'a'; /* accept */ pub const SMFIR_REPLBODY: u8 = b'b'; /* replace body (chunk) */ pub const SMFIR_CONTINUE: u8 = b'c'; /* continue */ pub const SMFIR_DISCARD: u8 = b'd'; /* discard */ pub const SMFIR_CHGFROM: u8 = b'e'; /* change envelope sender (from) */ pub const SMFIR_CONN_FAIL: u8 = b'f'; /* cause a connection failure */ pub const SMFIR_ADDHEADER: u8 = b'h'; /* add header */ pub const SMFIR_INSHEADER: u8 = b'i'; /* insert header */ pub const SMFIR_SETSYMLIST: u8 = b'l'; /* set list of symbols (macros) */ pub const SMFIR_CHGHEADER: u8 = b'm'; /* change header */ pub const SMFIR_PROGRESS: u8 = b'p'; /* progress */ pub const SMFIR_QUARANTINE: u8 = b'q'; /* quarantine */ pub const SMFIR_REJECT: u8 = b'r'; /* reject */ pub const SMFIR_SKIP: u8 = b's'; /* skip */ pub const SMFIR_TEMPFAIL: u8 = b't'; /* tempfail */ pub const SMFIR_REPLYCODE: u8 = b'y'; /* reply code etc */ pub const SMFIC_ABORT: u8 = b'A'; /* Abort */ pub const SMFIC_BODY: u8 = b'B'; /* Body chunk */ pub const SMFIC_CONNECT: u8 = b'C'; /* Connection information */ pub const SMFIC_MACRO: u8 = b'D'; /* Define macro */ pub const SMFIC_BODYEOB: u8 = b'E'; /* final body chunk (End) */ pub const SMFIC_HELO: u8 = b'H'; /* HELO/EHLO */ pub const SMFIC_QUIT_NC: u8 = b'K'; /* QUIT but new connection follows */ pub const SMFIC_HEADER: u8 = b'L'; /* Header */ pub const SMFIC_MAIL: u8 = b'M'; /* MAIL from */ pub const SMFIC_EOH: u8 = b'N'; /* EOH */ pub const SMFIC_OPTNEG: u8 = b'O'; /* Option negotiation */ pub const SMFIC_QUIT: u8 = b'Q'; /* QUIT */ pub const SMFIC_RCPT: u8 = b'R'; /* RCPT to */ pub const SMFIC_DATA: u8 = b'T'; /* DATA */ pub const SMFIC_UNKNOWN: u8 = b'U'; /* Any unknown command */ impl Command<'_> { fn build(command: u8, len: u32) -> Vec { let mut buf = Vec::with_capacity(len as usize + 1 + std::mem::size_of::()); buf.extend_from_slice((len + 1).to_be_bytes().as_ref()); buf.push(command); buf } pub fn serialize(self) -> Vec { match self { Command::Abort => Command::build(SMFIC_ABORT, 0), Command::Body { value } => { let mut buf = Command::build(SMFIC_BODY, value.len() as u32); buf.extend(value); buf } Command::EndOfBody => Command::build(SMFIC_BODYEOB, 0), Command::Connect { hostname, port, address, } => { /* char hostname[] Hostname, NUL terminated char family Protocol family (see below) uint16 port Port number (SMFIA_INET or SMFIA_INET6 only) char address[] IP address (ASCII) or unix socket path, NUL terminated */ let (address, family) = match address { IpAddr::V4(address) => (address.to_string(), b'4'), IpAddr::V6(address) => (address.to_string(), b'6'), }; let mut buf = Command::build( SMFIC_CONNECT, hostname.len() as u32 // hostname + 1 // NUL + 1 // family + std::mem::size_of::() as u32 // port + address.len() as u32 // address + 1, // NUL ); buf.extend(hostname); buf.push(0x00); buf.push(family); buf.extend(port.to_be_bytes().as_ref()); buf.extend(address.as_bytes()); buf.push(0x00); buf } Command::Macro { macros } => { let mut buf = Command::build( SMFIC_MACRO, macros.macros.iter().fold(1, |acc, macro_| { acc + macro_.name.len() as u32 + 1 + macro_.value.len() as u32 + 1 }), ); buf.push(macros.cmdcode); for macro_ in macros.macros { buf.extend(macro_.name); buf.push(0x00); buf.extend(macro_.value.as_ref()); buf.push(0x00); } buf } Command::Header { name, value } => { let mut buf = Command::build(SMFIC_HEADER, name.len() as u32 + 1 + value.len() as u32 + 1); buf.extend(name); buf.push(0x00); buf.extend(value); buf.push(0x00); buf } Command::EndOfHeader => Command::build(SMFIC_EOH, 0), Command::Helo { hostname } => { let mut buf = Command::build(SMFIC_HELO, hostname.len() as u32 + 1); buf.extend(hostname); buf.push(0x00); buf } Command::MailFrom { sender, args } => { let mut buf = Command::build( SMFIC_MAIL, sender.len() as u32 // sender + 1 // NUL + args.as_ref().map_or(0, |args| args.iter().fold(0, |acc, arg| acc + arg.len() as u32) + 1), // args ); buf.extend(sender); buf.push(0x00); if let Some(args) = args { for arg in args { buf.extend(arg); buf.push(0x00); } } buf } Command::Rcpt { recipient, args } => { let mut buf = Command::build( SMFIC_RCPT, recipient.len() as u32 // recipient + 1 // NUL + args.as_ref().map_or(0, |args| args.iter().fold(0, |acc, arg| acc + arg.len() as u32) + 1), // args ); buf.extend(recipient); buf.push(0x00); if let Some(args) = args { for arg in args { buf.extend(arg); buf.push(0x00); } } buf } Command::OptionNegotiation(opt) => { let mut buf = Command::build(SMFIC_OPTNEG, 3 * std::mem::size_of::() as u32); buf.extend(opt.version.to_be_bytes().as_ref()); buf.extend(opt.actions.to_be_bytes().as_ref()); buf.extend(opt.protocol.to_be_bytes().as_ref()); buf } Command::Quit => Command::build(SMFIC_QUIT, 0), // Version 6 Command::Data => Command::build(SMFIC_DATA, 0), Command::QuitNewConnection => Command::build(SMFIC_QUIT_NC, 0), } } #[cfg(feature = "test_mode")] pub fn deserialize(bytes: &[u8]) -> Command<'_> { let mut reader = PacketReader::new(bytes); match reader.byte() { SMFIC_ABORT => Command::Abort, SMFIC_BODY => Command::Body { value: &bytes[1..] }, SMFIC_BODYEOB => Command::EndOfBody, SMFIC_CONNECT => { let hostname = reader.read_nul_terminated().unwrap(); let family = reader.byte(); let port = reader.read_u16(); let address = std::str::from_utf8(reader.read_nul_terminated().unwrap()).unwrap(); Command::Connect { hostname, port, address: match family { b'4' => IpAddr::V4(address.parse().unwrap()), b'6' => IpAddr::V6(address.parse().unwrap()), _ => unreachable!(), }, } } SMFIC_MACRO => { let cmdcode = reader.byte(); let mut macros = Vec::new(); while let Some(name) = reader.read_nul_terminated() { let value = reader.read_nul_terminated().unwrap(); macros.push(super::Macro { name, value: value.into(), }); } Command::Macro { macros: super::Macros { cmdcode, macros }, } } SMFIC_HEADER => { let name = reader.read_nul_terminated().unwrap(); let value = reader.read_nul_terminated().unwrap(); Command::Header { name, value } } SMFIC_EOH => Command::EndOfHeader, SMFIC_HELO => { let hostname = reader.read_nul_terminated().unwrap(); Command::Helo { hostname } } SMFIC_MAIL => { let sender = reader.read_nul_terminated().unwrap(); let mut args = Vec::new(); while let Some(arg) = reader.read_nul_terminated() { args.push(arg); } Command::MailFrom { sender, args: Some(args), } } SMFIC_RCPT => { let recipient = reader.read_nul_terminated().unwrap(); let mut args = Vec::new(); while let Some(arg) = reader.read_nul_terminated() { args.push(arg); } Command::Rcpt { recipient, args: Some(args), } } SMFIC_OPTNEG => Command::OptionNegotiation(super::Options { version: reader.read_u32(), actions: reader.read_u32(), protocol: reader.read_u32(), }), SMFIC_QUIT => Command::Quit, SMFIC_DATA => Command::Data, SMFIC_QUIT_NC => Command::QuitNewConnection, c => panic!("Unknown command: {}", char::from(c)), } } } impl Response { pub fn deserialize(bytes: &[u8]) -> Option { let frame_len = bytes.len().saturating_sub(1); let mut bytes = bytes.iter(); match *bytes.next()? { SMFIR_ADDRCPT => Response::Modification(Modification::AddRcpt { recipient: read_nul_terminated(&mut bytes, frame_len)?, args: String::new(), }), SMFIR_DELRCPT => Response::Modification(Modification::DeleteRcpt { recipient: read_nul_terminated(&mut bytes, frame_len)?, }), SMFIR_ACCEPT => Response::Action(Action::Accept), SMFIR_REPLBODY => { let mut body = Vec::with_capacity(frame_len); body.extend(bytes); Response::Modification(Modification::ReplaceBody { value: body }) } SMFIR_CONTINUE => Response::Action(Action::Continue), SMFIR_DISCARD => Response::Action(Action::Discard), SMFIR_ADDHEADER => Response::Modification(Modification::AddHeader { name: read_nul_terminated(&mut bytes, 16)?, value: read_nul_terminated(&mut bytes, frame_len)?, }), SMFIR_CHGHEADER => Response::Modification(Modification::ChangeHeader { index: read_u32(&mut bytes)?, name: read_nul_terminated(&mut bytes, 16)?, value: read_nul_terminated(&mut bytes, frame_len)?, }), SMFIR_PROGRESS => Response::Progress, SMFIR_QUARANTINE => Response::Modification(Modification::Quarantine { reason: read_nul_terminated(&mut bytes, frame_len)?, }), SMFIR_REJECT => Response::Action(Action::Reject), SMFIR_TEMPFAIL => Response::Action(Action::TempFail), SMFIR_REPLYCODE => { let code = [*bytes.next()?, *bytes.next()?, *bytes.next()?]; bytes.next()?; // Space Response::Action(Action::ReplyCode { code, text: read_nul_terminated(&mut bytes, frame_len)?, }) } SMFIC_OPTNEG => Response::OptionNegotiation(Options { version: read_u32(&mut bytes)?, actions: read_u32(&mut bytes)?, protocol: read_u32(&mut bytes)?, }), // V6 SMFIR_ADDRCPT_PAR => Response::Modification(Modification::AddRcpt { recipient: read_nul_terminated(&mut bytes, frame_len)?, args: read_nul_terminated(&mut bytes, frame_len)?, }), SMFIR_CHGFROM => Response::Modification(Modification::ChangeFrom { sender: read_nul_terminated(&mut bytes, frame_len)?, args: read_nul_terminated(&mut bytes, frame_len)?, }), SMFIR_SKIP => Response::Skip, SMFIR_SETSYMLIST => Response::SetSymbols, SMFIR_SHUTDOWN => Response::Action(Action::Shutdown), SMFIR_CONN_FAIL => Response::Action(Action::ConnectionFailure), SMFIR_INSHEADER => Response::Modification(Modification::InsertHeader { index: read_u32(&mut bytes)?, name: read_nul_terminated(&mut bytes, 16)?, value: read_nul_terminated(&mut bytes, frame_len)?, }), _ => return None, } .into() } pub fn can_continue(&self) -> bool { matches!( self, Response::Progress | Response::Action(Action::Accept | Action::Continue) ) } pub fn into_action(self) -> super::Result { match self { Response::Action(action) => Ok(action), response => Err(Error::Unexpected(response)), } } #[cfg(feature = "test_mode")] pub fn serialize(&self) -> Vec { match self { Response::Action(action) => match action { Action::Accept => Command::build(SMFIR_ACCEPT, 0), Action::Continue => Command::build(SMFIR_CONTINUE, 0), Action::Discard => Command::build(SMFIR_DISCARD, 0), Action::Reject => Command::build(SMFIR_REJECT, 0), Action::TempFail => Command::build(SMFIR_TEMPFAIL, 0), Action::ReplyCode { code, text } => { let mut buf = Command::build(SMFIR_REPLYCODE, text.len() as u32 + 4 + 1); buf.extend(code); buf.push(b' '); buf.extend(text.as_bytes()); buf.push(0x00); buf } Action::Shutdown => Command::build(SMFIR_SHUTDOWN, 0), Action::ConnectionFailure => Command::build(SMFIR_CONN_FAIL, 0), }, Response::Modification(modif) => match modif { Modification::ChangeFrom { sender, args } => { let mut buf = Command::build(SMFIR_CHGFROM, sender.len() as u32 + args.len() as u32 + 2); buf.extend(sender.as_bytes()); buf.push(0x00); buf.extend(args.as_bytes()); buf.push(0x00); buf } Modification::AddRcpt { recipient, args } => { let mut buf = Command::build( SMFIR_ADDRCPT_PAR, recipient.len() as u32 + args.len() as u32 + 2, ); buf.extend(recipient.as_bytes()); buf.push(0x00); buf.extend(args.as_bytes()); buf.push(0x00); buf } Modification::DeleteRcpt { recipient } => { let mut buf = Command::build(SMFIR_DELRCPT, recipient.len() as u32 + 1); buf.extend(recipient.as_bytes()); buf.push(0x00); buf } Modification::ReplaceBody { value } => { let mut buf = Command::build(SMFIR_REPLBODY, value.len() as u32); buf.extend(value); buf } Modification::AddHeader { name, value } => { let mut buf = Command::build(SMFIR_ADDHEADER, name.len() as u32 + value.len() as u32 + 2); buf.extend(name.as_bytes()); buf.push(0x00); buf.extend(value.as_bytes()); buf.push(0x00); buf } Modification::InsertHeader { index, name, value } => { let mut buf = Command::build( SMFIR_INSHEADER, name.len() as u32 + value.len() as u32 + std::mem::size_of::() as u32 + 2, ); buf.extend(index.to_be_bytes().as_ref()); buf.extend(name.as_bytes()); buf.push(0x00); buf.extend(value.as_bytes()); buf.push(0x00); buf } Modification::ChangeHeader { index, name, value } => { let mut buf = Command::build( SMFIR_CHGHEADER, name.len() as u32 + value.len() as u32 + std::mem::size_of::() as u32 + 2, ); buf.extend(index.to_be_bytes().as_ref()); buf.extend(name.as_bytes()); buf.push(0x00); buf.extend(value.as_bytes()); buf.push(0x00); buf } Modification::Quarantine { reason } => { let mut buf = Command::build(SMFIR_QUARANTINE, reason.len() as u32 + 1); buf.extend(reason.as_bytes()); buf.push(0x00); buf } }, Response::Progress => Command::build(SMFIR_PROGRESS, 0), Response::Skip => Command::build(SMFIR_SKIP, 0), Response::SetSymbols => Command::build(SMFIR_SETSYMLIST, 0), Response::OptionNegotiation(opt) => { let mut buf = Command::build(SMFIC_OPTNEG, 3 * std::mem::size_of::() as u32); buf.extend(opt.version.to_be_bytes().as_ref()); buf.extend(opt.actions.to_be_bytes().as_ref()); buf.extend(opt.protocol.to_be_bytes().as_ref()); buf } } } } fn read_nul_terminated(bytes: &mut std::slice::Iter, expected_len: usize) -> Option { let mut buf = Vec::with_capacity(expected_len); loop { match bytes.next()? { 0x00 => break, byte => buf.push(*byte), } } String::from_utf8(buf).ok() } fn read_u32(bytes: &mut std::slice::Iter) -> Option { let mut buf = [0u8; 4]; for byte in buf.iter_mut() { *byte = *bytes.next()?; } Some(u32::from_be_bytes(buf)) } #[cfg(feature = "test_mode")] pub struct PacketReader<'x> { bytes: &'x [u8], iter: std::iter::Enumerate>, } #[cfg(feature = "test_mode")] impl<'x> PacketReader<'x> { pub fn new(bytes: &'x [u8]) -> PacketReader<'x> { Self { bytes, iter: bytes.iter().enumerate(), } } pub fn byte(&mut self) -> u8 { *self.iter.next().unwrap().1 } pub fn read_nul_terminated(&mut self) -> Option<&'x [u8]> { let (start_pos, ch) = self.iter.next()?; let mut end_pos = start_pos; if *ch != 0x00 { loop { match self.iter.next().unwrap().1 { 0x00 => break, _ => end_pos += 1, } } } Some(&self.bytes[start_pos..end_pos + 1]) } pub fn read_u32(&mut self) -> u32 { let mut buf = [0u8; 4]; for byte in buf.iter_mut() { *byte = self.byte(); } u32::from_be_bytes(buf) } pub fn read_u16(&mut self) -> u16 { let mut buf = [0u8; 2]; for byte in buf.iter_mut() { *byte = self.byte(); } u16::from_be_bytes(buf) } } ================================================ FILE: crates/smtp/src/inbound/milter/receiver.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::borrow::Cow; enum State { Len { buf: [u8; std::mem::size_of::()], bytes_read: usize, }, Frame { buf: Vec, frame_len: usize, }, } pub struct Receiver { packet_pos: usize, state: State, max_frame_len: usize, } pub enum FrameResult<'x> { Frame(Cow<'x, [u8]>), Incomplete, TooLarge(usize), } impl Default for State { fn default() -> Self { State::Len { buf: [0; std::mem::size_of::()], bytes_read: 0, } } } impl Receiver { pub fn with_max_frame_len(max_frame_len: usize) -> Self { Receiver { packet_pos: 0, state: State::default(), max_frame_len, } } pub fn read_frame<'x>(&mut self, packet: &'x [u8]) -> FrameResult<'x> { if !packet.is_empty() { match &mut self.state { State::Len { buf, bytes_read } => { while *bytes_read < std::mem::size_of::() { if let Some(byte) = packet.get(self.packet_pos) { buf[*bytes_read] = *byte; *bytes_read += 1; self.packet_pos += 1; } else { self.packet_pos = 0; return FrameResult::Incomplete; } } let length = u32::from_be_bytes(*buf) as usize; if length <= self.max_frame_len { if let Some(frame) = packet.get(self.packet_pos..self.packet_pos + length) { self.packet_pos += length; self.state = State::default(); FrameResult::Frame(frame.into()) } else { let mut buf = Vec::with_capacity(length); if let Some(bytes_available) = packet.get(self.packet_pos..) { buf.extend(bytes_available); } self.state = State::Frame { buf, frame_len: length, }; self.packet_pos = 0; FrameResult::Incomplete } } else { FrameResult::TooLarge(length) } } State::Frame { buf, frame_len } => { let bytes_pending = *frame_len - buf.len(); if let Some(bytes) = packet.get(self.packet_pos..self.packet_pos + bytes_pending) { let mut buf = std::mem::take(buf); buf.extend(bytes); self.packet_pos += bytes_pending; self.state = State::default(); FrameResult::Frame(buf.into()) } else if let Some(bytes_available) = packet.get(self.packet_pos..) { buf.extend(bytes_available); self.packet_pos = 0; FrameResult::Incomplete } else { self.packet_pos = 0; FrameResult::Incomplete } } } } else { FrameResult::Incomplete } } } ================================================ FILE: crates/smtp/src/inbound/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::borrow::Cow; use common::config::smtp::auth::{ArcSealer, DkimSigner}; use mail_auth::{ ArcOutput, AuthenticatedMessage, AuthenticationResults, DkimResult, DmarcResult, IprevResult, SpfResult, arc::ArcSet, dkim::Signature, dmarc::Policy, }; pub mod auth; pub mod data; pub mod ehlo; pub mod hooks; pub mod mail; pub mod milter; pub mod rcpt; pub mod session; pub mod spam; pub mod spawn; pub mod vrfy; #[derive(Debug, Default)] pub struct FilterResponse { pub message: Cow<'static, str>, pub disconnect: bool, } pub trait ArcSeal { fn seal<'x>( &self, message: &'x AuthenticatedMessage, results: &'x AuthenticationResults, arc_output: &'x ArcOutput, ) -> mail_auth::Result>; } impl ArcSeal for ArcSealer { fn seal<'x>( &self, message: &'x AuthenticatedMessage, results: &'x AuthenticationResults, arc_output: &'x ArcOutput, ) -> mail_auth::Result> { match self { ArcSealer::RsaSha256(sealer) => sealer.seal(message, results, arc_output), ArcSealer::Ed25519Sha256(sealer) => sealer.seal(message, results, arc_output), } } } pub trait DkimSign { fn sign(&self, message: &[u8]) -> mail_auth::Result; fn sign_chained(&self, message: &[&[u8]]) -> mail_auth::Result; } impl DkimSign for DkimSigner { fn sign(&self, message: &[u8]) -> mail_auth::Result { match self { DkimSigner::RsaSha256(signer) => signer.sign(message), DkimSigner::Ed25519Sha256(signer) => signer.sign(message), } } fn sign_chained(&self, message: &[&[u8]]) -> mail_auth::Result { match self { DkimSigner::RsaSha256(signer) => signer.sign_chained(message.iter().copied()), DkimSigner::Ed25519Sha256(signer) => signer.sign_chained(message.iter().copied()), } } } pub trait AuthResult { fn as_str(&self) -> &'static str; } impl AuthResult for SpfResult { fn as_str(&self) -> &'static str { match self { SpfResult::Pass => "pass", SpfResult::Fail => "fail", SpfResult::SoftFail => "softfail", SpfResult::Neutral => "neutral", SpfResult::None => "none", SpfResult::TempError => "temperror", SpfResult::PermError => "permerror", } } } impl AuthResult for IprevResult { fn as_str(&self) -> &'static str { match self { IprevResult::Pass => "pass", IprevResult::Fail(_) => "fail", IprevResult::TempError(_) => "temperror", IprevResult::PermError(_) => "permerror", IprevResult::None => "none", } } } impl AuthResult for DkimResult { fn as_str(&self) -> &'static str { match self { DkimResult::Pass => "pass", DkimResult::None => "none", DkimResult::Neutral(_) => "neutral", DkimResult::Fail(_) => "fail", DkimResult::PermError(_) => "permerror", DkimResult::TempError(_) => "temperror", } } } impl AuthResult for DmarcResult { fn as_str(&self) -> &'static str { match self { DmarcResult::Pass => "pass", DmarcResult::Fail(_) => "fail", DmarcResult::TempError(_) => "temperror", DmarcResult::PermError(_) => "permerror", DmarcResult::None => "none", } } } impl AuthResult for Policy { fn as_str(&self) -> &'static str { match self { Policy::Reject => "reject", Policy::Quarantine => "quarantine", Policy::None | Policy::Unspecified => "none", } } } impl FilterResponse { pub fn accept() -> Self { Self { message: Cow::Borrowed("250 2.0.0 Message queued for delivery.\r\n"), disconnect: false, } } pub fn reject() -> Self { Self { message: Cow::Borrowed("503 5.5.3 Message rejected.\r\n"), disconnect: false, } } pub fn temp_fail() -> Self { Self { message: Cow::Borrowed("451 4.3.5 Unable to accept message at this time.\r\n"), disconnect: false, } } pub fn shutdown() -> Self { Self { message: Cow::Borrowed("421 4.3.0 Server shutting down.\r\n"), disconnect: true, } } pub fn server_failure() -> Self { Self { message: Cow::Borrowed("451 4.3.5 Unable to accept message at this time.\r\n"), disconnect: false, } } pub fn disconnect(self) -> Self { Self { disconnect: true, ..self } } pub fn into_bytes(self) -> Cow<'static, [u8]> { match self.message { Cow::Borrowed(s) => Cow::Borrowed(s.as_bytes()), Cow::Owned(s) => Cow::Owned(s.into_bytes()), } } } ================================================ FILE: crates/smtp/src/inbound/rcpt.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ core::{Session, SessionAddress}, scripts::ScriptResult, }; use common::{ KV_GREYLIST, config::smtp::session::Stage, listener::SessionStream, scripts::ScriptModification, }; use directory::backend::RcptType; use smtp_proto::{ RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, RcptTo, }; use std::borrow::Cow; use store::dispatch::lookup::KeyValue; use trc::{SecurityEvent, SmtpEvent}; use utils::DomainPart; impl Session { pub async fn handle_rcpt_to(&mut self, to: RcptTo>) -> Result<(), ()> { #[cfg(feature = "test_mode")] if self.instance.id.ends_with("-debug") { if to.address.contains("fail@") { return self.write(b"503 5.5.1 Invalid recipient.\r\n").await; } else if (to.address.contains("delay-random@") && rand::random()) || to.address.contains("delay@") { return self.write(b"451 4.5.3 Try again later.\r\n").await; } else if to.address.contains("slow@") { tokio::time::sleep(std::time::Duration::from_secs( rand::random::() % 5 + 5, )) .await; } } if self.data.mail_from.is_none() { trc::event!( Smtp(SmtpEvent::MailFromMissing), SpanId = self.data.session_id, ); return self.write(b"503 5.5.1 MAIL is required first.\r\n").await; } else if self.data.rcpt_to.len() >= self.params.rcpt_max { trc::event!( Smtp(SmtpEvent::TooManyRecipients), SpanId = self.data.session_id, Limit = self.params.rcpt_max, ); return self.write(b"455 4.5.3 Too many recipients.\r\n").await; } // Verify parameters if ((to.flags & (RCPT_NOTIFY_DELAY | RCPT_NOTIFY_NEVER | RCPT_NOTIFY_SUCCESS | RCPT_NOTIFY_FAILURE) != 0) || to.orcpt.is_some()) && !self.params.rcpt_dsn { trc::event!(Smtp(SmtpEvent::DsnDisabled), SpanId = self.data.session_id,); return self .write(b"501 5.5.4 DSN extension has been disabled.\r\n") .await; } // Build RCPT let address_lcase = to.address.to_lowercase(); let rcpt = SessionAddress { domain: address_lcase.domain_part().into(), address_lcase, address: to.address.into_owned(), flags: to.flags, dsn_info: to.orcpt.map(|e| e.into_owned()), }; if self.data.rcpt_to.contains(&rcpt) { trc::event!( Smtp(SmtpEvent::RcptToDuplicate), SpanId = self.data.session_id, To = rcpt.address_lcase, ); self.data.rcpt_oks += 1; return self.write(b"250 2.1.5 OK\r\n").await; } self.data.rcpt_to.push(rcpt); // Address rewriting and Sieve filtering let rcpt_config = &self.server.core.smtp.session.rcpt; let rcpt_script = self .server .eval_if::(&rcpt_config.script, self, self.data.session_id) .await .and_then(|name| { self.server .get_trusted_sieve_script(&name, self.data.session_id) .map(|s| (s.clone(), name)) }); let session_config = &self.server.core.smtp.session; if rcpt_script.is_some() || !rcpt_config.rewrite.is_empty() || session_config .milters .iter() .any(|m| m.run_on_stage.contains(&Stage::Rcpt)) || session_config .hooks .iter() .any(|h| h.run_on_stage.contains(&Stage::Rcpt)) { // Sieve filtering if let Some((script, script_id)) = rcpt_script { match self .run_script( script_id, script.clone(), self.build_script_parameters("rcpt"), ) .await { ScriptResult::Accept { modifications } => { if !modifications.is_empty() { for modification in modifications { if let ScriptModification::SetEnvelope { name, value } = modification { self.data.apply_envelope_modification(name, value); } } } } ScriptResult::Reject(message) => { self.data.rcpt_to.pop(); return self.write(message.as_bytes()).await; } _ => (), } } // Milter filtering if let Err(message) = self.run_milters(Stage::Rcpt, None).await { self.data.rcpt_to.pop(); return self.write(message.message.as_bytes()).await; } // MTAHook filtering if let Err(message) = self.run_mta_hooks(Stage::Rcpt, None, None).await { self.data.rcpt_to.pop(); return self.write(message.message.as_bytes()).await; } // Address rewriting if let Some(new_address) = self .server .eval_if::(&rcpt_config.rewrite, self, self.data.session_id) .await { let rcpt = self.data.rcpt_to.last_mut().unwrap(); trc::event!( Smtp(SmtpEvent::RcptToRewritten), SpanId = self.data.session_id, Details = rcpt.address_lcase.clone(), To = new_address.clone(), ); if new_address.contains('@') { rcpt.address_lcase = new_address.to_lowercase(); rcpt.domain = rcpt.address_lcase.domain_part().into(); rcpt.address = new_address; } } // Check for duplicates let rcpt = self.data.rcpt_to.last().unwrap(); if self.data.rcpt_to.iter().filter(|r| r == &rcpt).count() > 1 { trc::event!( Smtp(SmtpEvent::RcptToDuplicate), SpanId = self.data.session_id, To = rcpt.address_lcase.clone(), ); self.data.rcpt_to.pop(); self.data.rcpt_oks += 1; return self.write(b"250 2.1.5 OK\r\n").await; } } // Verify address let rcpt = self.data.rcpt_to.last().unwrap(); let mut rcpt_members = None; if let Some(directory) = self .server .eval_if::(&rcpt_config.directory, self, self.data.session_id) .await .and_then(|name| self.server.get_directory(&name)) { match directory.is_local_domain(&rcpt.domain).await { Ok(true) => { match self .server .rcpt(directory, &rcpt.address_lcase, self.data.session_id) .await { Ok(RcptType::Mailbox) => {} Ok(RcptType::List(members)) => { rcpt_members = Some(members); } Ok(RcptType::Invalid) => { trc::event!( Smtp(SmtpEvent::MailboxDoesNotExist), SpanId = self.data.session_id, To = rcpt.address_lcase.clone(), ); let rcpt_to = self.data.rcpt_to.pop().unwrap().address_lcase; return self .rcpt_error(b"550 5.1.2 Mailbox does not exist.\r\n", rcpt_to) .await; } Err(err) => { trc::error!( err.span_id(self.data.session_id) .caused_by(trc::location!()) .details("Failed to verify address.") ); self.data.rcpt_to.pop(); return self .write(b"451 4.4.3 Unable to verify address at this time.\r\n") .await; } } } Ok(false) => { if !self .server .eval_if(&rcpt_config.relay, self, self.data.session_id) .await .unwrap_or(false) { trc::event!( Smtp(SmtpEvent::RelayNotAllowed), SpanId = self.data.session_id, To = rcpt.address_lcase.clone(), ); let rcpt_to = self.data.rcpt_to.pop().unwrap().address_lcase; return self .rcpt_error(b"550 5.1.2 Relay not allowed.\r\n", rcpt_to) .await; } } Err(err) => { trc::error!( err.span_id(self.data.session_id) .caused_by(trc::location!()) .details("Failed to verify address.") ); self.data.rcpt_to.pop(); return self .write(b"451 4.4.3 Unable to verify address at this time.\r\n") .await; } } } else if !self .server .eval_if(&rcpt_config.relay, self, self.data.session_id) .await .unwrap_or(false) { trc::event!( Smtp(SmtpEvent::RelayNotAllowed), SpanId = self.data.session_id, To = rcpt.address_lcase.clone(), ); let rcpt_to = self.data.rcpt_to.pop().unwrap().address_lcase; return self .rcpt_error(b"550 5.1.2 Relay not allowed.\r\n", rcpt_to) .await; } if self.is_allowed().await { // Greylist if let Some(greylist_duration) = self .server .core .spam .grey_list_expiry .filter(|_| self.data.authenticated_as.is_none()) { let from_addr = self .data .mail_from .as_ref() .unwrap() .address_lcase .as_bytes(); let to_addr = self.data.rcpt_to.last().unwrap().address_lcase.as_bytes(); let mut key = Vec::with_capacity(from_addr.len() + to_addr.len() + 1); key.push(KV_GREYLIST); key.extend_from_slice(from_addr); key.extend_from_slice(to_addr); match self.server.in_memory_store().key_exists(key.clone()).await { Ok(true) => (), Ok(false) => { match self .server .in_memory_store() .key_set(KeyValue::new(key, vec![]).expires(greylist_duration)) .await { Ok(_) => { let rcpt = self.data.rcpt_to.pop().unwrap(); trc::event!( Smtp(SmtpEvent::RcptToGreylisted), SpanId = self.data.session_id, To = rcpt.address_lcase, ); return self .write( concat!( "452 4.2.2 Greylisted, please try ", "again in a few moments.\r\n" ) .as_bytes(), ) .await; } Err(err) => { trc::error!( err.span_id(self.data.session_id) .caused_by(trc::location!()) .details("Failed to set greylist.") ); } } } Err(err) => { trc::error!( err.span_id(self.data.session_id) .caused_by(trc::location!()) .details("Failed to check greylist.") ); } } } trc::event!( Smtp(SmtpEvent::RcptTo), SpanId = self.data.session_id, To = self.data.rcpt_to.last().unwrap().address_lcase.clone(), ); } else { trc::event!( Smtp(SmtpEvent::RateLimitExceeded), SpanId = self.data.session_id, To = self.data.rcpt_to.last().unwrap().address_lcase.clone(), ); self.data.rcpt_to.pop(); return self .write(b"452 4.4.5 Rate limit exceeded, try again later.\r\n") .await; } // Expand list if let Some(members) = rcpt_members { let list_addr = self.data.rcpt_to.pop().unwrap(); let orcpt = format!("rfc822;{}", list_addr.address_lcase); for member in members { let mut member_addr = SessionAddress::new(member); if !self.data.rcpt_to.contains(&member_addr) && member_addr.address_lcase != list_addr.address_lcase { member_addr.dsn_info = orcpt.clone().into(); member_addr.flags = list_addr.flags; self.data.rcpt_to.push(member_addr); } } } self.data.rcpt_oks += 1; self.write(b"250 2.1.5 OK\r\n").await } async fn rcpt_error(&mut self, response: &[u8], rcpt: String) -> Result<(), ()> { tokio::time::sleep(self.params.rcpt_errors_wait).await; self.data.rcpt_errors += 1; let has_too_many_errors = self.data.rcpt_errors >= self.params.rcpt_errors_max; match self .server .is_rcpt_fail2banned(self.data.remote_ip, &rcpt) .await { Ok(true) => { trc::event!( Security(SecurityEvent::AbuseBan), SpanId = self.data.session_id, RemoteIp = self.data.remote_ip, To = rcpt, ); } Ok(false) => { if has_too_many_errors { trc::event!( Smtp(SmtpEvent::TooManyInvalidRcpt), SpanId = self.data.session_id, Limit = self.params.rcpt_errors_max, To = rcpt, ); } } Err(err) => { trc::error!( err.span_id(self.data.session_id) .caused_by(trc::location!()) .details("Failed to check if IP should be banned.") ); } } if !has_too_many_errors { self.write(response).await } else { self.write(b"451 4.3.0 Too many errors, disconnecting.\r\n") .await?; Err(()) } } } ================================================ FILE: crates/smtp/src/inbound/session.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{ config::{server::ServerProtocol, smtp::session::Mechanism}, expr::{self, functions::ResolveVariable, *}, listener::SessionStream, }; use compact_str::ToCompactString; use smtp_proto::{ request::receiver::{ BdatReceiver, DataReceiver, DummyDataReceiver, DummyLineReceiver, LineReceiver, MAX_LINE_LENGTH, }, *, }; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use trc::{NetworkEvent, SecurityEvent, SmtpEvent}; use crate::core::{Session, State}; use super::auth::SaslToken; impl Session { pub async fn ingest(&mut self, bytes: &[u8]) -> Result { let mut iter = bytes.iter(); let mut state = std::mem::replace(&mut self.state, State::None); 'outer: loop { match &mut state { State::Request(receiver) => loop { match receiver.ingest(&mut iter) { Ok(request) => match request { Request::Rcpt { to } => { self.handle_rcpt_to(to).await?; } Request::Mail { from } => { self.handle_mail_from(from).await?; } Request::Ehlo { host } => { if self.instance.protocol == ServerProtocol::Smtp { self.handle_ehlo(host, true).await?; } else { trc::event!( Smtp(SmtpEvent::LhloExpected), SpanId = self.data.session_id, ); self.write(b"500 5.5.1 Invalid command.\r\n").await?; } } Request::Data => { if self.can_send_data().await? { self.write(b"354 Start mail input; end with .\r\n") .await?; self.data.message = Vec::with_capacity(1024); state = State::Data(DataReceiver::new()); continue 'outer; } } Request::Bdat { chunk_size, is_last, } => { state = if chunk_size + self.data.message.len() < self.params.max_message_size { if self.data.message.is_empty() { self.data.message = Vec::with_capacity(chunk_size); } else { self.data.message.reserve(chunk_size); } State::Bdat(BdatReceiver::new(chunk_size, is_last)) } else { // Chunk is too large, ignore. State::DataTooLarge(DummyDataReceiver::new_bdat(chunk_size)) }; continue 'outer; } Request::Auth { mechanism, initial_response, } => { let auth: u64 = self .server .eval_if::( &self.server.core.smtp.session.auth.mechanisms, self, self.data.session_id, ) .await .unwrap_or_default() .into(); if auth == 0 || self.params.auth_directory.is_none() { trc::event!( Smtp(SmtpEvent::AuthNotAllowed), SpanId = self.data.session_id, ); self.write(b"503 5.5.1 AUTH not allowed.\r\n").await?; } else if let Some(authenticated_as) = self.authenticated_as() { trc::event!( Smtp(SmtpEvent::AlreadyAuthenticated), SpanId = self.data.session_id, AccountName = authenticated_as.to_string(), ); self.write(b"503 5.5.1 Already authenticated.\r\n").await?; } else if let Some(mut token) = SaslToken::from_mechanism(mechanism & auth) { if self .handle_sasl_response( &mut token, initial_response.as_bytes(), ) .await? { state = State::Sasl(LineReceiver::new(token)); continue 'outer; } } else { trc::event!( Smtp(SmtpEvent::AuthMechanismNotSupported), SpanId = self.data.session_id, ); self.write( b"554 5.7.8 Authentication mechanism not supported.\r\n", ) .await?; } } Request::Noop { .. } => { trc::event!(Smtp(SmtpEvent::Noop), SpanId = self.data.session_id,); self.write(b"250 2.0.0 OK\r\n").await?; } Request::Vrfy { value } => { self.handle_vrfy(value).await?; } Request::Expn { value } => { self.handle_expn(value).await?; } Request::StartTls => { if !self.stream.is_tls() { if self.instance.acceptor.is_tls() { trc::event!( Smtp(SmtpEvent::StartTls), SpanId = self.data.session_id, ); self.write(b"220 2.0.0 Ready to start TLS.\r\n").await?; #[cfg(any(test, feature = "test_mode"))] if self.data.helo_domain.contains("badtls") { return Err(()); } self.state = State::default(); return Ok(false); } else { trc::event!( Smtp(SmtpEvent::StartTlsUnavailable), SpanId = self.data.session_id, ); self.write(b"502 5.7.0 TLS not available.\r\n").await?; } } else { trc::event!( Smtp(SmtpEvent::StartTlsAlready), SpanId = self.data.session_id, ); self.write(b"504 5.7.4 Already in TLS mode.\r\n").await?; } } Request::Rset => { trc::event!(Smtp(SmtpEvent::Rset), SpanId = self.data.session_id,); self.reset(); self.write(b"250 2.0.0 OK\r\n").await?; } Request::Quit => { trc::event!(Smtp(SmtpEvent::Quit), SpanId = self.data.session_id,); self.write(b"221 2.0.0 Bye.\r\n").await?; return Err(()); } Request::Help { .. } => { trc::event!(Smtp(SmtpEvent::Help), SpanId = self.data.session_id,); self.write(b"250 2.0.0 Help can be found at https://stalw.art\r\n") .await?; } Request::Helo { host } => { if self.instance.protocol == ServerProtocol::Smtp { self.handle_ehlo(host, false).await?; } else { trc::event!( Smtp(SmtpEvent::LhloExpected), SpanId = self.data.session_id, ); self.write(b"500 5.5.1 Invalid command: LHLO expected.\r\n") .await?; } } Request::Lhlo { host } => { if self.instance.protocol == ServerProtocol::Lmtp { self.handle_ehlo(host, true).await?; } else { trc::event!( Smtp(SmtpEvent::EhloExpected), SpanId = self.data.session_id, ); self.write(b"502 5.5.1 Invalid command: EHLO expected.\r\n") .await?; } } cmd @ (Request::Etrn { .. } | Request::Atrn { .. } | Request::Burl { .. }) => { trc::event!( Smtp(SmtpEvent::CommandNotImplemented), SpanId = self.data.session_id, Details = format!("{cmd:?}"), ); self.write(b"502 5.5.1 Command not implemented.\r\n") .await?; } }, Err(err) => match err { Error::NeedsMoreData { .. } => break 'outer, Error::UnknownCommand | Error::InvalidResponse { .. } => { // Check for port scanners if !self.is_authenticated() { match self .server .is_scanner_fail2banned(self.data.remote_ip) .await { Ok(true) => { trc::event!( Security(SecurityEvent::ScanBan), SpanId = self.data.session_id, RemoteIp = self.data.remote_ip, Reason = "Invalid SMTP command", ); return Err(()); } Ok(false) => {} Err(err) => { trc::error!( err.span_id(self.data.session_id) .details("Failed to check for fail2ban") ); } } } trc::event!( Smtp(SmtpEvent::InvalidCommand), SpanId = self.data.session_id, ); self.write(b"500 5.5.1 Invalid command.\r\n").await?; } Error::InvalidSenderAddress => { trc::event!( Smtp(SmtpEvent::InvalidSenderAddress), SpanId = self.data.session_id, ); self.write(b"501 5.1.8 Bad sender's system address.\r\n") .await?; } Error::InvalidRecipientAddress => { trc::event!( Smtp(SmtpEvent::InvalidRecipientAddress), SpanId = self.data.session_id, ); self.write( b"501 5.1.3 Bad destination mailbox address syntax.\r\n", ) .await?; } Error::SyntaxError { syntax } => { trc::event!( Smtp(SmtpEvent::SyntaxError), SpanId = self.data.session_id, Details = syntax ); if !self.params.ehlo_reject_non_fqdn && syntax.starts_with("EHLO ") { self.handle_ehlo("null".into(), true).await? } else { self.write( format!("501 5.5.2 Syntax error, expected: {syntax}\r\n") .as_bytes(), ) .await?; } } Error::InvalidParameter { param } => { trc::event!( Smtp(SmtpEvent::InvalidParameter), SpanId = self.data.session_id, Details = param ); self.write( format!("501 5.5.4 Invalid parameter {param:?}.\r\n") .as_bytes(), ) .await?; } Error::UnsupportedParameter { param } => { trc::event!( Smtp(SmtpEvent::UnsupportedParameter), SpanId = self.data.session_id, Details = param.clone() ); self.write( format!("504 5.5.4 Unsupported parameter {param:?}.\r\n") .as_bytes(), ) .await?; } Error::ResponseTooLong => { state = State::RequestTooLarge(DummyLineReceiver::default()); continue 'outer; } }, } }, State::Data(receiver) => { if self.data.message.len() + bytes.len() < self.params.max_message_size { if receiver.ingest(&mut iter, &mut self.data.message) { let message = self.queue_message().await; let num_responses = if self.instance.protocol == ServerProtocol::Smtp { 1 } else { self.data.rcpt_oks }; if !message.is_empty() { for _ in 0..num_responses { self.write(message.as_ref()).await?; } self.reset(); state = State::default(); } else { // Disconnect requested return Err(()); } } else { break 'outer; } } else { state = State::DataTooLarge(DummyDataReceiver::new_data(receiver)); } } State::Bdat(receiver) => { if receiver.ingest(&mut iter, &mut self.data.message) { if self.can_send_data().await? { if receiver.is_last { let message = self.queue_message().await; if !message.is_empty() { let num_responses = if self.instance.protocol == ServerProtocol::Smtp { 1 } else { self.data.rcpt_oks }; for _ in 0..num_responses { self.write(message.as_ref()).await?; } self.reset(); } else { // Disconnect requested return Err(()); } } else { self.write(b"250 2.6.0 Chunk accepted.\r\n").await?; } } else { self.data.message = Vec::with_capacity(0); } state = State::default(); } else { break 'outer; } } State::Sasl(receiver) => { if receiver.ingest(&mut iter) { if receiver.buf.len() < MAX_LINE_LENGTH { if self .handle_sasl_response(&mut receiver.state, &receiver.buf) .await? { receiver.buf.clear(); continue 'outer; } } else { trc::event!( Smtp(SmtpEvent::AuthExchangeTooLong), SpanId = self.data.session_id, Limit = MAX_LINE_LENGTH, ); self.auth_error( b"500 5.5.6 Authentication Exchange line is too long.\r\n", ) .await?; } state = State::default(); } else { break 'outer; } } State::DataTooLarge(receiver) => { if receiver.ingest(&mut iter) { trc::event!( Smtp(SmtpEvent::MessageTooLarge), SpanId = self.data.session_id, ); self.data.message = Vec::with_capacity(0); self.write(b"552 5.3.4 Message too big for system.\r\n") .await?; state = State::default(); } else { break 'outer; } } State::RequestTooLarge(receiver) => { if receiver.ingest(&mut iter) { trc::event!( Smtp(SmtpEvent::RequestTooLarge), SpanId = self.data.session_id, ); self.write(b"554 5.3.4 Line is too long.\r\n").await?; state = State::default(); } else { break 'outer; } } State::None | State::Accepted(_) => unreachable!(), } } self.state = state; Ok(true) } } impl Session { pub fn reset(&mut self) { self.data.mail_from = None; self.data.spf_mail_from = None; self.data.rcpt_to.clear(); self.data.message = Vec::with_capacity(0); self.data.priority = 0; self.data.delivery_by = 0; self.data.future_release = 0; self.data.rcpt_oks = 0; } #[inline(always)] pub async fn write(&mut self, bytes: &[u8]) -> Result<(), ()> { match self.stream.write_all(bytes).await { Ok(_) => match self.stream.flush().await { Ok(_) => { trc::event!( Smtp(SmtpEvent::RawOutput), SpanId = self.data.session_id, Size = bytes.len(), Contents = trc::Value::from_maybe_string(bytes), ); Ok(()) } Err(err) => { trc::event!( Network(NetworkEvent::FlushError), SpanId = self.data.session_id, Reason = err.to_string(), ); Err(()) } }, Err(err) => { trc::event!( Network(NetworkEvent::WriteError), SpanId = self.data.session_id, Reason = err.to_string(), ); Err(()) } } } #[inline(always)] pub async fn read(&mut self, bytes: &mut [u8]) -> Result { match self.stream.read(bytes).await { Ok(len) => { trc::event!( Smtp(SmtpEvent::RawInput), SpanId = self.data.session_id, Size = len, Contents = String::from_utf8_lossy(bytes.get(0..len).unwrap_or_default()).into_owned(), ); Ok(len) } Err(err) => { trc::event!( Network(NetworkEvent::ReadError), SpanId = self.data.session_id, Reason = err.to_string(), ); Err(()) } } } } impl ResolveVariable for Session { fn resolve_variable(&self, variable: u32) -> expr::Variable<'_> { match variable { V_RECIPIENT => self .data .rcpt_to .last() .map(|r| r.address_lcase.as_str()) .unwrap_or_default() .into(), V_RECIPIENT_DOMAIN => self .data .rcpt_to .last() .map(|r| r.domain.as_str()) .unwrap_or_default() .into(), V_RECIPIENTS => self .data .rcpt_to .iter() .map(|r| Variable::from(r.address_lcase.as_str())) .collect::>() .into(), V_SENDER => self .data .mail_from .as_ref() .map(|m| m.address_lcase.as_str()) .unwrap_or_default() .into(), V_SENDER_DOMAIN => self .data .mail_from .as_ref() .map(|m| m.domain.as_str()) .unwrap_or_default() .into(), V_HELO_DOMAIN => self.data.helo_domain.as_str().into(), V_AUTHENTICATED_AS => self.authenticated_as().unwrap_or_default().into(), V_LISTENER => self.instance.id.as_str().into(), V_REMOTE_IP => self.data.remote_ip_str.as_str().into(), V_REMOTE_PORT => self.data.remote_port.into(), V_LOCAL_IP => self.data.local_ip_str.as_str().into(), V_LOCAL_PORT => self.data.local_port.into(), V_TLS => self.stream.is_tls().into(), V_PRIORITY => self.data.priority.to_compact_string().into(), V_PROTOCOL => self.instance.protocol.as_str().into(), V_ASN => self .data .asn_geo_data .asn .as_ref() .map(|a| a.id) .unwrap_or_default() .into(), V_COUNTRY => self .data .asn_geo_data .country .as_ref() .map(|c| c.as_str()) .unwrap_or_default() .into(), _ => expr::Variable::default(), } } fn resolve_global(&self, _: &str) -> Variable<'_> { Variable::Integer(0) } } ================================================ FILE: crates/smtp/src/inbound/spam.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{config::spamfilter::SpamFilterAction, listener::SessionStream}; use mail_auth::{ArcOutput, DkimOutput, DmarcResult, dmarc::Policy}; use mail_parser::Message; use spam_filter::{ SpamFilterInput, analysis::{ init::SpamFilterInit, score::{SpamFilterAnalyzeScore, SpamFilterScore}, }, }; use crate::core::Session; impl Session { pub async fn spam_classify<'x>( &'x self, message: &'x Message<'x>, dkim_result: &'x [DkimOutput<'x>], arc_result: Option<&'x ArcOutput<'x>>, dmarc_result: Option<&'x DmarcResult>, dmarc_policy: Option<&'x Policy>, ) -> SpamFilterAction { let server = &self.server; let mut ctx = server.spam_filter_init(self.build_spam_input( message, dkim_result, arc_result, dmarc_result, dmarc_policy, )); if !self.is_authenticated() { // Spam classification server.spam_filter_classify(&mut ctx).await } else { // Do not classify authenticated sessions SpamFilterAction::Disabled } } pub fn build_spam_input<'x>( &'x self, message: &'x Message<'x>, dkim_result: &'x [DkimOutput<'x>], arc_result: Option<&'x ArcOutput>, dmarc_result: Option<&'x DmarcResult>, dmarc_policy: Option<&'x Policy>, ) -> SpamFilterInput<'x> { SpamFilterInput { message, span_id: self.data.session_id, arc_result, spf_ehlo_result: self.data.spf_ehlo.as_ref(), spf_mail_from_result: self.data.spf_mail_from.as_ref(), dkim_result, dmarc_result, dmarc_policy, iprev_result: self.data.iprev.as_ref(), remote_ip: self.data.remote_ip, ehlo_domain: self.data.helo_domain.as_str().into(), authenticated_as: self.data.authenticated_as.as_ref().map(|a| a.name.as_str()), asn: self.data.asn_geo_data.asn.as_ref().map(|a| a.id), country: self.data.asn_geo_data.country.as_ref().map(|c| c.as_str()), is_tls: self.stream.is_tls(), env_from: self .data .mail_from .as_ref() .map(|m| m.address_lcase.as_str()) .unwrap_or_default(), env_from_flags: self .data .mail_from .as_ref() .map(|m| m.flags) .unwrap_or_default(), env_rcpt_to: self .data .rcpt_to .iter() .map(|r| r.address_lcase.as_str()) .collect(), is_test: false, is_train: false, } } } ================================================ FILE: crates/smtp/src/inbound/spawn.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Instant; use common::{ config::smtp::session::Stage, core::BuildServer, listener::{self, SessionManager, SessionStream}, }; use tokio_rustls::server::TlsStream; use trc::{SecurityEvent, SmtpEvent}; use crate::{ core::{Session, SessionData, SessionParameters, SmtpSessionManager, State}, scripts::ScriptResult, }; impl SessionManager for SmtpSessionManager { async fn handle(self, session: listener::SessionData) { // Build server and create session let server = self.inner.build_server(); let _in_flight = session.in_flight; let mut session = Session { data: SessionData::new( session.local_ip, session.local_port, session.remote_ip, session.remote_port, server.lookup_asn_country(session.remote_ip).await, session.session_id, ), hostname: "".into(), server, instance: session.instance, state: State::default(), stream: session.stream, params: SessionParameters::default(), }; // Enforce throttle if session.is_allowed().await && session.init_conn().await && session.handle_conn().await && session.instance.acceptor.is_tls() && let Ok(mut session) = session.into_tls().await { session.handle_conn().await; } } #[allow(clippy::manual_async_fn)] fn shutdown(&self) -> impl std::future::Future + Send { async { let _ = self .inner .ipc .queue_tx .send(common::ipc::QueueEvent::Stop) .await; let _ = self .inner .ipc .report_tx .send(common::ipc::ReportingEvent::Stop) .await; } } } impl Session { pub async fn init_conn(&mut self) -> bool { self.eval_session_params().await; let config = &self.server.core.smtp.session.connect; // Sieve filtering if let Some((script, script_id)) = self .server .eval_if::(&config.script, self, self.data.session_id) .await .and_then(|name| { self.server .get_trusted_sieve_script(&name, self.data.session_id) .map(|s| (s, name)) }) && let ScriptResult::Reject(message) = self .run_script( script_id, script.clone(), self.build_script_parameters("connect"), ) .await { let _ = self.write(message.as_bytes()).await; return false; } // Milter filtering if let Err(message) = self.run_milters(Stage::Connect, None).await { let _ = self.write(message.message.as_bytes()).await; return false; } // MTAHook filtering if let Err(message) = self.run_mta_hooks(Stage::Connect, None, None).await { let _ = self.write(message.message.as_bytes()).await; return false; } // Obtain hostname self.hostname = self .server .eval_if::(&config.hostname, self, self.data.session_id) .await .unwrap_or_default(); if self.hostname.is_empty() { trc::event!( Smtp(SmtpEvent::MissingLocalHostname), SpanId = self.data.session_id, ); self.hostname = "localhost".into(); } // Obtain greeting let greeting = self .server .eval_if::(&config.greeting, self, self.data.session_id) .await .filter(|g| !g.is_empty()) .map(|g| format!("220 {}\r\n", g)) .unwrap_or_else(|| "220 Stalwart ESMTP at your service.\r\n".to_string()); if self.write(greeting.as_bytes()).await.is_err() { return false; } true } pub async fn handle_conn(&mut self) -> bool { let mut buf = vec![0; 8192]; let mut shutdown_rx = self.instance.shutdown_rx.clone(); loop { tokio::select! { result = tokio::time::timeout( self.params.timeout, self.read(&mut buf)) => { match result { Ok(Ok(bytes_read)) => { if bytes_read > 0 { if Instant::now() < self.data.valid_until && bytes_read <= self.data.bytes_left { self.data.bytes_left -= bytes_read; match self.ingest(&buf[..bytes_read]).await { Ok(true) => (), Ok(false) => { return true; } Err(_) => { break; } } } else if bytes_read > self.data.bytes_left { self .write(format!("452 4.7.28 {} Session exceeded transfer quota.\r\n", self.hostname).as_bytes()) .await .ok(); trc::event!( Smtp(SmtpEvent::TransferLimitExceeded), SpanId = self.data.session_id, ); break; } else { self .write(format!("421 4.3.2 {} Session open for too long.\r\n", self.hostname).as_bytes()) .await .ok(); match self.server.is_loiter_fail2banned(self.data.remote_ip) .await { Ok(true) => { trc::event!( Security(SecurityEvent::LoiterBan), SpanId = self.data.session_id, RemoteIp = self.data.remote_ip, ); } Ok(false) => { trc::event!( Smtp(SmtpEvent::TimeLimitExceeded), SpanId = self.data.session_id, ); } Err(err) => { trc::error!(err .span_id(self.data.session_id) .caused_by(trc::location!()) .details("Failed to check if IP should be banned.")); } } break; } } else { trc::event!( Network(trc::NetworkEvent::Closed), SpanId = self.data.session_id, CausedBy = trc::location!() ); break; } } Ok(Err(_)) => { break; } Err(_) => { trc::event!( Network(trc::NetworkEvent::Timeout), SpanId = self.data.session_id, CausedBy = trc::location!() ); self .write(format!("221 2.0.0 {} Disconnecting inactive client.\r\n", self.hostname).as_bytes()) .await .ok(); break; } } }, _ = shutdown_rx.changed() => { trc::event!( Network(trc::NetworkEvent::Closed), SpanId = self.data.session_id, Reason = "Server shutting down", CausedBy = trc::location!() ); self.write(format!("421 4.3.0 {} Server shutting down.\r\n", self.hostname).as_bytes()).await.ok(); break; } }; } false } pub async fn into_tls(self) -> Result>, ()> { Ok(Session { hostname: self.hostname, stream: self .instance .tls_accept(self.stream, self.data.session_id) .await?, state: self.state, data: self.data, instance: self.instance, server: self.server, params: self.params, }) } } ================================================ FILE: crates/smtp/src/inbound/vrfy.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::core::Session; use common::listener::SessionStream; use std::{borrow::Cow, fmt::Write}; use trc::SmtpEvent; impl Session { pub async fn handle_vrfy(&mut self, address: Cow<'_, str>) -> Result<(), ()> { match self .server .eval_if::( &self.server.core.smtp.session.rcpt.directory, self, self.data.session_id, ) .await .and_then(|name| self.server.get_directory(&name)) { Some(directory) if self.params.can_vrfy => { match self .server .vrfy(directory, &address.to_lowercase(), self.data.session_id) .await { Ok(values) if !values.is_empty() => { let mut result = String::with_capacity(32); for (pos, value) in values.iter().enumerate() { let _ = write!( result, "250{}{}\r\n", if pos == values.len() - 1 { " " } else { "-" }, value ); } trc::event!( Smtp(SmtpEvent::Vrfy), SpanId = self.data.session_id, To = address.as_ref().to_string(), Result = values, ); self.write(result.as_bytes()).await } Ok(_) => { trc::event!( Smtp(SmtpEvent::VrfyNotFound), SpanId = self.data.session_id, To = address.as_ref().to_string(), ); self.write(b"550 5.1.2 Address not found.\r\n").await } Err(err) => { let is_not_supported = err.matches(trc::EventType::Store(trc::StoreEvent::NotSupported)); trc::error!(err.span_id(self.data.session_id).details("VRFY failed")); if !is_not_supported { self.write(b"252 2.4.3 Unable to verify address at this time.\r\n") .await } else { self.write(b"550 5.1.2 Address not found.\r\n").await } } } } _ => { trc::event!( Smtp(SmtpEvent::VrfyDisabled), SpanId = self.data.session_id, To = address.as_ref().to_string(), ); self.write(b"252 2.5.1 VRFY is disabled.\r\n").await } } } pub async fn handle_expn(&mut self, address: Cow<'_, str>) -> Result<(), ()> { match self .server .eval_if::( &self.server.core.smtp.session.rcpt.directory, self, self.data.session_id, ) .await .and_then(|name| self.server.get_directory(&name)) { Some(directory) if self.params.can_expn => { match self .server .expn(directory, &address.to_lowercase(), self.data.session_id) .await { Ok(values) if !values.is_empty() => { let mut result = String::with_capacity(32); for (pos, value) in values.iter().enumerate() { let _ = write!( result, "250{}{}\r\n", if pos == values.len() - 1 { " " } else { "-" }, value ); } trc::event!( Smtp(SmtpEvent::Expn), SpanId = self.data.session_id, To = address.as_ref().to_string(), Result = values, ); self.write(result.as_bytes()).await } Ok(_) => { trc::event!( Smtp(SmtpEvent::ExpnNotFound), SpanId = self.data.session_id, To = address.as_ref().to_string(), ); self.write(b"550 5.1.2 Mailing list not found.\r\n").await } Err(err) => { let is_not_supported = err.matches(trc::EventType::Store(trc::StoreEvent::NotSupported)); trc::error!(err.span_id(self.data.session_id).details("VRFY failed")); if !is_not_supported { self.write(b"252 2.4.3 Unable to expand mailing list at this time.\r\n") .await } else { self.write(b"550 5.1.2 Mailing list not found.\r\n").await } } } } _ => { trc::event!( Smtp(SmtpEvent::ExpnDisabled), SpanId = self.data.session_id, To = address.as_ref().to_string(), ); self.write(b"252 2.5.1 EXPN is disabled.\r\n").await } } } } ================================================ FILE: crates/smtp/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ #![warn(clippy::large_futures)] use common::{ Inner, manager::boot::{BootManager, IpcReceivers}, }; use queue::manager::SpawnQueue; use reporting::scheduler::SpawnReport; use std::sync::Arc; pub mod core; pub mod inbound; pub mod outbound; pub mod queue; pub mod reporting; pub mod scripts; pub trait StartQueueManager { fn start_queue_manager(&mut self); } pub trait SpawnQueueManager { fn spawn_queue_manager(&mut self, inner: Arc); } impl StartQueueManager for BootManager { fn start_queue_manager(&mut self) { self.ipc_rxs.spawn_queue_manager(self.inner.clone()); } } impl SpawnQueueManager for IpcReceivers { fn spawn_queue_manager(&mut self, inner: Arc) { // Spawn queue manager self.queue_rx.take().unwrap().spawn(inner.clone()); // Spawn report manager self.report_rx.take().unwrap().spawn(inner); } } ================================================ FILE: crates/smtp/src/outbound/client.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::session::SessionParams; use crate::queue::{Error, ErrorDetails, HostResponse, MessageWrapper, Status}; use mail_send::{Credentials, smtp::AssertReply}; use rustls::ClientConnection; use rustls_pki_types::ServerName; use smtp_proto::{ AUTH_CRAM_MD5, AUTH_DIGEST_MD5, AUTH_LOGIN, AUTH_OAUTHBEARER, AUTH_PLAIN, AUTH_XOAUTH2, EXT_START_TLS, EhloResponse, Response, response::{ generate::BitToString, parser::{MAX_RESPONSE_LENGTH, ResponseReceiver}, }, }; use std::{ net::{IpAddr, SocketAddr}, time::Duration, }; use tokio::{ io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}, net::{TcpSocket, TcpStream}, }; use tokio_rustls::{TlsConnector, client::TlsStream}; use trc::DeliveryEvent; pub struct SmtpClient { pub stream: T, pub timeout: Duration, pub session_id: u64, } impl SmtpClient { pub async fn authenticate( &mut self, credentials: impl AsRef>, capabilities: impl AsRef>, ) -> mail_send::Result<&mut Self> where U: AsRef + PartialEq + Eq + std::hash::Hash, { let credentials = credentials.as_ref(); let capabilities = capabilities.as_ref(); let mut available_mechanisms = match &credentials { Credentials::Plain { .. } => AUTH_CRAM_MD5 | AUTH_DIGEST_MD5 | AUTH_LOGIN | AUTH_PLAIN, Credentials::OAuthBearer { .. } => AUTH_OAUTHBEARER, Credentials::XOauth2 { .. } => AUTH_XOAUTH2, } & capabilities.auth_mechanisms; // Try authenticating from most secure to least secure let mut has_err = None; let mut has_failed = false; while available_mechanisms != 0 && !has_failed { let mechanism = 1 << ((63 - available_mechanisms.leading_zeros()) as u64); available_mechanisms ^= mechanism; match self.auth(mechanism, credentials).await { Ok(_) => { return Ok(self); } Err(err) => match err { mail_send::Error::UnexpectedReply(reply) => { has_failed = reply.code() == 535; has_err = reply.into(); } mail_send::Error::UnsupportedAuthMechanism => (), _ => return Err(err), }, } } if let Some(has_err) = has_err { Err(mail_send::Error::AuthenticationFailed(has_err)) } else { Err(mail_send::Error::UnsupportedAuthMechanism) } } pub(crate) async fn auth( &mut self, mechanism: u64, credentials: &Credentials, ) -> mail_send::Result<()> where U: AsRef + PartialEq + Eq + std::hash::Hash, { let mut reply = if (mechanism & (AUTH_PLAIN | AUTH_XOAUTH2 | AUTH_OAUTHBEARER)) != 0 { self.cmd( format!( "AUTH {} {}\r\n", mechanism.to_mechanism(), credentials.encode(mechanism, "")?, ) .as_bytes(), ) .await? } else { self.cmd(format!("AUTH {}\r\n", mechanism.to_mechanism()).as_bytes()) .await? }; for _ in 0..3 { match reply.code() { 334 => { reply = self .cmd( format!("{}\r\n", credentials.encode(mechanism, reply.message())?) .as_bytes(), ) .await?; } 235 => { return Ok(()); } _ => { return Err(mail_send::Error::UnexpectedReply(reply)); } } } Err(mail_send::Error::UnexpectedReply(reply)) } pub async fn read_greeting( &mut self, hostname: &str, ) -> Result<(), Status>, ErrorDetails>> { tokio::time::timeout(self.timeout, self.read()) .await .map_err(|_| Status::timeout(hostname, "reading greeting"))? .and_then(|r| r.assert_code(220)) .map_err(|err| Status::from_smtp_error(hostname, "", err)) } pub async fn read_smtp_data_response( &mut self, hostname: &str, bdat_cmd: &Option, ) -> Result, Status>, ErrorDetails>> { tokio::time::timeout(self.timeout, self.read()) .await .map_err(|_| Status::timeout(hostname, "reading SMTP DATA response"))? .map_err(|err| { Status::from_smtp_error(hostname, bdat_cmd.as_deref().unwrap_or("DATA"), err) }) } pub async fn read_lmtp_data_response( &mut self, hostname: &str, num_responses: usize, ) -> Result>>, Status>, ErrorDetails>> { tokio::time::timeout(self.timeout, async { self.read_many(num_responses).await }) .await .map_err(|_| Status::timeout(hostname, "reading LMTP DATA responses"))? .map_err(|err| Status::from_smtp_error(hostname, "", err)) } pub async fn write_chunks(&mut self, chunks: &[&[u8]]) -> Result<(), mail_send::Error> { for chunk in chunks { self.stream .write_all(chunk) .await .map_err(mail_send::Error::from)?; } self.stream.flush().await.map_err(mail_send::Error::from) } pub async fn send_message( &mut self, message: &MessageWrapper, bdat_cmd: &Option, params: &SessionParams<'_>, ) -> Result<(), Status>, ErrorDetails>> { match params .server .blob_store() .get_blob(message.message.blob_hash.as_slice(), 0..usize::MAX) .await { Ok(Some(raw_message)) => { tokio::time::timeout(params.conn_strategy.timeout_data, async { if let Some(bdat_cmd) = bdat_cmd { trc::event!( Delivery(DeliveryEvent::RawOutput), SpanId = self.session_id, Contents = bdat_cmd.clone(), Size = bdat_cmd.len() ); self.write_chunks(&[bdat_cmd.as_bytes(), &raw_message]) .await } else { trc::event!( Delivery(DeliveryEvent::RawOutput), SpanId = self.session_id, Contents = "DATA\r\n", Size = 6 ); self.write_chunks(&[b"DATA\r\n"]).await?; self.read().await?.assert_code(354)?; self.write_message(&raw_message) .await .map_err(mail_send::Error::from) } }) .await .map_err(|_| Status::timeout(params.hostname, "sending message"))? .map_err(|err| { Status::from_smtp_error( params.hostname, bdat_cmd.as_deref().unwrap_or("DATA"), err, ) }) } Ok(None) => { trc::event!( Queue(trc::QueueEvent::BlobNotFound), SpanId = message.span_id, BlobId = message.message.blob_hash.to_hex(), CausedBy = trc::location!() ); Err(Status::TemporaryFailure(ErrorDetails { entity: "localhost".into(), details: Error::Io("Queue system error.".into()), })) } Err(err) => { trc::error!( err.span_id(message.span_id) .details("Failed to fetch blobId") .caused_by(trc::location!()) ); Err(Status::TemporaryFailure(ErrorDetails { entity: "localhost".into(), details: Error::Io("Queue system error.".into()), })) } } } pub async fn say_helo( &mut self, params: &SessionParams<'_>, ) -> Result, Status>, ErrorDetails>> { let cmd = if params.is_smtp { format!("EHLO {}\r\n", params.local_hostname) } else { format!("LHLO {}\r\n", params.local_hostname) }; trc::event!( Delivery(DeliveryEvent::RawOutput), SpanId = self.session_id, Contents = cmd.clone(), Size = cmd.len() ); tokio::time::timeout(params.conn_strategy.timeout_ehlo, async { self.stream.write_all(cmd.as_bytes()).await?; self.stream.flush().await?; self.read_ehlo().await }) .await .map_err(|_| Status::timeout(params.hostname, "reading EHLO response"))? .map_err(|err| Status::from_smtp_error(params.hostname, &cmd, err)) } pub async fn quit(mut self: SmtpClient) { trc::event!( Delivery(DeliveryEvent::RawOutput), SpanId = self.session_id, Contents = "QUIT\r\n", Size = 6 ); let _ = tokio::time::timeout(Duration::from_secs(10), async { if self.stream.write_all(b"QUIT\r\n").await.is_ok() && self.stream.flush().await.is_ok() { let mut buf = [0u8; 128]; let _ = self.stream.read(&mut buf).await; } }) .await; } pub async fn read_ehlo(&mut self) -> mail_send::Result> { let mut buf = vec![0u8; 8192]; let mut buf_concat = Vec::with_capacity(0); loop { let br = self.stream.read(&mut buf).await?; if br == 0 { return Err(mail_send::Error::UnparseableReply); } trc::event!( Delivery(DeliveryEvent::RawInput), SpanId = self.session_id, Contents = trc::Value::from_maybe_string(&buf[..br]), Size = br, ); let mut iter = if buf_concat.is_empty() { buf[..br].iter() } else if br + buf_concat.len() < MAX_RESPONSE_LENGTH { buf_concat.extend_from_slice(&buf[..br]); buf_concat.iter() } else { return Err(mail_send::Error::UnparseableReply); }; match EhloResponse::parse(&mut iter) { Ok(reply) => return Ok(reply), Err(err) => match err { smtp_proto::Error::NeedsMoreData { .. } => { if buf_concat.is_empty() { buf_concat = buf[..br].to_vec(); } } smtp_proto::Error::InvalidResponse { code } => { match ResponseReceiver::from_code(code).parse(&mut iter) { Ok(response) => { return Err(mail_send::Error::UnexpectedReply(response)); } Err(smtp_proto::Error::NeedsMoreData { .. }) => { if buf_concat.is_empty() { buf_concat = buf[..br].to_vec(); } } Err(_) => return Err(mail_send::Error::UnparseableReply), } } _ => { return Err(mail_send::Error::UnparseableReply); } }, } } } pub async fn read(&mut self) -> mail_send::Result> { let mut buf = vec![0u8; 8192]; let mut parser = ResponseReceiver::default(); loop { let br = self.stream.read(&mut buf).await?; if br > 0 { trc::event!( Delivery(DeliveryEvent::RawInput), SpanId = self.session_id, Contents = trc::Value::from_maybe_string(&buf[..br]), Size = br ); match parser.parse(&mut buf[..br].iter()) { Ok(reply) => return Ok(reply), Err(err) => match err { smtp_proto::Error::NeedsMoreData { .. } => (), _ => { return Err(mail_send::Error::UnparseableReply); } }, } } else { return Err(mail_send::Error::UnparseableReply); } } } pub async fn read_many(&mut self, num: usize) -> mail_send::Result>>> { let mut buf = vec![0u8; 1024]; let mut response = Vec::with_capacity(num); let mut parser = ResponseReceiver::default(); 'outer: loop { let br = self.stream.read(&mut buf).await?; if br > 0 { let mut iter = buf[..br].iter(); trc::event!( Delivery(DeliveryEvent::RawInput), SpanId = self.session_id, Contents = trc::Value::from_maybe_string(&buf[..br]), Size = br ); loop { match parser.parse(&mut iter) { Ok(reply) => { response.push(reply.into_box()); if response.len() != num { parser.reset(); } else { break 'outer; } } Err(err) => match err { smtp_proto::Error::NeedsMoreData { .. } => break, _ => { return Err(mail_send::Error::UnparseableReply); } }, } } } else { return Err(mail_send::Error::UnparseableReply); } } Ok(response) } /// Sends a command to the SMTP server and waits for a reply. pub async fn cmd(&mut self, cmd: impl AsRef<[u8]>) -> mail_send::Result> { tokio::time::timeout(self.timeout, async { let cmd = cmd.as_ref(); trc::event!( Delivery(DeliveryEvent::RawOutput), SpanId = self.session_id, Contents = trc::Value::from_maybe_string(cmd), Size = cmd.len() ); self.stream.write_all(cmd).await?; self.stream.flush().await?; self.read().await }) .await .map_err(|_| mail_send::Error::Timeout)? } pub async fn write_message(&mut self, message: &[u8]) -> tokio::io::Result<()> { // Transparency procedure let mut is_cr_or_lf = false; // As per RFC 5322bis, section 2.3: // CR and LF MUST only occur together as CRLF; they MUST NOT appear // independently in the body. // For this reason, we apply the transparency procedure when there is // a CR or LF followed by a dot. trc::event!( Delivery(DeliveryEvent::RawOutput), SpanId = self.session_id, Contents = "[message]", Size = message.len() + 5 ); let mut last_pos = 0; for (pos, byte) in message.iter().enumerate() { if *byte == b'.' && is_cr_or_lf { if let Some(bytes) = message.get(last_pos..pos) { self.stream.write_all(bytes).await?; self.stream.write_all(b".").await?; last_pos = pos; } is_cr_or_lf = false; } else { is_cr_or_lf = *byte == b'\n' || *byte == b'\r'; } } if let Some(bytes) = message.get(last_pos..) { self.stream.write_all(bytes).await?; } self.stream.write_all("\r\n.\r\n".as_bytes()).await?; self.stream.flush().await } } impl SmtpClient { /// Upgrade the connection to TLS. pub async fn start_tls( mut self, tls_connector: &TlsConnector, hostname: &str, ) -> mail_send::Result>> { // Send STARTTLS command self.cmd(b"STARTTLS\r\n") .await? .assert_positive_completion()?; self.into_tls(tls_connector, hostname).await } pub async fn into_tls( self, tls_connector: &TlsConnector, hostname: &str, ) -> mail_send::Result>> { tokio::time::timeout(self.timeout, async { Ok(SmtpClient { stream: tls_connector .connect( ServerName::try_from(hostname) .map_err(|_| mail_send::Error::InvalidTLSName)? .to_owned(), self.stream, ) .await .map_err(|err| { let kind = err.kind(); if let Some(inner) = err.into_inner() { match inner.downcast::() { Ok(error) => mail_send::Error::Tls(error), Err(error) => { mail_send::Error::Io(std::io::Error::new(kind, error)) } } } else { mail_send::Error::Io(std::io::Error::new(kind, "Unspecified")) } })?, timeout: self.timeout, session_id: self.session_id, }) }) .await .map_err(|_| mail_send::Error::Timeout)? } } impl SmtpClient { /// Connects to a remote host address pub async fn connect( remote_addr: SocketAddr, timeout: Duration, session_id: u64, ) -> mail_send::Result { tokio::time::timeout(timeout, async { Ok(SmtpClient { stream: TcpStream::connect(remote_addr).await?, timeout, session_id, }) }) .await .map_err(|_| mail_send::Error::Timeout)? } /// Connects to a remote host address using the provided local IP pub async fn connect_using( local_ip: IpAddr, remote_addr: SocketAddr, timeout: Duration, session_id: u64, ) -> mail_send::Result { tokio::time::timeout(timeout, async { let socket = if local_ip.is_ipv4() { TcpSocket::new_v4()? } else { TcpSocket::new_v6()? }; socket.bind(SocketAddr::new(local_ip, 0))?; Ok(SmtpClient { stream: socket.connect(remote_addr).await?, timeout, session_id, }) }) .await .map_err(|_| mail_send::Error::Timeout)? } pub async fn try_start_tls( mut self, tls_connector: &TlsConnector, hostname: &str, capabilities: &EhloResponse, ) -> StartTlsResult { if capabilities.has_capability(EXT_START_TLS) { match self.cmd("STARTTLS\r\n").await { Ok(response) => { if response.code() == 220 { match self.into_tls(tls_connector, hostname).await { Ok(smtp_client) => StartTlsResult::Success { smtp_client }, Err(error) => StartTlsResult::Error { error }, } } else { StartTlsResult::Unavailable { response: response.into_box().into(), smtp_client: self, } } } Err(error) => StartTlsResult::Error { error }, } } else { StartTlsResult::Unavailable { smtp_client: self, response: None, } } } } impl SmtpClient> { pub fn tls_connection(&self) -> &ClientConnection { self.stream.get_ref().1 } } #[allow(clippy::large_enum_variant)] pub enum StartTlsResult { Success { smtp_client: SmtpClient>, }, Error { error: mail_send::Error, }, Unavailable { response: Option>>, smtp_client: SmtpClient, }, } pub(crate) trait BoxResponse { fn into_box(self) -> Response>; } impl BoxResponse for Response { fn into_box(self) -> Response> { Response { code: self.code, esc: self.esc, message: self.message.into_boxed_str(), } } } pub(crate) fn from_mail_send_error(error: &mail_send::Error) -> trc::Error { let event = trc::EventType::Smtp(trc::SmtpEvent::Error).into_err(); match error { mail_send::Error::Io(err) => event.details("I/O Error").reason(err), mail_send::Error::Tls(err) => event.details("TLS Error").reason(err), mail_send::Error::Base64(err) => event.details("Base64 Error").reason(err), mail_send::Error::Auth(err) => event.details("SMTP Authentication Error").reason(err), mail_send::Error::UnparseableReply => event.details("Unparseable SMTP Reply"), mail_send::Error::UnexpectedReply(reply) => event .details("Unexpected SMTP Response") .ctx(trc::Key::Code, reply.code) .ctx(trc::Key::Reason, reply.message.clone()), mail_send::Error::AuthenticationFailed(reply) => event .details("SMTP Authentication Failed") .ctx(trc::Key::Code, reply.code) .ctx(trc::Key::Reason, reply.message.clone()), mail_send::Error::InvalidTLSName => event.details("Invalid TLS Name"), mail_send::Error::MissingCredentials => event.details("Missing Authentication Credentials"), mail_send::Error::MissingMailFrom => event.details("Missing Message Sender"), mail_send::Error::MissingRcptTo => event.details("Missing Message Recipients"), mail_send::Error::UnsupportedAuthMechanism => { event.details("Unsupported Authentication Mechanism") } mail_send::Error::Timeout => event.details("Connection Timeout"), mail_send::Error::MissingStartTls => event.details("STARTTLS not available"), } } pub(crate) fn from_error_status(err: &Status>, ErrorDetails>) -> trc::Error { match err { Status::Scheduled | Status::Completed(_) => { trc::EventType::Smtp(trc::SmtpEvent::Error).into_err() } Status::TemporaryFailure(err) | Status::PermanentFailure(err) => { from_error_details(&err.details) } } } pub(crate) fn from_error_details(err: &Error) -> trc::Error { let event = trc::EventType::Smtp(trc::SmtpEvent::Error).into_err(); match err { Error::DnsError(err) => event.details("DNS Error").reason(err), Error::UnexpectedResponse(reply) => event .details("Unexpected SMTP Response") .ctx(trc::Key::Code, reply.response.code) .ctx(trc::Key::Details, reply.command.clone()) .ctx(trc::Key::Reason, reply.response.message.clone()), Error::ConnectionError(err) => event .details("Connection Error") .ctx(trc::Key::Reason, err.clone()), Error::TlsError(err) => event .details("TLS Error") .ctx(trc::Key::Reason, err.clone()), Error::DaneError(err) => event .details("DANE Error") .ctx(trc::Key::Reason, err.clone()), Error::MtaStsError(err) => event.details("MTA-STS Error").reason(err), Error::RateLimited => event.details("Rate Limited"), Error::ConcurrencyLimited => event.details("Concurrency Limited"), Error::Io(err) => event.details("I/O Error").reason(err), } } ================================================ FILE: crates/smtp/src/outbound/dane/dnssec.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{ Server, config::smtp::resolver::{Tlsa, TlsaEntry}, }; use mail_auth::{ common::resolver::IntoFqdn, hickory_resolver::{ Name, proto::rr::rdata::tlsa::{CertUsage, Matching, Selector}, }, }; use std::{future::Future, sync::Arc}; pub trait TlsaLookup: Sync + Send { fn tlsa_lookup<'x>( &self, key: impl IntoFqdn<'x> + Sync + Send, ) -> impl Future>>> + Send; } impl TlsaLookup for Server { async fn tlsa_lookup<'x>( &self, key: impl IntoFqdn<'x> + Sync + Send, ) -> mail_auth::Result>> { let key = key.into_fqdn(); if let Some(value) = self.inner.cache.dns_tlsa.get(key.as_ref()) { return Ok(Some(value)); } #[cfg(any(test, feature = "test_mode"))] if true { return mail_auth::common::resolver::mock_resolve(key.as_ref()); } let mut entries = Vec::new(); let tlsa_lookup = self .core .smtp .resolvers .dnssec .resolver .tlsa_lookup(Name::from_str_relaxed(key.as_ref())?) .await?; let mut has_end_entities = false; let mut has_intermediates = false; let mut found_insecure = false; for record in tlsa_lookup.as_lookup().record_iter() { if let Some(tlsa) = record.data().as_tlsa() { if record.proof().is_secure() { let is_end_entity = match tlsa.cert_usage() { CertUsage::DaneEe => true, CertUsage::DaneTa => false, _ => continue, }; if is_end_entity { has_end_entities = true; } else { has_intermediates = true; } entries.push(TlsaEntry { is_end_entity, is_sha256: match tlsa.matching() { Matching::Sha256 => true, Matching::Sha512 => false, _ => continue, }, is_spki: match tlsa.selector() { Selector::Spki => true, Selector::Full => false, _ => continue, }, data: tlsa.cert_data().to_vec(), }); } else { found_insecure = true; } } } if !entries.is_empty() || !found_insecure { let tlsa = Arc::new(Tlsa { entries, has_end_entities, has_intermediates, }); self.inner.cache.dns_tlsa.insert_with_expiry( key.into_owned(), tlsa.clone(), tlsa_lookup.valid_until(), ); Ok(Some(tlsa)) } else { Ok(None) } } } ================================================ FILE: crates/smtp/src/outbound/dane/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod dnssec; pub mod verify; ================================================ FILE: crates/smtp/src/outbound/dane/verify.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::config::smtp::resolver::Tlsa; use rustls_pki_types::CertificateDer; use sha1::Digest; use sha2::{Sha256, Sha512}; use trc::DaneEvent; use x509_parser::prelude::{FromDer, X509Certificate}; use crate::queue::{Error, ErrorDetails, HostResponse, Status}; pub trait TlsaVerify { fn verify( &self, session_id: u64, hostname: &str, certificates: Option<&[CertificateDer<'_>]>, ) -> Result<(), Status>, ErrorDetails>>; } impl TlsaVerify for Tlsa { fn verify( &self, session_id: u64, hostname: &str, certificates: Option<&[CertificateDer<'_>]>, ) -> Result<(), Status>, ErrorDetails>> { let certificates = if let Some(certificates) = certificates { certificates } else { trc::event!( Dane(DaneEvent::NoCertificatesFound), SpanId = session_id, Hostname = hostname.to_string(), ); return Err(Status::TemporaryFailure(ErrorDetails { entity: hostname.into(), details: Error::DaneError("No certificates were provided by host".into()), })); }; let mut matched_end_entity = false; let mut matched_intermediate = false; 'outer: for (pos, der_certificate) in certificates.iter().enumerate() { // Parse certificate let certificate = match X509Certificate::from_der(der_certificate.as_ref()) { Ok((_, certificate)) => certificate, Err(err) => { trc::event!( Dane(DaneEvent::CertificateParseError), SpanId = session_id, Hostname = hostname.to_string(), Reason = err.to_string(), ); return Err(Status::TemporaryFailure(ErrorDetails { entity: hostname.into(), details: Error::DaneError("Failed to parse X.509 certificate".into()), })); } }; // Match against TLSA records let is_end_entity = pos == 0; let mut sha256 = [None, None]; let mut sha512 = [None, None]; for record in self.entries.iter() { if record.is_end_entity == is_end_entity { let hash: &[u8] = if record.is_sha256 { &sha256[usize::from(record.is_spki)].get_or_insert_with(|| { let mut hasher = Sha256::new(); hasher.update(if record.is_spki { certificate.public_key().raw } else { der_certificate.as_ref() }); hasher.finalize() })[..] } else { &sha512[usize::from(record.is_spki)].get_or_insert_with(|| { let mut hasher = Sha512::new(); hasher.update(if record.is_spki { certificate.public_key().raw } else { der_certificate.as_ref() }); hasher.finalize() })[..] }; if hash == record.data { trc::event!( Dane(DaneEvent::TlsaRecordMatch), SpanId = session_id, Hostname = hostname.to_string(), Type = if is_end_entity { "end-entity" } else { "intermediate" }, Details = format!("{:x?}", hash), ); if is_end_entity { matched_end_entity = true; if !self.has_intermediates { break 'outer; } } else { matched_intermediate = true; break 'outer; } } } } } // DANE is valid if: // - EE matched even if no TA matched // - Both EE and TA matched // - EE is not present and TA matched if (self.has_end_entities && matched_end_entity) || ((self.has_end_entities == matched_end_entity) && (self.has_intermediates == matched_intermediate)) { trc::event!( Dane(DaneEvent::AuthenticationSuccess), SpanId = session_id, Hostname = hostname.to_string(), ); Ok(()) } else { trc::event!( Dane(DaneEvent::AuthenticationFailure), SpanId = session_id, Hostname = hostname.to_string(), ); Err(Status::PermanentFailure(ErrorDetails { entity: hostname.into(), details: Error::DaneError("No matching certificates found in TLSA records".into()), })) } } } ================================================ FILE: crates/smtp/src/outbound/delivery.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{NextHop, lookup::ToNextHop, mta_sts, session::SessionParams}; use crate::outbound::DeliveryResult; use crate::outbound::client::{ SmtpClient, from_error_details, from_error_status, from_mail_send_error, }; use crate::outbound::dane::dnssec::TlsaLookup; use crate::outbound::lookup::{DnsLookup, SourceIp}; use crate::outbound::mta_sts::lookup::MtaStsLookup; use crate::outbound::mta_sts::verify::VerifyPolicy; use crate::outbound::{client::StartTlsResult, dane::verify::TlsaVerify}; use crate::queue::dsn::SendDsn; use crate::queue::spool::SmtpSpool; use crate::queue::throttle::IsAllowed; use crate::queue::{ Error, FROM_REPORT, HostResponse, MessageWrapper, QueueEnvelope, QueuedMessage, Status, }; use crate::reporting::SmtpReporting; use crate::{queue::ErrorDetails, reporting::tls::TlsRptOptions}; use ahash::AHashMap; use common::Server; use common::config::smtp::queue::RoutingStrategy; use common::config::{server::ServerProtocol, smtp::report::AggregateFrequency}; use common::ipc::{PolicyType, QueueEvent, QueueEventStatus, TlsEvent}; use compact_str::ToCompactString; use mail_auth::{ mta_sts::TlsRpt, report::tlsrpt::{FailureDetails, ResultType}, }; use smtp_proto::MAIL_REQUIRETLS; use std::sync::Arc; use std::{ net::{IpAddr, Ipv4Addr, SocketAddr}, time::Instant, }; use store::write::{BatchBuilder, QueueClass, ValueClass, now}; use trc::{DaneEvent, DeliveryEvent, MtaStsEvent, ServerEvent, TlsRptEvent}; impl QueuedMessage { pub fn try_deliver(self, server: Server) { #![allow(clippy::large_futures)] tokio::spawn(async move { // Lock queue event let queue_id = self.queue_id; let status = if server.try_lock_event(queue_id, self.queue_name).await { if let Some(mut message) = server.read_message(queue_id, self.queue_name).await { // Generate span id message.span_id = server.inner.data.span_id_gen.generate(); let span_id = message.span_id; trc::event!( Delivery(DeliveryEvent::AttemptStart), SpanId = message.span_id, QueueId = message.queue_id, QueueName = message.queue_name.to_string(), From = if !message.message.return_path.is_empty() { trc::Value::String(message.message.return_path.as_ref().into()) } else { trc::Value::String("<>".into()) }, To = message .message .recipients .iter() .filter_map(|r| { if matches!( r.status, Status::Scheduled | Status::TemporaryFailure(_) ) && r.queue == message.queue_name { Some(trc::Value::String(r.address().into())) } else { None } }) .collect::>(), Size = message.message.size, Total = message.message.recipients.len(), ); // Attempt delivery let start_time = Instant::now(); let queue_event = self.deliver_task(server.clone(), message).await; trc::event!( Delivery(DeliveryEvent::AttemptEnd), SpanId = span_id, Elapsed = start_time.elapsed(), ); // Unlock event server.unlock_event(queue_id, self.queue_name).await; queue_event } else { // Message no longer exists, delete queue event. let mut batch = BatchBuilder::new(); batch.clear(ValueClass::Queue(QueueClass::MessageEvent( store::write::QueueEvent { due: self.due, queue_id: self.queue_id, queue_name: self.queue_name.into_inner(), }, ))); if let Err(err) = server.store().write(batch.build_all()).await { trc::error!( err.details("Failed to delete queue event.") .caused_by(trc::location!()) ); } // Unlock event server.unlock_event(queue_id, self.queue_name).await; QueueEventStatus::Completed } } else { QueueEventStatus::Locked }; // Notify queue manager if server .inner .ipc .queue_tx .send(QueueEvent::WorkerDone { queue_id, queue_name: self.queue_name, status, }) .await .is_err() { trc::event!( Server(ServerEvent::ThreadError), Reason = "Channel closed.", CausedBy = trc::location!(), ); } }); } async fn deliver_task(self, server: Server, mut message: MessageWrapper) -> QueueEventStatus { // Check that the message still has recipients to be delivered let has_pending_delivery = message.has_pending_delivery(); let span_id = message.span_id; // Send any due Delivery Status Notifications server.send_dsn(&mut message).await; match has_pending_delivery { PendingDelivery::Yes(true) if message .message .next_delivery_event(self.queue_name.into()) .is_some_and(|due| due <= now()) => {} PendingDelivery::No => { trc::event!( Delivery(DeliveryEvent::Completed), SpanId = span_id, Elapsed = trc::Value::Duration((now() - message.message.created) * 1000) ); // All message recipients expired, do not re-queue. (DSN has been already sent) message.remove(&server, self.due.into()).await; return QueueEventStatus::Completed; } _ => { // Re-queue the message if its not yet due for delivery message.save_changes(&server, self.due.into()).await; return QueueEventStatus::Deferred; } } // Throttle sender for throttle in &server.core.smtp.queue.outbound_limiters.sender { if let Err(retry_at) = server .is_allowed(throttle, &message.message, message.span_id) .await { trc::event!( Delivery(DeliveryEvent::RateLimitExceeded), Id = throttle.id.clone(), SpanId = span_id, NextRetry = trc::Value::Timestamp(retry_at) ); let now = now(); for rcpt in message.message.recipients.iter_mut() { if matches!( &rcpt.status, Status::Scheduled | Status::TemporaryFailure(_) ) && rcpt.retry.due <= now && rcpt.queue == message.queue_name { rcpt.retry.due = retry_at; rcpt.status = Status::TemporaryFailure(ErrorDetails { entity: "localhost".into(), details: Error::RateLimited, }); } } message.save_changes(&server, self.due.into()).await; return QueueEventStatus::Deferred; } } // Group recipients by route let queue_config = &server.core.smtp.queue; let now_ = now(); let mut routes: AHashMap<(&str, &RoutingStrategy), Vec> = AHashMap::new(); for (rcpt_idx, rcpt) in message.message.recipients.iter().enumerate() { if matches!( &rcpt.status, Status::Scheduled | Status::TemporaryFailure(_) ) && rcpt.retry.due <= now_ && rcpt.queue == message.queue_name { let envelope = QueueEnvelope::new(&message.message, rcpt); let route = server.get_route_or_default( &server .eval_if::(&queue_config.route, &envelope, message.span_id) .await .unwrap_or_else(|| "default".to_string()), message.span_id, ); routes .entry((rcpt.domain_part(), route)) .or_default() .push(rcpt_idx); } } let no_ip = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)); let mut delivery_results: Vec = Vec::new(); 'next_route: for ((domain, route), rcpt_idxs) in routes { trc::event!( Delivery(DeliveryEvent::DomainDeliveryStart), SpanId = message.span_id, Domain = domain.to_string(), ); // Build envelope let mut envelope = QueueEnvelope::new(&message.message, &message.message.recipients[rcpt_idxs[0]]); // Throttle recipient domain for throttle in &queue_config.outbound_limiters.rcpt { if let Err(retry_at) = server .is_allowed(throttle, &envelope, message.span_id) .await { trc::event!( Delivery(DeliveryEvent::RateLimitExceeded), Id = throttle.id.clone(), SpanId = span_id, Domain = domain.to_string(), ); delivery_results.push(DeliveryResult::rate_limited(rcpt_idxs, retry_at)); continue 'next_route; } } // Obtain next hop let (mut remote_hosts, mx_config, is_smtp) = match route { RoutingStrategy::Local => { // Deliver message locally message .deliver_local(&rcpt_idxs, &mut delivery_results, &server) .await; continue 'next_route; } RoutingStrategy::Mx(mx_config) => (Vec::with_capacity(0), Some(mx_config), true), RoutingStrategy::Relay(relay_config) => ( vec![NextHop::Relay(relay_config)], None, relay_config.protocol == ServerProtocol::Smtp, ), }; // Prepare TLS strategy let mut tls_strategy = server.get_tls_or_default( &server .eval_if::(&queue_config.tls, &envelope, message.span_id) .await .unwrap_or_else(|| "default".to_string()), message.span_id, ); // Obtain TLS reporting let tls_report = if is_smtp && mx_config.is_some() && (message.message.flags & FROM_REPORT == 0) { match server .eval_if( &server.core.smtp.report.tls.send, &envelope, message.span_id, ) .await .unwrap_or(AggregateFrequency::Never) { interval @ (AggregateFrequency::Hourly | AggregateFrequency::Daily | AggregateFrequency::Weekly) => { let time = Instant::now(); match server .core .smtp .resolvers .dns .txt_lookup::( format!("_smtp._tls.{domain}."), Some(&server.inner.cache.dns_txt), ) .await { Ok(record) => { trc::event!( TlsRpt(TlsRptEvent::RecordFetch), SpanId = message.span_id, Domain = domain.to_string(), Details = record .rua .iter() .map(|uri| trc::Value::from(match uri { mail_auth::mta_sts::ReportUri::Mail(uri) | mail_auth::mta_sts::ReportUri::Http(uri) => uri.to_string(), })) .collect::>(), Elapsed = time.elapsed(), ); TlsRptOptions { record, interval }.into() } Err(mail_auth::Error::DnsRecordNotFound(_)) => { trc::event!( TlsRpt(TlsRptEvent::RecordNotFound), SpanId = message.span_id, Domain = domain.to_string(), Elapsed = time.elapsed(), ); None } Err(err) => { trc::event!( TlsRpt(TlsRptEvent::RecordFetchError), SpanId = message.span_id, Domain = domain.to_string(), CausedBy = trc::Error::from(err), Elapsed = time.elapsed(), ); None } } } _ => None, } } else { None }; // Obtain MTA-STS policy for domain let mta_sts_policy = if mx_config.is_some() && tls_strategy.try_mta_sts() && is_smtp { let time = Instant::now(); match server .lookup_mta_sts_policy(domain, tls_strategy.timeout_mta_sts) .await { Ok(mta_sts_policy) => { trc::event!( MtaSts(MtaStsEvent::PolicyFetch), SpanId = message.span_id, Domain = domain.to_string(), Strict = mta_sts_policy.enforce(), Details = mta_sts_policy .mx .iter() .map(|mx| trc::Value::String(mx.to_compact_string())) .collect::>(), Elapsed = time.elapsed(), ); mta_sts_policy.into() } Err(err) => { // Report MTA-STS error let strict = tls_strategy.is_mta_sts_required(); if let Some(tls_report) = &tls_report { match &err { mta_sts::Error::Dns(mail_auth::Error::DnsRecordNotFound(_)) => { if strict { server.schedule_report(TlsEvent { policy: PolicyType::Sts(None), domain: domain.to_string(), failure: FailureDetails::new(ResultType::Other) .with_failure_reason_code( "MTA-STS is required and no policy was found.", ) .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, }) .await; } } mta_sts::Error::Dns(mail_auth::Error::DnsError(_)) => (), _ => { server .schedule_report(TlsEvent { policy: PolicyType::Sts(None), domain: domain.to_string(), failure: FailureDetails::new(&err) .with_failure_reason_code(err.to_string()) .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, }) .await; } } } match &err { mta_sts::Error::Dns(mail_auth::Error::DnsRecordNotFound(_)) => { trc::event!( MtaSts(MtaStsEvent::PolicyNotFound), SpanId = message.span_id, Domain = domain.to_string(), Strict = strict, Elapsed = time.elapsed(), ); } mta_sts::Error::Dns(err) => { trc::event!( MtaSts(MtaStsEvent::PolicyFetchError), SpanId = message.span_id, Domain = domain.to_string(), CausedBy = trc::Error::from(err.clone()), Strict = strict, Elapsed = time.elapsed(), ); } mta_sts::Error::Http(err) => { trc::event!( MtaSts(MtaStsEvent::PolicyFetchError), SpanId = message.span_id, Domain = domain.to_string(), Reason = err.to_string(), Strict = strict, Elapsed = time.elapsed(), ); } mta_sts::Error::InvalidPolicy(reason) => { trc::event!( MtaSts(MtaStsEvent::InvalidPolicy), SpanId = message.span_id, Domain = domain.to_string(), Reason = reason.clone(), Strict = strict, Elapsed = time.elapsed(), ); } } if strict { delivery_results.push(DeliveryResult::domain( Status::from_mta_sts_error(domain, err), rcpt_idxs, )); continue 'next_route; } None } } } else { None }; // Obtain remote hosts list let mx_list; if let Some(mx_config) = mx_config { // Lookup MX let time = Instant::now(); mx_list = match server .core .smtp .resolvers .dns .mx_lookup(domain, Some(&server.inner.cache.dns_mx)) .await { Ok(mx) => mx, Err(mail_auth::Error::DnsRecordNotFound(_)) => { trc::event!( Delivery(DeliveryEvent::MxLookupFailed), SpanId = message.span_id, Domain = domain.to_string(), Details = "No MX records were found, attempting implicit MX.", Elapsed = time.elapsed(), ); Arc::new(vec![]) } Err(err) => { trc::event!( Delivery(DeliveryEvent::MxLookupFailed), SpanId = message.span_id, Domain = domain.to_string(), CausedBy = trc::Error::from(err.clone()), Elapsed = time.elapsed(), ); delivery_results.push(DeliveryResult::domain( Status::from_mail_auth_error(domain, err), rcpt_idxs, )); continue 'next_route; } }; if let Some(remote_hosts_) = mx_list.to_remote_hosts(domain, mx_config) { trc::event!( Delivery(DeliveryEvent::MxLookup), SpanId = message.span_id, Domain = domain.to_string(), Details = remote_hosts_ .iter() .map(|h| trc::Value::String(h.hostname().into())) .collect::>(), Elapsed = time.elapsed(), ); remote_hosts = remote_hosts_; } else { trc::event!( Delivery(DeliveryEvent::NullMx), SpanId = message.span_id, Domain = domain.to_string(), Elapsed = time.elapsed(), ); delivery_results.push(DeliveryResult::domain( Status::PermanentFailure(ErrorDetails { entity: domain.into(), details: Error::DnsError( "Domain does not accept messages (null MX)".into(), ), }), rcpt_idxs, )); continue 'next_route; } } // Try delivering message let mut last_status: Status>, ErrorDetails> = Status::Scheduled; 'next_host: for remote_host in &remote_hosts { // Validate MTA-STS envelope.mx = remote_host.hostname(); if let Some(mta_sts_policy) = &mta_sts_policy { let strict = mta_sts_policy.enforce(); if !mta_sts_policy.verify(envelope.mx) { // Report MTA-STS failed verification if let Some(tls_report) = &tls_report { server .schedule_report(TlsEvent { policy: mta_sts_policy.into(), domain: domain.to_string(), failure: FailureDetails::new(ResultType::ValidationFailure) .with_receiving_mx_hostname(envelope.mx) .with_failure_reason_code("MX not authorized by policy.") .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, }) .await; } trc::event!( MtaSts(MtaStsEvent::NotAuthorized), SpanId = message.span_id, Domain = domain.to_string(), Hostname = envelope.mx.to_string(), Details = mta_sts_policy .mx .iter() .map(|mx| trc::Value::String(mx.to_compact_string())) .collect::>(), Strict = strict, ); if strict { last_status = Status::PermanentFailure(ErrorDetails { entity: envelope.mx.into(), details: Error::MtaStsError( format!("MX {:?} not authorized by policy.", envelope.mx) .into_boxed_str(), ), }); continue 'next_host; } } else { trc::event!( MtaSts(MtaStsEvent::Authorized), SpanId = message.span_id, Domain = domain.to_string(), Hostname = envelope.mx.to_string(), Details = mta_sts_policy .mx .iter() .map(|mx| trc::Value::String(mx.to_compact_string())) .collect::>(), Strict = strict, ); } } // Obtain source and remote IPs let time = Instant::now(); let resolve_result = match server.resolve_host(remote_host, &envelope).await { Ok(result) => { trc::event!( Delivery(DeliveryEvent::IpLookup), SpanId = message.span_id, Domain = domain.to_string(), Hostname = envelope.mx.to_string(), Details = result .remote_ips .iter() .map(|ip| trc::Value::from(*ip)) .collect::>(), Limit = remote_host.max_multi_homed(), Elapsed = time.elapsed(), ); result } Err(status) => { trc::event!( Delivery(DeliveryEvent::IpLookupFailed), SpanId = message.span_id, Domain = domain.to_string(), Hostname = envelope.mx.to_string(), Details = status.to_string(), Elapsed = time.elapsed(), ); last_status = status; continue 'next_host; } }; // Update TLS strategy tls_strategy = server.get_tls_or_default( &server .eval_if::(&queue_config.tls, &envelope, message.span_id) .await .unwrap_or_else(|| "default".to_string()), message.span_id, ); // Lookup DANE policy let dane_policy = if tls_strategy.try_dane() && is_smtp { let time = Instant::now(); let strict = tls_strategy.is_dane_required(); match server .tlsa_lookup(format!("_25._tcp.{}.", envelope.mx)) .await { Ok(Some(tlsa)) => { if tlsa.has_end_entities { trc::event!( Dane(DaneEvent::TlsaRecordFetch), SpanId = message.span_id, Domain = domain.to_string(), Hostname = envelope.mx.to_string(), Details = format!("{tlsa:?}"), Strict = strict, Elapsed = time.elapsed(), ); tlsa.into() } else { trc::event!( Dane(DaneEvent::TlsaRecordInvalid), SpanId = message.span_id, Domain = domain.to_string(), Hostname = envelope.mx.to_string(), Details = format!("{tlsa:?}"), Strict = strict, Elapsed = time.elapsed(), ); // Report invalid TLSA record if let Some(tls_report) = &tls_report { server .schedule_report(TlsEvent { policy: tlsa.into(), domain: domain.to_string(), failure: FailureDetails::new(ResultType::TlsaInvalid) .with_receiving_mx_hostname(envelope.mx) .with_failure_reason_code("Invalid TLSA record.") .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, }) .await; } if strict { last_status = Status::PermanentFailure(ErrorDetails { entity: envelope.mx.into(), details: Error::DaneError( "No valid TLSA records were found".into(), ), }); continue 'next_host; } None } } Ok(None) => { trc::event!( Dane(DaneEvent::TlsaRecordNotDnssecSigned), SpanId = message.span_id, Domain = domain.to_string(), Hostname = envelope.mx.to_string(), Strict = strict, Elapsed = time.elapsed(), ); if strict { // Report DANE required if let Some(tls_report) = &tls_report { server .schedule_report(TlsEvent { policy: PolicyType::Tlsa(None), domain: domain.to_string(), failure: FailureDetails::new(ResultType::DaneRequired) .with_receiving_mx_hostname(envelope.mx) .with_failure_reason_code( "No TLSA DNSSEC records found.", ) .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, }) .await; } last_status = Status::PermanentFailure(ErrorDetails { entity: envelope.mx.into(), details: Error::DaneError( "No TLSA DNSSEC records found".into(), ), }); continue 'next_host; } None } Err(err) => { let not_found = matches!(&err, mail_auth::Error::DnsRecordNotFound(_)); if not_found { trc::event!( Dane(DaneEvent::TlsaRecordNotFound), SpanId = message.span_id, Domain = domain.to_string(), Hostname = envelope.mx.to_string(), Strict = strict, Elapsed = time.elapsed(), ); } else { trc::event!( Dane(DaneEvent::TlsaRecordFetchError), SpanId = message.span_id, Domain = domain.to_string(), Hostname = envelope.mx.to_string(), CausedBy = trc::Error::from(err.clone()), Strict = strict, Elapsed = time.elapsed(), ); } if strict { last_status = if not_found { // Report DANE required if let Some(tls_report) = &tls_report { server .schedule_report(TlsEvent { policy: PolicyType::Tlsa(None), domain: domain.to_string(), failure: FailureDetails::new( ResultType::DaneRequired, ) .with_receiving_mx_hostname(envelope.mx) .with_failure_reason_code( "No TLSA records found for MX.", ) .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, }) .await; } Status::PermanentFailure(ErrorDetails { entity: envelope.mx.into(), details: Error::DaneError("No TLSA records found".into()), }) } else { Status::from_mail_auth_error(envelope.mx, err) }; continue 'next_host; } None } } } else { None }; // Try each IP address 'next_ip: for remote_ip in resolve_result.remote_ips { // Throttle remote host envelope.remote_ip = remote_ip; for throttle in &queue_config.outbound_limiters.remote { if let Err(retry_at) = server .is_allowed(throttle, &envelope, message.span_id) .await { trc::event!( Delivery(DeliveryEvent::RateLimitExceeded), SpanId = message.span_id, Id = throttle.id.clone(), RemoteIp = remote_ip, ); delivery_results .push(DeliveryResult::rate_limited(rcpt_idxs, retry_at)); continue 'next_route; } } // Obtain connection parameters let conn_strategy = server.get_connection_or_default( &server .eval_if::( &queue_config.connection, &envelope, message.span_id, ) .await .unwrap_or_else(|| "default".to_string()), message.span_id, ); // Set source IP, if any let ip_host = conn_strategy.source_ip(remote_ip.is_ipv4()); // Connect let time = Instant::now(); let mut smtp_client = match if let Some(ip_host) = ip_host { envelope.local_ip = ip_host.ip; SmtpClient::connect_using( ip_host.ip, SocketAddr::new(remote_ip, remote_host.port()), conn_strategy.timeout_connect, span_id, ) .await } else { envelope.local_ip = no_ip; SmtpClient::connect( SocketAddr::new(remote_ip, remote_host.port()), conn_strategy.timeout_connect, span_id, ) .await } { Ok(smtp_client) => { trc::event!( Delivery(DeliveryEvent::Connect), SpanId = message.span_id, Domain = domain.to_string(), Hostname = envelope.mx.to_string(), LocalIp = envelope.local_ip, RemoteIp = remote_ip, RemotePort = remote_host.port(), Elapsed = time.elapsed(), ); smtp_client } Err(err) => { trc::event!( Delivery(DeliveryEvent::ConnectError), SpanId = message.span_id, Domain = domain.to_string(), Hostname = envelope.mx.to_string(), LocalIp = envelope.local_ip, RemoteIp = remote_ip, RemotePort = remote_host.port(), CausedBy = from_mail_send_error(&err), Elapsed = time.elapsed(), ); last_status = Status::from_smtp_error(envelope.mx, "", err); continue 'next_ip; } }; // Obtain session parameters let local_hostname = ip_host .and_then(|ip| ip.host.as_deref()) .or(conn_strategy.ehlo_hostname.as_deref()) .unwrap_or(server.core.network.server_name.as_str()); let mut params = SessionParams { session_id: message.span_id, server: &server, credentials: remote_host.credentials(), is_smtp: remote_host.is_smtp(), hostname: envelope.mx, local_hostname, conn_strategy, capabilities: None, }; // Prepare TLS connector let is_strict_tls = tls_strategy.is_tls_required() || (message.message.flags & MAIL_REQUIRETLS) != 0 || mta_sts_policy.is_some() || dane_policy.is_some(); // As per RFC7671 Section 5.1, DANE-EE(3) allows name mismatch let tls_connector = if tls_strategy.allow_invalid_certs || remote_host.allow_invalid_certs() || dane_policy.as_ref().is_some_and(|t| t.has_end_entities) { &server.inner.data.smtp_connectors.dummy_verify } else { &server.inner.data.smtp_connectors.pki_verify }; if !remote_host.implicit_tls() { // Read greeting smtp_client.timeout = conn_strategy.timeout_greeting; if let Err(status) = smtp_client.read_greeting(envelope.mx).await { trc::event!( Delivery(DeliveryEvent::GreetingFailed), SpanId = message.span_id, Domain = domain.to_string(), Hostname = envelope.mx.to_string(), Details = status.to_string(), ); last_status = status; continue 'next_host; } // Say EHLO let time = Instant::now(); let capabilities = match smtp_client.say_helo(¶ms).await { Ok(capabilities) => { trc::event!( Delivery(DeliveryEvent::Ehlo), SpanId = message.span_id, Domain = domain.to_string(), Hostname = envelope.mx.to_string(), Details = capabilities.capabilities(), Elapsed = time.elapsed(), ); capabilities } Err(status) => { trc::event!( Delivery(DeliveryEvent::EhloRejected), SpanId = message.span_id, Domain = domain.to_string(), Hostname = envelope.mx.to_string(), Details = status.to_string(), Elapsed = time.elapsed(), ); last_status = status; continue 'next_host; } }; // Try starting TLS if tls_strategy.try_start_tls() { let time = Instant::now(); smtp_client.timeout = tls_strategy.timeout_tls; match smtp_client .try_start_tls(tls_connector, envelope.mx, &capabilities) .await { StartTlsResult::Success { smtp_client } => { trc::event!( Delivery(DeliveryEvent::StartTls), SpanId = message.span_id, Domain = domain.to_string(), Hostname = envelope.mx.to_string(), Version = format!( "{:?}", smtp_client .tls_connection() .protocol_version() .unwrap() ), Details = format!( "{:?}", smtp_client .tls_connection() .negotiated_cipher_suite() .unwrap() ), Elapsed = time.elapsed(), ); // Verify DANE if let Some(dane_policy) = &dane_policy && let Err(status) = dane_policy.verify( message.span_id, envelope.mx, smtp_client.tls_connection().peer_certificates(), ) { // Report DANE verification failure if let Some(tls_report) = &tls_report { server .schedule_report(TlsEvent { policy: dane_policy.into(), domain: domain.to_string(), failure: FailureDetails::new( ResultType::ValidationFailure, ) .with_receiving_mx_hostname(envelope.mx) .with_receiving_ip(remote_ip) .with_failure_reason_code( "No matching certificates found.", ) .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, }) .await; } last_status = status; continue 'next_host; } // Report TLS success if let Some(tls_report) = &tls_report { server .schedule_report(TlsEvent { policy: (&mta_sts_policy, &dane_policy).into(), domain: domain.to_string(), failure: None, tls_record: tls_report.record.clone(), interval: tls_report.interval, }) .await; } // Deliver message over TLS message .deliver( smtp_client, rcpt_idxs, &mut delivery_results, params, ) .await } StartTlsResult::Unavailable { response, smtp_client, } => { // Report unavailable STARTTLS let reason = response.as_ref().map(|r| r.to_string()).unwrap_or_else( || "STARTTLS was not advertised by host".to_string(), ); trc::event!( Delivery(DeliveryEvent::StartTlsUnavailable), SpanId = message.span_id, Domain = domain.to_string(), Hostname = envelope.mx.to_string(), Code = response.as_ref().map(|r| r.code()), Details = response .as_ref() .map(|r| r.message().as_ref()) .unwrap_or("STARTTLS was not advertised by host") .to_string(), Elapsed = time.elapsed(), ); if let Some(tls_report) = &tls_report { server .schedule_report(TlsEvent { policy: (&mta_sts_policy, &dane_policy).into(), domain: domain.to_string(), failure: FailureDetails::new( ResultType::StartTlsNotSupported, ) .with_receiving_mx_hostname(envelope.mx) .with_receiving_ip(remote_ip) .with_failure_reason_code(reason) .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, }) .await; } if is_strict_tls { last_status = Status::from_starttls_error(envelope.mx, response); continue 'next_host; } else { // TLS is not required, proceed in plain-text params.capabilities = Some(capabilities); message .deliver( smtp_client, rcpt_idxs, &mut delivery_results, params, ) .await } } StartTlsResult::Error { error } => { trc::event!( Delivery(DeliveryEvent::StartTlsError), SpanId = message.span_id, Domain = domain.to_string(), Hostname = envelope.mx.to_string(), Reason = from_mail_send_error(&error), Elapsed = time.elapsed(), ); // Report TLS failure if let (Some(tls_report), mail_send::Error::Tls(error)) = (&tls_report, &error) { server .schedule_report(TlsEvent { policy: (&mta_sts_policy, &dane_policy).into(), domain: domain.to_string(), failure: FailureDetails::new( ResultType::CertificateNotTrusted, ) .with_receiving_mx_hostname(envelope.mx) .with_receiving_ip(remote_ip) .with_failure_reason_code(error.to_string()) .into(), tls_record: tls_report.record.clone(), interval: tls_report.interval, }) .await; } last_status = if is_strict_tls { Status::from_tls_error(envelope.mx, error) } else { Status::from_tls_error(envelope.mx, error).into_temporary() }; continue 'next_host; } } } else { // TLS has been disabled trc::event!( Delivery(DeliveryEvent::StartTlsDisabled), SpanId = message.span_id, Domain = domain.to_string(), Hostname = envelope.mx.to_string(), ); message .deliver(smtp_client, rcpt_idxs, &mut delivery_results, params) .await } } else { // Start TLS smtp_client.timeout = tls_strategy.timeout_tls; let mut smtp_client = match smtp_client.into_tls(tls_connector, envelope.mx).await { Ok(smtp_client) => smtp_client, Err(error) => { trc::event!( Delivery(DeliveryEvent::ImplicitTlsError), SpanId = message.span_id, Domain = domain.to_string(), Hostname = envelope.mx.to_string(), Reason = from_mail_send_error(&error), ); last_status = Status::from_tls_error(envelope.mx, error); continue 'next_host; } }; // Read greeting smtp_client.timeout = conn_strategy.timeout_greeting; if let Err(status) = smtp_client.read_greeting(envelope.mx).await { trc::event!( Delivery(DeliveryEvent::GreetingFailed), SpanId = message.span_id, Domain = domain.to_string(), Hostname = envelope.mx.to_string(), Details = from_error_status(&status), ); last_status = status; continue 'next_host; } // Deliver message message .deliver(smtp_client, rcpt_idxs, &mut delivery_results, params) .await } // Continue with the next domain/route continue 'next_route; } } // Update status delivery_results.push(DeliveryResult::domain(last_status, rcpt_idxs)); } // Apply status changes for delivery_result in delivery_results { match delivery_result { DeliveryResult::Domain { status, rcpt_idxs } => { for rcpt_idx in rcpt_idxs { message .set_rcpt_status(status.clone(), rcpt_idx, &server) .await; } } DeliveryResult::Account { status, rcpt_idx } => { message.set_rcpt_status(status, rcpt_idx, &server).await; } DeliveryResult::RateLimited { rcpt_idxs, retry_at, } => { for rcpt_idx in rcpt_idxs { message.set_rcpt_rate_limit(rcpt_idx, retry_at); } } } } // Send Delivery Status Notifications server.send_dsn(&mut message).await; // Notify queue manager if message.message.next_event(None).is_some() { trc::event!( Queue(trc::QueueEvent::Rescheduled), SpanId = span_id, NextRetry = message .message .next_delivery_event(None) .map(trc::Value::Timestamp), NextDsn = message.message.next_dsn(None).map(trc::Value::Timestamp), Expires = message.message.expires(None).map(trc::Value::Timestamp), ); // Save changes to disk message.save_changes(&server, self.due.into()).await; QueueEventStatus::Deferred } else { trc::event!( Delivery(DeliveryEvent::Completed), SpanId = span_id, Elapsed = trc::Value::Duration((now() - message.message.created) * 1000) ); // Delete message from queue message.remove(&server, self.due.into()).await; QueueEventStatus::Completed } } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PendingDelivery { Yes(bool), No, } impl MessageWrapper { /// Marks as failed all domains that reached their expiration time pub fn has_pending_delivery(&mut self) -> PendingDelivery { let now = now(); let mut has_pending_delivery = false; let mut matches_queue = false; for rcpt in self.message.recipients.iter_mut() { match &rcpt.status { Status::TemporaryFailure(err) if rcpt.is_expired(self.message.created, now) => { trc::event!( Delivery(DeliveryEvent::Failed), SpanId = self.span_id, QueueId = self.queue_id, QueueName = self.queue_name.as_str().to_string(), To = rcpt.address().to_string(), Reason = from_error_details(&err.details), Details = trc::Value::Timestamp(now), Expires = rcpt .expiration_time(self.message.created) .map(trc::Value::Timestamp), NextRetry = trc::Value::Timestamp(rcpt.retry.due), NextDsn = trc::Value::Timestamp(rcpt.notify.due), ); rcpt.status = std::mem::replace(&mut rcpt.status, Status::Scheduled).into_permanent(); } Status::Scheduled if rcpt.is_expired(self.message.created, now) => { trc::event!( Delivery(DeliveryEvent::Failed), SpanId = self.span_id, QueueId = self.queue_id, QueueName = self.queue_name.as_str().to_string(), To = rcpt.address().to_string(), Reason = "Message expired without any delivery attempts made.", Details = trc::Value::Timestamp(now), Expires = rcpt .expiration_time(self.message.created) .map(trc::Value::Timestamp), NextRetry = trc::Value::Timestamp(rcpt.retry.due), NextDsn = trc::Value::Timestamp(rcpt.notify.due), ); rcpt.status = Status::PermanentFailure(ErrorDetails { entity: rcpt.domain_part().into(), details: Error::Io( "Message expired without any delivery attempts made.".into(), ), }); } Status::Completed(_) | Status::PermanentFailure(_) => (), _ => { has_pending_delivery = true; matches_queue = matches_queue || rcpt.queue == self.queue_name; } } } if has_pending_delivery { PendingDelivery::Yes(matches_queue) } else { PendingDelivery::No } } pub async fn set_rcpt_status( &mut self, status: Status>, ErrorDetails>, rcpt_idx: usize, server: &Server, ) { let needs_retry = matches!(&status, Status::TemporaryFailure(_) | Status::Scheduled); self.message.recipients[rcpt_idx].status = status; if needs_retry { let envelope = QueueEnvelope::new(&self.message, &self.message.recipients[rcpt_idx]); let queue = server.get_queue_or_default( &server .eval_if::(&server.core.smtp.queue.queue, &envelope, self.span_id) .await .unwrap_or_else(|| "default".to_string()), self.span_id, ); let rcpt = &mut self.message.recipients[rcpt_idx]; rcpt.retry.due = now() + queue.retry[std::cmp::min(rcpt.retry.inner as usize, queue.retry.len() - 1)]; rcpt.retry.inner += 1; rcpt.expires = queue.expiry; rcpt.queue = queue.virtual_queue; } } pub fn set_rcpt_rate_limit(&mut self, rcpt_idx: usize, retry_at: u64) { let rcpt = &mut self.message.recipients[rcpt_idx]; rcpt.retry.due = retry_at; rcpt.status = Status::TemporaryFailure(ErrorDetails { entity: "localhost".into(), details: Error::RateLimited, }); } } ================================================ FILE: crates/smtp/src/outbound/local.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ outbound::DeliveryResult, queue::{ Error, ErrorDetails, FROM_AUTHENTICATED, FROM_UNAUTHENTICATED_DMARC, HostResponse, MessageSource, MessageWrapper, RCPT_SPAM_PAYLOAD, Status, UnexpectedResponse, quota::HasQueueQuota, spool::SmtpSpool, }, reporting::SmtpReporting, }; use common::Server; use email::message::delivery::{IngestMessage, IngestRecipient, LocalDeliveryStatus, MailDelivery}; use smtp_proto::Response; use trc::SieveEvent; impl MessageWrapper { pub(super) async fn deliver_local( &self, rcpt_idxs: &[usize], statuses: &mut Vec, server: &Server, ) { // Prepare recipients list let mut pending_recipients = Vec::new(); let mut recipients = Vec::new(); for &rcpt_idx in rcpt_idxs { let rcpt = &self.message.recipients[rcpt_idx]; let rcpt_addr = rcpt.address(); recipients.push(IngestRecipient { address: rcpt_addr.to_lowercase(), is_spam: rcpt.flags & RCPT_SPAM_PAYLOAD != 0, }); pending_recipients.push((rcpt_idx, rcpt_addr)); } // Deliver message let delivery_result = server .deliver_message(IngestMessage { sender_address: self.message.return_path.to_string(), sender_authenticated: self.message.flags & (FROM_UNAUTHENTICATED_DMARC | FROM_AUTHENTICATED) != 0, recipients, message_blob: self.message.blob_hash.clone(), message_size: self.message.size, session_id: self.span_id, }) .await; // Process delivery results for ((rcpt_idx, rcpt_addr), result) in pending_recipients.into_iter().zip(delivery_result.status) { let status = match result { LocalDeliveryStatus::Success => Status::Completed(HostResponse { hostname: "localhost".into(), response: Response { code: 250, esc: [2, 1, 5], message: "OK".into(), }, }), LocalDeliveryStatus::TemporaryFailure { reason } => { Status::TemporaryFailure(ErrorDetails { entity: "localhost".into(), details: Error::UnexpectedResponse(UnexpectedResponse { command: format!("RCPT TO:<{rcpt_addr}>").into_boxed_str(), response: Response { code: 451, esc: [4, 3, 0], message: reason.into(), }, }), }) } LocalDeliveryStatus::PermanentFailure { code, reason } => { Status::PermanentFailure(ErrorDetails { entity: "localhost".into(), details: Error::UnexpectedResponse(UnexpectedResponse { command: format!("RCPT TO:<{rcpt_addr}>").into_boxed_str(), response: Response { code: 550, esc: code, message: reason.into(), }, }), }) } }; statuses.push(DeliveryResult::account(status, rcpt_idx)); } // Process autogenerated messages for autogenerated in delivery_result.autogenerated { let mut message = server.new_message(autogenerated.sender_address, self.span_id); for rcpt in autogenerated.recipients { message.add_recipient(rcpt, server).await; } // Sign message let signature = server .sign_message( &mut message, &server.core.sieve.sign, &autogenerated.message, ) .await; // Queue Message message.message.size = (autogenerated.message.len() + signature.as_ref().map_or(0, |s| s.len())) as u64; if server.has_quota(&mut message).await { message .queue( signature.as_deref(), &autogenerated.message, self.span_id, server, MessageSource::Autogenerated, ) .await; } else { trc::event!( Sieve(SieveEvent::QuotaExceeded), SpanId = self.span_id, From = message.message.return_path, To = message .message .recipients .into_iter() .map(|r| trc::Value::from(r.address().to_string())) .collect::>(), ); } } } } ================================================ FILE: crates/smtp/src/outbound/lookup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::NextHop; use crate::queue::{Error, ErrorDetails, HostResponse, Status}; use common::{ Server, config::smtp::queue::{ConnectionStrategy, IpAndHost, MxConfig}, expr::{V_MX, functions::ResolveVariable}, }; use mail_auth::{IpLookupStrategy, MX}; use rand::{Rng, seq::SliceRandom}; use std::{future::Future, net::IpAddr, sync::Arc}; pub struct IpLookupResult { pub remote_ips: Vec, } pub trait DnsLookup: Sync + Send { fn ip_lookup( &self, key: &str, strategy: IpLookupStrategy, max_results: usize, ) -> impl Future>> + Send; fn resolve_host( &self, remote_host: &NextHop<'_>, envelope: &impl ResolveVariable, ) -> impl Future>, ErrorDetails>>> + Send; } impl DnsLookup for Server { async fn ip_lookup( &self, key: &str, strategy: IpLookupStrategy, max_results: usize, ) -> mail_auth::Result> { let (has_ipv4, has_ipv6, v4_first) = match strategy { IpLookupStrategy::Ipv4Only => (true, false, false), IpLookupStrategy::Ipv6Only => (false, true, false), IpLookupStrategy::Ipv4thenIpv6 => (true, true, true), IpLookupStrategy::Ipv6thenIpv4 => (true, true, false), }; let ipv4_addrs = if has_ipv4 { match self .core .smtp .resolvers .dns .ipv4_lookup(key, Some(&self.inner.cache.dns_ipv4)) .await { Ok(addrs) => addrs, Err(_) if has_ipv6 => Arc::new(Vec::new()), Err(err) => return Err(err), } } else { Arc::new(Vec::new()) }; if has_ipv6 { let ipv6_addrs = match self .core .smtp .resolvers .dns .ipv6_lookup(key, Some(&self.inner.cache.dns_ipv6)) .await { Ok(addrs) => addrs, Err(_) if !ipv4_addrs.is_empty() => Arc::new(Vec::new()), Err(err) => return Err(err), }; if v4_first { Ok(ipv4_addrs .iter() .copied() .map(IpAddr::from) .chain(ipv6_addrs.iter().copied().map(IpAddr::from)) .take(max_results) .collect()) } else { Ok(ipv6_addrs .iter() .copied() .map(IpAddr::from) .chain(ipv4_addrs.iter().copied().map(IpAddr::from)) .take(max_results) .collect()) } } else { Ok(ipv4_addrs .iter() .take(max_results) .copied() .map(IpAddr::from) .collect()) } } #[allow(unused_mut)] async fn resolve_host( &self, remote_host: &NextHop<'_>, envelope: &impl ResolveVariable, ) -> Result>, ErrorDetails>> { let mut remote_ips = self .ip_lookup( remote_host.fqdn_hostname().as_ref(), remote_host.ip_lookup_strategy(), remote_host.max_multi_homed(), ) .await .map_err(|err| { if let mail_auth::Error::DnsRecordNotFound(_) = &err { if matches!( remote_host, NextHop::MX { is_implicit: true, .. } ) { Status::PermanentFailure(ErrorDetails { entity: remote_host.hostname().into(), details: Error::DnsError("no MX record found.".into()), }) } else { Status::PermanentFailure(ErrorDetails { entity: remote_host.hostname().into(), details: Error::ConnectionError("record not found for MX".into()), }) } } else { Status::TemporaryFailure(ErrorDetails { entity: remote_host.hostname().into(), details: Error::ConnectionError( format!("lookup error: {err}").into_boxed_str(), ), }) } })?; if !remote_ips.is_empty() { #[cfg(not(feature = "test_mode"))] if remote_ips.iter().any(|ip| ip.is_loopback()) { remote_ips.retain(|ip| !ip.is_loopback()); if remote_ips.is_empty() { return Err(Status::PermanentFailure(ErrorDetails { entity: remote_host.hostname().into(), details: Error::ConnectionError("host resolves loopback address".into()), })); } } Ok(IpLookupResult { remote_ips }) } else { Err(Status::TemporaryFailure(ErrorDetails { entity: remote_host.hostname().into(), details: Error::DnsError( format!( "No IP addresses found for {:?}.", envelope.resolve_variable(V_MX).to_string() ) .into_boxed_str(), ), })) } } } pub trait SourceIp { fn source_ip(&self, is_v4: bool) -> Option<&IpAndHost>; } impl SourceIp for ConnectionStrategy { fn source_ip(&self, is_v4: bool) -> Option<&IpAndHost> { let ips = if is_v4 { &self.source_ipv4 } else { &self.source_ipv6 }; match ips.len().cmp(&1) { std::cmp::Ordering::Equal => ips.first(), std::cmp::Ordering::Greater => Some(&ips[rand::rng().random_range(0..ips.len())]), std::cmp::Ordering::Less => None, } } } pub trait ToNextHop { fn to_remote_hosts<'x, 'y: 'x>( &'x self, domain: &'y str, config: &'x MxConfig, ) -> Option>>; } impl ToNextHop for Vec { fn to_remote_hosts<'x, 'y: 'x>( &'x self, domain: &'y str, config: &'x MxConfig, ) -> Option>> { if !self.is_empty() { // Obtain max number of MX hosts to process let mut remote_hosts = Vec::with_capacity(config.max_mx); 'outer: for mx in self.iter() { if mx.exchanges.len() > 1 { let mut slice = mx.exchanges.iter().collect::>(); slice.shuffle(&mut rand::rng()); for remote_host in slice { remote_hosts.push(NextHop::MX { host: remote_host.as_str(), is_implicit: false, config, }); if remote_hosts.len() == config.max_mx { break 'outer; } } } else if let Some(remote_host) = mx.exchanges.first() { // Check for Null MX if mx.preference == 0 && remote_host == "." { return None; } remote_hosts.push(NextHop::MX { host: remote_host.as_str(), is_implicit: false, config, }); if remote_hosts.len() == config.max_mx { break; } } } remote_hosts.into() } else { // If an empty list of MXs is returned, the address is treated as if it was // associated with an implicit MX RR with a preference of 0, pointing to that host. vec![NextHop::MX { host: domain, is_implicit: true, config, }] .into() } } } ================================================ FILE: crates/smtp/src/outbound/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ outbound::client::BoxResponse, queue::{Error, ErrorDetails, HostResponse, Status, UnexpectedResponse}, }; use common::config::{ server::ServerProtocol, smtp::queue::{MxConfig, RelayConfig}, }; use mail_auth::IpLookupStrategy; use mail_send::Credentials; use smtp_proto::{Response, Severity}; use std::borrow::Cow; pub mod client; pub mod dane; pub mod delivery; pub mod local; pub mod lookup; pub mod mta_sts; pub mod session; pub(super) enum DeliveryResult { Domain { status: Status>, ErrorDetails>, rcpt_idxs: Vec, }, Account { status: Status>, ErrorDetails>, rcpt_idx: usize, }, RateLimited { rcpt_idxs: Vec, retry_at: u64, }, } impl Status>, ErrorDetails> { pub fn from_smtp_error(hostname: &str, command: &str, err: mail_send::Error) -> Self { match err { mail_send::Error::Io(_) | mail_send::Error::Tls(_) | mail_send::Error::Base64(_) | mail_send::Error::UnparseableReply | mail_send::Error::AuthenticationFailed(_) | mail_send::Error::MissingCredentials | mail_send::Error::MissingMailFrom | mail_send::Error::MissingRcptTo | mail_send::Error::Timeout => Status::TemporaryFailure(ErrorDetails { entity: hostname.into(), details: Error::ConnectionError(err.to_string().into_boxed_str()), }), mail_send::Error::UnexpectedReply(response) => { if response.severity() == Severity::PermanentNegativeCompletion { Status::PermanentFailure(ErrorDetails { entity: hostname.into(), details: Error::UnexpectedResponse(UnexpectedResponse { command: command.trim().into(), response: response.into_box(), }), }) } else { Status::TemporaryFailure(ErrorDetails { entity: hostname.into(), details: Error::UnexpectedResponse(UnexpectedResponse { command: command.trim().into(), response: response.into_box(), }), }) } } mail_send::Error::Auth(_) | mail_send::Error::UnsupportedAuthMechanism | mail_send::Error::InvalidTLSName | mail_send::Error::MissingStartTls => Status::PermanentFailure(ErrorDetails { entity: hostname.into(), details: Error::ConnectionError(err.to_string().into_boxed_str()), }), } } pub fn from_starttls_error(hostname: &str, response: Option>>) -> Self { let entity = hostname.into(); if let Some(response) = response { if response.severity() == Severity::PermanentNegativeCompletion { Status::PermanentFailure(ErrorDetails { entity, details: Error::UnexpectedResponse(UnexpectedResponse { command: "STARTTLS".into(), response, }), }) } else { Status::TemporaryFailure(ErrorDetails { entity, details: Error::UnexpectedResponse(UnexpectedResponse { command: "STARTTLS".into(), response, }), }) } } else { Status::PermanentFailure(ErrorDetails { entity, details: Error::TlsError("STARTTLS not advertised by host.".into()), }) } } pub fn from_tls_error(hostname: &str, err: mail_send::Error) -> Self { match err { mail_send::Error::InvalidTLSName => Status::PermanentFailure(ErrorDetails { entity: hostname.into(), details: Error::TlsError("Invalid hostname".into()), }), mail_send::Error::Timeout => Status::TemporaryFailure(ErrorDetails { entity: hostname.into(), details: Error::TlsError("TLS handshake timed out".into()), }), mail_send::Error::Tls(err) => Status::TemporaryFailure(ErrorDetails { entity: hostname.into(), details: Error::TlsError(format!("Handshake failed: {err}").into_boxed_str()), }), mail_send::Error::Io(err) => Status::TemporaryFailure(ErrorDetails { entity: hostname.into(), details: Error::TlsError(format!("I/O error: {err}").into_boxed_str()), }), _ => Status::PermanentFailure(ErrorDetails { entity: hostname.into(), details: Error::TlsError("Other TLS error".into()), }), } } pub fn timeout(hostname: &str, stage: &str) -> Self { Status::TemporaryFailure(ErrorDetails { entity: hostname.into(), details: Error::ConnectionError(format!("Timeout while {stage}").into_boxed_str()), }) } pub fn local_error() -> Self { Status::TemporaryFailure(ErrorDetails { entity: "localhost".into(), details: Error::ConnectionError("Could not deliver message locally.".into()), }) } pub fn from_mail_auth_error(entity: &str, err: mail_auth::Error) -> Self { match &err { mail_auth::Error::DnsRecordNotFound(code) => Status::PermanentFailure(ErrorDetails { entity: entity.into(), details: Error::DnsError(format!("Domain not found: {code:?}").into_boxed_str()), }), _ => Status::TemporaryFailure(ErrorDetails { entity: entity.into(), details: Error::DnsError(err.to_string().into_boxed_str()), }), } } pub fn from_mta_sts_error(entity: &str, err: mta_sts::Error) -> Self { match &err { mta_sts::Error::Dns(err) => match err { mail_auth::Error::DnsRecordNotFound(code) => { Status::PermanentFailure(ErrorDetails { entity: entity.into(), details: Error::MtaStsError( format!("Record not found: {code:?}").into_boxed_str(), ), }) } mail_auth::Error::InvalidRecordType => Status::PermanentFailure(ErrorDetails { entity: entity.into(), details: Error::MtaStsError("Failed to parse MTA-STS DNS record.".into()), }), _ => Status::TemporaryFailure(ErrorDetails { entity: entity.into(), details: Error::MtaStsError( format!("DNS lookup error: {err}").into_boxed_str(), ), }), }, mta_sts::Error::Http(err) => { if err.is_timeout() { Status::TemporaryFailure(ErrorDetails { entity: entity.into(), details: Error::MtaStsError("Timeout fetching policy.".into()), }) } else if err.is_connect() { Status::TemporaryFailure(ErrorDetails { entity: entity.into(), details: Error::MtaStsError("Could not reach policy host.".into()), }) } else if err.is_status() & err .status() .is_some_and(|s| s == reqwest::StatusCode::NOT_FOUND) { Status::PermanentFailure(ErrorDetails { entity: entity.into(), details: Error::MtaStsError("Policy not found.".into()), }) } else { Status::TemporaryFailure(ErrorDetails { entity: entity.into(), details: Error::MtaStsError("Failed to fetch policy.".into()), }) } } mta_sts::Error::InvalidPolicy(err) => Status::PermanentFailure(ErrorDetails { entity: entity.into(), details: Error::MtaStsError( format!("Failed to parse policy: {err}").into_boxed_str(), ), }), } } } #[derive(Debug)] pub enum NextHop<'x> { Relay(&'x RelayConfig), MX { is_implicit: bool, host: &'x str, config: &'x MxConfig, }, } impl NextHop<'_> { #[inline(always)] pub fn hostname(&self) -> &str { match self { NextHop::MX { host, .. } => { if let Some(host) = host.strip_suffix('.') { host } else { host } } NextHop::Relay(host) => host.address.as_str(), } } #[inline(always)] pub fn fqdn_hostname(&self) -> Cow<'_, str> { match self { NextHop::MX { host, .. } => { if !host.ends_with('.') { format!("{host}.").into() } else { (*host).into() } } NextHop::Relay(host) => host.address.as_str().into(), } } #[inline(always)] pub fn max_multi_homed(&self) -> usize { match self { NextHop::MX { config, .. } => config.max_multi_homed, NextHop::Relay(_) => 10, } } #[inline(always)] pub fn ip_lookup_strategy(&self) -> IpLookupStrategy { match self { NextHop::MX { config, .. } => config.ip_lookup_strategy, NextHop::Relay(_) => IpLookupStrategy::Ipv4thenIpv6, } } #[inline(always)] fn port(&self) -> u16 { match self { #[cfg(feature = "test_mode")] NextHop::MX { .. } => 9925, #[cfg(not(feature = "test_mode"))] NextHop::MX { .. } => 25, NextHop::Relay(host) => host.port, } } #[inline(always)] fn credentials(&self) -> Option<&Credentials> { match self { NextHop::MX { .. } => None, NextHop::Relay(host) => host.auth.as_ref(), } } #[inline(always)] fn allow_invalid_certs(&self) -> bool { #[cfg(feature = "test_mode")] { true } #[cfg(not(feature = "test_mode"))] match self { NextHop::MX { .. } => false, NextHop::Relay(host) => host.tls_allow_invalid_certs, } } #[inline(always)] fn implicit_tls(&self) -> bool { match self { NextHop::MX { .. } => false, NextHop::Relay(host) => host.tls_implicit, } } #[inline(always)] fn is_smtp(&self) -> bool { match self { NextHop::MX { .. } => true, NextHop::Relay(host) => host.protocol == ServerProtocol::Smtp, } } } impl DeliveryResult { pub fn domain( status: Status>, ErrorDetails>, rcpt_idxs: Vec, ) -> Self { DeliveryResult::Domain { status, rcpt_idxs } } pub fn rate_limited(rcpt_idxs: Vec, retry_at: u64) -> Self { DeliveryResult::RateLimited { rcpt_idxs, retry_at, } } pub fn account(status: Status>, ErrorDetails>, rcpt_idx: usize) -> Self { DeliveryResult::Account { status, rcpt_idx } } } ================================================ FILE: crates/smtp/src/outbound/mta_sts/lookup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{fmt::Display, sync::Arc, time::Duration}; #[cfg(feature = "test_mode")] pub static STS_TEST_POLICY: parking_lot::Mutex> = parking_lot::Mutex::new(Vec::new()); use common::{Server, config::smtp::resolver::Policy}; use mail_auth::{mta_sts::MtaSts, report::tlsrpt::ResultType}; use super::{Error, parse::ParsePolicy}; #[cfg(not(feature = "test_mode"))] use utils::HttpLimitResponse; #[cfg(not(feature = "test_mode"))] const MAX_POLICY_SIZE: usize = 1024 * 1024; pub trait MtaStsLookup: Sync + Send { fn lookup_mta_sts_policy( &self, domain: &str, timeout: Duration, ) -> impl std::future::Future, Error>> + Send; } #[allow(unused_variables)] impl MtaStsLookup for Server { async fn lookup_mta_sts_policy( &self, domain: &str, timeout: Duration, ) -> Result, Error> { // Lookup MTA-STS TXT record let record = match self .core .smtp .resolvers .dns .txt_lookup::( format!("_mta-sts.{domain}."), Some(&self.inner.cache.dns_txt), ) .await { Ok(record) => record, Err(err) => { // Return the cached policy in case of failure return if let Some(value) = self.inner.cache.dbs_mta_sts.get(domain) { Ok(value) } else { Err(err.into()) }; } }; // Check if the policy has been cached if let Some(value) = self.inner.cache.dbs_mta_sts.get(domain) && value.id == record.id { return Ok(value); } // Fetch policy #[cfg(not(feature = "test_mode"))] let bytes = reqwest::Client::builder() .user_agent(common::USER_AGENT) .timeout(timeout) .redirect(reqwest::redirect::Policy::none()) .build()? .get(format!("https://mta-sts.{domain}/.well-known/mta-sts.txt")) .send() .await? .bytes_with_limit(MAX_POLICY_SIZE) .await? .ok_or_else(|| Error::InvalidPolicy("Policy too large".to_string()))?; #[cfg(feature = "test_mode")] let bytes = STS_TEST_POLICY.lock().clone(); // Parse policy let policy = Arc::new(Policy::parse( std::str::from_utf8(&bytes).map_err(|err| Error::InvalidPolicy(err.to_string()))?, record.id.clone(), )?); self.inner.cache.dbs_mta_sts.insert( domain.to_string(), policy.clone(), Duration::from_secs(if (3600..31557600).contains(&policy.max_age) { policy.max_age } else { 86400 }), ); Ok(policy) } } impl From<&Error> for ResultType { fn from(err: &Error) -> Self { match &err { Error::InvalidPolicy(_) => ResultType::StsPolicyInvalid, _ => ResultType::StsPolicyFetchError, } } } impl Display for Error { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Error::Dns(err) => match err { mail_auth::Error::DnsRecordNotFound(code) => { write!(f, "Record not found: {code:?}") } mail_auth::Error::InvalidRecordType => { f.write_str("Failed to parse MTA-STS DNS record.") } _ => write!(f, "DNS lookup error: {err}"), }, Error::Http(err) => { if err.is_timeout() { f.write_str("Timeout fetching policy.") } else if err.is_connect() { f.write_str("Could not reach policy host.") } else if err.is_status() && (err.status() == Some(reqwest::StatusCode::NOT_FOUND)) { f.write_str("Policy not found.") } else { f.write_str("Failed to fetch policy.") } } Error::InvalidPolicy(err) => write!(f, "Failed to parse policy: {err}"), } } } impl From for Error { fn from(value: mail_auth::Error) -> Self { Error::Dns(value) } } impl From for Error { fn from(value: reqwest::Error) -> Self { Error::Http(value) } } impl From for Error { fn from(value: String) -> Self { Error::InvalidPolicy(value) } } ================================================ FILE: crates/smtp/src/outbound/mta_sts/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod lookup; pub mod parse; pub mod verify; #[derive(Debug)] pub enum Error { Dns(mail_auth::Error), Http(reqwest::Error), InvalidPolicy(String), } ================================================ FILE: crates/smtp/src/outbound/mta_sts/parse.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::config::smtp::resolver::{Mode, MxPattern, Policy}; pub trait ParsePolicy { fn parse(data: &str, id: String) -> Result where Self: Sized; } impl ParsePolicy for Policy { fn parse(mut data: &str, id: String) -> Result { let mut mode = Mode::None; let mut max_age: u64 = 86400; let mut mx = Vec::new(); while !data.is_empty() { if let Some((key, next_data)) = data.split_once(':') { let value = if let Some((value, next_data)) = next_data.split_once('\n') { data = next_data; value.trim() } else { data = ""; next_data.trim() }; match key.trim() { "mx" => { if let Some(suffix) = value.strip_prefix("*.") { if !suffix.is_empty() { mx.push(MxPattern::StartsWith(suffix.to_lowercase())); } } else if !value.is_empty() { mx.push(MxPattern::Equals(value.to_lowercase())); } } "max_age" => { if let Ok(value) = value.parse() { max_age = value; } } "mode" => { mode = match value { "enforce" => Mode::Enforce, "testing" => Mode::Testing, "none" => Mode::None, _ => return Err(format!("Unsupported mode {value:?}.")), }; } "version" => { if !value.eq_ignore_ascii_case("STSv1") { return Err(format!("Unsupported version {value:?}.")); } } _ => (), } } else { break; } } if !mx.is_empty() { Ok(Policy { id, mode, mx, max_age, }) } else { Err("No 'mx' entries found.".to_string()) } } } ================================================ FILE: crates/smtp/src/outbound/mta_sts/verify.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::config::smtp::resolver::{Mode, MxPattern, Policy}; pub trait VerifyPolicy { fn verify(&self, mx_host: &str) -> bool; fn enforce(&self) -> bool; } impl VerifyPolicy for Policy { fn verify(&self, mx_host: &str) -> bool { if self.mode != Mode::None { for mx_pattern in &self.mx { match mx_pattern { MxPattern::Equals(host) => { if host == mx_host { return true; } } MxPattern::StartsWith(domain) => { if let Some((_, suffix)) = mx_host.split_once('.') && suffix == domain { return true; } } } } false } else { true } } fn enforce(&self) -> bool { self.mode == Mode::Enforce } } ================================================ FILE: crates/smtp/src/outbound/session.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::client::SmtpClient; use crate::outbound::DeliveryResult; use crate::outbound::client::{BoxResponse, from_error_status, from_mail_send_error}; use crate::queue::{Error, MessageWrapper, Recipient, Status}; use crate::queue::{ErrorDetails, HostResponse, UnexpectedResponse}; use common::Server; use common::config::smtp::queue::ConnectionStrategy; use mail_send::Credentials; use smtp_proto::{ EXT_CHUNKING, EXT_DSN, EXT_REQUIRE_TLS, EXT_SIZE, EXT_SMTP_UTF8, EhloResponse, MAIL_REQUIRETLS, MAIL_RET_FULL, MAIL_RET_HDRS, MAIL_SMTPUTF8, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, Severity, }; use std::{fmt::Write, time::Instant}; use tokio::io::{AsyncRead, AsyncWrite}; use trc::DeliveryEvent; pub struct SessionParams<'x> { pub server: &'x Server, pub hostname: &'x str, pub credentials: Option<&'x Credentials>, pub capabilities: Option>, pub is_smtp: bool, pub local_hostname: &'x str, pub conn_strategy: &'x ConnectionStrategy, pub session_id: u64, } impl MessageWrapper { pub(super) async fn deliver( &self, mut smtp_client: SmtpClient, rcpt_idxs: Vec, statuses: &mut Vec, mut params: SessionParams<'_>, ) { // Obtain capabilities let time = Instant::now(); let capabilities = if let Some(capabilities) = params.capabilities.take() { capabilities } else { match smtp_client.say_helo(¶ms).await { Ok(capabilities) => { trc::event!( Delivery(DeliveryEvent::Ehlo), SpanId = params.session_id, Hostname = params.hostname.to_string(), Details = capabilities.capabilities(), Elapsed = time.elapsed(), ); capabilities } Err(status) => { trc::event!( Delivery(DeliveryEvent::EhloRejected), SpanId = params.session_id, Hostname = params.hostname.to_string(), CausedBy = from_error_status(&status), Elapsed = time.elapsed(), ); smtp_client.quit().await; statuses.push(DeliveryResult::domain(status, rcpt_idxs)); return; } } }; // Authenticate if let Some(credentials) = params.credentials { let time = Instant::now(); if let Err(err) = smtp_client.authenticate(credentials, &capabilities).await { trc::event!( Delivery(DeliveryEvent::AuthFailed), SpanId = params.session_id, Hostname = params.hostname.to_string(), CausedBy = from_mail_send_error(&err), Elapsed = time.elapsed(), ); smtp_client.quit().await; statuses.push(DeliveryResult::domain( Status::from_smtp_error(params.hostname, "AUTH ...", err), rcpt_idxs, )); return; } trc::event!( Delivery(DeliveryEvent::Auth), SpanId = params.session_id, Hostname = params.hostname.to_string(), Elapsed = time.elapsed(), ); // Refresh capabilities // Disabled as some SMTP servers deauthenticate after EHLO /*capabilities = match say_helo(&mut smtp_client, ¶ms).await { Ok(capabilities) => capabilities, Err(status) => { trc::event!( context = "ehlo", event = "rejected", mx = ¶ms.hostname, reason = %status, ); smtp_client.quit().await; return status; } };*/ } // MAIL FROM let time = Instant::now(); smtp_client.timeout = params.conn_strategy.timeout_mail; let cmd = self.build_mail_from(&capabilities); match smtp_client.cmd(cmd.as_bytes()).await.and_then(|r| { if r.is_positive_completion() { Ok(r) } else { Err(mail_send::Error::UnexpectedReply(r)) } }) { Ok(response) => { trc::event!( Delivery(DeliveryEvent::MailFrom), SpanId = params.session_id, Hostname = params.hostname.to_string(), From = self.message.return_path.to_string(), Code = response.code, Details = response.message.to_string(), Elapsed = time.elapsed(), ); } Err(err) => { trc::event!( Delivery(DeliveryEvent::MailFromRejected), SpanId = params.session_id, Hostname = params.hostname.to_string(), CausedBy = from_mail_send_error(&err), Elapsed = time.elapsed(), ); smtp_client.quit().await; statuses.push(DeliveryResult::domain( Status::from_smtp_error(params.hostname, &cmd, err), rcpt_idxs, )); return; } } // RCPT TO let mut accepted_rcpts = Vec::new(); smtp_client.timeout = params.conn_strategy.timeout_rcpt; for rcpt_idx in &rcpt_idxs { let time = Instant::now(); let rcpt = &self.message.recipients[*rcpt_idx]; if matches!( &rcpt.status, Status::Completed(_) | Status::PermanentFailure(_) ) { continue; } let cmd = self.build_rcpt_to(rcpt, &capabilities); match smtp_client.cmd(cmd.as_bytes()).await { Ok(response) => match response.severity() { Severity::PositiveCompletion => { trc::event!( Delivery(DeliveryEvent::RcptTo), SpanId = params.session_id, Hostname = params.hostname.to_string(), To = rcpt.address().to_string(), Code = response.code, Details = response.message.to_string(), Elapsed = time.elapsed(), ); accepted_rcpts.push(( rcpt, rcpt_idx, Status::Completed(HostResponse { hostname: params.hostname.into(), response: response.into_box(), }), )); } severity => { trc::event!( Delivery(DeliveryEvent::RcptToRejected), SpanId = params.session_id, Hostname = params.hostname.to_string(), To = rcpt.address().to_string(), Code = response.code, Details = response.message.to_string(), Elapsed = time.elapsed(), ); let response = ErrorDetails { entity: params.hostname.into(), details: Error::UnexpectedResponse(UnexpectedResponse { command: cmd.trim().into(), response: response.into_box(), }), }; statuses.push(DeliveryResult::account( if severity == Severity::PermanentNegativeCompletion { Status::PermanentFailure(response) } else { Status::TemporaryFailure(response) }, *rcpt_idx, )); } }, Err(err) => { trc::event!( Delivery(DeliveryEvent::RcptToFailed), SpanId = params.session_id, Hostname = params.hostname.to_string(), To = rcpt.address().to_string(), CausedBy = from_mail_send_error(&err), Elapsed = time.elapsed(), ); // Something went wrong, abort. smtp_client.quit().await; statuses.push(DeliveryResult::domain( Status::from_smtp_error(params.hostname, "", err), rcpt_idxs, )); return; } } } // Send message if !accepted_rcpts.is_empty() { let time = Instant::now(); let bdat_cmd = capabilities .has_capability(EXT_CHUNKING) .then(|| format!("BDAT {} LAST\r\n", self.message.size)); if let Err(status) = smtp_client.send_message(self, &bdat_cmd, ¶ms).await { trc::event!( Delivery(DeliveryEvent::MessageRejected), SpanId = params.session_id, Hostname = params.hostname.to_string(), CausedBy = from_error_status(&status), Elapsed = time.elapsed(), ); smtp_client.quit().await; statuses.push(DeliveryResult::domain(status, rcpt_idxs)); return; } if params.is_smtp { // Handle SMTP response match smtp_client .read_smtp_data_response(params.hostname, &bdat_cmd) .await { Ok(response) => { // Mark recipients as delivered if response.code() == 250 { for (rcpt, rcpt_idx, status) in accepted_rcpts { trc::event!( Delivery(DeliveryEvent::Delivered), SpanId = params.session_id, Hostname = params.hostname.to_string(), To = rcpt.address().to_string(), Code = response.code, Details = response.message.to_string(), Elapsed = time.elapsed(), ); statuses.push(DeliveryResult::account(status, *rcpt_idx)); } } else { trc::event!( Delivery(DeliveryEvent::MessageRejected), SpanId = params.session_id, Hostname = params.hostname.to_string(), Code = response.code, Details = response.message.to_string(), Elapsed = time.elapsed(), ); smtp_client.quit().await; statuses.push(DeliveryResult::domain( Status::from_smtp_error( params.hostname, bdat_cmd.as_deref().unwrap_or("DATA"), mail_send::Error::UnexpectedReply(response), ), rcpt_idxs, )); return; } } Err(status) => { trc::event!( Delivery(DeliveryEvent::MessageRejected), SpanId = params.session_id, Hostname = params.hostname.to_string(), CausedBy = from_error_status(&status), Elapsed = time.elapsed(), ); smtp_client.quit().await; statuses.push(DeliveryResult::domain(status, rcpt_idxs)); return; } } } else { // Handle LMTP responses match smtp_client .read_lmtp_data_response(params.hostname, accepted_rcpts.len()) .await { Ok(responses) => { for ((rcpt, rcpt_idx, _), response) in accepted_rcpts.into_iter().zip(responses) { let status: Status>, ErrorDetails> = match response.severity() { Severity::PositiveCompletion => { trc::event!( Delivery(DeliveryEvent::Delivered), SpanId = params.session_id, Hostname = params.hostname.to_string(), To = rcpt.address().to_string(), Code = response.code, Details = response.message.to_string(), Elapsed = time.elapsed(), ); Status::Completed(HostResponse { hostname: params.hostname.into(), response, }) } severity => { trc::event!( Delivery(DeliveryEvent::RcptToRejected), SpanId = params.session_id, Hostname = params.hostname.to_string(), To = rcpt.address().to_string(), Code = response.code, Details = response.message.to_string(), Elapsed = time.elapsed(), ); let response = ErrorDetails { entity: params.hostname.into(), details: Error::UnexpectedResponse( UnexpectedResponse { command: bdat_cmd .as_deref() .unwrap_or("DATA") .into(), response, }, ), }; if severity == Severity::PermanentNegativeCompletion { Status::PermanentFailure(response) } else { Status::TemporaryFailure(response) } } }; statuses.push(DeliveryResult::account(status, *rcpt_idx)); } } Err(status) => { trc::event!( Delivery(DeliveryEvent::MessageRejected), SpanId = params.session_id, Hostname = params.hostname.to_string(), CausedBy = from_error_status(&status), Elapsed = time.elapsed(), ); smtp_client.quit().await; statuses.push(DeliveryResult::domain(status, rcpt_idxs)); return; } } } } smtp_client.quit().await; } fn build_mail_from(&self, capabilities: &EhloResponse) -> String { let mut mail_from = String::with_capacity(self.message.return_path.len() + 60); let _ = write!(mail_from, "MAIL FROM:<{}>", self.message.return_path); if capabilities.has_capability(EXT_SIZE) { let _ = write!(mail_from, " SIZE={}", self.message.size); } if self.has_flag(MAIL_REQUIRETLS) & capabilities.has_capability(EXT_REQUIRE_TLS) { mail_from.push_str(" REQUIRETLS"); } if self.has_flag(MAIL_SMTPUTF8) & capabilities.has_capability(EXT_SMTP_UTF8) { mail_from.push_str(" SMTPUTF8"); } if capabilities.has_capability(EXT_DSN) { if self.has_flag(MAIL_RET_FULL) { mail_from.push_str(" RET=FULL"); } else if self.has_flag(MAIL_RET_HDRS) { mail_from.push_str(" RET=HDRS"); } if let Some(env_id) = &self.message.env_id { let _ = write!(mail_from, " ENVID={env_id}"); } } mail_from.push_str("\r\n"); mail_from } fn build_rcpt_to(&self, rcpt: &Recipient, capabilities: &EhloResponse) -> String { let mut rcpt_to = String::with_capacity(rcpt.address().len() + 60); let _ = write!(rcpt_to, "RCPT TO:<{}>", rcpt.address()); if capabilities.has_capability(EXT_DSN) { if rcpt.has_flag(RCPT_NOTIFY_SUCCESS | RCPT_NOTIFY_FAILURE | RCPT_NOTIFY_DELAY) { rcpt_to.push_str(" NOTIFY="); let mut add_comma = if rcpt.has_flag(RCPT_NOTIFY_SUCCESS) { rcpt_to.push_str("SUCCESS"); true } else { false }; if rcpt.has_flag(RCPT_NOTIFY_DELAY) { if add_comma { rcpt_to.push(','); } else { add_comma = true; } rcpt_to.push_str("DELAY"); } if rcpt.has_flag(RCPT_NOTIFY_FAILURE) { if add_comma { rcpt_to.push(','); } rcpt_to.push_str("FAILURE"); } } else if rcpt.has_flag(RCPT_NOTIFY_NEVER) { rcpt_to.push_str(" NOTIFY=NEVER"); } } rcpt_to.push_str("\r\n"); rcpt_to } #[inline(always)] pub fn has_flag(&self, flag: u64) -> bool { (self.message.flags & flag) != 0 } } impl Recipient { #[inline(always)] pub fn has_flag(&self, flag: u64) -> bool { (self.flags & flag) != 0 } } ================================================ FILE: crates/smtp/src/queue/dsn.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::spool::SmtpSpool; use super::{ Error, ErrorDetails, HostResponse, Message, MessageSource, QueueEnvelope, RCPT_DSN_SENT, Recipient, Status, }; use crate::queue::{MessageWrapper, UnexpectedResponse}; use crate::reporting::SmtpReporting; use common::Server; use mail_builder::MessageBuilder; use mail_builder::headers::HeaderType; use mail_builder::headers::content_type::ContentType; use mail_builder::mime::{BodyPart, MimePart, make_boundary}; use mail_parser::DateTime; use smtp_proto::{ RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, Response, }; use std::fmt::Write; use std::future::Future; use store::write::now; pub trait SendDsn: Sync + Send { fn send_dsn(&self, message: &mut MessageWrapper) -> impl Future + Send; fn log_dsn(&self, message: &MessageWrapper) -> impl Future + Send; } impl SendDsn for Server { async fn send_dsn(&self, message: &mut MessageWrapper) { // Send DSN events self.log_dsn(message).await; if !message.message.return_path.is_empty() { // Build DSN if let Some(dsn) = message.build_dsn(self).await { let mut dsn_message = self.new_message("", message.span_id); dsn_message .add_recipient(message.message.return_path.as_ref(), self) .await; // Sign message let signature = self .sign_message(message, &self.core.smtp.queue.dsn.sign, &dsn) .await; // Queue DSN dsn_message .queue( signature.as_deref(), &dsn, message.span_id, self, MessageSource::Dsn, ) .await; } } else { // Handle double bounce message.handle_double_bounce(); } } async fn log_dsn(&self, message: &MessageWrapper) { let now = now(); for rcpt in &message.message.recipients { if rcpt.has_flag(RCPT_DSN_SENT) { continue; } match &rcpt.status { Status::Completed(response) => { trc::event!( Delivery(trc::DeliveryEvent::DsnSuccess), SpanId = message.span_id, To = rcpt.address.clone(), Hostname = response.hostname.clone(), Code = response.response.code, Details = response.response.message.to_string(), ); } Status::TemporaryFailure(response) if rcpt.notify.due <= now => { trc::event!( Delivery(trc::DeliveryEvent::DsnTempFail), SpanId = message.span_id, To = rcpt.address.clone(), Hostname = response.entity.clone(), Details = response.details.to_string(), NextRetry = trc::Value::Timestamp(rcpt.retry.due), Expires = rcpt .expiration_time(message.message.created) .map(trc::Value::Timestamp), Total = rcpt.retry.inner, ); } Status::PermanentFailure(response) => { trc::event!( Delivery(trc::DeliveryEvent::DsnPermFail), SpanId = message.span_id, To = rcpt.address.clone(), Hostname = response.entity.clone(), Details = response.details.to_string(), Total = rcpt.retry.inner, ); } Status::Scheduled if rcpt.notify.due <= now => { trc::event!( Delivery(trc::DeliveryEvent::DsnTempFail), SpanId = message.span_id, To = rcpt.address.clone(), Details = "Concurrency limited", NextRetry = trc::Value::Timestamp(rcpt.retry.due), Expires = rcpt .expiration_time(message.message.created) .map(trc::Value::Timestamp), Total = rcpt.retry.inner, ); } _ => continue, } } } } const MAX_HEADER_SIZE: usize = 4096; impl MessageWrapper { pub async fn build_dsn(&mut self, server: &Server) -> Option> { let config = &server.core.smtp.queue; let now = now(); let mut txt_success = String::new(); let mut txt_delay = String::new(); let mut txt_failed = String::new(); let mut dsn = String::new(); for rcpt in &mut self.message.recipients { if rcpt.has_flag(RCPT_DSN_SENT | RCPT_NOTIFY_NEVER) { continue; } match &rcpt.status { Status::Completed(response) => { rcpt.flags |= RCPT_DSN_SENT; if !rcpt.has_flag(RCPT_NOTIFY_SUCCESS) { continue; } rcpt.write_dsn(&mut dsn); rcpt.status.write_dsn(&mut dsn); response.write_dsn_text(&rcpt.address, &mut txt_success); } Status::TemporaryFailure(response) if rcpt.notify.due <= now && rcpt.has_flag(RCPT_NOTIFY_DELAY) => { rcpt.write_dsn(&mut dsn); rcpt.status.write_dsn(&mut dsn); rcpt.write_dsn_will_retry_until(self.message.created, &mut dsn); response.write_dsn_text(&rcpt.address, &mut txt_delay); } Status::PermanentFailure(response) => { rcpt.flags |= RCPT_DSN_SENT; if !rcpt.has_flag(RCPT_NOTIFY_FAILURE) { continue; } rcpt.write_dsn(&mut dsn); rcpt.status.write_dsn(&mut dsn); response.write_dsn_text(&rcpt.address, &mut txt_failed); } Status::Scheduled if rcpt.notify.due <= now && rcpt.has_flag(RCPT_NOTIFY_DELAY) => { // This case should not happen under normal circumstances rcpt.write_dsn(&mut dsn); rcpt.status.write_dsn(&mut dsn); rcpt.write_dsn_will_retry_until(self.message.created, &mut dsn); ErrorDetails { entity: "localhost".into(), details: Error::ConcurrencyLimited, } .write_dsn_text(&rcpt.address, &mut txt_delay); } _ => continue, } dsn.push_str("\r\n"); } // Build text response let txt_len = txt_success.len() + txt_delay.len() + txt_failed.len(); if txt_len == 0 { return None; } let has_success = !txt_success.is_empty(); let has_delay = !txt_delay.is_empty(); let has_failure = !txt_failed.is_empty(); let mut txt = String::with_capacity(txt_len + 128); let (subject, is_mixed) = if has_success && !has_delay && !has_failure { txt.push_str( "Your message has been successfully delivered to the following recipients:\r\n\r\n", ); ("Successfully delivered message", false) } else if has_delay && !has_success && !has_failure { txt.push_str("There was a temporary problem delivering your message to the following recipients:\r\n\r\n"); ("Warning: Delay in message delivery", false) } else if has_failure && !has_success && !has_delay { txt.push_str( "Your message could not be delivered to the following recipients:\r\n\r\n", ); ("Failed to deliver message", false) } else if has_success { txt.push_str("Your message has been partially delivered:\r\n\r\n"); ("Partially delivered message", true) } else { txt.push_str("Your message could not be delivered to some recipients:\r\n\r\n"); ( "Warning: Temporary and permanent failures during message delivery", true, ) }; if has_success { if is_mixed { txt.push_str( " ----- Delivery to the following addresses was successful -----\r\n", ); } txt.push_str(&txt_success); txt.push_str("\r\n"); } if has_delay { if is_mixed { txt.push_str( " ----- There was a temporary problem delivering to these addresses -----\r\n", ); } txt.push_str(&txt_delay); txt.push_str("\r\n"); } if has_failure { if is_mixed { txt.push_str(" ----- Delivery to the following addresses failed -----\r\n"); } txt.push_str(&txt_failed); txt.push_str("\r\n"); } // Update next delay notification time if has_delay { let mut changes = Vec::new(); for (rcpt_idx, rcpt) in self.message.recipients.iter().enumerate() { if matches!( &rcpt.status, Status::TemporaryFailure(_) | Status::Scheduled ) && rcpt.notify.due <= now { let envelope = QueueEnvelope::new(&self.message, rcpt); let queue_id = server .eval_if::( &server.core.smtp.queue.queue, &envelope, self.span_id, ) .await .unwrap_or_else(|| "default".to_string()); let queue = server.get_queue_or_default(&queue_id, self.span_id); if let Some(next_notify) = queue.notify.get((rcpt.notify.inner + 1) as usize).copied() { changes.push((rcpt_idx, 1, now + next_notify)); } else { changes.push((rcpt_idx, 0, u64::MAX)); } } } for (rcpt_idx, inner, due) in changes { let rcpt = &mut self.message.recipients[rcpt_idx]; rcpt.notify.inner += inner; rcpt.notify.due = due; } } // Obtain hostname and sender addresses let from_name = server .eval_if(&config.dsn.name, &self.message, self.span_id) .await .unwrap_or_else(|| String::from("Mail Delivery Subsystem")); let from_addr = server .eval_if(&config.dsn.address, &self.message, self.span_id) .await .unwrap_or_else(|| String::from("MAILER-DAEMON@localhost")); let reporting_mta = server .eval_if( &server.core.smtp.report.submitter, &self.message, self.span_id, ) .await .unwrap_or_else(|| String::from("localhost")); // Prepare DSN let mut dsn_header = String::with_capacity(dsn.len() + 128); self.message .write_dsn_headers(&mut dsn_header, &reporting_mta); let dsn = dsn_header + dsn.as_str(); // Fetch up to MAX_HEADER_SIZE bytes of message headers let headers = match server .blob_store() .get_blob(self.message.blob_hash.as_slice(), 0..MAX_HEADER_SIZE) .await { Ok(Some(mut buf)) => { let mut prev_ch = 0; let mut last_lf = buf.len(); for (pos, &ch) in buf.iter().enumerate() { match ch { b'\n' => { last_lf = pos + 1; if prev_ch != b'\n' { prev_ch = ch; } else { break; } } b'\r' => (), 0 => break, _ => { prev_ch = ch; } } } if last_lf < MAX_HEADER_SIZE { buf.truncate(last_lf); } String::from_utf8(buf).unwrap_or_default() } Ok(None) => { trc::event!( Queue(trc::QueueEvent::BlobNotFound), SpanId = self.span_id, BlobId = self.message.blob_hash.to_hex(), CausedBy = trc::location!() ); String::new() } Err(err) => { trc::error!( err.span_id(self.span_id) .details("Failed to fetch blobId") .caused_by(trc::location!()) ); String::new() } }; // Build message MessageBuilder::new() .from((from_name.as_str(), from_addr.as_str())) .header( "To", HeaderType::Text(self.message.return_path.as_ref().into()), ) .header("Auto-Submitted", HeaderType::Text("auto-generated".into())) .message_id(format!("<{}@{}>", make_boundary("."), reporting_mta)) .subject(subject) .body(MimePart::new( ContentType::new("multipart/report").attribute("report-type", "delivery-status"), BodyPart::Multipart(vec![ MimePart::new(ContentType::new("text/plain"), BodyPart::Text(txt.into())), MimePart::new( ContentType::new("message/delivery-status"), BodyPart::Text(dsn.into()), ), MimePart::new( ContentType::new("message/rfc822"), BodyPart::Text(headers.into()), ), ]), )) .write_to_vec() .unwrap_or_default() .into() } fn handle_double_bounce(&mut self) { let mut is_double_bounce = Vec::with_capacity(0); let now = now(); for rcpt in &mut self.message.recipients { if !rcpt.has_flag(RCPT_DSN_SENT | RCPT_NOTIFY_NEVER) && let Status::PermanentFailure(err) = &rcpt.status { rcpt.flags |= RCPT_DSN_SENT; let mut dsn = String::new(); err.write_dsn_text(&rcpt.address, &mut dsn); is_double_bounce.push(dsn); } if rcpt.notify.due <= now { rcpt.notify.due = rcpt .expiration_time(self.message.created) .map(|d| d + 10) .unwrap_or(u64::MAX); } } if !is_double_bounce.is_empty() { trc::event!( Delivery(trc::DeliveryEvent::DoubleBounce), SpanId = self.span_id, To = is_double_bounce ); } } } impl HostResponse> { fn write_dsn_text(&self, addr: &str, dsn: &mut String) { let _ = write!( dsn, "<{}> (delivered to '{}' with code {} ({}.{}.{}) '", addr, self.hostname, self.response.code, self.response.esc[0], self.response.esc[1], self.response.esc[2] ); self.response.write_response(dsn); dsn.push_str("')\r\n"); } } impl UnexpectedResponse { fn write_dsn_text(&self, host: &str, addr: &str, dsn: &mut String) { let _ = write!(dsn, "<{addr}> (host '{host}' rejected "); if !self.command.is_empty() { let _ = write!(dsn, "command '{}'", self.command); } else { dsn.push_str("transaction"); } let _ = write!( dsn, " with code {} ({}.{}.{}) '", self.response.code, self.response.esc[0], self.response.esc[1], self.response.esc[2] ); self.response.write_response(dsn); dsn.push_str("')\r\n"); } } impl ErrorDetails { fn write_dsn_text(&self, addr: &str, dsn: &mut String) { let entity = self.entity.as_ref(); match &self.details { Error::UnexpectedResponse(response) => { response.write_dsn_text(entity, addr, dsn); } Error::DnsError(err) => { let _ = write!(dsn, "<{addr}> (failed to lookup '{entity}': {err})\r\n",); } Error::ConnectionError(details) => { let _ = write!( dsn, "<{addr}> (connection to '{entity}' failed: {details})\r\n", ); } Error::TlsError(details) => { let _ = write!(dsn, "<{addr}> (TLS error from '{entity}': {details})\r\n",); } Error::DaneError(details) => { let _ = write!( dsn, "<{addr}> (DANE failed to authenticate '{entity}': {details})\r\n", ); } Error::MtaStsError(details) => { let _ = write!( dsn, "<{addr}> (MTA-STS failed to authenticate '{entity}': {details})\r\n", ); } Error::RateLimited => { let _ = write!(dsn, "<{addr}> (rate limited)\r\n"); } Error::ConcurrencyLimited => { let _ = write!( dsn, "<{addr}> (too many concurrent connections to remote server)\r\n", ); } Error::Io(err) => { let _ = write!(dsn, "<{addr}> (queue error: {err})\r\n"); } } } } impl Message { fn write_dsn_headers(&self, dsn: &mut String, reporting_mta: &str) { let _ = write!(dsn, "Reporting-MTA: dns;{reporting_mta}\r\n"); dsn.push_str("Arrival-Date: "); dsn.push_str(&DateTime::from_timestamp(self.created as i64).to_rfc822()); dsn.push_str("\r\n"); if let Some(env_id) = &self.env_id { let _ = write!(dsn, "Original-Envelope-Id: {env_id}\r\n"); } dsn.push_str("\r\n"); } } impl Recipient { fn write_dsn(&self, dsn: &mut String) { if let Some(orcpt) = &self.orcpt { let _ = write!(dsn, "Original-Recipient: rfc822;{orcpt}\r\n"); } let _ = write!(dsn, "Final-Recipient: rfc822;{}\r\n", self.address); } fn write_dsn_will_retry_until(&self, created: u64, dsn: &mut String) { if let Some(expires) = self.expiration_time(created) && expires > now() { dsn.push_str("Will-Retry-Until: "); dsn.push_str(&DateTime::from_timestamp(expires as i64).to_rfc822()); dsn.push_str("\r\n"); } } } impl Status { pub fn into_permanent(self) -> Self { match self { Status::TemporaryFailure(v) => Status::PermanentFailure(v), v => v, } } pub fn into_temporary(self) -> Self { match self { Status::PermanentFailure(err) => Status::TemporaryFailure(err), other => other, } } pub fn is_permanent(&self) -> bool { matches!(self, Status::PermanentFailure(_)) } fn write_dsn_action(&self, dsn: &mut String) { dsn.push_str("Action: "); dsn.push_str(match self { Status::Completed(_) => "delivered", Status::PermanentFailure(_) => "failed", Status::TemporaryFailure(_) | Status::Scheduled => "delayed", }); dsn.push_str("\r\n"); } } impl Status>, ErrorDetails> { fn write_dsn(&self, dsn: &mut String) { self.write_dsn_action(dsn); self.write_dsn_status(dsn); self.write_dsn_diagnostic(dsn); self.write_dsn_remote_mta(dsn); } fn write_dsn_status(&self, dsn: &mut String) { dsn.push_str("Status: "); match self { Status::Completed(response) => { response.response.write_dsn_status(dsn); } Status::TemporaryFailure(err) | Status::PermanentFailure(err) => { if let Error::UnexpectedResponse(response) = &err.details { response.response.write_dsn_status(dsn); } else { dsn.push_str(if matches!(self, Status::PermanentFailure(_)) { "5.0.0" } else { "4.0.0" }); } } Status::Scheduled => { dsn.push_str("4.0.0"); } } dsn.push_str("\r\n"); } fn write_dsn_remote_mta(&self, dsn: &mut String) { match self { Status::Completed(response) => { dsn.push_str("Remote-MTA: dns;"); dsn.push_str(&response.hostname); dsn.push_str("\r\n"); } Status::TemporaryFailure(err) | Status::PermanentFailure(err) => match &err.details { Error::UnexpectedResponse(_) | Error::ConnectionError(_) | Error::TlsError(_) | Error::DaneError(_) => { dsn.push_str("Remote-MTA: dns;"); dsn.push_str(&err.entity); dsn.push_str("\r\n"); } _ => (), }, Status::Scheduled => (), } } fn write_dsn_diagnostic(&self, dsn: &mut String) { if let Status::PermanentFailure(err) | Status::TemporaryFailure(err) = self && let Error::UnexpectedResponse(response) = &err.details { response.response.write_dsn_diagnostic(dsn); } } } impl WriteDsn for Response> { fn write_dsn_status(&self, dsn: &mut String) { if self.esc[0] > 0 { let _ = write!(dsn, "{}.{}.{}", self.esc[0], self.esc[1], self.esc[2]); } else { let _ = write!( dsn, "{}.{}.{}", self.code / 100, (self.code / 10) % 10, self.code % 10 ); } } fn write_dsn_diagnostic(&self, dsn: &mut String) { let _ = write!(dsn, "Diagnostic-Code: smtp;{} ", self.code); self.write_response(dsn); dsn.push_str("\r\n"); } fn write_response(&self, dsn: &mut String) { for ch in self.message.chars() { if ch != '\n' && ch != '\r' { dsn.push(ch); } } } } trait WriteDsn { fn write_dsn_status(&self, dsn: &mut String); fn write_dsn_diagnostic(&self, dsn: &mut String); fn write_response(&self, dsn: &mut String); } ================================================ FILE: crates/smtp/src/queue/manager.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{Message, QueueId, Status, spool::SmtpSpool}; use crate::queue::{Recipient, spool::LOCK_EXPIRY}; use ahash::AHashMap; use common::{ Inner, config::smtp::queue::{QueueExpiry, QueueName}, core::BuildServer, ipc::{QueueEvent, QueueEventStatus}, }; use rand::{Rng, seq::SliceRandom}; use std::{ collections::hash_map::Entry, sync::{Arc, atomic::Ordering}, time::{Duration, Instant}, }; use store::write::now; use tokio::sync::mpsc; pub struct Queue { pub core: Arc, pub locked: AHashMap<(QueueId, QueueName), LockedMessage>, pub locked_revision: u64, pub stats: AHashMap, pub next_refresh: Instant, pub rx: mpsc::Receiver, pub is_paused: bool, } #[derive(Debug)] pub struct QueueStats { pub in_flight: usize, pub max_in_flight: usize, pub last_warning: Instant, } #[derive(Debug)] pub struct LockedMessage { pub expires: u64, pub revision: u64, } impl SpawnQueue for mpsc::Receiver { fn spawn(self, core: Arc) { tokio::spawn(async move { Queue::new(core, self).start().await; }); } } const BACK_PRESSURE_WARN_INTERVAL: Duration = Duration::from_secs(60); impl Queue { pub fn new(core: Arc, rx: mpsc::Receiver) -> Self { Queue { core, locked: AHashMap::with_capacity(128), locked_revision: 0, stats: AHashMap::new(), next_refresh: Instant::now() + Duration::from_secs(1), is_paused: false, rx, } } pub async fn start(&mut self) { loop { let mut refresh_queue; match tokio::time::timeout( self.next_refresh.duration_since(Instant::now()), self.rx.recv(), ) .await { Ok(Some(event)) => { refresh_queue = self.handle_event(event).await; while let Ok(event) = self.rx.try_recv() { refresh_queue = self.handle_event(event).await || refresh_queue; } } Err(_) => { refresh_queue = true; } Ok(None) => { break; } }; if !self.is_paused { // Deliver scheduled messages if refresh_queue || self.next_refresh <= Instant::now() { // Process queue events let server = self.core.build_server(); let mut queue_events = server.next_event(self).await; if queue_events.messages.len() > 3 { queue_events.messages.shuffle(&mut rand::rng()); } for queue_event in &queue_events.messages { // Fetch queue stats let stats = match self.stats.get_mut(&queue_event.queue_name) { Some(stats) => stats, None => { let queue_config = server.get_virtual_queue_or_default(&queue_event.queue_name); self.stats.insert( queue_event.queue_name, QueueStats::new(queue_config.threads), ); self.stats.get_mut(&queue_event.queue_name).unwrap() } }; // Enforce concurrency limits if stats.has_capacity() { // Deliver message stats.in_flight += 1; queue_event.try_deliver(server.clone()); } else { if stats.last_warning.elapsed() >= BACK_PRESSURE_WARN_INTERVAL { stats.last_warning = Instant::now(); trc::event!( Queue(trc::QueueEvent::BackPressure), Reason = "Processing capacity for this queue exceeded.", QueueName = queue_event.queue_name.to_string(), Limit = stats.max_in_flight, ); } self.locked .remove(&(queue_event.queue_id, queue_event.queue_name)); } } // Remove expired locks let now = now(); self.locked.retain(|_, locked| { locked.expires > now && locked.revision == self.locked_revision }); self.next_refresh = Instant::now() + Duration::from_secs(queue_events.next_refresh.saturating_sub(now)); } } else { // Queue is paused self.next_refresh = Instant::now() + Duration::from_secs(86400); } } } async fn handle_event(&mut self, event: QueueEvent) -> bool { match event { QueueEvent::WorkerDone { queue_id, queue_name, status, } => { let queue_stats = self.stats.get_mut(&queue_name).unwrap(); queue_stats.in_flight -= 1; match status { QueueEventStatus::Completed => { self.locked.remove(&(queue_id, queue_name)); !self.locked.is_empty() || !queue_stats.has_capacity() } QueueEventStatus::Locked => { let expires = LOCK_EXPIRY + rand::rng().random_range(5..10); let due_in = Instant::now() + Duration::from_secs(expires); if due_in < self.next_refresh { self.next_refresh = due_in; } self.locked.insert( (queue_id, queue_name), LockedMessage { expires: now() + expires, revision: self.locked_revision, }, ); self.locked.len() > 1 || !queue_stats.has_capacity() } QueueEventStatus::Deferred => { self.locked.remove(&(queue_id, queue_name)); true } } } QueueEvent::Refresh => true, QueueEvent::Paused(paused) => { self.core .data .queue_status .store(!paused, Ordering::Relaxed); self.is_paused = paused; false } QueueEvent::ReloadSettings => { let server = self.core.build_server(); for (name, settings) in &server.core.smtp.queue.virtual_queues { if let Some(stats) = self.stats.get_mut(name) { stats.max_in_flight = settings.threads; } else { self.stats.insert(*name, QueueStats::new(settings.threads)); } } false } QueueEvent::Stop => { self.rx.close(); self.is_paused = true; false } } } } impl Message { pub fn next_event(&self, queue: Option) -> Option { let mut next_event = None; for rcpt in &self.recipients { if matches!(rcpt.status, Status::Scheduled | Status::TemporaryFailure(_)) && queue.is_none_or(|q| rcpt.queue == q) { let mut earlier_event = std::cmp::min(rcpt.retry.due, rcpt.notify.due); if let Some(expires) = rcpt.expiration_time(self.created) { earlier_event = std::cmp::min(earlier_event, expires); } if let Some(next_event) = &mut next_event { if earlier_event < *next_event { *next_event = earlier_event; } } else { next_event = Some(earlier_event); } } } next_event } pub fn next_delivery_event(&self, queue: Option) -> Option { let mut next_delivery = None; for rcpt in self.recipients.iter().filter(|rcpt| { matches!(rcpt.status, Status::Scheduled | Status::TemporaryFailure(_)) && queue.is_none_or(|q| rcpt.queue == q) }) { if let Some(next_delivery) = &mut next_delivery { if rcpt.retry.due < *next_delivery { *next_delivery = rcpt.retry.due; } } else { next_delivery = Some(rcpt.retry.due); } } next_delivery } pub fn next_dsn(&self, queue: Option) -> Option { let mut next_dsn = None; for rcpt in self.recipients.iter().filter(|rcpt| { matches!(rcpt.status, Status::Scheduled | Status::TemporaryFailure(_)) && queue.is_none_or(|q| rcpt.queue == q) }) { if let Some(next_dsn) = &mut next_dsn { if rcpt.notify.due < *next_dsn { *next_dsn = rcpt.notify.due; } } else { next_dsn = Some(rcpt.notify.due); } } next_dsn } pub fn expires(&self, queue: Option) -> Option { let mut expires = None; for rcpt in self.recipients.iter().filter(|d| { matches!(d.status, Status::Scheduled | Status::TemporaryFailure(_)) && queue.is_none_or(|q| d.queue == q) }) { if let Some(rcpt_expires) = rcpt.expiration_time(self.created) { if let Some(expires) = &mut expires { if rcpt_expires > *expires { *expires = rcpt_expires; } } else { expires = Some(rcpt_expires) } } } expires } pub fn next_events(&self) -> AHashMap { let mut next_events = AHashMap::new(); for rcpt in &self.recipients { if matches!(rcpt.status, Status::Scheduled | Status::TemporaryFailure(_)) { let mut earlier_event = std::cmp::min(rcpt.retry.due, rcpt.notify.due); if let Some(expires) = rcpt.expiration_time(self.created) { earlier_event = std::cmp::min(earlier_event, expires); } match next_events.entry(rcpt.queue) { Entry::Occupied(mut entry) => { let entry = entry.get_mut(); if earlier_event < *entry { *entry = earlier_event; } } Entry::Vacant(entry) => { entry.insert(earlier_event); } } } } next_events } } impl Recipient { pub fn expiration_time(&self, created: u64) -> Option { match self.expires { QueueExpiry::Ttl(time) => Some(created + time), QueueExpiry::Attempts(_) => None, } } pub fn is_expired(&self, created: u64, now: u64) -> bool { match self.expires { QueueExpiry::Ttl(time) => created + time <= now, QueueExpiry::Attempts(count) => self.retry.inner >= count, } } } pub trait SpawnQueue { fn spawn(self, core: Arc); } impl QueueStats { fn new(max_in_flight: usize) -> Self { QueueStats { in_flight: 0, max_in_flight, last_warning: Instant::now() - BACK_PRESSURE_WARN_INTERVAL, } } #[inline] pub fn has_capacity(&self) -> bool { self.in_flight < self.max_in_flight } } ================================================ FILE: crates/smtp/src/queue/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{ config::smtp::queue::{QueueExpiry, QueueName}, expr::{self, functions::ResolveVariable, *}, }; use compact_str::ToCompactString; use smtp_proto::Response; use std::{ fmt::Display, net::{IpAddr, Ipv4Addr}, time::{Duration, Instant, SystemTime}, }; use store::write::now; use types::blob_hash::BlobHash; use utils::DomainPart; pub mod dsn; pub mod manager; pub mod quota; pub mod spool; pub mod throttle; pub type QueueId = u64; #[derive(Debug, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, serde::Deserialize)] pub struct Schedule { pub due: u64, pub inner: T, } #[derive(Debug, Clone, Copy)] pub struct QueuedMessage { pub due: u64, pub queue_id: QueueId, pub queue_name: QueueName, } #[derive(Debug, Clone, Copy)] pub enum MessageSource { Authenticated, Unauthenticated { dmarc_pass: bool, train_spam: Option, }, Dsn, Report, Autogenerated, } #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Clone, PartialEq, Eq)] pub struct Message { pub created: u64, pub blob_hash: BlobHash, pub return_path: Box, pub recipients: Vec, pub received_from_ip: IpAddr, pub received_via_port: u16, pub flags: u64, pub env_id: Option>, pub priority: i16, pub size: u64, pub quota_keys: Box<[QuotaKey]>, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct MessageWrapper { pub queue_id: QueueId, pub queue_name: QueueName, pub is_multi_queue: bool, pub span_id: u64, pub message: Message, } #[derive( rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Clone, PartialEq, Eq, serde::Deserialize, )] pub enum QuotaKey { Size { key: Box<[u8]>, id: u64 }, Count { key: Box<[u8]>, id: u64 }, } #[derive( rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Clone, PartialEq, Eq, serde::Deserialize, )] pub struct Recipient { pub address: Box, pub retry: Schedule, pub notify: Schedule, pub expires: QueueExpiry, pub queue: QueueName, pub status: Status>, ErrorDetails>, pub flags: u64, pub orcpt: Option>, } pub const FROM_AUTHENTICATED: u64 = 1 << 32; pub const FROM_UNAUTHENTICATED: u64 = 1 << 33; pub const FROM_UNAUTHENTICATED_DMARC: u64 = 1 << 34; pub const FROM_DSN: u64 = 1 << 35; pub const FROM_REPORT: u64 = 1 << 36; pub const FROM_AUTOGENERATED: u64 = 1 << 37; pub const RCPT_DSN_SENT: u64 = 1 << 32; //pub const RCPT_STATUS_CHANGED: u64 = 1 << 33; pub const RCPT_SPAM_PAYLOAD: u64 = 1 << 34; #[derive( Debug, Clone, PartialEq, Eq, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, serde::Serialize, serde::Deserialize, )] pub enum Status { #[serde(rename = "scheduled")] Scheduled, #[serde(rename = "completed")] Completed(T), #[serde(rename = "temp_fail")] TemporaryFailure(E), #[serde(rename = "perm_fail")] PermanentFailure(E), } #[derive( Debug, Clone, PartialEq, Eq, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, serde::Deserialize, )] pub struct HostResponse { pub hostname: T, pub response: Response>, } #[derive( Debug, Clone, PartialEq, Eq, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, serde::Deserialize, Default, )] pub enum Error { DnsError(Box), UnexpectedResponse(UnexpectedResponse), ConnectionError(Box), TlsError(Box), DaneError(Box), MtaStsError(Box), RateLimited, #[default] ConcurrencyLimited, Io(Box), } #[derive( Debug, Clone, PartialEq, Eq, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, serde::Deserialize, )] pub struct UnexpectedResponse { pub command: Box, pub response: Response>, } #[derive( Debug, Clone, PartialEq, Eq, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Default, serde::Deserialize, )] pub struct ErrorDetails { pub entity: Box, pub details: Error, } impl Ord for Schedule { fn cmp(&self, other: &Self) -> std::cmp::Ordering { other.due.cmp(&self.due) } } impl PartialOrd for Schedule { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } impl PartialEq for Schedule { fn eq(&self, other: &Self) -> bool { self.due == other.due } } impl Eq for Schedule {} impl Schedule { pub fn now() -> Self { Schedule { due: now(), inner: T::default(), } } pub fn later(duration: u64) -> Self { Schedule { due: now() + duration, inner: T::default(), } } } pub struct QueueEnvelope<'x> { pub message: &'x Message, pub domain: &'x str, pub mx: &'x str, pub rcpt: &'x Recipient, pub remote_ip: IpAddr, pub local_ip: IpAddr, } impl<'x> QueueEnvelope<'x> { pub fn new(message: &'x Message, rcpt: &'x Recipient) -> Self { Self { message, domain: rcpt.address.domain_part(), rcpt, mx: "", remote_ip: IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), local_ip: IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), } } } impl<'x> ResolveVariable for QueueEnvelope<'x> { fn resolve_variable(&self, variable: u32) -> expr::Variable<'x> { match variable { V_SENDER => self.message.return_path.as_ref().into(), V_SENDER_DOMAIN => self.message.return_path.domain_part().into(), V_RECIPIENT_DOMAIN => self.domain.into(), V_RECIPIENT => self.rcpt.address.as_ref().into(), V_RECIPIENTS => self .message .recipients .iter() .map(|r| Variable::from(r.address.as_ref())) .collect::>() .into(), V_QUEUE_RETRY_NUM => self.rcpt.retry.inner.into(), V_QUEUE_NOTIFY_NUM => self.rcpt.notify.inner.into(), V_QUEUE_EXPIRES_IN => match &self.rcpt.expires { QueueExpiry::Ttl(time) => (*time + self.message.created).saturating_sub(now()), QueueExpiry::Attempts(count) => { (count.saturating_sub(self.rcpt.retry.inner)) as u64 } } .into(), V_QUEUE_LAST_STATUS => self.rcpt.status.to_compact_string().into(), V_QUEUE_LAST_ERROR => match &self.rcpt.status { Status::Scheduled | Status::Completed(_) => "none", Status::TemporaryFailure(err) | Status::PermanentFailure(err) => { match &err.details { Error::DnsError(_) => "dns", Error::UnexpectedResponse(_) => "unexpected-reply", Error::ConnectionError(_) => "connection", Error::TlsError(_) => "tls", Error::DaneError(_) => "dane", Error::MtaStsError(_) => "mta-sts", Error::RateLimited => "rate", Error::ConcurrencyLimited => "concurrency", Error::Io(_) => "io", } } } .into(), V_QUEUE_NAME => self.rcpt.queue.as_str().into(), V_QUEUE_AGE => now().saturating_sub(self.message.created).into(), V_SOURCE => if (self.message.flags & FROM_AUTHENTICATED) != 0 { "authenticated" } else if (self.message.flags & FROM_UNAUTHENTICATED_DMARC) != 0 { "dmarc_pass" } else if (self.message.flags & FROM_UNAUTHENTICATED) != 0 { "unauthenticated" } else if (self.message.flags & FROM_DSN) != 0 { "dsn" } else if (self.message.flags & FROM_REPORT) != 0 { "report" } else if (self.message.flags & FROM_AUTOGENERATED) != 0 { "autogenerated" } else { "unknown" } .into(), V_MX => self.mx.into(), V_PRIORITY => self.message.priority.into(), V_REMOTE_IP => self.remote_ip.to_compact_string().into(), V_LOCAL_IP => self.local_ip.to_compact_string().into(), V_RECEIVED_FROM_IP => self.message.received_from_ip.to_compact_string().into(), V_RECEIVED_VIA_PORT => self.message.received_via_port.into(), V_SIZE => self.message.size.into(), _ => "".into(), } } fn resolve_global(&self, _: &str) -> Variable<'_> { Variable::Integer(0) } } impl ResolveVariable for Message { fn resolve_variable(&self, variable: u32) -> expr::Variable<'_> { match variable { V_SENDER => self.return_path.as_ref().into(), V_SENDER_DOMAIN => self.return_path.domain_part().into(), V_RECIPIENTS => self .recipients .iter() .map(|r| Variable::from(r.address.as_ref())) .collect::>() .into(), V_PRIORITY => self.priority.into(), _ => "".into(), } } fn resolve_global(&self, _: &str) -> Variable<'_> { Variable::Integer(0) } } pub struct RecipientDomain<'x>(&'x str); impl<'x> RecipientDomain<'x> { pub fn new(domain: &'x str) -> Self { Self(domain) } } impl<'x> ResolveVariable for RecipientDomain<'x> { fn resolve_variable(&self, variable: u32) -> expr::Variable<'x> { match variable { V_RECIPIENT_DOMAIN => self.0.into(), _ => "".into(), } } fn resolve_global(&self, _: &str) -> Variable<'_> { Variable::Integer(0) } } #[inline(always)] pub fn instant_to_timestamp(now: Instant, time: Instant) -> u64 { SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()) + time.checked_duration_since(now).map_or(0, |d| d.as_secs()) } impl Recipient { pub fn new(address: impl AsRef) -> Self { Recipient { address: address.to_lowercase_domain().into_boxed_str(), status: Status::Scheduled, flags: 0, orcpt: None, retry: Schedule::now(), notify: Schedule::now(), expires: QueueExpiry::Attempts(0), queue: QueueName::default(), } } pub fn with_flags(mut self, flags: u64) -> Self { self.flags = flags; self } pub fn with_orcpt(mut self, orcpt: Option>) -> Self { self.orcpt = orcpt; self } pub fn address(&self) -> &str { &self.address } pub fn domain_part(&self) -> &str { self.address.domain_part() } } impl ArchivedRecipient { pub fn address(&self) -> &str { self.address.as_ref() } pub fn domain_part(&self) -> &str { self.address.domain_part() } } pub trait InstantFromTimestamp { fn to_instant(&self) -> Instant; } impl InstantFromTimestamp for u64 { fn to_instant(&self) -> Instant { let timestamp = *self; let current_timestamp = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()); if timestamp > current_timestamp { Instant::now() + Duration::from_secs(timestamp - current_timestamp) } else { Instant::now() } } } impl Display for Error { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Error::UnexpectedResponse(response) => { write!( f, "Unexpected response for {}: {}", response.command, response.response ) } Error::DnsError(err) => { write!(f, "DNS lookup failed: {err}") } Error::ConnectionError(details) => { write!(f, "Connection failed: {details}",) } Error::TlsError(details) => { write!(f, "TLS error: {details}",) } Error::DaneError(details) => { write!(f, "DANE authentication failure: {details}",) } Error::MtaStsError(details) => { write!(f, "MTA-STS auth failed: {details}") } Error::RateLimited => { write!(f, "Rate limited") } Error::ConcurrencyLimited => { write!(f, "Too many concurrent connections to remote server") } Error::Io(err) => { write!(f, "Queue error: {err}") } } } } impl Display for ArchivedError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { ArchivedError::UnexpectedResponse(response) => { write!( f, "Unexpected response for {}: {}", response.command, response.response ) } ArchivedError::DnsError(err) => { write!(f, "DNS lookup failed: {err}") } ArchivedError::ConnectionError(details) => { write!(f, "Connection failed: {details}",) } ArchivedError::TlsError(details) => { write!(f, "TLS error: {details}",) } ArchivedError::DaneError(details) => { write!(f, "DANE authentication failure: {details}",) } ArchivedError::MtaStsError(details) => { write!(f, "MTA-STS auth failed: {details}") } ArchivedError::RateLimited => { write!(f, "Rate limited") } ArchivedError::ConcurrencyLimited => { write!(f, "Too many concurrent connections to remote server") } ArchivedError::Io(err) => { write!(f, "Queue error: {err}") } } } } impl Display for Status>, ErrorDetails> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Status::Scheduled => write!(f, "Scheduled"), Status::Completed(response) => write!(f, "Delivered: {}", response.response), Status::TemporaryFailure(err) => { write!(f, "Temporary Failure for {}: {}", err.entity, err.details) } Status::PermanentFailure(err) => { write!(f, "Permanent Failure for {}: {}", err.entity, err.details) } } } } impl Display for ArchivedErrorDetails { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "Error for {}: {}", self.entity, self.details) } } /* pub trait DisplayArchivedResponse { fn to_string(&self) -> String; } impl DisplayArchivedResponse for ArchivedResponse> { fn to_string(&self) -> String { format!( "Code: {}, Enhanced code: {}.{}.{}, Message: {}", self.code, self.esc[0], self.esc[1], self.esc[2], self.message, ) } } */ ================================================ FILE: crates/smtp/src/queue/quota.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{QueueEnvelope, QuotaKey, Status}; use crate::{core::throttle::NewKey, queue::MessageWrapper}; use ahash::AHashSet; use common::{Server, config::smtp::queue::QueueQuota, expr::functions::ResolveVariable}; use std::future::Future; use store::{ ValueKey, write::{BatchBuilder, QueueClass, ValueClass}, }; use trc::QueueEvent; use utils::DomainPart; pub trait HasQueueQuota: Sync + Send { fn has_quota(&self, message: &mut MessageWrapper) -> impl Future + Send; fn check_quota<'x>( &'x self, quota: &'x QueueQuota, envelope: &impl ResolveVariable, size: u64, id: u64, refs: &mut Vec, session_id: u64, ) -> impl Future + Send; } impl HasQueueQuota for Server { async fn has_quota(&self, message: &mut MessageWrapper) -> bool { let mut quota_keys = Vec::new(); if !self.core.smtp.queue.quota.sender.is_empty() { for quota in &self.core.smtp.queue.quota.sender { if !self .check_quota( quota, &message.message, message.message.size, 0, &mut quota_keys, message.span_id, ) .await { trc::event!( Queue(QueueEvent::QuotaExceeded), SpanId = message.span_id, Id = quota.id.clone(), Type = "Sender" ); return false; } } } if !self.core.smtp.queue.quota.rcpt_domain.is_empty() { let mut seen_domains = AHashSet::new(); for quota in &self.core.smtp.queue.quota.rcpt_domain { for (rcpt_idx, rcpt) in message.message.recipients.iter().enumerate() { if seen_domains.insert(rcpt.address.domain_part()) && !self .check_quota( quota, &QueueEnvelope::new(&message.message, rcpt), message.message.size, ((rcpt_idx + 1) << 32) as u64, &mut quota_keys, message.span_id, ) .await { trc::event!( Queue(QueueEvent::QuotaExceeded), SpanId = message.span_id, Id = quota.id.clone(), Type = "Domain" ); return false; } } } } for quota in &self.core.smtp.queue.quota.rcpt { for (rcpt_idx, rcpt) in message.message.recipients.iter().enumerate() { if !self .check_quota( quota, &QueueEnvelope::new(&message.message, rcpt), message.message.size, (rcpt_idx + 1) as u64, &mut quota_keys, message.span_id, ) .await { trc::event!( Queue(QueueEvent::QuotaExceeded), SpanId = message.span_id, Id = quota.id.clone(), Type = "Recipient" ); return false; } } } message.message.quota_keys = quota_keys.into_boxed_slice(); true } async fn check_quota<'x>( &'x self, quota: &'x QueueQuota, envelope: &impl ResolveVariable, size: u64, id: u64, refs: &mut Vec, session_id: u64, ) -> bool { if !quota.expr.is_empty() && self .eval_expr("a.expr, envelope, "check_quota", session_id) .await .unwrap_or(false) { let key = quota.new_key(envelope, ""); if let Some(max_size) = quota.size { let used_size = self .core .storage .data .get_counter(ValueKey::from(ValueClass::Queue(QueueClass::QuotaSize( key.as_ref().to_vec(), )))) .await .unwrap_or(0) as u64; if used_size + size > max_size { return false; } else { refs.push(QuotaKey::Size { key: key.as_ref().into(), id, }); } } if let Some(max_messages) = quota.messages { let total_messages = self .core .storage .data .get_counter(ValueKey::from(ValueClass::Queue(QueueClass::QuotaCount( key.as_ref().to_vec(), )))) .await .unwrap_or(0) as u64; if total_messages + 1 > max_messages { return false; } else { refs.push(QuotaKey::Count { key: key.as_ref().into(), id, }); } } } true } } impl MessageWrapper { pub fn release_quota(&mut self, batch: &mut BatchBuilder) { if self.message.quota_keys.is_empty() { return; } let mut quota_ids = Vec::with_capacity(self.message.recipients.len()); let mut seen_domains = AHashSet::new(); for (pos, rcpt) in self.message.recipients.iter().enumerate() { if matches!( &rcpt.status, Status::Completed(_) | Status::PermanentFailure(_) ) { if seen_domains.insert(rcpt.address.domain_part()) { quota_ids.push(((pos + 1) as u64) << 32); } quota_ids.push((pos + 1) as u64); } } if !quota_ids.is_empty() { let mut quota_keys = Vec::new(); for quota_key in std::mem::take(&mut self.message.quota_keys) { match quota_key { QuotaKey::Count { id, key } if quota_ids.contains(&id) => { batch.add( ValueClass::Queue(QueueClass::QuotaCount(key.into_vec())), -1, ); } QuotaKey::Size { id, key } if quota_ids.contains(&id) => { batch.add( ValueClass::Queue(QueueClass::QuotaSize(key.into_vec())), -(self.message.size as i64), ); } _ => { quota_keys.push(quota_key); } } } self.message.quota_keys = quota_keys.into_boxed_slice(); } } } ================================================ FILE: crates/smtp/src/queue/spool.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ ArchivedMessage, ArchivedStatus, Message, MessageSource, QueueEnvelope, QueueId, QueuedMessage, QuotaKey, Recipient, Schedule, Status, }; use crate::queue::manager::{LockedMessage, Queue}; use crate::queue::{ FROM_AUTHENTICATED, FROM_AUTOGENERATED, FROM_DSN, FROM_REPORT, FROM_UNAUTHENTICATED, FROM_UNAUTHENTICATED_DMARC, MessageWrapper, }; use common::config::smtp::queue::QueueName; use common::ipc::QueueEvent; use common::{KV_LOCK_QUEUE_MESSAGE, Server}; use std::borrow::Cow; use std::collections::hash_map::Entry; use std::future::Future; use std::net::{IpAddr, Ipv4Addr}; use std::time::SystemTime; use store::write::key::DeserializeBigEndian; use store::write::serialize::rkyv_deserialize; use store::write::{ AlignedBytes, Archive, Archiver, BatchBuilder, BlobLink, BlobOp, MergeResult, Params, QueueClass, ValueClass, now, }; use store::{Deserialize, IterateParams, Serialize, U64_LEN, ValueKey}; use trc::{AddContext, ServerEvent, SpamEvent}; use types::blob_hash::BlobHash; use utils::DomainPart; pub const LOCK_EXPIRY: u64 = 10 * 60; // 10 minutes pub const QUEUE_REFRESH: u64 = 5 * 60; // 5 minutes const INFINITE_LOCK: u64 = 60 * 60 * 24 * 365; // 1 year pub struct QueuedMessages { pub messages: Vec, pub next_refresh: u64, } pub trait SmtpSpool: Sync + Send { fn new_message(&self, return_path: impl AsRef, span_id: u64) -> MessageWrapper; fn next_event(&self, queue: &mut Queue) -> impl Future + Send; fn try_lock_event( &self, queue_id: QueueId, queue_name: QueueName, ) -> impl Future + Send; fn unlock_event( &self, queue_id: QueueId, queue_name: QueueName, ) -> impl Future + Send; fn read_message( &self, id: QueueId, queue_name: QueueName, ) -> impl Future> + Send; fn read_message_archive( &self, id: QueueId, ) -> impl Future>>> + Send; } impl SmtpSpool for Server { fn new_message(&self, return_path: impl AsRef, span_id: u64) -> MessageWrapper { let created = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()); MessageWrapper { queue_id: self.inner.data.queue_id_gen.generate(), queue_name: QueueName::default(), is_multi_queue: false, span_id, message: Message { created, return_path: return_path.to_lowercase_domain().into_boxed_str(), recipients: Vec::with_capacity(1), flags: 0, env_id: None, priority: 0, size: 0, blob_hash: Default::default(), quota_keys: Default::default(), received_from_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), received_via_port: 0, }, } } async fn next_event(&self, queue: &mut Queue) -> QueuedMessages { let now = now(); let from_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( store::write::QueueEvent { due: 0, queue_id: 0, queue_name: [0; 8], }, ))); let to_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( store::write::QueueEvent { due: now + QUEUE_REFRESH, queue_id: u64::MAX, queue_name: [u8::MAX; 8], }, ))); let mut events = QueuedMessages { messages: Vec::new(), next_refresh: now + QUEUE_REFRESH, }; queue.locked_revision += 1; let result = self .store() .iterate( IterateParams::new(from_key, to_key).ascending().no_values(), |key, _| { let due = key.deserialize_be_u64(0)?; if due <= now { let queue_id = key.deserialize_be_u64(U64_LEN)?; let queue_name = key .get(U64_LEN + U64_LEN..) .and_then(QueueName::from_bytes) .ok_or_else(|| { trc::StoreEvent::DataCorruption .caused_by(trc::location!()) .ctx(trc::Key::Key, key) })?; let add_event = queue .stats .get(&queue_name) .is_none_or(|stats| stats.has_capacity()) && match queue.locked.entry((queue_id, queue_name)) { Entry::Occupied(mut entry) => { let locked = entry.get_mut(); locked.revision = queue.locked_revision; if locked.expires <= now { locked.expires = now + INFINITE_LOCK; true } else { if locked.expires < events.next_refresh { events.next_refresh = locked.expires; } false } } Entry::Vacant(entry) => { entry.insert(LockedMessage { expires: now + INFINITE_LOCK, revision: queue.locked_revision, }); true } }; if add_event { events.messages.push(QueuedMessage { due, queue_id, queue_name, }); } Ok(true) } else { if due < events.next_refresh { events.next_refresh = due; } Ok(false) } }, ) .await; if let Err(err) = result { trc::error!( err.details("Failed to read queue.") .caused_by(trc::location!()) ); } events } async fn try_lock_event(&self, queue_id: QueueId, queue_name: QueueName) -> bool { match self .in_memory_store() .try_lock( KV_LOCK_QUEUE_MESSAGE, &lock_id(queue_id, queue_name), LOCK_EXPIRY, ) .await { Ok(result) => { if !result { trc::event!( Queue(trc::QueueEvent::Locked), QueueId = queue_id, QueueName = queue_name.to_string() ); } result } Err(err) => { trc::error!( err.details("Failed to lock event.") .caused_by(trc::location!()) ); false } } } async fn unlock_event(&self, queue_id: QueueId, queue_name: QueueName) { if let Err(err) = self .in_memory_store() .remove_lock(KV_LOCK_QUEUE_MESSAGE, &lock_id(queue_id, queue_name)) .await { trc::error!( err.details("Failed to unlock event.") .caused_by(trc::location!()) ); } } async fn read_message( &self, queue_id: QueueId, queue_name: QueueName, ) -> Option { match self .read_message_archive(queue_id) .await .and_then(|a| match a { Some(a) => a.deserialize::().map(Some), None => Ok(None), }) { Ok(Some(message)) => Some(MessageWrapper { is_multi_queue: message.recipients.iter().any(|rcpt| { matches!(rcpt.status, Status::Scheduled | Status::TemporaryFailure(_)) && rcpt.queue != queue_name }), queue_id, queue_name, span_id: 0, message, }), Ok(None) => None, Err(err) => { trc::error!( err.details("Failed to read message.") .caused_by(trc::location!()) ); None } } } async fn read_message_archive( &self, id: QueueId, ) -> trc::Result>> { self.store() .get_value::>(ValueKey::from(ValueClass::Queue( QueueClass::Message(id), ))) .await } } fn lock_id(queue_id: QueueId, queue_name: QueueName) -> [u8; 16] { let mut id = [0; 16]; id[..8].copy_from_slice(&queue_id.to_be_bytes()); id[8..].copy_from_slice(queue_name.as_ref()); id } impl MessageWrapper { pub async fn queue( mut self, raw_headers: Option<&[u8]>, raw_message: &[u8], session_id: u64, server: &Server, source: MessageSource, ) -> bool { // Set flags let (flags, event, train_spam) = match source { MessageSource::Authenticated => ( FROM_AUTHENTICATED, trc::QueueEvent::QueueMessageAuthenticated, None, ), MessageSource::Unauthenticated { dmarc_pass: true, train_spam, } => ( FROM_UNAUTHENTICATED_DMARC, trc::QueueEvent::QueueMessage, train_spam, ), MessageSource::Unauthenticated { dmarc_pass: false, train_spam, } => ( FROM_UNAUTHENTICATED, trc::QueueEvent::QueueMessage, train_spam, ), MessageSource::Dsn => (FROM_DSN, trc::QueueEvent::QueueDsn, None), MessageSource::Report => (FROM_REPORT, trc::QueueEvent::QueueReport, None), MessageSource::Autogenerated => ( FROM_AUTOGENERATED, trc::QueueEvent::QueueAutogenerated, None, ), }; self.message.flags |= flags; // Write blob let message = if let Some(raw_headers) = raw_headers { let mut message = Vec::with_capacity(raw_headers.len() + raw_message.len()); message.extend_from_slice(raw_headers); message.extend_from_slice(raw_message); Cow::Owned(message) } else { raw_message.into() }; self.message.blob_hash = BlobHash::generate(message.as_ref()); // Generate id if self.message.size == 0 { self.message.size = message.len() as u64; } // Reserve and write blob let mut batch = BatchBuilder::new(); let now = now(); let reserve_until = now + 120; batch.set( BlobOp::Link { hash: self.message.blob_hash.clone(), to: BlobLink::Temporary { until: reserve_until, }, }, vec![], ); if let Err(err) = server.store().write(batch.build_all()).await { trc::error!( err.details("Failed to write to store.") .span_id(session_id) .caused_by(trc::location!()) ); return false; } if let Err(err) = server .blob_store() .put_blob(self.message.blob_hash.as_slice(), message.as_ref()) .await { trc::error!( err.details("Failed to write blob.") .span_id(session_id) .caused_by(trc::location!()) ); return false; } trc::event!( Queue(event), SpanId = session_id, QueueId = self.queue_id, From = if !self.message.return_path.is_empty() { trc::Value::String(self.message.return_path.as_ref().into()) } else { trc::Value::String("<>".into()) }, To = self .message .recipients .iter() .map(|r| trc::Value::String(r.address.as_ref().into())) .collect::>(), Size = self.message.size, NextRetry = self .message .next_delivery_event(None) .map(trc::Value::Timestamp), NextDsn = self.message.next_dsn(None).map(trc::Value::Timestamp), Expires = self.message.expires(None).map(trc::Value::Timestamp), ); // Write message to queue let mut batch = BatchBuilder::new(); // Reserve quotas for quota_key in &self.message.quota_keys { match quota_key { QuotaKey::Count { key, .. } => { batch.add(ValueClass::Queue(QueueClass::QuotaCount(key.to_vec())), 1); } QuotaKey::Size { key, .. } => { batch.add( ValueClass::Queue(QueueClass::QuotaSize(key.to_vec())), self.message.size as i64, ); } } } for (queue_name, due) in self.message.next_events() { batch.set( ValueClass::Queue(QueueClass::MessageEvent(store::write::QueueEvent { due, queue_id: self.queue_id, queue_name: queue_name.into_inner(), })), Vec::new(), ); } if let Some(is_spam) = train_spam && let Some(config) = &server.core.spam.classifier { let hold_period = now + config.hold_samples_for; batch .set( BlobOp::Link { hash: self.message.blob_hash.clone(), to: BlobLink::Temporary { until: hold_period }, }, vec![BlobLink::SPAM_SAMPLE_LINK], ) .set( BlobOp::SpamSample { hash: self.message.blob_hash.clone(), until: hold_period, }, vec![u8::from(is_spam), 1], ); trc::event!( Spam(SpamEvent::TrainSampleAdded), Details = if is_spam { "spam" } else { "ham" }, Expires = trc::Value::Timestamp(hold_period), SpanId = self.span_id, ); } batch .clear(BlobOp::Link { hash: self.message.blob_hash.clone(), to: BlobLink::Temporary { until: reserve_until, }, }) .set( BlobOp::Link { hash: self.message.blob_hash.clone(), to: BlobLink::Id { id: self.queue_id }, }, vec![], ) .set( BlobOp::Commit { hash: self.message.blob_hash.clone(), }, vec![], ) .set( ValueClass::Queue(QueueClass::Message(self.queue_id)), match Archiver::new(self.message).serialize() { Ok(data) => data, Err(err) => { trc::error!( err.details("Failed to serialize message.") .span_id(session_id) .caused_by(trc::location!()) ); return false; } }, ); if let Err(err) = server.store().write(batch.build_all()).await { trc::error!( err.details("Failed to write to store.") .span_id(session_id) .caused_by(trc::location!()) ); return false; } // Queue the message if server .inner .ipc .queue_tx .send(QueueEvent::Refresh) .await .is_err() { trc::event!( Server(ServerEvent::ThreadError), Reason = "Channel closed.", CausedBy = trc::location!(), SpanId = session_id, ); } true } pub async fn add_recipient(&mut self, rcpt: impl AsRef, server: &Server) { // Resolve queue self.message.recipients.push(Recipient::new(rcpt)); let queue = server.get_queue_or_default( &server .eval_if::( &server.core.smtp.queue.queue, &QueueEnvelope::new(&self.message, self.message.recipients.last().unwrap()), self.span_id, ) .await .unwrap_or_else(|| "default".to_string()), self.span_id, ); // Update expiration let now = now(); let recipient = self.message.recipients.last_mut().unwrap(); recipient.notify = Schedule::later(queue.notify.first().copied().unwrap_or(86400) + now); recipient.expires = queue.expiry; recipient.queue = queue.virtual_queue; } pub async fn save_changes(mut self, server: &Server, prev_event: Option) -> bool { // Release quota for completed deliveries let mut batch = BatchBuilder::new(); self.release_quota(&mut batch); // Update message queue if let Some(prev_event) = prev_event { batch.clear(ValueClass::Queue(QueueClass::MessageEvent( store::write::QueueEvent { due: prev_event, queue_id: self.queue_id, queue_name: self.queue_name.into_inner(), }, ))); } for (queue_name, due) in self.message.next_events() { batch.set( ValueClass::Queue(QueueClass::MessageEvent(store::write::QueueEvent { due, queue_id: self.queue_id, queue_name: queue_name.into_inner(), })), Vec::new(), ); } let message_bytes = match Archiver::new(self.message).serialize() { Ok(data) => data, Err(err) => { trc::error!( err.details("Failed to serialize message.") .span_id(self.span_id) .caused_by(trc::location!()) ); return false; } }; if self.is_multi_queue { batch.merge_fnc( ValueClass::Queue(QueueClass::Message(self.queue_id)), Params::with_capacity(3) .with_u64(self.queue_id) .with_bytes(self.queue_name.into_inner().to_vec()) .with_bytes(message_bytes), |params, _, bytes| { let mut cur_message = as Deserialize>::deserialize( bytes.ok_or_else(|| { trc::StoreEvent::NotFound .into_err() .details("Message no longer exists.") .caused_by(trc::location!()) .ctx(trc::Key::QueueId, params.u64(0)) })?, ) .and_then(|archive| archive.deserialize::()) .caused_by(trc::location!())?; let new_message_ = as Deserialize>::deserialize(params.bytes(2)) .caused_by(trc::location!())?; let new_message = new_message_ .unarchive::() .caused_by(trc::location!())?; if cur_message.blob_hash.as_slice() == new_message.blob_hash.0.as_slice() && cur_message.recipients.len() == new_message.recipients.len() { let queue_name = params.bytes(1); for (rcpt_idx, rcpt) in new_message .recipients .iter() .enumerate() .filter(|(_, rcpt)| rcpt.queue.as_slice() == queue_name) { cur_message.recipients[rcpt_idx] = rkyv_deserialize(rcpt).caused_by(trc::location!())?; } Archiver::new(cur_message) .serialize() .caused_by(trc::location!()) .map(MergeResult::Update) } else { Err(trc::StoreEvent::UnexpectedError .into_err() .details("Message blob hash or recipient count mismatch.") .caused_by(trc::location!()) .ctx(trc::Key::QueueId, params.u64(0))) } }, ); } else { batch.set( ValueClass::Queue(QueueClass::Message(self.queue_id)), message_bytes, ); } if let Err(err) = server.store().write(batch.build_all()).await { trc::error!( err.details("Failed to save changes.") .span_id(self.span_id) .caused_by(trc::location!()) ); false } else { true } } pub async fn remove(self, server: &Server, prev_event: Option) -> bool { let mut batch = BatchBuilder::new(); if let Some(prev_event) = prev_event { batch.clear(ValueClass::Queue(QueueClass::MessageEvent( store::write::QueueEvent { due: prev_event, queue_id: self.queue_id, queue_name: self.queue_name.into_inner(), }, ))); } else { for (queue_name, due) in self.message.next_events() { batch.clear(ValueClass::Queue(QueueClass::MessageEvent( store::write::QueueEvent { due, queue_id: self.queue_id, queue_name: queue_name.into_inner(), }, ))); } } // Release all quotas for quota_key in self.message.quota_keys { match quota_key { QuotaKey::Count { key, .. } => { batch.add(ValueClass::Queue(QueueClass::QuotaCount(key.to_vec())), -1); } QuotaKey::Size { key, .. } => { batch.add( ValueClass::Queue(QueueClass::QuotaSize(key.to_vec())), -(self.message.size as i64), ); } } } batch .clear(BlobOp::Link { hash: self.message.blob_hash.clone(), to: BlobLink::Id { id: self.queue_id }, }) .clear(ValueClass::Queue(QueueClass::Message(self.queue_id))); if let Err(err) = server.store().write(batch.build_all()).await { trc::error!( err.details("Failed to write to update queue.") .span_id(self.span_id) .caused_by(trc::location!()) ); false } else { true } } pub fn has_domain(&self, domains: &[String]) -> bool { self.message.recipients.iter().any(|r| { let domain = r.address.domain_part(); domains.iter().any(|dd| dd == domain) }) || self .message .return_path .rsplit_once('@') .is_some_and(|(_, domain)| domains.iter().any(|dd| dd == domain)) } } impl ArchivedMessage { pub fn has_domain(&self, domains: &[String]) -> bool { self.recipients.iter().any(|r| { let domain = r.address.domain_part(); domains.iter().any(|dd| dd == domain) }) || self .return_path .rsplit_once('@') .is_some_and(|(_, domain)| domains.iter().any(|dd| dd == domain)) } pub fn next_delivery_event(&self, queue: Option) -> Option { let mut next_delivery = None; for rcpt in self.recipients.iter().filter(|d| { matches!( d.status, ArchivedStatus::Scheduled | ArchivedStatus::TemporaryFailure(_) ) && queue.is_none_or(|q| d.queue == q) }) { let retry_due = rcpt.retry.due.to_native(); if let Some(next_delivery) = &mut next_delivery { if retry_due < *next_delivery { *next_delivery = retry_due; } } else { next_delivery = Some(retry_due); } } next_delivery } } ================================================ FILE: crates/smtp/src/queue/throttle.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::core::throttle::NewKey; use common::{ KV_RATE_LIMIT_SMTP, Server, config::smtp::QueueRateLimiter, expr::functions::ResolveVariable, }; use std::future::Future; use store::write::now; pub trait IsAllowed: Sync + Send { fn is_allowed<'x>( &'x self, throttle: &'x QueueRateLimiter, envelope: &impl ResolveVariable, session_id: u64, ) -> impl Future> + Send; } impl IsAllowed for Server { async fn is_allowed<'x>( &'x self, throttle: &'x QueueRateLimiter, envelope: &impl ResolveVariable, session_id: u64, ) -> Result<(), u64> { if throttle.expr.is_empty() || self .eval_expr(&throttle.expr, envelope, "throttle", session_id) .await .unwrap_or(false) { let key = throttle.new_key(envelope, "outbound"); match self .core .storage .lookup .is_rate_allowed(KV_RATE_LIMIT_SMTP, key.as_ref(), &throttle.rate, false) .await { Ok(Some(next_refill)) => { trc::event!( Queue(trc::QueueEvent::RateLimitExceeded), SpanId = session_id, Id = throttle.id.clone(), Limit = vec![ trc::Value::from(throttle.rate.requests), trc::Value::from(throttle.rate.period) ], ); return Err(now() + next_refill); } Err(err) => { trc::error!(err.span_id(session_id).caused_by(trc::location!())); } _ => (), } } Ok(()) } } ================================================ FILE: crates/smtp/src/reporting/analysis.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use ahash::AHashMap; use common::Server; use mail_auth::{ flate2::read::GzDecoder, report::{ActionDisposition, DmarcResult, Feedback, Report, tlsrpt::TlsReport}, zip, }; use mail_parser::{Message, MimeHeaders, PartType}; use std::{ borrow::Cow, collections::hash_map::Entry, io::{Cursor, Read}, }; use store::{ Serialize, write::{Archiver, BatchBuilder, ReportClass, ValueClass, now}, }; use trc::IncomingReportEvent; enum Compression { None, Gzip, Zip, } enum Format { Dmarc(D), Tls(T), Arf(A), } struct ReportData<'x> { compression: Compression, format: Format<(), (), ()>, data: &'x [u8], } #[derive( rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, serde::Serialize, serde::Deserialize, )] pub struct IncomingReport { pub from: String, pub to: Vec, pub subject: String, pub report: T, } pub trait AnalyzeReport: Sync + Send { fn analyze_report(&self, message: Message<'static>, session_id: u64); } impl AnalyzeReport for Server { fn analyze_report(&self, message: Message<'static>, session_id: u64) { let core = self.clone(); tokio::spawn(async move { let from: String = message .from() .and_then(|a| a.last()) .and_then(|a| a.address()) .unwrap_or_default() .into(); let to: Vec = message.to().map_or_else(Vec::new, |a| { a.iter() .filter_map(|a| a.address()) .map(|a| a.into()) .collect() }); let subject: String = message.subject().unwrap_or_default().into(); let mut reports = Vec::new(); for part in &message.parts { match &part.body { PartType::Text(report) => { if part .content_type() .and_then(|ct| ct.subtype()) .is_some_and(|t| t.eq_ignore_ascii_case("xml")) || part .attachment_name() .and_then(|n| n.rsplit_once('.')) .is_some_and(|(_, e)| e.eq_ignore_ascii_case("xml")) { reports.push(ReportData { compression: Compression::None, format: Format::Dmarc(()), data: report.as_bytes(), }); } else if part.is_content_type("message", "feedback-report") { reports.push(ReportData { compression: Compression::None, format: Format::Arf(()), data: report.as_bytes(), }); } } PartType::Binary(report) | PartType::InlineBinary(report) => { if part.is_content_type("message", "feedback-report") { reports.push(ReportData { compression: Compression::None, format: Format::Arf(()), data: report.as_ref(), }); continue; } let subtype = part .content_type() .and_then(|ct| ct.subtype()) .unwrap_or(""); let attachment_name = part.attachment_name(); let ext = attachment_name .and_then(|f| f.rsplit_once('.')) .map_or("", |(_, e)| e); let tls_parts = subtype.rsplit_once('+'); let compression = match (tls_parts.map(|(_, c)| c).unwrap_or(subtype), ext) { ("gzip", _) => Compression::Gzip, ("zip", _) => Compression::Zip, (_, "gz") => Compression::Gzip, (_, "zip") => Compression::Zip, _ => Compression::None, }; let format = match (tls_parts.map(|(c, _)| c).unwrap_or(subtype), ext) { ("xml", _) => Format::Dmarc(()), ("tlsrpt", _) | (_, "json") => Format::Tls(()), _ => { if attachment_name .is_some_and(|n| n.contains(".xml") || n.contains('!')) { Format::Dmarc(()) } else { continue; } } }; reports.push(ReportData { compression, format, data: report.as_ref(), }); } _ => (), } } for report in reports { let data = match report.compression { Compression::None => Cow::Borrowed(report.data), Compression::Gzip => { let mut file = GzDecoder::new(report.data); let mut buf = Vec::new(); if let Err(err) = file.read_to_end(&mut buf) { trc::event!( IncomingReport(IncomingReportEvent::DecompressError), SpanId = session_id, From = from.to_string(), Reason = err.to_string(), CausedBy = trc::location!() ); continue; } Cow::Owned(buf) } Compression::Zip => { let mut archive = match zip::ZipArchive::new(Cursor::new(report.data)) { Ok(archive) => archive, Err(err) => { trc::event!( IncomingReport(IncomingReportEvent::DecompressError), SpanId = session_id, From = from.to_string(), Reason = err.to_string(), CausedBy = trc::location!() ); continue; } }; let mut buf = Vec::with_capacity(0); for i in 0..archive.len() { match archive.by_index(i) { Ok(mut file) => { buf = Vec::with_capacity(file.compressed_size() as usize); if let Err(err) = file.read_to_end(&mut buf) { trc::event!( IncomingReport(IncomingReportEvent::DecompressError), SpanId = session_id, From = from.to_string(), Reason = err.to_string(), CausedBy = trc::location!() ); } break; } Err(err) => { trc::event!( IncomingReport(IncomingReportEvent::DecompressError), SpanId = session_id, From = from.to_string(), Reason = err.to_string(), CausedBy = trc::location!() ); } } } Cow::Owned(buf) } }; let report = match report.format { Format::Dmarc(_) => match Report::parse_xml(&data) { Ok(report) => { // Log report.log(); Format::Dmarc(report) } Err(err) => { trc::event!( IncomingReport(IncomingReportEvent::DmarcParseFailed), SpanId = session_id, From = from.to_string(), Reason = err, CausedBy = trc::location!() ); continue; } }, Format::Tls(_) => match TlsReport::parse_json(&data) { Ok(report) => { // Log report.log(); Format::Tls(report) } Err(err) => { trc::event!( IncomingReport(IncomingReportEvent::TlsRpcParseFailed), SpanId = session_id, From = from.to_string(), Reason = format!("{err:?}"), CausedBy = trc::location!() ); continue; } }, Format::Arf(_) => match Feedback::parse_arf(&data) { Some(report) => { // Log report.log(); Format::Arf(report.into_owned()) } None => { trc::event!( IncomingReport(IncomingReportEvent::ArfParseFailed), SpanId = session_id, From = from.to_string(), CausedBy = trc::location!() ); continue; } }, }; // Store report if let Some(expires_in) = &core.core.smtp.report.analysis.store { let expires = now() + expires_in.as_secs(); let id = core.inner.data.queue_id_gen.generate(); let mut batch = BatchBuilder::new(); match report { Format::Dmarc(report) => { batch.set( ValueClass::Report(ReportClass::Dmarc { id, expires }), Archiver::new(IncomingReport { from, to, subject, report, }) .serialize() .unwrap_or_default(), ); } Format::Tls(report) => { batch.set( ValueClass::Report(ReportClass::Tls { id, expires }), Archiver::new(IncomingReport { from, to, subject, report, }) .serialize() .unwrap_or_default(), ); } Format::Arf(report) => { batch.set( ValueClass::Report(ReportClass::Arf { id, expires }), Archiver::new(IncomingReport { from, to, subject, report, }) .serialize() .unwrap_or_default(), ); } } if let Err(err) = core.core.storage.data.write(batch.build_all()).await { trc::error!( err.span_id(session_id) .caused_by(trc::location!()) .details("Failed to write report") ); } } return; } }); } } trait LogReport { fn log(&self); } impl LogReport for Report { fn log(&self) { let mut dmarc_pass = 0; let mut dmarc_quarantine = 0; let mut dmarc_reject = 0; let mut dmarc_none = 0; let mut dkim_pass = 0; let mut dkim_fail = 0; let mut dkim_none = 0; let mut spf_pass = 0; let mut spf_fail = 0; let mut spf_none = 0; for record in self.records() { let count = std::cmp::min(record.count(), 1); match record.action_disposition() { ActionDisposition::Pass => { dmarc_pass += count; } ActionDisposition::Quarantine => { dmarc_quarantine += count; } ActionDisposition::Reject => { dmarc_reject += count; } ActionDisposition::None | ActionDisposition::Unspecified => { dmarc_none += count; } } match record.dmarc_dkim_result() { DmarcResult::Pass => { dkim_pass += count; } DmarcResult::Fail => { dkim_fail += count; } DmarcResult::Unspecified => { dkim_none += count; } } match record.dmarc_spf_result() { DmarcResult::Pass => { spf_pass += count; } DmarcResult::Fail => { spf_fail += count; } DmarcResult::Unspecified => { spf_none += count; } } } trc::event!( IncomingReport( if (dmarc_reject + dmarc_quarantine + dkim_fail + spf_fail) > 0 { IncomingReportEvent::DmarcReportWithWarnings } else { IncomingReportEvent::DmarcReport } ), RangeFrom = trc::Value::Timestamp(self.date_range_begin()), RangeTo = trc::Value::Timestamp(self.date_range_end()), Domain = self.domain().to_string(), From = self.email().to_string(), Id = self.report_id().to_string(), DmarcPass = dmarc_pass, DmarcQuarantine = dmarc_quarantine, DmarcReject = dmarc_reject, DmarcNone = dmarc_none, DkimPass = dkim_pass, DkimFail = dkim_fail, DkimNone = dkim_none, SpfPass = spf_pass, SpfFail = spf_fail, SpfNone = spf_none, ); } } impl LogReport for TlsReport { fn log(&self) { for policy in self.policies.iter().take(5) { let mut details = AHashMap::with_capacity(policy.failure_details.len()); for failure in &policy.failure_details { let num_failures = std::cmp::min(1, failure.failed_session_count); match details.entry(failure.result_type) { Entry::Occupied(mut e) => { *e.get_mut() += num_failures; } Entry::Vacant(e) => { e.insert(num_failures); } } } trc::event!( IncomingReport(if policy.summary.total_failure > 0 { IncomingReportEvent::TlsReportWithWarnings } else { IncomingReportEvent::TlsReport }), RangeFrom = trc::Value::Timestamp(self.date_range.start_datetime.to_timestamp() as u64), RangeTo = trc::Value::Timestamp(self.date_range.end_datetime.to_timestamp() as u64), Domain = policy.policy.policy_domain.clone(), From = self.contact_info.as_deref().unwrap_or_default().to_string(), Id = self.report_id.clone(), Policy = format!("{:?}", policy.policy.policy_type), TotalSuccesses = policy.summary.total_success, TotalFailures = policy.summary.total_failure, Details = format!("{details:?}"), ); } } } impl LogReport for Feedback<'_> { fn log(&self) { trc::event!( IncomingReport(match self.feedback_type() { mail_auth::report::FeedbackType::Abuse => IncomingReportEvent::AbuseReport, mail_auth::report::FeedbackType::AuthFailure => IncomingReportEvent::AuthFailureReport, mail_auth::report::FeedbackType::Fraud => IncomingReportEvent::FraudReport, mail_auth::report::FeedbackType::NotSpam => IncomingReportEvent::NotSpamReport, mail_auth::report::FeedbackType::Other => IncomingReportEvent::OtherReport, mail_auth::report::FeedbackType::Virus => IncomingReportEvent::VirusReport, }), RangeFrom = trc::Value::Timestamp( self.arrival_date() .map(|d| d as u64) .unwrap_or_else(|| { now() }) ), Domain = self .reported_domain() .iter() .map(|d| trc::Value::String(d.as_ref().into())) .collect::>(), Hostname = self.reporting_mta().map(|d| trc::Value::String(d.into())), Url = self .reported_uri() .iter() .map(|d| trc::Value::String(d.as_ref().into())) .collect::>(), RemoteIp = self.source_ip(), Total = self.incidents(), Result = format!("{:?}", self.delivery_result()), Details = self .authentication_results() .iter() .map(|d| trc::Value::String(d.as_ref().into())) .collect::>(), ); } } impl IncomingReport { pub fn has_domain(&self, domain: &[String]) -> bool { self.to .iter() .any(|to| domain.iter().any(|d| to.ends_with(d.as_str()))) || domain.iter().any(|d| self.from.ends_with(d.as_str())) } } ================================================ FILE: crates/smtp/src/reporting/dkim.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::listener::SessionStream; use mail_auth::{ AuthenticatedMessage, AuthenticationResults, DkimOutput, common::verify::VerifySignature, }; use trc::OutgoingReportEvent; use utils::config::Rate; use crate::{core::Session, reporting::SmtpReporting}; impl Session { pub async fn send_dkim_report( &self, rcpt: &str, message: &AuthenticatedMessage<'_>, rate: &Rate, rejected: bool, output: &DkimOutput<'_>, ) { // Generate report let signature = if let Some(signature) = output.signature() { signature } else { return; }; // Throttle recipient if !self.throttle_rcpt(rcpt, rate, "dkim").await { trc::event!( OutgoingReport(OutgoingReportEvent::DkimRateLimited), SpanId = self.data.session_id, To = rcpt.to_string(), Limit = vec![ trc::Value::from(rate.requests), trc::Value::from(rate.period) ], ); return; } let config = &self.server.core.smtp.report.dkim; let from_addr = self .server .eval_if(&config.address, self, self.data.session_id) .await .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()); let mut report = Vec::with_capacity(128); self.new_auth_failure(output.result().into(), rejected) .with_authentication_results( AuthenticationResults::new(&self.hostname) .with_dkim_result(output, message.from()) .to_string(), ) .with_dkim_domain(signature.domain()) .with_dkim_selector(signature.selector()) .with_dkim_identity(signature.identity()) .with_headers(std::str::from_utf8(message.raw_headers()).unwrap_or_default()) .write_rfc5322( ( self.server .eval_if(&config.name, self, self.data.session_id) .await .unwrap_or_else(|| "Mail Delivery Subsystem".to_string()) .as_str(), from_addr.as_str(), ), rcpt, &self .server .eval_if(&config.subject, self, self.data.session_id) .await .unwrap_or_else(|| "DKIM Report".to_string()), &mut report, ) .ok(); trc::event!( OutgoingReport(OutgoingReportEvent::DkimReport), SpanId = self.data.session_id, From = from_addr.to_string(), To = rcpt.to_string(), ); // Send report self.server .send_report( &from_addr, [rcpt].into_iter(), report, &config.sign, true, self.data.session_id, ) .await; } } ================================================ FILE: crates/smtp/src/reporting/dmarc.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{AggregateTimestamp, SerializedSize}; use crate::{core::Session, queue::RecipientDomain, reporting::SmtpReporting}; use ahash::AHashMap; use common::{ Server, config::smtp::report::AggregateFrequency, ipc::{DmarcEvent, ToHash}, listener::SessionStream, }; use compact_str::ToCompactString; use mail_auth::{ ArcOutput, AuthenticatedMessage, AuthenticationResults, DkimOutput, DkimResult, DmarcOutput, SpfResult, common::verify::VerifySignature, dmarc::{self, URI}, report::{AuthFailureType, IdentityAlignment, PolicyPublished, Record, Report, SPFDomainScope}, }; use std::{collections::hash_map::Entry, future::Future}; use store::{ Deserialize, IterateParams, Serialize, ValueKey, write::{AlignedBytes, Archive, Archiver, BatchBuilder, QueueClass, ReportEvent, ValueClass}, }; use trc::{AddContext, OutgoingReportEvent}; use utils::{DomainPart, config::Rate}; #[derive( Debug, PartialEq, Eq, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, serde::Serialize, serde::Deserialize, )] pub struct DmarcFormat { pub rua: Vec, pub policy: PolicyPublished, pub records: Vec, } impl Session { #[allow(clippy::too_many_arguments)] pub async fn send_dmarc_report( &self, message: &AuthenticatedMessage<'_>, auth_results: &AuthenticationResults<'_>, rejected: bool, dmarc_output: DmarcOutput, dkim_output: &[DkimOutput<'_>], arc_output: &Option>, ) { let dmarc_record = dmarc_output.dmarc_record_cloned().unwrap(); let config = &self.server.core.smtp.report.dmarc; // Send failure report if let (Some(failure_rate), Some(report_options)) = ( self.server .eval_if::(&config.send, self, self.data.session_id) .await, dmarc_output.failure_report(), ) { // Verify that any external reporting addresses are authorized let rcpts = match self .server .core .smtp .resolvers .dns .verify_dmarc_report_address( dmarc_output.domain(), dmarc_record.ruf(), Some(&self.server.inner.cache.dns_txt), ) .await { Some(rcpts) => { if !rcpts.is_empty() { let mut new_rcpts = Vec::with_capacity(rcpts.len()); for rcpt in rcpts { if self.throttle_rcpt(rcpt.uri(), &failure_rate, "dmarc").await { new_rcpts.push(rcpt.uri()); } } new_rcpts } else { if !dmarc_record.ruf().is_empty() { trc::event!( OutgoingReport(OutgoingReportEvent::UnauthorizedReportingAddress), SpanId = self.data.session_id, Url = dmarc_record .ruf() .iter() .map(|u| trc::Value::String(u.uri().to_compact_string())) .collect::>(), ); } vec![] } } None => { trc::event!( OutgoingReport(OutgoingReportEvent::ReportingAddressValidationError), SpanId = self.data.session_id, Url = dmarc_record .ruf() .iter() .map(|u| trc::Value::String(u.uri().to_compact_string())) .collect::>(), ); vec![] } }; // Throttle recipient if !rcpts.is_empty() { let mut report = Vec::with_capacity(128); let from_addr = self .server .eval_if(&config.address, self, self.data.session_id) .await .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_compact_string()); let mut auth_failure = self .new_auth_failure(AuthFailureType::Dmarc, rejected) .with_authentication_results(auth_results.to_string()) .with_headers(std::str::from_utf8(message.raw_headers()).unwrap_or_default()); // Report the first failed signature let dkim_failed = if let ( dmarc::Report::Dkim | dmarc::Report::DkimSpf | dmarc::Report::All | dmarc::Report::Any, Some(signature), ) = ( &report_options, dkim_output.iter().find_map(|o| { let s = o.signature()?; if !matches!(o.result(), DkimResult::Pass) { Some(s) } else { None } }), ) { auth_failure = auth_failure .with_dkim_domain(signature.domain()) .with_dkim_selector(signature.selector()) .with_dkim_identity(signature.identity()); true } else { false }; // Report SPF failure let spf_failed = if let ( dmarc::Report::Spf | dmarc::Report::DkimSpf | dmarc::Report::All | dmarc::Report::Any, Some(output), ) = ( &report_options, self.data .spf_ehlo .as_ref() .and_then(|s| { if s.result() != SpfResult::Pass { s.into() } else { None } }) .or_else(|| { self.data.spf_mail_from.as_ref().and_then(|s| { if s.result() != SpfResult::Pass { s.into() } else { None } }) }), ) { auth_failure = auth_failure.with_spf_dns(format!("txt : {} : v=SPF1", output.domain())); // TODO use DNS record true } else { false }; auth_failure .with_identity_alignment(if dkim_failed && spf_failed { IdentityAlignment::DkimSpf } else if dkim_failed { IdentityAlignment::Dkim } else { IdentityAlignment::Spf }) .write_rfc5322( ( self.server .eval_if(&config.name, self, self.data.session_id) .await .unwrap_or_else(|| "Mail Delivery Subsystem".to_compact_string()) .as_str(), from_addr.as_str(), ), &rcpts.join(", "), &self .server .eval_if(&config.subject, self, self.data.session_id) .await .unwrap_or_else(|| "DMARC Report".to_compact_string()), &mut report, ) .ok(); trc::event!( OutgoingReport(OutgoingReportEvent::DmarcReport), SpanId = self.data.session_id, From = from_addr.to_string(), To = rcpts .iter() .map(|a| trc::Value::String(a.to_compact_string())) .collect::>(), ); // Send report self.server .send_report( &from_addr, rcpts.into_iter(), report, &config.sign, true, self.data.session_id, ) .await; } else { trc::event!( OutgoingReport(OutgoingReportEvent::DmarcRateLimited), SpanId = self.data.session_id, Limit = vec![ trc::Value::from(failure_rate.requests), trc::Value::from(failure_rate.period) ], ); } } // Send aggregate reports let interval = self .server .eval_if( &self.server.core.smtp.report.dmarc_aggregate.send, self, self.data.session_id, ) .await .unwrap_or(AggregateFrequency::Never); if matches!(interval, AggregateFrequency::Never) || dmarc_record.rua().is_empty() { return; } // Create DMARC report record let mut report_record = Record::new() .with_dmarc_output(&dmarc_output) .with_dkim_output(dkim_output) .with_source_ip(self.data.remote_ip) .with_header_from(message.from().domain_part()) .with_envelope_from( self.data .mail_from .as_ref() .map(|mf| mf.domain.as_str()) .unwrap_or_else(|| self.data.helo_domain.as_str()), ); if let Some(spf_ehlo) = &self.data.spf_ehlo { report_record = report_record.with_spf_output(spf_ehlo, SPFDomainScope::Helo); } if let Some(spf_mail_from) = &self.data.spf_mail_from { report_record = report_record.with_spf_output(spf_mail_from, SPFDomainScope::MailFrom); } if let Some(arc_output) = arc_output { report_record = report_record.with_arc_output(arc_output); } // Submit DMARC report event self.server .schedule_report(DmarcEvent { domain: dmarc_output.into_domain(), report_record, dmarc_record, interval, }) .await; } } pub trait DmarcReporting: Sync + Send { fn send_dmarc_aggregate_report(&self, event: ReportEvent) -> impl Future + Send; fn generate_dmarc_aggregate_report( &self, event: &ReportEvent, rua: &mut Vec, serialized_size: Option<&mut serde_json::Serializer>, span_id: u64, ) -> impl Future>> + Send; fn delete_dmarc_report(&self, event: ReportEvent) -> impl Future + Send; fn schedule_dmarc(&self, event: Box) -> impl Future + Send; } impl DmarcReporting for Server { async fn send_dmarc_aggregate_report(&self, event: ReportEvent) { let span_id = self.inner.data.span_id_gen.generate(); trc::event!( OutgoingReport(OutgoingReportEvent::DmarcAggregateReport), SpanId = span_id, ReportId = event.seq_id, Domain = event.domain.clone(), RangeFrom = trc::Value::Timestamp(event.seq_id), RangeTo = trc::Value::Timestamp(event.due), ); // Generate report let mut serialized_size = serde_json::Serializer::new(SerializedSize::new( self.eval_if( &self.core.smtp.report.dmarc_aggregate.max_size, &RecipientDomain::new(event.domain.as_str()), span_id, ) .await .unwrap_or(25 * 1024 * 1024), )); let mut rua = Vec::new(); let report = match self .generate_dmarc_aggregate_report(&event, &mut rua, Some(&mut serialized_size), span_id) .await { Ok(Some(report)) => report, Ok(None) => { trc::event!( OutgoingReport(OutgoingReportEvent::NotFound), SpanId = span_id, CausedBy = trc::location!() ); return; } Err(err) => { trc::error!(err.span_id(span_id).details("Failed to read DMARC report")); return; } }; // Verify external reporting addresses let rua = match self .core .smtp .resolvers .dns .verify_dmarc_report_address(&event.domain, &rua, Some(&self.inner.cache.dns_txt)) .await { Some(rcpts) => { if !rcpts.is_empty() { rcpts .into_iter() .map(|u| u.uri().to_string()) .collect::>() } else { trc::event!( OutgoingReport(OutgoingReportEvent::UnauthorizedReportingAddress), SpanId = span_id, Url = rua .iter() .map(|u| trc::Value::String(u.uri().to_compact_string())) .collect::>(), ); self.delete_dmarc_report(event).await; return; } } None => { trc::event!( OutgoingReport(OutgoingReportEvent::ReportingAddressValidationError), SpanId = span_id, Url = rua .iter() .map(|u| trc::Value::String(u.uri().to_compact_string())) .collect::>(), ); self.delete_dmarc_report(event).await; return; } }; // Serialize report let config = &self.core.smtp.report.dmarc_aggregate; let from_addr = self .eval_if( &config.address, &RecipientDomain::new(event.domain.as_str()), span_id, ) .await .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_compact_string()); let mut message = Vec::with_capacity(2048); let _ = report.write_rfc5322( &self .eval_if( &self.core.smtp.report.submitter, &RecipientDomain::new(event.domain.as_str()), span_id, ) .await .unwrap_or_else(|| "localhost".to_compact_string()), ( self.eval_if( &config.name, &RecipientDomain::new(event.domain.as_str()), span_id, ) .await .unwrap_or_else(|| "Mail Delivery Subsystem".to_compact_string()) .as_str(), from_addr.as_str(), ), rua.iter().map(|a| a.as_str()), &mut message, ); // Send report self.send_report( &from_addr, rua.iter(), message, &config.sign, false, event.seq_id, ) .await; self.delete_dmarc_report(event).await; } async fn generate_dmarc_aggregate_report( &self, event: &ReportEvent, rua: &mut Vec, mut serialized_size: Option<&mut serde_json::Serializer>, span_id: u64, ) -> trc::Result> { // Deserialize report let dmarc = match self .store() .get_value::>(ValueKey::from(ValueClass::Queue( QueueClass::DmarcReportHeader(event.clone()), ))) .await? { Some(dmarc) => dmarc.deserialize::()?, None => { return Ok(None); } }; let _ = std::mem::replace(rua, dmarc.rua); // Create report let config = &self.core.smtp.report.dmarc_aggregate; let mut report = Report::new() .with_policy_published(dmarc.policy) .with_date_range_begin(event.seq_id) .with_date_range_end(event.due) .with_report_id(format!("{}_{}", event.policy_hash, event.seq_id)) .with_email( self.eval_if( &config.address, &RecipientDomain::new(event.domain.as_str()), span_id, ) .await .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_compact_string()), ); if let Some(org_name) = self .eval_if::( &config.org_name, &RecipientDomain::new(event.domain.as_str()), span_id, ) .await { report = report.with_org_name(org_name); } if let Some(contact_info) = self .eval_if::( &config.contact_info, &RecipientDomain::new(event.domain.as_str()), span_id, ) .await { report = report.with_extra_contact_info(contact_info); } if let Some(serialized_size) = serialized_size.as_deref_mut() { let _ = serde::Serialize::serialize(&report, serialized_size); } // Group duplicates let from_key = ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportEvent( ReportEvent { due: event.due, policy_hash: event.policy_hash, seq_id: 0, domain: event.domain.clone(), }, ))); let to_key = ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportEvent( ReportEvent { due: event.due, policy_hash: event.policy_hash, seq_id: u64::MAX, domain: event.domain.clone(), }, ))); let mut record_map = AHashMap::with_capacity(dmarc.records.len()); self.core .storage .data .iterate(IterateParams::new(from_key, to_key).ascending(), |_, v| { let archive = as Deserialize>::deserialize(v)?; match record_map.entry(archive.deserialize::()?) { Entry::Occupied(mut e) => { *e.get_mut() += 1; Ok(true) } Entry::Vacant(e) => { if serialized_size .as_deref_mut() .is_none_or(|serialized_size| { serde::Serialize::serialize(e.key(), serialized_size).is_ok() }) { e.insert(1u32); Ok(true) } else { Ok(false) } } } }) .await .caused_by(trc::location!())?; for (record, count) in record_map { report = report.with_record(record.with_count(count)); } Ok(Some(report)) } async fn delete_dmarc_report(&self, event: ReportEvent) { let from_key = ReportEvent { due: event.due, policy_hash: event.policy_hash, seq_id: 0, domain: event.domain.clone(), }; let to_key = ReportEvent { due: event.due, policy_hash: event.policy_hash, seq_id: u64::MAX, domain: event.domain.clone(), }; if let Err(err) = self .core .storage .data .delete_range( ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportEvent(from_key))), ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportEvent(to_key))), ) .await { trc::error!( err.caused_by(trc::location!()) .details("Failed to delete DMARC report") ); return; } let mut batch = BatchBuilder::new(); batch.clear(ValueClass::Queue(QueueClass::DmarcReportHeader(event))); if let Err(err) = self.core.storage.data.write(batch.build_all()).await { trc::error!( err.caused_by(trc::location!()) .details("Failed to delete DMARC report") ); } } async fn schedule_dmarc(&self, event: Box) { let created = event.interval.to_timestamp(); let deliver_at = created + event.interval.as_secs(); let mut report_event = ReportEvent { due: deliver_at, policy_hash: event.dmarc_record.to_hash(), seq_id: created, domain: event.domain, }; // Write policy if missing let mut builder = BatchBuilder::new(); if self .core .storage .data .get_value::<()>(ValueKey::from(ValueClass::Queue( QueueClass::DmarcReportHeader(report_event.clone()), ))) .await .unwrap_or_default() .is_none() { // Serialize report let entry = DmarcFormat { rua: event.dmarc_record.rua().to_vec(), policy: PolicyPublished::from_record( report_event.domain.to_string(), &event.dmarc_record, ), records: vec![], }; // Write report builder.set( ValueClass::Queue(QueueClass::DmarcReportHeader(report_event.clone())), match Archiver::new(entry).serialize() { Ok(data) => data.to_vec(), Err(err) => { trc::error!( err.caused_by(trc::location!()) .details("Failed to serialize DMARC report") ); return; } }, ); } // Write entry report_event.seq_id = self.inner.data.queue_id_gen.generate(); builder.set( ValueClass::Queue(QueueClass::DmarcReportEvent(report_event)), match Archiver::new(event.report_record).serialize() { Ok(data) => data.to_vec(), Err(err) => { trc::error!( err.caused_by(trc::location!()) .details("Failed to serialize DMARC report") ); return; } }, ); if let Err(err) = self.core.storage.data.write(builder.build_all()).await { trc::error!( err.caused_by(trc::location!()) .details("Failed to write DMARC report") ); } } } ================================================ FILE: crates/smtp/src/reporting/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ core::Session, inbound::DkimSign, queue::{MessageSource, MessageWrapper, spool::SmtpSpool}, }; use common::{ Server, USER_AGENT, config::smtp::report::{AddressMatch, AggregateFrequency}, expr::if_block::IfBlock, ipc::ReportingEvent, }; use mail_auth::{ common::headers::HeaderWriter, report::{AuthFailureType, DeliveryResult, Feedback, FeedbackType}, }; use mail_parser::DateTime; use std::{future::Future, io, time::SystemTime}; use store::write::{ReportEvent, key::KeySerializer}; use tokio::io::{AsyncRead, AsyncWrite}; pub mod analysis; pub mod dkim; pub mod dmarc; pub mod scheduler; pub mod spf; pub mod tls; impl Session { pub fn new_auth_failure(&self, ft: AuthFailureType, rejected: bool) -> Feedback<'_> { Feedback::new(FeedbackType::AuthFailure) .with_auth_failure(ft) .with_arrival_date( SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()) as i64, ) .with_source_ip(self.data.remote_ip) .with_reporting_mta(&self.hostname) .with_user_agent(USER_AGENT) .with_delivery_result(if rejected { DeliveryResult::Reject } else { DeliveryResult::Unspecified }) } pub fn is_report(&self) -> bool { for addr_match in &self.server.core.smtp.report.analysis.addresses { for addr in &self.data.rcpt_to { match addr_match { AddressMatch::StartsWith(prefix) if addr.address_lcase.starts_with(prefix) => { return true; } AddressMatch::EndsWith(suffix) if addr.address_lcase.ends_with(suffix) => { return true; } AddressMatch::Equals(value) if addr.address_lcase.eq(value) => return true, _ => (), } } } false } } pub trait SmtpReporting: Sync + Send { fn send_report( &self, from_addr: &str, rcpts: impl Iterator + Sync + Send> + Sync + Send, report: Vec, sign_config: &IfBlock, deliver_now: bool, parent_session_id: u64, ) -> impl Future + Send; fn send_autogenerated( &self, from_addr: impl AsRef + Sync + Send, rcpts: impl Iterator + Sync + Send> + Sync + Send, raw_message: Vec, sign_config: Option<&IfBlock>, parent_session_id: u64, ) -> impl Future + Send; fn schedule_report( &self, report: impl Into + Sync + Send, ) -> impl Future + Send; fn sign_message( &self, message: &mut MessageWrapper, config: &IfBlock, bytes: &[u8], ) -> impl Future>> + Send; } impl SmtpReporting for Server { async fn send_report( &self, from_addr: &str, rcpts: impl Iterator + Sync + Send> + Sync + Send, report: Vec, sign_config: &IfBlock, deliver_now: bool, parent_session_id: u64, ) { // Build message let mut message = self.new_message(from_addr, parent_session_id); for rcpt_ in rcpts { message.add_recipient(rcpt_.as_ref(), self).await; } // Sign message let signature = self.sign_message(&mut message, sign_config, &report).await; // Schedule delivery at a random time between now and the next 3 hours if !deliver_now { #[cfg(not(feature = "test_mode"))] { use common::config::smtp::queue::QueueExpiry; use rand::Rng; let delivery_time = rand::rng().random_range(0u64..10800u64); for rcpt in &mut message.message.recipients { rcpt.retry.due += delivery_time; rcpt.notify.due += delivery_time; if let QueueExpiry::Ttl(expires) = &mut rcpt.expires { *expires += delivery_time; } } } } // Queue message message .queue( signature.as_deref(), &report, parent_session_id, self, MessageSource::Report, ) .await; } async fn send_autogenerated( &self, from_addr: impl AsRef + Sync + Send, rcpts: impl Iterator + Sync + Send> + Sync + Send, raw_message: Vec, sign_config: Option<&IfBlock>, parent_session_id: u64, ) { // Build message let mut message = self.new_message(from_addr.as_ref(), parent_session_id); for rcpt in rcpts { message.add_recipient(rcpt, self).await; } // Sign message let signature = if let Some(sign_config) = sign_config { self.sign_message(&mut message, sign_config, &raw_message) .await } else { None }; // Queue message message .queue( signature.as_deref(), &raw_message, parent_session_id, self, MessageSource::Autogenerated, ) .await; } async fn schedule_report(&self, report: impl Into + Sync + Send) { if self.inner.ipc.report_tx.send(report.into()).await.is_err() { trc::event!( Server(trc::ServerEvent::ThreadError), CausedBy = trc::location!(), Details = "Failed to send event to ReportScheduler" ); } } async fn sign_message( &self, message: &mut MessageWrapper, config: &IfBlock, bytes: &[u8], ) -> Option> { let signers = self .eval_if::, _>(config, &message.message, message.span_id) .await .unwrap_or_default(); if !signers.is_empty() { let mut headers = Vec::with_capacity(64); for signer in signers.iter() { if let Some(signer) = self.get_dkim_signer(signer, message.span_id) { match signer.sign(bytes) { Ok(signature) => { signature.write_header(&mut headers); } Err(err) => { trc::error!( trc::Error::from(err) .span_id(message.span_id) .details("Failed to sign message") .caused_by(trc::location!()) ); } } } } if !headers.is_empty() { return Some(headers); } } None } } pub trait AggregateTimestamp { fn to_timestamp(&self) -> u64; fn to_timestamp_(&self, dt: DateTime) -> u64; fn as_secs(&self) -> u64; fn due(&self) -> u64; } impl AggregateTimestamp for AggregateFrequency { fn to_timestamp(&self) -> u64 { self.to_timestamp_(DateTime::from_timestamp( SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()) as i64, )) } fn to_timestamp_(&self, mut dt: DateTime) -> u64 { (match self { AggregateFrequency::Hourly => { dt.minute = 0; dt.second = 0; dt.to_timestamp() } AggregateFrequency::Daily => { dt.hour = 0; dt.minute = 0; dt.second = 0; dt.to_timestamp() } AggregateFrequency::Weekly => { let dow = dt.day_of_week(); dt.hour = 0; dt.minute = 0; dt.second = 0; dt.to_timestamp() - (86400 * dow as i64) } AggregateFrequency::Never => dt.to_timestamp(), }) as u64 } fn as_secs(&self) -> u64 { match self { AggregateFrequency::Hourly => 3600, AggregateFrequency::Daily => 86400, AggregateFrequency::Weekly => 7 * 86400, AggregateFrequency::Never => 0, } } fn due(&self) -> u64 { self.to_timestamp() + self.as_secs() } } pub struct SerializedSize { bytes_left: usize, } impl SerializedSize { pub fn new(bytes_left: usize) -> Self { Self { bytes_left } } } impl io::Write for SerializedSize { fn write(&mut self, buf: &[u8]) -> io::Result { //let c = print!(" (left: {}, buf: {})", self.bytes_left, buf.len()); let buf_len = buf.len(); if buf_len <= self.bytes_left { self.bytes_left -= buf_len; Ok(buf_len) } else { Err(io::Error::other("Size exceeded")) } } fn flush(&mut self) -> io::Result<()> { Ok(()) } } pub trait ReportLock { fn tls_lock(&self) -> Vec; fn dmarc_lock(&self) -> Vec; } impl ReportLock for ReportEvent { fn tls_lock(&self) -> Vec { KeySerializer::new(self.domain.len() + std::mem::size_of::() + 1) .write(0u8) .write(self.due) .write(self.domain.as_bytes()) .finalize() } fn dmarc_lock(&self) -> Vec { KeySerializer::new(self.domain.len() + (std::mem::size_of::() * 2) + 1) .write(1u8) .write(self.due) .write(self.policy_hash) .write(self.domain.as_bytes()) .finalize() } } ================================================ FILE: crates/smtp/src/reporting/scheduler.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use ahash::AHashMap; use common::{Inner, KV_LOCK_QUEUE_REPORT, Server, core::BuildServer, ipc::ReportingEvent}; use std::{ future::Future, sync::Arc, time::{Duration, SystemTime}, }; use store::{ Deserialize, IterateParams, Store, ValueKey, write::{BatchBuilder, QueueClass, ReportEvent, ValueClass, now}, }; use tokio::sync::mpsc; use crate::queue::spool::LOCK_EXPIRY; use super::{AggregateTimestamp, ReportLock, dmarc::DmarcReporting, tls::TlsReporting}; pub const REPORT_REFRESH: Duration = Duration::from_secs(86400); impl SpawnReport for mpsc::Receiver { fn spawn(mut self, inner: Arc) { tokio::spawn(async move { let mut next_wake_up = REPORT_REFRESH; let mut refresh_queue = true; loop { let server = inner.build_server(); if refresh_queue { // Read events let events = next_report_event(server.store()).await; let now = now(); next_wake_up = events .last() .and_then(|e| { e.due() .filter(|due| *due > now) .map(|due| Duration::from_secs(due - now)) }) .unwrap_or(REPORT_REFRESH); if events .first() .and_then(|e| e.due()) .is_some_and(|due| due <= now) { let server_ = server.clone(); tokio::spawn(async move { let mut tls_reports = AHashMap::new(); for report_event in events { match report_event { QueueClass::DmarcReportHeader(event) if event.due <= now => { let lock_name = event.dmarc_lock(); if server_.try_lock_report(&lock_name).await { server_.send_dmarc_aggregate_report(event).await; server_.unlock_report(&lock_name).await; } } QueueClass::TlsReportHeader(event) if event.due <= now => { tls_reports .entry(event.domain.clone()) .or_insert_with(Vec::new) .push(event); } _ => (), } } for (_, tls_report) in tls_reports { let lock_name = tls_report.first().unwrap().tls_lock(); if server_.try_lock_report(&lock_name).await { server_.send_tls_aggregate_report(tls_report).await; server_.unlock_report(&lock_name).await; } } }); } } match tokio::time::timeout(next_wake_up, self.recv()).await { Ok(Some(event)) => { refresh_queue = false; match event { ReportingEvent::Dmarc(event) => { next_wake_up = std::cmp::min( next_wake_up, Duration::from_secs(event.interval.due().saturating_sub(now())), ); server.schedule_dmarc(event).await; } ReportingEvent::Tls(event) => { next_wake_up = std::cmp::min( next_wake_up, Duration::from_secs(event.interval.due().saturating_sub(now())), ); server.schedule_tls(event).await; } ReportingEvent::Stop => break, } } Ok(None) => break, Err(_) => { refresh_queue = true; } } } }); } } async fn next_report_event(store: &Store) -> Vec { let now = now(); let from_key = ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportHeader( ReportEvent { due: 0, policy_hash: 0, seq_id: 0, domain: String::new(), }, ))); let to_key = ValueKey::from(ValueClass::Queue(QueueClass::TlsReportHeader( ReportEvent { due: now + REPORT_REFRESH.as_secs(), policy_hash: 0, seq_id: 0, domain: String::new(), }, ))); let mut events = Vec::new(); let mut old_locks = Vec::new(); let result = store .iterate( IterateParams::new(from_key, to_key).ascending().no_values(), |key, _| { let event = ReportEvent::deserialize(key)?; // TODO - REMOVEME - Part of v0.11 migration if event.seq_id == 0 { old_locks.push(if *key.last().unwrap() == 0 { QueueClass::DmarcReportHeader(event) } else { QueueClass::TlsReportHeader(event) }); return Ok(true); } let do_continue = event.due <= now; events.push(if *key.last().unwrap() == 0 { QueueClass::DmarcReportHeader(event) } else { QueueClass::TlsReportHeader(event) }); Ok(do_continue) }, ) .await; // TODO - REMOVEME - Part of v0.11 migration if !old_locks.is_empty() { let mut batch = BatchBuilder::new(); for event in old_locks { batch.clear(ValueClass::Queue(event)); } if let Err(err) = store.write(batch.build_all()).await { trc::error!( err.caused_by(trc::location!()) .details("Failed to remove old report events") ); } } if let Err(err) = result { trc::error!( err.caused_by(trc::location!()) .details("Failed to read from store") ); } events } pub trait LockReport: Sync + Send { fn try_lock_report(&self, lock: &[u8]) -> impl Future + Send; fn unlock_report(&self, lock: &[u8]) -> impl Future + Send; } impl LockReport for Server { async fn try_lock_report(&self, key: &[u8]) -> bool { match self .in_memory_store() .try_lock(KV_LOCK_QUEUE_REPORT, key, LOCK_EXPIRY) .await { Ok(result) => { if !result { trc::event!( OutgoingReport(trc::OutgoingReportEvent::Locked), Expires = trc::Value::Timestamp(now() + LOCK_EXPIRY), Key = key ); } result } Err(err) => { trc::error!( err.details("Failed to lock report.") .caused_by(trc::location!()) ); false } } } async fn unlock_report(&self, key: &[u8]) { if let Err(err) = self .in_memory_store() .remove_lock(KV_LOCK_QUEUE_REPORT, key) .await { trc::error!( err.details("Failed to unlock event.") .caused_by(trc::location!()) ); } } } pub trait ToTimestamp { fn to_timestamp(&self) -> u64; } impl ToTimestamp for Duration { fn to_timestamp(&self) -> u64 { SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()) + self.as_secs() } } pub trait SpawnReport { fn spawn(self, core: Arc); } ================================================ FILE: crates/smtp/src/reporting/spf.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::listener::SessionStream; use mail_auth::{AuthenticationResults, SpfOutput, report::AuthFailureType}; use trc::OutgoingReportEvent; use utils::config::Rate; use crate::{core::Session, reporting::SmtpReporting}; impl Session { pub async fn send_spf_report( &self, rcpt: &str, rate: &Rate, rejected: bool, output: &SpfOutput, ) { // Throttle recipient if !self.throttle_rcpt(rcpt, rate, "spf").await { trc::event!( OutgoingReport(OutgoingReportEvent::SpfRateLimited), SpanId = self.data.session_id, To = rcpt.to_string(), Limit = vec![ trc::Value::from(rate.requests), trc::Value::from(rate.period) ], ); return; } // Generate report let config = &self.server.core.smtp.report.spf; let from_addr = self .server .eval_if(&config.address, self, self.data.session_id) .await .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()); let mut report = Vec::with_capacity(128); self.new_auth_failure(AuthFailureType::Spf, rejected) .with_authentication_results( if let Some(mail_from) = &self.data.mail_from { AuthenticationResults::new(&self.hostname).with_spf_mailfrom_result( output, self.data.remote_ip, &mail_from.address, &self.data.helo_domain, ) } else { AuthenticationResults::new(&self.hostname).with_spf_ehlo_result( output, self.data.remote_ip, &self.data.helo_domain, ) } .to_string(), ) .with_spf_dns(format!("txt : {} : v=SPF1", output.domain())) // TODO use DNS record .write_rfc5322( ( self.server .eval_if(&config.name, self, self.data.session_id) .await .unwrap_or_else(|| "Mailer Daemon".to_string()) .as_str(), from_addr.as_str(), ), rcpt, &self .server .eval_if(&config.subject, self, self.data.session_id) .await .unwrap_or_else(|| "SPF Report".to_string()), &mut report, ) .ok(); trc::event!( OutgoingReport(OutgoingReportEvent::SpfReport), SpanId = self.data.session_id, To = rcpt.to_string(), From = from_addr.to_string(), ); // Send report self.server .send_report( &from_addr, [rcpt].into_iter(), report, &config.sign, true, self.data.session_id, ) .await; } } ================================================ FILE: crates/smtp/src/reporting/tls.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{AggregateTimestamp, SerializedSize}; use crate::{queue::RecipientDomain, reporting::SmtpReporting}; use ahash::AHashMap; use common::{ Server, USER_AGENT, config::smtp::{ report::AggregateFrequency, resolver::{Mode, MxPattern}, }, ipc::{TlsEvent, ToHash}, }; use mail_auth::{ flate2::{Compression, write::GzEncoder}, mta_sts::{ReportUri, TlsRpt}, report::tlsrpt::{ DateRange, FailureDetails, Policy, PolicyDetails, PolicyType, Summary, TlsReport, }, }; use mail_parser::DateTime; use reqwest::header::CONTENT_TYPE; use std::fmt::Write; use std::{collections::hash_map::Entry, future::Future, sync::Arc, time::Duration}; use store::{ Deserialize, IterateParams, Serialize, ValueKey, write::{AlignedBytes, Archive, Archiver, BatchBuilder, QueueClass, ReportEvent, ValueClass}, }; use trc::{AddContext, OutgoingReportEvent}; #[derive(Debug, Clone)] pub struct TlsRptOptions { pub record: Arc, pub interval: AggregateFrequency, } #[derive(Debug, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, serde::Serialize)] pub struct TlsFormat { pub rua: Vec, pub policy: PolicyDetails, pub records: Vec>, } #[cfg(feature = "test_mode")] pub static TLS_HTTP_REPORT: parking_lot::Mutex> = parking_lot::Mutex::new(Vec::new()); pub trait TlsReporting: Sync + Send { fn send_tls_aggregate_report( &self, events: Vec, ) -> impl Future + Send; fn generate_tls_aggregate_report( &self, events: &[ReportEvent], rua: &mut Vec, serialized_size: Option<&mut serde_json::Serializer>, span_id: u64, ) -> impl Future>> + Send; fn schedule_tls(&self, event: Box) -> impl Future + Send; fn delete_tls_report(&self, events: Vec) -> impl Future + Send; } impl TlsReporting for Server { async fn send_tls_aggregate_report(&self, events: Vec) { let (domain_name, event_from, event_to) = events .first() .map(|e| (e.domain.as_str(), e.seq_id, e.due)) .unwrap(); let span_id = self.inner.data.span_id_gen.generate(); trc::event!( OutgoingReport(OutgoingReportEvent::TlsAggregate), SpanId = span_id, ReportId = event_from, Domain = domain_name.to_string(), RangeFrom = trc::Value::Timestamp(event_from), RangeTo = trc::Value::Timestamp(event_to), ); // Generate report let mut rua = Vec::new(); let mut serialized_size = serde_json::Serializer::new(SerializedSize::new( self.eval_if( &self.core.smtp.report.tls.max_size, &RecipientDomain::new(domain_name), span_id, ) .await .unwrap_or(25 * 1024 * 1024), )); let report = match self .generate_tls_aggregate_report(&events, &mut rua, Some(&mut serialized_size), span_id) .await { Ok(Some(report)) => report, Ok(None) => { // This should not happen trc::event!( OutgoingReport(OutgoingReportEvent::NotFound), SpanId = span_id, CausedBy = trc::location!() ); self.delete_tls_report(events).await; return; } Err(err) => { trc::error!( err.span_id(span_id) .caused_by(trc::location!()) .details("Failed to read TLS report") ); return; } }; // Compress and serialize report let json = report.to_json(); let mut e = GzEncoder::new(Vec::with_capacity(json.len()), Compression::default()); let json = match std::io::Write::write_all(&mut e, json.as_bytes()).and_then(|_| e.finish()) { Ok(report) => report, Err(err) => { trc::event!( OutgoingReport(OutgoingReportEvent::SubmissionError), SpanId = span_id, Reason = err.to_string(), Details = "Failed to compress report" ); self.delete_tls_report(events).await; return; } }; // Try delivering report over HTTP let mut rcpts = Vec::with_capacity(rua.len()); for uri in &rua { match uri { ReportUri::Http(uri) => { if let Ok(client) = reqwest::Client::builder() .user_agent(USER_AGENT) .timeout(Duration::from_secs(2 * 60)) .build() { #[cfg(feature = "test_mode")] if uri == "https://127.0.0.1/tls" { TLS_HTTP_REPORT.lock().extend_from_slice(&json); self.delete_tls_report(events).await; return; } match client .post(uri) .header(CONTENT_TYPE, "application/tlsrpt+gzip") .body(json.to_vec()) .send() .await { Ok(response) => { if response.status().is_success() { trc::event!( OutgoingReport(OutgoingReportEvent::HttpSubmission), SpanId = span_id, Url = uri.to_string(), Code = response.status().as_u16(), ); self.delete_tls_report(events).await; return; } else { trc::event!( OutgoingReport(OutgoingReportEvent::SubmissionError), SpanId = span_id, Url = uri.to_string(), Code = response.status().as_u16(), Details = "Invalid HTTP response" ); } } Err(err) => { trc::event!( OutgoingReport(OutgoingReportEvent::SubmissionError), SpanId = span_id, Url = uri.to_string(), Reason = err.to_string(), Details = "HTTP submission error" ); } } } } ReportUri::Mail(mailto) => { rcpts.push(mailto.as_str()); } } } // Deliver report over SMTP if !rcpts.is_empty() { let config = &self.core.smtp.report.tls; let from_addr = self .eval_if(&config.address, &RecipientDomain::new(domain_name), span_id) .await .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()); let mut message = Vec::with_capacity(2048); let _ = report.write_rfc5322_from_bytes( domain_name, &self .eval_if( &self.core.smtp.report.submitter, &RecipientDomain::new(domain_name), span_id, ) .await .unwrap_or_else(|| "localhost".to_string()), ( self.eval_if(&config.name, &RecipientDomain::new(domain_name), span_id) .await .unwrap_or_else(|| "Mail Delivery Subsystem".to_string()) .as_str(), from_addr.as_str(), ), rcpts.iter().copied(), &json, &mut message, ); // Send report self.send_report( &from_addr, rcpts.iter(), message, &config.sign, false, span_id, ) .await; } else { trc::event!( OutgoingReport(OutgoingReportEvent::NoRecipientsFound), SpanId = span_id, ); } self.delete_tls_report(events).await; } async fn generate_tls_aggregate_report( &self, events: &[ReportEvent], rua: &mut Vec, mut serialized_size: Option<&mut serde_json::Serializer>, span_id: u64, ) -> trc::Result> { let (domain_name, event_from, event_to, policy) = events .first() .map(|e| (e.domain.as_str(), e.seq_id, e.due, e.policy_hash)) .unwrap(); let config = &self.core.smtp.report.tls; let mut report = TlsReport { organization_name: self .eval_if::( &config.org_name, &RecipientDomain::new(domain_name), span_id, ) .await .clone(), date_range: DateRange { start_datetime: DateTime::from_timestamp(event_from as i64), end_datetime: DateTime::from_timestamp(event_to as i64), }, contact_info: self .eval_if::( &config.contact_info, &RecipientDomain::new(domain_name), span_id, ) .await .clone(), report_id: format!("{}_{}", event_from, policy), policies: Vec::with_capacity(events.len()), }; if let Some(serialized_size) = serialized_size.as_deref_mut() { let _ = serde::Serialize::serialize(&report, serialized_size); } for event in events { let tls = if let Some(tls) = self .store() .get_value::>(ValueKey::from(ValueClass::Queue( QueueClass::TlsReportHeader(event.clone()), ))) .await? { tls.deserialize::()? } else { continue; }; if let Some(serialized_size) = serialized_size.as_deref_mut() && serde::Serialize::serialize(&tls, serialized_size).is_err() { continue; } // Group duplicates let mut total_success = 0; let mut total_failure = 0; let from_key = ValueKey::from(ValueClass::Queue(QueueClass::TlsReportEvent(ReportEvent { due: event.due, policy_hash: event.policy_hash, seq_id: 0, domain: event.domain.clone(), }))); let to_key = ValueKey::from(ValueClass::Queue(QueueClass::TlsReportEvent(ReportEvent { due: event.due, policy_hash: event.policy_hash, seq_id: u64::MAX, domain: event.domain.clone(), }))); let mut record_map = AHashMap::new(); self.core .storage .data .iterate(IterateParams::new(from_key, to_key).ascending(), |_, v| { let archive = as Deserialize>::deserialize(v)?; if let Some(failure_details) = archive.deserialize::>()? { match record_map.entry(failure_details) { Entry::Occupied(mut e) => { total_failure += 1; *e.get_mut() += 1; Ok(true) } Entry::Vacant(e) => { if serialized_size .as_deref_mut() .is_none_or(|serialized_size| { serde::Serialize::serialize(e.key(), serialized_size) .is_ok() }) { total_failure += 1; e.insert(1u32); Ok(true) } else { Ok(false) } } } } else { total_success += 1; Ok(true) } }) .await .caused_by(trc::location!())?; // Add policy report.policies.push(Policy { policy: tls.policy, summary: Summary { total_success, total_failure, }, failure_details: record_map .into_iter() .map(|(mut r, count)| { r.failed_session_count = count; r }) .collect(), }); // Add report URIs for entry in tls.rua { if !rua.contains(&entry) { rua.push(entry); } } } Ok(if !report.policies.is_empty() { Some(report) } else { None }) } async fn schedule_tls(&self, event: Box) { let created = event.interval.to_timestamp(); let deliver_at = created + event.interval.as_secs(); let mut report_event = ReportEvent { due: deliver_at, policy_hash: event.policy.to_hash(), seq_id: created, domain: event.domain, }; // Write policy if missing let mut builder = BatchBuilder::new(); if self .core .storage .data .get_value::<()>(ValueKey::from(ValueClass::Queue( QueueClass::TlsReportHeader(report_event.clone()), ))) .await .unwrap_or_default() .is_none() { // Serialize report let mut policy = PolicyDetails { policy_type: PolicyType::NoPolicyFound, policy_string: vec![], policy_domain: report_event.domain.clone(), mx_host: vec![], }; match event.policy { common::ipc::PolicyType::Tlsa(tlsa) => { policy.policy_type = PolicyType::Tlsa; if let Some(tlsa) = tlsa { for entry in &tlsa.entries { policy.policy_string.push(format!( "{} {} {} {}", if entry.is_end_entity { 3 } else { 2 }, i32::from(entry.is_spki), if entry.is_sha256 { 1 } else { 2 }, entry .data .iter() .fold(String::with_capacity(64), |mut s, b| { write!(s, "{b:02X}").ok(); s }) )); } } } common::ipc::PolicyType::Sts(sts) => { policy.policy_type = PolicyType::Sts; if let Some(sts) = sts { policy.policy_string.push("version: STSv1".to_string()); policy.policy_string.push(format!( "mode: {}", match sts.mode { Mode::Enforce => "enforce", Mode::Testing => "testing", Mode::None => "none", } )); policy .policy_string .push(format!("max_age: {}", sts.max_age)); for mx in &sts.mx { let mx = match mx { MxPattern::Equals(mx) => mx.to_string(), MxPattern::StartsWith(mx) => format!("*.{mx}"), }; policy.policy_string.push(format!("mx: {mx}")); policy.mx_host.push(mx); } } } _ => (), } // Create report entry let entry = TlsFormat { rua: event.tls_record.rua.clone(), policy, records: vec![], }; // Write report builder.set( ValueClass::Queue(QueueClass::TlsReportHeader(report_event.clone())), match Archiver::new(entry).serialize() { Ok(data) => data.to_vec(), Err(err) => { trc::error!( err.caused_by(trc::location!()) .details("Failed to serialize TLS report") ); return; } }, ); } // Write entry report_event.seq_id = self.inner.data.queue_id_gen.generate(); builder.set( ValueClass::Queue(QueueClass::TlsReportEvent(report_event)), match Archiver::new(event.failure).serialize() { Ok(data) => data.to_vec(), Err(err) => { trc::error!( err.caused_by(trc::location!()) .details("Failed to serialize TLS report") ); return; } }, ); if let Err(err) = self.core.storage.data.write(builder.build_all()).await { trc::error!( err.caused_by(trc::location!()) .details("Failed to write TLS report") ); } } async fn delete_tls_report(&self, events: Vec) { let mut batch = BatchBuilder::new(); for event in events { let from_key = ReportEvent { due: event.due, policy_hash: event.policy_hash, seq_id: 0, domain: event.domain.clone(), }; let to_key = ReportEvent { due: event.due, policy_hash: event.policy_hash, seq_id: u64::MAX, domain: event.domain.clone(), }; // Remove report events if let Err(err) = self .core .storage .data .delete_range( ValueKey::from(ValueClass::Queue(QueueClass::TlsReportEvent(from_key))), ValueKey::from(ValueClass::Queue(QueueClass::TlsReportEvent(to_key))), ) .await { trc::error!( err.caused_by(trc::location!()) .details("Failed to delete TLS reports") ); return; } // Remove report header batch.clear(ValueClass::Queue(QueueClass::TlsReportHeader(event))); } if let Err(err) = self.core.storage.data.write(batch.build_all()).await { trc::error!( err.caused_by(trc::location!()) .details("Failed to delete TLS reports") ); } } } ================================================ FILE: crates/smtp/src/scripts/envelope.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use sieve::Envelope; use smtp_proto::{ MAIL_BY_NOTIFY, MAIL_BY_RETURN, MAIL_BY_TRACE, MAIL_RET_FULL, MAIL_RET_HDRS, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, }; use utils::DomainPart; use crate::core::{SessionAddress, SessionData}; impl SessionData { pub fn apply_envelope_modification(&mut self, envelope: Envelope, value: String) { match envelope { Envelope::From => { let (address, address_lcase, domain) = if value.contains('@') { let address_lcase = value.to_lowercase(); let domain = address_lcase.domain_part().into(); (value, address_lcase, domain) } else if value.is_empty() { (String::new(), String::new(), String::new()) } else { return; }; if let Some(mail_from) = &mut self.mail_from { mail_from.address = address; mail_from.address_lcase = address_lcase; mail_from.domain = domain; } else { self.mail_from = SessionAddress { address, address_lcase, domain, flags: 0, dsn_info: None, } .into(); } } Envelope::To => { if value.contains('@') { let address_lcase = value.to_lowercase(); let domain = address_lcase.domain_part().into(); if let Some(rcpt_to) = self.rcpt_to.last_mut() { rcpt_to.address = value; rcpt_to.address_lcase = address_lcase; rcpt_to.domain = domain; } else { self.rcpt_to.push(SessionAddress { address: value, address_lcase, domain, flags: 0, dsn_info: None, }); } } } Envelope::ByMode => { if let Some(mail_from) = &mut self.mail_from { mail_from.flags &= !(MAIL_BY_NOTIFY | MAIL_BY_RETURN); if value == "N" { mail_from.flags |= MAIL_BY_NOTIFY; } else if value == "R" { mail_from.flags |= MAIL_BY_RETURN; } } } Envelope::ByTrace => { if let Some(mail_from) = &mut self.mail_from { if value == "T" { mail_from.flags |= MAIL_BY_TRACE; } else { mail_from.flags &= !MAIL_BY_TRACE; } } } Envelope::Notify => { if let Some(rcpt_to) = self.rcpt_to.last_mut() { rcpt_to.flags &= !(RCPT_NOTIFY_DELAY | RCPT_NOTIFY_FAILURE | RCPT_NOTIFY_SUCCESS | RCPT_NOTIFY_NEVER); if value == "NEVER" { rcpt_to.flags |= RCPT_NOTIFY_NEVER; } else { for value in value.split(',') { match value.trim() { "SUCCESS" => rcpt_to.flags |= RCPT_NOTIFY_SUCCESS, "FAILURE" => rcpt_to.flags |= RCPT_NOTIFY_FAILURE, "DELAY" => rcpt_to.flags |= RCPT_NOTIFY_DELAY, _ => (), } } } } } Envelope::Ret => { if let Some(mail_from) = &mut self.mail_from { mail_from.flags &= !(MAIL_RET_FULL | MAIL_RET_HDRS); if value == "FULL" { mail_from.flags |= MAIL_RET_FULL; } else if value == "HDRS" { mail_from.flags |= MAIL_RET_HDRS; } } } Envelope::Orcpt => { if let Some(rcpt_to) = self.rcpt_to.last_mut() { rcpt_to.dsn_info = value.into(); } } Envelope::Envid => { if let Some(mail_from) = &mut self.mail_from { mail_from.dsn_info = value.into(); } } Envelope::ByTimeAbsolute | Envelope::ByTimeRelative => (), } } } ================================================ FILE: crates/smtp/src/scripts/event_loop.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ inbound::DkimSign, queue::{MessageSource, quota::HasQueueQuota, spool::SmtpSpool}, }; use common::{Server, config::smtp::queue::QueueExpiry, scripts::plugins::PluginContext}; use mail_auth::common::headers::HeaderWriter; use mail_parser::{Encoding, Message, MessagePart, PartType}; use sieve::{ Event, Input, MatchAs, Recipient, Sieve, compiler::grammar::actions::action_redirect::{ByMode, ByTime, Notify, NotifyItem, Ret}, }; use smtp_proto::{ MAIL_BY_TRACE, MAIL_RET_FULL, MAIL_RET_HDRS, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, }; use std::{borrow::Cow, future::Future, sync::Arc, time::Instant}; use trc::SieveEvent; use super::{ScriptModification, ScriptParameters, ScriptResult}; pub trait RunScript: Sync + Send { fn run_script( &self, script_id: String, script: Arc, params: ScriptParameters<'_>, ) -> impl Future + Send; } impl RunScript for Server { async fn run_script( &self, script_id: String, script: Arc, params: ScriptParameters<'_>, ) -> ScriptResult { // Create filter instance let time = Instant::now(); let mut instance = self .core .sieve .trusted_runtime .filter_parsed(params.message.unwrap_or_else(|| Message { parts: vec![MessagePart { headers: vec![], is_encoding_problem: false, body: PartType::Text("".into()), encoding: Encoding::None, offset_header: 0, offset_body: 0, offset_end: 0, }], raw_message: b""[..].into(), ..Default::default() })) .with_vars_env(params.variables) .with_envelope_list(params.envelope) .with_user_address(¶ms.from_addr) .with_user_full_name(¶ms.from_name); let mut input = Input::script("__script", script); let mut messages: Vec> = Vec::new(); let session_id = params.session_id; let mut reject_reason = None; let mut modifications = vec![]; let mut keep_id = usize::MAX; // Start event loop while let Some(result) = instance.run(input) { match result { Ok(event) => match event { Event::IncludeScript { name, optional } => { let name_ = name.as_str().to_lowercase(); if let Some(script) = self.core.sieve.trusted_scripts.get(&name_) { input = Input::script(name, script.clone()); } else if optional { input = false.into(); } else { trc::event!( Sieve(SieveEvent::ScriptNotFound), Id = script_id.clone(), SpanId = session_id, Details = name_, ); break; } } Event::ListContains { lists, values, match_as, } => { input = false.into(); 'outer: for list in lists { if let Some(store) = self.core.storage.lookups.get(&list) { for value in &values { if let Ok(true) = store .key_exists(if !matches!(match_as, MatchAs::Lowercase) { value.clone() } else { value.to_lowercase() }) .await { input = true.into(); break 'outer; } } } else { trc::event!( Sieve(SieveEvent::ListNotFound), Id = script_id.clone(), SpanId = session_id, Details = list, ); } } } Event::Function { id, arguments } => { input = self .core .run_plugin( id, PluginContext { session_id, server: self, message: instance.message(), modifications: &mut modifications, access_token: params.access_token, arguments, }, ) .await; } Event::Keep { message_id, .. } => { keep_id = message_id; input = true.into(); } Event::Discard => { keep_id = usize::MAX - 1; input = true.into(); } Event::Reject { reason, .. } => { reject_reason = reason.into(); input = true.into(); } Event::SendMessage { recipient, notify, return_of_content, by_time, message_id, } => { // Build message let mut message = self.new_message(params.return_path.as_str(), session_id); match recipient { Recipient::Address(rcpt) => { message.add_recipient(rcpt, self).await; } Recipient::Group(rcpt_list) => { for rcpt in rcpt_list { message.add_recipient(rcpt, self).await; } } Recipient::List(list) => { trc::event!( Sieve(SieveEvent::NotSupported), Id = script_id.clone(), SpanId = session_id, Details = list, Reason = "Sending to lists is not supported.", ); } } // Set notify flags let mut flags = 0; match notify { Notify::Never => { flags = RCPT_NOTIFY_NEVER; } Notify::Items(items) => { for item in items { flags |= match item { NotifyItem::Success => RCPT_NOTIFY_SUCCESS, NotifyItem::Failure => RCPT_NOTIFY_FAILURE, NotifyItem::Delay => RCPT_NOTIFY_DELAY, }; } } Notify::Default => (), } if flags > 0 { for rcpt in &mut message.message.recipients { rcpt.flags |= flags; } } // Set ByTime flags match by_time { ByTime::Relative { rlimit, mode, trace, } => { if trace { message.message.flags |= MAIL_BY_TRACE; } match mode { ByMode::Notify => { for domain in &mut message.message.recipients { domain.notify.due += rlimit; } } ByMode::Return => { for domain in &mut message.message.recipients { domain.notify.due += rlimit; } } ByMode::Default => (), } } ByTime::Absolute { alimit, mode, trace, } => { if trace { message.message.flags |= MAIL_BY_TRACE; } match mode { ByMode::Notify => { for domain in &mut message.message.recipients { domain.notify.due = alimit as u64; } } ByMode::Return => { let expires = (alimit as u64).saturating_sub(message.message.created); if expires > 0 { for domain in &mut message.message.recipients { domain.expires = QueueExpiry::Ttl(expires); } } } ByMode::Default => (), } } ByTime::None => (), }; // Set ret match return_of_content { Ret::Full => { message.message.flags |= MAIL_RET_FULL; } Ret::Hdrs => { message.message.flags |= MAIL_RET_HDRS; } Ret::Default => (), } // Queue message let is_forward = message_id == 0; let raw_message = if !is_forward { messages.get(message_id - 1).map(|m| m.as_slice()) } else { instance.message().raw_message().into() }; if let Some(raw_message) = raw_message.filter(|m| !m.is_empty()) { let headers = if !params.sign.is_empty() { let mut headers = Vec::new(); for dkim in ¶ms.sign { if let Some(dkim) = self.get_dkim_signer(dkim, session_id) { match dkim.sign(raw_message) { Ok(signature) => { signature.write_header(&mut headers); } Err(err) => { trc::error!( trc::Error::from(err) .span_id(session_id) .caused_by(trc::location!()) .details("DKIM sign failed") ); } } } } if is_forward { headers.extend_from_slice(params.headers.unwrap_or_default()); } Some(Cow::Owned(headers)) } else if is_forward { params.headers.map(Cow::Borrowed) } else { None }; if self.has_quota(&mut message).await { message .queue( headers.as_deref(), raw_message, session_id, self, MessageSource::Autogenerated, ) .await; } else { trc::event!( Sieve(SieveEvent::QuotaExceeded), SpanId = session_id, Id = script_id.clone(), From = message.message.return_path, To = message .message .recipients .into_iter() .map(|r| trc::Value::from(r.address().to_string())) .collect::>(), ); } } input = true.into(); } Event::CreatedMessage { message, .. } => { messages.push(message); input = true.into(); } Event::SetEnvelope { envelope, value } => { modifications.push(ScriptModification::SetEnvelope { name: envelope, value, }); input = true.into(); } unsupported => { trc::event!( Sieve(SieveEvent::NotSupported), Id = script_id.clone(), SpanId = session_id, Reason = "Unsupported event", Details = format!("{unsupported:?}"), ); break; } }, Err(err) => { trc::event!( Sieve(SieveEvent::RuntimeError), Id = script_id.clone(), SpanId = session_id, Reason = err.to_string(), ); break; } } } // Keep id // 0 = use original message // MAX = implicit keep // MAX - 1 = discard message if keep_id == 0 { trc::event!( Sieve(SieveEvent::ActionAccept), SpanId = session_id, Id = script_id, Elapsed = time.elapsed(), ); ScriptResult::Accept { modifications } } else if let Some(mut reject_reason) = reject_reason { trc::event!( Sieve(SieveEvent::ActionReject), Id = script_id, SpanId = session_id, Details = reject_reason.clone(), Elapsed = time.elapsed(), ); if !reject_reason.ends_with('\n') { reject_reason.push_str("\r\n"); } let mut reject_bytes = reject_reason.as_bytes().iter(); if matches!(reject_bytes.next(), Some(ch) if ch.is_ascii_digit()) && matches!(reject_bytes.next(), Some(ch) if ch.is_ascii_digit()) && matches!(reject_bytes.next(), Some(ch) if ch.is_ascii_digit()) && matches!(reject_bytes.next(), Some(ch) if ch == &b' ' ) { ScriptResult::Reject(reject_reason) } else { ScriptResult::Reject(format!("503 5.5.3 {reject_reason}")) } } else if keep_id != usize::MAX - 1 { if let Some(message) = messages.into_iter().nth(keep_id - 1) { trc::event!( Sieve(SieveEvent::ActionAccept), SpanId = session_id, Id = script_id, Elapsed = time.elapsed(), ); ScriptResult::Replace { message, modifications, } } else { trc::event!( Sieve(SieveEvent::ActionAcceptReplace), SpanId = session_id, Id = script_id, Elapsed = time.elapsed(), ); ScriptResult::Accept { modifications } } } else { trc::event!( Sieve(SieveEvent::ActionDiscard), SpanId = session_id, Id = script_id, Elapsed = time.elapsed() ); ScriptResult::Discard } } } ================================================ FILE: crates/smtp/src/scripts/exec.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{sync::Arc, time::SystemTime}; use common::listener::SessionStream; use mail_auth::common::resolver::ToReverseName; use sieve::{Envelope, Sieve, runtime::Variable}; use smtp_proto::*; use crate::{core::Session, inbound::AuthResult}; use super::{ScriptParameters, ScriptResult, event_loop::RunScript}; impl Session { pub fn build_script_parameters(&self, stage: &'static str) -> ScriptParameters<'_> { let (tls_version, tls_cipher) = self.stream.tls_version_and_cipher(); let mut params = ScriptParameters::new() .set_variable("remote_ip", self.data.remote_ip.to_string()) .set_variable("remote_ip.reverse", self.data.remote_ip.to_reverse_name()) .set_variable("helo_domain", self.data.helo_domain.as_str().to_lowercase()) .set_variable( "authenticated_as", self.authenticated_as().unwrap_or_default().to_string(), ) .set_variable( "now", SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()), ) .set_variable( "asn", self.data .asn_geo_data .asn .as_ref() .map(|r| r.id) .unwrap_or_default(), ) .set_variable( "country", self.data .asn_geo_data .country .as_ref() .map(|r| r.as_str()) .unwrap_or_default(), ) .set_variable( "spf.result", self.data .spf_mail_from .as_ref() .map(|r| r.result().as_str()) .unwrap_or_default(), ) .set_variable( "spf_ehlo.result", self.data .spf_ehlo .as_ref() .map(|r| r.result().as_str()) .unwrap_or_default(), ) .set_variable("tls.version", tls_version) .set_variable("tls.cipher", tls_cipher) .set_variable("stage", stage); if let Some(ip_rev) = &self.data.iprev { params = params.set_variable("iprev.result", ip_rev.result().as_str()); if let Some(ptr) = ip_rev.ptr.as_ref().and_then(|addrs| addrs.first()) { params = params.set_variable( "iprev.ptr", ptr.strip_suffix('.').unwrap_or(ptr).to_lowercase(), ); } } if let Some(mail_from) = &self.data.mail_from { params .envelope .push((Envelope::From, mail_from.address_lcase.to_string().into())); if let Some(env_id) = &mail_from.dsn_info { params .envelope .push((Envelope::Envid, env_id.as_str().to_lowercase().into())); } if stage != "data" { if let Some(rcpt) = self.data.rcpt_to.last() { params .envelope .push((Envelope::To, rcpt.address_lcase.to_string().into())); if let Some(orcpt) = &rcpt.dsn_info { params .envelope .push((Envelope::Orcpt, orcpt.as_str().to_lowercase().into())); } } } else { // Build recipients list let mut recipients = vec![]; for rcpt in &self.data.rcpt_to { recipients.push(Variable::from(rcpt.address_lcase.to_string())); } params.envelope.push((Envelope::To, recipients.into())); } if (mail_from.flags & MAIL_RET_FULL) != 0 { params.envelope.push((Envelope::Ret, "FULL".into())); } else if (mail_from.flags & MAIL_RET_HDRS) != 0 { params.envelope.push((Envelope::Ret, "HDRS".into())); } if (mail_from.flags & MAIL_BY_NOTIFY) != 0 { params.envelope.push((Envelope::ByMode, "N".into())); } else if (mail_from.flags & MAIL_BY_RETURN) != 0 { params.envelope.push((Envelope::ByMode, "R".into())); } if (mail_from.flags & MAIL_BODY_7BIT) != 0 { params = params.set_variable("param.body", "7bit"); } else if (mail_from.flags & MAIL_BODY_8BITMIME) != 0 { params = params.set_variable("param.body", "8bitmime"); } else if (mail_from.flags & MAIL_BODY_BINARYMIME) != 0 { params = params.set_variable("param.body", "binarymime"); } if (mail_from.flags & MAIL_SMTPUTF8) != 0 { params = params.set_variable("param.smtputf8", Variable::Integer(1)); } if (mail_from.flags & MAIL_REQUIRETLS) != 0 { params = params.set_variable("param.requiretls", Variable::Integer(1)); } } params } pub async fn run_script( &self, script_id: String, script: Arc, params: ScriptParameters<'_>, ) -> ScriptResult { self.server .run_script( script_id, script, params .with_session_id(self.data.session_id) .with_envelope(&self.server, self, self.data.session_id) .await, ) .await } } ================================================ FILE: crates/smtp/src/scripts/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::borrow::Cow; use ahash::AHashMap; use common::{ Server, auth::AccessToken, expr::functions::ResolveVariable, scripts::ScriptModification, }; use mail_parser::Message; use sieve::{Envelope, runtime::Variable}; pub mod envelope; pub mod event_loop; pub mod exec; #[derive(Debug, serde::Serialize)] pub enum ScriptResult { Accept { modifications: Vec, }, Replace { message: Vec, modifications: Vec, }, Reject(String), Discard, } pub struct ScriptParameters<'x> { message: Option>, headers: Option<&'x [u8]>, variables: AHashMap, Variable>, envelope: Vec<(Envelope, Variable)>, from_addr: String, from_name: String, return_path: String, sign: Vec, access_token: Option<&'x AccessToken>, session_id: u64, } impl<'x> ScriptParameters<'x> { pub fn new() -> Self { ScriptParameters { variables: AHashMap::with_capacity(10), envelope: Vec::with_capacity(6), message: None, headers: None, from_addr: Default::default(), from_name: Default::default(), return_path: Default::default(), sign: Default::default(), access_token: None, session_id: Default::default(), } } pub async fn with_envelope( mut self, server: &Server, vars: &impl ResolveVariable, session_id: u64, ) -> Self { for (variable, expr) in [ (&mut self.from_addr, &server.core.sieve.from_addr), (&mut self.from_name, &server.core.sieve.from_name), (&mut self.return_path, &server.core.sieve.return_path), ] { if let Some(value) = server.eval_if(expr, vars, session_id).await { *variable = value; } } if let Some(value) = server .eval_if(&server.core.sieve.sign, vars, session_id) .await { self.sign = value; } self } pub fn with_message(self, message: Message<'x>) -> Self { Self { message: message.into(), ..self } } pub fn with_auth_headers(self, headers: &'x [u8]) -> Self { Self { headers: headers.into(), ..self } } pub fn set_variable( mut self, name: impl Into>, value: impl Into, ) -> Self { self.variables.insert(name.into(), value.into()); self } pub fn set_envelope(mut self, envelope: Envelope, value: impl Into) -> Self { self.envelope.push((envelope, value.into())); self } pub fn with_access_token(mut self, access_token: &'x AccessToken) -> Self { self.access_token = Some(access_token); self } pub fn with_session_id(mut self, session_id: u64) -> Self { self.session_id = session_id; self } } impl Default for ScriptParameters<'_> { fn default() -> Self { Self::new() } } ================================================ FILE: crates/spam-filter/Cargo.toml ================================================ [package] name = "spam-filter" version = "0.15.5" edition = "2024" [dependencies] utils = { path = "../utils" } types = { path = "../types" } nlp = { path = "../nlp" } store = { path = "../store" } trc = { path = "../trc" } common = { path = "../common" } smtp-proto = { version = "0.2", features = ["rkyv"] } mail-parser = { version = "0.11", features = ["full_encoding"] } mail-builder = { version = "0.4" } mail-auth = { version = "0.7.1" } mail-send = { version = "0.5", default-features = false, features = ["cram-md5", "ring", "tls12"] } tokio = { version = "1.47", features = ["net", "macros"] } psl = "2" hyper = { version = "1.0.1", features = ["server", "http1", "http2"] } idna = "1.0" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-webpki-roots", "http2", "stream"]} decancer = "3.0.1" unicode-security = "0.1.0" infer = "0.19" sha1 = "0.10" sha2 = "0.10.6" compact_str = "0.9.0" rkyv = { version = "0.8.10", features = ["little_endian"] } serde = { version = "1.0", features = ["derive"]} unicode-general-category = "1.1.0" unicode-normalization = "0.1.25" [features] test_mode = [] enterprise = [] [dev-dependencies] tokio = { version = "1.47", features = ["full"] } ================================================ FILE: crates/spam-filter/src/analysis/classifier.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{SpamFilterContext, modules::classifier::SpamClassifier}; use common::Server; use std::future::Future; pub trait SpamFilterAnalyzeClassify: Sync + Send { fn spam_filter_analyze_classify( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; fn spam_filter_analyze_spam_trap( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; } impl SpamFilterAnalyzeClassify for Server { async fn spam_filter_analyze_classify(&self, ctx: &mut SpamFilterContext<'_>) { if self.core.spam.classifier.is_some() && !ctx.result.has_tag("SPAM_TRAP") && let Err(err) = self.spam_classify(ctx).await { trc::error!(err.span_id(ctx.input.span_id).caused_by(trc::location!())); } } async fn spam_filter_analyze_spam_trap(&self, ctx: &mut SpamFilterContext<'_>) -> bool { if let Some(store) = self.get_in_memory_store("spam-traps") { for addr in &ctx.output.env_to_addr { match store.key_exists(addr.address.as_str()).await { Ok(true) => { ctx.result.add_tag("SPAM_TRAP"); return true; } Ok(false) => (), Err(err) => { trc::error!(err.span_id(ctx.input.span_id).caused_by(trc::location!())); } } } } false } } ================================================ FILE: crates/spam-filter/src/analysis/date.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::future::Future; use common::Server; use mail_parser::HeaderName; use store::write::now; use crate::SpamFilterContext; pub trait SpamFilterAnalyzeDate: Sync + Send { fn spam_filter_analyze_date( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; } impl SpamFilterAnalyzeDate for Server { async fn spam_filter_analyze_date(&self, ctx: &mut SpamFilterContext<'_>) { match ctx .input .message .header(HeaderName::Date) .map(|h| h.as_datetime()) { Some(Some(date)) => { let date = date.to_timestamp(); if date != 0 { let date_diff = now() as i64 - date; if date_diff > 86400 { // Older than a day ctx.result.add_tag("DATE_IN_PAST"); } else if -date_diff > 7200 { //# More than 2 hours in the future ctx.result.add_tag("DATE_IN_FUTURE"); } } else { ctx.result.add_tag("INVALID_DATE"); } } Some(None) => { ctx.result.add_tag("INVALID_DATE"); } None => { ctx.result.add_tag("MISSING_DATE"); } } } } ================================================ FILE: crates/spam-filter/src/analysis/dmarc.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::future::Future; use common::Server; use mail_auth::{DkimResult, DmarcResult, SpfResult, dmarc::Policy}; use crate::SpamFilterContext; pub trait SpamFilterAnalyzeDmarc: Sync + Send { fn spam_filter_analyze_dmarc( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; } impl SpamFilterAnalyzeDmarc for Server { async fn spam_filter_analyze_dmarc(&self, ctx: &mut SpamFilterContext<'_>) { ctx.result.add_tag( ctx.input .spf_mail_from_result .map_or("SPF_NA", |r| match r.result() { SpfResult::Pass => "SPF_ALLOW", SpfResult::Fail => "SPF_FAIL", SpfResult::SoftFail => "SPF_SOFTFAIL", SpfResult::Neutral => "SPF_NEUTRAL", SpfResult::TempError => "SPF_DNSFAIL", SpfResult::PermError => "SPF_PERMFAIL", SpfResult::None => "SPF_NA", }), ); ctx.result.add_tag( match ctx .input .dkim_result .iter() .find(|r| matches!(r.result(), DkimResult::Pass)) .or_else(|| ctx.input.dkim_result.first()) .map(|r| r.result()) .unwrap_or(&DkimResult::None) { DkimResult::Pass => "DKIM_ALLOW", DkimResult::Fail(_) => "DKIM_REJECT", DkimResult::PermError(_) => "DKIM_PERMFAIL", DkimResult::TempError(_) => "DKIM_TEMPFAIL", DkimResult::Neutral(_) | DkimResult::None => "DKIM_NA", }, ); ctx.result .add_tag(ctx.input.arc_result.map_or("ARC_NA", |r| match r.result() { DkimResult::Pass => "ARC_ALLOW", DkimResult::Fail(_) => "ARC_REJECT", DkimResult::PermError(_) => "ARC_INVALID", DkimResult::TempError(_) => "ARC_DNSFAIL", DkimResult::Neutral(_) | DkimResult::None => "ARC_NA", })); ctx.result .add_tag(ctx.input.dmarc_result.map_or("DMARC_NA", |r| match r { DmarcResult::Pass => "DMARC_POLICY_ALLOW", DmarcResult::TempError(_) => "DMARC_DNSFAIL", DmarcResult::PermError(_) => "DMARC_BAD_POLICY", DmarcResult::None => "DMARC_NA", DmarcResult::Fail(_) => ctx.input.dmarc_policy.map_or( "DMARC_POLICY_SOFTFAIL", |p| match p { Policy::Quarantine => "DMARC_POLICY_QUARANTINE", Policy::Reject => "DMARC_POLICY_REJECT", Policy::Unspecified | Policy::None => "DMARC_POLICY_SOFTFAIL", }, ), })); } } ================================================ FILE: crates/spam-filter/src/analysis/domain.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ElementLocation, is_trusted_domain}; use crate::{ Email, Hostname, Recipient, SpamFilterContext, TextPart, modules::{ dnsbl::check_dnsbl, expression::StringResolver, html::{A, HREF, HtmlToken}, }, }; use common::{ Server, config::spamfilter::{Element, Location}, }; use mail_auth::DkimResult; use mail_parser::{HeaderName, HeaderValue, Host, parsers::MessageStream}; use nlp::tokenizers::types::TokenType; use std::{collections::HashSet, future::Future}; pub trait SpamFilterAnalyzeDomain: Sync + Send { fn spam_filter_analyze_domain( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; } impl SpamFilterAnalyzeDomain for Server { async fn spam_filter_analyze_domain(&self, ctx: &mut SpamFilterContext<'_>) { // Obtain email addresses and domains let mut domains: HashSet> = HashSet::new(); let mut emails: HashSet> = HashSet::new(); // Add DKIM domains for dkim in ctx.input.dkim_result { if dkim.result() == &DkimResult::Pass && let Some(domain) = dkim.signature().map(|s| &s.d) { domains.insert(ElementLocation::new( domain.to_lowercase(), Location::HeaderDkimPass, )); } } // Add Received headers for header in ctx.input.message.headers() { match (&header.name, &header.value) { (HeaderName::Received, HeaderValue::Received(received)) => { for host in [&received.from, &received.helo, &received.by] .into_iter() .flatten() { if let Host::Name(name) = host { let host = Hostname::new(name.as_ref()); if host.sld.is_some() { domains.insert(ElementLocation::new( host.fqdn, Location::HeaderReceived, )); } } } } (HeaderName::MessageId, value) => { if let Some(mid_domain) = value .as_text() .and_then(|s| s.rsplit_once('@')) .and_then(|(_, d)| { let host = Hostname::new(d); if host.sld.is_some() { Some(host) } else { None } }) { domains.insert(ElementLocation::new(mid_domain.fqdn, Location::HeaderMid)); } } (HeaderName::Other(name), _) if name.eq_ignore_ascii_case("Disposition-Notification-To") => { if let Some(address) = MessageStream::new( ctx.input .message .raw_message .get(header.offset_start as usize..header.offset_end as usize) .unwrap_or_default(), ) .parse_address() .as_address() { for addr in address.iter() { if let Some(email) = addr.address() { emails.insert(ElementLocation::new( Recipient { email: Email::new(email), name: None, }, Location::HeaderDnt, )); } } } } _ => (), } } // Add EHLO domain if ctx.output.ehlo_host.sld.is_some() { domains.insert(ElementLocation::new( ctx.output.ehlo_host.fqdn.clone(), Location::Ehlo, )); } // Add PTR if let Some(ptr) = &ctx.output.iprev_ptr { domains.insert(ElementLocation::new(ptr.clone(), Location::Tcp)); } // Add From, Envelope From and Reply-To emails.insert(ElementLocation::new( ctx.output.from.clone(), Location::HeaderFrom, )); if let Some(reply_to) = &ctx.output.reply_to { emails.insert(ElementLocation::new( reply_to.clone(), Location::HeaderReplyTo, )); } emails.insert(ElementLocation::new( Recipient { email: ctx.output.env_from_addr.clone(), name: None, }, Location::EnvelopeFrom, )); // Add emails found in the message for (part_id, part) in ctx.output.text_parts.iter().enumerate() { let part_id = part_id as u32; let is_body = ctx.input.message.text_body.contains(&part_id) || ctx.input.message.html_body.contains(&part_id); let tokens = match part { TextPart::Plain { tokens, .. } => tokens, TextPart::Html { tokens, html_tokens, .. } => { emails.extend(html_tokens.iter().filter_map(|token| { if let HtmlToken::StartTag { name: A, attributes, .. } = token { attributes.iter().find_map(|(attr, value)| { if *attr == HREF { let value = value.as_deref()?.strip_prefix("mailto:")?; let email = Email::new(value.split_once('?').map_or(value, |(e, _)| e)); if email.is_valid() { return Some(ElementLocation::new( Recipient { email, name: None }, if is_body { Location::BodyHtml } else { Location::Attachment }, )); } } None }) } else { None } })); tokens } TextPart::None => continue, }; for token in tokens { if let TokenType::Email(email) = token { if !ctx.input.is_train && is_body && !ctx.result.has_tag("RCPT_IN_BODY") { for rcpt in ctx.output.all_recipients() { if rcpt.email.address == email.address { ctx.result.add_tag("RCPT_IN_BODY"); break; } } } if email.is_valid() { emails.insert(ElementLocation::new( Recipient { email: email.clone(), name: None, }, if is_body { Location::BodyText } else { Location::Attachment }, )); } } } } if !ctx.input.is_train { // Validate email for email in &emails { // Skip trusted domains if !email.element.email.is_valid() || is_trusted_domain( self, &email.element.email.domain_part.fqdn, ctx.input.span_id, ) .await { continue; } // Check Email DNSBL check_dnsbl(self, ctx, &email.element, Element::Email, email.location).await; domains.insert(ElementLocation::new( email.element.email.domain_part.fqdn.clone(), email.location, )); } // Validate domains for domain in &domains { // Skip trusted domains if !is_trusted_domain(self, &domain.element, ctx.input.span_id).await { // Check Domain DNSBL check_dnsbl( self, ctx, &StringResolver(domain.element.as_str()), Element::Domain, domain.location, ) .await; } } } ctx.output.emails = emails; ctx.output.domains = domains; } } ================================================ FILE: crates/spam-filter/src/analysis/ehlo.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::future::Future; use common::Server; use crate::SpamFilterContext; pub trait SpamFilterAnalyzeEhlo: Sync + Send { fn spam_filter_analyze_ehlo( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; } impl SpamFilterAnalyzeEhlo for Server { async fn spam_filter_analyze_ehlo(&self, ctx: &mut SpamFilterContext<'_>) { if let Some(ehlo_ip) = ctx.output.ehlo_host.ip { // Helo host is bare ip ctx.result.add_tag("HELO_BAREIP"); if ehlo_ip != ctx.input.remote_ip { // Helo A IP != hostname IP ctx.result.add_tag("HELO_IP_A"); } } else if ctx.output.ehlo_host.sld.is_some() { if ctx .output .iprev_ptr .as_ref() .is_some_and(|ptr| *ptr != ctx.output.ehlo_host.fqdn) { // Helo does not match reverse IP ctx.result.add_tag("HELO_IPREV_MISMATCH"); } if matches!( ( self.dns_exists_ip(&ctx.output.ehlo_host.fqdn).await, self.dns_exists_mx(&ctx.output.ehlo_host.fqdn).await ), (Ok(false), Ok(false)) ) { // Helo no resolve to A or MX ctx.result.add_tag("HELO_NORES_A_OR_MX"); } } else { if ctx.output.ehlo_host.fqdn.contains("user") { // Helo host contains 'user' ctx.result.add_tag("RCVD_HELO_USER"); } // Helo not FQDN ctx.result.add_tag("HELO_NOT_FQDN"); } } } ================================================ FILE: crates/spam-filter/src/analysis/from.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{Email, SpamFilterContext}; use common::Server; use mail_parser::HeaderName; use nlp::tokenizers::types::{TokenType, TypesTokenizer}; use smtp_proto::{MAIL_BODY_8BITMIME, MAIL_BODY_BINARYMIME, MAIL_SMTPUTF8}; use std::future::Future; pub trait SpamFilterAnalyzeFrom: Sync + Send { fn spam_filter_analyze_from( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; } impl SpamFilterAnalyzeFrom for Server { async fn spam_filter_analyze_from(&self, ctx: &mut SpamFilterContext<'_>) { let mut from_count = 0; let mut from_raw = b"".as_slice(); let mut crt = None; let mut dnt = None; for header in ctx.input.message.headers() { match &header.name { HeaderName::From => { from_count += 1; from_raw = ctx .input .message .raw_message() .get(header.offset_start as usize..header.offset_end as usize) .unwrap_or_default(); } HeaderName::Other(name) => { if name.eq_ignore_ascii_case("X-Confirm-Reading-To") { crt = ctx .input .header_as_address(header) .map(|s| s.to_lowercase()); } else if name.eq_ignore_ascii_case("Disposition-Notification-To") { dnt = ctx .input .header_as_address(header) .map(|s| s.to_lowercase()); } } _ => {} } } match from_count { 0 => { ctx.result.add_tag("MISSING_FROM"); } 1 => {} _ => { ctx.result.add_tag("MULTIPLE_FROM"); } } let env_from_empty = ctx.output.env_from_addr.address.is_empty(); let from_addr = &ctx.output.from.email; let from_name = ctx.output.from.name.as_deref().unwrap_or_default(); if from_count > 0 { // Validate address let from_addr_is_valid = from_addr.is_valid(); if !from_addr_is_valid { ctx.result.add_tag("FROM_INVALID"); } // Validate from name let from_name_trimmed = from_name.trim(); if from_name_trimmed.is_empty() { ctx.result.add_tag("FROM_NO_DN"); } else if from_name_trimmed == from_addr.address { ctx.result.add_tag("FROM_DN_EQ_ADDR"); } else { if from_addr_is_valid { ctx.result.add_tag("FROM_HAS_DN"); } if from_name_trimmed.contains('@') && let Some(from_name_addr) = TypesTokenizer::new(from_name_trimmed) .tokenize_numbers(false) .tokenize_urls(false) .tokenize_urls_without_scheme(false) .tokenize_emails(true) .filter_map(|t| match t.word { TokenType::Email(email) => { let email = Email::new(email); email.is_valid().then_some(email) } _ => None, }) .next() { if (from_addr_is_valid && from_name_addr.domain_part.sld != from_addr.domain_part.sld) || (!env_from_empty && ctx.output.env_from_addr.domain_part.sld != from_name_addr.domain_part.sld) || (env_from_empty && ctx.output.ehlo_host.sld != from_name_addr.domain_part.sld) { ctx.result.add_tag("SPOOF_DISPLAY_NAME"); } else { ctx.result.add_tag("FROM_NEQ_DISPLAY_NAME"); } } } // Check sender if ctx.output.env_from_postmaster { ctx.result.add_tag("FROM_BOUNCE"); } if !env_from_empty && ctx.output.env_from_addr.address == from_addr.address { ctx.result.add_tag("FROM_EQ_ENV_FROM"); } else if from_addr_is_valid { if from_addr.domain_part.sld == ctx.output.ehlo_host.sld { ctx.result.add_tag("FROMTLD_EQ_ENV_FROMTLD"); } else if !ctx.output.env_from_postmaster { ctx.result.add_tag("FORGED_SENDER"); ctx.result.add_tag("FROM_NEQ_ENV_FROM"); } } // Validate FROM/TO relationship if ctx.output.recipients_to.len() + ctx.output.recipients_cc.len() == 1 { let rcpt = ctx .output .recipients_to .first() .or_else(|| ctx.output.recipients_cc.first()) .unwrap(); if rcpt.email.address == from_addr.address { ctx.result.add_tag("TO_EQ_FROM"); } else if rcpt.email.domain_part.fqdn == from_addr.domain_part.fqdn { ctx.result.add_tag("TO_DOM_EQ_FROM_DOM"); } } // Validate encoding let from_raw_utf8 = std::str::from_utf8(from_raw); if !from_raw.is_ascii() { if (ctx.input.env_from_flags & (MAIL_SMTPUTF8 | MAIL_BODY_8BITMIME | MAIL_BODY_BINARYMIME)) == 0 { ctx.result.add_tag("FROM_NEEDS_ENCODING"); } if from_raw_utf8.is_err() { ctx.result.add_tag("INVALID_FROM_8BIT"); } } // Validate unnecessary encoding let from_raw_utf8 = from_raw_utf8.unwrap_or_default(); if from_name.is_ascii() && from_addr.address.is_ascii() && from_raw_utf8.contains("=?") && from_raw_utf8.contains("?=") { if from_raw_utf8.contains("?q?") || from_raw_utf8.contains("?Q?") { // From header is unnecessarily encoded in quoted-printable ctx.result.add_tag("FROM_EXCESS_QP"); } else if from_raw_utf8.contains("?b?") || from_raw_utf8.contains("?B?") { // From header is unnecessarily encoded in base64 ctx.result.add_tag("FROM_EXCESS_BASE64"); } } // Validate space in FROM if !from_name.is_empty() && !from_addr.address.is_empty() && from_raw_utf8 .as_bytes() .iter() .position(|&b| b == b'<') .and_then(|v| from_raw_utf8.as_bytes().get(v - 1)) .is_none_or(|v| !v.is_ascii_whitespace()) { ctx.result.add_tag("NO_SPACE_IN_FROM"); } // Check whether read confirmation address is different to from address if let Some(crt) = crt && crt != from_addr.address { ctx.result.add_tag("HEADER_RCONFIRM_MISMATCH"); } } if !env_from_empty { // Validate envelope address if ctx.output.env_from_addr.is_valid() { // Mail from no resolve to A or MX if matches!( ( self.dns_exists_ip(&ctx.output.env_from_addr.domain_part.fqdn) .await, self.dns_exists_mx(&ctx.output.env_from_addr.domain_part.fqdn) .await ), (Ok(false), Ok(false)) ) { // Helo no resolve to A or MX ctx.result.add_tag("FROMHOST_NORES_A_OR_MX"); } } else { ctx.result.add_tag("ENV_FROM_INVALID"); } // Check whether disposition notification address is different to return path if let Some(dnt) = dnt && dnt != ctx.output.env_from_addr.address { ctx.result.add_tag("HEADER_FORGED_MDN"); } } } } ================================================ FILE: crates/spam-filter/src/analysis/headers.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::future::Future; use common::Server; use mail_parser::HeaderName; use store::ahash::AHashSet; use crate::SpamFilterContext; pub trait SpamFilterAnalyzeHeaders: Sync + Send { fn spam_filter_analyze_headers( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; } impl SpamFilterAnalyzeHeaders for Server { async fn spam_filter_analyze_headers(&self, ctx: &mut SpamFilterContext<'_>) { let mut list_score = 0.0; let mut unique_headers = AHashSet::new(); let raw_message = ctx.input.message.raw_message(); for header in ctx.input.message.headers() { // Add header exists tag let hdr_name = header.name(); let mut tag: String = String::with_capacity(hdr_name.len() + 5); tag.push_str("X_HDR_"); for ch in hdr_name.chars() { if ch.is_ascii_alphanumeric() { tag.push(ch.to_ascii_uppercase()); } else if ch == '-' { tag.push('_'); } else { tag.push(' '); } } ctx.result.add_tag(tag); match &header.name { HeaderName::ContentType | HeaderName::ContentTransferEncoding | HeaderName::Date | HeaderName::From | HeaderName::Sender | HeaderName::To | HeaderName::Cc | HeaderName::Bcc | HeaderName::ReplyTo | HeaderName::Subject | HeaderName::MessageId | HeaderName::References | HeaderName::InReplyTo => { if !unique_headers.insert(header.name.clone()) { ctx.result.add_tag("MULTIPLE_UNIQUE_HEADERS"); } let mut value = raw_message .get(header.offset_start as usize..) .unwrap_or_default() .iter(); loop { match value.next() { Some(b' ' | b'\t') => { break; } Some(b'\r' | b'\n') => {} _ => { ctx.result.add_tag("HEADER_EMPTY_DELIMITER"); break; } } } } HeaderName::ListArchive | HeaderName::ListOwner | HeaderName::ListHelp | HeaderName::ListPost => { list_score += 0.125; } HeaderName::ListId => { list_score += 0.5125; } HeaderName::ListSubscribe => { list_score += 0.25; } HeaderName::ListUnsubscribe => { list_score += 0.25; ctx.result.add_tag("HAS_LIST_UNSUB"); } HeaderName::Other(name) => { let value = header .value() .as_text() .unwrap_or_default() .trim() .to_lowercase(); if name.eq_ignore_ascii_case("Precedence") { if value == "bulk" { list_score += 0.25; ctx.result.add_tag("PRECEDENCE_BULK"); } else if value == "list" { list_score += 0.25; } } else if name.eq_ignore_ascii_case("X-Loop") { list_score += 0.125; } else if name.eq_ignore_ascii_case("X-Priority") { match value.parse::().unwrap_or(i32::MAX) { 0 => { ctx.result.add_tag("HAS_X_PRIO_ZERO"); } 1 => { ctx.result.add_tag("HAS_X_PRIO_ONE"); } 2 => { ctx.result.add_tag("HAS_X_PRIO_TWO"); } 3 | 4 => { ctx.result.add_tag("HAS_X_PRIO_THREE"); } 4..=10000 => { ctx.result.add_tag("HAS_X_PRIO_FIVE"); } _ => {} } } } _ => {} } } if list_score >= 1.0 { ctx.result.add_tag("MAILLIST"); } if unique_headers.is_empty() { ctx.result.add_tag("MISSING_ESSENTIAL_HEADERS"); } } } ================================================ FILE: crates/spam-filter/src/analysis/html.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::future::Future; use common::Server; use hyper::Uri; use mail_parser::MimeHeaders; use nlp::tokenizers::types::{TokenType, TypesTokenizer}; use crate::{Hostname, SpamFilterContext, TextPart, modules::html::*}; pub trait SpamFilterAnalyzeHtml: Sync + Send { fn spam_filter_analyze_html( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; } #[derive(Debug)] struct Href { url_parsed: Option, host: Option, } impl SpamFilterAnalyzeHtml for Server { async fn spam_filter_analyze_html(&self, ctx: &mut SpamFilterContext<'_>) { // Message only has text/html MIME parts if ctx.input.message.content_type().is_some_and(|ct| { ct.ctype().eq_ignore_ascii_case("text") && ct .subtype() .unwrap_or_default() .eq_ignore_ascii_case("html") }) { ctx.result.add_tag("MIME_HTML_ONLY"); } for (part_id, part) in ctx.output.text_parts.iter().enumerate() { let part_id = part_id as u32; let is_body_part = ctx.input.message.text_body.contains(&part_id) || ctx.input.message.html_body.contains(&part_id); let (html_tokens, tokens) = if let TextPart::Html { html_tokens, tokens, .. } = part { (html_tokens, tokens) } else { continue; }; let mut has_link_to_img = false; let mut last_href: Option = None; let mut html_img_words = 0; let mut in_head: i32 = 0; let mut in_body: i32 = 0; for token in html_tokens { match token { HtmlToken::StartTag { name, attributes, is_self_closing, } => match *name { A => { if let Some(attr) = attributes.iter().find_map(|(attr, value)| { if *attr == HREF { value.as_deref() } else { None } }) { let url = attr.trim().to_lowercase(); let url_parsed = url.parse::().ok(); let href = Href { host: url_parsed .as_ref() .and_then(|uri| uri.host().map(Hostname::new)), url_parsed, }; if is_body_part && attr.starts_with("data:") && attr.contains(";base64,") { // Has Data URI encoding ctx.result.add_tag("HAS_DATA_URI"); if attr.contains("text/") { // Uses Data URI encoding to obfuscate plain or HTML in base64 ctx.result.add_tag("DATA_URI_OBFU"); } } else if href.host.as_ref().is_some_and(|h| h.ip.is_some()) { // HTML anchor points to an IP address ctx.result.add_tag("HTTP_TO_IP"); } if !*is_self_closing { last_href = Some(href); } } } IMG if is_body_part => { let mut img_width = 800; let mut img_height = 600; for (attr, value) in attributes { if let Some(value) = value.as_deref().map(|v| v.trim()).filter(|v| !v.is_empty()) { let dimension = match *attr { WIDTH => &mut img_width, HEIGHT => &mut img_height, SRC => { let src = value.to_ascii_lowercase(); if src.starts_with("data:") && src.contains(";base64,") { // Has Data URI encoding ctx.result.add_tag("HAS_DATA_URI"); } else if src.starts_with("https://") || src.starts_with("http://") { // Has external image ctx.result.add_tag("HAS_EXTERNAL_IMG"); } continue; } _ => { continue; } }; if let Some(pct) = value.strip_suffix('%') { if let Ok(pct) = pct.trim().parse::() { *dimension = (*dimension * pct) / 100; } } else if let Ok(value) = value.parse::() { *dimension = value; } } } let dimensions = img_width + img_height; if last_href.is_some() { if dimensions >= 210 { ctx.result.add_tag("HAS_LINK_TO_LARGE_IMG"); has_link_to_img = true; } else { ctx.result.add_tag("HAS_LINK_TO_IMG"); } } if dimensions > 100 { // We assume that a single picture 100x200 contains approx 3 words of text html_img_words += dimensions / 100; } } META => { let mut has_equiv_refresh = false; let mut has_content_url = false; for (attr, value) in attributes { if let Some(value) = value.as_deref().map(|v| v.trim()).filter(|v| !v.is_empty()) { if *attr == HTTP_EQUIV { if value.eq_ignore_ascii_case("refresh") { has_equiv_refresh = true; } } else if *attr == CONTENT && value.to_ascii_lowercase().contains("url=") { has_content_url = true; } } } if has_equiv_refresh && has_content_url { // HTML meta refresh tag ctx.result.add_tag("HTML_META_REFRESH_URL"); } } LINK if is_body_part => { let mut has_rel_style = false; let mut has_href_css = false; for (attr, value) in attributes { if let Some(value) = value.as_deref().map(|v| v.trim()).filter(|v| !v.is_empty()) { if *attr == REL { if value.to_ascii_lowercase().contains("stylesheet") { has_rel_style = true; } } else if *attr == HREF && value.to_ascii_lowercase().contains(".css") { has_href_css = true; } } } if has_rel_style || has_href_css { // Has external CSS ctx.result.add_tag("EXT_CSS"); } } HEAD if !*is_self_closing => { in_head += 1; } BODY if !*is_self_closing => { in_body += 1; } _ => {} }, HtmlToken::EndTag { name } => match *name { A => { last_href = None; } HEAD => { in_head -= 1; } BODY => { in_body -= 1; } _ => (), }, HtmlToken::Text { text } if in_head == 0 => { if let Some((href_url, href_host)) = last_href .as_ref() .and_then(|href| Some((href.url_parsed.as_ref()?, href.host.as_ref()?))) { for token in TypesTokenizer::new(text.as_ref()) .tokenize_numbers(false) .tokenize_urls(true) .tokenize_urls_without_scheme(true) .tokenize_emails(true) { let text_url = match token.word { TokenType::Url(url) => url.to_lowercase(), TokenType::UrlNoScheme(url) => { format!("http://{}", url.to_lowercase()) } _ => continue, }; let text_url_parsed = if let Ok(text_url_parsed) = text_url.parse::() { text_url_parsed } else { continue; }; if href_url.scheme().map(|s| s.as_str()).unwrap_or_default() == "http" && text_url_parsed .scheme() .map(|s| s.as_str()) .unwrap_or_default() == "https" { // The anchor text contains a distinct scheme compared to the target URL ctx.result.add_tag("HTTP_TO_HTTPS"); } if let Some(text_url_host) = text_url_parsed.host() { let text_url_host = Hostname::new(text_url_host); if text_url_host.sld_or_default() != href_host.sld_or_default() { // The anchor text contains a different domain than the target URL ctx.result.add_tag("PHISHING"); } } } } } _ => (), } } if is_body_part { if in_head != 0 || in_body != 0 { // HTML tags are not properly closed ctx.result.add_tag("HTML_UNBALANCED_TAG"); } let mut html_words = 0; let mut html_uris = 0; let mut html_text_chars = 0; for token in tokens { match token { TokenType::Alphabetic(s) | TokenType::Alphanumeric(s) => { html_words += 1; html_text_chars += s.len(); } TokenType::Email(s) => { html_words += 1; html_text_chars += s.address.len(); } TokenType::Url(_) | TokenType::UrlNoScheme(_) => { html_uris += 1; } _ => (), } } match html_text_chars { 0..1024 => { ctx.result.add_tag("HTML_SHORT_1"); } 1024..1536 => { ctx.result.add_tag("HTML_SHORT_2"); } 1536..2048 => { ctx.result.add_tag("HTML_SHORT_3"); } _ => (), } if (!has_link_to_img || html_text_chars >= 2048) && (html_img_words as f64 / (html_words as f64 + html_img_words as f64) > 0.5) { // Message contains more images than text ctx.result.add_tag("HTML_TEXT_IMG_RATIO"); } if html_uris > 0 && html_words == 0 { // Message only contains URIs in HTML ctx.result.add_tag("BODY_URI_ONLY"); } } } } } ================================================ FILE: crates/spam-filter/src/analysis/init.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::Server; use mail_auth::DmarcResult; use mail_parser::{HeaderName, PartType, parsers::fields::thread::thread_name}; use nlp::tokenizers::types::{TokenType, TypesTokenizer}; use crate::{ Email, Hostname, IpParts, Recipient, SpamFilterContext, SpamFilterInput, SpamFilterOutput, SpamFilterResult, TextPart, modules::html::{HEAD, HtmlToken, html_to_tokens}, }; use super::url::UrlParts; pub trait SpamFilterInit { fn spam_filter_init<'x>(&self, input: SpamFilterInput<'x>) -> SpamFilterContext<'x>; } const POSTMASTER_ADDRESSES: [&str; 3] = ["postmaster", "mailer-daemon", "root"]; impl SpamFilterInit for Server { fn spam_filter_init<'x>(&self, mut input: SpamFilterInput<'x>) -> SpamFilterContext<'x> { let mut subject = ""; let mut from = None; let mut reply_to = None; let mut recipients_to = Vec::new(); let mut recipients_cc = Vec::new(); let mut recipients_bcc = Vec::new(); let mut found_spam_status = false; for header in input.message.headers() { match &header.name { HeaderName::To | HeaderName::Cc | HeaderName::Bcc => { if let Some(addrs) = header.value().as_address() { for addr in addrs.iter() { let rcpt = Recipient { email: Email::new(addr.address().unwrap_or_default()), name: addr.name().and_then(|s| { let s = s.trim(); if !s.is_empty() { Some(s.to_lowercase()) } else { None } }), }; if header.name == HeaderName::To { recipients_to.push(rcpt); } else if header.name == HeaderName::Cc { recipients_cc.push(rcpt); } else { recipients_bcc.push(rcpt); } } } } HeaderName::ReplyTo => { reply_to = header .value() .as_address() .and_then(|addrs| addrs.first()) .and_then(|addr| { Some(Recipient { email: Email::new(addr.address()?), name: addr.name().and_then(|s| { let s = s.trim(); if !s.is_empty() { Some(s.to_lowercase()) } else { None } }), }) }); } HeaderName::Subject => { subject = header.value().as_text().unwrap_or_default(); } HeaderName::From => { from = header.value().as_address().and_then(|addrs| addrs.first()); } HeaderName::Other(name) if input.is_train && !found_spam_status && name.eq("X-Spam-Result") => { for token in header .value() .as_text() .unwrap_or_default() .split_ascii_whitespace() { if let Some(dmarc) = token.strip_prefix("DMARC_") { input.dmarc_result = if dmarc == "POLICY_ALLOW" { Some(&DmarcResult::Pass) } else { Some(&DmarcResult::None) }; } else if let Some(asn) = token .strip_prefix("SOURCE_ASN_") .and_then(|v| v.parse().ok()) { input.asn = Some(asn); } } found_spam_status = true; } _ => {} } } // Tokenize subject let subject_tokens = TypesTokenizer::new(subject) .tokenize_numbers(false) .tokenize_urls(true) .tokenize_urls_without_scheme(true) .tokenize_emails(true) .map(|t| match t.word { TokenType::Alphabetic(s) => TokenType::Alphabetic(s.into()), TokenType::Alphanumeric(s) => TokenType::Alphanumeric(s.into()), TokenType::Integer(s) => TokenType::Integer(s.into()), TokenType::Other(s) => TokenType::Other(s), TokenType::Punctuation(s) => TokenType::Punctuation(s), TokenType::Space => TokenType::Space, TokenType::Url(url) => TokenType::Url(UrlParts::new(url)), TokenType::UrlNoHost(s) => TokenType::UrlNoHost(s.into()), TokenType::UrlNoScheme(s) => { TokenType::UrlNoScheme(UrlParts::new(format!("https://{}", s.trim()))) } TokenType::IpAddr(i) => TokenType::IpAddr(IpParts::new(i)), TokenType::Email(e) => TokenType::Email(Email::new(e)), TokenType::Float(s) => TokenType::Float(s.into()), }) .collect::>(); // Tokenize and convert text parts let mut text_parts = Vec::new(); let mut text_parts_nested = Vec::new(); let mut message_stack = Vec::new(); let mut message_iter = input.message.parts.iter(); loop { while let Some(part) = message_iter.next() { let is_main_message = message_stack.is_empty(); let text_part = match &part.body { PartType::Text(text) => TextPart::Plain { text_body: text.as_ref(), tokens: TypesTokenizer::new(text.as_ref()) .tokenize_numbers(false) .tokenize_urls(true) .tokenize_urls_without_scheme(true) .tokenize_emails(true) .map(|t| match t.word { TokenType::Alphabetic(s) => TokenType::Alphabetic(s.into()), TokenType::Alphanumeric(s) => TokenType::Alphanumeric(s.into()), TokenType::Integer(s) => TokenType::Integer(s.into()), TokenType::Other(s) => TokenType::Other(s), TokenType::Punctuation(s) => TokenType::Punctuation(s), TokenType::Space => TokenType::Space, TokenType::Url(url) => TokenType::Url(UrlParts::new(url)), TokenType::UrlNoHost(s) => TokenType::UrlNoHost(s.into()), TokenType::UrlNoScheme(s) => TokenType::UrlNoScheme(UrlParts::new( format!("https://{}", s.trim()), )), TokenType::IpAddr(i) => TokenType::IpAddr(IpParts::new(i)), TokenType::Email(e) => TokenType::Email(Email::new(e)), TokenType::Float(s) => TokenType::Float(s.into()), }) .collect::>(), }, PartType::Html(html) => { let html_tokens = html_to_tokens(html); let text_body_len = html_tokens .iter() .filter_map(|t| match t { HtmlToken::Text { text } => text.len().into(), _ => None, }) .sum(); let mut text_body = String::with_capacity(text_body_len); let mut in_head = false; for token in &html_tokens { match token { HtmlToken::StartTag { name: HEAD, .. } => { in_head = true; } HtmlToken::EndTag { name: HEAD } => { in_head = false; } HtmlToken::Text { text } if !in_head => { if !text_body.is_empty() && !text_body.ends_with(' ') && !text.starts_with(' ') { text_body.push(' '); } text_body.push_str(text) } _ => {} } } TextPart::Html { tokens: TypesTokenizer::new(&text_body) .tokenize_numbers(false) .tokenize_urls(true) .tokenize_urls_without_scheme(true) .tokenize_emails(true) .map(|t| match t.word { TokenType::Alphabetic(s) => { TokenType::Alphabetic(s.to_string().into()) } TokenType::Alphanumeric(s) => { TokenType::Alphanumeric(s.to_string().into()) } TokenType::Integer(s) => { TokenType::Integer(s.to_string().into()) } TokenType::Other(s) => TokenType::Other(s), TokenType::Punctuation(s) => TokenType::Punctuation(s), TokenType::Space => TokenType::Space, TokenType::Url(url) => { TokenType::Url(UrlParts::new(url.to_string())) } TokenType::UrlNoHost(s) => { TokenType::UrlNoHost(s.to_string().into()) } TokenType::UrlNoScheme(s) => TokenType::UrlNoScheme( UrlParts::new(format!("https://{}", s.trim())), ), TokenType::IpAddr(i) => TokenType::IpAddr(IpParts::new(i)), TokenType::Email(e) => TokenType::Email(Email::new(e)), TokenType::Float(s) => TokenType::Float(s.to_string().into()), }) .collect::>(), html_tokens, text_body, } } PartType::Message(message) => { message_stack.push(message_iter); message_iter = message.parts.iter(); TextPart::None } _ => TextPart::None, }; if is_main_message { text_parts.push(text_part); } else if !matches!(text_part, TextPart::None) { text_parts_nested.push(text_part); } } if let Some(iter) = message_stack.pop() { message_iter = iter; } else { break; } } text_parts.extend(text_parts_nested); let subject_thread = thread_name(subject).to_string(); let env_from_addr = Email::new(input.env_from); SpamFilterContext { output: SpamFilterOutput { ehlo_host: Hostname::new(input.ehlo_domain.unwrap_or("unknown")), iprev_ptr: input.iprev_result.and_then(|r| { r.ptr .as_ref() .and_then(|ptr| ptr.first()) .map(|ptr| (ptr.strip_suffix('.').unwrap_or(ptr)).to_lowercase()) }), env_from_postmaster: env_from_addr.address.is_empty() || POSTMASTER_ADDRESSES.contains(&env_from_addr.local_part.as_str()), env_from_addr, env_to_addr: input .env_rcpt_to .iter() .map(|rcpt| Email::new(rcpt)) .collect(), from: Recipient { email: Email::new(from.and_then(|f| f.address()).unwrap_or_default()), name: from.and_then(|f| f.name()).map(|name| name.to_lowercase()), }, reply_to, subject_thread_lc: subject_thread.trim().to_lowercase(), subject_thread, subject_lc: subject.trim().to_lowercase(), subject: subject.to_string(), subject_tokens, recipients_to, recipients_cc, recipients_bcc, text_parts, ips: Default::default(), emails: Default::default(), urls: Default::default(), domains: Default::default(), }, input, result: SpamFilterResult::default(), } } } ================================================ FILE: crates/spam-filter/src/analysis/ip.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::ElementLocation; use crate::{IpParts, SpamFilterContext, TextPart, modules::dnsbl::check_dnsbl}; use common::{ Server, config::spamfilter::{Element, IpResolver, Location}, }; use mail_auth::IprevResult; use mail_parser::{HeaderName, HeaderValue, Host}; use nlp::tokenizers::types::TokenType; use std::future::Future; use store::ahash::AHashSet; pub trait SpamFilterAnalyzeIp: Sync + Send { fn spam_filter_analyze_ip( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; } impl SpamFilterAnalyzeIp for Server { async fn spam_filter_analyze_ip(&self, ctx: &mut SpamFilterContext<'_>) { // IP Address RBL let mut ips = AHashSet::new(); ips.insert(ElementLocation::new(ctx.input.remote_ip, Location::Tcp)); // Obtain IP addresses from Received headers for header in ctx.input.message.headers() { if let (HeaderName::Received, HeaderValue::Received(received)) = (&header.name, &header.value) { if let Some(ip) = received.from_ip() && !ip.is_loopback() && !self.is_ip_allowed(&ip) { ips.insert(ElementLocation::new(ip, Location::HeaderReceived)); } for host in [&received.from, &received.helo, &received.by] .into_iter() .flatten() { if let Host::IpAddr(ip) = host && !ip.is_loopback() && !self.is_ip_allowed(ip) { ips.insert(ElementLocation::new(*ip, Location::HeaderReceived)); } } } } // Obtain IP addresses from the message body for (part_id, part) in ctx.output.text_parts.iter().enumerate() { let part_id = part_id as u32; let is_body = ctx.input.message.text_body.contains(&part_id) || ctx.input.message.html_body.contains(&part_id); match part { TextPart::Plain { tokens, .. } | TextPart::Html { tokens, .. } => { ips.extend(tokens.iter().filter_map(|t| { if let TokenType::IpAddr(ip) = t { ip.ip.map(|ip| { ElementLocation::new( ip, if is_body { Location::BodyText } else { Location::Attachment }, ) }) } else { None } })) } TextPart::None => (), } } // Validate IP addresses for ip in &ips { if ip.element.is_loopback() || ip.element.is_multicast() || ip.element.is_unspecified() || self.is_ip_allowed(&ip.element) { continue; } else if self.is_ip_blocked(&ip.element) { ctx.result.add_tag("IP_BLOCKED"); continue; } check_dnsbl( self, ctx, &IpResolver::new(ip.element), Element::Ip, ip.location, ) .await; } ctx.output.ips = ips; // Reverse DNS validation if let Some(iprev) = ctx.input.iprev_result { match &iprev.result { IprevResult::TempError(_) => ctx.result.add_tag("RDNS_DNSFAIL"), IprevResult::Fail(_) | IprevResult::PermError(_) => ctx.result.add_tag("RDNS_NONE"), IprevResult::Pass | IprevResult::None => (), } } // Add ASN if let Some(asn_id) = &ctx.input.asn { ctx.result.add_tag(format!("SOURCE_ASN_{asn_id}")); } } } impl IpParts { pub fn new(text: &str) -> IpParts { IpParts { ip: text.parse().ok(), } } } ================================================ FILE: crates/spam-filter/src/analysis/llm.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: LicenseRef-SEL */ use std::{future::Future, time::Instant}; use common::Server; use trc::AiEvent; use crate::SpamFilterContext; pub trait SpamFilterAnalyzeLlm: Sync + Send { fn spam_filter_analyze_llm( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; } impl SpamFilterAnalyzeLlm for Server { async fn spam_filter_analyze_llm(&self, ctx: &mut SpamFilterContext<'_>) { if let Some(config) = self .core .enterprise .as_ref() .and_then(|c| c.spam_filter_llm.as_ref()) { let time = Instant::now(); let body = if let Some(body) = ctx.text_body() { body } else { return; }; let prompt = format!( "{}\n\nSubject: {}\n\n{}", config.prompt, ctx.output.subject, body ); match config .model .send_request(prompt, config.temperature.into()) .await { Ok(response) => { trc::event!( Ai(AiEvent::LlmResponse), Id = config.model.id.clone(), Details = response.clone(), Elapsed = time.elapsed(), SpanId = ctx.input.span_id, ); let mut category = None; let mut confidence = None; let mut explanation = None; for (idx, value) in response.split(config.separator).enumerate() { let value = value.trim(); if !value.is_empty() { if idx == config.index_category { let value = value.to_uppercase(); if config.categories.contains(value.as_str()) { category = Some(value); } } else if config.index_confidence.is_some_and(|i| i == idx) { let value = value.to_uppercase(); if config.confidence.contains(value.as_str()) { confidence = Some(value); } } else if config.index_explanation.is_some_and(|i| i == idx) { let explanation = explanation.get_or_insert_with(|| { String::with_capacity(std::cmp::min(value.len(), 255)) }); for value in value.chars() { if !value.is_whitespace() { explanation.push(value); } else { explanation.push(' '); } if explanation.len() == 255 { break; } } } } } let category = match (category, confidence) { (Some(category), Some(confidence)) => { ctx.result.add_tag(format!("LLM_{category}_{confidence}")); category } (Some(category), None) => { ctx.result.add_tag(format!("LLM_{category}")); category } _ => return, }; if let Some(explanation) = explanation { ctx.result.llm_result = Some((category, explanation)); } } Err(err) => { trc::error!(err.span_id(ctx.input.span_id)); } } } } } ================================================ FILE: crates/spam-filter/src/analysis/messageid.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::future::Future; use common::Server; use mail_parser::HeaderName; use crate::{Hostname, SpamFilterContext}; pub trait SpamFilterAnalyzeMid: Sync + Send { fn spam_filter_analyze_message_id( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; } impl SpamFilterAnalyzeMid for Server { async fn spam_filter_analyze_message_id(&self, ctx: &mut SpamFilterContext<'_>) { let mut mid = ""; let mut mid_raw = ""; for header in ctx.input.message.headers() { if let (HeaderName::MessageId, value) = (&header.name, &header.value) { mid = value.as_text().unwrap_or_default(); mid_raw = std::str::from_utf8( &ctx.input.message.raw_message() [header.offset_start as usize..header.offset_end as usize], ) .unwrap_or_default() .trim(); break; } } if !mid.is_empty() { let mid = mid.to_lowercase(); if let Some(mid_host) = mid.rsplit_once('@').map(|(_, host)| Hostname::new(host)) { if mid_host.ip.is_some() { if mid_host.fqdn.starts_with('[') { ctx.result.add_tag("MID_RHS_IP_LITERAL"); } else { ctx.result.add_tag("MID_BARE_IP"); } } else if !mid_host.fqdn.contains('.') { ctx.result.add_tag("MID_RHS_NOT_FQDN"); } else if mid_host.fqdn.starts_with("www.") { ctx.result.add_tag("MID_RHS_WWW"); } if !mid_raw.is_ascii() || mid_raw.contains('(') || mid.starts_with('@') { ctx.result.add_tag("INVALID_MSGID"); } if mid_host.fqdn.len() > 255 { ctx.result.add_tag("MID_RHS_TOO_LONG"); } // From address present in Message-ID checks for (part, sender) in [ ("FROM", &ctx.output.from.email), ("ENV_FROM", &ctx.output.env_from_addr), ] { if !sender.address.is_empty() { if mid.contains(sender.address.as_str()) { ctx.result.add_tag(format!("MID_CONTAINS_{part}")); } else if mid_host.fqdn == sender.domain_part.fqdn { ctx.result.add_tag(format!("MID_RHS_MATCH_{part}")); } else if matches!((&mid_host.sld, &sender.domain_part.sld), (Some(mid_sld), Some(sender_sld)) if mid_sld == sender_sld) { ctx.result.add_tag(format!("MID_RHS_MATCH_{part}TLD")); } } } // To/Cc addresses present in Message-ID checks for rcpt in ctx.output.all_recipients() { if mid.contains(rcpt.email.address.as_str()) { ctx.result.add_tag("MID_CONTAINS_TO"); } else if mid_host.fqdn == rcpt.email.domain_part.fqdn { ctx.result.add_tag("MID_RHS_MATCH_TO"); } } } else { ctx.result.add_tag("INVALID_MSGID"); } if !mid_raw.starts_with('<') || !mid_raw.contains('>') { ctx.result.add_tag("MID_MISSING_BRACKETS"); } } else { ctx.result.add_tag("MISSING_MID"); } } } ================================================ FILE: crates/spam-filter/src/analysis/mime.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{collections::HashSet, future::Future, vec}; use common::{ Server, scripts::{ IsMixedCharset, functions::{array::cosine_similarity, unicode::CharUtils}, }, }; use mail_parser::{HeaderName, MimeHeaders, PartType}; use nlp::tokenizers::types::TokenType; use crate::{SpamFilterContext, TextPart}; pub trait SpamFilterAnalyzeMime: Sync + Send { fn spam_filter_analyze_mime( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; } impl SpamFilterAnalyzeMime for Server { async fn spam_filter_analyze_mime(&self, ctx: &mut SpamFilterContext<'_>) { let mut has_mime_version = false; let mut has_ct = false; let mut has_cte = false; let mut had_cd = false; let mut is_plain_text = false; for header in ctx.input.message.headers() { match &header.name { HeaderName::MimeVersion => { if ctx .input .message .raw_message() .get(header.offset_field as usize..header.offset_start as usize - 1) != Some(b"MIME-Version") { ctx.result.add_tag("MV_CASE"); } has_mime_version = true; } HeaderName::ContentType => { has_ct = true; if let Some(ct) = header.value().as_content_type() { if ct.ctype().eq_ignore_ascii_case("multipart") && ct .subtype() .is_some_and(|s| s.eq_ignore_ascii_case("report")) && ct.attribute("report-type").is_some_and(|a| { a.eq_ignore_ascii_case("delivery-status") || a.eq_ignore_ascii_case("disposition-notification") }) { // Message is a DSN ctx.result.add_tag("IS_DSN"); } is_plain_text = ct.ctype().eq_ignore_ascii_case("text") && ct .subtype() .unwrap_or_default() .eq_ignore_ascii_case("plain"); } } HeaderName::ContentTransferEncoding => { has_cte = true; } HeaderName::ContentDisposition => { had_cd = true; } _ => (), } } if !has_mime_version && (has_ct || has_cte) { ctx.result.add_tag("MISSING_MIME_VERSION"); } if has_ct && !is_plain_text && !has_cte && !had_cd && !has_mime_version { // Only Content-Type header without other MIME headers ctx.result.add_tag("MIME_HEADER_CTYPE_ONLY"); } let raw_message = ctx.input.message.raw_message(); let mut has_text_part = false; let mut is_encrypted = false; let mut is_encrypted_smime = false; let mut is_encrypted_pgp = false; let mut num_parts = 0; let mut num_parts_size = 0; for (part_id, part) in ctx.input.message.parts.iter().enumerate() { let part_id = part_id as u32; let mut ct = None; let mut cd = None; let mut ct_type = String::new(); let mut ct_subtype = String::new(); let mut cte = String::new(); let mut is_attachment = ctx.input.message.attachments.contains(&part_id); let mut has_content_id = false; for header in part.headers() { match &header.name { HeaderName::ContentType => { if let Some(ct_) = header.value().as_content_type() { ct_type = ct_.ctype().to_ascii_lowercase(); ct_subtype = ct_.subtype().unwrap_or_default().to_ascii_lowercase(); ct = Some(ct_); } if ct_type.is_empty() { // Content-Type header can't be parsed ctx.result.add_tag("BROKEN_CONTENT_TYPE"); } else if (ct_type == "message" && ct_subtype == "rfc822") || (ct_type == "text" && ct_subtype == "rfc822-headers") { // Message has parts ctx.result.add_tag("HAS_MESSAGE_PARTS"); } if raw_message .get(header.offset_start as usize..header.offset_end as usize) .and_then(|s| s.trim_ascii_end().last()) == Some(&b';') { // Content-Type header ends with a semi-colon ctx.result.add_tag("CT_EXTRA_SEMI"); } } HeaderName::ContentTransferEncoding => { let cte_ = header.value().as_text().unwrap_or_default(); cte = cte_.to_ascii_lowercase(); if cte != cte_ { ctx.result.add_tag("CTE_CASE"); } } HeaderName::ContentDisposition => { cd = header.value().as_content_type(); } HeaderName::ContentId => { has_content_id = true; } _ => (), } } match ct_type.as_str() { "multipart" => { let part_ids = match &part.body { PartType::Multipart(parts) => parts.as_slice(), _ => &[], }; match ct_subtype.as_str() { "alternative" => { let mut has_plain_part = false; let mut has_html_part = false; let mut text_part_words = vec![]; let mut text_part_uris = 0; let mut html_part_words = vec![]; let mut html_part_uris = 0; for text_part in part_ids .iter() .map(|id| &ctx.output.text_parts[*id as usize]) { let (tokens, words, uri_count) = match text_part { TextPart::Plain { tokens, .. } if !has_plain_part => { has_plain_part = true; (tokens, &mut text_part_words, &mut text_part_uris) } TextPart::Html { tokens, .. } if !has_html_part => { has_html_part = true; (tokens, &mut html_part_words, &mut html_part_uris) } _ => continue, }; let mut uris = HashSet::new(); for token in tokens { match token { TokenType::Alphabetic(v) | TokenType::Alphanumeric(v) => { words.push(v.as_ref()); } TokenType::Url(v) => { if let Some(host) = v.url_parsed.as_ref().map(|uri| &uri.host) { uris.insert(host.sld_or_default()); } } _ => (), } } *uri_count = uris.len(); } // Multipart message mostly text/html MIME if has_html_part { if !has_plain_part { ctx.result.add_tag("MIME_MA_MISSING_TEXT"); } } else if has_plain_part { ctx.result.add_tag("MIME_MA_MISSING_HTML"); } // HTML and text parts are different if has_plain_part && has_html_part && (!text_part_words.is_empty() || !html_part_words.is_empty()) && cosine_similarity(&text_part_words, &html_part_words) < 0.95 { ctx.result.add_tag("PARTS_DIFFER"); } // Odd URI count between parts if text_part_uris != html_part_uris { ctx.result.add_tag("URI_COUNT_ODD"); } } "mixed" => { let mut num_text_parts = 0; let mut has_other_parts = false; for (sub_part_id, sub_part) in part_ids .iter() .map(|id| (*id, &ctx.input.message.parts[*id as usize])) { let ctype = sub_part .content_type() .map(|ct| ct.ctype()) .unwrap_or_default(); if ctype.eq_ignore_ascii_case("text") && !ctx.input.message.attachments.contains(&sub_part_id) { num_text_parts += 1; } else if !ctype.eq_ignore_ascii_case("multipart") { has_other_parts = true; } } // Found multipart/mixed without non-textual part if !has_other_parts && num_text_parts < 3 { ctx.result.add_tag("CTYPE_MIXED_BOGUS"); } } "encrypted" => { is_encrypted = true; } _ => (), } continue; } "text" => { let mut is_7bit = false; match cte.as_str() { "" | "7bit" => { if raw_message .get( part.raw_body_offset() as usize..part.raw_end_offset() as usize, ) .is_some_and(|bytes| !bytes.is_ascii()) { // MIME text part claims to be ASCII but isn't ctx.result.add_tag("BAD_CTE_7BIT"); } is_7bit = true; } "base64" => { if part.contents().is_ascii() { // Has text part encoded in base64 that does not contain any 8bit characters ctx.result.add_tag("MIME_BASE64_TEXT_BOGUS"); } else { // Has text part encoded in base64 ctx.result.add_tag("MIME_BASE64_TEXT"); } } _ => (), } if !is_7bit && ct_subtype == "plain" && ct .and_then(|ct| ct.attribute("charset")) .is_none_or(|c| c.is_empty()) { // Charset header is missing ctx.result.add_tag("MISSING_CHARSET"); } if ctx .output .text_parts .get(part_id as usize) .filter(|_| { ctx.input.message.text_body.contains(&part_id) || ctx.input.message.html_body.contains(&part_id) }) .is_some_and(|p| match p { TextPart::Plain { text_body, .. } => text_body.is_mixed_charset(), TextPart::Html { text_body, .. } => text_body.is_mixed_charset(), TextPart::None => false, }) { // Text part contains multiple scripts ctx.result.add_tag("MIXED_CHARSET"); } has_text_part = true; } "application" => match ct_subtype.as_str() { "pkcs7-mime" => { ctx.result.add_tag("ENCRYPTED_SMIME"); is_attachment = false; is_encrypted_smime = true; } "pkcs7-signature" => { ctx.result.add_tag("SIGNED_SMIME"); is_attachment = false; } "pgp-encrypted" => { ctx.result.add_tag("ENCRYPTED_PGP"); is_attachment = false; is_encrypted_pgp = true; } "pgp-signature" => { ctx.result.add_tag("SIGNED_PGP"); is_attachment = false; } "octet-stream" => { if !is_encrypted && !has_content_id && cd.is_none_or(|cd| { !cd.c_type.eq_ignore_ascii_case("attachment") && !cd.has_attribute("filename") }) { ctx.result.add_tag("CTYPE_MISSING_DISPOSITION"); } } _ => (), }, _ => (), } num_parts += 1; num_parts_size += part.len(); let ct_full = format!("{ct_type}/{ct_subtype}"); if is_attachment { // Has a MIME attachment ctx.result.add_tag("HAS_ATTACHMENT"); if ct_full != "application/octet-stream" && let Some(t) = infer::get(part.contents()) { if t.mime_type() == ct_full { // Known content-type ctx.result.add_tag("MIME_GOOD"); } else { // Known bad content-type ctx.result.add_tag("MIME_BAD"); } } } // Analyze attachment name if let Some(attach_name) = part.attachment_name() { if attach_name.chars().any(|c| c.is_obscured()) { // Attachment name contains zero-width space ctx.result.add_tag("MIME_BAD_UNICODE"); } let attach_name = attach_name.trim().to_lowercase(); if let Some((name, ext)) = attach_name.rsplit_once('.').and_then(|(name, ext)| { Some((name, self.core.spam.lists.file_extensions.get(ext)?)) }) { let sub_ext = name .rsplit_once('.') .and_then(|(_, ext)| self.core.spam.lists.file_extensions.get(ext)); if ext.is_bad { // Attachment has a bad extension if sub_ext.is_some_and(|e| e.is_bad) { ctx.result.add_tag("MIME_DOUBLE_BAD_EXTENSION"); } else { ctx.result.add_tag("MIME_BAD_EXTENSION"); } } if ext.is_archive && sub_ext.is_some_and(|e| e.is_archive) { // Archive in archive ctx.result.add_tag("MIME_ARCHIVE_IN_ARCHIVE"); } if !ext.known_types.is_empty() && ct_full != "application/octet-stream" && !ext.known_types.contains(&ct_full) { // Invalid attachment mime type ctx.result.add_tag("MIME_BAD_ATTACHMENT"); } } } } match num_parts_size { 0 => { // Message contains no parts ctx.result.add_tag("COMPLETELY_EMPTY"); } 1..64 if num_parts == 1 => { // Message contains only one short part ctx.result.add_tag("SINGLE_SHORT_PART"); } _ => (), } if has_text_part && (is_encrypted_pgp || is_encrypted_smime) { // Message contains both text and encrypted parts ctx.result.add_tag("BOGUS_ENCRYPTED_AND_TEXT"); } } } ================================================ FILE: crates/spam-filter/src/analysis/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ Recipient, SpamFilterContext, SpamFilterInput, SpamFilterOutput, SpamFilterResult, TextPart, }; use common::{Server, config::spamfilter::Location}; use mail_parser::{Header, parsers::MessageStream}; use std::{ borrow::Cow, hash::{Hash, Hasher}, }; pub mod classifier; pub mod date; pub mod dmarc; pub mod domain; pub mod ehlo; pub mod from; pub mod headers; pub mod html; pub mod init; pub mod ip; pub mod messageid; pub mod mime; pub mod pyzor; pub mod received; pub mod recipient; pub mod replyto; pub mod rules; pub mod score; pub mod subject; pub mod url; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] pub mod llm; // SPDX-SnippetEnd impl SpamFilterInput<'_> { pub fn header_as_address(&self, header: &Header<'_>) -> Option> { self.message .raw_message() .get(header.offset_start as usize..header.offset_end as usize) .map(|bytes| MessageStream::new(bytes).parse_address()) .and_then(|addr| addr.into_address()) .and_then(|addr| addr.into_list().into_iter().next()) .and_then(|addr| addr.address) } } impl SpamFilterOutput<'_> { pub fn all_recipients(&self) -> impl Iterator { self.recipients_to .iter() .chain(self.recipients_cc.iter()) .chain(self.recipients_bcc.iter()) } } impl SpamFilterContext<'_> { pub fn text_body(&self) -> Option<&str> { self.input .message .text_body .first() .or_else(|| self.input.message.html_body.first()) .and_then(|idx| self.output.text_parts.get(*idx as usize)) .and_then(|part| match part { TextPart::Plain { text_body, .. } => Some(*text_body), TextPart::Html { text_body, .. } => Some(text_body.as_str()), TextPart::None => None, }) } } impl SpamFilterResult { pub fn add_tag(&mut self, tag: impl Into) { self.tags.insert(tag.into()); } pub fn has_tag(&self, tag: impl AsRef) -> bool { self.tags.contains(tag.as_ref()) } } #[derive(Debug)] pub struct ElementLocation { pub element: T, pub location: Location, } impl Hash for ElementLocation { fn hash(&self, state: &mut H) { self.element.hash(state); } } impl PartialEq for ElementLocation { fn eq(&self, other: &Self) -> bool { self.element.eq(&other.element) } } impl Eq for ElementLocation {} impl ElementLocation { pub fn new(element: T, location: impl Into) -> Self { Self { element, location: location.into(), } } } pub(crate) async fn is_trusted_domain(server: &Server, domain: &str, span_id: u64) -> bool { if let Some(store) = server.core.storage.lookups.get("trusted-domains") { match store.key_exists(domain).await { Ok(true) => return true, Ok(false) => (), Err(err) => { trc::error!(err.span_id(span_id).caused_by(trc::location!())); } } } match server.core.storage.directory.is_local_domain(domain).await { Ok(result) => result, Err(err) => { trc::error!(err.span_id(span_id).caused_by(trc::location!())); false } } } pub(crate) async fn is_url_redirector(server: &Server, url: &str, span_id: u64) -> bool { if let Some(store) = server.core.storage.lookups.get("url-redirectors") { match store.key_exists(url).await { Ok(result) => result, Err(err) => { trc::error!(err.span_id(span_id).caused_by(trc::location!())); false } } } else { false } } ================================================ FILE: crates/spam-filter/src/analysis/pyzor.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{future::Future, time::Instant}; use common::Server; use crate::{SpamFilterContext, modules::pyzor::pyzor_check}; pub trait SpamFilterAnalyzePyzor: Sync + Send { fn spam_filter_analyze_pyzor( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; } impl SpamFilterAnalyzePyzor for Server { async fn spam_filter_analyze_pyzor(&self, ctx: &mut SpamFilterContext<'_>) { if let Some(config) = &self.core.spam.pyzor { let time = Instant::now(); match pyzor_check(ctx.input.message, config).await { Ok(Some(result)) => { let is_spam = result.code == 200 && result.count > config.min_count && (result.wl_count < config.min_wl_count || (result.wl_count as f64 / result.count as f64) < config.ratio); if is_spam { ctx.result.add_tag("PYZOR"); } trc::event!( Spam(trc::SpamEvent::Pyzor), Result = is_spam, Details = vec![ trc::Value::from(result.code), trc::Value::from(result.count), trc::Value::from(result.wl_count) ], SpanId = ctx.input.span_id, Elapsed = time.elapsed() ); } Ok(None) => {} Err(err) => { trc::error!( err.span_id(ctx.input.span_id) .ctx(trc::Key::Elapsed, time.elapsed()) ); } } } } } ================================================ FILE: crates/spam-filter/src/analysis/received.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::future::Future; use common::Server; use mail_parser::{HeaderName, Host}; use crate::SpamFilterContext; pub trait SpamFilterAnalyzeReceived: Sync + Send { fn spam_filter_analyze_received( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; } impl SpamFilterAnalyzeReceived for Server { async fn spam_filter_analyze_received(&self, ctx: &mut SpamFilterContext<'_>) { let mut rcvd_count = 0; let mut rcvd_from_ip = 0; let mut tls_count = 0; for header in ctx.input.message.headers() { if let HeaderName::Received = &header.name { if !ctx .input .message .raw_message() .get(header.offset_start as usize..header.offset_end as usize) .unwrap_or_default() .is_ascii() { // Received headers have non-ASCII characters ctx.result.add_tag("RCVD_ILLEGAL_CHARS"); } if let Some(received) = header.value().as_received() { let helo_domain = received.from().or_else(|| received.helo()); let ip_rev = received.from_iprev(); if matches!(&helo_domain, Some(Host::Name(hostname)) if hostname.eq_ignore_ascii_case("user")) { // HELO domain is "user" ctx.result.add_tag("RCVD_HELO_USER"); } else if let (Some(Host::Name(helo_domain)), Some(ip_rev)) = (helo_domain, ip_rev) && helo_domain.to_lowercase() != ip_rev.to_lowercase() { // HELO domain does not match PTR record ctx.result.add_tag("FORGED_RCVD_TRAIL"); } if let Some(delivered_for) = received.for_().map(|s| s.to_lowercase()) && ctx .output .all_recipients() .any(|r| r.email.address == delivered_for) { // Recipient appears on Received trail ctx.result.add_tag("PREVIOUSLY_DELIVERED"); } if matches!(received.from, Some(Host::IpAddr(_))) { // Received from an IP address rather than a FQDN rcvd_from_ip += 1; } if received.tls_version().is_some() { // Received with TLS tls_count += 1; } } else { // Received header is not RFC 5322 compliant ctx.result.add_tag("RCVD_UNPARSABLE"); } rcvd_count += 1; } } if rcvd_from_ip >= 2 || (rcvd_from_ip == 1 && ctx.output.ehlo_host.ip.is_some()) { // Has two or more Received headers containing bare IP addresses ctx.result.add_tag("RCVD_DOUBLE_IP_SPAM"); } // Received from an authenticated user if ctx.input.authenticated_as.is_some() { ctx.result.add_tag("RCVD_VIA_SMTP_AUTH"); } // Received with TLS checks if rcvd_count > 0 && rcvd_count == tls_count && ctx.input.is_tls { ctx.result.add_tag("RCVD_TLS_ALL"); } else if ctx.input.is_tls { ctx.result.add_tag("RCVD_TLS_LAST"); } else { ctx.result.add_tag("RCVD_NO_TLS_LAST"); } match rcvd_count { 0 => { ctx.result.add_tag("RCVD_COUNT_ZERO"); } 1 => { ctx.result.add_tag("RCVD_COUNT_ONE"); } 2 => { ctx.result.add_tag("RCVD_COUNT_TWO"); } 3 => { ctx.result.add_tag("RCVD_COUNT_THREE"); } 4 | 5 => { ctx.result.add_tag("RCVD_COUNT_FIVE"); } 6 | 7 => { ctx.result.add_tag("RCVD_COUNT_SEVEN"); } 8..=12 => { ctx.result.add_tag("RCVD_COUNT_TWELVE"); } _ => {} } } } ================================================ FILE: crates/spam-filter/src/analysis/recipient.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::future::Future; use common::{Server, scripts::functions::text::levenshtein_distance}; use mail_parser::HeaderName; use smtp_proto::{MAIL_BODY_8BITMIME, MAIL_BODY_BINARYMIME, MAIL_SMTPUTF8}; use store::ahash::HashSet; use crate::SpamFilterContext; pub trait SpamFilterAnalyzeRecipient: Sync + Send { fn spam_filter_analyze_recipient( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; } impl SpamFilterAnalyzeRecipient for Server { async fn spam_filter_analyze_recipient(&self, ctx: &mut SpamFilterContext<'_>) { let mut to_raw = b"".as_slice(); let mut cc_raw = b"".as_slice(); let mut bcc_raw = b"".as_slice(); let mut has_list_unsubscribe = false; let mut has_list_id = false; for header in ctx.input.message.headers() { match &header.name { HeaderName::To | HeaderName::Cc | HeaderName::Bcc => { let raw = ctx .input .message .raw_message() .get(header.offset_start as usize..header.offset_end as usize) .unwrap_or_default(); match header.name { HeaderName::To => to_raw = raw, HeaderName::Cc => cc_raw = raw, HeaderName::Bcc => bcc_raw = raw, _ => unreachable!(), } } HeaderName::ListUnsubscribe => { has_list_unsubscribe = true; } HeaderName::ListId => { has_list_id = true; } _ => {} } } if to_raw.is_empty() { ctx.result.add_tag("MISSING_TO"); } let to_raw_utf8 = std::str::from_utf8(to_raw); let cc_raw_utf8 = std::str::from_utf8(cc_raw); let bcc_raw_utf8 = std::str::from_utf8(bcc_raw); for (raw, raw_utf8, recipients) in [ (to_raw, &to_raw_utf8, &ctx.output.recipients_to), (cc_raw, &cc_raw_utf8, &ctx.output.recipients_cc), (bcc_raw, &bcc_raw_utf8, &ctx.output.recipients_bcc), ] { if !raw.is_empty() { // Validate non-ASCII characters in recipient headers if !raw.is_ascii() { if (ctx.input.env_from_flags & (MAIL_SMTPUTF8 | MAIL_BODY_8BITMIME | MAIL_BODY_BINARYMIME)) == 0 { ctx.result.add_tag("TO_NEEDS_ENCODING"); } if raw_utf8.is_err() { ctx.result.add_tag("INVALID_TO_8BIT"); } } // Validate unnecessary encoding in recipient headers let raw_utf8 = raw_utf8.unwrap_or_default(); if recipients.iter().all(|rcpt| { rcpt.name.as_ref().is_none_or(|name| name.is_ascii()) && rcpt.email.address.is_ascii() }) && raw_utf8.contains("=?") && raw_utf8.contains("?=") { if raw_utf8.contains("?q?") || raw_utf8.contains("?Q?") { // To header is unnecessarily encoded in quoted-printable ctx.result.add_tag("TO_EXCESS_QP"); } else if raw_utf8.contains("?b?") || raw_utf8.contains("?B?") { // To header is unnecessarily encoded in base64 ctx.result.add_tag("TO_EXCESS_BASE64"); } } // Check for spaces in recipient addresses for token in raw_utf8.split('<') { if let Some((addr, _)) = token.split_once('>') && (addr.starts_with(' ') || addr.ends_with(' ')) { ctx.result.add_tag("TO_WRAPPED_IN_SPACES"); break; } } } } let unique_recipients = ctx .output .all_recipients() .filter(|rcpt| !rcpt.email.address.is_empty()) .collect::>(); let rcpt_count = unique_recipients.len(); match unique_recipients.len() { 0 => { ctx.result.add_tag("RCPT_COUNT_ZERO"); return; } 1 => { ctx.result.add_tag("RCPT_COUNT_ONE"); } 2 => { ctx.result.add_tag("RCPT_COUNT_TWO"); } 3 => { ctx.result.add_tag("RCPT_COUNT_THREE"); } 4 | 5 => { ctx.result.add_tag("RCPT_COUNT_FIVE"); } 6 | 7 => { ctx.result.add_tag("RCPT_COUNT_SEVEN"); } 8..=12 => { ctx.result.add_tag("RCPT_COUNT_TWELVE"); } 13.. => { ctx.result.add_tag("RCPT_COUNT_GT_50"); } } let mut to_dn_eq_addr_count = 0; let mut to_dn_count = 0; let mut to_match_envrcpt = 0; for rcpt in &unique_recipients { // Validate name if let Some(rcpt_name) = &rcpt.name { if *rcpt_name == rcpt.email.address { to_dn_eq_addr_count += 1; } else { to_dn_count += 1; } } // Recipient is present in envelope if ctx.output.env_to_addr.contains(&rcpt.email) { to_match_envrcpt += 1; } // Check if the local part is present in the subject if !rcpt.email.local_part.is_empty() { if ctx.output.subject_lc.contains(rcpt.email.address.as_str()) { ctx.result.add_tag("RCPT_IN_SUBJECT"); } else if rcpt.email.local_part.len() > 3 && ctx .output .subject_lc .contains(rcpt.email.local_part.as_str()) { ctx.result.add_tag("RCPT_LOCAL_IN_SUBJECT"); } } } if to_dn_count == 0 && to_dn_eq_addr_count == 0 { ctx.result.add_tag("TO_DN_NONE"); } else if to_dn_count == rcpt_count { ctx.result.add_tag("TO_DN_ALL"); } else if to_dn_count > 0 { ctx.result.add_tag("TO_DN_SOME"); } if to_dn_eq_addr_count == rcpt_count { ctx.result.add_tag("TO_DN_EQ_ADDR_ALL"); } else if to_dn_eq_addr_count > 0 { ctx.result.add_tag("TO_DN_EQ_ADDR_SOME"); } if to_match_envrcpt == rcpt_count { ctx.result.add_tag("TO_MATCH_ENVRCPT_ALL"); } else { if to_match_envrcpt > 0 { ctx.result.add_tag("TO_MATCH_ENVRCPT_SOME"); } if !has_list_id && !has_list_unsubscribe { for env_rcpt in &ctx.output.env_to_addr { if !unique_recipients.iter().any(|rcpt| rcpt.email == *env_rcpt) && env_rcpt != &ctx.output.env_from_addr { ctx.result.add_tag("FORGED_RECIPIENTS"); break; } } } } // Message from bounce and over 1 recipient if rcpt_count > 1 && ctx.output.env_from_postmaster { ctx.result.add_tag("RCPT_BOUNCEMOREONE"); } let rcpts = ctx .output .recipients_to .iter() .chain(ctx.output.recipients_cc.iter()) .collect::>(); let mut is_sorted = false; if rcpts.len() >= 6 { // Check if the recipients list is sorted let mut sorted = true; for i in 1..rcpts.len() { if rcpts[i - 1].email.address > rcpts[i].email.address { sorted = false; break; } } if sorted { ctx.result.add_tag("SORTED_RECIPS"); is_sorted = true; } } if !is_sorted && rcpt_count >= 5 { // Look for similar recipients let mut hits = 0; let mut combinations = 0; for i in 0..rcpts.len() { for j in i + 1..rcpts.len() { let a = &rcpts[i].email; let b = &rcpts[j].email; if levenshtein_distance(&a.local_part, &b.local_part) < 3 || (a.domain_part.fqdn != b.domain_part.fqdn && levenshtein_distance(&a.domain_part.fqdn, &b.domain_part.fqdn) < 4) { hits += 1; } combinations += 1; } } if hits as f64 / combinations as f64 > 0.65 { ctx.result.add_tag("SUSPICIOUS_RECIPS"); } } } } ================================================ FILE: crates/spam-filter/src/analysis/replyto.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::future::Future; use common::Server; use mail_parser::HeaderName; use crate::SpamFilterContext; pub trait SpamFilterAnalyzeReplyTo: Sync + Send { fn spam_filter_analyze_reply_to( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; } impl SpamFilterAnalyzeReplyTo for Server { async fn spam_filter_analyze_reply_to(&self, ctx: &mut SpamFilterContext<'_>) { let mut reply_to_raw = b"".as_slice(); let mut is_from_list = false; for header in ctx.input.message.headers() { match &header.name { HeaderName::ReplyTo => { reply_to_raw = ctx .input .message .raw_message() .get(header.offset_start as usize..header.offset_end as usize) .unwrap_or_default(); } HeaderName::ListUnsubscribe | HeaderName::ListId => { is_from_list = true; } HeaderName::Other(name) => { if !is_from_list { is_from_list = name.eq_ignore_ascii_case("X-To-Get-Off-This-List") || name.eq_ignore_ascii_case("X-List") || name.eq_ignore_ascii_case("Auto-Submitted"); } } _ => {} } } if reply_to_raw.is_empty() { return; } if let Some(reply_to) = &ctx.output.reply_to { let reply_to_name = reply_to.name.as_deref().unwrap_or_default(); ctx.result.add_tag("HAS_REPLYTO"); if reply_to.email == ctx.output.from.email { ctx.result.add_tag("REPLYTO_EQ_FROM"); } else { if reply_to.email.domain_part.sld == ctx.output.from.email.domain_part.sld { ctx.result.add_tag("REPLYTO_DOM_EQ_FROM_DOM"); } else { if !is_from_list && ctx .output .all_recipients() .any(|r| r.email == reply_to.email) { ctx.result.add_tag("REPLYTO_EQ_TO_ADDR"); } else { ctx.result.add_tag("REPLYTO_DOM_NEQ_FROM_DOM"); } if !(is_from_list || ctx .output .recipients_to .iter() .any(|r| r.email == ctx.output.from.email) || ctx .output .env_to_addr .iter() .any(|r| r.domain_part.sld == ctx.output.from.email.domain_part.sld) || ctx.output.env_to_addr.len() == 1 && ctx.output.env_to_addr.contains(&ctx.output.from.email)) { ctx.result.add_tag("SPOOF_REPLYTO"); } } if !reply_to_name.is_empty() && reply_to_name == ctx.output.from.name.as_deref().unwrap_or_default() { ctx.result.add_tag("REPLYTO_DN_EQ_FROM_DN"); } } if reply_to.email == ctx.output.env_from_addr { ctx.result.add_tag("REPLYTO_ADDR_EQ_FROM"); } // Validate unnecessary encoding let reply_to_raw_utf8 = std::str::from_utf8(reply_to_raw).unwrap_or_default(); if reply_to.email.address.is_ascii() && reply_to_name.is_ascii() && reply_to_raw_utf8.contains("=?") && reply_to_raw_utf8.contains("?=") { if reply_to_raw_utf8.contains("?q?") || reply_to_raw_utf8.contains("?Q?") { // Reply-To header is unnecessarily encoded in quoted-printable ctx.result.add_tag("REPLYTO_EXCESS_QP"); } else if reply_to_raw_utf8.contains("?b?") || reply_to_raw_utf8.contains("?B?") { // Reply-To header is unnecessarily encoded in base64 ctx.result.add_tag("REPLYTO_EXCESS_BASE64"); } } } else { ctx.result.add_tag("REPLYTO_UNPARSABLE"); } } } ================================================ FILE: crates/spam-filter/src/analysis/rules.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::future::Future; use common::{ Server, config::spamfilter::{IpResolver, Location}, }; use crate::{ SpamFilterContext, TextPart, modules::expression::{EmailHeader, SpamFilterResolver, StringResolver}, }; pub trait SpamFilterAnalyzeRules: Sync + Send { fn spam_filter_analyze_rules( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; } impl SpamFilterAnalyzeRules for Server { async fn spam_filter_analyze_rules(&self, ctx: &mut SpamFilterContext<'_>) { if !self.core.spam.rules.url.is_empty() { for url in &ctx.output.urls { for rule in &self.core.spam.rules.url { if let Some(tag) = self .eval_if::( rule, &SpamFilterResolver::new(ctx, &url.element, url.location), ctx.input.span_id, ) .await { ctx.result.tags.insert(tag); } } } } if !self.core.spam.rules.domain.is_empty() { for domain in &ctx.output.domains { let resolver = StringResolver(domain.element.as_str()); for rule in &self.core.spam.rules.domain { if let Some(tag) = self .eval_if::( rule, &SpamFilterResolver::new(ctx, &resolver, domain.location), ctx.input.span_id, ) .await { ctx.result.tags.insert(tag); } } } } if !self.core.spam.rules.email.is_empty() { for email in &ctx.output.emails { for rule in &self.core.spam.rules.email { if let Some(tag) = self .eval_if::( rule, &SpamFilterResolver::new(ctx, &email.element, email.location), ctx.input.span_id, ) .await { ctx.result.tags.insert(tag); } } } for (rcpt, location) in [ (&ctx.output.recipients_to, Location::HeaderTo), (&ctx.output.recipients_cc, Location::HeaderCc), (&ctx.output.recipients_bcc, Location::HeaderBcc), ] { for email in rcpt { for rule in &self.core.spam.rules.email { if let Some(tag) = self .eval_if::( rule, &SpamFilterResolver::new(ctx, email, location), ctx.input.span_id, ) .await { ctx.result.tags.insert(tag); } } } } } if !self.core.spam.rules.ip.is_empty() { for ip in &ctx.output.ips { let ip_resolver = IpResolver::new(ip.element); for rule in &self.core.spam.rules.ip { if let Some(tag) = self .eval_if::( rule, &SpamFilterResolver::new(ctx, &ip_resolver, ip.location), ctx.input.span_id, ) .await { ctx.result.tags.insert(tag); } } } } if !self.core.spam.rules.header.is_empty() { for header in ctx.input.message.headers() { let raw = String::from_utf8_lossy( ctx.input .message .raw_message() .get(header.offset_start as usize..header.offset_end as usize) .unwrap_or_default(), ); let header_resolver = EmailHeader { header, raw: raw.as_ref(), }; for rule in &self.core.spam.rules.header { if let Some(tag) = self .eval_if::( rule, &SpamFilterResolver::new(ctx, &header_resolver, Location::BodyText), ctx.input.span_id, ) .await { ctx.result.tags.insert(tag); } } } } if !self.core.spam.rules.body.is_empty() { for (idx, part) in ctx.output.text_parts.iter().enumerate() { let text = match part { TextPart::Plain { text_body, .. } => *text_body, TextPart::Html { text_body, .. } => text_body.as_str(), TextPart::None => continue, }; let idx = idx as u32; let location = if ctx.input.message.text_body.contains(&idx) { Location::BodyText } else if ctx.input.message.html_body.contains(&idx) { Location::BodyHtml } else { Location::Attachment }; let string_resolver = StringResolver(text); for rule in &self.core.spam.rules.body { if let Some(tag) = self .eval_if::( rule, &SpamFilterResolver::new(ctx, &string_resolver, location), ctx.input.span_id, ) .await { ctx.result.tags.insert(tag); } } } } if !self.core.spam.rules.any.is_empty() { let dummy_resolver = StringResolver(""); for rule in &self.core.spam.rules.any { if let Some(tag) = self .eval_if::( rule, &SpamFilterResolver::new(ctx, &dummy_resolver, Location::BodyText), ctx.input.span_id, ) .await { ctx.result.tags.insert(tag); } } } } } ================================================ FILE: crates/spam-filter/src/analysis/score.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ SpamFilterContext, analysis::{ classifier::SpamFilterAnalyzeClassify, date::SpamFilterAnalyzeDate, dmarc::SpamFilterAnalyzeDmarc, domain::SpamFilterAnalyzeDomain, ehlo::SpamFilterAnalyzeEhlo, from::SpamFilterAnalyzeFrom, headers::SpamFilterAnalyzeHeaders, html::SpamFilterAnalyzeHtml, ip::SpamFilterAnalyzeIp, messageid::SpamFilterAnalyzeMid, mime::SpamFilterAnalyzeMime, pyzor::SpamFilterAnalyzePyzor, received::SpamFilterAnalyzeReceived, recipient::SpamFilterAnalyzeRecipient, replyto::SpamFilterAnalyzeReplyTo, rules::SpamFilterAnalyzeRules, subject::SpamFilterAnalyzeSubject, url::SpamFilterAnalyzeUrl, }, }; use common::{Server, config::spamfilter::SpamFilterAction}; use std::{fmt::Write, future::Future, vec}; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] use crate::analysis::llm::SpamFilterAnalyzeLlm; // SPDX-SnippetEnd pub trait SpamFilterAnalyzeScore: Sync + Send { fn spam_filter_finalize( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future> + Send; fn spam_filter_classify( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future> + Send; } #[derive(Debug, Default)] pub struct SpamFilterScore { pub results: Vec, pub headers: String, pub train_spam: Option, pub score: f32, } impl SpamFilterAnalyzeScore for Server { async fn spam_filter_finalize( &self, ctx: &mut SpamFilterContext<'_>, ) -> SpamFilterAction { // Calculate final score let mut results = vec![]; let mut header_len = 60; let mut is_spam_trap = false; let mut rbl_count = 0; for tag in &ctx.result.tags { let score = match self.core.spam.lists.scores.get(tag) { Some(SpamFilterAction::Allow(score)) => *score, Some(SpamFilterAction::Discard) => { return SpamFilterAction::Discard; } Some(SpamFilterAction::Reject) => { return SpamFilterAction::Reject; } None | Some(SpamFilterAction::Disabled) => 0.0, }; if tag == "SPAM_TRAP" { is_spam_trap = true; } else if score > 1.0 && tag.starts_with("RBL_") { rbl_count += 1; } ctx.result.score += score; header_len += tag.len() + 10; if score != 0.0 || !tag.starts_with("X_") { results.push((tag.as_str(), score)); } } let mut final_score = ctx.result.score; let mut avg_confidence: f32 = 0.0; let mut total_results = 0; let mut user_results = vec![ ctx.result.score >= self.core.spam.scores.spam_threshold; ctx.input.env_rcpt_to.len() ]; if !ctx.result.classifier_confidence.is_empty() { for (idx, &confidence) in ctx.result.classifier_confidence.iter().enumerate() { if let Some(confidence) = confidence { avg_confidence += confidence; total_results += 1; let user_score = self .core .spam .lists .scores .get(confidence.spam_tag()) .and_then(|v| v.as_score()) .copied() .unwrap_or_default(); user_results[idx] = ctx.result.score + user_score >= self.core.spam.scores.spam_threshold; } } if total_results > 0 { avg_confidence /= total_results as f32; let tag = avg_confidence.spam_tag(); let score = self .core .spam .lists .scores .get(tag) .and_then(|v| v.as_score()) .copied() .unwrap_or_default(); results.push((tag, score)); final_score += score; } } if self.core.spam.scores.reject_threshold > 0.0 && final_score >= self.core.spam.scores.reject_threshold { SpamFilterAction::Reject } else if self.core.spam.scores.discard_threshold > 0.0 && final_score >= self.core.spam.scores.discard_threshold { SpamFilterAction::Discard } else { let mut headers = String::with_capacity(header_len + 40); results.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap().then_with(|| a.0.cmp(b.0))); headers.push_str("X-Spam-Result: "); for (idx, (tag, score)) in results.into_iter().enumerate() { if idx > 0 { headers.push_str(",\r\n\t"); } let _ = write!(&mut headers, "{} ({:.2})", tag, score); } headers.push_str("\r\n"); if let Some((category, explanation)) = &ctx.result.llm_result { let _ = write!(&mut headers, "X-Spam-LLM: {category} ({explanation})\r\n",); } let is_spam = final_score >= self.core.spam.scores.spam_threshold; let class = if is_spam { "spam" } else { "ham" }; if avg_confidence != 0.0 { let _ = write!( &mut headers, "X-Spam-Score: {class}, score={final_score:.2}, avg_confidence={avg_confidence:.2}\r\n", ); } else { let _ = write!( &mut headers, "X-Spam-Score: {class}, score={final_score:.2}\r\n", ); } // Autolearn SPAM let mut train_spam = None; if is_spam && self.core.spam.classifier.as_ref().is_some_and(|c| { (c.auto_learn_spam_trap && is_spam_trap) || (c.auto_learn_spam_rbl_count > 0 && rbl_count >= c.auto_learn_spam_rbl_count) }) { train_spam = Some(true); } SpamFilterAction::Allow(SpamFilterScore { results: user_results, headers, train_spam, score: final_score, }) } } async fn spam_filter_classify( &self, ctx: &mut SpamFilterContext<'_>, ) -> SpamFilterAction { // IP address analysis self.spam_filter_analyze_ip(ctx).await; // DMARC/SPF/DKIM/ARC analysis self.spam_filter_analyze_dmarc(ctx).await; // EHLO hostname analysis self.spam_filter_analyze_ehlo(ctx).await; // Generic header analysis self.spam_filter_analyze_headers(ctx).await; // Received headers analysis self.spam_filter_analyze_received(ctx).await; // Message-ID analysis self.spam_filter_analyze_message_id(ctx).await; // Date header analysis self.spam_filter_analyze_date(ctx).await; // Subject analysis self.spam_filter_analyze_subject(ctx).await; // From and Envelope From analysis self.spam_filter_analyze_from(ctx).await; // Reply-To analysis self.spam_filter_analyze_reply_to(ctx).await; // Recipient analysis self.spam_filter_analyze_recipient(ctx).await; // E-mail and domain analysis self.spam_filter_analyze_domain(ctx).await; // URL analysis self.spam_filter_analyze_url(ctx).await; // MIME part analysis self.spam_filter_analyze_mime(ctx).await; // HTML content analysis self.spam_filter_analyze_html(ctx).await; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL // LLM classification #[cfg(feature = "enterprise")] self.spam_filter_analyze_llm(ctx).await; // SPDX-SnippetEnd // Spam trap self.spam_filter_analyze_spam_trap(ctx).await; // Pyzor checks self.spam_filter_analyze_pyzor(ctx).await; // Model classification self.spam_filter_analyze_classify(ctx).await; // User-defined rules self.spam_filter_analyze_rules(ctx).await; // Final score calculation self.spam_filter_finalize(ctx).await } } pub trait ConfidenceStore { fn spam_tag(&self) -> &'static str; } impl ConfidenceStore for f32 { fn spam_tag(&self) -> &'static str { match *self { p if p < 0.15 => "PROB_HAM_HIGH", p if p < 0.25 => "PROB_HAM_MEDIUM", p if p < 0.40 => "PROB_HAM_LOW", p if p < 0.60 => "PROB_SPAM_UNCERTAIN", p if p < 0.75 => "PROB_SPAM_LOW", p if p < 0.85 => "PROB_SPAM_MEDIUM", p => { if p.is_finite() { "PROB_SPAM_HIGH" } else { "PROB_SPAM_UNCERTAIN" } } } } } ================================================ FILE: crates/spam-filter/src/analysis/subject.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::future::Future; use common::Server; use mail_parser::HeaderName; use nlp::tokenizers::types::TokenType; use smtp_proto::{MAIL_BODY_8BITMIME, MAIL_BODY_BINARYMIME, MAIL_SMTPUTF8}; use crate::SpamFilterContext; pub trait SpamFilterAnalyzeSubject: Sync + Send { fn spam_filter_analyze_subject( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; } impl SpamFilterAnalyzeSubject for Server { async fn spam_filter_analyze_subject(&self, ctx: &mut SpamFilterContext<'_>) { let mut subject_raw = b"".as_slice(); for header in ctx.input.message.headers() { if header.name == HeaderName::Subject { subject_raw = ctx .input .message .raw_message() .get(header.offset_start as usize..header.offset_end as usize) .unwrap_or_default(); break; } } if subject_raw.is_empty() { // Missing subject header ctx.result.add_tag("MISSING_SUBJECT"); return; } let mut word_count = 0; let mut upper_count = 0; let mut lower_count = 0; let mut last_ch = ' '; let mut is_ascii = true; for ch in ctx.output.subject_thread.chars() { if !ch.is_whitespace() { if last_ch.is_whitespace() { word_count += 1; } match ch { '$' | '€' | '£' | '¥' | '₹' | '₽' | '₿' => { ctx.result.add_tag("SUBJECT_HAS_CURRENCY"); } _ => { if ch.is_alphabetic() { if ch.is_uppercase() { upper_count += 1; } else { lower_count += 1; } } } } } if !ch.is_ascii() { is_ascii = false; } last_ch = ch; } if ctx.output.subject_lc.is_empty() { // Subject is empty ctx.result.add_tag("EMPTY_SUBJECT"); } else if ctx.output.subject.ends_with(' ') { // Subject ends with whitespace ctx.result.add_tag("SUBJECT_ENDS_SPACES"); } else if ctx.output.subject == "XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X" { ctx.result.add_tag("GTUBE_TEST"); } if ctx.output.subject_thread.len() >= 10 && word_count > 1 && upper_count > 2 && lower_count == 0 { // Subject contains mostly capital letters ctx.result.add_tag("SUBJ_ALL_CAPS"); } for token in &ctx.output.subject_tokens { match token { TokenType::Url(url) => { // Subject contains URL ctx.result.add_tag("URL_IN_SUBJECT"); if let Some(url_parsed) = &url.url_parsed { let host = url_parsed.host.sld_or_default(); for rcpt in ctx.output.all_recipients() { if rcpt.email.domain_part.sld_or_default() == host { ctx.result.add_tag("RCPT_DOMAIN_IN_SUBJECT"); break; } } } } TokenType::UrlNoScheme(url) => { if let Some(url_parsed) = &url.url_parsed { let host = url_parsed.host.sld_or_default(); for rcpt in ctx.output.all_recipients() { if rcpt.email.domain_part.sld_or_default() == host { ctx.result.add_tag("RCPT_DOMAIN_IN_SUBJECT"); break; } } } } TokenType::Email(email) => { // Subject contains recipient if ctx.output.env_to_addr.contains(email) || ctx .output .all_recipients() .any(|r| r.email.address == email.address) { ctx.result.add_tag("RCPT_IN_SUBJECT"); } else { let host = email.domain_part.sld_or_default(); for rcpt in ctx.output.all_recipients() { if rcpt.email.address == email.address { ctx.result.add_tag("RCPT_IN_SUBJECT"); break; } else if rcpt.email.domain_part.sld_or_default() == host { ctx.result.add_tag("RCPT_DOMAIN_IN_SUBJECT"); break; } } } } _ => {} } } // Validate encoding let subject_raw_utf8 = std::str::from_utf8(subject_raw); if !subject_raw.is_ascii() { if (ctx.input.env_from_flags & (MAIL_SMTPUTF8 | MAIL_BODY_8BITMIME | MAIL_BODY_BINARYMIME)) == 0 { ctx.result.add_tag("SUBJECT_NEEDS_ENCODING"); } if subject_raw_utf8.is_err() { ctx.result.add_tag("INVALID_SUBJECT_8BIT"); } } // Validate unnecessary encoding let subject_raw_utf8 = subject_raw_utf8.unwrap_or_default(); if is_ascii && subject_raw_utf8.contains("=?") && subject_raw_utf8.contains("?=") { if subject_raw_utf8.contains("?q?") || subject_raw_utf8.contains("?Q?") { // Subject header is unnecessarily encoded in quoted-printable ctx.result.add_tag("SUBJ_EXCESS_QP"); } else if subject_raw_utf8.contains("?b?") || subject_raw_utf8.contains("?B?") { // Subject header is unnecessarily encoded in base64 ctx.result.add_tag("SUBJ_EXCESS_BASE64"); } } } } ================================================ FILE: crates/spam-filter/src/analysis/url.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ElementLocation, is_trusted_domain, is_url_redirector}; use crate::modules::dnsbl::check_dnsbl; use crate::modules::expression::StringResolver; use crate::modules::html::SRC; use crate::{ Hostname, SpamFilterContext, TextPart, modules::html::{A, HREF, HtmlToken}, }; use common::Server; use common::config::spamfilter::{Element, IpResolver, Location}; use common::scripts::IsMixedCharset; use common::scripts::functions::unicode::CharUtils; use hyper::{Uri, header::LOCATION}; use nlp::tokenizers::types::TokenType; use reqwest::redirect::Policy; use std::collections::HashSet; use std::hash::{Hash, Hasher}; use std::{borrow::Cow, future::Future, time::Duration}; pub trait SpamFilterAnalyzeUrl: Sync + Send { fn spam_filter_analyze_url( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future + Send; } #[derive(Clone, Debug)] pub struct UrlParts<'x> { pub url: String, pub url_original: Cow<'x, str>, pub url_parsed: Option, } #[derive(Clone, Debug)] pub struct UrlParsed { pub parts: Uri, pub host: Hostname, } impl SpamFilterAnalyzeUrl for Server { async fn spam_filter_analyze_url(&self, ctx: &mut SpamFilterContext<'_>) { // Extract URLs let mut urls: HashSet>> = HashSet::from_iter(ctx.output.subject_tokens.iter().filter_map(|t| match t { TokenType::Url(url) | TokenType::UrlNoScheme(url) => Some(ElementLocation::new( url.to_owned(), Location::HeaderSubject, )), _ => None, })); for (part_id, part) in ctx.output.text_parts.iter().enumerate() { let part_id = part_id as u32; let is_body = ctx.input.message.text_body.contains(&part_id) || ctx.input.message.html_body.contains(&part_id); let tokens = match part { TextPart::Plain { tokens, .. } => tokens, TextPart::Html { html_tokens, tokens, .. } => { for token in html_tokens { if let HtmlToken::StartTag { attributes, .. } = token { for (attr, value) in attributes { match value { Some(value) if [HREF, SRC].contains(attr) => { urls.insert(ElementLocation::new( UrlParts::new(value.trim().to_string()), if is_body { Location::BodyHtml } else { Location::Attachment }, )); } _ => {} } } } } tokens } TextPart::None => &[][..], }; for token in tokens { match token { TokenType::Url(url) | TokenType::UrlNoScheme(url) => { if !ctx.input.is_train && is_body && !ctx.result.has_tag("RCPT_DOMAIN_IN_BODY") && let Some(url_parsed) = &url.url_parsed { let host = url_parsed.host.sld_or_default(); for rcpt in ctx.output.all_recipients() { if rcpt.email.domain_part.sld_or_default() == host { ctx.result.add_tag("RCPT_DOMAIN_IN_BODY"); break; } } } urls.insert(ElementLocation::new( url.to_owned(), if is_body { Location::BodyHtml } else { Location::Attachment }, )); } _ => {} } } if is_body && !ctx.input.is_train { let is_single = match part { TextPart::Plain { tokens, .. } => is_single_url(tokens), TextPart::Html { html_tokens, tokens, .. } => is_single_html_url(html_tokens, tokens), TextPart::None => false, }; if is_single { ctx.result.add_tag("URL_ONLY"); } } } if !ctx.input.is_train { let mut redirected_urls = HashSet::new(); for url in &urls { for ch in url.element.url.chars() { if ch.is_zwsp() { ctx.result.add_tag("ZERO_WIDTH_SPACE_URL"); } if ch.is_obscured() { ctx.result.add_tag("SUSPICIOUS_URL"); } } // Skip non-URLs such as 'data:' and 'mailto:' if !url.element.url.contains("://") { continue; } // Obtain parse url let url_parsed = if let Some(url_parsed) = &url.element.url_parsed { url_parsed } else { // URL could not be parsed ctx.result.add_tag("UNPARSABLE_URL"); continue; }; let host_sld = url_parsed.host.sld_or_default(); // Skip local and trusted domains if is_trusted_domain(self, host_sld, ctx.input.span_id).await { continue; } if let Some(ip) = url_parsed.host.ip { // Check IP DNSBL check_dnsbl(self, ctx, &IpResolver::new(ip), Element::Ip, url.location).await; } else if is_url_redirector(self, host_sld, ctx.input.span_id).await { // Check for redirectors ctx.result.add_tag("REDIRECTOR_URL"); if !ctx.result.has_tag("URL_REDIRECTOR_NESTED") { let mut redirect_count = 1; let mut url_redirect = Cow::Borrowed(url.element.url.as_str()); while redirect_count <= 3 { match http_get_header( url_redirect.as_ref(), LOCATION, Duration::from_secs(5), ) .await { Ok(Some(location)) => { let location = UrlParts::new(location); if let Some(location_parsed) = &location.url_parsed { if is_url_redirector( self, location_parsed.host.sld_or_default(), ctx.input.span_id, ) .await { url_redirect = Cow::Owned(location.url); redirect_count += 1; continue; } else { redirected_urls.insert(ElementLocation::new( location, url.location, )); } } } Ok(None) => {} Err(err) => { trc::error!(err.span_id(ctx.input.span_id)); } } break; } if redirect_count > 3 { ctx.result.add_tag("URL_REDIRECTOR_NESTED"); } } } } urls.extend(redirected_urls); for (el, url_parsed) in urls.iter().filter_map(|el| { el.element .url_parsed .as_ref() .map(|url_parsed| (el, url_parsed)) }) { let host = &url_parsed.host; if host.ip.is_none() { if !host.fqdn.is_ascii() { if let Ok(cured_host) = decancer::cure(&host.fqdn, decancer::Options::default()) { let cured_host = cured_host.to_string(); if cured_host != host.fqdn && matches!(self.dns_exists_ip(&cured_host).await, Ok(true)) { ctx.result.add_tag("HOMOGRAPH_URL"); } } if host.fqdn.is_mixed_charset() { ctx.result.add_tag("MIXED_CHARSET_URL"); } } // Check Domain DNSBL if let Some(sld) = &host.sld { check_dnsbl( self, ctx, &StringResolver(sld), Element::Domain, el.location, ) .await; } } else { // URL is an ip address ctx.result.add_tag("SUSPICIOUS_URL"); } // Check URL DNSBL check_dnsbl(self, ctx, &el.element, Element::Url, el.location).await; } } // Update context ctx.output.urls = urls; } } #[allow(unreachable_code)] #[allow(unused_variables)] async fn http_get_header( url: &str, header: hyper::header::HeaderName, timeout: Duration, ) -> trc::Result> { #[cfg(feature = "test_mode")] { return if url.contains("redirect.") { Ok(url.split_once("/?").unwrap().1.to_string().into()) } else { Ok(None) }; } reqwest::Client::builder() .user_agent("Mozilla/5.0 (X11; Linux i686; rv:109.0) Gecko/20100101 Firefox/118.0") .timeout(timeout) .redirect(Policy::none()) .danger_accept_invalid_certs(true) .build() .map_err(|err| { trc::SieveEvent::RuntimeError .into_err() .reason(err) .details("Failed to build request") })? .get(url) .send() .await .map_err(|err| { trc::SieveEvent::RuntimeError .into_err() .reason(err) .details("Failed to send request") }) .map(|response| { response .headers() .get(header) .and_then(|h| h.to_str().ok()) .map(|h| h.to_string()) }) } fn is_single_url(tokens: &[TokenType]) -> bool { let mut url_count = 0; let mut word_count = 0; for token in tokens { match token { TokenType::Alphabetic(_) | TokenType::Alphanumeric(_) | TokenType::Integer(_) | TokenType::Email(_) | TokenType::Float(_) => { word_count += 1; } TokenType::Url(_) | TokenType::UrlNoScheme(_) => { url_count += 1; } _ => {} } } url_count == 1 && word_count <= 1 } fn is_single_html_url( html_tokens: &[HtmlToken], tokens: &[TokenType], ) -> bool { let mut url_count = 0; let mut word_count = 0; for token in tokens { match token { TokenType::Alphabetic(_) | TokenType::Alphanumeric(_) | TokenType::Integer(_) | TokenType::Email(_) | TokenType::Float(_) => { word_count += 1; } TokenType::Url(_) | TokenType::UrlNoScheme(_) => { url_count += 1; } _ => {} } } if word_count > 1 || url_count != 1 { return false; } url_count = 0; for token in html_tokens { if matches!(token, HtmlToken::StartTag { name, attributes, .. } if *name == A && attributes.iter().any(|(k, _)| *k == HREF)) { url_count += 1; } } url_count == 1 } impl PartialEq for UrlParts<'_> { fn eq(&self, other: &Self) -> bool { self.url == other.url } } impl Eq for UrlParts<'_> {} impl Hash for UrlParts<'_> { fn hash(&self, state: &mut H) { self.url.hash(state); } } impl<'x> UrlParts<'x> { pub fn new(url: impl Into>) -> Self { let url_original = url.into(); let url = url_original.trim().to_lowercase(); Self { url_parsed: url.parse::().ok().and_then(|url_parsed| { if url_parsed.host().is_some() { Some(UrlParsed { host: Hostname::new(url_parsed.host().unwrap()), parts: url_parsed, }) } else { None } }), url, url_original, } } pub fn to_owned(&self) -> UrlParts<'static> { UrlParts { url: self.url.clone(), url_original: Cow::Owned(self.url_original.clone().into_owned()), url_parsed: self.url_parsed.clone(), } } } ================================================ FILE: crates/spam-filter/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod analysis; pub mod modules; use analysis::ElementLocation; use analysis::url::UrlParts; use mail_auth::{ArcOutput, DkimOutput, DmarcResult, IprevOutput, SpfOutput, dmarc::Policy}; use mail_parser::Message; use modules::html::HtmlToken; use nlp::tokenizers::types::TokenType; use std::borrow::Cow; use std::collections::HashSet; use std::hash::{Hash, Hasher}; use std::net::{IpAddr, Ipv4Addr}; use store::ahash::AHashSet; pub struct SpamFilterInput<'x> { pub message: &'x Message<'x>, pub span_id: u64, // Sender authentication pub arc_result: Option<&'x ArcOutput<'x>>, pub spf_ehlo_result: Option<&'x SpfOutput>, pub spf_mail_from_result: Option<&'x SpfOutput>, pub dkim_result: &'x [DkimOutput<'x>], pub dmarc_result: Option<&'x DmarcResult>, pub dmarc_policy: Option<&'x Policy>, pub iprev_result: Option<&'x IprevOutput>, // Session details pub remote_ip: IpAddr, pub ehlo_domain: Option<&'x str>, pub authenticated_as: Option<&'x str>, pub asn: Option, pub country: Option<&'x str>, // TLS pub is_tls: bool, // Envelope pub env_from: &'x str, pub env_from_flags: u64, pub env_rcpt_to: Vec<&'x str>, pub is_train: bool, pub is_test: bool, } pub struct SpamFilterOutput<'x> { pub ehlo_host: Hostname, pub iprev_ptr: Option, pub env_from_addr: Email, pub env_from_postmaster: bool, pub env_to_addr: HashSet, pub from: Recipient, pub recipients_to: Vec, pub recipients_cc: Vec, pub recipients_bcc: Vec, pub reply_to: Option, pub subject: String, pub subject_lc: String, pub subject_thread: String, pub subject_thread_lc: String, pub subject_tokens: Vec, Email, UrlParts<'x>, IpParts>>, pub ips: AHashSet>, pub urls: HashSet>>, pub emails: HashSet>, pub domains: HashSet>, pub text_parts: Vec>, } #[derive(Debug)] pub struct IpParts { ip: Option, } pub enum TextPart<'x> { Plain { text_body: &'x str, tokens: Vec, Email, UrlParts<'x>, IpParts>>, }, Html { html_tokens: Vec, text_body: String, tokens: Vec, Email, UrlParts<'x>, IpParts>>, }, None, } #[derive(Debug, Default)] pub struct SpamFilterResult { pub tags: AHashSet, pub classifier_confidence: Vec>, pub score: f32, pub rbl_ip_checks: usize, pub rbl_domain_checks: usize, pub rbl_url_checks: usize, pub rbl_email_checks: usize, pub llm_result: Option<(String, String)>, } pub struct SpamFilterContext<'x> { pub input: SpamFilterInput<'x>, pub output: SpamFilterOutput<'x>, pub result: SpamFilterResult, } #[derive(Debug, Clone)] pub struct Hostname { pub fqdn: String, pub ip: Option, pub sld: Option, } #[derive(Debug, Clone)] pub struct Email { pub address: String, pub local_part: String, pub domain_part: Hostname, } #[derive(Debug, Clone)] pub struct Recipient { pub email: Email, pub name: Option, } impl<'x> SpamFilterInput<'x> { pub fn from_message(message: &'x Message<'x>, span_id: u64) -> Self { Self { message, span_id, arc_result: None, spf_ehlo_result: None, spf_mail_from_result: None, dkim_result: &[], dmarc_result: None, dmarc_policy: None, iprev_result: None, remote_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), ehlo_domain: None, authenticated_as: None, asn: None, country: None, is_tls: true, env_from: "", env_from_flags: 0, env_rcpt_to: vec![], is_test: false, is_train: false, } } pub fn train_mode(mut self) -> Self { self.is_train = true; self } } impl PartialEq for Hostname { fn eq(&self, other: &Self) -> bool { self.fqdn.eq(&other.fqdn) } } impl Eq for Hostname {} impl PartialEq for Email { fn eq(&self, other: &Self) -> bool { self.address.eq(&other.address) } } impl Eq for Email {} impl Hash for Hostname { fn hash(&self, state: &mut H) { self.fqdn.hash(state) } } impl Hash for Email { fn hash(&self, state: &mut H) { self.address.hash(state) } } impl Email { pub fn classifier_parts(&self) -> Option<(&str, &str)> { // Returns (local@, @domain) if self.is_valid() { let at_pos = self.address.find('@')?; Some((&self.address[..=at_pos], &self.address[at_pos..])) } else { None } } pub fn is_valid(&self) -> bool { self.domain_part.sld.is_some() && !self.local_part.is_empty() } } impl PartialEq for Recipient { fn eq(&self, other: &Self) -> bool { self.email.eq(&other.email) } } impl Eq for Recipient {} impl Hash for Recipient { fn hash(&self, state: &mut H) { self.email.hash(state) } } impl PartialOrd for Email { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } impl PartialOrd for Recipient { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } impl Ord for Email { fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.address.cmp(&other.address) } } impl Ord for Recipient { fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.email.cmp(&other.email) } } ================================================ FILE: crates/spam-filter/src/modules/classifier.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::analysis::domain::SpamFilterAnalyzeDomain; use crate::analysis::init::SpamFilterInit; use crate::analysis::is_trusted_domain; use crate::analysis::url::SpamFilterAnalyzeUrl; use crate::modules::html::{A, ALT, HREF, HtmlToken, IMG, SRC, TITLE}; use crate::{Email, SpamFilterContext, TextPart}; use crate::{Hostname, SpamFilterInput}; use common::config::spamfilter; use common::manager::{SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY}; use common::{Server, config::spamfilter::Location, ipc::BroadcastEvent}; use mail_auth::DmarcResult; use mail_parser::{MessageParser, MimeHeaders}; use nlp::classifier::feature::{ CcfhFeature, CcfhFeatureBuilder, FeatureBuilder, FhFeature, FhFeatureBuilder, Sample, UnprocessedFeature, }; use nlp::classifier::ftrl::Ftrl; use nlp::classifier::reservoir::SampleReservoir; use nlp::classifier::train::{CcfhTrainer, FhTrainer}; use nlp::tokenizers::types::TypesTokenizer; use nlp::tokenizers::{stream::WordStemTokenizer, types::TokenType}; use std::time::Instant; use std::{ borrow::Cow, collections::{HashMap, hash_map::Entry}, hash::{Hash, RandomState}, sync::Arc, }; use store::rand::seq::SliceRandom; use store::write::{BlobLink, now}; use store::{ Deserialize, IterateParams, Serialize, U32_LEN, U64_LEN, ValueKey, write::{ AlignedBytes, Archive, Archiver, BatchBuilder, BlobOp, ValueClass, key::DeserializeBigEndian, }, }; use tokio::sync::{mpsc, oneshot}; use trc::{AddContext, SpamEvent}; use types::blob_hash::BlobHash; use unicode_general_category::{GeneralCategory, get_general_category}; use unicode_normalization::UnicodeNormalization; use unicode_security::mixed_script::AugmentedScriptSet; pub trait SpamClassifier { fn spam_train(&self, retrain: bool) -> impl Future> + Send; fn spam_classify( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future> + Send; fn spam_build_tokens<'x>( &self, ctx: &'x SpamFilterContext<'_>, ) -> impl Future> + Send; } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Clone, PartialEq, Eq, Debug)] pub struct TrainingSample { hash: BlobHash, account_id: u32, } struct TrainingTask { sample: TrainingSample, is_spam: bool, is_replay: bool, remove: Option, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug)] pub struct SpamTrainer { pub trainer: SpamTrainerClass, pub reservoir: SampleReservoir, pub last_sample_expiry: u64, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug)] pub enum SpamTrainerClass { FtrlFh(Box>), FtrlCfh(Box>), } impl SpamClassifier for Server { async fn spam_train(&self, retrain: bool) -> trc::Result<()> { let Some(config) = &self.core.spam.classifier else { return Ok(()); }; let _permit = self .inner .ipc .train_task_controller .try_run() .ok_or_else(|| { trc::EventType::Spam(SpamEvent::TrainCompleted) .reason("Spam training task is already running") .caused_by(trc::location!()) })?; let started = Instant::now(); trc::event!(Spam(SpamEvent::TrainStarted)); // Fetch or build trainer let mut trainer = if !retrain && let Some(trainer) = self .blob_store() .get_blob(SPAM_TRAINER_KEY, 0..usize::MAX) .await .and_then(|archive| match archive { Some(archive) => as Deserialize>::deserialize(&archive) .and_then(|archive| archive.deserialize_untrusted::()) .map(Some), None => Ok(None), }) .caused_by(trc::location!())? { trainer } else { SpamTrainer { trainer: match &config.i_params { Some(i_params) => SpamTrainerClass::FtrlCfh(Box::new(CcfhTrainer::new( Ftrl::new(config.w_params.feature_hash_size), Ftrl::new(i_params.feature_hash_size).with_initial_weights(0.5), ))), None => SpamTrainerClass::FtrlFh(Box::new(FhTrainer::new(Ftrl::new( config.w_params.feature_hash_size, )))), }, reservoir: SampleReservoir::default(), last_sample_expiry: 0, } }; // Update hyperparameters match (&mut trainer.trainer, &config.i_params) { (SpamTrainerClass::FtrlFh(trainer), None) => { trainer.optimizer_mut().set_hyperparams( config.w_params.alpha, config.w_params.beta, config.w_params.l1_ratio, config.w_params.l2_ratio, ); } (SpamTrainerClass::FtrlCfh(trainer), Some(i_params)) => { trainer.w_optimizer_mut().set_hyperparams( config.w_params.alpha, config.w_params.beta, config.w_params.l1_ratio, config.w_params.l2_ratio, ); trainer.i_optimizer_mut().set_hyperparams( i_params.alpha, i_params.beta, i_params.l1_ratio, i_params.l2_ratio, ); } _ => {} } // Fetch blob hashes for samples let mut samples = Vec::new(); let mut remove_entries = false; let from_key = ValueKey { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Blob(BlobOp::SpamSample { hash: BlobHash::default(), until: trainer.last_sample_expiry + 1, }), }; let to_key = ValueKey { account_id: u32::MAX, collection: u8::MAX, document_id: u32::MAX, class: ValueClass::Blob(BlobOp::SpamSample { hash: BlobHash::new_max(), until: u64::MAX, }), }; let mut spam_count = 0; let mut ham_count = 0; self.store() .iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { let until = key.deserialize_be_u64(1)?; let account_id = key.deserialize_be_u32(U64_LEN + 1)?; let hash = BlobHash::try_from_hash_slice( key.get(U64_LEN + U32_LEN + 1..).ok_or_else(|| { trc::Error::corrupted_key(key, value.into(), trc::location!()) })?, ) .unwrap(); let (Some(is_spam), Some(hold)) = (value.first(), value.get(1)) else { return Err(trc::Error::corrupted_key( key, value.into(), trc::location!(), )); }; let do_remove = *hold == 0; let is_spam = *is_spam == 1; let sample = TrainingSample { hash, account_id }; // Add to reservoir if !do_remove { trainer.reservoir.update_reservoir( &sample, is_spam, config.reservoir_capacity, ); } else { trainer.reservoir.update_counts(is_spam); } samples.push(TrainingTask { sample, is_spam, is_replay: false, remove: do_remove.then_some(until), }); remove_entries |= do_remove; // Update trainer stats trainer.last_sample_expiry = until; if is_spam { spam_count += 1; } else { ham_count += 1; } Ok(true) }, ) .await .caused_by(trc::location!())?; if samples.is_empty() { trc::event!( Spam(SpamEvent::TrainCompleted), Total = 0, Elapsed = started.elapsed() ); return Ok(()); } else if (trainer.reservoir.ham.total_seen < config.min_ham_samples) || (trainer.reservoir.spam.total_seen < config.min_spam_samples) { trc::event!( Spam(SpamEvent::ModelNotReady), Reason = "Not enough samples for training", Details = vec![ trc::Value::from(trainer.reservoir.ham.total_seen), trc::Value::from(trainer.reservoir.spam.total_seen) ], Limit = vec![ trc::Value::from(config.min_ham_samples), trc::Value::from(config.min_spam_samples) ], Elapsed = started.elapsed() ); return Ok(()); } // Balance classes if needed if spam_count > ham_count { // We have too much spam this time. We need to replay old HAM. samples.extend( trainer .reservoir .replay_samples((spam_count - ham_count) as usize, false) .map(|sample| TrainingTask { sample: sample.clone(), is_spam: false, is_replay: true, remove: None, }), ); } else if ham_count > spam_count { // We have too much ham this time. We need to replay old SPAM. samples.extend( trainer .reservoir .replay_samples((ham_count - spam_count) as usize, true) .map(|sample| TrainingTask { sample: sample.clone(), is_spam: true, is_replay: true, remove: None, }), ); } let num_samples = samples.len(); samples.shuffle(&mut store::rand::rng()); // Spawn training task let epochs = match trainer .reservoir .ham .total_seen .min(trainer.reservoir.spam.total_seen) { 0..=50 => 3, // Bootstrap 51..=200 => 2, // Refinement _ => 1, // Full online training }; let task = trainer.trainer.spawn(epochs)?; let is_fh = matches!(task, TrainTask::Fh { .. }); // Train for chunk in samples.chunks(128) { let mut fh_samples = if is_fh { Vec::with_capacity(chunk.len()) } else { Vec::new() }; let mut ccfh_samples = if !is_fh { Vec::with_capacity(chunk.len()) } else { Vec::new() }; for sample in chunk { let account_id = if sample.sample.account_id != u32::MAX { Some(sample.sample.account_id) } else { None }; let Some(raw_message) = self .blob_store() .get_blob(sample.sample.hash.as_slice(), 0..usize::MAX) .await .caused_by(trc::location!())? else { if sample.is_replay { trainer .reservoir .remove_sample(&sample.sample, sample.is_spam); } else { trc::event!( Spam(SpamEvent::TrainSampleNotFound), Reason = "Blob not found", AccountId = account_id, BlobId = sample.sample.hash.to_hex(), ); } continue; }; // Build features let Some(message) = MessageParser::new().parse(&raw_message) else { if sample.is_replay { trainer .reservoir .remove_sample(&sample.sample, sample.is_spam); } trc::event!( Spam(SpamEvent::TrainSampleNotFound), Reason = "Failed to parse message", AccountId = account_id, BlobId = sample.sample.hash.to_hex(), ); continue; }; let mut ctx = self.spam_filter_init(SpamFilterInput::from_message(&message, 0).train_mode()); self.spam_filter_analyze_domain(&mut ctx).await; self.spam_filter_analyze_url(&mut ctx).await; let mut tokens = self.spam_build_tokens(&ctx).await.0; match &task { TrainTask::Fh { builder, .. } => { if config.log_scale { builder.scale(&mut tokens); } fh_samples.push(Sample::new( builder.build(&tokens, account_id, config.l2_normalize), sample.is_spam, )); } TrainTask::Ccfh { builder, .. } => { if config.log_scale { builder.scale(&mut tokens); } ccfh_samples.push(Sample::new( builder.build(&tokens, account_id, config.l2_normalize), sample.is_spam, )); } } // Look for stop requests if self.inner.ipc.train_task_controller.should_stop() { trc::event!( Spam(SpamEvent::TrainCompleted), Reason = "Training task was stopped", Total = fh_samples.len() + ccfh_samples.len(), Elapsed = started.elapsed() ); return Ok(()); } } // Send batch for training let (done_tx, done_rx) = oneshot::channel::<()>(); match &task { TrainTask::Fh { batch_tx, .. } => { batch_tx .send(FhTrainJob { samples: fh_samples, done: done_tx, }) .await .map_err(|err| { trc::EventType::Server(trc::ServerEvent::ThreadError) .reason(err) .details("Spam train task failed") .caused_by(trc::location!()) })?; } TrainTask::Ccfh { batch_tx, .. } => { batch_tx .send(CcfhTrainJob { samples: ccfh_samples, done: done_tx, }) .await .map_err(|err| { trc::EventType::Server(trc::ServerEvent::ThreadError) .reason(err) .details("Spam train task failed") .caused_by(trc::location!()) })?; } } done_rx.await.map_err(|err| { trc::EventType::Server(trc::ServerEvent::ThreadError) .reason(err) .details("Spam train task failed") .caused_by(trc::location!()) })?; } // Take ownership of trainer trainer.trainer = match task { TrainTask::Fh { batch_tx, trainer_rx, .. } => { drop(batch_tx); SpamTrainerClass::FtrlFh(trainer_rx.await.map_err(|err| { trc::EventType::Server(trc::ServerEvent::ThreadError) .reason(err) .details("Spam train task failed") .caused_by(trc::location!()) })?) } TrainTask::Ccfh { batch_tx, trainer_rx, .. } => { drop(batch_tx); SpamTrainerClass::FtrlCfh(trainer_rx.await.map_err(|err| { trc::EventType::Server(trc::ServerEvent::ThreadError) .reason(err) .details("Spam train task failed") .caused_by(trc::location!()) })?) } }; // Store updated trainer and classifier let ham_count = trainer.reservoir.ham.total_seen; let spam_count = trainer.reservoir.spam.total_seen; let classifier = Archiver::new(match &trainer.trainer { SpamTrainerClass::FtrlFh(fh_trainer) => spamfilter::SpamClassifier::FhClassifier { classifier: fh_trainer.build_classifier(), last_trained_at: now(), }, SpamTrainerClass::FtrlCfh(ccfh_trainer) => spamfilter::SpamClassifier::CcfhClassifier { classifier: ccfh_trainer.build_classifier(), last_trained_at: now(), }, }); self.blob_store() .put_blob( SPAM_TRAINER_KEY, &Archiver::new(trainer) .serialize() .caused_by(trc::location!())?, ) .await .caused_by(trc::location!())?; self.blob_store() .put_blob( SPAM_CLASSIFIER_KEY, &classifier.serialize().caused_by(trc::location!())?, ) .await .caused_by(trc::location!())?; self.inner .data .spam_classifier .store(Arc::new(classifier.inner)); self.cluster_broadcast(BroadcastEvent::ReloadSpamFilter) .await; trc::event!( Spam(SpamEvent::TrainCompleted), Total = num_samples, Details = vec![trc::Value::from(ham_count), trc::Value::from(spam_count)], Elapsed = started.elapsed() ); // Remove samples marked for deletion if remove_entries { let mut batch = BatchBuilder::new(); for sample in samples { if let Some(until) = sample.remove { batch .with_account_id(sample.sample.account_id) .clear(BlobOp::Link { hash: sample.sample.hash.clone(), to: BlobLink::Temporary { until }, }) .clear(BlobOp::SpamSample { hash: sample.sample.hash, until, }); if batch.is_large_batch() { self.store() .write(batch.build_all()) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); } } } if !batch.is_empty() { self.store() .write(batch.build_all()) .await .caused_by(trc::location!())?; } } Ok(()) } async fn spam_classify(&self, ctx: &mut SpamFilterContext<'_>) -> trc::Result<()> { let classifier = self.inner.data.spam_classifier.load_full(); let Some(config) = &self.core.spam.classifier else { return Ok(()); }; let started = Instant::now(); match classifier.as_ref() { spamfilter::SpamClassifier::FhClassifier { classifier, .. } => { let mut classifier_confidence = Vec::with_capacity(ctx.input.env_rcpt_to.len()); let mut has_prediction = false; let mut tokens = self.spam_build_tokens(ctx).await.0; let feature_builder = classifier.feature_builder(); if config.log_scale { feature_builder.scale(&mut tokens); } for rcpt in &ctx.input.env_rcpt_to { let prediction = if let Some(account_id) = self .directory() .email_to_id(rcpt) .await .caused_by(trc::location!())? { has_prediction = true; classifier .predict_proba_sample(&feature_builder.build( &tokens, account_id.into(), config.l2_normalize, )) .into() } else { None }; classifier_confidence.push(prediction); } if has_prediction { ctx.result.classifier_confidence = classifier_confidence; } else { // None of the recipients are local, default to global model prediction let prediction = classifier.predict_proba_sample(&feature_builder.build( &tokens, None, config.l2_normalize, )); ctx.result.classifier_confidence = vec![prediction.into(); ctx.input.env_rcpt_to.len()]; } } spamfilter::SpamClassifier::CcfhClassifier { classifier, .. } => { let mut classifier_confidence = Vec::with_capacity(ctx.input.env_rcpt_to.len()); let mut has_prediction = false; let mut tokens = self.spam_build_tokens(ctx).await.0; let feature_builder = classifier.feature_builder(); if config.log_scale { feature_builder.scale(&mut tokens); } for rcpt in &ctx.input.env_rcpt_to { let prediction = if let Some(account_id) = self .directory() .email_to_id(rcpt) .await .caused_by(trc::location!())? { has_prediction = true; classifier .predict_proba_sample(&feature_builder.build( &tokens, account_id.into(), config.l2_normalize, )) .into() } else { None }; classifier_confidence.push(prediction); } if has_prediction { ctx.result.classifier_confidence = classifier_confidence; } else { // None of the recipients are local, default to global model prediction let prediction = classifier.predict_proba_sample(&feature_builder.build( &tokens, None, config.l2_normalize, )); ctx.result.classifier_confidence = vec![prediction.into(); ctx.input.env_rcpt_to.len()]; } } spamfilter::SpamClassifier::Disabled => { return Ok(()); } } trc::event!( Spam(SpamEvent::Classify), Result = ctx .result .classifier_confidence .iter() .zip(ctx.input.env_rcpt_to.iter()) .map(|(v, rcpt)| trc::Value::Array(vec![ trc::Value::from(rcpt.to_string()), trc::Value::from(*v) ])) .collect::>(), SpanId = ctx.input.span_id, Elapsed = started.elapsed() ); Ok(()) } async fn spam_build_tokens<'x>(&self, ctx: &'x SpamFilterContext<'_>) -> Tokens<'x> { let mut tokens = Tokens::default(); // Add From addresses if ctx .input .dmarc_result .as_ref() .is_some_and(|result| **result != DmarcResult::Pass) { tokens.insert(Token::Sender { value: "!".into() }); } for email in [&ctx.output.env_from_addr, &ctx.output.from.email] { tokens.insert_email(email, true); } // Add Email addresses for email in &ctx.output.emails { let is_sender = match &email.location { Location::HeaderReplyTo | Location::HeaderDnt => true, Location::BodyText | Location::BodyHtml | Location::Attachment | Location::HeaderSubject => false, _ => continue, }; if is_sender || !is_trusted_domain( self, email.element.email.domain_part.sld_or_default(), ctx.input.span_id, ) .await { tokens.insert_email(&email.element.email, is_sender); } } // Add URLs for url in &ctx.output.urls { if let Some(url) = &url.element.url_parsed && !is_trusted_domain(self, url.host.sld_or_default(), ctx.input.span_id).await { if let Some(host) = &url.host.sld { tokens.insert(Token::Url { value: host.into() }); if host != &url.host.fqdn { tokens.insert(Token::Url { value: url.host.fqdn.as_str().into(), }); } } else { tokens.insert(Token::Url { value: url.host.fqdn.as_str().into(), }); } for token in url .parts .path() .split(['/', '.', '_']) .filter(|v| v.chars().all(|ch| ch.is_alphabetic())) { if token.len() > 2 { let token = truncate_word(token, MAX_TOKEN_LENGTH); tokens.insert(Token::Url { value: format!("_{token}").into(), }); } } } } // Add hostnames for domain in &ctx.output.domains { if matches!( domain.location, Location::HeaderReceived | Location::HeaderMid | Location::Ehlo | Location::Tcp ) { let host = Hostname::new(&domain.element); let host_sld = host.sld_or_default(); if !is_trusted_domain(self, host_sld, ctx.input.span_id).await { if !host_sld.is_empty() && host_sld != host.fqdn { tokens.insert(Token::Hostname { value: host_sld.to_string().into(), }); } tokens.insert(Token::Hostname { value: host.fqdn.into(), }); } } } // Add ASN if let Some(asn) = ctx.input.asn { tokens.insert(Token::Asn { number: asn.to_be_bytes(), }); } // Add MIME and attachment indicators for part in &ctx.input.message.parts { if let Some(name) = part.attachment_name() && let Some((name, ext)) = name.rsplit_once('.') { if !ext.is_empty() { tokens.insert(Token::Attachment { value: lower_prefix("!", truncate_word(ext, MAX_TOKEN_LENGTH)).into(), }); } let name = name.to_lowercase(); let word_tokenizer = WordStemTokenizer::new(&name); for token in TypesTokenizer::new(&name) { if let TokenType::Alphabetic(word) = token.word { word_tokenizer.tokenize(word, |token| { tokens.insert(Token::Attachment { value: format!( "_{}", truncate_word(token.as_ref(), MAX_TOKEN_LENGTH) ) .into(), }); }); } } } if let Some(ct) = part.content_type() { let mut ct_lower = String::with_capacity( ct.c_type.len() + ct.c_subtype.as_ref().map_or(0, |s| s.len()), ); for ch in ct.c_type.chars() { ct_lower.push(ch.to_ascii_lowercase()); } if let Some(st) = &ct.c_subtype { ct_lower.push('/'); for ch in st.chars() { ct_lower.push(ch.to_ascii_lowercase()); } } tokens.insert(Token::MimeType { value: ct_lower }); } } // Tokenize the subject for token in &ctx.output.subject_tokens { tokens.insert_type( &WordStemTokenizer::new(&ctx.output.subject_thread_lc), token, false, ); } // Tokenize the text parts let body_idx = ctx .input .message .html_body .first() .or_else(|| ctx.input.message.text_body.first()) .map(|idx| *idx as usize); let mut alt_tokens = Tokens::default(); for (idx, part) in ctx.output.text_parts.iter().enumerate() { let is_body = Some(idx) == body_idx; if is_body || (!ctx.input.message.text_body.contains(&(idx as u32)) && !ctx.input.message.html_body.contains(&(idx as u32))) { tokens.insert_text_part(part, is_body); } else { alt_tokens.insert_text_part(part, false); } } if !alt_tokens.0.is_empty() { for (token, count) in alt_tokens.0.into_iter() { if let Entry::Vacant(entry) = tokens.0.entry(token) { entry.insert(count); } } } tokens } } struct FhTrainJob { samples: Vec>, done: oneshot::Sender<()>, } struct CcfhTrainJob { samples: Vec>, done: oneshot::Sender<()>, } enum TrainTask { Fh { batch_tx: mpsc::Sender, trainer_rx: oneshot::Receiver>>, builder: FhFeatureBuilder, }, Ccfh { batch_tx: mpsc::Sender, trainer_rx: oneshot::Receiver>>, builder: CcfhFeatureBuilder, }, } impl SpamTrainerClass { fn spawn(self, num_epochs: usize) -> trc::Result { match self { SpamTrainerClass::FtrlFh(mut trainer) => { let builder = trainer.feature_builder(); let (batch_tx, mut batch_rx) = mpsc::channel::(1); let (trainer_tx, trainer_rx) = oneshot::channel(); std::thread::Builder::new() .name("FTRL Train Task".into()) .spawn(move || { while let Some(mut job) = batch_rx.blocking_recv() { trainer.fit(&mut job.samples, num_epochs); let _ = job.done.send(()); } // Send trainer back when done let _ = trainer_tx.send(trainer); }) .map_err(|err| { trc::EventType::Server(trc::ServerEvent::ThreadError) .reason(err) .details("Failed to spawn spam train task") .caused_by(trc::location!()) })?; Ok(TrainTask::Fh { batch_tx, trainer_rx, builder, }) } SpamTrainerClass::FtrlCfh(mut trainer) => { let builder = trainer.feature_builder(); let (batch_tx, mut batch_rx) = mpsc::channel::(1); let (trainer_tx, trainer_rx) = oneshot::channel(); std::thread::Builder::new() .name("FTRL Train Task".into()) .spawn(move || { while let Some(mut job) = batch_rx.blocking_recv() { trainer.fit(&mut job.samples, num_epochs); let _ = job.done.send(()); } // Send trainer back when done let _ = trainer_tx.send(trainer); }) .map_err(|err| { trc::EventType::Server(trc::ServerEvent::ThreadError) .reason(err) .details("Failed to spawn spam train task") .caused_by(trc::location!()) })?; Ok(TrainTask::Ccfh { batch_tx, trainer_rx, builder, }) } } } } const MAX_TOKEN_LENGTH: usize = 16; #[derive( Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, PartialOrd, Ord, )] #[serde(tag = "type", rename_all = "snake_case")] pub enum Token<'x> { Word { value: Cow<'x, str> }, Number { code: [u8; 2] }, Alphanumeric { code: [u8; 4] }, UnicodeCategory { value: &'x str }, Sender { value: Cow<'x, str> }, Asn { number: [u8; 4] }, Url { value: Cow<'x, str> }, Email { value: Cow<'x, str> }, Hostname { value: Cow<'x, str> }, Attachment { value: Cow<'x, str> }, MimeType { value: String }, HtmlImage { src: &'x str }, HtmlAnchor { href: &'x str }, } #[derive(Debug)] pub struct Tokens<'x>(pub HashMap, f32, RandomState>); impl<'x> Tokens<'x> { fn insert_text_part(&mut self, part: &'x TextPart<'x>, is_body: bool) { match part { TextPart::Plain { text_body, tokens } => { let word_tokenizer = WordStemTokenizer::new(text_body); for token in tokens { self.insert_type(&word_tokenizer, token, is_body); } if is_body && (tokens.is_empty() || !tokens.iter().any(|t| matches!(t, TokenType::Alphabetic(_)))) { self.insert(Token::Word { value: "_null".into(), }); } } TextPart::Html { text_body, tokens, html_tokens, } => { let word_tokenizer = WordStemTokenizer::new(text_body); for token in tokens { self.insert_type(&word_tokenizer, token, is_body); } if is_body { if tokens.is_empty() || !tokens.iter().any(|t| matches!(t, TokenType::Alphabetic(_))) { self.insert(Token::Word { value: "_null".into(), }); } for token in html_tokens { if let HtmlToken::StartTag { name: A | IMG, attributes, .. } = token { for (name, value) in attributes { match (*name, value) { (ALT | TITLE, Some(value)) => { for token in TypesTokenizer::new(value) { self.insert_type(&word_tokenizer, &token.word, is_body); } } (SRC, Some(value)) => { self.insert(Token::HtmlImage { src: value.split_once(':').unwrap_or_default().0, }); } (HREF, Some(value)) => { self.insert(Token::HtmlAnchor { href: value.split_once(':').unwrap_or_default().0, }); } _ => {} } } } } } } TextPart::None => (), } } fn insert_type, E, U, I>( &mut self, word_tokenizer: &WordStemTokenizer, token: &TokenType, is_body: bool, ) { match token { TokenType::Alphabetic(word) => { let word = word.as_ref(); let mut set: Option = None; let mut has_confusables = false; let mut upper_count = 0; for ch in word.chars() { if ch.is_uppercase() { upper_count += 1; } has_confusables |= !ch.is_ascii() && !std::iter::once(ch).nfc().eq(std::iter::once(ch).nfkc()); set.get_or_insert_default().intersect_with(ch.into()); } let is_mixed_script = set.is_some_and(|set| set.is_empty()); if (is_mixed_script || has_confusables) && let Ok(cured_word) = decancer::cure(word, decancer::Options::default()) { if word.len() > MAX_TOKEN_LENGTH { self.insert(Token::Word { value: truncate_word(cured_word.as_str(), MAX_TOKEN_LENGTH) .to_string() .into(), }); } else { self.insert(Token::Word { value: String::from(cured_word).into(), }); } } else { let word = word.to_lowercase(); word_tokenizer.tokenize(&word, |token| { self.insert(Token::Word { value: truncate_word(token.as_ref(), MAX_TOKEN_LENGTH) .to_string() .into(), }); }); } if is_body && word.len() == upper_count && word.len() > 3 { self.insert(Token::Word { value: "_allcaps".into(), }); } } TokenType::Alphanumeric(word) => { self.insert(Token::from_alphanumeric(word.as_ref())); } TokenType::UrlNoHost(url) => { for token in url .as_ref() .to_lowercase() .split(['/', '.', '_']) .filter(|v| v.chars().all(|ch| ch.is_alphabetic())) { if token.len() > 2 { let token = truncate_word(token, MAX_TOKEN_LENGTH); self.insert(Token::Url { value: format!("_{token}").into(), }); } } } TokenType::Other(ch) | TokenType::Punctuation(ch) => { let category = get_general_category(*ch); if !matches!( category, GeneralCategory::ClosePunctuation | GeneralCategory::ConnectorPunctuation | GeneralCategory::DashPunctuation | GeneralCategory::FinalPunctuation | GeneralCategory::InitialPunctuation | GeneralCategory::OpenPunctuation | GeneralCategory::OtherPunctuation | GeneralCategory::SpaceSeparator ) { self.insert(Token::UnicodeCategory { value: category.abbreviation(), }); } } TokenType::Integer(word) => { self.insert(Token::from_number(false, word.as_ref())); } TokenType::Float(word) => { self.insert(Token::from_number(true, word.as_ref())); } TokenType::IpAddr(_) => { self.insert(Token::Url { value: "!ip".into(), }); } TokenType::Email(_) | TokenType::Url(_) | TokenType::UrlNoScheme(_) | TokenType::Space => {} } } fn insert(&mut self, token: Token<'x>) { *self.0.entry(token).or_insert(0.0) += 1.0; } fn insert_if_missing(&mut self, token: Token<'x>) { self.0.entry(token).or_insert(1.0); } fn insert_email(&mut self, email: &'x Email, is_sender: bool) { if !email.address.is_empty() { if is_sender { self.insert_if_missing(Token::Sender { value: email.address.as_str().into(), }); self.insert_if_missing(Token::Sender { value: email.domain_part.fqdn.as_str().into(), }); if let Some(sld) = &email.domain_part.sld && sld != &email.domain_part.fqdn { self.insert_if_missing(Token::Sender { value: sld.into() }); } } else { self.insert_if_missing(Token::Email { value: email.address.as_str().into(), }); self.insert_if_missing(Token::Email { value: email.domain_part.fqdn.as_str().into(), }); if let Some(sld) = &email.domain_part.sld && !sld.is_empty() && sld != &email.domain_part.fqdn { self.insert_if_missing(Token::Email { value: sld.into() }); } } } } } impl Token<'static> { fn from_alphanumeric(s: &str) -> Self { let mut is_hex = true; let mut is_ascii = true; let mut digit_count = 0; for ch in s.chars() { match ch { 'a'..='f' | 'A'..='F' => {} '0'..='9' => { digit_count += 1; } _ => { is_ascii &= ch.is_ascii(); is_hex = false; } } } if is_hex { Token::Number { code: [b'X', s.len().min(u8::MAX as usize) as u8], } } else if !is_ascii { let word: String = if let Ok(cured) = decancer::cure(s, decancer::Options::default()) { cured .as_str() .chars() .filter(|ch| ch.is_alphabetic()) .take(MAX_TOKEN_LENGTH) .collect() } else { s.chars() .filter(|ch| ch.is_alphabetic()) .flat_map(|ch| ch.to_lowercase()) .take(MAX_TOKEN_LENGTH) .collect() }; Token::Word { value: word.into() } } else if s.len() > 3 && digit_count == 1 { let word: String = s .chars() .filter(|ch| ch.is_alphabetic()) .flat_map(|ch| ch.to_lowercase()) .take(MAX_TOKEN_LENGTH) .collect(); Token::Word { value: word.into() } } else { // Character class counts let mut upper = 0u32; let mut lower = 0u32; let mut digit = 0u32; let mut len = 0; let mut char_types = Vec::with_capacity(len); for c in s.chars() { let char_type = CharType::from_char(c); char_types.push(char_type); match char_type { CharType::Upper => upper += 1, CharType::Lower => lower += 1, CharType::Digit => digit += 1, CharType::Other => (), } len += 1; } // Determine dominant composition let composition = match (upper > 0, lower > 0, digit > 0) { (true, false, false) => b'U', // UPPERCASE only (false, true, false) => b'L', // lowercase only (false, false, true) => b'D', // digits only (true, true, false) => b'A', // Alphabetic mixed case (true, false, true) => b'H', // Upper + digits (common in codes) (false, true, true) => b'M', // lower + digits (common in identifiers) (true, true, true) => b'X', // eXtreme mix - all three (false, false, false) => b'E', // empty/invalid }; // Length bucket (log-ish scale) let len_code = match len { 1 => b'1', 2 => b'2', 3 => b'3', 4 => b'4', 5..=6 => b'5', 7..=8 => b'6', 9..=12 => b'7', 13..=16 => b'8', 17..=32 => b'9', _ => b'Z', }; // Ratio encoding (which class dominates) let max_count = upper.max(lower).max(digit); let dominance = (max_count * 100) / len.min(1) as u32; let ratio = match dominance { 0..=50 => b'B', // Balanced 51..=75 => b'P', // Partial dominance 76..=99 => b'D', // Dominant _ => b'O', // One class only (100%) }; // Run code let mut run_count = 0; if len > 1 { let mut prev_type = char_types[0]; for ¤t_type in char_types.iter().skip(1) { if current_type != prev_type { run_count += 1; prev_type = current_type; } } } let run_ratio = (run_count as f64) / ((len - 1) as f64); let run_code = match run_ratio { r if r <= 0.1 => b'0', // Very long runs (e.g., AAAABBBB) r if r <= 0.3 => b'1', // Moderate runs r if r <= 0.5 => b'2', // Balanced runs/alternation r if r <= 0.7 => b'3', // High alternation _ => b'4', // Near maximum alternation (e.g., A1A1A1) }; Token::Alphanumeric { code: [composition, len_code, ratio, run_code], } } } fn from_number(is_float: bool, num: &str) -> Self { Token::Number { code: [ if num.starts_with("-") { if is_float { b'F' } else { b'I' } } else if is_float { b'f' } else { b'i' }, num.as_bytes() .iter() .filter(|c| c.is_ascii_digit()) .count() .min(u8::MAX as usize) as u8, ], } } } fn lower_prefix(prefix: &str, value: &str) -> String { let mut result = String::with_capacity(prefix.len() + value.len()); result.push_str(prefix); for ch in value.chars() { for lower_ch in ch.to_lowercase() { result.push(lower_ch); } } result } fn truncate_word(word: &str, max_len: usize) -> &str { if word.len() <= max_len { word } else { let mut pos = 0; for (count, (idx, _)) in word.char_indices().enumerate() { pos = idx; if count == max_len { break; } } &word[..pos] } } impl UnprocessedFeature for Token<'_> { fn prefix(&self) -> u16 { match self { Token::Word { .. } => 0, Token::Number { .. } => 1, Token::Alphanumeric { .. } => 2, Token::UnicodeCategory { .. } => 3, Token::Sender { .. } => 4, Token::Asn { .. } => 5, Token::Url { .. } => 6, Token::Email { .. } => 7, Token::Hostname { .. } => 8, Token::Attachment { .. } => 9, Token::MimeType { .. } => 10, Token::HtmlImage { .. } => 11, Token::HtmlAnchor { .. } => 12, } } fn value(&self) -> &[u8] { match self { Token::Word { value } => value.as_bytes(), Token::Number { code } => code, Token::Alphanumeric { code } => code, Token::UnicodeCategory { value } => value.as_bytes(), Token::Sender { value } => value.as_bytes(), Token::Asn { number } => number, Token::Url { value } => value.as_bytes(), Token::Email { value } => value.as_bytes(), Token::Hostname { value } => value.as_bytes(), Token::Attachment { value } => value.as_bytes(), Token::MimeType { value } => value.as_bytes(), Token::HtmlImage { src } => src.as_bytes(), Token::HtmlAnchor { href } => href.as_bytes(), } } } #[derive(Debug, PartialEq, Eq, Clone, Copy)] enum CharType { Upper, Lower, Digit, Other, } impl CharType { fn from_char(c: char) -> CharType { match c { 'A'..='Z' => CharType::Upper, 'a'..='z' => CharType::Lower, '0'..='9' => CharType::Digit, _ => CharType::Other, } } } impl<'x> Default for Tokens<'x> { fn default() -> Self { Tokens(HashMap::with_capacity(128)) } } ================================================ FILE: crates/spam-filter/src/modules/dnsbl.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ net::Ipv4Addr, sync::Arc, time::{Duration, Instant}, }; use common::{ Server, config::spamfilter::{DnsBlServer, Element, IpResolver, Location}, expr::functions::ResolveVariable, }; use mail_auth::{Error, common::resolver::IntoFqdn}; use trc::SpamEvent; use crate::SpamFilterContext; use super::expression::SpamFilterResolver; pub(crate) async fn check_dnsbl( server: &Server, ctx: &mut SpamFilterContext<'_>, resolver: &impl ResolveVariable, scope: Element, location: Location, ) { let (mut checks, max_checks) = match scope { Element::Email => ( ctx.result.rbl_email_checks, server.core.spam.dnsbl.max_email_checks, ), Element::Ip => ( ctx.result.rbl_ip_checks, server.core.spam.dnsbl.max_ip_checks, ), Element::Url => ( ctx.result.rbl_url_checks, server.core.spam.dnsbl.max_url_checks, ), Element::Domain => ( ctx.result.rbl_domain_checks, server.core.spam.dnsbl.max_domain_checks, ), Element::Header | Element::Body | Element::Any => unreachable!(), }; for dnsbl in &server.core.spam.dnsbl.servers { if dnsbl.scope == scope && checks < max_checks && let Some(tag) = is_dnsbl( server, dnsbl, SpamFilterResolver::new(ctx, resolver, location), scope, &mut checks, ) .await { ctx.result.add_tag(tag); } } match scope { Element::Email => ctx.result.rbl_email_checks = checks, Element::Ip => ctx.result.rbl_ip_checks = checks, Element::Url => ctx.result.rbl_url_checks = checks, Element::Domain => ctx.result.rbl_domain_checks = checks, Element::Header | Element::Body | Element::Any => unreachable!(), } } async fn is_dnsbl( server: &Server, config: &DnsBlServer, resolver: SpamFilterResolver<'_, impl ResolveVariable>, element: Element, checks: &mut usize, ) -> Option { let time = Instant::now(); let zone = server .eval_if::(&config.zone, &resolver, resolver.ctx.input.span_id) .await?; #[cfg(feature = "test_mode")] { if zone.contains(".11.20.") { let parts = zone.split('.').collect::>(); return if config.tags.if_then.iter().any(|i| i.expr.items.len() == 3) && parts[0] != "2" { None } else { server .eval_if( &config.tags, &SpamFilterResolver::new( resolver.ctx, &IpResolver::new( format!("127.0.{}.{}", parts[1], parts[0]).parse().unwrap(), ), resolver.location, ), resolver.ctx.input.span_id, ) .await }; } } let result = match server.inner.cache.dns_rbl.get(zone.as_str()) { Some(Some(result)) => result, Some(None) => return None, None => { *checks += 1; match server .core .smtp .resolvers .dns .ipv4_lookup_raw((&zone).into_fqdn().as_ref()) .await { Ok(result) => { trc::event!( Spam(SpamEvent::Dnsbl), Hostname = zone.clone(), Result = result .entry .iter() .map(|ip| trc::Value::from(ip.to_string())) .collect::>(), Details = element.as_str(), Elapsed = time.elapsed() ); let entry = Arc::new(IpResolver::new( result .entry .iter() .copied() .next() .unwrap_or(Ipv4Addr::BROADCAST) .into(), )); server.inner.cache.dns_rbl.insert_with_expiry( zone.to_string(), Some(entry.clone()), result.expires, ); entry } Err(Error::DnsRecordNotFound(_)) => { trc::event!( Spam(SpamEvent::Dnsbl), Hostname = zone.clone(), Result = trc::Value::None, Details = element.as_str(), Elapsed = time.elapsed() ); server.inner.cache.dns_rbl.insert( zone.to_string(), None, Duration::from_secs(86400), ); return None; } Err(err) => { trc::event!( Spam(SpamEvent::DnsblError), Hostname = zone, Elapsed = time.elapsed(), Details = element.as_str(), CausedBy = err.to_string() ); return None; } } } }; server .eval_if( &config.tags, &SpamFilterResolver::new(resolver.ctx, result.as_ref(), resolver.location), resolver.ctx.input.span_id, ) .await } ================================================ FILE: crates/spam-filter/src/modules/expression.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{ config::spamfilter::*, expr::{StringCow, Variable, functions::ResolveVariable}, }; use compact_str::{CompactString, ToCompactString, format_compact}; use mail_parser::{Header, HeaderValue}; use nlp::tokenizers::types::TokenType; use crate::{Recipient, SpamFilterContext, TextPart, analysis::url::UrlParts}; pub(crate) struct SpamFilterResolver<'x, T: ResolveVariable> { pub ctx: &'x SpamFilterContext<'x>, pub item: &'x T, pub location: Location, } impl<'x, T: ResolveVariable> SpamFilterResolver<'x, T> { pub fn new(ctx: &'x SpamFilterContext<'x>, item: &'x T, location: Location) -> Self { Self { ctx, item, location, } } } impl ResolveVariable for SpamFilterResolver<'_, T> { fn resolve_variable(&self, variable: u32) -> Variable<'_> { match variable { 0..100 => self.item.resolve_variable(variable), V_SPAM_REMOTE_IP => self.ctx.input.remote_ip.to_compact_string().into(), V_SPAM_REMOTE_IP_PTR => self .ctx .output .iprev_ptr .as_deref() .unwrap_or_default() .into(), V_SPAM_EHLO_DOMAIN => self.ctx.output.ehlo_host.fqdn.as_str().into(), V_SPAM_AUTH_AS => self.ctx.input.authenticated_as.unwrap_or_default().into(), V_SPAM_ASN => self.ctx.input.asn.unwrap_or_default().into(), V_SPAM_COUNTRY => self.ctx.input.country.unwrap_or_default().into(), V_SPAM_IS_TLS => self.ctx.input.is_tls.into(), V_SPAM_ENV_FROM => self.ctx.output.env_from_addr.address.as_str().into(), V_SPAM_ENV_FROM_LOCAL => self.ctx.output.env_from_addr.local_part.as_str().into(), V_SPAM_ENV_FROM_DOMAIN => self .ctx .output .env_from_addr .domain_part .fqdn .as_str() .into(), V_SPAM_ENV_TO => self .ctx .output .env_to_addr .iter() .map(|e| Variable::from(e.address.as_str())) .collect::>() .into(), V_SPAM_FROM => self.ctx.output.from.email.address.as_str().into(), V_SPAM_FROM_NAME => self .ctx .output .from .name .as_deref() .unwrap_or_default() .into(), V_SPAM_FROM_LOCAL => self.ctx.output.from.email.local_part.as_str().into(), V_SPAM_FROM_DOMAIN => self.ctx.output.from.email.domain_part.fqdn.as_str().into(), V_SPAM_REPLY_TO => self .ctx .output .reply_to .as_ref() .map(|r| r.email.address.as_str()) .unwrap_or_default() .into(), V_SPAM_REPLY_TO_NAME => self .ctx .output .reply_to .as_ref() .and_then(|r| r.name.as_deref()) .unwrap_or_default() .into(), V_SPAM_REPLY_TO_LOCAL => self .ctx .output .reply_to .as_ref() .map(|r| r.email.local_part.as_str()) .unwrap_or_default() .into(), V_SPAM_REPLY_TO_DOMAIN => self .ctx .output .reply_to .as_ref() .map(|r| r.email.domain_part.fqdn.as_str()) .unwrap_or_default() .into(), V_SPAM_TO => self .ctx .output .recipients_to .iter() .map(|r| Variable::from(r.email.address.as_str())) .collect::>() .into(), V_SPAM_TO_NAME => self .ctx .output .recipients_to .iter() .filter_map(|r| Variable::from(r.name.as_deref()?).into()) .collect::>() .into(), V_SPAM_TO_LOCAL => self .ctx .output .recipients_to .iter() .map(|r| Variable::from(r.email.local_part.as_str())) .collect::>() .into(), V_SPAM_TO_DOMAIN => self .ctx .output .recipients_to .iter() .map(|r| Variable::from(r.email.domain_part.fqdn.as_str())) .collect::>() .into(), V_SPAM_CC => self .ctx .output .recipients_cc .iter() .map(|r| Variable::from(r.email.address.as_str())) .collect::>() .into(), V_SPAM_CC_NAME => self .ctx .output .recipients_cc .iter() .filter_map(|r| Variable::from(r.name.as_deref()?).into()) .collect::>() .into(), V_SPAM_CC_LOCAL => self .ctx .output .recipients_cc .iter() .map(|r| Variable::from(r.email.local_part.as_str())) .collect::>() .into(), V_SPAM_CC_DOMAIN => self .ctx .output .recipients_cc .iter() .map(|r| Variable::from(r.email.domain_part.fqdn.as_str())) .collect::>() .into(), V_SPAM_BCC => self .ctx .output .recipients_bcc .iter() .map(|r| Variable::from(r.email.address.as_str())) .collect::>() .into(), V_SPAM_BCC_NAME => self .ctx .output .recipients_bcc .iter() .filter_map(|r| Variable::from(r.name.as_deref()?).into()) .collect::>() .into(), V_SPAM_BCC_LOCAL => self .ctx .output .recipients_bcc .iter() .map(|r| Variable::from(r.email.local_part.as_str())) .collect::>() .into(), V_SPAM_BCC_DOMAIN => self .ctx .output .recipients_bcc .iter() .map(|r| Variable::from(r.email.domain_part.fqdn.as_str())) .collect::>() .into(), V_SPAM_BODY_TEXT => self.ctx.text_body().unwrap_or_default().into(), V_SPAM_BODY_HTML => self .ctx .input .message .html_body .first() .and_then(|idx| self.ctx.output.text_parts.get(*idx as usize)) .map(|part| { if let TextPart::Html { text_body, .. } = part { text_body.as_str() } else { "" } }) .unwrap_or_default() .into(), V_SPAM_BODY_RAW => Variable::from(CompactString::from_utf8_lossy( self.ctx.input.message.raw_message(), )), V_SPAM_SUBJECT => self.ctx.output.subject_lc.as_str().into(), V_SPAM_SUBJECT_THREAD => self.ctx.output.subject_thread_lc.as_str().into(), V_SPAM_LOCATION => self.location.as_str().into(), V_WORDS_SUBJECT => self .ctx .output .subject_tokens .iter() .filter_map(|w| match w { TokenType::Alphabetic(w) | TokenType::Alphanumeric(w) | TokenType::Integer(w) | TokenType::Float(w) => Some(Variable::from(w.as_ref())), _ => None, }) .collect::>() .into(), V_WORDS_BODY => self .ctx .input .message .html_body .first() .and_then(|idx| self.ctx.output.text_parts.get(*idx as usize)) .map(|part| match part { TextPart::Plain { tokens, .. } | TextPart::Html { tokens, .. } => tokens .iter() .filter_map(|w| match w { TokenType::Alphabetic(w) | TokenType::Alphanumeric(w) | TokenType::Integer(w) | TokenType::Float(w) => Some(Variable::from(w.as_ref())), _ => None, }) .collect::>(), TextPart::None => vec![], }) .unwrap_or_default() .into(), _ => Variable::Integer(0), } } fn resolve_global(&self, variable: &str) -> Variable<'_> { Variable::Integer(self.ctx.result.tags.contains(variable).into()) } } pub(crate) struct EmailHeader<'x> { pub header: &'x Header<'x>, pub raw: &'x str, } impl ResolveVariable for EmailHeader<'_> { fn resolve_variable(&self, variable: u32) -> Variable<'_> { match variable { V_HEADER_NAME => self.header.name().into(), V_HEADER_NAME_LOWER => CompactString::from_str_to_lowercase(self.header.name()).into(), V_HEADER_VALUE | V_HEADER_VALUE_LOWER | V_HEADER_PROPERTY => match &self.header.value { HeaderValue::Text(text) => { if variable == V_HEADER_VALUE_LOWER { CompactString::from_str_to_lowercase(text).into() } else { text.as_ref().into() } } HeaderValue::TextList(list) => Variable::Array( list.iter() .map(|text| { Variable::String(if variable == V_HEADER_VALUE_LOWER { StringCow::Owned(CompactString::from_str_to_lowercase(text)) } else { StringCow::Borrowed(text.as_ref()) }) }) .collect(), ), HeaderValue::Address(address) => Variable::Array(if variable == 1 { address .iter() .filter_map(|a| { a.address.as_ref().map(|text| { Variable::String(if variable == V_HEADER_VALUE_LOWER { StringCow::Owned(CompactString::from_str_to_lowercase(text)) } else { StringCow::Borrowed(text.as_ref()) }) }) }) .collect() } else { address .iter() .filter_map(|a| { a.name.as_ref().map(|text| { Variable::String(if variable == V_HEADER_VALUE_LOWER { StringCow::Owned(CompactString::from_str_to_lowercase(text)) } else { StringCow::Borrowed(text.as_ref()) }) }) }) .collect() }), HeaderValue::DateTime(date_time) => { CompactString::new(date_time.to_rfc3339()).into() } HeaderValue::ContentType(ct) => { if variable != V_HEADER_PROPERTY { if let Some(st) = ct.subtype() { format_compact!("{}/{}", ct.ctype(), st).into() } else { ct.ctype().into() } } else { Variable::Array( ct.attributes() .map(|attr| { attr.iter() .map(|attr| { Variable::from(format_compact!( "{}={}", attr.name, attr.value )) }) .collect::>() }) .unwrap_or_default(), ) } } HeaderValue::Received(_) => { if variable == V_HEADER_VALUE_LOWER { CompactString::from_str_to_lowercase(self.raw.trim()).into() } else { self.raw.trim().into() } } HeaderValue::Empty => "".into(), }, V_HEADER_RAW => self.raw.into(), V_HEADER_RAW_LOWER => CompactString::from_str_to_lowercase(self.raw).into(), _ => Variable::Integer(0), } } fn resolve_global(&self, _: &str) -> Variable<'_> { Variable::Integer(0) } } impl ResolveVariable for Recipient { fn resolve_variable(&self, variable: u32) -> Variable<'_> { match variable { V_RCPT_EMAIL => Variable::from(self.email.address.as_str()), V_RCPT_NAME => Variable::from(self.name.as_deref().unwrap_or_default()), V_RCPT_LOCAL => Variable::from(self.email.local_part.as_str()), V_RCPT_DOMAIN => Variable::from(self.email.domain_part.fqdn.as_str()), V_RCPT_DOMAIN_SLD => Variable::from(self.email.domain_part.sld_or_default()), _ => Variable::Integer(0), } } fn resolve_global(&self, _: &str) -> Variable<'_> { Variable::Integer(0) } } impl ResolveVariable for UrlParts<'_> { fn resolve_variable(&self, variable: u32) -> Variable<'_> { match variable { V_URL_FULL => Variable::from(self.url.as_str()), V_URL_PATH_QUERY => Variable::from( self.url_parsed .as_ref() .and_then(|p| p.parts.path_and_query().map(|p| p.as_str())) .unwrap_or_default(), ), V_URL_PATH => Variable::from( self.url_parsed .as_ref() .map(|p| p.parts.path()) .unwrap_or_default(), ), V_URL_QUERY => Variable::from( self.url_parsed .as_ref() .and_then(|p| p.parts.query()) .unwrap_or_default(), ), V_URL_SCHEME => Variable::from( self.url_parsed .as_ref() .and_then(|p| p.parts.scheme_str()) .unwrap_or_default(), ), V_URL_AUTHORITY => Variable::from( self.url_parsed .as_ref() .and_then(|p| p.parts.authority().map(|a| a.as_str())) .unwrap_or_default(), ), V_URL_HOST => Variable::from( self.url_parsed .as_ref() .map(|p| p.host.fqdn.as_str()) .unwrap_or_default(), ), V_URL_HOST_SLD => Variable::from( self.url_parsed .as_ref() .map(|p| p.host.sld_or_default()) .unwrap_or_default(), ), V_URL_PORT => Variable::Integer( self.url_parsed .as_ref() .and_then(|p| p.parts.port_u16()) .unwrap_or(0) as _, ), _ => Variable::Integer(0), } } fn resolve_global(&self, _: &str) -> Variable<'_> { Variable::Integer(0) } } pub struct StringResolver<'x>(pub &'x str); impl ResolveVariable for StringResolver<'_> { fn resolve_variable(&self, _: u32) -> Variable<'_> { Variable::from(self.0) } fn resolve_global(&self, _: &str) -> Variable<'_> { Variable::Integer(0) } } pub struct StringListResolver<'x>(pub &'x [String]); impl ResolveVariable for StringListResolver<'_> { fn resolve_variable(&self, _: u32) -> Variable<'_> { Variable::Array(self.0.iter().map(|v| Variable::from(v.as_str())).collect()) } fn resolve_global(&self, _: &str) -> Variable<'_> { Variable::Integer(0) } } ================================================ FILE: crates/spam-filter/src/modules/html.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use mail_parser::decoders::html::add_html_token; #[derive(Debug, Eq, PartialEq, Clone, serde::Serialize, serde::Deserialize)] #[serde(tag = "type")] pub enum HtmlToken { StartTag { name: u64, attributes: Vec<(u64, Option)>, is_self_closing: bool, }, EndTag { name: u64, }, Comment { text: String, }, Text { text: String, }, } pub(crate) const A: u64 = b'a' as u64; pub(crate) const IMG: u64 = (b'i' as u64) | ((b'm' as u64) << 8) | ((b'g' as u64) << 16); pub(crate) const HEAD: u64 = (b'h' as u64) | ((b'e' as u64) << 8) | ((b'a' as u64) << 16) | ((b'd' as u64) << 24); pub(crate) const BODY: u64 = (b'b' as u64) | ((b'o' as u64) << 8) | ((b'd' as u64) << 16) | ((b'y' as u64) << 24); pub(crate) const META: u64 = (b'm' as u64) | ((b'e' as u64) << 8) | ((b't' as u64) << 16) | ((b'a' as u64) << 24); pub(crate) const LINK: u64 = (b'l' as u64) | ((b'i' as u64) << 8) | ((b'n' as u64) << 16) | ((b'k' as u64) << 24); pub(crate) const ALT: u64 = (b'a' as u64) | ((b'l' as u64) << 8) | ((b't' as u64) << 16); pub(crate) const TITLE: u64 = (b't' as u64) | ((b'i' as u64) << 8) | ((b't' as u64) << 16) | ((b'l' as u64) << 24) | ((b'e' as u64) << 32); pub(crate) const HREF: u64 = (b'h' as u64) | ((b'r' as u64) << 8) | ((b'e' as u64) << 16) | ((b'f' as u64) << 24); pub(crate) const SRC: u64 = (b's' as u64) | ((b'r' as u64) << 8) | ((b'c' as u64) << 16); pub(crate) const WIDTH: u64 = (b'w' as u64) | ((b'i' as u64) << 8) | ((b'd' as u64) << 16) | ((b't' as u64) << 24) | ((b'h' as u64) << 32); pub(crate) const HEIGHT: u64 = (b'h' as u64) | ((b'e' as u64) << 8) | ((b'i' as u64) << 16) | ((b'g' as u64) << 24) | ((b'h' as u64) << 32) | ((b't' as u64) << 40); pub(crate) const REL: u64 = (b'r' as u64) | ((b'e' as u64) << 8) | ((b'l' as u64) << 16); pub(crate) const CONTENT: u64 = (b'c' as u64) | ((b'o' as u64) << 8) | ((b'n' as u64) << 16) | ((b't' as u64) << 24) | ((b'e' as u64) << 32) | ((b'n' as u64) << 40) | ((b't' as u64) << 48); pub(crate) const HTTP_EQUIV: u64 = (b'h' as u64) | ((b't' as u64) << 8) | ((b't' as u64) << 16) | ((b'p' as u64) << 24) | ((b'-' as u64) << 32) | ((b'e' as u64) << 40) | ((b'q' as u64) << 48) | ((b'u' as u64) << 56); pub fn html_to_tokens(input: &str) -> Vec { let input = input.as_bytes(); let mut iter = input.iter().enumerate().peekable(); let mut tags = vec![]; let mut is_token_start = true; let mut is_after_space = false; let mut is_new_line = true; let mut token_start = 0; let mut token_end = 0; let mut text = String::with_capacity(16); while let Some((mut pos, &ch)) = iter.next() { match ch { b'<' => { if !is_token_start { add_html_token( &mut text, &input[token_start..token_end + 1], is_after_space, ); is_after_space = false; is_token_start = true; } if !text.is_empty() { tags.push(HtmlToken::Text { text: text.as_str().into(), }); text.clear(); } while matches!(iter.peek(), Some(&(_, &ch)) if ch.is_ascii_whitespace()) { pos += 1; iter.next(); } if matches!(input.get(pos + 1..pos + 4), Some(b"!--")) { let mut comment = Vec::new(); let mut last_ch: u8 = 0; for (_, &ch) in iter.by_ref() { match ch { b'>' if comment.len() > 2 && matches!(comment.last(), Some(b'-')) && matches!(comment.get(comment.len() - 2), Some(b'-')) => { break; } b' ' | b'\t' | b'\r' | b'\n' => { if last_ch != b' ' { comment.push(b' '); } else { last_ch = b' '; } continue; } _ => { comment.push(ch); } } last_ch = ch; } tags.push(HtmlToken::Comment { text: String::from_utf8(comment).unwrap_or_default(), }); } else { let mut is_end_tag = false; loop { match iter.peek() { Some(&(_, &b'/')) => { is_end_tag = true; //pos += 1; iter.next(); } Some((_, ch)) if ch.is_ascii_whitespace() => { //pos += 1; iter.next(); } _ => break, } } let mut in_quote = false; let mut is_self_closing = false; let mut key: u64 = 0; let mut shift = 0; let mut tag = 0; let mut attributes: Vec<(u64, Option)> = vec![]; 'outer: while let Some((_, &ch)) = iter.next() { match ch { b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' if shift < 64 => { key |= (ch as u64) << shift; shift += 8; } b'A'..=b'Z' if shift < 64 => { key |= ((ch - b'A' + b'a') as u64) << shift; shift += 8; } b'/' if !in_quote => { is_self_closing = true; } b'>' if !in_quote => { if shift != 0 { if tag == 0 { tag = key; } else { attributes.push((key, None)); } } break; } b'"' => { in_quote = !in_quote; } b'=' if !in_quote => { while matches!(iter.peek(), Some(&(_, &ch)) if ch.is_ascii_whitespace()) { iter.next(); } if shift != 0 { attributes.push((key, None)); key = 0; shift = 0; } let mut value = vec![]; for (_, &ch) in iter.by_ref() { match ch { b'>' if !in_quote => { if !value.is_empty() { let value = String::from_utf8(value).unwrap_or_default(); if let Some((_, v)) = attributes.last_mut() { *v = value.into(); } else { // Broken attribute attributes.push((0, Some(value))); } } break 'outer; } b'"' => { if in_quote { in_quote = false; break; } else { in_quote = true; } } b' ' | b'\t' | b'\r' | b'\n' if !in_quote => { break; } _ => { value.push(ch); } } } if !value.is_empty() { let value = String::from_utf8(value).unwrap_or_default(); if let Some((_, v)) = attributes.last_mut() { *v = value.into(); } else { // Broken attribute attributes.push((0, Some(value))); } } } b' ' | b'\t' | b'\r' | b'\n' => { if shift != 0 { if tag == 0 { tag = key; } else { attributes.push((key, None)); } key = 0; shift = 0; } } _ => {} } } if tag != 0 { if is_end_tag { tags.push(HtmlToken::EndTag { name: tag }); } else { tags.push(HtmlToken::StartTag { name: tag, attributes, is_self_closing, }); } } } continue; } b' ' | b'\t' | b'\r' | b'\n' => { if !is_token_start { add_html_token( &mut text, &input[token_start..token_end + 1], is_after_space && !is_new_line, ); is_new_line = false; } is_after_space = true; is_token_start = true; continue; } b'&' if !is_token_start => { add_html_token( &mut text, &input[token_start..token_end + 1], is_after_space && !is_new_line, ); is_new_line = false; is_token_start = true; is_after_space = false; } b';' if !is_token_start => { add_html_token( &mut text, &input[token_start..pos + 1], is_after_space && !is_new_line, ); is_token_start = true; is_after_space = false; is_new_line = false; continue; } _ => (), } if is_token_start { token_start = pos; is_token_start = false; } token_end = pos; } if !is_token_start { add_html_token( &mut text, &input[token_start..token_end + 1], is_after_space && !is_new_line, ); } if !text.is_empty() { tags.push(HtmlToken::Text { text: text.as_str().into(), }); } tags } #[cfg(test)] mod tests { use super::*; #[test] fn test_html_to_tokens_text() { let input = "Hello, world!"; let tokens = html_to_tokens(input); assert_eq!( tokens, vec![HtmlToken::Text { text: "Hello, world!".into() }] ); } #[test] fn test_html_to_tokens_start_tag() { let input = "
"; let tokens = html_to_tokens(input); assert_eq!( tokens, vec![HtmlToken::StartTag { name: 7760228, attributes: vec![], is_self_closing: false }] ); } #[test] fn test_html_to_tokens_end_tag() { let input = "
"; let tokens = html_to_tokens(input); assert_eq!(tokens, vec![HtmlToken::EndTag { name: 7760228 }]); } #[test] fn test_html_to_tokens_comment() { let input = ""; let tokens = html_to_tokens(input); assert_eq!( tokens, vec![HtmlToken::Comment { text: "!-- This is a comment --".into() }] ); } #[test] fn test_html_to_tokens_mixed() { let input = "
Hello, " world " !
"; let tokens = html_to_tokens(input); assert_eq!( tokens, vec![ HtmlToken::StartTag { name: 7760228, attributes: vec![], is_self_closing: false }, HtmlToken::Text { text: "Hello,".into() }, HtmlToken::StartTag { name: 1851879539, attributes: vec![], is_self_closing: false }, HtmlToken::Text { text: " \" world \"".into() }, HtmlToken::EndTag { name: 1851879539 }, HtmlToken::Text { text: " !".into() }, HtmlToken::EndTag { name: 7760228 } ] ); } #[test] fn test_html_to_tokens_with_attributes() { let input = r#""#; let tokens = html_to_tokens(input); assert_eq!( tokens, vec![ HtmlToken::StartTag { name: 500186508905, attributes: vec![ (1701869940, Some("text".into())), (435761734006, Some("test".into())) ], is_self_closing: false }, HtmlToken::StartTag { name: 111516266162547, attributes: vec![], is_self_closing: true }, HtmlToken::StartTag { name: 6647407, attributes: vec![(1920234593, None)], is_self_closing: true }, HtmlToken::StartTag { name: 97, attributes: vec![(98, Some("1".into())), (98, None), (99, Some("123".into()))], is_self_closing: false } ] ); } } ================================================ FILE: crates/spam-filter/src/modules/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod classifier; pub mod dnsbl; pub mod expression; pub mod html; pub mod pyzor; pub mod sanitize; ================================================ FILE: crates/spam-filter/src/modules/pyzor.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ borrow::Cow, io::Write, net::SocketAddr, time::{Duration, SystemTime}, }; use common::config::spamfilter::PyzorConfig; use mail_parser::{Message, PartType, decoders::html::add_html_token}; use nlp::tokenizers::types::{TokenType, TypesTokenizer}; use sha1::{Digest, Sha1}; use tokio::net::UdpSocket; const MIN_LINE_LENGTH: usize = 8; const ATOMIC_NUM_LINES: usize = 4; const DIGEST_SPEC: &[(usize, usize)] = &[(20, 3), (60, 3)]; #[derive(Default, Debug, PartialEq, Eq)] pub(crate) struct PyzorResponse { pub code: u32, pub count: u64, pub wl_count: u64, } pub(crate) async fn pyzor_check( message: &Message<'_>, config: &PyzorConfig, ) -> trc::Result> { // Make sure there is at least one text part if !message .parts .iter() .any(|p| matches!(p.body, PartType::Text(_) | PartType::Html(_))) { return Ok(None); } // Hash message let request = message.pyzor_check_message(); #[cfg(feature = "test_mode")] { if request.contains("b5b476f0b5ba6e1c038361d3ded5818dd39c90a2") { return Ok(PyzorResponse { code: 200, count: 1000, wl_count: 0, } .into()); } else if request.contains("d67d4b8bfc3860449e3418bb6017e2612f3e2a99") { return Ok(PyzorResponse { code: 200, count: 60, wl_count: 10, } .into()); } else if request.contains("81763547012b75e57a20d18ce0b93014208cdfdb") { return Ok(PyzorResponse { code: 200, count: 50, wl_count: 20, } .into()); } } // Send message to address pyzor_send_message(config.address, config.timeout, &request) .await .map(Into::into) .map_err(|err| { trc::SpamEvent::PyzorError .into_err() .ctx(trc::Key::Url, config.address.to_string()) .reason(err) .details("Pyzor failed") }) } async fn pyzor_send_message( addr: SocketAddr, timeout: Duration, message: &str, ) -> std::io::Result { let socket = UdpSocket::bind("0.0.0.0:0").await?; tokio::time::timeout(timeout, socket.send_to(message.as_bytes(), addr)).await??; let mut buffer = vec![0u8; 1024]; let (size, _) = tokio::time::timeout(timeout, socket.recv_from(&mut buffer)).await??; let raw_response = std::str::from_utf8(&buffer[..size]) .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?; let mut response = PyzorResponse { code: u32::MAX, count: u64::MAX, wl_count: u64::MAX, }; for line in raw_response.lines() { if let Some((k, v)) = line.split_once(':') { if k.eq_ignore_ascii_case("code") { response.code = v.trim().parse().map_err(|_| { std::io::Error::new( std::io::ErrorKind::InvalidData, format!("Invalid line: {raw_response}"), ) })?; } else if k.eq_ignore_ascii_case("count") { response.count = v.trim().parse().map_err(|_| { std::io::Error::new( std::io::ErrorKind::InvalidData, format!("Invalid line: {raw_response}"), ) })?; } else if k.eq_ignore_ascii_case("wl-count") { response.wl_count = v.trim().parse().map_err(|_| { std::io::Error::new( std::io::ErrorKind::InvalidData, format!("Invalid line: {raw_response}"), ) })?; } } } if response.code != u32::MAX && response.count != u64::MAX && response.wl_count != u64::MAX { Ok(response) } else { Err(std::io::Error::new( std::io::ErrorKind::InvalidData, format!("Invalid response: {raw_response}"), )) } } trait PyzorDigest { fn pyzor_digest(&self, writer: W) -> W; } pub trait PyzorCheck { fn pyzor_check_message(&self) -> String; } impl PyzorDigest for Message<'_> { fn pyzor_digest(&self, writer: W) -> W { let parts = self .parts .iter() .filter_map(|part| match &part.body { PartType::Text(text) => Some(text.as_ref().into()), PartType::Html(html) => Some(html_to_text(html.as_ref()).into()), _ => None, }) .collect::>>(); pyzor_digest(writer, parts.iter().flat_map(|text| text.lines())) } } impl PyzorCheck for Message<'_> { fn pyzor_check_message(&self) -> String { let time = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()); pyzor_create_message( self, time, (time & 0xFFFF) as u16 ^ ((time >> 16) & 0xFFFF) as u16, ) } } fn pyzor_create_message(message: &Message<'_>, time: u64, thread: u16) -> String { // Hash message let hash = message.pyzor_digest(Sha1::new()).finalize(); // Hash key let mut hash_key = Sha1::new(); hash_key.update("anonymous:".as_bytes()); let hash_key = hash_key.finalize(); // Hash message let message = format!( "Op: check\nOp-Digest: {hash:x}\nThread: {thread}\nPV: 2.1\nUser: anonymous\nTime: {time}" ); let mut msg_hash = Sha1::new(); msg_hash.update(message.as_bytes()); let msg_hash = msg_hash.finalize(); // Sign let mut sig = Sha1::new(); sig.update(msg_hash); sig.update(format!(":{time}:{hash_key:x}")); let sig = sig.finalize(); format!("{message}\nSig: {sig:x}\n") } fn pyzor_digest<'x, I, W>(mut writer: W, lines: I) -> W where I: Iterator, W: Write, { let mut result = Vec::with_capacity(16); for line in lines { let mut clean_line = String::with_capacity(line.len()); let mut token_start = usize::MAX; let mut token_end = usize::MAX; let add_line = |line: &mut String, span: &str| { if !span.contains(char::from(0)) { if span.len() < 10 { line.push_str(span); } } else { let span = span.replace(char::from(0), ""); if span.len() < 10 { line.push_str(&span); } } }; for token in TypesTokenizer::new(line) { match token.word { TokenType::Alphabetic(_) | TokenType::Alphanumeric(_) | TokenType::Integer(_) | TokenType::Float(_) | TokenType::Other(_) | TokenType::Punctuation(_) => { if token_start == usize::MAX { token_start = token.from; } token_end = token.to; } TokenType::Space | TokenType::Url(_) | TokenType::UrlNoScheme(_) | TokenType::UrlNoHost(_) | TokenType::IpAddr(_) | TokenType::Email(_) => { if token_start != usize::MAX { add_line(&mut clean_line, &line[token_start..token_end]); token_start = usize::MAX; token_end = usize::MAX; } } } } if token_start != usize::MAX { add_line(&mut clean_line, &line[token_start..token_end]); } if clean_line.len() >= MIN_LINE_LENGTH { result.push(clean_line); } } if result.len() > ATOMIC_NUM_LINES { for (offset, length) in DIGEST_SPEC { for i in 0..*length { if let Some(line) = result.get((*offset * result.len() / 100) + i) { let _ = writer.write_all(line.as_bytes()); } } } } else { for line in result { let _ = writer.write_all(line.as_bytes()); } } writer } fn html_to_text(input: &str) -> String { let mut result = String::with_capacity(input.len()); let input = input.as_bytes(); let mut in_tag = false; let mut in_comment = false; let mut in_style = false; let mut in_script = false; let mut is_token_start = true; let mut is_after_space = false; let mut is_tag_close = false; let mut token_start = 0; let mut token_end = 0; let mut tag_token_pos = 0; let mut comment_pos = 0; for (pos, ch) in input.iter().enumerate() { if !in_comment { match ch { b'<' => { if !(in_tag || in_style || in_script || is_token_start) { add_html_token( &mut result, &input[token_start..token_end + 1], is_after_space, ); is_after_space = false; } tag_token_pos = 0; in_tag = true; is_token_start = true; is_tag_close = false; continue; } b'>' if in_tag => { if tag_token_pos == 1 && let Some(tag) = input.get(token_start..token_end + 1) { if tag.eq_ignore_ascii_case(b"style") { in_style = !is_tag_close; } else if tag.eq_ignore_ascii_case(b"script") { in_script = !is_tag_close; } } in_tag = false; is_token_start = true; is_after_space = !result.is_empty(); continue; } b'/' if in_tag => { if tag_token_pos == 0 { is_tag_close = true; } continue; } b'!' if in_tag && tag_token_pos == 0 => { if let Some(b"--") = input.get(pos + 1..pos + 3) { in_comment = true; continue; } } b' ' | b'\t' | b'\r' | b'\n' => { if !(in_tag || in_style || in_script) { if !is_token_start { add_html_token( &mut result, &input[token_start..token_end + 1], is_after_space, ); } is_after_space = true; } is_token_start = true; continue; } b'&' if !(in_tag || is_token_start || in_style || in_script) => { add_html_token( &mut result, &input[token_start..token_end + 1], is_after_space, ); is_token_start = true; is_after_space = false; } b';' if !(in_tag || is_token_start || in_style || in_script) => { add_html_token(&mut result, &input[token_start..pos + 1], is_after_space); is_token_start = true; is_after_space = false; continue; } _ => (), } if is_token_start { token_start = pos; is_token_start = false; if in_tag { tag_token_pos += 1; } } token_end = pos; } else { match ch { b'-' => comment_pos += 1, b'>' if comment_pos == 2 => { comment_pos = 0; in_comment = false; in_tag = false; is_token_start = true; } _ => comment_pos = 0, } } } if !(in_tag || is_token_start || in_style || in_script) { add_html_token( &mut result, &input[token_start..token_end + 1], is_after_space, ); } result.shrink_to_fit(); result } #[cfg(test)] mod test { use std::time::Duration; use mail_parser::MessageParser; use sha1::Digest; use sha1::Sha1; use super::pyzor_create_message; use super::pyzor_send_message; use super::{PyzorDigest, html_to_text, pyzor_digest}; use super::PyzorResponse; #[ignore] #[tokio::test] async fn send_message() { assert_eq!( pyzor_send_message( "public.pyzor.org:24441".parse().unwrap(), Duration::from_secs(10), concat!( "Op: check\n", "Op-Digest: b2c27325a034c581df0c9ef37e4a0d63208a3e7e\n", "Thread: 49005\n", "PV: 2.1\n", "User: anonymous\n", "Time: 1697468672\n", "Sig: 9cf4571b85d3887fdd0d4f444fd0c164e0290722\n" ), ) .await .unwrap(), PyzorResponse { code: 200, count: 0, wl_count: 0 } ); } #[test] fn message_pyzor() { let message = pyzor_create_message( &MessageParser::new().parse(HTML_TEXT_STYLE_SCRIPT).unwrap(), 1697468672, 49005, ); assert_eq!( message, concat!( "Op: check\n", "Op-Digest: b2c27325a034c581df0c9ef37e4a0d63208a3e7e\n", "Thread: 49005\n", "PV: 2.1\n", "User: anonymous\n", "Time: 1697468672\n", "Sig: 9cf4571b85d3887fdd0d4f444fd0c164e0290722\n" ) ); } #[test] fn digest_pyzor() { // HTML stripping assert_eq!(html_to_text(HTML_RAW), HTML_RAW_STRIPED); // Token stripping for strip_me in [ "t@abc.com", "t1@abc.com", "t+a@abc.com", "t.a@abc.com", "0A2D3f%a#S", "3sddkf9jdkd9", "@@#@@@@@@@@@", "http://spammer.com/special-offers?buy=now", ] { assert_eq!( String::from_utf8(pyzor_digest( Vec::new(), format!("Test {strip_me} Test2").lines(), )) .unwrap(), "TestTest2" ); } // Test short lines assert_eq!( String::from_utf8(pyzor_digest( Vec::new(), concat!("This line is included\n", "not this\n", "This also").lines(), )) .unwrap(), "ThislineisincludedThisalso" ); // Test atomic assert_eq!( String::from_utf8(pyzor_digest( Vec::new(), "All this message\nShould be included\nIn the digest".lines(), )) .unwrap(), "AllthismessageShouldbeincludedInthedigest" ); // Test spec let mut text = String::new(); for i in 0..100 { text += format!("Line{i} test test test\n").as_str(); } let mut expected = String::new(); for i in [20, 21, 22, 60, 61, 62] { expected += format!("Line{i}testtesttest").as_str(); } assert_eq!( String::from_utf8(pyzor_digest(Vec::new(), text.lines(),)).unwrap(), expected ); // Test email parsing for (input, expected) in [ ( HTML_TEXT, concat!( "Emailspam,alsoknownasjunkemailorbulkemail,isasubset", "ofspaminvolvingnearlyidenticalmessagessenttonumerous", "byemail.Clickingonlinksinspamemailmaysendusersto", "byemail.Clickingonlinksinspamemailmaysendusersto", "phishingwebsitesorsitesthatarehostingmalware.", "Emailspam.Emailspam,alsoknownasjunkemailorbulkemail,", "isasubsetofspaminvolvingnearlyidenticalmessage", "ssenttonumerousbyemail.Clickingonlinksinspamemailmaysenduse", "rstophishingwebsitesorsitesthatarehostingmalware." ), ), (HTML_TEXT_STYLE_SCRIPT, "Thisisatest.Thisisatest."), (TEXT_ATTACHMENT, "Thisisatestmailing"), (TEXT_ATTACHMENT_W_NULL, "Thisisatestmailing"), (TEXT_ATTACHMENT_W_MULTIPLE_NULLS, "Thisisatestmailing"), (TEXT_ATTACHMENT_W_SUBJECT_NULL, "Thisisatestmailing"), (TEXT_ATTACHMENT_W_CONTENTTYPE_NULL, "Thisisatestmailing"), ] { assert_eq!( String::from_utf8( MessageParser::new() .parse(input) .unwrap() .pyzor_digest(Vec::new(),) ) .unwrap(), expected, "failed for {input}" ) } // Test SHA hash assert_eq!( format!( "{:x}", MessageParser::new() .parse(HTML_TEXT_STYLE_SCRIPT) .unwrap() .pyzor_digest(Sha1::new(),) .finalize() ), "b2c27325a034c581df0c9ef37e4a0d63208a3e7e", ) } const HTML_TEXT: &str = r#"MIME-Version: 1.0 Sender: chirila@gapps.spamexperts.com Received: by 10.216.157.70 with HTTP; Thu, 16 Jan 2014 00:43:31 -0800 (PST) Date: Thu, 16 Jan 2014 10:43:31 +0200 Delivered-To: chirila@gapps.spamexperts.com X-Google-Sender-Auth: ybCmONS9U9D6ZUfjx-9_tY-hF2Q Message-ID: Subject: Test From: Alexandru Chirila To: Alexandru Chirila Content-Type: multipart/alternative; boundary=001a11c25ff293069304f0126bfd --001a11c25ff293069304f0126bfd Content-Type: text/plain; charset=ISO-8859-1 Email spam. Email spam, also known as junk email or unsolicited bulk email, is a subset of electronic spam involving nearly identical messages sent to numerous recipients by email. Clicking on links in spam email may send users to phishing web sites or sites that are hosting malware. --001a11c25ff293069304f0126bfd Content-Type: text/html; charset=ISO-8859-1 Content-Transfer-Encoding: quoted-printable
Email spam.

Email spam, also= known as junk email or unsolicited bulk email, is a subset of electronic s= pam involving nearly identical messages sent to numerous recipients by emai= l. Clicking on links in spam email may send users to phishing web sites or = sites that are hosting malware.
--001a11c25ff293069304f0126bfd-- "#; const HTML_TEXT_STYLE_SCRIPT: &str = r#"MIME-Version: 1.0 Sender: chirila@gapps.spamexperts.com Received: by 10.216.157.70 with HTTP; Thu, 16 Jan 2014 00:43:31 -0800 (PST) Date: Thu, 16 Jan 2014 10:43:31 +0200 Delivered-To: chirila@gapps.spamexperts.com X-Google-Sender-Auth: ybCmONS9U9D6ZUfjx-9_tY-hF2Q Message-ID: Subject: Test From: Alexandru Chirila To: Alexandru Chirila Content-Type: multipart/alternative; boundary=001a11c25ff293069304f0126bfd --001a11c25ff293069304f0126bfd Content-Type: text/plain; charset=ISO-8859-1 This is a test. --001a11c25ff293069304f0126bfd Content-Type: text/html; charset=ISO-8859-1 Content-Transfer-Encoding: quoted-printable
This is a test.
--001a11c25ff293069304f0126bfd-- "#; const TEXT_ATTACHMENT: &str = r#"MIME-Version: 1.0 Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) Date: Fri, 17 Jan 2014 12:21:43 +0200 Delivered-To: chirila.s.alexandru@gmail.com Message-ID: Subject: Test From: Alexandru Chirila To: Alexandru Chirila Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc --f46d040a62c49bb1c804f027e8cc Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca --f46d040a62c49bb1c404f027e8ca Content-Type: text/plain; charset=ISO-8859-1 This is a test mailing --f46d040a62c49bb1c404f027e8ca-- --f46d040a62c49bb1c804f027e8cc Content-Type: image/png; name="tar.png" Content-Disposition: attachment; filename="tar.png" Content-Transfer-Encoding: base64 X-Attachment-Id: f_hqjas5ad0 iVBORw0KGgoAAAANSUhEUgAAAskAAADlCAAAAACErzVVAAAACXBIWXMAAAsTAAALEwEAmpwYAAAD QmCC --f46d040a62c49bb1c804f027e8cc--"#; const TEXT_ATTACHMENT_W_NULL: &str = "MIME-Version: 1.0 Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) Date: Fri, 17 Jan 2014 12:21:43 +0200 Delivered-To: chirila.s.alexandru@gmail.com Message-ID: Subject: Test From: Alexandru Chirila To: Alexandru Chirila Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc --f46d040a62c49bb1c804f027e8cc Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca --f46d040a62c49bb1c404f027e8ca Content-Type: text/plain; charset=ISO-8859-1 This is a test ma\0iling --f46d040a62c49bb1c804f027e8cc--"; const TEXT_ATTACHMENT_W_MULTIPLE_NULLS: &str = "MIME-Version: 1.0 Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) Date: Fri, 17 Jan 2014 12:21:43 +0200 Delivered-To: chirila.s.alexandru@gmail.com Message-ID: Subject: Test From: Alexandru Chirila To: Alexandru Chirila Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc --f46d040a62c49bb1c804f027e8cc Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca --f46d040a62c49bb1c404f027e8ca Content-Type: text/plain; charset=ISO-8859-1 This is a test ma\0\0\0iling --f46d040a62c49bb1c804f027e8cc--"; const TEXT_ATTACHMENT_W_SUBJECT_NULL: &str = "MIME-Version: 1.0 Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) Date: Fri, 17 Jan 2014 12:21:43 +0200 Delivered-To: chirila.s.alexandru@gmail.com Message-ID: Subject: Te\0\0\0st From: Alexandru Chirila To: Alexandru Chirila Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc --f46d040a62c49bb1c804f027e8cc Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca --f46d040a62c49bb1c404f027e8ca Content-Type: text/plain; charset=ISO-8859-1 This is a test mailing --f46d040a62c49bb1c804f027e8cc--"; const TEXT_ATTACHMENT_W_CONTENTTYPE_NULL: &str = "MIME-Version: 1.0 Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) Date: Fri, 17 Jan 2014 12:21:43 +0200 Delivered-To: chirila.s.alexandru@gmail.com Message-ID: Subject: Test From: Alexandru Chirila To: Alexandru Chirila Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc --f46d040a62c49bb1c804f027e8cc Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca --f46d040a62c49bb1c404f027e8ca Content-Type: text/plain; charset=\"iso-8859-1\0\0\0\" This is a test mailing --f46d040a62c49bb1c804f027e8cc--"; const HTML_RAW: &str = r#"Email spam

Email spam, also known as junk email or unsolicited bulk email (UBE), is a subset of electronic spam involving nearly identical messages sent to numerous recipients by email. Clicking on links in spam email may send users to phishing web sites or sites that are hosting malware."#; const HTML_RAW_STRIPED: &str = concat!( "Email spam Email spam , also known as junk email or unsolicited bulk email ( UBE ),", " is a subset of electronic spam involving nearly identical messages sent to numerous recipients by email", " . Clicking on links in spam email may send users to phishing web sites or sites that are hosting malware ." ); } ================================================ FILE: crates/spam-filter/src/modules/sanitize.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::net::IpAddr; use crate::{Email, Hostname}; impl Hostname { pub fn new(host: &str) -> Self { let mut fqdn = host.trim_end_matches('.').to_lowercase(); // Decode punycode if fqdn.contains("xn--") { let mut decoded = String::with_capacity(fqdn.len()); for part in fqdn.split('.') { if !decoded.is_empty() { decoded.push('.'); } if let Some(puny) = part .strip_prefix("xn--") .and_then(idna::punycode::decode_to_string) { decoded.push_str(&puny); } else { decoded.push_str(part); } } fqdn = decoded; } let ip = fqdn .strip_prefix('[') .and_then(|ip| ip.strip_suffix(']')) .unwrap_or(&fqdn) .parse::() .ok(); Hostname { sld: if ip.is_none() { psl::domain(fqdn.as_bytes()).and_then(|domain| { if domain.suffix().typ().is_some() { std::str::from_utf8(domain.as_bytes()).ok().map(Into::into) } else { None } }) } else { None }, ip, fqdn, } } } impl Email { pub fn new(address: &str) -> Self { let address = address.to_lowercase(); let (local_part, domain) = address.rsplit_once('@').unwrap_or_default(); Email { local_part: local_part.into(), domain_part: Hostname::new(domain), address, } } } impl Hostname { pub fn sld_or_default(&self) -> &str { self.sld.as_deref().unwrap_or(self.fqdn.as_str()) } } ================================================ FILE: crates/store/Cargo.toml ================================================ [package] name = "store" version = "0.15.5" edition = "2024" [dependencies] utils = { path = "../utils" } types = { path = "../types" } nlp = { path = "../nlp" } trc = { path = "../trc" } rocksdb = { version = "0.24", optional = true, features = ["multi-threaded-cf"] } foundationdb = { version = "0.9.2", features = ["embedded-fdb-include", "fdb-7_3"], optional = true } rusqlite = { version = "0.37", features = ["bundled"], optional = true } #rust-s3 = { version = "0.37", default-features = false, features = ["tokio-rustls-tls"], optional = true } rust-s3 = { version = "0.35", default-features = false, features = ["tokio-rustls-tls", "no-verify-ssl"], optional = true } async-nats = { version = "0.44", default-features = false, features = ["server_2_10", "server_2_11", "ring"], optional = true } azure_core = { version = "0.21.0", optional = true } azure_storage = { version = "0.21.0", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"], optional = true } azure_storage_blobs = { version = "0.21.0", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"], optional = true } reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-webpki-roots", "http2", "stream"]} tokio = { version = "1.47", features = ["sync", "fs", "io-util"] } r2d2 = { version = "0.8.10", optional = true } futures = { version = "0.3", optional = true } rand = "0.9.0" roaring = "0.11" rayon = { version = "1.11", optional = true } serde = { version = "1.0", features = ["derive"]} ahash = { version = "0.8.2", features = ["serde"] } xxhash-rust = { version = "0.8.5", features = ["xxh3"] } farmhash = "1.1.5" parking_lot = "0.12" lru-cache = { version = "0.1.2", optional = true } num_cpus = { version = "1.17", optional = true } blake3 = "1.8" lz4_flex = { version = "0.12", default-features = false } deadpool-postgres = { version = "0.14", optional = true } tokio-postgres = { version = "0.7.10", features = ["with-serde_json-1"], optional = true } tokio-rustls = { version = "0.26", optional = true, default-features = false, features = ["ring", "tls12"] } rustls = { version = "0.23.5", optional = true, default-features = false, features = ["std", "ring", "tls12"] } rustls-pki-types = { version = "1", optional = true } ring = { version = "0.17", optional = true } bytes = { version = "1.10", optional = true } mysql_async = { version = "0.36", default-features = false, features = ["default-rustls-ring", "minimal"], optional = true } serde_json = { version = "1.0.64" } regex = "1.12" flate2 = "1.1" redis = { version = "0.32", features = [ "tokio-comp", "tokio-rustls-comp", "tls-rustls-insecure", "tls-rustls-webpki-roots", "cluster-async"], optional = true } deadpool = { version = "0.12", features = ["managed"], optional = true } arc-swap = "1.6.0" bitpacking = "0.9.2" memchr = { version = "2.7" } rkyv = { version = "0.8.10", features = ["little_endian"] } compact_str = "0.9.0" zenoh = { version = "1.3.4", default-features = false, features = ["auth_pubkey", "transport_multilink", "transport_compression", "transport_quic", "transport_tcp", "transport_tls", "transport_udp"], optional = true } rdkafka = { version = "0.38", features = ["cmake-build"], optional = true } rustls_021 = { package = "rustls", version = "0.21", default-features = false, features = ["dangerous_configuration"], optional = true } [dev-dependencies] tokio = { version = "1.47", features = ["full"] } [features] # Data Stores rocks = ["rocksdb", "rayon", "num_cpus"] sqlite = ["rusqlite", "rayon", "r2d2", "num_cpus", "lru-cache"] postgres = ["tokio-postgres", "deadpool", "deadpool-postgres", "tokio-rustls", "rustls", "ring", "rustls-pki-types", "futures", "bytes"] mysql = ["mysql_async", "futures"] foundation = ["foundationdb", "futures"] fdb-chunked-bm = [] # Blob stores s3 = ["rust-s3", "rustls_021"] azure = ["azure_core", "azure_storage", "azure_storage_blobs"] # In-memory stores redis = ["dep:redis", "deadpool", "futures"] # Pubsub nats = ["async-nats"] zenoh = ["dep:zenoh"] kafka = ["rdkafka"] enterprise = [] test_mode = [] ================================================ FILE: crates/store/src/backend/azure/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{fmt::Display, io::Write, ops::Range, time::Duration}; use azure_core::error::ErrorKind; use azure_core::{ExponentialRetryOptions, RetryOptions, StatusCode, TransportOptions}; use azure_storage::StorageCredentials; use azure_storage_blobs::prelude::{ClientBuilder, ContainerClient}; use futures::stream::StreamExt; use std::sync::Arc; use utils::{ codec::base32_custom::Base32Writer, config::{Config, utils::AsKey}, }; pub struct AzureStore { client: ContainerClient, prefix: Option, } impl AzureStore { pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { let prefix = prefix.as_key(); let storage_account = config .value_require((&prefix, "storage-account"))? .to_string(); let container = config.value_require((&prefix, "container"))?.to_string(); let credentials = match ( config.value((&prefix, "azure-access-key")), config.value((&prefix, "sas-token")), ) { (Some(access_key), None) => { StorageCredentials::access_key(storage_account.clone(), access_key.to_string()) } (None, Some(sas_token)) => match StorageCredentials::sas_token(sas_token) { Ok(cred) => cred, Err(err) => { config.new_build_error( prefix.as_str(), format!("Failed to create credentials: {err:?}"), ); return None; } }, _ => { config.new_build_error( prefix.as_str(), concat!( "Failed to create credentials: exactly one of ", "'azure-access-key' and 'sas-token' must be specified" ), ); return None; } }; let timeout = config .property_or_default::((&prefix, "timeout"), "30s") .unwrap_or_else(|| Duration::from_secs(30)); let transport = match reqwest::Client::builder().timeout(timeout).build() { Ok(client) => Arc::new(client), Err(err) => { config.new_build_error( prefix.as_str(), format!("Failed to create HTTP client: {err:?}"), ); return None; } }; // Take the configured number of retries and multiply by 2. This is intended to match the // precedent set by the S3 back end, where we do the indicated number of retries, // ourselves, but internally the rust-s3 crate is also retrying each of our requests up to // one additional time, itself. So our retries, and the S3 backend's retries, are // comparable to each other. let max_retries: u32 = config .property_or_default((&prefix, "max-retries"), "3") .unwrap_or(3) * 2; Some(AzureStore { client: ClientBuilder::new(storage_account, credentials) .transport(TransportOptions::new(transport)) .retry(RetryOptions::exponential( ExponentialRetryOptions::default().max_retries(max_retries), )) .container_client(container), prefix: config.value((&prefix, "key-prefix")).map(|s| s.to_string()), }) } pub(crate) async fn get_blob( &self, key: &[u8], range: Range, ) -> trc::Result>> { let blob_client = self.client.blob_client(self.build_key(key)); let mut stream = blob_client.get(); let mut buf = if range.end == usize::MAX { // Let's turn this into a proper RangeFrom. stream = stream.range(range.start..); // We don't know how big to expect the result to be. Vec::new() } else { stream = stream.range(range.clone()); Vec::with_capacity(range.end - range.start) }; let mut stream = stream.into_stream(); while let Some(response) = stream.next().await { let err = match response { Ok(chunks) => { let mut chunks = chunks.data; let mut err = None; while let Some(chunk) = chunks.next().await { match chunk { Ok(ref data) => { buf.extend(data); } Err(e) => { err = Some(e); break; } } } err } Err(e) => Some(e), }; if let Some(e) = err { return if matches!( e.kind(), ErrorKind::HttpResponse { status: StatusCode::NotFound, .. } ) { Ok(None) } else { Err(trc::StoreEvent::AzureError.reason(e)) }; } } Ok(Some(buf)) } pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { let blob_client = self.client.blob_client(self.build_key(key)); // We unfortunately have to make a copy of `data`. This is because the Azure SDK wants to // coerce the body into a value of type azure_core::Body, which doesn't have a lifetime // parameter and so cannot hold any non-static references (directly or indirectly). let data = data.to_vec(); blob_client .put_block_blob(data) .into_future() .await .map_err(into_error)?; Ok(()) } pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result { let blob_client = self.client.blob_client(self.build_key(key)); if let Err(e) = blob_client.delete().into_future().await { if matches!( e.kind(), ErrorKind::HttpResponse { status: StatusCode::NotFound, .. } ) { Ok(false) } else { Err(trc::StoreEvent::AzureError.reason(e)) } } else { Ok(true) } } fn build_key(&self, key: &[u8]) -> String { if let Some(prefix) = &self.prefix { let mut writer = Base32Writer::with_raw_capacity(prefix.len() + (key.len().div_ceil(4) * 5)); writer.push_string(prefix); writer.write_all(key).unwrap(); writer.finalize() } else { Base32Writer::from_bytes(key).finalize() } } } #[inline(always)] fn into_error(err: impl Display) -> trc::Error { trc::StoreEvent::AzureError.reason(err) } ================================================ FILE: crates/store/src/backend/composite/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: LicenseRef-SEL * * This file is subject to the Stalwart Enterprise License Agreement (SEL) and * is NOT open source software. * */ #[cfg(any(feature = "postgres", feature = "mysql"))] pub mod read_replica; pub mod sharded_blob; pub mod sharded_lookup; ================================================ FILE: crates/store/src/backend/composite/read_replica.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: LicenseRef-SEL * * This file is subject to the Stalwart Enterprise License Agreement (SEL) and * is NOT open source software. * */ use crate::{ Deserialize, IterateParams, Key, Store, Stores, ValueKey, search::{IndexDocument, SearchComparator, SearchDocumentId, SearchFilter, SearchQuery}, write::{AssignedIds, Batch, SearchIndex, ValueClass}, }; use std::{ future::Future, ops::Range, sync::atomic::{AtomicUsize, Ordering}, }; use utils::config::{Config, utils::AsKey}; pub struct SQLReadReplica { primary: Store, replicas: Vec, last_used_replica: AtomicUsize, } impl SQLReadReplica { pub async fn open( config: &mut Config, prefix: impl AsKey, stores: &Stores, create_store_tables: bool, create_search_tables: bool, ) -> Option { let prefix = prefix.as_key(); let primary_id = config.value_require((&prefix, "primary"))?.to_string(); let replica_ids = config .values((&prefix, "replicas")) .map(|(_, v)| v.to_string()) .collect::>(); let primary = if let Some(store) = stores.stores.get(&primary_id) { if store.is_pg_or_mysql() { store.clone() } else { config.new_build_error( (&prefix, "primary"), "Primary store must be a PostgreSQL or MySQL store", ); return None; } } else { config.new_build_error( (&prefix, "primary"), format!("Primary store {primary_id} not found"), ); return None; }; let mut replicas = Vec::with_capacity(replica_ids.len()); for replica_id in replica_ids { if let Some(store) = stores.stores.get(&replica_id) { if store.is_pg_or_mysql() { replicas.push(store.clone()); } else { config.new_build_error( (&prefix, "replicas"), "Replica store must be a PostgreSQL or MySQL store", ); return None; } } else { config.new_build_error( (&prefix, "replicas"), format!("Replica store {replica_id} not found"), ); return None; } } if !replicas.is_empty() { if create_store_tables { let result = match &primary { #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.create_storage_tables().await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.create_storage_tables().await, _ => panic!("Invalid store type"), }; if let Err(err) = result { config.new_build_error( (&prefix, "primary"), format!("Failed to create tables: {err}"), ); } } if create_search_tables { let result = match &primary { #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.create_search_tables().await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.create_search_tables().await, _ => panic!("Invalid store type"), }; if let Err(err) = result { config.new_build_warning( (&prefix, "primary"), format!("Failed to create search tables: {err}"), ); } } Some(Self { primary, replicas, last_used_replica: AtomicUsize::new(0), }) } else { config.new_build_error((&prefix, "replicas"), "No replica stores specified"); None } } async fn run_op<'x, F, T, R>(&'x self, f: F) -> trc::Result where F: Fn(&'x Store) -> R, R: Future>, T: 'static, { let mut last_error = None; for store in [ &self.replicas [self.last_used_replica.fetch_add(1, Ordering::Relaxed) % self.replicas.len()], &self.primary, ] { match f(store).await { Ok(result) => return Ok(result), Err(err) => { if err.is_assertion_failure() { return Err(err); } else { last_error = Some(err); } } } } Err(last_error.unwrap()) } pub async fn get_blob(&self, key: &[u8], range: Range) -> trc::Result>> { self.run_op(move |store| { let range = range.clone(); async move { match store { #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.get_blob(key, range).await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.get_blob(key, range).await, _ => panic!("Invalid store type"), } } }) .await } pub async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { match &self.primary { #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.put_blob(key, data).await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.put_blob(key, data).await, _ => panic!("Invalid store type"), } } pub async fn delete_blob(&self, key: &[u8]) -> trc::Result { match &self.primary { #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.delete_blob(key).await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.delete_blob(key).await, _ => panic!("Invalid store type"), } } pub async fn get_value(&self, key: impl Key) -> trc::Result> where U: Deserialize + 'static, { self.run_op(move |store| { let key = key.clone(); async move { match store { #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.get_value(key).await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.get_value(key).await, _ => panic!("Invalid store type"), } } }) .await } pub async fn iterate( &self, params: IterateParams, mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, ) -> trc::Result<()> { let mut last_error = None; for store in [ &self.replicas [self.last_used_replica.fetch_add(1, Ordering::Relaxed) % self.replicas.len()], &self.primary, ] { match match store { #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.iterate(params.clone(), &mut cb).await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.iterate(params.clone(), &mut cb).await, _ => panic!("Invalid store type"), } { Ok(result) => return Ok(result), Err(err) => { last_error = Some(err); } } } Err(last_error.unwrap()) } pub async fn get_counter( &self, key: impl Into> + Sync + Send, ) -> trc::Result { let key = key.into(); self.run_op(move |store| { let key = key.clone(); async move { match store { #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.get_counter(key).await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.get_counter(key).await, _ => panic!("Invalid store type"), } } }) .await } pub async fn write(&self, batch: Batch<'_>) -> trc::Result { match &self.primary { #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.write(batch).await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.write(batch).await, _ => panic!("Invalid store type"), } } pub async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> { match &self.primary { #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.delete_range(from, to).await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.delete_range(from, to).await, _ => panic!("Invalid store type"), } } pub async fn purge_store(&self) -> trc::Result<()> { match &self.primary { #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.purge_store().await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.purge_store().await, _ => panic!("Invalid store type"), } } pub async fn index(&self, documents: Vec) -> trc::Result<()> { match &self.primary { #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.index(documents).await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.index(documents).await, _ => panic!("Invalid store type"), } } pub async fn unindex(&self, query: SearchQuery) -> trc::Result { match &self.primary { #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.unindex(query).await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.unindex(query).await, _ => panic!("Invalid store type"), } } pub async fn query( &self, index: SearchIndex, filters: &[SearchFilter], sort: &[SearchComparator], ) -> trc::Result> { match &self.primary { #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.query(index, filters, sort).await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.query(index, filters, sort).await, _ => panic!("Invalid store type"), } } } ================================================ FILE: crates/store/src/backend/composite/sharded_blob.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: LicenseRef-SEL * * This file is subject to the Stalwart Enterprise License Agreement (SEL) and * is NOT open source software. * */ use std::ops::Range; use utils::config::{Config, utils::AsKey}; use crate::{BlobBackend, Store, Stores}; pub struct ShardedBlob { pub stores: Vec, } impl ShardedBlob { pub fn open(config: &mut Config, prefix: impl AsKey, stores: &Stores) -> Option { let prefix = prefix.as_key(); let store_ids = config .values((&prefix, "stores")) .map(|(_, v)| v.to_string()) .collect::>(); let mut blob_stores = Vec::with_capacity(store_ids.len()); for store_id in store_ids { if let Some(store) = stores.blob_stores.get(&store_id) { blob_stores.push(store.backend.clone()); } else { config.new_build_error( (&prefix, "stores"), format!("Blob store {store_id} not found"), ); return None; } } if !blob_stores.is_empty() { Some(Self { stores: blob_stores, }) } else { config.new_build_error((&prefix, "stores"), "No blob stores specified"); None } } #[inline(always)] fn get_store(&self, key: &[u8]) -> &BlobBackend { &self.stores[xxhash_rust::xxh3::xxh3_64(key) as usize % self.stores.len()] } pub async fn get_blob( &self, key: &[u8], read_range: Range, ) -> trc::Result>> { Box::pin(async move { match self.get_store(key) { BlobBackend::Store(store) => match store { #[cfg(feature = "sqlite")] Store::SQLite(store) => store.get_blob(key, read_range).await, #[cfg(feature = "foundation")] Store::FoundationDb(store) => store.get_blob(key, read_range).await, #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.get_blob(key, read_range).await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.get_blob(key, read_range).await, #[cfg(feature = "rocks")] Store::RocksDb(store) => store.get_blob(key, read_range).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all( feature = "enterprise", any(feature = "postgres", feature = "mysql") ))] Store::SQLReadReplica(store) => store.get_blob(key, read_range).await, // SPDX-SnippetEnd Store::None => Err(trc::StoreEvent::NotConfigured.into()), }, BlobBackend::Fs(store) => store.get_blob(key, read_range).await, #[cfg(feature = "s3")] BlobBackend::S3(store) => store.get_blob(key, read_range).await, #[cfg(feature = "azure")] BlobBackend::Azure(store) => store.get_blob(key, read_range).await, BlobBackend::Sharded(_) => unimplemented!(), } }) .await } pub async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { Box::pin(async move { match self.get_store(key) { BlobBackend::Store(store) => match store { #[cfg(feature = "sqlite")] Store::SQLite(store) => store.put_blob(key, data).await, #[cfg(feature = "foundation")] Store::FoundationDb(store) => store.put_blob(key, data).await, #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.put_blob(key, data).await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.put_blob(key, data).await, #[cfg(feature = "rocks")] Store::RocksDb(store) => store.put_blob(key, data).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all( feature = "enterprise", any(feature = "postgres", feature = "mysql") ))] // SPDX-SnippetEnd Store::SQLReadReplica(store) => store.put_blob(key, data).await, Store::None => Err(trc::StoreEvent::NotConfigured.into()), }, BlobBackend::Fs(store) => store.put_blob(key, data).await, #[cfg(feature = "s3")] BlobBackend::S3(store) => store.put_blob(key, data).await, #[cfg(feature = "azure")] BlobBackend::Azure(store) => store.put_blob(key, data).await, BlobBackend::Sharded(_) => unimplemented!(), } }) .await } pub async fn delete_blob(&self, key: &[u8]) -> trc::Result { Box::pin(async move { match self.get_store(key) { BlobBackend::Store(store) => match store { #[cfg(feature = "sqlite")] Store::SQLite(store) => store.delete_blob(key).await, #[cfg(feature = "foundation")] Store::FoundationDb(store) => store.delete_blob(key).await, #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.delete_blob(key).await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.delete_blob(key).await, #[cfg(feature = "rocks")] Store::RocksDb(store) => store.delete_blob(key).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all( feature = "enterprise", any(feature = "postgres", feature = "mysql") ))] Store::SQLReadReplica(store) => store.delete_blob(key).await, // SPDX-SnippetEnd Store::None => Err(trc::StoreEvent::NotConfigured.into()), }, BlobBackend::Fs(store) => store.delete_blob(key).await, #[cfg(feature = "s3")] BlobBackend::S3(store) => store.delete_blob(key).await, #[cfg(feature = "azure")] BlobBackend::Azure(store) => store.delete_blob(key).await, BlobBackend::Sharded(_) => unimplemented!(), } }) .await } } ================================================ FILE: crates/store/src/backend/composite/sharded_lookup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: LicenseRef-SEL * * This file is subject to the Stalwart Enterprise License Agreement (SEL) and * is NOT open source software. * */ use utils::config::{Config, utils::AsKey}; use crate::{ Deserialize, InMemoryStore, Stores, Value, dispatch::lookup::{KeyValue, LookupKey}, }; #[derive(Debug)] pub struct ShardedInMemory { pub stores: Vec, } impl ShardedInMemory { pub fn open(config: &mut Config, prefix: impl AsKey, stores: &Stores) -> Option { let prefix = prefix.as_key(); let store_ids = config .values((&prefix, "stores")) .map(|(_, v)| v.to_string()) .collect::>(); let mut in_memory_stores = Vec::with_capacity(store_ids.len()); for store_id in store_ids { if let Some(store) = stores .in_memory_stores .get(&store_id) .filter(|store| store.is_redis()) { in_memory_stores.push(store.clone()); } else { config.new_build_error( (&prefix, "stores"), format!("In-memory store {store_id} not found"), ); return None; } } if !in_memory_stores.is_empty() { Some(Self { stores: in_memory_stores, }) } else { config.new_build_error((&prefix, "stores"), "No in-memory stores specified"); None } } #[inline(always)] fn get_store(&self, key: &[u8]) -> &InMemoryStore { &self.stores[xxhash_rust::xxh3::xxh3_64(key) as usize % self.stores.len()] } pub async fn key_set(&self, kv: KeyValue>) -> trc::Result<()> { Box::pin(async move { match self.get_store(&kv.key) { #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_set(&kv.key, &kv.value, kv.expires).await, InMemoryStore::Static(_) => Err(trc::StoreEvent::NotSupported.into_err()), _ => Err(trc::StoreEvent::NotSupported.into_err()), } }) .await } pub async fn counter_incr(&self, kv: KeyValue) -> trc::Result { Box::pin(async move { match self.get_store(&kv.key) { #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_incr(&kv.key, kv.value, kv.expires).await, InMemoryStore::Static(_) => Err(trc::StoreEvent::NotSupported.into_err()), _ => Err(trc::StoreEvent::NotSupported.into_err()), } }) .await } pub async fn key_delete(&self, key: impl Into>) -> trc::Result<()> { let key_ = key.into(); let key = key_.as_bytes(); Box::pin(async move { match self.get_store(key) { #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_delete(key).await, InMemoryStore::Static(_) => Err(trc::StoreEvent::NotSupported.into_err()), _ => Err(trc::StoreEvent::NotSupported.into_err()), } }) .await } pub async fn counter_delete(&self, key: impl Into>) -> trc::Result<()> { let key_ = key.into(); let key = key_.as_bytes(); Box::pin(async move { match self.get_store(key) { #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_delete(key).await, InMemoryStore::Static(_) => Err(trc::StoreEvent::NotSupported.into_err()), _ => Err(trc::StoreEvent::NotSupported.into_err()), } }) .await } #[allow(unused_variables)] pub async fn key_delete_prefix(&self, prefix: &[u8]) -> trc::Result<()> { Box::pin(async move { #[cfg(feature = "redis")] for store in &self.stores { match store { InMemoryStore::Redis(store) => store.key_delete_prefix(prefix).await?, InMemoryStore::Static(_) => { return Err(trc::StoreEvent::NotSupported.into_err()); } _ => return Err(trc::StoreEvent::NotSupported.into_err()), } } Ok(()) }) .await } pub async fn key_get> + std::fmt::Debug + 'static>( &self, key: impl Into>, ) -> trc::Result> { let key_ = key.into(); let key = key_.as_bytes(); Box::pin(async move { match self.get_store(key) { #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_get(key).await, InMemoryStore::Static(_) => Err(trc::StoreEvent::NotSupported.into_err()), _ => Err(trc::StoreEvent::NotSupported.into_err()), } }) .await } pub async fn counter_get(&self, key: impl Into>) -> trc::Result { let key_ = key.into(); let key = key_.as_bytes(); Box::pin(async move { match self.get_store(key) { #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.counter_get(key).await, InMemoryStore::Static(_) => Err(trc::StoreEvent::NotSupported.into_err()), _ => Err(trc::StoreEvent::NotSupported.into_err()), } }) .await } pub async fn key_exists(&self, key: impl Into>) -> trc::Result { let key_ = key.into(); let key = key_.as_bytes(); Box::pin(async move { match self.get_store(key) { #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_exists(key).await, InMemoryStore::Static(_) => Err(trc::StoreEvent::NotSupported.into_err()), _ => Err(trc::StoreEvent::NotSupported.into_err()), } }) .await } } ================================================ FILE: crates/store/src/backend/elastic/main.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ backend::elastic::ElasticSearchStore, search::{ CalendarSearchField, ContactSearchField, EmailSearchField, SearchableField, TracingSearchField, }, }; use reqwest::{Error, Response, Url}; use serde_json::{Value, json}; use utils::config::{Config, http::build_http_client, utils::AsKey}; impl ElasticSearchStore { pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { let client = build_http_client(config, prefix.clone(), "application/json".into())?; let prefix = prefix.as_key(); let url = config .value_require((&prefix, "url"))? .trim_end_matches("/") .to_string(); Url::parse(&url) .map_err(|e| config.new_parse_error((&prefix, "url"), format!("Invalid URL: {e}",))) .ok()?; let es = Self { client, url }; let shards = config .property_or_default((&prefix, "index.shards"), "3") .unwrap_or(3); let replicas = config .property_or_default((&prefix, "index.replicas"), "0") .unwrap_or(0); let with_source = config .property_or_default((&prefix, "index.include-source"), "false") .unwrap_or(false); if let Err(err) = es.create_indexes(shards, replicas, with_source).await { config.new_build_error(prefix.as_str(), err.to_string()); } Some(es) } pub async fn create_indexes( &self, shards: usize, replicas: usize, with_source: bool, ) -> trc::Result<()> { self.create_index::(shards, replicas, with_source) .await?; self.create_index::(shards, replicas, with_source) .await?; self.create_index::(shards, replicas, with_source) .await?; self.create_index::(shards, replicas, with_source) .await?; Ok(()) } async fn create_index( &self, shards: usize, replicas: usize, with_source: bool, ) -> trc::Result<()> { let mut mappings = serde_json::Map::new(); mappings.insert( "properties".to_string(), Value::Object( T::primary_keys() .iter() .chain(T::all_fields()) .map(|field| (field.field_name().to_string(), field.es_schema())) .collect::>(), ), ); if !with_source { mappings.insert("_source".to_string(), json!({ "enabled": false })); } let body = json!({ "mappings": mappings, "settings": { "index.number_of_shards": shards, "index.number_of_replicas": replicas, "analysis": { "analyzer": { "default": { "type": "custom", "tokenizer": "standard", "filter": ["lowercase", "stemmer"] } } } } }); let response = self .client .put(format!("{}/{}", self.url, T::index().index_name())) .body(body.to_string()) .send() .await .map_err(|err| { trc::StoreEvent::ElasticsearchError .reason(err) .details("Failed to create index") })?; match response.status().as_u16() { 200..300 => Ok(()), status @ (400..500) => { let text = response.text().await.unwrap_or_default(); if text.contains("resource_already_exists_exception") { // Index already exists, ignore Ok(()) } else { Err(trc::StoreEvent::ElasticsearchError .reason(text) .ctx(trc::Key::Code, status)) } } status => { let text = response.text().await.unwrap_or_default(); Err(trc::StoreEvent::ElasticsearchError .reason(text) .ctx(trc::Key::Code, status)) } } } #[cfg(feature = "test_mode")] pub async fn drop_indexes(&self) -> trc::Result<()> { use crate::write::SearchIndex; for index in &[ SearchIndex::Email, SearchIndex::Calendar, SearchIndex::Contacts, SearchIndex::Tracing, ] { assert_success( self.client .delete(format!("{}/{}", self.url, index.index_name())) .send() .await, ) .await .map(|_| ())?; } Ok(()) } } pub(crate) async fn assert_success(response: Result) -> trc::Result { match response { Ok(response) => { let status = response.status(); if status.is_success() { Ok(response) } else { Err(trc::StoreEvent::ElasticsearchError .reason(response.text().await.unwrap_or_default()) .ctx(trc::Key::Code, status.as_u16())) } } Err(err) => Err(trc::StoreEvent::ElasticsearchError.reason(err)), } } ================================================ FILE: crates/store/src/backend/elastic/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::search::*; use reqwest::Client; use serde::{Deserialize, Deserializer}; use serde_json::{Value, json}; pub mod main; pub mod search; pub struct ElasticSearchStore { client: Client, url: String, } #[derive(Debug, Deserialize)] pub struct SearchResponse { pub hits: Hits, } #[derive(Debug, Deserialize)] pub struct Hits { pub total: Total, pub hits: Vec, } #[derive(Debug, Deserialize)] pub struct Total { pub value: u64, } #[derive(Debug, Deserialize)] pub struct Hit { #[serde(rename = "_id", deserialize_with = "deserialize_string_to_u64")] pub id: u64, pub sort: Option, } #[derive(Debug, Deserialize)] pub struct DeleteByQueryResponse { pub deleted: u64, } impl SearchField { pub fn es_schema(&self) -> Value { match self { SearchField::AccountId | SearchField::DocumentId | SearchField::Email(EmailSearchField::Size) => json!({ "type": "integer" }), SearchField::Id | SearchField::Email(EmailSearchField::SentAt | EmailSearchField::ReceivedAt) | SearchField::Calendar(CalendarSearchField::Start) | SearchField::Tracing(TracingSearchField::QueueId | TracingSearchField::EventType) => { json!({ "type": "long" }) } SearchField::Email(EmailSearchField::HasAttachment) => json!({ "type": "boolean" }), SearchField::Calendar(CalendarSearchField::Uid) | SearchField::Contact(ContactSearchField::Uid) => json!({ "type": "keyword", }), SearchField::Email( EmailSearchField::From | EmailSearchField::To | EmailSearchField::Subject, ) => json!({ "type": "text", "fields": { "keyword": { "type": "keyword" } } }), SearchField::Email(EmailSearchField::Headers) => { json!({ "type": "object", "enabled": true }) } #[cfg(feature = "test_mode")] SearchField::Email(EmailSearchField::Bcc | EmailSearchField::Cc) => { json!({ "type": "text", "fields": { "keyword": { "type": "keyword" } } }) } #[cfg(not(feature = "test_mode"))] SearchField::Email(EmailSearchField::Bcc | EmailSearchField::Cc) => { json!({ "type": "text" }) } SearchField::Email(EmailSearchField::Body | EmailSearchField::Attachment) | SearchField::Calendar( CalendarSearchField::Title | CalendarSearchField::Description | CalendarSearchField::Location | CalendarSearchField::Owner | CalendarSearchField::Attendee, ) | SearchField::Contact( ContactSearchField::Member | ContactSearchField::Kind | ContactSearchField::Name | ContactSearchField::Nickname | ContactSearchField::Organization | ContactSearchField::Email | ContactSearchField::Phone | ContactSearchField::OnlineService | ContactSearchField::Address | ContactSearchField::Note, ) | SearchField::File(FileSearchField::Name | FileSearchField::Content) | SearchField::Tracing(TracingSearchField::Keywords) => json!({ "type": "text" }), } } } fn deserialize_string_to_u64<'de, D>(deserializer: D) -> Result where D: Deserializer<'de>, { <&str>::deserialize(deserializer)? .parse::() .map_err(serde::de::Error::custom) } ================================================ FILE: crates/store/src/backend/elastic/search.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ backend::elastic::{ DeleteByQueryResponse, ElasticSearchStore, SearchResponse, main::assert_success, }, search::{ IndexDocument, SearchComparator, SearchDocumentId, SearchField, SearchFilter, SearchOperator, SearchQuery, SearchValue, }, write::SearchIndex, }; use serde_json::{Map, Value, json}; use std::fmt::Write; impl ElasticSearchStore { pub async fn index(&self, documents: Vec) -> trc::Result<()> { let mut request = String::with_capacity(512); for document in documents { let id = if let (Some(SearchValue::Uint(account_id)), Some(SearchValue::Uint(doc_id))) = ( document.fields.get(&SearchField::AccountId), document.fields.get(&SearchField::DocumentId), ) { *account_id << 32 | *doc_id } else if let Some(SearchValue::Uint(id)) = document.fields.get(&SearchField::Id) { *id } else { debug_assert!(false, "Document is missing required ID fields"); continue; }; let _ = writeln!( &mut request, "{{\"index\":{{\"_index\":\"{}\",\"_id\":{id}}}}}", document.index.index_name() ); json_serialize(&mut request, &document); request.push('\n'); } assert_success( self.client .post(format!("{}/_bulk", self.url)) .body(request) .send() .await, ) .await .map(|_| ()) } pub async fn query( &self, index: SearchIndex, filters: &[SearchFilter], sort: &[SearchComparator], ) -> trc::Result> { let mut search_after: Option = None; let mut results = Vec::new(); let mut has_more = true; while has_more { let query = Map::from_iter( [ Some(("query".to_string(), build_query(filters))), Some(("size".to_string(), Value::from(10_000))), Some(("_source".to_string(), Value::from(false))), Some(( "sort".to_string(), build_sort(sort, R::field().field_name()), )), search_after .take() .map(|sa| ("search_after".to_string(), sa)), ] .into_iter() .flatten(), ); let response = assert_success( self.client .post(format!("{}/{}/_search", self.url, index.index_name())) .body(serde_json::to_string(&query).unwrap_or_default()) .send() .await, ) .await?; let text = response .text() .await .map_err(|err| trc::StoreEvent::ElasticsearchError.reason(err))?; let response = serde_json::from_str::(&text).map_err(|err| { trc::StoreEvent::ElasticsearchError .reason(err) .details(text) })?; has_more = response.hits.hits.len() == 10_000 && response.hits.hits.last().unwrap().sort.is_some(); for hit in response.hits.hits { search_after = hit.sort; results.push(R::from_u64(hit.id)); } } Ok(results) } pub async fn unindex(&self, filter: SearchQuery) -> trc::Result { if filter.filters.is_empty() { return Err(trc::StoreEvent::ElasticsearchError .reason("Unindex operation requires at least one filter")); } let query = json!({ "query": build_query(&filter.filters), }); let response = assert_success( self.client .post(format!( "{}/{}/_delete_by_query", self.url, filter.index.index_name() )) .body(serde_json::to_string(&query).unwrap_or_default()) .send() .await, ) .await?; let response_body = response .text() .await .map_err(|err| trc::StoreEvent::ElasticsearchError.reason(err))?; serde_json::from_str::(&response_body) .map(|delete_response| delete_response.deleted) .map_err(|err| trc::StoreEvent::ElasticsearchError.reason(err)) } pub async fn refresh_index(&self, index: SearchIndex) -> trc::Result<()> { let url = format!("{}/{}/_refresh", self.url, index.index_name()); assert_success(self.client.post(url).send().await) .await .map(|_| ()) } } fn build_query(filters: &[SearchFilter]) -> Value { if filters.is_empty() { return json!({ "match_all": {} }); } let mut stack = Vec::new(); let mut conditions = Vec::new(); let mut logical_op = &SearchFilter::And; for filter in filters { match filter { SearchFilter::Operator { field, op, value } => { if field.is_text() && matches!(op, SearchOperator::Equal | SearchOperator::Contains) { let SearchValue::Text { value, .. } = value else { debug_assert!(false, "Invalid value type for text field"); continue; }; if op != &SearchOperator::Equal { conditions.push(json!({ "match": { field.field_name(): { "query": value, "operator": "and" } } })); } else { conditions.push(json!({ "match_phrase": { field.field_name(): value } })); } } else { let value = match value { SearchValue::Text { value, .. } => json!(value), SearchValue::Int(value) => json!(value), SearchValue::Uint(value) => json!(value), SearchValue::Boolean(value) => json!(value), SearchValue::KeyValues(kv) => { let (key, value) = kv.iter().next().unwrap(); let cond = if !value.is_empty() { if op == &SearchOperator::Equal { json!({ "term": { format!("{}.{}.keyword", field.field_name(), key): value } }) } else { json!({ "match": { format!("{}.{}", field.field_name(), key): value } }) } } else { json!({ "exists": { "field": format!("{}.{}", field.field_name(), key) } }) }; conditions.push(cond); continue; } }; let cond = match op { SearchOperator::Equal | SearchOperator::Contains => json!({ "term": { field.field_name(): value } }), op => { let op = match op { SearchOperator::LowerThan => "lt", SearchOperator::LowerEqualThan => "lte", SearchOperator::GreaterThan => "gt", SearchOperator::GreaterEqualThan => "gte", _ => unreachable!(), }; json!({ "range": { field.field_name(): { op: value } } }) } }; conditions.push(cond); } } SearchFilter::And | SearchFilter::Or | SearchFilter::Not => { stack.push((logical_op, conditions)); logical_op = filter; conditions = Vec::new(); } SearchFilter::End => { if let Some((prev_logical_op, mut prev_conditions)) = stack.pop() { if !conditions.is_empty() { match logical_op { SearchFilter::And => { prev_conditions.push(json!({ "bool": { "must": conditions } })); } SearchFilter::Or => { prev_conditions.push(json!({ "bool": { "should": conditions } })); } SearchFilter::Not => { prev_conditions.push(json!({ "bool": { "must_not": conditions } })); } _ => unreachable!(), } } logical_op = prev_logical_op; conditions = prev_conditions; } } SearchFilter::DocumentSet(_) => { debug_assert!( false, "DocumentSet filters are not supported in this backend" ); continue; } } } debug_assert!( !conditions.is_empty(), "No conditions were built for the query" ); if conditions.len() == 1 { conditions.pop().unwrap() } else { json!({ "bool": { "must": conditions } }) } } fn build_sort(sort: &[SearchComparator], tie_breaker: &str) -> Value { Value::Array( sort.iter() .filter_map(|comp| match comp { SearchComparator::Field { field, ascending } => { let field = if field.is_text() { format!("{}.keyword", field.field_name()) } else { field.field_name().to_string() }; Some(json!({ field: if *ascending { "asc" } else { "desc" } })) } _ => None, }) .chain([json!({ tie_breaker: "asc" })]) .collect(), ) } fn json_serialize(request: &mut String, document: &IndexDocument) { request.push('{'); for (idx, (k, v)) in document.fields.iter().enumerate() { if idx > 0 { request.push(','); } let _ = write!(request, "{:?}:", k.field_name()); match v { SearchValue::Text { value, .. } => { json_serialize_str(request, value); } SearchValue::KeyValues(map) => { request.push('{'); for (i, (key, value)) in map.iter().enumerate() { if i > 0 { request.push(','); } json_serialize_str(request, key); request.push(':'); json_serialize_str(request, value); } request.push('}'); } SearchValue::Int(v) => { let _ = write!(request, "{}", v); } SearchValue::Uint(v) => { let _ = write!(request, "{}", v); } SearchValue::Boolean(v) => { let _ = write!(request, "{}", v); } } } request.push('}'); } fn json_serialize_str(request: &mut String, value: &str) { request.push('"'); for c in value.chars() { match c { '"' => request.push_str("\\\""), '\\' => request.push_str("\\\\"), '\n' => request.push_str("\\n"), '\r' => request.push_str("\\r"), '\t' => request.push_str("\\t"), '\u{0008}' => request.push_str("\\b"), // backspace '\u{000C}' => request.push_str("\\f"), // form feed _ => { if !c.is_control() { request.push(c); } else { let _ = write!(request, "\\u{:04x}", c as u32); } } } } request.push('"'); } ================================================ FILE: crates/store/src/backend/foundationdb/blob.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{FdbStore, MAX_VALUE_SIZE}; use crate::{ IterateParams, SUBSPACE_BLOBS, backend::foundationdb::into_error, write::{AnyKey, key::KeySerializer}, }; use std::ops::Range; use trc::AddContext; use types::blob_hash::BLOB_HASH_LEN; impl FdbStore { pub(crate) async fn get_blob( &self, key: &[u8], range: Range, ) -> trc::Result>> { let block_start = range.start / MAX_VALUE_SIZE; let bytes_start = range.start % MAX_VALUE_SIZE; let block_end = (range.end / MAX_VALUE_SIZE) + 1; let begin = KeySerializer::new(key.len() + 2) .write(key) .write(block_start as u16) .finalize(); let end = KeySerializer::new(key.len() + 2) .write(key) .write(block_end as u16) .finalize(); let key_len = begin.len(); let mut blob_data: Option> = None; let blob_range = range.end - range.start; self.iterate( IterateParams::new( AnyKey { subspace: SUBSPACE_BLOBS, key: begin, }, AnyKey { subspace: SUBSPACE_BLOBS, key: end, }, ), |key, value| { if key.len() == key_len { if let Some(blob_data) = &mut blob_data { blob_data.extend_from_slice( value .get( ..std::cmp::min( blob_range.saturating_sub(blob_data.len()), value.len(), ), ) .unwrap_or(&[]), ); if blob_data.len() == blob_range { return Ok(false); } } else { let blob_size = if blob_range <= (5 * (1 << 20)) { blob_range } else if value.len() == MAX_VALUE_SIZE { MAX_VALUE_SIZE * 2 } else { value.len() }; let mut blob_data_ = Vec::with_capacity(blob_size); blob_data_.extend_from_slice( value .get( bytes_start ..std::cmp::min(bytes_start + blob_range, value.len()), ) .unwrap_or(&[]), ); let is_done = blob_data_.len() == blob_range; blob_data = blob_data_.into(); if is_done { return Ok(false); } } } Ok(true) }, ) .await .caused_by(trc::location!())?; Ok(blob_data) } pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { const N_CHUNKS: usize = (1 << 5) - 1; let last_chunk = std::cmp::max( (data.len() / MAX_VALUE_SIZE) + if !data.len().is_multiple_of(MAX_VALUE_SIZE) { 1 } else { 0 }, 1, ) - 1; let mut trx = self.db.create_trx().map_err(into_error)?; for (chunk_pos, chunk_bytes) in data.chunks(MAX_VALUE_SIZE).enumerate() { trx.set( &KeySerializer::new(key.len() + 3) .write(SUBSPACE_BLOBS) .write(key) .write(chunk_pos as u16) .finalize(), chunk_bytes, ); if chunk_pos == last_chunk || (chunk_pos > 0 && chunk_pos % N_CHUNKS == 0) { self.commit(trx, false).await?; if chunk_pos < last_chunk { trx = self.db.create_trx().map_err(into_error)?; } else { break; } } } Ok(()) } pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result { if key.len() < BLOB_HASH_LEN { return Ok(false); } let trx = self.db.create_trx().map_err(into_error)?; trx.clear_range( &KeySerializer::new(key.len() + 3) .write(SUBSPACE_BLOBS) .write(key) .write(0u16) .finalize(), &KeySerializer::new(key.len() + 3) .write(SUBSPACE_BLOBS) .write(key) .write(u16::MAX) .finalize(), ); self.commit(trx, false).await } } ================================================ FILE: crates/store/src/backend/foundationdb/main.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use foundationdb::{Database, api, options::DatabaseOption}; use utils::config::{Config, utils::AsKey}; use super::FdbStore; impl FdbStore { pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { let prefix = prefix.as_key(); let guard = unsafe { api::FdbApiBuilder::default() .build() .map_err(|err| { config.new_build_error( prefix.as_str(), format!("Failed to boot FoundationDB: {err:?}"), ) }) .ok()? .boot() .map_err(|err| { config.new_build_error( prefix.as_str(), format!("Failed to boot FoundationDB: {err:?}"), ) }) .ok()? }; let db = Database::new(config.value((&prefix, "cluster-file"))) .map_err(|err| { config.new_build_error( prefix.as_str(), format!("Failed to create FoundationDB database: {err:?}"), ) }) .ok()?; if let Some(value) = config .property::>((&prefix, "transaction.timeout")) .unwrap_or_default() { db.set_option(DatabaseOption::TransactionTimeout(value.as_millis() as i32)) .map_err(|err| { config.new_build_error( (&prefix, "transaction.timeout"), format!("Failed to set option: {err:?}"), ) }) .ok()?; } if let Some(value) = config.property((&prefix, "transaction.retry-limit")) { db.set_option(DatabaseOption::TransactionRetryLimit(value)) .map_err(|err| { config.new_build_error( (&prefix, "transaction.retry-limit"), format!("Failed to set option: {err:?}"), ) }) .ok()?; } if let Some(value) = config .property::>((&prefix, "transaction.max-retry-delay")) .unwrap_or_default() { db.set_option(DatabaseOption::TransactionMaxRetryDelay( value.as_millis() as i32 )) .map_err(|err| { config.new_build_error( (&prefix, "transaction.max-retry-delay"), format!("Failed to set option: {err:?}"), ) }) .ok()?; } if let Some(value) = config.property((&prefix, "ids.machine")) { db.set_option(DatabaseOption::MachineId(value)) .map_err(|err| { config.new_build_error( (&prefix, "ids.machine"), format!("Failed to set option: {err:?}"), ) }) .ok()?; } if let Some(value) = config.property((&prefix, "ids.datacenter")) { db.set_option(DatabaseOption::DatacenterId(value)) .map_err(|err| { config.new_build_error( (&prefix, "ids.datacenter"), format!("Failed to set option: {err:?}"), ) }) .ok()?; } Some(Self { guard, db, version: Default::default(), }) } } ================================================ FILE: crates/store/src/backend/foundationdb/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use foundationdb::{Database, FdbError, api::NetworkAutoStop}; use std::time::{Duration, Instant}; pub mod blob; pub mod main; pub mod read; pub mod write; const MAX_VALUE_SIZE: usize = 100000; pub const TRANSACTION_EXPIRY: Duration = Duration::from_secs(1); #[allow(dead_code)] pub struct FdbStore { db: Database, guard: NetworkAutoStop, version: parking_lot::Mutex, } pub(crate) struct ReadVersion { version: i64, expires: Instant, } impl ReadVersion { pub fn new(version: i64) -> Self { Self { version, expires: Instant::now() + TRANSACTION_EXPIRY, } } pub fn is_expired(&self) -> bool { self.expires < Instant::now() } } impl Default for ReadVersion { fn default() -> Self { Self { version: 0, expires: Instant::now(), } } } #[inline(always)] fn into_error(error: FdbError) -> trc::Error { trc::StoreEvent::FoundationdbError .reason(error.message()) .ctx(trc::Key::Code, error.code()) } ================================================ FILE: crates/store/src/backend/foundationdb/read.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{FdbStore, MAX_VALUE_SIZE, ReadVersion, into_error}; use crate::{ Deserialize, IterateParams, Key, ValueKey, WITH_SUBSPACE, backend::deserialize_i64_le, write::{ValueClass, key::KeySerializer}, }; use foundationdb::{ KeySelector, RangeOption, Transaction, future::FdbSlice, options::{self}, }; use futures::TryStreamExt; #[allow(dead_code)] pub(crate) enum ChunkedValue { Single(FdbSlice), Chunked { n_chunks: u8, bytes: Vec }, None, } struct ChunkedValueCollector { key: Vec, bytes: Vec, } impl FdbStore { pub(crate) async fn get_value(&self, key: impl Key) -> trc::Result> where U: Deserialize, { let key = key.serialize(WITH_SUBSPACE); let trx = self.read_trx().await?; match read_chunked_value(&key, &trx, true).await? { ChunkedValue::Single(bytes) => U::deserialize(&bytes).map(Some), ChunkedValue::Chunked { bytes, .. } => U::deserialize_owned(bytes).map(Some), ChunkedValue::None => Ok(None), } } pub(crate) async fn iterate( &self, params: IterateParams, mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, ) -> trc::Result<()> { let begin = params.begin.serialize(WITH_SUBSPACE); let end = params.end.serialize(WITH_SUBSPACE); if !params.first { let mut last_key = vec![]; let mut chunked_key: Option = None; 'outer: loop { let begin_selector = if last_key.is_empty() { KeySelector::first_greater_or_equal(&begin) } else { KeySelector::first_greater_than(&last_key) }; let trx = self.read_trx().await?; let mut values = trx.get_ranges( RangeOption { begin: begin_selector, end: KeySelector::first_greater_than(&end), mode: options::StreamingMode::WantAll, reverse: !params.ascending, ..Default::default() }, true, ); let mut last_key_ = vec![]; loop { match values.try_next().await { Ok(Some(values)) => { let mut key = &[] as &[u8]; for value in values.iter() { key = value.key(); // Check whether we are collecting a chunked value let cb_key = key.get(1..).unwrap_or_default(); let cb_value = value.value(); if let Some(chunk) = &mut chunked_key { if chunk.key.len() + 1 == cb_key.len() && cb_key[..chunk.key.len()] == chunk.key[..] { // This is a chunk of the current value chunk.bytes.extend_from_slice(cb_value); continue; } else { // Return collected chunked value if !cb(&chunk.key, &chunk.bytes)? { return Ok(()); } // Reset collector chunked_key = None; } } if cb_value.len() < MAX_VALUE_SIZE { if !cb(cb_key, cb_value)? { return Ok(()); } } else { // Start collecting chunked value chunked_key = Some(ChunkedValueCollector { key: cb_key.to_vec(), bytes: cb_value.to_vec(), }); } } if values.more() { last_key_ = key.to_vec(); } } Ok(None) => { // Return any chunked value collected if let Some(chunked_key) = chunked_key.take() { cb(&chunked_key.key, &chunked_key.bytes)?; } break 'outer; } Err(e) => { if e.code() == 1007 && !last_key_.is_empty() { // Transaction is too old to perform reads or be committed drop(values); last_key = last_key_; continue 'outer; } else { return Err(into_error(e)); } } } } } } else { let trx = self.read_trx().await?; let mut values = trx.get_ranges_keyvalues( RangeOption { begin: KeySelector::first_greater_or_equal(&begin), end: KeySelector::first_greater_than(&end), mode: options::StreamingMode::Small, reverse: !params.ascending, ..Default::default() }, true, ); if let Some(value) = values.try_next().await.map_err(into_error)? { cb(value.key().get(1..).unwrap_or_default(), value.value())?; } } Ok(()) } pub(crate) async fn get_counter( &self, key: impl Into> + Sync + Send, ) -> trc::Result { let key = key.into().serialize(WITH_SUBSPACE); if let Some(bytes) = self .read_trx() .await? .get(&key, true) .await .map_err(into_error)? { deserialize_i64_le(&key, &bytes) } else { Ok(0) } } pub(crate) async fn read_trx(&self) -> trc::Result { let (is_expired, mut read_version) = { let version = self.version.lock(); (version.is_expired(), version.version) }; let trx = self.db.create_trx().map_err(into_error)?; if is_expired { read_version = trx.get_read_version().await.map_err(into_error)?; *self.version.lock() = ReadVersion::new(read_version); } else { trx.set_read_version(read_version); } Ok(trx) } } pub(crate) async fn read_chunked_value( key: &[u8], trx: &Transaction, snapshot: bool, ) -> trc::Result { if let Some(bytes) = trx.get(key, snapshot).await.map_err(into_error)? { if bytes.len() < MAX_VALUE_SIZE { Ok(ChunkedValue::Single(bytes)) } else { let mut value = Vec::with_capacity(bytes.len() * 2); value.extend_from_slice(&bytes); let mut key = KeySerializer::new(key.len() + 1) .write(key) .write(0u8) .finalize(); while let Some(bytes) = trx.get(&key, snapshot).await.map_err(into_error)? { value.extend_from_slice(&bytes); *key.last_mut().unwrap() += 1; } Ok(ChunkedValue::Chunked { bytes: value, n_chunks: *key.last().unwrap(), }) } } else { Ok(ChunkedValue::None) } } ================================================ FILE: crates/store/src/backend/foundationdb/write.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ FdbStore, MAX_VALUE_SIZE, ReadVersion, into_error, read::{ChunkedValue, read_chunked_value}, }; use crate::{ backend::deserialize_i64_le, write::{ AssignedIds, Batch, DirectoryClass, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, MergeResult, Operation, TaskQueueClass, TelemetryClass, ValueClass, ValueOp, key::KeySerializer, }, *, }; use foundationdb::{ FdbError, KeySelector, RangeOption, Transaction, options::{self, MutationType}, }; use futures::TryStreamExt; use rand::Rng; use std::{ cmp::Ordering, time::{Duration, Instant}, }; use trc::AddContext; impl FdbStore { pub(crate) async fn write(&self, batch: Batch<'_>) -> trc::Result { let start = Instant::now(); let mut retry_count = 0; let has_changes = !batch.changes.is_empty(); loop { let mut account_id = u32::MAX; let mut collection = u8::MAX; let mut document_id = u32::MAX; let mut change_id = 0u64; let mut result = AssignedIds::default(); let trx = self.db.create_trx().map_err(into_error)?; if has_changes { for &account_id in batch.changes.keys() { debug_assert!(account_id != u32::MAX); let key = ValueClass::ChangeId.serialize(account_id, 0, 0, WITH_SUBSPACE); let change_id = if let Some(bytes) = trx.get(&key, false).await.map_err(into_error)? { deserialize_i64_le(&key, &bytes)? + 1 } else { 1 }; trx.set(&key, &change_id.to_le_bytes()[..]); result.push_change_id(account_id, change_id as u64); } } for op in batch.ops.iter_mut() { match op { Operation::AccountId { account_id: account_id_, } => { account_id = *account_id_; if has_changes { change_id = result.set_current_change_id(account_id)?; } } Operation::Collection { collection: collection_, } => { collection = u8::from(*collection_); } Operation::DocumentId { document_id: document_id_, } => { document_id = *document_id_; } Operation::Value { class, op } => { let mut key = class.serialize(account_id, collection, document_id, WITH_SUBSPACE); match op { ValueOp::Set(value) => { if !chunk_value(&trx, &mut key, value) { trx.cancel(); return Err(trc::StoreEvent::FoundationdbError .ctx(trc::Key::Reason, "Value is too large")); } } ValueOp::SetFnc(set_op) => { let value = (set_op.fnc)(&set_op.params, &result)?; if !chunk_value(&trx, &mut key, &value) { trx.cancel(); return Err(trc::StoreEvent::FoundationdbError .ctx(trc::Key::Reason, "Value is too large")); } } ValueOp::MergeFnc(merge_op) => { let (merge_result, is_chunked) = match read_chunked_value(&key, &trx, false) .await .caused_by(trc::location!())? { ChunkedValue::Single(slice) => ( (merge_op.fnc)( &merge_op.params, &result, Some(slice.as_ref()), )?, false, ), ChunkedValue::Chunked { bytes, .. } => ( (merge_op.fnc)( &merge_op.params, &result, Some(bytes.as_ref()), )?, true, ), ChunkedValue::None => ( (merge_op.fnc)(&merge_op.params, &result, None)?, false, ), }; match merge_result { MergeResult::Update(value) => { if !chunk_value(&trx, &mut key, &value) { trx.cancel(); return Err(trc::StoreEvent::FoundationdbError .ctx(trc::Key::Reason, "Value is too large")); } } MergeResult::Delete => { if is_chunked { trx.clear_range( &key, &KeySerializer::new(key.len() + 1) .write(key.as_slice()) .write(u8::MAX) .finalize(), ); } else { trx.clear(&key); } } MergeResult::Skip => (), } } ValueOp::AtomicAdd(by) => { trx.atomic_op(&key, &by.to_le_bytes()[..], MutationType::Add); } ValueOp::AddAndGet(by) => { let num = if let Some(bytes) = trx.get(&key, false).await.map_err(into_error)? { deserialize_i64_le(&key, &bytes)? + *by } else { *by }; trx.set(&key, &num.to_le_bytes()[..]); result.push_counter_id(num); } ValueOp::Clear => { if matches!( key[0], SUBSPACE_DIRECTORY | SUBSPACE_TASK_QUEUE | SUBSPACE_IN_MEMORY_VALUE | SUBSPACE_PROPERTY | SUBSPACE_QUEUE_MESSAGE | SUBSPACE_REPORT_OUT | SUBSPACE_REPORT_IN | SUBSPACE_TELEMETRY_SPAN | SUBSPACE_SEARCH_INDEX | SUBSPACE_LOGS ) && matches!( class, ValueClass::Property(_) | ValueClass::Queue(_) | ValueClass::Report(_) | ValueClass::Directory(DirectoryClass::Principal(_)) | ValueClass::ShareNotification { .. } | ValueClass::Telemetry(TelemetryClass::Metric { .. }) | ValueClass::TaskQueue(TaskQueueClass::SendImip { is_payload: true, .. }) | ValueClass::InMemory(_) ) { trx.clear_range( &key, &KeySerializer::new(key.len() + 1) .write(key.as_slice()) .write(u8::MAX) .finalize(), ); } else { trx.clear(&key); } } } } Operation::Index { field, key, set } => { let key = IndexKey { account_id, collection, document_id, field: *field, key: &*key, } .serialize(WITH_SUBSPACE); if *set { trx.set(&key, &[]); } else { trx.clear(&key); } } Operation::Log { collection, set } => { let key = LogKey { account_id, collection: u8::from(*collection), change_id, } .serialize(WITH_SUBSPACE); trx.set(&key, set); } Operation::AssertValue { class, assert_value, } => { let key = class.serialize(account_id, collection, document_id, WITH_SUBSPACE); let matches = match read_chunked_value(&key, &trx, false).await { Ok(ChunkedValue::Single(bytes)) => assert_value.matches(bytes.as_ref()), Ok(ChunkedValue::Chunked { bytes, .. }) => { assert_value.matches(bytes.as_ref()) } Ok(ChunkedValue::None) => assert_value.is_none(), Err(_) => false, }; if !matches { trx.cancel(); return Err(trc::StoreEvent::AssertValueFailed.into()); } } } } if self .commit( trx, retry_count < MAX_COMMIT_ATTEMPTS && start.elapsed() < MAX_COMMIT_TIME, ) .await? { return Ok(result); } else { let backoff = rand::rng().random_range(50..=100); tokio::time::sleep(Duration::from_millis(backoff)).await; retry_count += 1; } } } pub(crate) async fn commit(&self, trx: Transaction, will_retry: bool) -> trc::Result { match trx.commit().await { Ok(result) => { let commit_version = result.committed_version().map_err(into_error)?; let mut version = self.version.lock(); if commit_version > version.version { *version = ReadVersion::new(commit_version); } Ok(true) } Err(err) => { if will_retry { err.on_error().await.map_err(into_error)?; Ok(false) } else { Err(into_error(FdbError::from(err))) } } } } pub(crate) async fn purge_store(&self) -> trc::Result<()> { // Obtain all zero counters let mut delete_keys = Vec::new(); for subspace in [SUBSPACE_COUNTER, SUBSPACE_QUOTA, SUBSPACE_IN_MEMORY_COUNTER] { let trx = self.db.create_trx().map_err(into_error)?; let from_key = [subspace, 0u8]; let to_key = [subspace, u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX]; let mut values = trx.get_ranges_keyvalues( RangeOption { begin: KeySelector::first_greater_or_equal(&from_key[..]), end: KeySelector::first_greater_or_equal(&to_key[..]), mode: options::StreamingMode::WantAll, reverse: false, ..Default::default() }, true, ); while let Some(value) = values.try_next().await.map_err(into_error)? { if value.value().iter().all(|byte| *byte == 0) { delete_keys.push(value.key().to_vec()); } } } if delete_keys.is_empty() { return Ok(()); } // Delete keys let integer = 0i64.to_le_bytes(); for chunk in delete_keys.chunks(1024) { let mut retry_count = 0; loop { let trx = self.db.create_trx().map_err(into_error)?; for key in chunk { trx.atomic_op(key, &integer, MutationType::CompareAndClear); } if self.commit(trx, retry_count < MAX_COMMIT_ATTEMPTS).await? { break; } else { retry_count += 1; } } } Ok(()) } pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> { let from = from.serialize(WITH_SUBSPACE); let to = to.serialize(WITH_SUBSPACE); let trx = self.db.create_trx().map_err(into_error)?; trx.clear_range(&from, &to); self.commit(trx, false).await.map(|_| ()) } } fn chunk_value(trx: &Transaction, key: &mut Vec, value: &[u8]) -> bool { if !value.is_empty() && value.len() > MAX_VALUE_SIZE { for (pos, chunk) in value.chunks(MAX_VALUE_SIZE).enumerate() { match pos.cmp(&1) { Ordering::Less => {} Ordering::Equal => { key.push(0); } Ordering::Greater => { if pos < u8::MAX as usize { *key.last_mut().unwrap() += 1; } else { return false; } } } trx.set(key, chunk); } } else { trx.set(key, value.as_ref()); } true } ================================================ FILE: crates/store/src/backend/fs/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{io::SeekFrom, ops::Range, path::PathBuf}; use tokio::{ fs::{self, File}, io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}, }; use utils::{ codec::base32_custom::Base32Writer, config::{Config, utils::AsKey}, }; pub struct FsStore { path: PathBuf, hash_levels: usize, } impl FsStore { pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { let prefix = prefix.as_key(); let path = PathBuf::from(config.value_require((&prefix, "path"))?); if !path.exists() { fs::create_dir_all(&path) .await .map_err(|e| { config.new_build_error( (&prefix, "path"), format!("Failed to create directory: {e}"), ) }) .ok()?; } Some(FsStore { path, hash_levels: std::cmp::min( config .property_or_default((&prefix, "depth"), "2") .unwrap_or(2), 5, ), }) } pub(crate) async fn get_blob( &self, key: &[u8], range: Range, ) -> trc::Result>> { let blob_path = self.build_path(key); let blob_size = match fs::metadata(&blob_path).await { Ok(m) => m.len() as usize, Err(_) => return Ok(None), }; let mut blob = File::open(&blob_path).await.map_err(into_error)?; Ok(Some(if range.start != 0 || range.end != usize::MAX { let from_offset = if range.start < blob_size { range.start } else { 0 }; let mut buf = vec![0; (std::cmp::min(range.end, blob_size) - from_offset) as usize]; if from_offset > 0 { blob.seek(SeekFrom::Start(from_offset as u64)) .await .map_err(into_error)?; } blob.read_exact(&mut buf).await.map_err(into_error)?; buf } else { let mut buf = Vec::with_capacity(blob_size as usize); blob.read_to_end(&mut buf).await.map_err(into_error)?; buf })) } pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { let blob_path = self.build_path(key); if fs::metadata(&blob_path) .await .map_or(true, |m| m.len() as usize != data.len()) { fs::create_dir_all(blob_path.parent().unwrap()) .await .map_err(into_error)?; let mut blob_file = File::create(&blob_path).await.map_err(into_error)?; blob_file.write_all(data).await.map_err(into_error)?; blob_file.flush().await.map_err(into_error)?; } Ok(()) } pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result { let blob_path = self.build_path(key); if fs::metadata(&blob_path).await.is_ok() { fs::remove_file(&blob_path).await.map_err(into_error)?; Ok(true) } else { Ok(false) } } fn build_path(&self, key: &[u8]) -> PathBuf { let mut path = self.path.clone(); for byte in key.iter().take(self.hash_levels) { path.push(format!("{:x}", byte)); } path.push(Base32Writer::from_bytes(key).finalize()); path } } fn into_error(err: std::io::Error) -> trc::Error { trc::StoreEvent::FilesystemError.reason(err) } ================================================ FILE: crates/store/src/backend/http/config.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ collections::hash_map::Entry, sync::atomic::{AtomicBool, AtomicU64}, time::Duration, }; use ahash::AHashMap; use arc_swap::ArcSwap; use utils::config::Config; use crate::{InMemoryStore, Stores}; use super::{HttpStore, HttpStoreConfig, HttpStoreFormat}; impl Stores { pub fn parse_http_stores(&mut self, config: &mut Config, is_reload: bool) { // Parse remote lists for id in config.sub_keys("http-lookup", ".url") { let id_ = id.as_str(); if !config .property_or_default(("http-lookup", id_, "enable"), "true") .unwrap_or(true) { continue; } let format = match config .value_require(("http-lookup", id_, "format")) .unwrap_or_default() { "list" => HttpStoreFormat::List, "csv" => HttpStoreFormat::Csv { index_key: config .property_require(("http-lookup", id_, "index.key")) .unwrap_or(0), index_value: config.property(("http-lookup", id_, "index.value")), separator: config .property_or_default::(("http-lookup", id_, "separator"), ",") .unwrap_or_default() .chars() .next() .unwrap_or(','), skip_first: config .property_or_default::(("http-lookup", id_, "skip-first"), "false") .unwrap_or(false), }, other => { let message = format!("Invalid format: {other:?}"); config.new_build_error(("http-lookup", id_, "format"), message); continue; } }; let http_config = HttpStoreConfig { url: config .value_require(("http-lookup", id_, "url")) .unwrap_or_default() .to_string(), retry: config .property_or_default::(("http-lookup", id_, "retry"), "1h") .unwrap_or(Duration::from_secs(3600)) .as_secs(), refresh: config .property_or_default::(("http-lookup", id_, "refresh"), "12h") .unwrap_or(Duration::from_secs(43200)) .as_secs(), timeout: config .property_or_default::(("http-lookup", id_, "timeout"), "30s") .unwrap_or(Duration::from_secs(30)), gzipped: config .property_or_default::(("http-lookup", id_, "gzipped"), "false") .unwrap_or_default(), max_size: config .property_or_default::(("http-lookup", id_, "limits.size"), "104857600") .unwrap_or(104857600), max_entries: config .property_or_default::(("http-lookup", id_, "limits.entries"), "100000") .unwrap_or(100000), max_entry_size: config .property_or_default::(("http-lookup", id_, "limits.entry-size"), "512") .unwrap_or(512), format, id, }; match self.in_memory_stores.entry(http_config.id.clone()) { Entry::Vacant(entry) => { let store = HttpStore { entries: ArcSwap::from_pointee(AHashMap::new()), expires: AtomicU64::new(0), in_flight: AtomicBool::new(false), config: http_config, }; entry.insert(InMemoryStore::Http(store.into())); } Entry::Occupied(e) if !is_reload => { config.new_build_error( ("http-lookup", e.key().as_str()), "An in-memory store with this id already exists", ); } _ => {} } } } } ================================================ FILE: crates/store/src/backend/http/lookup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ io::{BufRead, BufReader}, sync::{Arc, atomic::Ordering}, time::Instant, }; use ahash::AHashMap; use compact_str::ToCompactString; use rand::seq::IndexedRandom; use utils::HttpLimitResponse; use crate::{Value, backend::http::HttpStoreFormat, write::now}; use super::HttpStore; const BROWSER_USER_AGENTS: [&str; 5] = [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/120.0.0.0 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", ]; pub(crate) trait HttpStoreGet { fn get(&self, key: &str) -> Option>; fn contains(&self, key: &str) -> bool; fn refresh(&self); } impl HttpStoreGet for Arc { fn get(&self, key: &str) -> Option> { self.refresh(); self.entries.load().get(key).cloned() } fn contains(&self, key: &str) -> bool { #[cfg(feature = "test_mode")] { if self.config.url.contains("phishtank.com") || self.config.url.contains("openphish.com") { return (self.config.url.contains("open") && key.contains("open")) || (self.config.url.contains("tank") && key.contains("tank")); } else if self.config.url.contains("disposable.github.io") { return key.ends_with("guerrillamail.com") || key.ends_with("disposable.org"); } else if self.config.url.contains("free_email_provider_domains.txt") { return key.ends_with("gmail.com") || key.ends_with("googlemail.com") || key.ends_with("yahoomail.com") || key.ends_with("outlook.com") || key.ends_with("freemail.org"); } } self.refresh(); self.entries.load().contains_key(key) } fn refresh(&self) { if self.expires.load(Ordering::Relaxed) <= now() { let in_flight = self.in_flight.swap(true, Ordering::Relaxed); if !in_flight { let this = self.clone(); tokio::spawn(async move { let expires = match this.try_refresh().await { Ok(list) => { this.entries.store(list.into()); this.config.refresh } Err(err) => { trc::error!(err); this.config.retry } }; this.expires.store(now() + expires, Ordering::Relaxed); this.in_flight.store(false, Ordering::Relaxed); }); } } } } impl HttpStore { async fn try_refresh(&self) -> trc::Result>> { let time = Instant::now(); let agent = BROWSER_USER_AGENTS.choose(&mut rand::rng()).unwrap(); let response = reqwest::Client::builder() .timeout(self.config.timeout) .user_agent(*agent) .build() .unwrap_or_default() .get(&self.config.url) .send() .await .map_err(|err| { trc::StoreEvent::HttpStoreError .into_err() .reason(err) .ctx(trc::Key::Url, self.config.url.to_compact_string()) .details("Failed to build request") })?; if !response.status().is_success() { trc::bail!( trc::StoreEvent::HttpStoreError .into_err() .ctx(trc::Key::Code, response.status().as_u16()) .ctx(trc::Key::Url, self.config.url.to_compact_string()) .ctx(trc::Key::Elapsed, time.elapsed()) .details("Failed to fetch HTTP list") ); } let bytes = response .bytes_with_limit(self.config.max_size) .await .map_err(|err| { trc::StoreEvent::HttpStoreError .into_err() .reason(err) .ctx(trc::Key::Url, self.config.url.to_compact_string()) .ctx(trc::Key::Elapsed, time.elapsed()) .details("Failed to fetch resource") })? .ok_or_else(|| { trc::StoreEvent::HttpStoreError .into_err() .ctx(trc::Key::Url, self.config.url.to_compact_string()) .ctx(trc::Key::Elapsed, time.elapsed()) .details("Resource is too large") })?; let reader: Box = if self.config.gzipped { Box::new(flate2::read::GzDecoder::new(&bytes[..])) } else { Box::new(&bytes[..]) }; let mut entries = AHashMap::new(); for (pos, line) in BufReader::new(reader).lines().enumerate() { let line_ = line.map_err(|err| { trc::StoreEvent::HttpStoreError .into_err() .reason(err) .ctx(trc::Key::Url, self.config.url.to_compact_string()) .ctx(trc::Key::Elapsed, time.elapsed()) .details("Failed to read line") })?; match &self.config.format { HttpStoreFormat::List => { let line = line_.trim(); if !line.is_empty() { entries.insert(line.to_string(), Value::Integer(1)); } } HttpStoreFormat::Csv { index_key, index_value, separator, skip_first, } if pos > 0 || !*skip_first => { let mut in_quote = false; let mut col_num = 0; let mut last_ch = ' '; let mut entry_key: String = String::new(); let mut entry_value: String = String::new(); for ch in line_.chars() { match ch { '"' if last_ch != '\\' => { in_quote = !in_quote; } '\\' if last_ch != '\\' => (), _ => { if ch == *separator && !in_quote { if col_num == *index_key && index_value.is_none() { break; } else { col_num += 1; } } else if col_num == *index_key { entry_key.push(ch); if entry_key.len() > self.config.max_entry_size { break; } } else if index_value.is_some_and(|v| col_num == v) { entry_value.push(ch); if entry_value.len() > self.config.max_entry_size { break; } } } } last_ch = ch; } if !entry_key.is_empty() { let entry_value = if !entry_value.is_empty() { Value::Text(entry_value.into()) } else { Value::Integer(1) }; entries.insert(entry_key, entry_value); } } _ => (), } if entries.len() == self.config.max_entries { break; } } trc::event!( Store(trc::StoreEvent::HttpStoreFetch), Url = self.config.url.to_compact_string(), Total = entries.len(), Elapsed = time.elapsed(), ); Ok(entries) } } ================================================ FILE: crates/store/src/backend/http/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod config; pub mod lookup; use std::{ sync::atomic::{AtomicBool, AtomicU64}, time::Duration, }; use ahash::AHashMap; use arc_swap::ArcSwap; use crate::Value; #[derive(Debug, Clone)] pub struct HttpStoreConfig { pub id: String, pub url: String, pub retry: u64, pub refresh: u64, pub timeout: Duration, pub gzipped: bool, pub max_size: usize, pub max_entries: usize, pub max_entry_size: usize, pub format: HttpStoreFormat, } #[derive(Debug, Clone)] pub enum HttpStoreFormat { List, Csv { index_key: u32, index_value: Option, separator: char, skip_first: bool, }, } #[derive(Debug)] pub struct HttpStore { pub entries: ArcSwap>>, pub expires: AtomicU64, pub in_flight: AtomicBool, pub config: HttpStoreConfig, } ================================================ FILE: crates/store/src/backend/kafka/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use rdkafka::{ ClientConfig, ClientContext, TopicPartitionList, consumer::{BaseConsumer, ConsumerContext, Rebalance, StreamConsumer}, error::KafkaResult, producer::FutureProducer, }; use std::{fmt::Debug, time::Duration}; use utils::config::{Config, utils::AsKey}; pub mod pubsub; pub(super) type LoggingConsumer = StreamConsumer; pub struct KafkaPubSub { consumer_builder: ClientConfig, producer: FutureProducer, } impl KafkaPubSub { pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { let prefix = prefix.as_key(); let brokers = config .values((&prefix, "brokers")) .map(|(_, v)| v.to_string()) .collect::>(); if brokers.is_empty() { config.new_build_error((&prefix, "brokers"), "No Kafka brokers specified"); return None; } let mut consumer_builder = ClientConfig::new(); consumer_builder .set( "group.id", config.value_require_non_empty((&prefix, "group-id"))?, ) .set( "bootstrap.servers", config.value_require_non_empty((&prefix, "brokers"))?, ) .set("enable.partition.eof", "false") .set( "session.timeout.ms", config .property_or_default((&prefix, "timeout.session"), "5s") .unwrap_or(Duration::from_secs(5)) .as_millis() .to_string(), ) .set("enable.auto.commit", "true"); let producer = ClientConfig::new() .set( "bootstrap.servers", config.value_require_non_empty((&prefix, "brokers"))?, ) .set( "message.timeout.ms", config .property_or_default((&prefix, "timeout.message"), "5s") .unwrap_or(Duration::from_secs(5)) .as_millis() .to_string(), ) .create() .map_err(|err| { config.new_build_error( (&prefix, "config"), format!("Failed to create Kafka producer: {}", err), ); }) .ok()?; KafkaPubSub { consumer_builder, producer, } .into() } } impl Debug for KafkaPubSub { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("KafkaPubSub").finish() } } pub(super) struct CustomContext; impl ClientContext for CustomContext {} impl ConsumerContext for CustomContext { fn pre_rebalance(&self, _: &BaseConsumer, _: &Rebalance) {} fn post_rebalance(&self, _: &BaseConsumer, _: &Rebalance) {} fn commit_callback(&self, _: KafkaResult<()>, _: &TopicPartitionList) {} } ================================================ FILE: crates/store/src/backend/kafka/pubsub.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use super::{CustomContext, KafkaPubSub, LoggingConsumer}; use crate::dispatch::pubsub::{Msg, PubSubStream}; use rdkafka::{ Message, consumer::{CommitMode, Consumer, StreamConsumer}, producer::FutureRecord, }; use trc::{ClusterEvent, Error, EventType}; pub struct KafkaPubSubStream { subs: LoggingConsumer, } impl KafkaPubSub { pub async fn publish(&self, topic: &'static str, message: Vec) -> trc::Result<()> { self.producer .send( FutureRecord::<(), [u8]>::to(topic).payload(message.as_slice()), Duration::from_secs(0), ) .await .map(|_| ()) .map_err(|(err, _)| { Error::new(EventType::Cluster(ClusterEvent::PublisherError)).reason(err) }) } pub async fn subscribe(&self, topic: &'static str) -> trc::Result { let subs: StreamConsumer = self .consumer_builder .create_with_context(CustomContext) .map_err(|err| { Error::new(EventType::Cluster(ClusterEvent::SubscriberError)).reason(err) })?; subs.subscribe(&[topic]).map_err(|err| { Error::new(EventType::Cluster(ClusterEvent::SubscriberError)).reason(err) })?; Ok(PubSubStream::Kafka(KafkaPubSubStream { subs })) } } impl KafkaPubSubStream { pub async fn next(&mut self) -> Option { let msg = self.subs.recv().await.ok()?; let _ = self.subs.commit_message(&msg, CommitMode::Async); Msg::Kafka(msg.payload().unwrap_or_default().to_vec()).into() } } ================================================ FILE: crates/store/src/backend/meili/main.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ backend::meili::{MeiliSearchStore, Task, TaskStatus, TaskUid}, search::{ CalendarSearchField, ContactSearchField, EmailSearchField, SearchableField, TracingSearchField, }, }; use reqwest::{Error, Response, Url}; use serde_json::{Value, json}; use std::time::Duration; use utils::config::{Config, http::build_http_client, utils::AsKey}; impl MeiliSearchStore { pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { let client = build_http_client(config, prefix.clone(), "application/json".into())?; let prefix = prefix.as_key(); let url = config .value_require((&prefix, "url"))? .trim_end_matches("/") .to_string(); Url::parse(&url) .map_err(|e| config.new_parse_error((&prefix, "url"), format!("Invalid URL: {e}",))) .ok()?; let task_poll_interval = config .property_or_default::((&prefix, "task.poll-interval"), "500ms") .unwrap_or(Duration::from_millis(500)); let task_poll_retries = config .property_or_default::((&prefix, "task.poll-retries"), "120") .unwrap_or(120); let task_fail_on_timeout = config .property_or_default::((&prefix, "task.fail-on-timeout"), "true") .unwrap_or(true); let ms = Self { client, url, task_poll_interval: Duration::from_millis(500), task_poll_retries: 120, task_fail_on_timeout: true, }; if let Err(err) = ms.create_indexes().await { config.new_build_error(prefix.as_str(), err.to_string()); } Some(Self { client: ms.client, url: ms.url, task_poll_interval, task_poll_retries, task_fail_on_timeout, }) } pub async fn create_indexes(&self) -> trc::Result<()> { self.create_index::().await?; self.create_index::().await?; self.create_index::().await?; self.create_index::().await?; Ok(()) } async fn create_index(&self) -> trc::Result<()> { let index_name = T::index().index_name(); let response = assert_success( self.client .post(format!("{}/indexes", self.url)) .body( json!({ "uid": index_name, "primaryKey": "id", }) .to_string(), ) .send() .await, ) .await?; if !self.wait_for_task(response).await? { // Index already exists return Ok(()); } let mut searchable = Vec::new(); let mut filterable = Vec::new(); let mut sortable = Vec::new(); for field in T::all_fields() { if field.is_indexed() { sortable.push(Value::String(field.field_name().to_string())); } if field.is_text() { searchable.push(Value::String(field.field_name().to_string())); } else { filterable.push(Value::String(field.field_name().to_string())); } } for key in T::primary_keys() { filterable.push(Value::String(key.field_name().to_string())); } #[cfg(feature = "test_mode")] filterable.push(Value::String("bcc".into())); if !searchable.is_empty() { self.update_index_settings( index_name, "searchable-attributes", Value::Array(searchable), ) .await?; } if !filterable.is_empty() { self.update_index_settings( index_name, "filterable-attributes", Value::Array(filterable), ) .await?; } if !sortable.is_empty() { self.update_index_settings(index_name, "sortable-attributes", Value::Array(sortable)) .await?; } Ok(()) } async fn update_index_settings( &self, index_uid: &str, setting: &str, value: Value, ) -> trc::Result { let response = assert_success( self.client .put(format!( "{}/indexes/{}/settings/{}", self.url, index_uid, setting )) .body(value.to_string()) .send() .await, ) .await?; self.wait_for_task(response).await } #[cfg(feature = "test_mode")] pub async fn drop_indexes(&self) -> trc::Result<()> { use crate::write::SearchIndex; for index in &[ SearchIndex::Email, SearchIndex::Calendar, SearchIndex::Contacts, SearchIndex::Tracing, ] { let response = self .client .delete(format!("{}/indexes/{}", self.url, index.index_name())) .send() .await .map_err(|err| trc::StoreEvent::MeilisearchError.reason(err))?; match response.status().as_u16() { 200..=299 => { self.wait_for_task(response).await?; } 400..=499 => { // Index does not exist return Ok(()); } _ => { let status = response.status(); let msg = response.text().await.unwrap_or_default(); return Err(trc::StoreEvent::MeilisearchError .reason(msg) .ctx(trc::Key::Code, status.as_u16())); } } } Ok(()) } pub(crate) async fn wait_for_task(&self, response: Response) -> trc::Result { let response_body = response.text().await.map_err(|err| { trc::StoreEvent::MeilisearchError .reason(err) .details("Request failed") })?; let task_uid = serde_json::from_str::(&response_body) .map_err(|err| trc::StoreEvent::MeilisearchError.reason(err))? .task_uid; let mut loop_count = 0; let url = format!("{}/tasks/{}", self.url, task_uid); while loop_count < self.task_poll_retries { let resp = assert_success(self.client.get(&url).send().await).await?; let text = resp .text() .await .map_err(|err| trc::StoreEvent::MeilisearchError.reason(err))?; let task = serde_json::from_str::(&text).map_err(|err| { trc::StoreEvent::MeilisearchError .reason(err) .details(text.clone()) })?; match task.status { TaskStatus::Succeeded => return Ok(true), TaskStatus::Failed => { let (code, message) = task .error .map(|e| (e.code, Some(e.message))) .unwrap_or((None, None)); return if matches!(code.as_deref(), Some("index_already_exists")) { Ok(false) } else { Err(trc::StoreEvent::MeilisearchError .reason("Meilisearch task failed.") .id(task_uid) .code(code) .details(message)) }; } TaskStatus::Canceled => { return Err(trc::StoreEvent::MeilisearchError .reason("Meilisearch task was canceled") .id(task_uid)); } TaskStatus::Enqueued | TaskStatus::Processing => { loop_count += 1; tokio::time::sleep(self.task_poll_interval).await; } TaskStatus::Unknown => { return Err(trc::StoreEvent::MeilisearchError .reason("Meilisearch task returned an unknown status") .id(task_uid) .details(text)); } } } if self.task_fail_on_timeout { Err(trc::StoreEvent::MeilisearchError .reason("Timed out waiting for Meilisearch task") .id(task_uid)) } else { Ok(true) } } } pub(crate) async fn assert_success(response: Result) -> trc::Result { match response { Ok(response) => { let status = response.status(); if status.is_success() { Ok(response) } else { Err(trc::StoreEvent::MeilisearchError .reason(response.text().await.unwrap_or_default()) .ctx(trc::Key::Code, status.as_u16())) } } Err(err) => Err(trc::StoreEvent::MeilisearchError.reason(err)), } } ================================================ FILE: crates/store/src/backend/meili/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use reqwest::Client; use serde::Deserialize; use std::time::Duration; pub mod main; pub mod search; pub struct MeiliSearchStore { client: Client, url: String, task_poll_interval: Duration, task_poll_retries: usize, task_fail_on_timeout: bool, } #[derive(Debug, Deserialize)] pub(crate) struct TaskUid { #[serde(rename = "taskUid")] pub task_uid: u64, } #[derive(Debug, Deserialize)] struct TaskError { message: String, #[serde(default)] code: Option, } #[derive(Debug, Deserialize)] struct Task { //#[serde(rename = "uid")] //uid: u64, status: TaskStatus, #[serde(default)] error: Option, } #[derive(Debug, Deserialize)] #[serde(rename_all = "lowercase")] enum TaskStatus { Enqueued, Processing, Succeeded, Failed, Canceled, #[serde(other)] Unknown, } #[derive(Debug, Deserialize)] struct MeiliSearchResponse { hits: Vec, //#[allow(dead_code)] //#[serde(default, rename = "estimatedTotalHits")] //estimated_total_hits: Option, } #[derive(Debug, Deserialize)] struct MeiliHit { id: u64, } ================================================ FILE: crates/store/src/backend/meili/search.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use ahash::AHashSet; use serde_json::{Map, Value, json}; use crate::{ backend::meili::{MeiliSearchResponse, MeiliSearchStore, main::assert_success}, search::*, write::SearchIndex, }; use std::fmt::{Display, Write}; impl MeiliSearchStore { pub async fn index(&self, documents: Vec) -> trc::Result<()> { let mut index_documents: [String; 5] = [ String::new(), String::new(), String::new(), String::new(), String::new(), ]; for document in documents { let request = &mut index_documents[document.index.array_pos()]; if !request.is_empty() { request.push(','); } else { request.reserve(1024); request.push('['); } json_serialize(request, &document); } for (mut payload, index) in index_documents.into_iter().zip([ SearchIndex::Email, SearchIndex::Calendar, SearchIndex::Contacts, SearchIndex::Tracing, SearchIndex::File, ]) { if payload.is_empty() { continue; } payload.push(']'); let response = assert_success( self.client .put(format!( "{}/indexes/{}/documents", self.url, index.index_name() )) .body(payload) .send() .await, ) .await?; self.wait_for_task(response).await?; } Ok(()) } pub async fn query( &self, index: SearchIndex, filters: &[SearchFilter], sort: &[SearchComparator], ) -> trc::Result> { let filter_group = build_query(filters); let mut body = Map::new(); body.insert("limit".to_string(), Value::from(10_000)); body.insert("offset".to_string(), Value::from(0)); body.insert( "attributesToRetrieve".to_string(), Value::Array(vec![Value::String("id".to_string())]), ); if !filter_group.filter.is_empty() { body.insert("filter".to_string(), Value::String(filter_group.filter)); } if !filter_group.q.is_empty() { body.insert("q".to_string(), Value::String(filter_group.q)); } if !sort.is_empty() { let sort_arr: Vec = sort .iter() .filter_map(|comp| match comp { SearchComparator::Field { field, ascending } => Some(Value::String(format!( "{}:{}", field.field_name(), if *ascending { "asc" } else { "desc" } ))), _ => None, }) .collect(); if !sort_arr.is_empty() { body.insert("sort".to_string(), Value::Array(sort_arr)); } } let resp = assert_success( self.client .post(format!( "{}/indexes/{}/search", self.url, index.index_name() )) .body(Value::Object(body).to_string()) .send() .await, ) .await?; let text = resp .text() .await .map_err(|err| trc::StoreEvent::MeilisearchError.reason(err))?; serde_json::from_str::(&text) .map(|results| { results .hits .into_iter() .map(|hit| R::from_u64(hit.id)) .collect() }) .map_err(|err| trc::StoreEvent::MeilisearchError.reason(err).details(text)) } pub async fn unindex(&self, filter: SearchQuery) -> trc::Result { let filter_group = build_query(&filter.filters); if filter_group.filter.is_empty() { return Err(trc::StoreEvent::MeilisearchError.reason( "Meilisearch delete-by-filter requires structured (non-text) filters only", )); } let url = format!( "{}/indexes/{}/documents/delete", self.url, filter.index.index_name() ); let response = assert_success( self.client .post(url) .body(json!({ "filter": filter_group.filter }).to_string()) .send() .await, ) .await?; self.wait_for_task(response).await?; Ok(0) } } #[derive(Default, Debug)] struct FilterGroup { q: String, filter: String, } fn build_query(filters: &[SearchFilter]) -> FilterGroup { if filters.is_empty() { return FilterGroup::default(); } let mut operator_stack = Vec::new(); let mut operator = &SearchFilter::And; let mut is_first = true; let mut filter = String::new(); let mut queries = AHashSet::new(); for f in filters { match f { SearchFilter::Operator { field, op, value } => { if field.is_text() && matches!(op, SearchOperator::Equal | SearchOperator::Contains) { let value = match value { SearchValue::Text { value, .. } => value, _ => { debug_assert!( false, "Text field search with non-text value is not supported" ); "" } }; if matches!(op, SearchOperator::Equal) { queries.insert(format!("{value:?}")); } else { for token in value.split_whitespace() { queries.insert(token.to_string()); } } } else { if !filter.is_empty() && !filter.ends_with('(') { match operator { SearchFilter::And => filter.push_str(" AND "), SearchFilter::Or => filter.push_str(" OR "), _ => (), } } match value { SearchValue::Text { value, .. } => { filter.push_str(field.field_name()); filter.push(' '); op.write_meli_op(&mut filter, format!("{value:?}")); } SearchValue::KeyValues(kv) => { let (key, value) = kv.iter().next().unwrap(); filter.push_str(field.field_name()); filter.push('.'); filter.push_str(key); filter.push(' '); op.write_meli_op(&mut filter, format!("{value:?}")); } SearchValue::Int(v) => { filter.push_str(field.field_name()); filter.push(' '); op.write_meli_op(&mut filter, v); } SearchValue::Uint(v) => { filter.push_str(field.field_name()); filter.push(' '); op.write_meli_op(&mut filter, v); } SearchValue::Boolean(v) => { filter.push_str(field.field_name()); filter.push(' '); op.write_meli_op(&mut filter, v); } } } } SearchFilter::And | SearchFilter::Or => { if !filter.is_empty() && !filter.ends_with('(') { match operator { SearchFilter::And => filter.push_str(" AND "), SearchFilter::Or => filter.push_str(" OR "), _ => (), } } operator_stack.push((operator, is_first)); operator = f; is_first = true; filter.push('('); } SearchFilter::Not => { if !filter.is_empty() && !filter.ends_with('(') { match operator { SearchFilter::And => filter.push_str(" AND "), SearchFilter::Or => filter.push_str(" OR "), _ => (), } } operator_stack.push((operator, is_first)); operator = &SearchFilter::And; is_first = true; filter.push_str("NOT ("); } SearchFilter::End => { let p = operator_stack.pop().unwrap_or((&SearchFilter::And, true)); operator = p.0; is_first = p.1; if !filter.ends_with('(') { filter.push(')'); } else { filter.pop(); if filter.ends_with("NOT ") { let len = filter.len(); filter.truncate(len - 4); } if filter.ends_with(" AND ") { let len = filter.len(); filter.truncate(len - 5); is_first = true; } else if filter.ends_with(" OR ") { let len = filter.len(); filter.truncate(len - 4); is_first = true; } } } SearchFilter::DocumentSet(_) => { debug_assert!(false, "DocumentSet filters are not supported") } } } let mut q = String::new(); if !queries.is_empty() { for (idx, term) in queries.into_iter().enumerate() { if idx > 0 { q.push(' '); } q.push_str(&term); } } FilterGroup { q, filter } } impl SearchOperator { fn write_meli_op(&self, query: &mut String, value: impl Display) { match self { SearchOperator::LowerThan => { let _ = write!(query, "< {value}"); } SearchOperator::LowerEqualThan => { let _ = write!(query, "<= {value}"); } SearchOperator::GreaterThan => { let _ = write!(query, "> {value}"); } SearchOperator::GreaterEqualThan => { let _ = write!(query, ">= {value}"); } SearchOperator::Equal | SearchOperator::Contains => { let _ = write!(query, "= {value}"); } } } } fn json_serialize(request: &mut String, document: &IndexDocument) { let mut id = 0u64; let mut is_first = true; request.push('{'); for (k, v) in document.fields.iter() { match k { SearchField::AccountId => { if let SearchValue::Uint(account_id) = v { id |= account_id << 32; } } SearchField::DocumentId => { if let SearchValue::Uint(doc_id) = v { id |= doc_id; } } SearchField::Id => { if let SearchValue::Uint(doc_id) = v { id = *doc_id; } continue; } _ => {} } if !is_first { request.push(','); } else { is_first = false; } let _ = write!(request, "{:?}:", k.field_name()); match v { SearchValue::Text { value, .. } => { json_serialize_str(request, value); } SearchValue::KeyValues(map) => { request.push('{'); for (i, (key, value)) in map.iter().enumerate() { if i > 0 { request.push(','); } json_serialize_str(request, key); request.push(':'); json_serialize_str(request, value); } request.push('}'); } SearchValue::Int(v) => { let _ = write!(request, "{}", v); } SearchValue::Uint(v) => { let _ = write!(request, "{}", v); } SearchValue::Boolean(v) => { let _ = write!(request, "{}", v); } } } /*if id == 0 { debug_assert!(false, "Document is missing required ID fields"); }*/ let _ = write!(request, ",\"id\":{id}}}"); } fn json_serialize_str(request: &mut String, value: &str) { request.push('"'); for c in value.chars() { match c { '"' => request.push_str("\\\""), '\\' => request.push_str("\\\\"), '\n' => request.push_str("\\n"), '\r' => request.push_str("\\r"), '\t' => request.push_str("\\t"), '\u{0008}' => request.push_str("\\b"), // backspace '\u{000C}' => request.push_str("\\f"), // form feed _ => { if !c.is_control() { request.push(c); } else { let _ = write!(request, "\\u{:04x}", c as u32); } } } } request.push('"'); } impl SearchIndex { #[inline(always)] fn array_pos(&self) -> usize { match self { SearchIndex::Email => 0, SearchIndex::Calendar => 1, SearchIndex::Contacts => 2, SearchIndex::Tracing => 3, SearchIndex::File => 4, SearchIndex::InMemory => unreachable!(), } } } ================================================ FILE: crates/store/src/backend/memory/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::collections::hash_map::Entry; use ahash::AHashMap; use utils::{config::Config, glob::GlobMap}; use crate::{InMemoryStore, Stores, Value}; pub type StaticMemoryStore = GlobMap>; impl Stores { pub fn parse_static_stores(&mut self, config: &mut Config, is_reload: bool) { let mut lookups = AHashMap::new(); let mut errors = Vec::new(); for (key, value) in config.iterate_prefix("lookup") { if let Some((id, key)) = key .split_once('.') .filter(|(id, key)| !id.is_empty() && !key.is_empty()) { // Detect value type let value = if !value.is_empty() { let mut has_integers = false; let mut has_floats = false; let mut has_others = false; for (pos, ch) in value.as_bytes().iter().enumerate() { match ch { b'.' if !has_floats && has_integers => { has_floats = true; } b'0'..=b'9' => { has_integers = true; } b'-' if pos == 0 && value.len() > 1 => {} _ => { has_others = true; } } } if has_others { if value == "true" { Value::Integer(1.into()) } else if value == "false" { Value::Integer(0.into()) } else { Value::Text(value.to_string().into()) } } else if has_floats { value .parse() .map(Value::Float) .unwrap_or_else(|_| Value::Text(value.to_string().into())) } else { value .parse() .map(Value::Integer) .unwrap_or_else(|_| Value::Text(value.to_string().into())) } } else { Value::Text("".into()) }; // Add entry lookups .entry(id.to_string()) .or_insert_with(StaticMemoryStore::default) .insert(key, value); } else { errors.push(key.to_string()); } } for error in errors { config.new_parse_error(error, "Invalid lookup key format"); } for (id, store) in lookups { match self.in_memory_stores.entry(id) { Entry::Vacant(entry) => { entry.insert(InMemoryStore::Static(store.into())); } Entry::Occupied(e) if !is_reload => { config.new_build_error( ("lookup", e.key().as_str()), "An in-memory store with this id already exists", ); } _ => {} } } } } ================================================ FILE: crates/store/src/backend/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ #[cfg(feature = "azure")] pub mod azure; pub mod elastic; #[cfg(feature = "foundation")] pub mod foundationdb; pub mod fs; pub mod http; #[cfg(feature = "kafka")] pub mod kafka; pub mod meili; pub mod memory; #[cfg(feature = "mysql")] pub mod mysql; #[cfg(feature = "nats")] pub mod nats; #[cfg(feature = "postgres")] pub mod postgres; #[cfg(feature = "redis")] pub mod redis; #[cfg(feature = "rocks")] pub mod rocksdb; #[cfg(feature = "s3")] pub mod s3; #[cfg(feature = "sqlite")] pub mod sqlite; #[cfg(feature = "zenoh")] pub mod zenoh; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] pub mod composite; // SPDX-SnippetEnd pub const MAX_TOKEN_LENGTH: usize = (u8::MAX >> 1) as usize; pub const MAX_TOKEN_MASK: usize = MAX_TOKEN_LENGTH - 1; #[allow(dead_code)] fn deserialize_i64_le(key: &[u8], bytes: &[u8]) -> trc::Result { Ok(i64::from_le_bytes(bytes[..].try_into().map_err(|_| { trc::Error::corrupted_key(key, bytes.into(), trc::location!()) })?)) } ================================================ FILE: crates/store/src/backend/mysql/blob.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::ops::Range; use mysql_async::prelude::Queryable; use super::{MysqlStore, into_error}; impl MysqlStore { pub(crate) async fn get_blob( &self, key: &[u8], range: Range, ) -> trc::Result>> { let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; let s = conn .prep("SELECT v FROM t WHERE k = ?") .await .map_err(into_error)?; conn.exec_first::, _, _>(&s, (key,)) .await .map(|bytes| { if range.start == 0 && range.end == usize::MAX { bytes } else { bytes.map(|bytes| { bytes .get(range.start..std::cmp::min(bytes.len(), range.end)) .unwrap_or_default() .to_vec() }) } }) .map_err(into_error) } pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; let s = conn .prep("INSERT INTO t (k, v) VALUES (?, ?) ON DUPLICATE KEY UPDATE v = VALUES(v)") .await .map_err(into_error)?; conn.exec_drop(&s, (key, data)) .await .map_err(into_error) .map(|_| ()) } pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result { let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; let s = conn .prep("DELETE FROM t WHERE k = ?") .await .map_err(into_error)?; conn.exec_iter(&s, (key,)) .await .map_err(into_error) .map(|hits| hits.affected_rows() > 0) } } ================================================ FILE: crates/store/src/backend/mysql/lookup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use mysql_async::{Params, Row, prelude::Queryable}; use crate::{IntoRows, QueryResult, QueryType, Value}; use super::{MysqlStore, into_error}; impl MysqlStore { pub(crate) async fn sql_query( &self, query: &str, params: &[Value<'_>], ) -> trc::Result { let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; let s = conn.prep(query).await.map_err(into_error)?; let params = Params::Positional(params.iter().map(Into::into).collect()); match T::query_type() { QueryType::Execute => conn.exec_drop(s, params).await.map_or_else( |e| Err(into_error(e)), |_| Ok(T::from_exec(conn.affected_rows() as usize)), ), QueryType::Exists => conn .exec_first::(s, params) .await .map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_exists(r.is_some()))), QueryType::QueryOne => conn .exec_first::(s, params) .await .map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_query_one(r))), QueryType::QueryAll => conn .exec::(s, params) .await .map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_query_all(r))), } } } impl From> for mysql_async::Value { fn from(value: crate::Value) -> Self { match value { crate::Value::Integer(i) => mysql_async::Value::Int(i), crate::Value::Bool(b) => mysql_async::Value::Int(b as i64), crate::Value::Float(f) => mysql_async::Value::Double(f), crate::Value::Text(t) => mysql_async::Value::Bytes(t.into_owned().into_bytes()), crate::Value::Blob(b) => mysql_async::Value::Bytes(b.into_owned()), crate::Value::Null => mysql_async::Value::NULL, } } } impl From for crate::Value<'static> { fn from(value: mysql_async::Value) -> Self { match value { mysql_async::Value::Int(i) => Self::Integer(i), mysql_async::Value::UInt(i) => Self::Integer(i as i64), mysql_async::Value::Double(f) => Self::Float(f), mysql_async::Value::Bytes(b) => String::from_utf8(b).map_or_else( |e| Self::Blob(e.into_bytes().into()), |s| Self::Text(s.into()), ), mysql_async::Value::NULL => Self::Null, mysql_async::Value::Float(f) => Self::Float(f as f64), mysql_async::Value::Date(_, _, _, _, _, _, _) | mysql_async::Value::Time(_, _, _, _, _, _) => Self::Text(value.as_sql(true).into()), } } } impl IntoRows for Vec { fn into_rows(self) -> crate::Rows { crate::Rows { rows: self .into_iter() .map(|r| crate::Row { values: r .unwrap_raw() .into_iter() .flatten() .map(Into::into) .collect(), }) .collect(), } } fn into_named_rows(self) -> crate::NamedRows { crate::NamedRows { names: self .first() .map(|r| r.columns().iter().map(|c| c.name_str().into()).collect()) .unwrap_or_default(), rows: self .into_iter() .map(|r| crate::Row { values: r .unwrap_raw() .into_iter() .flatten() .map(Into::into) .collect(), }) .collect(), } } fn into_row(self) -> Option { unreachable!() } } impl IntoRows for Option { fn into_row(self) -> Option { self.map(|row| crate::Row { values: row .unwrap_raw() .into_iter() .flatten() .map(Into::into) .collect(), }) } fn into_rows(self) -> crate::Rows { unreachable!() } fn into_named_rows(self) -> crate::NamedRows { unreachable!() } } ================================================ FILE: crates/store/src/backend/mysql/main.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use mysql_async::{ Conn, OptsBuilder, Pool, PoolConstraints, PoolOpts, SslOpts, prelude::Queryable, }; use utils::config::{Config, utils::AsKey}; use crate::{ backend::mysql::MysqlSearchField, search::{ CalendarSearchField, ContactSearchField, EmailSearchField, SearchableField, TracingSearchField, }, *, }; use super::{MysqlStore, into_error}; impl MysqlStore { pub async fn open( config: &mut Config, prefix: impl AsKey, create_store_tables: bool, create_search_tables: bool, ) -> Option { let prefix = prefix.as_key(); let mut opts = OptsBuilder::default() .ip_or_hostname(config.value_require((&prefix, "host"))?.to_string()) .user(config.value((&prefix, "user")).map(|s| s.to_string())) .pass(config.value((&prefix, "password")).map(|s| s.to_string())) .db_name( config .value_require((&prefix, "database"))? .to_string() .into(), ) .max_allowed_packet(config.property((&prefix, "max-allowed-packet"))) .wait_timeout( config .property::>((&prefix, "timeout")) .unwrap_or_default() .map(|t| t.as_secs() as usize), ) .client_found_rows(true); if let Some(port) = config.property((&prefix, "port")) { opts = opts.tcp_port(port); } if config .property_or_default::((&prefix, "tls.enable"), "false") .unwrap_or_default() { let allow_invalid = config .property_or_default::((&prefix, "tls.allow-invalid-certs"), "false") .unwrap_or_default(); opts = opts.ssl_opts(Some( SslOpts::default() .with_danger_accept_invalid_certs(allow_invalid) .with_danger_skip_domain_validation(allow_invalid), )); } // Configure connection pool let mut pool_min = PoolConstraints::default().min(); let mut pool_max = PoolConstraints::default().max(); if let Some(n_size) = config .property::((&prefix, "pool.min-connections")) .filter(|&n| n > 0) { pool_min = n_size; } if let Some(n_size) = config .property::((&prefix, "pool.max-connections")) .filter(|&n| n > 0) { pool_max = n_size; } opts = opts.pool_opts( PoolOpts::default().with_constraints(PoolConstraints::new(pool_min, pool_max).unwrap()), ); let db = Self { conn_pool: Pool::new(opts), }; if create_store_tables && let Err(err) = db.create_storage_tables().await { config.new_build_error(prefix.as_str(), format!("Failed to create tables: {err}")); } if create_search_tables && let Err(err) = db.create_search_tables().await { config.new_build_warning( prefix.as_str(), format!("Failed to create search tables: {err}"), ); } Some(db) } pub(crate) async fn create_storage_tables(&self) -> trc::Result<()> { let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; for table in [ SUBSPACE_ACL, SUBSPACE_DIRECTORY, SUBSPACE_TASK_QUEUE, SUBSPACE_BLOB_EXTRA, SUBSPACE_BLOB_LINK, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_PROPERTY, SUBSPACE_SETTINGS, SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUEUE_EVENT, SUBSPACE_REPORT_OUT, SUBSPACE_REPORT_IN, SUBSPACE_LOGS, SUBSPACE_TELEMETRY_SPAN, SUBSPACE_TELEMETRY_METRIC, ] { let table = char::from(table); conn.query_drop(format!( "CREATE TABLE IF NOT EXISTS {table} ( k TINYBLOB, v MEDIUMBLOB NOT NULL, PRIMARY KEY (k(255)) ) ENGINE=InnoDB" )) .await .map_err(into_error)?; } conn.query_drop(format!( "CREATE TABLE IF NOT EXISTS {} ( k TINYBLOB, v LONGBLOB NOT NULL, PRIMARY KEY (k(255)) ) ENGINE=InnoDB", char::from(SUBSPACE_BLOBS), )) .await .map_err(into_error)?; for table in [SUBSPACE_INDEXES] { let table = char::from(table); conn.query_drop(format!( "CREATE TABLE IF NOT EXISTS {table} ( k BLOB, PRIMARY KEY (k(400)) ) ENGINE=InnoDB" )) .await .map_err(into_error)?; } for table in [SUBSPACE_COUNTER, SUBSPACE_QUOTA, SUBSPACE_IN_MEMORY_COUNTER] { conn.query_drop(format!( "CREATE TABLE IF NOT EXISTS {} ( k TINYBLOB, v BIGINT NOT NULL DEFAULT 0, PRIMARY KEY (k(255)) ) ENGINE=InnoDB", char::from(table) )) .await .map_err(into_error)?; } Ok(()) } pub(crate) async fn create_search_tables(&self) -> trc::Result<()> { let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; create_search_tables::(&mut conn).await?; create_search_tables::(&mut conn).await?; create_search_tables::(&mut conn).await?; //create_search_tables::(&mut conn).await?; create_search_tables::(&mut conn).await?; Ok(()) } } async fn create_search_tables( conn: &mut Conn, ) -> trc::Result<()> { let table_name = T::index().mysql_table(); let mut query = format!("CREATE TABLE IF NOT EXISTS {} (", table_name); // Add primary key columns let pkeys = T::primary_keys(); for pkey in pkeys { query.push_str(&format!("{} {}, ", pkey.column(), pkey.column_type())); } // Add other columns for field in T::all_fields() { query.push_str(&format!("{} {}, ", field.column(), field.column_type())); } // Add primary key constraint query.push_str("PRIMARY KEY ("); for (i, pkey) in pkeys.iter().enumerate() { if i > 0 { query.push_str(", "); } query.push_str(pkey.column()); } query.push_str(")) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"); conn.query_drop(&query).await.map_err(into_error)?; // Create indexes for field in T::all_fields() { if field.is_text() { let column_name = field.column(); let create_index_query = format!( "CREATE FULLTEXT INDEX fts_{table_name}_{column_name} ON {table_name}({column_name})", ); let _ = conn.query_drop(&create_index_query).await; } if field.is_indexed() { let column_name = field.column(); let create_index_query = format!( "CREATE INDEX idx_{table_name}_{column_name} ON {table_name}({column_name})", ); let _ = conn.query_drop(&create_index_query).await; } } Ok(()) } ================================================ FILE: crates/store/src/backend/mysql/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ search::{ CalendarSearchField, ContactSearchField, EmailSearchField, FileSearchField, SearchField, TracingSearchField, }, write::SearchIndex, }; use mysql_async::Pool; use std::fmt::Display; pub mod blob; pub mod lookup; pub mod main; pub mod read; pub mod search; pub mod write; pub struct MysqlStore { pub(crate) conn_pool: Pool, } #[inline(always)] fn into_error(err: impl Display) -> trc::Error { trc::StoreEvent::MysqlError.reason(err) } impl SearchIndex { pub fn mysql_table(&self) -> &'static str { match self { SearchIndex::Email => "s_email", SearchIndex::Calendar => "s_cal", SearchIndex::Contacts => "s_card", SearchIndex::File => "s_file", SearchIndex::Tracing => "s_trace", SearchIndex::InMemory => "", } } } trait MysqlSearchField { fn column(&self) -> &'static str; fn column_type(&self) -> &'static str; } impl MysqlSearchField for EmailSearchField { fn column(&self) -> &'static str { match self { EmailSearchField::From => "fadr", EmailSearchField::To => "tadr", EmailSearchField::Cc => "cc", EmailSearchField::Bcc => "bcc", EmailSearchField::Subject => "subj", EmailSearchField::Body => "body", EmailSearchField::Attachment => "atta", EmailSearchField::ReceivedAt => "rcvd", EmailSearchField::SentAt => "sent", EmailSearchField::Size => "size", EmailSearchField::HasAttachment => "hatt", EmailSearchField::Headers => "hdrs", } } fn column_type(&self) -> &'static str { match self { EmailSearchField::ReceivedAt | EmailSearchField::SentAt => "BIGINT", EmailSearchField::Size => "INT", EmailSearchField::HasAttachment => "BOOLEAN", EmailSearchField::Headers => "JSON", EmailSearchField::From => "TEXT", EmailSearchField::To => "TEXT", EmailSearchField::Cc => "TEXT", EmailSearchField::Bcc => "TEXT", EmailSearchField::Subject => "TEXT", EmailSearchField::Body => "MEDIUMTEXT", EmailSearchField::Attachment => "MEDIUMTEXT", } } } impl MysqlSearchField for CalendarSearchField { fn column(&self) -> &'static str { match self { CalendarSearchField::Title => "titl", CalendarSearchField::Description => "dscd", CalendarSearchField::Location => "locn", CalendarSearchField::Owner => "ownr", CalendarSearchField::Attendee => "atnd", CalendarSearchField::Start => "strt", CalendarSearchField::Uid => "uid", } } fn column_type(&self) -> &'static str { match self { CalendarSearchField::Start => "BIGINT NOT NULL", _ => "TEXT", } } } impl MysqlSearchField for ContactSearchField { fn column(&self) -> &'static str { match self { ContactSearchField::Member => "mmbr", ContactSearchField::Name => "name", ContactSearchField::Nickname => "nick", ContactSearchField::Organization => "orgn", ContactSearchField::Email => "eml", ContactSearchField::Phone => "phon", ContactSearchField::OnlineService => "olsv", ContactSearchField::Address => "addr", ContactSearchField::Note => "note", ContactSearchField::Kind => "kind", ContactSearchField::Uid => "uid", } } fn column_type(&self) -> &'static str { match self { ContactSearchField::Kind | ContactSearchField::Uid => "TEXT", _ => "TEXT", } } } impl MysqlSearchField for FileSearchField { fn column(&self) -> &'static str { match self { FileSearchField::Name => "name", FileSearchField::Content => "body", } } fn column_type(&self) -> &'static str { match self { FileSearchField::Name => "TEXT", FileSearchField::Content => "MEDIUMTEXT", } } } impl MysqlSearchField for TracingSearchField { fn column(&self) -> &'static str { match self { TracingSearchField::QueueId => "qid", TracingSearchField::EventType => "etyp", TracingSearchField::Keywords => "kwds", } } fn column_type(&self) -> &'static str { match self { TracingSearchField::EventType => "BIGINT", TracingSearchField::QueueId => "BIGINT", TracingSearchField::Keywords => "TEXT", } } } impl MysqlSearchField for SearchField { fn column(&self) -> &'static str { match self { SearchField::AccountId => "accid", SearchField::DocumentId => "docid", SearchField::Id => "id", SearchField::Email(field) => field.column(), SearchField::Calendar(field) => field.column(), SearchField::Contact(field) => field.column(), SearchField::File(field) => field.column(), SearchField::Tracing(field) => field.column(), } } fn column_type(&self) -> &'static str { match self { SearchField::AccountId => "INT NOT NULL", SearchField::DocumentId => "INT NOT NULL", SearchField::Id => "BIGINT NOT NULL", SearchField::Email(field) => field.column_type(), SearchField::Calendar(field) => field.column_type(), SearchField::Contact(field) => field.column_type(), SearchField::File(field) => field.column_type(), SearchField::Tracing(field) => field.column_type(), } } } ================================================ FILE: crates/store/src/backend/mysql/read.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{MysqlStore, into_error}; use crate::{Deserialize, IterateParams, Key, ValueKey, write::ValueClass}; use futures::TryStreamExt; use mysql_async::{Row, prelude::Queryable}; impl MysqlStore { pub(crate) async fn get_value(&self, key: impl Key) -> trc::Result> where U: Deserialize + 'static, { let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; let s = conn .prep(format!( "SELECT v FROM {} WHERE k = ?", char::from(key.subspace()) )) .await .map_err(into_error)?; let key = key.serialize(0); conn.exec_first::, _, _>(&s, (key,)) .await .map_err(into_error) .and_then(|r| { if let Some(r) = r { Ok(Some(U::deserialize_owned(r)?)) } else { Ok(None) } }) } pub(crate) async fn iterate( &self, params: IterateParams, mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, ) -> trc::Result<()> { let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; let table = char::from(params.begin.subspace()); let begin = params.begin.serialize(0); let end = params.end.serialize(0); let keys = if params.values { "k, v" } else { "k" }; let s = conn .prep(&match (params.first, params.ascending) { (true, true) => { format!( "SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC LIMIT 1" ) } (true, false) => { format!( "SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC LIMIT 1" ) } (false, true) => { format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC") } (false, false) => { format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC") } }) .await .map_err(into_error)?; let mut rows = conn .exec_stream::(&s, (begin, end)) .await .map_err(into_error)?; if params.values { while let Some(mut row) = rows.try_next().await.map_err(into_error)? { let value = row .take_opt::, _>(1) .unwrap_or_else(|| Ok(vec![])) .map_err(into_error)?; let key = row .take_opt::, _>(0) .unwrap_or_else(|| Ok(vec![])) .map_err(into_error)?; if !cb(&key, &value)? { break; } } } else { while let Some(mut row) = rows.try_next().await.map_err(into_error)? { if !cb( &row.take_opt::, _>(0) .unwrap_or_else(|| Ok(vec![])) .map_err(into_error)?, b"", )? { break; } } } Ok(()) } pub(crate) async fn get_counter( &self, key: impl Into> + Sync + Send, ) -> trc::Result { let key = key.into(); let table = char::from(key.subspace()); let key = key.serialize(0); let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; let s = conn .prep(format!("SELECT v FROM {table} WHERE k = ?")) .await .map_err(into_error)?; match conn.exec_first::(&s, (key,)).await { Ok(Some(num)) => Ok(num), Ok(None) => Ok(0), Err(e) => Err(into_error(e)), } } } ================================================ FILE: crates/store/src/backend/mysql/search.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ backend::{ MAX_TOKEN_LENGTH, mysql::{MysqlSearchField, MysqlStore, into_error}, }, search::{ IndexDocument, SearchComparator, SearchDocumentId, SearchFilter, SearchOperator, SearchQuery, SearchValue, }, write::SearchIndex, }; use mysql_async::{IsolationLevel, TxOpts, Value, prelude::Queryable}; use nlp::tokenizers::word::WordTokenizer; use std::fmt::Write; impl MysqlStore { pub async fn index(&self, documents: Vec) -> trc::Result<()> { let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; let mut tx_opts = TxOpts::default(); tx_opts .with_consistent_snapshot(false) .with_isolation_level(IsolationLevel::ReadCommitted); let mut trx = conn.start_transaction(tx_opts).await.map_err(into_error)?; for document in documents { let index = document.index; let primary_keys = index.primary_keys(); let all_fields = index.all_fields(); let mut fields = document.fields; let mut values = Vec::with_capacity(fields.len() + 2); let mut query = format!("INSERT INTO {} (", index.mysql_table()); for (i, field) in primary_keys.iter().chain(all_fields).enumerate() { if i > 0 { query.push(','); } query.push_str(field.column()); } query.push_str(") VALUES ("); for (i, field) in primary_keys.iter().chain(all_fields).enumerate() { if i > 0 { query.push(','); } if let Some(value) = fields.remove(field) { query.push('?'); values.push(value); } else { query.push_str("NULL"); } } query.push_str(") ON DUPLICATE KEY UPDATE "); for (i, field) in all_fields.iter().enumerate() { if i > 0 { query.push(','); } let column = field.column(); let _ = write!(&mut query, "{column} = VALUES({column})"); } let s = trx.prep(&query).await.map_err(into_error)?; trx.exec_drop(&s, values).await.map_err(into_error)?; } trx.commit().await.map_err(into_error) } pub async fn query( &self, index: SearchIndex, filters: &[SearchFilter], sort: &[SearchComparator], ) -> trc::Result> { let mut query = format!( "SELECT {} FROM {}", R::field().column(), index.mysql_table() ); let params = build_filter(&mut query, filters); if !sort.is_empty() { build_sort(&mut query, sort); } let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; let s = conn.prep(query).await.map_err(into_error)?; conn.exec::(s, params) .await .map(|r| r.into_iter().map(|r| R::from_u64(r as u64)).collect()) .map_err(into_error) } pub async fn unindex(&self, filter: SearchQuery) -> trc::Result { let mut query = format!("DELETE FROM {} ", filter.index.mysql_table()); let params = build_filter(&mut query, &filter.filters); let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; let s = conn.prep(query).await.map_err(into_error)?; conn.exec_drop(s, params) .await .map(|_| conn.affected_rows() as u64) .map_err(into_error) } } fn build_filter(query: &mut String, filters: &[SearchFilter]) -> Vec { if filters.is_empty() { return Vec::new(); } query.push_str(" WHERE "); let mut operator_stack = Vec::new(); let mut operator = &SearchFilter::And; let mut is_first = true; let mut values: Vec = Vec::new(); for filter in filters { match filter { SearchFilter::Operator { field, op, value } => { if !is_first { match operator { SearchFilter::And => query.push_str(" AND "), SearchFilter::Or => query.push_str(" OR "), _ => (), } } else { is_first = false; } if field.is_text() && matches!(op, SearchOperator::Equal | SearchOperator::Contains) { let (value, mode) = match (value, op) { (SearchValue::Text { value, .. }, SearchOperator::Equal) => { (Value::Bytes(format!("{value:?}").into_bytes()), "BOOLEAN") } (SearchValue::Text { value, .. }, ..) => { let mut text_query = String::with_capacity(value.len() + 1); for item in WordTokenizer::new(value, MAX_TOKEN_LENGTH) { if !text_query.is_empty() { text_query.push(' '); } text_query.push('+'); text_query.push_str(&item.word); } (Value::Bytes(text_query.into_bytes()), "BOOLEAN") } _ => { debug_assert!(false, "Invalid search value for text field"); continue; } }; let _ = write!(query, "MATCH({}) AGAINST(? IN {mode} MODE)", field.column()); values.push(value); } else if let SearchValue::KeyValues(kv) = value { let (key, value) = kv.iter().next().unwrap(); values.push(Value::Bytes(format!("$.{key:?}").into_bytes())); if !value.is_empty() { if op == &SearchOperator::Equal { let _ = write!(query, "JSON_EXTRACT({}, ?) = ?", field.column()); values.push(Value::Bytes(value.as_bytes().to_vec())); } else { let _ = write!(query, "JSON_EXTRACT({}, ?) LIKE ?", field.column(),); values.push(Value::Bytes(format!("%{value}%").into_bytes())); } } else { let _ = write!(query, "JSON_CONTAINS_PATH({}, 'one', ?)", field.column(),); } } else { query.push_str(field.column()); query.push(' '); op.write_mysql(query); values.push(to_mysql(value)); } } SearchFilter::And | SearchFilter::Or => { if !is_first { match operator { SearchFilter::And => query.push_str(" AND "), SearchFilter::Or => query.push_str(" OR "), _ => (), } } else { is_first = false; } operator_stack.push((operator, is_first)); operator = filter; is_first = true; query.push('('); } SearchFilter::Not => { if !is_first { match operator { SearchFilter::And => query.push_str(" AND "), SearchFilter::Or => query.push_str(" OR "), _ => (), } } else { is_first = false; } operator_stack.push((operator, is_first)); operator = &SearchFilter::And; is_first = true; query.push_str("NOT ("); } SearchFilter::End => { let p = operator_stack.pop().unwrap_or((&SearchFilter::And, true)); operator = p.0; is_first = p.1; query.push(')'); } SearchFilter::DocumentSet(_) => { debug_assert!( false, "DocumentSet filters are not supported in Postgres backend" ) } } } values } fn build_sort(query: &mut String, sort: &[SearchComparator]) { query.push_str(" ORDER BY "); for (i, comparator) in sort.iter().enumerate() { if i > 0 { query.push_str(", "); } match comparator { SearchComparator::Field { field, ascending } => { query.push_str(field.column()); if *ascending { query.push_str(" ASC"); } else { query.push_str(" DESC"); } } SearchComparator::DocumentSet { .. } | SearchComparator::SortedSet { .. } => { debug_assert!( false, "DocumentSet and SortedSet comparators are not supported " ); } } } } impl SearchOperator { fn write_mysql(&self, query: &mut String) { match self { SearchOperator::LowerThan => { let _ = write!(query, "< ?"); } SearchOperator::LowerEqualThan => { let _ = write!(query, "<= ?"); } SearchOperator::GreaterThan => { let _ = write!(query, "> ?"); } SearchOperator::GreaterEqualThan => { let _ = write!(query, ">= ?"); } SearchOperator::Equal => { let _ = write!(query, "= ?"); } SearchOperator::Contains => { let _ = write!(query, "LIKE '%' CONCAT('%', ?, '%')"); } } } } impl From for Value { fn from(value: SearchValue) -> Self { match value { SearchValue::Text { mut value, .. } => { // Truncate values larger than 16MB to avoid MySQL errors if value.len() > 16_777_214 { let pos = value.floor_char_boundary(16_777_214); value.truncate(pos); } Value::Bytes(value.into_bytes()) } SearchValue::KeyValues(vec_map) => serde_json::to_string(&vec_map) .map(|v| Value::Bytes(v.into_bytes())) .unwrap_or(Value::NULL), SearchValue::Int(i) => Value::Int(i), SearchValue::Uint(i) => Value::Int(i as i64), SearchValue::Boolean(b) => Value::Int(b as i64), } } } fn to_mysql(value: &SearchValue) -> Value { match value { SearchValue::Text { value, .. } => Value::Bytes(value.as_bytes().to_vec()), SearchValue::KeyValues(vec_map) => serde_json::to_string(&vec_map) .map(|v| Value::Bytes(v.into_bytes())) .unwrap_or(Value::NULL), SearchValue::Int(i) => Value::Int(*i), SearchValue::Uint(i) => Value::Int(*i as i64), SearchValue::Boolean(b) => Value::Int(*b as i64), } } ================================================ FILE: crates/store/src/backend/mysql/write.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{MysqlStore, into_error}; use crate::{ IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, write::{ AssignedIds, Batch, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, MergeResult, Operation, ValueClass, ValueOp, }, }; use ahash::AHashMap; use mysql_async::{Conn, Error, IsolationLevel, TxOpts, params, prelude::Queryable}; use rand::Rng; use std::time::{Duration, Instant}; #[derive(Debug)] enum CommitError { Mysql(mysql_async::Error), Internal(trc::Error), //Retry, } impl MysqlStore { pub(crate) async fn write(&self, mut batch: Batch<'_>) -> trc::Result { let start = Instant::now(); let mut retry_count = 0; let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; loop { let err = match self.write_trx(&mut conn, &mut batch).await { Ok(result) => { return Ok(result); } Err(err) => err, }; let _ = conn.query_drop("ROLLBACK;").await; match err { CommitError::Mysql(Error::Server(err)) if [1062, 1213].contains(&err.code) && retry_count < MAX_COMMIT_ATTEMPTS && start.elapsed() < MAX_COMMIT_TIME => {} /*CommitError::Retry => { if retry_count > MAX_COMMIT_ATTEMPTS || start.elapsed() > MAX_COMMIT_TIME { return Err(trc::StoreEvent::AssertValueFailed .into_err() .caused_by(trc::location!())); } }*/ CommitError::Mysql(err) => { return Err(into_error(err)); } CommitError::Internal(err) => { return Err(err); } } let backoff = rand::rng().random_range(50..=300); tokio::time::sleep(Duration::from_millis(backoff)).await; retry_count += 1; } } async fn write_trx( &self, conn: &mut Conn, batch: &mut Batch<'_>, ) -> Result { let has_changes = !batch.changes.is_empty(); let mut account_id = u32::MAX; let mut collection = u8::MAX; let mut document_id = u32::MAX; let mut change_id = 0u64; let mut asserted_values = AHashMap::new(); let mut tx_opts = TxOpts::default(); tx_opts .with_consistent_snapshot(false) .with_isolation_level(IsolationLevel::ReadCommitted); let mut trx = conn.start_transaction(tx_opts).await?; let mut result = AssignedIds::default(); if has_changes { for &account_id in batch.changes.keys() { let key = ValueClass::ChangeId.serialize(account_id, 0, 0, 0); let s = trx .prep(concat!( "INSERT INTO n (k, v) VALUES (:k, LAST_INSERT_ID(1)) ", "ON DUPLICATE KEY UPDATE v = LAST_INSERT_ID(v + 1)" )) .await?; trx.exec_drop(&s, params! {"k" => key}).await?; let s = trx.prep("SELECT LAST_INSERT_ID()").await?; let change_id = trx.exec_first::(&s, ()).await?.ok_or_else(|| { mysql_async::Error::Io(mysql_async::IoError::Io(std::io::Error::other( "LAST_INSERT_ID() did not return a value", ))) })?; result.push_change_id(account_id, change_id as u64); } } for op in batch.ops.iter_mut() { match op { Operation::AccountId { account_id: account_id_, } => { account_id = *account_id_; if has_changes { change_id = result.set_current_change_id(account_id)?; } } Operation::Collection { collection: collection_, } => { collection = u8::from(*collection_); } Operation::DocumentId { document_id: document_id_, } => { document_id = *document_id_; } Operation::Value { class, op } => { let key = class.serialize(account_id, collection, document_id, 0); let table = char::from(class.subspace(collection)); match op { ValueOp::Set(value) => { let exists = asserted_values.get(&key); let s = if let Some(exists) = exists { if *exists { trx.prep(format!("UPDATE {} SET v = :v WHERE k = :k", table)) .await? } else { trx.prep(format!( "INSERT INTO {} (k, v) VALUES (:k, :v)", table )) .await? } } else { trx .prep( format!("INSERT INTO {} (k, v) VALUES (:k, :v) ON DUPLICATE KEY UPDATE v = VALUES(v)", table), ) .await? }; match trx .exec_drop(&s, params! {"k" => key, "v" => &*value}) .await { Ok(_) => { if trx.affected_rows() == 0 { trx.rollback().await?; return Err(trc::StoreEvent::AssertValueFailed .into_err() .caused_by(trc::location!()) .into()); } } Err(err) => { trx.rollback().await?; return Err(err.into()); } } } ValueOp::SetFnc(set_op) => { let value = (set_op.fnc)(&set_op.params, &result)?; let exists = asserted_values.get(&key); let s = if let Some(exists) = exists { if *exists { trx.prep(format!("UPDATE {} SET v = :v WHERE k = :k", table)) .await? } else { trx.prep(format!( "INSERT INTO {} (k, v) VALUES (:k, :v)", table )) .await? } } else { trx .prep( format!("INSERT INTO {} (k, v) VALUES (:k, :v) ON DUPLICATE KEY UPDATE v = VALUES(v)", table), ) .await? }; match trx.exec_drop(&s, params! {"k" => key, "v" => &value}).await { Ok(_) => { if trx.affected_rows() == 0 { trx.rollback().await?; return Err(trc::StoreEvent::AssertValueFailed .into_err() .caused_by(trc::location!()) .into()); } } Err(err) => { trx.rollback().await?; return Err(err.into()); } } } ValueOp::MergeFnc(merge_op) => { let s = trx .prep(format!("SELECT v FROM {} WHERE k = ? FOR UPDATE", table)) .await?; let (exists, merge_result) = trx .exec_first::, _, _>(&s, (&key,)) .await? .map(|bytes| { (merge_op.fnc)(&merge_op.params, &result, Some(bytes.as_ref())) .map(|v| (true, v)) .map_err(CommitError::from) }) .unwrap_or_else(|| { (merge_op.fnc)(&merge_op.params, &result, None) .map(|v| (false, v)) .map_err(CommitError::from) })?; let s = if exists { trx.prep(format!("UPDATE {} SET v = :v WHERE k = :k", table)) .await? } else { trx.prep(format!("INSERT INTO {} (k, v) VALUES (:k, :v)", table)) .await? }; match merge_result { MergeResult::Update(value) => { if let Err(err) = trx.exec_drop(&s, params! {"k" => key, "v" => &value}).await { trx.rollback().await?; return Err(err.into()); } } MergeResult::Delete if exists => { // Update asserted value if let Some(exists) = asserted_values.get_mut(&key) { *exists = false; } let s = trx .prep(format!("DELETE FROM {} WHERE k = ?", table)) .await?; trx.exec_drop(&s, (key,)).await?; } _ => (), } } ValueOp::AtomicAdd(by) => { if *by >= 0 { let s = trx .prep(format!( concat!( "INSERT INTO {} (k, v) VALUES (?, ?) ", "ON DUPLICATE KEY UPDATE v = v + VALUES(v)" ), table )) .await?; trx.exec_drop(&s, (key, &*by)).await?; } else { let s = trx .prep(format!("UPDATE {table} SET v = v + ? WHERE k = ?")) .await?; trx.exec_drop(&s, (&*by, key)).await?; } } ValueOp::AddAndGet(by) => { let s = trx .prep(format!( concat!( "INSERT INTO {} (k, v) VALUES (:k, LAST_INSERT_ID(:v)) ", "ON DUPLICATE KEY UPDATE v = LAST_INSERT_ID(v + :v)" ), table )) .await?; trx.exec_drop(&s, params! {"k" => key, "v" => &*by}).await?; let s = trx.prep("SELECT LAST_INSERT_ID()").await?; result.push_counter_id( trx.exec_first::(&s, ()).await?.ok_or_else(|| { mysql_async::Error::Io(mysql_async::IoError::Io( std::io::Error::other( "LAST_INSERT_ID() did not return a value", ), )) })?, ); } ValueOp::Clear => { // Update asserted value if let Some(exists) = asserted_values.get_mut(&key) { *exists = false; } let s = trx .prep(format!("DELETE FROM {} WHERE k = ?", table)) .await?; trx.exec_drop(&s, (key,)).await?; } } } Operation::Index { field, key, set } => { let key = IndexKey { account_id, collection, document_id, field: *field, key: &*key, } .serialize(0); let s = if *set { trx.prep("INSERT IGNORE INTO i (k) VALUES (?)").await? } else { trx.prep("DELETE FROM i WHERE k = ?").await? }; trx.exec_drop(&s, (key,)).await?; } Operation::Log { collection, set } => { let key = LogKey { account_id, collection: u8::from(*collection), change_id, } .serialize(0); let s = trx .prep("INSERT INTO l (k, v) VALUES (?, ?) ON DUPLICATE KEY UPDATE v = VALUES(v)") .await?; trx.exec_drop(&s, (key, &*set)).await?; } Operation::AssertValue { class, assert_value, } => { let key = class.serialize(account_id, collection, document_id, 0); let table = char::from(class.subspace(collection)); let s = trx .prep(format!("SELECT v FROM {} WHERE k = ? FOR UPDATE", table)) .await?; let (exists, matches) = trx .exec_first::, _, _>(&s, (&key,)) .await? .map(|bytes| (true, assert_value.matches(&bytes))) .unwrap_or_else(|| (false, assert_value.is_none())); if !matches { trx.rollback().await?; return Err(trc::StoreEvent::AssertValueFailed .into_err() .caused_by(trc::location!()) .into()); } asserted_values.insert(key, exists); } } } trx.commit().await.map(|_| result).map_err(Into::into) } pub(crate) async fn purge_store(&self) -> trc::Result<()> { let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] { let s = conn .prep(format!("DELETE FROM {} WHERE v = 0", char::from(subspace),)) .await .map_err(into_error)?; conn.exec_drop(&s, ()).await.map_err(into_error)?; } Ok(()) } pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> { let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; let s = conn .prep(format!( "DELETE FROM {} WHERE k >= ? AND k < ?", char::from(from.subspace()), )) .await .map_err(into_error)?; conn.exec_drop(&s, (&from.serialize(0), &to.serialize(0))) .await .map_err(into_error) } } impl From for CommitError { fn from(err: trc::Error) -> Self { CommitError::Internal(err) } } impl From for CommitError { fn from(err: mysql_async::Error) -> Self { CommitError::Mysql(err) } } ================================================ FILE: crates/store/src/backend/nats/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use async_nats::Client; use utils::config::{Config, utils::AsKey}; pub mod pubsub; #[derive(Debug)] pub struct NatsPubSub { client: Client, } impl NatsPubSub { pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { let prefix = prefix.as_key(); let urls = config .values((&prefix, "address")) .map(|(_, v)| v.to_string()) .collect::>(); if urls.is_empty() { config.new_build_error((&prefix, "address"), "No Nats addresses specified"); return None; } let mut opts = async_nats::ConnectOptions::new() .max_reconnects( config .property_or_default::>((&prefix, "max-reconnects"), "false") .unwrap_or_default(), ) .connection_timeout( config .property_or_default((&prefix, "timeout.connection"), "5s") .unwrap_or_else(|| Duration::from_secs(5)), ) .request_timeout( config .property_or_default::>((&prefix, "timeout.request"), "10s") .unwrap_or_else(|| Some(Duration::from_secs(10))), ) .ping_interval( config .property_or_default((&prefix, "ping-interval"), "60s") .unwrap_or_else(|| Duration::from_secs(5)), ) .client_capacity( config .property_or_default((&prefix, "capacity.client"), "2048") .unwrap_or(2048), ) .subscription_capacity( config .property_or_default((&prefix, "capacity.subscription"), "65536") .unwrap_or(65536), ) .read_buffer_capacity( config .property_or_default((&prefix, "capacity.read-buffer"), "65535") .unwrap_or(65535), ) .require_tls( config .property_or_default((&prefix, "tls.enable"), "false") .unwrap_or_default(), ); if config .property_or_default((&prefix, "no-echo"), "true") .unwrap_or(true) { opts = opts.no_echo(); } if let (Some(user), Some(pass)) = ( config.value((&prefix, "user")), config.value((&prefix, "password")), ) { opts = opts.user_and_password(user.to_string(), pass.to_string()); } else if let Some(credentials) = config.value((&prefix, "credentials")) { opts = opts .credentials(credentials) .map_err(|err| { config.new_build_error( (&prefix, "credentials"), format!("Failed to parse Nats credentials: {}", err), ); }) .ok()?; } async_nats::connect_with_options(urls, opts) .await .map_err(|err| { config.new_build_error( (&prefix, "urls"), format!("Failed to connect to Nats: {}", err), ); }) .map(|client| NatsPubSub { client }) .ok() } } ================================================ FILE: crates/store/src/backend/nats/pubsub.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::NatsPubSub; use crate::dispatch::pubsub::{Msg, PubSubStream}; use futures::StreamExt; use trc::{ClusterEvent, Error, EventType}; pub struct NatsPubSubStream { subs: async_nats::Subscriber, } impl NatsPubSub { pub async fn publish(&self, topic: &'static str, message: Vec) -> trc::Result<()> { self.client .publish(topic, message.into()) .await .map_err(|err| Error::new(EventType::Cluster(ClusterEvent::PublisherError)).reason(err)) } pub async fn subscribe(&self, topic: &'static str) -> trc::Result { self.client .subscribe(topic) .await .map(|subs| PubSubStream::Nats(NatsPubSubStream { subs })) .map_err(|err| { Error::new(EventType::Cluster(ClusterEvent::SubscriberError)).reason(err) }) } } impl NatsPubSubStream { pub async fn next(&mut self) -> Option { self.subs.next().await.map(Msg::Nats) } } ================================================ FILE: crates/store/src/backend/postgres/blob.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::ops::Range; use crate::backend::postgres::into_pool_error; use super::{PostgresStore, into_error}; impl PostgresStore { pub(crate) async fn get_blob( &self, key: &[u8], range: Range, ) -> trc::Result>> { let conn = self.conn_pool.get().await.map_err(into_pool_error)?; let s = conn .prepare_cached("SELECT v FROM t WHERE k = $1") .await .map_err(into_error)?; conn.query_opt(&s, &[&key]) .await .and_then(|row| { if let Some(row) = row { Ok(Some(if range.start == 0 && range.end == usize::MAX { row.try_get::<_, Vec>(0)? } else { let bytes = row.try_get::<_, &[u8]>(0)?; bytes .get(range.start..std::cmp::min(bytes.len(), range.end)) .unwrap_or_default() .to_vec() })) } else { Ok(None) } }) .map_err(into_error) } pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { let conn = self.conn_pool.get().await.map_err(into_pool_error)?; let s = conn .prepare_cached( "INSERT INTO t (k, v) VALUES ($1, $2) ON CONFLICT (k) DO UPDATE SET v = EXCLUDED.v", ) .await .map_err(into_error)?; conn.execute(&s, &[&key, &data]) .await .map_err(into_error) .map(|_| ()) } pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result { let conn = self.conn_pool.get().await.map_err(into_pool_error)?; let s = conn .prepare_cached("DELETE FROM t WHERE k = $1") .await .map_err(into_error)?; conn.execute(&s, &[&key]) .await .map_err(into_error) .map(|hits| hits > 0) } } ================================================ FILE: crates/store/src/backend/postgres/lookup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{QueryResult, QueryType, backend::postgres::into_pool_error}; use bytes::BytesMut; use futures::{TryStreamExt, pin_mut}; use tokio_postgres::types::{FromSql, ToSql, Type}; use crate::IntoRows; use super::{PostgresStore, into_error}; impl PostgresStore { pub(crate) async fn sql_query( &self, query: &str, params_: &[crate::Value<'_>], ) -> trc::Result { let conn = self.conn_pool.get().await.map_err(into_pool_error)?; let s = conn.prepare_cached(query).await.map_err(into_error)?; let params = params_ .iter() .map(|v| v as &(dyn tokio_postgres::types::ToSql + Sync)) .collect::>(); match T::query_type() { QueryType::Execute => conn .execute(&s, params.as_slice()) .await .map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_exec(r as usize))), QueryType::Exists => { let rows = conn .query_raw(&s, params.into_iter()) .await .map_err(into_error)?; pin_mut!(rows); rows.try_next() .await .map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_exists(r.is_some()))) } QueryType::QueryOne => conn .query_opt(&s, params.as_slice()) .await .map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_query_one(r))), QueryType::QueryAll => conn .query(&s, params.as_slice()) .await .map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_query_all(r))), } } } impl ToSql for crate::Value<'_> { fn to_sql( &self, ty: &tokio_postgres::types::Type, out: &mut BytesMut, ) -> Result> where Self: Sized, { match self { crate::Value::Integer(v) => match *ty { Type::CHAR => (*v as i8).to_sql(ty, out), Type::INT2 => (*v as i16).to_sql(ty, out), Type::INT4 => (*v as i32).to_sql(ty, out), _ => v.to_sql(ty, out), }, crate::Value::Bool(v) => v.to_sql(ty, out), crate::Value::Float(v) => { if matches!(ty, &Type::FLOAT4) { (*v as f32).to_sql(ty, out) } else { v.to_sql(ty, out) } } crate::Value::Text(v) => v.to_sql(ty, out), crate::Value::Blob(v) => v.to_sql(ty, out), crate::Value::Null => None::.to_sql(ty, out), } } fn accepts(_: &tokio_postgres::types::Type) -> bool where Self: Sized, { true } fn to_sql_checked( &self, ty: &tokio_postgres::types::Type, out: &mut BytesMut, ) -> Result> { match self { crate::Value::Integer(v) => match *ty { Type::CHAR => (*v as i8).to_sql_checked(ty, out), Type::INT2 => (*v as i16).to_sql_checked(ty, out), Type::INT4 => (*v as i32).to_sql_checked(ty, out), _ => v.to_sql_checked(ty, out), }, crate::Value::Bool(v) => v.to_sql_checked(ty, out), crate::Value::Float(v) => { if matches!(ty, &Type::FLOAT4) { (*v as f32).to_sql_checked(ty, out) } else { v.to_sql_checked(ty, out) } } crate::Value::Text(v) => v.to_sql_checked(ty, out), crate::Value::Blob(v) => v.to_sql_checked(ty, out), crate::Value::Null => None::.to_sql_checked(ty, out), } } } impl IntoRows for Vec { fn into_rows(self) -> crate::Rows { crate::Rows { rows: self .into_iter() .map(|r| crate::Row { values: (0..r.len()) .map(|idx| r.try_get(idx).unwrap_or(crate::Value::Null)) .collect(), }) .collect(), } } fn into_named_rows(self) -> crate::NamedRows { crate::NamedRows { names: self .first() .map(|r| r.columns().iter().map(|c| c.name().to_string()).collect()) .unwrap_or_default(), rows: self .into_iter() .map(|r| crate::Row { values: (0..r.len()) .map(|idx| r.try_get(idx).unwrap_or(crate::Value::Null)) .collect(), }) .collect(), } } fn into_row(self) -> Option { unreachable!() } } impl IntoRows for Option { fn into_row(self) -> Option { self.map(|row| crate::Row { values: (0..row.len()) .map(|idx| row.try_get(idx).unwrap_or(crate::Value::Null)) .collect(), }) } fn into_rows(self) -> crate::Rows { unreachable!() } fn into_named_rows(self) -> crate::NamedRows { unreachable!() } } impl FromSql<'_> for crate::Value<'static> { fn from_sql( ty: &tokio_postgres::types::Type, raw: &'_ [u8], ) -> Result> { match ty { &Type::VARCHAR | &Type::TEXT | &Type::BPCHAR | &Type::NAME | &Type::UNKNOWN => { String::from_sql(ty, raw).map(|s| crate::Value::Text(s.into())) } &Type::BOOL => bool::from_sql(ty, raw).map(crate::Value::Bool), &Type::CHAR => i8::from_sql(ty, raw).map(|v| crate::Value::Integer(v as i64)), &Type::INT2 => i16::from_sql(ty, raw).map(|v| crate::Value::Integer(v as i64)), &Type::INT4 => i32::from_sql(ty, raw).map(|v| crate::Value::Integer(v as i64)), &Type::INT8 | &Type::OID => i64::from_sql(ty, raw).map(crate::Value::Integer), &Type::FLOAT4 | &Type::FLOAT8 => f64::from_sql(ty, raw).map(crate::Value::Float), ty if (ty.name() == "citext" || ty.name() == "ltree" || ty.name() == "lquery" || ty.name() == "ltxtquery") => { String::from_sql(ty, raw).map(|s| crate::Value::Text(s.into())) } _ => Vec::::from_sql(ty, raw).map(|b| crate::Value::Blob(b.into())), } } fn accepts(_: &tokio_postgres::types::Type) -> bool { true } } ================================================ FILE: crates/store/src/backend/postgres/main.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{PostgresStore, into_error}; use crate::{ backend::postgres::{PsqlSearchField, into_pool_error, tls::MakeRustlsConnect}, search::{ CalendarSearchField, ContactSearchField, EmailSearchField, SearchableField, TracingSearchField, }, *, }; use deadpool::managed::Object; use deadpool_postgres::{Config, Manager, ManagerConfig, PoolConfig, RecyclingMethod, Runtime}; use nlp::language::Language; use std::time::Duration; use tokio_postgres::NoTls; use utils::{config::utils::AsKey, rustls_client_config}; impl PostgresStore { pub async fn open( config: &mut utils::config::Config, prefix: impl AsKey, create_store_tables: bool, create_search_tables: bool, ) -> Option { let prefix = prefix.as_key(); let mut cfg = Config::new(); cfg.dbname = config .value_require((&prefix, "database"))? .to_string() .into(); cfg.host = config.value((&prefix, "host")).map(|s| s.to_string()); cfg.user = config.value((&prefix, "user")).map(|s| s.to_string()); cfg.password = config.value((&prefix, "password")).map(|s| s.to_string()); cfg.port = config.property((&prefix, "port")); cfg.connect_timeout = config .property::>((&prefix, "timeout")) .unwrap_or_default(); cfg.options = config.value((&prefix, "options")).map(|s| s.to_string()); cfg.manager = Some(ManagerConfig { recycling_method: RecyclingMethod::Clean, }); if let Some(max_conn) = config.property::((&prefix, "pool.max-connections")) { cfg.pool = PoolConfig::new(max_conn).into(); } let mut db = Self { conn_pool: if config .property_or_default::((&prefix, "tls.enable"), "false") .unwrap_or_default() { cfg.create_pool( Some(Runtime::Tokio1), MakeRustlsConnect::new(rustls_client_config( config .property_or_default((&prefix, "tls.allow-invalid-certs"), "false") .unwrap_or_default(), )), ) } else { cfg.create_pool(Some(Runtime::Tokio1), NoTls) } .map_err(|e| { config.new_build_error( prefix.as_str(), format!("Failed to create connection pool: {e}"), ) }) .ok()?, languages: config .properties::((&prefix, "languages")) .into_iter() .map(|(_, v)| v) .collect(), }; if db.languages.is_empty() { db.languages.insert(Language::English); } if create_store_tables && let Err(err) = db.create_storage_tables().await { config.new_build_error(prefix.as_str(), format!("Failed to create tables: {err}")); } if create_search_tables && let Err(err) = db.create_search_tables().await { config.new_build_warning( prefix.as_str(), format!("Failed to create search tables: {err}"), ); } Some(db) } pub(crate) async fn create_storage_tables(&self) -> trc::Result<()> { let conn = self.conn_pool.get().await.map_err(into_pool_error)?; for table in [ SUBSPACE_ACL, SUBSPACE_DIRECTORY, SUBSPACE_TASK_QUEUE, SUBSPACE_BLOB_EXTRA, SUBSPACE_BLOB_LINK, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_PROPERTY, SUBSPACE_SETTINGS, SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUEUE_EVENT, SUBSPACE_REPORT_OUT, SUBSPACE_REPORT_IN, SUBSPACE_LOGS, SUBSPACE_BLOBS, SUBSPACE_TELEMETRY_SPAN, SUBSPACE_TELEMETRY_METRIC, ] { let table = char::from(table); conn.execute( &format!( "CREATE TABLE IF NOT EXISTS {table} ( k BYTEA PRIMARY KEY, v BYTEA NOT NULL )" ), &[], ) .await .map_err(into_error)?; } for table in [SUBSPACE_INDEXES] { let table = char::from(table); conn.execute( &format!( "CREATE TABLE IF NOT EXISTS {table} ( k BYTEA PRIMARY KEY )" ), &[], ) .await .map_err(into_error)?; } for table in [SUBSPACE_COUNTER, SUBSPACE_QUOTA, SUBSPACE_IN_MEMORY_COUNTER] { conn.execute( &format!( "CREATE TABLE IF NOT EXISTS {} ( k BYTEA PRIMARY KEY, v BIGINT NOT NULL DEFAULT 0 )", char::from(table) ), &[], ) .await .map_err(into_error)?; } Ok(()) } pub(crate) async fn create_search_tables(&self) -> trc::Result<()> { let conn = self.conn_pool.get().await.map_err(into_pool_error)?; create_search_tables::(&conn).await?; create_search_tables::(&conn).await?; create_search_tables::(&conn).await?; //create_search_tables::(&conn).await?; create_search_tables::(&conn).await?; Ok(()) } } async fn create_search_tables( conn: &Object, ) -> trc::Result<()> { let table_name = T::index().psql_table(); let mut query = format!("CREATE TABLE IF NOT EXISTS {} (", table_name); // Add primary key columns let pkeys = T::primary_keys(); for pkey in pkeys { query.push_str(&format!("{} {}, ", pkey.column(), pkey.column_type())); } // Add other columns for field in T::all_fields() { query.push_str(&format!("{} {}", field.column(), field.column_type())); if let Some(sort_type) = field.sort_column_type() { query.push_str(&format!(", {} {}", field.sort_column().unwrap(), sort_type)); } query.push_str(", "); } // Add primary key constraint query.push_str("PRIMARY KEY ("); for (i, pkey) in pkeys.iter().enumerate() { if i > 0 { query.push_str(", "); } query.push_str(pkey.column()); } query.push_str("))"); conn.execute(&query, &[]).await.map_err(into_error)?; // Create indexes for field in T::all_fields() { if field.is_text() || field.is_json() { let column_name = field.column(); let create_index_query = format!( "CREATE INDEX IF NOT EXISTS gin_{table_name}_{column_name} ON {table_name} USING GIN({column_name})", ); conn.execute(&create_index_query, &[]) .await .map_err(into_error)?; } if field.is_indexed() { let column_name = field.sort_column().unwrap_or(field.column()); let create_index_query = format!( "CREATE INDEX IF NOT EXISTS idx_{table_name}_{column_name} ON {table_name}({column_name})", ); conn.execute(&create_index_query, &[]) .await .map_err(into_error)?; } } Ok(()) } ================================================ FILE: crates/store/src/backend/postgres/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ search::{ CalendarSearchField, ContactSearchField, EmailSearchField, FileSearchField, SearchField, TracingSearchField, }, write::SearchIndex, }; use ahash::AHashSet; use deadpool_postgres::Pool; use nlp::language::Language; pub mod blob; pub mod lookup; pub mod main; pub mod read; pub mod search; pub mod tls; pub mod write; pub struct PostgresStore { pub(crate) conn_pool: Pool, pub(crate) languages: AHashSet, } #[inline(always)] fn into_error(err: tokio_postgres::error::Error) -> trc::Error { let mut local_err = trc::StoreEvent::PostgresqlError.reason(err.to_string()); if let Some(db_err) = err.as_db_error() { local_err = local_err.code(db_err.code().code().to_string()); if let Some(detail) = db_err.detail() { local_err = local_err.details(detail.to_string()); } if let Some(hint) = db_err.hint() { local_err = local_err.caused_by(hint.to_string()); } } local_err } #[inline(always)] fn into_pool_error(err: deadpool::managed::PoolError) -> trc::Error { trc::StoreEvent::PostgresqlError.reason(err) } impl SearchIndex { pub fn psql_table(&self) -> &'static str { match self { SearchIndex::Email => "s_email", SearchIndex::Calendar => "s_cal", SearchIndex::Contacts => "s_card", SearchIndex::File => "s_file", SearchIndex::Tracing => "s_trace", SearchIndex::InMemory => "", } } } trait PsqlSearchField { fn column(&self) -> &'static str; fn column_type(&self) -> &'static str; fn sort_column_type(&self) -> Option<&'static str>; fn sort_column(&self) -> Option<&'static str>; } impl PsqlSearchField for EmailSearchField { fn column(&self) -> &'static str { match self { EmailSearchField::From => "fadr", EmailSearchField::To => "tadr", EmailSearchField::Cc => "cc", EmailSearchField::Bcc => "bcc", EmailSearchField::Subject => "subj", EmailSearchField::Body => "body", EmailSearchField::Attachment => "atta", EmailSearchField::ReceivedAt => "rcvd", EmailSearchField::SentAt => "sent", EmailSearchField::Size => "size", EmailSearchField::HasAttachment => "hatt", EmailSearchField::Headers => "hdrs", } } fn column_type(&self) -> &'static str { match self { EmailSearchField::ReceivedAt | EmailSearchField::SentAt => "BIGINT", EmailSearchField::Size => "INTEGER", EmailSearchField::HasAttachment => "BOOLEAN", EmailSearchField::Headers => "JSONB", _ => "TSVECTOR", } } fn sort_column_type(&self) -> Option<&'static str> { match self { EmailSearchField::From | EmailSearchField::To | EmailSearchField::Subject => { Some("TEXT") } #[cfg(feature = "test_mode")] EmailSearchField::Cc | EmailSearchField::Bcc => Some("TEXT"), _ => None, } } fn sort_column(&self) -> Option<&'static str> { match self { EmailSearchField::From => Some("s_fr"), EmailSearchField::To => Some("s_to"), EmailSearchField::Subject => Some("s_sj"), #[cfg(feature = "test_mode")] EmailSearchField::Bcc => Some("s_bc"), #[cfg(feature = "test_mode")] EmailSearchField::Cc => Some("s_cc"), _ => None, } } } impl PsqlSearchField for CalendarSearchField { fn column(&self) -> &'static str { match self { CalendarSearchField::Title => "titl", CalendarSearchField::Description => "dscd", CalendarSearchField::Location => "locn", CalendarSearchField::Owner => "ownr", CalendarSearchField::Attendee => "atnd", CalendarSearchField::Start => "strt", CalendarSearchField::Uid => "uid", } } fn column_type(&self) -> &'static str { match self { CalendarSearchField::Start => "BIGINT", CalendarSearchField::Uid => "TEXT", _ => "TSVECTOR", } } fn sort_column_type(&self) -> Option<&'static str> { None } fn sort_column(&self) -> Option<&'static str> { None } } impl PsqlSearchField for ContactSearchField { fn column(&self) -> &'static str { match self { ContactSearchField::Member => "mmbr", ContactSearchField::Name => "name", ContactSearchField::Nickname => "nick", ContactSearchField::Organization => "orgn", ContactSearchField::Email => "eml", ContactSearchField::Phone => "phon", ContactSearchField::OnlineService => "olsv", ContactSearchField::Address => "addr", ContactSearchField::Note => "note", ContactSearchField::Kind => "kind", ContactSearchField::Uid => "uid", } } fn column_type(&self) -> &'static str { match self { ContactSearchField::Kind | ContactSearchField::Uid => "TEXT", _ => "TSVECTOR", } } fn sort_column_type(&self) -> Option<&'static str> { None } fn sort_column(&self) -> Option<&'static str> { None } } impl PsqlSearchField for FileSearchField { fn column(&self) -> &'static str { match self { FileSearchField::Name => "name", FileSearchField::Content => "body", } } fn column_type(&self) -> &'static str { "TSVECTOR" } fn sort_column_type(&self) -> Option<&'static str> { None } fn sort_column(&self) -> Option<&'static str> { None } } impl PsqlSearchField for TracingSearchField { fn column(&self) -> &'static str { match self { TracingSearchField::QueueId => "qid", TracingSearchField::EventType => "etyp", TracingSearchField::Keywords => "kwds", } } fn column_type(&self) -> &'static str { match self { TracingSearchField::EventType => "BIGINT", TracingSearchField::QueueId => "BIGINT", TracingSearchField::Keywords => "TSVECTOR", } } fn sort_column_type(&self) -> Option<&'static str> { None } fn sort_column(&self) -> Option<&'static str> { None } } impl PsqlSearchField for SearchField { fn column(&self) -> &'static str { match self { SearchField::AccountId => "accid", SearchField::DocumentId => "docid", SearchField::Id => "id", SearchField::Email(field) => field.column(), SearchField::Calendar(field) => field.column(), SearchField::Contact(field) => field.column(), SearchField::File(field) => field.column(), SearchField::Tracing(field) => field.column(), } } fn column_type(&self) -> &'static str { match self { SearchField::AccountId => "INTEGER NOT NULL", SearchField::DocumentId => "INTEGER NOT NULL", SearchField::Id => "BIGINT NOT NULL", SearchField::Email(field) => field.column_type(), SearchField::Calendar(field) => field.column_type(), SearchField::Contact(field) => field.column_type(), SearchField::File(field) => field.column_type(), SearchField::Tracing(field) => field.column_type(), } } fn sort_column_type(&self) -> Option<&'static str> { match self { SearchField::Email(field) => field.sort_column_type(), SearchField::Calendar(field) => field.sort_column_type(), SearchField::Contact(field) => field.sort_column_type(), SearchField::File(field) => field.sort_column_type(), SearchField::Tracing(field) => field.sort_column_type(), SearchField::AccountId | SearchField::DocumentId | SearchField::Id => None, } } fn sort_column(&self) -> Option<&'static str> { match self { SearchField::Email(field) => field.sort_column(), SearchField::Calendar(field) => field.sort_column(), SearchField::Contact(field) => field.sort_column(), SearchField::File(field) => field.sort_column(), SearchField::Tracing(field) => field.sort_column(), SearchField::AccountId | SearchField::DocumentId | SearchField::Id => None, } } } ================================================ FILE: crates/store/src/backend/postgres/read.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{PostgresStore, into_error}; use crate::{ Deserialize, IterateParams, Key, ValueKey, backend::postgres::into_pool_error, write::ValueClass, }; use futures::{TryStreamExt, pin_mut}; impl PostgresStore { pub(crate) async fn get_value(&self, key: impl Key) -> trc::Result> where U: Deserialize + 'static, { let conn = self.conn_pool.get().await.map_err(into_pool_error)?; let s = conn .prepare_cached(&format!( "SELECT v FROM {} WHERE k = $1", char::from(key.subspace()) )) .await .map_err(into_error)?; let key = key.serialize(0); conn.query_opt(&s, &[&key]) .await .map_err(into_error) .and_then(|r| { if let Some(r) = r { Ok(Some(U::deserialize(r.get(0))?)) } else { Ok(None) } }) } pub(crate) async fn iterate( &self, params: IterateParams, mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, ) -> trc::Result<()> { let conn = self.conn_pool.get().await.map_err(into_pool_error)?; let table = char::from(params.begin.subspace()); let begin = params.begin.serialize(0); let end = params.end.serialize(0); let keys = if params.values { "k, v" } else { "k" }; let s = conn .prepare_cached(&match (params.first, params.ascending) { (true, true) => { format!( "SELECT {keys} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k ASC LIMIT 1" ) } (true, false) => { format!( "SELECT {keys} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k DESC LIMIT 1" ) } (false, true) => { format!("SELECT {keys} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k ASC") } (false, false) => { format!("SELECT {keys} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k DESC") } }) .await.map_err(into_error)?; let rows = conn .query_raw(&s, &[&begin, &end]) .await .map_err(into_error)?; pin_mut!(rows); if params.values { while let Some(row) = rows.try_next().await.map_err(into_error)? { let key = row.try_get::<_, &[u8]>(0).map_err(into_error)?; let value = row.try_get::<_, &[u8]>(1).map_err(into_error)?; if !cb(key, value)? { break; } } } else { while let Some(row) = rows.try_next().await.map_err(into_error)? { if !cb(row.try_get::<_, &[u8]>(0).map_err(into_error)?, b"")? { break; } } } Ok(()) } pub(crate) async fn get_counter( &self, key: impl Into> + Sync + Send, ) -> trc::Result { let key = key.into(); let table = char::from(key.subspace()); let key = key.serialize(0); let conn = self.conn_pool.get().await.map_err(into_pool_error)?; let s = conn .prepare_cached(&format!("SELECT v FROM {table} WHERE k = $1")) .await .map_err(into_error)?; match conn.query_opt(&s, &[&key]).await { Ok(Some(row)) => row.try_get(0).map_err(into_error), Ok(None) => Ok(0), Err(e) => Err(into_error(e)), } } } ================================================ FILE: crates/store/src/backend/postgres/search.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ backend::postgres::{PostgresStore, PsqlSearchField, into_error, into_pool_error}, search::{ IndexDocument, SearchComparator, SearchDocumentId, SearchFilter, SearchOperator, SearchQuery, SearchValue, }, write::SearchIndex, }; use nlp::language::Language; use std::fmt::Write; use tokio_postgres::{ IsolationLevel, types::{FromSql, ToSql, Type, WrongType}, }; impl PostgresStore { pub async fn index(&self, documents: Vec) -> trc::Result<()> { let mut conn = self.conn_pool.get().await.map_err(into_pool_error)?; let trx = conn .build_transaction() .isolation_level(IsolationLevel::ReadCommitted) .start() .await .map_err(into_error)?; for document in documents { let index = document.index; let primary_keys = index.primary_keys(); let all_fields = index.all_fields(); let fields = document.fields; let mut values = Vec::with_capacity(fields.len() + 2); let mut query = format!("INSERT INTO {} (", index.psql_table()); for (i, field) in primary_keys.iter().chain(all_fields).enumerate() { if i > 0 { query.push(','); } query.push_str(field.column()); if let Some(sort_column) = field.sort_column() { query.push(','); query.push_str(sort_column); } } query.push_str(") VALUES ("); for (i, field) in primary_keys.iter().chain(all_fields).enumerate() { if i > 0 { query.push(','); } if let Some(value) = fields.get(field) { let value_ref = format!("${}", values.len() + 1); let (text_len, language) = if let SearchValue::Text { value, language } = value { ( value.len(), if self.languages.contains(language) { pg_lang(language).unwrap_or("simple") } else { "simple" }, ) } else { (0, "simple") }; if field.is_text() { let _ = write!(&mut query, "to_tsvector('{language}',{value_ref})"); } else if text_len > 512 { query.push_str("left("); query.push_str(&value_ref); query.push_str(",512)"); } else { query.push_str(&value_ref); } if field.sort_column().is_some() { if text_len > 255 { query.push_str(",left("); query.push_str(&value_ref); query.push_str(",255)"); } else { query.push(','); query.push_str(&value_ref); } } values.push(value as &(dyn ToSql + Sync)); } else { query.push_str("NULL"); if field.sort_column().is_some() { query.push_str(",NULL"); } } } query.push_str(") ON CONFLICT ("); for (i, pkey) in primary_keys.iter().enumerate() { if i > 0 { query.push(','); } query.push_str(pkey.column()); } query.push_str(") DO UPDATE SET "); for (i, field) in all_fields.iter().enumerate() { if i > 0 { query.push(','); } let column = field.column(); let _ = write!(&mut query, "{column} = EXCLUDED.{column}"); } trx.execute(&query, &values).await.map_err(into_error)?; } trx.commit().await.map_err(into_error) } pub async fn query( &self, index: SearchIndex, filters: &[SearchFilter], sort: &[SearchComparator], ) -> trc::Result> { let mut query = format!("SELECT {} FROM {}", R::field().column(), index.psql_table()); let params = self.build_filter(&mut query, filters); if !sort.is_empty() { build_sort(&mut query, sort); } let conn = self.conn_pool.get().await.map_err(into_pool_error)?; let s = conn.prepare_cached(&query).await.map_err(into_error)?; conn.query(&s, params.as_slice()) .await .and_then(|rows| { rows.into_iter() .map(|row| row.try_get::<_, DocId>(0).map(|v| R::from_u64(v.0))) .collect::, _>>() }) .map_err(into_error) } pub async fn unindex(&self, filter: SearchQuery) -> trc::Result { debug_assert!(!filter.filters.is_empty()); let mut query = format!("DELETE FROM {} ", filter.index.psql_table()); let params = self.build_filter(&mut query, &filter.filters); let conn = self.conn_pool.get().await.map_err(into_pool_error)?; let s = conn.prepare_cached(&query).await.map_err(into_error)?; conn.execute(&s, params.as_slice()) .await .map_err(into_error) } fn build_filter<'x>( &self, query: &mut String, filters: &'x [SearchFilter], ) -> Vec<&'x (dyn ToSql + Sync)> { if filters.is_empty() { return Vec::new(); } query.push_str(" WHERE "); let mut operator_stack = Vec::new(); let mut operator = &SearchFilter::And; let mut is_first = true; let mut values = Vec::new(); for filter in filters { match filter { SearchFilter::Operator { field, op, value } => { if !is_first { match operator { SearchFilter::And => query.push_str(" AND "), SearchFilter::Or => query.push_str(" OR "), _ => (), } } else { is_first = false; } let value_pos = values.len() + 1; if field.is_text() && matches!(op, SearchOperator::Equal | SearchOperator::Contains) { query.push_str(field.column()); query.push(' '); let language = match &value { SearchValue::Text { language, .. } if self.languages.contains(language) => { pg_lang(language).unwrap_or("simple") } _ => "simple", }; let method = match op { SearchOperator::Equal => "phraseto_tsquery", _ => "plainto_tsquery", }; let _ = write!(query, "@@ {method}('{language}', ${value_pos})"); values.push(value as &(dyn ToSql + Sync)); } else if let SearchValue::KeyValues(kv) = value { query.push_str(field.column()); query.push(' '); let (key, value) = kv.iter().next().unwrap(); values.push(key as &(dyn ToSql + Sync)); if !value.is_empty() { let _ = write!(query, "->> ${value_pos} "); op.write_pqsql(query, values.len() + 1); values.push(value as &(dyn ToSql + Sync)); } else { let _ = write!(query, " ? ${value_pos}"); } } else { query.push_str(field.sort_column().unwrap_or(field.column())); query.push(' '); op.write_pqsql(query, value_pos); values.push(value as &(dyn ToSql + Sync)); } } SearchFilter::And | SearchFilter::Or => { if !is_first { match operator { SearchFilter::And => query.push_str(" AND "), SearchFilter::Or => query.push_str(" OR "), _ => (), } } else { is_first = false; } operator_stack.push((operator, is_first)); operator = filter; is_first = true; query.push('('); } SearchFilter::Not => { if !is_first { match operator { SearchFilter::And => query.push_str(" AND "), SearchFilter::Or => query.push_str(" OR "), _ => (), } } else { is_first = false; } operator_stack.push((operator, is_first)); operator = &SearchFilter::And; is_first = true; query.push_str("NOT ("); } SearchFilter::End => { let p = operator_stack.pop().unwrap_or((&SearchFilter::And, true)); operator = p.0; is_first = p.1; query.push(')'); } SearchFilter::DocumentSet(_) => { debug_assert!( false, "DocumentSet filters are not supported in Postgres backend" ) } } } values } } fn build_sort(query: &mut String, sort: &[SearchComparator]) { query.push_str(" ORDER BY "); for (i, comparator) in sort.iter().enumerate() { if i > 0 { query.push_str(", "); } match comparator { SearchComparator::Field { field, ascending } => { query.push_str(field.sort_column().unwrap_or(field.column())); if *ascending { query.push_str(" ASC"); } else { query.push_str(" DESC"); } } SearchComparator::DocumentSet { .. } | SearchComparator::SortedSet { .. } => { debug_assert!( false, "DocumentSet and SortedSet comparators are not supported " ); } } } } impl ToSql for SearchValue { fn to_sql( &self, ty: &tokio_postgres::types::Type, out: &mut bytes::BytesMut, ) -> Result> where Self: Sized, { match self { SearchValue::Text { value, .. } => { // Truncate large text fields to avoid Postgres errors (see https://www.postgresql.org/docs/current/textsearch-limitations.html) if value.len() > 650_000 { (&value[..value.floor_char_boundary(650_000)]).to_sql(ty, out) } else { value.to_sql(ty, out) } } SearchValue::Int(v) => match *ty { Type::INT4 => (*v as i32).to_sql(ty, out), _ => v.to_sql(ty, out), }, SearchValue::Uint(v) => match *ty { Type::INT4 => (*v as i32).to_sql(ty, out), _ => (*v as i64).to_sql(ty, out), }, SearchValue::Boolean(v) => v.to_sql(ty, out), SearchValue::KeyValues(kv) => { serde_json::to_value(kv).unwrap_or_default().to_sql(ty, out) } } } fn accepts(_: &tokio_postgres::types::Type) -> bool where Self: Sized, { true } fn to_sql_checked( &self, ty: &tokio_postgres::types::Type, out: &mut bytes::BytesMut, ) -> Result> { match self { SearchValue::Text { value, .. } => { // Truncate large text fields to avoid Postgres errors (see https://www.postgresql.org/docs/current/textsearch-limitations.html) if value.len() > 650_000 { (&value[..value.floor_char_boundary(650_000)]).to_sql_checked(ty, out) } else { value.to_sql_checked(ty, out) } } SearchValue::Int(v) => match *ty { Type::INT4 => (*v as i32).to_sql_checked(ty, out), _ => v.to_sql_checked(ty, out), }, SearchValue::Uint(v) => match *ty { Type::INT4 => (*v as i32).to_sql_checked(ty, out), _ => (*v as i64).to_sql_checked(ty, out), }, SearchValue::Boolean(v) => v.to_sql_checked(ty, out), SearchValue::KeyValues(kv) => serde_json::to_value(kv) .unwrap_or_default() .to_sql_checked(ty, out), } } } struct DocId(u64); impl FromSql<'_> for DocId { fn from_sql( ty: &tokio_postgres::types::Type, raw: &'_ [u8], ) -> Result> { match ty { &Type::INT4 => i32::from_sql(ty, raw).map(|v| DocId(v as u64)), &Type::INT8 | &Type::OID => i64::from_sql(ty, raw).map(|v| DocId(v as u64)), _ => Err(Box::new(WrongType::new::(ty.clone()))), } } fn accepts(typ: &Type) -> bool { matches!(typ, &Type::INT4 | &Type::INT8 | &Type::OID) } } impl SearchOperator { fn write_pqsql(&self, query: &mut String, value_pos: usize) { match self { SearchOperator::LowerThan => { let _ = write!(query, "< ${value_pos}"); } SearchOperator::LowerEqualThan => { let _ = write!(query, "<= ${value_pos}"); } SearchOperator::GreaterThan => { let _ = write!(query, "> ${value_pos}"); } SearchOperator::GreaterEqualThan => { let _ = write!(query, ">= ${value_pos}"); } SearchOperator::Equal => { let _ = write!(query, "= ${value_pos}"); } SearchOperator::Contains => { let _ = write!(query, "LIKE '%' || ${value_pos} || '%'"); } } } } #[inline(always)] fn pg_lang(lang: &Language) -> Option<&'static str> { match lang { Language::Esperanto => None, Language::English => Some("english"), Language::Russian => Some("russian"), Language::Mandarin => None, Language::Spanish => Some("spanish"), Language::Portuguese => Some("portuguese"), Language::Italian => Some("italian"), Language::Bengali => None, Language::French => Some("french"), Language::German => Some("german"), Language::Ukrainian => None, Language::Georgian => None, Language::Arabic => Some("arabic"), Language::Hindi => Some("hindi"), Language::Japanese => None, Language::Hebrew => None, Language::Yiddish => Some("yiddish"), Language::Polish => Some("polish"), Language::Amharic => None, Language::Javanese => None, Language::Korean => None, Language::Bokmal => Some("norwegian"), // Norwegian covers Bokmål Language::Danish => Some("danish"), Language::Swedish => Some("swedish"), Language::Finnish => Some("finnish"), Language::Turkish => Some("turkish"), Language::Dutch => Some("dutch"), Language::Hungarian => Some("hungarian"), Language::Czech => Some("czech"), Language::Greek => Some("greek"), Language::Bulgarian => None, Language::Belarusian => None, Language::Marathi => None, Language::Kannada => None, Language::Romanian => Some("romanian"), Language::Slovene => None, Language::Croatian => None, Language::Serbian => Some("serbian"), Language::Macedonian => None, Language::Lithuanian => Some("lithuanian"), Language::Latvian => None, Language::Estonian => None, Language::Tamil => Some("tamil"), Language::Vietnamese => None, Language::Urdu => None, Language::Thai => None, Language::Gujarati => None, Language::Uzbek => None, Language::Punjabi => None, Language::Azerbaijani => None, Language::Indonesian => Some("indonesian"), Language::Telugu => None, Language::Persian => None, Language::Malayalam => None, Language::Oriya => None, Language::Burmese => None, Language::Nepali => Some("nepali"), Language::Sinhalese => None, Language::Khmer => None, Language::Turkmen => None, Language::Akan => None, Language::Zulu => None, Language::Shona => None, Language::Afrikaans => None, Language::Latin => None, Language::Slovak => None, Language::Catalan => Some("catalan"), Language::Tagalog => None, Language::Armenian => Some("armenian"), Language::Unknown | Language::None => None, } } ================================================ FILE: crates/store/src/backend/postgres/tls.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ // Credits: https://github.com/jbg/tokio-postgres-rustls use std::{ convert::TryFrom, future::Future, io, pin::Pin, sync::Arc, task::{Context, Poll}, }; use futures::future::{FutureExt, TryFutureExt}; use ring::digest; use rustls::ClientConfig; use rustls_pki_types::ServerName; use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; use tokio_postgres::tls::{ChannelBinding, MakeTlsConnect, TlsConnect}; use tokio_rustls::{TlsConnector, client::TlsStream}; #[derive(Clone)] pub struct MakeRustlsConnect { config: Arc, } impl MakeRustlsConnect { pub fn new(config: ClientConfig) -> Self { Self { config: Arc::new(config), } } } impl MakeTlsConnect for MakeRustlsConnect where S: AsyncRead + AsyncWrite + Unpin + Send + 'static, { type Stream = RustlsStream; type TlsConnect = RustlsConnect; type Error = io::Error; fn make_tls_connect(&mut self, hostname: &str) -> io::Result { ServerName::try_from(hostname.to_string()) .map(|dns_name| { RustlsConnect(Some(RustlsConnectData { hostname: dns_name, connector: Arc::clone(&self.config).into(), })) }) .or(Ok(RustlsConnect(None))) } } pub struct RustlsConnect(Option); struct RustlsConnectData { hostname: ServerName<'static>, connector: TlsConnector, } impl TlsConnect for RustlsConnect where S: AsyncRead + AsyncWrite + Unpin + Send + 'static, { type Stream = RustlsStream; type Error = io::Error; type Future = Pin>> + Send>>; fn connect(self, stream: S) -> Self::Future { match self.0 { None => Box::pin(core::future::ready(Err(io::ErrorKind::InvalidInput.into()))), Some(c) => c .connector .connect(c.hostname, stream) .map_ok(|s| RustlsStream(Box::pin(s))) .boxed(), } } } pub struct RustlsStream(Pin>>); impl tokio_postgres::tls::TlsStream for RustlsStream where S: AsyncRead + AsyncWrite + Unpin, { fn channel_binding(&self) -> ChannelBinding { let (_, session) = self.0.get_ref(); match session.peer_certificates() { Some(certs) if !certs.is_empty() => { let sha256 = digest::digest(&digest::SHA256, certs[0].as_ref()); ChannelBinding::tls_server_end_point(sha256.as_ref().into()) } _ => ChannelBinding::none(), } } } impl AsyncRead for RustlsStream where S: AsyncRead + AsyncWrite + Unpin, { fn poll_read( mut self: Pin<&mut Self>, cx: &mut Context, buf: &mut ReadBuf<'_>, ) -> Poll> { self.0.as_mut().poll_read(cx, buf) } } impl AsyncWrite for RustlsStream where S: AsyncRead + AsyncWrite + Unpin, { fn poll_write( mut self: Pin<&mut Self>, cx: &mut Context, buf: &[u8], ) -> Poll> { self.0.as_mut().poll_write(cx, buf) } fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { self.0.as_mut().poll_flush(cx) } fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { self.0.as_mut().poll_shutdown(cx) } } ================================================ FILE: crates/store/src/backend/postgres/write.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{PostgresStore, into_error}; use crate::{ IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, backend::postgres::into_pool_error, write::{ AssignedIds, Batch, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, MergeResult, Operation, ValueClass, ValueOp, }, }; use ahash::AHashMap; use deadpool_postgres::Object; use rand::Rng; use std::time::{Duration, Instant}; use tokio_postgres::{IsolationLevel, error::SqlState}; #[derive(Debug)] enum CommitError { Postgres(tokio_postgres::Error), Internal(trc::Error), //Retry, } impl PostgresStore { pub(crate) async fn write(&self, mut batch: Batch<'_>) -> trc::Result { let mut conn = self.conn_pool.get().await.map_err(into_pool_error)?; let start = Instant::now(); let mut retry_count = 0; loop { match self.write_trx(&mut conn, &mut batch).await { Ok(result) => { return Ok(result); } Err(err) => { match err { CommitError::Postgres(err) => match err.code() { Some( &SqlState::T_R_SERIALIZATION_FAILURE | &SqlState::T_R_DEADLOCK_DETECTED, ) if retry_count < MAX_COMMIT_ATTEMPTS && start.elapsed() < MAX_COMMIT_TIME => {} Some(&SqlState::UNIQUE_VIOLATION) => { return Err(trc::StoreEvent::AssertValueFailed .into_err() .reason("Unique violation") .caused_by(trc::location!())); } _ => return Err(into_error(err)), }, CommitError::Internal(err) => return Err(err), /*CommitError::Retry => { if retry_count > MAX_COMMIT_ATTEMPTS || start.elapsed() > MAX_COMMIT_TIME { return Err(trc::StoreEvent::AssertValueFailed .into_err() .caused_by(trc::location!())); } }*/ } let backoff = rand::rng().random_range(50..=300); tokio::time::sleep(Duration::from_millis(backoff)).await; retry_count += 1; } } } } async fn write_trx( &self, conn: &mut Object, batch: &mut Batch<'_>, ) -> Result { let mut account_id = u32::MAX; let mut collection = u8::MAX; let mut document_id = u32::MAX; let mut change_id = 0u64; let mut asserted_values = AHashMap::new(); let trx = conn .build_transaction() .isolation_level(IsolationLevel::ReadCommitted) .start() .await?; let mut result = AssignedIds::default(); let has_changes = !batch.changes.is_empty(); if has_changes { for &account_id in batch.changes.keys() { let key = ValueClass::ChangeId.serialize(account_id, 0, 0, 0); let s = trx .prepare_cached(concat!( "INSERT INTO n (k, v) VALUES ($1, 1) ", "ON CONFLICT(k) DO UPDATE SET v = n.v + 1 RETURNING v" )) .await?; let change_id = trx .query_one(&s, &[&key]) .await .and_then(|row| row.try_get::<_, i64>(0))?; result.push_change_id(account_id, change_id as u64); } } for op in batch.ops.iter_mut() { match op { Operation::AccountId { account_id: account_id_, } => { account_id = *account_id_; if has_changes { change_id = result.set_current_change_id(account_id)?; } } Operation::Collection { collection: collection_, } => { collection = u8::from(*collection_); } Operation::DocumentId { document_id: document_id_, } => { document_id = *document_id_; } Operation::Value { class, op } => { let key = class.serialize(account_id, collection, document_id, 0); let table = char::from(class.subspace(collection)); match op { ValueOp::Set(value) => { let s = if let Some(exists) = asserted_values.get(&key) { if *exists { trx.prepare_cached(&format!( "UPDATE {} SET v = $2 WHERE k = $1", table )) .await? } else { trx.prepare_cached(&format!( "INSERT INTO {} (k, v) VALUES ($1, $2)", table )) .await? } } else { trx.prepare_cached(&format!( concat!( "INSERT INTO {} (k, v) VALUES ($1, $2) ", "ON CONFLICT (k) DO UPDATE SET v = EXCLUDED.v" ), table )) .await? }; if trx.execute(&s, &[&key, &(*value)]).await? == 0 { return Err(trc::StoreEvent::AssertValueFailed .into_err() .caused_by(trc::location!()) .into()); } } ValueOp::SetFnc(set_op) => { let value = (set_op.fnc)(&set_op.params, &result)?; let s = if let Some(exists) = asserted_values.get(&key) { if *exists { trx.prepare_cached(&format!( "UPDATE {} SET v = $2 WHERE k = $1", table )) .await? } else { trx.prepare_cached(&format!( "INSERT INTO {} (k, v) VALUES ($1, $2)", table )) .await? } } else { trx.prepare_cached(&format!( concat!( "INSERT INTO {} (k, v) VALUES ($1, $2) ", "ON CONFLICT (k) DO UPDATE SET v = EXCLUDED.v" ), table )) .await? }; if trx.execute(&s, &[&key, &value]).await? == 0 { return Err(trc::StoreEvent::AssertValueFailed .into_err() .caused_by(trc::location!()) .into()); } } ValueOp::MergeFnc(merge_op) => { let s = trx .prepare_cached(&format!( "SELECT v FROM {} WHERE k = $1 FOR UPDATE", table )) .await?; let (exists, merge_result) = trx .query_opt(&s, &[&key]) .await? .map(|row| { row.try_get::<_, &[u8]>(0) .map_err(CommitError::from) .and_then(|v| { (merge_op.fnc)(&merge_op.params, &result, Some(v)) .map(|v| (true, v)) .map_err(CommitError::from) }) }) .unwrap_or_else(|| { (merge_op.fnc)(&merge_op.params, &result, None) .map(|v| (false, v)) .map_err(CommitError::from) })?; match merge_result { MergeResult::Update(value) => { let s = if exists { trx.prepare_cached(&format!( "UPDATE {} SET v = $2 WHERE k = $1", table )) .await? } else { trx.prepare_cached(&format!( "INSERT INTO {} (k, v) VALUES ($1, $2)", table )) .await? }; trx.execute(&s, &[&key, &value]).await?; } MergeResult::Delete if exists => { let s = trx .prepare_cached(&format!( "DELETE FROM {} WHERE k = $1", table )) .await?; trx.execute(&s, &[&key]).await?; // Update asserted value if let Some(exists) = asserted_values.get_mut(&key) { *exists = false; } } _ => (), } } ValueOp::AtomicAdd(by) => { if *by >= 0 { let s = trx .prepare_cached(&format!( concat!( "INSERT INTO {} (k, v) VALUES ($1, $2) ", "ON CONFLICT(k) DO UPDATE SET v = {}.v + EXCLUDED.v" ), table, table )) .await?; trx.execute(&s, &[&key, &*by]).await?; } else { let s = trx .prepare_cached(&format!( "UPDATE {table} SET v = v + $1 WHERE k = $2" )) .await?; trx.execute(&s, &[&*by, &key]).await?; } } ValueOp::AddAndGet(by) => { let s = trx .prepare_cached(&format!( concat!( "INSERT INTO {} (k, v) VALUES ($1, $2) ", "ON CONFLICT(k) DO UPDATE SET v = {}.v + EXCLUDED.v RETURNING v" ), table, table )) .await?; result.push_counter_id( trx.query_one(&s, &[&key, &*by]) .await .and_then(|row| row.try_get::<_, i64>(0))?, ); } ValueOp::Clear => { let s = trx .prepare_cached(&format!("DELETE FROM {} WHERE k = $1", table)) .await?; trx.execute(&s, &[&key]).await?; // Update asserted value if let Some(exists) = asserted_values.get_mut(&key) { *exists = false; } } } } Operation::Index { field, key, set } => { let key = IndexKey { account_id, collection, document_id, field: *field, key: &*key, } .serialize(0); let s = if *set { trx.prepare_cached( "INSERT INTO i (k) VALUES ($1) ON CONFLICT (k) DO NOTHING", ) .await? } else { trx.prepare_cached("DELETE FROM i WHERE k = $1").await? }; trx.execute(&s, &[&key]).await?; } Operation::Log { collection, set } => { let key = LogKey { account_id, collection: u8::from(*collection), change_id, } .serialize(0); let s = trx .prepare_cached(concat!( "INSERT INTO l (k, v) VALUES ($1, $2) ", "ON CONFLICT (k) DO UPDATE SET v = EXCLUDED.v" )) .await?; trx.execute(&s, &[&key, &*set]).await?; } Operation::AssertValue { class, assert_value, } => { let key = class.serialize(account_id, collection, document_id, 0); let table = char::from(class.subspace(collection)); let s = trx .prepare_cached(&format!("SELECT v FROM {} WHERE k = $1 FOR UPDATE", table)) .await?; let (exists, matches) = trx .query_opt(&s, &[&key]) .await? .map(|row| { row.try_get::<_, &[u8]>(0) .map_or((true, false), |v| (true, assert_value.matches(v))) }) .unwrap_or_else(|| (false, assert_value.is_none())); if !matches { return Err(trc::StoreEvent::AssertValueFailed .into_err() .caused_by(trc::location!()) .into()); } asserted_values.insert(key, exists); } } } trx.commit().await.map(|_| result).map_err(Into::into) } pub(crate) async fn purge_store(&self) -> trc::Result<()> { let conn = self.conn_pool.get().await.map_err(into_pool_error)?; for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] { let s = conn .prepare_cached(&format!("DELETE FROM {} WHERE v = 0", char::from(subspace),)) .await .map_err(into_error)?; conn.execute(&s, &[]) .await .map(|_| ()) .map_err(into_error)? } Ok(()) } pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> { let conn = self.conn_pool.get().await.map_err(into_pool_error)?; let s = conn .prepare_cached(&format!( "DELETE FROM {} WHERE k >= $1 AND k < $2", char::from(from.subspace()), )) .await .map_err(into_error)?; conn.execute(&s, &[&from.serialize(0), &to.serialize(0)]) .await .map(|_| ()) .map_err(into_error) } } impl From for CommitError { fn from(err: trc::Error) -> Self { CommitError::Internal(err) } } impl From for CommitError { fn from(err: tokio_postgres::Error) -> Self { CommitError::Postgres(err) } } ================================================ FILE: crates/store/src/backend/redis/lookup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use redis::AsyncCommands; use crate::Deserialize; use super::{RedisPool, RedisStore, into_error}; impl RedisStore { pub async fn key_set(&self, key: &[u8], value: &[u8], expires: Option) -> trc::Result<()> { match &self.pool { RedisPool::Single(pool) => { self.key_set_( pool.get().await.map_err(into_error)?.as_mut(), key, value, expires, ) .await } RedisPool::Cluster(pool) => { self.key_set_( pool.get().await.map_err(into_error)?.as_mut(), key, value, expires, ) .await } } } pub async fn key_incr(&self, key: &[u8], value: i64, expires: Option) -> trc::Result { match &self.pool { RedisPool::Single(pool) => { self.key_incr_( pool.get().await.map_err(into_error)?.as_mut(), key, value, expires, ) .await } RedisPool::Cluster(pool) => { self.key_incr_( pool.get().await.map_err(into_error)?.as_mut(), key, value, expires, ) .await } } } pub async fn key_delete(&self, key: &[u8]) -> trc::Result<()> { match &self.pool { RedisPool::Single(pool) => { self.key_delete_(pool.get().await.map_err(into_error)?.as_mut(), key) .await } RedisPool::Cluster(pool) => { self.key_delete_(pool.get().await.map_err(into_error)?.as_mut(), key) .await } } } pub async fn key_delete_prefix(&self, prefix: &[u8]) -> trc::Result<()> { match &self.pool { RedisPool::Single(pool) => { self.key_delete_prefix_(pool.get().await.map_err(into_error)?.as_mut(), prefix) .await } RedisPool::Cluster(pool) => { self.key_delete_prefix_(pool.get().await.map_err(into_error)?.as_mut(), prefix) .await } } } pub async fn key_get( &self, key: &[u8], ) -> trc::Result> { match &self.pool { RedisPool::Single(pool) => { self.key_get_(pool.get().await.map_err(into_error)?.as_mut(), key) .await } RedisPool::Cluster(pool) => { self.key_get_(pool.get().await.map_err(into_error)?.as_mut(), key) .await } } } pub async fn counter_get(&self, key: &[u8]) -> trc::Result { match &self.pool { RedisPool::Single(pool) => { self.counter_get_(pool.get().await.map_err(into_error)?.as_mut(), key) .await } RedisPool::Cluster(pool) => { self.counter_get_(pool.get().await.map_err(into_error)?.as_mut(), key) .await } } } pub async fn key_exists(&self, key: &[u8]) -> trc::Result { match &self.pool { RedisPool::Single(pool) => { self.key_exists_(pool.get().await.map_err(into_error)?.as_mut(), key) .await } RedisPool::Cluster(pool) => { self.key_exists_(pool.get().await.map_err(into_error)?.as_mut(), key) .await } } } async fn key_get_( &self, conn: &mut impl AsyncCommands, key: &[u8], ) -> trc::Result> { if let Some(value) = redis::cmd("GET") .arg(key) .query_async::>>(conn) .await .map_err(into_error)? { T::deserialize_owned(value).map(Some) } else { Ok(None) } } async fn counter_get_(&self, conn: &mut impl AsyncCommands, key: &[u8]) -> trc::Result { redis::cmd("GET") .arg(key) .query_async::>(conn) .await .map(|x| x.unwrap_or(0)) .map_err(into_error) } async fn key_exists_(&self, conn: &mut impl AsyncCommands, key: &[u8]) -> trc::Result { conn.exists(key).await.map_err(into_error) } async fn key_set_( &self, conn: &mut impl AsyncCommands, key: &[u8], value: &[u8], expires: Option, ) -> trc::Result<()> { if let Some(expires) = expires { conn.set_ex(key, value, expires).await.map_err(into_error) } else { conn.set(key, value).await.map_err(into_error) } } async fn key_incr_( &self, conn: &mut impl AsyncCommands, key: &[u8], value: i64, expires: Option, ) -> trc::Result { if let Some(expires) = expires { redis::pipe() .atomic() .incr(key, value) .expire(key, expires as i64) .ignore() .query_async::>(conn) .await .map_err(into_error) .map(|v| v.first().copied().unwrap_or(0)) } else { conn.incr(key, value).await.map_err(into_error) } } async fn key_delete_(&self, conn: &mut impl AsyncCommands, key: &[u8]) -> trc::Result<()> { conn.del(key).await.map_err(into_error) } async fn key_delete_prefix_( &self, conn: &mut impl AsyncCommands, prefix: &[u8], ) -> trc::Result<()> { let mut pattern = Vec::with_capacity(prefix.len() + 1); pattern.extend_from_slice(prefix); pattern.push(b'*'); let mut cursor = 0; loop { let (new_cursor, keys): (u64, Vec>) = redis::cmd("SCAN") .cursor_arg(cursor) .arg("MATCH") .arg(&pattern) .arg("COUNT") .arg(100) .query_async(conn) .await .map_err(into_error)?; if !keys.is_empty() { conn.del::<_, ()>(&keys).await.map_err(into_error)?; } if new_cursor != 0 { cursor = new_cursor; } else { return Ok(()); } } } } ================================================ FILE: crates/store/src/backend/redis/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{fmt::Display, time::Duration}; use deadpool::{ Runtime, managed::{Manager, Pool}, }; use redis::{ Client, ProtocolVersion, cluster::{ClusterClient, ClusterClientBuilder}, }; use utils::config::{Config, utils::AsKey}; pub mod lookup; pub mod pool; pub mod pubsub; #[derive(Debug)] pub struct RedisStore { pool: RedisPool, } struct RedisConnectionManager { client: Client, timeout: Duration, } struct RedisClusterConnectionManager { client: ClusterClient, timeout: Duration, } enum RedisPool { Single(Pool), Cluster(Pool), } impl RedisStore { pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { let prefix = prefix.as_key(); let urls = config .values((&prefix, "urls")) .map(|(_, v)| v.to_string()) .collect::>(); if urls.is_empty() { config.new_build_error((&prefix, "urls"), "No Redis URLs specified"); return None; } Some( match config.value((&prefix, "redis-type")).unwrap_or("single") { "single" => { let client = Client::open(urls.into_iter().next().unwrap()) .map_err(|err| { config.new_build_error( prefix.as_str(), format!("Failed to open Redis client: {err:?}"), ) }) .ok()?; let timeout = config .property_or_default((&prefix, "timeout"), "10s") .unwrap_or_default(); Self { pool: RedisPool::Single( build_pool(config, &prefix, RedisConnectionManager { client, timeout }) .map_err(|err| { config.new_build_error( prefix.as_str(), format!("Failed to build Redis pool: {err:?}"), ) }) .ok()?, ), } } "cluster" => { let mut builder = ClusterClientBuilder::new(urls.into_iter()); if let Some(value) = config.property((&prefix, "user")) { builder = builder.username(value); } if let Some(value) = config.property((&prefix, "password")) { builder = builder.password(value); } if let Some(value) = config.property((&prefix, "retry.total")) { builder = builder.retries(value); } if let Some(value) = config .property::>((&prefix, "retry.max-wait")) .unwrap_or_default() { builder = builder.max_retry_wait(value.as_millis() as u64); } if let Some(value) = config .property::>((&prefix, "retry.min-wait")) .unwrap_or_default() { builder = builder.min_retry_wait(value.as_millis() as u64); } if let Some(true) = config.property::((&prefix, "read-from-replicas")) { builder = builder.read_from_replicas(); } if config .value((&prefix, "protocol-version")) .unwrap_or("resp2") == "resp3" { builder = builder.use_protocol(ProtocolVersion::RESP3); } let client = builder .build() .map_err(|err| { config.new_build_error( prefix.as_str(), format!("Failed to open Redis client: {err:?}"), ) }) .ok()?; let timeout = config .property_or_default::((&prefix, "timeout"), "10s") .unwrap_or_else(|| Duration::from_secs(10)); Self { pool: RedisPool::Cluster( build_pool( config, &prefix, RedisClusterConnectionManager { client, timeout }, ) .map_err(|err| { config.new_build_error( prefix.as_str(), format!("Failed to build Redis pool: {err:?}"), ) }) .ok()?, ), } } invalid => { let err = format!("Invalid Redis type {invalid:?}"); config.new_parse_error((&prefix, "redis-type"), err); return None; } }, ) } } fn build_pool( config: &mut Config, prefix: &str, manager: M, ) -> Result, String> { Pool::builder(manager) .runtime(Runtime::Tokio1) .max_size( config .property_or_default((prefix, "pool.max-connections"), "10") .unwrap_or(10), ) .create_timeout( config .property_or_default::>((prefix, "pool.create-timeout"), "30s") .unwrap_or_default(), ) .wait_timeout( config .property_or_default::>((prefix, "pool.wait-timeout"), "30s") .unwrap_or_default(), ) .recycle_timeout( config .property_or_default::>((prefix, "pool.recycle-timeout"), "30s") .unwrap_or_default(), ) .build() .map_err(|err| { format!( "Failed to build pool for {prefix:?}: {err}", prefix = prefix, err = err ) }) } #[inline(always)] fn into_error(err: impl Display) -> trc::Error { trc::StoreEvent::RedisError.reason(err) } impl std::fmt::Debug for RedisPool { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Single(_) => f.debug_tuple("Single").finish(), Self::Cluster(_) => f.debug_tuple("Cluster").finish(), } } } ================================================ FILE: crates/store/src/backend/redis/pool.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use deadpool::managed; use redis::{ aio::{ConnectionLike, MultiplexedConnection}, cluster_async::ClusterConnection, }; use super::{RedisClusterConnectionManager, RedisConnectionManager, into_error}; impl managed::Manager for RedisConnectionManager { type Type = MultiplexedConnection; type Error = trc::Error; async fn create(&self) -> Result { match tokio::time::timeout(self.timeout, self.client.get_multiplexed_tokio_connection()) .await { Ok(conn) => conn.map_err(into_error), Err(_) => Err(trc::StoreEvent::RedisError.ctx(trc::Key::Details, "Connection Timeout")), } } async fn recycle( &self, conn: &mut MultiplexedConnection, _: &managed::Metrics, ) -> managed::RecycleResult { conn.req_packed_command(&redis::cmd("PING")) .await .map(|_| ()) .map_err(|err| managed::RecycleError::Backend(into_error(err))) } } impl managed::Manager for RedisClusterConnectionManager { type Type = ClusterConnection; type Error = trc::Error; async fn create(&self) -> Result { match tokio::time::timeout(self.timeout, self.client.get_async_connection()).await { Ok(conn) => conn.map_err(into_error), Err(_) => Err(trc::StoreEvent::RedisError.ctx(trc::Key::Details, "Connection Timeout")), } } async fn recycle( &self, conn: &mut ClusterConnection, _: &managed::Metrics, ) -> managed::RecycleResult { conn.req_packed_command(&redis::cmd("PING")) .await .map(|_| ()) .map_err(|err| managed::RecycleError::Backend(into_error(err))) } } ================================================ FILE: crates/store/src/backend/redis/pubsub.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{RedisPool, RedisStore, into_error}; use crate::dispatch::pubsub::{Msg, PubSubStream}; use futures::StreamExt; use redis::{AsyncCommands, PushInfo, cluster::ClusterConfig, cluster_async::ClusterConnection}; use tokio::sync::mpsc::UnboundedReceiver; pub struct RedisPubSubStream { stream: redis::aio::PubSubStream, } pub struct RedisClusterPubSubStream { _conn: ClusterConnection, rx: UnboundedReceiver, } impl RedisStore { pub async fn publish(&self, topic: &'static str, message: Vec) -> trc::Result<()> { match &self.pool { RedisPool::Single(pool) => pool .get() .await .map_err(into_error)? .as_mut() .publish(topic, message) .await .map_err(into_error), RedisPool::Cluster(pool) => pool .get() .await .map_err(into_error)? .as_mut() .publish(topic, message) .await .map_err(into_error), } } pub async fn subscribe(&self, topic: &'static str) -> trc::Result { match &self.pool { RedisPool::Single(pool) => { let mut pubsub = pool .manager() .client .get_async_pubsub() .await .map_err(into_error)?; pubsub.subscribe(topic).await.map_err(into_error)?; Ok(PubSubStream::Redis(RedisPubSubStream { stream: pubsub.into_on_message(), })) } RedisPool::Cluster(pool) => { let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); let mut _conn = pool .manager() .client .get_async_connection_with_config(ClusterConfig::default().set_push_sender(tx)) .await .map_err(into_error)?; _conn.subscribe(topic).await.map_err(into_error)?; Ok(PubSubStream::RedisCluster(RedisClusterPubSubStream { _conn, rx, })) } } } } impl RedisPubSubStream { pub async fn next(&mut self) -> Option { self.stream.next().await.map(Msg::Redis) } } impl RedisClusterPubSubStream { pub async fn next(&mut self) -> Option { loop { if let Some(msg) = redis::Msg::from_push_info(self.rx.recv().await?) { return Some(Msg::Redis(msg)); } } } } ================================================ FILE: crates/store/src/backend/rocksdb/blob.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::ops::Range; use super::{CF_BLOBS, RocksDbStore, into_error}; impl RocksDbStore { pub(crate) async fn get_blob( &self, key: &[u8], range: Range, ) -> trc::Result>> { let db = self.db.clone(); self.spawn_worker(move || { db.get_pinned_cf(&db.cf_handle(CF_BLOBS).unwrap(), key) .map(|obj| { obj.map(|bytes| { if range.start == 0 && range.end == usize::MAX { bytes.to_vec() } else { bytes .get(range.start..std::cmp::min(bytes.len(), range.end)) .unwrap_or_default() .to_vec() } }) }) .map_err(into_error) }) .await } pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { let db = self.db.clone(); self.spawn_worker(move || { db.put_cf(&db.cf_handle(CF_BLOBS).unwrap(), key, data) .map_err(into_error) }) .await } pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result { let db = self.db.clone(); self.spawn_worker(move || { db.delete_cf(&db.cf_handle(CF_BLOBS).unwrap(), key) .map_err(into_error) .map(|_| true) }) .await } } ================================================ FILE: crates/store/src/backend/rocksdb/main.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{CF_BLOBS, RocksDbStore}; use crate::*; use rocksdb::{ColumnFamilyDescriptor, MergeOperands, OptimisticTransactionDB, Options}; use std::path::PathBuf; use tokio::sync::oneshot; use utils::config::{Config, utils::AsKey}; impl RocksDbStore { pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { let prefix = prefix.as_key(); // Create the database directory if it doesn't exist let idx_path: PathBuf = PathBuf::from(config.value_require((&prefix, "path"))?); std::fs::create_dir_all(&idx_path) .map_err(|err| { config.new_build_error( (&prefix, "path"), format!( "Failed to create database directory {}: {:?}", idx_path.display(), err ), ) }) .ok()?; let mut cfs = Vec::new(); // Counters for subspace in [SUBSPACE_COUNTER, SUBSPACE_QUOTA, SUBSPACE_IN_MEMORY_COUNTER] { let mut cf_opts = Options::default(); cf_opts.set_merge_operator_associative("merge", numeric_value_merge); cfs.push(ColumnFamilyDescriptor::new( std::str::from_utf8(&[subspace]).unwrap(), cf_opts, )); } // Blobs let mut cf_opts = Options::default(); cf_opts.set_enable_blob_files(true); cf_opts.set_min_blob_size( config .property_or_default((&prefix, "min-blob-size"), "16834") .unwrap_or(16834), ); cfs.push(ColumnFamilyDescriptor::new(CF_BLOBS, cf_opts)); // Other cfs for subspace in [ SUBSPACE_INDEXES, SUBSPACE_ACL, SUBSPACE_DIRECTORY, SUBSPACE_TASK_QUEUE, SUBSPACE_BLOB_EXTRA, SUBSPACE_BLOB_LINK, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_PROPERTY, SUBSPACE_SETTINGS, SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUEUE_EVENT, SUBSPACE_REPORT_OUT, SUBSPACE_REPORT_IN, SUBSPACE_LOGS, SUBSPACE_BLOBS, SUBSPACE_TELEMETRY_SPAN, SUBSPACE_TELEMETRY_METRIC, SUBSPACE_SEARCH_INDEX, LEGACY_SUBSPACE_BITMAP_ID, LEGACY_SUBSPACE_BITMAP_TAG, LEGACY_SUBSPACE_BITMAP_TEXT, LEGACY_SUBSPACE_FTS_INDEX, LEGACY_SUBSPACE_TELEMETRY_INDEX, ] { let cf_opts = Options::default(); cfs.push(ColumnFamilyDescriptor::new( std::str::from_utf8(&[subspace]).unwrap(), cf_opts, )); } let mut db_opts = Options::default(); db_opts.create_missing_column_families(true); db_opts.create_if_missing(true); db_opts.set_max_background_jobs(std::cmp::max(num_cpus::get() as i32, 3)); db_opts.increase_parallelism(std::cmp::max(num_cpus::get() as i32, 3)); db_opts.set_level_zero_file_num_compaction_trigger(1); db_opts.set_level_compaction_dynamic_level_bytes(true); //db_opts.set_keep_log_file_num(100); //db_opts.set_max_successive_merges(100); db_opts.set_write_buffer_size( config .property_or_default((&prefix, "write-buffer-size"), "134217728") .unwrap_or(134217728), ); Some(RocksDbStore { db: OptimisticTransactionDB::open_cf_descriptors(&db_opts, idx_path, cfs) .map_err(|err| { config.new_build_error( prefix.as_str(), format!("Failed to open database: {:?}", err), ) }) .ok()? .into(), worker_pool: rayon::ThreadPoolBuilder::new() .num_threads(std::cmp::max( config .property::((&prefix, "pool.workers")) .filter(|v| *v > 0) .unwrap_or_else(num_cpus::get), 4, )) .build() .map_err(|err| { config.new_build_error( (&prefix, "pool.workers"), format!("Failed to build worker pool: {:?}", err), ) }) .ok()?, }) } pub async fn spawn_worker(&self, mut f: U) -> trc::Result where U: FnMut() -> trc::Result + Send, V: Sync + Send + 'static, { let (tx, rx) = oneshot::channel(); self.worker_pool.scope(|s| { s.spawn(|_| { tx.send(f()).ok(); }); }); match rx.await { Ok(result) => result, Err(err) => Err(trc::EventType::Server(trc::ServerEvent::ThreadError).reason(err)), } } } pub fn numeric_value_merge( _key: &[u8], value: Option<&[u8]>, operands: &MergeOperands, ) -> Option> { let mut value = if let Some(value) = value { i64::from_le_bytes(value.try_into().ok()?) } else { 0 }; for op in operands.iter() { value += i64::from_le_bytes(op.try_into().ok()?); } let mut bytes = Vec::with_capacity(std::mem::size_of::()); bytes.extend_from_slice(&value.to_le_bytes()); Some(bytes) } ================================================ FILE: crates/store/src/backend/rocksdb/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::Arc; use rocksdb::{BoundColumnFamily, MultiThreaded, OptimisticTransactionDB}; use crate::{SUBSPACE_BLOBS, SUBSPACE_INDEXES, SUBSPACE_LOGS}; pub mod blob; pub mod main; pub mod read; pub mod write; static CF_LOGS: &str = unsafe { std::str::from_utf8_unchecked(&[SUBSPACE_LOGS]) }; static CF_INDEXES: &str = unsafe { std::str::from_utf8_unchecked(&[SUBSPACE_INDEXES]) }; static CF_BLOBS: &str = unsafe { std::str::from_utf8_unchecked(&[SUBSPACE_BLOBS]) }; pub(crate) trait CfHandle { fn subspace_handle(&self, subspace: u8) -> Arc>; } impl CfHandle for OptimisticTransactionDB { #[inline(always)] fn subspace_handle(&self, subspace: u8) -> Arc> { let subspace = &[subspace]; self.cf_handle(unsafe { std::str::from_utf8_unchecked(subspace) }) .unwrap() } } pub struct RocksDbStore { db: Arc>, worker_pool: rayon::ThreadPool, } #[inline(always)] fn into_error(err: rocksdb::Error) -> trc::Error { trc::StoreEvent::RocksdbError.reason(err) } ================================================ FILE: crates/store/src/backend/rocksdb/read.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{RocksDbStore, into_error}; use crate::{ Deserialize, IterateParams, Key, ValueKey, backend::rocksdb::CfHandle, write::ValueClass, }; use rocksdb::{Direction, IteratorMode}; impl RocksDbStore { pub(crate) async fn get_value(&self, key: impl Key) -> trc::Result> where U: Deserialize + 'static, { let db = self.db.clone(); self.spawn_worker(move || { db.get_pinned_cf( &db.cf_handle(std::str::from_utf8(&[key.subspace()]).unwrap()) .unwrap(), key.serialize(0), ) .map_err(into_error) .and_then(|value| { if let Some(value) = value { U::deserialize(&value).map(Some) } else { Ok(None) } }) }) .await } pub(crate) async fn iterate( &self, params: IterateParams, mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, ) -> trc::Result<()> { let db = self.db.clone(); self.spawn_worker(move || { let cf = db.subspace_handle(params.begin.subspace()); let begin = params.begin.serialize(0); let end = params.end.serialize(0); let it_mode = if params.ascending { IteratorMode::From(&begin, Direction::Forward) } else { IteratorMode::From(&end, Direction::Reverse) }; for row in db.iterator_cf(&cf, it_mode) { let (key, value) = row.map_err(into_error)?; if key.as_ref() < begin.as_slice() || key.as_ref() > end.as_slice() || !cb(&key, &value)? || params.first { break; } } Ok(()) }) .await } pub(crate) async fn get_counter( &self, key: impl Into> + Sync + Send, ) -> trc::Result { let key = key.into(); let db = self.db.clone(); self.spawn_worker(move || { let cf = self.db.subspace_handle(key.subspace()); let key = key.serialize(0); db.get_pinned_cf(&cf, &key) .map_err(into_error) .and_then(|bytes| { Ok(if let Some(bytes) = bytes { i64::from_le_bytes(bytes[..].try_into().map_err(|_| { trc::Error::corrupted_key(&key, (&bytes[..]).into(), trc::location!()) })?) } else { 0 }) }) }) .await } } ================================================ FILE: crates/store/src/backend/rocksdb/write.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{CF_INDEXES, CF_LOGS, CfHandle, RocksDbStore, into_error}; use crate::{ Deserialize, IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, backend::deserialize_i64_le, write::{ AssignedIds, Batch, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, MergeResult, Operation, ValueClass, ValueOp, }, }; use rand::Rng; use rocksdb::{ BoundColumnFamily, ErrorKind, IteratorMode, OptimisticTransactionDB, OptimisticTransactionOptions, WriteOptions, }; use std::{ sync::Arc, thread::sleep, time::{Duration, Instant}, }; impl RocksDbStore { pub(crate) async fn write(&self, mut batch: Batch<'_>) -> trc::Result { let db = self.db.clone(); self.spawn_worker(move || { let mut txn = RocksDBTransaction { db: &db, cf_indexes: db.cf_handle(CF_INDEXES).unwrap(), cf_logs: db.cf_handle(CF_LOGS).unwrap(), txn_opts: OptimisticTransactionOptions::default(), batch: &mut batch, }; txn.txn_opts.set_snapshot(true); // Begin write let mut retry_count = 0; let start = Instant::now(); loop { match txn.commit() { Ok(result) => { return Ok(result); } Err(CommitError::Internal(err)) => return Err(err), Err(CommitError::RocksDB(err)) => match err.kind() { ErrorKind::Busy | ErrorKind::MergeInProgress | ErrorKind::TryAgain if retry_count < MAX_COMMIT_ATTEMPTS && start.elapsed() < MAX_COMMIT_TIME => { let backoff = rand::rng().random_range(50..=300); sleep(Duration::from_millis(backoff)); retry_count += 1; } _ => return Err(into_error(err)), }, } } }) .await } pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> { let db = self.db.clone(); self.spawn_worker(move || { db.delete_range_cf( &db.cf_handle(std::str::from_utf8(&[from.subspace()]).unwrap()) .unwrap(), from.serialize(0), to.serialize(0), ) .map_err(into_error) }) .await } pub(crate) async fn purge_store(&self) -> trc::Result<()> { let db = self.db.clone(); self.spawn_worker(move || { for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] { let cf = db .cf_handle(std::str::from_utf8(&[subspace]).unwrap()) .unwrap(); let mut delete_keys = Vec::new(); for row in db.iterator_cf(&cf, IteratorMode::Start) { let (key, value) = row.map_err(into_error)?; if i64::deserialize(&value)? == 0 { delete_keys.push(key); } } let txn_opts = OptimisticTransactionOptions::default(); for key in delete_keys { let txn = db.transaction_opt(&WriteOptions::default(), &txn_opts); if txn .get_pinned_for_update_cf(&cf, &key, true) .map_err(into_error)? .map(|value| i64::deserialize(&value).map(|v| v == 0).unwrap_or(false)) .unwrap_or(false) { txn.delete_cf(&cf, key).map_err(into_error)?; txn.commit().map_err(into_error)?; } else { txn.rollback().map_err(into_error)?; } } } Ok(()) }) .await } } struct RocksDBTransaction<'x, 'y> { db: &'x OptimisticTransactionDB, cf_indexes: Arc>, cf_logs: Arc>, txn_opts: OptimisticTransactionOptions, batch: &'x mut Batch<'y>, } enum CommitError { Internal(trc::Error), RocksDB(rocksdb::Error), } impl RocksDBTransaction<'_, '_> { fn commit(&mut self) -> Result { let mut account_id = u32::MAX; let mut collection = u8::MAX; let mut document_id = u32::MAX; let mut change_id = 0u64; let mut result = AssignedIds::default(); let has_changes = !self.batch.changes.is_empty(); let txn = self .db .transaction_opt(&WriteOptions::default(), &self.txn_opts); if has_changes { let cf = self.db.cf_handle("n").unwrap(); for &account_id in self.batch.changes.keys() { let key = ValueClass::ChangeId.serialize(account_id, 0, 0, 0); let change_id = txn .get_pinned_for_update_cf(&cf, &key, true) .map_err(CommitError::from) .and_then(|bytes| { if let Some(bytes) = bytes { deserialize_i64_le(&key, &bytes) .map(|v| v + 1) .map_err(CommitError::from) } else { Ok(1) } })?; txn.put_cf(&cf, &key, &change_id.to_le_bytes()[..])?; result.push_change_id(account_id, change_id as u64); } } for op in self.batch.ops.iter_mut() { match op { Operation::AccountId { account_id: account_id_, } => { account_id = *account_id_; if has_changes { change_id = result.set_current_change_id(account_id)?; } } Operation::Collection { collection: collection_, } => { collection = u8::from(*collection_); } Operation::DocumentId { document_id: document_id_, } => { document_id = *document_id_; } Operation::Value { class, op } => { let key = class.serialize(account_id, collection, document_id, 0); let cf = self.db.subspace_handle(class.subspace(collection)); match op { ValueOp::Set(value) => { txn.put_cf(&cf, &key, value)?; } ValueOp::SetFnc(set_op) => { let value = (set_op.fnc)(&set_op.params, &result)?; txn.put_cf(&cf, &key, value)?; } ValueOp::MergeFnc(merge_op) => { let merge_result = (merge_op.fnc)( &merge_op.params, &result, txn.get_pinned_for_update_cf(&cf, &key, true)?.as_deref(), )?; match merge_result { MergeResult::Update(value) => { txn.put_cf(&cf, &key, value)?; } MergeResult::Delete => { txn.delete_cf(&cf, &key)?; } MergeResult::Skip => (), } } ValueOp::AtomicAdd(by) => { txn.merge_cf(&cf, &key, &by.to_le_bytes()[..])?; } ValueOp::AddAndGet(by) => { let num = txn .get_pinned_for_update_cf(&cf, &key, true) .map_err(CommitError::from) .and_then(|bytes| { if let Some(bytes) = bytes { deserialize_i64_le(&key, &bytes) .map(|v| v + *by) .map_err(CommitError::from) } else { Ok(*by) } })?; txn.put_cf(&cf, &key, &num.to_le_bytes()[..])?; result.push_counter_id(num); } ValueOp::Clear => { txn.delete_cf(&cf, &key)?; } } } Operation::Index { field, key, set } => { let key = IndexKey { account_id, collection, document_id, field: *field, key: &*key, } .serialize(0); if *set { txn.put_cf(&self.cf_indexes, &key, [])?; } else { txn.delete_cf(&self.cf_indexes, &key)?; } } Operation::Log { collection, set } => { let key = LogKey { account_id, collection: u8::from(*collection), change_id, } .serialize(0); txn.put_cf(&self.cf_logs, &key, set)?; } Operation::AssertValue { class, assert_value, } => { let key = class.serialize(account_id, collection, document_id, 0); let cf = self.db.subspace_handle(class.subspace(collection)); let matches = txn .get_pinned_for_update_cf(&cf, &key, true)? .map(|value| assert_value.matches(&value)) .unwrap_or_else(|| assert_value.is_none()); if !matches { txn.rollback()?; return Err(CommitError::Internal( trc::StoreEvent::AssertValueFailed.into(), )); } } } } txn.commit().map(|_| result).map_err(Into::into) } } impl From for CommitError { fn from(err: rocksdb::Error) -> Self { CommitError::RocksDB(err) } } impl From for CommitError { fn from(err: trc::Error) -> Self { CommitError::Internal(err) } } ================================================ FILE: crates/store/src/backend/s3/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use s3::{Bucket, Region, creds::Credentials}; use std::{fmt::Display, io::Write, ops::Range, time::Duration}; use utils::{ codec::base32_custom::Base32Writer, config::{Config, utils::AsKey}, }; pub struct S3Store { bucket: Box, prefix: Option, max_retries: u32, } impl S3Store { pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { // Obtain region and endpoint from config let prefix = prefix.as_key(); let region = config.value_require((&prefix, "region"))?.to_string(); let region = if let Some(endpoint) = config.value((&prefix, "endpoint")) { Region::Custom { region: region.to_string(), endpoint: endpoint.to_string(), } } else { region.parse().unwrap() }; let credentials = Credentials::new( config.value((&prefix, "access-key")), config.value((&prefix, "secret-key")), config.value((&prefix, "security-token")), config.value((&prefix, "session-token")), config.value((&prefix, "profile")), ) .map_err(|err| { config.new_build_error( prefix.as_str(), format!("Failed to create credentials: {err:?}"), ) }) .ok()?; let timeout = config .property_or_default::((&prefix, "timeout"), "30s") .unwrap_or_else(|| Duration::from_secs(30)); /*let allow_invalid = config .property_or_default::((&prefix, "tls.allow-invalid"), "false") .unwrap_or_default();*/ Some(S3Store { bucket: Bucket::new( config.value_require((&prefix, "bucket"))?, region, credentials, ) .map_err(|err| { config.new_build_error(prefix.as_str(), format!("Failed to create bucket: {err:?}")) }) .ok()? .with_path_style() /*.set_dangereous_config(allow_invalid, allow_invalid) .map_err(|err| { config.new_build_error(prefix.as_str(), format!("Failed to create bucket: {err:?}")) }) .ok()?*/ .with_request_timeout(timeout) .map_err(|err| { config.new_build_error(prefix.as_str(), format!("Failed to create bucket: {err:?}")) }) .ok()?, max_retries: config .property_or_default((&prefix, "max-retries"), "3") .unwrap_or(3), prefix: config.value((&prefix, "key-prefix")).map(|s| s.to_string()), }) } pub(crate) async fn get_blob( &self, key: &[u8], range: Range, ) -> trc::Result>> { let path = self.build_key(key); let mut retries_left = self.max_retries; loop { let response = if range.start != 0 || range.end != usize::MAX { self.bucket .get_object_range( &path, range.start as u64, Some(range.end.saturating_sub(1) as u64), ) .await } else { self.bucket.get_object(&path).await } .map_err(into_error)?; match response.status_code() { 200..=299 => return Ok(Some(response.to_vec())), 404 => return Ok(None), 500..=599 if retries_left > 0 => { // wait backoff tokio::time::sleep(Duration::from_secs( 1 << (self.max_retries - retries_left).min(6), )) .await; retries_left -= 1; } code => { return Err(trc::StoreEvent::S3Error .reason(String::from_utf8_lossy(response.as_slice())) .ctx(trc::Key::Code, code)); } } } } pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { let mut retries_left = self.max_retries; loop { let response = self .bucket .put_object(self.build_key(key), data) .await .map_err(into_error)?; match response.status_code() { 200..=299 => return Ok(()), 500..=599 if retries_left > 0 => { // wait backoff tokio::time::sleep(Duration::from_secs( 1 << (self.max_retries - retries_left).min(6), )) .await; retries_left -= 1; } code => { return Err(trc::StoreEvent::S3Error .reason(String::from_utf8_lossy(response.as_slice())) .ctx(trc::Key::Code, code)); } } } } pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result { let mut retries_left = self.max_retries; loop { let response = self .bucket .delete_object(self.build_key(key)) .await .map_err(into_error)?; match response.status_code() { 200..=299 => return Ok(true), 404 => return Ok(false), 500..=599 if retries_left > 0 => { // wait backoff tokio::time::sleep(Duration::from_secs( 1 << (self.max_retries - retries_left).min(6), )) .await; retries_left -= 1; } code => { return Err(trc::StoreEvent::S3Error .reason(String::from_utf8_lossy(response.as_slice())) .ctx(trc::Key::Code, code)); } } } } fn build_key(&self, key: &[u8]) -> String { if let Some(prefix) = &self.prefix { let mut writer = Base32Writer::with_raw_capacity(prefix.len() + (key.len().div_ceil(4) * 5)); writer.push_string(prefix); writer.write_all(key).unwrap(); writer.finalize() } else { Base32Writer::from_bytes(key).finalize() } } } #[inline(always)] fn into_error(err: impl Display) -> trc::Error { trc::StoreEvent::S3Error.reason(err) } ================================================ FILE: crates/store/src/backend/sqlite/blob.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::ops::Range; use rusqlite::OptionalExtension; use super::{SqliteStore, into_error}; impl SqliteStore { pub(crate) async fn get_blob( &self, key: &[u8], range: Range, ) -> trc::Result>> { let conn = self.conn_pool.get().map_err(into_error)?; self.spawn_worker(move || { let mut result = conn .prepare_cached("SELECT v FROM t WHERE k = ?") .map_err(into_error)?; result .query_row([&key], |row| { Ok({ let bytes = row.get_ref(0)?.as_bytes()?; if range.start == 0 && range.end == usize::MAX { bytes.to_vec() } else { bytes .get(range.start..std::cmp::min(bytes.len(), range.end)) .unwrap_or_default() .to_vec() } }) }) .optional() .map_err(into_error) }) .await } pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { let conn = self.conn_pool.get().map_err(into_error)?; self.spawn_worker(move || { conn.prepare_cached("INSERT OR REPLACE INTO t (k, v) VALUES (?, ?)") .map_err(into_error)? .execute([key, data]) .map_err(into_error) .map(|_| ()) }) .await } pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result { let conn = self.conn_pool.get().map_err(into_error)?; self.spawn_worker(move || { conn.prepare_cached("DELETE FROM t WHERE k = ?") .map_err(into_error)? .execute([key]) .map_err(into_error) .map(|_| true) }) .await } } ================================================ FILE: crates/store/src/backend/sqlite/lookup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use rusqlite::{Row, Rows, ToSql, types::FromSql}; use crate::{IntoRows, QueryResult, QueryType, Value}; use super::{SqliteStore, into_error}; impl SqliteStore { pub(crate) async fn sql_query( &self, query: &str, params_: &[Value<'_>], ) -> trc::Result { let conn = self.conn_pool.get().map_err(into_error)?; self.spawn_worker(move || { let mut s = conn.prepare_cached(query).map_err(into_error)?; let params = params_ .iter() .map(|v| v as &dyn rusqlite::types::ToSql) .collect::>(); match T::query_type() { QueryType::Execute => s .execute(params.as_slice()) .map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_exec(r))), QueryType::Exists => s .exists(params.as_slice()) .map(T::from_exists) .map_err(into_error), QueryType::QueryOne => s .query(params.as_slice()) .and_then(|mut rows| Ok(T::from_query_one(rows.next()?))) .map_err(into_error), QueryType::QueryAll => Ok(T::from_query_all( s.query(params.as_slice()).map_err(into_error)?, )), } }) .await } } impl ToSql for Value<'_> { fn to_sql(&self) -> rusqlite::Result> { match self { Value::Integer(value) => value.to_sql(), Value::Bool(value) => value.to_sql(), Value::Float(value) => value.to_sql(), Value::Text(value) => value.to_sql(), Value::Blob(value) => value.to_sql(), Value::Null => Ok(rusqlite::types::ToSqlOutput::Owned( rusqlite::types::Value::Null, )), } } } impl FromSql for Value<'static> { fn column_result(value: rusqlite::types::ValueRef<'_>) -> rusqlite::types::FromSqlResult { Ok(match value { rusqlite::types::ValueRef::Null => Value::Null, rusqlite::types::ValueRef::Integer(v) => Value::Integer(v), rusqlite::types::ValueRef::Real(v) => Value::Float(v), rusqlite::types::ValueRef::Text(v) => { Value::Text(String::from_utf8_lossy(v).into_owned().into()) } rusqlite::types::ValueRef::Blob(v) => Value::Blob(v.to_vec().into()), }) } } impl IntoRows for Rows<'_> { fn into_rows(mut self) -> crate::Rows { let column_count = self.as_ref().map(|s| s.column_count()).unwrap_or_default(); let mut rows = crate::Rows { rows: Vec::new() }; while let Ok(Some(row)) = self.next() { rows.rows.push(crate::Row { values: (0..column_count) .map(|idx| row.get::<_, Value>(idx).unwrap_or(Value::Null)) .collect(), }); } rows } fn into_named_rows(mut self) -> crate::NamedRows { let (column_count, names) = self .as_ref() .map(|s| { ( s.column_count(), s.column_names() .into_iter() .map(String::from) .collect::>(), ) }) .unwrap_or((0, Vec::new())); let mut rows = crate::NamedRows { names, rows: Vec::new(), }; while let Ok(Some(row)) = self.next() { rows.rows.push(crate::Row { values: (0..column_count) .map(|idx| row.get::<_, Value>(idx).unwrap_or(Value::Null)) .collect(), }); } rows } fn into_row(self) -> Option { unreachable!() } } impl IntoRows for Option<&Row<'_>> { fn into_row(self) -> Option { self.map(|row| crate::Row { values: (0..row.as_ref().column_count()) .map(|idx| row.get::<_, Value>(idx).unwrap_or(Value::Null)) .collect(), }) } fn into_rows(self) -> crate::Rows { unreachable!() } fn into_named_rows(self) -> crate::NamedRows { unreachable!() } } ================================================ FILE: crates/store/src/backend/sqlite/main.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use r2d2::Pool; use tokio::sync::oneshot; use utils::config::{Config, utils::AsKey}; use crate::*; use super::{SqliteStore, into_error, pool::SqliteConnectionManager}; impl SqliteStore { pub fn open(config: &mut Config, prefix: impl AsKey) -> Option { let prefix = prefix.as_key(); let db = Self { conn_pool: Pool::builder() .max_size( config .property((&prefix, "pool.max-connections")) .unwrap_or_else(|| (num_cpus::get() * 4) as u32), ) .build( SqliteConnectionManager::file(config.value_require((&prefix, "path"))?) .with_init(|c| { c.execute_batch(concat!( "PRAGMA journal_mode = WAL; ", "PRAGMA synchronous = NORMAL; ", "PRAGMA temp_store = memory;", "PRAGMA busy_timeout = 30000;" )) }), ) .map_err(|err| { config.new_build_error( prefix.as_str(), format!("Failed to build connection pool: {err}"), ) }) .ok()?, worker_pool: rayon::ThreadPoolBuilder::new() .num_threads(std::cmp::max( config .property::((&prefix, "pool.workers")) .filter(|v| *v > 0) .unwrap_or_else(num_cpus::get), 4, )) .build() .map_err(|err| { config.new_build_error( prefix.as_str(), format!("Failed to build worker pool: {err}"), ) }) .ok()?, }; if let Err(err) = db.create_tables() { config.new_build_error(prefix.as_str(), format!("Failed to create tables: {err}")); } Some(db) } #[cfg(feature = "test_mode")] pub fn open_memory() -> trc::Result { use super::into_error; let db = Self { conn_pool: Pool::builder() .max_size(1) .build(SqliteConnectionManager::memory()) .map_err(into_error)?, worker_pool: rayon::ThreadPoolBuilder::new() .num_threads(num_cpus::get()) .build() .map_err(|err| { into_error(err).ctx(trc::Key::Reason, "Failed to build worker pool") })?, }; db.create_tables()?; Ok(db) } pub(super) fn create_tables(&self) -> trc::Result<()> { let conn = self.conn_pool.get().map_err(into_error)?; for table in [ SUBSPACE_ACL, SUBSPACE_DIRECTORY, SUBSPACE_TASK_QUEUE, SUBSPACE_BLOB_EXTRA, SUBSPACE_BLOB_LINK, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_PROPERTY, SUBSPACE_SETTINGS, SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUEUE_EVENT, SUBSPACE_REPORT_OUT, SUBSPACE_REPORT_IN, SUBSPACE_LOGS, SUBSPACE_BLOBS, SUBSPACE_TELEMETRY_SPAN, SUBSPACE_TELEMETRY_METRIC, SUBSPACE_SEARCH_INDEX, ] { let table = char::from(table); conn.execute( &format!( "CREATE TABLE IF NOT EXISTS {table} ( k BLOB PRIMARY KEY, v BLOB NOT NULL )" ), [], ) .map_err(into_error)?; } let table = char::from(SUBSPACE_INDEXES); conn.execute( &format!( "CREATE TABLE IF NOT EXISTS {table} ( k BLOB PRIMARY KEY )" ), [], ) .map_err(into_error)?; for table in [SUBSPACE_COUNTER, SUBSPACE_QUOTA, SUBSPACE_IN_MEMORY_COUNTER] { conn.execute( &format!( "CREATE TABLE IF NOT EXISTS {} ( k BLOB PRIMARY KEY, v INTEGER NOT NULL DEFAULT 0 )", char::from(table) ), [], ) .map_err(into_error)?; } Ok(()) } pub async fn spawn_worker(&self, mut f: U) -> trc::Result where U: FnMut() -> trc::Result + Send, V: Sync + Send + 'static, { let (tx, rx) = oneshot::channel(); self.worker_pool.scope(|s| { s.spawn(|_| { tx.send(f()).ok(); }); }); match rx.await { Ok(result) => result, Err(err) => Err(trc::EventType::Server(trc::ServerEvent::ThreadError).reason(err)), } } } ================================================ FILE: crates/store/src/backend/sqlite/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::fmt::Display; use r2d2::Pool; use self::pool::SqliteConnectionManager; pub mod blob; pub mod lookup; pub mod main; pub mod pool; pub mod read; pub mod write; pub struct SqliteStore { pub(crate) conn_pool: Pool, pub(crate) worker_pool: rayon::ThreadPool, } #[inline(always)] fn into_error(err: impl Display) -> trc::Error { trc::StoreEvent::SqliteError.reason(err) } ================================================ FILE: crates/store/src/backend/sqlite/pool.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use rusqlite::{Connection, Error, OpenFlags}; use std::fmt; use std::path::{Path, PathBuf}; #[derive(Debug)] enum Source { File(PathBuf), Memory, } type InitFn = dyn Fn(&mut Connection) -> Result<(), rusqlite::Error> + Send + Sync + 'static; /// An `r2d2::ManageConnection` for `rusqlite::Connection`s. pub struct SqliteConnectionManager { source: Source, flags: OpenFlags, init: Option>, } impl fmt::Debug for SqliteConnectionManager { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let mut builder = f.debug_struct("SqliteConnectionManager"); let _ = builder.field("source", &self.source); let _ = builder.field("flags", &self.source); let _ = builder.field("init", &self.init.as_ref().map(|_| "InitFn")); builder.finish() } } impl SqliteConnectionManager { /// Creates a new `SqliteConnectionManager` from file. /// /// See `rusqlite::Connection::open` pub fn file>(path: P) -> Self { Self { source: Source::File(path.as_ref().to_path_buf()), flags: OpenFlags::default(), init: None, } } /// Creates a new `SqliteConnectionManager` from memory. pub fn memory() -> Self { Self { source: Source::Memory, flags: OpenFlags::default(), init: None, } } /// Converts `SqliteConnectionManager` into one that sets OpenFlags upon /// connection creation. /// /// See `rustqlite::OpenFlags` for a list of available flags. pub fn with_flags(self, flags: OpenFlags) -> Self { Self { flags, ..self } } /// Converts `SqliteConnectionManager` into one that calls an initialization /// function upon connection creation. Could be used to set PRAGMAs, for /// example. /// /// ### Example /// /// Make a `SqliteConnectionManager` that sets the `foreign_keys` pragma to /// true for every connection. /// /// ```rust,no_run /// # use r2d2_sqlite::{SqliteConnectionManager}; /// let manager = SqliteConnectionManager::file("app.db") /// .with_init(|c| c.execute_batch("PRAGMA foreign_keys=1;")); /// ``` pub fn with_init(self, init: F) -> Self where F: Fn(&mut Connection) -> Result<(), rusqlite::Error> + Send + Sync + 'static, { let init: Option> = Some(Box::new(init)); Self { init, ..self } } } fn sleeper(_: i32) -> bool { std::thread::sleep(std::time::Duration::from_millis(200)); true } impl r2d2::ManageConnection for SqliteConnectionManager { type Connection = Connection; type Error = rusqlite::Error; fn connect(&self) -> Result { match self.source { Source::File(ref path) => Connection::open_with_flags(path, self.flags), Source::Memory => Connection::open_in_memory_with_flags(self.flags), } .and_then(|mut c| { c.busy_handler(Some(sleeper))?; match self.init { None => Ok(c), Some(ref init) => init(&mut c).map(|_| c), } }) } fn is_valid(&self, conn: &mut Connection) -> Result<(), Error> { conn.execute_batch("") } fn has_broken(&self, _: &mut Connection) -> bool { false } } ================================================ FILE: crates/store/src/backend/sqlite/read.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{SqliteStore, into_error}; use crate::{Deserialize, IterateParams, Key, ValueKey, write::ValueClass}; use rusqlite::OptionalExtension; impl SqliteStore { pub(crate) async fn get_value(&self, key: impl Key) -> trc::Result> where U: Deserialize + 'static, { let conn = self.conn_pool.get().map_err(into_error)?; self.spawn_worker(move || { let mut result = conn .prepare_cached(&format!( "SELECT v FROM {} WHERE k = ?", char::from(key.subspace()) )) .map_err(into_error)?; let key = key.serialize(0); result .query_row([&key], |row| { U::deserialize(row.get_ref(0)?.as_bytes()?) .map_err(|err| rusqlite::Error::ToSqlConversionFailure(err.into())) }) .optional() .map_err(into_error) }) .await } pub(crate) async fn iterate( &self, params: IterateParams, mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, ) -> trc::Result<()> { let conn = self.conn_pool.get().map_err(into_error)?; self.spawn_worker(move || { let table = char::from(params.begin.subspace()); let begin = params.begin.serialize(0); let end = params.end.serialize(0); let keys = if params.values { "k, v" } else { "k" }; let mut query = conn .prepare_cached(&match (params.first, params.ascending) { (true, true) => { format!( "SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC LIMIT 1" ) } (true, false) => { format!( "SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC LIMIT 1" ) } (false, true) => { format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC") } (false, false) => { format!( "SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC" ) } }) .map_err(into_error)?; let mut rows = query.query([&begin, &end]).map_err(into_error)?; if params.values { while let Some(row) = rows.next().map_err(into_error)? { let key = row .get_ref(0) .map_err(into_error)? .as_bytes() .map_err(into_error)?; let value = row .get_ref(1) .map_err(into_error)? .as_bytes() .map_err(into_error)?; if !cb(key, value)? { break; } } } else { while let Some(row) = rows.next().map_err(into_error)? { if !cb( row.get_ref(0) .map_err(into_error)? .as_bytes() .map_err(into_error)?, b"", )? { break; } } } Ok(()) }) .await } pub(crate) async fn get_counter( &self, key: impl Into> + Sync + Send, ) -> trc::Result { let key = key.into(); let table = char::from(key.subspace()); let key = key.serialize(0); let conn = self.conn_pool.get().map_err(into_error)?; self.spawn_worker(move || { match conn .prepare_cached(&format!("SELECT v FROM {table} WHERE k = ?")) .map_err(into_error)? .query_row([&key], |row| row.get::<_, i64>(0)) { Ok(value) => Ok(value), Err(rusqlite::Error::QueryReturnedNoRows) => Ok(0), Err(e) => Err(into_error(e)), } }) .await } } ================================================ FILE: crates/store/src/backend/sqlite/write.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{SqliteStore, into_error}; use crate::{ IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, write::{AssignedIds, Batch, MergeResult, Operation, ValueClass, ValueOp}, }; use rusqlite::{OptionalExtension, TransactionBehavior, params}; use trc::AddContext; impl SqliteStore { pub(crate) async fn write(&self, batch: Batch<'_>) -> trc::Result { let mut conn = self .conn_pool .get() .map_err(into_error) .caused_by(trc::location!())?; self.spawn_worker(move || { let mut account_id = u32::MAX; let mut collection = u8::MAX; let mut document_id = u32::MAX; let mut change_id = 0u64; let trx = conn .transaction_with_behavior(TransactionBehavior::Immediate) .map_err(into_error) .caused_by(trc::location!())?; let mut result = AssignedIds::default(); let has_changes = !batch.changes.is_empty(); if has_changes { for &account_id in batch.changes.keys() { let key = ValueClass::ChangeId.serialize(account_id, 0, 0, 0); let change_id = trx .prepare_cached(concat!( "INSERT INTO n (k, v) VALUES (?, ?) ", "ON CONFLICT(k) DO UPDATE SET v = v + ", "excluded.v RETURNING v" )) .map_err(into_error) .caused_by(trc::location!())? .query_row(params![&key, &1i64], |row| row.get::<_, i64>(0)) .map_err(into_error) .caused_by(trc::location!())?; result.push_change_id(account_id, change_id as u64); } } for op in batch.ops.iter_mut() { match op { Operation::AccountId { account_id: account_id_, } => { account_id = *account_id_; if has_changes { change_id = result.set_current_change_id(account_id)?; } } Operation::Collection { collection: collection_, } => { collection = u8::from(*collection_); } Operation::DocumentId { document_id: document_id_, } => { document_id = *document_id_; } Operation::Value { class, op } => { let key = class.serialize(account_id, collection, document_id, 0); let table = char::from(class.subspace(collection)); match op { ValueOp::Set(value) => { trx.prepare_cached(&format!( "INSERT OR REPLACE INTO {} (k, v) VALUES (?, ?)", table )) .map_err(into_error) .caused_by(trc::location!())? .execute([&key, value]) .map_err(into_error) .caused_by(trc::location!())?; } ValueOp::SetFnc(set_op) => { let value = (set_op.fnc)(&set_op.params, &result)?; trx.prepare_cached(&format!( "INSERT OR REPLACE INTO {} (k, v) VALUES (?, ?)", table )) .map_err(into_error) .caused_by(trc::location!())? .execute([&key, &value]) .map_err(into_error) .caused_by(trc::location!())?; } ValueOp::MergeFnc(merge_op) => { let merge_result = trx .prepare_cached(&format!("SELECT v FROM {} WHERE k = ?", table)) .map_err(into_error) .caused_by(trc::location!())? .query_row([&key], |row| { Ok((merge_op.fnc)( &merge_op.params, &result, Some(row.get_ref(0)?.as_bytes()?), )) }) .optional() .map_err(into_error) .caused_by(trc::location!())? .unwrap_or_else(|| { (merge_op.fnc)(&merge_op.params, &result, None) })?; match merge_result { MergeResult::Update(value) => { trx.prepare_cached(&format!( "INSERT OR REPLACE INTO {} (k, v) VALUES (?, ?)", table )) .map_err(into_error) .caused_by(trc::location!())? .execute([&key, &value]) .map_err(into_error) .caused_by(trc::location!())?; } MergeResult::Delete => { trx.prepare_cached(&format!( "DELETE FROM {} WHERE k = ?", table )) .map_err(into_error) .caused_by(trc::location!())? .execute([&key]) .map_err(into_error) .caused_by(trc::location!())?; } MergeResult::Skip => (), } } ValueOp::AtomicAdd(by) => { if *by >= 0 { trx.prepare_cached(&format!( concat!( "INSERT INTO {} (k, v) VALUES (?, ?) ", "ON CONFLICT(k) DO UPDATE SET v = v + excluded.v" ), table )) .map_err(into_error) .caused_by(trc::location!())? .execute(params![&key, *by]) .map_err(into_error) .caused_by(trc::location!())?; } else { trx.prepare_cached(&format!( "UPDATE {table} SET v = v + ? WHERE k = ?" )) .map_err(into_error) .caused_by(trc::location!())? .execute(params![*by, &key]) .map_err(into_error) .caused_by(trc::location!())?; } } ValueOp::AddAndGet(by) => { result.push_counter_id( trx.prepare_cached(&format!( concat!( "INSERT INTO {} (k, v) VALUES (?, ?) ", "ON CONFLICT(k) DO UPDATE SET v = v + ", "excluded.v RETURNING v" ), table )) .map_err(into_error) .caused_by(trc::location!())? .query_row(params![&key, &*by], |row| row.get::<_, i64>(0)) .map_err(into_error) .caused_by(trc::location!())?, ); } ValueOp::Clear => { trx.prepare_cached(&format!("DELETE FROM {} WHERE k = ?", table)) .map_err(into_error) .caused_by(trc::location!())? .execute([&key]) .map_err(into_error) .caused_by(trc::location!())?; } } } Operation::Index { field, key, set } => { let key = IndexKey { account_id, collection, document_id, field: *field, key: &*key, } .serialize(0); if *set { trx.prepare_cached("INSERT OR IGNORE INTO i (k) VALUES (?)") .map_err(into_error) .caused_by(trc::location!())? .execute([&key]) .map_err(into_error) .caused_by(trc::location!())?; } else { trx.prepare_cached("DELETE FROM i WHERE k = ?") .map_err(into_error) .caused_by(trc::location!())? .execute([&key]) .map_err(into_error) .caused_by(trc::location!())?; } } Operation::Log { collection, set } => { let key = LogKey { account_id, collection: u8::from(*collection), change_id, } .serialize(0); trx.prepare_cached("INSERT OR REPLACE INTO l (k, v) VALUES (?, ?)") .map_err(into_error) .caused_by(trc::location!())? .execute([&key, set]) .map_err(into_error) .caused_by(trc::location!())?; } Operation::AssertValue { class, assert_value, } => { let key = class.serialize(account_id, collection, document_id, 0); let table = char::from(class.subspace(collection)); let matches = trx .prepare_cached(&format!("SELECT v FROM {} WHERE k = ?", table)) .map_err(into_error) .caused_by(trc::location!())? .query_row([&key], |row| { Ok(assert_value.matches(row.get_ref(0)?.as_bytes()?)) }) .optional() .map_err(into_error) .caused_by(trc::location!())? .unwrap_or_else(|| assert_value.is_none()); if !matches { trx.rollback() .map_err(into_error) .caused_by(trc::location!())?; return Err(trc::StoreEvent::AssertValueFailed .into_err() .caused_by(trc::location!())); } } } } trx.commit().map(|_| result).map_err(into_error) }) .await } pub(crate) async fn purge_store(&self) -> trc::Result<()> { let conn = self .conn_pool .get() .map_err(into_error) .caused_by(trc::location!())?; self.spawn_worker(move || { for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] { conn.prepare_cached(&format!("DELETE FROM {} WHERE v = 0", char::from(subspace),)) .map_err(into_error) .caused_by(trc::location!())? .execute([]) .map_err(into_error) .caused_by(trc::location!())?; } Ok(()) }) .await } pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> { let conn = self .conn_pool .get() .map_err(into_error) .caused_by(trc::location!())?; self.spawn_worker(move || { conn.prepare_cached(&format!( "DELETE FROM {} WHERE k >= ? AND k < ?", char::from(from.subspace()), )) .map_err(into_error) .caused_by(trc::location!())? .execute([from.serialize(0), to.serialize(0)]) .map_err(into_error) .caused_by(trc::location!())?; Ok(()) }) .await } } ================================================ FILE: crates/store/src/backend/zenoh/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use utils::config::{Config, utils::AsKey}; pub mod pubsub; #[derive(Debug)] pub struct ZenohPubSub { session: zenoh::Session, } impl ZenohPubSub { pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { let prefix = prefix.as_key(); let zenoh_config = zenoh::Config::from_json5(config.value_require_non_empty((&prefix, "config"))?) .map_err(|err| { config.new_build_error( (&prefix, "config"), format!("Invalid zenoh config: {}", err), ); }) .ok()?; zenoh::open(zenoh_config) .await .map_err(|err| { config.new_build_error( (&prefix, "config"), format!("Failed to create zenoh session: {}", err), ); }) .map(|session| ZenohPubSub { session }) .ok() } } ================================================ FILE: crates/store/src/backend/zenoh/pubsub.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::ZenohPubSub; use crate::dispatch::pubsub::{Msg, PubSubStream}; use trc::{ClusterEvent, Error, EventType}; pub struct ZenohPubSubStream { subs: zenoh::pubsub::Subscriber>, } impl ZenohPubSub { pub async fn publish(&self, topic: &'static str, message: Vec) -> trc::Result<()> { self.session .declare_publisher(topic) .await .map_err(|err| { Error::new(EventType::Cluster(ClusterEvent::PublisherError)).reason(err) })? .put(message) .await .map_err(|err| Error::new(EventType::Cluster(ClusterEvent::PublisherError)).reason(err)) } pub async fn subscribe(&self, topic: &'static str) -> trc::Result { self.session .declare_subscriber(topic) .await .map(|subs| PubSubStream::Zenoh(ZenohPubSubStream { subs })) .map_err(|err| { Error::new(EventType::Cluster(ClusterEvent::SubscriberError)).reason(err) }) } } impl ZenohPubSubStream { pub async fn next(&mut self) -> Option { self.subs .recv_async() .await .map(|sample| Msg::Zenoh(sample.payload().to_bytes().into_owned())) .ok() } } ================================================ FILE: crates/store/src/config.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ BlobStore, CompressionAlgo, InMemoryStore, PurgeSchedule, PurgeStore, Store, Stores, backend::{elastic::ElasticSearchStore, fs::FsStore, meili::MeiliSearchStore}, }; use utils::config::{Config, cron::SimpleCron, utils::ParseValue}; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] enum CompositeStore { #[cfg(any(feature = "postgres", feature = "mysql"))] SQLReadReplica(String), ShardedBlob(String), ShardedInMemory(String), } // SPDX-SnippetEnd impl Stores { pub async fn parse_all(config: &mut Config, is_reload: bool) -> Self { let mut stores = Self::parse(config).await; stores.parse_in_memory(config, is_reload).await; stores } pub async fn parse(config: &mut Config) -> Self { let mut stores = Self::default(); stores.parse_stores(config).await; stores } pub async fn parse_stores(&mut self, config: &mut Config) { let is_reload = !self.stores.is_empty(); // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] let mut composite_stores = Vec::new(); // SPDX-SnippetEnd for store_id in config.sub_keys("store", ".type") { let id = store_id.as_str(); // Parse store #[cfg(feature = "test_mode")] { if config .property_or_default::(("store", id, "disable"), "false") .unwrap_or(false) { continue; } } let protocol = if let Some(protocol) = config.value_require(("store", id, "type")) { protocol.to_ascii_lowercase() } else { continue; }; let prefix = ("store", id); let compression_algo = config .property_or_default::(("store", id, "compression"), "none") .unwrap_or(CompressionAlgo::None); match protocol.as_str() { #[cfg(feature = "rocks")] "rocksdb" => { // Avoid opening the same store twice if is_reload && self .stores .values() .any(|store| matches!(store, Store::RocksDb(_))) { continue; } if let Some(db) = crate::backend::rocksdb::RocksDbStore::open(config, prefix) .await .map(Store::from) { self.stores.insert(store_id.clone(), db.clone()); self.search_stores .insert(store_id.clone(), db.clone().into()); self.blob_stores.insert( store_id.clone(), BlobStore::from(db.clone()).with_compression(compression_algo), ); self.in_memory_stores.insert(store_id, db.into()); } } #[cfg(feature = "foundation")] "foundationdb" => { // Avoid opening the same store twice if is_reload && self .stores .values() .any(|store| matches!(store, Store::FoundationDb(_))) { continue; } if let Some(db) = crate::backend::foundationdb::FdbStore::open(config, prefix) .await .map(Store::from) { self.stores.insert(store_id.clone(), db.clone()); self.search_stores .insert(store_id.clone(), db.clone().into()); self.blob_stores.insert( store_id.clone(), BlobStore::from(db.clone()).with_compression(compression_algo), ); self.in_memory_stores.insert(store_id, db.into()); } } #[cfg(feature = "postgres")] "postgresql" => { if let Some(db) = crate::backend::postgres::PostgresStore::open( config, prefix, config.is_active_store(id), config.is_active_search_store(id), ) .await .map(Store::from) { self.stores.insert(store_id.clone(), db.clone()); self.search_stores .insert(store_id.clone(), db.clone().into()); self.blob_stores.insert( store_id.clone(), BlobStore::from(db.clone()).with_compression(compression_algo), ); self.in_memory_stores.insert(store_id.clone(), db.into()); } } #[cfg(feature = "mysql")] "mysql" => { if let Some(db) = crate::backend::mysql::MysqlStore::open( config, prefix, config.is_active_store(id), config.is_active_search_store(id), ) .await .map(Store::from) { self.stores.insert(store_id.clone(), db.clone()); self.search_stores .insert(store_id.clone(), db.clone().into()); self.blob_stores.insert( store_id.clone(), BlobStore::from(db.clone()).with_compression(compression_algo), ); self.in_memory_stores.insert(store_id.clone(), db.into()); } } #[cfg(feature = "sqlite")] "sqlite" => { // Avoid opening the same store twice if is_reload && self .stores .values() .any(|store| matches!(store, Store::SQLite(_))) { continue; } if let Some(db) = crate::backend::sqlite::SqliteStore::open(config, prefix).map(Store::from) { self.stores.insert(store_id.clone(), db.clone()); self.search_stores .insert(store_id.clone(), db.clone().into()); self.blob_stores.insert( store_id.clone(), BlobStore::from(db.clone()).with_compression(compression_algo), ); self.in_memory_stores.insert(store_id.clone(), db.into()); } } "fs" => { if let Some(db) = FsStore::open(config, prefix).await.map(BlobStore::from) { self.blob_stores .insert(store_id, db.with_compression(compression_algo)); } } #[cfg(feature = "s3")] "s3" => { if let Some(db) = crate::backend::s3::S3Store::open(config, prefix) .await .map(BlobStore::from) { self.blob_stores .insert(store_id, db.with_compression(compression_algo)); } } "elasticsearch" => { if let Some(db) = ElasticSearchStore::open(config, prefix) .await .map(crate::SearchStore::from) { self.search_stores.insert(store_id, db); } } "meilisearch" => { if let Some(db) = MeiliSearchStore::open(config, prefix) .await .map(crate::SearchStore::from) { self.search_stores.insert(store_id, db); } } #[cfg(feature = "redis")] "redis" => { if let Some(db) = crate::backend::redis::RedisStore::open(config, prefix) .await .map(std::sync::Arc::new) { self.in_memory_stores .insert(store_id.clone(), InMemoryStore::Redis(db.clone())); self.pubsub_stores .insert(store_id, crate::PubSubStore::Redis(db)); } } #[cfg(feature = "nats")] "nats" => { if let Some(db) = crate::backend::nats::NatsPubSub::open(config, prefix) .await .map(std::sync::Arc::new) { self.pubsub_stores .insert(store_id, crate::PubSubStore::Nats(db)); } } #[cfg(feature = "zenoh")] "zenoh" => { if let Some(db) = crate::backend::zenoh::ZenohPubSub::open(config, prefix) .await .map(std::sync::Arc::new) { self.pubsub_stores .insert(store_id, crate::PubSubStore::Zenoh(db)); } } #[cfg(feature = "kafka")] "kafka" => { if let Some(db) = crate::backend::kafka::KafkaPubSub::open(config, prefix) .await .map(std::sync::Arc::new) { self.pubsub_stores .insert(store_id, crate::PubSubStore::Kafka(db)); } } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] "sql-read-replica" => { #[cfg(any(feature = "postgres", feature = "mysql"))] composite_stores.push(CompositeStore::SQLReadReplica(store_id)); } #[cfg(feature = "enterprise")] "distributed-blob" | "sharded-blob" => { composite_stores.push(CompositeStore::ShardedBlob(store_id)); } #[cfg(feature = "enterprise")] "sharded-in-memory" => { composite_stores.push(CompositeStore::ShardedInMemory(store_id)); } // SPDX-SnippetEnd #[cfg(feature = "azure")] "azure" => { if let Some(db) = crate::backend::azure::AzureStore::open(config, prefix) .await .map(BlobStore::from) { self.blob_stores .insert(store_id, db.with_compression(compression_algo)); } } unknown => { config.new_parse_warning( ("store", id, "type"), format!("Unknown directory type: {unknown:?}"), ); } } } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] for composite_store in composite_stores { match composite_store { #[cfg(any(feature = "postgres", feature = "mysql"))] CompositeStore::SQLReadReplica(id) => { let prefix = ("store", id.as_str()); if let Some(db) = crate::backend::composite::read_replica::SQLReadReplica::open( config, prefix, self, config.is_active_store(&id), config.is_active_search_store(&id), ) .await { let db = Store::SQLReadReplica(db.into()); self.stores.insert(id.to_string(), db.clone()); self.search_stores.insert(id.to_string(), db.clone().into()); self.blob_stores.insert( id.to_string(), BlobStore::from(db.clone()).with_compression( config .property_or_default::( ("store", id.as_str(), "compression"), "none", ) .unwrap_or(CompressionAlgo::None), ), ); self.in_memory_stores.insert(id, db.into()); } } CompositeStore::ShardedBlob(id) => { let prefix = ("store", id.as_str()); if let Some(db) = crate::backend::composite::sharded_blob::ShardedBlob::open( config, prefix, self, ) { let store = BlobStore { backend: crate::BlobBackend::Sharded(db.into()), compression: config .property_or_default::( ("store", id.as_str(), "compression"), "none", ) .unwrap_or(CompressionAlgo::None), }; self.blob_stores.insert(id, store); } } CompositeStore::ShardedInMemory(id) => { let prefix = ("store", id.as_str()); if let Some(db) = crate::backend::composite::sharded_lookup::ShardedInMemory::open( config, prefix, self, ) { self.in_memory_stores .insert(id, InMemoryStore::Sharded(db.into())); } } } } // SPDX-SnippetEnd } pub async fn parse_in_memory(&mut self, config: &mut Config, is_reload: bool) { // Parse memory stores self.parse_static_stores(config, is_reload); // Parse http stores self.parse_http_stores(config, is_reload); // Parse purge schedules if let Some(store) = config .value("storage.data") .and_then(|store_id| self.stores.get(store_id)) { let store_id = config.value("storage.data").unwrap().to_string(); self.purge_schedules.push(PurgeSchedule { cron: config .property_or_default::( ("store", store_id.as_str(), "purge.frequency"), "0 3 *", ) .unwrap_or_else(|| SimpleCron::parse_value("0 3 *").unwrap()), store_id, store: PurgeStore::Data(store.clone()), }); if let Some(blob_store) = config .value("storage.blob") .and_then(|blob_store_id| self.blob_stores.get(blob_store_id)) { let store_id = config.value("storage.blob").unwrap().to_string(); self.purge_schedules.push(PurgeSchedule { cron: config .property_or_default::( ("store", store_id.as_str(), "purge.frequency"), "0 4 *", ) .unwrap_or_else(|| SimpleCron::parse_value("0 4 *").unwrap()), store_id, store: PurgeStore::Blobs { store: store.clone(), blob_store: blob_store.clone(), }, }); } } for (store_id, store) in &self.in_memory_stores { if matches!(store, InMemoryStore::Store(_)) && config.is_active_in_memory_store(store_id) { self.purge_schedules.push(PurgeSchedule { cron: config .property_or_default::( ("store", store_id.as_str(), "purge.frequency"), "0 5 *", ) .unwrap_or_else(|| SimpleCron::parse_value("0 5 *").unwrap()), store_id: store_id.clone(), store: PurgeStore::Lookup(store.clone()), }); } } } } #[allow(dead_code)] trait IsActiveStore { fn is_active_store(&self, id: &str) -> bool; fn is_active_in_memory_store(&self, id: &str) -> bool; fn is_active_search_store(&self, id: &str) -> bool; } impl IsActiveStore for Config { fn is_active_store(&self, id: &str) -> bool { for key in [ "storage.data", "storage.blob", "storage.lookup", "tracing.history.store", "metrics.history.store", ] { if let Some(store_id) = self.value(key) && store_id == id { return true; } } false } fn is_active_search_store(&self, id: &str) -> bool { self.value("storage.fts") .is_some_and(|store_id| store_id == id) } fn is_active_in_memory_store(&self, id: &str) -> bool { self.value("storage.lookup") .is_some_and(|store_id| store_id == id) } } ================================================ FILE: crates/store/src/dispatch/blob.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{borrow::Cow, ops::Range, time::Instant}; use trc::{AddContext, StoreEvent}; use utils::config::utils::ParseValue; use crate::{BlobBackend, BlobStore, CompressionAlgo, Store}; impl BlobStore { pub async fn get_blob(&self, key: &[u8], range: Range) -> trc::Result>> { let read_range = match self.compression { CompressionAlgo::None => range.clone(), CompressionAlgo::Lz4 => 0..usize::MAX, }; let start_time = Instant::now(); let result = match &self.backend { BlobBackend::Store(store) => match store { #[cfg(feature = "sqlite")] Store::SQLite(store) => store.get_blob(key, read_range).await, #[cfg(feature = "foundation")] Store::FoundationDb(store) => store.get_blob(key, read_range).await, #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.get_blob(key, read_range).await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.get_blob(key, read_range).await, #[cfg(feature = "rocks")] Store::RocksDb(store) => store.get_blob(key, read_range).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Store::SQLReadReplica(store) => store.get_blob(key, read_range).await, // SPDX-SnippetEnd Store::None => Err(trc::StoreEvent::NotConfigured.into()), }, BlobBackend::Fs(store) => store.get_blob(key, read_range).await, #[cfg(feature = "s3")] BlobBackend::S3(store) => store.get_blob(key, read_range).await, #[cfg(feature = "azure")] BlobBackend::Azure(store) => store.get_blob(key, read_range).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] BlobBackend::Sharded(store) => store.get_blob(key, read_range).await, // SPDX-SnippetEnd }; trc::event!( Store(StoreEvent::BlobRead), Key = key, Elapsed = start_time.elapsed(), Size = result .as_ref() .map_or(0, |data| data.as_ref().map_or(0, |data| data.len())), ); let decompressed = match self.compression { CompressionAlgo::Lz4 => match result.caused_by(trc::location!())? { Some(data) if data.last().copied().unwrap_or_default() == CompressionAlgo::Lz4.marker() => { lz4_flex::decompress_size_prepended( data.get(..data.len() - 1).unwrap_or_default(), ) .map_err(|err| { trc::StoreEvent::DecompressError .reason(err) .ctx(trc::Key::Key, key) .ctx(trc::Key::CausedBy, trc::location!()) })? } Some(data) => { trc::event!(Store(StoreEvent::BlobMissingMarker), Key = key,); data } None => return Ok(None), }, _ => return result, }; if range.end > decompressed.len() { Ok(Some(decompressed)) } else { Ok(Some( decompressed .get(range.start..range.end) .unwrap_or_default() .to_vec(), )) } } pub async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { let data: Cow<[u8]> = match self.compression { CompressionAlgo::None => data.into(), CompressionAlgo::Lz4 => { let mut compressed = lz4_flex::compress_prepend_size(data); compressed.push(CompressionAlgo::Lz4.marker()); compressed.into() } }; let start_time = Instant::now(); let result = match &self.backend { BlobBackend::Store(store) => match store { #[cfg(feature = "sqlite")] Store::SQLite(store) => store.put_blob(key, data.as_ref()).await, #[cfg(feature = "foundation")] Store::FoundationDb(store) => store.put_blob(key, data.as_ref()).await, #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.put_blob(key, data.as_ref()).await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.put_blob(key, data.as_ref()).await, #[cfg(feature = "rocks")] Store::RocksDb(store) => store.put_blob(key, data.as_ref()).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Store::SQLReadReplica(store) => store.put_blob(key, data.as_ref()).await, // SPDX-SnippetEnd Store::None => Err(trc::StoreEvent::NotConfigured.into()), }, BlobBackend::Fs(store) => store.put_blob(key, data.as_ref()).await, #[cfg(feature = "s3")] BlobBackend::S3(store) => store.put_blob(key, data.as_ref()).await, #[cfg(feature = "azure")] BlobBackend::Azure(store) => store.put_blob(key, data.as_ref()).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] BlobBackend::Sharded(store) => store.put_blob(key, data.as_ref()).await, // SPDX-SnippetEnd } .caused_by(trc::location!()); trc::event!( Store(StoreEvent::BlobWrite), Key = key, Elapsed = start_time.elapsed(), Size = data.len(), ); result } pub async fn delete_blob(&self, key: &[u8]) -> trc::Result { let start_time = Instant::now(); let result = match &self.backend { BlobBackend::Store(store) => match store { #[cfg(feature = "sqlite")] Store::SQLite(store) => store.delete_blob(key).await, #[cfg(feature = "foundation")] Store::FoundationDb(store) => store.delete_blob(key).await, #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.delete_blob(key).await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.delete_blob(key).await, #[cfg(feature = "rocks")] Store::RocksDb(store) => store.delete_blob(key).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Store::SQLReadReplica(store) => store.delete_blob(key).await, // SPDX-SnippetEnd Store::None => Err(trc::StoreEvent::NotConfigured.into()), }, BlobBackend::Fs(store) => store.delete_blob(key).await, #[cfg(feature = "s3")] BlobBackend::S3(store) => store.delete_blob(key).await, #[cfg(feature = "azure")] BlobBackend::Azure(store) => store.delete_blob(key).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] BlobBackend::Sharded(store) => store.delete_blob(key).await, // SPDX-SnippetEnd } .caused_by(trc::location!()); trc::event!( Store(StoreEvent::BlobWrite), Key = key, Elapsed = start_time.elapsed(), ); result } pub fn with_compression(self, compression: CompressionAlgo) -> Self { Self { backend: self.backend, compression, } } } const MAGIC_MARKER: u8 = 0xa0; impl CompressionAlgo { pub fn marker(&self) -> u8 { match self { CompressionAlgo::Lz4 => MAGIC_MARKER | 0x01, //CompressionAlgo::Zstd => MAGIC_MARKER | 0x02, CompressionAlgo::None => 0, } } } impl ParseValue for CompressionAlgo { fn parse_value(value: &str) -> Result { match value { "lz4" => Ok(CompressionAlgo::Lz4), //"zstd" => Ok(CompressionAlgo::Zstd), "none" | "false" | "disable" | "disabled" => Ok(CompressionAlgo::None), algo => Err(format!("Invalid compression algorithm: {algo}",)), } } } ================================================ FILE: crates/store/src/dispatch/lookup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::borrow::Cow; use trc::AddContext; use utils::config::Rate; #[allow(unused_imports)] use crate::{ Deserialize, InMemoryStore, IterateParams, QueryResult, Store, U64_LEN, Value, ValueKey, write::{ BatchBuilder, Operation, ValueClass, ValueOp, key::{DeserializeBigEndian, KeySerializer}, now, }, }; use crate::{ SerializeInfallible, backend::http::lookup::HttpStoreGet, write::{InMemoryClass, assert::AssertValue}, }; pub struct KeyValue { pub key: Vec, pub value: T, pub expires: Option, } impl InMemoryStore { pub async fn key_set(&self, kv: KeyValue>) -> trc::Result<()> { match self { InMemoryStore::Store(store) => { let mut batch = BatchBuilder::new(); batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Key(kv.key)), op: ValueOp::Set( KeySerializer::new(kv.value.len() + U64_LEN) .write(kv.expires.map_or(u64::MAX, |expires| now() + expires)) .write(kv.value.as_slice()) .finalize(), ), }); store.write(batch.build_all()).await.map(|_| ()) } #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_set(&kv.key, &kv.value, kv.expires).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] InMemoryStore::Sharded(store) => store.key_set(kv).await, // SPDX-SnippetEnd InMemoryStore::Static(_) | InMemoryStore::Http(_) => { Err(trc::StoreEvent::NotSupported.into_err()) } } .caused_by(trc::location!()) } pub async fn counter_incr(&self, kv: KeyValue, return_value: bool) -> trc::Result { match self { InMemoryStore::Store(store) => { let mut batch = BatchBuilder::new(); if let Some(expires) = kv.expires { batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Key(kv.key.clone())), op: ValueOp::Set( KeySerializer::new(U64_LEN * 2) .write(0u64) .write(now() + expires) .finalize(), ), }); } if return_value { batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Counter(kv.key)), op: ValueOp::AddAndGet(kv.value), }); store .write(batch.build_all()) .await .and_then(|r| r.last_counter_id()) } else { batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Counter(kv.key)), op: ValueOp::AtomicAdd(kv.value), }); store.write(batch.build_all()).await.map(|_| 0) } } #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_incr(&kv.key, kv.value, kv.expires).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] InMemoryStore::Sharded(store) => store.counter_incr(kv).await, // SPDX-SnippetEnd InMemoryStore::Static(_) | InMemoryStore::Http(_) => { Err(trc::StoreEvent::NotSupported.into_err()) } } .caused_by(trc::location!()) } pub async fn key_delete(&self, key: impl Into>) -> trc::Result<()> { match self { InMemoryStore::Store(store) => { let mut batch = BatchBuilder::new(); batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Key(key.into().into_bytes())), op: ValueOp::Clear, }); store.write(batch.build_all()).await.map(|_| ()) } #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_delete(key.into().as_bytes()).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] InMemoryStore::Sharded(store) => store.key_delete(key).await, // SPDX-SnippetEnd InMemoryStore::Static(_) | InMemoryStore::Http(_) => { Err(trc::StoreEvent::NotSupported.into_err()) } } .caused_by(trc::location!()) } pub async fn counter_delete(&self, key: impl Into>) -> trc::Result<()> { match self { InMemoryStore::Store(store) => { let mut batch = BatchBuilder::new(); batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Counter(key.into().into_bytes())), op: ValueOp::Clear, }); store.write(batch.build_all()).await.map(|_| ()) } #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_delete(key.into().as_bytes()).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] InMemoryStore::Sharded(store) => store.counter_delete(key).await, // SPDX-SnippetEnd InMemoryStore::Static(_) | InMemoryStore::Http(_) => { Err(trc::StoreEvent::NotSupported.into_err()) } } .caused_by(trc::location!()) } pub async fn key_delete_prefix(&self, prefix: &[u8]) -> trc::Result<()> { match self { InMemoryStore::Store(store) => { if prefix.is_empty() { return Ok(()); } let from_range = prefix.to_vec(); let mut to_range = Vec::with_capacity(prefix.len() + 3); to_range.extend_from_slice(prefix); to_range.extend_from_slice([u8::MAX, u8::MAX, u8::MAX].as_ref()); store .delete_range( ValueKey::from(ValueClass::InMemory(InMemoryClass::Counter( from_range.clone(), ))), ValueKey::from(ValueClass::InMemory(InMemoryClass::Counter( to_range.clone(), ))), ) .await?; store .delete_range( ValueKey::from(ValueClass::InMemory(InMemoryClass::Key(from_range))), ValueKey::from(ValueClass::InMemory(InMemoryClass::Key(to_range))), ) .await } #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_delete_prefix(prefix).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] InMemoryStore::Sharded(store) => store.key_delete_prefix(prefix).await, // SPDX-SnippetEnd InMemoryStore::Static(_) | InMemoryStore::Http(_) => { Err(trc::StoreEvent::NotSupported.into_err()) } } .caused_by(trc::location!()) } pub async fn key_get> + std::fmt::Debug + 'static>( &self, key: impl Into>, ) -> trc::Result> { match self { InMemoryStore::Store(store) => store .get_value::>(ValueKey::from(ValueClass::InMemory( InMemoryClass::Key(key.into().into_bytes()), ))) .await .map(|value| value.and_then(|v| v.into())), #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_get(key.into().as_bytes()).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] InMemoryStore::Sharded(store) => store.key_get(key).await, // SPDX-SnippetEnd InMemoryStore::Static(store) => Ok(store .get(key.into().as_str()) .map(|value| T::from(value.clone()))), InMemoryStore::Http(store) => { Ok(store.get(key.into().as_str()).map(|value| T::from(value))) } } .caused_by(trc::location!()) } pub async fn counter_get(&self, key: impl Into>) -> trc::Result { match self { InMemoryStore::Store(store) => { store .get_counter(ValueKey::from(ValueClass::InMemory( InMemoryClass::Counter(key.into().into_bytes()), ))) .await } #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.counter_get(key.into().as_bytes()).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] InMemoryStore::Sharded(store) => store.counter_get(key).await, // SPDX-SnippetEnd InMemoryStore::Static(_) | InMemoryStore::Http(_) => { Err(trc::StoreEvent::NotSupported.into_err()) } } .caused_by(trc::location!()) } pub async fn key_exists(&self, key: impl Into>) -> trc::Result { match self { InMemoryStore::Store(store) => store .get_value::>(ValueKey::from(ValueClass::InMemory( InMemoryClass::Key(key.into().into_bytes()), ))) .await .map(|value| matches!(value, Some(LookupValue::Value(())))), #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store.key_exists(key.into().as_bytes()).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] InMemoryStore::Sharded(store) => store.key_exists(key).await, // SPDX-SnippetEnd InMemoryStore::Static(store) => Ok(store.get(key.into().as_str()).is_some()), InMemoryStore::Http(store) => Ok(store.contains(key.into().as_str())), } .caused_by(trc::location!()) } pub async fn is_rate_allowed( &self, prefix: u8, key: &[u8], rate: &Rate, soft_check: bool, ) -> trc::Result> { let now = now(); let range_start = now / rate.period.as_secs(); let range_end = (range_start * rate.period.as_secs()) + rate.period.as_secs(); let expires_in = range_end - now; let mut bucket = Vec::with_capacity(key.len() + U64_LEN + 1); bucket.push(prefix); bucket.extend_from_slice(key); bucket.extend_from_slice(range_start.to_be_bytes().as_slice()); let requests = if !soft_check { self.counter_incr(KeyValue::new(bucket, 1).expires(expires_in), true) .await .caused_by(trc::location!())? } else { self.counter_get(bucket).await.caused_by(trc::location!())? + 1 }; if requests <= rate.requests as i64 { Ok(None) } else { Ok(Some(expires_in)) } } pub async fn try_lock(&self, prefix: u8, key: &[u8], duration: u64) -> trc::Result { match self { InMemoryStore::Store(store) => { let key = KeyValue::<()>::build_key(prefix, key); let lock_expiry = match store .get_value::(ValueKey::from(ValueClass::InMemory(InMemoryClass::Key( key.clone(), )))) .await { Ok(lock_expiry) => lock_expiry, Err(err) if err.matches(trc::EventType::Store(trc::StoreEvent::DataCorruption)) => { // TODO remove in 1.0 let mut batch = BatchBuilder::new(); batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Key(key.clone())), op: ValueOp::Clear, }); store .write(batch.build_all()) .await .caused_by(trc::location!())?; None } Err(err) => { return Err(err .details("Failed to read lock.") .caused_by(trc::location!())); } }; let now = now(); if lock_expiry.is_some_and(|expiry| expiry > now) { return Ok(false); } let key: ValueClass = ValueClass::InMemory(InMemoryClass::Key(key)); let mut batch = BatchBuilder::new(); batch.assert_value( key.clone(), match lock_expiry { Some(value) => AssertValue::U64(value), None => AssertValue::None, }, ); batch.set(key.clone(), (now + duration).serialize()); match store.write(batch.build_all()).await { Ok(_) => Ok(true), Err(err) if err.is_assertion_failure() => Ok(false), Err(err) => Err(err .details("Failed to lock event.") .caused_by(trc::location!())), } } #[cfg(feature = "redis")] InMemoryStore::Redis(store) => store .key_incr(&KeyValue::<()>::build_key(prefix, key), 1, duration.into()) .await .map(|count| count == 1), // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] InMemoryStore::Sharded(store) => store .counter_incr(KeyValue::with_prefix(prefix, key, 1).expires(duration)) .await .map(|count| count == 1), // SPDX-SnippetEnd InMemoryStore::Static(_) | InMemoryStore::Http(_) => { Err(trc::StoreEvent::NotSupported.into_err()) } } } pub async fn remove_lock(&self, prefix: u8, key: &[u8]) -> trc::Result<()> { self.key_delete(KeyValue::<()>::build_key(prefix, key)) .await } pub async fn purge_in_memory_store(&self) -> trc::Result<()> { match self { InMemoryStore::Store(store) => { // Delete expired keys and counters let from_key = ValueKey::from(ValueClass::InMemory(InMemoryClass::Key(vec![0u8]))); let to_key = ValueKey::from(ValueClass::InMemory(InMemoryClass::Key(vec![u8::MAX; 10]))); let current_time = now(); let mut expired_keys = Vec::new(); let mut expired_counters = Vec::new(); store .iterate(IterateParams::new(from_key, to_key), |key, value| { let expiry = value.deserialize_be_u64(0).caused_by(trc::location!())?; if expiry == 0 { if value .deserialize_be_u64(U64_LEN) .caused_by(trc::location!())? <= current_time { expired_counters.push(key.to_vec()); } } else if expiry <= current_time { expired_keys.push(key.to_vec()); } Ok(true) }) .await .caused_by(trc::location!())?; if !expired_keys.is_empty() { let mut batch = BatchBuilder::new(); for key in expired_keys { batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Key(key)), op: ValueOp::Clear, }); if batch.is_large_batch() { store .write(batch.build_all()) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); } } if !batch.is_empty() { store .write(batch.build_all()) .await .caused_by(trc::location!())?; } } if !expired_counters.is_empty() { let mut batch = BatchBuilder::new(); for key in expired_counters { batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Counter(key.clone())), op: ValueOp::Clear, }); batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Key(key)), op: ValueOp::Clear, }); if batch.is_large_batch() { store .write(batch.build_all()) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); } } if !batch.is_empty() { store .write(batch.build_all()) .await .caused_by(trc::location!())?; } } } #[cfg(feature = "redis")] InMemoryStore::Redis(_) => {} // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] InMemoryStore::Sharded(_) => {} // SPDX-SnippetEnd InMemoryStore::Static(_) | InMemoryStore::Http(_) => {} } Ok(()) } pub fn is_sql(&self) -> bool { match self { InMemoryStore::Store(store) => store.is_sql(), _ => false, } } pub fn is_redis(&self) -> bool { match self { #[cfg(feature = "redis")] InMemoryStore::Redis(_) => true, InMemoryStore::Static(_) => false, _ => false, } } } pub enum LookupKey<'x> { String(String), StringRef(&'x str), Bytes(Vec), BytesRef(&'x [u8]), } impl<'x> From<&'x str> for LookupKey<'x> { fn from(key: &'x str) -> Self { LookupKey::StringRef(key) } } impl<'x> From<&'x String> for LookupKey<'x> { fn from(key: &'x String) -> Self { LookupKey::StringRef(key.as_str()) } } impl<'x> From<&'x [u8]> for LookupKey<'x> { fn from(key: &'x [u8]) -> Self { LookupKey::BytesRef(key) } } impl<'x> From> for LookupKey<'x> { fn from(key: Cow<'x, str>) -> Self { match key { Cow::Borrowed(key) => LookupKey::StringRef(key), Cow::Owned(key) => LookupKey::String(key), } } } impl From for LookupKey<'static> { fn from(key: String) -> Self { LookupKey::String(key) } } impl From> for LookupKey<'static> { fn from(key: Vec) -> Self { LookupKey::Bytes(key) } } impl LookupKey<'_> { pub fn as_str(&self) -> &str { match self { LookupKey::String(string) => string, LookupKey::StringRef(string) => string, LookupKey::Bytes(bytes) => std::str::from_utf8(bytes).unwrap_or_default(), LookupKey::BytesRef(bytes) => std::str::from_utf8(bytes).unwrap_or_default(), } } pub fn into_bytes(self) -> Vec { match self { LookupKey::String(string) => string.into_bytes(), LookupKey::StringRef(string) => string.as_bytes().to_vec(), LookupKey::Bytes(bytes) => bytes, LookupKey::BytesRef(bytes) => bytes.to_vec(), } } pub fn as_bytes(&self) -> &[u8] { match self { LookupKey::String(string) => string.as_bytes(), LookupKey::StringRef(string) => string.as_bytes(), LookupKey::Bytes(bytes) => bytes.as_slice(), LookupKey::BytesRef(bytes) => bytes, } } } impl KeyValue { pub fn build_key(prefix: u8, key: impl AsRef<[u8]>) -> Vec { let key_ = key.as_ref(); let mut key = Vec::with_capacity(key_.len() + 1); key.push(prefix); key.extend_from_slice(key_); key } pub fn with_prefix(prefix: u8, key: impl AsRef<[u8]>, value: T) -> Self { Self { key: Self::build_key(prefix, key), value, expires: None, } } pub fn new(key: impl Into>, value: T) -> Self { Self { key: key.into(), value, expires: None, } } pub fn expires(mut self, expires: u64) -> Self { self.expires = expires.into(); self } pub fn expires_opt(mut self, expires: Option) -> Self { self.expires = expires; self } } enum LookupValue { Value(T), None, } impl Deserialize for LookupValue { fn deserialize(bytes: &[u8]) -> trc::Result { bytes.deserialize_be_u64(0).and_then(|expires| { Ok(if expires > now() { LookupValue::Value( T::deserialize(bytes.get(U64_LEN..).unwrap_or_default()) .caused_by(trc::location!())?, ) } else { LookupValue::None }) }) } } impl From> for Option { fn from(value: LookupValue) -> Self { match value { LookupValue::Value(value) => Some(value), LookupValue::None => None, } } } impl From> for String { fn from(value: Value<'static>) -> Self { match value { Value::Text(string) => string.into_owned(), Value::Blob(bytes) => String::from_utf8_lossy(bytes.as_ref()).into_owned(), Value::Bool(boolean) => boolean.to_string(), Value::Null => String::new(), Value::Integer(num) => num.to_string(), Value::Float(num) => num.to_string(), } } } ================================================ FILE: crates/store/src/dispatch/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use roaring::RoaringBitmap; use crate::Store; pub mod blob; pub mod lookup; pub mod pubsub; pub mod search; pub mod store; impl Store { pub fn id(&self) -> &'static str { match self { #[cfg(feature = "sqlite")] Self::SQLite(_) => "sqlite", #[cfg(feature = "foundation")] Self::FoundationDb(_) => "foundationdb", #[cfg(feature = "postgres")] Self::PostgreSQL(_) => "postgresql", #[cfg(feature = "mysql")] Self::MySQL(_) => "mysql", #[cfg(feature = "rocks")] Self::RocksDb(_) => "rocksdb", // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(_) => "read_replica", // SPDX-SnippetEnd Self::None => "none", } } } #[allow(clippy::len_without_is_empty)] pub trait DocumentSet: Sync + Send { fn min(&self) -> u32; fn max(&self) -> u32; fn contains(&self, id: u32) -> bool; fn len(&self) -> usize; fn iterate(&self) -> impl Iterator; } impl DocumentSet for RoaringBitmap { fn min(&self) -> u32 { self.min().unwrap_or(0) } fn max(&self) -> u32 { self.max().map(|m| m + 1).unwrap_or(0) } fn contains(&self, id: u32) -> bool { self.contains(id) } fn len(&self) -> usize { self.len() as usize } fn iterate(&self) -> impl Iterator { self.iter() } } impl DocumentSet for Vec { fn contains(&self, id: u32) -> bool { self.binary_search(&id).is_ok() } fn min(&self) -> u32 { self.first().copied().unwrap_or(0) } fn max(&self) -> u32 { self.last().copied().map(|m| m + 1).unwrap_or(0) } fn len(&self) -> usize { self.len() } fn iterate(&self) -> impl Iterator { self.iter().copied() } } impl DocumentSet for () { fn min(&self) -> u32 { 0 } fn max(&self) -> u32 { u32::MAX } fn contains(&self, _: u32) -> bool { true } fn len(&self) -> usize { 0 } fn iterate(&self) -> impl Iterator { std::iter::empty() } } ================================================ FILE: crates/store/src/dispatch/pubsub.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::PubSubStore; pub enum PubSubStream { #[cfg(feature = "redis")] Redis(crate::backend::redis::pubsub::RedisPubSubStream), #[cfg(feature = "redis")] RedisCluster(crate::backend::redis::pubsub::RedisClusterPubSubStream), #[cfg(feature = "nats")] Nats(crate::backend::nats::pubsub::NatsPubSubStream), #[cfg(feature = "zenoh")] Zenoh(crate::backend::zenoh::pubsub::ZenohPubSubStream), #[cfg(feature = "kafka")] Kafka(crate::backend::kafka::pubsub::KafkaPubSubStream), #[cfg(not(any(feature = "redis", feature = "nats")))] Unimplemented, } pub enum Msg { #[cfg(feature = "redis")] Redis(redis::Msg), #[cfg(feature = "nats")] Nats(async_nats::Message), #[cfg(feature = "zenoh")] Zenoh(Vec), #[cfg(feature = "kafka")] Kafka(Vec), #[cfg(not(any(feature = "redis", feature = "nats")))] Unimplemented, } #[allow(unused_variables)] impl PubSubStore { pub async fn publish(&self, topic: &'static str, message: Vec) -> trc::Result<()> { match self { #[cfg(feature = "redis")] PubSubStore::Redis(store) => store.publish(topic, message).await, #[cfg(feature = "nats")] PubSubStore::Nats(store) => store.publish(topic, message).await, #[cfg(feature = "zenoh")] PubSubStore::Zenoh(store) => store.publish(topic, message).await, #[cfg(feature = "kafka")] PubSubStore::Kafka(store) => store.publish(topic, message).await, PubSubStore::None => Err(trc::StoreEvent::NotSupported.into_err()), } } pub async fn subscribe(&self, topic: &'static str) -> trc::Result { match self { #[cfg(feature = "redis")] PubSubStore::Redis(store) => store.subscribe(topic).await, #[cfg(feature = "nats")] PubSubStore::Nats(store) => store.subscribe(topic).await, #[cfg(feature = "zenoh")] PubSubStore::Zenoh(store) => store.subscribe(topic).await, #[cfg(feature = "kafka")] PubSubStore::Kafka(store) => store.subscribe(topic).await, PubSubStore::None => Err(trc::StoreEvent::NotSupported.into_err()), } } pub fn is_none(&self) -> bool { matches!(self, PubSubStore::None) } } impl PubSubStream { pub async fn next(&mut self) -> Option { match self { #[cfg(feature = "redis")] PubSubStream::Redis(stream) => stream.next().await, #[cfg(feature = "redis")] PubSubStream::RedisCluster(stream) => stream.next().await, #[cfg(feature = "nats")] PubSubStream::Nats(stream) => stream.next().await, #[cfg(feature = "zenoh")] PubSubStream::Zenoh(stream) => stream.next().await, #[cfg(feature = "kafka")] PubSubStream::Kafka(stream) => stream.next().await, #[cfg(not(any(feature = "redis", feature = "nats")))] PubSubStream::Unimplemented => None, } } } impl Msg { pub fn payload(&self) -> &[u8] { match self { #[cfg(feature = "redis")] Msg::Redis(msg) => msg.get_payload_bytes(), #[cfg(feature = "nats")] Msg::Nats(msg) => msg.payload.as_ref(), #[cfg(feature = "zenoh")] Msg::Zenoh(msg) => msg.as_slice(), #[cfg(feature = "kafka")] Msg::Kafka(msg) => msg.as_slice(), #[cfg(not(any(feature = "redis", feature = "nats")))] Msg::Unimplemented => &[], } } pub fn topic(&self) -> &str { match self { #[cfg(feature = "redis")] Msg::Redis(msg) => msg.get_channel_name(), #[cfg(feature = "nats")] Msg::Nats(msg) => msg.subject.as_str(), #[cfg(feature = "zenoh")] Msg::Zenoh(_) => "", #[cfg(feature = "kafka")] Msg::Kafka(_) => "", #[cfg(not(any(feature = "redis", feature = "nats")))] Msg::Unimplemented => "", } } } ================================================ FILE: crates/store/src/dispatch/search.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use trc::AddContext; use crate::{ SearchStore, Store, search::{ IndexDocument, SearchComparator, SearchField, SearchFilter, SearchOperator, SearchQuery, SearchValue, split::{SplitFilter, split_filters}, }, write::SearchIndex, }; use std::cmp::Ordering; impl SearchStore { pub async fn query_account(&self, query: SearchQuery) -> trc::Result> { // Pre-filter by mask if query.mask.is_empty() { return Ok(vec![]); } // If the store does not support FTS, use the internal FTS store if let Some(store) = self.internal_fts() { return store.query_account(query).await; } // If all filters and comparators are external, delegate to the underlying store let mut account_id = u32::MAX; let mut has_local_filters = false; let mut has_external_filters = false; for filter in &query.filters { match filter { SearchFilter::Operator { field: SearchField::AccountId, op: SearchOperator::Equal, value: SearchValue::Uint(id), } => { account_id = *id as u32; } SearchFilter::DocumentSet(_) => { has_local_filters = true; } SearchFilter::Operator { .. } => { has_external_filters = true; } _ => (), } } if account_id == u32::MAX { return Err(trc::StoreEvent::UnexpectedError .reason("Account ID filter is required for account queries") .caused_by(trc::location!())); } if !has_local_filters && !has_external_filters && query.comparators.is_empty() { return Ok(query.mask.iter().collect()); } if !has_local_filters && query.comparators.iter().all(|c| c.is_external()) { return self .sub_query(query.index, &query.filters, &query.comparators) .await .map(|results| { if !results.is_empty() || has_external_filters { results .into_iter() .filter(|id| query.mask.contains(*id)) .collect() } else { // Database sort is broken, return masked results query.mask.iter().collect() } }) .caused_by(trc::location!()); } let filters = if has_external_filters { // Split filters let split_filters = split_filters(query.filters).ok_or_else(|| { trc::StoreEvent::UnexpectedError .reason("Invalid filter query") .caused_by(trc::location!()) })?; let mut filters = Vec::with_capacity(split_filters.len()); for split_filter in split_filters { match split_filter { SplitFilter::External(external) => { // Execute sub-query filters.push(SearchFilter::DocumentSet( self.sub_query(query.index, &external, &[]) .await? .into_iter() .collect(), )); } SplitFilter::Internal(filter) => { filters.push(filter); } } } filters } else { query.filters }; // Merge results locally let results = SearchQuery::new(query.index) .with_filters(filters) .with_mask(query.mask) .filter(); let total_results = results.results().len(); match total_results.cmp(&1) { Ordering::Equal => Ok(vec![results.results().min().unwrap()]), Ordering::Less => Ok(vec![]), Ordering::Greater => { if !query.comparators.is_empty() { let mut local = Vec::with_capacity(query.comparators.len()); let mut external = Vec::with_capacity(query.comparators.len()); let mut external_first = false; for (pos, comparator) in query.comparators.into_iter().enumerate() { if comparator.is_external() { external.push(comparator); if pos == 0 { external_first = true; } } else { local.push(comparator); } } if !external.is_empty() { let mut results = results.results().clone(); let filters = vec![ SearchFilter::Operator { field: SearchField::AccountId, op: SearchOperator::Equal, value: SearchValue::Uint(account_id as u64), }, SearchFilter::Operator { field: SearchField::DocumentId, op: SearchOperator::GreaterEqualThan, value: SearchValue::Uint(results.min().unwrap() as u64), }, SearchFilter::Operator { field: SearchField::DocumentId, op: SearchOperator::LowerEqualThan, value: SearchValue::Uint(results.max().unwrap() as u64), }, ]; let mut ordered_results = Vec::with_capacity(total_results as usize); for ordered_result in self.sub_query(query.index, &filters, &external).await? { if results.remove(ordered_result) { ordered_results.push(ordered_result); } } // Add any remaining results not yet in the index ordered_results.extend(results.into_iter()); if local.is_empty() { return Ok(ordered_results); } let comparator = SearchComparator::SortedSet { set: ordered_results .into_iter() .enumerate() .map(|(pos, id)| (id, pos as u32)) .collect(), ascending: true, }; if external_first { local.insert(0, comparator); } else { local.push(comparator); } } Ok(results.with_comparators(local).into_sorted()) } else { Ok(results.results().iter().collect()) } } } } async fn sub_query( &self, index: SearchIndex, filters: &[SearchFilter], sort: &[SearchComparator], ) -> trc::Result> { match self { SearchStore::Store(store) => match store { #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.query(index, filters, sort).await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.query(index, filters, sort).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Store::SQLReadReplica(store) => store.query(index, filters, sort).await, // SPDX-SnippetEnd _ => unreachable!(), }, SearchStore::ElasticSearch(store) => store.query(index, filters, sort).await, SearchStore::MeiliSearch(store) => store.query(index, filters, sort).await, } } pub async fn query_global(&self, query: SearchQuery) -> trc::Result> { match self { SearchStore::Store(store) => match store { #[cfg(feature = "postgres")] Store::PostgreSQL(store) => { store .query(query.index, &query.filters, &query.comparators) .await } #[cfg(feature = "mysql")] Store::MySQL(store) => { store .query(query.index, &query.filters, &query.comparators) .await } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Store::SQLReadReplica(store) => { store .query(query.index, &query.filters, &query.comparators) .await } // SPDX-SnippetEnd store => store.query_global(query).await, }, SearchStore::ElasticSearch(store) => { store .query(query.index, &query.filters, &query.comparators) .await } SearchStore::MeiliSearch(store) => { store .query(query.index, &query.filters, &query.comparators) .await } } } pub async fn index(&self, documents: Vec) -> trc::Result<()> { match self { SearchStore::Store(store) => match store { #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.index(documents).await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.index(documents).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Store::SQLReadReplica(store) => store.index(documents).await, // SPDX-SnippetEnd store => store.index(documents).await, }, SearchStore::ElasticSearch(store) => store.index(documents).await, SearchStore::MeiliSearch(store) => store.index(documents).await, } } pub async fn unindex(&self, query: SearchQuery) -> trc::Result { match self { SearchStore::Store(store) => match store { #[cfg(feature = "postgres")] Store::PostgreSQL(store) => store.unindex(query).await, #[cfg(feature = "mysql")] Store::MySQL(store) => store.unindex(query).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Store::SQLReadReplica(store) => store.unindex(query).await, // SPDX-SnippetEnd store => store.unindex(query).await.map(|_| 0), }, SearchStore::ElasticSearch(store) => store.unindex(query).await, SearchStore::MeiliSearch(store) => store.unindex(query).await, } } pub fn internal_fts(&self) -> Option<&Store> { match self { SearchStore::Store(store) => match store { #[cfg(feature = "postgres")] Store::PostgreSQL(_) => None, #[cfg(feature = "mysql")] Store::MySQL(_) => None, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Store::SQLReadReplica(_) => None, // SPDX-SnippetEnd store => Some(store), }, _ => None, } } pub fn is_mysql(&self) -> bool { match self { #[cfg(feature = "mysql")] SearchStore::Store(Store::MySQL(_)) => true, _ => false, } } pub fn is_postgres(&self) -> bool { match self { #[cfg(feature = "postgres")] SearchStore::Store(Store::PostgreSQL(_)) => true, _ => false, } } pub fn is_elasticsearch(&self) -> bool { matches!(self, SearchStore::ElasticSearch(_)) } pub fn is_meilisearch(&self) -> bool { matches!(self, SearchStore::MeiliSearch(_)) } } impl SearchFilter { pub fn is_external(&self) -> bool { matches!(self, SearchFilter::Operator { .. }) } } impl SearchComparator { pub fn is_external(&self) -> bool { matches!(self, SearchComparator::Field { .. }) } } ================================================ FILE: crates/store/src/dispatch/store.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::DocumentSet; use crate::{ Deserialize, IterateParams, Key, QueryResult, SUBSPACE_BLOB_EXTRA, SUBSPACE_COUNTER, SUBSPACE_INDEXES, SUBSPACE_LOGS, Store, U32_LEN, Value, ValueKey, write::{ AnyClass, AnyKey, AssignedIds, Batch, BatchBuilder, Operation, ReportClass, ValueClass, ValueOp, key::{DeserializeBigEndian, KeySerializer}, now, }, }; use compact_str::ToCompactString; use std::{ops::Range, time::Instant}; use trc::{AddContext, StoreEvent}; use types::collection::Collection; impl Store { pub async fn get_value(&self, key: impl Key) -> trc::Result> where U: Deserialize + 'static, { match self { #[cfg(feature = "sqlite")] Self::SQLite(store) => store.get_value(key).await, #[cfg(feature = "foundation")] Self::FoundationDb(store) => store.get_value(key).await, #[cfg(feature = "postgres")] Self::PostgreSQL(store) => store.get_value(key).await, #[cfg(feature = "mysql")] Self::MySQL(store) => store.get_value(key).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.get_value(key).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(store) => store.get_value(key).await, // SPDX-SnippetEnd Self::None => Err(trc::StoreEvent::NotConfigured.into()), } .caused_by(trc::location!()) } pub async fn iterate( &self, params: IterateParams, cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, ) -> trc::Result<()> { let start_time = Instant::now(); let result = match self { #[cfg(feature = "sqlite")] Self::SQLite(store) => store.iterate(params, cb).await, #[cfg(feature = "foundation")] Self::FoundationDb(store) => store.iterate(params, cb).await, #[cfg(feature = "postgres")] Self::PostgreSQL(store) => store.iterate(params, cb).await, #[cfg(feature = "mysql")] Self::MySQL(store) => store.iterate(params, cb).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.iterate(params, cb).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(store) => store.iterate(params, cb).await, // SPDX-SnippetEnd Self::None => Err(trc::StoreEvent::NotConfigured.into()), } .caused_by(trc::location!()); trc::event!( Store(StoreEvent::DataIterate), Elapsed = start_time.elapsed(), ); result } pub async fn get_counter( &self, key: impl Into> + Sync + Send, ) -> trc::Result { match self { #[cfg(feature = "sqlite")] Self::SQLite(store) => store.get_counter(key).await, #[cfg(feature = "foundation")] Self::FoundationDb(store) => store.get_counter(key).await, #[cfg(feature = "postgres")] Self::PostgreSQL(store) => store.get_counter(key).await, #[cfg(feature = "mysql")] Self::MySQL(store) => store.get_counter(key).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.get_counter(key).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(store) => store.get_counter(key).await, // SPDX-SnippetEnd Self::None => Err(trc::StoreEvent::NotConfigured.into()), } .caused_by(trc::location!()) } #[allow(unreachable_patterns)] #[allow(unused_variables)] pub async fn sql_query( &self, query: &str, params: Vec>, ) -> trc::Result { let result = match self { #[cfg(feature = "sqlite")] Self::SQLite(store) => store.sql_query(query, ¶ms).await, #[cfg(feature = "postgres")] Self::PostgreSQL(store) => store.sql_query(query, ¶ms).await, #[cfg(feature = "mysql")] Self::MySQL(store) => store.sql_query(query, ¶ms).await, _ => Err(trc::StoreEvent::NotSupported.into_err()), }; trc::event!( Store(trc::StoreEvent::SqlQuery), Details = query.to_compact_string(), Value = params.as_slice(), Result = &result, ); result.caused_by(trc::location!()) } pub async fn write(&self, batch: Batch<'_>) -> trc::Result { let start_time = Instant::now(); let ops = batch.ops.len(); let result = match self { #[cfg(feature = "sqlite")] Self::SQLite(store) => store.write(batch).await, #[cfg(feature = "foundation")] Self::FoundationDb(store) => store.write(batch).await, #[cfg(feature = "postgres")] Self::PostgreSQL(store) => store.write(batch).await, #[cfg(feature = "mysql")] Self::MySQL(store) => store.write(batch).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.write(batch).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(store) => store.write(batch).await, // SPDX-SnippetEnd Self::None => Err(trc::StoreEvent::NotConfigured.into()), }; trc::event!( Store(StoreEvent::DataWrite), Elapsed = start_time.elapsed(), Total = ops, ); result } pub async fn assign_document_ids( &self, account_id: u32, collection: Collection, num_ids: u64, ) -> trc::Result { // Increment UID next let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(collection) .add_and_get(ValueClass::DocumentId, num_ids as i64); self.write(batch.build_all()).await.and_then(|v| { v.last_counter_id().map(|id| { debug_assert!(id >= num_ids as i64, "{} < {}", id, num_ids); id as u32 }) }) } pub async fn purge_store(&self) -> trc::Result<()> { // Delete expired reports let now = now(); self.delete_range( ValueKey::from(ValueClass::Report(ReportClass::Dmarc { id: 0, expires: 0 })), ValueKey::from(ValueClass::Report(ReportClass::Dmarc { id: u64::MAX, expires: now, })), ) .await .caused_by(trc::location!())?; self.delete_range( ValueKey::from(ValueClass::Report(ReportClass::Tls { id: 0, expires: 0 })), ValueKey::from(ValueClass::Report(ReportClass::Tls { id: u64::MAX, expires: now, })), ) .await .caused_by(trc::location!())?; self.delete_range( ValueKey::from(ValueClass::Report(ReportClass::Arf { id: 0, expires: 0 })), ValueKey::from(ValueClass::Report(ReportClass::Arf { id: u64::MAX, expires: now, })), ) .await .caused_by(trc::location!())?; match self { #[cfg(feature = "sqlite")] Self::SQLite(store) => store.purge_store().await, #[cfg(feature = "foundation")] Self::FoundationDb(store) => store.purge_store().await, #[cfg(feature = "postgres")] Self::PostgreSQL(store) => store.purge_store().await, #[cfg(feature = "mysql")] Self::MySQL(store) => store.purge_store().await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.purge_store().await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(store) => store.purge_store().await, // SPDX-SnippetEnd Self::None => Err(trc::StoreEvent::NotConfigured.into()), } .caused_by(trc::location!()) } pub async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> { match self { #[cfg(feature = "sqlite")] Self::SQLite(store) => store.delete_range(from, to).await, #[cfg(feature = "foundation")] Self::FoundationDb(store) => store.delete_range(from, to).await, #[cfg(feature = "postgres")] Self::PostgreSQL(store) => store.delete_range(from, to).await, #[cfg(feature = "mysql")] Self::MySQL(store) => store.delete_range(from, to).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.delete_range(from, to).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(store) => store.delete_range(from, to).await, // SPDX-SnippetEnd Self::None => Err(trc::StoreEvent::NotConfigured.into()), } .caused_by(trc::location!()) } pub async fn delete_documents( &self, subspace: u8, account_id: u32, collection: u8, collection_offset: Option, document_ids: &impl DocumentSet, ) -> trc::Result<()> { // Serialize keys let (from_key, to_key) = if collection_offset.is_some() { ( KeySerializer::new(U32_LEN + 2) .write(account_id) .write(collection), KeySerializer::new(U32_LEN + 2) .write(account_id) .write(collection + 1), ) } else { ( KeySerializer::new(U32_LEN).write(account_id), KeySerializer::new(U32_LEN).write(account_id + 1), ) }; // Find keys to delete let mut delete_keys = Vec::new(); self.iterate( IterateParams::new( AnyKey { subspace, key: from_key.finalize(), }, AnyKey { subspace, key: to_key.finalize(), }, ) .no_values(), |key, _| { if collection_offset.is_none_or(|offset| { key.get(key.len() - U32_LEN - offset).copied() == Some(collection) }) { let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; if document_ids.contains(document_id) { delete_keys.push(key.to_vec()); } } Ok(true) }, ) .await .caused_by(trc::location!())?; // Remove keys let mut batch = BatchBuilder::new(); for key in delete_keys { if batch.is_large_batch() { self.write(std::mem::take(&mut batch).build_all()) .await .caused_by(trc::location!())?; } batch.any_op(Operation::Value { class: ValueClass::Any(AnyClass { subspace, key }), op: ValueOp::Clear, }); } if !batch.is_empty() { self.write(batch.build_all()) .await .caused_by(trc::location!())?; } Ok(()) } pub async fn danger_destroy_account(&self, account_id: u32) -> trc::Result<()> { for subspace in [ SUBSPACE_LOGS, SUBSPACE_INDEXES, SUBSPACE_COUNTER, SUBSPACE_BLOB_EXTRA, ] { self.delete_range( AnyKey { subspace, key: KeySerializer::new(U32_LEN).write(account_id).finalize(), }, AnyKey { subspace, key: KeySerializer::new(U32_LEN).write(account_id + 1).finalize(), }, ) .await .caused_by(trc::location!())?; } for (from_class, to_class) in [ (ValueClass::Acl(account_id), ValueClass::Acl(account_id + 1)), (ValueClass::Property(0), ValueClass::Property(0)), ] { self.delete_range( ValueKey { account_id, collection: 0, document_id: 0, class: from_class, }, ValueKey { account_id: account_id + 1, collection: 0, document_id: 0, class: to_class, }, ) .await .caused_by(trc::location!())?; } Ok(()) } pub async fn get_blob(&self, key: &[u8], range: Range) -> trc::Result>> { match self { #[cfg(feature = "sqlite")] Self::SQLite(store) => store.get_blob(key, range).await, #[cfg(feature = "foundation")] Self::FoundationDb(store) => store.get_blob(key, range).await, #[cfg(feature = "postgres")] Self::PostgreSQL(store) => store.get_blob(key, range).await, #[cfg(feature = "mysql")] Self::MySQL(store) => store.get_blob(key, range).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.get_blob(key, range).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(store) => store.get_blob(key, range).await, // SPDX-SnippetEnd Self::None => Err(trc::StoreEvent::NotConfigured.into()), } .caused_by(trc::location!()) } pub async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> { match self { #[cfg(feature = "sqlite")] Self::SQLite(store) => store.put_blob(key, data).await, #[cfg(feature = "foundation")] Self::FoundationDb(store) => store.put_blob(key, data).await, #[cfg(feature = "postgres")] Self::PostgreSQL(store) => store.put_blob(key, data).await, #[cfg(feature = "mysql")] Self::MySQL(store) => store.put_blob(key, data).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.put_blob(key, data).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(store) => store.put_blob(key, data).await, // SPDX-SnippetEnd Self::None => Err(trc::StoreEvent::NotConfigured.into()), } .caused_by(trc::location!()) } pub async fn delete_blob(&self, key: &[u8]) -> trc::Result { match self { #[cfg(feature = "sqlite")] Self::SQLite(store) => store.delete_blob(key).await, #[cfg(feature = "foundation")] Self::FoundationDb(store) => store.delete_blob(key).await, #[cfg(feature = "postgres")] Self::PostgreSQL(store) => store.delete_blob(key).await, #[cfg(feature = "mysql")] Self::MySQL(store) => store.delete_blob(key).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.delete_blob(key).await, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(store) => store.delete_blob(key).await, // SPDX-SnippetEnd Self::None => Err(trc::StoreEvent::NotConfigured.into()), } .caused_by(trc::location!()) } } ================================================ FILE: crates/store/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod backend; pub mod config; pub mod dispatch; pub mod query; pub mod search; pub mod write; pub use ahash; pub use blake3; pub use parking_lot; pub use rand; pub use rkyv; pub use roaring; pub use xxhash_rust; use ahash::AHashMap; use backend::{fs::FsStore, http::HttpStore, memory::StaticMemoryStore}; use std::{borrow::Cow, sync::Arc}; use utils::config::cron::SimpleCron; use write::ValueClass; use crate::backend::{elastic::ElasticSearchStore, meili::MeiliSearchStore}; pub trait Deserialize: Sized + Sync + Send { fn deserialize(bytes: &[u8]) -> trc::Result; fn deserialize_owned(bytes: Vec) -> trc::Result { Self::deserialize(&bytes) } } pub trait Serialize { fn serialize(&self) -> trc::Result>; } pub trait SerializeInfallible { fn serialize(&self) -> Vec; } // Key serialization flags pub(crate) const WITH_SUBSPACE: u32 = 1; pub trait Key: Sync + Send + Clone { fn serialize(&self, flags: u32) -> Vec; fn subspace(&self) -> u8; } #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct IndexKey> { pub account_id: u32, pub collection: u8, pub document_id: u32, pub field: u8, pub key: T, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct IndexKeyPrefix { pub account_id: u32, pub collection: u8, pub field: u8, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct ValueKey> { pub account_id: u32, pub collection: u8, pub document_id: u32, pub class: T, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct LogKey { pub account_id: u32, pub collection: u8, pub change_id: u64, } pub const U64_LEN: usize = std::mem::size_of::(); pub const U32_LEN: usize = std::mem::size_of::(); pub const U16_LEN: usize = std::mem::size_of::(); pub const SUBSPACE_ACL: u8 = b'a'; pub const SUBSPACE_DIRECTORY: u8 = b'd'; pub const SUBSPACE_TASK_QUEUE: u8 = b'f'; pub const SUBSPACE_INDEXES: u8 = b'i'; pub const SUBSPACE_BLOB_EXTRA: u8 = b'j'; pub const SUBSPACE_BLOB_LINK: u8 = b'k'; pub const SUBSPACE_BLOBS: u8 = b't'; pub const SUBSPACE_LOGS: u8 = b'l'; pub const SUBSPACE_COUNTER: u8 = b'n'; pub const SUBSPACE_IN_MEMORY_VALUE: u8 = b'm'; pub const SUBSPACE_IN_MEMORY_COUNTER: u8 = b'y'; pub const SUBSPACE_PROPERTY: u8 = b'p'; pub const SUBSPACE_SETTINGS: u8 = b's'; pub const SUBSPACE_QUEUE_MESSAGE: u8 = b'e'; pub const SUBSPACE_QUEUE_EVENT: u8 = b'q'; pub const SUBSPACE_QUOTA: u8 = b'u'; pub const SUBSPACE_REPORT_OUT: u8 = b'h'; pub const SUBSPACE_REPORT_IN: u8 = b'r'; pub const SUBSPACE_TELEMETRY_SPAN: u8 = b'o'; pub const SUBSPACE_TELEMETRY_METRIC: u8 = b'x'; pub const SUBSPACE_SEARCH_INDEX: u8 = b'z'; // TODO: Remove in v1.0 pub const LEGACY_SUBSPACE_BITMAP_ID: u8 = b'b'; pub const LEGACY_SUBSPACE_BITMAP_TAG: u8 = b'c'; pub const LEGACY_SUBSPACE_BITMAP_TEXT: u8 = b'v'; pub const LEGACY_SUBSPACE_FTS_INDEX: u8 = b'g'; pub const LEGACY_SUBSPACE_TELEMETRY_INDEX: u8 = b'w'; #[derive(Clone)] pub struct IterateParams { begin: T, end: T, first: bool, ascending: bool, values: bool, } #[derive(Clone, Default)] pub struct Stores { pub stores: AHashMap, pub blob_stores: AHashMap, pub search_stores: AHashMap, pub in_memory_stores: AHashMap, pub pubsub_stores: AHashMap, pub purge_schedules: Vec, } #[derive(Clone, Default)] pub enum Store { #[cfg(feature = "sqlite")] SQLite(Arc), #[cfg(feature = "foundation")] FoundationDb(Arc), #[cfg(feature = "postgres")] PostgreSQL(Arc), #[cfg(feature = "mysql")] MySQL(Arc), #[cfg(feature = "rocks")] RocksDb(Arc), // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] SQLReadReplica(Arc), // SPDX-SnippetEnd #[default] None, } #[derive(Clone)] pub struct BlobStore { pub backend: BlobBackend, pub compression: CompressionAlgo, } #[derive(Clone, Copy, Debug)] pub enum CompressionAlgo { None, Lz4, } #[derive(Clone)] pub enum BlobBackend { Store(Store), Fs(Arc), #[cfg(feature = "s3")] S3(Arc), #[cfg(feature = "azure")] Azure(Arc), // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] Sharded(Arc), // SPDX-SnippetEnd } #[derive(Clone)] pub enum SearchStore { Store(Store), ElasticSearch(Arc), MeiliSearch(Arc), } #[derive(Clone, Debug)] pub enum InMemoryStore { Store(Store), #[cfg(feature = "redis")] Redis(Arc), Http(Arc), Static(Arc), // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] Sharded(Arc), // SPDX-SnippetEnd } #[derive(Clone, Default)] pub enum PubSubStore { #[cfg(feature = "redis")] Redis(Arc), #[cfg(feature = "nats")] Nats(Arc), #[cfg(feature = "zenoh")] Zenoh(Arc), #[cfg(feature = "kafka")] Kafka(Arc), #[default] None, } #[cfg(feature = "sqlite")] impl From for Store { fn from(store: backend::sqlite::SqliteStore) -> Self { Self::SQLite(Arc::new(store)) } } #[cfg(feature = "foundation")] impl From for Store { fn from(store: backend::foundationdb::FdbStore) -> Self { Self::FoundationDb(Arc::new(store)) } } #[cfg(feature = "postgres")] impl From for Store { fn from(store: backend::postgres::PostgresStore) -> Self { Self::PostgreSQL(Arc::new(store)) } } #[cfg(feature = "mysql")] impl From for Store { fn from(store: backend::mysql::MysqlStore) -> Self { Self::MySQL(Arc::new(store)) } } #[cfg(feature = "rocks")] impl From for Store { fn from(store: backend::rocksdb::RocksDbStore) -> Self { Self::RocksDb(Arc::new(store)) } } impl From for BlobStore { fn from(store: FsStore) -> Self { BlobStore { backend: BlobBackend::Fs(Arc::new(store)), compression: CompressionAlgo::None, } } } #[cfg(feature = "s3")] impl From for BlobStore { fn from(store: backend::s3::S3Store) -> Self { BlobStore { backend: BlobBackend::S3(Arc::new(store)), compression: CompressionAlgo::None, } } } #[cfg(feature = "azure")] impl From for BlobStore { fn from(store: backend::azure::AzureStore) -> Self { BlobStore { backend: BlobBackend::Azure(Arc::new(store)), compression: CompressionAlgo::None, } } } impl From for SearchStore { fn from(store: ElasticSearchStore) -> Self { Self::ElasticSearch(Arc::new(store)) } } impl From for SearchStore { fn from(store: MeiliSearchStore) -> Self { Self::MeiliSearch(Arc::new(store)) } } #[cfg(feature = "redis")] impl From for InMemoryStore { fn from(store: backend::redis::RedisStore) -> Self { Self::Redis(Arc::new(store)) } } impl From for SearchStore { fn from(store: Store) -> Self { Self::Store(store) } } impl From for BlobStore { fn from(store: Store) -> Self { BlobStore { backend: BlobBackend::Store(store), compression: CompressionAlgo::None, } } } impl From for InMemoryStore { fn from(store: Store) -> Self { Self::Store(store) } } impl Default for BlobStore { fn default() -> Self { Self { backend: BlobBackend::Store(Store::None), compression: CompressionAlgo::None, } } } impl Default for InMemoryStore { fn default() -> Self { Self::Store(Store::None) } } impl Default for SearchStore { fn default() -> Self { Self::Store(Store::None) } } #[derive(Clone)] pub enum PurgeStore { Data(Store), Blobs { store: Store, blob_store: BlobStore }, Lookup(InMemoryStore), } #[derive(Clone)] pub struct PurgeSchedule { pub cron: SimpleCron, pub store_id: String, pub store: PurgeStore, } #[derive(Clone, Debug, PartialEq)] pub enum Value<'x> { Integer(i64), Bool(bool), Float(f64), Text(Cow<'x, str>), Blob(Cow<'x, [u8]>), Null, } impl Eq for Value<'_> {} impl<'x> Value<'x> { pub fn to_str<'y: 'x>(&'y self) -> Cow<'x, str> { match self { Value::Text(s) => s.as_ref().into(), Value::Integer(i) => Cow::Owned(i.to_string()), Value::Bool(b) => Cow::Owned(b.to_string()), Value::Float(f) => Cow::Owned(f.to_string()), Value::Blob(b) => String::from_utf8_lossy(b.as_ref()), Value::Null => Cow::Borrowed(""), } } } #[derive(Clone, Debug)] pub struct Row { pub values: Vec>, } #[derive(Clone, Debug)] pub struct Rows { pub rows: Vec, } #[derive(Clone, Debug)] pub struct NamedRows { pub names: Vec, pub rows: Vec, } #[derive(Clone, Copy)] pub enum QueryType { Execute, Exists, QueryAll, QueryOne, } pub trait QueryResult: Sync + Send + 'static { fn from_exec(items: usize) -> Self; fn from_exists(exists: bool) -> Self; fn from_query_one(items: impl IntoRows) -> Self; fn from_query_all(items: impl IntoRows) -> Self; fn query_type() -> QueryType; } pub trait IntoRows { fn into_row(self) -> Option; fn into_rows(self) -> Rows; fn into_named_rows(self) -> NamedRows; } impl QueryResult for Option { fn query_type() -> QueryType { QueryType::QueryOne } fn from_exec(_: usize) -> Self { unreachable!() } fn from_exists(_: bool) -> Self { unreachable!() } fn from_query_all(_: impl IntoRows) -> Self { unreachable!() } fn from_query_one(items: impl IntoRows) -> Self { items.into_row() } } impl QueryResult for Rows { fn query_type() -> QueryType { QueryType::QueryAll } fn from_exec(_: usize) -> Self { unreachable!() } fn from_exists(_: bool) -> Self { unreachable!() } fn from_query_all(items: impl IntoRows) -> Self { items.into_rows() } fn from_query_one(_: impl IntoRows) -> Self { unreachable!() } } impl QueryResult for NamedRows { fn query_type() -> QueryType { QueryType::QueryAll } fn from_exec(_: usize) -> Self { unreachable!() } fn from_exists(_: bool) -> Self { unreachable!() } fn from_query_all(items: impl IntoRows) -> Self { items.into_named_rows() } fn from_query_one(_: impl IntoRows) -> Self { unreachable!() } } impl QueryResult for bool { fn query_type() -> QueryType { QueryType::Exists } fn from_exec(_: usize) -> Self { unreachable!() } fn from_exists(exists: bool) -> Self { exists } fn from_query_all(_: impl IntoRows) -> Self { unreachable!() } fn from_query_one(_: impl IntoRows) -> Self { unreachable!() } } impl QueryResult for usize { fn query_type() -> QueryType { QueryType::Execute } fn from_exec(items: usize) -> Self { items } fn from_exists(_: bool) -> Self { unreachable!() } fn from_query_all(_: impl IntoRows) -> Self { unreachable!() } fn from_query_one(_: impl IntoRows) -> Self { unreachable!() } } impl<'x> From<&'x str> for Value<'x> { fn from(value: &'x str) -> Self { Self::Text(value.into()) } } impl From for Value<'_> { fn from(value: String) -> Self { Self::Text(value.into()) } } impl<'x> From<&'x String> for Value<'x> { fn from(value: &'x String) -> Self { Self::Text(value.into()) } } impl<'x> From> for Value<'x> { fn from(value: Cow<'x, str>) -> Self { Self::Text(value) } } impl From for Value<'_> { fn from(value: bool) -> Self { Self::Bool(value) } } impl From for Value<'_> { fn from(value: i64) -> Self { Self::Integer(value) } } impl From> for i64 { fn from(value: Value<'static>) -> Self { if let Value::Integer(value) = value { value } else { 0 } } } impl From for Value<'_> { fn from(value: u64) -> Self { Self::Integer(value as i64) } } impl From for Value<'_> { fn from(value: u32) -> Self { Self::Integer(value as i64) } } impl From for Value<'_> { fn from(value: f64) -> Self { Self::Float(value) } } impl<'x> From<&'x [u8]> for Value<'x> { fn from(value: &'x [u8]) -> Self { Self::Blob(value.into()) } } impl From> for Value<'_> { fn from(value: Vec) -> Self { Self::Blob(value.into()) } } impl Value<'_> { pub fn into_string(self) -> String { match self { Value::Text(s) => s.into_owned(), Value::Integer(i) => i.to_string(), Value::Bool(b) => b.to_string(), Value::Float(f) => f.to_string(), Value::Blob(b) => String::from_utf8_lossy(b.as_ref()).into_owned(), Value::Null => "".into(), } } pub fn into_lower_string(self) -> String { match self { Value::Text(s) => s.as_ref().to_lowercase(), Value::Integer(i) => i.to_string(), Value::Bool(b) => b.to_string(), Value::Float(f) => f.to_string(), Value::Blob(b) => String::from_utf8_lossy(b.as_ref()).to_lowercase(), Value::Null => "".into(), } } } impl From for Vec { fn from(value: Row) -> Self { value.values.into_iter().map(|v| v.into_string()).collect() } } impl From for Vec { fn from(value: Row) -> Self { value .values .into_iter() .filter_map(|v| { if let Value::Integer(v) = v { Some(v as u32) } else { None } }) .collect() } } impl From for Vec { fn from(value: Rows) -> Self { value .rows .into_iter() .flat_map(|v| v.values.into_iter().map(|v| v.into_string())) .collect() } } impl From for Vec { fn from(value: Rows) -> Self { value .rows .into_iter() .flat_map(|v| { v.values.into_iter().filter_map(|v| { if let Value::Integer(v) = v { Some(v as u32) } else { None } }) }) .collect() } } impl Store { #[inline(always)] pub fn is_none(&self) -> bool { matches!(self, Self::None) } #[inline(always)] pub fn is_sql(&self) -> bool { match self { #[cfg(feature = "sqlite")] Store::SQLite(_) => true, #[cfg(feature = "postgres")] Store::PostgreSQL(_) => true, #[cfg(feature = "mysql")] Store::MySQL(_) => true, // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Store::SQLReadReplica(_) => true, // SPDX-SnippetEnd _ => false, } } #[inline(always)] pub fn is_pg_or_mysql(&self) -> bool { match self { #[cfg(feature = "mysql")] Store::MySQL(_) => true, #[cfg(feature = "postgres")] Store::PostgreSQL(_) => true, _ => false, } } #[inline(always)] pub fn is_foundationdb(&self) -> bool { match self { #[cfg(feature = "foundation")] Store::FoundationDb(_) => true, _ => false, } } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] pub fn is_enterprise_store(&self) -> bool { match self { #[cfg(any(feature = "postgres", feature = "mysql"))] Store::SQLReadReplica(_) => true, _ => false, } } // SPDX-SnippetEnd #[cfg(not(feature = "enterprise"))] pub fn is_enterprise_store(&self) -> bool { false } } impl std::fmt::Debug for Store { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { #[cfg(feature = "sqlite")] Self::SQLite(_) => f.debug_tuple("SQLite").finish(), #[cfg(feature = "foundation")] Self::FoundationDb(_) => f.debug_tuple("FoundationDb").finish(), #[cfg(feature = "postgres")] Self::PostgreSQL(_) => f.debug_tuple("PostgreSQL").finish(), #[cfg(feature = "mysql")] Self::MySQL(_) => f.debug_tuple("MySQL").finish(), #[cfg(feature = "rocks")] Self::RocksDb(_) => f.debug_tuple("RocksDb").finish(), // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] Self::SQLReadReplica(_) => f.debug_tuple("SQLReadReplica").finish(), // SPDX-SnippetEnd Self::None => f.debug_tuple("None").finish(), } } } impl From> for trc::Value { fn from(value: Value) -> Self { match value { Value::Integer(v) => trc::Value::Int(v), Value::Bool(v) => trc::Value::Bool(v), Value::Float(v) => trc::Value::Float(v), Value::Text(v) => trc::Value::String(match v { Cow::Borrowed(v) => v.into(), Cow::Owned(v) => v.into(), }), Value::Blob(v) => trc::Value::Bytes(v.into_owned()), Value::Null => trc::Value::None, } } } impl From> for () { fn from(_: Value<'static>) -> Self { unreachable!() } } impl Stores { pub fn disable_enterprise_only(&mut self) { // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] { #[cfg(any(feature = "postgres", feature = "mysql"))] self.stores .retain(|_, store| !matches!(store, Store::SQLReadReplica(_))); self.blob_stores .retain(|_, store| !matches!(store.backend, BlobBackend::Sharded(_))); } // SPDX-SnippetEnd } } ================================================ FILE: crates/store/src/query/acl.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use ahash::AHashSet; use trc::AddContext; use types::collection::Collection; use crate::{ Deserialize, IterateParams, Store, U32_LEN, ValueKey, write::{BatchBuilder, ValueClass, key::DeserializeBigEndian}, }; pub enum AclQuery { SharedWith { grant_account_id: u32, to_account_id: u32, to_collection: u8, }, HasAccess { grant_account_id: u32, }, } #[derive(Debug)] pub struct AclItem { pub to_account_id: u32, pub to_collection: Collection, pub to_document_id: u32, pub permissions: u64, } impl Store { pub async fn acl_query(&self, query: AclQuery) -> trc::Result> { let mut results = Vec::new(); let (from_key, to_key) = match query { AclQuery::SharedWith { grant_account_id, to_account_id, to_collection, } => { let from_key = ValueKey { account_id: to_account_id, collection: to_collection, document_id: 0, class: ValueClass::Acl(grant_account_id), }; let mut to_key = from_key.clone(); to_key.document_id = u32::MAX; (from_key, to_key) } AclQuery::HasAccess { grant_account_id } => ( ValueKey { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Acl(grant_account_id), }, ValueKey { account_id: u32::MAX, collection: u8::MAX, document_id: u32::MAX, class: ValueClass::Acl(grant_account_id), }, ), }; self.iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { results.push(AclItem::deserialize(key)?.with_permissions(u64::deserialize(value)?)); Ok(true) }, ) .await .caused_by(trc::location!()) .map(|_| results) } pub async fn acl_revoke_all(&self, account_id: u32) -> trc::Result> { let from_key = ValueKey { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Acl(0), }; let to_key = ValueKey { account_id: u32::MAX, collection: u8::MAX, document_id: u32::MAX, class: ValueClass::Acl(u32::MAX), }; let mut delete_keys = Vec::new(); let mut revoked_accounts = AHashSet::new(); self.iterate( IterateParams::new(from_key, to_key).ascending().no_values(), |key, _| { if account_id == key.deserialize_be_u32(U32_LEN)? { let owner_account_id = key.deserialize_be_u32(0)?; revoked_accounts.insert(owner_account_id); delete_keys.push((owner_account_id, AclItem::deserialize(key)?)); } Ok(true) }, ) .await .caused_by(trc::location!())?; // Remove permissions let mut batch = BatchBuilder::new(); batch.with_account_id(account_id); let mut last_collection = Collection::None; for (revoke_account_id, acl_item) in delete_keys.into_iter() { if batch.is_large_batch() { self.write(batch.build_all()) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); batch.with_account_id(account_id); last_collection = Collection::None; } if acl_item.to_collection != last_collection { batch.with_collection(acl_item.to_collection); last_collection = acl_item.to_collection; } batch .with_document(acl_item.to_document_id) .acl_revoke(revoke_account_id); } if !batch.is_empty() { self.write(batch.build_all()) .await .caused_by(trc::location!())?; } Ok(revoked_accounts) } } impl Deserialize for AclItem { fn deserialize(bytes: &[u8]) -> trc::Result { Ok(AclItem { to_account_id: bytes.deserialize_be_u32(U32_LEN)?, to_collection: bytes .get(U32_LEN * 2) .map(|b| Collection::from(*b)) .ok_or_else(|| trc::StoreEvent::DataCorruption.caused_by(trc::location!()))?, to_document_id: bytes.deserialize_be_u32((U32_LEN * 2) + 1)?, permissions: 0, }) } } impl AclItem { fn with_permissions(mut self, permissions: u64) -> Self { self.permissions = permissions; self } } ================================================ FILE: crates/store/src/query/log.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use trc::AddContext; use types::collection::{SyncCollection, VanishedCollection}; use utils::codec::leb128::Leb128Iterator; use crate::{ IterateParams, LogKey, Store, U32_LEN, U64_LEN, write::{LogCollection, key::DeserializeBigEndian}, }; #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum Change { InsertContainer(u64), UpdateContainer(u64), UpdateContainerProperty(u64), DeleteContainer(u64), InsertItem(u64), UpdateItem(u64), DeleteItem(u64), } #[derive(Debug)] pub struct Changes { pub changes: Vec, pub from_change_id: u64, pub to_change_id: u64, pub container_change_id: Option, pub item_change_id: Option, pub is_truncated: bool, } #[derive(Debug, Clone, Copy)] pub enum Query { All, Since(u64), SinceInclusive(u64), RangeInclusive(u64, u64), } pub trait DeserializeVanished: Sized + Sync + Send { fn deserialize_vanished<'x>(bytes: &mut impl Iterator) -> Option; } impl Default for Changes { fn default() -> Self { Self { changes: Vec::with_capacity(10), from_change_id: 0, to_change_id: 0, container_change_id: None, item_change_id: None, is_truncated: false, } } } impl Store { pub async fn changes( &self, account_id: u32, collection_: LogCollection, query: Query, ) -> trc::Result { let is_share_log = matches!( collection_, LogCollection::Sync(SyncCollection::ShareNotification) ); let collection = u8::from(collection_); let (is_inclusive, from_change_id, to_change_id) = match query { Query::All => (true, 0, u64::MAX), Query::Since(change_id) => (false, change_id, u64::MAX), Query::SinceInclusive(change_id) => (true, change_id, u64::MAX), Query::RangeInclusive(from_change_id, to_change_id) => { (true, from_change_id, to_change_id) } }; let from_key = LogKey { account_id, collection, change_id: from_change_id, }; let to_key = LogKey { account_id, collection, change_id: to_change_id, }; let mut changelog = Changes::default(); self.iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { let change_id = key.deserialize_be_u64(key.len() - U64_LEN)?; if is_inclusive || change_id != from_change_id { if value.is_empty() { changelog.is_truncated = true; return Ok(true); } if changelog.changes.is_empty() { changelog.from_change_id = change_id; } changelog.to_change_id = change_id; if !is_share_log { let (has_container_changes, has_item_changes) = changelog.deserialize(value).ok_or_else(|| { trc::Error::corrupted_key(key, value.into(), trc::location!()) })?; if has_container_changes { changelog.container_change_id = Some(change_id); } if has_item_changes { changelog.item_change_id = Some(change_id); } } else { changelog.changes.push(Change::InsertItem(change_id)); } } else { changelog.from_change_id = change_id; changelog.to_change_id = change_id; } Ok(true) }, ) .await .caused_by(trc::location!())?; Ok(changelog) } pub async fn vanished( &self, account_id: u32, collection: LogCollection, query: Query, ) -> trc::Result> { let collection = u8::from(collection); let (is_inclusive, from_change_id, to_change_id) = match query { Query::All => (true, 0, u64::MAX), Query::Since(change_id) => (false, change_id, u64::MAX), Query::SinceInclusive(change_id) => (true, change_id, u64::MAX), Query::RangeInclusive(from_change_id, to_change_id) => { (true, from_change_id, to_change_id) } }; let from_key = LogKey { account_id, collection, change_id: from_change_id, }; let to_key = LogKey { account_id, collection, change_id: to_change_id, }; let mut vanished = Vec::default(); self.iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { let change_id = key.deserialize_be_u64(key.len() - U64_LEN)?; if is_inclusive || change_id != from_change_id { let mut iter = value.iter().peekable(); while iter.peek().is_some() { if let Some(item) = T::deserialize_vanished(&mut iter) { vanished.push(item); } else { return Err(trc::Error::corrupted_key( key, value.into(), trc::location!(), )); } } } Ok(true) }, ) .await .caused_by(trc::location!())?; Ok(vanished) } pub async fn get_last_change_id( &self, account_id: u32, collection: LogCollection, ) -> trc::Result> { let collection = u8::from(collection); let from_key = LogKey { account_id, collection, change_id: 0, }; let to_key = LogKey { account_id, collection, change_id: u64::MAX, }; let mut last_change_id = None; self.iterate( IterateParams::new(from_key, to_key) .descending() .no_values() .only_first(), |key, _| { last_change_id = key.deserialize_be_u64(key.len() - U64_LEN)?.into(); Ok(false) }, ) .await .caused_by(trc::location!())?; Ok(last_change_id) } } impl From for LogCollection { fn from(value: VanishedCollection) -> Self { LogCollection::Vanished(value) } } impl From for LogCollection { fn from(value: SyncCollection) -> Self { LogCollection::Sync(value) } } impl Changes { pub fn deserialize(&mut self, bytes: &[u8]) -> Option<(bool, bool)> { let mut bytes_it = bytes.iter(); let container_inserts: usize = bytes_it.next_leb128()?; let container_updates: usize = bytes_it.next_leb128()?; let container_property_changes: usize = bytes_it.next_leb128()?; let container_deletes: usize = bytes_it.next_leb128()?; let item_inserts: usize = bytes_it.next_leb128()?; let item_updates: usize = bytes_it.next_leb128()?; let item_deletes: usize = bytes_it.next_leb128()?; let has_container_changes = container_inserts + container_updates + container_property_changes + container_deletes > 0; let has_item_changes = item_inserts + item_updates + item_deletes > 0; if container_inserts > 0 { for _ in 0..container_inserts { self.changes .push(Change::InsertContainer(bytes_it.next_leb128()?)); } } if container_updates > 0 || container_property_changes > 0 { 'update_outer: for change_pos in 0..(container_updates + container_property_changes) { let id = bytes_it.next_leb128()?; let mut is_property_change = change_pos >= container_updates; for (idx, change) in self.changes.iter().enumerate() { match change { Change::InsertContainer(insert_id) if *insert_id == id => { // Item updated after inserted, no need to count this change. continue 'update_outer; } Change::UpdateContainer(update_id) if *update_id == id => { // Move update to the front is_property_change = false; self.changes.remove(idx); break; } Change::UpdateContainerProperty(update_id) if *update_id == id => { // Move update to the front self.changes.remove(idx); break; } _ => (), } } self.changes.push(if !is_property_change { Change::UpdateContainer(id) } else { Change::UpdateContainerProperty(id) }); } } if container_deletes > 0 { 'delete_outer: for _ in 0..container_deletes { let id = bytes_it.next_leb128()?; 'delete_inner: for (idx, change) in self.changes.iter().enumerate() { match change { Change::InsertContainer(insert_id) if *insert_id == id => { self.changes.remove(idx); continue 'delete_outer; } Change::UpdateContainer(update_id) if *update_id == id => { self.changes.remove(idx); break 'delete_inner; } _ => (), } } self.changes.push(Change::DeleteContainer(id)); } } // Item changes if item_inserts > 0 { for _ in 0..item_inserts { self.changes .push(Change::InsertItem(bytes_it.next_leb128()?)); } } if item_updates > 0 { 'update_outer: for _ in 0..item_updates { let id = bytes_it.next_leb128()?; for (idx, change) in self.changes.iter().enumerate() { match change { Change::InsertItem(insert_id) if *insert_id == id => { // Item updated after inserted, no need to count this change. continue 'update_outer; } Change::UpdateItem(update_id) if *update_id == id => { // Move update to the front self.changes.remove(idx); break; } _ => (), } } self.changes.push(Change::UpdateItem(id)); } } if item_deletes > 0 { 'delete_outer: for _ in 0..item_deletes { let id = bytes_it.next_leb128()?; 'delete_inner: for (idx, change) in self.changes.iter().enumerate() { match change { Change::InsertItem(insert_id) if *insert_id == id => { self.changes.remove(idx); continue 'delete_outer; } Change::UpdateItem(update_id) if *update_id == id => { self.changes.remove(idx); break 'delete_inner; } _ => (), } } self.changes.push(Change::DeleteItem(id)); } } Some((has_container_changes, has_item_changes)) } } impl Changes { pub fn total_container_changes(&self) -> usize { self.changes .iter() .filter(|change| change.is_container_change()) .count() } pub fn total_item_changes(&self) -> usize { self.changes .iter() .filter(|change| change.is_item_change()) .count() } } impl Change { pub fn item_id(&self) -> Option { match self { Change::InsertItem(id) => Some(*id), Change::UpdateItem(id) => Some(*id), Change::DeleteItem(id) => Some(*id), _ => None, } } pub fn container_id(&self) -> Option { match self { Change::InsertContainer(id) => Some(*id), Change::UpdateContainer(id) => Some(*id), Change::UpdateContainerProperty(id) => Some(*id), Change::DeleteContainer(id) => Some(*id), _ => None, } } pub fn try_unwrap_item_id(self) -> Option { match self { Change::InsertItem(id) => Some(id), Change::UpdateItem(id) => Some(id), Change::DeleteItem(id) => Some(id), _ => None, } } pub fn try_unwrap_container_id(self) -> Option { match self { Change::InsertContainer(id) => Some(id), Change::UpdateContainer(id) => Some(id), Change::UpdateContainerProperty(id) => Some(id), Change::DeleteContainer(id) => Some(id), _ => None, } } pub fn is_container_change(&self) -> bool { matches!( self, Change::InsertContainer(_) | Change::UpdateContainer(_) | Change::UpdateContainerProperty(_) | Change::DeleteContainer(_) ) } pub fn is_item_change(&self) -> bool { matches!( self, Change::InsertItem(_) | Change::UpdateItem(_) | Change::DeleteItem(_) ) } } impl DeserializeVanished for u64 { fn deserialize_vanished<'x>(bytes: &mut impl Iterator) -> Option { let mut num = [0u8; U64_LEN]; for i in num.iter_mut() { *i = *bytes.next()?; } Some(u64::from_be_bytes(num)) } } impl DeserializeVanished for (u32, u32) { fn deserialize_vanished<'x>(bytes: &mut impl Iterator) -> Option { let mut num1 = [0u8; U32_LEN]; let mut num2 = [0u8; U32_LEN]; for i in num1.iter_mut().chain(num2.iter_mut()) { *i = *bytes.next()?; } Some((u32::from_be_bytes(num1), u32::from_be_bytes(num2))) } } impl DeserializeVanished for String { fn deserialize_vanished<'x>(bytes: &mut impl Iterator) -> Option { let mut name = Vec::with_capacity(16); loop { let byte = bytes.next()?; if *byte != 0 { name.push(*byte); } else { break; } } String::from_utf8(name).ok() } } ================================================ FILE: crates/store/src/query/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod acl; pub mod log; use crate::{IterateParams, Key}; impl IterateParams { pub fn new(begin: T, end: T) -> Self { IterateParams { begin, end, first: false, ascending: true, values: true, } } pub fn set_ascending(mut self, ascending: bool) -> Self { self.ascending = ascending; self } pub fn set_values(mut self, values: bool) -> Self { self.values = values; self } pub fn ascending(mut self) -> Self { self.ascending = true; self } pub fn descending(mut self) -> Self { self.ascending = false; self } pub fn only_first(mut self) -> Self { self.first = true; self } pub fn no_values(mut self) -> Self { self.values = false; self } } ================================================ FILE: crates/store/src/search/bm_u32.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ IterateParams, Store, U32_LEN, ValueKey, search::*, write::{ SEARCH_INDEX_MAX_FIELD_LEN, SearchIndex, SearchIndexClass, SearchIndexField, SearchIndexId, SearchIndexType, ValueClass, key::{DeserializeBigEndian, KeySerializer}, }, }; use ahash::AHashMap; use roaring::RoaringBitmap; use std::{ collections::hash_map::Entry, ops::{BitAndAssign, BitOrAssign}, }; use trc::AddContext; use utils::cheeky_hash::CheekyHash; #[derive(Default)] pub(super) struct BitmapCache { cache: AHashMap<(CheekyHash, u8), Option>, } impl BitmapCache { pub async fn merge_bitmaps( &mut self, store: &Store, index: SearchIndex, account_id: u32, hashes: impl Iterator, field: u8, is_union: bool, ) -> trc::Result> { let mut result = RoaringBitmap::new(); for (idx, hash) in hashes.enumerate() { match self.cache.entry((hash, field)) { Entry::Occupied(entry) => { if let Some(bm) = entry.get() { if is_union { result.bitor_assign(bm); } else if idx == 0 { result = bm.clone(); } else { result.bitand_assign(bm); if result.is_empty() { return Ok(None); } } } else if !is_union { return Ok(None); } } Entry::Vacant(entry) => { let from_key = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Account { account_id, document_id: 0, }, typ: SearchIndexType::Term { hash, field }, })); let to_key = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Account { account_id, document_id: u32::MAX, }, typ: SearchIndexType::Term { hash, field }, })); let key_len = (U32_LEN * 2) + hash.len() + 2; let mut documents = RoaringBitmap::new(); store .iterate( IterateParams::new(from_key, to_key).no_values().ascending(), |key, _| { if key.len() == key_len { documents.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); } Ok(true) }, ) .await .caused_by(trc::location!())?; if !documents.is_empty() { if is_union { result.bitor_assign(&documents); } else if idx == 0 { result = documents.clone(); } else { result.bitand_assign(&documents); if result.is_empty() { entry.insert(Some(documents)); return Ok(None); } } entry.insert(Some(documents)); } else if !is_union { entry.insert(None); return Ok(None); } } } } if !result.is_empty() { Ok(Some(result)) } else { Ok(None) } } } pub(crate) async fn range_to_bitmap( store: &Store, index: SearchIndex, account_id: u32, field_id: u8, match_value: &[u8], op: SearchOperator, ) -> trc::Result> { let ((from_value, from_doc_id, from_field), (end_value, end_doc_id, end_field)) = match op { SearchOperator::LowerThan => ((&[][..], 0, field_id), (match_value, 0, field_id)), SearchOperator::LowerEqualThan => { ((&[][..], 0, field_id), (match_value, u32::MAX, field_id)) } SearchOperator::GreaterThan => ( (match_value, u32::MAX, field_id), (&[][..], u32::MAX, field_id + 1), ), SearchOperator::GreaterEqualThan => ( (match_value, 0, field_id), (&[][..], u32::MAX, field_id + 1), ), SearchOperator::Equal | SearchOperator::Contains => ( (match_value, 0, field_id), (match_value, u32::MAX, field_id), ), }; let begin = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Account { account_id, document_id: from_doc_id, }, typ: SearchIndexType::Index { field: SearchIndexField { field_id: from_field, data: from_value.to_vec(), }, }, })); let end = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Account { account_id, document_id: end_doc_id, }, typ: SearchIndexType::Index { field: SearchIndexField { field_id: end_field, data: end_value.to_vec(), }, }, })); let mut bm = RoaringBitmap::new(); let prefix = KeySerializer::new(U32_LEN + 2) .write(index.as_u8() | 1 << 6) .write(account_id) .write(field_id) .finalize(); let prefix_len = prefix.len(); store .iterate( IterateParams::new(begin, end).no_values().ascending(), |key, _| { if !key.starts_with(&prefix) { return Ok(false); } let id_pos = key.len() - U32_LEN; let value = key .get(prefix_len..id_pos) .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?; let matches = match op { SearchOperator::LowerThan => value < match_value, SearchOperator::LowerEqualThan => value <= match_value, SearchOperator::GreaterThan => value > match_value, SearchOperator::GreaterEqualThan => value >= match_value, SearchOperator::Equal | SearchOperator::Contains => value == match_value, }; if matches { bm.insert(key.deserialize_be_u32(id_pos)?); } Ok(true) }, ) .await .caused_by(trc::location!())?; if !bm.is_empty() { Ok(Some(bm)) } else { Ok(None) } } pub(crate) async fn sort_order( store: &Store, index: SearchIndex, account_id: u32, field_id: u8, ) -> trc::Result> { let begin = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Account { account_id, document_id: 0, }, typ: SearchIndexType::Index { field: SearchIndexField { field_id, data: vec![0u8], }, }, })); let end = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Account { account_id, document_id: u32::MAX, }, typ: SearchIndexType::Index { field: SearchIndexField { field_id, data: vec![u8::MAX; SEARCH_INDEX_MAX_FIELD_LEN], }, }, })); let mut last_value = Vec::new(); let mut results = AHashMap::new(); let mut pos = 0; store .iterate( IterateParams::new(begin, end).no_values().ascending(), |key, _| { let value = key .get(U32_LEN + 2..key.len() - U32_LEN) .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?; if value != last_value { pos += 1; last_value = value.to_vec(); } results.insert(key.deserialize_be_u32(key.len() - U32_LEN)?, pos); Ok(true) }, ) .await .caused_by(trc::location!())?; Ok(results) } ================================================ FILE: crates/store/src/search/bm_u64.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ IterateParams, Store, U64_LEN, ValueKey, search::*, write::{ SearchIndex, SearchIndexClass, SearchIndexField, SearchIndexId, SearchIndexType, ValueClass, key::{DeserializeBigEndian, KeySerializer}, }, }; use ahash::AHashMap; use roaring::RoaringTreemap; use std::{ collections::hash_map::Entry, ops::{BitAndAssign, BitOrAssign}, }; use trc::AddContext; use utils::cheeky_hash::CheekyHash; #[derive(Default)] pub(super) struct TreemapCache { cache: AHashMap<(CheekyHash, u8), Option>, } impl TreemapCache { pub async fn merge_treemaps( &mut self, store: &Store, index: SearchIndex, hashes: impl Iterator, field: u8, is_union: bool, ) -> trc::Result> { let mut result = RoaringTreemap::new(); for (idx, hash) in hashes.enumerate() { match self.cache.entry((hash, field)) { Entry::Occupied(entry) => { if let Some(bm) = entry.get() { if is_union { result.bitor_assign(bm); } else if idx == 0 { result = bm.clone(); } else { result.bitand_assign(bm); if result.is_empty() { return Ok(None); } } } else if !is_union { return Ok(None); } } Entry::Vacant(entry) => { let from_key = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Global { id: 0 }, typ: SearchIndexType::Term { hash, field }, })); let to_key = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Global { id: u64::MAX }, typ: SearchIndexType::Term { hash, field }, })); let key_len = U64_LEN + hash.len() + 2; let mut documents = RoaringTreemap::new(); store .iterate( IterateParams::new(from_key, to_key).no_values().ascending(), |key, _| { if key.len() == key_len { documents.insert(key.deserialize_be_u64(key.len() - U64_LEN)?); } Ok(true) }, ) .await .caused_by(trc::location!())?; if !documents.is_empty() { if is_union { result.bitor_assign(&documents); } else if idx == 0 { result = documents.clone(); } else { result.bitand_assign(&documents); if result.is_empty() { entry.insert(Some(documents)); return Ok(None); } } entry.insert(Some(documents)); } else if !is_union { entry.insert(None); return Ok(None); } } } } if !result.is_empty() { Ok(Some(result)) } else { Ok(None) } } } pub(crate) async fn range_to_treemap( store: &Store, index: SearchIndex, field_id: u8, match_value: &[u8], op: SearchOperator, ) -> trc::Result> { let ((from_value, from_id, from_field), (end_value, end_id, end_field)) = match op { SearchOperator::LowerThan => ((&[][..], 0, field_id), (match_value, 0, field_id)), SearchOperator::LowerEqualThan => { ((&[][..], 0, field_id), (match_value, u64::MAX, field_id)) } SearchOperator::GreaterThan => ( (match_value, u64::MAX, field_id), (&[][..], u64::MAX, field_id + 1), ), SearchOperator::GreaterEqualThan => ( (match_value, 0, field_id), (&[][..], u64::MAX, field_id + 1), ), SearchOperator::Equal | SearchOperator::Contains => ( (match_value, 0, field_id), (match_value, u64::MAX, field_id), ), }; let begin = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Global { id: from_id }, typ: SearchIndexType::Index { field: SearchIndexField { field_id: from_field, data: from_value.to_vec(), }, }, })); let end = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Global { id: end_id }, typ: SearchIndexType::Index { field: SearchIndexField { field_id: end_field, data: end_value.to_vec(), }, }, })); let mut bm = RoaringTreemap::new(); let prefix = KeySerializer::new(U64_LEN + 2) .write(index.as_u8() | 1 << 6) .write(field_id) .finalize(); let prefix_len = prefix.len(); store .iterate( IterateParams::new(begin, end).no_values().ascending(), |key, _| { if !key.starts_with(&prefix) { return Ok(false); } let id_pos = key.len() - U64_LEN; let value = key .get(prefix_len..id_pos) .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?; let matches = match op { SearchOperator::LowerThan => value < match_value, SearchOperator::LowerEqualThan => value <= match_value, SearchOperator::GreaterThan => value > match_value, SearchOperator::GreaterEqualThan => value >= match_value, SearchOperator::Equal | SearchOperator::Contains => value == match_value, }; if matches { bm.insert(key.deserialize_be_u64(id_pos)?); } Ok(true) }, ) .await .caused_by(trc::location!())?; if !bm.is_empty() { Ok(Some(bm)) } else { Ok(None) } } ================================================ FILE: crates/store/src/search/document.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::search::*; impl IndexDocument { pub fn new(index: SearchIndex) -> Self { Self { fields: Default::default(), index, } } pub fn with_account_id(mut self, account_id: u32) -> Self { self.fields .insert(SearchField::AccountId, SearchValue::Uint(account_id as u64)); self } pub fn with_document_id(mut self, document_id: u32) -> Self { self.fields.insert( SearchField::DocumentId, SearchValue::Uint(document_id as u64), ); self } pub fn with_id(mut self, id: u64) -> Self { self.fields.insert(SearchField::Id, SearchValue::Uint(id)); self } pub fn index_text(&mut self, field: impl Into, value: &str, language: Language) { match self.fields.entry(field.into()) { Entry::Occupied(mut entry) => { if let SearchValue::Text { value: existing_value, .. } = entry.get_mut() { existing_value.push(' '); sanitize_text_to_buf(existing_value, value); } } Entry::Vacant(entry) => { entry.insert(SearchValue::Text { value: sanitize_text(value), language, }); } } } pub fn index_bool(&mut self, field: impl Into, value: bool) { self.fields .insert(field.into(), SearchValue::Boolean(value)); } pub fn index_integer>(&mut self, field: impl Into, value: N) { self.fields .insert(field.into(), SearchValue::Int(value.into())); } pub fn index_unsigned>(&mut self, field: impl Into, value: N) { self.fields .insert(field.into(), SearchValue::Uint(value.into())); } pub fn index_keyword(&mut self, field: impl Into, value: impl AsRef) { self.fields.insert( field.into(), SearchValue::Text { value: sanitize_text(value.as_ref()), language: Language::None, }, ); } pub fn insert_key_value( &mut self, field: impl Into, key: impl AsRef, value: impl AsRef, ) { let search_field = field.into(); let key = key .as_ref() .chars() .filter(|ch| !ch.is_control()) .map(|ch| ch.to_ascii_lowercase()) .collect::(); let value = value.as_ref(); match self.fields.entry(search_field) { Entry::Occupied(mut entry) => { if let SearchValue::KeyValues(existing_key_values) = entry.get_mut() { if let Some(existing_value) = existing_key_values.get_mut(&key) { existing_value.push(' '); sanitize_text_to_buf(existing_value, value); } else { existing_key_values.append(key, sanitize_text(value)); } } } Entry::Vacant(entry) => { let mut new_key_values = VecMap::new(); new_key_values.append(key, sanitize_text(value)); entry.insert(SearchValue::KeyValues(new_key_values)); } } } pub fn is_empty(&self) -> bool { self.fields.is_empty() } pub fn has_field(&self, field: &SearchField) -> bool { self.fields.contains_key(field) } pub fn fields(&self) -> impl Iterator { self.fields.iter() } pub fn set_unknown_language(&mut self, lang: Language) { for value in self.fields.values_mut() { if let SearchValue::Text { language, .. } = value && language.is_unknown() { *language = lang; } } } } impl SearchFilter { pub fn cond( field: impl Into, op: SearchOperator, value: impl Into, ) -> Self { SearchFilter::Operator { field: field.into(), op, value: value.into(), } } pub fn eq(field: impl Into, value: impl Into) -> Self { SearchFilter::Operator { field: field.into(), op: SearchOperator::Equal, value: value.into(), } } pub fn lt(field: impl Into, value: impl Into) -> Self { SearchFilter::Operator { field: field.into(), op: SearchOperator::LowerThan, value: value.into(), } } pub fn le(field: impl Into, value: impl Into) -> Self { SearchFilter::Operator { field: field.into(), op: SearchOperator::LowerEqualThan, value: value.into(), } } pub fn gt(field: impl Into, value: impl Into) -> Self { SearchFilter::Operator { field: field.into(), op: SearchOperator::GreaterThan, value: value.into(), } } pub fn ge(field: impl Into, value: impl Into) -> Self { SearchFilter::Operator { field: field.into(), op: SearchOperator::GreaterEqualThan, value: value.into(), } } pub fn has_text_detect( field: impl Into, text: impl Into, default_language: Language, ) -> Self { let (text, language) = Language::detect(text.into(), default_language); Self::has_text(field, text, language) } pub fn has_text( field: impl Into, text: impl Into, language: Language, ) -> Self { let text = text.into(); let (is_exact, text) = if let Some(text) = text .strip_prefix('"') .and_then(|t| t.strip_suffix('"')) .or_else(|| text.strip_prefix('\'').and_then(|t| t.strip_suffix('\''))) { (true, text.to_string()) } else { (false, text) }; if !matches!(language, Language::None) && is_exact { SearchFilter::Operator { field: field.into(), op: SearchOperator::Equal, value: SearchValue::Text { value: text, language, }, } } else { SearchFilter::Operator { field: field.into(), op: SearchOperator::Contains, value: SearchValue::Text { value: text, language, }, } } } #[inline(always)] pub fn has_english_text(field: impl Into, text: impl Into) -> Self { Self::has_text(field, text, Language::English) } #[inline(always)] pub fn has_keyword(field: impl Into, text: impl Into) -> Self { Self::has_text(field, text, Language::None) } pub fn is_in_set(set: RoaringBitmap) -> Self { SearchFilter::DocumentSet(set) } } impl SearchComparator { pub fn field(field: impl Into, ascending: bool) -> Self { Self::Field { field: field.into(), ascending, } } pub fn set(set: RoaringBitmap, ascending: bool) -> Self { Self::DocumentSet { set, ascending } } pub fn sorted_set(set: AHashMap, ascending: bool) -> Self { Self::SortedSet { set, ascending } } pub fn ascending(field: impl Into) -> Self { Self::Field { field: field.into(), ascending: true, } } pub fn descending(field: impl Into) -> Self { Self::Field { field: field.into(), ascending: false, } } } #[inline(always)] fn write_sanitized(out: &mut String, text: &str) { let mut last_is_space = true; for ch in text.chars() { match ch { ' ' | '\x09'..='\x0d' => { if !last_is_space { out.push(' '); last_is_space = true; } } '\0'..='\x1f' | '\x7f'..='\u{9f}' => {} ch => { out.push(ch); last_is_space = false; } } } } #[inline(always)] fn sanitize_text_to_buf(out: &mut String, text: &str) { out.reserve_exact(text.len()); write_sanitized(out, text); } #[inline(always)] fn sanitize_text(text: &str) -> String { let mut out = String::with_capacity(text.len()); write_sanitized(&mut out, text); out } ================================================ FILE: crates/store/src/search/fields.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::search::*; impl SearchableField for EmailSearchField { fn index() -> SearchIndex { SearchIndex::Email } fn primary_keys() -> &'static [SearchField] { &[SearchField::AccountId, SearchField::DocumentId] } fn all_fields() -> &'static [SearchField] { &[ SearchField::Email(EmailSearchField::From), SearchField::Email(EmailSearchField::To), SearchField::Email(EmailSearchField::Cc), SearchField::Email(EmailSearchField::Bcc), SearchField::Email(EmailSearchField::Subject), SearchField::Email(EmailSearchField::Body), SearchField::Email(EmailSearchField::Attachment), SearchField::Email(EmailSearchField::ReceivedAt), SearchField::Email(EmailSearchField::SentAt), SearchField::Email(EmailSearchField::Size), SearchField::Email(EmailSearchField::HasAttachment), SearchField::Email(EmailSearchField::Headers), ] } fn is_indexed(&self) -> bool { #[cfg(not(feature = "test_mode"))] { matches!( self, EmailSearchField::From | EmailSearchField::To | EmailSearchField::Subject | EmailSearchField::ReceivedAt | EmailSearchField::SentAt | EmailSearchField::Size | EmailSearchField::HasAttachment, ) } #[cfg(feature = "test_mode")] { matches!( self, EmailSearchField::From | EmailSearchField::To | EmailSearchField::Subject | EmailSearchField::ReceivedAt | EmailSearchField::SentAt | EmailSearchField::Size | EmailSearchField::HasAttachment | EmailSearchField::Bcc | EmailSearchField::Cc ) } } fn is_text(&self) -> bool { matches!( self, EmailSearchField::From | EmailSearchField::To | EmailSearchField::Cc | EmailSearchField::Bcc | EmailSearchField::Subject | EmailSearchField::Body | EmailSearchField::Attachment, ) } } impl SearchableField for CalendarSearchField { fn index() -> SearchIndex { SearchIndex::Calendar } fn primary_keys() -> &'static [SearchField] { &[SearchField::AccountId, SearchField::DocumentId] } fn all_fields() -> &'static [SearchField] { &[ SearchField::Calendar(CalendarSearchField::Title), SearchField::Calendar(CalendarSearchField::Description), SearchField::Calendar(CalendarSearchField::Location), SearchField::Calendar(CalendarSearchField::Owner), SearchField::Calendar(CalendarSearchField::Attendee), SearchField::Calendar(CalendarSearchField::Start), SearchField::Calendar(CalendarSearchField::Uid), ] } fn is_indexed(&self) -> bool { matches!(self, CalendarSearchField::Start | CalendarSearchField::Uid) } fn is_text(&self) -> bool { !self.is_indexed() } } impl SearchableField for ContactSearchField { fn index() -> SearchIndex { SearchIndex::Contacts } fn primary_keys() -> &'static [SearchField] { &[SearchField::AccountId, SearchField::DocumentId] } fn all_fields() -> &'static [SearchField] { &[ SearchField::Contact(ContactSearchField::Member), SearchField::Contact(ContactSearchField::Kind), SearchField::Contact(ContactSearchField::Name), SearchField::Contact(ContactSearchField::Nickname), SearchField::Contact(ContactSearchField::Organization), SearchField::Contact(ContactSearchField::Email), SearchField::Contact(ContactSearchField::Phone), SearchField::Contact(ContactSearchField::OnlineService), SearchField::Contact(ContactSearchField::Address), SearchField::Contact(ContactSearchField::Note), SearchField::Contact(ContactSearchField::Uid), ] } fn is_indexed(&self) -> bool { matches!(self, ContactSearchField::Uid | ContactSearchField::Kind) } fn is_text(&self) -> bool { !self.is_indexed() } } impl SearchableField for FileSearchField { fn index() -> SearchIndex { SearchIndex::File } fn primary_keys() -> &'static [SearchField] { &[SearchField::AccountId, SearchField::DocumentId] } fn all_fields() -> &'static [SearchField] { &[ SearchField::File(FileSearchField::Name), SearchField::File(FileSearchField::Content), ] } fn is_indexed(&self) -> bool { false } fn is_text(&self) -> bool { true } } impl SearchableField for TracingSearchField { fn index() -> SearchIndex { SearchIndex::Tracing } fn primary_keys() -> &'static [SearchField] { &[SearchField::Id] } fn all_fields() -> &'static [SearchField] { &[ SearchField::Tracing(TracingSearchField::EventType), SearchField::Tracing(TracingSearchField::QueueId), SearchField::Tracing(TracingSearchField::Keywords), ] } fn is_indexed(&self) -> bool { matches!( self, TracingSearchField::QueueId | TracingSearchField::EventType ) } fn is_text(&self) -> bool { matches!(self, TracingSearchField::Keywords) } } impl SearchField { pub(crate) fn is_indexed(&self) -> bool { match self { SearchField::Email(field) => field.is_indexed(), SearchField::Calendar(field) => field.is_indexed(), SearchField::Contact(field) => field.is_indexed(), SearchField::File(field) => field.is_indexed(), SearchField::Tracing(field) => field.is_indexed(), SearchField::AccountId | SearchField::DocumentId | SearchField::Id => false, } } pub(crate) fn is_text(&self) -> bool { match self { SearchField::Email(field) => field.is_text(), SearchField::Calendar(field) => field.is_text(), SearchField::Contact(field) => field.is_text(), SearchField::File(field) => field.is_text(), SearchField::Tracing(field) => field.is_text(), SearchField::AccountId | SearchField::DocumentId | SearchField::Id => false, } } pub(crate) fn is_json(&self) -> bool { matches!(self, SearchField::Email(EmailSearchField::Headers)) } } impl SearchIndex { pub fn all_fields(&self) -> &[SearchField] { match self { SearchIndex::Email => EmailSearchField::all_fields(), SearchIndex::Calendar => CalendarSearchField::all_fields(), SearchIndex::Contacts => ContactSearchField::all_fields(), SearchIndex::File => FileSearchField::all_fields(), SearchIndex::Tracing => TracingSearchField::all_fields(), SearchIndex::InMemory => unreachable!(), } } pub fn primary_keys(&self) -> &'static [SearchField] { match self { SearchIndex::Email => EmailSearchField::primary_keys(), SearchIndex::Calendar => CalendarSearchField::primary_keys(), SearchIndex::Contacts => ContactSearchField::primary_keys(), SearchIndex::File => FileSearchField::primary_keys(), SearchIndex::Tracing => TracingSearchField::primary_keys(), SearchIndex::InMemory => unreachable!(), } } } ================================================ FILE: crates/store/src/search/index.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ Deserialize, IterateParams, Store, U64_LEN, ValueKey, search::{ IndexDocument, SearchField, SearchFilter, SearchOperator, SearchQuery, SearchValue, term::{TermIndex, TermIndexBuilder}, }, write::{ AlignedBytes, Archive, BatchBuilder, SEARCH_INDEX_MAX_FIELD_LEN, SearchIndex, SearchIndexClass, SearchIndexField, SearchIndexId, SearchIndexType, ValueClass, key::DeserializeBigEndian, }, }; use ahash::AHashMap; use trc::AddContext; use utils::cheeky_hash::CheekyHash; impl Store { pub(crate) async fn index(&self, documents: Vec) -> trc::Result<()> { let truncate_at = if self.is_foundationdb() { 1_048_576 } else { 0 }; for document in documents { let mut batch = BatchBuilder::new(); let index = document.index; let mut old_term_index = None; if matches!(index, SearchIndex::Calendar | SearchIndex::Contacts) { let mut account_id = None; let mut document_id = None; for (field, value) in &document.fields { if let SearchValue::Uint(id) = value { match field { SearchField::AccountId => { account_id = Some(*id as u32); } SearchField::DocumentId => { document_id = Some(*id as u32); } _ => {} } } } if let (Some(account_id), Some(document_id)) = (account_id, document_id) && let Some(archive) = self .get_value::>(ValueKey::from( ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Account { account_id, document_id, }, typ: SearchIndexType::Document, }), )) .await .caused_by(trc::location!())? { old_term_index = Some(archive); } } let term_index_builder = TermIndexBuilder::build(document, truncate_at); if let Some(old_term_index) = old_term_index { let old_term_index = old_term_index .unarchive::() .caused_by(trc::location!())?; term_index_builder .index .merge_index(&mut batch, index, term_index_builder.id, old_term_index) .caused_by(trc::location!())?; } else { term_index_builder .index .write_index(&mut batch, index, term_index_builder.id) .caused_by(trc::location!())?; } let mut commit_points = batch.commit_points(); for commit_point in commit_points.iter() { let batch = batch.build_one(commit_point); self.write(batch).await.caused_by(trc::location!())?; } } Ok(()) } pub(crate) async fn unindex(&self, query: SearchQuery) -> trc::Result<()> { let index = query.index; let mut account_documents: AHashMap> = AHashMap::new(); let mut ids = vec![]; let mut to_id = None; let mut last_account_id = None; for filter in query.filters { match filter { SearchFilter::Operator { field, op, value } => match (field, value) { (SearchField::AccountId, SearchValue::Uint(id)) if op == SearchOperator::Equal => { last_account_id = Some(id as u32); account_documents.entry(id as u32).or_default(); } (SearchField::DocumentId, SearchValue::Uint(id)) if op == SearchOperator::Equal && last_account_id.is_some() => { account_documents .get_mut(&last_account_id.unwrap()) .unwrap() .push(id as u32); } (SearchField::Id, SearchValue::Uint(id)) => match op { SearchOperator::LowerThan => { to_id = Some(id.saturating_sub(1)); } SearchOperator::LowerEqualThan => { to_id = Some(id); } SearchOperator::Equal => { ids.push(id); } _ => { return Err(trc::StoreEvent::UnexpectedError .into_err() .reason("Unsupported operator for Id field")); } }, filter => { return Err(trc::StoreEvent::UnexpectedError .into_err() .details(format!("Unsupported unindex filter {filter:?}"))); } }, SearchFilter::And | SearchFilter::Or | SearchFilter::End => {} SearchFilter::Not | SearchFilter::DocumentSet(_) => { return Err(trc::StoreEvent::UnexpectedError .into_err() .details(format!("Unsupported unindex filter {filter:?}"))); } } } // Delete by account and document ids for (account_id, document_ids) in account_documents { if !document_ids.is_empty() { for document_id in document_ids { let Some(archive) = self .get_value::>(ValueKey::from( ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Account { account_id, document_id, }, typ: SearchIndexType::Document, }), )) .await .caused_by(trc::location!())? else { continue; }; let term_index = archive .unarchive::() .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); term_index.delete_index( &mut batch, index, SearchIndexId::Account { account_id, document_id, }, ); self.write(batch.build_all()) .await .caused_by(trc::location!())?; } } else { // Delete all documents for the account self.delete_range( ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Account { account_id, document_id: 0, }, typ: SearchIndexType::Document, })), ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Account { account_id, document_id: u32::MAX, }, typ: SearchIndexType::Document, })), ) .await .caused_by(trc::location!())?; self.delete_range( ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Account { account_id, document_id: 0, }, typ: SearchIndexType::Index { field: SearchIndexField { field_id: 0, data: vec![0u8], }, }, })), ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Account { account_id, document_id: u32::MAX, }, typ: SearchIndexType::Index { field: SearchIndexField { field_id: u8::MAX, data: vec![u8::MAX; SEARCH_INDEX_MAX_FIELD_LEN], }, }, })), ) .await .caused_by(trc::location!())?; self.delete_range( ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Account { account_id, document_id: 0, }, typ: SearchIndexType::Term { hash: CheekyHash::NULL, field: 0, }, })), ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Account { account_id, document_id: u32::MAX, }, typ: SearchIndexType::Term { hash: CheekyHash::FULL, field: u8::MAX, }, })), ) .await .caused_by(trc::location!())?; } } // Delete by global ids for id in ids { let Some(archive) = self .get_value::>(ValueKey::from(ValueClass::SearchIndex( SearchIndexClass { index, id: SearchIndexId::Global { id }, typ: SearchIndexType::Document, }, ))) .await .caused_by(trc::location!())? else { continue; }; let term_index = archive .unarchive::() .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); term_index.delete_index(&mut batch, index, SearchIndexId::Global { id }); self.write(batch.build_all()) .await .caused_by(trc::location!())?; } // Delete ranges if let Some(to_id) = to_id { let mut batches = Vec::new(); self.iterate( IterateParams::new( ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Global { id: 0 }, typ: SearchIndexType::Document, })), ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Global { id: to_id }, typ: SearchIndexType::Document, })), ), |key, value| { let archive = as Deserialize>::deserialize(value)?; let term_index = archive.unarchive::()?; let mut batch = BatchBuilder::new(); term_index.delete_index( &mut batch, index, SearchIndexId::Global { id: key.deserialize_be_u64(key.len() - U64_LEN)?, }, ); batches.push(batch); Ok(true) }, ) .await .caused_by(trc::location!())?; for mut batch in batches { self.write(batch.build_all()) .await .caused_by(trc::location!())?; } } Ok(()) } } ================================================ FILE: crates/store/src/search/local.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::search::*; use roaring::RoaringBitmap; struct State { pub op: SearchFilter, pub bm: Option, } impl SearchQuery { pub fn new(index: SearchIndex) -> Self { Self { index, filters: Vec::new(), comparators: Vec::new(), mask: RoaringBitmap::new(), } } pub fn with_filters(mut self, filters: Vec) -> Self { if self.filters.is_empty() { self.filters = filters; } else { self.filters.extend(filters); } self } pub fn with_comparators(mut self, comparators: Vec) -> Self { if self.comparators.is_empty() { self.comparators = comparators; } else { self.comparators.extend(comparators); } self } pub fn with_filter(mut self, filter: SearchFilter) -> Self { self.filters.push(filter); self } pub fn add_filter(&mut self, filter: SearchFilter) -> &mut Self { self.filters.push(filter); self } pub fn with_comparator(mut self, comparator: SearchComparator) -> Self { self.comparators.push(comparator); self } pub fn with_mask(mut self, mask: RoaringBitmap) -> Self { self.mask = mask; self } pub fn with_account_id(mut self, account_id: u32) -> Self { self.filters.push(SearchFilter::cond( SearchField::AccountId, SearchOperator::Equal, SearchValue::Uint(account_id as u64), )); self } pub fn filter(self) -> QueryResults { if self.filters.is_empty() { return QueryResults { results: self.mask, comparators: self.comparators, }; } let mut state: State = State { op: SearchFilter::And, bm: None, }; let mut stack = Vec::new(); let mut filters = self.filters.into_iter().peekable(); let mask = self.mask; while let Some(filter) = filters.next() { let mut result = match filter { SearchFilter::DocumentSet(set) => Some(set), op @ (SearchFilter::And | SearchFilter::Or | SearchFilter::Not) => { stack.push(state); state = State { op, bm: None }; continue; } SearchFilter::End => { if let Some(prev_state) = stack.pop() { let bm = state.bm; state = prev_state; bm } else { break; } } SearchFilter::Operator { .. } => { continue; } }; // Apply logical operation if let Some(dest) = &mut state.bm { match state.op { SearchFilter::And => { if let Some(result) = result { dest.bitand_assign(result); } else { dest.clear(); } } SearchFilter::Or => { if let Some(result) = result { dest.bitor_assign(result); } } SearchFilter::Not => { if let Some(mut result) = result { result.bitxor_assign(&mask); dest.bitand_assign(result); } } _ => unreachable!(), } } else if let Some(ref mut result_) = result { if let SearchFilter::Not = state.op { result_.bitxor_assign(&mask); } state.bm = result; } else if let SearchFilter::Not = state.op { state.bm = Some(mask.clone()); } else { state.bm = Some(RoaringBitmap::new()); } // And short-circuit if matches!(state.op, SearchFilter::And) && state.bm.as_ref().unwrap().is_empty() { while let Some(filter) = filters.peek() { if matches!(filter, SearchFilter::End) { break; } else { filters.next(); } } } } // AND with mask let mut results = state.bm.unwrap_or_default(); results.bitand_assign(&mask); QueryResults { results, comparators: self.comparators, } } } impl QueryResults { pub fn new(results: RoaringBitmap, comparators: Vec) -> Self { Self { results, comparators, } } pub fn with_comparators(mut self, comparators: Vec) -> Self { if self.comparators.is_empty() { self.comparators = comparators; } else { self.comparators.extend(comparators); } self } pub fn results(&self) -> &RoaringBitmap { &self.results } pub fn update_results(&mut self, results: RoaringBitmap) { self.results = results; } pub fn into_bitmap(self) -> RoaringBitmap { self.results } pub fn into_sorted(self) -> Vec { let comparators = self.comparators; let mut results = self.results.into_iter().collect::>(); if !results.is_empty() && !comparators.is_empty() { results.sort_by(|a, b| { for comparator in &comparators { let (a, b, is_ascending) = match comparator { SearchComparator::DocumentSet { set, ascending } => ( !set.contains(*a) as u32, !set.contains(*b) as u32, *ascending, ), SearchComparator::SortedSet { set, ascending } => ( *set.get(a).unwrap_or(&u32::MAX), *set.get(b).unwrap_or(&u32::MAX), *ascending, ), SearchComparator::Field { .. } => continue, }; let ordering = if is_ascending { a.cmp(&b) } else { b.cmp(&a) }; if ordering != Ordering::Equal { return ordering; } } Ordering::Equal }); } results } } ================================================ FILE: crates/store/src/search/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod bm_u32; pub mod bm_u64; pub mod document; pub mod fields; pub mod index; pub mod local; pub mod query; pub mod split; pub mod term; use crate::write::SearchIndex; use ahash::AHashMap; use nlp::language::Language; use roaring::RoaringBitmap; use std::cmp::Ordering; use std::collections::hash_map::Entry; use std::fmt::Display; use std::ops::{BitAndAssign, BitOrAssign, BitXorAssign}; use utils::config::utils::ParseValue; use utils::map::vec_map::VecMap; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SearchOperator { LowerThan, LowerEqualThan, GreaterThan, GreaterEqualThan, Equal, Contains, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum SearchField { AccountId, DocumentId, Id, Email(EmailSearchField), Calendar(CalendarSearchField), Contact(ContactSearchField), File(FileSearchField), Tracing(TracingSearchField), } #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum EmailSearchField { From, To, Cc, Bcc, Subject, Body, Attachment, ReceivedAt, SentAt, Size, HasAttachment, Headers, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum CalendarSearchField { Title, Description, Location, Owner, Attendee, Start, Uid, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum ContactSearchField { Member, Kind, Name, Nickname, Organization, Email, Phone, OnlineService, Address, Note, Uid, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum FileSearchField { Name, Content, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum TracingSearchField { EventType, QueueId, Keywords, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum SearchValue { Text { value: String, language: Language }, KeyValues(VecMap), Int(i64), Uint(u64), Boolean(bool), } pub trait SearchDocumentId: Sized + Copy + Display { fn from_u64(id: u64) -> Self; fn field() -> SearchField; } #[derive(Debug)] pub struct SearchQuery { pub(crate) index: SearchIndex, pub(crate) filters: Vec, pub(crate) comparators: Vec, pub(crate) mask: RoaringBitmap, } #[derive(Debug, PartialEq, Clone, Default)] pub enum SearchFilter { Operator { field: SearchField, op: SearchOperator, value: SearchValue, }, DocumentSet(RoaringBitmap), And, Or, Not, #[default] End, } #[derive(Debug)] pub enum SearchComparator { Field { field: SearchField, ascending: bool, }, DocumentSet { set: RoaringBitmap, ascending: bool, }, SortedSet { set: AHashMap, ascending: bool, }, } #[derive(Debug)] pub struct IndexDocument { pub(crate) index: SearchIndex, pub(crate) fields: AHashMap, } #[derive(Debug)] pub struct QueryResults { results: RoaringBitmap, comparators: Vec, } impl From for SearchField { fn from(field: EmailSearchField) -> Self { SearchField::Email(field) } } impl From for SearchField { fn from(field: CalendarSearchField) -> Self { SearchField::Calendar(field) } } impl From for SearchField { fn from(field: ContactSearchField) -> Self { SearchField::Contact(field) } } impl From for SearchField { fn from(field: FileSearchField) -> Self { SearchField::File(field) } } impl From for SearchField { fn from(field: TracingSearchField) -> Self { SearchField::Tracing(field) } } impl From for SearchValue { fn from(value: u64) -> Self { SearchValue::Uint(value) } } impl From for SearchValue { fn from(value: i64) -> Self { SearchValue::Int(value) } } impl From for SearchValue { fn from(value: u32) -> Self { SearchValue::Uint(value as u64) } } impl From for SearchValue { fn from(value: i32) -> Self { SearchValue::Int(value as i64) } } impl From for SearchValue { fn from(value: usize) -> Self { SearchValue::Uint(value as u64) } } impl From for SearchValue { fn from(value: bool) -> Self { SearchValue::Boolean(value) } } impl From for SearchValue { fn from(value: String) -> Self { SearchValue::Text { value, language: Language::None, } } } impl SearchDocumentId for u32 { fn from_u64(id: u64) -> Self { id as u32 } fn field() -> SearchField { SearchField::DocumentId } } impl SearchDocumentId for u64 { fn from_u64(id: u64) -> Self { id } fn field() -> SearchField { SearchField::Id } } pub trait SearchableField: Sized { fn index() -> SearchIndex; fn primary_keys() -> &'static [SearchField]; fn all_fields() -> &'static [SearchField]; fn is_indexed(&self) -> bool; fn is_text(&self) -> bool; } impl ParseValue for SearchField { fn parse_value(value: &str) -> utils::config::Result { Ok(match value { // Email "email-from" => Self::Email(EmailSearchField::From), "email-to" => Self::Email(EmailSearchField::To), "email-cc" => Self::Email(EmailSearchField::Cc), "email-bcc" => Self::Email(EmailSearchField::Bcc), "email-subject" => Self::Email(EmailSearchField::Subject), "email-body" => Self::Email(EmailSearchField::Body), "email-attachment" => Self::Email(EmailSearchField::Attachment), "email-received-at" => Self::Email(EmailSearchField::ReceivedAt), "email-sent-at" => Self::Email(EmailSearchField::SentAt), "email-size" => Self::Email(EmailSearchField::Size), "email-has-attachment" => Self::Email(EmailSearchField::HasAttachment), "email-headers" => Self::Email(EmailSearchField::Headers), // Calendar "cal-title" => Self::Calendar(CalendarSearchField::Title), "cal-desc" => Self::Calendar(CalendarSearchField::Description), "cal-location" => Self::Calendar(CalendarSearchField::Location), "cal-owner" => Self::Calendar(CalendarSearchField::Owner), "cal-attendee" => Self::Calendar(CalendarSearchField::Attendee), "cal-start" => Self::Calendar(CalendarSearchField::Start), "cal-uid" => Self::Calendar(CalendarSearchField::Uid), // Contact "contact-member" => Self::Contact(ContactSearchField::Member), "contact-kind" => Self::Contact(ContactSearchField::Kind), "contact-name" => Self::Contact(ContactSearchField::Name), "contact-nickname" => Self::Contact(ContactSearchField::Nickname), "contact-org" => Self::Contact(ContactSearchField::Organization), "contact-email" => Self::Contact(ContactSearchField::Email), "contact-phone" => Self::Contact(ContactSearchField::Phone), "contact-online-service" => Self::Contact(ContactSearchField::OnlineService), "contact-address" => Self::Contact(ContactSearchField::Address), "contact-note" => Self::Contact(ContactSearchField::Note), "contact-uid" => Self::Contact(ContactSearchField::Uid), // File "file-name" => Self::File(FileSearchField::Name), "file-content" => Self::File(FileSearchField::Content), // Tracing "trace-event-type" => Self::Tracing(TracingSearchField::EventType), "trace-queue-id" => Self::Tracing(TracingSearchField::QueueId), "trace-keywords" => Self::Tracing(TracingSearchField::Keywords), _ => return Err(format!("Unknown search field: {value}")), }) } } impl Eq for SearchFilter {} impl SearchIndex { pub fn index_name(&self) -> &'static str { match self { SearchIndex::Email => "st_email", SearchIndex::Calendar => "st_calendar", SearchIndex::Contacts => "st_contact", SearchIndex::File => "st_file", SearchIndex::Tracing => "st_tracing", SearchIndex::InMemory => unreachable!(), } } } impl SearchField { pub fn field_name(&self) -> &'static str { match self { SearchField::AccountId => "acc_id", SearchField::DocumentId => "doc_id", SearchField::Id => "id", SearchField::Email(field) => match field { EmailSearchField::From => "from", EmailSearchField::To => "to", EmailSearchField::Cc => "cc", EmailSearchField::Bcc => "bcc", EmailSearchField::Subject => "subj", EmailSearchField::Body => "body", EmailSearchField::Attachment => "attach", EmailSearchField::ReceivedAt => "rcvd", EmailSearchField::SentAt => "sent", EmailSearchField::Size => "size", EmailSearchField::HasAttachment => "has_att", EmailSearchField::Headers => "headers", }, SearchField::Calendar(field) => match field { CalendarSearchField::Title => "title", CalendarSearchField::Description => "desc", CalendarSearchField::Location => "loc", CalendarSearchField::Owner => "owner", CalendarSearchField::Attendee => "attendee", CalendarSearchField::Start => "start", CalendarSearchField::Uid => "uid", }, SearchField::Contact(field) => match field { ContactSearchField::Member => "member", ContactSearchField::Kind => "kind", ContactSearchField::Name => "name", ContactSearchField::Nickname => "nick", ContactSearchField::Organization => "org", ContactSearchField::Email => "email", ContactSearchField::Phone => "phone", ContactSearchField::OnlineService => "online", ContactSearchField::Address => "addr", ContactSearchField::Note => "note", ContactSearchField::Uid => "uid", }, SearchField::File(field) => match field { FileSearchField::Name => "name", FileSearchField::Content => "content", }, SearchField::Tracing(field) => match field { TracingSearchField::EventType => "ev_type", TracingSearchField::QueueId => "queue_id", TracingSearchField::Keywords => "keywords", }, } } } ================================================ FILE: crates/store/src/search/query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ Store, backend::MAX_TOKEN_LENGTH, search::{ QueryResults, SearchComparator, SearchField, SearchFilter, SearchOperator, SearchQuery, SearchValue, bm_u32::{BitmapCache, range_to_bitmap, sort_order}, bm_u64::{TreemapCache, range_to_treemap}, }, write::SEARCH_INDEX_MAX_FIELD_LEN, }; use nlp::{language::stemmer::Stemmer, tokenizers::space::SpaceTokenizer}; use roaring::{RoaringBitmap, RoaringTreemap}; use std::ops::{BitAndAssign, BitOrAssign, BitXorAssign}; use utils::cheeky_hash::CheekyHash; impl Store { pub(crate) async fn query_account(&self, query: SearchQuery) -> trc::Result> { struct State { pub op: SearchFilter, pub bm: Option, } let mut state: State = State { op: SearchFilter::And, bm: None, }; let mut stack = Vec::new(); let mask = query.mask; let mut bitmaps = BitmapCache::default(); let mut account_id = u32::MAX; for filter in &query.filters { if let SearchFilter::Operator { field: SearchField::AccountId, value: SearchValue::Uint(id), .. } = filter { account_id = *id as u32; break; } } if account_id == u32::MAX { return Err(trc::StoreEvent::UnexpectedError .into_err() .details("Account ID must be specified before other filters")); } let mut results; if query.filters.len() > 1 { let mut filters = query.filters.into_iter().peekable(); while let Some(filter) = filters.next() { let mut result = match filter { SearchFilter::Operator { field, op, value } => { if matches!(field, SearchField::AccountId) { continue; } if field.is_text() && matches!(op, SearchOperator::Contains | SearchOperator::Equal) { let (value, language) = match value { SearchValue::Text { value, language } => (value, language), _ => { return Err(trc::StoreEvent::UnexpectedError .into_err() .details("Expected text value for text field")); } }; if op == SearchOperator::Equal { bitmaps .merge_bitmaps( self, query.index, account_id, language .tokenize_text(&value, MAX_TOKEN_LENGTH) .map(|token| CheekyHash::new(token.word.as_bytes())), field.u8_id(), false, ) .await? } else { let mut result = RoaringBitmap::new(); for token in Stemmer::new(&value, language, MAX_TOKEN_LENGTH) { let mut tokens = Vec::with_capacity(3); tokens.push(CheekyHash::new(token.word.as_bytes())); tokens.push(CheekyHash::new( format!("{}*", token.word).as_bytes(), )); if let Some(stemmed_word) = token.stemmed_word { tokens.push(CheekyHash::new( format!("{stemmed_word}*").as_bytes(), )); } let union = bitmaps .merge_bitmaps( self, query.index, account_id, tokens.into_iter(), field.u8_id(), true, ) .await?; if let Some(union) = union { if result.is_empty() { result = union; } else { result.bitand_assign(&union); if result.is_empty() { break; } } } else { result.clear(); break; } } if !result.is_empty() { Some(result) } else { None } } } else if field.is_json() { let (key, value) = match value { SearchValue::KeyValues(kv) => kv.into_iter().next().unwrap(), _ => { return Err(trc::StoreEvent::UnexpectedError .into_err() .details("Expected text value for text field")); } }; if !value.is_empty() { bitmaps .merge_bitmaps( self, query.index, account_id, SpaceTokenizer::new(value.as_str(), MAX_TOKEN_LENGTH).map( |value| { CheekyHash::new(format!("{key} {value}").as_bytes()) }, ), field.u8_id(), true, ) .await? } else { bitmaps .merge_bitmaps( self, query.index, account_id, [CheekyHash::new(key.as_bytes())].into_iter(), field.u8_id(), false, ) .await? } } else if field.is_indexed() { let value = match value { SearchValue::Text { value, .. } => { let mut value = value.into_bytes(); value.truncate(SEARCH_INDEX_MAX_FIELD_LEN); value } SearchValue::Int(v) => (v as u64).to_be_bytes().to_vec(), SearchValue::Uint(v) => v.to_be_bytes().to_vec(), SearchValue::Boolean(v) => vec![v as u8], SearchValue::KeyValues(_) => { return Err(trc::StoreEvent::UnexpectedError .into_err() .details("Expected non key-value for non-text field")); } }; range_to_bitmap( self, query.index, account_id, field.u8_id(), &value, op, ) .await? } else { return Err(trc::StoreEvent::UnexpectedError .into_err() .details(format!("Field {field:?} is not indexed"))); } } SearchFilter::DocumentSet(bitmap) => Some(bitmap), op @ (SearchFilter::And | SearchFilter::Or | SearchFilter::Not) => { stack.push(state); state = State { op, bm: None }; continue; } SearchFilter::End => { if let Some(prev_state) = stack.pop() { let bm = state.bm; state = prev_state; bm } else { break; } } }; // Apply logical operation if let Some(dest) = &mut state.bm { match state.op { SearchFilter::And => { if let Some(result) = result { dest.bitand_assign(result); } else { dest.clear(); } } SearchFilter::Or => { if let Some(result) = result { dest.bitor_assign(result); } } SearchFilter::Not => { if let Some(mut result) = result { result.bitxor_assign(&mask); dest.bitand_assign(result); } } _ => unreachable!(), } } else if let Some(result_) = &mut result { if let SearchFilter::Not = state.op { result_.bitxor_assign(&mask); } state.bm = result; } else if let SearchFilter::Not = state.op { state.bm = Some(mask.clone()); } else { state.bm = Some(RoaringBitmap::new()); } // And short circuit if matches!(state.op, SearchFilter::And) && state.bm.as_ref().unwrap().is_empty() { while let Some(filter) = filters.peek() { if matches!(filter, SearchFilter::End) { break; } else { filters.next(); } } } } results = state.bm.unwrap_or_default(); results.bitand_assign(&mask); } else { results = mask; } if results.len() > 1 && !query.comparators.is_empty() { let mut comparators = Vec::with_capacity(query.comparators.len()); for comparator in query.comparators { let comparator = match comparator { SearchComparator::Field { field, ascending } => SearchComparator::SortedSet { set: sort_order(self, query.index, account_id, field.u8_id()).await?, ascending, }, _ => comparator, }; comparators.push(comparator); } Ok(QueryResults::new(results, comparators).into_sorted()) } else { Ok(results.into_iter().collect::>()) } } pub(crate) async fn query_global(&self, query: SearchQuery) -> trc::Result> { struct State { pub op: SearchFilter, pub bm: Option, } let mut state: State = State { op: SearchFilter::And, bm: None, }; let mut stack = Vec::new(); let mut filters = query.filters.into_iter().peekable(); let mut bitmaps = TreemapCache::default(); while let Some(filter) = filters.next() { let result = match filter { SearchFilter::Operator { field, op, value } => { if field.is_text() { let value = match value { SearchValue::Text { value, .. } => value, _ => { return Err(trc::StoreEvent::UnexpectedError .into_err() .details("Expected text value for text field")); } }; bitmaps .merge_treemaps( self, query.index, SpaceTokenizer::new(value.as_str(), MAX_TOKEN_LENGTH) .map(|word| CheekyHash::new(word.as_bytes())), field.u8_id(), false, ) .await? } else if field.is_indexed() || matches!(field, SearchField::Id) { let value = match value { SearchValue::Text { value, .. } => value.into_bytes(), SearchValue::Int(v) => (v as u64).to_be_bytes().to_vec(), SearchValue::Uint(v) => v.to_be_bytes().to_vec(), SearchValue::Boolean(v) => vec![v as u8], SearchValue::KeyValues(_) => { return Err(trc::StoreEvent::UnexpectedError .into_err() .details("Expected non key-value for non-text field")); } }; range_to_treemap(self, query.index, field.u8_id(), &value, op).await? } else { return Err(trc::StoreEvent::UnexpectedError .into_err() .details(format!("Field {field:?} is not indexed"))); } } SearchFilter::DocumentSet(_) | SearchFilter::Not => { return Err(trc::StoreEvent::UnexpectedError .into_err() .details("Unsupported filter in global search")); } op @ (SearchFilter::And | SearchFilter::Or) => { stack.push(state); state = State { op, bm: None }; continue; } SearchFilter::End => { if let Some(prev_state) = stack.pop() { let bm = state.bm; state = prev_state; bm } else { break; } } }; // Apply logical operation if let Some(dest) = &mut state.bm { match state.op { SearchFilter::And => { if let Some(result) = result { dest.bitand_assign(result); } else { dest.clear(); } } SearchFilter::Or => { if let Some(result) = result { dest.bitor_assign(result); } } _ => unreachable!(), } } else if result.is_some() { state.bm = result; } else { state.bm = Some(RoaringTreemap::new()); } // And short circuit if matches!(state.op, SearchFilter::And) && state.bm.as_ref().unwrap().is_empty() { while let Some(filter) = filters.peek() { if matches!(filter, SearchFilter::End) { break; } else { filters.next(); } } } } if query.comparators.iter().all(|c| { matches!( c, SearchComparator::Field { field: SearchField::Id, ascending: false } ) }) { Ok(state .bm .unwrap_or_default() .into_iter() .rev() .collect::>()) } else { Ok(state.bm.unwrap_or_default().into_iter().collect::>()) } } } ================================================ FILE: crates/store/src/search/split.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::search::*; #[derive(Debug, PartialEq, Eq)] pub(crate) enum SplitFilter { Internal(SearchFilter), External(Vec), } pub(crate) fn split_filters(filters_in: Vec) -> Option> { let mut account_id = u64::MAX; let mut filters: Vec = Vec::with_capacity(filters_in.len()); let mut op_stack = Vec::new(); let mut document_sets: AHashMap = AHashMap::new(); let mut operators: AHashMap> = AHashMap::new(); for filter in filters_in { match filter { op @ (SearchFilter::And | SearchFilter::Or | SearchFilter::Not) => { op_stack.push(op.clone()); filters.push(op); } SearchFilter::End => { if let Some(ops) = operators.remove(&op_stack.len()) { filters.extend(ops); } if let Some(docs) = document_sets.remove(&op_stack.len()) { filters.push(SearchFilter::DocumentSet(docs)); } filters.push(SearchFilter::End); op_stack.pop()?; } SearchFilter::Operator { field: SearchField::AccountId, value: SearchValue::Uint(id), .. } => { account_id = id; } SearchFilter::Operator { .. } => { operators.entry(op_stack.len()).or_default().push(filter); } SearchFilter::DocumentSet(docs) => match document_sets.entry(op_stack.len()) { Entry::Occupied(mut entry) => { if matches!(op_stack.last(), Some(SearchFilter::Or)) { entry.get_mut().bitor_assign(&docs); } else { entry.get_mut().bitand_assign(&docs); } } Entry::Vacant(entry) => { entry.insert(docs); } }, } } if let Some(ops) = operators.remove(&0) { filters.extend(ops); } if let Some(docs) = document_sets.remove(&0) { filters.push(SearchFilter::DocumentSet(docs)); } if account_id == u64::MAX { return None; } let mut split: Vec = Vec::new(); let mut i = 0; 'outer: while i < filters.len() { let mut j = i; let mut depth = 0; while j < filters.len() { match &filters[j] { SearchFilter::And | SearchFilter::Or | SearchFilter::Not => { depth += 1; } SearchFilter::End => { depth -= 1; if depth < 0 { if j > i { break; } else { split.push(SplitFilter::Internal(SearchFilter::End)); i += 1; continue 'outer; } } } SearchFilter::Operator { .. } => {} SearchFilter::DocumentSet(_) => { if depth == 0 && j > i { break; } else { split.push(SplitFilter::Internal(std::mem::take(&mut filters[i]))); i += 1; continue 'outer; } } } j += 1; } let mut external_filters = vec![SearchFilter::Operator { field: SearchField::AccountId, op: SearchOperator::Equal, value: SearchValue::Uint(account_id), }]; let add_or = matches!(split.last(), Some(SplitFilter::Internal(SearchFilter::Or))) && j > i + 1; if add_or { external_filters.push(SearchFilter::Or); } external_filters.extend(&mut filters[i..j].iter_mut().map(std::mem::take)); if add_or { external_filters.push(SearchFilter::End); } split.push(SplitFilter::External(external_filters)); i = j; } Some(split) } // Test cases #[cfg(test)] mod tests { use super::*; #[test] fn test_split_filters_exhaustive() { let test_cases: Vec<(&str, Vec, Vec)> = vec![ // Test 1: Operator followed by document set at depth 0 ( "Operator then document set at depth 0", vec![account_id(42), other_op("test"), doc_set(&[1, 2, 3])], vec![ SplitFilter::External(vec![account_id(42), other_op("test")]), SplitFilter::Internal(doc_set(&[1, 2, 3])), ], ), // Test 2: Document set followed by operator at depth 0 ( "Document set then operator at depth 0", vec![account_id(42), doc_set(&[1, 2, 3]), other_op("test")], vec![ SplitFilter::External(vec![account_id(42), other_op("test")]), SplitFilter::Internal(doc_set(&[1, 2, 3])), ], ), // Test 3: Multiple document sets with operator in between ( "Multiple document sets at depth 0 with operator", vec![ account_id(42), doc_set(&[1, 2]), other_op("middle"), doc_set(&[2, 4]), ], vec![ SplitFilter::External(vec![account_id(42), other_op("middle")]), SplitFilter::Internal(doc_set(&[2])), ], ), // Test 4: Document set at depth 0, then AND group ( "Document set then AND group", vec![ account_id(42), doc_set(&[1, 2]), SearchFilter::And, other_op("a"), other_op("b"), SearchFilter::End, ], vec![ SplitFilter::External(vec![ account_id(42), SearchFilter::And, other_op("a"), other_op("b"), SearchFilter::End, ]), SplitFilter::Internal(doc_set(&[1, 2])), ], ), // Test 5: AND group followed by document set at depth 0 ( "AND group then document set", vec![ account_id(42), SearchFilter::And, other_op("a"), other_op("b"), SearchFilter::End, doc_set(&[1, 2]), ], vec![ SplitFilter::External(vec![ account_id(42), SearchFilter::And, other_op("a"), other_op("b"), SearchFilter::End, ]), SplitFilter::Internal(doc_set(&[1, 2])), ], ), // Test 6: Operator at depth 0, then OR group, then document set ( "Operator, OR group, then document set", vec![ account_id(42), other_op("pre"), SearchFilter::Or, other_op("a"), other_op("b"), SearchFilter::End, doc_set(&[1, 2, 3]), ], vec![ SplitFilter::External(vec![ account_id(42), SearchFilter::Or, other_op("a"), other_op("b"), SearchFilter::End, other_op("pre"), ]), SplitFilter::Internal(doc_set(&[1, 2, 3])), ], ), // Test 7: Document set, OR group, operator ( "Document set, OR group, operator", vec![ account_id(42), doc_set(&[1, 2]), SearchFilter::Or, other_op("a"), other_op("b"), SearchFilter::End, other_op("post"), ], vec![ SplitFilter::External(vec![ account_id(42), SearchFilter::Or, other_op("a"), other_op("b"), SearchFilter::End, other_op("post"), ]), SplitFilter::Internal(doc_set(&[1, 2])), ], ), // Test 8: Multiple OR branches with document sets between ( "Multiple OR branches with document sets between", vec![ account_id(42), SearchFilter::Or, other_op("a"), SearchFilter::End, doc_set(&[1, 2]), SearchFilter::Or, other_op("b"), SearchFilter::End, doc_set(&[1, 2, 5, 6]), ], vec![ SplitFilter::External(vec![ account_id(42), SearchFilter::Or, other_op("a"), SearchFilter::End, SearchFilter::Or, other_op("b"), SearchFilter::End, ]), SplitFilter::Internal(doc_set(&[1, 2])), ], ), // Test 9: Document sets at different depths - depth 0 and inside AND ( "Document sets at different depths in AND", vec![ account_id(42), doc_set(&[1, 2]), SearchFilter::And, other_op("a"), doc_set(&[2, 3]), SearchFilter::End, ], vec![ SplitFilter::Internal(SearchFilter::And), SplitFilter::External(vec![account_id(42), other_op("a")]), SplitFilter::Internal(doc_set(&[2, 3])), SplitFilter::Internal(SearchFilter::End), SplitFilter::Internal(doc_set(&[1, 2])), ], ), // Test 10: Operator, AND group with doc set inside, operator ( "Operator, AND(operator, doc_set), operator", vec![ account_id(42), other_op("pre"), SearchFilter::And, other_op("a"), doc_set(&[1, 2, 3]), SearchFilter::End, other_op("post"), ], vec![ SplitFilter::Internal(SearchFilter::And), SplitFilter::External(vec![account_id(42), other_op("a")]), SplitFilter::Internal(doc_set(&[1, 2, 3])), SplitFilter::Internal(SearchFilter::End), SplitFilter::External(vec![account_id(42), other_op("pre"), other_op("post")]), ], ), // Test 11: Document set, nested groups, document set ( "Doc set, AND(OR(a,b)), doc set", vec![ account_id(42), SearchFilter::Or, doc_set(&[1, 2]), other_op("c"), SearchFilter::And, other_op("a"), other_op("b"), SearchFilter::End, doc_set(&[3, 4]), SearchFilter::End, ], vec![ SplitFilter::Internal(SearchFilter::Or), SplitFilter::External(vec![ account_id(42), SearchFilter::Or, SearchFilter::And, other_op("a"), other_op("b"), SearchFilter::End, other_op("c"), SearchFilter::End, ]), SplitFilter::Internal(doc_set(&[1, 2, 3, 4])), SplitFilter::Internal(SearchFilter::End), ], ), // Test 12: OR with nested AND containing document sets, followed by operator ( "OR(AND(doc_set, doc_set), operator) followed by operator", vec![ account_id(42), SearchFilter::Or, SearchFilter::And, doc_set(&[1, 2]), doc_set(&[2, 3]), SearchFilter::End, other_op("b"), SearchFilter::End, other_op("post"), ], vec![ SplitFilter::Internal(SearchFilter::Or), SplitFilter::Internal(SearchFilter::And), SplitFilter::Internal(doc_set(&[2])), SplitFilter::Internal(SearchFilter::End), SplitFilter::External(vec![account_id(42), other_op("b")]), SplitFilter::Internal(SearchFilter::End), SplitFilter::External(vec![account_id(42), other_op("post")]), ], ), // Test 13: Complex: doc set, AND group, doc set, OR group, doc set ( "Complex: doc, AND, doc, OR, doc", vec![ account_id(42), doc_set(&[1, 2, 3]), SearchFilter::And, other_op("a"), SearchFilter::End, doc_set(&[1, 2, 3, 5]), SearchFilter::Or, other_op("b"), SearchFilter::End, doc_set(&[1, 2, 3, 6]), ], vec![ SplitFilter::External(vec![ account_id(42), SearchFilter::And, other_op("a"), SearchFilter::End, SearchFilter::Or, other_op("b"), SearchFilter::End, ]), SplitFilter::Internal(doc_set(&[1, 2, 3])), ], ), // Test 14: Operator, NOT group, document set ( "Operator, NOT(operator), document set", vec![ account_id(42), other_op("pre"), SearchFilter::Not, other_op("a"), SearchFilter::End, doc_set(&[1, 2]), ], vec![ SplitFilter::External(vec![ account_id(42), SearchFilter::Not, other_op("a"), SearchFilter::End, other_op("pre"), ]), SplitFilter::Internal(doc_set(&[1, 2])), ], ), // Test 15: Document set, NOT group, operator ( "Document set, NOT(operator), operator", vec![ account_id(42), doc_set(&[1, 2]), SearchFilter::Not, other_op("a"), doc_set(&[3, 4]), SearchFilter::End, other_op("post"), ], vec![ SplitFilter::Internal(SearchFilter::Not), SplitFilter::External(vec![account_id(42), other_op("a")]), SplitFilter::Internal(doc_set(&[3, 4])), SplitFilter::Internal(SearchFilter::End), SplitFilter::External(vec![account_id(42), other_op("post")]), SplitFilter::Internal(doc_set(&[1, 2])), ], ), // Test 16: Alternating doc sets and operators ( "Alternating: doc, op, doc, op, doc", vec![ account_id(42), doc_set(&[1]), other_op("a"), doc_set(&[1, 2]), other_op("b"), doc_set(&[1, 3]), ], vec![ SplitFilter::External(vec![account_id(42), other_op("a"), other_op("b")]), SplitFilter::Internal(doc_set(&[1])), ], ), // Test 17: Multiple operators, then OR group with doc set inside, then doc set ( "Multiple ops, OR(op, doc_set), doc", vec![ account_id(42), other_op("a"), SearchFilter::Or, other_op("c"), doc_set(&[1, 2]), SearchFilter::End, other_op("b"), doc_set(&[3, 4]), ], vec![ SplitFilter::Internal(SearchFilter::Or), SplitFilter::External(vec![account_id(42), other_op("c")]), SplitFilter::Internal(doc_set(&[1, 2])), SplitFilter::Internal(SearchFilter::End), SplitFilter::External(vec![account_id(42), other_op("a"), other_op("b")]), SplitFilter::Internal(doc_set(&[3, 4])), ], ), // Test 18: Doc set before and after nested OR(AND(op)) ( "Doc, OR(AND(op)), doc", vec![ account_id(42), doc_set(&[1]), SearchFilter::Or, SearchFilter::And, other_op("a"), other_op("c"), SearchFilter::End, other_op("b"), SearchFilter::End, doc_set(&[2]), ], vec![ SplitFilter::External(vec![ account_id(42), SearchFilter::Or, SearchFilter::And, other_op("a"), other_op("c"), SearchFilter::End, other_op("b"), SearchFilter::End, ]), SplitFilter::Internal(doc_set(&[])), ], ), // Test 19: AND group with doc set, operator between, OR group with doc set ( "AND(op, doc), op, OR(op, doc)", vec![ account_id(42), SearchFilter::And, other_op("a"), doc_set(&[1, 2]), SearchFilter::End, other_op("middle"), SearchFilter::Or, other_op("b"), other_op("c"), doc_set(&[3, 4]), SearchFilter::End, ], vec![ SplitFilter::Internal(SearchFilter::And), SplitFilter::External(vec![account_id(42), other_op("a")]), SplitFilter::Internal(doc_set(&[1, 2])), SplitFilter::Internal(SearchFilter::End), SplitFilter::Internal(SearchFilter::Or), SplitFilter::External(vec![ account_id(42), SearchFilter::Or, other_op("b"), other_op("c"), SearchFilter::End, ]), SplitFilter::Internal(doc_set(&[3, 4])), SplitFilter::Internal(SearchFilter::End), SplitFilter::External(vec![account_id(42), other_op("middle")]), ], ), // Test 20: Deep nesting with document sets at multiple levels ( "Deep nesting: doc, AND(doc, OR(doc, AND(op, doc)))", vec![ account_id(42), doc_set(&[1]), SearchFilter::And, doc_set(&[2]), SearchFilter::Or, doc_set(&[3]), SearchFilter::And, other_op("a"), doc_set(&[4]), SearchFilter::End, SearchFilter::End, SearchFilter::End, ], vec![ SplitFilter::Internal(SearchFilter::And), SplitFilter::Internal(SearchFilter::Or), SplitFilter::Internal(SearchFilter::And), SplitFilter::External(vec![account_id(42), other_op("a")]), SplitFilter::Internal(doc_set(&[4])), SplitFilter::Internal(SearchFilter::End), SplitFilter::Internal(doc_set(&[3])), SplitFilter::Internal(SearchFilter::End), SplitFilter::Internal(doc_set(&[2])), SplitFilter::Internal(SearchFilter::End), SplitFilter::Internal(doc_set(&[1])), ], ), ]; for (description, input, expected) in test_cases { println!("------ Running test: {} ------", description); let result = split_filters(input.clone()); assert!(result.is_some(), "Test '{}' returned None", description); let result = result.unwrap(); if result != expected { print_split_filter_code(&result); } assert_eq!(result, expected, "Test '{description}' failed",); } } fn account_id(id: u64) -> SearchFilter { SearchFilter::Operator { field: SearchField::AccountId, op: SearchOperator::Equal, value: SearchValue::Uint(id), } } fn other_op(value: &str) -> SearchFilter { SearchFilter::Operator { field: SearchField::DocumentId, op: SearchOperator::Equal, value: SearchValue::Text { value: value.to_string(), language: Language::None, }, } } fn doc_set(ids: &[u32]) -> SearchFilter { let mut bitmap = RoaringBitmap::new(); for id in ids { bitmap.insert(*id); } SearchFilter::DocumentSet(bitmap) } fn print_split_filter_code(splits: &[SplitFilter]) { println!("vec!["); for split in splits { match split { SplitFilter::Internal(filter) => { print!(" SplitFilter::Internal("); print_search_filter_code(filter, 0); println!("),"); } SplitFilter::External(filters) => { println!(" SplitFilter::External(vec!["); for filter in filters { print!(" "); print_search_filter_code(filter, 2); println!(","); } println!(" ]),"); } } } println!("]"); } fn print_search_filter_code(filter: &SearchFilter, indent_level: usize) { let indent = " ".repeat(indent_level); match filter { SearchFilter::Operator { field, op, value } => match (field, op, value) { (SearchField::AccountId, SearchOperator::Equal, SearchValue::Uint(id)) => { print!("account_id({})", id); } ( SearchField::DocumentId, SearchOperator::Equal, SearchValue::Text { value, .. }, ) => { print!("other_op(\"{}\")", value); } _ => { println!("SearchFilter::Operator {{"); println!("{} field: {:?},", indent, field); println!("{} op: {:?},", indent, op); println!("{} value: {:?},", indent, value); print!("{}}}", indent); } }, SearchFilter::DocumentSet(bitmap) => { let ids: Vec = bitmap.iter().collect(); if ids.is_empty() { print!("doc_set(&[])"); } else if ids.len() <= 5 { print!("doc_set(&["); for (i, id) in ids.iter().enumerate() { if i > 0 { print!(", "); } print!("{}", id); } print!("])"); } else { // For large bitmaps, create inline println!("{{"); println!("{} let mut bitmap = RoaringBitmap::new();", indent); for id in ids { println!("{} bitmap.insert({});", indent, id); } print!("{} doc_set_bitmap(bitmap)", indent); println!(); print!("{}}}", indent); } } SearchFilter::And => print!("SearchFilter::And"), SearchFilter::Or => print!("SearchFilter::Or"), SearchFilter::Not => print!("SearchFilter::Not"), SearchFilter::End => print!("SearchFilter::End"), } } } ================================================ FILE: crates/store/src/search/term.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ Serialize, backend::MAX_TOKEN_LENGTH, search::*, write::{ Archiver, BatchBuilder, SEARCH_INDEX_MAX_FIELD_LEN, SearchIndexClass, SearchIndexField, SearchIndexId, SearchIndexType, ValueClass, }, }; use ahash::AHashSet; use nlp::{ language::stemmer::Stemmer, tokenizers::{space::SpaceTokenizer, word::WordTokenizer}, }; use utils::{ cheeky_hash::{CheekyBTreeMap, CheekyHash}, map::bitmap::BitPop, }; #[derive(Debug, PartialEq, Eq, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] pub(crate) struct TermIndex { terms: Vec, fields: Vec, } #[derive(Debug, PartialEq, Eq, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] pub(crate) struct Term { hash: CheekyHash, fields: u32, } pub(crate) struct TermIndexBuilder { pub(crate) index: TermIndex, pub(crate) id: SearchIndexId, } impl TermIndexBuilder { pub fn build(document: IndexDocument, truncate_at: usize) -> Self { let mut terms: CheekyBTreeMap = CheekyBTreeMap::new(); let mut fields: Vec = Vec::new(); let mut account_id = None; let mut document_id = None; let mut id = None; for (field, value) in document.fields { match field { SearchField::Id => { if let SearchValue::Uint(v) = value { fields.push(SearchIndexField { field_id: field.u8_id(), data: v.to_be_bytes().to_vec(), }); id = Some(v); } continue; } SearchField::AccountId => { if let SearchValue::Uint(v) = value { account_id = Some(v); } continue; } SearchField::DocumentId => { if let SearchValue::Uint(v) = value { document_id = Some(v); } continue; } _ => {} } let field = match value { SearchValue::Text { value, language } => { if field.is_text() { let value = if truncate_at > 0 && value.len() > truncate_at { let pos = value.floor_char_boundary(truncate_at); &value[..pos] } else { &value }; match language { Language::Unknown => { for token in WordTokenizer::new(value, MAX_TOKEN_LENGTH) { terms .entry(CheekyHash::new(token.word.as_bytes())) .or_default() .bit_push(field.u8_id()); } } Language::None => { for token in SpaceTokenizer::new(value, MAX_TOKEN_LENGTH) { terms .entry(CheekyHash::new(token.as_bytes())) .or_default() .bit_push(field.u8_id()); } } _ => { for token in Stemmer::new(value, language, MAX_TOKEN_LENGTH) { terms .entry(CheekyHash::new(token.word.as_bytes())) .or_default() .bit_push(field.u8_id()); if let Some(stemmed_word) = token.stemmed_word { terms .entry(CheekyHash::new( format!("{}*", stemmed_word).as_bytes(), )) .or_default() .bit_push(field.u8_id()); } } } } } if field.is_indexed() { let mut data = value.into_bytes(); data.truncate(SEARCH_INDEX_MAX_FIELD_LEN); SearchIndexField { field_id: field.u8_id(), data, } } else { continue; } } SearchValue::KeyValues(map) => { for (key, value) in map { terms .entry(CheekyHash::new(key.as_bytes())) .or_default() .bit_push(field.u8_id()); for token in SpaceTokenizer::new(value.as_str(), MAX_TOKEN_LENGTH) { terms .entry(CheekyHash::new(format!("{key} {token}").as_bytes())) .or_default() .bit_push(field.u8_id()); } } continue; } SearchValue::Int(v) => SearchIndexField { field_id: field.u8_id(), data: (v as u64).to_be_bytes().to_vec(), }, SearchValue::Uint(v) => SearchIndexField { field_id: field.u8_id(), data: v.to_be_bytes().to_vec(), }, SearchValue::Boolean(v) => SearchIndexField { field_id: field.u8_id(), data: vec![v as u8], }, }; fields.push(field); } TermIndexBuilder { index: TermIndex { terms: terms .into_iter() .map(|(k, v)| Term { hash: k, fields: v }) .collect(), fields, }, id: match (account_id, document_id, id) { (Some(account_id), Some(document_id), None) => SearchIndexId::Account { account_id: account_id as u32, document_id: document_id as u32, }, (None, None, Some(id)) => SearchIndexId::Global { id }, _ => { debug_assert!( false, "Invalid combination of AccountId {account_id:?}, DocumentId {document_id:?} and Id {id:?} fields" ); SearchIndexId::Global { id: 0 } } }, } } } impl TermIndex { pub fn write_index( self, batch: &mut BatchBuilder, index: SearchIndex, id: SearchIndexId, ) -> trc::Result<()> { let archive = Archiver::new(self); batch .set( ValueClass::SearchIndex(SearchIndexClass { index, id, typ: SearchIndexType::Document, }), archive.serialize()?, ) .commit_point(); for term in archive.inner.terms { let mut fields = term.fields; while let Some(field) = fields.bit_pop() { batch .set( ValueClass::SearchIndex(SearchIndexClass { index, id, typ: SearchIndexType::Term { hash: term.hash, field, }, }), vec![], ) .commit_point(); } } for field in archive.inner.fields { batch .set( ValueClass::SearchIndex(SearchIndexClass { index, id, typ: SearchIndexType::Index { field }, }), vec![], ) .commit_point(); } Ok(()) } pub fn merge_index( self, batch: &mut BatchBuilder, index: SearchIndex, id: SearchIndexId, old_term: &ArchivedTermIndex, ) -> trc::Result<()> { let archive = Archiver::new(self); batch .set( ValueClass::SearchIndex(SearchIndexClass { index, id, typ: SearchIndexType::Document, }), archive.serialize()?, ) .commit_point(); let mut old_terms = AHashSet::with_capacity(old_term.terms.len()); let mut old_fields = AHashSet::with_capacity(old_term.fields.len()); for term in old_term.terms.iter() { let mut fields = term.fields.to_native(); while let Some(field) = fields.bit_pop() { old_terms.insert(SearchIndexType::Term { hash: term.hash.to_native(), field, }); } } for field in old_term.fields.iter() { old_fields.insert(SearchIndexField { field_id: field.field_id, data: field.data.to_vec(), }); } for term in archive.inner.terms { let mut fields = term.fields; while let Some(field) = fields.bit_pop() { let typ = SearchIndexType::Term { hash: term.hash, field, }; if !old_terms.remove(&typ) { batch .set( ValueClass::SearchIndex(SearchIndexClass { index, id, typ }), vec![], ) .commit_point(); } } } for field in archive.inner.fields { if !old_fields.remove(&field) { batch .set( ValueClass::SearchIndex(SearchIndexClass { index, id, typ: SearchIndexType::Index { field }, }), vec![], ) .commit_point(); } } for typ in old_terms { batch .clear(ValueClass::SearchIndex(SearchIndexClass { index, id, typ })) .commit_point(); } for field in old_fields { batch .clear(ValueClass::SearchIndex(SearchIndexClass { index, id, typ: SearchIndexType::Index { field }, })) .commit_point(); } Ok(()) } } impl ArchivedTermIndex { pub fn delete_index(&self, batch: &mut BatchBuilder, index: SearchIndex, id: SearchIndexId) { batch .clear(ValueClass::SearchIndex(SearchIndexClass { index, id, typ: SearchIndexType::Document, })) .commit_point(); for term in self.terms.iter() { let mut fields = term.fields.to_native(); while let Some(field) = fields.bit_pop() { batch .clear(ValueClass::SearchIndex(SearchIndexClass { index, id, typ: SearchIndexType::Term { hash: term.hash.to_native(), field, }, })) .commit_point(); } } for field in self.fields.iter() { batch .clear(ValueClass::SearchIndex(SearchIndexClass { index, id, typ: SearchIndexType::Index { field: SearchIndexField { field_id: field.field_id, data: field.data.to_vec(), }, }, })) .commit_point(); } } } impl SearchIndex { pub(crate) fn as_u8(&self) -> u8 { match self { SearchIndex::Email => 0, SearchIndex::Calendar => 1, SearchIndex::Contacts => 2, SearchIndex::File => 3, SearchIndex::Tracing => 4, SearchIndex::InMemory => unreachable!(), } } } impl SearchField { pub(crate) fn u8_id(&self) -> u8 { match self { SearchField::AccountId => 0, SearchField::DocumentId => 1, SearchField::Id => 2, SearchField::Email(field) => match field { EmailSearchField::From => 3, EmailSearchField::To => 4, EmailSearchField::Cc => 5, EmailSearchField::Bcc => 6, EmailSearchField::Subject => 7, EmailSearchField::Body => 8, EmailSearchField::Attachment => 9, EmailSearchField::ReceivedAt => 10, EmailSearchField::SentAt => 11, EmailSearchField::Size => 12, EmailSearchField::HasAttachment => 13, EmailSearchField::Headers => 14, }, SearchField::Calendar(field) => match field { CalendarSearchField::Title => 3, CalendarSearchField::Description => 4, CalendarSearchField::Location => 5, CalendarSearchField::Owner => 6, CalendarSearchField::Attendee => 7, CalendarSearchField::Start => 8, CalendarSearchField::Uid => 9, }, SearchField::Contact(field) => match field { ContactSearchField::Member => 3, ContactSearchField::Kind => 4, ContactSearchField::Name => 5, ContactSearchField::Nickname => 6, ContactSearchField::Organization => 7, ContactSearchField::Email => 8, ContactSearchField::Phone => 9, ContactSearchField::OnlineService => 10, ContactSearchField::Address => 11, ContactSearchField::Note => 12, ContactSearchField::Uid => 13, }, SearchField::File(field) => match field { FileSearchField::Name => 3, FileSearchField::Content => 4, }, SearchField::Tracing(field) => match field { TracingSearchField::EventType => 3, TracingSearchField::QueueId => 4, TracingSearchField::Keywords => 5, }, } } } ================================================ FILE: crates/store/src/write/assert.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{Archive, ArchiveVersion}; use crate::{U32_LEN, U64_LEN}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum AssertValue { U32(u32), U64(u64), Archive(ArchiveVersion), Some, None, } pub trait ToAssertValue { fn to_assert_value(&self) -> AssertValue; } impl ToAssertValue for AssertValue { fn to_assert_value(&self) -> AssertValue { *self } } impl ToAssertValue for () { fn to_assert_value(&self) -> AssertValue { AssertValue::None } } impl ToAssertValue for u64 { fn to_assert_value(&self) -> AssertValue { AssertValue::U64(*self) } } impl ToAssertValue for u32 { fn to_assert_value(&self) -> AssertValue { AssertValue::U32(*self) } } impl ToAssertValue for Archive { fn to_assert_value(&self) -> AssertValue { AssertValue::Archive(self.version) } } impl ToAssertValue for &Archive { fn to_assert_value(&self) -> AssertValue { AssertValue::Archive(self.version) } } impl AssertValue { pub fn matches(&self, bytes: &[u8]) -> bool { match self { AssertValue::U32(v) => bytes .get(bytes.len() - U32_LEN..) .is_some_and(|b| b == v.to_be_bytes()), AssertValue::U64(v) => bytes .get(bytes.len() - U64_LEN..) .is_some_and(|b| b == v.to_be_bytes()), AssertValue::Archive(v) => match v { ArchiveVersion::Versioned { hash, .. } => bytes .get(bytes.len() - U32_LEN - U64_LEN - 1..bytes.len() - U64_LEN - 1) .is_some_and(|b| b == hash.to_be_bytes()), ArchiveVersion::Hashed { hash } => bytes .get(bytes.len() - U32_LEN - 1..bytes.len() - 1) .is_some_and(|b| b == hash.to_be_bytes()), ArchiveVersion::Unversioned => false, }, AssertValue::None => false, AssertValue::Some => true, } } pub fn is_none(&self) -> bool { matches!(self, AssertValue::None) } } ================================================ FILE: crates/store/src/write/batch.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ Batch, BatchBuilder, ChangedCollection, IntoOperations, Operation, ValueClass, ValueOp, assert::ToAssertValue, log::VanishedItem, }; use crate::{ SerializeInfallible, U32_LEN, write::{LogCollection, MergeFnc, MergeOperation, Params, SetFnc, SetOperation}, }; use types::{ collection::{Collection, SyncCollection, VanishedCollection}, field::FieldType, }; use utils::map::vec_map::VecMap; impl BatchBuilder { pub fn new() -> Self { Self { ops: Vec::with_capacity(32), current_account_id: None, current_collection: None, current_document_id: None, changes: Default::default(), changed_collections: Default::default(), batch_size: 0, batch_ops: 0, has_assertions: false, commit_points: Vec::new(), } } pub fn with_account_id(&mut self, account_id: u32) -> &mut Self { if self .current_account_id .is_none_or(|current_account_id| current_account_id != account_id) { self.current_account_id = account_id.into(); self.ops.push(Operation::AccountId { account_id }); } self } pub fn with_collection(&mut self, collection: Collection) -> &mut Self { let collection_ = Some(collection); if collection_ != self.current_collection { self.current_collection = collection_; self.ops.push(Operation::Collection { collection }); } self } pub fn with_document(&mut self, document_id: u32) -> &mut Self { self.ops.push(Operation::DocumentId { document_id }); self.current_document_id = Some(document_id); self.has_assertions = false; self } pub fn assert_value( &mut self, class: impl Into, value: impl ToAssertValue, ) -> &mut Self { self.ops.push(Operation::AssertValue { class: class.into(), assert_value: value.to_assert_value(), }); self.batch_ops += 1; self.has_assertions = true; self } pub fn index(&mut self, field: impl FieldType, value: impl Into>) -> &mut Self { let field = field.into(); let value = value.into(); let value_len = value.len(); self.ops.push(Operation::Index { field, key: value, set: true, }); self.batch_size += (U32_LEN * 3) + value_len; self.batch_ops += 1; self } pub fn unindex(&mut self, field: impl FieldType, value: impl Into>) -> &mut Self { let field = field.into(); let value = value.into(); let value_len = value.len(); self.ops.push(Operation::Index { field, key: value, set: false, }); self.batch_size += (U32_LEN * 3) + value_len; self.batch_ops += 1; self } #[inline(always)] pub fn tag(&mut self, field: impl FieldType) -> &mut Self { self.index(field, vec![]) } #[inline(always)] pub fn untag(&mut self, field: impl FieldType) -> &mut Self { self.unindex(field, vec![]) } pub fn add(&mut self, class: impl Into, value: i64) -> &mut Self { let class = class.into(); self.batch_size += class.serialized_size() + std::mem::size_of::(); self.ops.push(Operation::Value { class, op: ValueOp::AtomicAdd(value), }); self.batch_ops += 1; self } pub fn add_and_get(&mut self, class: impl Into, value: i64) -> &mut Self { let class = class.into(); self.batch_size += class.serialized_size() + (std::mem::size_of::() * 2); self.ops.push(Operation::Value { class, op: ValueOp::AddAndGet(value), }); self.batch_ops += 1; self } pub fn set(&mut self, class: impl Into, value: impl Into>) -> &mut Self { let class = class.into(); let value = value.into(); self.batch_size += class.serialized_size() + value.len(); self.ops.push(Operation::Value { class, op: ValueOp::Set(value), }); self.batch_ops += 1; self } pub fn set_fnc( &mut self, class: impl Into, params: Params, fnc: SetFnc, ) -> &mut Self { self.ops.push(Operation::Value { class: class.into(), op: ValueOp::SetFnc(SetOperation { fnc, params }), }); self } pub fn merge_fnc( &mut self, class: impl Into, params: Params, fnc: MergeFnc, ) -> &mut Self { self.ops.push(Operation::Value { class: class.into(), op: ValueOp::MergeFnc(MergeOperation { fnc, params }), }); self } pub fn clear(&mut self, class: impl Into) -> &mut Self { let class = class.into(); self.batch_size += class.serialized_size(); self.ops.push(Operation::Value { class, op: ValueOp::Clear, }); self.batch_ops += 1; self } pub fn acl_grant(&mut self, grant_account_id: u32, op: Vec) -> &mut Self { self.batch_size += (U32_LEN * 3) + op.len(); self.ops.push(Operation::Value { class: ValueClass::Acl(grant_account_id), op: ValueOp::Set(op), }); self.batch_ops += 1; self } pub fn acl_revoke(&mut self, grant_account_id: u32) -> &mut Self { self.batch_size += U32_LEN * 3; self.ops.push(Operation::Value { class: ValueClass::Acl(grant_account_id), op: ValueOp::Clear, }); self.batch_ops += 1; self } pub fn log_item_insert( &mut self, collection: SyncCollection, prefix: Option, ) -> &mut Self { if let (Some(account_id), Some(document_id)) = (self.current_account_id, self.current_document_id) { self.changes.get_mut_or_insert(account_id).log_item_insert( collection, prefix, document_id, ); } self } pub fn log_item_update( &mut self, collection: SyncCollection, prefix: Option, ) -> &mut Self { if let (Some(account_id), Some(document_id)) = (self.current_account_id, self.current_document_id) { self.changes.get_mut_or_insert(account_id).log_item_update( collection, prefix, document_id, ); } self } pub fn log_item_delete( &mut self, collection: SyncCollection, prefix: Option, ) -> &mut Self { if let (Some(account_id), Some(document_id)) = (self.current_account_id, self.current_document_id) { self.changes.get_mut_or_insert(account_id).log_item_delete( collection, prefix, document_id, ); } self } pub fn log_container_insert(&mut self, collection: SyncCollection) -> &mut Self { if let (Some(account_id), Some(document_id)) = (self.current_account_id, self.current_document_id) { self.changes .get_mut_or_insert(account_id) .log_container_insert(collection, document_id); } self } pub fn log_container_update(&mut self, collection: SyncCollection) -> &mut Self { if let (Some(account_id), Some(document_id)) = (self.current_account_id, self.current_document_id) { self.changes .get_mut_or_insert(account_id) .log_container_update(collection, document_id); } self } pub fn log_container_delete(&mut self, collection: SyncCollection) -> &mut Self { if let (Some(account_id), Some(document_id)) = (self.current_account_id, self.current_document_id) { self.changes .get_mut_or_insert(account_id) .log_container_delete(collection, document_id); } self } pub fn log_container_property_change( &mut self, collection: SyncCollection, document_id: u32, ) -> &mut Self { if let Some(account_id) = self.current_account_id { self.changes .get_mut_or_insert(account_id) .log_container_property_update(collection, document_id); } self } pub fn log_vanished_item( &mut self, collection: VanishedCollection, item: impl Into, ) -> &mut Self { if let Some(account_id) = self.current_account_id { let item = item.into(); self.batch_size += item.serialized_size(); self.changes .get_mut_or_insert(account_id) .log_vanished_item(collection, item); } self } pub fn log_share_notification( &mut self, notification_id: u64, notify_account_id: u32, value: impl SerializeInfallible, ) -> &mut Self { self.changed_collections .get_mut_or_insert(notify_account_id) .share_notification_id = Some(notification_id); self.set( ValueClass::ShareNotification { notification_id, notify_account_id, }, value.serialize(), ) } fn serialize_changes(&mut self) { if !self.changes.is_empty() { for (account_id, changelog) in std::mem::take(&mut self.changes) { self.with_account_id(account_id); // Serialize changes for (collection, changes) in changelog.changes.into_iter() { let cc = self.changed_collections.get_mut_or_insert(account_id); if changes.has_container_changes() { cc.changed_containers.insert(collection); } if changes.has_item_changes() { cc.changed_items.insert(collection); } self.ops.push(Operation::Log { collection: LogCollection::Sync(collection), set: changes.serialize(), }); } // Serialize vanished items for (collection, vanished) in changelog.vanished.into_iter() { self.ops.push(Operation::Log { collection: LogCollection::Vanished(collection), set: vanished.serialize(), }); } } } } pub fn commit_point(&mut self) -> &mut Self { if self.is_large_batch() { self.serialize_changes(); self.commit_points.push(self.ops.len()); self.batch_ops = 0; self.batch_size = 0; if let Some(account_id) = self.current_account_id { self.ops.push(Operation::AccountId { account_id }); } if let Some(collection) = self.current_collection { self.ops.push(Operation::Collection { collection }); } } self } #[inline] pub fn is_large_batch(&self) -> bool { self.batch_size > 5_000_000 || self.batch_ops > 1000 } pub fn any_op(&mut self, op: Operation) -> &mut Self { self.ops.push(op); self.batch_ops += 1; self } pub fn custom(&mut self, value: impl IntoOperations) -> trc::Result<&mut Self> { value.build(self)?; Ok(self) } pub fn last_account_id(&self) -> Option { self.current_account_id } pub fn last_collection(&self) -> Option { self.current_collection } pub fn last_document_id(&self) -> Option { self.current_document_id } pub fn commit_points(&mut self) -> CommitPointIterator { self.serialize_changes(); CommitPointIterator { commit_points: std::mem::take(&mut self.commit_points), commit_point_last: self.ops.len(), offset_start: 0, } } pub fn build_one(&mut self, commit_point: CommitPoint) -> Batch<'_> { Batch { changes: &self.changed_collections, ops: &mut self.ops[commit_point.offset_start..commit_point.offset_end], } } pub fn build_all(&mut self) -> Batch<'_> { self.serialize_changes(); Batch { changes: &self.changed_collections, ops: self.ops.as_mut_slice(), } } pub fn changes(self) -> Option> { if self.has_changes() { Some(self.changed_collections) } else { None } } pub fn has_changes(&self) -> bool { !self.changed_collections.is_empty() } pub fn ops(&self) -> &[Operation] { self.ops.as_slice() } pub fn len(&self) -> usize { self.batch_size } pub fn is_empty(&self) -> bool { self.batch_ops == 0 } } pub struct CommitPointIterator { commit_points: Vec, commit_point_last: usize, offset_start: usize, } pub struct CommitPoint { pub offset_start: usize, pub offset_end: usize, } impl CommitPointIterator { pub fn iter(&mut self) -> impl Iterator { self.commit_points .iter() .copied() .chain([self.commit_point_last]) .map(|offset_end| { let point = CommitPoint { offset_start: self.offset_start, offset_end, }; self.offset_start = offset_end; point }) } } impl Batch<'_> { pub fn is_atomic(&self) -> bool { !self.ops.iter().any(|op| { matches!( op, Operation::AssertValue { .. } | Operation::Value { op: ValueOp::AddAndGet(_), .. } ) }) } pub fn first_account_id(&self) -> Option { self.ops.iter().find_map(|op| match op { Operation::AccountId { account_id } => Some(*account_id), _ => None, }) } } impl Default for BatchBuilder { fn default() -> Self { Self::new() } } ================================================ FILE: crates/store/src/write/bitpack.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use bitpacking::{BitPacker, BitPacker1x, BitPacker4x, BitPacker8x}; use utils::codec::leb128::Leb128Reader; use super::key::KeySerializer; #[derive(Default)] pub struct BitpackIterator<'x> { pub(crate) bytes: &'x [u8], pub(crate) bytes_offset: usize, pub(crate) chunk: Vec, pub(crate) chunk_offset: usize, pub items_left: u32, } #[derive(Clone, Copy)] pub(crate) struct BitBlockPacker { bitpacker_1: BitPacker1x, bitpacker_4: BitPacker4x, bitpacker_8: BitPacker8x, block_len: usize, } impl KeySerializer { pub fn bitpack_sorted(self, items: &[u32]) -> Self { let mut serializer = self; let mut bitpacker = BitBlockPacker::new(); let mut compressed = vec![0u8; 4 * BitPacker8x::BLOCK_LEN]; let mut pos = 0; let len = items.len(); let mut initial_value = None; serializer = serializer.write_leb128(len as u32); while pos < len { let block_len = match len - pos { 0..=31 => { for val in &items[pos..] { serializer = serializer.write_leb128(*val); } break; } 32..=127 => BitPacker1x::BLOCK_LEN, 128..=255 => BitPacker4x::BLOCK_LEN, _ => BitPacker8x::BLOCK_LEN, }; let chunk = &items[pos..pos + block_len]; bitpacker.block_len(block_len); let num_bits: u8 = bitpacker.num_bits_strictly_sorted(initial_value, chunk); let compressed_len = bitpacker.compress_strictly_sorted( initial_value, chunk, &mut compressed[..], num_bits, ); serializer = serializer .write(num_bits) .write(&compressed[..compressed_len]); initial_value = chunk[chunk.len() - 1].into(); pos += block_len; } serializer } } impl<'x> BitpackIterator<'x> { pub fn from_bytes_and_offset(bytes: &'x [u8], bytes_offset: usize, items_left: u32) -> Self { BitpackIterator { bytes, bytes_offset, items_left, ..Default::default() } } pub fn new(bytes: &'x [u8]) -> Option { bytes .read_leb128::() .map(|(items_left, bytes_offset)| BitpackIterator { bytes, bytes_offset, items_left, ..Default::default() }) } } impl Iterator for BitpackIterator<'_> { type Item = u32; fn next(&mut self) -> Option { if let Some(item) = self.chunk.get(self.chunk_offset) { self.chunk_offset += 1; return Some(*item); } let block_len = match self.items_left { 0 => return None, 1..=31 => { self.items_left -= 1; let (item, bytes_read) = self.bytes.get(self.bytes_offset..)?.read_leb128()?; self.bytes_offset += bytes_read; return Some(item); } 32..=127 => BitPacker1x::BLOCK_LEN, 128..=255 => BitPacker4x::BLOCK_LEN, _ => BitPacker8x::BLOCK_LEN, }; let bitpacker = BitBlockPacker::with_block_len(block_len); let num_bits = *self.bytes.get(self.bytes_offset)?; let bytes_read = ((num_bits as usize) * block_len / 8) + 1; let initial_value = self.chunk.last().copied(); self.chunk = vec![0u32; block_len]; self.chunk_offset = 1; bitpacker.decompress_strictly_sorted( initial_value, self.bytes .get(self.bytes_offset + 1..self.bytes_offset + bytes_read)?, &mut self.chunk[..], num_bits, ); self.bytes_offset += bytes_read; self.items_left -= block_len as u32; self.chunk.first().copied() } } impl BitBlockPacker { pub fn with_block_len(block_len: usize) -> Self { BitBlockPacker { bitpacker_1: BitPacker1x::new(), bitpacker_4: BitPacker4x::new(), bitpacker_8: BitPacker8x::new(), block_len, } } pub fn block_len(&mut self, num: usize) { self.block_len = num; } } impl BitPacker for BitBlockPacker { const BLOCK_LEN: usize = 0; fn new() -> Self { BitBlockPacker { bitpacker_1: BitPacker1x::new(), bitpacker_4: BitPacker4x::new(), bitpacker_8: BitPacker8x::new(), block_len: 1, } } fn compress(&self, decompressed: &[u32], compressed: &mut [u8], num_bits: u8) -> usize { match self.block_len { BitPacker8x::BLOCK_LEN => self .bitpacker_8 .compress(decompressed, compressed, num_bits), BitPacker4x::BLOCK_LEN => self .bitpacker_4 .compress(decompressed, compressed, num_bits), _ => self .bitpacker_1 .compress(decompressed, compressed, num_bits), } } fn compress_sorted( &self, initial: u32, decompressed: &[u32], compressed: &mut [u8], num_bits: u8, ) -> usize { match self.block_len { BitPacker8x::BLOCK_LEN => { self.bitpacker_8 .compress_sorted(initial, decompressed, compressed, num_bits) } BitPacker4x::BLOCK_LEN => { self.bitpacker_4 .compress_sorted(initial, decompressed, compressed, num_bits) } _ => self .bitpacker_1 .compress_sorted(initial, decompressed, compressed, num_bits), } } fn decompress(&self, compressed: &[u8], decompressed: &mut [u32], num_bits: u8) -> usize { match self.block_len { BitPacker8x::BLOCK_LEN => { self.bitpacker_8 .decompress(compressed, decompressed, num_bits) } BitPacker4x::BLOCK_LEN => { self.bitpacker_4 .decompress(compressed, decompressed, num_bits) } _ => self .bitpacker_1 .decompress(compressed, decompressed, num_bits), } } fn decompress_sorted( &self, initial: u32, compressed: &[u8], decompressed: &mut [u32], num_bits: u8, ) -> usize { match self.block_len { BitPacker8x::BLOCK_LEN => { self.bitpacker_8 .decompress_sorted(initial, compressed, decompressed, num_bits) } BitPacker4x::BLOCK_LEN => { self.bitpacker_4 .decompress_sorted(initial, compressed, decompressed, num_bits) } _ => self .bitpacker_1 .decompress_sorted(initial, compressed, decompressed, num_bits), } } fn num_bits(&self, decompressed: &[u32]) -> u8 { match self.block_len { BitPacker8x::BLOCK_LEN => self.bitpacker_8.num_bits(decompressed), BitPacker4x::BLOCK_LEN => self.bitpacker_4.num_bits(decompressed), _ => self.bitpacker_1.num_bits(decompressed), } } fn num_bits_sorted(&self, initial: u32, decompressed: &[u32]) -> u8 { match self.block_len { BitPacker8x::BLOCK_LEN => self.bitpacker_8.num_bits_sorted(initial, decompressed), BitPacker4x::BLOCK_LEN => self.bitpacker_4.num_bits_sorted(initial, decompressed), _ => self.bitpacker_1.num_bits_sorted(initial, decompressed), } } fn compress_strictly_sorted( &self, initial: Option, decompressed: &[u32], compressed: &mut [u8], num_bits: u8, ) -> usize { match self.block_len { BitPacker8x::BLOCK_LEN => self.bitpacker_8.compress_strictly_sorted( initial, decompressed, compressed, num_bits, ), BitPacker4x::BLOCK_LEN => self.bitpacker_4.compress_strictly_sorted( initial, decompressed, compressed, num_bits, ), _ => self.bitpacker_1.compress_strictly_sorted( initial, decompressed, compressed, num_bits, ), } } fn decompress_strictly_sorted( &self, initial: Option, compressed: &[u8], decompressed: &mut [u32], num_bits: u8, ) -> usize { match self.block_len { BitPacker8x::BLOCK_LEN => self.bitpacker_8.decompress_strictly_sorted( initial, compressed, decompressed, num_bits, ), BitPacker4x::BLOCK_LEN => self.bitpacker_4.decompress_strictly_sorted( initial, compressed, decompressed, num_bits, ), _ => self.bitpacker_1.decompress_strictly_sorted( initial, compressed, decompressed, num_bits, ), } } fn num_bits_strictly_sorted(&self, initial: Option, decompressed: &[u32]) -> u8 { match self.block_len { BitPacker8x::BLOCK_LEN => self .bitpacker_8 .num_bits_strictly_sorted(initial, decompressed), BitPacker4x::BLOCK_LEN => self .bitpacker_4 .num_bits_strictly_sorted(initial, decompressed), _ => self .bitpacker_1 .num_bits_strictly_sorted(initial, decompressed), } } } #[cfg(test)] mod tests { use super::*; #[test] fn bitpack_roundtrip() { for num_positions in [ 1, 10, BitPacker1x::BLOCK_LEN, BitPacker4x::BLOCK_LEN, BitPacker8x::BLOCK_LEN, BitPacker8x::BLOCK_LEN + BitPacker4x::BLOCK_LEN + BitPacker1x::BLOCK_LEN, BitPacker8x::BLOCK_LEN + BitPacker4x::BLOCK_LEN + BitPacker1x::BLOCK_LEN + 1, (BitPacker8x::BLOCK_LEN * 3) + (BitPacker4x::BLOCK_LEN * 3) + (BitPacker1x::BLOCK_LEN * 3) + 1, (BitPacker8x::BLOCK_LEN * 32) + 1, ] { let serialized = KeySerializer::new(num_positions * std::mem::size_of::()) .bitpack_sorted( &(0..num_positions) .map(|i| (i * i) as u32) .collect::>(), ) .finalize(); println!( "Testing block {num_positions} with {} size...", serialized.len() ); let mut iter = BitpackIterator::new(&serialized).unwrap(); assert_eq!( iter.items_left, num_positions as u32, "failed for num_positions: {}", num_positions ); for i in 0..num_positions { assert_eq!( iter.next(), Some((i * i) as u32), "failed for position: {}", i ); } assert_eq!(iter.next(), None, "expected end of iterator"); } } } ================================================ FILE: crates/store/src/write/blob.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Instant; use super::{BlobOp, Operation, ValueClass, ValueOp, key::DeserializeBigEndian, now}; use crate::{ BlobStore, IterateParams, Store, U32_LEN, U64_LEN, ValueKey, write::{BatchBuilder, BlobLink}, }; use trc::{AddContext, PurgeEvent}; use types::{ blob::BlobClass, blob_hash::{BLOB_HASH_LEN, BlobHash}, }; #[derive(Debug, PartialEq, Eq)] pub struct BlobQuota { pub bytes: usize, pub count: usize, } impl Store { pub async fn blob_exists(&self, hash: impl AsRef + Sync + Send) -> trc::Result { self.get_value::<()>(ValueKey { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Blob(BlobOp::Commit { hash: hash.as_ref().clone(), }), }) .await .map(|v| v.is_some()) .caused_by(trc::location!()) } pub async fn blob_quota(&self, account_id: u32) -> trc::Result { let from_key = ValueKey { account_id, collection: 0, document_id: 0, class: ValueClass::Blob(BlobOp::Quota { hash: BlobHash::default(), until: 0, }), }; let to_key = ValueKey { account_id: account_id + 1, collection: 0, document_id: 0, class: ValueClass::Blob(BlobOp::Quota { hash: BlobHash::default(), until: u64::MAX, }), }; let now = now(); let mut quota = BlobQuota { bytes: 0, count: 0 }; self.iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { let until = key.deserialize_be_u64(key.len() - U64_LEN)?; if until > now { let bytes = value.deserialize_be_u32(0)?; if bytes > 0 { quota.bytes += bytes as usize; quota.count += 1; } } Ok(true) }, ) .await .caused_by(trc::location!())?; Ok(quota) } pub async fn blob_has_access( &self, hash: impl AsRef + Sync + Send, class: impl AsRef + Sync + Send, ) -> trc::Result { let key = match class.as_ref() { BlobClass::Reserved { account_id, expires, } if *expires > now() => ValueKey { account_id: *account_id, collection: 0, document_id: 0, class: ValueClass::Blob(BlobOp::Link { hash: hash.as_ref().clone(), to: BlobLink::Temporary { until: *expires }, }), }, BlobClass::Linked { account_id, collection, document_id, } => ValueKey { account_id: *account_id, collection: *collection, document_id: *document_id, class: ValueClass::Blob(BlobOp::Link { hash: hash.as_ref().clone(), to: BlobLink::Document, }), }, _ => return Ok(false), }; self.get_value::<()>(key).await.map(|v| v.is_some()) } pub async fn purge_blobs(&self, blob_store: BlobStore) -> trc::Result<()> { let mut total_active = 0; let mut total_deleted = 0; let started = Instant::now(); for byte in 0..=u8::MAX { // Validate linked blobs let mut from_hash = BlobHash::default(); let mut to_hash = BlobHash::new_max(); from_hash.0[0] = byte; to_hash.0[0] = byte; let from_key = ValueKey { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Blob(BlobOp::Commit { hash: from_hash }), }; let to_key = ValueKey { account_id: u32::MAX, collection: u8::MAX, document_id: u32::MAX, class: ValueClass::Blob(BlobOp::Link { hash: to_hash, to: BlobLink::Document, }), }; let mut state = BlobPurgeState::new(); self.iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { let hash = BlobHash::try_from_hash_slice(key.get(0..BLOB_HASH_LEN).ok_or_else( || trc::Error::corrupted_key(key, value.into(), trc::location!()), )?) .unwrap(); state.update_hash(hash); state.process_key(key, value)?; Ok(true) }, ) .await .caused_by(trc::location!())?; state.finalize(BlobHash::default()); // Delete expired or unlinked blobs for (_, op) in &state.delete_keys { if let BlobOp::Commit { hash } = op { blob_store .delete_blob(hash.as_ref()) .await .caused_by(trc::location!())?; } } // Delete hashes let mut batch = BatchBuilder::new(); for (account_id, op) in state.delete_keys { if batch.is_large_batch() { self.write(batch.build_all()) .await .caused_by(trc::location!())?; batch = BatchBuilder::new(); } if let Some(account_id) = account_id { batch.with_account_id(account_id); } batch.any_op(Operation::Value { class: ValueClass::Blob(op), op: ValueOp::Clear, }); } if !batch.is_empty() { self.write(batch.build_all()) .await .caused_by(trc::location!())?; } total_active += state.total_active - 1; // Exclude default hash total_deleted += state.total_deleted; } trc::event!( Purge(PurgeEvent::BlobCleanup), Expires = total_deleted, Total = total_active, Elapsed = started.elapsed() ); Ok(()) } } struct BlobPurgeState { last_hash: BlobHash, last_hash_is_linked: bool, delete_keys: Vec<(Option, BlobOp)>, spam_train_samples: Vec<(u32, u64)>, now: u64, total_deleted: u64, total_active: u64, } impl BlobPurgeState { fn new() -> Self { Self { last_hash: BlobHash::default(), last_hash_is_linked: true, // Avoid deleting non-existing last_hash on first iteration delete_keys: Vec::new(), spam_train_samples: Vec::new(), now: now(), total_deleted: 0, total_active: 0, } } pub fn update_hash(&mut self, hash: BlobHash) { if self.last_hash != hash { self.finalize(hash); self.last_hash_is_linked = false; } } pub fn finalize(&mut self, new_hash: BlobHash) { if !self.last_hash_is_linked { self.total_deleted += 1; self.delete_keys.push(( None, BlobOp::Commit { hash: std::mem::replace(&mut self.last_hash, new_hash), }, )); } else { self.total_active += 1; if !self.spam_train_samples.is_empty() { if self.spam_train_samples.len() > 1 { // Sort by account_id ascending, then until descending self.spam_train_samples .sort_unstable_by(|(a_id, a_until), (b_id, b_until)| { a_id.cmp(b_id).then_with(|| b_until.cmp(a_until)) }); let mut samples = self.spam_train_samples.iter().peekable(); while let Some((account_id, _)) = samples.next() { // Keep only the latest sample per account while let Some((next_account_id, next_until)) = samples.peek() { if next_account_id == account_id { self.delete_keys.push(( Some(*account_id), BlobOp::SpamSample { hash: self.last_hash.clone(), until: *next_until, }, )); self.delete_keys.push(( Some(*account_id), BlobOp::Link { hash: self.last_hash.clone(), to: BlobLink::Temporary { until: *next_until }, }, )); samples.next(); } else { break; } } } } self.spam_train_samples.clear(); } self.last_hash = new_hash; } } pub fn process_key(&mut self, key: &[u8], value: &[u8]) -> trc::Result<()> { const TEMP_LINK: usize = BLOB_HASH_LEN + U32_LEN + U64_LEN; const DOC_LINK: usize = BLOB_HASH_LEN + U64_LEN + 1; const ID_LINK: usize = BLOB_HASH_LEN + U64_LEN; match key.len() { BLOB_HASH_LEN => { // Main blob entry Ok(()) } TEMP_LINK => { // Temporary link let until = key.deserialize_be_u64(BLOB_HASH_LEN + U32_LEN)?; if until <= self.now { let account_id = key.deserialize_be_u32(BLOB_HASH_LEN)?; self.delete_keys.push(( Some(account_id), BlobOp::Link { hash: self.last_hash.clone(), to: BlobLink::Temporary { until }, }, )); match value.first().copied() { Some(BlobLink::QUOTA_LINK) => { self.delete_keys.push(( Some(account_id), BlobOp::Quota { hash: self.last_hash.clone(), until, }, )); } Some(BlobLink::UNDELETE_LINK) => { self.delete_keys.push(( Some(account_id), BlobOp::Undelete { hash: self.last_hash.clone(), until, }, )); } Some(BlobLink::SPAM_SAMPLE_LINK) => { self.delete_keys.push(( Some(account_id), BlobOp::SpamSample { hash: self.last_hash.clone(), until, }, )); } _ => {} } } else { // Delete attempts to train the same message multiple times if matches!(value.first(), Some(&BlobLink::SPAM_SAMPLE_LINK)) { let account_id = key.deserialize_be_u32(BLOB_HASH_LEN)?; self.spam_train_samples.push((account_id, until)); } self.last_hash_is_linked = true; } Ok(()) } DOC_LINK | ID_LINK => { // Document/Id link self.last_hash_is_linked = true; Ok(()) } _ => Err(trc::Error::corrupted_key( key, value.into(), trc::location!(), )), } } } ================================================ FILE: crates/store/src/write/key.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ AnyKey, BlobOp, DirectoryClass, InMemoryClass, QueueClass, ReportClass, ReportEvent, TaskQueueClass, TelemetryClass, ValueClass, }; use crate::{ Deserialize, IndexKey, IndexKeyPrefix, Key, LogKey, SUBSPACE_ACL, SUBSPACE_BLOB_EXTRA, SUBSPACE_BLOB_LINK, SUBSPACE_COUNTER, SUBSPACE_DIRECTORY, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_PROPERTY, SUBSPACE_QUEUE_EVENT, SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUOTA, SUBSPACE_REPORT_IN, SUBSPACE_REPORT_OUT, SUBSPACE_SEARCH_INDEX, SUBSPACE_SETTINGS, SUBSPACE_TASK_QUEUE, SUBSPACE_TELEMETRY_METRIC, SUBSPACE_TELEMETRY_SPAN, U16_LEN, U32_LEN, U64_LEN, ValueKey, WITH_SUBSPACE, write::{BlobLink, IndexPropertyClass, SearchIndex, SearchIndexId, SearchIndexType}, }; use std::convert::TryInto; use types::{blob_hash::BLOB_HASH_LEN, collection::SyncCollection, field::Field}; use utils::codec::leb128::Leb128_; pub struct KeySerializer { pub buf: Vec, } pub trait KeySerialize { fn serialize(&self, buf: &mut Vec); } pub trait DeserializeBigEndian { fn deserialize_be_u16(&self, index: usize) -> trc::Result; fn deserialize_be_u32(&self, index: usize) -> trc::Result; fn deserialize_be_u64(&self, index: usize) -> trc::Result; } impl KeySerializer { pub fn new(capacity: usize) -> Self { Self { buf: Vec::with_capacity(capacity), } } pub fn write(mut self, value: T) -> Self { value.serialize(&mut self.buf); self } pub fn write_leb128(mut self, value: T) -> Self { T::to_leb128_bytes(value, &mut self.buf); self } pub fn finalize(self) -> Vec { self.buf } } impl KeySerialize for u8 { fn serialize(&self, buf: &mut Vec) { buf.push(*self); } } impl KeySerialize for &str { fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(self.as_bytes()); } } impl KeySerialize for &String { fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(self.as_bytes()); } } impl KeySerialize for &[u8] { fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(self); } } impl KeySerialize for u32 { fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(&self.to_be_bytes()); } } impl KeySerialize for u16 { fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(&self.to_be_bytes()); } } impl KeySerialize for u64 { fn serialize(&self, buf: &mut Vec) { buf.extend_from_slice(&self.to_be_bytes()); } } impl DeserializeBigEndian for &[u8] { fn deserialize_be_u16(&self, index: usize) -> trc::Result { self.get(index..index + U16_LEN) .ok_or_else(|| { trc::StoreEvent::DataCorruption .caused_by(trc::location!()) .ctx(trc::Key::Value, *self) }) .and_then(|bytes| { bytes.try_into().map_err(|_| { trc::StoreEvent::DataCorruption .caused_by(trc::location!()) .ctx(trc::Key::Value, *self) }) }) .map(u16::from_be_bytes) } fn deserialize_be_u32(&self, index: usize) -> trc::Result { self.get(index..index + U32_LEN) .ok_or_else(|| { trc::StoreEvent::DataCorruption .caused_by(trc::location!()) .ctx(trc::Key::Value, *self) }) .and_then(|bytes| { bytes.try_into().map_err(|_| { trc::StoreEvent::DataCorruption .caused_by(trc::location!()) .ctx(trc::Key::Value, *self) }) }) .map(u32::from_be_bytes) } fn deserialize_be_u64(&self, index: usize) -> trc::Result { self.get(index..index + U64_LEN) .ok_or_else(|| { trc::StoreEvent::DataCorruption .caused_by(trc::location!()) .ctx(trc::Key::Value, *self) }) .and_then(|bytes| { bytes.try_into().map_err(|_| { trc::StoreEvent::DataCorruption .caused_by(trc::location!()) .ctx(trc::Key::Value, *self) }) }) .map(u64::from_be_bytes) } } impl> ValueKey { pub fn with_document_id(self, document_id: u32) -> Self { Self { document_id, ..self } } pub fn is_counter(&self) -> bool { self.class.as_ref().is_counter(self.collection) } } impl ValueKey { pub fn property( account_id: u32, collection: impl Into, document_id: u32, field: impl Into, ) -> ValueKey { ValueKey { account_id, collection: collection.into(), document_id, class: ValueClass::Property(field.into()), } } pub fn archive( account_id: u32, collection: impl Into, document_id: u32, ) -> ValueKey { ValueKey { account_id, collection: collection.into(), document_id, class: ValueClass::Property(Field::ARCHIVE.into()), } } } impl Key for IndexKeyPrefix { fn serialize(&self, flags: u32) -> Vec { { if (flags & WITH_SUBSPACE) != 0 { KeySerializer::new(std::mem::size_of::() + 1) .write(crate::SUBSPACE_INDEXES) } else { KeySerializer::new(std::mem::size_of::()) } } .write(self.account_id) .write(self.collection) .write(self.field) .finalize() } fn subspace(&self) -> u8 { SUBSPACE_INDEXES } } impl IndexKeyPrefix { pub fn len() -> usize { U32_LEN + 2 } } impl Key for LogKey { fn subspace(&self) -> u8 { SUBSPACE_LOGS } fn serialize(&self, flags: u32) -> Vec { { if (flags & WITH_SUBSPACE) != 0 { KeySerializer::new(std::mem::size_of::() + 1).write(crate::SUBSPACE_LOGS) } else { KeySerializer::new(std::mem::size_of::()) } } .write(self.account_id) .write(self.collection) .write(self.change_id) .finalize() } } impl + Sync + Send + Clone> Key for ValueKey { fn subspace(&self) -> u8 { self.class.as_ref().subspace(self.collection) } fn serialize(&self, flags: u32) -> Vec { self.class .as_ref() .serialize(self.account_id, self.collection, self.document_id, flags) } } impl ValueClass { pub fn serialize( &self, account_id: u32, collection: u8, document_id: u32, flags: u32, ) -> Vec { let serializer = if (flags & WITH_SUBSPACE) != 0 { KeySerializer::new(self.serialized_size() + 2).write(self.subspace(collection)) } else { KeySerializer::new(self.serialized_size() + 1) }; match self { ValueClass::Property(property) => serializer .write(account_id) .write(collection) .write(*property) .write(document_id), ValueClass::IndexProperty(property) => match property { IndexPropertyClass::Hash { property, hash } => serializer .write(account_id) .write(collection) .write(*property) .write(hash.as_bytes()) .write(document_id), IndexPropertyClass::Integer { property, value } => serializer .write(account_id) .write(collection) .write(*property) .write(*value) .write(document_id), }, ValueClass::Acl(grant_account_id) => serializer .write(*grant_account_id) .write(account_id) .write(collection) .write(document_id), ValueClass::TaskQueue(task) => match task { TaskQueueClass::UpdateIndex { index, is_insert, due, } => serializer .write(due.inner()) .write(account_id) .write(if *is_insert { 7u8 } else { 8u8 }) .write(document_id) .write(index.to_u8()), TaskQueueClass::SendAlarm { due, event_id, alarm_id, is_email_alert, } => serializer .write(due.inner()) .write(account_id) .write(if *is_email_alert { 3u8 } else { 6u8 }) .write(document_id) .write(*event_id) .write(*alarm_id), TaskQueueClass::SendImip { due, is_payload } => { if !*is_payload { serializer .write(due.inner()) .write(account_id) .write(4u8) .write(document_id) } else { serializer .write(u64::MAX) .write(account_id) .write(5u8) .write(document_id) .write(due.inner()) } } TaskQueueClass::MergeThreads { due } => serializer .write(due.inner()) .write(account_id) .write(9u8) .write(document_id), }, ValueClass::Blob(op) => match op { BlobOp::Commit { hash } => serializer.write::<&[u8]>(hash.as_ref()), BlobOp::Link { hash, to } => match to { BlobLink::Id { id } => serializer.write::<&[u8]>(hash.as_ref()).write(*id), BlobLink::Document => serializer .write::<&[u8]>(hash.as_ref()) .write(account_id) .write(collection) .write(document_id), BlobLink::Temporary { until } => serializer .write::<&[u8]>(hash.as_ref()) .write(account_id) .write(*until), }, BlobOp::Quota { hash, until } => serializer .write(BlobLink::QUOTA_LINK) .write(account_id) .write::<&[u8]>(hash.as_ref()) .write(*until), BlobOp::Undelete { hash, until } => serializer .write(BlobLink::UNDELETE_LINK) .write(account_id) .write::<&[u8]>(hash.as_ref()) .write(*until), BlobOp::SpamSample { hash, until } => serializer .write(BlobLink::SPAM_SAMPLE_LINK) .write(*until) .write(account_id) .write::<&[u8]>(hash.as_ref()), }, ValueClass::Config(key) => serializer.write(key.as_slice()), ValueClass::InMemory(lookup) => match lookup { InMemoryClass::Key(key) => serializer.write(key.as_slice()), InMemoryClass::Counter(key) => serializer.write(key.as_slice()), }, ValueClass::Directory(directory) => match directory { DirectoryClass::NameToId(name) => serializer.write(0u8).write(name.as_slice()), DirectoryClass::EmailToId(email) => serializer.write(1u8).write(email.as_slice()), DirectoryClass::Principal(uid) => serializer.write(2u8).write_leb128(*uid), DirectoryClass::UsedQuota(uid) => serializer.write(4u8).write_leb128(*uid), DirectoryClass::MemberOf { principal_id, member_of, } => serializer.write(5u8).write(*principal_id).write(*member_of), DirectoryClass::Members { principal_id, has_member, } => serializer .write(6u8) .write(*principal_id) .write(*has_member), DirectoryClass::Index { word, principal_id } => serializer .write(7u8) .write(word.as_slice()) .write(*principal_id), }, ValueClass::Queue(queue) => match queue { QueueClass::Message(queue_id) => serializer.write(*queue_id), QueueClass::MessageEvent(event) => serializer .write(event.due) .write(event.queue_id) .write(event.queue_name.as_slice()), QueueClass::DmarcReportHeader(event) => serializer .write(0u8) .write(event.due) .write(event.domain.as_bytes()) .write(event.policy_hash) .write(event.seq_id) .write(0u8), QueueClass::TlsReportHeader(event) => serializer .write(0u8) .write(event.due) .write(event.domain.as_bytes()) .write(event.policy_hash) .write(event.seq_id) .write(1u8), QueueClass::DmarcReportEvent(event) => serializer .write(1u8) .write(event.due) .write(event.domain.as_bytes()) .write(event.policy_hash) .write(event.seq_id), QueueClass::TlsReportEvent(event) => serializer .write(2u8) .write(event.due) .write(event.domain.as_bytes()) .write(event.policy_hash) .write(event.seq_id), QueueClass::QuotaCount(key) => serializer.write(0u8).write(key.as_slice()), QueueClass::QuotaSize(key) => serializer.write(1u8).write(key.as_slice()), }, ValueClass::Report(report) => match report { ReportClass::Tls { id, expires } => { serializer.write(0u8).write(*expires).write(*id) } ReportClass::Dmarc { id, expires } => { serializer.write(1u8).write(*expires).write(*id) } ReportClass::Arf { id, expires } => { serializer.write(2u8).write(*expires).write(*id) } }, ValueClass::Telemetry(telemetry) => match telemetry { TelemetryClass::Span { span_id } => serializer.write(*span_id), TelemetryClass::Metric { timestamp, metric_id, node_id, } => serializer .write(*timestamp) .write_leb128(*metric_id) .write_leb128(*node_id), }, ValueClass::DocumentId => serializer.write(account_id).write(collection), ValueClass::ChangeId => serializer.write(account_id), ValueClass::ShareNotification { notification_id, notify_account_id, } => serializer .write(*notify_account_id) .write(u8::from(SyncCollection::ShareNotification)) .write(*notification_id), ValueClass::SearchIndex(index) => match &index.typ { SearchIndexType::Term { field, hash } => { let class = index.index.as_u8(); match &index.id { SearchIndexId::Account { account_id, document_id, } => serializer .write(class) .write(*account_id) .write(hash.payload()) .write(hash.payload_len()) .write(*field) .write(*document_id), SearchIndexId::Global { id } => serializer .write(class) .write(hash.payload()) .write(hash.payload_len()) .write(*field) .write(*id), } } SearchIndexType::Index { field } => { let class = index.index.as_u8() | 1 << 6; match &index.id { SearchIndexId::Account { account_id, document_id, } => serializer .write(class) .write(*account_id) .write(field.field_id) .write(field.data.as_slice()) .write(*document_id), SearchIndexId::Global { id } => serializer .write(class) .write(field.field_id) .write(field.data.as_slice()) .write(*id), } } SearchIndexType::Document => { let class = index.index.as_u8() | 2 << 6; match &index.id { SearchIndexId::Account { account_id, document_id, } => serializer .write(class) .write(*account_id) .write(*document_id), SearchIndexId::Global { id } => serializer.write(class).write(*id), } } }, ValueClass::Any(any) => serializer.write(any.key.as_slice()), } .finalize() } } impl BlobLink { pub const QUOTA_LINK: u8 = 0; pub const UNDELETE_LINK: u8 = 1; pub const SPAM_SAMPLE_LINK: u8 = 2; } impl + Sync + Send + Clone> Key for IndexKey { fn subspace(&self) -> u8 { SUBSPACE_INDEXES } fn serialize(&self, flags: u32) -> Vec { let key = self.key.as_ref(); { if (flags & WITH_SUBSPACE) != 0 { KeySerializer::new(std::mem::size_of::>() + key.len() + 1) .write(crate::SUBSPACE_INDEXES) } else { KeySerializer::new(std::mem::size_of::>() + key.len()) } } .write(self.account_id) .write(self.collection) .write(self.field) .write(key) .write(self.document_id) .finalize() } } impl + Sync + Send + Clone> Key for AnyKey { fn serialize(&self, flags: u32) -> Vec { let key = self.key.as_ref(); if (flags & WITH_SUBSPACE) != 0 { KeySerializer::new(key.len() + 1).write(self.subspace) } else { KeySerializer::new(key.len()) } .write(key) .finalize() } fn subspace(&self) -> u8 { self.subspace } } impl ValueClass { pub fn serialized_size(&self) -> usize { match self { ValueClass::Property(_) => U32_LEN * 2 + 3, ValueClass::IndexProperty(p) => match p { IndexPropertyClass::Hash { hash, .. } => U32_LEN * 2 + 3 + hash.len(), IndexPropertyClass::Integer { .. } => U32_LEN * 2 + 3 + U64_LEN, }, ValueClass::Acl(_) => U32_LEN * 3 + 2, ValueClass::InMemory(InMemoryClass::Counter(v) | InMemoryClass::Key(v)) | ValueClass::Config(v) => v.len(), ValueClass::Directory(d) => match d { DirectoryClass::NameToId(v) | DirectoryClass::EmailToId(v) => v.len(), DirectoryClass::Principal(_) | DirectoryClass::UsedQuota(_) => U32_LEN, DirectoryClass::Members { .. } | DirectoryClass::MemberOf { .. } => U32_LEN * 2, DirectoryClass::Index { word, .. } => word.len() + U32_LEN, }, ValueClass::Blob(op) => match op { BlobOp::Commit { .. } => BLOB_HASH_LEN, BlobOp::Link { to, .. } => { BLOB_HASH_LEN + match to { BlobLink::Id { .. } => U64_LEN, BlobLink::Document => U32_LEN * 2 + 1, BlobLink::Temporary { .. } => U32_LEN + U64_LEN, } } BlobOp::Quota { .. } | BlobOp::Undelete { .. } => { BLOB_HASH_LEN + U32_LEN + U64_LEN + 1 } BlobOp::SpamSample { .. } => BLOB_HASH_LEN + U32_LEN + 2, }, ValueClass::TaskQueue(e) => match e { TaskQueueClass::UpdateIndex { .. } => (U64_LEN * 2) + 2, TaskQueueClass::SendAlarm { .. } | TaskQueueClass::MergeThreads { .. } => { U64_LEN + (U32_LEN * 3) + 1 } TaskQueueClass::SendImip { is_payload, .. } => { if *is_payload { (U64_LEN * 2) + (U32_LEN * 2) + 1 } else { U64_LEN + (U32_LEN * 2) + 1 } } }, ValueClass::Queue(q) => match q { QueueClass::Message(_) => U64_LEN, QueueClass::MessageEvent(_) => U64_LEN * 3, QueueClass::DmarcReportEvent(event) | QueueClass::TlsReportEvent(event) => { event.domain.len() + U64_LEN * 3 } QueueClass::DmarcReportHeader(event) | QueueClass::TlsReportHeader(event) => { event.domain.len() + (U64_LEN * 3) + 1 } QueueClass::QuotaCount(v) | QueueClass::QuotaSize(v) => v.len(), }, ValueClass::Report(_) => U64_LEN * 2 + 1, ValueClass::Telemetry(telemetry) => match telemetry { TelemetryClass::Span { .. } => U64_LEN + 1, TelemetryClass::Metric { .. } => U64_LEN * 2 + 1, }, ValueClass::DocumentId => U32_LEN + 1, ValueClass::ChangeId => U32_LEN, ValueClass::ShareNotification { .. } => U32_LEN + U64_LEN + 1, ValueClass::SearchIndex(v) => match &v.typ { SearchIndexType::Term { hash, .. } => U64_LEN + hash.len() + 2, SearchIndexType::Index { field, .. } => 1 + field.data.len() + U64_LEN, SearchIndexType::Document => match &v.id { SearchIndexId::Account { .. } => 1 + U32_LEN * 2, SearchIndexId::Global { .. } => 1 + U64_LEN, }, }, ValueClass::Any(v) => v.key.len(), } } pub fn subspace(&self, collection: u8) -> u8 { match self { ValueClass::Property(field) => { if *field == 84 && collection == 1 { SUBSPACE_COUNTER } else { SUBSPACE_PROPERTY } } ValueClass::IndexProperty { .. } => SUBSPACE_PROPERTY, ValueClass::Acl(_) => SUBSPACE_ACL, ValueClass::TaskQueue { .. } => SUBSPACE_TASK_QUEUE, ValueClass::Blob(op) => match op { BlobOp::Commit { .. } | BlobOp::Link { .. } => SUBSPACE_BLOB_LINK, BlobOp::Quota { .. } | BlobOp::Undelete { .. } | BlobOp::SpamSample { .. } => { SUBSPACE_BLOB_EXTRA } }, ValueClass::Config(_) => SUBSPACE_SETTINGS, ValueClass::InMemory(lookup) => match lookup { InMemoryClass::Key(_) => SUBSPACE_IN_MEMORY_VALUE, InMemoryClass::Counter(_) => SUBSPACE_IN_MEMORY_COUNTER, }, ValueClass::Directory(directory) => match directory { DirectoryClass::UsedQuota(_) => SUBSPACE_QUOTA, _ => SUBSPACE_DIRECTORY, }, ValueClass::Queue(queue) => match queue { QueueClass::Message(_) => SUBSPACE_QUEUE_MESSAGE, QueueClass::MessageEvent(_) => SUBSPACE_QUEUE_EVENT, QueueClass::DmarcReportHeader(_) | QueueClass::TlsReportHeader(_) | QueueClass::DmarcReportEvent(_) | QueueClass::TlsReportEvent(_) => SUBSPACE_REPORT_OUT, QueueClass::QuotaCount(_) | QueueClass::QuotaSize(_) => SUBSPACE_QUOTA, }, ValueClass::Report(_) => SUBSPACE_REPORT_IN, ValueClass::Telemetry(telemetry) => match telemetry { TelemetryClass::Span { .. } => SUBSPACE_TELEMETRY_SPAN, TelemetryClass::Metric { .. } => SUBSPACE_TELEMETRY_METRIC, }, ValueClass::DocumentId | ValueClass::ChangeId => SUBSPACE_COUNTER, ValueClass::ShareNotification { .. } => SUBSPACE_LOGS, ValueClass::SearchIndex(_) => SUBSPACE_SEARCH_INDEX, ValueClass::Any(any) => any.subspace, } } pub fn is_counter(&self, collection: u8) -> bool { match self { ValueClass::Directory(DirectoryClass::UsedQuota(_)) | ValueClass::InMemory(InMemoryClass::Counter(_)) | ValueClass::Queue(QueueClass::QuotaCount(_) | QueueClass::QuotaSize(_)) | ValueClass::DocumentId | ValueClass::ChangeId => true, ValueClass::Property(84) if collection == 1 => true, // TODO: Find a more elegant way to do this _ => false, } } } impl From for ValueKey { fn from(class: ValueClass) -> Self { ValueKey { account_id: 0, collection: 0, document_id: 0, class, } } } impl From for ValueKey { fn from(value: DirectoryClass) -> Self { ValueKey { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Directory(value), } } } impl From for ValueClass { fn from(value: DirectoryClass) -> Self { ValueClass::Directory(value) } } impl From for ValueClass { fn from(value: BlobOp) -> Self { ValueClass::Blob(value) } } impl Deserialize for ReportEvent { fn deserialize(key: &[u8]) -> trc::Result { Ok(ReportEvent { due: key.deserialize_be_u64(1)?, policy_hash: key.deserialize_be_u64(key.len() - (U64_LEN * 2 + 1))?, seq_id: key.deserialize_be_u64(key.len() - (U64_LEN + 1))?, domain: key .get(U64_LEN + 1..key.len() - (U64_LEN * 2 + 1)) .and_then(|domain| std::str::from_utf8(domain).ok()) .map(|s| s.to_string()) .ok_or_else(|| { trc::StoreEvent::DataCorruption .caused_by(trc::location!()) .ctx(trc::Key::Key, key) })?, }) } } impl SearchIndex { pub fn to_u8(&self) -> u8 { match self { SearchIndex::Email => 0, SearchIndex::Calendar => 1, SearchIndex::Contacts => 2, SearchIndex::File => 3, SearchIndex::Tracing => 4, SearchIndex::InMemory => unreachable!(), } } pub fn try_from_u8(value: u8) -> Option { match value { 0 => Some(SearchIndex::Email), 1 => Some(SearchIndex::Calendar), 2 => Some(SearchIndex::Contacts), 3 => Some(SearchIndex::File), 4 => Some(SearchIndex::Tracing), _ => None, } } pub fn name(&self) -> &'static str { match self { SearchIndex::Email => "email", SearchIndex::Calendar => "calendar", SearchIndex::Contacts => "contacts", SearchIndex::File => "file", SearchIndex::Tracing => "tracing", SearchIndex::InMemory => "in_memory", } } pub fn try_from_str(value: &str) -> Option { match value { "email" => Some(SearchIndex::Email), "calendar" => Some(SearchIndex::Calendar), "contacts" => Some(SearchIndex::Contacts), "file" => Some(SearchIndex::File), "tracing" => Some(SearchIndex::Tracing), _ => None, } } } ================================================ FILE: crates/store/src/write/log.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{SerializeInfallible, U64_LEN}; use ahash::AHashSet; use types::collection::{SyncCollection, VanishedCollection}; use utils::{codec::leb128::Leb128Vec, map::vec_map::VecMap}; use super::key::KeySerializer; #[derive(Default, Debug)] pub(crate) struct ChangeLogBuilder { pub changes: VecMap, pub vanished: VecMap, } #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub enum VanishedItem { Name(String), Id(u64), IdPair(u32, u32), } #[derive(Default, Debug)] pub(crate) struct VanishedItems(Vec); #[derive(Default, Debug)] pub struct Changes { pub item_inserts: AHashSet, pub item_updates: AHashSet, pub item_deletes: AHashSet, pub container_inserts: AHashSet, pub container_updates: AHashSet, pub container_deletes: AHashSet, pub container_property_changes: AHashSet, } impl ChangeLogBuilder { pub fn log_container_insert(&mut self, collection: SyncCollection, document_id: u32) { let changes = self.changes.get_mut_or_insert(collection); if changes.container_deletes.remove(&document_id) { changes.container_updates.insert(document_id); } else { changes.container_inserts.insert(document_id); } } pub fn log_item_insert( &mut self, collection: SyncCollection, prefix: Option, document_id: u32, ) { let id = build_id(prefix, document_id); let changes = self.changes.get_mut_or_insert(collection); if changes.item_deletes.remove(&id) { changes.item_updates.insert(id); } else { changes.item_inserts.insert(id); } } pub fn log_container_update(&mut self, collection: SyncCollection, document_id: u32) { self.changes .get_mut_or_insert(collection) .container_updates .insert(document_id); } pub fn log_container_property_update(&mut self, collection: SyncCollection, document_id: u32) { self.changes .get_mut_or_insert(collection) .container_property_changes .insert(document_id); } pub fn log_item_update( &mut self, collection: SyncCollection, prefix: Option, document_id: u32, ) { self.changes .get_mut_or_insert(collection) .item_updates .insert(build_id(prefix, document_id)); } pub fn log_container_delete(&mut self, collection: SyncCollection, document_id: u32) { let changes = self.changes.get_mut_or_insert(collection); let id = document_id; changes.container_updates.remove(&id); changes.container_property_changes.remove(&id); changes.container_deletes.insert(id); } pub fn log_item_delete( &mut self, collection: SyncCollection, prefix: Option, document_id: u32, ) { let changes = self.changes.get_mut_or_insert(collection); let id = build_id(prefix, document_id); changes.item_updates.remove(&id); changes.item_deletes.insert(id); } pub fn log_vanished_item( &mut self, collection: VanishedCollection, item: impl Into, ) { self.vanished .get_mut_or_insert(collection) .0 .push(item.into()); } } #[inline(always)] fn build_id(prefix: Option, document_id: u32) -> u64 { if let Some(prefix) = prefix { ((prefix as u64) << 32) | document_id as u64 } else { document_id as u64 } } impl Changes { pub fn has_container_changes(&self) -> bool { !self.container_inserts.is_empty() || !self.container_updates.is_empty() || !self.container_property_changes.is_empty() || !self.container_deletes.is_empty() } pub fn has_item_changes(&self) -> bool { !self.item_inserts.is_empty() || !self.item_updates.is_empty() || !self.item_deletes.is_empty() } } impl SerializeInfallible for Changes { fn serialize(&self) -> Vec { let mut buf = Vec::with_capacity( 1 + (self.item_inserts.len() + self.item_updates.len() + self.item_deletes.len() + self.container_inserts.len() + self.container_updates.len() + self.container_property_changes.len() + self.container_deletes.len() + 4) * std::mem::size_of::(), ); buf.push_leb128(self.container_inserts.len()); buf.push_leb128(self.container_updates.len()); buf.push_leb128(self.container_property_changes.len()); buf.push_leb128(self.container_deletes.len()); buf.push_leb128(self.item_inserts.len()); buf.push_leb128(self.item_updates.len()); buf.push_leb128(self.item_deletes.len()); for list in [ &self.container_inserts, &self.container_updates, &self.container_property_changes, &self.container_deletes, ] { for id in list { buf.push_leb128(*id); } } for list in [&self.item_inserts, &self.item_updates, &self.item_deletes] { for id in list { buf.push_leb128(*id); } } buf } } impl From for VanishedItem { fn from(value: String) -> Self { VanishedItem::Name(value) } } impl From for VanishedItem { fn from(value: u64) -> Self { VanishedItem::Id(value) } } impl From<(u32, u32)> for VanishedItem { fn from(value: (u32, u32)) -> Self { VanishedItem::Id((value.0 as u64) << 32 | value.1 as u64) } } impl VanishedItem { pub fn serialized_size(&self) -> usize { match self { VanishedItem::Name(name) => name.len() + 1, VanishedItem::Id(_) | VanishedItem::IdPair(..) => U64_LEN, } } } impl SerializeInfallible for VanishedItems { fn serialize(&self) -> Vec { let mut buf = KeySerializer::new(64); for item in &self.0 { buf = match item { VanishedItem::Name(name) => buf.write(name.as_bytes()).write(0u8), VanishedItem::Id(id) => buf.write(id.to_be_bytes().as_slice()), VanishedItem::IdPair(a, b) => buf .write(a.to_be_bytes().as_slice()) .write(b.to_be_bytes().as_slice()), }; } buf.finalize() } } ================================================ FILE: crates/store/src/write/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use self::assert::AssertValue; use crate::backend::MAX_TOKEN_LENGTH; use log::ChangeLogBuilder; use nlp::tokenizers::word::WordTokenizer; use rkyv::util::AlignedVec; use std::{ collections::HashSet, hash::Hash, time::{Duration, SystemTime}, }; use types::{ blob_hash::BlobHash, collection::{Collection, SyncCollection, VanishedCollection}, field::{ CalendarEventField, CalendarNotificationField, ContactField, EmailField, EmailSubmissionField, Field, MailboxField, PrincipalField, SieveField, }, }; use utils::{ cheeky_hash::CheekyHash, map::{bitmap::Bitmap, vec_map::VecMap}, }; pub mod assert; pub mod batch; pub mod bitpack; pub mod blob; pub mod key; pub mod log; pub mod serialize; pub(crate) const ARCHIVE_ALIGNMENT: usize = 16; #[derive(Debug, Clone)] pub struct Archive { pub inner: T, pub version: ArchiveVersion, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum ArchiveVersion { Versioned { change_id: u64, hash: u32 }, Hashed { hash: u32 }, Unversioned, } #[derive(Debug, Clone)] pub enum AlignedBytes { Aligned(AlignedVec), Vec(Vec), } pub struct Archiver where T: rkyv::Archive + for<'a> rkyv::Serialize< rkyv::api::high::HighSerializer< rkyv::util::AlignedVec, rkyv::ser::allocator::ArenaHandle<'a>, rkyv::rancor::Error, >, >, { pub inner: T, pub flags: u8, } #[derive(Debug, Default)] pub struct AssignedIds { pub ids: Vec, current_change_id: Option, } #[derive(Debug)] pub enum AssignedId { Counter(i64), ChangeId(ChangeId), } #[derive(Debug, Clone, Copy)] pub struct ChangeId { pub account_id: u32, pub change_id: u64, } #[cfg(not(feature = "test_mode"))] pub(crate) const MAX_COMMIT_ATTEMPTS: u32 = 10; #[cfg(not(feature = "test_mode"))] pub(crate) const MAX_COMMIT_TIME: Duration = Duration::from_secs(10); #[cfg(feature = "test_mode")] pub(crate) const MAX_COMMIT_ATTEMPTS: u32 = 1000; #[cfg(feature = "test_mode")] pub(crate) const MAX_COMMIT_TIME: Duration = Duration::from_secs(3600); #[derive(Debug)] pub struct Batch<'x> { pub(crate) changes: &'x VecMap, pub(crate) ops: &'x mut [Operation], } #[derive(Debug)] pub struct BatchBuilder { current_account_id: Option, current_collection: Option, current_document_id: Option, changes: VecMap, changed_collections: VecMap, has_assertions: bool, batch_size: usize, batch_ops: usize, commit_points: Vec, ops: Vec, } #[derive(Debug, Default)] pub struct ChangedCollection { pub changed_containers: Bitmap, pub changed_items: Bitmap, pub share_notification_id: Option, } #[derive(Debug, PartialEq, Eq, Hash)] pub enum Operation { AccountId { account_id: u32, }, Collection { collection: Collection, }, DocumentId { document_id: u32, }, AssertValue { class: ValueClass, assert_value: AssertValue, }, Value { class: ValueClass, op: ValueOp, }, Index { field: u8, key: Vec, set: bool, }, Log { collection: LogCollection, set: Vec, }, } #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] pub enum LogCollection { Sync(SyncCollection), Vanished(VanishedCollection), } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum ValueClass { Property(u8), IndexProperty(IndexPropertyClass), Acl(u32), InMemory(InMemoryClass), TaskQueue(TaskQueueClass), Directory(DirectoryClass), Blob(BlobOp), Config(Vec), Queue(QueueClass), Report(ReportClass), Telemetry(TelemetryClass), SearchIndex(SearchIndexClass), Any(AnyClass), ShareNotification { notification_id: u64, notify_account_id: u32, }, DocumentId, ChangeId, } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum IndexPropertyClass { Hash { property: u8, hash: CheekyHash }, Integer { property: u8, value: u64 }, } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub struct SearchIndexClass { pub index: SearchIndex, pub id: SearchIndexId, pub typ: SearchIndexType, } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum SearchIndexType { Term { field: u8, hash: CheekyHash }, Index { field: SearchIndexField }, Document, } pub(crate) const SEARCH_INDEX_MAX_FIELD_LEN: usize = 128; #[derive(Debug, PartialEq, Eq, Clone, Hash, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] pub struct SearchIndexField { pub(crate) field_id: u8, pub(crate) data: Vec, } #[derive(Debug, PartialEq, Clone, Copy, Eq, Hash)] pub enum SearchIndexId { Account { account_id: u32, document_id: u32 }, Global { id: u64 }, } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum TaskQueueClass { UpdateIndex { due: TaskEpoch, index: SearchIndex, is_insert: bool, }, SendAlarm { due: TaskEpoch, event_id: u16, alarm_id: u16, is_email_alert: bool, }, SendImip { due: TaskEpoch, is_payload: bool, }, MergeThreads { due: TaskEpoch, }, } #[derive(Debug, PartialEq, Clone, Copy, Eq, Hash)] #[repr(transparent)] pub struct TaskEpoch(pub(crate) u64); #[derive(Debug, PartialEq, Clone, Copy, Eq, Hash)] pub enum SearchIndex { Email, Calendar, Contacts, File, Tracing, InMemory, } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub struct AnyClass { pub subspace: u8, pub key: Vec, } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum InMemoryClass { Key(Vec), Counter(Vec), } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum DirectoryClass { NameToId(Vec), EmailToId(Vec), Index { word: Vec, principal_id: u32 }, MemberOf { principal_id: u32, member_of: u32 }, Members { principal_id: u32, has_member: u32 }, Principal(u32), UsedQuota(u32), } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum QueueClass { Message(u64), MessageEvent(QueueEvent), DmarcReportHeader(ReportEvent), DmarcReportEvent(ReportEvent), TlsReportHeader(ReportEvent), TlsReportEvent(ReportEvent), QuotaCount(Vec), QuotaSize(Vec), } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum ReportClass { Tls { id: u64, expires: u64 }, Dmarc { id: u64, expires: u64 }, Arf { id: u64, expires: u64 }, } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum TelemetryClass { Span { span_id: u64, }, Metric { timestamp: u64, metric_id: u64, node_id: u64, }, } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub struct QueueEvent { pub due: u64, pub queue_id: u64, pub queue_name: [u8; 8], } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub struct ReportEvent { pub due: u64, pub policy_hash: u64, pub seq_id: u64, pub domain: String, } #[derive(Debug, PartialEq, Eq, Hash, Default)] pub enum ValueOp { Set(Vec), SetFnc(SetOperation), MergeFnc(MergeOperation), AtomicAdd(i64), AddAndGet(i64), #[default] Clear, } pub enum MergeResult { Update(Vec), Skip, Delete, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum Param { I64(i64), U64(u64), String(String), Bytes(Vec), Bool(bool), } #[derive(Debug, Clone, PartialEq, Eq, Hash)] #[repr(transparent)] pub struct Params(Vec); pub type SetFnc = fn(&Params, &AssignedIds) -> trc::Result>; pub type MergeFnc = fn(&Params, &AssignedIds, Option<&[u8]>) -> trc::Result; #[derive(Debug, Clone)] pub struct MergeOperation { pub(crate) fnc: MergeFnc, pub(crate) params: Params, } #[derive(Debug, Clone)] pub struct SetOperation { pub(crate) fnc: SetFnc, pub(crate) params: Params, } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum BlobOp { Commit { hash: BlobHash }, Link { hash: BlobHash, to: BlobLink }, Quota { hash: BlobHash, until: u64 }, Undelete { hash: BlobHash, until: u64 }, SpamSample { hash: BlobHash, until: u64 }, } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum BlobLink { Id { id: u64 }, Document, Temporary { until: u64 }, } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub struct AnyKey> { pub subspace: u8, pub key: T, } pub trait TokenizeText { fn tokenize_into(&self, tokens: &mut HashSet); fn to_tokens(&self) -> HashSet; } impl TokenizeText for &str { fn tokenize_into(&self, tokens: &mut HashSet) { for token in WordTokenizer::new(self, MAX_TOKEN_LENGTH) { tokens.insert(token.word.into_owned()); } } fn to_tokens(&self) -> HashSet { let mut tokens = HashSet::new(); self.tokenize_into(&mut tokens); tokens } } pub trait IntoOperations { fn build(self, batch: &mut BatchBuilder) -> trc::Result<()>; } #[inline(always)] pub fn now() -> u64 { SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()) } impl AsRef for ValueClass { fn as_ref(&self) -> &ValueClass { self } } impl AssignedIds { pub fn push_counter_id(&mut self, id: i64) { self.ids.push(AssignedId::Counter(id)); } pub fn push_change_id(&mut self, account_id: u32, change_id: u64) { self.ids.push(AssignedId::ChangeId(ChangeId { account_id, change_id, })); } pub fn last_change_id(&self, account_id: u32) -> trc::Result { self.ids .iter() .filter_map(|id| match id { AssignedId::ChangeId(change_id) if change_id.account_id == account_id => { Some(change_id.change_id) } _ => None, }) .next_back() .ok_or_else(|| { trc::StoreEvent::UnexpectedError .caused_by(trc::location!()) .ctx(trc::Key::Reason, "No change ids were created") }) } pub fn current_change_id(&self) -> trc::Result { self.current_change_id.ok_or_else(|| { trc::StoreEvent::UnexpectedError .caused_by(trc::location!()) .ctx(trc::Key::Reason, "No current change id is set") }) } pub(crate) fn set_current_change_id(&mut self, account_id: u32) -> trc::Result { let change_id = self.last_change_id(account_id)?; self.current_change_id = Some(change_id); Ok(change_id) } pub fn last_counter_id(&self) -> trc::Result { self.ids .iter() .filter_map(|id| match id { AssignedId::Counter(counter_id) => Some(*counter_id), _ => None, }) .next_back() .ok_or_else(|| { trc::StoreEvent::UnexpectedError .caused_by(trc::location!()) .ctx(trc::Key::Reason, "No counter ids were created") }) } } impl QueueClass { pub fn due(&self) -> Option { match self { QueueClass::DmarcReportHeader(report_event) => report_event.due.into(), QueueClass::TlsReportHeader(report_event) => report_event.due.into(), _ => None, } } } impl> AsRef<[u8]> for Archive { fn as_ref(&self) -> &[u8] { self.inner.as_ref() } } impl ArchiveVersion { pub fn hash(&self) -> Option { match self { ArchiveVersion::Versioned { hash, .. } => Some(*hash), ArchiveVersion::Hashed { hash } => Some(*hash), ArchiveVersion::Unversioned => None, } } pub fn change_id(&self) -> Option { match self { ArchiveVersion::Versioned { change_id, .. } => Some(*change_id), _ => None, } } } impl From for u8 { fn from(value: LogCollection) -> Self { match value { LogCollection::Sync(col) => col as u8, LogCollection::Vanished(col) => col as u8, } } } impl From for ValueClass { fn from(value: ContactField) -> Self { ValueClass::Property(value.into()) } } impl From for ValueClass { fn from(value: CalendarEventField) -> Self { ValueClass::Property(value.into()) } } impl From for ValueClass { fn from(value: CalendarNotificationField) -> Self { ValueClass::Property(value.into()) } } impl From for ValueClass { fn from(value: EmailField) -> Self { ValueClass::Property(value.into()) } } impl From for ValueClass { fn from(value: MailboxField) -> Self { ValueClass::Property(value.into()) } } impl From for ValueClass { fn from(value: PrincipalField) -> Self { ValueClass::Property(value.into()) } } impl From for ValueClass { fn from(value: SieveField) -> Self { ValueClass::Property(value.into()) } } impl From for ValueClass { fn from(value: EmailSubmissionField) -> Self { ValueClass::Property(value.into()) } } impl From for ValueClass { fn from(value: Field) -> Self { ValueClass::Property(value.into()) } } impl PartialEq for MergeOperation { fn eq(&self, other: &Self) -> bool { self.params == other.params } } impl Eq for MergeOperation {} impl PartialEq for SetOperation { fn eq(&self, other: &Self) -> bool { self.params == other.params } } impl Eq for SetOperation {} impl Hash for MergeOperation { fn hash(&self, state: &mut H) { self.params.hash(state); } } impl Hash for SetOperation { fn hash(&self, state: &mut H) { self.params.hash(state); } } impl SetOperation { pub fn params(&self) -> &Params { &self.params } } impl MergeOperation { pub fn params(&self) -> &Params { &self.params } } impl Params { pub fn with_capacity(capacity: usize) -> Self { Self(Vec::with_capacity(capacity)) } pub fn new() -> Self { Self(Vec::new()) } pub fn with_i64(mut self, value: i64) -> Self { self.0.push(Param::I64(value)); self } pub fn with_u64(mut self, value: u64) -> Self { self.0.push(Param::U64(value)); self } pub fn with_string(mut self, value: String) -> Self { self.0.push(Param::String(value)); self } pub fn with_str(mut self, value: &str) -> Self { self.0.push(Param::String(value.to_string())); self } pub fn with_bytes(mut self, value: Vec) -> Self { self.0.push(Param::Bytes(value)); self } pub fn with_bool(mut self, value: bool) -> Self { self.0.push(Param::Bool(value)); self } pub fn i64(&self, idx: usize) -> i64 { match &self.0[idx] { Param::I64(v) => *v, _ => panic!("Param at index {} is not an i64", idx), } } pub fn u64(&self, idx: usize) -> u64 { match &self.0[idx] { Param::U64(v) => *v, _ => panic!("Param at index {} is not a u64", idx), } } pub fn string(&self, idx: usize) -> &str { match &self.0[idx] { Param::String(v) => v.as_str(), _ => panic!("Param at index {} is not a String", idx), } } pub fn bytes(&self, idx: usize) -> &[u8] { match &self.0[idx] { Param::Bytes(v) => v.as_slice(), _ => panic!("Param at index {} is not Bytes", idx), } } pub fn bool(&self, idx: usize) -> bool { match &self.0[idx] { Param::Bool(v) => *v, _ => panic!("Param at index {} is not a bool", idx), } } pub fn len(&self) -> usize { self.0.len() } pub fn is_empty(&self) -> bool { self.0.is_empty() } pub fn as_slice(&self) -> &[Param] { &self.0 } } impl Default for Params { fn default() -> Self { Self::new() } } impl AsRef<[Param]> for Params { fn as_ref(&self) -> &[Param] { &self.0 } } impl TaskEpoch { /* Structure of the 64-bit epoch: 4 bytes: seconds since custom epoch (1632280000) 2 bytes: attempt number 2 bytes: sequence id */ const EPOCH_OFFSET: u64 = 1632280000; pub fn now() -> Self { Self::new(now()) } pub fn new(timestamp: u64) -> Self { Self(timestamp.saturating_sub(Self::EPOCH_OFFSET) << 32) } pub fn with_attempt(mut self, attempt: u16) -> Self { self.0 |= (attempt as u64) << 16; self } pub fn with_sequence_id(mut self, sequence_id: u16) -> Self { self.0 |= sequence_id as u64; self } pub fn with_random_sequence_id(self) -> Self { self.with_sequence_id(rand::random()) } pub fn due(&self) -> u64 { (self.0 >> 32) + Self::EPOCH_OFFSET } pub fn attempt(&self) -> u16 { (self.0 >> 16) as u16 } pub fn sequence_id(&self) -> u16 { self.0 as u16 } pub fn inner(&self) -> u64 { self.0 } pub fn from_inner(inner: u64) -> Self { Self(inner) } } ================================================ FILE: crates/store/src/write/serialize.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{ARCHIVE_ALIGNMENT, AlignedBytes, Archive, ArchiveVersion, Archiver}; use crate::{Deserialize, Serialize, SerializeInfallible, U32_LEN, U64_LEN, Value}; use compact_str::format_compact; use rkyv::util::AlignedVec; use roaring::{RoaringBitmap, RoaringTreemap}; const MAGIC_MARKER: u8 = 1 << 7; const VERSIONED: u8 = 1 << 6; const HASHED: u8 = 1 << 5; const LZ4_COMPRESSED: u8 = 1 << 4; const COMPRESS_WATERMARK: usize = 8192; fn validate_marker_and_contents(bytes: &[u8]) -> Option<(bool, &[u8], ArchiveVersion)> { let (marker, contents) = bytes .split_last() .filter(|(marker, _)| (**marker & MAGIC_MARKER) != 0)?; let is_uncompressed = (marker & LZ4_COMPRESSED) == 0; if marker & VERSIONED != 0 { let (contents, change_id) = contents .split_at_checked(contents.len() - U64_LEN) .and_then(|(contents, change_id)| { change_id .try_into() .ok() .map(|change_id| (contents, u64::from_be_bytes(change_id))) })?; contents .split_at_checked(contents.len() - U32_LEN) .and_then(|(contents, archive_hash)| { let hash = xxhash_rust::xxh3::xxh3_64(contents) as u32; if hash.to_be_bytes().as_slice() == archive_hash { Some(( is_uncompressed, contents, ArchiveVersion::Versioned { change_id, hash }, )) } else { None } }) } else if marker & HASHED != 0 { contents .split_at_checked(contents.len() - U32_LEN) .and_then(|(contents, archive_hash)| { let hash = xxhash_rust::xxh3::xxh3_64(contents) as u32; if hash.to_be_bytes().as_slice() == archive_hash { Some((is_uncompressed, contents, ArchiveVersion::Hashed { hash })) } else { None } }) } else { Some((is_uncompressed, contents, ArchiveVersion::Unversioned)) } } impl Deserialize for Archive { fn deserialize(bytes: &[u8]) -> trc::Result { let (is_uncompressed, contents, version) = validate_marker_and_contents(bytes).ok_or_else(|| { trc::StoreEvent::DataCorruption .into_err() .details("Archive integrity compromised") .ctx(trc::Key::Value, bytes) .caused_by(trc::location!()) })?; if is_uncompressed { let mut bytes = AlignedVec::with_capacity(contents.len()); bytes.extend_from_slice(contents); Ok(Archive { version, inner: AlignedBytes::Aligned(bytes), }) } else { aligned_lz4_deflate(contents).map(|inner| Archive { version, inner }) } } fn deserialize_owned(mut bytes: Vec) -> trc::Result { let (is_uncompressed, contents, version) = validate_marker_and_contents(&bytes) .ok_or_else(|| { trc::StoreEvent::DataCorruption .into_err() .details("Archive integrity compromised") .ctx(trc::Key::Value, bytes.as_slice()) .caused_by(trc::location!()) })?; if is_uncompressed { bytes.truncate(contents.len()); if bytes.as_ptr().addr() & (ARCHIVE_ALIGNMENT - 1) == 0 { Ok(Archive { version, inner: AlignedBytes::Vec(bytes), }) } else { let mut aligned = AlignedVec::with_capacity(bytes.len()); aligned.extend_from_slice(&bytes); Ok(Archive { version, inner: AlignedBytes::Aligned(aligned), }) } } else { aligned_lz4_deflate(contents).map(|inner| Archive { version, inner }) } } } #[inline] fn aligned_lz4_deflate(archive: &[u8]) -> trc::Result { lz4_flex::block::uncompressed_size(archive) .and_then(|(uncompressed_size, archive)| { let mut bytes = AlignedVec::with_capacity(uncompressed_size); unsafe { // SAFETY: `new_len` is equal to `capacity` and vector is initialized by lz4_flex. bytes.set_len(uncompressed_size); } lz4_flex::decompress_into(archive, &mut bytes)?; Ok(AlignedBytes::Aligned(bytes)) }) .map_err(|err| { trc::StoreEvent::DecompressError .ctx(trc::Key::Value, archive) .caused_by(trc::location!()) .reason(err) }) } impl Serialize for Archiver where T: rkyv::Archive + for<'a> rkyv::Serialize< rkyv::api::high::HighSerializer< rkyv::util::AlignedVec, rkyv::ser::allocator::ArenaHandle<'a>, rkyv::rancor::Error, >, >, { fn serialize(&self) -> trc::Result> { rkyv::to_bytes::(&self.inner) .map_err(|err| { trc::StoreEvent::DeserializeError .caused_by(trc::location!()) .reason(err) }) .map(|input| { let input = input.as_ref(); let input_len = input.len(); let version_offset = ((self.flags & VERSIONED != 0) as usize) * U64_LEN; let mut bytes = if input_len > COMPRESS_WATERMARK { let mut bytes = vec![ self.flags | LZ4_COMPRESSED; lz4_flex::block::get_maximum_output_size(input_len) + (U32_LEN * 2) + version_offset + 1 ]; // Compress the data let compressed_len = lz4_flex::compress_into(input, &mut bytes[U32_LEN..]).unwrap(); if compressed_len < input_len { // Prepend the length of the uncompressed data bytes[..U32_LEN].copy_from_slice(&(input_len as u32).to_le_bytes()); if self.flags & HASHED != 0 { // Hash the compressed data including the length let hash = xxhash_rust::xxh3::xxh3_64(&bytes[..compressed_len + U32_LEN]) as u32; // Add the hash bytes[compressed_len + U32_LEN..compressed_len + (U32_LEN * 2)] .copy_from_slice(&hash.to_be_bytes()); // Truncate to the actual size bytes.truncate(compressed_len + (U32_LEN * 2) + version_offset + 1); } else { // Truncate to the actual size bytes.truncate(compressed_len + U32_LEN + 1); } return bytes; } bytes.clear(); bytes } else { Vec::with_capacity(input_len + U32_LEN + version_offset + 1) }; bytes.extend_from_slice(input); if self.flags & HASHED != 0 { bytes.extend_from_slice( &(xxhash_rust::xxh3::xxh3_64(input) as u32).to_be_bytes(), ); } if version_offset != 0 { bytes.extend_from_slice(0u64.to_be_bytes().as_slice()); } bytes.push(self.flags); bytes }) } } impl Archive { #[inline] pub fn as_bytes(&self) -> &[u8] { match &self.inner { AlignedBytes::Vec(bytes) => bytes.as_slice(), AlignedBytes::Aligned(bytes) => bytes.as_slice(), } } pub fn unarchive(&self) -> trc::Result<&::Archived> where T: rkyv::Archive, T::Archived: for<'a> rkyv::bytecheck::CheckBytes< rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, > + rkyv::Deserialize>, { let bytes = self.as_bytes(); if self.version != ArchiveVersion::Unversioned { if bytes.len() >= std::mem::size_of::() { // SAFETY: Trusted input with integrity hash Ok(unsafe { rkyv::access_unchecked::(bytes) }) } else { Err(trc::StoreEvent::DataCorruption .into_err() .details(format_compact!( "Archive size mismatch, expected {} bytes but got {} bytes.", std::mem::size_of::(), bytes.len() )) .ctx(trc::Key::Value, bytes) .caused_by(trc::location!())) } } else { rkyv::access::(bytes).map_err(|err| { trc::StoreEvent::DeserializeError .ctx(trc::Key::Value, self.as_bytes()) .details("Archive access failed") .caused_by(trc::location!()) .reason(err) }) } } pub fn unarchive_untrusted(&self) -> trc::Result<&::Archived> where T: rkyv::Archive, T::Archived: for<'a> rkyv::bytecheck::CheckBytes< rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, > + rkyv::Deserialize>, { let bytes = self.as_bytes(); if bytes.len() >= std::mem::size_of::() { rkyv::access::(bytes).map_err(|err| { trc::StoreEvent::DeserializeError .ctx(trc::Key::Value, self.as_bytes()) .details("Archive access failed") .caused_by(trc::location!()) .reason(err) }) } else { Err(trc::StoreEvent::DataCorruption .into_err() .details(format_compact!( "Archive size mismatch, expected {} bytes but got {} bytes.", std::mem::size_of::(), bytes.len() )) .ctx(trc::Key::Value, bytes) .caused_by(trc::location!())) } } pub fn deserialize(&self) -> trc::Result where T: rkyv::Archive, T::Archived: for<'a> rkyv::bytecheck::CheckBytes< rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, > + rkyv::Deserialize>, { self.unarchive::().and_then(|input| { rkyv::deserialize(input).map_err(|err| { trc::StoreEvent::DeserializeError .ctx(trc::Key::Value, self.as_bytes()) .caused_by(trc::location!()) .reason(err) }) }) } pub fn deserialize_untrusted(&self) -> trc::Result where T: rkyv::Archive, T::Archived: for<'a> rkyv::bytecheck::CheckBytes< rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, > + rkyv::Deserialize>, { self.unarchive_untrusted::().and_then(|input| { rkyv::deserialize(input).map_err(|err| { trc::StoreEvent::DeserializeError .ctx(trc::Key::Value, self.as_bytes()) .caused_by(trc::location!()) .reason(err) }) }) } pub fn to_unarchived(&self) -> trc::Result::Archived>> where T: rkyv::Archive, T::Archived: for<'a> rkyv::bytecheck::CheckBytes< rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, > + rkyv::Deserialize>, { self.unarchive::().map(|inner| Archive { version: self.version, inner, }) } pub fn into_deserialized(&self) -> trc::Result> where T: rkyv::Archive, T::Archived: for<'a> rkyv::bytecheck::CheckBytes< rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, > + rkyv::Deserialize>, { self.deserialize::().map(|inner| Archive { version: self.version, inner, }) } pub fn into_inner(self) -> Vec { let mut bytes = match self.inner { AlignedBytes::Vec(bytes) => bytes, AlignedBytes::Aligned(bytes) => bytes.to_vec(), }; match self.version { ArchiveVersion::Versioned { change_id, hash } => { bytes.extend_from_slice(&change_id.to_be_bytes()); bytes.extend_from_slice(&hash.to_be_bytes()); bytes.push(MAGIC_MARKER | VERSIONED | HASHED); } ArchiveVersion::Hashed { hash } => { bytes.extend_from_slice(&hash.to_be_bytes()); bytes.push(MAGIC_MARKER | HASHED); } ArchiveVersion::Unversioned => { bytes.push(MAGIC_MARKER); } } bytes } pub fn extract_hash(bytes: &[u8]) -> Option { let marker = *bytes.last()?; if marker & VERSIONED != 0 { bytes .get(bytes.len() - U32_LEN - U64_LEN - 1..bytes.len() - U64_LEN - 1) .and_then(|slice| slice.try_into().ok().map(u32::from_be_bytes)) } else if marker & HASHED != 0 { bytes .get(bytes.len() - U32_LEN - 1..bytes.len() - 1) .and_then(|slice| slice.try_into().ok().map(u32::from_be_bytes)) } else { None } } } impl Archiver where T: rkyv::Archive + for<'a> rkyv::Serialize< rkyv::api::high::HighSerializer< rkyv::util::AlignedVec, rkyv::ser::allocator::ArenaHandle<'a>, rkyv::rancor::Error, >, >, { pub fn new(inner: T) -> Self { Self { inner, flags: MAGIC_MARKER | HASHED, } } pub fn into_inner(self) -> T { self.inner } pub fn with_version(self) -> Self { Self { inner: self.inner, flags: self.flags | VERSIONED, } } pub fn untrusted(self) -> Self { Self { inner: self.inner, flags: MAGIC_MARKER, } } pub fn serialize_versioned(self) -> trc::Result<(u64, Vec)> { self.with_version() .serialize() .map(|bytes| ((bytes.len() - U64_LEN - 1) as u64, bytes)) } } impl Archive<&T> where T: rkyv::Portable + for<'a> rkyv::bytecheck::CheckBytes> + Sync + Send, { pub fn to_deserialized(&self) -> trc::Result> where T: rkyv::Deserialize>, { rkyv::deserialize::(self.inner) .map_err(|err| { trc::StoreEvent::DeserializeError .caused_by(trc::location!()) .reason(err) }) .map(|inner| Archive { version: self.version, inner, }) } pub fn deserialize(&self) -> trc::Result where T: rkyv::Deserialize>, { rkyv::deserialize::(self.inner).map_err(|err| { trc::StoreEvent::DeserializeError .caused_by(trc::location!()) .reason(err) }) } } #[inline] pub fn rkyv_deserialize(input: &T) -> trc::Result where T: rkyv::Portable + for<'a> rkyv::bytecheck::CheckBytes> + Sync + Send + rkyv::Deserialize>, { rkyv::deserialize::(input).map_err(|err| { trc::StoreEvent::DeserializeError .caused_by(trc::location!()) .reason(err) }) } pub fn rkyv_unarchive(input: &[u8]) -> trc::Result<&::Archived> where T: rkyv::Archive, T::Archived: for<'a> rkyv::bytecheck::CheckBytes> + rkyv::Deserialize>, { rkyv::access::(input).map_err(|err| { trc::StoreEvent::DataCorruption .caused_by(trc::location!()) .ctx(trc::Key::Value, input) .reason(err) }) } impl SerializeInfallible for u32 { fn serialize(&self) -> Vec { self.to_be_bytes().to_vec() } } impl SerializeInfallible for u64 { fn serialize(&self) -> Vec { self.to_be_bytes().to_vec() } } impl SerializeInfallible for i64 { fn serialize(&self) -> Vec { self.to_be_bytes().to_vec() } } impl SerializeInfallible for u16 { fn serialize(&self) -> Vec { self.to_be_bytes().to_vec() } } impl SerializeInfallible for f64 { fn serialize(&self) -> Vec { self.to_be_bytes().to_vec() } } impl SerializeInfallible for &str { fn serialize(&self) -> Vec { self.as_bytes().to_vec() } } impl Deserialize for String { fn deserialize(bytes: &[u8]) -> trc::Result { Ok(String::from_utf8_lossy(bytes).into_owned()) } fn deserialize_owned(bytes: Vec) -> trc::Result { Ok(String::from_utf8(bytes) .unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned())) } } impl Deserialize for u64 { fn deserialize(bytes: &[u8]) -> trc::Result { Ok(u64::from_be_bytes(bytes.try_into().map_err(|_| { trc::StoreEvent::DataCorruption.caused_by(trc::location!()) })?)) } } impl Deserialize for i64 { fn deserialize(bytes: &[u8]) -> trc::Result { Ok(i64::from_be_bytes(bytes.try_into().map_err(|_| { trc::StoreEvent::DataCorruption.caused_by(trc::location!()) })?)) } } impl Deserialize for u32 { fn deserialize(bytes: &[u8]) -> trc::Result { Ok(u32::from_be_bytes(bytes.try_into().map_err(|_| { trc::StoreEvent::DataCorruption.caused_by(trc::location!()) })?)) } } impl Deserialize for () { fn deserialize(_bytes: &[u8]) -> trc::Result { Ok(()) } } impl From> for Archive { fn from(_: Value<'static>) -> Self { unimplemented!() } } impl Default for Archive { fn default() -> Self { Archive { version: ArchiveVersion::Unversioned, inner: AlignedBytes::Aligned(AlignedVec::new()), } } } impl Serialize for RoaringBitmap { fn serialize(&self) -> trc::Result> { let mut bytes = Vec::with_capacity(self.serialized_size()); self.serialize_into(&mut bytes) .map_err(|err| { trc::StoreEvent::UnexpectedError .caused_by(trc::location!()) .reason(err) }) .map(|_| bytes) } } impl Deserialize for RoaringBitmap { fn deserialize(bytes: &[u8]) -> trc::Result { RoaringBitmap::deserialize_from(bytes).map_err(|err| { trc::StoreEvent::DeserializeError .caused_by(trc::location!()) .reason(err) }) } } impl Serialize for RoaringTreemap { fn serialize(&self) -> trc::Result> { let mut bytes = Vec::with_capacity(self.serialized_size()); self.serialize_into(&mut bytes) .map_err(|err| { trc::StoreEvent::UnexpectedError .caused_by(trc::location!()) .reason(err) }) .map(|_| bytes) } } impl Deserialize for RoaringTreemap { fn deserialize(bytes: &[u8]) -> trc::Result { RoaringTreemap::deserialize_from(bytes).map_err(|err| { trc::StoreEvent::DeserializeError .caused_by(trc::location!()) .reason(err) }) } } ================================================ FILE: crates/trc/Cargo.toml ================================================ [package] name = "trc" version = "0.15.5" edition = "2024" [dependencies] event_macro = { path = "./event-macro" } mail-auth = { version = "0.7.1" } mail-parser = { version = "0.11", features = ["full_encoding"] } base64 = "0.22.1" serde = "1.0" serde_json = "1.0.120" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-webpki-roots", "http2"]} rtrb = "0.3.1" parking_lot = "0.12.3" tokio = { version = "1.47", features = ["net", "macros"] } ahash = "0.8.11" rkyv = { version = "0.8.10", features = ["little_endian"] } compact_str = "0.9.0" [features] test_mode = [] enterprise = [] [dev-dependencies] ================================================ FILE: crates/trc/event-macro/Cargo.toml ================================================ [package] name = "event_macro" version = "0.15.5" edition = "2024" [lib] proc-macro = true [dependencies] syn = { version = "2.0", features = ["full"] } quote = "1.0" proc-macro2 = "1.0" ================================================ FILE: crates/trc/event-macro/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use proc_macro::TokenStream; use quote::quote; use syn::{ Data, DeriveInput, Expr, ExprPath, Fields, Ident, Token, parse::Parse, parse_macro_input, }; static mut GLOBAL_ID_COUNTER: usize = 0; #[proc_macro_attribute] pub fn event_type(_attr: TokenStream, item: TokenStream) -> TokenStream { let input = parse_macro_input!(item as DeriveInput); let name = &input.ident; let name_str = name.to_string(); let prefix = to_snake_case(name_str.strip_suffix("Event").unwrap_or(&name_str)); let enum_variants = match &input.data { Data::Enum(data_enum) => &data_enum.variants, _ => panic!("This macro only works with enums"), }; let mut variant_ids = Vec::new(); let mut variant_names = Vec::new(); let mut event_names = Vec::new(); for variant in enum_variants { unsafe { variant_ids.push(GLOBAL_ID_COUNTER); GLOBAL_ID_COUNTER += 1; } let variant_name = &variant.ident; event_names.push(format!( "{prefix}.{}", to_snake_case(&variant_name.to_string()) )); variant_names.push(variant_name); } let id_fn = quote! { pub const fn id(&self) -> usize { match self { #(Self::#variant_names => #variant_ids,)* } } }; let name_fn = quote! { pub fn name(&self) -> &'static str { match self { #(Self::#variant_names => #event_names,)* } } }; let parse_fn = quote! { pub fn try_parse(name: &str) -> Option { match name { #(#event_names => Some(Self::#variant_names),)* _ => None, } } }; let variants_fn = quote! { pub const fn variants() -> &'static [Self] { &[ #(#name::#variant_names,)* ] } }; let expanded = quote! { #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum #name { #(#variant_names),* } impl #name { #id_fn #name_fn #parse_fn #variants_fn } }; TokenStream::from(expanded) } #[proc_macro_attribute] pub fn event_family(_attr: TokenStream, item: TokenStream) -> TokenStream { let input = parse_macro_input!(item as DeriveInput); let name = &input.ident; let variants = match &input.data { Data::Enum(data_enum) => &data_enum.variants, _ => panic!("EventType must be an enum"), }; let variant_idents: Vec<_> = variants.iter().map(|v| &v.ident).collect(); let event_types: Vec<_> = variants .iter() .map(|v| match &v.fields { Fields::Unnamed(fields) => &fields.unnamed[0], _ => panic!("EventType variants must be unnamed and contain a single type"), }) .map(|f| &f.ty) .collect(); let variant_names: Vec<_> = variant_idents .iter() .map(|ident| { let name_str = ident.to_string(); to_snake_case(name_str.strip_suffix("Event").unwrap_or(&name_str)) }) .collect(); let expanded = quote! { pub enum #name { #(#variant_idents(#event_types)),* } impl #name { pub const fn id(&self) -> usize { match self { #(#name::#variant_idents(e) => e.id()),* } } pub fn name(&self) -> &'static str { match self { #(#name::#variant_idents(e) => e.name()),* } } pub fn try_parse(name: &str) -> Option { match name.trim().split_once('.')?.0 { #( #variant_names => <#event_types>::try_parse(&name).map(#name::#variant_idents), )* _ => None, } } pub const fn variants() -> [#name; crate::TOTAL_EVENT_COUNT] { let mut variants = [crate::EventType::Eval(crate::EvalEvent::Error); crate::TOTAL_EVENT_COUNT]; #( { let sub_variants = <#event_types>::variants(); let mut i = 0; while i < sub_variants.len() { variants[sub_variants[i].id()] = #name::#variant_idents(sub_variants[i]); i += 1; } } )* variants } } }; TokenStream::from(expanded) } #[proc_macro_attribute] pub fn key_names(_attr: TokenStream, item: TokenStream) -> TokenStream { let input = parse_macro_input!(item as DeriveInput); let name = &input.ident; let enum_variants = match &input.data { Data::Enum(data_enum) => &data_enum.variants, _ => panic!("This macro only works with enums"), }; let mut variant_names = Vec::new(); let mut camel_case_names = Vec::new(); let mut snake_case_names = Vec::new(); for variant in enum_variants.iter() { let variant_name = &variant.ident; variant_names.push(variant_name); snake_case_names.push(to_snake_case(&variant_name.to_string())); camel_case_names.push( variant_name .to_string() .char_indices() .map(|(i, c)| if i == 0 { c.to_ascii_lowercase() } else { c }) .collect::(), ); } let id_fn = quote! { pub fn id(&self) -> &'static str { match self { #(Self::#variant_names => #snake_case_names,)* } } }; let name_fn = quote! { pub fn name(&self) -> &'static str { match self { #(Self::#variant_names => #camel_case_names,)* } } }; let parse_fn = quote! { pub fn try_parse(name: &str) -> Option { match name { #(#snake_case_names => Some(Self::#variant_names),)* _ => None, } } }; let expanded = quote! { #input impl #name { #name_fn #id_fn #parse_fn } }; TokenStream::from(expanded) } #[proc_macro] pub fn total_event_count(_item: TokenStream) -> TokenStream { let count = unsafe { GLOBAL_ID_COUNTER }; let expanded = quote! { #count }; TokenStream::from(expanded) } fn to_snake_case(name: &str) -> String { let mut out = String::with_capacity(name.len()); for (idx, ch) in name.char_indices() { if ch.is_ascii_uppercase() { if idx > 0 { out.push('-'); } out.push(ch.to_ascii_lowercase()); } else { out.push(ch); } } out } struct EventMacroInput { event: Ident, param: Expr, key_values: Vec<(Ident, Expr)>, } impl Parse for EventMacroInput { fn parse(input: syn::parse::ParseStream) -> syn::Result { let event: Ident = input.parse()?; let content; syn::parenthesized!(content in input); let param: Expr = content.parse()?; let mut key_values = Vec::new(); while !input.is_empty() { input.parse::()?; if input.is_empty() { break; } let key: Ident = input.parse()?; input.parse::()?; let value: Expr = input.parse()?; key_values.push((key, value)); } Ok(EventMacroInput { event, param, key_values, }) } } #[proc_macro] pub fn event(input: TokenStream) -> TokenStream { let EventMacroInput { event, param, key_values, } = parse_macro_input!(input as EventMacroInput); let key_value_tokens = key_values.iter().map(|(key, value)| { quote! { (trc::Key::#key, trc::Value::from(#value)) } }); // This avoids having to evaluate expensive values when we know we are not interested in the event let key_value_metric_tokens = key_values.iter().filter_map(|(key, value)| { if key.is_metric_key() { Some(quote! { (trc::Key::#key, trc::Value::from(#value)) }) } else { None } }); let expanded = if matches!(¶m, Expr::Path(ExprPath { path, .. }) if path.segments.len() > 1 && path.segments.last().unwrap().arguments.is_empty() ) { quote! {{ const ET: trc::EventType = trc::EventType::#event(#param); const ET_ID: usize = ET.id(); if trc::Collector::has_interest(ET_ID) { let keys = vec![#(#key_value_tokens),*]; if trc::Collector::is_metric(ET_ID) { trc::Collector::record_metric(ET, ET_ID, &keys); } trc::Event::with_keys(ET, keys).send(); } else if trc::Collector::is_metric(ET_ID) { trc::Collector::record_metric(ET, ET_ID, &[#(#key_value_metric_tokens),*]); } }} } else { quote! {{ let et = trc::EventType::#event(#param); let et_id = et.id(); if trc::Collector::has_interest(et_id) { let keys = vec![#(#key_value_tokens),*]; if trc::Collector::is_metric(et_id) { trc::Collector::record_metric(et, et_id, &keys); } trc::Event::with_keys(et, keys).send(); } else if trc::Collector::is_metric(et_id) { trc::Collector::record_metric(et, et_id, &[#(#key_value_metric_tokens),*]); } }} }; TokenStream::from(expanded) } trait IsMetricKey { fn is_metric_key(&self) -> bool; } impl IsMetricKey for Ident { fn is_metric_key(&self) -> bool { matches!( self.to_string().as_ref(), "Total" | "Elapsed" | "Size" | "TotalSuccesses" | "TotalFailures" | "DmarcPass" | "DmarcQuarantine" | "DmarcReject" | "DmarcNone" | "DkimPass" | "DkimFail" | "DkimNone" | "SpfPass" | "SpfFail" | "SpfNone" | "Protocol" | "Code" ) } } ================================================ FILE: crates/trc/src/atomics/array.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; pub struct AtomicU32Array([AtomicU32; N]); pub struct AtomicU64Array([AtomicU64; N]); impl AtomicU32Array { #[allow(clippy::new_without_default)] #[allow(clippy::declare_interior_mutable_const)] pub const fn new() -> Self { Self({ const INIT: AtomicU32 = AtomicU32::new(0); let mut array = [INIT; N]; let mut i = 0; while i < N { array[i] = AtomicU32::new(0); i += 1; } array }) } #[inline(always)] pub fn get(&self, index: usize) -> u32 { self.0[index].load(Ordering::Relaxed) } #[inline(always)] pub fn set(&self, index: usize, value: u32) { self.0[index].store(value, Ordering::Relaxed); } #[inline(always)] pub fn add(&self, index: usize, value: u32) { self.0[index].fetch_add(value, Ordering::Relaxed); } pub fn inner(&self) -> &[AtomicU32; N] { &self.0 } } impl AtomicU64Array { #[allow(clippy::new_without_default)] #[allow(clippy::declare_interior_mutable_const)] pub const fn new() -> Self { Self({ const INIT: AtomicU64 = AtomicU64::new(0); let mut array = [INIT; N]; let mut i = 0; while i < N { array[i] = AtomicU64::new(0); i += 1; } array }) } #[inline(always)] pub fn get(&self, index: usize) -> u64 { self.0[index].load(Ordering::Relaxed) } #[inline(always)] pub fn set(&self, index: usize, value: u64) { self.0[index].store(value, Ordering::Relaxed); } #[inline(always)] pub fn add(&self, index: usize, value: u64) { self.0[index].fetch_add(value, Ordering::Relaxed); } pub fn inner(&self) -> &[AtomicU64; N] { &self.0 } } ================================================ FILE: crates/trc/src/atomics/bitset.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::atomic::{AtomicUsize, Ordering}; use crate::ipc::{USIZE_BITS, USIZE_BITS_MASK, bitset::Bitset}; pub struct AtomicBitset([AtomicUsize; N]); impl AtomicBitset { #[allow(clippy::new_without_default)] #[allow(clippy::declare_interior_mutable_const)] pub const fn new() -> Self { Self({ const INIT: AtomicUsize = AtomicUsize::new(0); let mut array = [INIT; N]; let mut i = 0; while i < N { array[i] = AtomicUsize::new(0); i += 1; } array }) } #[inline(always)] pub fn set(&self, index: impl Into) { let index = index.into(); self.0[index / USIZE_BITS].fetch_or(1 << (index & USIZE_BITS_MASK), Ordering::Relaxed); } #[inline(always)] pub fn clear(&self, index: impl Into) { let index = index.into(); self.0[index / USIZE_BITS].fetch_and(!(1 << (index & USIZE_BITS_MASK)), Ordering::Relaxed); } #[inline(always)] pub fn get(&self, index: impl Into) -> bool { let index = index.into(); self.0[index / USIZE_BITS].load(Ordering::Relaxed) & (1 << (index & USIZE_BITS_MASK)) != 0 } pub fn update(&self, bitset: impl AsRef>) { let bitset = bitset.as_ref(); for i in 0..N { self.0[i].store(bitset.0[i], Ordering::Relaxed); } } pub fn union(&self, bitset: impl AsRef>) { let bitset = bitset.as_ref(); for i in 0..N { self.0[i].fetch_or(bitset.0[i], Ordering::Relaxed); } } pub fn clear_all(&self) { for i in 0..N { self.0[i].store(0, Ordering::Relaxed); } } pub fn is_empty(&self) -> bool { for i in 0..N { if self.0[i].load(Ordering::Relaxed) != 0 { return false; } } true } } #[cfg(test)] mod tests { use super::*; const TEST_SIZE: usize = 1000; type TestBitset = AtomicBitset<{ TEST_SIZE.div_ceil(USIZE_BITS) }>; static BITSET: TestBitset = TestBitset::new(); #[test] fn test_atomic_bitset() { for i in 0..TEST_SIZE { assert!(!BITSET.get(i), "Bit {} should be unset in new BITSET", i); } for i in 0..TEST_SIZE { assert!(!BITSET.get(i), "Bit {} should be initially unset", i); BITSET.set(i); assert!(BITSET.get(i), "Bit {} should be set after setting", i); } BITSET.clear_all(); for i in 0..TEST_SIZE { BITSET.set(i); assert!(BITSET.get(i), "Bit {} should be set before clearing", i); BITSET.clear(i); assert!(!BITSET.get(i), "Bit {} should be unset after clearing", i); } BITSET.clear_all(); // Set even bits for i in (0..TEST_SIZE).step_by(2) { BITSET.set(i); } // Check all bits for i in 0..TEST_SIZE { if i % 2 == 0 { assert!(BITSET.get(i), "Even bit {} should be set", i); } else { assert!(!BITSET.get(i), "Odd bit {} should be unset", i); } } // Clear even bits and set odd bits for i in 0..TEST_SIZE { if i % 2 == 0 { BITSET.clear(i); } else { BITSET.set(i); } } // Check all bits again for i in 0..TEST_SIZE { if i % 2 == 0 { assert!(!BITSET.get(i), "Even bit {} should now be unset", i); } else { assert!(BITSET.get(i), "Odd bit {} should now be set", i); } } } } ================================================ FILE: crates/trc/src/atomics/counter.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::atomic::{AtomicU64, Ordering}; pub struct AtomicCounter { id: &'static str, description: &'static str, unit: &'static str, value: AtomicU64, } impl AtomicCounter { pub const fn new(id: &'static str, description: &'static str, unit: &'static str) -> Self { Self { id, description, unit, value: AtomicU64::new(0), } } #[inline(always)] pub fn increment(&self) { self.value.fetch_add(1, Ordering::Relaxed); } #[inline(always)] pub fn increment_by(&self, value: u64) { self.value.fetch_add(value, Ordering::Relaxed); } #[inline(always)] pub fn decrement(&self) { self.value.fetch_sub(1, Ordering::Relaxed); } #[inline(always)] pub fn decrement_by(&self, value: u64) { self.value.fetch_sub(value, Ordering::Relaxed); } #[inline(always)] pub fn get(&self) -> u64 { self.value.load(Ordering::Relaxed) } pub fn id(&self) -> &'static str { self.id } pub fn description(&self) -> &'static str { self.description } pub fn unit(&self) -> &'static str { self.unit } pub fn is_active(&self) -> bool { self.value.load(Ordering::Relaxed) > 0 } } ================================================ FILE: crates/trc/src/atomics/gauge.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::atomic::{AtomicU64, Ordering}; use crate::MetricType; pub struct AtomicGauge { id: MetricType, value: AtomicU64, } impl AtomicGauge { pub const fn new(id: MetricType) -> Self { Self { id, value: AtomicU64::new(0), } } #[inline(always)] pub fn increment(&self) { self.value.fetch_add(1, Ordering::Relaxed); } #[inline(always)] pub fn set(&self, value: u64) { self.value.store(value, Ordering::Relaxed); } #[inline(always)] pub fn decrement(&self) { self.value.fetch_sub(1, Ordering::Relaxed); } #[inline(always)] pub fn get(&self) -> u64 { self.value.load(Ordering::Relaxed) } #[inline(always)] pub fn add(&self, value: u64) { self.value.fetch_add(value, Ordering::Relaxed); } #[inline(always)] pub fn subtract(&self, value: u64) { self.value.fetch_sub(value, Ordering::Relaxed); } pub fn id(&self) -> MetricType { self.id } } ================================================ FILE: crates/trc/src/atomics/histogram.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::atomic::{AtomicU64, Ordering}; use crate::MetricType; use super::array::AtomicU32Array; pub struct AtomicHistogram { id: MetricType, buckets: AtomicU32Array, upper_bounds: [u64; N], sum: AtomicU64, count: AtomicU64, min: AtomicU64, max: AtomicU64, } impl AtomicHistogram { pub const fn new(id: MetricType, upper_bounds: [u64; N]) -> Self { Self { buckets: AtomicU32Array::new(), upper_bounds, sum: AtomicU64::new(0), count: AtomicU64::new(0), min: AtomicU64::new(u64::MAX), max: AtomicU64::new(0), id, } } pub fn observe(&self, value: u64) { self.sum.fetch_add(value, Ordering::Relaxed); self.count.fetch_add(1, Ordering::Relaxed); self.min.fetch_min(value, Ordering::Relaxed); self.max.fetch_max(value, Ordering::Relaxed); for (idx, upper_bound) in self.upper_bounds.iter().enumerate() { if value < *upper_bound { self.buckets.add(idx, 1); return; } } unreachable!() } pub fn id(&self) -> MetricType { self.id } pub fn sum(&self) -> u64 { self.sum.load(Ordering::Relaxed) } pub fn count(&self) -> u64 { self.count.load(Ordering::Relaxed) } pub fn average(&self) -> f64 { let sum = self.sum(); let count = self.count(); if count > 0 { sum as f64 / count as f64 } else { 0.0 } } pub fn min(&self) -> Option { let min = self.min.load(Ordering::Relaxed); if min != u64::MAX { Some(min) } else { None } } pub fn max(&self) -> Option { let max = self.max.load(Ordering::Relaxed); if max != 0 { Some(max) } else { None } } pub fn buckets_iter(&self) -> impl IntoIterator + '_ { self.buckets .inner() .iter() .map(|bucket| bucket.load(Ordering::Relaxed) as u64) } pub fn buckets_vec(&self) -> Vec { let mut vec = Vec::with_capacity(N); for bucket in self.buckets.inner().iter() { vec.push(bucket.load(Ordering::Relaxed) as u64); } vec } pub fn buckets_len(&self) -> usize { N } pub fn upper_bounds_iter(&self) -> impl IntoIterator + '_ { self.upper_bounds.iter().copied() } pub fn upper_bounds_vec(&self) -> Vec { let mut vec = Vec::with_capacity(N - 1); for upper_bound in self.upper_bounds.iter().take(N - 1) { vec.push(*upper_bound as f64); } vec } pub fn is_active(&self) -> bool { self.count.load(Ordering::Relaxed) > 0 } pub const fn new_message_sizes(id: MetricType) -> AtomicHistogram<12> { AtomicHistogram::new( id, [ 500, // 500 bytes 1_000, // 1 KB 10_000, // 10 KB 100_000, // 100 KB 1_000_000, // 1 MB 5_000_000, // 5 MB 10_000_000, // 10 MB 25_000_000, // 25 MB 50_000_000, // 50 MB 100_000_000, // 100 MB 500_000_000, // 500 MB u64::MAX, // Catch-all for any larger sizes ], ) } pub const fn new_short_durations(id: MetricType) -> AtomicHistogram<12> { AtomicHistogram::new( id, [ 5, // 5 milliseconds 10, // 10 milliseconds 50, // 50 milliseconds 100, // 100 milliseconds 500, // 0.5 seconds 1_000, // 1 second 2_000, // 2 seconds 5_000, // 5 seconds 10_000, // 10 seconds 30_000, // 30 seconds 60_000, // 1 minute u64::MAX, // Catch-all for any longer durations ], ) } pub const fn new_medium_durations(id: MetricType) -> AtomicHistogram<12> { AtomicHistogram::new( id, [ 250, 500, 1_000, 5_000, 10_000, // For quick connections (seconds) 60_000, (60 * 5) * 1_000, (60 * 10) * 1_000, (60 * 30) * 1_000, // For medium-length connections (minutes) (60 * 60) * 1_000, (60 * 60 * 5) * 1_000, u64::MAX, // For extreme cases (8 hours and 1 day) ], ) } pub const fn new_long_durations(id: MetricType) -> AtomicHistogram<12> { AtomicHistogram::new( id, [ 1_000, // 1 second 30_000, // 30 seconds 300_000, // 5 minutes 600_000, // 10 minutes 1_800_000, // 30 minutes 3_600_000, // 1 hour 14_400_000, // 5 hours 28_800_000, // 8 hours 43_200_000, // 12 hours 86_400_000, // 1 day 604_800_000, // 1 week u64::MAX, // Catch-all for any longer durations ], ) } } ================================================ FILE: crates/trc/src/atomics/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod array; pub mod bitset; pub mod counter; pub mod gauge; pub mod histogram; ================================================ FILE: crates/trc/src/event/conv.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{borrow::Cow, fmt::Debug, str::FromStr, time::Duration}; use compact_str::{CompactString, ToCompactString, format_compact}; use mail_auth::common::verify::VerifySignature; use crate::*; impl AsRef for Error { fn as_ref(&self) -> &EventType { &self.0.inner } } impl From<&'static str> for Value { fn from(value: &'static str) -> Self { Self::String(CompactString::const_new(value)) } } impl From for Value { fn from(value: String) -> Self { Self::String(CompactString::from_string_buffer(value)) } } impl From for Value { fn from(value: CompactString) -> Self { Self::String(value) } } impl From> for Value { fn from(value: Box) -> Self { Self::String(CompactString::from(value)) } } impl From for Value { fn from(value: u64) -> Self { Self::UInt(value) } } impl From for Value { fn from(value: i64) -> Self { Self::Int(value) } } impl From for Value { fn from(value: f64) -> Self { Self::Float(value) } } impl From for Value { fn from(value: f32) -> Self { Self::Float(value.into()) } } impl From for Value { fn from(value: u16) -> Self { Self::UInt(value.into()) } } impl From for Value { fn from(value: i32) -> Self { Self::Int(value.into()) } } impl From for Value { fn from(value: u32) -> Self { Self::UInt(value.into()) } } impl From for Value { fn from(value: usize) -> Self { Self::UInt(value as u64) } } impl From for Value { fn from(value: bool) -> Self { Self::Bool(value) } } impl From for Value { fn from(value: IpAddr) -> Self { match value { IpAddr::V4(ip) => Value::Ipv4(ip), IpAddr::V6(ip) => Value::Ipv6(ip), } } } impl> From> for Value { fn from(value: Option) -> Self { match value { Some(value) => value.into(), None => Self::None, } } } impl From for Value { fn from(value: Duration) -> Self { Self::Duration(value.as_millis() as u64) } } impl From for Value { fn from(value: Error) -> Self { Self::Event(value) } } impl From for Error { fn from(value: EventType) -> Self { Error::new(value) } } impl From for Error { fn from(value: StoreEvent) -> Self { Error::new(EventType::Store(value)) } } impl From for Error { fn from(value: AuthEvent) -> Self { Error::new(EventType::Auth(value)) } } impl From> for Value { fn from(value: Vec) -> Self { Self::Bytes(value) } } impl From<&[u8]> for Value { fn from(value: &[u8]) -> Self { Self::Bytes(value.to_vec()) } } impl From> for Value { fn from(value: Cow<'static, str>) -> Self { match value { Cow::Borrowed(value) => Self::String(CompactString::const_new(value)), Cow::Owned(value) => Self::String(value.into()), } } } impl From<&crate::Result> for Value where T: Debug, { fn from(value: &crate::Result) -> Self { match value { Ok(value) => format_compact!("{:?}", value).into(), Err(err) => Value::Event(err.clone()), } } } impl From> for Value where T: Into, { fn from(value: Vec) -> Self { Self::Array(value.into_iter().map(Into::into).collect()) } } impl From<&[T]> for Value where T: Into + Clone, { fn from(value: &[T]) -> Self { Self::Array(value.iter().map(|v| v.clone().into()).collect()) } } impl EventType { pub fn from_io_error(self, err: std::io::Error) -> Error { self.reason(err).details("I/O error") } pub fn from_json_error(self, err: serde_json::Error) -> Error { self.reason(err).details("JSON deserialization failed") } pub fn from_base64_error(self, err: base64::DecodeError) -> Error { self.reason(err).details("Base64 decoding failed") } pub fn from_http_error(self, err: reqwest::Error) -> Error { self.into_err() .ctx_opt( Key::Url, err.url().map(|url| url.as_ref().to_compact_string()), ) .ctx_opt(Key::Code, err.status().map(|status| status.as_u16())) .reason(err) } pub fn from_http_str_error(self, err: reqwest::header::ToStrError) -> Error { self.reason(err) .details("Failed to convert header to string") } } impl From for Error { fn from(err: mail_auth::Error) -> Self { match err { mail_auth::Error::ParseError => { EventType::MailAuth(MailAuthEvent::ParseError).into_err() } mail_auth::Error::MissingParameters => { EventType::MailAuth(MailAuthEvent::MissingParameters).into_err() } mail_auth::Error::NoHeadersFound => { EventType::MailAuth(MailAuthEvent::NoHeadersFound).into_err() } mail_auth::Error::CryptoError(details) => EventType::MailAuth(MailAuthEvent::Crypto) .into_err() .details(CompactString::from(details)), mail_auth::Error::Io(details) => EventType::MailAuth(MailAuthEvent::Io) .into_err() .details(CompactString::from(details)), mail_auth::Error::Base64 => EventType::MailAuth(MailAuthEvent::Base64).into_err(), mail_auth::Error::UnsupportedVersion => { EventType::Dkim(DkimEvent::UnsupportedVersion).into_err() } mail_auth::Error::UnsupportedAlgorithm => { EventType::Dkim(DkimEvent::UnsupportedAlgorithm).into_err() } mail_auth::Error::UnsupportedCanonicalization => { EventType::Dkim(DkimEvent::UnsupportedCanonicalization).into_err() } mail_auth::Error::UnsupportedKeyType => { EventType::Dkim(DkimEvent::UnsupportedKeyType).into_err() } mail_auth::Error::FailedBodyHashMatch => { EventType::Dkim(DkimEvent::FailedBodyHashMatch).into_err() } mail_auth::Error::FailedVerification => { EventType::Dkim(DkimEvent::FailedVerification).into_err() } mail_auth::Error::FailedAuidMatch => { EventType::Dkim(DkimEvent::FailedAuidMatch).into_err() } mail_auth::Error::RevokedPublicKey => { EventType::Dkim(DkimEvent::RevokedPublicKey).into_err() } mail_auth::Error::IncompatibleAlgorithms => { EventType::Dkim(DkimEvent::IncompatibleAlgorithms).into_err() } mail_auth::Error::SignatureExpired => { EventType::Dkim(DkimEvent::SignatureExpired).into_err() } mail_auth::Error::SignatureLength => { EventType::Dkim(DkimEvent::SignatureLength).into_err() } mail_auth::Error::DnsError(details) => EventType::MailAuth(MailAuthEvent::DnsError) .into_err() .details(CompactString::from(details)), mail_auth::Error::DnsRecordNotFound(code) => { EventType::MailAuth(MailAuthEvent::DnsRecordNotFound) .into_err() .code(code.to_str()) } mail_auth::Error::ArcChainTooLong => EventType::Arc(ArcEvent::ChainTooLong).into_err(), mail_auth::Error::ArcInvalidInstance(instance) => { EventType::Arc(ArcEvent::InvalidInstance).ctx(Key::Id, instance) } mail_auth::Error::ArcInvalidCV => EventType::Arc(ArcEvent::InvalidCv).into_err(), mail_auth::Error::ArcHasHeaderTag => EventType::Arc(ArcEvent::HasHeaderTag).into_err(), mail_auth::Error::ArcBrokenChain => EventType::Arc(ArcEvent::BrokenChain).into_err(), mail_auth::Error::NotAligned => { EventType::MailAuth(MailAuthEvent::PolicyNotAligned).into_err() } mail_auth::Error::InvalidRecordType => { EventType::MailAuth(MailAuthEvent::DnsInvalidRecordType).into_err() } } } } impl From<&mail_auth::DkimResult> for Error { fn from(value: &mail_auth::DkimResult) -> Self { match value.clone() { mail_auth::DkimResult::Pass => Error::new(EventType::Dkim(DkimEvent::Pass)), mail_auth::DkimResult::Neutral(err) => { Error::new(EventType::Dkim(DkimEvent::Neutral)).caused_by(Error::from(err)) } mail_auth::DkimResult::Fail(err) => { Error::new(EventType::Dkim(DkimEvent::Fail)).caused_by(Error::from(err)) } mail_auth::DkimResult::PermError(err) => { Error::new(EventType::Dkim(DkimEvent::PermError)).caused_by(Error::from(err)) } mail_auth::DkimResult::TempError(err) => { Error::new(EventType::Dkim(DkimEvent::TempError)).caused_by(Error::from(err)) } mail_auth::DkimResult::None => Error::new(EventType::Dkim(DkimEvent::None)), } } } impl From<&mail_auth::DmarcResult> for Error { fn from(value: &mail_auth::DmarcResult) -> Self { match value.clone() { mail_auth::DmarcResult::Pass => Error::new(EventType::Dmarc(DmarcEvent::Pass)), mail_auth::DmarcResult::Fail(err) => { Error::new(EventType::Dmarc(DmarcEvent::Fail)).caused_by(Error::from(err)) } mail_auth::DmarcResult::PermError(err) => { Error::new(EventType::Dmarc(DmarcEvent::PermError)).caused_by(Error::from(err)) } mail_auth::DmarcResult::TempError(err) => { Error::new(EventType::Dmarc(DmarcEvent::TempError)).caused_by(Error::from(err)) } mail_auth::DmarcResult::None => Error::new(EventType::Dmarc(DmarcEvent::None)), } } } impl From<&mail_auth::DkimOutput<'_>> for Error { fn from(value: &mail_auth::DkimOutput<'_>) -> Self { Error::from(value.result()).ctx_opt( Key::Domain, value.signature().map(|s| s.domain().to_compact_string()), ) } } impl From<&mail_auth::IprevOutput> for Error { fn from(value: &mail_auth::IprevOutput) -> Self { match value.result().clone() { mail_auth::IprevResult::Pass => Error::new(EventType::Iprev(IprevEvent::Pass)), mail_auth::IprevResult::Fail(err) => { Error::new(EventType::Iprev(IprevEvent::Fail)).caused_by(Error::from(err)) } mail_auth::IprevResult::PermError(err) => { Error::new(EventType::Iprev(IprevEvent::PermError)).caused_by(Error::from(err)) } mail_auth::IprevResult::TempError(err) => { Error::new(EventType::Iprev(IprevEvent::TempError)).caused_by(Error::from(err)) } mail_auth::IprevResult::None => Error::new(EventType::Iprev(IprevEvent::None)), } .ctx_opt( Key::Details, value.ptr.as_ref().map(|s| { s.iter() .map(|v| Value::String(v.into())) .collect::>() }), ) } } impl From<&mail_auth::SpfOutput> for Error { fn from(value: &mail_auth::SpfOutput) -> Self { Error::new(EventType::Spf(match value.result() { mail_auth::SpfResult::Pass => SpfEvent::Pass, mail_auth::SpfResult::Fail => SpfEvent::Fail, mail_auth::SpfResult::SoftFail => SpfEvent::SoftFail, mail_auth::SpfResult::Neutral => SpfEvent::Neutral, mail_auth::SpfResult::PermError => SpfEvent::PermError, mail_auth::SpfResult::TempError => SpfEvent::TempError, mail_auth::SpfResult::None => SpfEvent::None, })) .ctx_opt( Key::Details, value.explanation().map(|s| s.to_compact_string()), ) } } impl From for Error { fn from(value: rkyv::rancor::Error) -> Self { Error::new(EventType::Store(StoreEvent::DeserializeError)) .reason(value) .details("Rkyv de/serialization failed") } } pub trait AssertSuccess where Self: Sized, { fn assert_success( self, cause: EventType, ) -> impl std::future::Future> + Send; } impl AssertSuccess for reqwest::Response { async fn assert_success(self, cause: EventType) -> crate::Result { let status = self.status(); if status.is_success() { Ok(self) } else { Err(cause .ctx(Key::Code, status.as_u16()) .details("HTTP request failed") .ctx_opt(Key::Reason, self.text().await.map(CompactString::from).ok())) } } } impl FromStr for EventType { type Err = (); fn from_str(s: &str) -> std::result::Result { EventType::try_parse(s).ok_or(()) } } impl FromStr for Key { type Err = (); fn from_str(s: &str) -> std::result::Result { Key::try_parse(s).ok_or(()) } } ================================================ FILE: crates/trc/src/event/description.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::*; impl EventType { pub fn description(&self) -> &'static str { match self { EventType::Store(event) => event.description(), EventType::Jmap(event) => event.description(), EventType::Imap(event) => event.description(), EventType::ManageSieve(event) => event.description(), EventType::Pop3(event) => event.description(), EventType::Smtp(event) => event.description(), EventType::Network(event) => event.description(), EventType::Limit(event) => event.description(), EventType::Manage(event) => event.description(), EventType::Auth(event) => event.description(), EventType::Config(event) => event.description(), EventType::Resource(event) => event.description(), EventType::Sieve(event) => event.description(), EventType::Spam(event) => event.description(), EventType::Server(event) => event.description(), EventType::Purge(event) => event.description(), EventType::Eval(event) => event.description(), EventType::Acme(event) => event.description(), EventType::Http(event) => event.description(), EventType::Arc(event) => event.description(), EventType::Dkim(event) => event.description(), EventType::Dmarc(event) => event.description(), EventType::Iprev(event) => event.description(), EventType::Dane(event) => event.description(), EventType::Spf(event) => event.description(), EventType::MailAuth(event) => event.description(), EventType::Tls(event) => event.description(), EventType::PushSubscription(event) => event.description(), EventType::Cluster(event) => event.description(), EventType::Housekeeper(event) => event.description(), EventType::TaskQueue(event) => event.description(), EventType::Milter(event) => event.description(), EventType::MtaHook(event) => event.description(), EventType::Delivery(event) => event.description(), EventType::Queue(event) => event.description(), EventType::TlsRpt(event) => event.description(), EventType::MtaSts(event) => event.description(), EventType::IncomingReport(event) => event.description(), EventType::OutgoingReport(event) => event.description(), EventType::Telemetry(event) => event.description(), EventType::MessageIngest(event) => event.description(), EventType::Security(event) => event.description(), EventType::Ai(event) => event.description(), EventType::WebDav(event) => event.description(), EventType::Calendar(event) => event.description(), } } pub fn explain(&self) -> &'static str { match self { EventType::Store(event) => event.explain(), EventType::Jmap(event) => event.explain(), EventType::Imap(event) => event.explain(), EventType::ManageSieve(event) => event.explain(), EventType::Pop3(event) => event.explain(), EventType::Smtp(event) => event.explain(), EventType::Network(event) => event.explain(), EventType::Limit(event) => event.explain(), EventType::Manage(event) => event.explain(), EventType::Auth(event) => event.explain(), EventType::Config(event) => event.explain(), EventType::Resource(event) => event.explain(), EventType::Sieve(event) => event.explain(), EventType::Spam(event) => event.explain(), EventType::Server(event) => event.explain(), EventType::Purge(event) => event.explain(), EventType::Eval(event) => event.explain(), EventType::Acme(event) => event.explain(), EventType::Http(event) => event.explain(), EventType::Arc(event) => event.explain(), EventType::Dkim(event) => event.explain(), EventType::Dmarc(event) => event.explain(), EventType::Iprev(event) => event.explain(), EventType::Dane(event) => event.explain(), EventType::Spf(event) => event.explain(), EventType::MailAuth(event) => event.explain(), EventType::Tls(event) => event.explain(), EventType::PushSubscription(event) => event.explain(), EventType::Cluster(event) => event.explain(), EventType::Housekeeper(event) => event.explain(), EventType::TaskQueue(event) => event.explain(), EventType::Milter(event) => event.explain(), EventType::MtaHook(event) => event.explain(), EventType::Delivery(event) => event.explain(), EventType::Queue(event) => event.explain(), EventType::TlsRpt(event) => event.explain(), EventType::MtaSts(event) => event.explain(), EventType::IncomingReport(event) => event.explain(), EventType::OutgoingReport(event) => event.explain(), EventType::Telemetry(event) => event.explain(), EventType::MessageIngest(event) => event.explain(), EventType::Security(event) => event.explain(), EventType::Ai(event) => event.explain(), EventType::WebDav(event) => event.explain(), EventType::Calendar(event) => event.explain(), } } } impl HttpEvent { pub fn description(&self) -> &'static str { match self { HttpEvent::Error => "HTTP error occurred", HttpEvent::RequestUrl => "HTTP request URL", HttpEvent::RequestBody => "HTTP request body", HttpEvent::ResponseBody => "HTTP response body", HttpEvent::XForwardedMissing => "X-Forwarded-For header is missing", HttpEvent::ConnectionStart => "HTTP connection started", HttpEvent::ConnectionEnd => "HTTP connection ended", } } pub fn explain(&self) -> &'static str { match self { HttpEvent::Error => "An error occurred during an HTTP request", HttpEvent::RequestUrl => "The URL of an HTTP request", HttpEvent::RequestBody => "The body of an HTTP request", HttpEvent::ResponseBody => "The body of an HTTP response", HttpEvent::XForwardedMissing => "The X-Forwarded-For header is missing", HttpEvent::ConnectionStart => "An HTTP connection was started", HttpEvent::ConnectionEnd => "An HTTP connection was ended", } } } impl ClusterEvent { pub fn description(&self) -> &'static str { match self { ClusterEvent::SubscriberStart => "PubSub subscriber started", ClusterEvent::SubscriberStop => "PubSub subscriber stopped", ClusterEvent::SubscriberError => "PubSub subscriber error", ClusterEvent::SubscriberDisconnected => "PubSub subscriber disconnected", ClusterEvent::PublisherStart => "PubSub publisher started", ClusterEvent::PublisherStop => "PubSub publisher stopped", ClusterEvent::PublisherError => "PubSub publisher error", ClusterEvent::MessageReceived => "PubSub message received", ClusterEvent::MessageSkipped => "PubSub message skipped", ClusterEvent::MessageInvalid => "Invalid PubSub message", } } pub fn explain(&self) -> &'static str { match self { ClusterEvent::SubscriberStart => "The PubSub subscriber has started", ClusterEvent::SubscriberStop => "The PubSub subscriber has stopped", ClusterEvent::SubscriberError => "An error occurred while subscribing to PubSub", ClusterEvent::SubscriberDisconnected => "The PubSub subscriber has disconnected", ClusterEvent::PublisherStart => "The PubSub publisher has started", ClusterEvent::PublisherStop => "The PubSub publisher has stopped", ClusterEvent::PublisherError => "An error occurred while publishing to PubSub", ClusterEvent::MessageReceived => "A message was received from the PubSub server", ClusterEvent::MessageSkipped => "A message originating from this node was skipped", ClusterEvent::MessageInvalid => { "An invalid message was received from the PubSub server" } } } } impl HousekeeperEvent { pub fn description(&self) -> &'static str { match self { HousekeeperEvent::Start => "Housekeeper process started", HousekeeperEvent::Stop => "Housekeeper process stopped", HousekeeperEvent::Schedule => "Housekeeper task scheduled", HousekeeperEvent::Run => "Housekeeper task run", } } pub fn explain(&self) -> &'static str { match self { HousekeeperEvent::Start => "The housekeeper process has started", HousekeeperEvent::Stop => "The housekeeper process has stopped", HousekeeperEvent::Schedule => "A housekeeper task has been scheduled", HousekeeperEvent::Run => "A housekeeper task is running", } } } impl TaskQueueEvent { pub fn description(&self) -> &'static str { match self { TaskQueueEvent::TaskAcquired => "Task acquired from queue", TaskQueueEvent::TaskLocked => "Task is locked by another process", TaskQueueEvent::BlobNotFound => "Blob not found for task", TaskQueueEvent::MetadataNotFound => "Metadata not found for task", TaskQueueEvent::TaskIgnored => "Task ignored based on current server roles", TaskQueueEvent::TaskFailed => "Task failed during processing", } } pub fn explain(&self) -> &'static str { match self { TaskQueueEvent::TaskAcquired => "A task has been acquired from the queue", TaskQueueEvent::TaskLocked => "The task id is locked by another process", TaskQueueEvent::BlobNotFound => "The requested blob was not found for task", TaskQueueEvent::MetadataNotFound => "The metadata was not found for task", TaskQueueEvent::TaskIgnored => "The task was ignored based on the current server roles", TaskQueueEvent::TaskFailed => "The task failed during processing", } } } impl ImapEvent { pub fn description(&self) -> &'static str { match self { ImapEvent::GetAcl => "IMAP GET ACL command", ImapEvent::SetAcl => "IMAP SET ACL command", ImapEvent::MyRights => "IMAP MYRIGHTS command", ImapEvent::ListRights => "IMAP LISTRIGHTS command", ImapEvent::Append => "IMAP APPEND command", ImapEvent::Capabilities => "IMAP CAPABILITIES command", ImapEvent::Id => "IMAP ID command", ImapEvent::Close => "IMAP CLOSE command", ImapEvent::Copy => "IMAP COPY command", ImapEvent::Move => "IMAP MOVE command", ImapEvent::CreateMailbox => "IMAP CREATE mailbox command", ImapEvent::DeleteMailbox => "IMAP DELETE mailbox command", ImapEvent::RenameMailbox => "IMAP RENAME mailbox command", ImapEvent::Enable => "IMAP ENABLE command", ImapEvent::Expunge => "IMAP EXPUNGE command", ImapEvent::Fetch => "IMAP FETCH command", ImapEvent::IdleStart => "IMAP IDLE start", ImapEvent::IdleStop => "IMAP IDLE stop", ImapEvent::List => "IMAP LIST command", ImapEvent::Lsub => "IMAP LSUB command", ImapEvent::Logout => "IMAP LOGOUT command", ImapEvent::Namespace => "IMAP NAMESPACE command", ImapEvent::Noop => "IMAP NOOP command", ImapEvent::Search => "IMAP SEARCH command", ImapEvent::Sort => "IMAP SORT command", ImapEvent::Select => "IMAP SELECT command", ImapEvent::Status => "IMAP STATUS command", ImapEvent::Store => "IMAP STORE command", ImapEvent::Subscribe => "IMAP SUBSCRIBE command", ImapEvent::Unsubscribe => "IMAP UNSUBSCRIBE command", ImapEvent::Thread => "IMAP THREAD command", ImapEvent::Error => "IMAP error occurred", ImapEvent::RawInput => "Raw IMAP input received", ImapEvent::RawOutput => "Raw IMAP output sent", ImapEvent::ConnectionStart => "IMAP connection started", ImapEvent::ConnectionEnd => "IMAP connection ended", ImapEvent::GetQuota => "IMAP GETQUOTA command", } } pub fn explain(&self) -> &'static str { match self { ImapEvent::GetAcl => "Client requested mailbox ACL", ImapEvent::SetAcl => "Client set mailbox ACL", ImapEvent::MyRights => "Client requested mailbox rights", ImapEvent::ListRights => "Client requested mailbox rights list", ImapEvent::Append => "Client appended a message to a mailbox", ImapEvent::Capabilities => "Client requested server capabilities", ImapEvent::Id => "Client sent an ID command", ImapEvent::Close => "Client closed a mailbox", ImapEvent::Copy => "Client copied messages between mailboxes", ImapEvent::Move => "Client moved messages between mailboxes", ImapEvent::CreateMailbox => "Client created a mailbox", ImapEvent::DeleteMailbox => "Client deleted a mailbox", ImapEvent::RenameMailbox => "Client renamed a mailbox", ImapEvent::Enable => "Client enabled an extension", ImapEvent::Expunge => "Client expunged messages", ImapEvent::Fetch => "Client fetched messages", ImapEvent::IdleStart => "Client started IDLE", ImapEvent::IdleStop => "Client stopped IDLE", ImapEvent::List => "Client listed mailboxes", ImapEvent::Lsub => "Client listed subscribed mailboxes", ImapEvent::Logout => "Client logged out", ImapEvent::Namespace => "Client requested namespace", ImapEvent::Noop => "Client sent a NOOP command", ImapEvent::Search => "Client searched for messages", ImapEvent::Sort => "Client sorted messages", ImapEvent::Select => "Client selected a mailbox", ImapEvent::Status => "Client requested mailbox status", ImapEvent::Store => "Client stored flags", ImapEvent::Subscribe => "Client subscribed to a mailbox", ImapEvent::Unsubscribe => "Client unsubscribed from a mailbox", ImapEvent::Thread => "Client requested message threads", ImapEvent::Error => "An error occurred during an IMAP command", ImapEvent::RawInput => "Raw IMAP input received", ImapEvent::RawOutput => "Raw IMAP output sent", ImapEvent::ConnectionStart => "IMAP connection started", ImapEvent::ConnectionEnd => "IMAP connection ended", ImapEvent::GetQuota => "Client requested mailbox quota", } } } impl Pop3Event { pub fn description(&self) -> &'static str { match self { Pop3Event::Delete => "POP3 DELETE command", Pop3Event::Reset => "POP3 RESET command", Pop3Event::Quit => "POP3 QUIT command", Pop3Event::Fetch => "POP3 FETCH command", Pop3Event::List => "POP3 LIST command", Pop3Event::ListMessage => "POP3 LIST specific message command", Pop3Event::Uidl => "POP3 UIDL command", Pop3Event::UidlMessage => "POP3 UIDL specific message command", Pop3Event::Stat => "POP3 STAT command", Pop3Event::Noop => "POP3 NOOP command", Pop3Event::Capabilities => "POP3 CAPABILITIES command", Pop3Event::StartTls => "POP3 STARTTLS command", Pop3Event::Utf8 => "POP3 UTF8 command", Pop3Event::Error => "POP3 error occurred", Pop3Event::RawInput => "Raw POP3 input received", Pop3Event::RawOutput => "Raw POP3 output sent", Pop3Event::ConnectionStart => "POP3 connection started", Pop3Event::ConnectionEnd => "POP3 connection ended", } } pub fn explain(&self) -> &'static str { match self { Pop3Event::Delete => "Client deleted a message", Pop3Event::Reset => "Client reset the session", Pop3Event::Quit => "Client quit the session", Pop3Event::Fetch => "Client fetched a message", Pop3Event::List => "Client listed messages", Pop3Event::ListMessage => "Client listed a specific message", Pop3Event::Uidl => "Client requested unique identifiers", Pop3Event::UidlMessage => "Client requested a specific unique identifier", Pop3Event::Stat => "Client requested mailbox status", Pop3Event::Noop => "Client sent a NOOP command", Pop3Event::Capabilities => "Client requested server capabilities", Pop3Event::StartTls => "Client requested TLS", Pop3Event::Utf8 => "Client requested UTF-8 support", Pop3Event::Error => "An error occurred during a POP3 command", Pop3Event::RawInput => "Raw POP3 input received", Pop3Event::RawOutput => "Raw POP3 output sent", Pop3Event::ConnectionStart => "POP3 connection started", Pop3Event::ConnectionEnd => "POP3 connection ended", } } } impl ManageSieveEvent { pub fn description(&self) -> &'static str { match self { ManageSieveEvent::CreateScript => "ManageSieve CREATE script command", ManageSieveEvent::UpdateScript => "ManageSieve UPDATE script command", ManageSieveEvent::GetScript => "ManageSieve GET script command", ManageSieveEvent::DeleteScript => "ManageSieve DELETE script command", ManageSieveEvent::RenameScript => "ManageSieve RENAME script command", ManageSieveEvent::CheckScript => "ManageSieve CHECK script command", ManageSieveEvent::HaveSpace => "ManageSieve HAVESPACE command", ManageSieveEvent::ListScripts => "ManageSieve LIST scripts command", ManageSieveEvent::SetActive => "ManageSieve SET ACTIVE command", ManageSieveEvent::Capabilities => "ManageSieve CAPABILITIES command", ManageSieveEvent::StartTls => "ManageSieve STARTTLS command", ManageSieveEvent::Unauthenticate => "ManageSieve UNAUTHENTICATE command", ManageSieveEvent::Logout => "ManageSieve LOGOUT command", ManageSieveEvent::Noop => "ManageSieve NOOP command", ManageSieveEvent::Error => "ManageSieve error occurred", ManageSieveEvent::RawInput => "Raw ManageSieve input received", ManageSieveEvent::RawOutput => "Raw ManageSieve output sent", ManageSieveEvent::ConnectionStart => "ManageSieve connection started", ManageSieveEvent::ConnectionEnd => "ManageSieve connection ended", } } pub fn explain(&self) -> &'static str { match self { ManageSieveEvent::CreateScript => "Client created a script", ManageSieveEvent::UpdateScript => "Client updated a script", ManageSieveEvent::GetScript => "Client fetched a script", ManageSieveEvent::DeleteScript => "Client deleted a script", ManageSieveEvent::RenameScript => "Client renamed a script", ManageSieveEvent::CheckScript => "Client checked a script", ManageSieveEvent::HaveSpace => "Client checked for space", ManageSieveEvent::ListScripts => "Client listed scripts", ManageSieveEvent::SetActive => "Client set an active script", ManageSieveEvent::Capabilities => "Client requested server capabilities", ManageSieveEvent::StartTls => "Client requested TLS", ManageSieveEvent::Unauthenticate => "Client unauthenticated", ManageSieveEvent::Logout => "Client logged out", ManageSieveEvent::Noop => "Client sent a NOOP command", ManageSieveEvent::Error => "An error occurred during a ManageSieve command", ManageSieveEvent::RawInput => "Raw ManageSieve input received", ManageSieveEvent::RawOutput => "Raw ManageSieve output sent", ManageSieveEvent::ConnectionStart => "ManageSieve connection started", ManageSieveEvent::ConnectionEnd => "ManageSieve connection ended", } } } impl SmtpEvent { pub fn description(&self) -> &'static str { match self { SmtpEvent::Error => "SMTP error occurred", SmtpEvent::IdNotFound => "Strategy not found", SmtpEvent::ConcurrencyLimitExceeded => "Concurrency limit exceeded", SmtpEvent::TransferLimitExceeded => "Transfer limit exceeded", SmtpEvent::RateLimitExceeded => "Rate limit exceeded", SmtpEvent::TimeLimitExceeded => "Time limit exceeded", SmtpEvent::MissingAuthDirectory => "Missing auth directory", SmtpEvent::MessageParseFailed => "Message parsing failed", SmtpEvent::MessageTooLarge => "Message too large", SmtpEvent::LoopDetected => "Mail loop detected", SmtpEvent::DkimPass => "DKIM verification passed", SmtpEvent::DkimFail => "DKIM verification failed", SmtpEvent::ArcPass => "ARC verification passed", SmtpEvent::ArcFail => "ARC verification failed", SmtpEvent::SpfEhloPass => "SPF EHLO check passed", SmtpEvent::SpfEhloFail => "SPF EHLO check failed", SmtpEvent::SpfFromPass => "SPF From check passed", SmtpEvent::SpfFromFail => "SPF From check failed", SmtpEvent::DmarcPass => "DMARC check passed", SmtpEvent::DmarcFail => "DMARC check failed", SmtpEvent::IprevPass => "IPREV check passed", SmtpEvent::IprevFail => "IPREV check failed", SmtpEvent::TooManyMessages => "Too many messages", SmtpEvent::Ehlo => "SMTP EHLO command", SmtpEvent::InvalidEhlo => "Invalid EHLO command", SmtpEvent::DidNotSayEhlo => "Client did not say EHLO", SmtpEvent::EhloExpected => "EHLO command expected", SmtpEvent::LhloExpected => "LHLO command expected", SmtpEvent::MailFromUnauthenticated => "MAIL FROM without authentication", SmtpEvent::MailFromUnauthorized => "MAIL FROM unauthorized", SmtpEvent::MailFromRewritten => "MAIL FROM address rewritten", SmtpEvent::MailFromMissing => "MAIL FROM address missing", SmtpEvent::MailFromNotAllowed => "MAIL FROM not allowed", SmtpEvent::MailFrom => "SMTP MAIL FROM command", SmtpEvent::MultipleMailFrom => "Multiple MAIL FROM commands", SmtpEvent::MailboxDoesNotExist => "Mailbox does not exist", SmtpEvent::RelayNotAllowed => "Relay not allowed", SmtpEvent::RcptTo => "SMTP RCPT TO command", SmtpEvent::RcptToDuplicate => "Duplicate RCPT TO", SmtpEvent::RcptToRewritten => "RCPT TO address rewritten", SmtpEvent::RcptToMissing => "RCPT TO address missing", SmtpEvent::RcptToGreylisted => "RCPT TO greylisted", SmtpEvent::TooManyRecipients => "Too many recipients", SmtpEvent::TooManyInvalidRcpt => "Too many invalid recipients", SmtpEvent::RawInput => "Raw SMTP input received", SmtpEvent::RawOutput => "Raw SMTP output sent", SmtpEvent::MissingLocalHostname => "Missing local hostname", SmtpEvent::Vrfy => "SMTP VRFY command", SmtpEvent::VrfyNotFound => "VRFY address not found", SmtpEvent::VrfyDisabled => "VRFY command disabled", SmtpEvent::Expn => "SMTP EXPN command", SmtpEvent::ExpnNotFound => "EXPN address not found", SmtpEvent::ExpnDisabled => "EXPN command disabled", SmtpEvent::RequireTlsDisabled => "REQUIRETLS extension disabled", SmtpEvent::DeliverByDisabled => "DELIVERBY extension disabled", SmtpEvent::DeliverByInvalid => "Invalid DELIVERBY parameter", SmtpEvent::FutureReleaseDisabled => "FUTURE RELEASE extension disabled", SmtpEvent::FutureReleaseInvalid => "Invalid FUTURE RELEASE parameter", SmtpEvent::MtPriorityDisabled => "MT-PRIORITY extension disabled", SmtpEvent::MtPriorityInvalid => "Invalid MT-PRIORITY parameter", SmtpEvent::DsnDisabled => "DSN extension disabled", SmtpEvent::AuthNotAllowed => "Authentication not allowed", SmtpEvent::AuthMechanismNotSupported => "Auth mechanism not supported", SmtpEvent::AuthExchangeTooLong => "Auth exchange too long", SmtpEvent::AlreadyAuthenticated => "Already authenticated", SmtpEvent::Noop => "SMTP NOOP command", SmtpEvent::StartTls => "SMTP STARTTLS command", SmtpEvent::StartTlsUnavailable => "STARTTLS unavailable", SmtpEvent::StartTlsAlready => "TLS already active", SmtpEvent::Rset => "SMTP RSET command", SmtpEvent::Quit => "SMTP QUIT command", SmtpEvent::Help => "SMTP HELP command", SmtpEvent::CommandNotImplemented => "Command not implemented", SmtpEvent::InvalidCommand => "Invalid command", SmtpEvent::InvalidSenderAddress => "Invalid sender address", SmtpEvent::InvalidRecipientAddress => "Invalid recipient address", SmtpEvent::InvalidParameter => "Invalid parameter", SmtpEvent::UnsupportedParameter => "Unsupported parameter", SmtpEvent::SyntaxError => "Syntax error", SmtpEvent::RequestTooLarge => "Request too large", SmtpEvent::ConnectionStart => "SMTP connection started", SmtpEvent::ConnectionEnd => "SMTP connection ended", } } pub fn explain(&self) -> &'static str { match self { SmtpEvent::Error => "An error occurred during an SMTP command", SmtpEvent::IdNotFound => "The strategy ID was not found in the configuration", SmtpEvent::ConcurrencyLimitExceeded => "The concurrency limit was exceeded", SmtpEvent::TransferLimitExceeded => { "The remote host transferred more data than allowed" } SmtpEvent::RateLimitExceeded => "The rate limit was exceeded", SmtpEvent::TimeLimitExceeded => "The remote host kept the SMTP session open too long", SmtpEvent::MissingAuthDirectory => "The auth directory was missing", SmtpEvent::MessageParseFailed => "Failed to parse the message", SmtpEvent::MessageTooLarge => "The message was rejected because it was too large", SmtpEvent::LoopDetected => { "A mail loop was detected, the message contains too many Received headers" } SmtpEvent::DkimPass => "Successful DKIM verification", SmtpEvent::DkimFail => "Failed to verify DKIM signature", SmtpEvent::ArcPass => "Successful ARC verification", SmtpEvent::ArcFail => "Failed to verify ARC signature", SmtpEvent::SpfEhloPass => "EHLO identity passed SPF check", SmtpEvent::SpfEhloFail => "EHLO identity failed SPF check", SmtpEvent::SpfFromPass => "MAIL FROM identity passed SPF check", SmtpEvent::SpfFromFail => "MAIL FROM identity failed SPF check", SmtpEvent::DmarcPass => "Successful DMARC verification", SmtpEvent::DmarcFail => "Failed to verify DMARC policy", SmtpEvent::IprevPass => "Reverse IP check passed", SmtpEvent::IprevFail => "Reverse IP check failed", SmtpEvent::TooManyMessages => { "The remote server exceeded the number of messages allowed per session" } SmtpEvent::Ehlo => "The remote server sent an EHLO command", SmtpEvent::InvalidEhlo => "The remote server sent an invalid EHLO command", SmtpEvent::DidNotSayEhlo => "The remote server did not send EHLO command", SmtpEvent::EhloExpected => { "The remote server sent a LHLO command while EHLO was expected" } SmtpEvent::LhloExpected => { "The remote server sent an EHLO command while LHLO was expected" } SmtpEvent::MailFromUnauthenticated => { "The remote client did not authenticate before sending MAIL FROM" } SmtpEvent::MailFromUnauthorized => { "The remote client is not authorized to send mail from the given address" } SmtpEvent::MailFromRewritten => "The envelope sender address was rewritten", SmtpEvent::MailFromMissing => { "The remote client issued an RCPT TO command before MAIL FROM" } SmtpEvent::MailFromNotAllowed => { "The remote client is not allowed to send mail from this address" } SmtpEvent::MailFrom => "The remote client sent a MAIL FROM command", SmtpEvent::MultipleMailFrom => "The remote client already sent a MAIL FROM command", SmtpEvent::MailboxDoesNotExist => "The mailbox does not exist on the server", SmtpEvent::RelayNotAllowed => "The server does not allow relaying", SmtpEvent::RcptTo => "The remote client sent an RCPT TO command", SmtpEvent::RcptToDuplicate => { "The remote client already sent an RCPT TO command for this recipient" } SmtpEvent::RcptToRewritten => "The envelope recipient address was rewritten", SmtpEvent::RcptToMissing => "The remote client issued a DATA command before RCPT TO", SmtpEvent::RcptToGreylisted => "The recipient was greylisted", SmtpEvent::TooManyRecipients => { "The remote client exceeded the number of recipients allowed" } SmtpEvent::TooManyInvalidRcpt => { "The remote client exceeded the number of invalid RCPT TO commands allowed" } SmtpEvent::RawInput => "Raw SMTP input received", SmtpEvent::RawOutput => "Raw SMTP output sent", SmtpEvent::MissingLocalHostname => "The local hostname is missing in the configuration", SmtpEvent::Vrfy => "The remote client sent a VRFY command", SmtpEvent::VrfyNotFound => { "The remote client sent a VRFY command for an address that was not found" } SmtpEvent::VrfyDisabled => "The VRFY command is disabled", SmtpEvent::Expn => "The remote client sent an EXPN command", SmtpEvent::ExpnNotFound => { "The remote client sent an EXPN command for an address that was not found" } SmtpEvent::ExpnDisabled => "The EXPN command is disabled", SmtpEvent::RequireTlsDisabled => "The REQUIRETLS extension is disabled", SmtpEvent::DeliverByDisabled => "The DELIVERBY extension is disabled", SmtpEvent::DeliverByInvalid => "The DELIVERBY parameter is invalid", SmtpEvent::FutureReleaseDisabled => "The FUTURE RELEASE extension is disabled", SmtpEvent::FutureReleaseInvalid => "The FUTURE RELEASE parameter is invalid", SmtpEvent::MtPriorityDisabled => "The MT-PRIORITY extension is disabled", SmtpEvent::MtPriorityInvalid => "The MT-PRIORITY parameter is invalid", SmtpEvent::DsnDisabled => "The DSN extension is disabled", SmtpEvent::AuthNotAllowed => "Authentication is not allowed on this listener", SmtpEvent::AuthMechanismNotSupported => { "The requested authentication mechanism is not supported" } SmtpEvent::AuthExchangeTooLong => "The authentication exchange was too long", SmtpEvent::AlreadyAuthenticated => "The client is already authenticated", SmtpEvent::Noop => "The remote client sent a NOOP command", SmtpEvent::StartTls => "The remote client requested a TLS connection", SmtpEvent::StartTlsUnavailable => { "The remote client requested a TLS connection but it is not available" } SmtpEvent::Rset => "The remote client sent a RSET command", SmtpEvent::Quit => "The remote client sent a QUIT command", SmtpEvent::Help => "The remote client sent a HELP command", SmtpEvent::CommandNotImplemented => { "The server does not implement the requested command" } SmtpEvent::InvalidCommand => "The remote client sent an invalid command", SmtpEvent::InvalidSenderAddress => "The specified sender address is invalid", SmtpEvent::InvalidRecipientAddress => "The specified recipient address is invalid", SmtpEvent::InvalidParameter => "The command contained an invalid parameter", SmtpEvent::UnsupportedParameter => "The command contained an unsupported parameter", SmtpEvent::SyntaxError => "The command contained a syntax error", SmtpEvent::RequestTooLarge => "The request was too large", SmtpEvent::ConnectionStart => "A new SMTP connection was started", SmtpEvent::ConnectionEnd => "The SMTP connection was ended", SmtpEvent::StartTlsAlready => "TLS is already active", } } } impl DeliveryEvent { pub fn description(&self) -> &'static str { match self { DeliveryEvent::AttemptStart => "Delivery attempt started", DeliveryEvent::AttemptEnd => "Delivery attempt ended", DeliveryEvent::Completed => "Delivery completed", DeliveryEvent::Failed => "Delivery failed", DeliveryEvent::DomainDeliveryStart => "New delivery attempt for domain", DeliveryEvent::MxLookup => "MX record lookup", DeliveryEvent::MxLookupFailed => "MX record lookup failed", DeliveryEvent::IpLookup => "IP address lookup", DeliveryEvent::IpLookupFailed => "IP address lookup failed", DeliveryEvent::NullMx => "Null MX record found", DeliveryEvent::Connect => "Connecting to remote server", DeliveryEvent::ConnectError => "Connection error", DeliveryEvent::MissingOutboundHostname => "Missing outbound hostname in configuration", DeliveryEvent::GreetingFailed => "SMTP greeting failed", DeliveryEvent::Ehlo => "SMTP EHLO command", DeliveryEvent::EhloRejected => "SMTP EHLO rejected", DeliveryEvent::Auth => "SMTP authentication", DeliveryEvent::AuthFailed => "SMTP authentication failed", DeliveryEvent::MailFrom => "SMTP MAIL FROM command", DeliveryEvent::MailFromRejected => "SMTP MAIL FROM rejected", DeliveryEvent::Delivered => "Message delivered", DeliveryEvent::RcptTo => "SMTP RCPT TO command", DeliveryEvent::RcptToRejected => "SMTP RCPT TO rejected", DeliveryEvent::RcptToFailed => "SMTP RCPT TO failed", DeliveryEvent::MessageRejected => "Message rejected by remote server", DeliveryEvent::StartTls => "SMTP STARTTLS command", DeliveryEvent::StartTlsUnavailable => "STARTTLS unavailable", DeliveryEvent::StartTlsError => "STARTTLS error", DeliveryEvent::StartTlsDisabled => "STARTTLS disabled", DeliveryEvent::ImplicitTlsError => "Implicit TLS error", DeliveryEvent::ConcurrencyLimitExceeded => "Concurrency limit exceeded", DeliveryEvent::RateLimitExceeded => "Rate limit exceeded", DeliveryEvent::DoubleBounce => "Discarding message after double bounce", DeliveryEvent::DsnSuccess => "DSN success notification", DeliveryEvent::DsnTempFail => "DSN temporary failure notification", DeliveryEvent::DsnPermFail => "DSN permanent failure notification", DeliveryEvent::RawInput => "Raw SMTP input received", DeliveryEvent::RawOutput => "Raw SMTP output sent", } } pub fn explain(&self) -> &'static str { match self { DeliveryEvent::AttemptStart => "A new delivery attempt for the message has started", DeliveryEvent::AttemptEnd => "The delivery attempt has ended", DeliveryEvent::Completed => "Delivery was completed for all recipients", DeliveryEvent::Failed => "Message delivery failed due to a temporary error", DeliveryEvent::DomainDeliveryStart => "A new delivery attempt for a domain has started", DeliveryEvent::MxLookup => "Looking up MX records for the domain", DeliveryEvent::MxLookupFailed => "Failed to look up MX records for the domain", DeliveryEvent::IpLookup => "Looking up IP address for the domain", DeliveryEvent::IpLookupFailed => "Failed to look up IP address for the domain", DeliveryEvent::NullMx => "The domain has a null MX record, delivery is impossible", DeliveryEvent::Connect => "Connecting to the remote server", DeliveryEvent::ConnectError => "Error connecting to the remote server", DeliveryEvent::MissingOutboundHostname => { "The outbound hostname is missing in the configuration" } DeliveryEvent::GreetingFailed => { "Failed to read the SMTP greeting from the remote server" } DeliveryEvent::Ehlo => "The EHLO command was sent to the remote server", DeliveryEvent::EhloRejected => "The remote server rejected the EHLO command", DeliveryEvent::Auth => "Authenticating with the remote server", DeliveryEvent::AuthFailed => "Authentication with the remote server failed", DeliveryEvent::MailFrom => "The MAIL FROM command was sent to the remote server", DeliveryEvent::MailFromRejected => "The remote server rejected the MAIL FROM command", DeliveryEvent::Delivered => "The message was delivered to the recipient", DeliveryEvent::RcptTo => "The RCPT TO command was sent to the remote server", DeliveryEvent::RcptToRejected => "The remote server rejected the RCPT TO command", DeliveryEvent::RcptToFailed => { "Failed to send the RCPT TO command to the remote server" } DeliveryEvent::MessageRejected => "The remote server rejected the message", DeliveryEvent::StartTls => "Requesting a TLS connection with the remote server", DeliveryEvent::StartTlsUnavailable => "The remote server does not support STARTTLS", DeliveryEvent::StartTlsError => "It was not possible to establish a TLS connection", DeliveryEvent::StartTlsDisabled => { "STARTTLS has been disabled in the configuration for this host" } DeliveryEvent::ImplicitTlsError => "Error starting implicit TLS", DeliveryEvent::ConcurrencyLimitExceeded => { "The concurrency limit was exceeded for the remote host" } DeliveryEvent::RateLimitExceeded => "The rate limit was exceeded for the remote host", DeliveryEvent::DoubleBounce => "The message was discarded after a double bounce", DeliveryEvent::DsnSuccess => "A success delivery status notification was created", DeliveryEvent::DsnTempFail => { "A temporary failure delivery status notification was created" } DeliveryEvent::DsnPermFail => { "A permanent failure delivery status notification was created" } DeliveryEvent::RawInput => "Raw SMTP input received", DeliveryEvent::RawOutput => "Raw SMTP output sent", } } } impl QueueEvent { pub fn description(&self) -> &'static str { match self { QueueEvent::Rescheduled => "Message rescheduled for delivery", QueueEvent::Locked => "Queue event is locked by another process", QueueEvent::BlobNotFound => "Message blob not found", QueueEvent::RateLimitExceeded => "Rate limit exceeded", QueueEvent::ConcurrencyLimitExceeded => "Concurrency limit exceeded", QueueEvent::QuotaExceeded => "Quota exceeded", QueueEvent::QueueMessage => "Queued message for delivery", QueueEvent::QueueMessageAuthenticated => "Queued message submission for delivery", QueueEvent::QueueReport => "Queued report for delivery", QueueEvent::QueueDsn => "Queued DSN for delivery", QueueEvent::QueueAutogenerated => "Queued autogenerated message for delivery", QueueEvent::BackPressure => "Queue backpressure detected", } } pub fn explain(&self) -> &'static str { match self { QueueEvent::Rescheduled => "The message was rescheduled for delivery", QueueEvent::Locked => "The queue event is locked by another process", QueueEvent::BlobNotFound => "The message blob was not found", QueueEvent::RateLimitExceeded => "The queue rate limit was exceeded", QueueEvent::ConcurrencyLimitExceeded => "The queue concurrency limit was exceeded", QueueEvent::QuotaExceeded => "The queue quota was exceeded", QueueEvent::QueueMessage => "A new message was queued for delivery", QueueEvent::QueueMessageAuthenticated => { "A new message was queued for delivery from an authenticated client" } QueueEvent::QueueReport => "A new report was queued for delivery", QueueEvent::QueueDsn => "A delivery status notification was queued for delivery", QueueEvent::QueueAutogenerated => "A system generated message was queued for delivery", QueueEvent::BackPressure => { "Queue congested, processing can't keep up with incoming message rate" } } } } impl IncomingReportEvent { pub fn description(&self) -> &'static str { match self { IncomingReportEvent::DmarcReport => "DMARC report received", IncomingReportEvent::DmarcReportWithWarnings => "DMARC report received with warnings", IncomingReportEvent::TlsReport => "TLS report received", IncomingReportEvent::TlsReportWithWarnings => "TLS report received with warnings", IncomingReportEvent::AbuseReport => "Abuse report received", IncomingReportEvent::AuthFailureReport => "Authentication failure report received", IncomingReportEvent::FraudReport => "Fraud report received", IncomingReportEvent::NotSpamReport => "Not spam report received", IncomingReportEvent::VirusReport => "Virus report received", IncomingReportEvent::OtherReport => "Other type of report received", IncomingReportEvent::MessageParseFailed => "Failed to parse incoming report message", IncomingReportEvent::DmarcParseFailed => "Failed to parse DMARC report", IncomingReportEvent::TlsRpcParseFailed => "Failed to parse TLS RPC report", IncomingReportEvent::ArfParseFailed => "Failed to parse ARF report", IncomingReportEvent::DecompressError => "Error decompressing report", } } pub fn explain(&self) -> &'static str { match self { IncomingReportEvent::DmarcReport => "A DMARC report has been received", IncomingReportEvent::DmarcReportWithWarnings => { "A DMARC report with warnings has been received" } IncomingReportEvent::TlsReport => "A TLS report has been received", IncomingReportEvent::TlsReportWithWarnings => { "A TLS report with warnings has been received" } IncomingReportEvent::AbuseReport => "An abuse report has been received", IncomingReportEvent::AuthFailureReport => { "An authentication failure report has been received" } IncomingReportEvent::FraudReport => "A fraud report has been received", IncomingReportEvent::NotSpamReport => "A not spam report has been received", IncomingReportEvent::VirusReport => "A virus report has been received", IncomingReportEvent::OtherReport => "An unknown type of report has been received", IncomingReportEvent::MessageParseFailed => { "Failed to parse the incoming report message" } IncomingReportEvent::DmarcParseFailed => "Failed to parse the DMARC report", IncomingReportEvent::TlsRpcParseFailed => "Failed to parse the TLS RPC report", IncomingReportEvent::ArfParseFailed => "Failed to parse the ARF report", IncomingReportEvent::DecompressError => "Error decompressing the report", } } } impl OutgoingReportEvent { pub fn description(&self) -> &'static str { match self { OutgoingReportEvent::SpfReport => "SPF report sent", OutgoingReportEvent::SpfRateLimited => "SPF report rate limited", OutgoingReportEvent::DkimReport => "DKIM report sent", OutgoingReportEvent::DkimRateLimited => "DKIM report rate limited", OutgoingReportEvent::DmarcReport => "DMARC report sent", OutgoingReportEvent::DmarcRateLimited => "DMARC report rate limited", OutgoingReportEvent::DmarcAggregateReport => "DMARC aggregate is being prepared", OutgoingReportEvent::TlsAggregate => "TLS aggregate report is being prepared", OutgoingReportEvent::HttpSubmission => "Report submitted via HTTP", OutgoingReportEvent::UnauthorizedReportingAddress => "Unauthorized reporting address", OutgoingReportEvent::ReportingAddressValidationError => { "Error validating reporting address" } OutgoingReportEvent::NotFound => "Report not found", OutgoingReportEvent::SubmissionError => "Error submitting report", OutgoingReportEvent::NoRecipientsFound => "No recipients found for report", OutgoingReportEvent::Locked => "Report is locked by another process", } } pub fn explain(&self) -> &'static str { match self { OutgoingReportEvent::SpfReport => "An SPF report has been sent", OutgoingReportEvent::SpfRateLimited => "The SPF report was rate limited", OutgoingReportEvent::DkimReport => "A DKIM report has been sent", OutgoingReportEvent::DkimRateLimited => "The DKIM report was rate limited", OutgoingReportEvent::DmarcReport => "A DMARC report has been sent", OutgoingReportEvent::DmarcRateLimited => "The DMARC report was rate limited", OutgoingReportEvent::DmarcAggregateReport => "A DMARC aggregate report will be sent", OutgoingReportEvent::TlsAggregate => "A TLS aggregate report will be sent", OutgoingReportEvent::HttpSubmission => "The report was submitted via HTTP", OutgoingReportEvent::UnauthorizedReportingAddress => { "The reporting address is not authorized to send reports" } OutgoingReportEvent::ReportingAddressValidationError => { "Error validating the reporting address" } OutgoingReportEvent::NotFound => "The report was not found", OutgoingReportEvent::SubmissionError => "Error submitting the report", OutgoingReportEvent::NoRecipientsFound => "No recipients found for the report", OutgoingReportEvent::Locked => "The report is locked by another process", } } } impl MtaStsEvent { pub fn description(&self) -> &'static str { match self { MtaStsEvent::Authorized => "Host authorized by MTA-STS policy", MtaStsEvent::NotAuthorized => "Host not authorized by MTA-STS policy", MtaStsEvent::PolicyFetch => "Fetched MTA-STS policy", MtaStsEvent::PolicyNotFound => "MTA-STS policy not found", MtaStsEvent::PolicyFetchError => "Error fetching MTA-STS policy", MtaStsEvent::InvalidPolicy => "Invalid MTA-STS policy", } } pub fn explain(&self) -> &'static str { match self { MtaStsEvent::Authorized => "The host is authorized by the MTA-STS policy", MtaStsEvent::NotAuthorized => "The host is not authorized by the MTA-STS policy", MtaStsEvent::PolicyFetch => "The MTA-STS policy has been fetched", MtaStsEvent::PolicyNotFound => "An MTA-STS policy was not found", MtaStsEvent::PolicyFetchError => "An error occurred while fetching the MTA-STS policy", MtaStsEvent::InvalidPolicy => "The MTA-STS policy is invalid", } } } impl TlsRptEvent { pub fn description(&self) -> &'static str { match self { TlsRptEvent::RecordFetch => "Fetched TLS-RPT record", TlsRptEvent::RecordFetchError => "Error fetching TLS-RPT record", TlsRptEvent::RecordNotFound => "TLS-RPT record not found", } } pub fn explain(&self) -> &'static str { match self { TlsRptEvent::RecordFetch => "The TLS-RPT record has been fetched", TlsRptEvent::RecordFetchError => "An error occurred while fetching the TLS-RPT record", TlsRptEvent::RecordNotFound => "No TLS-RPT records were found", } } } impl DaneEvent { pub fn description(&self) -> &'static str { match self { DaneEvent::AuthenticationSuccess => "DANE authentication successful", DaneEvent::AuthenticationFailure => "DANE authentication failed", DaneEvent::NoCertificatesFound => "No certificates found for DANE", DaneEvent::CertificateParseError => "Error parsing certificate for DANE", DaneEvent::TlsaRecordMatch => "TLSA record match found", DaneEvent::TlsaRecordFetch => "Fetching TLSA record", DaneEvent::TlsaRecordFetchError => "Error fetching TLSA record", DaneEvent::TlsaRecordNotFound => "TLSA record not found", DaneEvent::TlsaRecordNotDnssecSigned => "TLSA record not DNSSEC signed", DaneEvent::TlsaRecordInvalid => "Invalid TLSA record", } } pub fn explain(&self) -> &'static str { match self { DaneEvent::AuthenticationSuccess => "Successful DANE authentication", DaneEvent::AuthenticationFailure => "Failed DANE authentication", DaneEvent::NoCertificatesFound => "No certificates were found for DANE", DaneEvent::CertificateParseError => "An error occurred while parsing the certificate", DaneEvent::TlsaRecordMatch => "A TLSA record match was found", DaneEvent::TlsaRecordFetch => "The TLSA record has been fetched", DaneEvent::TlsaRecordFetchError => "An error occurred while fetching the TLSA record", DaneEvent::TlsaRecordNotFound => "The TLSA record was not found", DaneEvent::TlsaRecordNotDnssecSigned => "The TLSA record is not DNSSEC signed", DaneEvent::TlsaRecordInvalid => "The TLSA record is invalid", } } } impl MilterEvent { pub fn description(&self) -> &'static str { match self { MilterEvent::Read => "Reading from Milter", MilterEvent::Write => "Writing to Milter", MilterEvent::ActionAccept => "Milter action: Accept", MilterEvent::ActionDiscard => "Milter action: Discard", MilterEvent::ActionReject => "Milter action: Reject", MilterEvent::ActionTempFail => "Milter action: Temporary failure", MilterEvent::ActionReplyCode => "Milter action: Reply code", MilterEvent::ActionConnectionFailure => "Milter action: Connection failure", MilterEvent::ActionShutdown => "Milter action: Shutdown", MilterEvent::IoError => "Milter I/O error", MilterEvent::FrameTooLarge => "Milter frame too large", MilterEvent::FrameInvalid => "Invalid Milter frame", MilterEvent::UnexpectedResponse => "Unexpected Milter response", MilterEvent::Timeout => "Milter timeout", MilterEvent::TlsInvalidName => "Invalid TLS name for Milter", MilterEvent::Disconnected => "Milter disconnected", MilterEvent::ParseError => "Milter parse error", } } pub fn explain(&self) -> &'static str { match self { MilterEvent::Read => "Reading from the Milter", MilterEvent::Write => "Writing to the Milter", MilterEvent::ActionAccept => "The Milter requested to accept the message", MilterEvent::ActionDiscard => "The Milter requested to discard the message", MilterEvent::ActionReject => "The Milter requested to reject the message", MilterEvent::ActionTempFail => "The Milter requested to temporarily fail the message", MilterEvent::ActionReplyCode => "The Milter requested a reply code", MilterEvent::ActionConnectionFailure => "The Milter requested a connection failure", MilterEvent::ActionShutdown => "The Milter requested a shutdown", MilterEvent::IoError => "An I/O error occurred with the Milter", MilterEvent::FrameTooLarge => "The Milter frame was too large", MilterEvent::FrameInvalid => "The Milter frame was invalid", MilterEvent::UnexpectedResponse => { "An unexpected response was received from the Milter" } MilterEvent::Timeout => "A timeout occurred with the Milter", MilterEvent::TlsInvalidName => "The Milter TLS name is invalid", MilterEvent::Disconnected => "The Milter disconnected", MilterEvent::ParseError => "An error occurred while parsing the Milter response", } } } impl MtaHookEvent { pub fn description(&self) -> &'static str { match self { MtaHookEvent::ActionAccept => "MTA hook action: Accept", MtaHookEvent::ActionDiscard => "MTA hook action: Discard", MtaHookEvent::ActionReject => "MTA hook action: Reject", MtaHookEvent::ActionQuarantine => "MTA hook action: Quarantine", MtaHookEvent::Error => "MTA hook error", } } pub fn explain(&self) -> &'static str { match self { MtaHookEvent::ActionAccept => "The MTA hook requested to accept the message", MtaHookEvent::ActionDiscard => "The MTA hook requested to discard the message", MtaHookEvent::ActionReject => "The MTA hook requested to reject the message", MtaHookEvent::ActionQuarantine => "The MTA hook requested to quarantine the message", MtaHookEvent::Error => "An error occurred with the MTA hook", } } } impl PushSubscriptionEvent { pub fn description(&self) -> &'static str { match self { PushSubscriptionEvent::Success => "Push subscription successful", PushSubscriptionEvent::Error => "Push subscription error", PushSubscriptionEvent::NotFound => "Push subscription not found", } } pub fn explain(&self) -> &'static str { match self { PushSubscriptionEvent::Success => "The push subscription was successful", PushSubscriptionEvent::Error => "An error occurred with the push subscription", PushSubscriptionEvent::NotFound => "The push subscription was not found", } } } impl SpamEvent { pub fn description(&self) -> &'static str { match self { SpamEvent::Pyzor => "Pyzor success", SpamEvent::PyzorError => "Pyzor error", SpamEvent::Classify => "Classifying message for spam", SpamEvent::Dnsbl => "DNSBL query", SpamEvent::DnsblError => "Error querying DNSBL", SpamEvent::TrainStarted => "Spam classifier training started", SpamEvent::TrainCompleted => "Spam classifier training completed", SpamEvent::TrainSampleAdded => "New training sample added", SpamEvent::TrainSampleNotFound => "Training sample not found", SpamEvent::ModelLoaded => "Spam classifier model loaded", SpamEvent::ModelNotReady => "Spam classifier model not ready", SpamEvent::ModelNotFound => "Spam classifier model not found", } } pub fn explain(&self) -> &'static str { match self { SpamEvent::PyzorError => "An error occurred with Pyzor", SpamEvent::Classify => "The message is being classified for spam", SpamEvent::Pyzor => "Pyzor query successful", SpamEvent::Dnsbl => "The DNSBL query was successful", SpamEvent::DnsblError => "An error occurred while querying the DNSBL", SpamEvent::TrainStarted => "SGD logistic regression training has started", SpamEvent::TrainCompleted => "SGD logistic regression training has completed", SpamEvent::TrainSampleAdded => "A new training sample has been added", SpamEvent::TrainSampleNotFound => "A training sample was not found", SpamEvent::ModelLoaded => "The spam classifier model has been loaded", SpamEvent::ModelNotReady => { "The spam classifier model has not been trained with enough data" } SpamEvent::ModelNotFound => "The spam classifier model has not been trained yet", } } } impl SieveEvent { pub fn description(&self) -> &'static str { match self { SieveEvent::ActionAccept => "Sieve action: Accept", SieveEvent::ActionAcceptReplace => "Sieve action: Accept and replace", SieveEvent::ActionDiscard => "Sieve action: Discard", SieveEvent::ActionReject => "Sieve action: Reject", SieveEvent::SendMessage => "Sieve sending message", SieveEvent::MessageTooLarge => "Sieve message too large", SieveEvent::ScriptNotFound => "Sieve script not found", SieveEvent::ListNotFound => "Sieve list not found", SieveEvent::RuntimeError => "Sieve runtime error", SieveEvent::UnexpectedError => "Unexpected Sieve error", SieveEvent::NotSupported => "Sieve action not supported", SieveEvent::QuotaExceeded => "Sieve quota exceeded", } } pub fn explain(&self) -> &'static str { match self { SieveEvent::ActionAccept => "The Sieve script requested to accept the message", SieveEvent::ActionAcceptReplace => { "The Sieve script requested to accept the message and replace its contents" } SieveEvent::ActionDiscard => "The Sieve script requested to discard the message", SieveEvent::ActionReject => "The Sieve script requested to reject the message", SieveEvent::SendMessage => "The Sieve script is sending a message", SieveEvent::MessageTooLarge => "The Sieve message is too large", SieveEvent::ScriptNotFound => "The Sieve script was not found", SieveEvent::ListNotFound => "The Sieve list was not found", SieveEvent::RuntimeError => "A runtime error occurred with the Sieve script", SieveEvent::UnexpectedError => "An unexpected error occurred with the Sieve script", SieveEvent::NotSupported => "The Sieve action is not supported", SieveEvent::QuotaExceeded => "The Sieve quota was exceeded", } } } impl TlsEvent { pub fn description(&self) -> &'static str { match self { TlsEvent::Handshake => "TLS handshake", TlsEvent::HandshakeError => "TLS handshake error", TlsEvent::NotConfigured => "TLS not configured", TlsEvent::CertificateNotFound => "TLS certificate not found", TlsEvent::NoCertificatesAvailable => "No TLS certificates available", TlsEvent::MultipleCertificatesAvailable => "Multiple TLS certificates available", } } pub fn explain(&self) -> &'static str { match self { TlsEvent::Handshake => "Successful TLS handshake", TlsEvent::HandshakeError => "An error occurred during the TLS handshake", TlsEvent::NotConfigured => "TLS is not configured", TlsEvent::CertificateNotFound => "The TLS certificate was not found", TlsEvent::NoCertificatesAvailable => "No TLS certificates are available", TlsEvent::MultipleCertificatesAvailable => "Multiple TLS certificates are available", } } } impl NetworkEvent { pub fn description(&self) -> &'static str { match self { NetworkEvent::ListenStart => "Network listener started", NetworkEvent::ListenStop => "Network listener stopped", NetworkEvent::ListenError => "Network listener error", NetworkEvent::BindError => "Network bind error", NetworkEvent::ReadError => "Network read error", NetworkEvent::WriteError => "Network write error", NetworkEvent::FlushError => "Network flush error", NetworkEvent::AcceptError => "Network accept error", NetworkEvent::SplitError => "Network split error", NetworkEvent::Timeout => "Network timeout", NetworkEvent::Closed => "Network connection closed", NetworkEvent::ProxyError => "Proxy protocol error", NetworkEvent::SetOptError => "Network set option error", } } pub fn explain(&self) -> &'static str { match self { NetworkEvent::ListenStart => "The network listener has started", NetworkEvent::ListenStop => "The network listener has stopped", NetworkEvent::ListenError => "An error occurred with the network listener", NetworkEvent::BindError => "An error occurred while binding the network listener", NetworkEvent::ReadError => "An error occurred while reading from the network", NetworkEvent::WriteError => "An error occurred while writing to the network", NetworkEvent::FlushError => "An error occurred while flushing the network", NetworkEvent::AcceptError => "An error occurred while accepting a network connection", NetworkEvent::SplitError => "An error occurred while splitting the network connection", NetworkEvent::Timeout => "A network timeout occurred", NetworkEvent::Closed => "The network connection was closed", NetworkEvent::ProxyError => "An error occurred with the proxy protocol", NetworkEvent::SetOptError => "An error occurred while setting network options", } } } impl ServerEvent { pub fn description(&self) -> &'static str { match self { ServerEvent::Startup => { concat!("Starting Stalwart Server v", env!("CARGO_PKG_VERSION")) } ServerEvent::Shutdown => { concat!("Shutting down Stalwart Server v", env!("CARGO_PKG_VERSION")) } ServerEvent::StartupError => "Server startup error", ServerEvent::ThreadError => "Server thread error", ServerEvent::Licensing => "Server licensing event", } } pub fn explain(&self) -> &'static str { match self { ServerEvent::Startup => "Stalwart Server has started", ServerEvent::Shutdown => "Stalwart Server is shutting down", ServerEvent::StartupError => "An error occurred while starting the server", ServerEvent::ThreadError => "An error occurred with a server thread", ServerEvent::Licensing => "A licensing event occurred", } } } impl TelemetryEvent { pub fn description(&self) -> &'static str { match self { TelemetryEvent::Alert => "Alert triggered", TelemetryEvent::LogError => "Log collector error", TelemetryEvent::WebhookError => "Webhook collector error", TelemetryEvent::JournalError => "Journal collector error", TelemetryEvent::OtelExporterError => "OpenTelemetry exporter error", TelemetryEvent::OtelMetricsExporterError => "OpenTelemetry metrics exporter error", TelemetryEvent::PrometheusExporterError => "Prometheus exporter error", } } pub fn explain(&self) -> &'static str { match self { TelemetryEvent::Alert => "An alert was triggered", TelemetryEvent::LogError => "An error occurred with the log collector", TelemetryEvent::WebhookError => "An error occurred with the webhook collector", TelemetryEvent::JournalError => "An error occurred with the journal collector", TelemetryEvent::OtelExporterError => { "An error occurred with the OpenTelemetry exporter" } TelemetryEvent::OtelMetricsExporterError => { "An error occurred with the OpenTelemetry metrics exporter" } TelemetryEvent::PrometheusExporterError => { "An error occurred with the Prometheus exporter" } } } } impl AcmeEvent { pub fn description(&self) -> &'static str { match self { AcmeEvent::AuthStart => "ACME authentication started", AcmeEvent::AuthPending => "ACME authentication pending", AcmeEvent::AuthValid => "ACME authentication valid", AcmeEvent::AuthCompleted => "ACME authentication completed", AcmeEvent::AuthError => "ACME authentication error", AcmeEvent::AuthTooManyAttempts => "Too many ACME authentication attempts", AcmeEvent::ProcessCert => "Processing ACME certificate", AcmeEvent::OrderStart => "ACME order started", AcmeEvent::OrderProcessing => "ACME order processing", AcmeEvent::OrderCompleted => "ACME order completed", AcmeEvent::OrderReady => "ACME order ready", AcmeEvent::OrderValid => "ACME order valid", AcmeEvent::OrderInvalid => "ACME order invalid", AcmeEvent::RenewBackoff => "ACME renew backoff", AcmeEvent::DnsRecordCreated => "ACME DNS record created", AcmeEvent::DnsRecordCreationFailed => "ACME DNS record creation failed", AcmeEvent::DnsRecordDeletionFailed => "ACME DNS record deletion failed", AcmeEvent::DnsRecordNotPropagated => "ACME DNS record not propagated", AcmeEvent::DnsRecordLookupFailed => "ACME DNS record lookup failed", AcmeEvent::DnsRecordPropagated => "ACME DNS record propagated", AcmeEvent::DnsRecordPropagationTimeout => "ACME DNS record propagation timeout", AcmeEvent::ClientSuppliedSni => "ACME client supplied SNI", AcmeEvent::ClientMissingSni => "ACME client missing SNI", AcmeEvent::TlsAlpnReceived => "ACME TLS ALPN received", AcmeEvent::TlsAlpnError => "ACME TLS ALPN error", AcmeEvent::TokenNotFound => "ACME token not found", AcmeEvent::Error => "ACME error", } } pub fn explain(&self) -> &'static str { match self { AcmeEvent::AuthStart => "ACME authentication has started", AcmeEvent::AuthPending => "ACME authentication is pending", AcmeEvent::AuthValid => "ACME authentication is valid", AcmeEvent::AuthCompleted => "ACME authentication has completed", AcmeEvent::AuthError => "An error occurred with ACME authentication", AcmeEvent::AuthTooManyAttempts => "Too many ACME authentication attempts", AcmeEvent::ProcessCert => "Processing the ACME certificate", AcmeEvent::OrderStart => "ACME order has started", AcmeEvent::OrderProcessing => "ACME order is processing", AcmeEvent::OrderCompleted => "ACME order has completed", AcmeEvent::OrderReady => "ACME order is ready", AcmeEvent::OrderValid => "ACME order is valid", AcmeEvent::OrderInvalid => "ACME order is invalid", AcmeEvent::RenewBackoff => "ACME renew backoff", AcmeEvent::DnsRecordCreated => "ACME DNS record has been created", AcmeEvent::DnsRecordCreationFailed => "Failed to create ACME DNS record", AcmeEvent::DnsRecordDeletionFailed => "Failed to delete ACME DNS record", AcmeEvent::DnsRecordNotPropagated => "ACME DNS record has not propagated", AcmeEvent::DnsRecordLookupFailed => "Failed to look up ACME DNS record", AcmeEvent::DnsRecordPropagated => "ACME DNS record has propagated", AcmeEvent::DnsRecordPropagationTimeout => "ACME DNS record propagation timeout", AcmeEvent::ClientSuppliedSni => "ACME client supplied SNI", AcmeEvent::ClientMissingSni => "ACME client missing SNI", AcmeEvent::TlsAlpnReceived => "ACME TLS ALPN received", AcmeEvent::TlsAlpnError => "ACME TLS ALPN error", AcmeEvent::TokenNotFound => "ACME token not found", AcmeEvent::Error => "An error occurred with ACME", } } } impl PurgeEvent { pub fn description(&self) -> &'static str { match self { PurgeEvent::Started => "Purge started", PurgeEvent::Finished => "Purge finished", PurgeEvent::Running => "Purge running", PurgeEvent::Error => "Purge error", PurgeEvent::InProgress => "Active purge in progress", PurgeEvent::AutoExpunge => "Auto-expunge executed", PurgeEvent::BlobCleanup => "Blob storage cleanup completed", } } pub fn explain(&self) -> &'static str { match self { PurgeEvent::Started => "The purge has started", PurgeEvent::Finished => "The purge has finished", PurgeEvent::Running => "The purge is running", PurgeEvent::Error => "An error occurred with the purge", PurgeEvent::InProgress => "An active purge is in progress", PurgeEvent::AutoExpunge => "Auto-expunge has been executed", PurgeEvent::BlobCleanup => "Blob storage cleanup has completed", } } } impl EvalEvent { pub fn description(&self) -> &'static str { match self { EvalEvent::Result => "Expression evaluation result", EvalEvent::Error => "Expression evaluation error", EvalEvent::DirectoryNotFound => "Directory not found while evaluating expression", EvalEvent::StoreNotFound => "Store not found while evaluating expression", } } pub fn explain(&self) -> &'static str { match self { EvalEvent::Result => "The expression evaluation has a result", EvalEvent::Error => "An error occurred while evaluating the expression", EvalEvent::DirectoryNotFound => { "The directory was not found while evaluating the expression" } EvalEvent::StoreNotFound => "The store was not found while evaluating the expression", } } } impl ConfigEvent { pub fn description(&self) -> &'static str { match self { ConfigEvent::ParseError => "Configuration parse error", ConfigEvent::BuildError => "Configuration build error", ConfigEvent::MacroError => "Configuration macro error", ConfigEvent::WriteError => "Configuration write error", ConfigEvent::FetchError => "Configuration fetch error", ConfigEvent::DefaultApplied => "Default configuration applied", ConfigEvent::MissingSetting => "Missing configuration setting", ConfigEvent::UnusedSetting => "Unused configuration setting", ConfigEvent::ParseWarning => "Configuration parse warning", ConfigEvent::BuildWarning => "Configuration build warning", ConfigEvent::ImportExternal => "Importing external configuration", ConfigEvent::AlreadyUpToDate => "Configuration already up to date", } } pub fn explain(&self) -> &'static str { match self { ConfigEvent::ParseError => "An error occurred while parsing the configuration", ConfigEvent::BuildError => "An error occurred while building the configuration", ConfigEvent::MacroError => "An error occurred with a configuration macro", ConfigEvent::WriteError => "An error occurred while writing the configuration", ConfigEvent::FetchError => "An error occurred while fetching the configuration", ConfigEvent::DefaultApplied => "The default configuration has been applied", ConfigEvent::MissingSetting => "A configuration setting is missing", ConfigEvent::UnusedSetting => "A configuration setting is unused", ConfigEvent::ParseWarning => "A warning occurred while parsing the configuration", ConfigEvent::BuildWarning => "A warning occurred while building the configuration", ConfigEvent::ImportExternal => "An external configuration is being imported", ConfigEvent::AlreadyUpToDate => "The configuration is already up to date", } } } impl ArcEvent { pub fn description(&self) -> &'static str { match self { ArcEvent::ChainTooLong => "ARC chain too long", ArcEvent::InvalidInstance => "Invalid ARC instance", ArcEvent::InvalidCv => "Invalid ARC CV", ArcEvent::HasHeaderTag => "ARC has header tag", ArcEvent::BrokenChain => "Broken ARC chain", ArcEvent::SealerNotFound => "ARC sealer not found", } } pub fn explain(&self) -> &'static str { match self { ArcEvent::ChainTooLong => "The ARC chain is too long", ArcEvent::InvalidInstance => "The ARC instance is invalid", ArcEvent::InvalidCv => "The ARC CV is invalid", ArcEvent::HasHeaderTag => "The ARC has a header tag", ArcEvent::BrokenChain => "The ARC chain is broken", ArcEvent::SealerNotFound => "The ARC sealer was not found", } } } impl DkimEvent { pub fn description(&self) -> &'static str { match self { DkimEvent::Pass => "DKIM verification passed", DkimEvent::Neutral => "DKIM verification neutral", DkimEvent::Fail => "DKIM verification failed", DkimEvent::PermError => "DKIM permanent error", DkimEvent::TempError => "DKIM temporary error", DkimEvent::None => "No DKIM signature", DkimEvent::UnsupportedVersion => "Unsupported DKIM version", DkimEvent::UnsupportedAlgorithm => "Unsupported DKIM algorithm", DkimEvent::UnsupportedCanonicalization => "Unsupported DKIM canonicalization", DkimEvent::UnsupportedKeyType => "Unsupported DKIM key type", DkimEvent::FailedBodyHashMatch => "DKIM body hash mismatch", DkimEvent::FailedVerification => "DKIM verification failed", DkimEvent::FailedAuidMatch => "DKIM AUID mismatch", DkimEvent::RevokedPublicKey => "DKIM public key revoked", DkimEvent::IncompatibleAlgorithms => "Incompatible DKIM algorithms", DkimEvent::SignatureExpired => "DKIM signature expired", DkimEvent::SignatureLength => "DKIM signature length issue", DkimEvent::SignerNotFound => "DKIM signer not found", } } pub fn explain(&self) -> &'static str { match self { DkimEvent::Pass => "DKIM verification has passed", DkimEvent::Neutral => "DKIM verification is neutral", DkimEvent::Fail => "DKIM verification has failed", DkimEvent::PermError => "A permanent error occurred with DKIM", DkimEvent::TempError => "A temporary error occurred with DKIM", DkimEvent::None => "No DKIM signature was found", DkimEvent::UnsupportedVersion => "The DKIM version is unsupported", DkimEvent::UnsupportedAlgorithm => "The DKIM algorithm is unsupported", DkimEvent::UnsupportedCanonicalization => "The DKIM canonicalization is unsupported", DkimEvent::UnsupportedKeyType => "The DKIM key type is unsupported", DkimEvent::FailedBodyHashMatch => "The DKIM body hash does not match", DkimEvent::FailedVerification => "The DKIM verification has failed", DkimEvent::FailedAuidMatch => "The DKIM AUID does not match", DkimEvent::RevokedPublicKey => "The DKIM public key has been revoked", DkimEvent::IncompatibleAlgorithms => "The DKIM algorithms are incompatible", DkimEvent::SignatureExpired => "The DKIM signature has expired", DkimEvent::SignatureLength => "The DKIM signature length is incorrect", DkimEvent::SignerNotFound => "The DKIM signer was not found", } } } impl SpfEvent { pub fn description(&self) -> &'static str { match self { SpfEvent::Pass => "SPF check passed", SpfEvent::Fail => "SPF check failed", SpfEvent::SoftFail => "SPF soft fail", SpfEvent::Neutral => "SPF neutral result", SpfEvent::TempError => "SPF temporary error", SpfEvent::PermError => "SPF permanent error", SpfEvent::None => "No SPF record", } } pub fn explain(&self) -> &'static str { match self { SpfEvent::Pass => "The SPF check has passed", SpfEvent::Fail => "The SPF check has failed", SpfEvent::SoftFail => "The SPF check has soft failed", SpfEvent::Neutral => "The SPF result is neutral", SpfEvent::TempError => "A temporary error occurred with SPF", SpfEvent::PermError => "A permanent error occurred with SPF", SpfEvent::None => "No SPF record was found", } } } impl DmarcEvent { pub fn description(&self) -> &'static str { match self { DmarcEvent::Pass => "DMARC check passed", DmarcEvent::Fail => "DMARC check failed", DmarcEvent::PermError => "DMARC permanent error", DmarcEvent::TempError => "DMARC temporary error", DmarcEvent::None => "No DMARC record", } } pub fn explain(&self) -> &'static str { match self { DmarcEvent::Pass => "The DMARC check has passed", DmarcEvent::Fail => "The DMARC check has failed", DmarcEvent::PermError => "A permanent error occurred with DMARC", DmarcEvent::TempError => "A temporary error occurred with DMARC", DmarcEvent::None => "No DMARC record was found", } } } impl IprevEvent { pub fn description(&self) -> &'static str { match self { IprevEvent::Pass => "IPREV check passed", IprevEvent::Fail => "IPREV check failed", IprevEvent::PermError => "IPREV permanent error", IprevEvent::TempError => "IPREV temporary error", IprevEvent::None => "No IPREV record", } } pub fn explain(&self) -> &'static str { match self { IprevEvent::Pass => "The IPREV check has passed", IprevEvent::Fail => "The IPREV check has failed", IprevEvent::PermError => "A permanent error occurred with IPREV", IprevEvent::TempError => "A temporary error occurred with IPREV", IprevEvent::None => "No IPREV record was found", } } } impl MailAuthEvent { pub fn description(&self) -> &'static str { match self { MailAuthEvent::ParseError => "Mail authentication parse error", MailAuthEvent::MissingParameters => "Missing mail authentication parameters", MailAuthEvent::NoHeadersFound => "No headers found in message", MailAuthEvent::Crypto => "Crypto error during mail authentication", MailAuthEvent::Io => "I/O error during mail authentication", MailAuthEvent::Base64 => "Base64 error during mail authentication", MailAuthEvent::DnsError => "DNS error", MailAuthEvent::DnsRecordNotFound => "DNS record not found", MailAuthEvent::DnsInvalidRecordType => "Invalid DNS record type", MailAuthEvent::PolicyNotAligned => "Policy not aligned", } } pub fn explain(&self) -> &'static str { match self { MailAuthEvent::ParseError => "An error occurred while parsing mail authentication", MailAuthEvent::MissingParameters => "Mail authentication parameters are missing", MailAuthEvent::NoHeadersFound => "No headers were found in the message", MailAuthEvent::Crypto => "A crypto error occurred during mail authentication", MailAuthEvent::Io => "An I/O error occurred during mail authentication", MailAuthEvent::Base64 => "A base64 error occurred during mail authentication", MailAuthEvent::DnsError => "A DNS error occurred", MailAuthEvent::DnsRecordNotFound => "The DNS record was not found", MailAuthEvent::DnsInvalidRecordType => "The DNS record type is invalid", MailAuthEvent::PolicyNotAligned => "The policy is not aligned", } } } impl StoreEvent { pub fn description(&self) -> &'static str { match self { StoreEvent::AssertValueFailed => "Another process modified the record", StoreEvent::FoundationdbError => "FoundationDB error", StoreEvent::MysqlError => "MySQL error", StoreEvent::PostgresqlError => "PostgreSQL error", StoreEvent::RocksdbError => "RocksDB error", StoreEvent::SqliteError => "SQLite error", StoreEvent::LdapError => "LDAP error", StoreEvent::ElasticsearchError => "ElasticSearch error", StoreEvent::RedisError => "Redis error", StoreEvent::S3Error => "S3 error", StoreEvent::AzureError => "Azure error", StoreEvent::FilesystemError => "Filesystem error", StoreEvent::PoolError => "Connection pool error", StoreEvent::DataCorruption => "Data corruption detected", StoreEvent::DecompressError => "Decompression error", StoreEvent::DeserializeError => "Deserialization error", StoreEvent::NotFound => "Record not found in database", StoreEvent::NotConfigured => "Store not configured", StoreEvent::NotSupported => "Operation not supported by store", StoreEvent::UnexpectedError => "Unexpected store error", StoreEvent::CryptoError => "Store crypto error", StoreEvent::BlobMissingMarker => "Blob missing marker", StoreEvent::SqlQuery => "SQL query executed", StoreEvent::LdapQuery => "LDAP query executed", StoreEvent::LdapWarning => "LDAP authentication warning", StoreEvent::DataWrite => "Write batch operation", StoreEvent::BlobRead => "Blob read operation", StoreEvent::BlobWrite => "Blob write operation", StoreEvent::BlobDelete => "Blob delete operation", StoreEvent::DataIterate => "Data store iteration operation", StoreEvent::HttpStoreFetch => "HTTP store updated", StoreEvent::HttpStoreError => "Error updating HTTP store", StoreEvent::CacheMiss => "Cache miss", StoreEvent::CacheHit => "Cache hit", StoreEvent::CacheStale => "Cache is stale", StoreEvent::CacheUpdate => "Cache update", StoreEvent::MeilisearchError => "Meilisearch error", } } pub fn explain(&self) -> &'static str { match self { StoreEvent::AssertValueFailed => "Another process modified the record", StoreEvent::FoundationdbError => "A FoundationDB error occurred", StoreEvent::MysqlError => "A MySQL error occurred", StoreEvent::PostgresqlError => "A PostgreSQL error occurred", StoreEvent::RocksdbError => "A RocksDB error occurred", StoreEvent::SqliteError => "An SQLite error occurred", StoreEvent::LdapError => "An LDAP error occurred", StoreEvent::ElasticsearchError => "An ElasticSearch error occurred", StoreEvent::RedisError => "A Redis error occurred", StoreEvent::S3Error => "An S3 error occurred", StoreEvent::AzureError => "An Azure error occurred", StoreEvent::FilesystemError => "A filesystem error occurred", StoreEvent::PoolError => "A connection pool error occurred", StoreEvent::DataCorruption => "Data corruption was detected", StoreEvent::DecompressError => "A decompression error occurred", StoreEvent::DeserializeError => "A deserialization error occurred", StoreEvent::NotFound => "The record was not found in the database", StoreEvent::NotConfigured => "The store is not configured", StoreEvent::NotSupported => "The operation is not supported by the store", StoreEvent::UnexpectedError => "An unexpected store error occurred", StoreEvent::CryptoError => "A store crypto error occurred", StoreEvent::BlobMissingMarker => "The blob is missing a marker", StoreEvent::SqlQuery => "An SQL query was executed", StoreEvent::LdapQuery => "An LDAP query was executed", StoreEvent::LdapWarning => "An LDAP authentication warning occurred", StoreEvent::DataWrite => "A write batch operation was executed", StoreEvent::BlobRead => "A blob read operation was executed", StoreEvent::BlobWrite => "A blob write operation was executed", StoreEvent::BlobDelete => "A blob delete operation was executed", StoreEvent::DataIterate => "A data store iteration operation was executed", StoreEvent::HttpStoreFetch => "The HTTP store was updated", StoreEvent::HttpStoreError => "An error occurred while updating the HTTP store", StoreEvent::CacheMiss => "No cache entry found for the account", StoreEvent::CacheHit => "Cache entry found for the account, no update needed", StoreEvent::CacheStale => "Cache is too old, rebuilding", StoreEvent::CacheUpdate => "Cache updated with latest database changes", StoreEvent::MeilisearchError => "A Meilisearch error occurred", } } } impl MessageIngestEvent { pub fn description(&self) -> &'static str { match self { MessageIngestEvent::Ham => "Message ingested", MessageIngestEvent::Spam => "Possible spam message ingested", MessageIngestEvent::ImapAppend => "Message appended via IMAP", MessageIngestEvent::JmapAppend => "Message appended via JMAP", MessageIngestEvent::Duplicate => "Skipping duplicate message", MessageIngestEvent::Error => "Message ingestion error", MessageIngestEvent::FtsIndex => "Full-text search index updated", } } pub fn explain(&self) -> &'static str { match self { MessageIngestEvent::Ham => "The message has been ingested", MessageIngestEvent::Spam => "A possible spam message has been ingested", MessageIngestEvent::ImapAppend => "The message has been appended via IMAP", MessageIngestEvent::JmapAppend => "The message has been appended via JMAP", MessageIngestEvent::Duplicate => "The message is a duplicate and has been skipped", MessageIngestEvent::Error => "An error occurred while ingesting the message", MessageIngestEvent::FtsIndex => "The full-text search index has been updated", } } } impl JmapEvent { pub fn description(&self) -> &'static str { match self { JmapEvent::MethodCall => "JMAP method call", JmapEvent::InvalidArguments => "Invalid JMAP arguments", JmapEvent::RequestTooLarge => "JMAP request too large", JmapEvent::StateMismatch => "JMAP state mismatch", JmapEvent::AnchorNotFound => "JMAP anchor not found", JmapEvent::UnsupportedFilter => "Unsupported JMAP filter", JmapEvent::UnsupportedSort => "Unsupported JMAP sort", JmapEvent::UnknownMethod => "Unknown JMAP method", JmapEvent::InvalidResultReference => "Invalid JMAP result reference", JmapEvent::Forbidden => "JMAP operation forbidden", JmapEvent::AccountNotFound => "JMAP account not found", JmapEvent::AccountNotSupportedByMethod => "JMAP account not supported by method", JmapEvent::AccountReadOnly => "JMAP account is read-only", JmapEvent::NotFound => "JMAP resource not found", JmapEvent::CannotCalculateChanges => "Cannot calculate JMAP changes", JmapEvent::UnknownDataType => "Unknown JMAP data type", JmapEvent::UnknownCapability => "Unknown JMAP capability", JmapEvent::NotJson => "JMAP request is not JSON", JmapEvent::NotRequest => "JMAP input is not a request", JmapEvent::WebsocketStart => "JMAP WebSocket connection started", JmapEvent::WebsocketStop => "JMAP WebSocket connection stopped", JmapEvent::WebsocketError => "JMAP WebSocket error", } } pub fn explain(&self) -> &'static str { match self { JmapEvent::MethodCall => "A JMAP method call has been made", JmapEvent::InvalidArguments => "The JMAP arguments are invalid", JmapEvent::RequestTooLarge => "The JMAP request is too large", JmapEvent::StateMismatch => "The JMAP state is mismatched", JmapEvent::AnchorNotFound => "The JMAP anchor was not found", JmapEvent::UnsupportedFilter => "The JMAP filter is unsupported", JmapEvent::UnsupportedSort => "The JMAP sort is unsupported", JmapEvent::UnknownMethod => "The JMAP method is unknown", JmapEvent::InvalidResultReference => "The JMAP result reference is invalid", JmapEvent::Forbidden => "The JMAP operation is forbidden", JmapEvent::AccountNotFound => "The JMAP account was not found", JmapEvent::AccountNotSupportedByMethod => { "The JMAP account is not supported by the method" } JmapEvent::AccountReadOnly => "The JMAP account is read-only", JmapEvent::NotFound => "The JMAP resource was not found", JmapEvent::CannotCalculateChanges => "Cannot calculate JMAP changes", JmapEvent::UnknownDataType => "The JMAP data type is unknown", JmapEvent::UnknownCapability => "The JMAP capability is unknown", JmapEvent::NotJson => "The JMAP request is not JSON", JmapEvent::NotRequest => "The JMAP input is not a request", JmapEvent::WebsocketStart => "The JMAP WebSocket connection has started", JmapEvent::WebsocketStop => "The JMAP WebSocket connection has stopped", JmapEvent::WebsocketError => "An error occurred with the JMAP WebSocket connection", } } } impl LimitEvent { pub fn description(&self) -> &'static str { match self { LimitEvent::SizeRequest => "Request size limit reached", LimitEvent::SizeUpload => "Upload size limit reached", LimitEvent::CallsIn => "Incoming calls limit reached", LimitEvent::ConcurrentRequest => "Concurrent request limit reached", LimitEvent::ConcurrentUpload => "Concurrent upload limit reached", LimitEvent::ConcurrentConnection => "Concurrent connection limit reached", LimitEvent::Quota => "Quota limit reached", LimitEvent::BlobQuota => "Blob quota limit reached", LimitEvent::TooManyRequests => "Too many requests", LimitEvent::TenantQuota => "Tenant quota limit reached", } } pub fn explain(&self) -> &'static str { match self { LimitEvent::SizeRequest => "The request size limit has been reached", LimitEvent::SizeUpload => "The upload size limit has been reached", LimitEvent::CallsIn => "The incoming calls limit has been reached", LimitEvent::ConcurrentRequest => "The concurrent request limit has been reached", LimitEvent::ConcurrentUpload => "The concurrent upload limit has been reached", LimitEvent::ConcurrentConnection => "The concurrent connection limit has been reached", LimitEvent::Quota => "The quota limit has been reached", LimitEvent::BlobQuota => "The blob quota limit has been reached", LimitEvent::TooManyRequests => "Too many requests have been made", LimitEvent::TenantQuota => "One of the tenant quota limits has been reached", } } } impl ManageEvent { pub fn description(&self) -> &'static str { match self { ManageEvent::MissingParameter => "Missing parameter", ManageEvent::AlreadyExists => "Record already exists", ManageEvent::AssertFailed => "Assertion failed", ManageEvent::NotFound => "Resource not found", ManageEvent::NotSupported => "Management operation not supported", ManageEvent::Error => "Management error", } } pub fn explain(&self) -> &'static str { match self { ManageEvent::MissingParameter => "A parameter is missing", ManageEvent::AlreadyExists => "A record with the same name already exists", ManageEvent::AssertFailed => "A management assertion has failed", ManageEvent::NotFound => "The managed resource was not found", ManageEvent::NotSupported => "The management operation is not supported", ManageEvent::Error => "A management error occurred", } } } impl AuthEvent { pub fn description(&self) -> &'static str { match self { AuthEvent::Success => "Authentication successful", AuthEvent::Failed => "Authentication failed", AuthEvent::MissingTotp => "Missing TOTP for authentication", AuthEvent::TooManyAttempts => "Too many authentication attempts", AuthEvent::Error => "Authentication error", AuthEvent::TokenExpired => "OAuth token expired", AuthEvent::ClientRegistration => "OAuth Client registration", } } pub fn explain(&self) -> &'static str { match self { AuthEvent::Success => "Successful authentication", AuthEvent::Failed => "Failed authentication", AuthEvent::MissingTotp => "TOTP is missing for authentication", AuthEvent::TooManyAttempts => "Too many authentication attempts have been made", AuthEvent::Error => "An error occurred with authentication", AuthEvent::TokenExpired => "OAuth authentication token has expired", AuthEvent::ClientRegistration => "OAuth client successfully registered", } } } impl ResourceEvent { pub fn description(&self) -> &'static str { match self { ResourceEvent::NotFound => "Resource not found", ResourceEvent::BadParameters => "Bad resource parameters", ResourceEvent::Error => "Resource error", ResourceEvent::DownloadExternal => "Downloading external resource", ResourceEvent::WebadminUnpacked => "Webadmin resource unpacked", } } pub fn explain(&self) -> &'static str { match self { ResourceEvent::NotFound => "The resource was not found", ResourceEvent::BadParameters => "The resource parameters are bad", ResourceEvent::Error => "An error occurred with the resource", ResourceEvent::DownloadExternal => "The external resource is being downloaded", ResourceEvent::WebadminUnpacked => "The webadmin resource has been unpacked", } } } impl SecurityEvent { pub fn description(&self) -> &'static str { match self { SecurityEvent::AuthenticationBan => "Banned due to authentication errors", SecurityEvent::AbuseBan => "Banned due to abuse", SecurityEvent::LoiterBan => "Banned due to loitering", SecurityEvent::IpBlocked => "Blocked IP address", SecurityEvent::ScanBan => "Banned due to scan", SecurityEvent::Unauthorized => "Unauthorized access", } } pub fn explain(&self) -> &'static str { match self { SecurityEvent::AuthenticationBan => { "IP address was banned due to multiple authentication errors" } SecurityEvent::AbuseBan => { "IP address was banned due to abuse, such as RCPT TO attacks" } SecurityEvent::ScanBan => "IP address was banned due to exploit scanning", SecurityEvent::LoiterBan => "IP address was banned due to multiple loitering events", SecurityEvent::IpBlocked => "Rejected connection from blocked IP address", SecurityEvent::Unauthorized => "Account does not have permission to access resource", } } } impl AiEvent { pub fn description(&self) -> &'static str { match self { AiEvent::LlmResponse => "LLM response", AiEvent::ApiError => "AI API error", } } pub fn explain(&self) -> &'static str { match self { AiEvent::LlmResponse => "An LLM response has been received", AiEvent::ApiError => "An AI API error occurred", } } } impl WebDavEvent { pub fn description(&self) -> &'static str { match self { WebDavEvent::Propfind => "WebDAV PROPFIND request", WebDavEvent::Proppatch => "WebDAV PROPPATCH request", WebDavEvent::Get => "WebDAV GET request", WebDavEvent::Report => "WebDAV REPORT request", WebDavEvent::Mkcol => "WebDAV MKCOL request", WebDavEvent::Delete => "WebDAV DELETE request", WebDavEvent::Put => "WebDAV PUT request", WebDavEvent::Post => "WebDAV POST request", WebDavEvent::Patch => "WebDAV PATCH request", WebDavEvent::Copy => "WebDAV COPY request", WebDavEvent::Move => "WebDAV MOVE request", WebDavEvent::Lock => "WebDAV LOCK request", WebDavEvent::Unlock => "WebDAV UNLOCK request", WebDavEvent::Acl => "WebDAV ACL request", WebDavEvent::Error => "WebDAV error", WebDavEvent::Head => "WebDAV HEAD request", WebDavEvent::Mkcalendar => "WebDAV MKCALENDAR request", WebDavEvent::Options => "WebDAV OPTIONS request", } } pub fn explain(&self) -> &'static str { match self { WebDavEvent::Propfind => "A PROPFIND request has been made to the server", WebDavEvent::Proppatch => "A PROPPATCH request has been made to the server", WebDavEvent::Get => "A GET request has been made to the server", WebDavEvent::Report => "A REPORT request has been made to the server", WebDavEvent::Mkcol => "A MKCOL request has been made to the server", WebDavEvent::Delete => "A DELETE request has been made to the server", WebDavEvent::Put => "A PUT request has been made to the server", WebDavEvent::Post => "A POST request has been made to the server", WebDavEvent::Patch => "A PATCH request has been made to the server", WebDavEvent::Copy => "A COPY request has been made to the server", WebDavEvent::Move => "A MOVE request has been made to the server", WebDavEvent::Lock => "A LOCK request has been made to the server", WebDavEvent::Unlock => "An UNLOCK request has been made to the server", WebDavEvent::Acl => { "An ACL request has been made to the server" } WebDavEvent::Error => "An error occurred with the WebDAV request", WebDavEvent::Head => "A HEAD request has been made to the server", WebDavEvent::Mkcalendar => "A MKCALENDAR request has been made to the server", WebDavEvent::Options => "An OPTIONS request has been made to the server", } } } impl CalendarEvent { pub fn description(&self) -> &'static str { match self { CalendarEvent::RuleExpansionError => "Calendar rule expansion error", CalendarEvent::AlarmSent => "Calendar alarm sent", CalendarEvent::AlarmSkipped => "Calendar alarm skipped", CalendarEvent::AlarmRecipientOverride => "Calendar alarm recipient overriden", CalendarEvent::AlarmFailed => "Calendar alarm could not be sent", CalendarEvent::ItipMessageSent => "Calendar iTIP message sent", CalendarEvent::ItipMessageReceived => "Calendar iTIP message received", CalendarEvent::ItipMessageError => "iTIP message error", } } pub fn explain(&self) -> &'static str { match self { CalendarEvent::RuleExpansionError => { "An error occurred while expanding calendar recurrences" } CalendarEvent::AlarmSent => "A calendar alarm has been sent to the recipient", CalendarEvent::AlarmSkipped => "A calendar alarm was skipped", CalendarEvent::AlarmRecipientOverride => "A calendar alarm recipient was overridden", CalendarEvent::AlarmFailed => "A calendar alarm could not be sent to the recipient", CalendarEvent::ItipMessageSent => "A calendar iTIP message has been sent", CalendarEvent::ItipMessageReceived => "A calendar iTIP/iMIP message has been received", CalendarEvent::ItipMessageError => { "An error occurred while processing an iTIP/iMIP message" } } } } ================================================ FILE: crates/trc/src/event/level.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{cmp::Ordering, fmt::Display, str::FromStr}; use super::*; impl EventType { pub fn level(&self) -> Level { match self { EventType::Store(event) => match event { StoreEvent::DataWrite | StoreEvent::DataIterate | StoreEvent::BlobRead | StoreEvent::BlobWrite | StoreEvent::BlobDelete | StoreEvent::SqlQuery | StoreEvent::LdapQuery => Level::Trace, StoreEvent::CacheMiss | StoreEvent::CacheHit | StoreEvent::CacheStale | StoreEvent::CacheUpdate | StoreEvent::NotFound | StoreEvent::HttpStoreFetch | StoreEvent::LdapWarning => Level::Debug, StoreEvent::AssertValueFailed | StoreEvent::FoundationdbError | StoreEvent::MysqlError | StoreEvent::PostgresqlError | StoreEvent::RocksdbError | StoreEvent::SqliteError | StoreEvent::LdapError | StoreEvent::ElasticsearchError | StoreEvent::MeilisearchError | StoreEvent::RedisError | StoreEvent::S3Error | StoreEvent::AzureError | StoreEvent::FilesystemError | StoreEvent::PoolError | StoreEvent::DataCorruption | StoreEvent::DecompressError | StoreEvent::DeserializeError | StoreEvent::NotConfigured | StoreEvent::NotSupported | StoreEvent::UnexpectedError | StoreEvent::CryptoError => Level::Error, StoreEvent::BlobMissingMarker | StoreEvent::HttpStoreError => Level::Warn, }, EventType::Jmap(_) => Level::Debug, EventType::Imap(event) => match event { ImapEvent::ConnectionStart | ImapEvent::ConnectionEnd => Level::Debug, ImapEvent::GetAcl | ImapEvent::SetAcl | ImapEvent::MyRights | ImapEvent::ListRights | ImapEvent::Append | ImapEvent::Capabilities | ImapEvent::Id | ImapEvent::Close | ImapEvent::Copy | ImapEvent::Move | ImapEvent::CreateMailbox | ImapEvent::DeleteMailbox | ImapEvent::RenameMailbox | ImapEvent::Enable | ImapEvent::Expunge | ImapEvent::Fetch | ImapEvent::List | ImapEvent::Lsub | ImapEvent::Logout | ImapEvent::Namespace | ImapEvent::Noop | ImapEvent::Search | ImapEvent::Sort | ImapEvent::Select | ImapEvent::Status | ImapEvent::Store | ImapEvent::Subscribe | ImapEvent::Unsubscribe | ImapEvent::Thread | ImapEvent::Error | ImapEvent::IdleStart | ImapEvent::IdleStop | ImapEvent::GetQuota => Level::Debug, ImapEvent::RawInput | ImapEvent::RawOutput => Level::Trace, }, EventType::ManageSieve(event) => match event { ManageSieveEvent::ConnectionStart | ManageSieveEvent::ConnectionEnd => Level::Debug, ManageSieveEvent::CreateScript | ManageSieveEvent::UpdateScript | ManageSieveEvent::GetScript | ManageSieveEvent::DeleteScript | ManageSieveEvent::RenameScript | ManageSieveEvent::CheckScript | ManageSieveEvent::HaveSpace | ManageSieveEvent::ListScripts | ManageSieveEvent::SetActive | ManageSieveEvent::Capabilities | ManageSieveEvent::StartTls | ManageSieveEvent::Unauthenticate | ManageSieveEvent::Logout | ManageSieveEvent::Noop | ManageSieveEvent::Error => Level::Debug, ManageSieveEvent::RawInput | ManageSieveEvent::RawOutput => Level::Trace, }, EventType::Pop3(event) => match event { Pop3Event::ConnectionStart | Pop3Event::ConnectionEnd => Level::Debug, Pop3Event::Delete | Pop3Event::Reset | Pop3Event::Quit | Pop3Event::Fetch | Pop3Event::List | Pop3Event::ListMessage | Pop3Event::Uidl | Pop3Event::UidlMessage | Pop3Event::Stat | Pop3Event::Noop | Pop3Event::Capabilities | Pop3Event::StartTls | Pop3Event::Utf8 | Pop3Event::Error => Level::Debug, Pop3Event::RawInput | Pop3Event::RawOutput => Level::Trace, }, EventType::Smtp(event) => match event { SmtpEvent::ConnectionStart | SmtpEvent::ConnectionEnd => Level::Debug, SmtpEvent::DidNotSayEhlo | SmtpEvent::EhloExpected | SmtpEvent::LhloExpected | SmtpEvent::MailFromUnauthenticated | SmtpEvent::MailFromUnauthorized | SmtpEvent::MailFromRewritten | SmtpEvent::MailFromMissing | SmtpEvent::MultipleMailFrom | SmtpEvent::MailFromNotAllowed | SmtpEvent::RcptToDuplicate | SmtpEvent::RcptToRewritten | SmtpEvent::RcptToMissing | SmtpEvent::RequireTlsDisabled | SmtpEvent::DeliverByDisabled | SmtpEvent::DeliverByInvalid | SmtpEvent::FutureReleaseDisabled | SmtpEvent::FutureReleaseInvalid | SmtpEvent::MtPriorityDisabled | SmtpEvent::MtPriorityInvalid | SmtpEvent::DsnDisabled | SmtpEvent::AuthExchangeTooLong | SmtpEvent::AlreadyAuthenticated | SmtpEvent::Noop | SmtpEvent::StartTls | SmtpEvent::StartTlsUnavailable | SmtpEvent::StartTlsAlready | SmtpEvent::Rset | SmtpEvent::Quit | SmtpEvent::Help | SmtpEvent::CommandNotImplemented | SmtpEvent::InvalidCommand | SmtpEvent::InvalidSenderAddress | SmtpEvent::InvalidRecipientAddress | SmtpEvent::InvalidParameter | SmtpEvent::UnsupportedParameter | SmtpEvent::SyntaxError | SmtpEvent::Error => Level::Debug, SmtpEvent::MissingLocalHostname | SmtpEvent::IdNotFound => Level::Warn, SmtpEvent::ConcurrencyLimitExceeded | SmtpEvent::TransferLimitExceeded | SmtpEvent::RateLimitExceeded | SmtpEvent::TimeLimitExceeded | SmtpEvent::MissingAuthDirectory | SmtpEvent::MessageParseFailed | SmtpEvent::MessageTooLarge | SmtpEvent::LoopDetected | SmtpEvent::DkimPass | SmtpEvent::DkimFail | SmtpEvent::ArcPass | SmtpEvent::ArcFail | SmtpEvent::SpfEhloPass | SmtpEvent::SpfEhloFail | SmtpEvent::SpfFromPass | SmtpEvent::SpfFromFail | SmtpEvent::DmarcPass | SmtpEvent::DmarcFail | SmtpEvent::IprevPass | SmtpEvent::IprevFail | SmtpEvent::TooManyMessages | SmtpEvent::Ehlo | SmtpEvent::InvalidEhlo | SmtpEvent::MailFrom | SmtpEvent::MailboxDoesNotExist | SmtpEvent::RelayNotAllowed | SmtpEvent::RcptTo | SmtpEvent::RcptToGreylisted | SmtpEvent::TooManyInvalidRcpt | SmtpEvent::Vrfy | SmtpEvent::VrfyNotFound | SmtpEvent::VrfyDisabled | SmtpEvent::Expn | SmtpEvent::ExpnNotFound | SmtpEvent::AuthNotAllowed | SmtpEvent::AuthMechanismNotSupported | SmtpEvent::ExpnDisabled | SmtpEvent::RequestTooLarge | SmtpEvent::TooManyRecipients => Level::Info, SmtpEvent::RawInput | SmtpEvent::RawOutput => Level::Trace, }, EventType::Network(event) => match event { NetworkEvent::ReadError | NetworkEvent::WriteError | NetworkEvent::FlushError | NetworkEvent::Closed => Level::Trace, NetworkEvent::Timeout | NetworkEvent::AcceptError => Level::Debug, NetworkEvent::ListenStart | NetworkEvent::ListenStop => Level::Info, NetworkEvent::ListenError | NetworkEvent::BindError | NetworkEvent::SetOptError | NetworkEvent::SplitError => Level::Error, NetworkEvent::ProxyError => Level::Warn, }, EventType::Limit(cause) => match cause { LimitEvent::SizeRequest => Level::Debug, LimitEvent::SizeUpload => Level::Debug, LimitEvent::CallsIn => Level::Debug, LimitEvent::ConcurrentRequest => Level::Debug, LimitEvent::ConcurrentUpload => Level::Debug, LimitEvent::ConcurrentConnection => Level::Warn, LimitEvent::Quota => Level::Debug, LimitEvent::BlobQuota => Level::Debug, LimitEvent::TooManyRequests => Level::Warn, LimitEvent::TenantQuota => Level::Info, }, EventType::Manage(_) => Level::Debug, EventType::Auth(cause) => match cause { AuthEvent::Failed | AuthEvent::TokenExpired => Level::Debug, AuthEvent::MissingTotp => Level::Trace, AuthEvent::TooManyAttempts => Level::Warn, AuthEvent::Error => Level::Error, AuthEvent::Success | AuthEvent::ClientRegistration => Level::Info, }, EventType::Config(cause) => match cause { ConfigEvent::ParseError | ConfigEvent::BuildError | ConfigEvent::MacroError | ConfigEvent::WriteError | ConfigEvent::FetchError => Level::Error, ConfigEvent::DefaultApplied | ConfigEvent::MissingSetting | ConfigEvent::UnusedSetting | ConfigEvent::AlreadyUpToDate => Level::Debug, ConfigEvent::ParseWarning | ConfigEvent::BuildWarning => Level::Warn, ConfigEvent::ImportExternal => Level::Info, }, EventType::Resource(cause) => match cause { ResourceEvent::NotFound => Level::Debug, ResourceEvent::BadParameters | ResourceEvent::Error => Level::Error, ResourceEvent::DownloadExternal | ResourceEvent::WebadminUnpacked => Level::Info, }, EventType::Arc(event) => match event { ArcEvent::ChainTooLong | ArcEvent::InvalidInstance | ArcEvent::InvalidCv | ArcEvent::HasHeaderTag | ArcEvent::BrokenChain => Level::Debug, ArcEvent::SealerNotFound => Level::Warn, }, EventType::Dkim(event) => match event { DkimEvent::SignerNotFound => Level::Warn, _ => Level::Debug, }, EventType::MailAuth(_) => Level::Debug, EventType::Purge(event) => match event { PurgeEvent::Started => Level::Debug, PurgeEvent::Finished => Level::Debug, PurgeEvent::Running => Level::Info, PurgeEvent::Error => Level::Error, PurgeEvent::BlobCleanup => Level::Info, PurgeEvent::InProgress | PurgeEvent::AutoExpunge => Level::Debug, }, EventType::Eval(event) => match event { EvalEvent::Error | EvalEvent::StoreNotFound => Level::Debug, EvalEvent::Result => Level::Trace, EvalEvent::DirectoryNotFound => Level::Warn, }, EventType::Server(event) => match event { ServerEvent::Startup | ServerEvent::Shutdown | ServerEvent::Licensing => { Level::Info } ServerEvent::StartupError | ServerEvent::ThreadError => Level::Error, }, EventType::Acme(event) => match event { AcmeEvent::DnsRecordCreated | AcmeEvent::DnsRecordPropagated | AcmeEvent::TlsAlpnReceived | AcmeEvent::AuthStart | AcmeEvent::AuthPending | AcmeEvent::AuthValid | AcmeEvent::AuthCompleted | AcmeEvent::ProcessCert | AcmeEvent::OrderProcessing | AcmeEvent::OrderReady | AcmeEvent::OrderValid | AcmeEvent::OrderStart | AcmeEvent::OrderCompleted => Level::Info, AcmeEvent::Error => Level::Error, AcmeEvent::OrderInvalid | AcmeEvent::AuthError | AcmeEvent::AuthTooManyAttempts | AcmeEvent::TokenNotFound | AcmeEvent::DnsRecordPropagationTimeout | AcmeEvent::TlsAlpnError | AcmeEvent::DnsRecordCreationFailed => Level::Warn, AcmeEvent::RenewBackoff | AcmeEvent::DnsRecordDeletionFailed | AcmeEvent::ClientSuppliedSni | AcmeEvent::ClientMissingSni | AcmeEvent::DnsRecordNotPropagated | AcmeEvent::DnsRecordLookupFailed => Level::Debug, }, EventType::Tls(event) => match event { TlsEvent::Handshake => Level::Info, TlsEvent::HandshakeError | TlsEvent::CertificateNotFound => Level::Debug, TlsEvent::NotConfigured => Level::Error, TlsEvent::NoCertificatesAvailable | TlsEvent::MultipleCertificatesAvailable => { Level::Warn } }, EventType::Sieve(event) => match event { SieveEvent::NotSupported | SieveEvent::QuotaExceeded | SieveEvent::ListNotFound | SieveEvent::ScriptNotFound | SieveEvent::MessageTooLarge => Level::Warn, SieveEvent::SendMessage => Level::Info, SieveEvent::UnexpectedError => Level::Error, SieveEvent::ActionAccept | SieveEvent::RuntimeError | SieveEvent::ActionAcceptReplace | SieveEvent::ActionDiscard | SieveEvent::ActionReject => Level::Debug, }, EventType::Spam(event) => match event { SpamEvent::Pyzor | SpamEvent::PyzorError | SpamEvent::Dnsbl | SpamEvent::DnsblError | SpamEvent::Classify | SpamEvent::TrainSampleAdded => Level::Debug, SpamEvent::TrainSampleNotFound => Level::Warn, SpamEvent::TrainStarted | SpamEvent::TrainCompleted | SpamEvent::ModelLoaded | SpamEvent::ModelNotReady | SpamEvent::ModelNotFound => Level::Info, }, EventType::Http(event) => match event { HttpEvent::ConnectionStart | HttpEvent::ConnectionEnd => Level::Debug, HttpEvent::XForwardedMissing => Level::Warn, HttpEvent::Error | HttpEvent::RequestUrl => Level::Debug, HttpEvent::RequestBody | HttpEvent::ResponseBody => Level::Trace, }, EventType::PushSubscription(event) => match event { PushSubscriptionEvent::Error | PushSubscriptionEvent::NotFound => Level::Debug, PushSubscriptionEvent::Success => Level::Trace, }, EventType::Cluster(event) => match event { ClusterEvent::SubscriberStart | ClusterEvent::SubscriberStop | ClusterEvent::PublisherStart | ClusterEvent::PublisherStop => Level::Info, ClusterEvent::SubscriberDisconnected => Level::Warn, ClusterEvent::MessageReceived | ClusterEvent::MessageSkipped => Level::Trace, ClusterEvent::PublisherError | ClusterEvent::SubscriberError | ClusterEvent::MessageInvalid => Level::Error, }, EventType::Housekeeper(event) => match event { HousekeeperEvent::Start | HousekeeperEvent::Stop => Level::Info, HousekeeperEvent::Run | HousekeeperEvent::Schedule => Level::Debug, }, EventType::TaskQueue(event) => match event { TaskQueueEvent::BlobNotFound | TaskQueueEvent::TaskAcquired | TaskQueueEvent::TaskLocked | TaskQueueEvent::TaskIgnored | TaskQueueEvent::MetadataNotFound => Level::Debug, TaskQueueEvent::TaskFailed => Level::Warn, }, EventType::Dmarc(_) => Level::Debug, EventType::Spf(_) => Level::Debug, EventType::Iprev(_) => Level::Debug, EventType::Milter(event) => match event { MilterEvent::Read | MilterEvent::Write => Level::Trace, MilterEvent::ActionAccept | MilterEvent::ActionDiscard | MilterEvent::ActionReject | MilterEvent::ActionTempFail | MilterEvent::ActionReplyCode | MilterEvent::ActionConnectionFailure | MilterEvent::ActionShutdown => Level::Info, MilterEvent::IoError | MilterEvent::FrameTooLarge | MilterEvent::FrameInvalid | MilterEvent::UnexpectedResponse | MilterEvent::Timeout | MilterEvent::TlsInvalidName | MilterEvent::Disconnected | MilterEvent::ParseError => Level::Warn, }, EventType::MtaHook(event) => match event { MtaHookEvent::ActionAccept | MtaHookEvent::ActionDiscard | MtaHookEvent::ActionReject | MtaHookEvent::ActionQuarantine => Level::Info, MtaHookEvent::Error => Level::Warn, }, EventType::Dane(event) => match event { DaneEvent::AuthenticationSuccess | DaneEvent::AuthenticationFailure | DaneEvent::NoCertificatesFound | DaneEvent::CertificateParseError | DaneEvent::TlsaRecordMatch | DaneEvent::TlsaRecordFetch | DaneEvent::TlsaRecordFetchError | DaneEvent::TlsaRecordNotFound | DaneEvent::TlsaRecordNotDnssecSigned | DaneEvent::TlsaRecordInvalid => Level::Info, }, EventType::Delivery(event) => match event { DeliveryEvent::AttemptStart | DeliveryEvent::AttemptEnd | DeliveryEvent::Completed | DeliveryEvent::Failed | DeliveryEvent::DomainDeliveryStart | DeliveryEvent::MxLookupFailed | DeliveryEvent::IpLookupFailed | DeliveryEvent::NullMx | DeliveryEvent::Connect | DeliveryEvent::ConnectError | DeliveryEvent::GreetingFailed | DeliveryEvent::EhloRejected | DeliveryEvent::AuthFailed | DeliveryEvent::MailFromRejected | DeliveryEvent::Delivered | DeliveryEvent::RcptToRejected | DeliveryEvent::RcptToFailed | DeliveryEvent::MessageRejected | DeliveryEvent::StartTls | DeliveryEvent::StartTlsUnavailable | DeliveryEvent::StartTlsError | DeliveryEvent::StartTlsDisabled | DeliveryEvent::ImplicitTlsError | DeliveryEvent::DoubleBounce => Level::Info, DeliveryEvent::ConcurrencyLimitExceeded | DeliveryEvent::RateLimitExceeded | DeliveryEvent::MissingOutboundHostname => Level::Warn, DeliveryEvent::DsnSuccess | DeliveryEvent::DsnTempFail | DeliveryEvent::DsnPermFail => Level::Info, DeliveryEvent::MxLookup | DeliveryEvent::IpLookup | DeliveryEvent::Ehlo | DeliveryEvent::Auth | DeliveryEvent::MailFrom | DeliveryEvent::RcptTo => Level::Debug, DeliveryEvent::RawInput | DeliveryEvent::RawOutput => Level::Trace, }, EventType::Queue(event) => match event { QueueEvent::BackPressure => Level::Warn, QueueEvent::QueueMessage | QueueEvent::QueueMessageAuthenticated | QueueEvent::QueueReport | QueueEvent::QueueDsn | QueueEvent::QueueAutogenerated | QueueEvent::RateLimitExceeded | QueueEvent::ConcurrencyLimitExceeded | QueueEvent::Rescheduled | QueueEvent::QuotaExceeded => Level::Info, QueueEvent::Locked | QueueEvent::BlobNotFound => Level::Debug, }, EventType::TlsRpt(event) => match event { TlsRptEvent::RecordFetch | TlsRptEvent::RecordFetchError | TlsRptEvent::RecordNotFound => Level::Info, }, EventType::MtaSts(event) => match event { MtaStsEvent::PolicyFetch | MtaStsEvent::PolicyNotFound | MtaStsEvent::PolicyFetchError | MtaStsEvent::InvalidPolicy | MtaStsEvent::NotAuthorized | MtaStsEvent::Authorized => Level::Info, }, EventType::IncomingReport(event) => match event { IncomingReportEvent::DmarcReportWithWarnings | IncomingReportEvent::TlsReportWithWarnings => Level::Warn, IncomingReportEvent::DmarcReport | IncomingReportEvent::TlsReport | IncomingReportEvent::AbuseReport | IncomingReportEvent::AuthFailureReport | IncomingReportEvent::FraudReport | IncomingReportEvent::NotSpamReport | IncomingReportEvent::VirusReport | IncomingReportEvent::OtherReport | IncomingReportEvent::MessageParseFailed | IncomingReportEvent::DmarcParseFailed | IncomingReportEvent::TlsRpcParseFailed | IncomingReportEvent::ArfParseFailed | IncomingReportEvent::DecompressError => Level::Info, }, EventType::OutgoingReport(event) => match event { OutgoingReportEvent::Locked | OutgoingReportEvent::NotFound => Level::Info, OutgoingReportEvent::SpfReport | OutgoingReportEvent::SpfRateLimited | OutgoingReportEvent::DkimReport | OutgoingReportEvent::DkimRateLimited | OutgoingReportEvent::DmarcReport | OutgoingReportEvent::DmarcRateLimited | OutgoingReportEvent::DmarcAggregateReport | OutgoingReportEvent::TlsAggregate | OutgoingReportEvent::HttpSubmission | OutgoingReportEvent::UnauthorizedReportingAddress | OutgoingReportEvent::ReportingAddressValidationError | OutgoingReportEvent::SubmissionError | OutgoingReportEvent::NoRecipientsFound => Level::Info, }, EventType::Telemetry(_) => Level::Warn, EventType::MessageIngest(event) => match event { MessageIngestEvent::Ham | MessageIngestEvent::Spam | MessageIngestEvent::ImapAppend | MessageIngestEvent::JmapAppend | MessageIngestEvent::Duplicate | MessageIngestEvent::FtsIndex => Level::Info, MessageIngestEvent::Error => Level::Error, }, EventType::Security(_) => Level::Info, EventType::Ai(event) => match event { AiEvent::LlmResponse => Level::Trace, AiEvent::ApiError => Level::Warn, }, EventType::WebDav(_) => Level::Debug, EventType::Calendar(event) => match event { CalendarEvent::ItipMessageSent | CalendarEvent::ItipMessageReceived | CalendarEvent::AlarmSent => Level::Info, CalendarEvent::AlarmFailed => Level::Warn, CalendarEvent::RuleExpansionError | CalendarEvent::AlarmSkipped | CalendarEvent::AlarmRecipientOverride | CalendarEvent::ItipMessageError => Level::Debug, }, } } } impl PartialOrd for Level { #[inline(always)] fn partial_cmp(&self, other: &Level) -> Option { Some(self.cmp(other)) } #[inline(always)] fn lt(&self, other: &Level) -> bool { (*other as usize) < (*self as usize) } #[inline(always)] fn le(&self, other: &Level) -> bool { (*other as usize) <= (*self as usize) } #[inline(always)] fn gt(&self, other: &Level) -> bool { (*other as usize) > (*self as usize) } #[inline(always)] fn ge(&self, other: &Level) -> bool { (*other as usize) >= (*self as usize) } } impl Ord for Level { #[inline(always)] fn cmp(&self, other: &Self) -> Ordering { (*other as usize).cmp(&(*self as usize)) } } impl FromStr for Level { type Err = String; fn from_str(s: &str) -> std::result::Result { match s.to_ascii_lowercase().as_str() { "disable" => Ok(Self::Disable), "trace" => Ok(Self::Trace), "debug" => Ok(Self::Debug), "info" => Ok(Self::Info), "warn" => Ok(Self::Warn), "error" => Ok(Self::Error), _ => Err(s.to_string()), } } } impl Level { pub fn as_str(&self) -> &'static str { match self { Self::Disable => "DISABLE", Self::Trace => "TRACE", Self::Debug => "DEBUG", Self::Info => "INFO", Self::Warn => "WARN", Self::Error => "ERROR", } } pub fn is_contained(&self, other: Self) -> bool { *self >= other && other != Level::Disable && *self != Level::Disable } } impl Display for Level { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { self.as_str().fmt(f) } } ================================================ FILE: crates/trc/src/event/metrics.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::MetricType; impl MetricType { pub fn name(&self) -> &'static str { match self { Self::MessageIngestionTime => "message-ingest.time", Self::MessageFtsIndexTime => "message-ingest.index-time", Self::DeliveryTotalTime => "delivery.total-time", Self::DeliveryTime => "delivery.attempt-time", Self::MessageSize => "message.size", Self::MessageAuthSize => "message.authenticated-size", Self::ReportOutgoingSize => "outgoing-report.size", Self::StoreReadTime => "store.data-read-time", Self::StoreWriteTime => "store.data-write-time", Self::BlobReadTime => "store.blob-read-time", Self::BlobWriteTime => "store.blob-write-time", Self::DnsLookupTime => "dns.lookup-time", Self::HttpRequestTime => "http.request-time", Self::ImapRequestTime => "imap.request-time", Self::Pop3RequestTime => "pop3.request-time", Self::SmtpRequestTime => "smtp.request-time", Self::SieveRequestTime => "sieve.request-time", Self::HttpActiveConnections => "http.active-connections", Self::ImapActiveConnections => "imap.active-connections", Self::Pop3ActiveConnections => "pop3.active-connections", Self::SmtpActiveConnections => "smtp.active-connections", Self::SieveActiveConnections => "sieve.active-connections", Self::DeliveryActiveConnections => "delivery.active-connections", Self::ServerMemory => "server.memory", Self::QueueCount => "queue.count", Self::UserCount => "user.count", Self::DomainCount => "domain.count", } } pub fn description(&self) -> &'static str { match self { Self::MessageIngestionTime => "Message ingestion time", Self::MessageFtsIndexTime => "Message full-text indexing time", Self::DeliveryTotalTime => "Total message delivery time from submission to delivery", Self::DeliveryTime => "Message delivery time", Self::MessageSize => "Received message size", Self::MessageAuthSize => "Received message size from authenticated users", Self::ReportOutgoingSize => "Outgoing report size", Self::StoreReadTime => "Data store read time", Self::StoreWriteTime => "Data store write time", Self::BlobReadTime => "Blob store read time", Self::BlobWriteTime => "Blob store write time", Self::DnsLookupTime => "DNS lookup time", Self::HttpRequestTime => "HTTP request duration", Self::ImapRequestTime => "IMAP request duration", Self::Pop3RequestTime => "POP3 request duration", Self::SmtpRequestTime => "SMTP request duration", Self::SieveRequestTime => "ManageSieve request duration", Self::HttpActiveConnections => "Active HTTP connections", Self::ImapActiveConnections => "Active IMAP connections", Self::Pop3ActiveConnections => "Active POP3 connections", Self::SmtpActiveConnections => "Active SMTP connections", Self::SieveActiveConnections => "Active ManageSieve connections", Self::DeliveryActiveConnections => "Active delivery connections", Self::ServerMemory => "Server memory usage", Self::QueueCount => "Total number of messages in the queue", Self::UserCount => "Total number of users", Self::DomainCount => "Total number of domains", } } pub fn unit(&self) -> &'static str { match self { Self::MessageIngestionTime | Self::MessageFtsIndexTime | Self::DeliveryTotalTime | Self::DeliveryTime | Self::StoreReadTime | Self::StoreWriteTime | Self::BlobReadTime | Self::BlobWriteTime | Self::DnsLookupTime | Self::HttpRequestTime | Self::ImapRequestTime | Self::Pop3RequestTime | Self::SmtpRequestTime | Self::SieveRequestTime => "milliseconds", Self::MessageSize | Self::MessageAuthSize | Self::ReportOutgoingSize | Self::ServerMemory => "bytes", Self::HttpActiveConnections | Self::ImapActiveConnections | Self::Pop3ActiveConnections | Self::SmtpActiveConnections | Self::SieveActiveConnections | Self::DeliveryActiveConnections => "connections", Self::QueueCount => "messages", Self::UserCount => "users", Self::DomainCount => "domains", } } pub fn code(&self) -> u64 { match self { Self::MessageIngestionTime => 0, Self::MessageFtsIndexTime => 1, Self::DeliveryTotalTime => 2, Self::DeliveryTime => 3, Self::MessageSize => 4, Self::MessageAuthSize => 5, Self::ReportOutgoingSize => 6, Self::StoreReadTime => 7, Self::StoreWriteTime => 8, Self::BlobReadTime => 9, Self::BlobWriteTime => 10, Self::DnsLookupTime => 11, Self::HttpRequestTime => 12, Self::ImapRequestTime => 13, Self::Pop3RequestTime => 14, Self::SmtpRequestTime => 15, Self::SieveRequestTime => 16, Self::HttpActiveConnections => 17, Self::ImapActiveConnections => 18, Self::Pop3ActiveConnections => 19, Self::SmtpActiveConnections => 20, Self::SieveActiveConnections => 21, Self::DeliveryActiveConnections => 22, Self::ServerMemory => 23, Self::QueueCount => 24, Self::UserCount => 25, Self::DomainCount => 26, } } pub fn from_code(code: u64) -> Option { match code { 0 => Some(Self::MessageIngestionTime), 1 => Some(Self::MessageFtsIndexTime), 2 => Some(Self::DeliveryTotalTime), 3 => Some(Self::DeliveryTime), 4 => Some(Self::MessageSize), 5 => Some(Self::MessageAuthSize), 6 => Some(Self::ReportOutgoingSize), 7 => Some(Self::StoreReadTime), 8 => Some(Self::StoreWriteTime), 9 => Some(Self::BlobReadTime), 10 => Some(Self::BlobWriteTime), 11 => Some(Self::DnsLookupTime), 12 => Some(Self::HttpRequestTime), 13 => Some(Self::ImapRequestTime), 14 => Some(Self::Pop3RequestTime), 15 => Some(Self::SmtpRequestTime), 16 => Some(Self::SieveRequestTime), 17 => Some(Self::HttpActiveConnections), 18 => Some(Self::ImapActiveConnections), 19 => Some(Self::Pop3ActiveConnections), 20 => Some(Self::SmtpActiveConnections), 21 => Some(Self::SieveActiveConnections), 22 => Some(Self::DeliveryActiveConnections), 23 => Some(Self::ServerMemory), 24 => Some(Self::QueueCount), 25 => Some(Self::UserCount), 26 => Some(Self::DomainCount), _ => None, } } pub fn try_parse(name: &str) -> Option { match name { "message-ingest.time" => Some(Self::MessageIngestionTime), "message-ingest.index-time" => Some(Self::MessageFtsIndexTime), "delivery.total-time" => Some(Self::DeliveryTotalTime), "delivery.attempt-time" => Some(Self::DeliveryTime), "message.size" => Some(Self::MessageSize), "message.authenticated-size" => Some(Self::MessageAuthSize), "outgoing-report.size" => Some(Self::ReportOutgoingSize), "store.data-read-time" => Some(Self::StoreReadTime), "store.data-write-time" => Some(Self::StoreWriteTime), "store.blob-read-time" => Some(Self::BlobReadTime), "store.blob-write-time" => Some(Self::BlobWriteTime), "dns.lookup-time" => Some(Self::DnsLookupTime), "http.request-time" => Some(Self::HttpRequestTime), "imap.request-time" => Some(Self::ImapRequestTime), "pop3.request-time" => Some(Self::Pop3RequestTime), "smtp.request-time" => Some(Self::SmtpRequestTime), "sieve.request-time" => Some(Self::SieveRequestTime), "http.active-connections" => Some(Self::HttpActiveConnections), "imap.active-connections" => Some(Self::ImapActiveConnections), "pop3.active-connections" => Some(Self::Pop3ActiveConnections), "smtp.active-connections" => Some(Self::SmtpActiveConnections), "sieve.active-connections" => Some(Self::SieveActiveConnections), "delivery.active-connections" => Some(Self::DeliveryActiveConnections), "server.memory" => Some(Self::ServerMemory), "queue.count" => Some(Self::QueueCount), "user.count" => Some(Self::UserCount), "domain.count" => Some(Self::DomainCount), _ => None, } } pub fn variants() -> &'static [Self] { &[ Self::MessageIngestionTime, Self::MessageFtsIndexTime, Self::DeliveryTotalTime, Self::DeliveryTime, Self::MessageSize, Self::MessageAuthSize, Self::ReportOutgoingSize, Self::StoreReadTime, Self::StoreWriteTime, Self::BlobReadTime, Self::BlobWriteTime, Self::DnsLookupTime, Self::HttpRequestTime, Self::ImapRequestTime, Self::Pop3RequestTime, Self::SmtpRequestTime, Self::SieveRequestTime, Self::HttpActiveConnections, Self::ImapActiveConnections, Self::Pop3ActiveConnections, Self::SmtpActiveConnections, Self::SieveActiveConnections, Self::DeliveryActiveConnections, Self::ServerMemory, Self::QueueCount, Self::UserCount, Self::DomainCount, ] } } ================================================ FILE: crates/trc/src/event/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod conv; pub mod description; pub mod level; pub mod metrics; use compact_str::ToCompactString; use std::fmt::Display; use crate::*; impl Event { pub fn with_capacity(inner: T, capacity: usize) -> Self { Self { inner, keys: Vec::with_capacity(capacity), } } pub fn with_keys(inner: T, keys: Vec<(Key, Value)>) -> Self { Self { inner, keys } } pub fn new(inner: T) -> Self { Self { inner, keys: Vec::with_capacity(5), } } pub fn value(&self, key: Key) -> Option<&Value> { self.keys .iter() .find_map(|(k, v)| if *k == key { Some(v) } else { None }) } pub fn value_as_str(&self, key: Key) -> Option<&str> { self.value(key).and_then(|v| v.as_str()) } pub fn value_as_uint(&self, key: Key) -> Option { self.value(key).and_then(|v| v.to_uint()) } pub fn take_value(&mut self, key: Key) -> Option { self.keys.iter_mut().find_map(|(k, v)| { if *k == key { Some(std::mem::take(v)) } else { None } }) } pub fn into_boxed(self) -> Box { Box::new(self) } } impl Error { #[inline(always)] pub fn new(inner: EventType) -> Self { Error(Box::new(Event::new(inner))) } #[inline(always)] pub fn set_ctx(&mut self, key: Key, value: impl Into) { self.0.keys.push((key, value.into())); } #[inline(always)] pub fn ctx(mut self, key: Key, value: impl Into) -> Self { self.0.keys.push((key, value.into())); self } #[inline(always)] pub fn ctx_unique(mut self, key: Key, value: impl Into) -> Self { if self.0.keys.iter().all(|(k, _)| *k != key) { self.0.keys.push((key, value.into())); } self } #[inline(always)] pub fn ctx_opt(self, key: Key, value: Option>) -> Self { match value { Some(value) => self.ctx(key, value), None => self, } } #[inline(always)] pub fn matches(&self, inner: EventType) -> bool { self.0.inner == inner } #[inline(always)] pub fn event_type(&self) -> EventType { self.0.inner } #[inline(always)] pub fn span_id(self, session_id: u64) -> Self { self.ctx(Key::SpanId, session_id) } #[inline(always)] pub fn caused_by(self, error: impl Into) -> Self { self.ctx(Key::CausedBy, error) } #[inline(always)] pub fn details(self, error: impl Into) -> Self { self.ctx(Key::Details, error) } #[inline(always)] pub fn code(self, error: impl Into) -> Self { self.ctx(Key::Code, error) } #[inline(always)] pub fn id(self, error: impl Into) -> Self { self.ctx(Key::Id, error) } #[inline(always)] pub fn reason(self, error: impl Display) -> Self { self.ctx(Key::Reason, error.to_compact_string()) } #[inline(always)] pub fn document_id(self, id: u32) -> Self { self.ctx(Key::DocumentId, id) } #[inline(always)] pub fn account_id(self, id: u32) -> Self { self.ctx(Key::AccountId, id) } #[inline(always)] pub fn collection(self, id: impl Into) -> Self { self.ctx(Key::Collection, id.into() as u64) } #[inline(always)] pub fn wrap(self, cause: EventType) -> Self { Error::new(cause).caused_by(self) } #[inline(always)] pub fn keys(&self) -> &[(Key, Value)] { &self.0.keys } #[inline(always)] pub fn value(&self, key: Key) -> Option<&Value> { self.0.value(key) } #[inline(always)] pub fn value_as_str(&self, key: Key) -> Option<&str> { self.0.value_as_str(key) } #[inline(always)] pub fn value_as_uint(&self, key: Key) -> Option { self.0.value_as_uint(key) } #[inline(always)] pub fn take_value(&mut self, key: Key) -> Option { self.0.take_value(key) } #[inline(always)] pub fn is_assertion_failure(&self) -> bool { self.0.inner == EventType::Store(StoreEvent::AssertValueFailed) } pub fn key(&self, key: Key) -> Option<&Value> { self.0 .keys .iter() .find_map(|(k, v)| if *k == key { Some(v) } else { None }) } #[inline(always)] pub fn is_jmap_method_error(&self) -> bool { !matches!( self.0.inner, EventType::Jmap( JmapEvent::UnknownCapability | JmapEvent::NotJson | JmapEvent::NotRequest ) ) } #[inline(always)] pub fn must_disconnect(&self) -> bool { matches!( self.0.inner, EventType::Network(_) | EventType::Auth(AuthEvent::TooManyAttempts) | EventType::Limit(LimitEvent::ConcurrentRequest | LimitEvent::TooManyRequests) | EventType::Security(_) ) } #[inline(always)] pub fn should_write_err(&self) -> bool { !matches!(self.0.inner, EventType::Network(_) | EventType::Security(_)) } pub fn corrupted_key(key: &[u8], value: Option<&[u8]>, caused_by: &'static str) -> Error { EventType::Store(StoreEvent::DataCorruption) .ctx(Key::Key, key) .ctx_opt(Key::Value, value) .ctx(Key::CausedBy, caused_by) } } impl Event { pub fn span_id(&self) -> Option { for (key, value) in &self.keys { match (key, value) { (Key::SpanId, Value::UInt(value)) => return Some(*value), (Key::SpanId, Value::Int(value)) => return Some(*value as u64), _ => {} } } None } } impl EventType { #[inline(always)] pub fn is_span_start(&self) -> bool { matches!( self, EventType::Smtp(SmtpEvent::ConnectionStart) | EventType::Imap(ImapEvent::ConnectionStart) | EventType::ManageSieve(ManageSieveEvent::ConnectionStart) | EventType::Pop3(Pop3Event::ConnectionStart) | EventType::Http(HttpEvent::ConnectionStart) | EventType::Delivery(DeliveryEvent::AttemptStart) ) } #[inline(always)] pub fn is_span_end(&self) -> bool { matches!( self, EventType::Smtp(SmtpEvent::ConnectionEnd) | EventType::Imap(ImapEvent::ConnectionEnd) | EventType::ManageSieve(ManageSieveEvent::ConnectionEnd) | EventType::Pop3(Pop3Event::ConnectionEnd) | EventType::Http(HttpEvent::ConnectionEnd) | EventType::Delivery(DeliveryEvent::AttemptEnd) ) } pub fn is_raw_io(&self) -> bool { matches!( self, EventType::Imap(ImapEvent::RawInput | ImapEvent::RawOutput) | EventType::Smtp(SmtpEvent::RawInput | SmtpEvent::RawOutput) | EventType::Pop3(Pop3Event::RawInput | Pop3Event::RawOutput) | EventType::ManageSieve(ManageSieveEvent::RawInput | ManageSieveEvent::RawOutput) | EventType::Delivery(DeliveryEvent::RawInput | DeliveryEvent::RawOutput) | EventType::Milter(MilterEvent::Read | MilterEvent::Write) ) } #[inline(always)] pub fn ctx(self, key: Key, value: impl Into) -> Error { self.into_err().ctx(key, value) } #[inline(always)] pub fn caused_by(self, error: impl Into) -> Error { self.into_err().caused_by(error) } #[inline(always)] pub fn reason(self, error: impl Display) -> Error { self.into_err().reason(error) } #[inline(always)] pub fn into_err(self) -> Error { Error::new(self) } pub fn message(&self) -> &'static str { match self { EventType::Store(cause) => cause.message(), EventType::Jmap(cause) => cause.message(), EventType::Imap(_) => "IMAP error", EventType::ManageSieve(_) => "ManageSieve error", EventType::Pop3(_) => "POP3 error", EventType::Smtp(_) => "SMTP error", EventType::Network(_) => "Network error", EventType::Limit(cause) => cause.message(), EventType::Manage(cause) => cause.message(), EventType::Auth(cause) => cause.message(), EventType::Config(_) => "Configuration error", EventType::Resource(cause) => cause.message(), EventType::Security(_) => "Insufficient permissions", _ => "Internal server error", } } } impl StoreEvent { #[inline(always)] pub fn ctx(self, key: Key, value: impl Into) -> Error { self.into_err().ctx(key, value) } #[inline(always)] pub fn caused_by(self, error: impl Into) -> Error { self.into_err().caused_by(error) } #[inline(always)] pub fn reason(self, error: impl Display) -> Error { self.into_err().reason(error) } #[inline(always)] pub fn into_err(self) -> Error { Error::new(EventType::Store(self)) } pub fn message(&self) -> &'static str { match self { Self::AssertValueFailed => "Another process has modified the value", Self::BlobMissingMarker => "Blob is missing marker", Self::FoundationdbError => "FoundationDB error", Self::MysqlError => "MySQL error", Self::PostgresqlError => "PostgreSQL error", Self::RocksdbError => "RocksDB error", Self::SqliteError => "SQLite error", Self::LdapError => "LDAP error", Self::ElasticsearchError => "ElasticSearch error", Self::RedisError => "Redis error", Self::S3Error => "S3 error", Self::AzureError => "Azure error", Self::FilesystemError => "Filesystem error", Self::PoolError => "Connection pool error", Self::DataCorruption => "Data corruption", Self::DecompressError => "Decompression error", Self::DeserializeError => "Deserialization error", Self::NotFound => "Not found", Self::NotConfigured => "Not configured", Self::NotSupported => "Operation not supported", Self::UnexpectedError => "Unexpected error", Self::CryptoError => "Crypto error", _ => "Store error", } } } impl SecurityEvent { #[inline(always)] pub fn into_err(self) -> Error { Error::new(EventType::Security(self)) } } impl AuthEvent { #[inline(always)] pub fn ctx(self, key: Key, value: impl Into) -> Error { self.into_err().ctx(key, value) } #[inline(always)] pub fn caused_by(self, error: impl Into) -> Error { self.into_err().caused_by(error) } #[inline(always)] pub fn reason(self, error: impl Display) -> Error { self.into_err().reason(error) } #[inline(always)] pub fn into_err(self) -> Error { Error::new(EventType::Auth(self)) } pub fn message(&self) -> &'static str { match self { Self::Failed => "Authentication failed", Self::MissingTotp => concat!( "A TOTP code is required to authenticate this account. ", "Try authenticating again using 'secret$totp_token'." ), Self::TooManyAttempts => "Too many authentication attempts", _ => "Authentication error", } } } impl ManageEvent { #[inline(always)] pub fn ctx(self, key: Key, value: impl Into) -> Error { self.into_err().ctx(key, value) } #[inline(always)] pub fn caused_by(self, error: impl Into) -> Error { self.into_err().caused_by(error) } #[inline(always)] pub fn reason(self, error: impl Display) -> Error { self.into_err().reason(error) } #[inline(always)] pub fn into_err(self) -> Error { Error::new(EventType::Manage(self)) } pub fn message(&self) -> &'static str { match self { Self::MissingParameter => "Missing parameter", Self::AlreadyExists => "Already exists", Self::AssertFailed => "Assertion failed", Self::NotFound => "Not found", Self::NotSupported => "Operation not supported", Self::Error => "Management API Error", } } } impl JmapEvent { #[inline(always)] pub fn ctx(self, key: Key, value: impl Into) -> Error { self.into_err().ctx(key, value) } #[inline(always)] pub fn caused_by(self, error: impl Into) -> Error { self.into_err().caused_by(error) } #[inline(always)] pub fn reason(self, error: impl Display) -> Error { self.into_err().reason(error) } #[inline(always)] pub fn into_err(self) -> Error { Error::new(EventType::Jmap(self)) } pub fn message(&self) -> &'static str { match self { Self::InvalidArguments => "Invalid arguments", Self::RequestTooLarge => "Request too large", Self::StateMismatch => "State mismatch", Self::AnchorNotFound => "Anchor not found", Self::UnsupportedFilter => "Unsupported filter", Self::UnsupportedSort => "Unsupported sort", Self::UnknownMethod => "Unknown method", Self::InvalidResultReference => "Invalid result reference", Self::Forbidden => "Forbidden", Self::AccountNotFound => "Account not found", Self::AccountNotSupportedByMethod => "Account not supported by method", Self::AccountReadOnly => "Account read-only", Self::NotFound => "Not found", Self::CannotCalculateChanges => "Cannot calculate changes", Self::UnknownDataType => "Unknown data type", Self::UnknownCapability => "Unknown capability", Self::NotJson => "Not JSON", Self::NotRequest => "Not a request", _ => "Other message", } } } impl LimitEvent { #[inline(always)] pub fn ctx(self, key: Key, value: impl Into) -> Error { self.into_err().ctx(key, value) } #[inline(always)] pub fn caused_by(self, error: impl Into) -> Error { self.into_err().caused_by(error) } #[inline(always)] pub fn reason(self, error: impl Display) -> Error { self.into_err().reason(error) } #[inline(always)] pub fn into_err(self) -> Error { Error::new(EventType::Limit(self)) } pub fn message(&self) -> &'static str { match self { Self::SizeRequest => "Request too large", Self::SizeUpload => "Upload too large", Self::CallsIn => "Too many calls in", Self::ConcurrentRequest => "Too many concurrent requests", Self::ConcurrentConnection => "Too many concurrent connections", Self::ConcurrentUpload => "Too many concurrent uploads", Self::Quota => "Quota exceeded", Self::BlobQuota => "Blob quota exceeded", Self::TooManyRequests => "Too many requests", Self::TenantQuota => "Tenant quota exceeded", } } } impl ResourceEvent { #[inline(always)] pub fn ctx(self, key: Key, value: impl Into) -> Error { self.into_err().ctx(key, value) } #[inline(always)] pub fn caused_by(self, error: impl Into) -> Error { self.into_err().caused_by(error) } #[inline(always)] pub fn reason(self, error: impl Display) -> Error { self.into_err().reason(error) } #[inline(always)] pub fn into_err(self) -> Error { Error::new(EventType::Resource(self)) } pub fn message(&self) -> &'static str { match self { Self::NotFound => "Not found", Self::BadParameters => "Bad parameters", Self::Error => "Resource error", _ => "Other status", } } } impl SmtpEvent { #[inline(always)] pub fn ctx(self, key: Key, value: impl Into) -> Error { self.into_err().ctx(key, value) } #[inline(always)] pub fn into_err(self) -> Error { Error::new(EventType::Smtp(self)) } } impl SieveEvent { #[inline(always)] pub fn ctx(self, key: Key, value: impl Into) -> Error { self.into_err().ctx(key, value) } #[inline(always)] pub fn into_err(self) -> Error { Error::new(EventType::Sieve(self)) } } impl SpamEvent { #[inline(always)] pub fn ctx(self, key: Key, value: impl Into) -> Error { self.into_err().ctx(key, value) } #[inline(always)] pub fn into_err(self) -> Error { Error::new(EventType::Spam(self)) } } impl ImapEvent { #[inline(always)] pub fn ctx(self, key: Key, value: impl Into) -> Error { self.into_err().ctx(key, value) } #[inline(always)] pub fn into_err(self) -> Error { Error::new(EventType::Imap(self)) } #[inline(always)] pub fn caused_by(self, error: impl Into) -> Error { self.into_err().caused_by(error) } #[inline(always)] pub fn reason(self, error: impl Display) -> Error { self.into_err().reason(error) } } impl Pop3Event { #[inline(always)] pub fn ctx(self, key: Key, value: impl Into) -> Error { self.into_err().ctx(key, value) } #[inline(always)] pub fn into_err(self) -> Error { Error::new(EventType::Pop3(self)) } } impl ManageSieveEvent { #[inline(always)] pub fn ctx(self, key: Key, value: impl Into) -> Error { self.into_err().ctx(key, value) } #[inline(always)] pub fn into_err(self) -> Error { Error::new(EventType::ManageSieve(self)) } } impl NetworkEvent { #[inline(always)] pub fn ctx(self, key: Key, value: impl Into) -> Error { self.into_err().ctx(key, value) } #[inline(always)] pub fn into_err(self) -> Error { Error::new(EventType::Network(self)) } } impl Value { pub fn from_maybe_string(value: &[u8]) -> Self { if let Ok(value) = std::str::from_utf8(value) { Self::String(value.into()) } else { Self::Bytes(value.to_vec()) } } pub fn to_uint(&self) -> Option { match self { Self::UInt(value) => Some(*value), Self::Int(value) => Some(*value as u64), _ => None, } } pub fn as_str(&self) -> Option<&str> { match self { Self::String(value) => Some(value.as_str()), _ => None, } } pub fn into_string(self) -> Option { match self { Self::String(value) => Some(value), _ => None, } } } impl AddContext for Result { #[inline(always)] fn caused_by(self, location: &'static str) -> Result { match self { Ok(value) => Ok(value), Err(mut err) => { err.set_ctx(Key::CausedBy, location); Err(err) } } } #[inline(always)] fn add_context(self, f: F) -> Result where F: FnOnce(Error) -> Error, { match self { Ok(value) => Ok(value), Err(err) => Err(f(err)), } } } impl std::error::Error for Error {} impl Eq for Error {} impl PartialEq for Error { fn eq(&self, other: &Self) -> bool { if self.0.inner == other.0.inner && self.0.keys.len() == other.0.keys.len() { for kv in self.0.keys.iter() { if !other.0.keys.iter().any(|okv| kv == okv) { return false; } } true } else { false } } } impl PartialEq for Value { fn eq(&self, other: &Self) -> bool { match (self, other) { (Self::String(l0), Self::String(r0)) => l0 == r0, (Self::UInt(l0), Self::UInt(r0)) => l0 == r0, (Self::Int(l0), Self::Int(r0)) => l0 == r0, (Self::Float(l0), Self::Float(r0)) => l0 == r0, (Self::Bytes(l0), Self::Bytes(r0)) => l0 == r0, (Self::Bool(l0), Self::Bool(r0)) => l0 == r0, (Self::Ipv4(l0), Self::Ipv4(r0)) => l0 == r0, (Self::Ipv6(l0), Self::Ipv6(r0)) => l0 == r0, (Self::Event(l0), Self::Event(r0)) => l0 == r0, (Self::Array(l0), Self::Array(r0)) => l0 == r0, _ => false, } } } impl Eq for Value {} impl From for usize { fn from(value: EventType) -> Self { value.id() } } impl AsRef> for Event { fn as_ref(&self) -> &Event { self } } ================================================ FILE: crates/trc/src/ipc/bitset.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{USIZE_BITS, USIZE_BITS_MASK}; #[derive(Clone, Debug, PartialEq, Eq)] pub struct Bitset(pub(crate) [usize; N]); impl Bitset { #[allow(clippy::new_without_default)] pub const fn new() -> Self { Self([0; N]) } pub const fn all() -> Self { Self([usize::MAX; N]) } #[inline(always)] pub fn set(&mut self, index: impl Into) { let index = index.into(); self.0[index / USIZE_BITS] |= 1 << (index & USIZE_BITS_MASK); } #[inline(always)] pub fn clear(&mut self, index: impl Into) { let index = index.into(); self.0[index / USIZE_BITS] &= !(1 << (index & USIZE_BITS_MASK)); } #[inline(always)] pub fn get(&self, index: impl Into) -> bool { let index = index.into(); self.0[index / USIZE_BITS] & (1 << (index & USIZE_BITS_MASK)) != 0 } pub fn union(&mut self, other: &Self) { for i in 0..N { self.0[i] |= other.0[i]; } } pub fn intersection(&mut self, other: &Self) { for i in 0..N { self.0[i] &= other.0[i]; } } pub fn difference(&mut self, other: &Self) { for i in 0..N { self.0[i] &= !other.0[i]; } } pub fn clear_all(&mut self) { for i in 0..N { self.0[i] = 0; } } pub fn is_empty(&self) -> bool { for i in 0..N { if self.0[i] != 0 { return false; } } true } pub fn inner(&self) -> &[usize; N] { &self.0 } } impl Default for Bitset { fn default() -> Self { Self::new() } } ================================================ FILE: crates/trc/src/ipc/channel.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ cell::UnsafeCell, sync::{ Arc, atomic::{AtomicU64, Ordering}, }, }; use rtrb::{Consumer, Producer, PushError, RingBuffer}; use crate::{ Error, Event, EventType, ipc::collector::{COLLECTOR_THREAD, COLLECTOR_UPDATES, Update}, }; use super::collector::{Collector, CollectorThread}; pub(crate) static CHANNEL_FLAGS: AtomicU64 = AtomicU64::new(0); pub(crate) const CHANNEL_SIZE: usize = 10240; pub(crate) const CHANNEL_UPDATE_MARKER: u64 = 1 << 63; thread_local! { static EVENT_TX: UnsafeCell = { // Create channel. let (tx, rx) = RingBuffer::new(CHANNEL_SIZE); // Register receiver with collector. COLLECTOR_UPDATES.lock().push(Update::RegisterReceiver { receiver: Receiver { rx } }); // Spawn collector thread. let collector = COLLECTOR_THREAD.clone(); CHANNEL_FLAGS.fetch_or(CHANNEL_UPDATE_MARKER, Ordering::Relaxed); collector.thread().unpark(); // Return sender. UnsafeCell::new(Sender { tx, collector, overflow: Vec::with_capacity(0), }) }; } pub struct Sender { tx: Producer>, collector: Arc, overflow: Vec>, } pub struct Receiver { rx: Consumer>, } #[derive(Debug)] pub struct ChannelError; impl Sender { pub fn send(&mut self, event: Event) -> Result<(), ChannelError> { while let Some(event) = self.overflow.pop() { if let Err(PushError::Full(event)) = self.tx.push(event) { self.overflow.push(event); break; } } if let Err(PushError::Full(event)) = self.tx.push(event) { if self.overflow.len() <= CHANNEL_SIZE * 2 { self.overflow.push(event); } else { return Err(ChannelError); } } Ok(()) } } impl Receiver { pub fn try_recv(&mut self) -> Result>, ChannelError> { match self.rx.pop() { Ok(event) => Ok(Some(event)), Err(_) => { if !self.rx.is_abandoned() { Ok(None) } else { Err(ChannelError) } } } } } impl Event { pub fn send(self) { // SAFETY: EVENT_TX is thread-local. let _ = EVENT_TX.try_with(|tx| unsafe { let tx = &mut *tx.get(); if tx.send(self).is_ok() { CHANNEL_FLAGS.fetch_add(1, Ordering::Relaxed); tx.collector.thread().unpark(); } }); } pub fn send_with_metrics(self) { Collector::record_metric(self.inner, self.inner.id(), &self.keys); self.send(); } } impl Error { pub fn send(self) { self.0.send(); } } ================================================ FILE: crates/trc/src/ipc/collector.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ sync::{Arc, LazyLock, atomic::Ordering}, thread::{Builder, JoinHandle, park}, time::SystemTime, }; use ahash::AHashMap; use atomics::bitset::AtomicBitset; use ipc::{ USIZE_BITS, channel::{CHANNEL_FLAGS, CHANNEL_UPDATE_MARKER, Receiver}, subscriber::{Interests, Subscriber}, }; use parking_lot::Mutex; use crate::*; pub(crate) type GlobalInterests = AtomicBitset<{ TOTAL_EVENT_COUNT.div_ceil(USIZE_BITS) }>; pub(crate) static TRACE_INTERESTS: GlobalInterests = GlobalInterests::new(); pub(crate) type CollectorThread = JoinHandle<()>; pub(crate) static ACTIVE_SUBSCRIBERS: Mutex> = Mutex::new(Vec::new()); pub(crate) static COLLECTOR_UPDATES: Mutex> = Mutex::new(Vec::new()); pub(crate) const EVENT_TYPES: [EventType; TOTAL_EVENT_COUNT] = EventType::variants(); #[allow(clippy::enum_variant_names)] pub(crate) enum Update { RegisterReceiver { receiver: Receiver, }, RegisterSubscriber { subscriber: Subscriber, }, UnregisterSubscriber { id: String, }, UpdateSubscriber { id: String, interests: Interests, lossy: bool, }, UpdateLevels { levels: AHashMap, }, Shutdown, } pub struct Collector { receivers: Vec, subscribers: Vec, levels: [Level; TOTAL_EVENT_COUNT], active_spans: AHashMap>>, } const HTTP_CONN_START: usize = EventType::Http(HttpEvent::ConnectionStart).id(); const HTTP_CONN_END: usize = EventType::Http(HttpEvent::ConnectionEnd).id(); const IMAP_CONN_START: usize = EventType::Imap(ImapEvent::ConnectionStart).id(); const IMAP_CONN_END: usize = EventType::Imap(ImapEvent::ConnectionEnd).id(); const POP3_CONN_START: usize = EventType::Pop3(Pop3Event::ConnectionStart).id(); const POP3_CONN_END: usize = EventType::Pop3(Pop3Event::ConnectionEnd).id(); const SMTP_CONN_START: usize = EventType::Smtp(SmtpEvent::ConnectionStart).id(); const SMTP_CONN_END: usize = EventType::Smtp(SmtpEvent::ConnectionEnd).id(); const MANAGE_SIEVE_CONN_START: usize = EventType::ManageSieve(ManageSieveEvent::ConnectionStart).id(); const MANAGE_SIEVE_CONN_END: usize = EventType::ManageSieve(ManageSieveEvent::ConnectionEnd).id(); const EV_ATTEMPT_START: usize = EventType::Delivery(DeliveryEvent::AttemptStart).id(); const EV_ATTEMPT_END: usize = EventType::Delivery(DeliveryEvent::AttemptEnd).id(); const STALE_SPAN_CHECK_WATERMARK: usize = 8000; const SPAN_MAX_HOLD: u64 = 60 * 60 * 24; // 1 day pub(crate) static COLLECTOR_THREAD: LazyLock> = LazyLock::new(|| { Arc::new( Builder::new() .name("stalwart-collector".to_string()) .spawn(move || { Collector::default().collect(); }) .expect("Failed to start event collector"), ) }); impl Collector { fn collect(&mut self) { let mut do_continue = true; // Update self.update(); while do_continue { match CHANNEL_FLAGS.swap(0, Ordering::Relaxed) { 0 => { park(); } CHANNEL_UPDATE_MARKER..=u64::MAX => { do_continue = self.update(); } _ => {} } // Collect all events let mut closed_rxs = Vec::new(); for (rx_idx, rx) in self.receivers.iter_mut().enumerate() { let timestamp = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()); loop { match rx.try_recv() { Ok(Some(event)) => { // Build event let event_id = event.inner.id(); let mut event = Event { inner: EventDetails { level: self.levels[event_id], typ: event.inner, timestamp, span: None, }, keys: event.keys, }; // Track spans let event = match event_id { HTTP_CONN_START | IMAP_CONN_START | POP3_CONN_START | SMTP_CONN_START | MANAGE_SIEVE_CONN_START | EV_ATTEMPT_START => { let event = Arc::new(event); self.active_spans.insert( event.span_id().unwrap_or_else(|| { panic!("Missing span ID: {event:?}") }), event.clone(), ); if self.active_spans.len() > STALE_SPAN_CHECK_WATERMARK { self.active_spans.retain(|_, span| { timestamp.saturating_sub(span.inner.timestamp) < SPAN_MAX_HOLD }); } event } HTTP_CONN_END | IMAP_CONN_END | POP3_CONN_END | SMTP_CONN_END | MANAGE_SIEVE_CONN_END | EV_ATTEMPT_END => { if let Some(span) = self .active_spans .remove(&event.span_id().expect("Missing span ID")) { event.inner.span = Some(span.clone()); } else { #[cfg(debug_assertions)] { if event.span_id().unwrap() != 0 { eprintln!("Unregistered span ID: {event:?}"); } } } Arc::new(event) } _ => { if let Some(span_id) = event.span_id() { if let Some(span) = self.active_spans.get(&span_id) { event.inner.span = Some(span.clone()); } else { #[cfg(debug_assertions)] { if span_id != 0 { eprintln!("Unregistered span ID: {event:?}"); } } } } Arc::new(event) } }; // Send to subscribers for subscriber in self.subscribers.iter_mut() { subscriber.push_event(event_id, event.clone()); } } Ok(None) => { break; } Err(_) => { closed_rxs.push(rx_idx); // Channel is closed, remove. break; } } } } if do_continue { // Remove closed receivers (should be rare in Tokio) if !closed_rxs.is_empty() { let mut receivers = Vec::with_capacity(self.receivers.len() - closed_rxs.len()); for (rx_idx, rx) in self.receivers.drain(..).enumerate() { if !closed_rxs.contains(&rx_idx) { receivers.push(rx); } } self.receivers = receivers; } // Send batched events if !self.subscribers.is_empty() { self.subscribers .retain_mut(|subscriber| subscriber.send_batch().is_ok()); } } } // Send remaining events for mut subscriber in self.subscribers.drain(..) { let _ = subscriber.send_batch(); } } fn update(&mut self) -> bool { for update in COLLECTOR_UPDATES.lock().drain(..) { match update { Update::RegisterReceiver { receiver } => { self.receivers.push(receiver); } Update::RegisterSubscriber { subscriber } => { ACTIVE_SUBSCRIBERS.lock().push(subscriber.id.clone()); self.subscribers.push(subscriber); } Update::UnregisterSubscriber { id } => { ACTIVE_SUBSCRIBERS.lock().retain(|s| s != &id); self.subscribers.retain(|s| s.id != id); } Update::UpdateSubscriber { id, interests, lossy, } => { for subscriber in self.subscribers.iter_mut() { if subscriber.id == id { subscriber.interests = interests; subscriber.lossy = lossy; break; } } } Update::UpdateLevels { levels } => { for event in EVENT_TYPES.iter() { let event_id = event.id(); if let Some(level) = levels.get(event) { self.levels[event_id] = *level; } else { self.levels[event_id] = event.level(); } } } Update::Shutdown => return false, } } true } pub fn set_interests(mut interests: Interests) { if !interests.is_empty() { for event_type in EVENT_TYPES.iter() { if event_type.is_span_start() || event_type.is_span_end() { interests.set(*event_type); } } } TRACE_INTERESTS.update(interests); } pub fn union_interests(interests: Interests) { TRACE_INTERESTS.union(interests); } #[inline(always)] pub fn has_interest(event: impl Into) -> bool { TRACE_INTERESTS.get(event) } pub fn get_subscribers() -> Vec { ACTIVE_SUBSCRIBERS.lock().clone() } pub fn update_custom_levels(levels: AHashMap) { COLLECTOR_UPDATES .lock() .push(Update::UpdateLevels { levels }); } pub fn update_subscriber(id: String, interests: Interests, lossy: bool) { COLLECTOR_UPDATES.lock().push(Update::UpdateSubscriber { id, interests, lossy, }); } pub fn remove_subscriber(id: String) { COLLECTOR_UPDATES .lock() .push(Update::UnregisterSubscriber { id }); } pub fn shutdown() { COLLECTOR_UPDATES.lock().push(Update::Shutdown); Collector::reload(); } pub fn is_enabled() -> bool { !TRACE_INTERESTS.is_empty() } pub fn reload() { CHANNEL_FLAGS.fetch_or(CHANNEL_UPDATE_MARKER, Ordering::Relaxed); COLLECTOR_THREAD.thread().unpark(); } } impl Default for Collector { fn default() -> Self { let mut c = Collector { subscribers: Vec::new(), levels: [Level::Disable; TOTAL_EVENT_COUNT], active_spans: AHashMap::new(), receivers: Vec::new(), }; for event in EVENT_TYPES.iter() { let event_id = event.id(); c.levels[event_id] = event.level(); } c } } ================================================ FILE: crates/trc/src/ipc/metrics.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::atomic::Ordering; use atomics::{array::AtomicU32Array, gauge::AtomicGauge, histogram::AtomicHistogram}; use ipc::{ collector::{Collector, EVENT_TYPES, GlobalInterests}, subscriber::Interests, }; use crate::*; pub(crate) static METRIC_INTERESTS: GlobalInterests = GlobalInterests::new(); static EVENT_COUNTERS: AtomicU32Array = AtomicU32Array::new(); static CONNECTION_METRICS: [ConnectionMetrics; TOTAL_CONN_TYPES] = init_conn_metrics(); static MESSAGE_INGESTION_TIME: AtomicHistogram<12> = AtomicHistogram::<10>::new_short_durations(MetricType::MessageIngestionTime); static MESSAGE_INDEX_TIME: AtomicHistogram<12> = AtomicHistogram::<10>::new_short_durations(MetricType::MessageFtsIndexTime); static MESSAGE_DELIVERY_TIME: AtomicHistogram<12> = AtomicHistogram::<18>::new_long_durations(MetricType::DeliveryTotalTime); static MESSAGE_INCOMING_SIZE: AtomicHistogram<12> = AtomicHistogram::<12>::new_message_sizes(MetricType::MessageSize); static MESSAGE_SUBMISSION_SIZE: AtomicHistogram<12> = AtomicHistogram::<12>::new_message_sizes(MetricType::MessageAuthSize); static MESSAGE_OUT_REPORT_SIZE: AtomicHistogram<12> = AtomicHistogram::<12>::new_message_sizes(MetricType::ReportOutgoingSize); static STORE_DATA_READ_TIME: AtomicHistogram<12> = AtomicHistogram::<10>::new_short_durations(MetricType::StoreReadTime); static STORE_DATA_WRITE_TIME: AtomicHistogram<12> = AtomicHistogram::<10>::new_short_durations(MetricType::StoreWriteTime); static STORE_BLOB_READ_TIME: AtomicHistogram<12> = AtomicHistogram::<10>::new_short_durations(MetricType::BlobReadTime); static STORE_BLOB_WRITE_TIME: AtomicHistogram<12> = AtomicHistogram::<10>::new_short_durations(MetricType::BlobWriteTime); static DNS_LOOKUP_TIME: AtomicHistogram<12> = AtomicHistogram::<10>::new_short_durations(MetricType::DnsLookupTime); static SERVER_MEMORY: AtomicGauge = AtomicGauge::new(MetricType::ServerMemory); static QUEUE_COUNT: AtomicGauge = AtomicGauge::new(MetricType::QueueCount); static USER_COUNT: AtomicGauge = AtomicGauge::new(MetricType::UserCount); static DOMAIN_COUNT: AtomicGauge = AtomicGauge::new(MetricType::DomainCount); const CONN_SMTP_IN: usize = 0; const CONN_SMTP_OUT: usize = 1; const CONN_IMAP: usize = 2; const CONN_POP3: usize = 3; const CONN_HTTP: usize = 4; const CONN_SIEVE: usize = 5; const TOTAL_CONN_TYPES: usize = 6; pub struct ConnectionMetrics { pub active_connections: AtomicGauge, pub elapsed: AtomicHistogram<12>, } pub struct EventCounter { id: EventType, value: u32, } impl Collector { pub fn record_metric(event: EventType, event_id: usize, keys: &[(Key, Value)]) { // Increment the event counter if !event.is_span_end() && !event.is_raw_io() { EVENT_COUNTERS.add(event_id, 1); } // Extract variables let mut elapsed = 0; let mut size = 0; for (key, value) in keys { match (key, value) { (Key::Elapsed, Value::Duration(d)) => elapsed = *d, (Key::Size, Value::UInt(s)) => size = *s, _ => {} } } match event { EventType::Smtp(SmtpEvent::ConnectionStart) => { let conn = &CONNECTION_METRICS[CONN_SMTP_IN]; conn.active_connections.increment(); } EventType::Smtp(SmtpEvent::ConnectionEnd) => { let conn = &CONNECTION_METRICS[CONN_SMTP_IN]; conn.active_connections.decrement(); conn.elapsed.observe(elapsed); } EventType::Imap(ImapEvent::ConnectionStart) => { let conn = &CONNECTION_METRICS[CONN_IMAP]; conn.active_connections.increment(); } EventType::Imap(ImapEvent::ConnectionEnd) => { let conn = &CONNECTION_METRICS[CONN_IMAP]; conn.active_connections.decrement(); conn.elapsed.observe(elapsed); } EventType::Pop3(Pop3Event::ConnectionStart) => { let conn = &CONNECTION_METRICS[CONN_POP3]; conn.active_connections.increment(); } EventType::Pop3(Pop3Event::ConnectionEnd) => { let conn = &CONNECTION_METRICS[CONN_POP3]; conn.active_connections.decrement(); conn.elapsed.observe(elapsed); } EventType::Http(HttpEvent::ConnectionStart) => { let conn = &CONNECTION_METRICS[CONN_HTTP]; conn.active_connections.increment(); } EventType::Http(HttpEvent::ConnectionEnd) => { let conn = &CONNECTION_METRICS[CONN_HTTP]; conn.active_connections.decrement(); conn.elapsed.observe(elapsed); } EventType::ManageSieve(ManageSieveEvent::ConnectionStart) => { let conn = &CONNECTION_METRICS[CONN_SIEVE]; conn.active_connections.increment(); } EventType::ManageSieve(ManageSieveEvent::ConnectionEnd) => { let conn = &CONNECTION_METRICS[CONN_SIEVE]; conn.active_connections.decrement(); conn.elapsed.observe(elapsed); } EventType::Delivery(DeliveryEvent::AttemptStart) => { let conn = &CONNECTION_METRICS[CONN_SMTP_OUT]; conn.active_connections.increment(); } EventType::Delivery(DeliveryEvent::AttemptEnd) => { let conn = &CONNECTION_METRICS[CONN_SMTP_OUT]; conn.active_connections.decrement(); conn.elapsed.observe(elapsed); } EventType::Delivery(DeliveryEvent::Completed) => { QUEUE_COUNT.decrement(); MESSAGE_DELIVERY_TIME.observe(elapsed); } EventType::Delivery( DeliveryEvent::MxLookup | DeliveryEvent::IpLookup | DeliveryEvent::NullMx, ) | EventType::TlsRpt(_) | EventType::MtaSts(_) | EventType::Dane(_) => { if elapsed > 0 { DNS_LOOKUP_TIME.observe(elapsed); } } EventType::MessageIngest( MessageIngestEvent::Ham | MessageIngestEvent::Spam | MessageIngestEvent::ImapAppend | MessageIngestEvent::JmapAppend, ) => { MESSAGE_INGESTION_TIME.observe(elapsed); } EventType::Queue(QueueEvent::QueueMessage) => { MESSAGE_INCOMING_SIZE.observe(size); QUEUE_COUNT.increment(); } EventType::Queue(QueueEvent::QueueMessageAuthenticated) => { MESSAGE_SUBMISSION_SIZE.observe(size); QUEUE_COUNT.increment(); } EventType::Queue(QueueEvent::QueueReport) => { MESSAGE_OUT_REPORT_SIZE.observe(size); QUEUE_COUNT.increment(); } EventType::Queue(QueueEvent::QueueAutogenerated | QueueEvent::QueueDsn) => { QUEUE_COUNT.increment(); } EventType::MessageIngest(MessageIngestEvent::FtsIndex) => { MESSAGE_INDEX_TIME.observe(elapsed); } EventType::Store(StoreEvent::BlobWrite) => { STORE_BLOB_WRITE_TIME.observe(elapsed); } EventType::Store(StoreEvent::BlobRead) => { STORE_BLOB_READ_TIME.observe(elapsed); } EventType::Store(StoreEvent::DataWrite) => { STORE_DATA_WRITE_TIME.observe(elapsed); } EventType::Store(StoreEvent::DataIterate) => { STORE_DATA_READ_TIME.observe(elapsed); } _ => {} } } #[inline(always)] pub fn is_metric(event: impl Into) -> bool { METRIC_INTERESTS.get(event) } pub fn set_metrics(interests: Interests) { METRIC_INTERESTS.update(interests); } pub fn collect_counters(_is_enterprise: bool) -> impl Iterator { EVENT_COUNTERS .inner() .iter() .enumerate() .filter_map(|(event_id, value)| { let value = value.load(Ordering::Relaxed); if value > 0 { Some(EventCounter { id: EVENT_TYPES[event_id], value, }) } else { None } }) } pub fn collect_gauges(is_enterprise: bool) -> impl Iterator { static E_GAUGES: &[&AtomicGauge] = &[&SERVER_MEMORY, &QUEUE_COUNT, &USER_COUNT, &DOMAIN_COUNT]; static C_GAUGES: &[&AtomicGauge] = &[&SERVER_MEMORY, &USER_COUNT, &DOMAIN_COUNT]; if is_enterprise { E_GAUGES } else { C_GAUGES } .iter() .copied() .chain(CONNECTION_METRICS.iter().map(|m| &m.active_connections)) } pub fn collect_histograms( is_enterprise: bool, ) -> impl Iterator> { static E_HISTOGRAMS: &[&AtomicHistogram<12>] = &[ &MESSAGE_INGESTION_TIME, &MESSAGE_INDEX_TIME, &MESSAGE_DELIVERY_TIME, &MESSAGE_INCOMING_SIZE, &MESSAGE_SUBMISSION_SIZE, &MESSAGE_OUT_REPORT_SIZE, &STORE_DATA_READ_TIME, &STORE_DATA_WRITE_TIME, &STORE_BLOB_READ_TIME, &STORE_BLOB_WRITE_TIME, &DNS_LOOKUP_TIME, ]; static C_HISTOGRAMS: &[&AtomicHistogram<12>] = &[ &MESSAGE_DELIVERY_TIME, &MESSAGE_INCOMING_SIZE, &MESSAGE_SUBMISSION_SIZE, ]; if is_enterprise { E_HISTOGRAMS } else { C_HISTOGRAMS } .iter() .copied() .chain(CONNECTION_METRICS.iter().map(|m| &m.elapsed)) .filter(|h| h.is_active()) } #[inline(always)] pub fn read_event_metric(metric_id: usize) -> u32 { EVENT_COUNTERS.get(metric_id) } pub fn read_metric(metric_type: MetricType) -> f64 { match metric_type { MetricType::ServerMemory => SERVER_MEMORY.get() as f64, MetricType::MessageIngestionTime => MESSAGE_INGESTION_TIME.average(), MetricType::MessageFtsIndexTime => MESSAGE_INDEX_TIME.average(), MetricType::MessageSize => MESSAGE_INCOMING_SIZE.average(), MetricType::MessageAuthSize => MESSAGE_SUBMISSION_SIZE.average(), MetricType::DeliveryTotalTime => MESSAGE_DELIVERY_TIME.average(), MetricType::DeliveryTime => CONNECTION_METRICS[CONN_SMTP_OUT].elapsed.average(), MetricType::DeliveryActiveConnections => { CONNECTION_METRICS[CONN_SMTP_OUT].active_connections.get() as f64 } MetricType::QueueCount => QUEUE_COUNT.get() as f64, MetricType::ReportOutgoingSize => MESSAGE_OUT_REPORT_SIZE.average(), MetricType::StoreReadTime => STORE_DATA_READ_TIME.average(), MetricType::StoreWriteTime => STORE_DATA_WRITE_TIME.average(), MetricType::BlobReadTime => STORE_BLOB_READ_TIME.average(), MetricType::BlobWriteTime => STORE_BLOB_WRITE_TIME.average(), MetricType::DnsLookupTime => DNS_LOOKUP_TIME.average(), MetricType::HttpActiveConnections => { CONNECTION_METRICS[CONN_HTTP].active_connections.get() as f64 } MetricType::HttpRequestTime => CONNECTION_METRICS[CONN_HTTP].elapsed.average(), MetricType::ImapActiveConnections => { CONNECTION_METRICS[CONN_IMAP].active_connections.get() as f64 } MetricType::ImapRequestTime => CONNECTION_METRICS[CONN_IMAP].elapsed.average(), MetricType::Pop3ActiveConnections => { CONNECTION_METRICS[CONN_POP3].active_connections.get() as f64 } MetricType::Pop3RequestTime => CONNECTION_METRICS[CONN_POP3].elapsed.average(), MetricType::SmtpActiveConnections => { CONNECTION_METRICS[CONN_SMTP_IN].active_connections.get() as f64 } MetricType::SmtpRequestTime => CONNECTION_METRICS[CONN_SMTP_IN].elapsed.average(), MetricType::SieveActiveConnections => { CONNECTION_METRICS[CONN_SIEVE].active_connections.get() as f64 } MetricType::SieveRequestTime => CONNECTION_METRICS[CONN_SIEVE].elapsed.average(), MetricType::UserCount => USER_COUNT.get() as f64, MetricType::DomainCount => DOMAIN_COUNT.get() as f64, } } pub fn update_gauge(metric_type: MetricType, value: u64) { match metric_type { MetricType::ServerMemory => SERVER_MEMORY.set(value), MetricType::QueueCount => QUEUE_COUNT.set(value), MetricType::UserCount => USER_COUNT.set(value), MetricType::DomainCount => DOMAIN_COUNT.set(value), _ => {} } } pub fn update_event_counter(event_type: EventType, value: u32) { EVENT_COUNTERS.add(event_type.into(), value); } pub fn update_histogram(metric_type: MetricType, value: u64) { match metric_type { MetricType::MessageIngestionTime => MESSAGE_INGESTION_TIME.observe(value), MetricType::MessageFtsIndexTime => MESSAGE_INDEX_TIME.observe(value), MetricType::DeliveryTotalTime => MESSAGE_DELIVERY_TIME.observe(value), MetricType::DeliveryTime => CONNECTION_METRICS[CONN_SMTP_OUT].elapsed.observe(value), MetricType::DnsLookupTime => DNS_LOOKUP_TIME.observe(value), _ => {} } } } impl EventCounter { pub fn id(&self) -> EventType { self.id } pub fn value(&self) -> u64 { self.value as u64 } } impl ConnectionMetrics { #[allow(clippy::new_without_default)] pub const fn new() -> Self { Self { active_connections: AtomicGauge::new(MetricType::BlobReadTime), elapsed: AtomicHistogram::<18>::new_medium_durations(MetricType::BlobReadTime), } } } #[allow(clippy::declare_interior_mutable_const)] const fn init_conn_metrics() -> [ConnectionMetrics; TOTAL_CONN_TYPES] { const INIT: ConnectionMetrics = ConnectionMetrics::new(); let mut array = [INIT; TOTAL_CONN_TYPES]; let mut i = 0; while i < TOTAL_CONN_TYPES { let metric = match i { CONN_HTTP => &[ MetricType::HttpRequestTime, MetricType::HttpActiveConnections, ], CONN_IMAP => &[ MetricType::ImapRequestTime, MetricType::ImapActiveConnections, ], CONN_POP3 => &[ MetricType::Pop3RequestTime, MetricType::Pop3ActiveConnections, ], CONN_SMTP_IN => &[ MetricType::SmtpRequestTime, MetricType::SmtpActiveConnections, ], CONN_SMTP_OUT => &[ MetricType::DeliveryTime, MetricType::DeliveryActiveConnections, ], CONN_SIEVE => &[ MetricType::SieveRequestTime, MetricType::SieveActiveConnections, ], _ => &[MetricType::BlobReadTime, MetricType::BlobReadTime], }; array[i] = ConnectionMetrics { elapsed: AtomicHistogram::<18>::new_medium_durations(metric[0]), active_connections: AtomicGauge::new(metric[1]), }; i += 1; } array } impl EventType { pub fn is_metric(&self) -> bool { match self { EventType::Server(ServerEvent::ThreadError) => true, EventType::Purge(PurgeEvent::Error) => true, EventType::Eval( EvalEvent::Error | EvalEvent::StoreNotFound | EvalEvent::DirectoryNotFound, ) => true, EventType::Acme( AcmeEvent::TlsAlpnError | AcmeEvent::OrderCompleted | AcmeEvent::AuthError | AcmeEvent::AuthTooManyAttempts | AcmeEvent::DnsRecordCreationFailed | AcmeEvent::DnsRecordDeletionFailed | AcmeEvent::DnsRecordPropagationTimeout | AcmeEvent::ClientMissingSni | AcmeEvent::TokenNotFound | AcmeEvent::DnsRecordLookupFailed | AcmeEvent::OrderInvalid | AcmeEvent::Error, ) => true, EventType::Store( StoreEvent::AssertValueFailed | StoreEvent::FoundationdbError | StoreEvent::MysqlError | StoreEvent::PostgresqlError | StoreEvent::RocksdbError | StoreEvent::SqliteError | StoreEvent::LdapError | StoreEvent::ElasticsearchError | StoreEvent::RedisError | StoreEvent::S3Error | StoreEvent::AzureError | StoreEvent::FilesystemError | StoreEvent::PoolError | StoreEvent::DataCorruption | StoreEvent::DecompressError | StoreEvent::DeserializeError | StoreEvent::NotFound | StoreEvent::NotConfigured | StoreEvent::NotSupported | StoreEvent::UnexpectedError | StoreEvent::CryptoError | StoreEvent::BlobMissingMarker | StoreEvent::DataWrite | StoreEvent::DataIterate | StoreEvent::BlobRead | StoreEvent::BlobWrite | StoreEvent::BlobDelete | StoreEvent::HttpStoreError, ) => true, EventType::MessageIngest(_) => true, EventType::Jmap( JmapEvent::MethodCall | JmapEvent::WebsocketStart | JmapEvent::WebsocketError | JmapEvent::UnsupportedFilter | JmapEvent::UnsupportedSort | JmapEvent::Forbidden | JmapEvent::NotJson | JmapEvent::NotRequest | JmapEvent::InvalidArguments | JmapEvent::RequestTooLarge | JmapEvent::UnknownMethod, ) => true, EventType::Imap(ImapEvent::ConnectionStart | ImapEvent::ConnectionEnd) => true, EventType::ManageSieve( ManageSieveEvent::ConnectionStart | ManageSieveEvent::ConnectionEnd, ) => true, EventType::Pop3(Pop3Event::ConnectionStart | Pop3Event::ConnectionEnd) => true, EventType::Smtp( SmtpEvent::ConnectionStart | SmtpEvent::ConnectionEnd | SmtpEvent::Error | SmtpEvent::ConcurrencyLimitExceeded | SmtpEvent::TransferLimitExceeded | SmtpEvent::RateLimitExceeded | SmtpEvent::TimeLimitExceeded | SmtpEvent::MessageParseFailed | SmtpEvent::MessageTooLarge | SmtpEvent::LoopDetected | SmtpEvent::DkimPass | SmtpEvent::DkimFail | SmtpEvent::ArcPass | SmtpEvent::ArcFail | SmtpEvent::SpfEhloPass | SmtpEvent::SpfEhloFail | SmtpEvent::SpfFromPass | SmtpEvent::SpfFromFail | SmtpEvent::DmarcPass | SmtpEvent::DmarcFail | SmtpEvent::IprevPass | SmtpEvent::IprevFail | SmtpEvent::TooManyMessages | SmtpEvent::InvalidEhlo | SmtpEvent::DidNotSayEhlo | SmtpEvent::MailFromUnauthenticated | SmtpEvent::MailFromUnauthorized | SmtpEvent::MailFromMissing | SmtpEvent::MultipleMailFrom | SmtpEvent::MailboxDoesNotExist | SmtpEvent::RelayNotAllowed | SmtpEvent::RcptToDuplicate | SmtpEvent::RcptToMissing | SmtpEvent::TooManyRecipients | SmtpEvent::TooManyInvalidRcpt | SmtpEvent::AuthMechanismNotSupported | SmtpEvent::AuthExchangeTooLong | SmtpEvent::CommandNotImplemented | SmtpEvent::InvalidCommand | SmtpEvent::SyntaxError | SmtpEvent::RequestTooLarge, ) => true, EventType::Http( HttpEvent::Error | HttpEvent::RequestBody | HttpEvent::ResponseBody | HttpEvent::XForwardedMissing, ) => true, EventType::Network(NetworkEvent::Timeout) => true, EventType::Security(_) => true, EventType::Limit(_) => true, EventType::Manage(_) => false, EventType::Auth( AuthEvent::Success | AuthEvent::Failed | AuthEvent::TooManyAttempts | AuthEvent::Error, ) => true, EventType::Config(_) => false, EventType::Resource( ResourceEvent::NotFound | ResourceEvent::BadParameters | ResourceEvent::Error, ) => true, EventType::Arc( ArcEvent::ChainTooLong | ArcEvent::InvalidInstance | ArcEvent::InvalidCv | ArcEvent::HasHeaderTag | ArcEvent::BrokenChain, ) => true, EventType::Dkim(_) => true, EventType::Dmarc(_) => true, EventType::Iprev(_) => true, EventType::Dane( DaneEvent::AuthenticationSuccess | DaneEvent::AuthenticationFailure | DaneEvent::NoCertificatesFound | DaneEvent::CertificateParseError | DaneEvent::TlsaRecordFetchError | DaneEvent::TlsaRecordNotFound | DaneEvent::TlsaRecordNotDnssecSigned | DaneEvent::TlsaRecordInvalid, ) => true, EventType::Spf(_) => true, EventType::MailAuth(_) => true, EventType::Tls(TlsEvent::HandshakeError) => true, EventType::Sieve( SieveEvent::ActionAccept | SieveEvent::ActionAcceptReplace | SieveEvent::ActionDiscard | SieveEvent::ActionReject | SieveEvent::SendMessage | SieveEvent::MessageTooLarge | SieveEvent::RuntimeError | SieveEvent::UnexpectedError | SieveEvent::NotSupported | SieveEvent::QuotaExceeded, ) => true, EventType::Spam( SpamEvent::PyzorError | SpamEvent::TrainCompleted | SpamEvent::TrainSampleAdded | SpamEvent::Classify | SpamEvent::ModelNotReady | SpamEvent::DnsblError, ) => true, EventType::PushSubscription(_) => true, EventType::Cluster( ClusterEvent::SubscriberError | ClusterEvent::PublisherError | ClusterEvent::SubscriberDisconnected, ) => true, EventType::Housekeeper(_) => false, EventType::TaskQueue( TaskQueueEvent::BlobNotFound | TaskQueueEvent::MetadataNotFound, ) => true, EventType::Milter( MilterEvent::ActionAccept | MilterEvent::ActionDiscard | MilterEvent::ActionReject | MilterEvent::ActionTempFail | MilterEvent::ActionReplyCode | MilterEvent::ActionConnectionFailure | MilterEvent::ActionShutdown, ) => true, EventType::MtaHook(_) => true, EventType::Delivery( DeliveryEvent::AttemptStart | DeliveryEvent::Completed | DeliveryEvent::AttemptEnd | DeliveryEvent::MxLookupFailed | DeliveryEvent::IpLookupFailed | DeliveryEvent::NullMx | DeliveryEvent::GreetingFailed | DeliveryEvent::EhloRejected | DeliveryEvent::AuthFailed | DeliveryEvent::MailFromRejected | DeliveryEvent::Delivered | DeliveryEvent::RcptToRejected | DeliveryEvent::RcptToFailed | DeliveryEvent::MessageRejected | DeliveryEvent::StartTlsUnavailable | DeliveryEvent::StartTlsError | DeliveryEvent::StartTlsDisabled | DeliveryEvent::ImplicitTlsError | DeliveryEvent::ConcurrencyLimitExceeded | DeliveryEvent::RateLimitExceeded | DeliveryEvent::DoubleBounce | DeliveryEvent::DsnSuccess | DeliveryEvent::DsnTempFail | DeliveryEvent::DsnPermFail, ) => true, EventType::Queue( QueueEvent::QueueMessage | QueueEvent::QueueMessageAuthenticated | QueueEvent::QueueReport | QueueEvent::QueueDsn | QueueEvent::QueueAutogenerated | QueueEvent::Rescheduled | QueueEvent::BlobNotFound | QueueEvent::RateLimitExceeded | QueueEvent::ConcurrencyLimitExceeded | QueueEvent::QuotaExceeded, ) => true, EventType::TlsRpt(_) => false, EventType::MtaSts( MtaStsEvent::Authorized | MtaStsEvent::NotAuthorized | MtaStsEvent::InvalidPolicy, ) => true, EventType::IncomingReport(_) => true, EventType::OutgoingReport( OutgoingReportEvent::SpfReport | OutgoingReportEvent::SpfRateLimited | OutgoingReportEvent::DkimReport | OutgoingReportEvent::DkimRateLimited | OutgoingReportEvent::DmarcReport | OutgoingReportEvent::DmarcRateLimited | OutgoingReportEvent::DmarcAggregateReport | OutgoingReportEvent::TlsAggregate | OutgoingReportEvent::HttpSubmission | OutgoingReportEvent::UnauthorizedReportingAddress | OutgoingReportEvent::ReportingAddressValidationError | OutgoingReportEvent::NotFound | OutgoingReportEvent::SubmissionError | OutgoingReportEvent::NoRecipientsFound, ) => true, EventType::Telemetry( TelemetryEvent::LogError | TelemetryEvent::WebhookError | TelemetryEvent::OtelExporterError | TelemetryEvent::OtelMetricsExporterError | TelemetryEvent::PrometheusExporterError | TelemetryEvent::JournalError, ) => true, EventType::Calendar( CalendarEvent::AlarmSent | CalendarEvent::AlarmFailed | CalendarEvent::ItipMessageReceived | CalendarEvent::ItipMessageSent | CalendarEvent::ItipMessageError, ) => true, _ => false, } } } ================================================ FILE: crates/trc/src/ipc/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod bitset; pub mod channel; pub mod collector; pub mod metrics; pub mod subscriber; pub(crate) const USIZE_BITS: usize = std::mem::size_of::() * 8; pub(crate) const USIZE_BITS_MASK: usize = USIZE_BITS - 1; ================================================ FILE: crates/trc/src/ipc/subscriber.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::Arc; use tokio::sync::mpsc::{self, error::TrySendError}; use crate::{Event, EventDetails, EventType, Level, TOTAL_EVENT_COUNT}; use super::{ USIZE_BITS, bitset::Bitset, channel::ChannelError, collector::{COLLECTOR_UPDATES, Collector, Update}, }; const MAX_BATCH_SIZE: usize = 32768; pub type Interests = Box>; pub type EventBatch = Vec>>; #[derive(Debug)] pub(crate) struct Subscriber { pub id: String, pub interests: Interests, pub tx: mpsc::Sender, pub lossy: bool, pub batch: EventBatch, } pub struct SubscriberBuilder { pub id: String, pub interests: Interests, pub lossy: bool, } impl Subscriber { #[inline(always)] pub fn push_event(&mut self, event_id: usize, trace: Arc>) { if self.interests.get(event_id) { self.batch.push(trace); } } pub fn send_batch(&mut self) -> Result<(), ChannelError> { if !self.batch.is_empty() { match self .tx .try_send(std::mem::replace(&mut self.batch, Vec::with_capacity(128))) { Ok(_) => Ok(()), Err(TrySendError::Full(mut events)) => { if self.lossy && events.len() > MAX_BATCH_SIZE { events.retain(|e| e.inner.level == Level::Error); if events.len() > MAX_BATCH_SIZE { events.truncate(MAX_BATCH_SIZE); } } self.batch = events; Ok(()) } Err(TrySendError::Closed(_)) => Err(ChannelError), } } else { Ok(()) } } } impl SubscriberBuilder { pub fn new(id: String) -> Self { Self { id, interests: Default::default(), lossy: true, } } pub fn with_default_interests(mut self, level: Level) -> Self { for event in EventType::variants() { if event.level() >= level { self.interests.set(event); } } self } pub fn with_interests(mut self, interests: Interests) -> Self { self.interests = interests; self } pub fn set_interests(mut self, interest: impl IntoIterator>) -> Self { for level in interest { self.interests.set(level); } self } pub fn with_lossy(mut self, lossy: bool) -> Self { self.lossy = lossy; self } pub fn register(self) -> (mpsc::Sender, mpsc::Receiver) { let (tx, rx) = mpsc::channel(8192); COLLECTOR_UPDATES.lock().push(Update::RegisterSubscriber { subscriber: Subscriber { id: self.id, interests: self.interests, tx: tx.clone(), lossy: self.lossy, batch: Vec::new(), }, }); // Notify collector Collector::reload(); (tx, rx) } } ================================================ FILE: crates/trc/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod atomics; pub mod event; pub mod ipc; pub mod macros; pub mod serializers; use std::{ net::{IpAddr, Ipv4Addr, Ipv6Addr}, sync::Arc, }; pub use crate::ipc::collector::Collector; use compact_str::CompactString; pub use event_macro::event; use event_macro::{event_family, event_type, key_names, total_event_count}; pub type Result = std::result::Result; #[derive(Debug, Clone)] #[repr(transparent)] pub struct Error(Box>); #[derive(Debug, Clone)] pub struct Event { pub inner: T, pub keys: Vec<(Key, Value)>, } #[derive(Debug, Clone)] pub struct EventDetails { pub typ: EventType, pub timestamp: u64, pub level: Level, pub span: Option>>, } #[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)] #[repr(usize)] pub enum Level { Trace = 0, Debug = 1, Info = 2, Warn = 3, Error = 4, Disable = 5, } #[derive(Debug, Default, Clone)] pub enum Value { String(CompactString), UInt(u64), Int(i64), Float(f64), Timestamp(u64), Duration(u64), Bytes(Vec), Bool(bool), Ipv4(Ipv4Addr), Ipv6(Ipv6Addr), Event(Error), Array(Vec), #[default] None, } #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] #[key_names] pub enum Key { AccountName, AccountId, BlobId, #[default] CausedBy, ChangeId, Code, Collection, Contents, Details, DkimFail, DkimNone, DkimPass, DmarcNone, DmarcPass, DmarcQuarantine, DmarcReject, DocumentId, Domain, Due, Elapsed, Expires, From, Hostname, Id, Key, Limit, ListenerId, LocalIp, LocalPort, MailboxName, MailboxId, MessageId, NextDsn, NextRetry, Path, Policy, QueueId, RangeFrom, RangeTo, Reason, RemoteIp, RemotePort, ReportId, Result, Size, Source, SpanId, SpfFail, SpfNone, SpfPass, Strict, Tls, To, Total, TotalFailures, TotalSuccesses, Type, Uid, UidNext, UidValidity, Url, ValidFrom, ValidTo, Value, Version, QueueName, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[event_family] pub enum EventType { Server(ServerEvent), Purge(PurgeEvent), Eval(EvalEvent), Acme(AcmeEvent), Store(StoreEvent), MessageIngest(MessageIngestEvent), Jmap(JmapEvent), Imap(ImapEvent), ManageSieve(ManageSieveEvent), Pop3(Pop3Event), Smtp(SmtpEvent), Http(HttpEvent), Network(NetworkEvent), Limit(LimitEvent), Manage(ManageEvent), Auth(AuthEvent), Config(ConfigEvent), Resource(ResourceEvent), Arc(ArcEvent), Dkim(DkimEvent), Dmarc(DmarcEvent), Iprev(IprevEvent), Dane(DaneEvent), Spf(SpfEvent), MailAuth(MailAuthEvent), Tls(TlsEvent), Sieve(SieveEvent), Spam(SpamEvent), PushSubscription(PushSubscriptionEvent), Cluster(ClusterEvent), Housekeeper(HousekeeperEvent), TaskQueue(TaskQueueEvent), Milter(MilterEvent), MtaHook(MtaHookEvent), Delivery(DeliveryEvent), Queue(QueueEvent), TlsRpt(TlsRptEvent), MtaSts(MtaStsEvent), IncomingReport(IncomingReportEvent), OutgoingReport(OutgoingReportEvent), Telemetry(TelemetryEvent), Security(SecurityEvent), Ai(AiEvent), WebDav(WebDavEvent), Calendar(CalendarEvent), } #[event_type] pub enum HttpEvent { ConnectionStart, ConnectionEnd, Error, RequestUrl, RequestBody, ResponseBody, XForwardedMissing, } #[event_type] pub enum SecurityEvent { AuthenticationBan, AbuseBan, ScanBan, LoiterBan, IpBlocked, Unauthorized, } #[event_type] pub enum ClusterEvent { SubscriberStart, SubscriberStop, SubscriberError, SubscriberDisconnected, PublisherStart, PublisherStop, PublisherError, MessageReceived, MessageSkipped, MessageInvalid, } #[event_type] pub enum HousekeeperEvent { Start, Stop, Schedule, Run, } #[event_type] pub enum TaskQueueEvent { TaskAcquired, TaskLocked, TaskIgnored, TaskFailed, BlobNotFound, MetadataNotFound, } #[event_type] pub enum ImapEvent { ConnectionStart, ConnectionEnd, // Commands GetAcl, SetAcl, MyRights, ListRights, Append, Capabilities, Id, Close, Copy, Move, CreateMailbox, DeleteMailbox, RenameMailbox, Enable, Expunge, Fetch, IdleStart, IdleStop, List, Lsub, Logout, Namespace, Noop, Search, Sort, Select, Status, Store, Subscribe, Unsubscribe, Thread, GetQuota, // Errors Error, // Debugging RawInput, RawOutput, } #[event_type] pub enum Pop3Event { ConnectionStart, ConnectionEnd, // Commands Delete, Reset, Quit, Fetch, List, ListMessage, Uidl, UidlMessage, Stat, Noop, Capabilities, StartTls, Utf8, // Errors Error, // Debugging RawInput, RawOutput, } #[event_type] pub enum ManageSieveEvent { ConnectionStart, ConnectionEnd, // Commands CreateScript, UpdateScript, GetScript, DeleteScript, RenameScript, CheckScript, HaveSpace, ListScripts, SetActive, Capabilities, StartTls, Unauthenticate, Logout, Noop, // Errors Error, // Debugging RawInput, RawOutput, } #[event_type] pub enum SmtpEvent { ConnectionStart, ConnectionEnd, Error, IdNotFound, ConcurrencyLimitExceeded, TransferLimitExceeded, RateLimitExceeded, TimeLimitExceeded, MissingAuthDirectory, MessageParseFailed, MessageTooLarge, LoopDetected, DkimPass, DkimFail, ArcPass, ArcFail, SpfEhloPass, SpfEhloFail, SpfFromPass, SpfFromFail, DmarcPass, DmarcFail, IprevPass, IprevFail, TooManyMessages, Ehlo, InvalidEhlo, DidNotSayEhlo, EhloExpected, LhloExpected, MailFromUnauthenticated, MailFromUnauthorized, MailFromNotAllowed, MailFromRewritten, MailFromMissing, MailFrom, MultipleMailFrom, MailboxDoesNotExist, RelayNotAllowed, RcptTo, RcptToDuplicate, RcptToRewritten, RcptToMissing, RcptToGreylisted, TooManyRecipients, TooManyInvalidRcpt, RawInput, RawOutput, MissingLocalHostname, Vrfy, VrfyNotFound, VrfyDisabled, Expn, ExpnNotFound, ExpnDisabled, RequireTlsDisabled, DeliverByDisabled, DeliverByInvalid, FutureReleaseDisabled, FutureReleaseInvalid, MtPriorityDisabled, MtPriorityInvalid, DsnDisabled, AuthNotAllowed, AuthMechanismNotSupported, AuthExchangeTooLong, AlreadyAuthenticated, Noop, StartTls, StartTlsUnavailable, StartTlsAlready, Rset, Quit, Help, CommandNotImplemented, InvalidCommand, InvalidSenderAddress, InvalidRecipientAddress, InvalidParameter, UnsupportedParameter, SyntaxError, RequestTooLarge, } #[event_type] pub enum DeliveryEvent { AttemptStart, AttemptEnd, Completed, Failed, DomainDeliveryStart, MxLookup, MxLookupFailed, IpLookup, IpLookupFailed, NullMx, Connect, ConnectError, MissingOutboundHostname, GreetingFailed, Ehlo, EhloRejected, Auth, AuthFailed, MailFrom, MailFromRejected, Delivered, RcptTo, RcptToRejected, RcptToFailed, MessageRejected, StartTls, StartTlsUnavailable, StartTlsError, StartTlsDisabled, ImplicitTlsError, ConcurrencyLimitExceeded, RateLimitExceeded, DoubleBounce, DsnSuccess, DsnTempFail, DsnPermFail, RawInput, RawOutput, } #[event_type] pub enum QueueEvent { QueueMessage, QueueMessageAuthenticated, QueueReport, QueueDsn, QueueAutogenerated, Rescheduled, Locked, BlobNotFound, RateLimitExceeded, ConcurrencyLimitExceeded, QuotaExceeded, BackPressure, } #[event_type] pub enum IncomingReportEvent { DmarcReport, DmarcReportWithWarnings, TlsReport, TlsReportWithWarnings, AbuseReport, AuthFailureReport, FraudReport, NotSpamReport, VirusReport, OtherReport, MessageParseFailed, DmarcParseFailed, TlsRpcParseFailed, ArfParseFailed, DecompressError, } #[event_type] pub enum OutgoingReportEvent { SpfReport, SpfRateLimited, DkimReport, DkimRateLimited, DmarcReport, DmarcRateLimited, DmarcAggregateReport, TlsAggregate, HttpSubmission, UnauthorizedReportingAddress, ReportingAddressValidationError, NotFound, SubmissionError, NoRecipientsFound, Locked, } #[event_type] pub enum MtaStsEvent { Authorized, NotAuthorized, PolicyFetch, PolicyNotFound, PolicyFetchError, InvalidPolicy, } #[event_type] pub enum TlsRptEvent { RecordFetch, RecordFetchError, RecordNotFound, } #[event_type] pub enum DaneEvent { AuthenticationSuccess, AuthenticationFailure, NoCertificatesFound, CertificateParseError, TlsaRecordMatch, TlsaRecordFetch, TlsaRecordFetchError, TlsaRecordNotFound, TlsaRecordNotDnssecSigned, TlsaRecordInvalid, } #[event_type] pub enum MilterEvent { Read, Write, ActionAccept, ActionDiscard, ActionReject, ActionTempFail, ActionReplyCode, ActionConnectionFailure, ActionShutdown, IoError, FrameTooLarge, FrameInvalid, UnexpectedResponse, Timeout, TlsInvalidName, Disconnected, ParseError, } #[event_type] pub enum MtaHookEvent { ActionAccept, ActionDiscard, ActionReject, ActionQuarantine, Error, } #[event_type] pub enum PushSubscriptionEvent { Success, Error, NotFound, } #[event_type] pub enum SpamEvent { Pyzor, PyzorError, Dnsbl, DnsblError, TrainStarted, TrainCompleted, TrainSampleAdded, TrainSampleNotFound, Classify, ModelLoaded, ModelNotReady, ModelNotFound, } #[event_type] pub enum SieveEvent { ActionAccept, ActionAcceptReplace, ActionDiscard, ActionReject, SendMessage, MessageTooLarge, ScriptNotFound, ListNotFound, RuntimeError, UnexpectedError, NotSupported, QuotaExceeded, } #[event_type] pub enum TlsEvent { Handshake, HandshakeError, NotConfigured, CertificateNotFound, NoCertificatesAvailable, MultipleCertificatesAvailable, } #[event_type] pub enum NetworkEvent { ListenStart, ListenStop, ListenError, BindError, ReadError, WriteError, FlushError, AcceptError, SplitError, Timeout, Closed, ProxyError, SetOptError, } #[event_type] pub enum ServerEvent { Startup, Shutdown, StartupError, ThreadError, Licensing, } #[event_type] pub enum TelemetryEvent { Alert, LogError, WebhookError, OtelExporterError, OtelMetricsExporterError, PrometheusExporterError, JournalError, } #[event_type] pub enum AcmeEvent { AuthStart, AuthPending, AuthValid, AuthCompleted, AuthError, AuthTooManyAttempts, ProcessCert, OrderStart, OrderProcessing, OrderCompleted, OrderReady, OrderValid, OrderInvalid, RenewBackoff, DnsRecordCreated, DnsRecordCreationFailed, DnsRecordDeletionFailed, DnsRecordNotPropagated, DnsRecordLookupFailed, DnsRecordPropagated, DnsRecordPropagationTimeout, ClientSuppliedSni, ClientMissingSni, TlsAlpnReceived, TlsAlpnError, TokenNotFound, Error, } #[event_type] pub enum PurgeEvent { Started, Finished, Running, Error, InProgress, AutoExpunge, BlobCleanup, } #[event_type] pub enum EvalEvent { Result, Error, DirectoryNotFound, StoreNotFound, } #[event_type] pub enum ConfigEvent { ParseError, BuildError, MacroError, WriteError, FetchError, DefaultApplied, MissingSetting, UnusedSetting, ParseWarning, BuildWarning, ImportExternal, AlreadyUpToDate, } #[event_type] pub enum ArcEvent { ChainTooLong, InvalidInstance, InvalidCv, HasHeaderTag, BrokenChain, SealerNotFound, } #[event_type] pub enum DkimEvent { Pass, Neutral, Fail, PermError, TempError, None, UnsupportedVersion, UnsupportedAlgorithm, UnsupportedCanonicalization, UnsupportedKeyType, FailedBodyHashMatch, FailedVerification, FailedAuidMatch, RevokedPublicKey, IncompatibleAlgorithms, SignatureExpired, SignatureLength, SignerNotFound, } #[event_type] pub enum SpfEvent { Pass, Fail, SoftFail, Neutral, TempError, PermError, None, } #[event_type] pub enum DmarcEvent { Pass, Fail, PermError, TempError, None, } #[event_type] pub enum IprevEvent { Pass, Fail, PermError, TempError, None, } #[event_type] pub enum MailAuthEvent { ParseError, MissingParameters, NoHeadersFound, Crypto, Io, Base64, DnsError, DnsRecordNotFound, DnsInvalidRecordType, PolicyNotAligned, } #[event_type] pub enum StoreEvent { // Errors AssertValueFailed, FoundationdbError, MysqlError, PostgresqlError, RocksdbError, SqliteError, LdapError, ElasticsearchError, MeilisearchError, RedisError, S3Error, AzureError, FilesystemError, PoolError, DataCorruption, DecompressError, DeserializeError, NotFound, NotConfigured, NotSupported, UnexpectedError, CryptoError, HttpStoreError, // Caching CacheMiss, CacheHit, CacheStale, CacheUpdate, // Warnings BlobMissingMarker, // Traces DataWrite, DataIterate, BlobRead, BlobWrite, BlobDelete, SqlQuery, LdapQuery, LdapWarning, HttpStoreFetch, } #[event_type] pub enum MessageIngestEvent { // Events Ham, Spam, ImapAppend, JmapAppend, Duplicate, Error, FtsIndex, } #[event_type] pub enum JmapEvent { // Calls MethodCall, // Method errors InvalidArguments, RequestTooLarge, StateMismatch, AnchorNotFound, UnsupportedFilter, UnsupportedSort, UnknownMethod, InvalidResultReference, Forbidden, AccountNotFound, AccountNotSupportedByMethod, AccountReadOnly, NotFound, CannotCalculateChanges, UnknownDataType, // Request errors UnknownCapability, NotJson, NotRequest, // Not JMAP standard WebsocketStart, WebsocketStop, WebsocketError, } #[event_type] pub enum LimitEvent { SizeRequest, SizeUpload, CallsIn, ConcurrentRequest, ConcurrentUpload, ConcurrentConnection, // Used by listener Quota, BlobQuota, TenantQuota, TooManyRequests, } #[event_type] pub enum ManageEvent { MissingParameter, AlreadyExists, AssertFailed, NotFound, NotSupported, Error, } #[event_type] pub enum AuthEvent { Success, Failed, TokenExpired, MissingTotp, TooManyAttempts, ClientRegistration, Error, } #[event_type] pub enum ResourceEvent { NotFound, BadParameters, Error, DownloadExternal, WebadminUnpacked, } #[event_type] pub enum AiEvent { LlmResponse, ApiError, } #[event_type] pub enum WebDavEvent { // Requests Propfind, Proppatch, Get, Head, Report, Mkcol, Mkcalendar, Delete, Put, Post, Patch, Copy, Move, Lock, Unlock, Acl, Options, // Errors Error, } #[event_type] pub enum CalendarEvent { RuleExpansionError, AlarmSent, AlarmSkipped, AlarmRecipientOverride, AlarmFailed, ItipMessageSent, ItipMessageReceived, ItipMessageError, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum MetricType { ServerMemory, MessageIngestionTime, MessageFtsIndexTime, MessageSize, MessageAuthSize, DeliveryTotalTime, DeliveryTime, DeliveryActiveConnections, QueueCount, ReportOutgoingSize, StoreReadTime, StoreWriteTime, BlobReadTime, BlobWriteTime, DnsLookupTime, HttpActiveConnections, HttpRequestTime, ImapActiveConnections, ImapRequestTime, Pop3ActiveConnections, Pop3RequestTime, SmtpActiveConnections, SmtpRequestTime, SieveActiveConnections, SieveRequestTime, UserCount, DomainCount, } pub const TOTAL_EVENT_COUNT: usize = total_event_count!(); pub trait AddContext { fn caused_by(self, location: &'static str) -> Result; fn add_context(self, f: F) -> Result where F: FnOnce(Error) -> Error; } ================================================ FILE: crates/trc/src/macros.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ #[macro_export] macro_rules! location { () => {{ concat!(file!(), ":", line!()) }}; } #[macro_export] macro_rules! bail { ($err:expr $(,)?) => { return Err($err); }; } #[macro_export] macro_rules! error { ($err:expr $(,)?) => { let err = $err; let event_id = err.as_ref().id(); if $crate::Collector::is_metric(event_id) { $crate::Collector::record_metric(*err.as_ref(), event_id, err.keys()); } if $crate::Collector::has_interest(event_id) { err.send(); } }; } ================================================ FILE: crates/trc/src/serializers/binary.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: LicenseRef-SEL * * This file is subject to the Stalwart Enterprise License Agreement (SEL) and * is NOT open source software. * */ use crate::*; use compact_str::format_compact; use std::net::{Ipv4Addr, Ipv6Addr}; const VERSION: u8 = 1; pub fn serialize_events<'x>( events: impl IntoIterator>, num_events: usize, ) -> Vec { let mut buf = Vec::with_capacity(num_events * 64); buf.push(VERSION); leb128_write(&mut buf, num_events as u64); for event in events { event.serialize(&mut buf); } buf } pub fn deserialize_events(bytes: &[u8]) -> crate::Result>> { let mut iter = bytes.iter(); if *iter.next().ok_or_else(|| { StoreEvent::DataCorruption .caused_by(crate::location!()) .details("EOF while reading version") })? != VERSION { crate::bail!( StoreEvent::DataCorruption .caused_by(crate::location!()) .details("Invalid version") ); } let len = leb128_read(&mut iter).ok_or_else(|| { StoreEvent::DataCorruption .caused_by(crate::location!()) .details("EOF while size") })? as usize; let mut events = Vec::with_capacity(len); for n in 0..len { events.push(Event::deserialize(&mut iter).ok_or_else(|| { StoreEvent::DataCorruption .caused_by(crate::location!()) .details(format_compact!("Failed to deserialize event {n}")) })?); } Ok(events) } pub fn deserialize_single_event(bytes: &[u8]) -> crate::Result> { let mut iter = bytes.iter(); if *iter.next().ok_or_else(|| { StoreEvent::DataCorruption .caused_by(crate::location!()) .details("EOF while reading version") })? != VERSION { crate::bail!( StoreEvent::DataCorruption .caused_by(crate::location!()) .details("Invalid version") ); } let _ = leb128_read(&mut iter).ok_or_else(|| { StoreEvent::DataCorruption .caused_by(crate::location!()) .details("EOF while size") })?; Event::deserialize(&mut iter).ok_or_else(|| { StoreEvent::DataCorruption .caused_by(crate::location!()) .details("Failed to deserialize event") }) } impl Event { pub fn serialize(&self, buf: &mut Vec) { leb128_write(buf, self.inner.typ.code()); buf.extend_from_slice(self.inner.timestamp.to_le_bytes().as_ref()); leb128_write(buf, self.keys.len() as u64); for (k, v) in &self.keys { leb128_write(buf, k.code()); v.serialize(buf); } } pub fn deserialize<'x>(iter: &mut impl Iterator) -> Option { let typ = EventType::from_code(leb128_read(iter)?)?; let timestamp = u64::from_le_bytes([ *iter.next()?, *iter.next()?, *iter.next()?, *iter.next()?, *iter.next()?, *iter.next()?, *iter.next()?, *iter.next()?, ]); let keys_len = leb128_read(iter)?; let mut keys = Vec::with_capacity(keys_len as usize); for _ in 0..keys_len { let key = Key::from_code(leb128_read(iter)?)?; let value = Value::deserialize(iter)?; keys.push((key, value)); } Some(Event { inner: EventDetails { typ, timestamp, level: Level::Info, span: None, }, keys, }) } } impl Value { fn serialize(&self, buf: &mut Vec) { match self { Value::String(v) => { buf.push(0u8); leb128_write(buf, v.len() as u64); buf.extend(v.as_bytes()); } Value::UInt(v) => { buf.push(1u8); leb128_write(buf, *v); } Value::Int(v) => { buf.push(2u8); buf.extend(&v.to_le_bytes()); } Value::Float(v) => { buf.push(3u8); buf.extend(&v.to_le_bytes()); } Value::Timestamp(v) => { buf.push(4u8); buf.extend(&v.to_le_bytes()); } Value::Duration(v) => { buf.push(5u8); leb128_write(buf, *v); } Value::Bytes(v) => { buf.push(6u8); leb128_write(buf, v.len() as u64); buf.extend(v); } Value::Bool(true) => { buf.push(7u8); } Value::Bool(false) => { buf.push(8u8); } Value::Ipv4(v) => { buf.push(9u8); buf.extend(&v.octets()); } Value::Ipv6(v) => { buf.push(10u8); buf.extend(&v.octets()); } Value::Event(v) => { buf.push(11u8); leb128_write(buf, v.0.inner.code()); leb128_write(buf, v.0.keys.len() as u64); for (k, v) in &v.0.keys { leb128_write(buf, k.code()); v.serialize(buf); } } Value::Array(v) => { buf.push(12u8); leb128_write(buf, v.len() as u64); for value in v { value.serialize(buf); } } Value::None => { buf.push(13u8); } } } fn deserialize<'x>(iter: &mut impl Iterator) -> Option { match iter.next()? { 0 => { let mut buf = vec![0u8; leb128_read(iter)? as usize]; for byte in buf.iter_mut() { *byte = *iter.next()?; } Some(Value::String(CompactString::from_utf8(buf).ok()?)) } 1 => Some(Value::UInt(leb128_read(iter)?)), 2 => { let mut buf = [0u8; std::mem::size_of::()]; for byte in buf.iter_mut() { *byte = *iter.next()?; } Some(Value::Int(i64::from_le_bytes(buf))) } 3 => { let mut buf = [0u8; std::mem::size_of::()]; for byte in buf.iter_mut() { *byte = *iter.next()?; } Some(Value::Float(f64::from_le_bytes(buf))) } 4 => { let mut buf = [0u8; std::mem::size_of::()]; for byte in buf.iter_mut() { *byte = *iter.next()?; } Some(Value::Timestamp(u64::from_le_bytes(buf))) } 5 => Some(Value::Duration(leb128_read(iter)?)), 6 => { let mut buf = vec![0u8; leb128_read(iter)? as usize]; for byte in buf.iter_mut() { *byte = *iter.next()?; } Some(Value::Bytes(buf)) } 7 => Some(Value::Bool(true)), 8 => Some(Value::Bool(false)), 9 => { let mut buf = [0u8; 4]; for byte in buf.iter_mut() { *byte = *iter.next()?; } Some(Value::Ipv4(Ipv4Addr::from(buf))) } 10 => { let mut buf = [0u8; 16]; for byte in buf.iter_mut() { *byte = *iter.next()?; } Some(Value::Ipv6(Ipv6Addr::from(buf))) } 11 => { let code = EventType::from_code(leb128_read(iter)?)?; let keys_len = leb128_read(iter)?; let mut keys = Vec::with_capacity(keys_len as usize); for _ in 0..keys_len { let key = Key::from_code(leb128_read(iter)?)?; let value = Value::deserialize(iter)?; keys.push((key, value)); } Some(Value::Event(Error( Event::with_keys(code, keys).into_boxed(), ))) } 12 => { let len = leb128_read(iter)?; let mut values = Vec::with_capacity(len as usize); for _ in 0..len { values.push(Value::deserialize(iter)?); } Some(Value::Array(values)) } 13 => Some(Value::None), _ => None, } } } fn leb128_write(buf: &mut Vec, mut value: u64) { loop { if value < 0x80 { buf.push(value as u8); break; } else { buf.push(((value & 0x7f) | 0x80) as u8); value >>= 7; } } } fn leb128_read<'x>(iter: &mut impl Iterator) -> Option { let mut result = 0; for shift in [0, 7, 14, 21, 28, 35, 42, 49, 56, 63] { let byte = iter.next()?; if (byte & 0x80) == 0 { result |= (*byte as u64) << shift; return Some(result); } else { result |= ((byte & 0x7F) as u64) << shift; } } None } impl EventType { pub fn code(&self) -> u64 { match self { EventType::Acme(AcmeEvent::AuthCompleted) => 0, EventType::Acme(AcmeEvent::AuthError) => 1, EventType::Acme(AcmeEvent::AuthPending) => 2, EventType::Acme(AcmeEvent::AuthStart) => 3, EventType::Acme(AcmeEvent::AuthTooManyAttempts) => 4, EventType::Acme(AcmeEvent::AuthValid) => 5, EventType::Acme(AcmeEvent::ClientMissingSni) => 6, EventType::Acme(AcmeEvent::ClientSuppliedSni) => 7, EventType::Acme(AcmeEvent::DnsRecordCreated) => 8, EventType::Acme(AcmeEvent::DnsRecordCreationFailed) => 9, EventType::Acme(AcmeEvent::DnsRecordDeletionFailed) => 10, EventType::Acme(AcmeEvent::DnsRecordLookupFailed) => 11, EventType::Acme(AcmeEvent::DnsRecordNotPropagated) => 12, EventType::Acme(AcmeEvent::DnsRecordPropagated) => 13, EventType::Acme(AcmeEvent::DnsRecordPropagationTimeout) => 14, EventType::Acme(AcmeEvent::Error) => 15, EventType::Acme(AcmeEvent::OrderCompleted) => 16, EventType::Acme(AcmeEvent::OrderInvalid) => 17, EventType::Acme(AcmeEvent::OrderProcessing) => 18, EventType::Acme(AcmeEvent::OrderReady) => 19, EventType::Acme(AcmeEvent::OrderStart) => 20, EventType::Acme(AcmeEvent::OrderValid) => 21, EventType::Acme(AcmeEvent::ProcessCert) => 22, EventType::Acme(AcmeEvent::RenewBackoff) => 23, EventType::Acme(AcmeEvent::TlsAlpnError) => 24, EventType::Acme(AcmeEvent::TlsAlpnReceived) => 25, EventType::Acme(AcmeEvent::TokenNotFound) => 26, EventType::Arc(ArcEvent::BrokenChain) => 27, EventType::Arc(ArcEvent::ChainTooLong) => 28, EventType::Arc(ArcEvent::HasHeaderTag) => 29, EventType::Arc(ArcEvent::InvalidCv) => 30, EventType::Arc(ArcEvent::InvalidInstance) => 31, EventType::Arc(ArcEvent::SealerNotFound) => 32, EventType::Security(SecurityEvent::AuthenticationBan) => 33, EventType::Auth(AuthEvent::Error) => 34, EventType::Auth(AuthEvent::Failed) => 35, EventType::Auth(AuthEvent::MissingTotp) => 36, EventType::Auth(AuthEvent::Success) => 37, EventType::Auth(AuthEvent::TooManyAttempts) => 38, EventType::Cluster(ClusterEvent::SubscriberStart) => 39, EventType::Cluster(ClusterEvent::SubscriberStop) => 40, EventType::Cluster(ClusterEvent::SubscriberError) => 41, EventType::Cluster(ClusterEvent::SubscriberDisconnected) => 42, EventType::Cluster(ClusterEvent::PublisherStart) => 43, EventType::Cluster(ClusterEvent::PublisherStop) => 44, EventType::Cluster(ClusterEvent::PublisherError) => 45, EventType::Cluster(ClusterEvent::MessageReceived) => 46, EventType::Cluster(ClusterEvent::MessageSkipped) => 47, EventType::Cluster(ClusterEvent::MessageInvalid) => 49, EventType::Config(ConfigEvent::AlreadyUpToDate) => 53, EventType::Config(ConfigEvent::BuildError) => 54, EventType::Config(ConfigEvent::BuildWarning) => 55, EventType::Config(ConfigEvent::DefaultApplied) => 56, EventType::Config(ConfigEvent::FetchError) => 58, EventType::Config(ConfigEvent::ImportExternal) => 59, EventType::Config(ConfigEvent::MacroError) => 60, EventType::Config(ConfigEvent::MissingSetting) => 61, EventType::Config(ConfigEvent::ParseError) => 62, EventType::Config(ConfigEvent::ParseWarning) => 63, EventType::Config(ConfigEvent::UnusedSetting) => 64, EventType::Config(ConfigEvent::WriteError) => 65, EventType::Dane(DaneEvent::AuthenticationFailure) => 66, EventType::Dane(DaneEvent::AuthenticationSuccess) => 67, EventType::Dane(DaneEvent::CertificateParseError) => 68, EventType::Dane(DaneEvent::NoCertificatesFound) => 69, EventType::Dane(DaneEvent::TlsaRecordFetch) => 70, EventType::Dane(DaneEvent::TlsaRecordFetchError) => 71, EventType::Dane(DaneEvent::TlsaRecordInvalid) => 72, EventType::Dane(DaneEvent::TlsaRecordMatch) => 73, EventType::Dane(DaneEvent::TlsaRecordNotDnssecSigned) => 74, EventType::Dane(DaneEvent::TlsaRecordNotFound) => 75, EventType::Delivery(DeliveryEvent::AttemptEnd) => 76, EventType::Delivery(DeliveryEvent::AttemptStart) => 77, EventType::Delivery(DeliveryEvent::Auth) => 78, EventType::Delivery(DeliveryEvent::AuthFailed) => 79, EventType::Delivery(DeliveryEvent::Completed) => 80, EventType::Delivery(DeliveryEvent::ConcurrencyLimitExceeded) => 81, EventType::Delivery(DeliveryEvent::Connect) => 82, EventType::Delivery(DeliveryEvent::ConnectError) => 83, EventType::Delivery(DeliveryEvent::Delivered) => 84, EventType::Delivery(DeliveryEvent::DomainDeliveryStart) => 85, EventType::Delivery(DeliveryEvent::DoubleBounce) => 86, EventType::Delivery(DeliveryEvent::DsnPermFail) => 87, EventType::Delivery(DeliveryEvent::DsnSuccess) => 88, EventType::Delivery(DeliveryEvent::DsnTempFail) => 89, EventType::Delivery(DeliveryEvent::Ehlo) => 90, EventType::Delivery(DeliveryEvent::EhloRejected) => 91, EventType::Delivery(DeliveryEvent::Failed) => 92, EventType::Delivery(DeliveryEvent::GreetingFailed) => 93, EventType::Delivery(DeliveryEvent::ImplicitTlsError) => 94, EventType::Delivery(DeliveryEvent::IpLookup) => 95, EventType::Delivery(DeliveryEvent::IpLookupFailed) => 96, EventType::Delivery(DeliveryEvent::MailFrom) => 97, EventType::Delivery(DeliveryEvent::MailFromRejected) => 98, EventType::Delivery(DeliveryEvent::MessageRejected) => 99, EventType::Delivery(DeliveryEvent::MissingOutboundHostname) => 100, EventType::Delivery(DeliveryEvent::MxLookup) => 101, EventType::Delivery(DeliveryEvent::MxLookupFailed) => 102, EventType::Delivery(DeliveryEvent::NullMx) => 103, EventType::Delivery(DeliveryEvent::RateLimitExceeded) => 104, EventType::Delivery(DeliveryEvent::RawInput) => 105, EventType::Delivery(DeliveryEvent::RawOutput) => 106, EventType::Delivery(DeliveryEvent::RcptTo) => 107, EventType::Delivery(DeliveryEvent::RcptToFailed) => 108, EventType::Delivery(DeliveryEvent::RcptToRejected) => 109, EventType::Delivery(DeliveryEvent::StartTls) => 110, EventType::Delivery(DeliveryEvent::StartTlsDisabled) => 111, EventType::Delivery(DeliveryEvent::StartTlsError) => 112, EventType::Delivery(DeliveryEvent::StartTlsUnavailable) => 113, EventType::Dkim(DkimEvent::Fail) => 114, EventType::Dkim(DkimEvent::FailedAuidMatch) => 115, EventType::Dkim(DkimEvent::FailedBodyHashMatch) => 116, EventType::Dkim(DkimEvent::FailedVerification) => 117, EventType::Dkim(DkimEvent::IncompatibleAlgorithms) => 118, EventType::Dkim(DkimEvent::Neutral) => 119, EventType::Dkim(DkimEvent::None) => 120, EventType::Dkim(DkimEvent::Pass) => 121, EventType::Dkim(DkimEvent::PermError) => 122, EventType::Dkim(DkimEvent::RevokedPublicKey) => 123, EventType::Dkim(DkimEvent::SignatureExpired) => 124, EventType::Dkim(DkimEvent::SignatureLength) => 125, EventType::Dkim(DkimEvent::SignerNotFound) => 126, EventType::Dkim(DkimEvent::TempError) => 127, EventType::Dkim(DkimEvent::UnsupportedAlgorithm) => 128, EventType::Dkim(DkimEvent::UnsupportedCanonicalization) => 129, EventType::Dkim(DkimEvent::UnsupportedKeyType) => 130, EventType::Dkim(DkimEvent::UnsupportedVersion) => 131, EventType::Dmarc(DmarcEvent::Fail) => 132, EventType::Dmarc(DmarcEvent::None) => 133, EventType::Dmarc(DmarcEvent::Pass) => 134, EventType::Dmarc(DmarcEvent::PermError) => 135, EventType::Dmarc(DmarcEvent::TempError) => 136, EventType::Eval(EvalEvent::DirectoryNotFound) => 137, EventType::Eval(EvalEvent::Error) => 138, EventType::Eval(EvalEvent::Result) => 139, EventType::Eval(EvalEvent::StoreNotFound) => 140, EventType::TaskQueue(TaskQueueEvent::BlobNotFound) => 141, EventType::MessageIngest(MessageIngestEvent::FtsIndex) => 142, EventType::Spam(SpamEvent::TrainSampleAdded) => 143, EventType::TaskQueue(TaskQueueEvent::TaskLocked) => 144, EventType::TaskQueue(TaskQueueEvent::MetadataNotFound) => 145, EventType::Housekeeper(HousekeeperEvent::Run) => 146, EventType::Housekeeper(HousekeeperEvent::Schedule) => 149, EventType::Housekeeper(HousekeeperEvent::Start) => 150, EventType::Housekeeper(HousekeeperEvent::Stop) => 151, EventType::Http(HttpEvent::ConnectionEnd) => 152, EventType::Http(HttpEvent::ConnectionStart) => 153, EventType::Http(HttpEvent::Error) => 154, EventType::Http(HttpEvent::RequestBody) => 155, EventType::Http(HttpEvent::RequestUrl) => 156, EventType::Http(HttpEvent::ResponseBody) => 157, EventType::Http(HttpEvent::XForwardedMissing) => 158, EventType::Imap(ImapEvent::Append) => 159, EventType::Imap(ImapEvent::Capabilities) => 160, EventType::Imap(ImapEvent::Close) => 161, EventType::Imap(ImapEvent::ConnectionEnd) => 162, EventType::Imap(ImapEvent::ConnectionStart) => 163, EventType::Imap(ImapEvent::Copy) => 164, EventType::Imap(ImapEvent::CreateMailbox) => 165, EventType::Imap(ImapEvent::DeleteMailbox) => 166, EventType::Imap(ImapEvent::Enable) => 167, EventType::Imap(ImapEvent::Error) => 168, EventType::Imap(ImapEvent::Expunge) => 169, EventType::Imap(ImapEvent::Fetch) => 170, EventType::Imap(ImapEvent::GetAcl) => 171, EventType::Imap(ImapEvent::Id) => 172, EventType::Imap(ImapEvent::IdleStart) => 173, EventType::Imap(ImapEvent::IdleStop) => 174, EventType::Imap(ImapEvent::List) => 175, EventType::Imap(ImapEvent::ListRights) => 176, EventType::Imap(ImapEvent::Logout) => 177, EventType::Imap(ImapEvent::Lsub) => 178, EventType::Imap(ImapEvent::Move) => 179, EventType::Imap(ImapEvent::MyRights) => 180, EventType::Imap(ImapEvent::Namespace) => 181, EventType::Imap(ImapEvent::Noop) => 182, EventType::Imap(ImapEvent::RawInput) => 183, EventType::Imap(ImapEvent::RawOutput) => 184, EventType::Imap(ImapEvent::RenameMailbox) => 185, EventType::Imap(ImapEvent::Search) => 186, EventType::Imap(ImapEvent::Select) => 187, EventType::Imap(ImapEvent::SetAcl) => 188, EventType::Imap(ImapEvent::Sort) => 189, EventType::Imap(ImapEvent::Status) => 190, EventType::Imap(ImapEvent::Store) => 191, EventType::Imap(ImapEvent::Subscribe) => 192, EventType::Imap(ImapEvent::Thread) => 193, EventType::Imap(ImapEvent::Unsubscribe) => 194, EventType::IncomingReport(IncomingReportEvent::AbuseReport) => 195, EventType::IncomingReport(IncomingReportEvent::ArfParseFailed) => 196, EventType::IncomingReport(IncomingReportEvent::AuthFailureReport) => 197, EventType::IncomingReport(IncomingReportEvent::DecompressError) => 198, EventType::IncomingReport(IncomingReportEvent::DmarcParseFailed) => 199, EventType::IncomingReport(IncomingReportEvent::DmarcReport) => 200, EventType::IncomingReport(IncomingReportEvent::DmarcReportWithWarnings) => 201, EventType::IncomingReport(IncomingReportEvent::FraudReport) => 202, EventType::IncomingReport(IncomingReportEvent::MessageParseFailed) => 203, EventType::IncomingReport(IncomingReportEvent::NotSpamReport) => 204, EventType::IncomingReport(IncomingReportEvent::OtherReport) => 205, EventType::IncomingReport(IncomingReportEvent::TlsReport) => 206, EventType::IncomingReport(IncomingReportEvent::TlsReportWithWarnings) => 207, EventType::IncomingReport(IncomingReportEvent::TlsRpcParseFailed) => 208, EventType::IncomingReport(IncomingReportEvent::VirusReport) => 209, EventType::Iprev(IprevEvent::Fail) => 210, EventType::Iprev(IprevEvent::None) => 211, EventType::Iprev(IprevEvent::Pass) => 212, EventType::Iprev(IprevEvent::PermError) => 213, EventType::Iprev(IprevEvent::TempError) => 214, EventType::Jmap(JmapEvent::AccountNotFound) => 215, EventType::Jmap(JmapEvent::AccountNotSupportedByMethod) => 216, EventType::Jmap(JmapEvent::AccountReadOnly) => 217, EventType::Jmap(JmapEvent::AnchorNotFound) => 218, EventType::Jmap(JmapEvent::CannotCalculateChanges) => 219, EventType::Jmap(JmapEvent::Forbidden) => 220, EventType::Jmap(JmapEvent::InvalidArguments) => 221, EventType::Jmap(JmapEvent::InvalidResultReference) => 222, EventType::Jmap(JmapEvent::MethodCall) => 223, EventType::Jmap(JmapEvent::NotFound) => 224, EventType::Jmap(JmapEvent::NotJson) => 225, EventType::Jmap(JmapEvent::NotRequest) => 226, EventType::Jmap(JmapEvent::RequestTooLarge) => 227, EventType::Jmap(JmapEvent::StateMismatch) => 228, EventType::Jmap(JmapEvent::UnknownCapability) => 229, EventType::Jmap(JmapEvent::UnknownDataType) => 230, EventType::Jmap(JmapEvent::UnknownMethod) => 231, EventType::Jmap(JmapEvent::UnsupportedFilter) => 232, EventType::Jmap(JmapEvent::UnsupportedSort) => 233, EventType::Jmap(JmapEvent::WebsocketError) => 234, EventType::Jmap(JmapEvent::WebsocketStart) => 235, EventType::Jmap(JmapEvent::WebsocketStop) => 236, EventType::Limit(LimitEvent::BlobQuota) => 237, EventType::Limit(LimitEvent::CallsIn) => 238, EventType::Limit(LimitEvent::ConcurrentConnection) => 239, EventType::Limit(LimitEvent::ConcurrentRequest) => 240, EventType::Limit(LimitEvent::ConcurrentUpload) => 241, EventType::Limit(LimitEvent::Quota) => 242, EventType::Limit(LimitEvent::SizeRequest) => 243, EventType::Limit(LimitEvent::SizeUpload) => 244, EventType::Limit(LimitEvent::TooManyRequests) => 245, EventType::MailAuth(MailAuthEvent::Base64) => 246, EventType::MailAuth(MailAuthEvent::Crypto) => 247, EventType::MailAuth(MailAuthEvent::DnsError) => 248, EventType::MailAuth(MailAuthEvent::DnsInvalidRecordType) => 249, EventType::MailAuth(MailAuthEvent::DnsRecordNotFound) => 250, EventType::MailAuth(MailAuthEvent::Io) => 251, EventType::MailAuth(MailAuthEvent::MissingParameters) => 252, EventType::MailAuth(MailAuthEvent::NoHeadersFound) => 253, EventType::MailAuth(MailAuthEvent::ParseError) => 254, EventType::MailAuth(MailAuthEvent::PolicyNotAligned) => 255, EventType::ManageSieve(ManageSieveEvent::Capabilities) => 256, EventType::ManageSieve(ManageSieveEvent::CheckScript) => 257, EventType::ManageSieve(ManageSieveEvent::ConnectionEnd) => 258, EventType::ManageSieve(ManageSieveEvent::ConnectionStart) => 259, EventType::ManageSieve(ManageSieveEvent::CreateScript) => 260, EventType::ManageSieve(ManageSieveEvent::DeleteScript) => 261, EventType::ManageSieve(ManageSieveEvent::Error) => 262, EventType::ManageSieve(ManageSieveEvent::GetScript) => 263, EventType::ManageSieve(ManageSieveEvent::HaveSpace) => 264, EventType::ManageSieve(ManageSieveEvent::ListScripts) => 265, EventType::ManageSieve(ManageSieveEvent::Logout) => 266, EventType::ManageSieve(ManageSieveEvent::Noop) => 267, EventType::ManageSieve(ManageSieveEvent::RawInput) => 268, EventType::ManageSieve(ManageSieveEvent::RawOutput) => 269, EventType::ManageSieve(ManageSieveEvent::RenameScript) => 270, EventType::ManageSieve(ManageSieveEvent::SetActive) => 271, EventType::ManageSieve(ManageSieveEvent::StartTls) => 272, EventType::ManageSieve(ManageSieveEvent::Unauthenticate) => 273, EventType::ManageSieve(ManageSieveEvent::UpdateScript) => 274, EventType::Manage(ManageEvent::AlreadyExists) => 275, EventType::Manage(ManageEvent::AssertFailed) => 276, EventType::Manage(ManageEvent::Error) => 277, EventType::Manage(ManageEvent::MissingParameter) => 278, EventType::Manage(ManageEvent::NotFound) => 279, EventType::Manage(ManageEvent::NotSupported) => 280, EventType::MessageIngest(MessageIngestEvent::Duplicate) => 281, EventType::MessageIngest(MessageIngestEvent::Error) => 282, EventType::MessageIngest(MessageIngestEvent::Ham) => 283, EventType::MessageIngest(MessageIngestEvent::ImapAppend) => 284, EventType::MessageIngest(MessageIngestEvent::JmapAppend) => 285, EventType::MessageIngest(MessageIngestEvent::Spam) => 286, EventType::Milter(MilterEvent::ActionAccept) => 287, EventType::Milter(MilterEvent::ActionConnectionFailure) => 288, EventType::Milter(MilterEvent::ActionDiscard) => 289, EventType::Milter(MilterEvent::ActionReject) => 290, EventType::Milter(MilterEvent::ActionReplyCode) => 291, EventType::Milter(MilterEvent::ActionShutdown) => 292, EventType::Milter(MilterEvent::ActionTempFail) => 293, EventType::Milter(MilterEvent::Disconnected) => 294, EventType::Milter(MilterEvent::FrameInvalid) => 295, EventType::Milter(MilterEvent::FrameTooLarge) => 296, EventType::Milter(MilterEvent::IoError) => 297, EventType::Milter(MilterEvent::ParseError) => 298, EventType::Milter(MilterEvent::Read) => 299, EventType::Milter(MilterEvent::Timeout) => 300, EventType::Milter(MilterEvent::TlsInvalidName) => 301, EventType::Milter(MilterEvent::UnexpectedResponse) => 302, EventType::Milter(MilterEvent::Write) => 303, EventType::MtaHook(MtaHookEvent::ActionAccept) => 304, EventType::MtaHook(MtaHookEvent::ActionDiscard) => 305, EventType::MtaHook(MtaHookEvent::ActionQuarantine) => 306, EventType::MtaHook(MtaHookEvent::ActionReject) => 307, EventType::MtaHook(MtaHookEvent::Error) => 308, EventType::MtaSts(MtaStsEvent::Authorized) => 309, EventType::MtaSts(MtaStsEvent::InvalidPolicy) => 310, EventType::MtaSts(MtaStsEvent::NotAuthorized) => 311, EventType::MtaSts(MtaStsEvent::PolicyFetch) => 312, EventType::MtaSts(MtaStsEvent::PolicyFetchError) => 313, EventType::MtaSts(MtaStsEvent::PolicyNotFound) => 314, EventType::Network(NetworkEvent::AcceptError) => 315, EventType::Network(NetworkEvent::BindError) => 316, EventType::Network(NetworkEvent::Closed) => 317, EventType::Security(SecurityEvent::IpBlocked) => 318, EventType::Network(NetworkEvent::FlushError) => 319, EventType::Network(NetworkEvent::ListenError) => 320, EventType::Network(NetworkEvent::ListenStart) => 321, EventType::Network(NetworkEvent::ListenStop) => 322, EventType::Network(NetworkEvent::ProxyError) => 323, EventType::Network(NetworkEvent::ReadError) => 324, EventType::Network(NetworkEvent::SetOptError) => 325, EventType::Network(NetworkEvent::SplitError) => 326, EventType::Network(NetworkEvent::Timeout) => 327, EventType::Network(NetworkEvent::WriteError) => 328, EventType::OutgoingReport(OutgoingReportEvent::DkimRateLimited) => 329, EventType::OutgoingReport(OutgoingReportEvent::DkimReport) => 330, EventType::OutgoingReport(OutgoingReportEvent::DmarcAggregateReport) => 331, EventType::OutgoingReport(OutgoingReportEvent::DmarcRateLimited) => 332, EventType::OutgoingReport(OutgoingReportEvent::DmarcReport) => 333, EventType::OutgoingReport(OutgoingReportEvent::HttpSubmission) => 334, EventType::OutgoingReport(OutgoingReportEvent::Locked) => 337, EventType::OutgoingReport(OutgoingReportEvent::NoRecipientsFound) => 338, EventType::OutgoingReport(OutgoingReportEvent::NotFound) => 339, EventType::OutgoingReport(OutgoingReportEvent::ReportingAddressValidationError) => 340, EventType::OutgoingReport(OutgoingReportEvent::SpfRateLimited) => 341, EventType::OutgoingReport(OutgoingReportEvent::SpfReport) => 342, EventType::OutgoingReport(OutgoingReportEvent::SubmissionError) => 343, EventType::OutgoingReport(OutgoingReportEvent::TlsAggregate) => 344, EventType::OutgoingReport(OutgoingReportEvent::UnauthorizedReportingAddress) => 345, EventType::Pop3(Pop3Event::Capabilities) => 346, EventType::Pop3(Pop3Event::ConnectionEnd) => 347, EventType::Pop3(Pop3Event::ConnectionStart) => 348, EventType::Pop3(Pop3Event::Delete) => 349, EventType::Pop3(Pop3Event::Error) => 350, EventType::Pop3(Pop3Event::Fetch) => 351, EventType::Pop3(Pop3Event::List) => 352, EventType::Pop3(Pop3Event::ListMessage) => 353, EventType::Pop3(Pop3Event::Noop) => 354, EventType::Pop3(Pop3Event::Quit) => 355, EventType::Pop3(Pop3Event::RawInput) => 356, EventType::Pop3(Pop3Event::RawOutput) => 357, EventType::Pop3(Pop3Event::Reset) => 358, EventType::Pop3(Pop3Event::StartTls) => 359, EventType::Pop3(Pop3Event::Stat) => 360, EventType::Pop3(Pop3Event::Uidl) => 361, EventType::Pop3(Pop3Event::UidlMessage) => 362, EventType::Pop3(Pop3Event::Utf8) => 363, EventType::Purge(PurgeEvent::AutoExpunge) => 364, EventType::Purge(PurgeEvent::Error) => 365, EventType::Purge(PurgeEvent::Finished) => 366, EventType::Purge(PurgeEvent::InProgress) => 367, EventType::Purge(PurgeEvent::Running) => 368, EventType::Purge(PurgeEvent::Started) => 369, EventType::Purge(PurgeEvent::BlobCleanup) => 370, EventType::PushSubscription(PushSubscriptionEvent::Error) => 371, EventType::PushSubscription(PushSubscriptionEvent::NotFound) => 372, EventType::PushSubscription(PushSubscriptionEvent::Success) => 373, EventType::Queue(QueueEvent::BlobNotFound) => 374, EventType::Queue(QueueEvent::ConcurrencyLimitExceeded) => 375, EventType::Queue(QueueEvent::Locked) => 377, EventType::Queue(QueueEvent::QueueAutogenerated) => 378, EventType::Queue(QueueEvent::QueueDsn) => 379, EventType::Queue(QueueEvent::QueueMessage) => 380, EventType::Queue(QueueEvent::QueueMessageAuthenticated) => 381, EventType::Queue(QueueEvent::QueueReport) => 382, EventType::Queue(QueueEvent::QuotaExceeded) => 383, EventType::Queue(QueueEvent::RateLimitExceeded) => 384, EventType::Queue(QueueEvent::Rescheduled) => 385, EventType::Resource(ResourceEvent::BadParameters) => 386, EventType::Resource(ResourceEvent::DownloadExternal) => 387, EventType::Resource(ResourceEvent::Error) => 388, EventType::Resource(ResourceEvent::NotFound) => 389, EventType::Resource(ResourceEvent::WebadminUnpacked) => 390, EventType::Server(ServerEvent::Licensing) => 391, EventType::Server(ServerEvent::Shutdown) => 392, EventType::Server(ServerEvent::Startup) => 393, EventType::Server(ServerEvent::StartupError) => 394, EventType::Server(ServerEvent::ThreadError) => 395, EventType::Sieve(SieveEvent::ActionAccept) => 396, EventType::Sieve(SieveEvent::ActionAcceptReplace) => 397, EventType::Sieve(SieveEvent::ActionDiscard) => 398, EventType::Sieve(SieveEvent::ActionReject) => 399, EventType::Sieve(SieveEvent::ListNotFound) => 400, EventType::Sieve(SieveEvent::MessageTooLarge) => 401, EventType::Sieve(SieveEvent::NotSupported) => 402, EventType::Sieve(SieveEvent::QuotaExceeded) => 403, EventType::Sieve(SieveEvent::RuntimeError) => 404, EventType::Sieve(SieveEvent::ScriptNotFound) => 405, EventType::Sieve(SieveEvent::SendMessage) => 406, EventType::Sieve(SieveEvent::UnexpectedError) => 407, EventType::Smtp(SmtpEvent::AlreadyAuthenticated) => 408, EventType::Smtp(SmtpEvent::ArcFail) => 409, EventType::Smtp(SmtpEvent::ArcPass) => 410, EventType::Smtp(SmtpEvent::AuthExchangeTooLong) => 411, EventType::Smtp(SmtpEvent::AuthMechanismNotSupported) => 412, EventType::Smtp(SmtpEvent::AuthNotAllowed) => 413, EventType::Smtp(SmtpEvent::CommandNotImplemented) => 414, EventType::Smtp(SmtpEvent::ConcurrencyLimitExceeded) => 415, EventType::Smtp(SmtpEvent::ConnectionEnd) => 416, EventType::Smtp(SmtpEvent::ConnectionStart) => 417, EventType::Smtp(SmtpEvent::DeliverByDisabled) => 418, EventType::Smtp(SmtpEvent::DeliverByInvalid) => 419, EventType::Smtp(SmtpEvent::DidNotSayEhlo) => 420, EventType::Smtp(SmtpEvent::DkimFail) => 421, EventType::Smtp(SmtpEvent::DkimPass) => 422, EventType::Smtp(SmtpEvent::DmarcFail) => 423, EventType::Smtp(SmtpEvent::DmarcPass) => 424, EventType::Smtp(SmtpEvent::DsnDisabled) => 425, EventType::Smtp(SmtpEvent::Ehlo) => 426, EventType::Smtp(SmtpEvent::EhloExpected) => 427, EventType::Smtp(SmtpEvent::Error) => 428, EventType::Smtp(SmtpEvent::Expn) => 429, EventType::Smtp(SmtpEvent::ExpnDisabled) => 430, EventType::Smtp(SmtpEvent::ExpnNotFound) => 431, EventType::Smtp(SmtpEvent::FutureReleaseDisabled) => 432, EventType::Smtp(SmtpEvent::FutureReleaseInvalid) => 433, EventType::Smtp(SmtpEvent::Help) => 434, EventType::Smtp(SmtpEvent::InvalidCommand) => 435, EventType::Smtp(SmtpEvent::InvalidEhlo) => 436, EventType::Smtp(SmtpEvent::InvalidParameter) => 437, EventType::Smtp(SmtpEvent::InvalidRecipientAddress) => 438, EventType::Smtp(SmtpEvent::InvalidSenderAddress) => 439, EventType::Smtp(SmtpEvent::IprevFail) => 440, EventType::Smtp(SmtpEvent::IprevPass) => 441, EventType::Smtp(SmtpEvent::LhloExpected) => 442, EventType::Smtp(SmtpEvent::LoopDetected) => 443, EventType::Smtp(SmtpEvent::MailFrom) => 444, EventType::Smtp(SmtpEvent::MailFromMissing) => 445, EventType::Smtp(SmtpEvent::MailFromRewritten) => 446, EventType::Smtp(SmtpEvent::MailFromUnauthenticated) => 447, EventType::Smtp(SmtpEvent::MailFromUnauthorized) => 448, EventType::Smtp(SmtpEvent::MailboxDoesNotExist) => 449, EventType::Smtp(SmtpEvent::MessageParseFailed) => 450, EventType::Smtp(SmtpEvent::MessageTooLarge) => 451, EventType::Smtp(SmtpEvent::MissingAuthDirectory) => 452, EventType::Smtp(SmtpEvent::MissingLocalHostname) => 453, EventType::Smtp(SmtpEvent::MtPriorityDisabled) => 454, EventType::Smtp(SmtpEvent::MtPriorityInvalid) => 455, EventType::Smtp(SmtpEvent::MultipleMailFrom) => 456, EventType::Smtp(SmtpEvent::Noop) => 457, EventType::Smtp(SmtpEvent::Quit) => 460, EventType::Smtp(SmtpEvent::RateLimitExceeded) => 461, EventType::Smtp(SmtpEvent::RawInput) => 462, EventType::Smtp(SmtpEvent::RawOutput) => 463, EventType::Smtp(SmtpEvent::RcptTo) => 464, EventType::Smtp(SmtpEvent::RcptToDuplicate) => 465, EventType::Smtp(SmtpEvent::RcptToMissing) => 466, EventType::Smtp(SmtpEvent::RcptToRewritten) => 467, EventType::Smtp(SmtpEvent::RelayNotAllowed) => 468, EventType::Smtp(SmtpEvent::IdNotFound) => 469, EventType::Smtp(SmtpEvent::RequestTooLarge) => 470, EventType::Smtp(SmtpEvent::RequireTlsDisabled) => 471, EventType::Smtp(SmtpEvent::Rset) => 472, EventType::Smtp(SmtpEvent::SpfEhloFail) => 473, EventType::Smtp(SmtpEvent::SpfEhloPass) => 474, EventType::Smtp(SmtpEvent::SpfFromFail) => 475, EventType::Smtp(SmtpEvent::SpfFromPass) => 476, EventType::Smtp(SmtpEvent::StartTls) => 477, EventType::Smtp(SmtpEvent::StartTlsAlready) => 478, EventType::Smtp(SmtpEvent::StartTlsUnavailable) => 479, EventType::Smtp(SmtpEvent::SyntaxError) => 480, EventType::Smtp(SmtpEvent::TimeLimitExceeded) => 481, EventType::Smtp(SmtpEvent::TooManyInvalidRcpt) => 482, EventType::Smtp(SmtpEvent::TooManyMessages) => 483, EventType::Smtp(SmtpEvent::TooManyRecipients) => 484, EventType::Smtp(SmtpEvent::TransferLimitExceeded) => 485, EventType::Smtp(SmtpEvent::UnsupportedParameter) => 486, EventType::Smtp(SmtpEvent::Vrfy) => 487, EventType::Smtp(SmtpEvent::VrfyDisabled) => 488, EventType::Smtp(SmtpEvent::VrfyNotFound) => 489, EventType::Spam(SpamEvent::Classify) => 490, EventType::Spam(SpamEvent::TrainSampleNotFound) => 491, EventType::Store(StoreEvent::HttpStoreFetch) => 492, EventType::Store(StoreEvent::HttpStoreError) => 493, EventType::Spam(SpamEvent::PyzorError) => 494, EventType::Spam(SpamEvent::TrainCompleted) => 495, EventType::Spam(SpamEvent::ModelNotReady) => 496, EventType::Spam(SpamEvent::ModelNotFound) => 497, EventType::Spf(SpfEvent::Fail) => 498, EventType::Spf(SpfEvent::Neutral) => 499, EventType::Spf(SpfEvent::None) => 500, EventType::Spf(SpfEvent::Pass) => 501, EventType::Spf(SpfEvent::PermError) => 502, EventType::Spf(SpfEvent::SoftFail) => 503, EventType::Spf(SpfEvent::TempError) => 504, EventType::Store(StoreEvent::AssertValueFailed) => 505, EventType::Store(StoreEvent::BlobDelete) => 506, EventType::Store(StoreEvent::BlobMissingMarker) => 507, EventType::Store(StoreEvent::BlobRead) => 508, EventType::Store(StoreEvent::BlobWrite) => 509, EventType::Store(StoreEvent::CryptoError) => 510, EventType::Store(StoreEvent::DataCorruption) => 511, EventType::Store(StoreEvent::DataIterate) => 512, EventType::Store(StoreEvent::DataWrite) => 513, EventType::Store(StoreEvent::DecompressError) => 514, EventType::Store(StoreEvent::DeserializeError) => 515, EventType::Store(StoreEvent::ElasticsearchError) => 516, EventType::Store(StoreEvent::FilesystemError) => 517, EventType::Store(StoreEvent::FoundationdbError) => 518, EventType::Store(StoreEvent::LdapWarning) => 519, EventType::Store(StoreEvent::LdapError) => 520, EventType::Store(StoreEvent::LdapQuery) => 521, EventType::Store(StoreEvent::MysqlError) => 522, EventType::Store(StoreEvent::NotConfigured) => 523, EventType::Store(StoreEvent::NotFound) => 524, EventType::Store(StoreEvent::NotSupported) => 525, EventType::Store(StoreEvent::PoolError) => 526, EventType::Store(StoreEvent::PostgresqlError) => 527, EventType::Store(StoreEvent::RedisError) => 528, EventType::Store(StoreEvent::RocksdbError) => 529, EventType::Store(StoreEvent::S3Error) => 530, EventType::Store(StoreEvent::SqlQuery) => 531, EventType::Store(StoreEvent::SqliteError) => 532, EventType::Store(StoreEvent::UnexpectedError) => 533, EventType::Telemetry(TelemetryEvent::JournalError) => 534, EventType::Telemetry(TelemetryEvent::LogError) => 535, EventType::Telemetry(TelemetryEvent::OtelExporterError) => 536, EventType::Telemetry(TelemetryEvent::OtelMetricsExporterError) => 537, EventType::Telemetry(TelemetryEvent::PrometheusExporterError) => 538, EventType::Telemetry(TelemetryEvent::WebhookError) => 539, EventType::TlsRpt(TlsRptEvent::RecordFetch) => 540, EventType::TlsRpt(TlsRptEvent::RecordFetchError) => 541, EventType::Tls(TlsEvent::CertificateNotFound) => 542, EventType::Tls(TlsEvent::Handshake) => 543, EventType::Tls(TlsEvent::HandshakeError) => 544, EventType::Tls(TlsEvent::MultipleCertificatesAvailable) => 545, EventType::Tls(TlsEvent::NoCertificatesAvailable) => 546, EventType::Tls(TlsEvent::NotConfigured) => 547, EventType::Telemetry(TelemetryEvent::Alert) => 548, EventType::Security(SecurityEvent::AbuseBan) => 549, EventType::Security(SecurityEvent::LoiterBan) => 550, EventType::Smtp(SmtpEvent::MailFromNotAllowed) => 551, EventType::Security(SecurityEvent::Unauthorized) => 552, EventType::Limit(LimitEvent::TenantQuota) => 553, EventType::Auth(AuthEvent::TokenExpired) => 554, EventType::Auth(AuthEvent::ClientRegistration) => 555, EventType::Ai(AiEvent::LlmResponse) => 556, EventType::Ai(AiEvent::ApiError) => 557, EventType::Security(SecurityEvent::ScanBan) => 558, EventType::Store(StoreEvent::AzureError) => 559, EventType::TlsRpt(TlsRptEvent::RecordNotFound) => 560, EventType::Smtp(SmtpEvent::RcptToGreylisted) => 561, EventType::Spam(SpamEvent::Dnsbl) => 562, EventType::Spam(SpamEvent::DnsblError) => 563, EventType::Spam(SpamEvent::Pyzor) => 564, EventType::Queue(QueueEvent::BackPressure) => 48, EventType::Imap(ImapEvent::GetQuota) => 57, EventType::WebDav(WebDavEvent::Propfind) => 147, EventType::WebDav(WebDavEvent::Proppatch) => 148, EventType::WebDav(WebDavEvent::Get) => 335, EventType::WebDav(WebDavEvent::Report) => 336, EventType::WebDav(WebDavEvent::Mkcol) => 376, EventType::WebDav(WebDavEvent::Delete) => 458, EventType::WebDav(WebDavEvent::Put) => 459, EventType::WebDav(WebDavEvent::Post) => 565, EventType::WebDav(WebDavEvent::Patch) => 566, EventType::WebDav(WebDavEvent::Copy) => 567, EventType::WebDav(WebDavEvent::Move) => 568, EventType::WebDav(WebDavEvent::Lock) => 569, EventType::WebDav(WebDavEvent::Unlock) => 570, EventType::WebDav(WebDavEvent::Acl) => 571, EventType::WebDav(WebDavEvent::Error) => 572, EventType::WebDav(WebDavEvent::Options) => 573, EventType::WebDav(WebDavEvent::Head) => 574, EventType::WebDav(WebDavEvent::Mkcalendar) => 575, EventType::Calendar(CalendarEvent::RuleExpansionError) => 576, EventType::Store(StoreEvent::CacheMiss) => 50, EventType::Store(StoreEvent::CacheHit) => 51, EventType::Store(StoreEvent::CacheStale) => 52, EventType::Store(StoreEvent::CacheUpdate) => 577, EventType::TaskQueue(TaskQueueEvent::TaskAcquired) => 578, EventType::Calendar(CalendarEvent::AlarmSent) => 579, EventType::Calendar(CalendarEvent::AlarmSkipped) => 580, EventType::Calendar(CalendarEvent::AlarmRecipientOverride) => 581, EventType::Calendar(CalendarEvent::AlarmFailed) => 582, EventType::Calendar(CalendarEvent::ItipMessageSent) => 583, EventType::Calendar(CalendarEvent::ItipMessageReceived) => 584, EventType::Calendar(CalendarEvent::ItipMessageError) => 585, EventType::TaskQueue(TaskQueueEvent::TaskIgnored) => 586, EventType::TaskQueue(TaskQueueEvent::TaskFailed) => 587, EventType::Spam(SpamEvent::TrainStarted) => 588, EventType::Spam(SpamEvent::ModelLoaded) => 589, EventType::Store(StoreEvent::MeilisearchError) => 590, } } pub fn from_code(code: u64) -> Option { match code { 0 => Some(EventType::Acme(AcmeEvent::AuthCompleted)), 1 => Some(EventType::Acme(AcmeEvent::AuthError)), 2 => Some(EventType::Acme(AcmeEvent::AuthPending)), 3 => Some(EventType::Acme(AcmeEvent::AuthStart)), 4 => Some(EventType::Acme(AcmeEvent::AuthTooManyAttempts)), 5 => Some(EventType::Acme(AcmeEvent::AuthValid)), 6 => Some(EventType::Acme(AcmeEvent::ClientMissingSni)), 7 => Some(EventType::Acme(AcmeEvent::ClientSuppliedSni)), 8 => Some(EventType::Acme(AcmeEvent::DnsRecordCreated)), 9 => Some(EventType::Acme(AcmeEvent::DnsRecordCreationFailed)), 10 => Some(EventType::Acme(AcmeEvent::DnsRecordDeletionFailed)), 11 => Some(EventType::Acme(AcmeEvent::DnsRecordLookupFailed)), 12 => Some(EventType::Acme(AcmeEvent::DnsRecordNotPropagated)), 13 => Some(EventType::Acme(AcmeEvent::DnsRecordPropagated)), 14 => Some(EventType::Acme(AcmeEvent::DnsRecordPropagationTimeout)), 15 => Some(EventType::Acme(AcmeEvent::Error)), 16 => Some(EventType::Acme(AcmeEvent::OrderCompleted)), 17 => Some(EventType::Acme(AcmeEvent::OrderInvalid)), 18 => Some(EventType::Acme(AcmeEvent::OrderProcessing)), 19 => Some(EventType::Acme(AcmeEvent::OrderReady)), 20 => Some(EventType::Acme(AcmeEvent::OrderStart)), 21 => Some(EventType::Acme(AcmeEvent::OrderValid)), 22 => Some(EventType::Acme(AcmeEvent::ProcessCert)), 23 => Some(EventType::Acme(AcmeEvent::RenewBackoff)), 24 => Some(EventType::Acme(AcmeEvent::TlsAlpnError)), 25 => Some(EventType::Acme(AcmeEvent::TlsAlpnReceived)), 26 => Some(EventType::Acme(AcmeEvent::TokenNotFound)), 27 => Some(EventType::Arc(ArcEvent::BrokenChain)), 28 => Some(EventType::Arc(ArcEvent::ChainTooLong)), 29 => Some(EventType::Arc(ArcEvent::HasHeaderTag)), 30 => Some(EventType::Arc(ArcEvent::InvalidCv)), 31 => Some(EventType::Arc(ArcEvent::InvalidInstance)), 32 => Some(EventType::Arc(ArcEvent::SealerNotFound)), 33 => Some(EventType::Security(SecurityEvent::AuthenticationBan)), 34 => Some(EventType::Auth(AuthEvent::Error)), 35 => Some(EventType::Auth(AuthEvent::Failed)), 36 => Some(EventType::Auth(AuthEvent::MissingTotp)), 37 => Some(EventType::Auth(AuthEvent::Success)), 38 => Some(EventType::Auth(AuthEvent::TooManyAttempts)), 39 => Some(EventType::Cluster(ClusterEvent::SubscriberStart)), 40 => Some(EventType::Cluster(ClusterEvent::SubscriberStop)), 41 => Some(EventType::Cluster(ClusterEvent::SubscriberError)), 42 => Some(EventType::Cluster(ClusterEvent::SubscriberDisconnected)), 43 => Some(EventType::Cluster(ClusterEvent::PublisherStart)), 44 => Some(EventType::Cluster(ClusterEvent::PublisherStop)), 45 => Some(EventType::Cluster(ClusterEvent::PublisherError)), 46 => Some(EventType::Cluster(ClusterEvent::MessageReceived)), 47 => Some(EventType::Cluster(ClusterEvent::MessageSkipped)), 49 => Some(EventType::Cluster(ClusterEvent::MessageInvalid)), 53 => Some(EventType::Config(ConfigEvent::AlreadyUpToDate)), 54 => Some(EventType::Config(ConfigEvent::BuildError)), 55 => Some(EventType::Config(ConfigEvent::BuildWarning)), 56 => Some(EventType::Config(ConfigEvent::DefaultApplied)), 58 => Some(EventType::Config(ConfigEvent::FetchError)), 59 => Some(EventType::Config(ConfigEvent::ImportExternal)), 60 => Some(EventType::Config(ConfigEvent::MacroError)), 61 => Some(EventType::Config(ConfigEvent::MissingSetting)), 62 => Some(EventType::Config(ConfigEvent::ParseError)), 63 => Some(EventType::Config(ConfigEvent::ParseWarning)), 64 => Some(EventType::Config(ConfigEvent::UnusedSetting)), 65 => Some(EventType::Config(ConfigEvent::WriteError)), 66 => Some(EventType::Dane(DaneEvent::AuthenticationFailure)), 67 => Some(EventType::Dane(DaneEvent::AuthenticationSuccess)), 68 => Some(EventType::Dane(DaneEvent::CertificateParseError)), 69 => Some(EventType::Dane(DaneEvent::NoCertificatesFound)), 70 => Some(EventType::Dane(DaneEvent::TlsaRecordFetch)), 71 => Some(EventType::Dane(DaneEvent::TlsaRecordFetchError)), 72 => Some(EventType::Dane(DaneEvent::TlsaRecordInvalid)), 73 => Some(EventType::Dane(DaneEvent::TlsaRecordMatch)), 74 => Some(EventType::Dane(DaneEvent::TlsaRecordNotDnssecSigned)), 75 => Some(EventType::Dane(DaneEvent::TlsaRecordNotFound)), 76 => Some(EventType::Delivery(DeliveryEvent::AttemptEnd)), 77 => Some(EventType::Delivery(DeliveryEvent::AttemptStart)), 78 => Some(EventType::Delivery(DeliveryEvent::Auth)), 79 => Some(EventType::Delivery(DeliveryEvent::AuthFailed)), 80 => Some(EventType::Delivery(DeliveryEvent::Completed)), 81 => Some(EventType::Delivery(DeliveryEvent::ConcurrencyLimitExceeded)), 82 => Some(EventType::Delivery(DeliveryEvent::Connect)), 83 => Some(EventType::Delivery(DeliveryEvent::ConnectError)), 84 => Some(EventType::Delivery(DeliveryEvent::Delivered)), 85 => Some(EventType::Delivery(DeliveryEvent::DomainDeliveryStart)), 86 => Some(EventType::Delivery(DeliveryEvent::DoubleBounce)), 87 => Some(EventType::Delivery(DeliveryEvent::DsnPermFail)), 88 => Some(EventType::Delivery(DeliveryEvent::DsnSuccess)), 89 => Some(EventType::Delivery(DeliveryEvent::DsnTempFail)), 90 => Some(EventType::Delivery(DeliveryEvent::Ehlo)), 91 => Some(EventType::Delivery(DeliveryEvent::EhloRejected)), 92 => Some(EventType::Delivery(DeliveryEvent::Failed)), 93 => Some(EventType::Delivery(DeliveryEvent::GreetingFailed)), 94 => Some(EventType::Delivery(DeliveryEvent::ImplicitTlsError)), 95 => Some(EventType::Delivery(DeliveryEvent::IpLookup)), 96 => Some(EventType::Delivery(DeliveryEvent::IpLookupFailed)), 97 => Some(EventType::Delivery(DeliveryEvent::MailFrom)), 98 => Some(EventType::Delivery(DeliveryEvent::MailFromRejected)), 99 => Some(EventType::Delivery(DeliveryEvent::MessageRejected)), 100 => Some(EventType::Delivery(DeliveryEvent::MissingOutboundHostname)), 101 => Some(EventType::Delivery(DeliveryEvent::MxLookup)), 102 => Some(EventType::Delivery(DeliveryEvent::MxLookupFailed)), 103 => Some(EventType::Delivery(DeliveryEvent::NullMx)), 104 => Some(EventType::Delivery(DeliveryEvent::RateLimitExceeded)), 105 => Some(EventType::Delivery(DeliveryEvent::RawInput)), 106 => Some(EventType::Delivery(DeliveryEvent::RawOutput)), 107 => Some(EventType::Delivery(DeliveryEvent::RcptTo)), 108 => Some(EventType::Delivery(DeliveryEvent::RcptToFailed)), 109 => Some(EventType::Delivery(DeliveryEvent::RcptToRejected)), 110 => Some(EventType::Delivery(DeliveryEvent::StartTls)), 111 => Some(EventType::Delivery(DeliveryEvent::StartTlsDisabled)), 112 => Some(EventType::Delivery(DeliveryEvent::StartTlsError)), 113 => Some(EventType::Delivery(DeliveryEvent::StartTlsUnavailable)), 114 => Some(EventType::Dkim(DkimEvent::Fail)), 115 => Some(EventType::Dkim(DkimEvent::FailedAuidMatch)), 116 => Some(EventType::Dkim(DkimEvent::FailedBodyHashMatch)), 117 => Some(EventType::Dkim(DkimEvent::FailedVerification)), 118 => Some(EventType::Dkim(DkimEvent::IncompatibleAlgorithms)), 119 => Some(EventType::Dkim(DkimEvent::Neutral)), 120 => Some(EventType::Dkim(DkimEvent::None)), 121 => Some(EventType::Dkim(DkimEvent::Pass)), 122 => Some(EventType::Dkim(DkimEvent::PermError)), 123 => Some(EventType::Dkim(DkimEvent::RevokedPublicKey)), 124 => Some(EventType::Dkim(DkimEvent::SignatureExpired)), 125 => Some(EventType::Dkim(DkimEvent::SignatureLength)), 126 => Some(EventType::Dkim(DkimEvent::SignerNotFound)), 127 => Some(EventType::Dkim(DkimEvent::TempError)), 128 => Some(EventType::Dkim(DkimEvent::UnsupportedAlgorithm)), 129 => Some(EventType::Dkim(DkimEvent::UnsupportedCanonicalization)), 130 => Some(EventType::Dkim(DkimEvent::UnsupportedKeyType)), 131 => Some(EventType::Dkim(DkimEvent::UnsupportedVersion)), 132 => Some(EventType::Dmarc(DmarcEvent::Fail)), 133 => Some(EventType::Dmarc(DmarcEvent::None)), 134 => Some(EventType::Dmarc(DmarcEvent::Pass)), 135 => Some(EventType::Dmarc(DmarcEvent::PermError)), 136 => Some(EventType::Dmarc(DmarcEvent::TempError)), 137 => Some(EventType::Eval(EvalEvent::DirectoryNotFound)), 138 => Some(EventType::Eval(EvalEvent::Error)), 139 => Some(EventType::Eval(EvalEvent::Result)), 140 => Some(EventType::Eval(EvalEvent::StoreNotFound)), 141 => Some(EventType::TaskQueue(TaskQueueEvent::BlobNotFound)), 142 => Some(EventType::MessageIngest(MessageIngestEvent::FtsIndex)), 143 => Some(EventType::Spam(SpamEvent::TrainSampleAdded)), 144 => Some(EventType::TaskQueue(TaskQueueEvent::TaskLocked)), 145 => Some(EventType::TaskQueue(TaskQueueEvent::MetadataNotFound)), 146 => Some(EventType::Housekeeper(HousekeeperEvent::Run)), 149 => Some(EventType::Housekeeper(HousekeeperEvent::Schedule)), 150 => Some(EventType::Housekeeper(HousekeeperEvent::Start)), 151 => Some(EventType::Housekeeper(HousekeeperEvent::Stop)), 152 => Some(EventType::Http(HttpEvent::ConnectionEnd)), 153 => Some(EventType::Http(HttpEvent::ConnectionStart)), 154 => Some(EventType::Http(HttpEvent::Error)), 155 => Some(EventType::Http(HttpEvent::RequestBody)), 156 => Some(EventType::Http(HttpEvent::RequestUrl)), 157 => Some(EventType::Http(HttpEvent::ResponseBody)), 158 => Some(EventType::Http(HttpEvent::XForwardedMissing)), 159 => Some(EventType::Imap(ImapEvent::Append)), 160 => Some(EventType::Imap(ImapEvent::Capabilities)), 161 => Some(EventType::Imap(ImapEvent::Close)), 162 => Some(EventType::Imap(ImapEvent::ConnectionEnd)), 163 => Some(EventType::Imap(ImapEvent::ConnectionStart)), 164 => Some(EventType::Imap(ImapEvent::Copy)), 165 => Some(EventType::Imap(ImapEvent::CreateMailbox)), 166 => Some(EventType::Imap(ImapEvent::DeleteMailbox)), 167 => Some(EventType::Imap(ImapEvent::Enable)), 168 => Some(EventType::Imap(ImapEvent::Error)), 169 => Some(EventType::Imap(ImapEvent::Expunge)), 170 => Some(EventType::Imap(ImapEvent::Fetch)), 171 => Some(EventType::Imap(ImapEvent::GetAcl)), 172 => Some(EventType::Imap(ImapEvent::Id)), 173 => Some(EventType::Imap(ImapEvent::IdleStart)), 174 => Some(EventType::Imap(ImapEvent::IdleStop)), 175 => Some(EventType::Imap(ImapEvent::List)), 176 => Some(EventType::Imap(ImapEvent::ListRights)), 177 => Some(EventType::Imap(ImapEvent::Logout)), 178 => Some(EventType::Imap(ImapEvent::Lsub)), 179 => Some(EventType::Imap(ImapEvent::Move)), 180 => Some(EventType::Imap(ImapEvent::MyRights)), 181 => Some(EventType::Imap(ImapEvent::Namespace)), 182 => Some(EventType::Imap(ImapEvent::Noop)), 183 => Some(EventType::Imap(ImapEvent::RawInput)), 184 => Some(EventType::Imap(ImapEvent::RawOutput)), 185 => Some(EventType::Imap(ImapEvent::RenameMailbox)), 186 => Some(EventType::Imap(ImapEvent::Search)), 187 => Some(EventType::Imap(ImapEvent::Select)), 188 => Some(EventType::Imap(ImapEvent::SetAcl)), 189 => Some(EventType::Imap(ImapEvent::Sort)), 190 => Some(EventType::Imap(ImapEvent::Status)), 191 => Some(EventType::Imap(ImapEvent::Store)), 192 => Some(EventType::Imap(ImapEvent::Subscribe)), 193 => Some(EventType::Imap(ImapEvent::Thread)), 194 => Some(EventType::Imap(ImapEvent::Unsubscribe)), 195 => Some(EventType::IncomingReport(IncomingReportEvent::AbuseReport)), 196 => Some(EventType::IncomingReport( IncomingReportEvent::ArfParseFailed, )), 197 => Some(EventType::IncomingReport( IncomingReportEvent::AuthFailureReport, )), 198 => Some(EventType::IncomingReport( IncomingReportEvent::DecompressError, )), 199 => Some(EventType::IncomingReport( IncomingReportEvent::DmarcParseFailed, )), 200 => Some(EventType::IncomingReport(IncomingReportEvent::DmarcReport)), 201 => Some(EventType::IncomingReport( IncomingReportEvent::DmarcReportWithWarnings, )), 202 => Some(EventType::IncomingReport(IncomingReportEvent::FraudReport)), 203 => Some(EventType::IncomingReport( IncomingReportEvent::MessageParseFailed, )), 204 => Some(EventType::IncomingReport( IncomingReportEvent::NotSpamReport, )), 205 => Some(EventType::IncomingReport(IncomingReportEvent::OtherReport)), 206 => Some(EventType::IncomingReport(IncomingReportEvent::TlsReport)), 207 => Some(EventType::IncomingReport( IncomingReportEvent::TlsReportWithWarnings, )), 208 => Some(EventType::IncomingReport( IncomingReportEvent::TlsRpcParseFailed, )), 209 => Some(EventType::IncomingReport(IncomingReportEvent::VirusReport)), 210 => Some(EventType::Iprev(IprevEvent::Fail)), 211 => Some(EventType::Iprev(IprevEvent::None)), 212 => Some(EventType::Iprev(IprevEvent::Pass)), 213 => Some(EventType::Iprev(IprevEvent::PermError)), 214 => Some(EventType::Iprev(IprevEvent::TempError)), 215 => Some(EventType::Jmap(JmapEvent::AccountNotFound)), 216 => Some(EventType::Jmap(JmapEvent::AccountNotSupportedByMethod)), 217 => Some(EventType::Jmap(JmapEvent::AccountReadOnly)), 218 => Some(EventType::Jmap(JmapEvent::AnchorNotFound)), 219 => Some(EventType::Jmap(JmapEvent::CannotCalculateChanges)), 220 => Some(EventType::Jmap(JmapEvent::Forbidden)), 221 => Some(EventType::Jmap(JmapEvent::InvalidArguments)), 222 => Some(EventType::Jmap(JmapEvent::InvalidResultReference)), 223 => Some(EventType::Jmap(JmapEvent::MethodCall)), 224 => Some(EventType::Jmap(JmapEvent::NotFound)), 225 => Some(EventType::Jmap(JmapEvent::NotJson)), 226 => Some(EventType::Jmap(JmapEvent::NotRequest)), 227 => Some(EventType::Jmap(JmapEvent::RequestTooLarge)), 228 => Some(EventType::Jmap(JmapEvent::StateMismatch)), 229 => Some(EventType::Jmap(JmapEvent::UnknownCapability)), 230 => Some(EventType::Jmap(JmapEvent::UnknownDataType)), 231 => Some(EventType::Jmap(JmapEvent::UnknownMethod)), 232 => Some(EventType::Jmap(JmapEvent::UnsupportedFilter)), 233 => Some(EventType::Jmap(JmapEvent::UnsupportedSort)), 234 => Some(EventType::Jmap(JmapEvent::WebsocketError)), 235 => Some(EventType::Jmap(JmapEvent::WebsocketStart)), 236 => Some(EventType::Jmap(JmapEvent::WebsocketStop)), 237 => Some(EventType::Limit(LimitEvent::BlobQuota)), 238 => Some(EventType::Limit(LimitEvent::CallsIn)), 239 => Some(EventType::Limit(LimitEvent::ConcurrentConnection)), 240 => Some(EventType::Limit(LimitEvent::ConcurrentRequest)), 241 => Some(EventType::Limit(LimitEvent::ConcurrentUpload)), 242 => Some(EventType::Limit(LimitEvent::Quota)), 243 => Some(EventType::Limit(LimitEvent::SizeRequest)), 244 => Some(EventType::Limit(LimitEvent::SizeUpload)), 245 => Some(EventType::Limit(LimitEvent::TooManyRequests)), 246 => Some(EventType::MailAuth(MailAuthEvent::Base64)), 247 => Some(EventType::MailAuth(MailAuthEvent::Crypto)), 248 => Some(EventType::MailAuth(MailAuthEvent::DnsError)), 249 => Some(EventType::MailAuth(MailAuthEvent::DnsInvalidRecordType)), 250 => Some(EventType::MailAuth(MailAuthEvent::DnsRecordNotFound)), 251 => Some(EventType::MailAuth(MailAuthEvent::Io)), 252 => Some(EventType::MailAuth(MailAuthEvent::MissingParameters)), 253 => Some(EventType::MailAuth(MailAuthEvent::NoHeadersFound)), 254 => Some(EventType::MailAuth(MailAuthEvent::ParseError)), 255 => Some(EventType::MailAuth(MailAuthEvent::PolicyNotAligned)), 256 => Some(EventType::ManageSieve(ManageSieveEvent::Capabilities)), 257 => Some(EventType::ManageSieve(ManageSieveEvent::CheckScript)), 258 => Some(EventType::ManageSieve(ManageSieveEvent::ConnectionEnd)), 259 => Some(EventType::ManageSieve(ManageSieveEvent::ConnectionStart)), 260 => Some(EventType::ManageSieve(ManageSieveEvent::CreateScript)), 261 => Some(EventType::ManageSieve(ManageSieveEvent::DeleteScript)), 262 => Some(EventType::ManageSieve(ManageSieveEvent::Error)), 263 => Some(EventType::ManageSieve(ManageSieveEvent::GetScript)), 264 => Some(EventType::ManageSieve(ManageSieveEvent::HaveSpace)), 265 => Some(EventType::ManageSieve(ManageSieveEvent::ListScripts)), 266 => Some(EventType::ManageSieve(ManageSieveEvent::Logout)), 267 => Some(EventType::ManageSieve(ManageSieveEvent::Noop)), 268 => Some(EventType::ManageSieve(ManageSieveEvent::RawInput)), 269 => Some(EventType::ManageSieve(ManageSieveEvent::RawOutput)), 270 => Some(EventType::ManageSieve(ManageSieveEvent::RenameScript)), 271 => Some(EventType::ManageSieve(ManageSieveEvent::SetActive)), 272 => Some(EventType::ManageSieve(ManageSieveEvent::StartTls)), 273 => Some(EventType::ManageSieve(ManageSieveEvent::Unauthenticate)), 274 => Some(EventType::ManageSieve(ManageSieveEvent::UpdateScript)), 275 => Some(EventType::Manage(ManageEvent::AlreadyExists)), 276 => Some(EventType::Manage(ManageEvent::AssertFailed)), 277 => Some(EventType::Manage(ManageEvent::Error)), 278 => Some(EventType::Manage(ManageEvent::MissingParameter)), 279 => Some(EventType::Manage(ManageEvent::NotFound)), 280 => Some(EventType::Manage(ManageEvent::NotSupported)), 281 => Some(EventType::MessageIngest(MessageIngestEvent::Duplicate)), 282 => Some(EventType::MessageIngest(MessageIngestEvent::Error)), 283 => Some(EventType::MessageIngest(MessageIngestEvent::Ham)), 284 => Some(EventType::MessageIngest(MessageIngestEvent::ImapAppend)), 285 => Some(EventType::MessageIngest(MessageIngestEvent::JmapAppend)), 286 => Some(EventType::MessageIngest(MessageIngestEvent::Spam)), 287 => Some(EventType::Milter(MilterEvent::ActionAccept)), 288 => Some(EventType::Milter(MilterEvent::ActionConnectionFailure)), 289 => Some(EventType::Milter(MilterEvent::ActionDiscard)), 290 => Some(EventType::Milter(MilterEvent::ActionReject)), 291 => Some(EventType::Milter(MilterEvent::ActionReplyCode)), 292 => Some(EventType::Milter(MilterEvent::ActionShutdown)), 293 => Some(EventType::Milter(MilterEvent::ActionTempFail)), 294 => Some(EventType::Milter(MilterEvent::Disconnected)), 295 => Some(EventType::Milter(MilterEvent::FrameInvalid)), 296 => Some(EventType::Milter(MilterEvent::FrameTooLarge)), 297 => Some(EventType::Milter(MilterEvent::IoError)), 298 => Some(EventType::Milter(MilterEvent::ParseError)), 299 => Some(EventType::Milter(MilterEvent::Read)), 300 => Some(EventType::Milter(MilterEvent::Timeout)), 301 => Some(EventType::Milter(MilterEvent::TlsInvalidName)), 302 => Some(EventType::Milter(MilterEvent::UnexpectedResponse)), 303 => Some(EventType::Milter(MilterEvent::Write)), 304 => Some(EventType::MtaHook(MtaHookEvent::ActionAccept)), 305 => Some(EventType::MtaHook(MtaHookEvent::ActionDiscard)), 306 => Some(EventType::MtaHook(MtaHookEvent::ActionQuarantine)), 307 => Some(EventType::MtaHook(MtaHookEvent::ActionReject)), 308 => Some(EventType::MtaHook(MtaHookEvent::Error)), 309 => Some(EventType::MtaSts(MtaStsEvent::Authorized)), 310 => Some(EventType::MtaSts(MtaStsEvent::InvalidPolicy)), 311 => Some(EventType::MtaSts(MtaStsEvent::NotAuthorized)), 312 => Some(EventType::MtaSts(MtaStsEvent::PolicyFetch)), 313 => Some(EventType::MtaSts(MtaStsEvent::PolicyFetchError)), 314 => Some(EventType::MtaSts(MtaStsEvent::PolicyNotFound)), 315 => Some(EventType::Network(NetworkEvent::AcceptError)), 316 => Some(EventType::Network(NetworkEvent::BindError)), 317 => Some(EventType::Network(NetworkEvent::Closed)), 318 => Some(EventType::Security(SecurityEvent::IpBlocked)), 319 => Some(EventType::Network(NetworkEvent::FlushError)), 320 => Some(EventType::Network(NetworkEvent::ListenError)), 321 => Some(EventType::Network(NetworkEvent::ListenStart)), 322 => Some(EventType::Network(NetworkEvent::ListenStop)), 323 => Some(EventType::Network(NetworkEvent::ProxyError)), 324 => Some(EventType::Network(NetworkEvent::ReadError)), 325 => Some(EventType::Network(NetworkEvent::SetOptError)), 326 => Some(EventType::Network(NetworkEvent::SplitError)), 327 => Some(EventType::Network(NetworkEvent::Timeout)), 328 => Some(EventType::Network(NetworkEvent::WriteError)), 329 => Some(EventType::OutgoingReport( OutgoingReportEvent::DkimRateLimited, )), 330 => Some(EventType::OutgoingReport(OutgoingReportEvent::DkimReport)), 331 => Some(EventType::OutgoingReport( OutgoingReportEvent::DmarcAggregateReport, )), 332 => Some(EventType::OutgoingReport( OutgoingReportEvent::DmarcRateLimited, )), 333 => Some(EventType::OutgoingReport(OutgoingReportEvent::DmarcReport)), 334 => Some(EventType::OutgoingReport( OutgoingReportEvent::HttpSubmission, )), 337 => Some(EventType::OutgoingReport(OutgoingReportEvent::Locked)), 338 => Some(EventType::OutgoingReport( OutgoingReportEvent::NoRecipientsFound, )), 339 => Some(EventType::OutgoingReport(OutgoingReportEvent::NotFound)), 340 => Some(EventType::OutgoingReport( OutgoingReportEvent::ReportingAddressValidationError, )), 341 => Some(EventType::OutgoingReport( OutgoingReportEvent::SpfRateLimited, )), 342 => Some(EventType::OutgoingReport(OutgoingReportEvent::SpfReport)), 343 => Some(EventType::OutgoingReport( OutgoingReportEvent::SubmissionError, )), 344 => Some(EventType::OutgoingReport(OutgoingReportEvent::TlsAggregate)), 345 => Some(EventType::OutgoingReport( OutgoingReportEvent::UnauthorizedReportingAddress, )), 346 => Some(EventType::Pop3(Pop3Event::Capabilities)), 347 => Some(EventType::Pop3(Pop3Event::ConnectionEnd)), 348 => Some(EventType::Pop3(Pop3Event::ConnectionStart)), 349 => Some(EventType::Pop3(Pop3Event::Delete)), 350 => Some(EventType::Pop3(Pop3Event::Error)), 351 => Some(EventType::Pop3(Pop3Event::Fetch)), 352 => Some(EventType::Pop3(Pop3Event::List)), 353 => Some(EventType::Pop3(Pop3Event::ListMessage)), 354 => Some(EventType::Pop3(Pop3Event::Noop)), 355 => Some(EventType::Pop3(Pop3Event::Quit)), 356 => Some(EventType::Pop3(Pop3Event::RawInput)), 357 => Some(EventType::Pop3(Pop3Event::RawOutput)), 358 => Some(EventType::Pop3(Pop3Event::Reset)), 359 => Some(EventType::Pop3(Pop3Event::StartTls)), 360 => Some(EventType::Pop3(Pop3Event::Stat)), 361 => Some(EventType::Pop3(Pop3Event::Uidl)), 362 => Some(EventType::Pop3(Pop3Event::UidlMessage)), 363 => Some(EventType::Pop3(Pop3Event::Utf8)), 364 => Some(EventType::Purge(PurgeEvent::AutoExpunge)), 365 => Some(EventType::Purge(PurgeEvent::Error)), 366 => Some(EventType::Purge(PurgeEvent::Finished)), 367 => Some(EventType::Purge(PurgeEvent::InProgress)), 368 => Some(EventType::Purge(PurgeEvent::Running)), 369 => Some(EventType::Purge(PurgeEvent::Started)), 370 => Some(EventType::Purge(PurgeEvent::BlobCleanup)), 371 => Some(EventType::PushSubscription(PushSubscriptionEvent::Error)), 372 => Some(EventType::PushSubscription(PushSubscriptionEvent::NotFound)), 373 => Some(EventType::PushSubscription(PushSubscriptionEvent::Success)), 374 => Some(EventType::Queue(QueueEvent::BlobNotFound)), 375 => Some(EventType::Queue(QueueEvent::ConcurrencyLimitExceeded)), 377 => Some(EventType::Queue(QueueEvent::Locked)), 378 => Some(EventType::Queue(QueueEvent::QueueAutogenerated)), 379 => Some(EventType::Queue(QueueEvent::QueueDsn)), 380 => Some(EventType::Queue(QueueEvent::QueueMessage)), 381 => Some(EventType::Queue(QueueEvent::QueueMessageAuthenticated)), 382 => Some(EventType::Queue(QueueEvent::QueueReport)), 383 => Some(EventType::Queue(QueueEvent::QuotaExceeded)), 384 => Some(EventType::Queue(QueueEvent::RateLimitExceeded)), 385 => Some(EventType::Queue(QueueEvent::Rescheduled)), 386 => Some(EventType::Resource(ResourceEvent::BadParameters)), 387 => Some(EventType::Resource(ResourceEvent::DownloadExternal)), 388 => Some(EventType::Resource(ResourceEvent::Error)), 389 => Some(EventType::Resource(ResourceEvent::NotFound)), 390 => Some(EventType::Resource(ResourceEvent::WebadminUnpacked)), 391 => Some(EventType::Server(ServerEvent::Licensing)), 392 => Some(EventType::Server(ServerEvent::Shutdown)), 393 => Some(EventType::Server(ServerEvent::Startup)), 394 => Some(EventType::Server(ServerEvent::StartupError)), 395 => Some(EventType::Server(ServerEvent::ThreadError)), 396 => Some(EventType::Sieve(SieveEvent::ActionAccept)), 397 => Some(EventType::Sieve(SieveEvent::ActionAcceptReplace)), 398 => Some(EventType::Sieve(SieveEvent::ActionDiscard)), 399 => Some(EventType::Sieve(SieveEvent::ActionReject)), 400 => Some(EventType::Sieve(SieveEvent::ListNotFound)), 401 => Some(EventType::Sieve(SieveEvent::MessageTooLarge)), 402 => Some(EventType::Sieve(SieveEvent::NotSupported)), 403 => Some(EventType::Sieve(SieveEvent::QuotaExceeded)), 404 => Some(EventType::Sieve(SieveEvent::RuntimeError)), 405 => Some(EventType::Sieve(SieveEvent::ScriptNotFound)), 406 => Some(EventType::Sieve(SieveEvent::SendMessage)), 407 => Some(EventType::Sieve(SieveEvent::UnexpectedError)), 408 => Some(EventType::Smtp(SmtpEvent::AlreadyAuthenticated)), 409 => Some(EventType::Smtp(SmtpEvent::ArcFail)), 410 => Some(EventType::Smtp(SmtpEvent::ArcPass)), 411 => Some(EventType::Smtp(SmtpEvent::AuthExchangeTooLong)), 412 => Some(EventType::Smtp(SmtpEvent::AuthMechanismNotSupported)), 413 => Some(EventType::Smtp(SmtpEvent::AuthNotAllowed)), 414 => Some(EventType::Smtp(SmtpEvent::CommandNotImplemented)), 415 => Some(EventType::Smtp(SmtpEvent::ConcurrencyLimitExceeded)), 416 => Some(EventType::Smtp(SmtpEvent::ConnectionEnd)), 417 => Some(EventType::Smtp(SmtpEvent::ConnectionStart)), 418 => Some(EventType::Smtp(SmtpEvent::DeliverByDisabled)), 419 => Some(EventType::Smtp(SmtpEvent::DeliverByInvalid)), 420 => Some(EventType::Smtp(SmtpEvent::DidNotSayEhlo)), 421 => Some(EventType::Smtp(SmtpEvent::DkimFail)), 422 => Some(EventType::Smtp(SmtpEvent::DkimPass)), 423 => Some(EventType::Smtp(SmtpEvent::DmarcFail)), 424 => Some(EventType::Smtp(SmtpEvent::DmarcPass)), 425 => Some(EventType::Smtp(SmtpEvent::DsnDisabled)), 426 => Some(EventType::Smtp(SmtpEvent::Ehlo)), 427 => Some(EventType::Smtp(SmtpEvent::EhloExpected)), 428 => Some(EventType::Smtp(SmtpEvent::Error)), 429 => Some(EventType::Smtp(SmtpEvent::Expn)), 430 => Some(EventType::Smtp(SmtpEvent::ExpnDisabled)), 431 => Some(EventType::Smtp(SmtpEvent::ExpnNotFound)), 432 => Some(EventType::Smtp(SmtpEvent::FutureReleaseDisabled)), 433 => Some(EventType::Smtp(SmtpEvent::FutureReleaseInvalid)), 434 => Some(EventType::Smtp(SmtpEvent::Help)), 435 => Some(EventType::Smtp(SmtpEvent::InvalidCommand)), 436 => Some(EventType::Smtp(SmtpEvent::InvalidEhlo)), 437 => Some(EventType::Smtp(SmtpEvent::InvalidParameter)), 438 => Some(EventType::Smtp(SmtpEvent::InvalidRecipientAddress)), 439 => Some(EventType::Smtp(SmtpEvent::InvalidSenderAddress)), 440 => Some(EventType::Smtp(SmtpEvent::IprevFail)), 441 => Some(EventType::Smtp(SmtpEvent::IprevPass)), 442 => Some(EventType::Smtp(SmtpEvent::LhloExpected)), 443 => Some(EventType::Smtp(SmtpEvent::LoopDetected)), 444 => Some(EventType::Smtp(SmtpEvent::MailFrom)), 445 => Some(EventType::Smtp(SmtpEvent::MailFromMissing)), 446 => Some(EventType::Smtp(SmtpEvent::MailFromRewritten)), 447 => Some(EventType::Smtp(SmtpEvent::MailFromUnauthenticated)), 448 => Some(EventType::Smtp(SmtpEvent::MailFromUnauthorized)), 449 => Some(EventType::Smtp(SmtpEvent::MailboxDoesNotExist)), 450 => Some(EventType::Smtp(SmtpEvent::MessageParseFailed)), 451 => Some(EventType::Smtp(SmtpEvent::MessageTooLarge)), 452 => Some(EventType::Smtp(SmtpEvent::MissingAuthDirectory)), 453 => Some(EventType::Smtp(SmtpEvent::MissingLocalHostname)), 454 => Some(EventType::Smtp(SmtpEvent::MtPriorityDisabled)), 455 => Some(EventType::Smtp(SmtpEvent::MtPriorityInvalid)), 456 => Some(EventType::Smtp(SmtpEvent::MultipleMailFrom)), 457 => Some(EventType::Smtp(SmtpEvent::Noop)), 460 => Some(EventType::Smtp(SmtpEvent::Quit)), 461 => Some(EventType::Smtp(SmtpEvent::RateLimitExceeded)), 462 => Some(EventType::Smtp(SmtpEvent::RawInput)), 463 => Some(EventType::Smtp(SmtpEvent::RawOutput)), 464 => Some(EventType::Smtp(SmtpEvent::RcptTo)), 465 => Some(EventType::Smtp(SmtpEvent::RcptToDuplicate)), 466 => Some(EventType::Smtp(SmtpEvent::RcptToMissing)), 467 => Some(EventType::Smtp(SmtpEvent::RcptToRewritten)), 468 => Some(EventType::Smtp(SmtpEvent::RelayNotAllowed)), 469 => Some(EventType::Smtp(SmtpEvent::IdNotFound)), 470 => Some(EventType::Smtp(SmtpEvent::RequestTooLarge)), 471 => Some(EventType::Smtp(SmtpEvent::RequireTlsDisabled)), 472 => Some(EventType::Smtp(SmtpEvent::Rset)), 473 => Some(EventType::Smtp(SmtpEvent::SpfEhloFail)), 474 => Some(EventType::Smtp(SmtpEvent::SpfEhloPass)), 475 => Some(EventType::Smtp(SmtpEvent::SpfFromFail)), 476 => Some(EventType::Smtp(SmtpEvent::SpfFromPass)), 477 => Some(EventType::Smtp(SmtpEvent::StartTls)), 478 => Some(EventType::Smtp(SmtpEvent::StartTlsAlready)), 479 => Some(EventType::Smtp(SmtpEvent::StartTlsUnavailable)), 480 => Some(EventType::Smtp(SmtpEvent::SyntaxError)), 481 => Some(EventType::Smtp(SmtpEvent::TimeLimitExceeded)), 482 => Some(EventType::Smtp(SmtpEvent::TooManyInvalidRcpt)), 483 => Some(EventType::Smtp(SmtpEvent::TooManyMessages)), 484 => Some(EventType::Smtp(SmtpEvent::TooManyRecipients)), 485 => Some(EventType::Smtp(SmtpEvent::TransferLimitExceeded)), 486 => Some(EventType::Smtp(SmtpEvent::UnsupportedParameter)), 487 => Some(EventType::Smtp(SmtpEvent::Vrfy)), 488 => Some(EventType::Smtp(SmtpEvent::VrfyDisabled)), 489 => Some(EventType::Smtp(SmtpEvent::VrfyNotFound)), 490 => Some(EventType::Spam(SpamEvent::Classify)), 491 => Some(EventType::Spam(SpamEvent::TrainSampleNotFound)), 492 => Some(EventType::Store(StoreEvent::HttpStoreFetch)), 493 => Some(EventType::Store(StoreEvent::HttpStoreError)), 494 => Some(EventType::Spam(SpamEvent::PyzorError)), 495 => Some(EventType::Spam(SpamEvent::TrainCompleted)), 496 => Some(EventType::Spam(SpamEvent::ModelNotReady)), 497 => Some(EventType::Spam(SpamEvent::ModelNotFound)), 498 => Some(EventType::Spf(SpfEvent::Fail)), 499 => Some(EventType::Spf(SpfEvent::Neutral)), 500 => Some(EventType::Spf(SpfEvent::None)), 501 => Some(EventType::Spf(SpfEvent::Pass)), 502 => Some(EventType::Spf(SpfEvent::PermError)), 503 => Some(EventType::Spf(SpfEvent::SoftFail)), 504 => Some(EventType::Spf(SpfEvent::TempError)), 505 => Some(EventType::Store(StoreEvent::AssertValueFailed)), 506 => Some(EventType::Store(StoreEvent::BlobDelete)), 507 => Some(EventType::Store(StoreEvent::BlobMissingMarker)), 508 => Some(EventType::Store(StoreEvent::BlobRead)), 509 => Some(EventType::Store(StoreEvent::BlobWrite)), 510 => Some(EventType::Store(StoreEvent::CryptoError)), 511 => Some(EventType::Store(StoreEvent::DataCorruption)), 512 => Some(EventType::Store(StoreEvent::DataIterate)), 513 => Some(EventType::Store(StoreEvent::DataWrite)), 514 => Some(EventType::Store(StoreEvent::DecompressError)), 515 => Some(EventType::Store(StoreEvent::DeserializeError)), 516 => Some(EventType::Store(StoreEvent::ElasticsearchError)), 517 => Some(EventType::Store(StoreEvent::FilesystemError)), 518 => Some(EventType::Store(StoreEvent::FoundationdbError)), 519 => Some(EventType::Store(StoreEvent::LdapWarning)), 520 => Some(EventType::Store(StoreEvent::LdapError)), 521 => Some(EventType::Store(StoreEvent::LdapQuery)), 522 => Some(EventType::Store(StoreEvent::MysqlError)), 523 => Some(EventType::Store(StoreEvent::NotConfigured)), 524 => Some(EventType::Store(StoreEvent::NotFound)), 525 => Some(EventType::Store(StoreEvent::NotSupported)), 526 => Some(EventType::Store(StoreEvent::PoolError)), 527 => Some(EventType::Store(StoreEvent::PostgresqlError)), 528 => Some(EventType::Store(StoreEvent::RedisError)), 529 => Some(EventType::Store(StoreEvent::RocksdbError)), 530 => Some(EventType::Store(StoreEvent::S3Error)), 531 => Some(EventType::Store(StoreEvent::SqlQuery)), 532 => Some(EventType::Store(StoreEvent::SqliteError)), 533 => Some(EventType::Store(StoreEvent::UnexpectedError)), 534 => Some(EventType::Telemetry(TelemetryEvent::JournalError)), 535 => Some(EventType::Telemetry(TelemetryEvent::LogError)), 536 => Some(EventType::Telemetry(TelemetryEvent::OtelExporterError)), 537 => Some(EventType::Telemetry( TelemetryEvent::OtelMetricsExporterError, )), 538 => Some(EventType::Telemetry( TelemetryEvent::PrometheusExporterError, )), 539 => Some(EventType::Telemetry(TelemetryEvent::WebhookError)), 540 => Some(EventType::TlsRpt(TlsRptEvent::RecordFetch)), 541 => Some(EventType::TlsRpt(TlsRptEvent::RecordFetchError)), 542 => Some(EventType::Tls(TlsEvent::CertificateNotFound)), 543 => Some(EventType::Tls(TlsEvent::Handshake)), 544 => Some(EventType::Tls(TlsEvent::HandshakeError)), 545 => Some(EventType::Tls(TlsEvent::MultipleCertificatesAvailable)), 546 => Some(EventType::Tls(TlsEvent::NoCertificatesAvailable)), 547 => Some(EventType::Tls(TlsEvent::NotConfigured)), 548 => Some(EventType::Telemetry(TelemetryEvent::Alert)), 549 => Some(EventType::Security(SecurityEvent::AbuseBan)), 550 => Some(EventType::Security(SecurityEvent::LoiterBan)), 551 => Some(EventType::Smtp(SmtpEvent::MailFromNotAllowed)), 552 => Some(EventType::Security(SecurityEvent::Unauthorized)), 553 => Some(EventType::Limit(LimitEvent::TenantQuota)), 554 => Some(EventType::Auth(AuthEvent::TokenExpired)), 555 => Some(EventType::Auth(AuthEvent::ClientRegistration)), 556 => Some(EventType::Ai(AiEvent::LlmResponse)), 557 => Some(EventType::Ai(AiEvent::ApiError)), 558 => Some(EventType::Security(SecurityEvent::ScanBan)), 559 => Some(EventType::Store(StoreEvent::AzureError)), 560 => Some(EventType::TlsRpt(TlsRptEvent::RecordNotFound)), 561 => Some(EventType::Smtp(SmtpEvent::RcptToGreylisted)), 562 => Some(EventType::Spam(SpamEvent::Dnsbl)), 563 => Some(EventType::Spam(SpamEvent::DnsblError)), 564 => Some(EventType::Spam(SpamEvent::Pyzor)), 48 => Some(EventType::Queue(QueueEvent::BackPressure)), 57 => Some(EventType::Imap(ImapEvent::GetQuota)), 147 => Some(EventType::WebDav(WebDavEvent::Propfind)), 148 => Some(EventType::WebDav(WebDavEvent::Proppatch)), 335 => Some(EventType::WebDav(WebDavEvent::Get)), 336 => Some(EventType::WebDav(WebDavEvent::Report)), 376 => Some(EventType::WebDav(WebDavEvent::Mkcol)), 458 => Some(EventType::WebDav(WebDavEvent::Delete)), 459 => Some(EventType::WebDav(WebDavEvent::Put)), 565 => Some(EventType::WebDav(WebDavEvent::Post)), 566 => Some(EventType::WebDav(WebDavEvent::Patch)), 567 => Some(EventType::WebDav(WebDavEvent::Copy)), 568 => Some(EventType::WebDav(WebDavEvent::Move)), 569 => Some(EventType::WebDav(WebDavEvent::Lock)), 570 => Some(EventType::WebDav(WebDavEvent::Unlock)), 571 => Some(EventType::WebDav(WebDavEvent::Acl)), 572 => Some(EventType::WebDav(WebDavEvent::Error)), 573 => Some(EventType::WebDav(WebDavEvent::Options)), 574 => Some(EventType::WebDav(WebDavEvent::Head)), 575 => Some(EventType::WebDav(WebDavEvent::Mkcalendar)), 576 => Some(EventType::Calendar(CalendarEvent::RuleExpansionError)), 50 => Some(EventType::Store(StoreEvent::CacheMiss)), 51 => Some(EventType::Store(StoreEvent::CacheHit)), 52 => Some(EventType::Store(StoreEvent::CacheStale)), 577 => Some(EventType::Store(StoreEvent::CacheUpdate)), 578 => Some(EventType::TaskQueue(TaskQueueEvent::TaskAcquired)), 579 => Some(EventType::Calendar(CalendarEvent::AlarmSent)), 580 => Some(EventType::Calendar(CalendarEvent::AlarmSkipped)), 581 => Some(EventType::Calendar(CalendarEvent::AlarmRecipientOverride)), 582 => Some(EventType::Calendar(CalendarEvent::AlarmFailed)), 583 => Some(EventType::Calendar(CalendarEvent::ItipMessageSent)), 584 => Some(EventType::Calendar(CalendarEvent::ItipMessageReceived)), 585 => Some(EventType::Calendar(CalendarEvent::ItipMessageError)), 586 => Some(EventType::TaskQueue(TaskQueueEvent::TaskIgnored)), 587 => Some(EventType::TaskQueue(TaskQueueEvent::TaskFailed)), 588 => Some(EventType::Spam(SpamEvent::TrainStarted)), 589 => Some(EventType::Spam(SpamEvent::ModelLoaded)), 590 => Some(EventType::Store(StoreEvent::MeilisearchError)), _ => None, } } } impl Key { fn code(&self) -> u64 { match self { Key::AccountName => 0, Key::AccountId => 1, Key::BlobId => 2, Key::CausedBy => 3, Key::ChangeId => 4, Key::Code => 5, Key::Collection => 6, Key::Contents => 7, Key::Details => 8, Key::DkimFail => 9, Key::DkimNone => 10, Key::DkimPass => 11, Key::DmarcNone => 12, Key::DmarcPass => 13, Key::DmarcQuarantine => 14, Key::DmarcReject => 15, Key::DocumentId => 16, Key::Domain => 17, Key::Due => 18, Key::Elapsed => 19, Key::Expires => 20, Key::From => 21, Key::Hostname => 22, Key::Id => 23, Key::Key => 24, Key::Limit => 25, Key::ListenerId => 26, Key::LocalIp => 27, Key::LocalPort => 28, Key::MailboxName => 29, Key::MailboxId => 30, Key::MessageId => 31, Key::NextDsn => 32, Key::NextRetry => 33, Key::Path => 34, Key::Policy => 35, Key::QueueId => 36, Key::RangeFrom => 37, Key::RangeTo => 38, Key::Reason => 39, Key::RemoteIp => 40, Key::RemotePort => 41, Key::ReportId => 42, Key::Result => 43, Key::Size => 44, Key::Source => 45, Key::SpanId => 46, Key::SpfFail => 47, Key::SpfNone => 48, Key::SpfPass => 49, Key::Strict => 50, Key::Tls => 51, Key::To => 52, Key::Total => 53, Key::TotalFailures => 54, Key::TotalSuccesses => 55, Key::Type => 56, Key::Uid => 57, Key::UidNext => 58, Key::UidValidity => 59, Key::Url => 60, Key::ValidFrom => 61, Key::ValidTo => 62, Key::Value => 63, Key::Version => 64, Key::QueueName => 65, } } fn from_code(code: u64) -> Option { match code { 0 => Some(Key::AccountName), 1 => Some(Key::AccountId), 2 => Some(Key::BlobId), 3 => Some(Key::CausedBy), 4 => Some(Key::ChangeId), 5 => Some(Key::Code), 6 => Some(Key::Collection), 7 => Some(Key::Contents), 8 => Some(Key::Details), 9 => Some(Key::DkimFail), 10 => Some(Key::DkimNone), 11 => Some(Key::DkimPass), 12 => Some(Key::DmarcNone), 13 => Some(Key::DmarcPass), 14 => Some(Key::DmarcQuarantine), 15 => Some(Key::DmarcReject), 16 => Some(Key::DocumentId), 17 => Some(Key::Domain), 18 => Some(Key::Due), 19 => Some(Key::Elapsed), 20 => Some(Key::Expires), 21 => Some(Key::From), 22 => Some(Key::Hostname), 23 => Some(Key::Id), 24 => Some(Key::Key), 25 => Some(Key::Limit), 26 => Some(Key::ListenerId), 27 => Some(Key::LocalIp), 28 => Some(Key::LocalPort), 29 => Some(Key::MailboxName), 30 => Some(Key::MailboxId), 31 => Some(Key::MessageId), 32 => Some(Key::NextDsn), 33 => Some(Key::NextRetry), 34 => Some(Key::Path), 35 => Some(Key::Policy), 36 => Some(Key::QueueId), 37 => Some(Key::RangeFrom), 38 => Some(Key::RangeTo), 39 => Some(Key::Reason), 40 => Some(Key::RemoteIp), 41 => Some(Key::RemotePort), 42 => Some(Key::ReportId), 43 => Some(Key::Result), 44 => Some(Key::Size), 45 => Some(Key::Source), 46 => Some(Key::SpanId), 47 => Some(Key::SpfFail), 48 => Some(Key::SpfNone), 49 => Some(Key::SpfPass), 50 => Some(Key::Strict), 51 => Some(Key::Tls), 52 => Some(Key::To), 53 => Some(Key::Total), 54 => Some(Key::TotalFailures), 55 => Some(Key::TotalSuccesses), 56 => Some(Key::Type), 57 => Some(Key::Uid), 58 => Some(Key::UidNext), 59 => Some(Key::UidValidity), 60 => Some(Key::Url), 61 => Some(Key::ValidFrom), 62 => Some(Key::ValidTo), 63 => Some(Key::Value), 64 => Some(Key::Version), 65 => Some(Key::QueueName), _ => None, } } } ================================================ FILE: crates/trc/src/serializers/json.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{Error, Event, EventDetails, Key, Value}; use ahash::AHashSet; use base64::{Engine, engine::general_purpose::STANDARD}; use mail_parser::DateTime; use serde::{ Serialize, Serializer, ser::{SerializeMap, SerializeSeq}, }; struct Keys<'x> { keys: &'x [(Key, Value)], span_keys: &'x [(Key, Value)], } pub struct JsonEventSerializer { inner: T, with_id: bool, with_spans: bool, with_description: bool, with_explanation: bool, } impl JsonEventSerializer { pub fn new(inner: T) -> Self { Self { inner, with_id: false, with_spans: false, with_description: false, with_explanation: false, } } pub fn with_id(mut self) -> Self { self.with_id = true; self } pub fn with_spans(mut self) -> Self { self.with_spans = true; self } pub fn with_description(mut self) -> Self { self.with_description = true; self } pub fn with_explanation(mut self) -> Self { self.with_explanation = true; self } pub fn into_inner(self) -> T { self.inner } } impl>> Serialize for JsonEventSerializer> { fn serialize(&self, serializer: S) -> Result where S: Serializer, { let mut seq = serializer.serialize_seq(Some(self.inner.len()))?; for event in &self.inner { seq.serialize_element(&JsonEventSerializer { inner: event, with_id: self.with_id, with_spans: self.with_spans, with_description: self.with_description, with_explanation: self.with_explanation, })?; } seq.end() } } impl>> Serialize for JsonEventSerializer { fn serialize(&self, serializer: S) -> Result where S: Serializer, { let event = self.inner.as_ref(); let mut map = serializer.serialize_map(None)?; if self.with_id { map.serialize_entry( "id", &format!("{}{}", event.inner.timestamp, event.inner.typ.id()), )?; } if self.with_description { map.serialize_entry("text", event.inner.typ.description())?; } if self.with_explanation { map.serialize_entry("details", event.inner.typ.explain())?; } map.serialize_entry( "createdAt", &DateTime::from_timestamp(event.inner.timestamp as i64).to_rfc3339(), )?; map.serialize_entry("type", event.inner.typ.name())?; map.serialize_entry( "data", &JsonEventSerializer { inner: Keys { keys: event.keys.as_slice(), span_keys: event .inner .span .as_ref() .map(|s| &s.keys[..]) .unwrap_or(&[]), }, with_spans: self.with_spans, with_description: self.with_description, with_explanation: self.with_explanation, with_id: self.with_id, }, )?; map.end() } } impl Serialize for JsonEventSerializer> { fn serialize(&self, serializer: S) -> Result where S: Serializer, { let keys_len = self.inner.keys.len() + self.inner.span_keys.len(); let mut seen_keys = AHashSet::with_capacity(keys_len); let mut keys = serializer.serialize_map(Some(keys_len))?; for (key, value) in self.inner.keys.iter().chain(self.inner.span_keys.iter()) { if !matches!(value, Value::None) && (self.with_spans || !matches!(key, Key::SpanId)) && seen_keys.insert(*key) { keys.serialize_entry( key.name(), &JsonEventSerializer { inner: value, with_spans: self.with_spans, with_description: self.with_description, with_explanation: self.with_explanation, with_id: self.with_id, }, )?; } } keys.end() } } impl Serialize for JsonEventSerializer<&Error> { fn serialize(&self, serializer: S) -> Result where S: Serializer, { let mut map = serializer.serialize_map(None)?; map.serialize_entry("type", self.inner.0.inner.name())?; if self.with_description { map.serialize_entry("text", self.inner.0.inner.description())?; } if self.with_explanation { map.serialize_entry("details", self.inner.0.inner.explain())?; } map.serialize_entry( "data", &JsonEventSerializer { inner: Keys { keys: self.inner.0.keys.as_slice(), span_keys: &[], }, with_spans: self.with_spans, with_description: self.with_description, with_explanation: self.with_explanation, with_id: self.with_id, }, )?; map.end() } } impl Serialize for JsonEventSerializer<&Value> { fn serialize(&self, serializer: S) -> Result where S: Serializer, { match &self.inner { Value::String(value) => value.serialize(serializer), Value::UInt(value) => value.serialize(serializer), Value::Int(value) => value.serialize(serializer), Value::Float(value) => value.serialize(serializer), Value::Timestamp(value) => DateTime::from_timestamp(*value as i64) .to_rfc3339() .serialize(serializer), Value::Duration(value) => value.serialize(serializer), Value::Bytes(value) => STANDARD.encode(value).serialize(serializer), Value::Bool(value) => value.serialize(serializer), Value::Ipv4(value) => value.serialize(serializer), Value::Ipv6(value) => value.serialize(serializer), Value::Event(value) => JsonEventSerializer { inner: value, with_spans: self.with_spans, with_description: self.with_description, with_explanation: self.with_explanation, with_id: self.with_id, } .serialize(serializer), Value::Array(value) => JsonEventSerializer { inner: value, with_spans: self.with_spans, with_description: self.with_description, with_explanation: self.with_explanation, with_id: self.with_id, } .serialize(serializer), Value::None => unreachable!(), } } } impl Serialize for JsonEventSerializer<&Vec> { fn serialize(&self, serializer: S) -> Result where S: Serializer, { let mut seq = serializer.serialize_seq(Some(self.inner.len()))?; for value in self.inner { seq.serialize_element(&JsonEventSerializer { inner: value, with_spans: self.with_spans, with_description: self.with_description, with_explanation: self.with_explanation, with_id: self.with_id, })?; } seq.end() } } ================================================ FILE: crates/trc/src/serializers/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod json; pub mod text; // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL #[cfg(feature = "enterprise")] pub mod binary; // SPDX-SnippetEnd ================================================ FILE: crates/trc/src/serializers/text.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::fmt::Display; use mail_parser::DateTime; use tokio::io::{AsyncWrite, AsyncWriteExt}; use crate::{Error, Event, EventDetails, Key, Level, Value}; use base64::{Engine, engine::general_purpose::STANDARD}; pub struct FmtWriter { writer: T, ansi: bool, multiline: bool, } #[allow(dead_code)] enum Color { Black, Red, Green, Yellow, Blue, Magenta, Cyan, White, } impl FmtWriter { pub fn new(writer: T) -> Self { Self { writer, ansi: false, multiline: false, } } pub fn with_ansi(self, ansi: bool) -> Self { Self { ansi, ..self } } pub fn with_multiline(self, multiline: bool) -> Self { Self { multiline, ..self } } pub async fn write(&mut self, event: &Event) -> std::io::Result<()> { // Write timestamp if self.ansi { self.writer .write_all(Color::White.as_code().as_bytes()) .await?; } self.writer .write_all( DateTime::from_timestamp(event.inner.timestamp as i64) .to_rfc3339() .as_bytes(), ) .await?; if self.ansi { self.writer.write_all(Color::reset().as_bytes()).await?; } self.writer.write_all(" ".as_bytes()).await?; // Write level if self.ansi { self.writer .write_all( match event.inner.level { Level::Error => Color::Red, Level::Warn => Color::Yellow, Level::Info => Color::Green, Level::Debug => Color::Blue, Level::Trace => Color::Magenta, Level::Disable => return Ok(()), } .as_code_bold() .as_bytes(), ) .await?; } self.writer .write_all(event.inner.level.as_str().as_bytes()) .await?; if self.ansi { self.writer.write_all(Color::reset().as_bytes()).await?; } self.writer.write_all(" ".as_bytes()).await?; // Write message if self.ansi { self.writer .write_all(Color::White.as_code_bold().as_bytes()) .await?; } self.writer .write_all(event.inner.typ.description().as_bytes()) .await?; if self.ansi { self.writer.write_all(Color::reset().as_bytes()).await?; } self.writer.write_all(" (".as_bytes()).await?; self.writer .write_all(event.inner.typ.name().as_bytes()) .await?; self.writer .write_all(if self.multiline { ")\n" } else { ") " }.as_bytes()) .await?; // Write keys if let Some(parent_event) = &event.inner.span { self.write_keys(&parent_event.keys, &event.keys, 1).await?; } else { self.write_keys(&[], &event.keys, 1).await?; } if !self.multiline { self.writer.write_all("\n".as_bytes()).await?; } Ok(()) } async fn write_keys( &mut self, span_keys: &[(Key, Value)], keys: &[(Key, Value)], indent: usize, ) -> std::io::Result<()> { Box::pin(async move { let mut is_first = true; for (key, value) in span_keys.iter().chain(keys.iter()) { if matches!(key, Key::SpanId) { continue; } else if is_first { is_first = false; } else if !self.multiline { self.writer.write_all(", ".as_bytes()).await?; } // Write key if self.multiline { for _ in 0..indent { self.writer.write_all("\t".as_bytes()).await?; } } if self.ansi { self.writer .write_all(Color::Cyan.as_code().as_bytes()) .await?; } self.writer.write_all(key.name().as_bytes()).await?; if self.ansi { self.writer.write_all(Color::reset().as_bytes()).await?; } // Write value self.writer.write_all(" = ".as_bytes()).await?; self.write_value(value, indent).await?; if self.multiline && !matches!(value, Value::Event(_)) { self.writer.write_all("\n".as_bytes()).await?; } } Ok(()) }) .await } async fn write_value(&mut self, value: &Value, indent: usize) -> std::io::Result<()> { Box::pin(async move { match value { Value::String(v) => { self.writer.write_all("\"".as_bytes()).await?; for ch in v.as_bytes() { match ch { b'\r' => { self.writer.write_all("\\r".as_bytes()).await?; } b'\n' => { self.writer.write_all("\\n".as_bytes()).await?; } b'\t' => { self.writer.write_all("\\t".as_bytes()).await?; } b'\\' => { self.writer.write_all("\\\\".as_bytes()).await?; } _ => { self.writer.write_all(&[*ch]).await?; } } } self.writer.write_all("\"".as_bytes()).await?; } Value::UInt(v) => { self.writer.write_all(v.to_string().as_bytes()).await?; } Value::Int(v) => { self.writer.write_all(v.to_string().as_bytes()).await?; } Value::Float(v) => { self.writer.write_all(v.to_string().as_bytes()).await?; } Value::Timestamp(v) => { self.writer .write_all(DateTime::from_timestamp(*v as i64).to_rfc3339().as_bytes()) .await?; } Value::Duration(v) => { self.writer.write_all(v.to_string().as_bytes()).await?; self.writer.write_all("ms".as_bytes()).await?; } Value::Bytes(bytes) => { self.writer.write_all("base64:".as_bytes()).await?; self.writer .write_all(STANDARD.encode(bytes).as_bytes()) .await?; } Value::Bool(true) => { self.writer.write_all("true".as_bytes()).await?; } Value::Bool(false) => { self.writer.write_all("false".as_bytes()).await?; } Value::Ipv4(v) => { self.writer.write_all(v.to_string().as_bytes()).await?; } Value::Ipv6(v) => { self.writer.write_all(v.to_string().as_bytes()).await?; } Value::Event(e) => { self.writer .write_all(e.0.inner.description().as_bytes()) .await?; self.writer.write_all(" (".as_bytes()).await?; self.writer.write_all(e.0.inner.name().as_bytes()).await?; self.writer.write_all(")".as_bytes()).await?; if !e.0.keys.is_empty() { self.writer .write_all(if self.multiline { "\n" } else { " { " }.as_bytes()) .await?; self.write_keys(&e.0.keys, &[], indent + 1).await?; if !self.multiline { self.writer.write_all(" }".as_bytes()).await?; } } else if self.multiline { self.writer.write_all("\n".as_bytes()).await?; } } Value::Array(arr) => { self.writer.write_all("[".as_bytes()).await?; for (pos, value) in arr.iter().enumerate() { if pos > 0 { self.writer.write_all(", ".as_bytes()).await?; } self.write_value(value, indent).await?; } self.writer.write_all("]".as_bytes()).await?; } Value::None => { self.writer.write_all("(null)".as_bytes()).await?; } } Ok(()) }) .await } pub async fn flush(&mut self) -> std::io::Result<()> { self.writer.flush().await } pub fn update_writer(&mut self, writer: T) { self.writer = writer; } } impl Color { pub fn as_code(&self) -> &'static str { match self { Color::Black => "\x1b[30m", Color::Red => "\x1b[31m", Color::Green => "\x1b[32m", Color::Yellow => "\x1b[33m", Color::Blue => "\x1b[34m", Color::Magenta => "\x1b[35m", Color::Cyan => "\x1b[36m", Color::White => "\x1b[37m", } } pub fn as_code_bold(&self) -> &'static str { match self { Color::Black => "\x1b[30;1m", Color::Red => "\x1b[31;1m", Color::Green => "\x1b[32;1m", Color::Yellow => "\x1b[33;1m", Color::Blue => "\x1b[34;1m", Color::Magenta => "\x1b[35;1m", Color::Cyan => "\x1b[36;1m", Color::White => "\x1b[37;1m", } } pub fn reset() -> &'static str { "\x1b[0m" } } impl Display for Value { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Value::String(value) => value.fmt(f), Value::UInt(value) => value.fmt(f), Value::Int(value) => value.fmt(f), Value::Float(value) => value.fmt(f), Value::Timestamp(value) => value.fmt(f), Value::Duration(value) => value.fmt(f), Value::Bytes(value) => STANDARD.encode(value).fmt(f), Value::Bool(value) => value.fmt(f), Value::Ipv4(value) => value.fmt(f), Value::Ipv6(value) => value.fmt(f), Value::Event(value) => { "{".fmt(f)?; value.fmt(f)?; "}".fmt(f) } Value::Array(value) => { f.write_str("[")?; for (i, value) in value.iter().enumerate() { if i > 0 { f.write_str(", ")?; } value.fmt(f)?; } f.write_str("]") } Value::None => "(null)".fmt(f), } } } impl Display for Error { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { self.0.inner.description().fmt(f)?; " (".fmt(f)?; self.0.inner.name().fmt(f)?; ")".fmt(f)?; if !self.0.keys.is_empty() { f.write_str(": ")?; for (i, (key, value)) in self.0.keys.iter().enumerate() { if i > 0 { f.write_str(", ")?; } key.name().fmt(f)?; f.write_str(" = ")?; value.fmt(f)?; } } Ok(()) } } #[cfg(test)] mod tests { use crate::{EventType, Level}; #[allow(dead_code)] fn to_camel_case(name: &str) -> String { let mut out = String::with_capacity(name.len()); let mut upper = true; for ch in name.chars() { if ch.is_alphanumeric() { if upper { out.push(ch.to_ascii_uppercase()); upper = false; } else { out.push(ch); } } else { upper = true; } } out } #[allow(dead_code)] fn event_to_class(name: &str) -> String { let (group, name) = name.split_once('.').unwrap(); let group = to_camel_case(group); format!( "EventType::{}({}Event::{})", group, group, to_camel_case(name) ) } #[allow(dead_code)] fn event_to_webadmin_class(name: &str) -> String { let (group, name) = name.split_once('.').unwrap(); format!("{}{}", to_camel_case(group), to_camel_case(name)) } #[test] fn print_all_events() { assert!(!Level::Disable.is_contained(Level::Warn)); assert!(Level::Trace.is_contained(Level::Error)); assert!(Level::Trace.is_contained(Level::Debug)); assert!(!Level::Error.is_contained(Level::Trace)); assert!(!Level::Debug.is_contained(Level::Trace)); let mut names = Vec::with_capacity(100); for event in EventType::variants() { names.push((event.name(), event.description(), event.level().as_str())); assert_eq!(EventType::try_parse(event.name()).unwrap(), event); } // sort by name names.sort_by(|a, b| a.0.cmp(b.0)); for (name, description, level) in names { //println!("{:?},", name); println!("|`{name}`|{description}|`{level}`|") } //for (pos, (name, _, _)) in names.iter().enumerate() { //println!("{:?},", name); //println!("{} => Some({}),", pos, event_to_class(name)); //println!("{} => {},", event_to_class(name), pos); /*println!( "#[serde(rename = \"{name}\")]\n{},", event_to_webadmin_class(name) );*/ //} } } ================================================ FILE: crates/types/Cargo.toml ================================================ [package] name = "types" version = "0.15.5" edition = "2024" [dependencies] utils = { path = "../utils" } trc = { path = "../trc" } jmap-tools = { version = "0.1" } hashify = "0.2" serde = { version = "1.0", features = ["derive"]} rkyv = { version = "0.8.10", features = ["little_endian"] } compact_str = { version = "0.9.0", features = ["rkyv", "serde"] } blake3 = "1.3.3" [features] test_mode = [] ================================================ FILE: crates/types/src/acl.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::fmt::{self, Display}; use utils::map::bitmap::{Bitmap, BitmapItem}; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Copy, )] #[rkyv(compare(PartialEq), derive(Debug))] #[repr(u8)] pub enum Acl { Read = 0, Modify = 1, Delete = 2, ReadItems = 3, AddItems = 4, ModifyItems = 5, RemoveItems = 6, CreateChild = 7, Share = 8, Submit = 9, SchedulingReadFreeBusy = 10, SchedulingInvite = 11, SchedulingReply = 12, ModifyItemsOwn = 13, ModifyPrivateProperties = 14, ModifyRSVP = 15, None = 16, } #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq, serde::Serialize, Default, )] #[rkyv(compare(PartialEq), derive(Debug))] pub struct AclGrant { pub account_id: u32, pub grants: Bitmap, } impl Acl { fn as_str(&self) -> &'static str { match self { Acl::Read => "read", Acl::Modify => "modify", Acl::Delete => "delete", Acl::ReadItems => "readItems", Acl::AddItems => "addItems", Acl::ModifyItems => "modifyItems", Acl::RemoveItems => "removeItems", Acl::CreateChild => "createChild", Acl::Share => "share", Acl::Submit => "submit", Acl::ModifyItemsOwn => "modifyItemsOwn", Acl::ModifyPrivateProperties => "modifyPrivateProperties", Acl::None => "", Acl::SchedulingReadFreeBusy => "schedulingReadFreeBusy", Acl::SchedulingInvite => "schedulingInvite", Acl::SchedulingReply => "schedulingReply", Acl::ModifyRSVP => "modifyRSVP", } } } impl Display for Acl { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{}", self.as_str()) } } impl serde::Serialize for Acl { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { serializer.serialize_str(self.as_str()) } } impl BitmapItem for Acl { fn max() -> u64 { Acl::None as u64 } fn is_valid(&self) -> bool { !matches!(self, Acl::None) } } impl From for u64 { fn from(value: Acl) -> Self { value as u64 } } impl From for Acl { fn from(value: u64) -> Self { match value { 0 => Acl::Read, 1 => Acl::Modify, 2 => Acl::Delete, 3 => Acl::ReadItems, 4 => Acl::AddItems, 5 => Acl::ModifyItems, 6 => Acl::RemoveItems, 7 => Acl::CreateChild, 8 => Acl::Share, 9 => Acl::Submit, 10 => Acl::SchedulingReadFreeBusy, 11 => Acl::SchedulingInvite, 12 => Acl::SchedulingReply, 13 => Acl::ModifyItemsOwn, 14 => Acl::ModifyPrivateProperties, 15 => Acl::ModifyRSVP, _ => Acl::None, } } } impl From<&ArchivedAclGrant> for AclGrant { fn from(value: &ArchivedAclGrant) -> Self { Self { account_id: u32::from(value.account_id), grants: (&value.grants).into(), } } } ================================================ FILE: crates/types/src/blob.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use jmap_tools::{Element, Property, Value}; use std::{borrow::Borrow, str::FromStr, time::SystemTime}; use utils::codec::{ base32_custom::{Base32Reader, Base32Writer}, leb128::{Leb128Iterator, Leb128Writer}, }; use crate::blob_hash::BlobHash; const B_LINKED: u8 = 0x10; const B_RESERVED: u8 = 0x20; #[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum BlobClass { Reserved { account_id: u32, expires: u64, }, Linked { account_id: u32, collection: u8, document_id: u32, }, } impl Default for BlobClass { fn default() -> Self { BlobClass::Reserved { account_id: 0, expires: 0, } } } impl AsRef for BlobClass { fn as_ref(&self) -> &BlobClass { self } } impl BlobClass { pub fn account_id(&self) -> u32 { match self { BlobClass::Reserved { account_id, .. } | BlobClass::Linked { account_id, .. } => { *account_id } } } pub fn is_valid(&self) -> bool { match self { BlobClass::Reserved { expires, .. } => { *expires > SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()) } BlobClass::Linked { .. } => true, } } } #[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct BlobId { pub hash: BlobHash, pub class: BlobClass, pub section: Option, } #[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct BlobSection { pub offset_start: usize, pub size: usize, pub encoding: u8, } impl FromStr for BlobId { type Err = (); fn from_str(s: &str) -> Result { BlobId::from_base32(s).ok_or(()) } } impl BlobId { pub fn new(hash: BlobHash, class: BlobClass) -> Self { BlobId { hash, class, section: None, } } pub fn new_section( hash: BlobHash, class: BlobClass, offset_start: usize, offset_end: usize, encoding: impl Into, ) -> Self { BlobId { hash, class, section: BlobSection { offset_start, size: offset_end - offset_start, encoding: encoding.into(), } .into(), } } pub fn with_section_size(mut self, size: usize) -> Self { self.section.get_or_insert_with(Default::default).size = size; self } #[inline] pub fn from_base32(value: impl AsRef<[u8]>) -> Option { BlobId::from_iter(&mut Base32Reader::new(value.as_ref())) } #[allow(clippy::should_implement_trait)] pub fn from_iter(it: &mut T) -> Option where T: Iterator + Leb128Iterator, U: Borrow, { let class = *it.next()?.borrow(); let encoding = class & 0x0F; let mut hash = BlobHash::default(); for byte in hash.as_mut().iter_mut() { *byte = *it.next()?.borrow(); } let account_id: u32 = it.next_leb128()?; BlobId { hash, class: if (class & B_LINKED) != 0 { BlobClass::Linked { account_id, collection: *it.next()?.borrow(), document_id: it.next_leb128()?, } } else { BlobClass::Reserved { account_id, expires: it.next_leb128()?, } }, section: if encoding != 0 { BlobSection { offset_start: it.next_leb128()?, size: it.next_leb128()?, encoding: encoding - 1, } .into() } else { None }, } .into() } fn serialize_as(&self, writer: &mut impl Leb128Writer) { let marker = self .section .as_ref() .map_or(0, |section| section.encoding + 1) | if matches!( self, BlobId { class: BlobClass::Linked { .. }, .. } ) { B_LINKED } else { B_RESERVED }; let _ = writer.write(&[marker]); let _ = writer.write(self.hash.as_ref()); match &self.class { BlobClass::Reserved { account_id, expires, } => { let _ = writer.write_leb128(*account_id); let _ = writer.write_leb128(*expires); } BlobClass::Linked { account_id, collection, document_id, } => { let _ = writer.write_leb128(*account_id); let _ = writer.write(&[*collection]); let _ = writer.write_leb128(*document_id); } } if let Some(section) = &self.section { let _ = writer.write_leb128(section.offset_start); let _ = writer.write_leb128(section.size); } } pub fn start_offset(&self) -> usize { if let Some(section) = &self.section { section.offset_start } else { 0 } } } impl serde::Serialize for BlobId { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { serializer.serialize_str(self.to_string().as_str()) } } impl std::fmt::Display for BlobId { #[allow(clippy::unused_io_amount)] fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let mut writer = Base32Writer::with_capacity(std::mem::size_of::() * 2); self.serialize_as(&mut writer); f.write_str(&writer.finalize()) } } impl<'x, P: Property, E: Element + From> From for Value<'x, P, E> { fn from(id: BlobId) -> Self { Value::Element(E::from(id)) } } ================================================ FILE: crates/types/src/blob_hash.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub const BLOB_HASH_LEN: usize = 32; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Clone, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize, )] #[rkyv(derive(Debug))] #[repr(transparent)] pub struct BlobHash(pub [u8; BLOB_HASH_LEN]); impl BlobHash { pub fn new_max() -> Self { BlobHash([u8::MAX; BLOB_HASH_LEN]) } pub fn generate(value: impl AsRef<[u8]>) -> Self { BlobHash(blake3::hash(value.as_ref()).into()) } pub fn try_from_hash_slice(value: &[u8]) -> Result { value.try_into().map(BlobHash) } pub fn as_slice(&self) -> &[u8] { self.0.as_ref() } pub fn to_hex(&self) -> String { let mut hex = String::with_capacity(BLOB_HASH_LEN * 2); for byte in self.0.iter() { hex.push_str(&format!("{:02x}", byte)); } hex } pub fn is_empty(&self) -> bool { self.0 == [0; BLOB_HASH_LEN] } } impl From<&ArchivedBlobHash> for BlobHash { fn from(value: &ArchivedBlobHash) -> Self { BlobHash(value.0) } } impl AsRef for BlobHash { fn as_ref(&self) -> &BlobHash { self } } impl From for Vec { fn from(value: BlobHash) -> Self { value.0.to_vec() } } impl AsRef<[u8]> for BlobHash { fn as_ref(&self) -> &[u8] { self.0.as_ref() } } impl AsMut<[u8]> for BlobHash { fn as_mut(&mut self) -> &mut [u8] { self.0.as_mut() } } ================================================ FILE: crates/types/src/collection.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::type_state::DataType; use compact_str::CompactString; use std::{ fmt::{self, Display, Formatter}, str::FromStr, }; use utils::map::bitmap::BitmapItem; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, Copy, Hash, PartialEq, Eq, Default, )] #[repr(u8)] pub enum Collection { Email = 0, Mailbox = 1, Thread = 2, Identity = 3, EmailSubmission = 4, SieveScript = 5, PushSubscription = 6, Principal = 7, Calendar = 8, CalendarEvent = 9, AddressBook = 10, ContactCard = 11, FileNode = 12, CalendarEventNotification = 13, #[default] None = 14, } #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Default)] #[repr(u8)] pub enum SyncCollection { Email = 0, Thread = 1, Calendar = 2, AddressBook = 3, FileNode = 4, Identity = 5, EmailSubmission = 6, SieveScript = 7, CalendarEventNotification = 8, ShareNotification = 9, #[default] None = 10, } #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] #[repr(u8)] pub enum VanishedCollection { Email = 251, Calendar = 252, AddressBook = 253, FileNode = 254, } impl Collection { pub const MAX: usize = Collection::None as usize; pub fn main_collection(&self) -> Collection { match self { Collection::Email => Collection::Mailbox, Collection::CalendarEvent => Collection::Calendar, Collection::ContactCard => Collection::AddressBook, _ => *self, } } pub fn parent_collection(&self) -> Option { match self { Collection::Email => Some(Collection::Mailbox), Collection::CalendarEvent => Some(Collection::Calendar), Collection::ContactCard => Some(Collection::AddressBook), Collection::FileNode => Some(Collection::FileNode), Collection::CalendarEventNotification => Some(Collection::CalendarEventNotification), _ => None, } } pub fn child_collection(&self) -> Option { match self { Collection::Mailbox => Some(Collection::Email), Collection::Calendar => Some(Collection::CalendarEvent), Collection::AddressBook => Some(Collection::ContactCard), Collection::FileNode => Some(Collection::FileNode), Collection::CalendarEventNotification => Some(Collection::CalendarEventNotification), _ => None, } } } impl SyncCollection { pub fn collection(&self, is_container: bool) -> Collection { match self { SyncCollection::Email => { if is_container { Collection::Mailbox } else { Collection::Email } } SyncCollection::Thread => Collection::Thread, SyncCollection::Calendar => { if is_container { Collection::Calendar } else { Collection::CalendarEvent } } SyncCollection::AddressBook => { if is_container { Collection::AddressBook } else { Collection::ContactCard } } SyncCollection::FileNode => Collection::FileNode, SyncCollection::Identity => Collection::Identity, SyncCollection::EmailSubmission => Collection::EmailSubmission, SyncCollection::SieveScript => Collection::SieveScript, SyncCollection::CalendarEventNotification => Collection::CalendarEventNotification, SyncCollection::ShareNotification | SyncCollection::None => Collection::None, } } pub fn vanished_collection(&self) -> Option { match self { SyncCollection::Email => Some(VanishedCollection::Email), SyncCollection::Calendar => Some(VanishedCollection::Calendar), SyncCollection::AddressBook => Some(VanishedCollection::AddressBook), SyncCollection::FileNode => Some(VanishedCollection::FileNode), _ => None, } } } impl From for SyncCollection { fn from(v: Collection) -> Self { match v { Collection::Email => SyncCollection::Email, Collection::Mailbox => SyncCollection::Email, Collection::Thread => SyncCollection::Thread, Collection::Identity => SyncCollection::Identity, Collection::EmailSubmission => SyncCollection::EmailSubmission, Collection::SieveScript => SyncCollection::SieveScript, Collection::PushSubscription => SyncCollection::None, Collection::Principal => SyncCollection::None, Collection::Calendar => SyncCollection::Calendar, Collection::CalendarEvent => SyncCollection::Calendar, Collection::CalendarEventNotification => SyncCollection::CalendarEventNotification, Collection::AddressBook => SyncCollection::AddressBook, Collection::ContactCard => SyncCollection::AddressBook, Collection::FileNode => SyncCollection::FileNode, _ => SyncCollection::None, } } } impl From for Collection { fn from(v: u8) -> Self { match v { 0 => Collection::Email, 1 => Collection::Mailbox, 2 => Collection::Thread, 3 => Collection::Identity, 4 => Collection::EmailSubmission, 5 => Collection::SieveScript, 6 => Collection::PushSubscription, 7 => Collection::Principal, 8 => Collection::Calendar, 9 => Collection::CalendarEvent, 10 => Collection::AddressBook, 11 => Collection::ContactCard, 12 => Collection::FileNode, 13 => Collection::CalendarEventNotification, _ => Collection::None, } } } impl From for SyncCollection { fn from(v: u8) -> Self { match v { 0 => SyncCollection::Email, 1 => SyncCollection::Thread, 2 => SyncCollection::Calendar, 3 => SyncCollection::AddressBook, 4 => SyncCollection::FileNode, 5 => SyncCollection::Identity, 6 => SyncCollection::EmailSubmission, 7 => SyncCollection::SieveScript, 8 => SyncCollection::CalendarEventNotification, 9 => SyncCollection::ShareNotification, _ => SyncCollection::None, } } } impl From for SyncCollection { fn from(v: u64) -> Self { match v { 0 => SyncCollection::Email, 1 => SyncCollection::Thread, 2 => SyncCollection::Calendar, 3 => SyncCollection::AddressBook, 4 => SyncCollection::FileNode, 5 => SyncCollection::Identity, 6 => SyncCollection::EmailSubmission, 7 => SyncCollection::SieveScript, 8 => SyncCollection::CalendarEventNotification, 9 => SyncCollection::ShareNotification, _ => SyncCollection::None, } } } impl From for Collection { fn from(v: u64) -> Self { match v { 0 => Collection::Email, 1 => Collection::Mailbox, 2 => Collection::Thread, 3 => Collection::Identity, 4 => Collection::EmailSubmission, 5 => Collection::SieveScript, 6 => Collection::PushSubscription, 7 => Collection::Principal, 8 => Collection::Calendar, 9 => Collection::CalendarEvent, 10 => Collection::AddressBook, 11 => Collection::ContactCard, 12 => Collection::FileNode, 13 => Collection::CalendarEventNotification, _ => Collection::None, } } } impl From for u8 { fn from(v: Collection) -> Self { v as u8 } } impl From for u8 { fn from(v: SyncCollection) -> Self { v as u8 } } impl From for u64 { fn from(v: SyncCollection) -> Self { v as u64 } } impl From for u8 { fn from(v: VanishedCollection) -> Self { v as u8 } } impl From for u64 { fn from(collection: Collection) -> u64 { collection as u64 } } impl TryFrom for DataType { type Error = (); fn try_from(value: Collection) -> Result { match value { Collection::Email => Ok(DataType::Email), Collection::Mailbox => Ok(DataType::Mailbox), Collection::Thread => Ok(DataType::Thread), Collection::Identity => Ok(DataType::Identity), Collection::EmailSubmission => Ok(DataType::EmailSubmission), Collection::SieveScript => Ok(DataType::SieveScript), Collection::PushSubscription => Ok(DataType::PushSubscription), Collection::Principal => Ok(DataType::Principal), Collection::Calendar => Ok(DataType::Calendar), Collection::CalendarEvent => Ok(DataType::CalendarEvent), Collection::AddressBook => Ok(DataType::AddressBook), Collection::ContactCard => Ok(DataType::ContactCard), Collection::FileNode => Ok(DataType::FileNode), Collection::CalendarEventNotification => Ok(DataType::CalendarEventNotification), _ => Err(()), } } } impl TryFrom for Collection { type Error = (); fn try_from(value: DataType) -> Result { match value { DataType::Email => Ok(Collection::Email), DataType::Mailbox => Ok(Collection::Mailbox), DataType::Thread => Ok(Collection::Thread), DataType::Identity => Ok(Collection::Identity), DataType::EmailSubmission => Ok(Collection::EmailSubmission), DataType::SieveScript => Ok(Collection::SieveScript), DataType::PushSubscription => Ok(Collection::PushSubscription), DataType::Principal => Ok(Collection::Principal), DataType::Calendar => Ok(Collection::Calendar), DataType::CalendarEvent => Ok(Collection::CalendarEvent), DataType::AddressBook => Ok(Collection::AddressBook), DataType::ContactCard => Ok(Collection::ContactCard), DataType::FileNode => Ok(Collection::FileNode), DataType::CalendarEventNotification => Ok(Collection::CalendarEventNotification), _ => Err(()), } } } impl Display for Collection { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { self.as_str().fmt(f) } } impl Collection { pub fn as_str(&self) -> &'static str { match self { Collection::PushSubscription => "pushSubscription", Collection::Email => "email", Collection::Mailbox => "mailbox", Collection::Thread => "thread", Collection::Identity => "identity", Collection::EmailSubmission => "emailSubmission", Collection::SieveScript => "sieveScript", Collection::Principal => "principal", Collection::Calendar => "calendar", Collection::CalendarEvent => "calendarEvent", Collection::AddressBook => "addressBook", Collection::ContactCard => "contactCard", Collection::FileNode => "fileNode", Collection::CalendarEventNotification => "calendarEventNotification", Collection::None => "", } } pub fn as_config_case(&self) -> &'static str { match self { Collection::PushSubscription => "push-subscription", Collection::Email => "email", Collection::Mailbox => "mailbox", Collection::Thread => "thread", Collection::Identity => "identity", Collection::EmailSubmission => "email-submission", Collection::SieveScript => "sieve-script", Collection::Principal => "principal", Collection::Calendar => "calendar", Collection::CalendarEvent => "calendar-event", Collection::AddressBook => "address-book", Collection::ContactCard => "contact-card", Collection::FileNode => "file-node", Collection::CalendarEventNotification => "calendar-event-notification", Collection::None => "", } } } impl FromStr for Collection { type Err = (); fn from_str(s: &str) -> Result { hashify::tiny_map!(s.as_bytes(), "pushSubscription" => Collection::PushSubscription, "email" => Collection::Email, "mailbox" => Collection::Mailbox, "thread" => Collection::Thread, "identity" => Collection::Identity, "emailSubmission" => Collection::EmailSubmission, "sieveScript" => Collection::SieveScript, "principal" => Collection::Principal, "calendar" => Collection::Calendar, "calendarEvent" => Collection::CalendarEvent, "addressBook" => Collection::AddressBook, "contactCard" => Collection::ContactCard, "fileNode" => Collection::FileNode, "calendarEventNotification" => Collection::CalendarEventNotification, ) .ok_or(()) } } impl From for trc::Value { fn from(value: Collection) -> Self { trc::Value::String(CompactString::const_new(value.as_str())) } } impl BitmapItem for Collection { fn max() -> u64 { Collection::None as u64 } fn is_valid(&self) -> bool { !matches!(self, Collection::None) } } impl BitmapItem for SyncCollection { fn max() -> u64 { SyncCollection::None as u64 } fn is_valid(&self) -> bool { !matches!(self, SyncCollection::None) } } impl SyncCollection { pub fn as_str(&self) -> &'static str { match self { SyncCollection::Email => "email", SyncCollection::Thread => "thread", SyncCollection::Calendar => "calendar", SyncCollection::AddressBook => "addressBook", SyncCollection::FileNode => "fileNode", SyncCollection::Identity => "identity", SyncCollection::EmailSubmission => "emailSubmission", SyncCollection::SieveScript => "sieveScript", SyncCollection::CalendarEventNotification => "calendarEventNotification", SyncCollection::ShareNotification => "shareNotification", SyncCollection::None => "", } } } ================================================ FILE: crates/types/src/dead_property.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ #[derive(Debug, Clone, PartialEq, Eq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] #[cfg_attr(feature = "test_mode", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(feature = "test_mode", serde(tag = "type", content = "data"))] #[rkyv(derive(Debug))] pub enum DeadPropertyTag { ElementStart(DeadElementTag), ElementEnd, Text(String), } #[derive(Debug, Clone, PartialEq, Eq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] #[cfg_attr(feature = "test_mode", derive(serde::Serialize, serde::Deserialize))] #[rkyv(derive(Debug))] pub struct DeadElementTag { pub name: String, pub attrs: Option, } #[derive(Debug, Clone, PartialEq, Eq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] #[cfg_attr(feature = "test_mode", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(feature = "test_mode", serde(transparent))] #[rkyv(derive(Debug))] #[repr(transparent)] pub struct DeadProperty(pub Vec); impl From<&ArchivedDeadProperty> for DeadProperty { fn from(value: &ArchivedDeadProperty) -> Self { DeadProperty(value.0.iter().map(|tag| tag.into()).collect::>()) } } impl From<&ArchivedDeadPropertyTag> for DeadPropertyTag { fn from(tag: &ArchivedDeadPropertyTag) -> Self { match tag { ArchivedDeadPropertyTag::ElementStart(tag) => DeadPropertyTag::ElementStart(tag.into()), ArchivedDeadPropertyTag::ElementEnd => DeadPropertyTag::ElementEnd, ArchivedDeadPropertyTag::Text(tag) => DeadPropertyTag::Text(tag.to_string()), } } } impl From<&ArchivedDeadElementTag> for DeadElementTag { fn from(tag: &ArchivedDeadElementTag) -> Self { DeadElementTag { name: tag.name.to_string(), attrs: tag.attrs.as_ref().map(|s| s.to_string()), } } } impl ArchivedDeadProperty { pub fn find_tag(&self, needle: &str) -> Option { let mut depth: u32 = 0; let mut tags = Vec::new(); let mut found_tag = false; for tag in self.0.iter() { match tag { ArchivedDeadPropertyTag::ElementStart(start) => { if depth == 0 && start.name == needle { found_tag = true; } else if found_tag { tags.push(tag.into()); } depth += 1; } ArchivedDeadPropertyTag::ElementEnd => { if found_tag { if depth == 1 { break; } else { tags.push(tag.into()); } } depth = depth.saturating_sub(1); } ArchivedDeadPropertyTag::Text(_) => { if found_tag { tags.push(tag.into()); } } } } if found_tag { Some(DeadProperty(tags)) } else { None } } } impl DeadProperty { pub fn remove_element(&mut self, element: &DeadElementTag) { let mut depth = 0; let mut remove = false; self.0.retain(|item| match item { DeadPropertyTag::ElementStart(tag) => { if depth == 0 && !remove && tag.name == element.name { remove = true; } depth += 1; !remove } DeadPropertyTag::ElementEnd => { depth -= 1; if remove && depth == 0 { remove = false; false } else { !remove } } _ => !remove, }); } pub fn add_element(&mut self, element: DeadElementTag, values: Vec) { self.0.push(DeadPropertyTag::ElementStart(element)); self.0.extend(values); self.0.push(DeadPropertyTag::ElementEnd); } pub fn size(&self) -> usize { let mut size = 0; for item in &self.0 { match item { DeadPropertyTag::ElementStart(tag) => { size += tag.size(); } DeadPropertyTag::ElementEnd => { size += 1; } DeadPropertyTag::Text(text) => { size += text.len(); } } } size } } impl ArchivedDeadProperty { pub fn size(&self) -> usize { let mut size = 0; for item in self.0.iter() { match item { ArchivedDeadPropertyTag::ElementStart(tag) => { size += tag.size(); } ArchivedDeadPropertyTag::ElementEnd => { size += 1; } ArchivedDeadPropertyTag::Text(text) => { size += text.len(); } } } size } } impl DeadElementTag { pub fn new(name: String, attrs: Option) -> Self { DeadElementTag { name, attrs } } pub fn size(&self) -> usize { self.name.len() + self.attrs.as_ref().map_or(0, |attrs| attrs.len()) } } impl ArchivedDeadElementTag { pub fn size(&self) -> usize { self.name.len() + self.attrs.as_ref().map_or(0, |attrs| attrs.len()) } } impl Default for DeadProperty { fn default() -> Self { DeadProperty(Vec::with_capacity(4)) } } ================================================ FILE: crates/types/src/field.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ const ARCHIVE_FIELD: u8 = 50; pub trait FieldType: Into + Copy + std::fmt::Debug + PartialEq + Eq {} #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(transparent)] pub struct Field(u8); #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] pub enum ContactField { Uid, Email, Archive, CreatedToUpdated, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] pub enum CalendarEventField { Uid, Archive, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] pub enum CalendarNotificationField { CreatedToId, Archive, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] pub enum EmailField { Archive, Metadata, Threading, DeletedAt, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] pub enum MailboxField { UidCounter, Archive, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] pub enum SieveField { Name, Ids, Archive, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] pub enum EmailSubmissionField { Archive, Metadata, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] pub enum IdentityField { Archive, DocumentId, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] pub enum PrincipalField { Archive, EncryptionKeys, ParticipantIdentities, DefaultCalendarId, DefaultAddressBookId, ActiveScriptId, PushSubscriptions, } impl From for u8 { fn from(value: ContactField) -> Self { match value { ContactField::Uid => 0, ContactField::Email => 1, ContactField::CreatedToUpdated => 2, ContactField::Archive => ARCHIVE_FIELD, } } } impl From for u8 { fn from(value: CalendarEventField) -> Self { match value { CalendarEventField::Uid => 0, CalendarEventField::Archive => ARCHIVE_FIELD, } } } impl From for u8 { fn from(value: CalendarNotificationField) -> Self { match value { CalendarNotificationField::CreatedToId => 0, CalendarNotificationField::Archive => ARCHIVE_FIELD, } } } impl From for u8 { fn from(value: EmailField) -> Self { match value { EmailField::Metadata => 71, EmailField::Threading => 90, EmailField::DeletedAt => 91, EmailField::Archive => ARCHIVE_FIELD, } } } impl From for u8 { fn from(value: MailboxField) -> Self { match value { MailboxField::UidCounter => 84, MailboxField::Archive => ARCHIVE_FIELD, } } } impl From for u8 { fn from(value: SieveField) -> Self { match value { SieveField::Name => 13, SieveField::Ids => 84, SieveField::Archive => ARCHIVE_FIELD, } } } impl From for u8 { fn from(value: EmailSubmissionField) -> Self { match value { EmailSubmissionField::Metadata => 49, EmailSubmissionField::Archive => ARCHIVE_FIELD, } } } impl From for u8 { fn from(value: PrincipalField) -> Self { match value { PrincipalField::ParticipantIdentities => 45, PrincipalField::EncryptionKeys => 46, PrincipalField::DefaultCalendarId => 47, PrincipalField::DefaultAddressBookId => 48, PrincipalField::ActiveScriptId => 49, PrincipalField::PushSubscriptions => 44, PrincipalField::Archive => ARCHIVE_FIELD, } } } impl From for u8 { fn from(value: IdentityField) -> Self { match value { IdentityField::Archive => ARCHIVE_FIELD, IdentityField::DocumentId => 51, } } } impl From for u8 { fn from(value: Field) -> Self { value.0 } } impl From for Field { fn from(value: ContactField) -> Self { Field(u8::from(value)) } } impl From for Field { fn from(value: CalendarEventField) -> Self { Field(u8::from(value)) } } impl From for Field { fn from(value: CalendarNotificationField) -> Self { Field(u8::from(value)) } } impl From for Field { fn from(value: EmailField) -> Self { Field(u8::from(value)) } } impl From for Field { fn from(value: MailboxField) -> Self { Field(u8::from(value)) } } impl From for Field { fn from(value: PrincipalField) -> Self { Field(u8::from(value)) } } impl From for Field { fn from(value: SieveField) -> Self { Field(u8::from(value)) } } impl From for Field { fn from(value: EmailSubmissionField) -> Self { Field(u8::from(value)) } } impl From for Field { fn from(value: IdentityField) -> Self { Field(u8::from(value)) } } impl Field { pub const ARCHIVE: Field = Field(ARCHIVE_FIELD); pub fn new(value: u8) -> Self { Field(value) } pub fn inner(&self) -> u8 { self.0 } } impl FieldType for Field {} impl FieldType for ContactField {} impl FieldType for CalendarEventField {} impl FieldType for CalendarNotificationField {} impl FieldType for EmailField {} impl FieldType for MailboxField {} impl FieldType for PrincipalField {} impl FieldType for SieveField {} impl FieldType for EmailSubmissionField {} impl FieldType for IdentityField {} ================================================ FILE: crates/types/src/id.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::DocumentId; use jmap_tools::{Element, Property, Value}; use std::{ops::Deref, str::FromStr}; use utils::codec::base32_custom::{BASE32_ALPHABET, BASE32_INVERSE}; #[derive(Debug, Clone, PartialEq, Eq, Hash, Copy, PartialOrd, Ord)] #[repr(transparent)] pub struct Id(u64); impl Default for Id { fn default() -> Self { Id(u64::MAX) } } impl FromStr for Id { type Err = (); fn from_str(s: &str) -> Result { let mut id = 0; for &ch in s.as_bytes() { let i = BASE32_INVERSE[ch as usize]; if i != u8::MAX { id = (id << 5) | i as u64; } else { return Err(()); } } Ok(Id(id)) } } impl Id { pub fn new(id: u64) -> Self { Self(id) } pub fn singleton() -> Self { Self::new(20080258862541) } // From https://github.com/archer884/crockford by J/A // License: MIT/Apache 2.0 pub fn as_string(&self) -> String { match self.0 { 0 => "a".to_string(), mut n => { // Used for the initial shift. const QUAD_SHIFT: usize = 60; const QUAD_RESET: usize = 4; // Used for all subsequent shifts. const FIVE_SHIFT: usize = 59; const FIVE_RESET: usize = 5; // After we clear the four most significant bits, the four least significant bits will be // replaced with 0001. We can then know to stop once the four most significant bits are, // likewise, 0001. const STOP_BIT: u64 = 1 << QUAD_SHIFT; let mut buf = String::with_capacity(7); // Start by getting the most significant four bits. We get four here because these would be // leftovers when starting from the least significant bits. In either case, tag the four least // significant bits with our stop bit. match (n >> QUAD_SHIFT) as usize { // Eat leading zero-bits. This should not be done if the first four bits were non-zero. // Additionally, we *must* do this in increments of five bits. 0 => { n <<= QUAD_RESET; n |= 1; n <<= n.leading_zeros() / 5 * 5; } // Write value of first four bytes. i => { n <<= QUAD_RESET; n |= 1; buf.push(char::from(BASE32_ALPHABET[i])); } } // From now until we reach the stop bit, take the five most significant bits and then shift // left by five bits. while n != STOP_BIT { buf.push(char::from(BASE32_ALPHABET[(n >> FIVE_SHIFT) as usize])); n <<= FIVE_RESET; } buf } } } #[inline(always)] pub fn from_parts(prefix_id: DocumentId, doc_id: DocumentId) -> Id { Id(((prefix_id as u64) << 32) | doc_id as u64) } #[inline(always)] pub fn id(&self) -> u64 { self.0 } #[inline(always)] pub fn document_id(&self) -> DocumentId { (self.0 & 0xFFFFFFFF) as DocumentId } #[inline(always)] pub fn prefix_id(&self) -> DocumentId { (self.0 >> 32) as DocumentId } #[inline(always)] pub fn is_singleton(&self) -> bool { self.0 == 20080258862541 } #[inline(always)] pub fn is_valid(&self) -> bool { self.0 != u64::MAX } } impl From for Id { fn from(id: u64) -> Self { Id(id) } } impl From for Id { fn from(id: u32) -> Self { Id(id as u64) } } impl From for u64 { fn from(id: Id) -> Self { id.0 } } impl From<&Id> for u64 { fn from(id: &Id) -> Self { id.0 } } impl From<(u32, u32)> for Id { fn from(id: (u32, u32)) -> Self { Id::from_parts(id.0, id.1) } } impl Deref for Id { type Target = u64; fn deref(&self) -> &Self::Target { &self.0 } } impl AsRef for Id { fn as_ref(&self) -> &u64 { &self.0 } } impl From for u32 { fn from(id: Id) -> Self { id.document_id() } } impl From for String { fn from(id: Id) -> Self { id.as_string() } } impl serde::Serialize for Id { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { serializer.serialize_str(self.as_string().as_str()) } } impl<'de> serde::Deserialize<'de> for Id { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { Id::from_str(<&str>::deserialize(deserializer)?) .map_err(|_| serde::de::Error::custom("invalid JMAP ID")) } } impl std::fmt::Display for Id { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(&self.as_string()) } } impl<'x, P: Property, E: Element + From> From for Value<'x, P, E> { fn from(id: Id) -> Self { Value::Element(E::from(id)) } } #[cfg(test)] mod tests { use std::str::FromStr; use crate::id::Id; #[test] fn parse_jmap_id() { for number in [ 0, 1, 10, 1000, Id::singleton().id(), u64::MAX / 2, u64::MAX - 1, u64::MAX, ] { let id = Id::from(number); assert_eq!(Id::from_str(&id.to_string()).unwrap(), id); } Id::from_str("p333333333333p333333333333").unwrap(); } } ================================================ FILE: crates/types/src/keyword.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use jmap_tools::{Element, Property, Value}; use std::{fmt::Display, str::FromStr}; pub const SEEN: usize = 0; pub const DRAFT: usize = 1; pub const FLAGGED: usize = 2; pub const ANSWERED: usize = 3; pub const RECENT: usize = 4; pub const IMPORTANT: usize = 5; pub const PHISHING: usize = 6; pub const JUNK: usize = 7; pub const NOTJUNK: usize = 8; pub const DELETED: usize = 9; pub const FORWARDED: usize = 10; pub const MDN_SENT: usize = 11; pub const AUTOSENT: usize = 12; pub const CANUNSUBSCRIBE: usize = 13; pub const FOLLOWED: usize = 14; pub const HASATTACHMENT: usize = 15; pub const HASMEMO: usize = 16; pub const HASNOATTACHMENT: usize = 17; pub const IMPORTED: usize = 18; pub const ISTRUSTED: usize = 19; pub const MAILFLAGBIT0: usize = 20; pub const MAILFLAGBIT1: usize = 21; pub const MAILFLAGBIT2: usize = 22; pub const MASKEDEMAIL: usize = 23; pub const MEMO: usize = 24; pub const MUTED: usize = 25; pub const NEW: usize = 26; pub const NOTIFY: usize = 27; pub const UNSUBSCRIBED: usize = 28; pub const OTHER: usize = 29; #[derive( rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Clone, PartialEq, Eq, Hash, Default, PartialOrd, Ord, serde::Serialize, )] #[serde(untagged)] #[rkyv(derive(PartialEq), compare(PartialEq))] pub enum Keyword { Other(Box), #[serde(rename(serialize = "$seen"))] Seen, #[serde(rename(serialize = "$draft"))] Draft, #[serde(rename(serialize = "$flagged"))] Flagged, #[serde(rename(serialize = "$answered"))] Answered, #[default] #[serde(rename(serialize = "$recent"))] Recent, #[serde(rename(serialize = "$important"))] Important, #[serde(rename(serialize = "$phishing"))] Phishing, #[serde(rename(serialize = "$junk"))] Junk, #[serde(rename(serialize = "$notjunk"))] NotJunk, #[serde(rename(serialize = "$deleted"))] Deleted, #[serde(rename(serialize = "$forwarded"))] Forwarded, #[serde(rename(serialize = "$mdnsent"))] MdnSent, #[serde(rename(serialize = "$autosent"))] Autosent, #[serde(rename(serialize = "$canunsubscribe"))] CanUnsubscribe, #[serde(rename(serialize = "$followed"))] Followed, #[serde(rename(serialize = "$hasattachment"))] HasAttachment, #[serde(rename(serialize = "$hasmemo"))] HasMemo, #[serde(rename(serialize = "$hasnoattachment"))] HasNoAttachment, #[serde(rename(serialize = "$imported"))] Imported, #[serde(rename(serialize = "$istrusted"))] IsTrusted, #[serde(rename(serialize = "$MailFlagBit0"))] MailFlagBit0, #[serde(rename(serialize = "$MailFlagBit1"))] MailFlagBit1, #[serde(rename(serialize = "$MailFlagBit2"))] MailFlagBit2, #[serde(rename(serialize = "$maskedemail"))] MaskedEmail, #[serde(rename(serialize = "$memo"))] Memo, #[serde(rename(serialize = "$muted"))] Muted, #[serde(rename(serialize = "$new"))] New, #[serde(rename(serialize = "$notify"))] Notify, #[serde(rename(serialize = "$unsubscribed"))] Unsubscribed, } impl Keyword { pub const MAX_LENGTH: usize = 128; pub fn parse(value: &str) -> Self { Self::try_parse(value) .unwrap_or_else(|| Keyword::Other(value.chars().take(Keyword::MAX_LENGTH).collect())) } pub fn from_other(value: String) -> Self { if value.len() <= Keyword::MAX_LENGTH { Keyword::Other(value.into_boxed_str()) } else { Keyword::Other(value.chars().take(Keyword::MAX_LENGTH).collect()) } } pub fn from_boxed_other(value: Box) -> Self { if value.len() <= Keyword::MAX_LENGTH { Keyword::Other(value) } else { Keyword::Other(value.chars().take(Keyword::MAX_LENGTH).collect()) } } pub fn try_parse(value: &str) -> Option { value .split_at_checked(1) .filter(|(prefix, _)| matches!(*prefix, "$" | "\\")) .and_then(|(_, rest)| { hashify::tiny_map_ignore_case!(rest.as_bytes(), "seen" => Keyword::Seen, "draft" => Keyword::Draft, "flagged" => Keyword::Flagged, "answered" => Keyword::Answered, "recent" => Keyword::Recent, "important" => Keyword::Important, "phishing" => Keyword::Phishing, "junk" => Keyword::Junk, "notjunk" => Keyword::NotJunk, "deleted" => Keyword::Deleted, "forwarded" => Keyword::Forwarded, "mdnsent" => Keyword::MdnSent, "autosent" => Keyword::Autosent, "canunsubscribe" => Keyword::CanUnsubscribe, "followed" => Keyword::Followed, "hasattachment" => Keyword::HasAttachment, "hasmemo" => Keyword::HasMemo, "hasnoattachment" => Keyword::HasNoAttachment, "imported" => Keyword::Imported, "istrusted" => Keyword::IsTrusted, "mailflagbit0" => Keyword::MailFlagBit0, "mailflagbit1" => Keyword::MailFlagBit1, "mailflagbit2" => Keyword::MailFlagBit2, "maskedemail" => Keyword::MaskedEmail, "memo" => Keyword::Memo, "muted" => Keyword::Muted, "new" => Keyword::New, "notify" => Keyword::Notify, "unsubscribed" => Keyword::Unsubscribed, ) }) } pub fn id(&self) -> Result { match self { Keyword::Seen => Ok(SEEN as u32), Keyword::Draft => Ok(DRAFT as u32), Keyword::Flagged => Ok(FLAGGED as u32), Keyword::Answered => Ok(ANSWERED as u32), Keyword::Recent => Ok(RECENT as u32), Keyword::Important => Ok(IMPORTANT as u32), Keyword::Phishing => Ok(PHISHING as u32), Keyword::Junk => Ok(JUNK as u32), Keyword::NotJunk => Ok(NOTJUNK as u32), Keyword::Deleted => Ok(DELETED as u32), Keyword::Forwarded => Ok(FORWARDED as u32), Keyword::MdnSent => Ok(MDN_SENT as u32), Keyword::Autosent => Ok(AUTOSENT as u32), Keyword::CanUnsubscribe => Ok(CANUNSUBSCRIBE as u32), Keyword::Followed => Ok(FOLLOWED as u32), Keyword::HasAttachment => Ok(HASATTACHMENT as u32), Keyword::HasMemo => Ok(HASMEMO as u32), Keyword::HasNoAttachment => Ok(HASNOATTACHMENT as u32), Keyword::Imported => Ok(IMPORTED as u32), Keyword::IsTrusted => Ok(ISTRUSTED as u32), Keyword::MailFlagBit0 => Ok(MAILFLAGBIT0 as u32), Keyword::MailFlagBit1 => Ok(MAILFLAGBIT1 as u32), Keyword::MailFlagBit2 => Ok(MAILFLAGBIT2 as u32), Keyword::MaskedEmail => Ok(MASKEDEMAIL as u32), Keyword::Memo => Ok(MEMO as u32), Keyword::Muted => Ok(MUTED as u32), Keyword::New => Ok(NEW as u32), Keyword::Notify => Ok(NOTIFY as u32), Keyword::Unsubscribed => Ok(UNSUBSCRIBED as u32), Keyword::Other(string) => Err(string.as_ref()), } } pub fn into_id(self) -> Result> { match self { Keyword::Seen => Ok(SEEN as u32), Keyword::Draft => Ok(DRAFT as u32), Keyword::Flagged => Ok(FLAGGED as u32), Keyword::Answered => Ok(ANSWERED as u32), Keyword::Recent => Ok(RECENT as u32), Keyword::Important => Ok(IMPORTANT as u32), Keyword::Phishing => Ok(PHISHING as u32), Keyword::Junk => Ok(JUNK as u32), Keyword::NotJunk => Ok(NOTJUNK as u32), Keyword::Deleted => Ok(DELETED as u32), Keyword::Forwarded => Ok(FORWARDED as u32), Keyword::MdnSent => Ok(MDN_SENT as u32), Keyword::Autosent => Ok(AUTOSENT as u32), Keyword::CanUnsubscribe => Ok(CANUNSUBSCRIBE as u32), Keyword::Followed => Ok(FOLLOWED as u32), Keyword::HasAttachment => Ok(HASATTACHMENT as u32), Keyword::HasMemo => Ok(HASMEMO as u32), Keyword::HasNoAttachment => Ok(HASNOATTACHMENT as u32), Keyword::Imported => Ok(IMPORTED as u32), Keyword::IsTrusted => Ok(ISTRUSTED as u32), Keyword::MailFlagBit0 => Ok(MAILFLAGBIT0 as u32), Keyword::MailFlagBit1 => Ok(MAILFLAGBIT1 as u32), Keyword::MailFlagBit2 => Ok(MAILFLAGBIT2 as u32), Keyword::MaskedEmail => Ok(MASKEDEMAIL as u32), Keyword::Memo => Ok(MEMO as u32), Keyword::Muted => Ok(MUTED as u32), Keyword::New => Ok(NEW as u32), Keyword::Notify => Ok(NOTIFY as u32), Keyword::Unsubscribed => Ok(UNSUBSCRIBED as u32), Keyword::Other(string) => Err(string), } } pub fn try_from_id(id: usize) -> Result { match id { SEEN => Ok(Keyword::Seen), DRAFT => Ok(Keyword::Draft), FLAGGED => Ok(Keyword::Flagged), ANSWERED => Ok(Keyword::Answered), RECENT => Ok(Keyword::Recent), IMPORTANT => Ok(Keyword::Important), PHISHING => Ok(Keyword::Phishing), JUNK => Ok(Keyword::Junk), NOTJUNK => Ok(Keyword::NotJunk), DELETED => Ok(Keyword::Deleted), FORWARDED => Ok(Keyword::Forwarded), MDN_SENT => Ok(Keyword::MdnSent), AUTOSENT => Ok(Keyword::Autosent), CANUNSUBSCRIBE => Ok(Keyword::CanUnsubscribe), FOLLOWED => Ok(Keyword::Followed), HASATTACHMENT => Ok(Keyword::HasAttachment), HASMEMO => Ok(Keyword::HasMemo), HASNOATTACHMENT => Ok(Keyword::HasNoAttachment), IMPORTED => Ok(Keyword::Imported), ISTRUSTED => Ok(Keyword::IsTrusted), MAILFLAGBIT0 => Ok(Keyword::MailFlagBit0), MAILFLAGBIT1 => Ok(Keyword::MailFlagBit1), MAILFLAGBIT2 => Ok(Keyword::MailFlagBit2), MASKEDEMAIL => Ok(Keyword::MaskedEmail), MEMO => Ok(Keyword::Memo), MUTED => Ok(Keyword::Muted), NEW => Ok(Keyword::New), NOTIFY => Ok(Keyword::Notify), UNSUBSCRIBED => Ok(Keyword::Unsubscribed), _ => Err(id), } } } impl From for Keyword { fn from(value: String) -> Self { Keyword::try_parse(&value).unwrap_or_else(|| Keyword::from_other(value)) } } impl Display for Keyword { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Keyword::Seen => write!(f, "$seen"), Keyword::Draft => write!(f, "$draft"), Keyword::Flagged => write!(f, "$flagged"), Keyword::Answered => write!(f, "$answered"), Keyword::Recent => write!(f, "$recent"), Keyword::Important => write!(f, "$important"), Keyword::Phishing => write!(f, "$phishing"), Keyword::Junk => write!(f, "$junk"), Keyword::NotJunk => write!(f, "$notjunk"), Keyword::Deleted => write!(f, "$deleted"), Keyword::Forwarded => write!(f, "$forwarded"), Keyword::MdnSent => write!(f, "$mdnsent"), Keyword::Autosent => write!(f, "$autosent"), Keyword::CanUnsubscribe => write!(f, "$canunsubscribe"), Keyword::Followed => write!(f, "$followed"), Keyword::HasAttachment => write!(f, "$hasattachment"), Keyword::HasMemo => write!(f, "$hasmemo"), Keyword::HasNoAttachment => write!(f, "$hasnoattachment"), Keyword::Imported => write!(f, "$imported"), Keyword::IsTrusted => write!(f, "$istrusted"), Keyword::MailFlagBit0 => write!(f, "$MailFlagBit0"), Keyword::MailFlagBit1 => write!(f, "$MailFlagBit1"), Keyword::MailFlagBit2 => write!(f, "$MailFlagBit2"), Keyword::MaskedEmail => write!(f, "$maskedemail"), Keyword::Memo => write!(f, "$memo"), Keyword::Muted => write!(f, "$muted"), Keyword::New => write!(f, "$new"), Keyword::Notify => write!(f, "$notify"), Keyword::Unsubscribed => write!(f, "$unsubscribed"), Keyword::Other(s) => write!(f, "{}", s), } } } impl Display for ArchivedKeyword { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { ArchivedKeyword::Seen => write!(f, "$seen"), ArchivedKeyword::Draft => write!(f, "$draft"), ArchivedKeyword::Flagged => write!(f, "$flagged"), ArchivedKeyword::Answered => write!(f, "$answered"), ArchivedKeyword::Recent => write!(f, "$recent"), ArchivedKeyword::Important => write!(f, "$important"), ArchivedKeyword::Phishing => write!(f, "$phishing"), ArchivedKeyword::Junk => write!(f, "$junk"), ArchivedKeyword::NotJunk => write!(f, "$notjunk"), ArchivedKeyword::Deleted => write!(f, "$deleted"), ArchivedKeyword::Forwarded => write!(f, "$forwarded"), ArchivedKeyword::MdnSent => write!(f, "$mdnsent"), ArchivedKeyword::Autosent => write!(f, "$autosent"), ArchivedKeyword::CanUnsubscribe => write!(f, "$canunsubscribe"), ArchivedKeyword::Followed => write!(f, "$followed"), ArchivedKeyword::HasAttachment => write!(f, "$hasattachment"), ArchivedKeyword::HasMemo => write!(f, "$hasmemo"), ArchivedKeyword::HasNoAttachment => write!(f, "$hasnoattachment"), ArchivedKeyword::Imported => write!(f, "$imported"), ArchivedKeyword::IsTrusted => write!(f, "$istrusted"), ArchivedKeyword::MailFlagBit0 => write!(f, "$MailFlagBit0"), ArchivedKeyword::MailFlagBit1 => write!(f, "$MailFlagBit1"), ArchivedKeyword::MailFlagBit2 => write!(f, "$MailFlagBit2"), ArchivedKeyword::MaskedEmail => write!(f, "$maskedemail"), ArchivedKeyword::Memo => write!(f, "$memo"), ArchivedKeyword::Muted => write!(f, "$muted"), ArchivedKeyword::New => write!(f, "$new"), ArchivedKeyword::Notify => write!(f, "$notify"), ArchivedKeyword::Unsubscribed => write!(f, "$unsubscribed"), ArchivedKeyword::Other(s) => write!(f, "{}", s), } } } impl From for Vec { fn from(keyword: Keyword) -> Self { match keyword { Keyword::Seen => vec![SEEN as u8], Keyword::Draft => vec![DRAFT as u8], Keyword::Flagged => vec![FLAGGED as u8], Keyword::Answered => vec![ANSWERED as u8], Keyword::Recent => vec![RECENT as u8], Keyword::Important => vec![IMPORTANT as u8], Keyword::Phishing => vec![PHISHING as u8], Keyword::Junk => vec![JUNK as u8], Keyword::NotJunk => vec![NOTJUNK as u8], Keyword::Deleted => vec![DELETED as u8], Keyword::Forwarded => vec![FORWARDED as u8], Keyword::MdnSent => vec![MDN_SENT as u8], Keyword::Autosent => vec![AUTOSENT as u8], Keyword::CanUnsubscribe => vec![CANUNSUBSCRIBE as u8], Keyword::Followed => vec![FOLLOWED as u8], Keyword::HasAttachment => vec![HASATTACHMENT as u8], Keyword::HasMemo => vec![HASMEMO as u8], Keyword::HasNoAttachment => vec![HASNOATTACHMENT as u8], Keyword::Imported => vec![IMPORTED as u8], Keyword::IsTrusted => vec![ISTRUSTED as u8], Keyword::MailFlagBit0 => vec![MAILFLAGBIT0 as u8], Keyword::MailFlagBit1 => vec![MAILFLAGBIT1 as u8], Keyword::MailFlagBit2 => vec![MAILFLAGBIT2 as u8], Keyword::MaskedEmail => vec![MASKEDEMAIL as u8], Keyword::Memo => vec![MEMO as u8], Keyword::Muted => vec![MUTED as u8], Keyword::New => vec![NEW as u8], Keyword::Notify => vec![NOTIFY as u8], Keyword::Unsubscribed => vec![UNSUBSCRIBED as u8], Keyword::Other(string) => string.as_bytes().to_vec(), } } } impl FromStr for Keyword { type Err = (); fn from_str(s: &str) -> Result { Ok(Keyword::parse(s)) } } impl ArchivedKeyword { pub fn id(&self) -> Result { match self { ArchivedKeyword::Seen => Ok(SEEN as u32), ArchivedKeyword::Draft => Ok(DRAFT as u32), ArchivedKeyword::Flagged => Ok(FLAGGED as u32), ArchivedKeyword::Answered => Ok(ANSWERED as u32), ArchivedKeyword::Recent => Ok(RECENT as u32), ArchivedKeyword::Important => Ok(IMPORTANT as u32), ArchivedKeyword::Phishing => Ok(PHISHING as u32), ArchivedKeyword::Junk => Ok(JUNK as u32), ArchivedKeyword::NotJunk => Ok(NOTJUNK as u32), ArchivedKeyword::Deleted => Ok(DELETED as u32), ArchivedKeyword::Forwarded => Ok(FORWARDED as u32), ArchivedKeyword::MdnSent => Ok(MDN_SENT as u32), ArchivedKeyword::Autosent => Ok(AUTOSENT as u32), ArchivedKeyword::CanUnsubscribe => Ok(CANUNSUBSCRIBE as u32), ArchivedKeyword::Followed => Ok(FOLLOWED as u32), ArchivedKeyword::HasAttachment => Ok(HASATTACHMENT as u32), ArchivedKeyword::HasMemo => Ok(HASMEMO as u32), ArchivedKeyword::HasNoAttachment => Ok(HASNOATTACHMENT as u32), ArchivedKeyword::Imported => Ok(IMPORTED as u32), ArchivedKeyword::IsTrusted => Ok(ISTRUSTED as u32), ArchivedKeyword::MailFlagBit0 => Ok(MAILFLAGBIT0 as u32), ArchivedKeyword::MailFlagBit1 => Ok(MAILFLAGBIT1 as u32), ArchivedKeyword::MailFlagBit2 => Ok(MAILFLAGBIT2 as u32), ArchivedKeyword::MaskedEmail => Ok(MASKEDEMAIL as u32), ArchivedKeyword::Memo => Ok(MEMO as u32), ArchivedKeyword::Muted => Ok(MUTED as u32), ArchivedKeyword::New => Ok(NEW as u32), ArchivedKeyword::Notify => Ok(NOTIFY as u32), ArchivedKeyword::Unsubscribed => Ok(UNSUBSCRIBED as u32), ArchivedKeyword::Other(string) => Err(string.as_ref()), } } pub fn to_native(&self) -> Keyword { match self { ArchivedKeyword::Seen => Keyword::Seen, ArchivedKeyword::Draft => Keyword::Draft, ArchivedKeyword::Flagged => Keyword::Flagged, ArchivedKeyword::Answered => Keyword::Answered, ArchivedKeyword::Recent => Keyword::Recent, ArchivedKeyword::Important => Keyword::Important, ArchivedKeyword::Phishing => Keyword::Phishing, ArchivedKeyword::Junk => Keyword::Junk, ArchivedKeyword::NotJunk => Keyword::NotJunk, ArchivedKeyword::Deleted => Keyword::Deleted, ArchivedKeyword::Forwarded => Keyword::Forwarded, ArchivedKeyword::MdnSent => Keyword::MdnSent, ArchivedKeyword::Autosent => Keyword::Autosent, ArchivedKeyword::CanUnsubscribe => Keyword::CanUnsubscribe, ArchivedKeyword::Followed => Keyword::Followed, ArchivedKeyword::HasAttachment => Keyword::HasAttachment, ArchivedKeyword::HasMemo => Keyword::HasMemo, ArchivedKeyword::HasNoAttachment => Keyword::HasNoAttachment, ArchivedKeyword::Imported => Keyword::Imported, ArchivedKeyword::IsTrusted => Keyword::IsTrusted, ArchivedKeyword::MailFlagBit0 => Keyword::MailFlagBit0, ArchivedKeyword::MailFlagBit1 => Keyword::MailFlagBit1, ArchivedKeyword::MailFlagBit2 => Keyword::MailFlagBit2, ArchivedKeyword::MaskedEmail => Keyword::MaskedEmail, ArchivedKeyword::Memo => Keyword::Memo, ArchivedKeyword::Muted => Keyword::Muted, ArchivedKeyword::New => Keyword::New, ArchivedKeyword::Notify => Keyword::Notify, ArchivedKeyword::Unsubscribed => Keyword::Unsubscribed, ArchivedKeyword::Other(other) => Keyword::Other(other.as_ref().into()), } } } impl From<&ArchivedKeyword> for Keyword { fn from(value: &ArchivedKeyword) -> Self { match value { ArchivedKeyword::Seen => Keyword::Seen, ArchivedKeyword::Draft => Keyword::Draft, ArchivedKeyword::Flagged => Keyword::Flagged, ArchivedKeyword::Answered => Keyword::Answered, ArchivedKeyword::Recent => Keyword::Recent, ArchivedKeyword::Important => Keyword::Important, ArchivedKeyword::Phishing => Keyword::Phishing, ArchivedKeyword::Junk => Keyword::Junk, ArchivedKeyword::NotJunk => Keyword::NotJunk, ArchivedKeyword::Deleted => Keyword::Deleted, ArchivedKeyword::Forwarded => Keyword::Forwarded, ArchivedKeyword::MdnSent => Keyword::MdnSent, ArchivedKeyword::Autosent => Keyword::Autosent, ArchivedKeyword::CanUnsubscribe => Keyword::CanUnsubscribe, ArchivedKeyword::Followed => Keyword::Followed, ArchivedKeyword::HasAttachment => Keyword::HasAttachment, ArchivedKeyword::HasMemo => Keyword::HasMemo, ArchivedKeyword::HasNoAttachment => Keyword::HasNoAttachment, ArchivedKeyword::Imported => Keyword::Imported, ArchivedKeyword::IsTrusted => Keyword::IsTrusted, ArchivedKeyword::MailFlagBit0 => Keyword::MailFlagBit0, ArchivedKeyword::MailFlagBit1 => Keyword::MailFlagBit1, ArchivedKeyword::MailFlagBit2 => Keyword::MailFlagBit2, ArchivedKeyword::MaskedEmail => Keyword::MaskedEmail, ArchivedKeyword::Memo => Keyword::Memo, ArchivedKeyword::Muted => Keyword::Muted, ArchivedKeyword::New => Keyword::New, ArchivedKeyword::Notify => Keyword::Notify, ArchivedKeyword::Unsubscribed => Keyword::Unsubscribed, ArchivedKeyword::Other(string) => Keyword::Other(string.as_ref().into()), } } } impl<'de> serde::Deserialize<'de> for Keyword { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { Ok(Keyword::parse(<&str>::deserialize(deserializer)?)) } } impl<'x, P: Property, E: Element + From> From for Value<'x, P, E> { fn from(id: Keyword) -> Self { Value::Element(E::from(id)) } } ================================================ FILE: crates/types/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod acl; pub mod blob; pub mod blob_hash; pub mod collection; pub mod dead_property; pub mod field; pub mod id; pub mod keyword; pub mod semver; pub mod special_use; pub mod type_state; pub type DocumentId = u32; pub type ChangeId = u64; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[cfg_attr(feature = "test_mode", derive(serde::Serialize, serde::Deserialize))] pub struct TimeRange { pub start: i64, pub end: i64, } impl TimeRange { pub fn new(start: i64, end: i64) -> Self { Self { start, end } } pub fn is_in_range(&self, match_overlap: bool, start: i64, end: i64) -> bool { if !match_overlap { // RFC4791#9.9: (start < DTEND AND end > DTSTART) self.start < end && self.end > start } else { // RFC4791#9.9: ((start < DUE) OR (start <= DTSTART)) AND ((end > DTSTART) OR (end >= DUE)) ((start < self.end) || (start <= self.start)) && (end > self.start || end >= self.end) } } } impl Default for TimeRange { fn default() -> Self { Self { start: i64::MIN, end: i64::MAX, } } } ================================================ FILE: crates/types/src/semver.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::fmt::Display; #[derive(Clone, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)] #[repr(transparent)] pub struct Semver(u64); impl Semver { pub fn current() -> Self { env!("CARGO_PKG_VERSION").try_into().unwrap() } pub fn new(major: u16, minor: u16, patch: u16) -> Self { let mut version: u64 = 0; version |= (major as u64) << 32; version |= (minor as u64) << 16; version |= patch as u64; Semver(version) } pub fn unpack(&self) -> (u16, u16, u16) { let version = self.0; let major = ((version >> 32) & 0xFFFF) as u16; let minor = ((version >> 16) & 0xFFFF) as u16; let patch = (version & 0xFFFF) as u16; (major, minor, patch) } pub fn major(&self) -> u16 { (self.0 >> 32) as u16 } pub fn minor(&self) -> u16 { (self.0 >> 16) as u16 } pub fn patch(&self) -> u16 { self.0 as u16 } pub fn is_valid(&self) -> bool { self.0 > 0 } } impl AsRef for Semver { fn as_ref(&self) -> &u64 { &self.0 } } impl From for Semver { fn from(value: u64) -> Self { Semver(value) } } impl TryFrom<&str> for Semver { type Error = (); fn try_from(value: &str) -> Result { let mut parts = value.splitn(3, '.'); let major = parts.next().ok_or(())?.parse().map_err(|_| ())?; let minor = parts.next().ok_or(())?.parse().map_err(|_| ())?; let patch = parts.next().ok_or(())?.parse().map_err(|_| ())?; Ok(Semver::new(major, minor, patch)) } } impl Display for Semver { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let (major, minor, patch) = self.unpack(); write!(f, "{major}.{minor}.{patch}") } } ================================================ FILE: crates/types/src/special_use.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use jmap_tools::{Element, Property, Value}; use utils::config::utils::ParseValue; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Clone, Copy, PartialEq, Eq, Hash, Debug, PartialOrd, Ord, )] #[rkyv(derive(Debug))] pub enum SpecialUse { Inbox, Trash, Junk, Drafts, Archive, Sent, Shared, Important, None, Memos, Scheduled, Snoozed, } impl SpecialUse { pub fn parse(s: &str) -> Option { hashify::tiny_map_ignore_case!(s.as_bytes(), b"inbox" => SpecialUse::Inbox, b"trash" => SpecialUse::Trash, b"junk" => SpecialUse::Junk, b"drafts" => SpecialUse::Drafts, b"archive" => SpecialUse::Archive, b"sent" => SpecialUse::Sent, b"shared" => SpecialUse::Shared, b"important" => SpecialUse::Important, b"memos" => SpecialUse::Memos, b"scheduled" => SpecialUse::Scheduled, b"snoozed" => SpecialUse::Snoozed, ) } pub fn as_str(&self) -> Option<&'static str> { match self { SpecialUse::Inbox => Some("inbox"), SpecialUse::Trash => Some("trash"), SpecialUse::Junk => Some("junk"), SpecialUse::Drafts => Some("drafts"), SpecialUse::Archive => Some("archive"), SpecialUse::Sent => Some("sent"), SpecialUse::Shared => Some("shared"), SpecialUse::Important => Some("important"), SpecialUse::Memos => Some("memos"), SpecialUse::Scheduled => Some("scheduled"), SpecialUse::Snoozed => Some("snoozed"), SpecialUse::None => None, } } } impl ArchivedSpecialUse { pub fn as_str(&self) -> Option<&'static str> { match self { ArchivedSpecialUse::Inbox => Some("inbox"), ArchivedSpecialUse::Trash => Some("trash"), ArchivedSpecialUse::Junk => Some("junk"), ArchivedSpecialUse::Drafts => Some("drafts"), ArchivedSpecialUse::Archive => Some("archive"), ArchivedSpecialUse::Sent => Some("sent"), ArchivedSpecialUse::Shared => Some("shared"), ArchivedSpecialUse::Important => Some("important"), ArchivedSpecialUse::Memos => Some("memos"), ArchivedSpecialUse::Scheduled => Some("scheduled"), ArchivedSpecialUse::Snoozed => Some("snoozed"), ArchivedSpecialUse::None => None, } } } impl From<&ArchivedSpecialUse> for SpecialUse { fn from(value: &ArchivedSpecialUse) -> Self { match value { ArchivedSpecialUse::Inbox => SpecialUse::Inbox, ArchivedSpecialUse::Trash => SpecialUse::Trash, ArchivedSpecialUse::Junk => SpecialUse::Junk, ArchivedSpecialUse::Drafts => SpecialUse::Drafts, ArchivedSpecialUse::Archive => SpecialUse::Archive, ArchivedSpecialUse::Sent => SpecialUse::Sent, ArchivedSpecialUse::Shared => SpecialUse::Shared, ArchivedSpecialUse::Important => SpecialUse::Important, ArchivedSpecialUse::Memos => SpecialUse::Memos, ArchivedSpecialUse::Scheduled => SpecialUse::Scheduled, ArchivedSpecialUse::Snoozed => SpecialUse::Snoozed, ArchivedSpecialUse::None => SpecialUse::None, } } } impl ParseValue for SpecialUse { fn parse_value(value: &str) -> Result { SpecialUse::parse(value).ok_or_else(|| format!("Unknown folder role {:?}", value)) } } impl<'x, P: Property, E: Element + From> From for Value<'x, P, E> { fn from(id: SpecialUse) -> Self { Value::Element(E::from(id)) } } ================================================ FILE: crates/types/src/type_state.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::collection::SyncCollection; use jmap_tools::{Element, Property, Value}; use serde::Serialize; use std::{fmt::Display, str::FromStr}; use utils::map::bitmap::{Bitmap, BitmapItem}; #[derive(Debug, Eq, PartialEq, Hash, Clone, Copy, Serialize, PartialOrd, Ord)] #[repr(u8)] pub enum DataType { #[serde(rename = "Email")] Email = 0, #[serde(rename = "EmailDelivery")] EmailDelivery = 1, #[serde(rename = "EmailSubmission")] EmailSubmission = 2, #[serde(rename = "Mailbox")] Mailbox = 3, #[serde(rename = "Thread")] Thread = 4, #[serde(rename = "Identity")] Identity = 5, #[serde(rename = "Core")] Core = 6, #[serde(rename = "PushSubscription")] PushSubscription = 7, #[serde(rename = "SearchSnippet")] SearchSnippet = 8, #[serde(rename = "VacationResponse")] VacationResponse = 9, #[serde(rename = "MDN")] Mdn = 10, #[serde(rename = "Quota")] Quota = 11, #[serde(rename = "SieveScript")] SieveScript = 12, #[serde(rename = "Calendar")] Calendar = 13, #[serde(rename = "CalendarEvent")] CalendarEvent = 14, #[serde(rename = "CalendarEventNotification")] CalendarEventNotification = 15, #[serde(rename = "AddressBook")] AddressBook = 16, #[serde(rename = "ContactCard")] ContactCard = 17, #[serde(rename = "FileNode")] FileNode = 18, #[serde(rename = "Principal")] Principal = 19, #[serde(rename = "ShareNotification")] ShareNotification = 20, #[serde(rename = "ParticipantIdentity")] ParticipantIdentity = 21, #[serde(rename = "CalendarAlert")] CalendarAlert = 22, None = 23, } #[derive(Debug, Clone, Copy)] pub struct StateChange { pub account_id: u32, pub change_id: u64, pub types: Bitmap, } impl StateChange { pub fn new(account_id: u32) -> Self { Self { account_id, change_id: 0, types: Default::default(), } } pub fn set_change(&mut self, type_state: DataType) { self.types.insert(type_state); } pub fn with_change(mut self, type_state: DataType) -> Self { self.set_change(type_state); self } pub fn with_change_id(mut self, change_id: u64) -> Self { self.change_id = change_id; self } pub fn has_changes(&self) -> bool { !self.types.is_empty() } } impl BitmapItem for DataType { fn max() -> u64 { DataType::None as u64 } fn is_valid(&self) -> bool { !matches!(self, DataType::None) } } impl From for DataType { fn from(value: u64) -> Self { match value { 0 => DataType::Email, 1 => DataType::EmailDelivery, 2 => DataType::EmailSubmission, 3 => DataType::Mailbox, 4 => DataType::Thread, 5 => DataType::Identity, 6 => DataType::Core, 7 => DataType::PushSubscription, 8 => DataType::SearchSnippet, 9 => DataType::VacationResponse, 10 => DataType::Mdn, 11 => DataType::Quota, 12 => DataType::SieveScript, 13 => DataType::Calendar, 14 => DataType::CalendarEvent, 15 => DataType::CalendarEventNotification, 16 => DataType::AddressBook, 17 => DataType::ContactCard, 18 => DataType::FileNode, 19 => DataType::Principal, 20 => DataType::ShareNotification, 21 => DataType::ParticipantIdentity, 22 => DataType::CalendarAlert, _ => { debug_assert!(false, "Invalid type_state value: {}", value); DataType::None } } } } impl From for u64 { fn from(type_state: DataType) -> u64 { type_state as u64 } } impl DataType { pub fn try_from_sync(value: SyncCollection, is_container: bool) -> Option { match (value, is_container) { (SyncCollection::Email, false) => DataType::Email.into(), (SyncCollection::Email, true) => DataType::Mailbox.into(), (SyncCollection::Thread, _) => DataType::Thread.into(), (SyncCollection::Calendar, true) => DataType::Calendar.into(), (SyncCollection::Calendar, false) => DataType::CalendarEvent.into(), (SyncCollection::AddressBook, true) => DataType::AddressBook.into(), (SyncCollection::AddressBook, false) => DataType::ContactCard.into(), (SyncCollection::FileNode, _) => DataType::FileNode.into(), (SyncCollection::Identity, _) => DataType::Identity.into(), (SyncCollection::EmailSubmission, _) => DataType::EmailSubmission.into(), (SyncCollection::SieveScript, _) => DataType::SieveScript.into(), _ => None, } } } impl DataType { pub fn parse(value: &str) -> Option { hashify::tiny_map!(value.as_bytes(), b"Email" => DataType::Email, b"EmailDelivery" => DataType::EmailDelivery, b"EmailSubmission" => DataType::EmailSubmission, b"Mailbox" => DataType::Mailbox, b"Thread" => DataType::Thread, b"Identity" => DataType::Identity, b"Core" => DataType::Core, b"PushSubscription" => DataType::PushSubscription, b"SearchSnippet" => DataType::SearchSnippet, b"VacationResponse" => DataType::VacationResponse, b"MDN" => DataType::Mdn, b"Quota" => DataType::Quota, b"SieveScript" => DataType::SieveScript, b"Calendar" => DataType::Calendar, b"CalendarEvent" => DataType::CalendarEvent, b"CalendarEventNotification" => DataType::CalendarEventNotification, b"AddressBook" => DataType::AddressBook, b"ContactCard" => DataType::ContactCard, b"FileNode" => DataType::FileNode, b"Principal" => DataType::Principal, b"ShareNotification" => DataType::ShareNotification, b"ParticipantIdentity" => DataType::ParticipantIdentity, b"CalendarAlert" => DataType::CalendarAlert, ) } pub fn as_str(&self) -> &'static str { match self { DataType::Email => "Email", DataType::EmailDelivery => "EmailDelivery", DataType::EmailSubmission => "EmailSubmission", DataType::Mailbox => "Mailbox", DataType::Thread => "Thread", DataType::Identity => "Identity", DataType::Core => "Core", DataType::PushSubscription => "PushSubscription", DataType::SearchSnippet => "SearchSnippet", DataType::VacationResponse => "VacationResponse", DataType::Mdn => "MDN", DataType::Quota => "Quota", DataType::SieveScript => "SieveScript", DataType::Calendar => "Calendar", DataType::CalendarEvent => "CalendarEvent", DataType::CalendarEventNotification => "CalendarEventNotification", DataType::AddressBook => "AddressBook", DataType::ContactCard => "ContactCard", DataType::FileNode => "FileNode", DataType::Principal => "Principal", DataType::ShareNotification => "ShareNotification", DataType::ParticipantIdentity => "ParticipantIdentity", DataType::CalendarAlert => "CalendarAlert", DataType::None => "", } } } impl FromStr for DataType { type Err = (); fn from_str(s: &str) -> Result { DataType::parse(s).ok_or(()) } } impl Display for DataType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.as_str()) } } impl<'de> serde::Deserialize<'de> for DataType { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { DataType::parse(<&str>::deserialize(deserializer)?) .ok_or_else(|| serde::de::Error::custom("invalid JMAP data type")) } } impl<'x, P: Property, E: Element + From> From for Value<'x, P, E> { fn from(id: DataType) -> Self { Value::Element(E::from(id)) } } ================================================ FILE: crates/utils/Cargo.toml ================================================ [package] name = "utils" version = "0.15.5" edition = "2024" [dependencies] trc = { path = "../trc" } rustls = { version = "0.23.5", default-features = false, features = ["std", "ring", "tls12"] } rustls-pemfile = "2.0" rustls-pki-types = { version = "1" } tokio = { version = "1.47", features = ["net", "macros", "signal"] } tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } serde = { version = "1.0", features = ["derive"]} mail-auth = { version = "0.7.1" } smtp-proto = { version = "0.2" } mail-send = { version = "0.5", default-features = false, features = ["cram-md5", "ring", "tls12"] } ahash = { version = "0.8", features = ["serde"] } chrono = "0.4" rand = "0.9.0" webpki-roots = { version = "1.0"} ring = { version = "0.17" } base64 = "0.22" serde_json = "1.0" rcgen = "0.14" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-webpki-roots", "http2", "stream"]} x509-parser = "0.18" pem = "3.0" parking_lot = "0.12" futures = "0.3" regex = "1.7.0" blake3 = "1.3.3" http-body-util = "0.1.0" form_urlencoded = "1.1.0" psl = "2" quick_cache = "0.6.9" fast-float = "0.2.0" rkyv = { version = "0.8.10", features = ["little_endian"] } compact_str = "0.9.0" xxhash-rust = { version = "0.8.5", features = ["xxh3"] } farmhash = "1.1.5" nohash-hasher = "0.2.0" [target.'cfg(unix)'.dependencies] privdrop = "0.5.3" [features] test_mode = [] [dev-dependencies] tokio = { version = "1.47", features = ["full"] } ================================================ FILE: crates/utils/proc-macros/Cargo.toml ================================================ [package] name = "proc_macros" version = "0.15.5" edition = "2024" [lib] proc-macro = true [dependencies] syn = { version = "2.0", features = ["full"] } quote = "1.0" proc-macro2 = "1.0" ================================================ FILE: crates/utils/proc-macros/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use proc_macro::TokenStream; use quote::quote; use syn::{Data, DeriveInput, parse_macro_input}; #[proc_macro_derive(EnumMethods)] pub fn enum_id(input: TokenStream) -> TokenStream { let input = parse_macro_input!(input as DeriveInput); let name = &input.ident; let variants = match input.data { Data::Enum(ref data) => &data.variants, _ => panic!("EnumMethods only works on enums"), }; let variant_count = variants.len(); let variant_names: Vec<_> = variants.iter().map(|v| &v.ident).collect(); let variant_ids: Vec = (0..(variant_count as u32)).collect(); let snake_case_names: Vec = variant_names .iter() .map(|name| to_snake_case(&name.to_string())) .collect(); let expanded = quote! { impl #name { pub const COUNT: usize = #variant_count; pub const fn id(&self) -> u32 { match self { #(#name::#variant_names => #variant_ids,)* } } pub fn from_id(id: u32) -> Option { match id { #(#variant_ids => Some(#name::#variant_names),)* _ => None, } } pub fn name(&self) -> &'static str { match self { #(#name::#variant_names => #snake_case_names,)* } } pub fn from_name(name: &str) -> Option { match name { #(#snake_case_names => Some(#name::#variant_names),)* _ => None, } } } }; TokenStream::from(expanded) } fn to_snake_case(s: &str) -> String { let mut result = String::new(); for (i, ch) in s.char_indices() { if ch.is_uppercase() { if i > 0 { result.push('-'); } result.push(ch.to_ascii_lowercase()); } else { result.push(ch); } } result } ================================================ FILE: crates/utils/src/bimap.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{borrow::Borrow, hash::Hash, rc::Rc}; use ahash::AHashMap; #[derive(Debug)] #[repr(transparent)] struct StringRef(Rc); #[derive(Debug)] #[repr(transparent)] struct IdRef(Rc); #[derive(Debug, Default)] pub struct IdBimap { id_to_name: AHashMap, Rc>, name_to_id: AHashMap, Rc>, } impl IdBimap { pub fn with_capacity(capacity: usize) -> Self { Self { id_to_name: AHashMap::with_capacity(capacity), name_to_id: AHashMap::with_capacity(capacity), } } pub fn insert(&mut self, item: T) { let item = Rc::new(item); self.id_to_name.insert(IdRef(item.clone()), item.clone()); self.name_to_id.insert(StringRef(item.clone()), item); } pub fn by_name(&self, name: &str) -> Option<&T> { self.name_to_id.get(name).map(|v| v.as_ref()) } pub fn by_id(&self, id: u32) -> Option<&T> { self.id_to_name.get(&id).map(|v| v.as_ref()) } pub fn iter(&self) -> impl Iterator { self.name_to_id.values().map(|v| v.as_ref()) } pub fn is_empty(&self) -> bool { self.name_to_id.is_empty() } } // SAFETY: Safe because Rc<> are never returned from the struct unsafe impl Send for IdBimap {} unsafe impl Sync for IdBimap {} pub trait IdBimapItem: std::fmt::Debug { fn id(&self) -> &u32; fn name(&self) -> &str; } impl Borrow for StringRef { fn borrow(&self) -> &str { self.0.name() } } impl Borrow for IdRef { fn borrow(&self) -> &u32 { self.0.id() } } impl PartialEq for StringRef { fn eq(&self, other: &Self) -> bool { self.0.name() == other.0.name() } } impl Eq for StringRef {} impl PartialEq for IdRef { fn eq(&self, other: &Self) -> bool { self.0.id() == other.0.id() } } impl Eq for IdRef {} impl Hash for StringRef { fn hash(&self, state: &mut H) { self.0.name().hash(state) } } impl Hash for IdRef { fn hash(&self, state: &mut H) { self.0.id().hash(state) } } ================================================ FILE: crates/utils/src/cache.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::config::Config; use mail_auth::{MX, ResolverCache, Txt}; use quick_cache::{ Equivalent, Weighter, sync::{DefaultLifecycle, PlaceholderGuard}, }; use std::{ borrow::Borrow, hash::Hash, net::{IpAddr, Ipv4Addr, Ipv6Addr}, sync::Arc, time::{Duration, Instant}, }; pub struct Cache( quick_cache::sync::Cache, ); pub struct CacheWithTtl( quick_cache::sync::Cache, CacheItemWeighter>, ); #[derive(Clone)] pub struct TtlEntry { value: V, expires: Instant, } impl Cache { pub fn from_config( config: &mut Config, key: &str, max_weight: u64, estimated_weight: u64, ) -> Self { let weight_capacity = config .property(("cache", key, "size")) .unwrap_or(max_weight); let estimated_items_capacity = config .property(("cache", key, "capacity")) .unwrap_or_else(|| weight_capacity as usize / estimated_weight as usize); Self::new(estimated_items_capacity, weight_capacity) } pub fn new(estimated_items_capacity: usize, weight_capacity: u64) -> Self { Self(quick_cache::sync::Cache::with_weighter( estimated_items_capacity, weight_capacity, CacheItemWeighter, )) } #[inline(always)] pub fn get(&self, key: &Q) -> Option where Q: Hash + Equivalent + ?Sized, { self.0.get(key) } #[inline(always)] pub async fn get_value_or_guard_async<'a, Q>( &'a self, key: &Q, ) -> Result< V, PlaceholderGuard<'a, K, V, CacheItemWeighter, ahash::RandomState, DefaultLifecycle>, > where Q: Hash + Equivalent + ToOwned + ?Sized, { self.0.get_value_or_guard_async(key).await } #[inline(always)] pub fn insert(&self, key: K, value: V) { self.0.insert(key, value); } #[inline(always)] pub fn remove(&self, key: &Q) -> Option where Q: Hash + Equivalent + ?Sized, { self.0.remove(key).map(|(_, v)| v) } #[inline(always)] pub fn clear(&self) { self.0.clear(); } } impl CacheWithTtl { pub fn from_config( config: &mut Config, key: &str, max_weight: u64, estimated_weight: u64, ) -> Self { let weight_capacity = config .property(("cache", key, "size")) .unwrap_or(max_weight); let estimated_items_capacity = config .property(("cache", key, "capacity")) .unwrap_or_else(|| weight_capacity as usize / estimated_weight as usize); Self::new(estimated_items_capacity, weight_capacity) } pub fn new(estimated_items_capacity: usize, weight_capacity: u64) -> Self { Self(quick_cache::sync::Cache::with_weighter( estimated_items_capacity, weight_capacity, CacheItemWeighter, )) } #[inline(always)] pub fn get(&self, key: &Q) -> Option where Q: Hash + Equivalent + ?Sized, { self.0.get(key).and_then(|v| { if v.expires > Instant::now() { Some(v.value) } else { self.0.remove(key); None } }) } #[inline(always)] pub async fn get_value_or_guard_async<'a, Q>( &'a self, key: &Q, ) -> Result< V, PlaceholderGuard< 'a, K, TtlEntry, CacheItemWeighter, ahash::RandomState, DefaultLifecycle>, >, > where Q: Hash + Equivalent + ToOwned + ?Sized, { match self.0.get_value_or_guard_async(key).await { Ok(value) => { if value.expires > Instant::now() { Ok(value.value) } else { self.0.remove(key); self.0.get_value_or_guard_async(key).await.map(|v| v.value) } } Err(err) => Err(err), } } #[inline(always)] pub fn insert(&self, key: K, value: V, expires: Duration) { self.0.insert(key, TtlEntry::new(value, expires)); } #[inline(always)] pub fn insert_with_expiry(&self, key: K, value: V, expires: Instant) { self.0.insert(key, TtlEntry::with_expiry(value, expires)); } #[inline(always)] pub fn remove(&self, key: &Q) -> Option where Q: Hash + Equivalent + ?Sized, { self.0.remove(key).map(|(_, v)| v.value) } #[inline(always)] pub fn clear(&self) { self.0.clear(); } } #[derive(Clone)] pub struct CacheItemWeighter; impl Weighter for CacheItemWeighter { fn weight(&self, key: &K, val: &V) -> u64 { key.weight() + val.weight() } } pub trait CacheItemWeight { fn weight(&self) -> u64; } impl CacheItemWeight for TtlEntry { fn weight(&self) -> u64 { self.value.weight() + std::mem::size_of::() as u64 } } impl CacheItemWeight for Option { fn weight(&self) -> u64 { match self { Some(v) => v.weight(), None => std::mem::size_of::() as u64, } } } impl CacheItemWeight for Arc { fn weight(&self) -> u64 { self.as_ref().weight() } } impl CacheItemWeight for u64 { fn weight(&self) -> u64 { std::mem::size_of::() as u64 } } impl CacheItemWeight for String { fn weight(&self) -> u64 { self.len() as u64 + std::mem::size_of::() as u64 } } impl CacheItemWeight for u32 { fn weight(&self) -> u64 { std::mem::size_of::() as u64 } } impl CacheItemWeight for Vec { fn weight(&self) -> u64 { (self.len() * std::mem::size_of::()) as u64 + std::mem::size_of::>() as u64 } } impl CacheItemWeight for Vec { fn weight(&self) -> u64 { (self.len() * std::mem::size_of::()) as u64 + std::mem::size_of::>() as u64 } } impl CacheItemWeight for Vec { fn weight(&self) -> u64 { (self.len() * std::mem::size_of::()) as u64 + std::mem::size_of::>() as u64 } } impl CacheItemWeight for Vec { fn weight(&self) -> u64 { self.iter() .map(|mx| { mx.exchanges .iter() .map(|e| e.len() + std::mem::size_of::()) .sum::() }) .sum::() as u64 + std::mem::size_of::>() as u64 } } impl CacheItemWeight for Vec { fn weight(&self) -> u64 { self.iter().map(|s| s.len()).sum::() as u64 + std::mem::size_of::>() as u64 } } impl CacheItemWeight for Txt { fn weight(&self) -> u64 { std::mem::size_of::() as u64 } } impl CacheItemWeight for IpAddr { fn weight(&self) -> u64 { std::mem::size_of::() as u64 } } impl CacheItemWeight for bool { fn weight(&self) -> u64 { std::mem::size_of::() as u64 } } impl TtlEntry { pub fn new(value: T, expires: Duration) -> Self { Self { value, expires: Instant::now() + expires, } } pub fn with_expiry(value: T, expires: Instant) -> Self { Self { value, expires } } } impl ResolverCache for CacheWithTtl { fn get(&self, key: &Q) -> Option where K: Borrow, Q: Hash + Eq + ?Sized, { CacheWithTtl::get(self, key) } fn remove(&self, key: &Q) -> Option where K: Borrow, Q: Hash + Eq + ?Sized, { CacheWithTtl::remove(self, key) } fn insert(&self, key: K, value: V, expires: Instant) { self.0.insert(key, TtlEntry::with_expiry(value, expires)); } } ================================================ FILE: crates/utils/src/chained_bytes.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{borrow::Cow, ops::Range}; #[derive(Debug, Clone)] pub struct ChainedBytes<'x> { first: &'x [u8], last: &'x [u8], } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SliceRange<'x> { Single(&'x [u8]), Split(&'x [u8], &'x [u8]), None, } impl<'x> ChainedBytes<'x> { pub fn new(first: &'x [u8]) -> Self { Self { first, last: &[] } } pub fn append(&mut self, bytes: &'x [u8]) { self.last = bytes; } pub fn with_last(mut self, bytes: &'x [u8]) -> Self { self.last = bytes; self } pub fn get(&self, index: Range) -> Option> { let start = index.start; let end = index.end; if let Some(bytes) = self.first.get(start..end) { Some(Cow::Borrowed(bytes)) } else if start >= self.first.len() { self.last .get(start - self.first.len()..end - self.first.len()) .map(Cow::Borrowed) } else if let (Some(first), Some(last)) = ( self.first.get(start..), self.last.get(..end - self.first.len()), ) { let mut vec = vec![0u8; first.len() + last.len()]; vec[..first.len()].copy_from_slice(first); vec[first.len()..].copy_from_slice(last); Some(Cow::Owned(vec)) } else { None } } pub fn get_slice_range(&self, index: Range) -> SliceRange<'x> { let start = index.start; let end = index.end; if let Some(bytes) = self.first.get(start..end) { SliceRange::Single(bytes) } else if start >= self.first.len() { self.last .get(start - self.first.len()..end - self.first.len()) .map(SliceRange::Single) .unwrap_or(SliceRange::None) } else if let (Some(first), Some(last)) = ( self.first.get(start..), self.last.get(..end - self.first.len()), ) { SliceRange::Split(first, last) } else { SliceRange::None } } pub fn get_full_range(&self) -> SliceRange<'x> { if self.last.is_empty() { SliceRange::Single(self.first) } else { SliceRange::Split(self.first, self.last) } } pub fn to_bytes(&self) -> Vec { let mut bytes = vec![0u8; self.first.len() + self.last.len()]; bytes[..self.first.len()].copy_from_slice(self.first); bytes[self.first.len()..].copy_from_slice(self.last); bytes } pub fn len(&self) -> usize { self.first.len() + self.last.len() } pub fn is_empty(&self) -> bool { self.len() == 0 } } impl<'x> SliceRange<'x> { pub fn len(&self) -> usize { match self { SliceRange::Single(bytes) => bytes.len(), SliceRange::Split(first, last) => first.len() + last.len(), SliceRange::None => 0, } } pub fn try_into_bytes(self) -> Option> { match self { SliceRange::Single(bytes) => Some(Cow::Borrowed(bytes)), SliceRange::Split(first, last) => { let mut vec = vec![0u8; first.len() + last.len()]; vec[..first.len()].copy_from_slice(first); vec[first.len()..].copy_from_slice(last); Some(Cow::Owned(vec)) } SliceRange::None => None, } } pub fn is_empty(&self) -> bool { self.len() == 0 } fn into_pairs(self) -> (&'x [u8], &'x [u8]) { match self { SliceRange::Single(bytes) => (bytes, &[][..]), SliceRange::Split(first, last) => (first, last), SliceRange::None => (&[][..], &[][..]), } } pub fn is_none(&self) -> bool { matches!(self, SliceRange::None) } pub fn is_some(&self) -> bool { !self.is_none() } } impl<'x> IntoIterator for SliceRange<'x> { type Item = &'x u8; type IntoIter = std::iter::Chain, std::slice::Iter<'x, u8>>; fn into_iter(self) -> Self::IntoIter { let (first, last) = self.into_pairs(); first.iter().chain(last.iter()) } } ================================================ FILE: crates/utils/src/cheeky_hash.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use nohash_hasher::IsEnabled; use std::{ collections::{BTreeMap, HashMap, HashSet}, fmt::Debug, hash::Hash, }; // A hash that can cheekily store small inputs directly without hashing them. #[derive( Copy, Clone, PartialEq, Eq, PartialOrd, Ord, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, )] #[repr(transparent)] pub struct CheekyHash([u8; HASH_SIZE]); const HASH_SIZE: usize = std::mem::size_of::() * 2; const HASH_PAYLOAD: usize = HASH_SIZE - 1; pub type CheekyHashSet = HashSet>; pub type CheekyHashMap = HashMap>; pub type CheekyBTreeMap = BTreeMap; impl CheekyHash { pub const HASH_SIZE: usize = HASH_SIZE; pub const NULL: CheekyHash = CheekyHash([0u8; HASH_SIZE]); pub const FULL: CheekyHash = CheekyHash([u8::MAX; HASH_SIZE]); pub fn new(bytes: impl AsRef<[u8]>) -> Self { let mut hash = [0u8; HASH_SIZE]; let bytes = bytes.as_ref(); if bytes.len() <= HASH_PAYLOAD { hash[0] = bytes.len() as u8; hash[1..1 + bytes.len()].copy_from_slice(bytes); } else { let h1 = xxhash_rust::xxh3::xxh3_64(bytes).to_be_bytes(); let h2 = farmhash::fingerprint64(bytes).to_be_bytes(); hash[0] = bytes.len().min(u8::MAX as usize) as u8; hash[1..1 + std::mem::size_of::()].copy_from_slice(&h1); hash[1 + std::mem::size_of::()..] .copy_from_slice(&h2[..std::mem::size_of::() - 1]); } CheekyHash(hash) } pub fn deserialize(bytes: &[u8]) -> Option { let len = *bytes.first()?; let mut hash = [0u8; HASH_SIZE]; let hash_len = 1 + (len as usize).min(HASH_PAYLOAD); hash[0] = len; hash[1..hash_len].copy_from_slice(bytes.get(1..hash_len)?); Some(CheekyHash(hash)) } #[allow(clippy::len_without_is_empty)] #[inline(always)] pub fn len(&self) -> usize { (self.0[0] as usize).min(HASH_PAYLOAD) + 1 } #[inline(always)] pub fn as_bytes(&self) -> &[u8] { &self.0[..self.len()] } #[inline(always)] pub fn as_raw_bytes(&self) -> &[u8; HASH_SIZE] { &self.0 } pub fn into_inner(self) -> [u8; HASH_SIZE] { self.0 } pub fn payload(&self) -> &[u8] { let len = self.0[0] as usize; if len <= HASH_PAYLOAD { &self.0[1..1 + len] } else { &self.0[1..] } } pub fn payload_len(&self) -> u8 { self.0[0] } } impl AsRef<[u8]> for CheekyHash { fn as_ref(&self) -> &[u8] { self.as_bytes() } } impl Hash for CheekyHash { fn hash(&self, state: &mut H) { let len = self.0[0] as usize; if len <= HASH_PAYLOAD { state.write_u64(xxhash_rust::xxh3::xxh3_64(&self.0[1..1 + len])); } else { state.write_u64(u64::from_be_bytes( self.0[1..1 + std::mem::size_of::()] .try_into() .unwrap(), )); } } } impl IsEnabled for CheekyHash {} impl ArchivedCheekyHash { #[inline(always)] pub fn as_raw_bytes(&self) -> &[u8; HASH_SIZE] { &self.0 } #[inline(always)] pub fn as_bytes(&self) -> &[u8] { let len = self.0[0] as usize; &self.0[..1 + len.min(HASH_PAYLOAD)] } #[inline(always)] pub fn to_native(&self) -> CheekyHash { CheekyHash(self.0) } } impl Debug for CheekyHash { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let len = self.payload_len(); let payload = self.payload(); let payload_str = if len <= HASH_PAYLOAD as u8 { std::str::from_utf8(payload).unwrap_or("") } else { "" }; f.debug_struct("CheekyHash") .field("length", &len) .field("bytes", &payload_str) .finish() } } #[cfg(test)] mod tests { use super::*; #[test] fn test_cheeky_hash_all() { // Test 1: Empty input let hash_empty = CheekyHash::new([]); assert_eq!( hash_empty.as_bytes()[0], 0, "Empty input should have length 0" ); assert_eq!( hash_empty.as_bytes().len(), 1, "Empty input should only have length byte" ); // Test 2: Single byte input let hash_single = CheekyHash::new([42]); assert_eq!( hash_single.as_bytes()[0], 1, "Single byte should have length 1" ); assert_eq!( hash_single.as_bytes()[1], 42, "Single byte value should be preserved" ); assert_eq!(hash_single.as_bytes().len(), 2); // Test 3: Small input (less than HASH_LEN) let small_data = b"hello"; let hash_small = CheekyHash::new(small_data); assert_eq!(hash_small.as_bytes()[0], 5, "Length should be 5"); assert_eq!( &hash_small.as_bytes()[1..6], small_data, "Small data should be stored directly" ); assert_eq!(hash_small.as_bytes().len(), 6); // Test 4: Input exactly at HASH_PAYLOAD boundary let boundary_data = vec![1u8; HASH_PAYLOAD - 1]; let hash_boundary = CheekyHash::new(&boundary_data); assert_eq!( hash_boundary.as_bytes()[0], (HASH_PAYLOAD - 1) as u8, "Length should be HASH_LEN" ); assert_eq!( &hash_boundary.as_bytes()[1..], &boundary_data[..], "Boundary data should be stored directly" ); // Test 5: Large input (greater than HASH_LEN) - uses hashing let large_data = vec![7u8; HASH_SIZE]; let hash_large = CheekyHash::new(&large_data); assert_eq!( hash_large.as_bytes()[0], HASH_SIZE as u8, "Large data should have length byte set to HASH_LEN" ); assert_eq!( hash_large.as_bytes().len(), HASH_SIZE, "Large data hash should be full length" ); // Verify it's actually hashed (not raw data) assert_ne!( &hash_large.as_bytes()[1..], &large_data[..HASH_PAYLOAD], "Large data should be hashed, not stored directly" ); // Test 6: AsRef<[u8]> trait let hash = CheekyHash::new(b"test"); let bytes_ref: &[u8] = hash.as_ref(); assert_eq!(bytes_ref, hash.as_bytes(), "AsRef should match as_bytes"); // Test 7: Copy, Clone, PartialEq traits let hash1 = CheekyHash::new(b"identical"); let hash2 = hash1; // Copy assert_eq!(hash1, hash2, "Copied hashes should be equal"); // Test 8: Different inputs produce different hashes let hash_a = CheekyHash::new(b"abc"); let hash_b = CheekyHash::new(b"def"); assert_ne!( hash_a, hash_b, "Different inputs should produce different hashes" ); // Test 9: Same input produces same hash (deterministic) let hash_x1 = CheekyHash::new(b"deterministic"); let hash_x2 = CheekyHash::new(b"deterministic"); assert_eq!( hash_x1, hash_x2, "Same input should produce identical hashes" ); // Test 10: Large inputs with different content produce different hashes let large1 = vec![1u8; 100]; let large2 = vec![2u8; 100]; let hash_large1 = CheekyHash::new(&large1); let hash_large2 = CheekyHash::new(&large2); assert_ne!( hash_large1, hash_large2, "Different large inputs should produce different hashes" ); // Test 11: Hash trait (can be used in HashMap/HashSet) use std::collections::HashMap; let mut map = HashMap::new(); let key = CheekyHash::new(b"key"); map.insert(key, "value"); assert_eq!( map.get(&key), Some(&"value"), "CheekyHash should work as HashMap key" ); // Test 12: Debug trait let hash = CheekyHash::new(b"debug"); let debug_str = format!("{:?}", hash); assert!( debug_str.contains("CheekyHash"), "Debug output should contain type name" ); // Test 13: CheekyHashSet and CheekyHashMap let mut cheeky_set: CheekyHashSet = CheekyHashSet::default(); cheeky_set.insert(CheekyHash::new(b"set_item")); assert!(cheeky_set.contains(&CheekyHash::new(b"set_item"))); let mut cheeky_map: CheekyHashMap<&str> = CheekyHashMap::default(); cheeky_map.insert(CheekyHash::new(b"map_key"), "map_value"); assert_eq!( cheeky_map.get(&CheekyHash::new(b"map_key")), Some(&"map_value") ); println!("All CheekyHash tests passed!"); } } ================================================ FILE: crates/utils/src/codec/base32_custom.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{io::Write, slice::Iter}; use super::leb128::{Leb128Iterator, Leb128Writer}; pub static BASE32_ALPHABET: &[u8] = b"abcdefghijklmnopqrstuvwxyz792013"; pub static BASE32_INVERSE: [u8; 256] = [ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 29, 30, 28, 31, 255, 255, 255, 26, 255, 27, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, ]; pub struct Base32Writer { last_byte: u8, pos: usize, result: String, } impl Base32Writer { pub fn from_bytes(bytes: impl AsRef<[u8]>) -> Self { let bytes = bytes.as_ref(); let mut writer = Base32Writer::with_capacity(bytes.len()); writer.write_all(bytes).unwrap(); writer } pub fn with_capacity(capacity: usize) -> Self { Self::with_raw_capacity(capacity.div_ceil(4) * 5) } pub fn with_raw_capacity(capacity: usize) -> Self { Base32Writer { result: String::with_capacity(capacity), last_byte: 0, pos: 0, } } pub fn push_char(&mut self, ch: char) { self.result.push(ch); } pub fn push_string(&mut self, string: &str) { self.result.push_str(string); } fn push_byte(&mut self, byte: u8, is_remainder: bool) { let (ch1, ch2) = match self.pos % 5 { 0 => ((byte & 0xF8) >> 3, u8::MAX), 1 => ( (((self.last_byte & 0x07) << 2) | ((byte & 0xC0) >> 6)), ((byte & 0x3E) >> 1), ), 2 => ( (((self.last_byte & 0x01) << 4) | ((byte & 0xF0) >> 4)), u8::MAX, ), 3 => ( (((self.last_byte & 0x0F) << 1) | (byte >> 7)), ((byte & 0x7C) >> 2), ), 4 => ( (((self.last_byte & 0x03) << 3) | ((byte & 0xE0) >> 5)), (byte & 0x1F), ), _ => unreachable!(), }; self.result.push(char::from(BASE32_ALPHABET[ch1 as usize])); if !is_remainder { if ch2 != u8::MAX { self.result.push(char::from(BASE32_ALPHABET[ch2 as usize])); } self.last_byte = byte; self.pos += 1; } } pub fn finalize(mut self) -> String { if !self.pos.is_multiple_of(5) { self.push_byte(0, true); } self.result } } impl std::io::Write for Base32Writer { fn write(&mut self, bytes: &[u8]) -> std::io::Result { let start_pos = self.pos; for &byte in bytes { self.push_byte(byte, false); } Ok(self.pos - start_pos) } fn flush(&mut self) -> std::io::Result<()> { Ok(()) } } #[derive(Debug)] pub struct Base32Reader<'x> { bytes: Iter<'x, u8>, last_byte: u8, pos: usize, } impl<'x> Base32Reader<'x> { pub fn new(bytes: &'x [u8]) -> Self { Base32Reader { bytes: bytes.iter(), pos: 0, last_byte: 0, } } #[allow(clippy::should_implement_trait)] pub fn from_iter(bytes: Iter<'x, u8>) -> Self { Base32Reader { bytes, pos: 0, last_byte: 0, } } #[inline(always)] fn map_byte(&mut self) -> Option { match self.bytes.next() { Some(&byte) => match BASE32_INVERSE[byte as usize] { byte if byte != u8::MAX => { self.last_byte = byte; Some(byte) } _ => None, }, _ => None, } } } impl Iterator for Base32Reader<'_> { type Item = u8; fn next(&mut self) -> Option { let pos = self.pos % 5; let last_byte = self.last_byte; let byte = self.map_byte()?; self.pos += 1; match pos { 0 => ((byte << 3) | (self.map_byte().unwrap_or(0) >> 2)).into(), 1 => ((last_byte << 6) | (byte << 1) | (self.map_byte().unwrap_or(0) >> 4)).into(), 2 => ((last_byte << 4) | (byte >> 1)).into(), 3 => ((last_byte << 7) | (byte << 2) | (self.map_byte().unwrap_or(0) >> 3)).into(), 4 => ((last_byte << 5) | byte).into(), _ => None, } } } impl Leb128Iterator for Base32Reader<'_> {} impl Leb128Writer for Base32Writer {} #[cfg(test)] mod tests { use std::io::Write; use crate::codec::base32_custom::{Base32Reader, Base32Writer}; #[test] fn base32_roundtrip() { let mut bytes = Vec::with_capacity(100); for byte in 0..100 { bytes.push((100 - byte) as u8); let mut writer = Base32Writer::with_capacity(10); writer.write_all(&bytes).unwrap(); let result = writer.finalize(); let mut bytes_result = Vec::new(); for byte in Base32Reader::new(result.as_bytes()) { bytes_result.push(byte); } assert_eq!(bytes, bytes_result); } for bytes in [ vec![0], vec![32, 43, 55, 99, 43, 55], vec![84, 4, 43, 77, 62, 55, 92], vec![84, 4, 43, 77, 62, 55, 92], ] { let mut writer = Base32Writer::with_capacity(10); writer.write_all(&bytes).unwrap(); let result = writer.finalize(); let mut bytes_result = Vec::new(); for byte in Base32Reader::new(result.as_bytes()) { bytes_result.push(byte); } assert_eq!(bytes, bytes_result); } } } ================================================ FILE: crates/utils/src/codec/leb128.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ #![allow(dead_code)] use std::{borrow::Borrow, io::Write}; pub trait Leb128_ { fn to_leb128_writer(self, out: &mut impl Write) -> std::io::Result; fn to_leb128_bytes(self, out: &mut Vec); fn from_leb128_bytes_pos(slice: &[u8]) -> Option<(Self, usize)> where Self: std::marker::Sized; fn from_leb128_bytes(slice: &[u8]) -> Option where Self: std::marker::Sized; fn from_leb128_it(it: T) -> Option where Self: std::marker::Sized, T: Iterator, I: Borrow; } pub trait Leb128Vec { fn push_leb128(&mut self, value: T); } pub trait Leb128Writer: Write + Sized { #[inline(always)] fn write_leb128(&mut self, value: T) -> std::io::Result { T::to_leb128_writer(value, self) } } impl Leb128Vec for Vec { #[inline(always)] fn push_leb128(&mut self, value: T) { T::to_leb128_bytes(value, self); } } pub trait Leb128Iterator: Iterator where I: Borrow, { #[inline(always)] fn next_leb128(&mut self) -> Option { T::from_leb128_it(self) } #[inline(always)] fn skip_leb128(&mut self) -> Option<()> { for byte in self { if (byte.borrow() & 0x80) == 0 { return Some(()); } } None } } pub trait Leb128Reader: AsRef<[u8]> { #[inline(always)] fn read_leb128(&self) -> Option<(T, usize)> { T::from_leb128_bytes_pos(self.as_ref()) } #[inline(always)] fn skip_leb128(&self) -> Option { for (pos, byte) in self.as_ref().iter().enumerate() { if (byte & 0x80) == 0 { return (pos + 1).into(); } } None } } impl Leb128Reader for &[u8] {} impl Leb128Reader for Vec {} impl Leb128Reader for Box<[u8]> {} impl<'x> Leb128Iterator<&'x u8> for std::slice::Iter<'x, u8> {} // Based on leb128.rs from rustc macro_rules! impl_unsigned_leb128 { ($int_ty:ident, $shifts:expr) => { impl Leb128_ for $int_ty { #[inline(always)] fn to_leb128_writer(self, out: &mut impl Write) -> std::io::Result { let mut value = self; let mut bytes_written = 0; loop { if value < 0x80 { bytes_written += out.write(&[value as u8])?; break; } else { bytes_written += out.write(&[((value & 0x7f) | 0x80) as u8])?; value >>= 7; } } Ok(bytes_written) } #[inline(always)] fn to_leb128_bytes(self, out: &mut Vec) { let mut value = self; loop { if value < 0x80 { out.push(value as u8); break; } else { out.push(((value & 0x7f) | 0x80) as u8); value >>= 7; } } } #[inline(always)] fn from_leb128_bytes_pos(slice: &[u8]) -> Option<($int_ty, usize)> { let mut result = 0; for (shift, (pos, &byte)) in $shifts.into_iter().zip(slice.iter().enumerate()) { if (byte & 0x80) == 0 { result |= (byte as $int_ty) << shift; return Some((result, pos + 1)); } else { result |= ((byte & 0x7F) as $int_ty) << shift; } } None } #[inline(always)] fn from_leb128_bytes(slice: &[u8]) -> Option<$int_ty> { let mut result = 0; for (shift, &byte) in $shifts.into_iter().zip(slice.iter()) { if (byte & 0x80) == 0 { result |= (byte as $int_ty) << shift; return Some(result); } else { result |= ((byte & 0x7F) as $int_ty) << shift; } } None } #[inline(always)] fn from_leb128_it(it: T) -> Option<$int_ty> where T: Iterator, I: Borrow, { let mut result = 0; for (shift, byte_) in $shifts.into_iter().zip(it) { let byte = byte_.borrow(); if (byte & 0x80) == 0 { result |= (*byte as $int_ty) << shift; return Some(result); } else { result |= ((byte & 0x7F) as $int_ty) << shift; } } None } } }; } impl_unsigned_leb128!(u8, [0]); impl_unsigned_leb128!(u16, [0, 7, 14]); impl_unsigned_leb128!(u32, [0, 7, 14, 21, 28]); impl_unsigned_leb128!(u64, [0, 7, 14, 21, 28, 35, 42, 49, 56, 63]); impl_unsigned_leb128!(usize, [0, 7, 14, 21, 28, 35, 42, 49, 56, 63]); impl Leb128Writer for Vec { fn write_leb128(&mut self, value: T) -> std::io::Result { T::to_leb128_writer(value, self) } } ================================================ FILE: crates/utils/src/codec/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod base32_custom; pub mod leb128; ================================================ FILE: crates/utils/src/config/cron.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use chrono::{Datelike, Local, TimeDelta, TimeZone, Timelike}; use super::utils::ParseValue; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SimpleCron { Day { hour: u32, minute: u32 }, Week { day: u32, hour: u32, minute: u32 }, Hour { minute: u32 }, } impl SimpleCron { pub fn time_to_next(&self) -> Duration { let now = Local::now(); let next = match self { SimpleCron::Day { hour, minute } => { let next = Local .with_ymd_and_hms(now.year(), now.month(), now.day(), *hour, *minute, 0) .earliest() .unwrap_or_else(|| now - TimeDelta::try_seconds(1).unwrap_or_default()); if next <= now { next + TimeDelta::try_days(1).unwrap_or_default() } else { next } } SimpleCron::Week { day, hour, minute } => { let next = Local .with_ymd_and_hms(now.year(), now.month(), now.day(), *hour, *minute, 0) .earliest() .unwrap_or_else(|| now - TimeDelta::try_seconds(1).unwrap_or_default()); if next <= now { next + TimeDelta::try_days( (7 - now.weekday().number_from_monday() + *day).into(), ) .unwrap_or_default() } else { next } } SimpleCron::Hour { minute } => { let next = Local .with_ymd_and_hms(now.year(), now.month(), now.day(), now.hour(), *minute, 0) .earliest() .unwrap_or_else(|| now - TimeDelta::try_seconds(1).unwrap_or_default()); if next <= now { next + TimeDelta::try_hours(1).unwrap_or_default() } else { next } } }; (next - now).to_std().unwrap_or_else(|_| self.as_duration()) } pub fn as_duration(&self) -> Duration { match self { SimpleCron::Day { .. } => Duration::from_secs(24 * 60 * 60), SimpleCron::Week { .. } => Duration::from_secs(7 * 24 * 60 * 60), SimpleCron::Hour { .. } => Duration::from_secs(60 * 60), } } } impl ParseValue for SimpleCron { fn parse_value(value: &str) -> super::Result { let mut hour = 0; let mut minute = 0; for (pos, value) in value.split(' ').enumerate() { if pos == 0 { minute = value .parse::() .map_err(|_| "Invalid cron key: failed to parse cron minute".to_string())?; if !(0..=59).contains(&minute) { return Err(format!( "Invalid cron key: failed to parse minute, invalid value: {minute}" )); } } else if pos == 1 { if value .as_bytes() .first() .ok_or_else(|| "Invalid cron key: failed to parse cron hour".to_string())? == &b'*' { return Ok(SimpleCron::Hour { minute }); } else { hour = value .parse::() .map_err(|_| "Invalid cron key: failed to parse cron hour".to_string())?; if !(0..=23).contains(&hour) { return Err(format!( "Invalid cron key: failed to parse hour, invalid value: {hour}" )); } } } else if pos == 2 { if value .as_bytes() .first() .ok_or_else(|| "Invalid cron key: failed to parse cron weekday".to_string())? == &b'*' { return Ok(SimpleCron::Day { hour, minute }); } else { let day = value.parse::().map_err(|_| { "Invalid cron key: failed to parse cron weekday".to_string() })?; if !(1..=7).contains(&hour) { return Err(format!( "Invalid cron key: failed to parse weekday, invalid value: {}, range is 1 (Monday) to 7 (Sunday).", hour, )); } return Ok(SimpleCron::Week { day, hour, minute }); } } } Err("Invalid cron key: parse cron expression.".to_string()) } } impl Default for SimpleCron { fn default() -> Self { SimpleCron::Hour { minute: 0 } } } ================================================ FILE: crates/utils/src/config/http.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::config::{Config, utils::AsKey}; use base64::{Engine, engine::general_purpose}; use reqwest::{ Client, header::{AUTHORIZATION, CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue, USER_AGENT}, }; use std::{str::FromStr, time::Duration}; pub fn build_http_client( config: &mut Config, prefix: impl AsKey, content_type: Option<&str>, ) -> Option { let mut headers = parse_http_headers(config, prefix.clone()); headers.insert(USER_AGENT, "Stalwart/1.0.0".parse().unwrap()); if let Some(content_type) = content_type { headers.insert(CONTENT_TYPE, HeaderValue::from_str(content_type).unwrap()); } let prefix = prefix.as_key(); match Client::builder() .connect_timeout( config .property_or_default::((&prefix, "timeout"), "30s") .unwrap_or(Duration::from_secs(30)), ) .danger_accept_invalid_certs( config .property_or_default::((&prefix, "tls.allow-invalid-certs"), "false") .unwrap_or(false), ) .default_headers(headers) .build() { Ok(client) => Some(client), Err(err) => { config.new_build_error(&prefix, format!("Failed to build HTTP client: {err}")); None } } } pub fn parse_http_headers(config: &mut Config, prefix: impl AsKey) -> HeaderMap { let prefix = prefix.as_key(); let mut headers = HeaderMap::new(); for (header, value) in config .values((&prefix, "headers")) .map(|(_, v)| { if let Some((k, v)) = v.split_once(':') { Ok(( HeaderName::from_str(k.trim()).map_err(|err| { format!("Invalid header found in property \"{prefix}.headers\": {err}",) })?, HeaderValue::from_str(v.trim()).map_err(|err| { format!("Invalid header found in property \"{prefix}.headers\": {err}",) })?, )) } else { Err(format!( "Invalid header found in property \"{prefix}.headers\": {v}", )) } }) .collect::, String>>() .map_err(|e| config.new_parse_error((&prefix, "headers"), e)) .unwrap_or_default() { headers.insert(header, value); } if let (Some(name), Some(secret)) = ( config.value((&prefix, "auth.username")), config.value((&prefix, "auth.secret")), ) { headers.insert( AUTHORIZATION, format!( "Basic {}", general_purpose::STANDARD.encode(format!("{}:{}", name, secret)) ) .parse() .unwrap(), ); } else if let Some(token) = config.value((&prefix, "auth.token")) { headers.insert(AUTHORIZATION, format!("Bearer {}", token).parse().unwrap()); } headers } ================================================ FILE: crates/utils/src/config/ipmask.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use rustls::{SupportedCipherSuite, crypto::ring::cipher_suite::*}; use super::utils::ParseValue; #[derive(Debug, Clone, PartialEq, Eq)] pub enum IpAddrMask { V4 { addr: Ipv4Addr, mask: u32 }, V6 { addr: Ipv6Addr, mask: u128 }, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum IpAddrOrMask { Ip(IpAddr), Mask(IpAddrMask), } impl IpAddrMask { pub fn matches(&self, remote: &IpAddr) -> bool { match self { IpAddrMask::V4 { addr, mask } => match *mask { u32::MAX => match remote { IpAddr::V4(remote) => addr == remote, IpAddr::V6(remote) => { if let Some(remote) = remote.to_ipv4_mapped() { addr == &remote } else { false } } }, 0 => { matches!(remote, IpAddr::V4(_)) } _ => { u32::from_be_bytes(match remote { IpAddr::V4(ip) => ip.octets(), IpAddr::V6(ip) => { if let Some(ip) = ip.to_ipv4() { ip.octets() } else { return false; } } }) & mask == u32::from_be_bytes(addr.octets()) & mask } }, IpAddrMask::V6 { addr, mask } => match *mask { u128::MAX => match remote { IpAddr::V6(remote) => remote == addr, IpAddr::V4(remote) => &remote.to_ipv6_mapped() == addr, }, 0 => { matches!(remote, IpAddr::V6(_)) } _ => { u128::from_be_bytes(match remote { IpAddr::V6(ip) => ip.octets(), IpAddr::V4(ip) => ip.to_ipv6_mapped().octets(), }) & mask == u128::from_be_bytes(addr.octets()) & mask } }, } } } impl ParseValue for IpAddrMask { fn parse_value(value: &str) -> super::Result { if let Some((addr, mask)) = value.rsplit_once('/') { if let (Ok(addr), Ok(mask)) = (addr.trim().parse::(), mask.trim().parse::()) { match addr { IpAddr::V4(addr) if (8..=32).contains(&mask) => { return Ok(IpAddrMask::V4 { addr, mask: u32::MAX << (32 - mask), }); } IpAddr::V6(addr) if (8..=128).contains(&mask) => { return Ok(IpAddrMask::V6 { addr, mask: u128::MAX << (128 - mask), }); } _ => (), } } } else { match value.trim().parse::() { Ok(IpAddr::V4(addr)) => { return Ok(IpAddrMask::V4 { addr, mask: u32::MAX, }); } Ok(IpAddr::V6(addr)) => { return Ok(IpAddrMask::V6 { addr, mask: u128::MAX, }); } _ => (), } } Err(format!("Invalid IP address {:?}", value,)) } } impl ParseValue for IpAddrOrMask { fn parse_value(ip: &str) -> super::Result { if ip.contains('/') { IpAddrMask::parse_value(ip).map(IpAddrOrMask::Mask) } else { IpAddr::parse_value(ip).map(IpAddrOrMask::Ip) } } } impl ParseValue for SocketAddr { fn parse_value(value: &str) -> super::Result { value .parse() .map_err(|_| format!("Invalid socket address {:?}.", value,)) } } impl ParseValue for SupportedCipherSuite { fn parse_value(value: &str) -> super::Result { Ok(match value { // TLS1.3 suites "TLS13_AES_256_GCM_SHA384" => TLS13_AES_256_GCM_SHA384, "TLS13_AES_128_GCM_SHA256" => TLS13_AES_128_GCM_SHA256, "TLS13_CHACHA20_POLY1305_SHA256" => TLS13_CHACHA20_POLY1305_SHA256, // TLS1.2 suites "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" => TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256" => TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256" => { TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 } "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384" => TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" => TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256" => { TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 } cipher => return Err(format!("Unsupported TLS cipher suite {:?}", cipher,)), }) } } #[cfg(test)] mod tests { use super::*; #[test] fn test_ipaddrmask() { for (mask, ip) in [ ("10.0.0.0/8", "10.30.20.11"), ("10.0.0.0/8", "10.0.13.73"), ("192.168.1.1", "192.168.1.1"), ] { let mask = IpAddrMask::parse_value(mask).unwrap(); let ip = ip.parse::().unwrap(); assert!(mask.matches(&ip)); } for (mask, ip) in [ ("10.0.0.0/8", "11.30.20.11"), ("192.168.1.1", "193.168.1.1"), ] { let mask = IpAddrMask::parse_value(mask).unwrap(); let ip = ip.parse::().unwrap(); assert!(!mask.matches(&ip)); } } } ================================================ FILE: crates/utils/src/config/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod cron; pub mod http; pub mod ipmask; pub mod parser; pub mod utils; use ahash::AHashMap; use compact_str::CompactString; use serde::Serialize; use std::{collections::BTreeMap, time::Duration}; #[derive(Debug, Default, Serialize)] pub struct Config { #[serde(skip)] pub keys: BTreeMap, pub warnings: AHashMap, pub errors: AHashMap, #[cfg(debug_assertions)] #[serde(skip)] pub keys_read: parking_lot::Mutex>, } #[derive(Debug, Clone, PartialEq, Eq, Serialize)] #[serde(tag = "type")] #[serde(rename_all = "camelCase")] pub enum ConfigWarning { Missing, AppliedDefault { default: String }, Unread { value: String }, Build { error: String }, Parse { error: String }, } #[derive(Debug, Clone, PartialEq, Eq, Serialize)] #[serde(tag = "type")] #[serde(rename_all = "camelCase")] pub enum ConfigError { Parse { error: String }, Build { error: String }, Macro { error: String }, } #[derive(Debug, Default, PartialEq, Eq)] pub struct ConfigKey { pub key: String, pub value: String, } #[derive(Debug, Default, PartialEq, Eq, Clone)] pub struct Rate { pub requests: u64, pub period: Duration, } pub type Result = std::result::Result; impl Config { pub async fn resolve_macros(&mut self, classes: &[&str]) { for macro_class in classes { self.resolve_macro_type(macro_class).await; } } pub async fn resolve_all_macros(&mut self) { self.resolve_macros(&["env", "file", "cfg"]).await; } async fn resolve_macro_type(&mut self, class: &str) { let macro_start = format!("%{{{class}:"); let mut replacements = AHashMap::new(); 'outer: for (key, value) in &self.keys { if value.contains(¯o_start) && value.contains("}%") { let mut result = String::with_capacity(value.len()); let mut snippet: &str = value.as_str(); loop { if let Some((suffix, macro_name)) = snippet.split_once(¯o_start) { if !suffix.is_empty() { result.push_str(suffix); } if let Some((location, rest)) = macro_name.split_once("}%") { match class { "cfg" => { if let Some(value) = replacements .get(location) .or_else(|| self.keys.get(location)) { result.push_str(value); } else { self.errors.insert( key.clone(), ConfigError::Macro { error: format!("Unknown key {location:?}"), }, ); } } "env" => match std::env::var(location) { Ok(value) => { result.push_str(&value); } Err(_) => { self.errors.insert( key.clone(), ConfigError::Macro { error : format!( "Failed to obtain environment variable {location:?}" )}, ); } }, "file" => { let file_name = location.strip_prefix("//").unwrap_or(location); match tokio::fs::read(file_name).await { Ok(value) => match String::from_utf8(value) { Ok(value) => { result.push_str(&value); } Err(err) => { self.errors.insert( key.clone(), ConfigError::Macro { error: format!( "Failed to read file {file_name:?}: {err}" ), }, ); continue 'outer; } }, Err(err) => { self.errors.insert( key.clone(), ConfigError::Macro { error: format!( "Failed to read file {file_name:?}: {err}" ), }, ); continue 'outer; } } } _ => { unreachable!() } }; snippet = rest; } } else { result.push_str(snippet); break; } } replacements.insert(key.clone(), result); } } if !replacements.is_empty() { for (key, value) in replacements { self.keys.insert(key, value); } } } pub fn update(&mut self, settings: Vec<(String, String)>) { self.keys.extend(settings); } pub fn log_errors(&self) { for (key, err) in &self.errors { let (cause, message) = match err { ConfigError::Parse { error } => ( trc::ConfigEvent::ParseError, format!("Failed to parse setting {key:?}: {error}"), ), ConfigError::Build { error } => ( trc::ConfigEvent::BuildError, format!("Build error for key {key:?}: {error}"), ), ConfigError::Macro { error } => ( trc::ConfigEvent::MacroError, format!("Macro expansion error for setting {key:?}: {error}"), ), }; trc::error!( trc::EventType::Config(cause) .into_err() .details(CompactString::from(message)) ); } } pub fn log_warnings(&mut self) { #[cfg(debug_assertions)] self.warn_unread_keys(); for (key, warn) in &self.warnings { let (cause, message) = match warn { ConfigWarning::AppliedDefault { default } => ( trc::ConfigEvent::DefaultApplied, format!("WARNING: Missing setting {key:?}, applied default {default:?}"), ), ConfigWarning::Missing => ( trc::ConfigEvent::MissingSetting, format!("WARNING: Missing setting {key:?}"), ), ConfigWarning::Unread { value } => ( trc::ConfigEvent::UnusedSetting, format!("WARNING: Unused setting {key:?} with value {value:?}"), ), ConfigWarning::Parse { error } => ( trc::ConfigEvent::ParseWarning, format!("WARNING: Failed to parse {key:?}: {error}"), ), ConfigWarning::Build { error } => ( trc::ConfigEvent::BuildWarning, format!("WARNING for {key:?}: {error}"), ), }; trc::error!( trc::EventType::Config(cause) .into_err() .details(CompactString::from(message)) ); } } } impl Clone for Config { fn clone(&self) -> Self { Self { keys: self.keys.clone(), warnings: self.warnings.clone(), errors: self.errors.clone(), #[cfg(debug_assertions)] keys_read: Default::default(), } } } impl PartialEq for Config { fn eq(&self, other: &Self) -> bool { self.keys == other.keys && self.warnings == other.warnings && self.errors == other.errors } } impl Eq for Config {} impl From<(String, String)> for ConfigKey { fn from((key, value): (String, String)) -> Self { Self { key, value } } } impl From<(&str, &str)> for ConfigKey { fn from((key, value): (&str, &str)) -> Self { Self { key: key.to_string(), value: value.to_string(), } } } impl From<(&str, String)> for ConfigKey { fn from((key, value): (&str, String)) -> Self { Self { key: key.to_string(), value, } } } impl From<(String, &str)> for ConfigKey { fn from((key, value): (String, &str)) -> Self { Self { key, value: value.to_string(), } } } ================================================ FILE: crates/utils/src/config/parser.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ collections::{BTreeMap, btree_map::Entry}, iter::Peekable, str::Chars, }; use super::{Config, Result}; use std::fmt::Write; const MAX_NEST_LEVEL: usize = 10; // Simple TOML parser for Stalwart Server configuration files. impl Config { pub fn new(toml: impl AsRef) -> Result { let mut config = Config::default(); config.parse(toml.as_ref())?; Ok(config) } pub fn parse(&mut self, toml: &str) -> Result<()> { let mut parser = TomlParser::new(&mut self.keys, toml); let mut table_name = String::new(); let mut last_array_name = String::new(); let mut last_array_pos = 0; while parser.seek_next_char() { match parser.peek_char()? { '[' => { parser.next_char(true, false)?; table_name.clear(); let mut is_array = match parser.next_char(true, false)? { '[' => true, ch => { table_name.push(ch); false } }; let mut in_quote = false; let mut last_ch = char::from(0); loop { let ch = parser.next_char(!in_quote, false)?; match ch { '\"' if !in_quote || last_ch != '\\' => { in_quote = !in_quote; } '\\' if in_quote => (), ']' if !in_quote => { if table_name.is_empty() { return Err(format!( "Empty table name at line {}.", parser.line )); } if is_array { if table_name == last_array_name { last_array_pos += 1; } else { last_array_pos = 0; last_array_name = table_name.to_string(); } is_array = false; write!(table_name, ".{last_array_pos:04}").ok(); } else { break; } } _ => { if !in_quote { if ch.is_alphanumeric() || ['.', '-', '_'].contains(&ch) { table_name.push(ch.to_ascii_lowercase()); } else { return Err(format!( "Unexpected character {:?} at line {}.", ch, parser.line )); } } else { table_name.push(ch); } } } last_ch = ch; } parser.skip_line(); } 'a'..='z' | 'A'..='Z' | '0'..='9' | '\"' => { let (key, _) = parser.key( if !table_name.is_empty() { format!("{table_name}.") } else { String::with_capacity(10) }, false, )?; parser.value(key, &['\n'], 0)?; } '#' => { parser.skip_line(); } ch => { let ch = *ch; return Err(format!( "Unexpected character {:?} at line {}.", ch, parser.line )); } } } Ok(()) } } struct TomlParser<'x, 'y> { keys: &'y mut BTreeMap, iter: Peekable>, line: usize, } impl<'x, 'y> TomlParser<'x, 'y> { fn new(keys: &'y mut BTreeMap, toml: &'x str) -> Self { Self { keys, iter: toml.chars().peekable(), line: 1, } } fn seek_next_char(&mut self) -> bool { while let Some(ch) = self.iter.peek() { match ch { '\n' => { self.iter.next(); self.line += 1; } '\r' | ' ' | '\t' => { self.iter.next(); } '#' => { self.skip_line(); } _ => { return true; } } } false } fn peek_char(&mut self) -> Result<&char> { self.iter.peek().ok_or_else(|| "".to_string()) } fn next_char(&mut self, skip_wsp: bool, allow_lf: bool) -> Result { for ch in &mut self.iter { match ch { '\r' => (), ' ' | '\t' if skip_wsp => (), '\n' => { return if allow_lf { self.line += 1; Ok(ch) } else { Err(format!("Unexpected end of line at line: {}", self.line)) }; } _ => { return Ok(ch); } } } Err(format!("Unexpected EOF at line: {}", self.line)) } fn skip_line(&mut self) { for ch in &mut self.iter { if ch == '\n' { self.line += 1; break; } } } #[allow(clippy::while_let_on_iterator)] fn key(&mut self, mut key: String, in_curly: bool) -> Result<(String, char)> { let start_key_len = key.len(); while let Some(ch) = self.iter.next() { match ch { '=' => { if start_key_len != key.len() { return Ok((key, ch)); } else { return Err(format!("Empty key at line: {}", self.line)); } } ',' | '}' if in_curly => { if start_key_len != key.len() { return Ok((key, ch)); } else { return Err(format!("Empty key at line: {}", self.line)); } } /*'a'..='z' | '.' | 'A'..='Z' | '0'..='9' | '_' | '-' => { key.push(ch); }*/ '\"' => { let mut last_ch = char::from(0); while let Some(ch) = self.iter.next() { match ch { '\\' => (), '\"' if last_ch != '\\' => { break; } '\n' => { return Err(format!( "Unexpected end of line while parsing quoted key at line: {}", self.line )); } _ => { key.push(ch); } } last_ch = ch; } } ' ' | '\t' | '\r' => (), '\n' => { if start_key_len == key.len() { self.line += 1; } else { return Err(format!( "Unexpected end of line while parsing key {:?} at line: {}", key, self.line )); } } _ => { key.push(ch); } } } Err(format!("Unexpected EOF at line: {}", self.line)) } fn value(&mut self, key: String, stop_chars: &[char], nest_level: usize) -> Result { if nest_level == MAX_NEST_LEVEL { return Err(format!("Too many nested structures at line {}.", self.line)); } match self.next_char(true, false)? { '[' => { let mut array_pos = 0; self.seek_next_char(); loop { match self.value( format!("{key}.{array_pos:04}"), &[',', ']'], nest_level + 1, )? { ',' => { self.seek_next_char(); array_pos += 1; } ']' => break, ch => { return Err(format!( "Unexpected character {:?} found in array for property {:?} at line {}.", ch, key, self.line )); } } } } '{' => { let base_key = format!("{key}."); let base_key_len = base_key.len(); loop { let (sub_key, stop_char) = self.key(base_key.clone(), true)?; match stop_char { '=' => { // Key value self.seek_next_char(); match self.value(sub_key, &[',', '}'], nest_level + 1)? { ',' => { self.seek_next_char(); } '}' => break, ch => { return Err(format!( "Unexpected character {:?} found in inline table for property {:?} at line {}.", ch, key, self.line )); } } } ',' => { // Set if sub_key.len() > base_key_len { self.insert_key(sub_key, String::new())?; } } '}' => { // Set if sub_key.len() > base_key_len { self.insert_key(sub_key, String::new())?; } break; } _ => unreachable!(), } } } qch @ ('\'' | '\"') => { let mut value = String::new(); if matches!(self.iter.peek(), Some(ch) if ch == &qch) { self.iter.next(); if matches!(self.iter.peek(), Some(ch) if ch == &qch) { self.iter.next(); if matches!(self.iter.peek(), Some(ch) if ch == &'\n') { self.iter.next(); self.line += 1; } let mut last_ch = char::from(0); let mut prev_last_ch = char::from(0); loop { let ch = self.next_char(false, true)?; if !(ch == qch && last_ch == qch && prev_last_ch == qch) { value.push(ch); prev_last_ch = last_ch; last_ch = ch; } else { value.truncate(value.len() - 2); break; } } } } else { let mut last_ch = char::from(0); loop { let ch = self.next_char(false, true)?; match ch { '\\' if last_ch != '\\' => (), 't' if last_ch == '\\' => { value.push('\t'); } 'r' if last_ch == '\\' => { value.push('\r'); } 'n' if last_ch == '\\' => { value.push('\n'); } ch => { if ch != qch || last_ch == '\\' { value.push(ch); } else { break; } } } last_ch = ch; } } self.insert_key(key, value)?; } ch if ch.is_alphanumeric() || ['.', '+', '-'].contains(&ch) => { let mut value = String::with_capacity(4); value.push(ch); while let Some(ch) = self.iter.peek() { if ch.is_alphanumeric() || ['.', '+', '-'].contains(ch) { value.push(self.next_char(true, false)?); } else { break; } } self.insert_key(key, value)?; } ch => { return if stop_chars.contains(&ch) { Ok(ch) } else { Err(format!( "Expected {:?} but found {:?} in value at line {}.", stop_chars, ch, self.line )) }; } } loop { match self.next_char(true, true)? { '#' => { self.skip_line(); if stop_chars.contains(&'\n') { return Ok('\n'); } } ch if stop_chars.contains(&ch) => { return Ok(ch); } '\n' if !stop_chars.contains(&'\n') => (), ch => { return Err(format!( "Expected {:?} but found {:?} in value at line {}.", stop_chars, ch, self.line )); } } } } fn insert_key(&mut self, key: String, mut value: String) -> Result<()> { match self.keys.entry(key) { Entry::Vacant(e) => { value.shrink_to_fit(); e.insert(value); Ok(()) } Entry::Occupied(e) => Err(format!( "Duplicate key {:?} at line {}.", e.key(), self.line )), } } } #[cfg(test)] mod tests { use std::{collections::BTreeMap, fs, path::PathBuf}; use crate::config::Config; #[test] fn toml_parse() { let file = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .parent() .unwrap() .parent() .unwrap() .to_path_buf() .join("tests") .join("resources") .join("smtp") .join("config") .join("toml-parser.toml"); let mut config = Config::default(); config.parse(&fs::read_to_string(file).unwrap()).unwrap(); let expected = BTreeMap::from_iter( [ ("arrays.colors.0000", "red"), ("arrays.colors.0001", "yellow"), ("arrays.colors.0002", "green"), ("arrays.contributors.0000", "Foo Bar "), ("arrays.contributors.0001.email", "bazqux@example.com"), ("arrays.contributors.0001.name", "Baz Qux"), ("arrays.contributors.0001.url", "https://example.com/bazqux"), ("arrays.integers.0000", "1"), ("arrays.integers.0001", "2"), ("arrays.integers.0002", "3"), ("arrays.integers2.0000", "1"), ("arrays.integers2.0001", "2"), ("arrays.integers2.0002", "3"), ("arrays.integers3.0000", "4"), ("arrays.integers3.0001", "5"), ("arrays.nested_arrays_of_ints.0000.0000", "1"), ("arrays.nested_arrays_of_ints.0000.0001", "2"), ("arrays.nested_arrays_of_ints.0001.0000", "3"), ("arrays.nested_arrays_of_ints.0001.0001", "4"), ("arrays.nested_arrays_of_ints.0001.0002", "5"), ("arrays.nested_mixed_array.0000.0000", "1"), ("arrays.nested_mixed_array.0000.0001", "2"), ("arrays.nested_mixed_array.0001.0000", "a"), ("arrays.nested_mixed_array.0001.0001", "b"), ("arrays.nested_mixed_array.0001.0002", "c"), ("arrays.numbers.0000", "0.1"), ("arrays.numbers.0001", "0.2"), ("arrays.numbers.0002", "0.5"), ("arrays.numbers.0003", "1"), ("arrays.numbers.0004", "2"), ("arrays.numbers.0005", "5"), ("arrays.string_array.0000", "all"), ("arrays.string_array.0001", "strings"), ("arrays.string_array.0002", "are the same"), ("arrays.string_array.0003", "type"), ("database.data.0000.0000", "delta"), ("database.data.0000.0001", "phi"), ("database.data.0001.0000", "3.14"), ("database.enabled", "true"), ("database.ports.0000", "8000"), ("database.ports.0001", "8001"), ("database.ports.0002", "8002"), ("database.temp_targets.case", "72.0"), ("database.temp_targets.cpu", "79.5"), ("products.0000.name", "Hammer"), ("products.0000.sku", "738594937"), ("products.0002.color", "gray"), ("products.0002.name", "Nail"), ("products.0002.sku", "284758393"), ("servers.127.0.0.1", "value"), ("servers.alpha.ip", "10.0.0.1"), ("servers.alpha.role", "frontend"), ("servers.beta.ip", "10.0.0.2"), ("servers.beta.role", "backend"), ("servers.character encoding", "value"), ( "strings.my \"string\" test.lines", concat!( "The first newline is\ntrimmed in raw strings.\n", "All other whitespace\nis preserved.\n" ), ), ("strings.my \"string\" test.str1", "I'm a string."), ("strings.my \"string\" test.str2", "You can \"quote\" me."), ("strings.my \"string\" test.str3", "Name\tTabs\nNew Line."), ("env.var1", "utils"), ("env.var2", "utils"), ("sets.integer.1", ""), ("sets.integers.1", ""), ("sets.integers.2", ""), ("sets.integers.3", ""), ("sets.string.red", ""), ("sets.strings.red", ""), ("sets.strings.yellow", ""), ("sets.strings.green", ""), ] .map(|(k, v)| (k.to_string(), v.to_string())), ); if config.keys != expected { for (key, value) in &config.keys { if let Some(expected_value) = expected.get(key) { if value != expected_value { panic!( "Expected value {:?} for key {:?} but found {:?}.", expected_value, key, value ); } } else { panic!( "Unexpected key {:?} found in config with value {:?}.", key, value ); } } for (key, value) in &expected { if let Some(config_value) = config.keys.get(key) { if value != config_value { panic!( "Expected value {:?} for key {:?} but found {:?}.", value, key, config_value ); } } else { panic!( "Expected key {:?} not found in config with value {:?}.", key, value ); } } } assert_eq!( config.set_values("sets.strings").collect::>(), vec!["green", "red", "yellow"] ); assert_eq!( config.sub_keys("sets.strings", ""), vec!["green", "red", "yellow"] ); assert_eq!(config.sub_keys("sets", ".red"), vec!["string", "strings"]); } } ================================================ FILE: crates/utils/src/config/utils.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ net::{IpAddr, Ipv4Addr, Ipv6Addr}, path::PathBuf, str::FromStr, time::Duration, }; use mail_auth::{ IpLookupStrategy, common::crypto::{Algorithm, HashAlgorithm}, dkim::Canonicalization, }; use smtp_proto::MtPriority; use super::{Config, ConfigError, ConfigWarning, Rate}; impl Config { pub fn property(&mut self, key: impl AsKey) -> Option { let key = key.as_key(); #[cfg(debug_assertions)] self.keys_read.lock().insert(key.clone()); if let Some(value) = self.keys.get(&key) { match T::parse_value(value) { Ok(value) => Some(value), Err(err) => { self.new_parse_error(key, err); None } } } else { None } } pub fn property_or_default( &mut self, key: impl AsKey, default: &str, ) -> Option { let key = key.as_key(); #[cfg(debug_assertions)] self.keys_read.lock().insert(key.clone()); let value = match self.keys.get(&key) { Some(value) => value.as_str(), None => default, }; match T::parse_value(value) { Ok(value) => Some(value), Err(err) => { self.new_parse_error(key, err); None } } } pub fn property_or_else( &mut self, key: impl AsKey, or_else: impl AsKey, default: &str, ) -> Option { let key = key.as_key(); let value = match self.value_or_else(key.as_str(), or_else.clone()) { Some(value) => value, None => default, }; match T::parse_value(value) { Ok(value) => Some(value), Err(err) => { self.new_parse_error(key, err); None } } } pub fn property_require(&mut self, key: impl AsKey) -> Option { let key = key.as_key(); #[cfg(debug_assertions)] self.keys_read.lock().insert(key.clone()); if let Some(value) = self.keys.get(&key) { match T::parse_value(value) { Ok(value) => Some(value), Err(err) => { self.new_parse_error(key, err); None } } } else { self.new_parse_error(key, "Missing property"); None } } pub fn sub_keys(&self, prefix: impl AsKey, suffix: &str) -> Vec { let mut last_key = ""; let prefix = prefix.as_prefix(); self.keys .keys() .filter_map(move |key| { let key = key.strip_prefix(&prefix)?; let key = if !suffix.is_empty() { key.strip_suffix(suffix)? } else if let Some((key, _)) = key.split_once('.') { key } else { key }; if last_key != key { last_key = key; Some(key.to_string()) } else { None } }) .collect() } pub fn sub_keys_with_suffixes(&self, prefix: impl AsKey, suffixes: &[&str]) -> Vec { let mut last_key = ""; let prefix = prefix.as_prefix(); self.keys .keys() .filter_map(move |key| { let key = key.strip_prefix(&prefix)?; let key = suffixes .iter() .filter_map(|suffix| key.strip_suffix(suffix)) .next()?; if last_key != key { last_key = key; Some(key.to_string()) } else { None } }) .collect() } pub fn prefix<'x, 'y: 'x>(&'y self, prefix: impl AsKey) -> impl Iterator + 'x { let prefix = prefix.as_prefix(); self.keys .keys() .filter_map(move |key| key.strip_prefix(&prefix)) } pub fn set_values<'x, 'y: 'x>( &'y self, prefix: impl AsKey, ) -> impl Iterator + 'x { let prefix = prefix.as_prefix(); #[cfg(debug_assertions)] self.keys_read.lock().insert(prefix.clone()); self.keys .keys() .filter_map(move |key| key.strip_prefix(&prefix)) } pub fn properties(&mut self, prefix: impl AsKey) -> Vec<(String, T)> { let full_prefix = prefix.as_key(); let prefix = prefix.as_prefix(); let mut results = Vec::new(); #[cfg(debug_assertions)] self.keys_read.lock().insert(prefix.clone()); for (key, value) in &self.keys { if key.starts_with(&prefix) || key == &full_prefix { match T::parse_value(value) { Ok(value) => { results.push((key.to_string(), value)); } Err(error) => { self.errors .insert(key.to_string(), ConfigError::Parse { error }); } } } } results } pub fn value(&self, key: impl AsKey) -> Option<&str> { let key = key.as_key(); #[cfg(debug_assertions)] self.keys_read.lock().insert(key.clone()); self.keys.get(&key).map(|s| s.as_str()) } pub fn contains_key(&self, key: impl AsKey) -> bool { self.keys.contains_key(&key.as_key()) } pub fn value_require(&mut self, key: impl AsKey) -> Option<&str> { let key = key.as_key(); #[cfg(debug_assertions)] self.keys_read.lock().insert(key.clone()); if let Some(value) = self.keys.get(&key) { Some(value.as_str()) } else { self.errors.insert( key, ConfigError::Parse { error: "Missing property".to_string(), }, ); None } } pub fn value_require_non_empty(&mut self, key: impl AsKey) -> Option<&str> { let key = key.as_key(); #[cfg(debug_assertions)] self.keys_read.lock().insert(key.clone()); if let Some(value) = self.keys.get(&key).and_then(|v| { let v = v.trim(); if !v.is_empty() { Some(v) } else { None } }) { Some(value) } else { self.errors.insert( key, ConfigError::Parse { error: "Missing property".to_string(), }, ); None } } pub fn try_parse_value(&mut self, key: impl AsKey, value: &str) -> Option { match T::parse_value(value) { Ok(value) => Some(value), Err(error) => { self.errors .insert(key.as_key(), ConfigError::Parse { error }); None } } } pub fn value_or_else(&self, key: impl AsKey, or_else: impl AsKey) -> Option<&str> { let key = key.as_key(); #[cfg(debug_assertions)] { self.keys_read.lock().insert(key.clone()); self.keys_read.lock().insert(or_else.clone().as_key()); } self.keys .get(&key) .or_else(|| self.keys.get(&or_else.as_key())) .map(|s| s.as_str()) } pub fn values(&self, prefix: impl AsKey) -> impl Iterator { let full_prefix = prefix.as_key(); let prefix = prefix.as_prefix(); #[cfg(debug_assertions)] self.keys_read.lock().insert(prefix.clone()); self.keys.iter().filter_map(move |(key, value)| { if key.starts_with(&prefix) || key == &full_prefix { (key.as_str(), value.as_str()).into() } else { None } }) } pub fn iterate_prefix(&self, prefix: impl AsKey) -> impl Iterator { let prefix = prefix.as_prefix(); #[cfg(debug_assertions)] self.keys_read.lock().insert(prefix.clone()); self.keys .iter() .filter_map(move |(key, value)| Some((key.strip_prefix(&prefix)?, value.as_str()))) } pub fn values_or_else( &self, prefix: impl AsKey, or_else: impl AsKey, ) -> impl Iterator { let mut prefix = prefix.as_prefix(); #[cfg(debug_assertions)] { self.keys_read.lock().insert(prefix.clone()); self.keys_read.lock().insert(or_else.clone().as_prefix()); } self.values(if self.keys.keys().any(|k| k.starts_with(&prefix)) { prefix.truncate(prefix.len() - 1); prefix } else { or_else.as_key() }) } pub fn has_prefix(&self, prefix: impl AsKey) -> bool { let prefix = prefix.as_prefix(); self.keys.keys().any(|k| k.starts_with(&prefix)) } pub fn new_parse_error(&mut self, key: impl AsKey, details: impl Into) { self.errors.insert( key.as_key(), ConfigError::Parse { error: details.into(), }, ); } pub fn new_build_error(&mut self, key: impl AsKey, details: impl Into) { self.errors.insert( key.as_key(), ConfigError::Build { error: details.into(), }, ); } pub fn new_parse_warning(&mut self, key: impl AsKey, details: impl Into) { self.warnings.insert( key.as_key(), ConfigWarning::Parse { error: details.into(), }, ); } pub fn new_build_warning(&mut self, key: impl AsKey, details: impl Into) { self.warnings.insert( key.as_key(), ConfigWarning::Build { error: details.into(), }, ); } pub fn new_missing_property(&mut self, key: impl AsKey) { self.warnings.insert(key.as_key(), ConfigWarning::Missing); } #[cfg(debug_assertions)] pub fn warn_unread_keys(&mut self) { let mut keys = self.keys.clone(); for key in self.keys_read.lock().iter() { if let Some(base_key) = key.strip_suffix('.') { keys.remove(base_key); keys.retain(|k, _| !k.starts_with(key)); } else { keys.remove(key); } } for (key, value) in keys { self.warnings.insert(key, ConfigWarning::Unread { value }); } } } pub trait ParseValue: Sized { fn parse_value(value: &str) -> super::Result; } impl ParseValue for Option { fn parse_value(value: &str) -> super::Result { if !value.is_empty() && !value.eq_ignore_ascii_case("false") && !value.eq_ignore_ascii_case("disable") && !value.eq_ignore_ascii_case("disabled") && !value.eq_ignore_ascii_case("never") && !value.eq("0") { T::parse_value(value).map(Some) } else { Ok(None) } } } impl ParseValue for String { fn parse_value(value: &str) -> super::Result { Ok(value.to_string()) } } impl ParseValue for u64 { fn parse_value(value: &str) -> super::Result { value .parse() .map_err(|_| format!("Invalid integer value {:?}.", value,)) } } impl ParseValue for f64 { fn parse_value(value: &str) -> super::Result { value .parse() .map_err(|_| format!("Invalid floating point value {:?}.", value)) } } impl ParseValue for u16 { fn parse_value(value: &str) -> super::Result { value .parse() .map_err(|_| format!("Invalid integer value {:?}.", value)) } } impl ParseValue for i16 { fn parse_value(value: &str) -> super::Result { value .parse() .map_err(|_| format!("Invalid integer value {:?}.", value)) } } impl ParseValue for u32 { fn parse_value(value: &str) -> super::Result { value .parse() .map_err(|_| format!("Invalid integer value {:?}.", value)) } } impl ParseValue for i32 { fn parse_value(value: &str) -> super::Result { value .parse() .map_err(|_| format!("Invalid integer value {:?}.", value)) } } impl ParseValue for f32 { fn parse_value(value: &str) -> super::Result { value .parse() .map_err(|_| format!("Invalid floating point value {:?}.", value)) } } impl ParseValue for IpAddr { fn parse_value(value: &str) -> super::Result { value .parse() .map_err(|_| format!("Invalid IP address value {:?}.", value)) } } impl ParseValue for usize { fn parse_value(value: &str) -> super::Result { value .parse() .map_err(|_| format!("Invalid integer value {:?}.", value)) } } impl ParseValue for bool { fn parse_value(value: &str) -> super::Result { value .parse() .map_err(|_| format!("Invalid boolean value {:?}.", value)) } } impl ParseValue for Ipv4Addr { fn parse_value(value: &str) -> super::Result { value .parse() .map_err(|_| format!("Invalid IPv4 value {:?}.", value)) } } impl ParseValue for Ipv6Addr { fn parse_value(value: &str) -> super::Result { value .parse() .map_err(|_| format!("Invalid IPv6 value {:?}.", value)) } } impl ParseValue for PathBuf { fn parse_value(value: &str) -> super::Result { let path = PathBuf::from(value); if path.exists() { Ok(path) } else { Err(format!("Directory {} does not exist.", path.display())) } } } impl ParseValue for MtPriority { fn parse_value(value: &str) -> super::Result { match value.to_ascii_lowercase().as_str() { "mixer" => Ok(MtPriority::Mixer), "stanag4406" => Ok(MtPriority::Stanag4406), "nsep" => Ok(MtPriority::Nsep), _ => Err(format!("Invalid priority value {:?}.", value)), } } } impl ParseValue for Canonicalization { fn parse_value(value: &str) -> super::Result { match value { "relaxed" => Ok(Canonicalization::Relaxed), "simple" => Ok(Canonicalization::Simple), _ => Err(format!("Invalid canonicalization value {:?}.", value)), } } } impl ParseValue for IpLookupStrategy { fn parse_value(value: &str) -> super::Result { Ok(match value.to_lowercase().as_str() { "ipv4_only" => IpLookupStrategy::Ipv4Only, "ipv6_only" => IpLookupStrategy::Ipv6Only, //"ipv4_and_ipv6" => IpLookupStrategy::Ipv4AndIpv6, "ipv6_then_ipv4" => IpLookupStrategy::Ipv6thenIpv4, "ipv4_then_ipv6" => IpLookupStrategy::Ipv4thenIpv6, _ => return Err(format!("Invalid IP lookup strategy {:?}.", value)), }) } } impl ParseValue for Algorithm { fn parse_value(value: &str) -> super::Result { match value { "ed25519-sha256" | "ed25519-sha-256" => Ok(Algorithm::Ed25519Sha256), "rsa-sha-256" | "rsa-sha256" => Ok(Algorithm::RsaSha256), "rsa-sha-1" | "rsa-sha1" => Ok(Algorithm::RsaSha1), _ => Err(format!("Invalid algorithm {:?}.", value)), } } } impl ParseValue for HashAlgorithm { fn parse_value(value: &str) -> super::Result { match value { "sha256" | "sha-256" => Ok(HashAlgorithm::Sha256), "sha-1" | "sha1" => Ok(HashAlgorithm::Sha1), _ => Err(format!("Invalid hash algorithm {:?}.", value)), } } } impl ParseValue for Duration { fn parse_value(value: &str) -> super::Result { let mut digits = String::new(); let mut multiplier = String::new(); for ch in value.chars() { if ch.is_ascii_digit() { digits.push(ch); } else if !ch.is_ascii_whitespace() { multiplier.push(ch.to_ascii_lowercase()); } } let multiplier = match multiplier.as_str() { "d" => 24 * 60 * 60 * 1000, "h" => 60 * 60 * 1000, "m" => 60 * 1000, "s" => 1000, "ms" | "" => 1, _ => return Err(format!("Invalid duration value {:?}.", value)), }; digits .parse::() .ok() .and_then(|num| { if num > 0 { Some(Duration::from_millis(num * multiplier)) } else { None } }) .ok_or_else(|| format!("Invalid duration value {:?}.", value)) } } impl ParseValue for Rate { fn parse_value(value: &str) -> super::Result { if let Some((requests, period)) = value.split_once('/') { Ok(Rate { requests: requests .trim() .parse::() .ok() .and_then(|r| if r > 0 { Some(r) } else { None }) .ok_or_else(|| format!("Invalid rate value {:?}.", value))?, period: std::cmp::max(Duration::parse_value(period)?, Duration::from_secs(1)), }) } else if ["false", "none", "unlimited"].contains(&value) { Ok(Rate::default()) } else { Err(format!("Invalid rate value {:?}.", value)) } } } impl ParseValue for trc::Level { fn parse_value(value: &str) -> super::Result { trc::Level::from_str(value).map_err(|err| format!("Invalid log level: {err}")) } } impl ParseValue for trc::EventType { fn parse_value(value: &str) -> super::Result { trc::EventType::try_parse(value).ok_or_else(|| format!("Unknown event type: {value}")) } } impl ParseValue for () { fn parse_value(_: &str) -> super::Result { Ok(()) } } pub trait AsKey: Clone { fn as_key(&self) -> String; fn as_prefix(&self) -> String; } impl AsKey for String { fn as_key(&self) -> String { self.to_string() } fn as_prefix(&self) -> String { format!("{self}.") } } impl AsKey for &String { fn as_key(&self) -> String { self.to_string() } fn as_prefix(&self) -> String { format!("{self}.") } } impl AsKey for &str { fn as_key(&self) -> String { self.to_string() } fn as_prefix(&self) -> String { format!("{self}.") } } impl AsKey for (A, B) where A: AsRef + Clone, B: AsRef + Clone, { fn as_key(&self) -> String { format!("{}.{}", self.0.as_ref(), self.1.as_ref(),) } fn as_prefix(&self) -> String { format!("{}.{}.", self.0.as_ref(), self.1.as_ref(),) } } impl AsKey for (A, B, C) where A: AsRef + Clone, B: AsRef + Clone, C: AsRef + Clone, { fn as_key(&self) -> String { format!( "{}.{}.{}", self.0.as_ref(), self.1.as_ref(), self.2.as_ref() ) } fn as_prefix(&self) -> String { format!( "{}.{}.{}.", self.0.as_ref(), self.1.as_ref(), self.2.as_ref() ) } } impl AsKey for (A, B, C, D) where A: AsRef + Clone, B: AsRef + Clone, C: AsRef + Clone, D: AsRef + Clone, { fn as_key(&self) -> String { format!( "{}.{}.{}.{}", self.0.as_ref(), self.1.as_ref(), self.2.as_ref(), self.3.as_ref() ) } fn as_prefix(&self) -> String { format!( "{}.{}.{}.{}.", self.0.as_ref(), self.1.as_ref(), self.2.as_ref(), self.3.as_ref() ) } } impl AsKey for (A, B, C, D, E) where A: AsRef + Clone, B: AsRef + Clone, C: AsRef + Clone, D: AsRef + Clone, E: AsRef + Clone, { fn as_key(&self) -> String { format!( "{}.{}.{}.{}.{}", self.0.as_ref(), self.1.as_ref(), self.2.as_ref(), self.3.as_ref(), self.4.as_ref() ) } fn as_prefix(&self) -> String { format!( "{}.{}.{}.{}.{}.", self.0.as_ref(), self.1.as_ref(), self.2.as_ref(), self.3.as_ref(), self.4.as_ref() ) } } #[cfg(test)] mod tests { use std::net::IpAddr; use crate::config::Config; #[test] fn toml_utils() { let toml = r#" [queues."z"] retry = [0, 1, 15, 60, 90] value = "hi" [queues."x"] retry = [3, 60] value = "hi 2" [queues.a] retry = [1, 2, 3, 4] value = "hi 3" [servers."my relay"] hostname = "mx.example.org" [[servers."my relay".transaction.auth.limits]] idle = 10 [[servers."my relay".transaction.auth.limits]] idle = 20 [servers."submissions"] hostname = "submit.example.org" ip = "a:b::1:1" "#; let mut config = Config::default(); config.parse(toml).unwrap(); assert_eq!(config.sub_keys("queues", ""), ["a", "x", "z"]); assert_eq!(config.sub_keys("servers", ""), ["my relay", "submissions"]); assert_eq!( config.sub_keys("queues.z.retry", ""), ["0000", "0001", "0002", "0003", "0004"] ); assert_eq!( config .property::("servers.my relay.transaction.auth.limits.0001.idle") .unwrap(), 20 ); assert_eq!( config .property::(("servers", "submissions", "ip")) .unwrap(), "a:b::1:1".parse::().unwrap() ); } } ================================================ FILE: crates/utils/src/glob.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::borrow::Cow; use ahash::{AHashMap, AHashSet}; use serde::Deserialize; #[derive(Debug, Clone, PartialEq, Eq)] pub struct GlobPattern { pattern: Vec, to_lower: bool, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum PatternChar { WildcardMany { num: usize, match_pos: usize }, WildcardSingle { match_pos: usize }, Char { char: char, match_pos: usize }, } impl GlobPattern { pub fn compile(pattern: &str, to_lower: bool) -> Self { let mut chars = Vec::new(); let mut is_escaped = false; let mut str = pattern.chars().peekable(); while let Some(char) = str.next() { match char { '*' if !is_escaped => { let mut num = 1; while let Some('*') = str.peek() { num += 1; str.next(); } chars.push(PatternChar::WildcardMany { num, match_pos: 0 }); } '?' if !is_escaped => { chars.push(PatternChar::WildcardSingle { match_pos: 0 }); } '\\' if !is_escaped => { is_escaped = true; continue; } _ => { if is_escaped { is_escaped = false; } if to_lower && char.is_uppercase() { for char in char.to_lowercase() { chars.push(PatternChar::Char { char, match_pos: 0 }); } } else { chars.push(PatternChar::Char { char, match_pos: 0 }); } } } } GlobPattern { pattern: chars, to_lower, } } pub fn try_compile(pattern: &str, to_lower: bool) -> Result { // Detect if the key is a glob pattern let mut last_ch = '\0'; let mut has_escape = false; let mut is_glob = false; for ch in pattern.chars() { match ch { '\\' => { has_escape = true; } '*' | '?' if last_ch != '\\' => { is_glob = true; } _ => {} } last_ch = ch; } if is_glob { Ok(GlobPattern::compile(pattern, to_lower)) } else { Err(if has_escape { pattern.replace('\\', "") } else { pattern.to_string() }) } } // Credits: Algorithm ported from https://research.swtch.com/glob pub fn matches(&self, value: &str) -> bool { let value = if self.to_lower { value.to_lowercase().chars().collect::>() } else { value.chars().collect::>() }; let mut px = 0; let mut nx = 0; let mut next_px = 0; let mut next_nx = 0; while px < self.pattern.len() || nx < value.len() { match self.pattern.get(px) { Some(PatternChar::Char { char, .. }) => { if matches!(value.get(nx), Some(nc) if nc == char ) { px += 1; nx += 1; continue; } } Some(PatternChar::WildcardSingle { .. }) => { if nx < value.len() { px += 1; nx += 1; continue; } } Some(PatternChar::WildcardMany { .. }) => { next_px = px; next_nx = nx + 1; px += 1; continue; } _ => (), } if 0 < next_nx && next_nx <= value.len() { px = next_px; nx = next_nx; continue; } return false; } true } } #[derive(Debug, Clone, Default)] pub struct GlobSet { entries: AHashSet, patterns: Vec, } #[derive(Debug, Clone)] pub struct GlobMap { entries: AHashMap, patterns: Vec<(GlobPattern, V)>, } impl GlobSet { pub fn new() -> Self { GlobSet::default() } pub fn insert(&mut self, pattern: &str) { match GlobPattern::try_compile(pattern, false) { Ok(glob) => { self.patterns.push(glob); } Err(entry) => { self.entries.insert(entry); } } } pub fn contains(&self, key: &str) -> bool { self.entries.contains(key) || self.patterns.iter().any(|pattern| pattern.matches(key)) } } impl GlobMap { pub fn new() -> Self { GlobMap { entries: AHashMap::new(), patterns: Vec::new(), } } pub fn insert(&mut self, pattern: &str, value: V) { match GlobPattern::try_compile(pattern, false) { Ok(glob) => { self.patterns.push((glob, value)); } Err(entry) => { self.entries.insert(entry, value); } } } pub fn get(&self, key: &str) -> Option<&V> { self.entries.get(key).or_else(|| { self.patterns .iter() .find_map(|(pattern, value)| pattern.matches(key).then_some(value)) }) } } impl Default for GlobMap { fn default() -> Self { GlobMap::new() } } impl<'de> Deserialize<'de> for GlobPattern { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { Ok(GlobPattern::compile( >::deserialize(deserializer)?.as_ref(), true, )) } } ================================================ FILE: crates/utils/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod bimap; pub mod cache; pub mod chained_bytes; pub mod cheeky_hash; pub mod codec; pub mod config; pub mod glob; pub mod map; pub mod snowflake; pub mod template; pub mod topological; pub mod url_params; use compact_str::ToCompactString; use futures::StreamExt; use reqwest::Response; use rustls::{ ClientConfig, RootCertStore, SignatureScheme, client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}, }; use rustls_pki_types::TrustAnchor; use std::sync::Arc; pub trait HttpLimitResponse: Sync + Send { fn bytes_with_limit( self, limit: usize, ) -> impl std::future::Future>>> + Send; } impl HttpLimitResponse for Response { async fn bytes_with_limit(self, limit: usize) -> reqwest::Result>> { if self .content_length() .is_some_and(|len| len as usize > limit) { return Ok(None); } let mut bytes = Vec::with_capacity(std::cmp::min(limit, 1024)); let mut stream = self.bytes_stream(); while let Some(chunk) = stream.next().await { let chunk = chunk?; if bytes.len() + chunk.len() > limit { return Ok(None); } bytes.extend_from_slice(&chunk); } Ok(Some(bytes)) } } pub trait UnwrapFailure { fn failed(self, action: &str) -> T; } impl UnwrapFailure for Option { fn failed(self, message: &str) -> T { match self { Some(result) => result, None => { trc::event!( Server(trc::ServerEvent::StartupError), Details = message.to_compact_string() ); eprintln!("{message}"); std::process::exit(1); } } } } impl UnwrapFailure for Result { fn failed(self, message: &str) -> T { match self { Ok(result) => result, Err(err) => { trc::event!( Server(trc::ServerEvent::StartupError), Details = message.to_compact_string(), Reason = err.to_compact_string() ); #[cfg(feature = "test_mode")] panic!("{message}: {err}"); #[cfg(not(feature = "test_mode"))] { eprintln!("{message}: {err}"); std::process::exit(1); } } } } } pub fn failed(message: &str) -> ! { trc::event!( Server(trc::ServerEvent::StartupError), Details = message.to_compact_string(), ); eprintln!("{message}"); std::process::exit(1); } pub async fn wait_for_shutdown() { #[cfg(not(target_env = "msvc"))] let signal = { use tokio::signal::unix::{SignalKind, signal}; let mut h_term = signal(SignalKind::terminate()).failed("start signal handler"); let mut h_int = signal(SignalKind::interrupt()).failed("start signal handler"); tokio::select! { _ = h_term.recv() => "SIGTERM", _ = h_int.recv() => "SIGINT", } }; #[cfg(target_env = "msvc")] let signal = { match tokio::signal::ctrl_c().await { Ok(()) => "SIGINT", Err(err) => { trc::event!( Server(trc::ServerEvent::ThreadError), Details = "Unable to listen for shutdown signal", Reason = err.to_string(), ); "Error" } } }; trc::event!(Server(trc::ServerEvent::Shutdown), CausedBy = signal); } pub fn rustls_client_config(allow_invalid_certs: bool) -> ClientConfig { let config = ClientConfig::builder(); if !allow_invalid_certs { let mut root_cert_store = RootCertStore::empty(); root_cert_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().map(|ta| TrustAnchor { subject: ta.subject.clone(), subject_public_key_info: ta.subject_public_key_info.clone(), name_constraints: ta.name_constraints.clone(), })); config .with_root_certificates(root_cert_store) .with_no_client_auth() } else { config .dangerous() .with_custom_certificate_verifier(Arc::new(DummyVerifier {})) .with_no_client_auth() } } pub trait DomainPart { fn to_lowercase_domain(&self) -> String; fn domain_part(&self) -> &str; fn try_domain_part(&self) -> Option<&str>; fn try_local_part(&self) -> Option<&str>; } impl> DomainPart for T { fn to_lowercase_domain(&self) -> String { let address = self.as_ref(); if let Some((local, domain)) = address.rsplit_once('@') { let mut address = String::with_capacity(address.len()); address.push_str(local); address.push('@'); for ch in domain.chars() { for ch in ch.to_lowercase() { address.push(ch); } } address } else { address.to_string() } } #[inline(always)] fn try_domain_part(&self) -> Option<&str> { self.as_ref().rsplit_once('@').map(|(_, d)| d) } #[inline(always)] fn try_local_part(&self) -> Option<&str> { self.as_ref().rsplit_once('@').map(|(l, _)| l) } #[inline(always)] fn domain_part(&self) -> &str { self.as_ref() .rsplit_once('@') .map(|(_, d)| d) .unwrap_or_default() } } #[derive(Debug)] struct DummyVerifier; impl ServerCertVerifier for DummyVerifier { fn verify_server_cert( &self, _end_entity: &rustls_pki_types::CertificateDer<'_>, _intermediates: &[rustls_pki_types::CertificateDer<'_>], _server_name: &rustls_pki_types::ServerName<'_>, _ocsp_response: &[u8], _now: rustls_pki_types::UnixTime, ) -> Result { Ok(ServerCertVerified::assertion()) } fn verify_tls12_signature( &self, _message: &[u8], _cert: &rustls_pki_types::CertificateDer<'_>, _dss: &rustls::DigitallySignedStruct, ) -> Result { Ok(HandshakeSignatureValid::assertion()) } fn verify_tls13_signature( &self, _message: &[u8], _cert: &rustls_pki_types::CertificateDer<'_>, _dss: &rustls::DigitallySignedStruct, ) -> Result { Ok(HandshakeSignatureValid::assertion()) } fn supported_verify_schemes(&self) -> Vec { vec![ SignatureScheme::RSA_PKCS1_SHA1, SignatureScheme::ECDSA_SHA1_Legacy, SignatureScheme::RSA_PKCS1_SHA256, SignatureScheme::ECDSA_NISTP256_SHA256, SignatureScheme::RSA_PKCS1_SHA384, SignatureScheme::ECDSA_NISTP384_SHA384, SignatureScheme::RSA_PKCS1_SHA512, SignatureScheme::ECDSA_NISTP521_SHA512, SignatureScheme::RSA_PSS_SHA256, SignatureScheme::RSA_PSS_SHA384, SignatureScheme::RSA_PSS_SHA512, SignatureScheme::ED25519, SignatureScheme::ED448, ] } } // Basic email sanitizer pub fn sanitize_email(email: &str) -> Option { let mut result = String::with_capacity(email.len()); let mut found_local = false; let mut found_domain = false; let mut last_ch = char::from(0); for ch in email.chars() { if !ch.is_whitespace() { if ch == '@' { if !result.is_empty() && !found_local { found_local = true; } else { return None; } } else if ch == '.' { if !(last_ch.is_alphanumeric() || last_ch == '-' || last_ch == '_') { return None; } else if found_local { found_domain = true; } } last_ch = ch; for ch in ch.to_lowercase() { result.push(ch); } } } if found_domain && last_ch != '.' && psl::domain(result.as_bytes()).is_some_and(|d| d.suffix().typ().is_some()) { Some(result) } else { None } } ================================================ FILE: crates/utils/src/map/bitmap.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::ops::Deref; #[derive( Debug, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, serde::Serialize, serde::Deserialize, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash, )] #[rkyv(compare(PartialEq), derive(Debug))] #[repr(transparent)] pub struct Bitmap { pub bitmap: u64, #[serde(skip)] #[rkyv(omit_bounds)] _state: std::marker::PhantomData, } pub trait BitmapItem: From + Into + Sized + Copy { fn max() -> u64; fn is_valid(&self) -> bool; } pub trait BitPop { fn bit_push(&mut self, item: u8); fn bit_pop(&mut self) -> Option; } impl Bitmap { pub fn new() -> Self { Self::default() } #[inline(always)] pub fn all() -> Self { Self { bitmap: u64::MAX >> (64 - T::max()), _state: std::marker::PhantomData, } } #[inline(always)] pub fn union(&mut self, items: &Bitmap) { self.bitmap |= items.bitmap; } #[inline(always)] pub fn union_raw(&mut self, items: impl Into) { self.bitmap |= items.into(); } #[inline(always)] pub fn intersection(&mut self, items: &Bitmap) { self.bitmap &= items.bitmap; } #[inline(always)] pub fn insert(&mut self, item: T) { debug_assert!(item.is_valid()); self.bitmap |= 1 << item.into(); } pub fn insert_many(&mut self, items: impl IntoIterator) { for item in items.into_iter() { self.insert(item); } } #[inline(always)] pub fn with_item(mut self, item: T) -> Self { self.insert(item); self } #[inline(always)] pub fn remove(&mut self, item: T) { debug_assert!(item.is_valid()); self.bitmap ^= 1 << item.into(); } #[inline(always)] pub fn pop(&mut self) -> Option { if self.bitmap != 0 { let item = 63 - self.bitmap.leading_zeros(); self.bitmap ^= 1 << item; Some((item as u64).into()) } else { None } } #[inline(always)] pub fn contains(&self, item: T) -> bool { self.bitmap & (1 << item.into()) != 0 } #[inline(always)] pub fn contains_any(&self, items: impl Iterator) -> bool { for item in items { if self.bitmap & (1 << item.into()) != 0 { return true; } } false } #[inline(always)] pub fn contains_all(&self, items: impl Iterator) -> bool { if !self.is_empty() { for item in items { if self.bitmap & (1 << item.into()) == 0 { return false; } } true } else { false } } #[inline(always)] pub fn is_empty(&self) -> bool { self.bitmap == 0 } #[inline(always)] pub fn clear(&mut self) -> Self { let bitmap = self.bitmap; self.bitmap = 0; Bitmap { bitmap, _state: std::marker::PhantomData, } } pub fn into_inner(self) -> u64 { self.bitmap } } impl BitPop for u32 { fn bit_push(&mut self, item: u8) { *self |= 1 << item; } fn bit_pop(&mut self) -> Option { if *self != 0 { let item = 31 - self.leading_zeros(); *self ^= 1 << item; Some(item as u8) } else { None } } } impl BitPop for u64 { fn bit_push(&mut self, item: u8) { *self |= 1 << item; } fn bit_pop(&mut self) -> Option { if *self != 0 { let item = 63 - self.leading_zeros(); *self ^= 1 << item; Some(item as u8) } else { None } } } impl From> for Bitmap { fn from(value: ArchivedBitmap) -> Self { Self { bitmap: value.bitmap.into(), _state: std::marker::PhantomData, } } } impl From<&ArchivedBitmap> for Bitmap { fn from(value: &ArchivedBitmap) -> Self { Self { bitmap: value.bitmap.into(), _state: std::marker::PhantomData, } } } impl From for Bitmap { fn from(value: u64) -> Self { Self { bitmap: value, _state: std::marker::PhantomData, } } } impl AsRef for Bitmap { fn as_ref(&self) -> &u64 { &self.bitmap } } impl Deref for Bitmap { type Target = u64; fn deref(&self) -> &Self::Target { &self.bitmap } } impl From> for u64 { fn from(value: Bitmap) -> Self { value.bitmap } } impl Iterator for Bitmap { type Item = T; fn next(&mut self) -> Option { if self.bitmap != 0 { let item = 63 - self.bitmap.leading_zeros(); self.bitmap ^= 1 << item; Some((item as u64).into()) } else { None } } } impl From> for Bitmap { fn from(values: Vec) -> Self { let mut bitmap = Bitmap::default(); for value in values { if value.is_valid() { bitmap.insert(value); } } bitmap } } impl FromIterator for Bitmap { fn from_iter>(iter: U) -> Self { let mut bitmap = Bitmap::new(); for value in iter { if value.is_valid() { bitmap.insert(value); } } bitmap } } impl From<&Vec> for Bitmap { fn from(values: &Vec) -> Self { let mut bitmap = Bitmap::default(); for value in values { if value.is_valid() { bitmap.insert(*value); } } bitmap } } impl From for Bitmap { fn from(value: T) -> Self { let mut bitmap = Bitmap::default(); bitmap.insert(value); bitmap } } impl From> for Vec { fn from(values: Bitmap) -> Self { let mut list = Vec::new(); for item in values { list.push(item); } list } } impl Default for Bitmap { fn default() -> Self { Bitmap { bitmap: 0, _state: std::marker::PhantomData, } } } ================================================ FILE: crates/utils/src/map/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod bitmap; pub mod mutex_map; pub mod vec_map; ================================================ FILE: crates/utils/src/map/mutex_map.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use core::hash::Hash; use std::hash::Hasher; use ahash::AHasher; use tokio::sync::{Mutex, MutexGuard}; pub struct MutexMap { map: Box<[Mutex]>, mask: u64, hasher: AHasher, } pub struct MutexMapLockError; pub type Result = std::result::Result; #[allow(clippy::mutex_atomic)] impl MutexMap { pub fn with_capacity(size: usize) -> MutexMap { let size = size.next_power_of_two(); MutexMap { map: (0..size) .map(|_| T::default().into()) .collect::>>() .into_boxed_slice(), mask: (size - 1) as u64, hasher: AHasher::default(), } } pub async fn lock(&self, key: U) -> MutexGuard<'_, T> where U: Into + Copy, { let hash = key.into() & self.mask; self.map[hash as usize].lock().await } /*pub async fn try_lock(&self, key: U, timeout: Duration) -> Option> where U: Into + Copy, { let hash = key.into() & self.mask; self.map[hash as usize].try_lock(timeout).await }*/ pub async fn lock_hash(&self, key: U) -> MutexGuard<'_, T> where U: Hash, { let mut hasher = self.hasher.clone(); key.hash(&mut hasher); let hash = hasher.finish() & self.mask; self.map[hash as usize].lock().await } /*pub async fn try_lock_hash(&self, key: U, timeout: Duration) -> Option> where U: Hash, { let mut hasher = self.hasher.clone(); key.hash(&mut hasher); let hash = hasher.finish() & self.mask; self.map[hash as usize].try_lock_for(timeout).await }*/ } ================================================ FILE: crates/utils/src/map/vec_map.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use rkyv::Archive; use serde::{Deserialize, Serialize, ser::SerializeMap}; use std::{borrow::Borrow, cmp::Ordering, fmt, hash::Hash}; // A map implemented using vectors // used for small datasets of less than 20 items // and when deserializing from JSON #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] pub struct VecMap { inner: Vec>, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq, Hash)] pub struct KeyValue { key: K, value: V, } impl Default for VecMap { fn default() -> Self { VecMap { inner: Vec::new() } } } impl VecMap { pub fn new() -> Self { Self::default() } pub fn with_capacity(capacity: usize) -> Self { Self { inner: Vec::with_capacity(capacity), } } #[inline(always)] pub fn set(&mut self, key: K, value: V) -> bool { if let Some(kv) = self.inner.iter_mut().find(|kv| kv.key == key) { kv.value = value; false } else { self.inner.push(KeyValue { key, value }); true } } #[inline(always)] pub fn append(&mut self, key: K, value: V) { self.inner.push(KeyValue { key, value }); } #[inline(always)] pub fn with_append(mut self, key: K, value: V) -> Self { self.append(key, value); self } #[inline(always)] pub fn insert(&mut self, idx: usize, key: K, value: V) { self.inner.insert(idx, KeyValue { key, value }); } #[inline(always)] pub fn get(&self, key: &Q) -> Option<&V> where K: Borrow + PartialEq, { self.inner.iter().find_map(|kv| { if &kv.key == key { Some(&kv.value) } else { None } }) } #[inline(always)] pub fn get_mut(&mut self, key: &K) -> Option<&mut V> { self.inner.iter_mut().find_map(|kv| { if &kv.key == key { Some(&mut kv.value) } else { None } }) } #[inline(always)] pub fn contains_key(&self, key: &K) -> bool { self.inner.iter().any(|kv| kv.key == *key) } #[inline(always)] pub fn remove(&mut self, key: &Q) -> Option where K: Borrow + PartialEq, { self.inner .iter() .position(|kv| kv.key == *key) .map(|pos| self.inner.remove(pos).value) } #[inline(always)] pub fn remove_all(&mut self, key: &K) { self.inner.retain(|kv| kv.key != *key); } #[inline(always)] pub fn remove_entry(&mut self, key: &K) -> Option<(K, V)> { self.inner.iter().position(|k| &k.key == key).map(|pos| { let kv = self.inner.remove(pos); (kv.key, kv.value) }) } #[inline(always)] pub fn swap_remove(&mut self, index: usize) -> V { self.inner.swap_remove(index).value } #[inline(always)] pub fn is_empty(&self) -> bool { self.inner.is_empty() } #[inline(always)] pub fn len(&self) -> usize { self.inner.len() } #[inline(always)] pub fn clear(&mut self) { self.inner.clear(); } #[inline(always)] pub fn iter(&self) -> impl Iterator { self.inner.iter().map(|kv| (&kv.key, &kv.value)) } #[inline(always)] pub fn iter_by_key<'x, 'y: 'x>(&'x self, key: &'y K) -> impl Iterator + 'x { self.inner.iter().filter_map(move |kv| { if &kv.key == key { Some(&kv.value) } else { None } }) } #[inline(always)] pub fn iter_mut(&mut self) -> impl Iterator { self.inner.iter_mut().map(|kv| (&mut kv.key, &mut kv.value)) } #[inline(always)] pub fn iter_mut_by_key<'x, 'y: 'x>( &'x mut self, key: &'y K, ) -> impl Iterator + 'x { self.inner.iter_mut().filter_map(move |kv| { if &kv.key == key { Some(&mut kv.value) } else { None } }) } #[inline(always)] pub fn keys(&self) -> impl Iterator { self.inner.iter().map(|kv| &kv.key) } #[inline(always)] pub fn values(&self) -> impl Iterator { self.inner.iter().map(|kv| &kv.value) } #[inline(always)] pub fn values_mut(&mut self) -> impl Iterator { self.inner.iter_mut().map(|kv| &mut kv.value) } pub fn get_mut_or_insert_with(&mut self, key: K, fnc: impl FnOnce() -> V) -> &mut V { if let Some(pos) = self.inner.iter().position(|kv| kv.key == key) { &mut self.inner[pos].value } else { self.inner.push(KeyValue { key, value: fnc() }); &mut self.inner.last_mut().unwrap().value } } pub fn with_key_value(mut self, key: K, value: V) -> Self { self.append(key, value); self } pub fn sort_unstable(&mut self) where K: Ord, V: Ord, { self.inner.sort_unstable_by(|a, b| match a.key.cmp(&b.key) { Ordering::Equal => a.value.cmp(&b.value), cmp => cmp, }); } } impl VecMap { pub fn get_mut_or_insert(&mut self, key: K) -> &mut V { if let Some(pos) = self.inner.iter().position(|kv| kv.key == key) { &mut self.inner[pos].value } else { self.inner.push(KeyValue { key, value: V::default(), }); &mut self.inner.last_mut().unwrap().value } } } impl ArchivedVecMap { pub fn len(&self) -> usize { self.inner.len() } pub fn is_empty(&self) -> bool { self.inner.is_empty() } #[inline(always)] pub fn iter( &self, ) -> impl Iterator< Item = ( &::Archived, &::Archived, ), > { self.inner.iter().map(|kv| (&kv.key, &kv.value)) } } impl IntoIterator for VecMap { type Item = (K, V); type IntoIter = std::iter::Map>, fn(KeyValue) -> (K, V)>; fn into_iter(self) -> Self::IntoIter { self.inner.into_iter().map(|kv| (kv.key, kv.value)) } } impl<'x, K: Eq + PartialEq, V> IntoIterator for &'x VecMap { type Item = (&'x K, &'x V); type IntoIter = std::iter::Map< std::slice::Iter<'x, KeyValue>, fn(&'x KeyValue) -> (&'x K, &'x V), >; fn into_iter(self) -> Self::IntoIter { self.inner.iter().map(|kv| (&kv.key, &kv.value)) } } impl Hash for VecMap where K: Eq + PartialEq + Hash, V: Hash, { fn hash(&self, state: &mut H) { self.inner.hash(state); } } impl FromIterator<(K, V)> for VecMap { fn from_iter(iter: T) -> Self where T: IntoIterator, { let mut map = VecMap::new(); for (k, v) in iter { map.append(k, v); } map } } struct VecMapVisitor { phantom: std::marker::PhantomData<(K, V)>, } impl<'de, K: Eq + PartialEq + Deserialize<'de>, V: Deserialize<'de>> serde::de::Visitor<'de> for VecMapVisitor { type Value = VecMap; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("a valid map") } fn visit_map(self, mut map: A) -> Result where A: serde::de::MapAccess<'de>, { // Duplicates are not checked during deserialization let mut vec_map = VecMap::new(); while let Some(key) = map.next_key::()? { vec_map.append(key, map.next_value()?); } Ok(vec_map) } } impl<'de, K: Eq + PartialEq + Deserialize<'de>, V: Deserialize<'de>> Deserialize<'de> for VecMap { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { deserializer.deserialize_map(VecMapVisitor { phantom: std::marker::PhantomData, }) } } impl Serialize for VecMap { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { let mut map = serializer.serialize_map(self.len().into())?; for (key, value) in self { map.serialize_entry(key, value)? } map.end() } } ================================================ FILE: crates/utils/src/snowflake.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ sync::atomic::{AtomicU64, Ordering}, time::{Duration, SystemTime}, }; #[derive(Debug)] pub struct SnowflakeIdGenerator { epoch: SystemTime, node_id: u64, sequence: AtomicU64, } const SEQUENCE_LEN: u64 = 12; const NODE_ID_LEN: u64 = 9; const SEQUENCE_MASK: u64 = (1 << SEQUENCE_LEN) - 1; const NODE_ID_MASK: u64 = (1 << NODE_ID_LEN) - 1; const DEFAULT_EPOCH: u64 = 1632280000; // 52 years after UNIX_EPOCH //const DEFAULT_EPOCH_MS: u128 = (DEFAULT_EPOCH as u128) * 1000; // 52 years after UNIX_EPOCH in milliseconds /* ID characteristics: - 43 bits for milliseconds since January 1st, 2022: 2^43 / (1000 * 60 * 60 * 24 * 365) = 278.92 years (from year 2022 until 2300) - 9 bits for a node id: 2^9 = 512 nodes - 12 bits for a sequence number: 2^12 = 4096 ids per millisecond */ impl SnowflakeIdGenerator { pub fn new() -> Self { Self::with_node_id(rand::random::()) } pub fn from_duration(period: Duration) -> Option { (SystemTime::UNIX_EPOCH + Duration::from_secs(DEFAULT_EPOCH)) .elapsed() .ok() .and_then(|elapsed| elapsed.checked_sub(period)) .map(|elapsed| (elapsed.as_millis() as u64) << (SEQUENCE_LEN + NODE_ID_LEN)) } pub fn from_timestamp(timestamp: u64) -> Option { SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .ok() .and_then(|now| now.as_secs().checked_sub(timestamp)) .and_then(|diff| Self::from_duration(Duration::from_secs(diff))) } pub fn from_sequence_and_node_id(sequence: u64, node_id: Option) -> Option { let node_id = node_id.unwrap_or_else(rand::random::); let sequence = sequence & SEQUENCE_MASK; (SystemTime::UNIX_EPOCH + Duration::from_secs(DEFAULT_EPOCH)) .elapsed() .ok() .map(|elapsed| { ((elapsed.as_millis() as u64) << (SEQUENCE_LEN + NODE_ID_LEN)) | (sequence << NODE_ID_LEN) | (node_id & NODE_ID_MASK) }) } pub fn to_timestamp(id: u64) -> u64 { (id >> (SEQUENCE_LEN + NODE_ID_LEN)) / 1000 + DEFAULT_EPOCH } pub fn with_node_id(node_id: u64) -> Self { Self { epoch: SystemTime::UNIX_EPOCH + Duration::from_secs(DEFAULT_EPOCH), // 52 years after UNIX_EPOCH node_id, sequence: 0.into(), } } #[inline(always)] pub fn past_id(&self, period: Duration) -> Option { self.epoch .elapsed() .ok() .and_then(|elapsed| elapsed.checked_sub(period)) .map(|elapsed| (elapsed.as_millis() as u64) << (SEQUENCE_LEN + NODE_ID_LEN)) } pub fn is_valid(&self) -> bool { self.epoch.elapsed().is_ok() } #[inline(always)] pub fn generate(&self) -> u64 { let elapsed = self .epoch .elapsed() .map(|e| e.as_millis()) .unwrap_or_default() as u64; let sequence = self.sequence.fetch_add(1, Ordering::Relaxed) & SEQUENCE_MASK; (elapsed << (SEQUENCE_LEN + NODE_ID_LEN)) | (sequence << NODE_ID_LEN) | (self.node_id & NODE_ID_MASK) } } impl Default for SnowflakeIdGenerator { fn default() -> Self { Self::new() } } impl Clone for SnowflakeIdGenerator { fn clone(&self) -> Self { Self { epoch: self.epoch, node_id: self.node_id, sequence: 0.into(), } } } ================================================ FILE: crates/utils/src/suffixlist.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::io::Read; use ahash::AHashSet; use mail_auth::flate2::read::GzDecoder; use crate::config::Config; #[derive(Debug, Clone, Default)] pub struct PublicSuffix { pub suffixes: AHashSet, pub exceptions: AHashSet, pub wildcards: Vec, } #[derive(PartialEq, Eq, Clone, Copy)] pub enum DomainPart { Sld, Tld, Host, } impl PublicSuffix { pub fn contains(&self, suffix: &str) -> bool { self.suffixes.contains(suffix) || (!self.exceptions.contains(suffix) && self.wildcards.iter().any(|w| suffix.ends_with(w))) } pub fn domain_part(&self, domain: &str, part: DomainPart) -> Option { let d = domain.trim().to_lowercase(); let mut seen_dot = false; for (pos, ch) in d.as_bytes().iter().enumerate().rev() { if *ch == b'.' { if seen_dot { let maybe_domain = std::str::from_utf8(&d.as_bytes()[pos + 1..]).unwrap_or_default(); if !self.contains(maybe_domain) { return if part == DomainPart::Sld { maybe_domain } else { std::str::from_utf8(&d.as_bytes()[..pos]).unwrap_or_default() } .to_string() .into(); } } else if part == DomainPart::Tld { return std::str::from_utf8(&d.as_bytes()[pos + 1..]) .unwrap_or_default() .to_string() .into(); } else { seen_dot = true; } } } if seen_dot { if part == DomainPart::Sld { d.into() } else { None } } else if part == DomainPart::Host { d.into() } else { None } } } impl From<&str> for PublicSuffix { fn from(list: &str) -> Self { let mut ps = PublicSuffix::default(); for line in list.lines() { let line = line.trim().to_lowercase(); if !line.starts_with("//") { if let Some(domain) = line.strip_prefix('*') { ps.wildcards.push(domain.to_string()); } else if let Some(domain) = line.strip_prefix('!') { ps.exceptions.insert(domain.to_string()); } else { ps.suffixes.insert(line.to_string()); } } } ps.suffixes.insert("onion".to_string()); ps } } impl PublicSuffix { #[allow(unused_variables)] pub async fn parse(config: &mut Config, key: &str) -> PublicSuffix { let mut values = config .values(key) .map(|(_, s)| s.to_string()) .collect::>(); if values.is_empty() { values = vec![ "https://publicsuffix.org/list/public_suffix_list.dat".to_string(), "https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat" .to_string(), ] } for (idx, value) in values.into_iter().enumerate() { let bytes = if value.starts_with("https://") || value.starts_with("http://") { let result = match reqwest::get(&value).await { Ok(r) => { if r.status().is_success() { r.bytes().await } else { config.new_build_warning( format!("{value}.{idx}"), format!( "Failed to fetch public suffixes from {value:?}: Status {status}", value = value, status = r.status() ), ); continue; } } Err(err) => Err(err), }; match result { Ok(bytes) => bytes.to_vec(), Err(err) => { config.new_build_warning( format!("{value}.{idx}"), format!("Failed to fetch public suffixes from {value:?}: {err}",), ); continue; } } } else if let Some(filename) = value.strip_prefix("file://") { match std::fs::read(filename) { Ok(bytes) => bytes, Err(err) => { config.new_build_warning( format!("{value}.{idx}"), format!("Failed to read public suffixes from {value:?}: {err}",), ); continue; } } } else { config.new_parse_error(key, format!("Invalid public suffix file {value:?}")); continue; }; let bytes = if value.ends_with(".gz") { match GzDecoder::new(&bytes[..]) .bytes() .collect::, _>>() { Ok(bytes) => bytes, Err(err) => { config.new_build_warning( format!("{value}.{idx}"), format!( "Failed to decompress public suffixes from {value:?}: {err}", value = value, err = err ), ); continue; } } } else { bytes }; match String::from_utf8(bytes) { Ok(list) => { return PublicSuffix::from(list.as_str()); } Err(err) => { config.new_build_warning( format!("{value}.{idx}"), format!( "Failed to parse public suffixes from {value:?}: {err}", value = value, err = err ), ); } } } #[cfg(not(feature = "test_mode"))] config.new_build_warning(key, "Failed to parse public suffixes from any source."); PublicSuffix::default() } } ================================================ FILE: crates/utils/src/template.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use ahash::AHashMap; use std::{hash::Hash, str::FromStr}; #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct Template { pub items: Vec>, pub size: usize, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum TemplateItem { Static(String), Variable { name: T, escape: bool }, If { variable: T, block_end: usize }, ForEach { variable: T, block_end: usize }, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum Variable> { Single(V), Block(Vec>), } #[derive(Debug, Clone, PartialEq, Eq)] pub struct Variables> { pub items: AHashMap>, } impl Template { pub fn parse(mut template: &str) -> Result { let mut items = Vec::new(); let mut block_stack = vec![]; let mut size = 0; loop { if let Some((start, end)) = template.split_once("{{") { if !start.is_empty() { items.push(TemplateItem::Static(start.to_string())); size += start.len(); } let (var, rest) = end.split_once("}}").ok_or("Unmatched {{")?; template = rest; let var = var.trim(); if let Some(var_name) = var.strip_prefix("#").map(|v| v.trim()) { let (is_each, var_name) = if let Some(each) = var_name.strip_prefix("each ") { (true, each) } else if let Some(if_cond) = var_name.strip_prefix("if ") { (false, if_cond) } else { return Err(format!("Invalid block start: {}", var_name)); }; let var = T::from_str(var_name) .map_err(|_| format!("Invalid variable: {}", var_name))?; block_stack.push((var_name, items.len())); if is_each { items.push(TemplateItem::ForEach { variable: var, block_end: 0, }); } else { items.push(TemplateItem::If { variable: var, block_end: 0, }); } } else if let Some(var_name) = var.strip_prefix("/").map(|v| v.trim()) { let (is_each, var_name) = if let Some(each) = var_name.strip_prefix("each ") { (true, each) } else if let Some(if_cond) = var_name.strip_prefix("if ") { (false, if_cond) } else { return Err(format!("Invalid block end: {}", var_name)); }; if let Some((expected_name, if_pos)) = block_stack.pop() { if expected_name != var_name { return Err(format!( "Block end does not match start: expected {}, got {}", expected_name, var_name )); } let block_end_idx = items.len(); match &mut items[if_pos] { TemplateItem::If { block_end, .. } if !is_each => { *block_end = block_end_idx; } TemplateItem::ForEach { block_end, .. } if is_each => { *block_end = block_end_idx; } _ => { return Err(format!( "Block end does not match start type for {}", var_name )); } } } } else { let (name, escape) = var.strip_prefix("!").map_or((var, true), |v| (v, false)); let name = T::from_str(name).map_err(|_| format!("Invalid variable: {}", name))?; items.push(TemplateItem::Variable { name, escape }); } } else { if !template.is_empty() { items.push(TemplateItem::Static(template.to_string())); size += template.len(); } break; } } if block_stack.is_empty() { Ok(Template { items, size }) } else { Err(format!("Unmatched {{: {}", block_stack.last().unwrap().0)) } } pub fn eval(&self, variables: &Variables) -> String where V: AsRef, { let mut result = String::with_capacity(self.size); let mut items = self.items.iter().enumerate(); let mut base_offset = 0; while let Some((idx, item)) = items.next() { let idx = idx + base_offset; match item { TemplateItem::Static(s) => result.push_str(s), TemplateItem::Variable { name, escape } => { if let Some(Variable::Single(variable)) = variables.items.get(name) { if *escape { html_escape(&mut result, variable.as_ref()) } else { result.push_str(variable.as_ref()); } } } TemplateItem::If { variable, block_end, } => { if !variables.items.contains_key(variable) { items = self.items[*block_end..].iter().enumerate(); base_offset = *block_end; } } TemplateItem::ForEach { variable, block_end, } => { if let Some(Variable::Block(entries)) = variables.items.get(variable) { let slice = &self.items[idx + 1..*block_end]; for entry in entries { let mut slice = slice.iter(); while let Some(sub_item) = slice.next() { match sub_item { TemplateItem::Static(s) => result.push_str(s), TemplateItem::Variable { name, escape } => { if let Some(variable) = entry.get(name) { if *escape { html_escape(&mut result, variable.as_ref()) } else { result.push_str(variable.as_ref()); } } } TemplateItem::If { variable, block_end: start_pos, } => { if !entry.contains_key(variable) { slice = self.items[*start_pos..*block_end].iter(); } } _ => {} } } } } items = self.items[*block_end..].iter().enumerate(); base_offset = *block_end; } } } result } } fn html_escape(result: &mut String, input: &str) { for c in input.chars() { match c { '&' => result.push_str("&"), '<' => result.push_str("<"), '>' => result.push_str(">"), '"' => result.push_str("""), '\'' => result.push_str("'"), _ => result.push(c), } } } impl> Variables { pub fn new() -> Self { Self { items: AHashMap::new(), } } pub fn insert_single(&mut self, key: T, value: V) { self.items.insert(key, Variable::Single(value)); } pub fn insert_block(&mut self, key: T, value: V1) where V1: IntoIterator, V2: IntoIterator, { self.items.insert( key, Variable::Block(value.into_iter().map(AHashMap::from_iter).collect()), ); } } impl> Default for Variables { fn default() -> Self { Self::new() } } #[cfg(test)] mod tests { use super::*; #[test] fn test_simple_variable_substitution() { let template = Template::parse("Hello {{name}}!").unwrap(); let mut vars = Variables::::new(); vars.insert_single("name".to_string(), "World".to_string()); let result = template.eval(&vars); assert_eq!(result, "Hello World!"); } #[test] fn test_multiple_variables() { let template = Template::parse("{{greeting}} {{name}}, today is {{day}}").unwrap(); let mut vars = Variables::::new(); vars.insert_single("greeting".to_string(), "Hello".to_string()); vars.insert_single("name".to_string(), "Alice".to_string()); vars.insert_single("day".to_string(), "Monday".to_string()); let result = template.eval(&vars); assert_eq!(result, "Hello Alice, today is Monday"); } #[test] fn test_missing_variable() { let template = Template::parse("Hello {{name}}!").unwrap(); let vars = Variables::::new(); let result = template.eval(&vars); assert_eq!(result, "Hello !"); } #[test] fn test_static_text_only() { let template = Template::parse("This is just static text").unwrap(); let vars = Variables::::new(); let result = template.eval(&vars); assert_eq!(result, "This is just static text"); } #[test] fn test_empty_template() { let template = Template::parse("").unwrap(); let vars = Variables::::new(); let result = template.eval(&vars); assert_eq!(result, ""); } #[test] fn test_if_block_with_existing_variable() { let template = Template::parse("{{#if show_message}}Hello World!{{/if show_message}}").unwrap(); let mut vars = Variables::::new(); vars.insert_single("show_message".to_string(), "true".to_string()); let result = template.eval(&vars); assert_eq!(result, "Hello World!"); } #[test] fn test_if_block_with_missing_variable() { let template = Template::parse("{{#if show_message}}Hello World!{{/if show_message}}").unwrap(); let vars = Variables::::new(); let result = template.eval(&vars); assert_eq!(result, ""); } #[test] fn test_if_block_with_content_and_variables() { let template = Template::parse( "{{#if notifications}}You have notifications: {{count}}{{/if notifications}}", ) .unwrap(); let mut vars = Variables::::new(); vars.insert_single("notifications".to_string(), "true".to_string()); vars.insert_single("count".to_string(), "5".to_string()); let result = template.eval(&vars); assert_eq!(result, "You have notifications: 5"); } #[test] fn test_foreach_block_basic() { let template = Template::parse("{{#each items}}{{name}} {{/each items}}").unwrap(); let mut vars = Variables::::new(); let items = vec![ vec![("name".to_string(), "Item1".to_string())], vec![("name".to_string(), "Item2".to_string())], vec![("name".to_string(), "Item3".to_string())], ]; vars.insert_block("items".to_string(), items); let result = template.eval(&vars); assert_eq!(result, "Item1 Item2 Item3 "); } #[test] fn test_foreach_block_multiple_variables() { let template = Template::parse( "{{#each notifications}}* {{name}} at {{time}}\n{{/each notifications}}", ) .unwrap(); let mut vars = Variables::::new(); let notifications = vec![ vec![ ("name".to_string(), "Meeting".to_string()), ("time".to_string(), "10:00".to_string()), ], vec![ ("name".to_string(), "Call".to_string()), ("time".to_string(), "14:30".to_string()), ], ]; vars.insert_block("notifications".to_string(), notifications); let result = template.eval(&vars); assert_eq!(result, "* Meeting at 10:00\n* Call at 14:30\n"); } #[test] fn test_foreach_block_empty() { let template = Template::parse("{{#each items}}{{name}}{{/each items}}").unwrap(); let mut vars = Variables::::new(); vars.insert_block("items".to_string(), Vec::>::new()); let result = template.eval(&vars); assert_eq!(result, ""); } #[test] fn test_foreach_block_missing_variable() { let template = Template::parse("{{#each items}}{{name}}{{/each items}}").unwrap(); let vars = Variables::::new(); let result = template.eval(&vars); assert_eq!(result, ""); } #[test] fn test_complex_template_example() { let template_str = r#"Hello {{name}}, {{#if notifications}}You have the following notifications: {{#each notifications}}* {{name}} at {{time}} {{/each notifications}}{{/if notifications}} Best regards"#; let template = Template::parse(template_str).unwrap(); let mut vars = Variables::::new(); vars.insert_single("name".to_string(), "Alice".to_string()); vars.insert_single("notifications".to_string(), "true".to_string()); let notifications = vec![ vec![ ("name".to_string(), "Team Meeting".to_string()), ("time".to_string(), "09:00".to_string()), ], vec![ ("name".to_string(), "Doctor Appointment".to_string()), ("time".to_string(), "15:30".to_string()), ], ]; vars.insert_block("notifications".to_string(), notifications); let result = template.eval(&vars); let expected = r#"Hello Alice, You have the following notifications: * Team Meeting at 09:00 * Doctor Appointment at 15:30 Best regards"#; assert_eq!(result, expected); } #[test] fn test_complex_template_no_notifications() { let template_str = r#"Hello {{name}}, {{#if notifications}} You have the following notifications: {{#each notifications}} * {{name}} at {{time}} {{/each notifications}}{{/if notifications}} Best regards"#; let template = Template::parse(template_str).unwrap(); let mut vars = Variables::::new(); vars.insert_single("name".to_string(), "Bob".to_string()); let result = template.eval(&vars); let expected = r#"Hello Bob, Best regards"#; assert_eq!(result, expected); } #[test] fn test_whitespace_handling() { let template = Template::parse("{{ name }}").unwrap(); let mut vars = Variables::::new(); vars.insert_single("name".to_string(), "Test".to_string()); let result = template.eval(&vars); assert_eq!(result, "Test"); } #[test] fn test_whitespace_in_blocks() { let template = Template::parse("{{# if condition }}Content{{/ if condition }}").unwrap(); let mut vars = Variables::::new(); vars.insert_single("condition".to_string(), "true".to_string()); let result = template.eval(&vars); assert_eq!(result, "Content"); } // Error handling tests #[test] fn test_unmatched_opening_brace() { let result = Template::::parse("Hello {{name"); assert!(result.is_err()); assert!(result.unwrap_err().contains("Unmatched {{")); } #[test] fn test_invalid_block_start() { let result = Template::::parse("{{#invalid block}}{{/invalid block}}"); assert!(result.is_err()); assert!(result.unwrap_err().contains("Invalid block start")); } #[test] fn test_invalid_block_end() { let result = Template::::parse("{{#if test}}{{\\/invalid block}}"); assert!(result.is_err()); assert!(result.unwrap_err().contains("Unmatched")); } #[test] fn test_mismatched_block_names() { let result = Template::::parse("{{#if test}}{{/if different}}"); assert!(result.is_err()); assert!( result .unwrap_err() .contains("Block end does not match start") ); } #[test] fn test_mismatched_block_types() { let result = Template::::parse("{{#if test}}{{/each test}}"); assert!(result.is_err()); assert!( result .unwrap_err() .contains("Block end does not match start") ); } #[test] fn test_consecutive_braces() { let template = Template::parse("{{}}").unwrap(); let vars = Variables::::new(); let result = template.eval(&vars); assert_eq!(result, ""); } #[test] fn test_foreach_with_missing_inner_variables() { let template = Template::parse("{{#each items}}{{name}}: {{missing}}{{/each items}}").unwrap(); let mut vars = Variables::::new(); let items = vec![ vec![("name".to_string(), "Item1".to_string())], vec![("name".to_string(), "Item2".to_string())], ]; vars.insert_block("items".to_string(), items); let result = template.eval(&vars); assert_eq!(result, "Item1: Item2: "); } /*#[test] fn test_full() { // Load static html in memory from resources/email-templates/calendar-alarm.html let template_str = include_str!("../../../resources/email-templates/calendar-alarm.html"); let template: Template = Template::parse(template_str).unwrap(); let mut vars = Variables::::new(); vars.insert_single( CalendarTemplateVariable::PageTitle, "Test Event".to_string(), ); vars.insert_single(CalendarTemplateVariable::Header, "Event Header".to_string()); vars.insert_single(CalendarTemplateVariable::Footer, "Event Footer".to_string()); vars.insert_single( CalendarTemplateVariable::EventTitle, "Meeting with Team".to_string(), ); vars.insert_single( CalendarTemplateVariable::EventDescription, "Discuss project updates".to_string(), ); vars.insert_single( CalendarTemplateVariable::EventDetails, "Details about the event".to_string(), ); vars.insert_single( CalendarTemplateVariable::ActionUrl, "http://example.com/action".to_string(), ); vars.insert_single( CalendarTemplateVariable::ActionName, "Join Meeting".to_string(), ); vars.insert_single( CalendarTemplateVariable::AttendeesTitle, "Attendees".to_string(), ); vars.insert_block( CalendarTemplateVariable::EventDetails, vec![ vec![ (CalendarTemplateVariable::Key, "Location".to_string()), ( CalendarTemplateVariable::Value, "Conference Room A".to_string(), ), ], vec![ (CalendarTemplateVariable::Key, "Time".to_string()), ( CalendarTemplateVariable::Value, "10:00 AM - 11:00 AM".to_string(), ), ], ], ); vars.insert_block( CalendarTemplateVariable::Attendees, vec![ vec![ (CalendarTemplateVariable::Key, "Alice".to_string()), ( CalendarTemplateVariable::Value, "alice@domain.org".to_string(), ), ], vec![ (CalendarTemplateVariable::Key, "Bob".to_string()), ( CalendarTemplateVariable::Value, "bob@domain.org".to_string(), ), ], ], ); let result = template.eval(&vars); // Write result to test.html std::fs::write("test.html", result).expect("Unable to write file"); }*/ } ================================================ FILE: crates/utils/src/topological.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use ahash::AHashMap; use std::{collections::VecDeque, hash::Hash}; #[derive(Debug)] pub struct TopologicalSort { edges: AHashMap>, count: AHashMap, } impl TopologicalSort { pub fn with_capacity(capacity: usize) -> Self { Self { edges: AHashMap::with_capacity(capacity), count: AHashMap::with_capacity(capacity), } } pub fn insert(&mut self, from: T, to: T) { self.count.entry(from).or_insert(0); self.edges.entry(from).or_default().push(to); *self.count.entry(to).or_insert(0) += 1; } pub fn into_iterator(mut self) -> TopologicalSortIterator { let mut no_edges = VecDeque::with_capacity(self.count.len()); self.count.retain(|node, count| { if *count == 0 { no_edges.push_back(*node); false } else { true } }); TopologicalSortIterator { edges: self.edges, count: self.count, no_edges, } } } #[derive(Debug)] pub struct TopologicalSortIterator { edges: AHashMap>, count: AHashMap, no_edges: VecDeque, } impl Iterator for TopologicalSortIterator { type Item = T; fn next(&mut self) -> Option { let no_edge = self.no_edges.pop_back()?; if let Some(edges) = self.edges.get(&no_edge) { for neighbor in edges { if let Some(count) = self.count.get_mut(neighbor) { *count -= 1; if *count == 0 { self.count.remove(neighbor); self.no_edges.push_front(*neighbor); } } } } Some(no_edge) } } impl TopologicalSortIterator { pub fn is_valid(&self) -> bool { self.count.is_empty() } } #[cfg(test)] mod tests { use super::*; #[test] fn test_topological_sort() { let mut sort = TopologicalSort::with_capacity(6); sort.insert(1, 2); sort.insert(1, 3); sort.insert(2, 4); sort.insert(3, 4); sort.insert(4, 5); sort.insert(5, 6); let mut iter = sort.into_iterator(); assert_eq!(iter.next(), Some(1)); assert_eq!(iter.next(), Some(2)); assert_eq!(iter.next(), Some(3)); assert_eq!(iter.next(), Some(4)); assert_eq!(iter.next(), Some(5)); assert_eq!(iter.next(), Some(6)); assert_eq!(iter.next(), None); assert!(iter.is_valid(), "{:?}", iter); } #[test] fn test_topological_sort_cycle() { let mut sort = TopologicalSort::with_capacity(6); sort.insert(1, 2); sort.insert(2, 3); sort.insert(3, 1); let mut iter = sort.into_iterator(); assert_eq!(iter.next(), None); assert!(!iter.is_valid()); } } ================================================ FILE: crates/utils/src/url_params.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{borrow::Cow, collections::HashMap}; #[derive(Default)] pub struct UrlParams<'x> { params: HashMap, Cow<'x, str>>, } impl<'x> UrlParams<'x> { pub fn new(query: Option<&'x str>) -> Self { if let Some(query) = query { Self { params: form_urlencoded::parse(query.as_bytes()) .filter(|(_, value)| !value.is_empty()) .collect(), } } else { Self::default() } } pub fn get(&self, key: &str) -> Option<&str> { self.params.get(key).map(|v| v.as_ref()) } pub fn has_key(&self, key: &str) -> bool { self.params.contains_key(key) } pub fn parse(&self, key: &str) -> Option where T: std::str::FromStr, { self.get(key).and_then(|v| v.parse().ok()) } pub fn into_inner(self) -> HashMap, Cow<'x, str>> { self.params } } ================================================ FILE: docker-bake.hcl ================================================ variable "TARGET" { default = "$TARGET" } variable "GHCR_REPO" { default = "$GHCR_REPO" } variable "BUILD_ENV" { default = "$BUILD_ENV" } variable "SUFFIX" { default = "$SUFFIX" } variable "DOCKER_PLATFORM" { default = "$DOCKER_PLATFORM" } target "docker-metadata-action" {} target "build" { secret = [ "type=env,id=ACTIONS_RESULTS_URL", "type=env,id=ACTIONS_RUNTIME_TOKEN" ] args = { TARGET = "${TARGET}" BUILD_ENV = equal("", "${BUILD_ENV}") ? null : "${BUILD_ENV}" } target = "binaries" cache-from = [ "type=registry,ref=${GHCR_REPO}-buildcache:${TARGET}" ] cache-to = [ "type=registry,ref=${GHCR_REPO}-buildcache:${TARGET},mode=max,compression=zstd,compression-level=9,force-compression=true,oci-mediatypes=true,image-manifest=false" ] context = "./" dockerfile = "Dockerfile.build" output = ["./artifact"] } target "image" { inherits = ["build","docker-metadata-action"] cache-to = [""] cache-from = [ "type=registry,ref=${GHCR_REPO}-buildcache:${TARGET}" ] target = equal("", "${SUFFIX}") ? "gnu" : "musl" platforms = [ "${DOCKER_PLATFORM}" ] output = [ "" ] } ================================================ FILE: install.sh ================================================ #!/usr/bin/env sh # shellcheck shell=dash # # SPDX-FileCopyrightText: 2020 Stalwart Labs LLC # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL # # Stalwart install script -- based on the rustup installation script. set -e set -u readonly BASE_URL="https://github.com/stalwartlabs/stalwart/releases/latest/download" main() { downloader --check need_cmd uname need_cmd mktemp need_cmd chmod need_cmd mkdir need_cmd rm need_cmd rmdir need_cmd tar # Make sure we are running as root if [ "$(id -u)" -ne 0 ] ; then err "❌ Install failed: This program needs to run as root." fi # Detect OS local _os="unknown" local _uname="$(uname)" _account="stalwart" if [ "${_uname}" = "Linux" ]; then _os="linux" elif [ "${_uname}" = "Darwin" ]; then _os="macos" _account="_stalwart" fi # Read arguments local _dir="/opt/stalwart" # Default component setting local _component="stalwart" # Loop through the arguments for arg in "$@"; do case "$arg" in --fdb) _component="stalwart-foundationdb" ;; *) if [ -n "$arg" ]; then _dir=$arg fi ;; esac done # Detect platform architecture get_architecture || return 1 local _arch="$RETVAL" assert_nz "$_arch" "arch" # Create directories ensure mkdir -p "$_dir" "$_dir/bin" "$_dir/etc" "$_dir/logs" # Download latest binary say "⏳ Downloading ${_component} for ${_arch}..." local _file="${_dir}/bin/stalwart.tar.gz" local _url="${BASE_URL}/${_component}-${_arch}.tar.gz" ensure mkdir -p "$_dir" ensure downloader "$_url" "$_file" "$_arch" ensure tar zxvf "$_file" -C "$_dir/bin" if [ "$_component" = "stalwart-foundationdb" ]; then ignore mv "$_dir/bin/stalwart-foundationdb" "$_dir/bin/stalwart" fi ignore chmod +x "$_dir/bin/stalwart" ignore rm "$_file" # Create system account if ! id -u ${_account} > /dev/null 2>&1; then say "🖥️ Creating '${_account}' account..." if [ "${_os}" = "macos" ]; then local _last_uid="$(dscacheutil -q user | grep uid | awk '{print $2}' | sort -n | tail -n 1)" local _last_gid="$(dscacheutil -q group | grep gid | awk '{print $2}' | sort -n | tail -n 1)" local _uid="$((_last_uid+1))" local _gid="$((_last_gid+1))" ensure dscl /Local/Default -create Groups/_stalwart ensure dscl /Local/Default -create Groups/_stalwart Password \* ensure dscl /Local/Default -create Groups/_stalwart PrimaryGroupID $_gid ensure dscl /Local/Default -create Groups/_stalwart RealName "Stalwart service" ensure dscl /Local/Default -create Groups/_stalwart RecordName _stalwart stalwart ensure dscl /Local/Default -create Users/_stalwart ensure dscl /Local/Default -create Users/_stalwart NFSHomeDirectory /Users/_stalwart ensure dscl /Local/Default -create Users/_stalwart Password \* ensure dscl /Local/Default -create Users/_stalwart PrimaryGroupID $_gid ensure dscl /Local/Default -create Users/_stalwart RealName "Stalwart service" ensure dscl /Local/Default -create Users/_stalwart RecordName _stalwart stalwart ensure dscl /Local/Default -create Users/_stalwart UniqueID $_uid ensure dscl /Local/Default -create Users/_stalwart UserShell /bin/bash ensure dscl /Local/Default -delete /Users/_stalwart AuthenticationAuthority ensure dscl /Local/Default -delete /Users/_stalwart PasswordPolicyOptions else ensure useradd ${_account} -s /usr/sbin/nologin -M -r -U fi fi # Run init ignore $_dir/bin/stalwart --init "$_dir" # Set permissions say "🔐 Setting permissions..." ensure chown -R ${_account}:${_account} "$_dir" ensure chmod -R 755 "$_dir" ensure chmod 700 "$_dir/etc/config.toml" # Create service file say "🚀 Starting service..." if [ "${_os}" = "linux" ]; then local _issystemdlinux=$(command -v systemctl) if [ -n "$_issystemdlinux" ]; then create_service_linux_systemd "$_dir" else create_service_linux_initd "$_dir" fi elif [ "${_os}" = "macos" ]; then create_service_macos "$_dir" fi # Installation complete local _host=$(hostname -f) say "🎉 Installation complete! Continue the setup at http://$_host:8080/login" return 0 } # Functions to create service files create_service_linux_systemd() { local _dir="$1" cat < /etc/systemd/system/stalwart.service [Unit] Description=Stalwart Conflicts=postfix.service sendmail.service exim4.service ConditionPathExists=__PATH__/etc/config.toml After=network-online.target [Service] Type=simple LimitNOFILE=65536 KillMode=process KillSignal=SIGINT Restart=on-failure RestartSec=5 ExecStart=__PATH__/bin/stalwart --config=__PATH__/etc/config.toml SyslogIdentifier=stalwart User=stalwart Group=stalwart AmbientCapabilities=CAP_NET_BIND_SERVICE [Install] WantedBy=multi-user.target EOF systemctl daemon-reload systemctl enable stalwart.service systemctl restart stalwart.service } create_service_linux_initd() { local _dir="$1" cat <<"EOF" | sed "s|__PATH__|$_dir|g" > /etc/init.d/stalwart #!/bin/sh ### BEGIN INIT INFO # Provides: stalwart # Required-Start: $network # Required-Stop: $network # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: Stalwart Server # Description: Starts and stops the Stalwart Server # Conflicts: postfix sendmail ### END INIT INFO PATH=/sbin:/usr/sbin:/bin:/usr/bin . /lib/init/vars.sh . /lib/lsb/init-functions # Service Config DAEMON=__PATH__/bin/stalwart DAEMON_ARGS="--config=__PATH__/etc/config.toml" PIDFILE=/var/run/stalwart.pid ULIMIT_NOFILE=65536 # Exit if the package is not installed [ -x "$DAEMON" ] || exit 0 # Exit if config file doesn't exist [ -f "__PATH__/etc/config.toml" ] || exit 0 # Read configuration variable file if it is present [ -r /etc/default/stalwart ] && . /etc/default/stalwart # Increase file descriptor limit ulimit -n $ULIMIT_NOFILE do_start() { # Return # 0 if daemon has been started # 1 if daemon was already running # 2 if daemon could not be started start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON --test > /dev/null \ || return 1 start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON \ --background --make-pidfile --chuid stalwart:stalwart \ -- $DAEMON_ARGS \ || return 2 } do_stop() { # Return # 0 if daemon has been stopped # 1 if daemon was already stopped # 2 if daemon could not be stopped # other if a failure occurred start-stop-daemon --stop --quiet --retry=INT/30/KILL/5 --pidfile $PIDFILE --name stalwart RETVAL="$?" [ "$RETVAL" = 2 ] && return 2 # Wait for children to finish too if this is a daemon that forks # and if the daemon is only ever run from this initscript. start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --exec $DAEMON [ "$?" = 2 ] && return 2 # Many daemons don't delete their pidfiles when they exit. rm -f $PIDFILE return "$RETVAL" } case "$1" in start) [ "$VERBOSE" != no ] && log_daemon_msg "Starting Stalwart Server" "stalwart" do_start case "$?" in 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; esac ;; stop) [ "$VERBOSE" != no ] && log_daemon_msg "Stopping Stalwart Server" "stalwart" do_stop case "$?" in 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; esac ;; status) status_of_proc "$DAEMON" "stalwart" && exit 0 || exit $? ;; restart) log_daemon_msg "Restarting Stalwart Server" "stalwart" do_stop case "$?" in 0|1) do_start case "$?" in 0) log_end_msg 0 ;; 1) log_end_msg 1 ;; # Old process is still running *) log_end_msg 1 ;; # Failed to start esac ;; *) # Failed to stop log_end_msg 1 ;; esac ;; *) echo "Usage: /etc/init.d/stalwart {start|stop|status|restart}" >&2 exit 3 ;; esac exit 0 EOF chmod +x /etc/init.d/stalwart cat < /etc/default/stalwart # Configuration for Stalwart init script being run during # the boot sequence # Set to 'yes' to enable additional verbosity #VERBOSE=no EOF update-rc.d stalwart defaults service stalwart start } create_service_macos() { local _dir="$1" cat < /Library/LaunchAgents/stalwart.mail.plist Label stalwart.mail ServiceDescription Stalwart ProgramArguments __PATH__/bin/stalwart --config=__PATH__/etc/config.toml RunAtLoad KeepAlive EOF launchctl load /Library/LaunchAgents/stalwart.mail.plist launchctl enable system/stalwart.mail launchctl start system/stalwart.mail } get_architecture() { local _ostype _cputype _bitness _arch _clibtype _ostype="$(uname -s)" _cputype="$(uname -m)" _clibtype="gnu" if [ "$_ostype" = Linux ]; then if [ "$(uname -o)" = Android ]; then _ostype=Android fi if ldd --version 2>&1 | grep -q 'musl'; then _clibtype="musl" fi fi if [ "$_ostype" = Darwin ] && [ "$_cputype" = i386 ]; then # Darwin `uname -m` lies if sysctl hw.optional.x86_64 | grep -q ': 1'; then _cputype=x86_64 fi fi if [ "$_ostype" = SunOS ]; then # Both Solaris and illumos presently announce as "SunOS" in "uname -s" # so use "uname -o" to disambiguate. We use the full path to the # system uname in case the user has coreutils uname first in PATH, # which has historically sometimes printed the wrong value here. if [ "$(/usr/bin/uname -o)" = illumos ]; then _ostype=illumos fi # illumos systems have multi-arch userlands, and "uname -m" reports the # machine hardware name; e.g., "i86pc" on both 32- and 64-bit x86 # systems. Check for the native (widest) instruction set on the # running kernel: if [ "$_cputype" = i86pc ]; then _cputype="$(isainfo -n)" fi fi case "$_ostype" in Android) _ostype=linux-android ;; Linux) check_proc _ostype=unknown-linux-$_clibtype _bitness=$(get_bitness) ;; FreeBSD) _ostype=unknown-freebsd ;; NetBSD) _ostype=unknown-netbsd ;; DragonFly) _ostype=unknown-dragonfly ;; Darwin) _ostype=apple-darwin ;; illumos) _ostype=unknown-illumos ;; MINGW* | MSYS* | CYGWIN* | Windows_NT) _ostype=pc-windows-gnu ;; *) err "unrecognized OS type: $_ostype" ;; esac case "$_cputype" in i386 | i486 | i686 | i786 | x86) _cputype=i686 ;; xscale | arm) _cputype=arm if [ "$_ostype" = "linux-android" ]; then _ostype=linux-androideabi fi ;; armv6l) _cputype=arm if [ "$_ostype" = "linux-android" ]; then _ostype=linux-androideabi else _ostype="${_ostype}eabihf" fi ;; armv7l | armv8l) _cputype=armv7 if [ "$_ostype" = "linux-android" ]; then _ostype=linux-androideabi else _ostype="${_ostype}eabihf" fi ;; aarch64 | arm64) _cputype=aarch64 ;; x86_64 | x86-64 | x64 | amd64) _cputype=x86_64 ;; mips) _cputype=$(get_endianness mips '' el) ;; mips64) if [ "$_bitness" -eq 64 ]; then # only n64 ABI is supported for now _ostype="${_ostype}abi64" _cputype=$(get_endianness mips64 '' el) fi ;; ppc) _cputype=powerpc ;; ppc64) _cputype=powerpc64 ;; ppc64le) _cputype=powerpc64le ;; s390x) _cputype=s390x ;; riscv64) _cputype=riscv64gc ;; *) err "unknown CPU type: $_cputype" esac # Detect 64-bit linux with 32-bit userland if [ "${_ostype}" = unknown-linux-gnu ] && [ "${_bitness}" -eq 32 ]; then case $_cputype in x86_64) if [ -n "${RUSTUP_CPUTYPE:-}" ]; then _cputype="$RUSTUP_CPUTYPE" else { # 32-bit executable for amd64 = x32 if is_host_amd64_elf; then { echo "This host is running an x32 userland; as it stands, x32 support is poor," 1>&2 echo "and there isn't a native toolchain -- you will have to install" 1>&2 echo "multiarch compatibility with i686 and/or amd64, then select one" 1>&2 echo "by re-running this script with the RUSTUP_CPUTYPE environment variable" 1>&2 echo "set to i686 or x86_64, respectively." 1>&2 echo 1>&2 echo "You will be able to add an x32 target after installation by running" 1>&2 echo " rustup target add x86_64-unknown-linux-gnux32" 1>&2 exit 1 }; else _cputype=i686 fi }; fi ;; mips64) _cputype=$(get_endianness mips '' el) ;; powerpc64) _cputype=powerpc ;; aarch64) _cputype=armv7 if [ "$_ostype" = "linux-android" ]; then _ostype=linux-androideabi else _ostype="${_ostype}eabihf" fi ;; riscv64gc) err "riscv64 with 32-bit userland unsupported" ;; esac fi # Detect armv7 but without the CPU features Rust needs in that build, # and fall back to arm. # See https://github.com/rust-lang/rustup.rs/issues/587. if [ "$_ostype" = "unknown-linux-gnueabihf" ] && [ "$_cputype" = armv7 ]; then if ensure grep '^Features' /proc/cpuinfo | grep -q -v neon; then # At least one processor does not have NEON. _cputype=arm fi fi _arch="${_cputype}-${_ostype}" RETVAL="$_arch" } check_proc() { # Check for /proc by looking for the /proc/self/exe link # This is only run on Linux if ! test -L /proc/self/exe ; then err "fatal: Unable to find /proc/self/exe. Is /proc mounted? Installation cannot proceed without /proc." fi } get_bitness() { need_cmd head # Architecture detection without dependencies beyond coreutils. # ELF files start out "\x7fELF", and the following byte is # 0x01 for 32-bit and # 0x02 for 64-bit. # The printf builtin on some shells like dash only supports octal # escape sequences, so we use those. local _current_exe_head _current_exe_head=$(head -c 5 /proc/self/exe ) if [ "$_current_exe_head" = "$(printf '\177ELF\001')" ]; then echo 32 elif [ "$_current_exe_head" = "$(printf '\177ELF\002')" ]; then echo 64 else err "unknown platform bitness" fi } is_host_amd64_elf() { need_cmd head need_cmd tail # ELF e_machine detection without dependencies beyond coreutils. # Two-byte field at offset 0x12 indicates the CPU, # but we're interested in it being 0x3E to indicate amd64, or not that. local _current_exe_machine _current_exe_machine=$(head -c 19 /proc/self/exe | tail -c 1) [ "$_current_exe_machine" = "$(printf '\076')" ] } get_endianness() { local cputype=$1 local suffix_eb=$2 local suffix_el=$3 # detect endianness without od/hexdump, like get_bitness() does. need_cmd head need_cmd tail local _current_exe_endianness _current_exe_endianness="$(head -c 6 /proc/self/exe | tail -c 1)" if [ "$_current_exe_endianness" = "$(printf '\001')" ]; then echo "${cputype}${suffix_el}" elif [ "$_current_exe_endianness" = "$(printf '\002')" ]; then echo "${cputype}${suffix_eb}" else err "unknown platform endianness" fi } say() { printf '%s\n' "$1" } err() { say "$1" >&2 exit 1 } need_cmd() { if ! check_cmd "$1"; then err "need '$1' (command not found)" fi } check_cmd() { command -v "$1" > /dev/null 2>&1 } assert_nz() { if [ -z "$1" ]; then err "assert_nz $2"; fi } # Run a command that should never fail. If the command fails execution # will immediately terminate with an error showing the failing # command. ensure() { if ! "$@"; then err "command failed: $*"; fi } # This wraps curl or wget. Try curl first, if not installed, # use wget instead. downloader() { local _dld local _ciphersuites local _err local _status local _retry if check_cmd curl; then _dld=curl elif check_cmd wget; then _dld=wget else _dld='curl or wget' # to be used in error message of need_cmd fi if [ "$1" = --check ]; then need_cmd "$_dld" elif [ "$_dld" = curl ]; then check_curl_for_retry_support _retry="$RETVAL" get_ciphersuites_for_curl _ciphersuites="$RETVAL" if [ -n "$_ciphersuites" ]; then _err=$(curl $_retry --proto '=https' --tlsv1.2 --ciphers "$_ciphersuites" --silent --show-error --fail --location "$1" --output "$2" 2>&1) _status=$? else echo "Warning: Not enforcing strong cipher suites for TLS, this is potentially less secure" if ! check_help_for "$3" curl --proto --tlsv1.2; then echo "Warning: Not enforcing TLS v1.2, this is potentially less secure" _err=$(curl $_retry --silent --show-error --fail --location "$1" --output "$2" 2>&1) _status=$? else _err=$(curl $_retry --proto '=https' --tlsv1.2 --silent --show-error --fail --location "$1" --output "$2" 2>&1) _status=$? fi fi if [ -n "$_err" ]; then if echo "$_err" | grep -q 404; then err "❌ Binary for platform '$3' not found, this platform may be unsupported." else echo "$_err" >&2 fi fi return $_status elif [ "$_dld" = wget ]; then if [ "$(wget -V 2>&1|head -2|tail -1|cut -f1 -d" ")" = "BusyBox" ]; then echo "Warning: using the BusyBox version of wget. Not enforcing strong cipher suites for TLS or TLS v1.2, this is potentially less secure" _err=$(wget "$1" -O "$2" 2>&1) _status=$? else get_ciphersuites_for_wget _ciphersuites="$RETVAL" if [ -n "$_ciphersuites" ]; then _err=$(wget --https-only --secure-protocol=TLSv1_2 --ciphers "$_ciphersuites" "$1" -O "$2" 2>&1) _status=$? else echo "Warning: Not enforcing strong cipher suites for TLS, this is potentially less secure" if ! check_help_for "$3" wget --https-only --secure-protocol; then echo "Warning: Not enforcing TLS v1.2, this is potentially less secure" _err=$(wget "$1" -O "$2" 2>&1) _status=$? else _err=$(wget --https-only --secure-protocol=TLSv1_2 "$1" -O "$2" 2>&1) _status=$? fi fi fi if [ -n "$_err" ]; then if echo "$_err" | grep -q ' 404 Not Found'; then err "❌ Binary for platform '$3' not found, this platform may be unsupported." else echo "$_err" >&2 fi fi return $_status else err "Unknown downloader" # should not reach here fi } # Check if curl supports the --retry flag, then pass it to the curl invocation. check_curl_for_retry_support() { local _retry_supported="" # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. if check_help_for "notspecified" "curl" "--retry"; then _retry_supported="--retry 3" fi RETVAL="$_retry_supported" } check_help_for() { local _arch local _cmd local _arg _arch="$1" shift _cmd="$1" shift local _category if "$_cmd" --help | grep -q 'For all options use the manual or "--help all".'; then _category="all" else _category="" fi case "$_arch" in *darwin*) if check_cmd sw_vers; then case $(sw_vers -productVersion) in 10.*) # If we're running on macOS, older than 10.13, then we always # fail to find these options to force fallback if [ "$(sw_vers -productVersion | cut -d. -f2)" -lt 13 ]; then # Older than 10.13 echo "Warning: Detected macOS platform older than 10.13" return 1 fi ;; 11.*) # We assume Big Sur will be OK for now ;; *) # Unknown product version, warn and continue echo "Warning: Detected unknown macOS major version: $(sw_vers -productVersion)" echo "Warning TLS capabilities detection may fail" ;; esac fi ;; esac for _arg in "$@"; do if ! "$_cmd" --help $_category | grep -q -- "$_arg"; then return 1 fi done true # not strictly needed } # Return cipher suite string specified by user, otherwise return strong TLS 1.2-1.3 cipher suites # if support by local tools is detected. Detection currently supports these curl backends: # GnuTLS and OpenSSL (possibly also LibreSSL and BoringSSL). Return value can be empty. get_ciphersuites_for_curl() { if [ -n "${RUSTUP_TLS_CIPHERSUITES-}" ]; then # user specified custom cipher suites, assume they know what they're doing RETVAL="$RUSTUP_TLS_CIPHERSUITES" return fi local _openssl_syntax="no" local _gnutls_syntax="no" local _backend_supported="yes" if curl -V | grep -q ' OpenSSL/'; then _openssl_syntax="yes" elif curl -V | grep -iq ' LibreSSL/'; then _openssl_syntax="yes" elif curl -V | grep -iq ' BoringSSL/'; then _openssl_syntax="yes" elif curl -V | grep -iq ' GnuTLS/'; then _gnutls_syntax="yes" else _backend_supported="no" fi local _args_supported="no" if [ "$_backend_supported" = "yes" ]; then # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. if check_help_for "notspecified" "curl" "--tlsv1.2" "--ciphers" "--proto"; then _args_supported="yes" fi fi local _cs="" if [ "$_args_supported" = "yes" ]; then if [ "$_openssl_syntax" = "yes" ]; then _cs=$(get_strong_ciphersuites_for "openssl") elif [ "$_gnutls_syntax" = "yes" ]; then _cs=$(get_strong_ciphersuites_for "gnutls") fi fi RETVAL="$_cs" } # Return cipher suite string specified by user, otherwise return strong TLS 1.2-1.3 cipher suites # if support by local tools is detected. Detection currently supports these wget backends: # GnuTLS and OpenSSL (possibly also LibreSSL and BoringSSL). Return value can be empty. get_ciphersuites_for_wget() { if [ -n "${RUSTUP_TLS_CIPHERSUITES-}" ]; then # user specified custom cipher suites, assume they know what they're doing RETVAL="$RUSTUP_TLS_CIPHERSUITES" return fi local _cs="" if wget -V | grep -q '\-DHAVE_LIBSSL'; then # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. if check_help_for "notspecified" "wget" "TLSv1_2" "--ciphers" "--https-only" "--secure-protocol"; then _cs=$(get_strong_ciphersuites_for "openssl") fi elif wget -V | grep -q '\-DHAVE_LIBGNUTLS'; then # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. if check_help_for "notspecified" "wget" "TLSv1_2" "--ciphers" "--https-only" "--secure-protocol"; then _cs=$(get_strong_ciphersuites_for "gnutls") fi fi RETVAL="$_cs" } # Return strong TLS 1.2-1.3 cipher suites in OpenSSL or GnuTLS syntax. TLS 1.2 # excludes non-ECDHE and non-AEAD cipher suites. DHE is excluded due to bad # DH params often found on servers (see RFC 7919). Sequence matches or is # similar to Firefox 68 ESR with weak cipher suites disabled via about:config. # $1 must be openssl or gnutls. get_strong_ciphersuites_for() { if [ "$1" = "openssl" ]; then # OpenSSL is forgiving of unknown values, no problems with TLS 1.3 values on versions that don't support it yet. echo "TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_256_GCM_SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384" elif [ "$1" = "gnutls" ]; then # GnuTLS isn't forgiving of unknown values, so this may require a GnuTLS version that supports TLS 1.3 even if wget doesn't. # Begin with SECURE128 (and higher) then remove/add to build cipher suites. Produces same 9 cipher suites as OpenSSL but in slightly different order. echo "SECURE128:-VERS-SSL3.0:-VERS-TLS1.0:-VERS-TLS1.1:-VERS-DTLS-ALL:-CIPHER-ALL:-MAC-ALL:-KX-ALL:+AEAD:+ECDHE-ECDSA:+ECDHE-RSA:+AES-128-GCM:+CHACHA20-POLY1305:+AES-256-GCM" fi } # This is just for indicating that commands' results are being # intentionally ignored. Usually, because it's being executed # as part of error handling. ignore() { "$@" } main "$@" || exit 1 ================================================ FILE: resources/apparmor.d/stalwart-mail ================================================ #include profile stalwart flags=(attach_disconnected) { #include #include #include # Allow network access network inet stream, network inet6 stream, network inet dgram, network inet6 dgram, # Outgoing access to port 25 and 443 network tcp, network udp, owner /proc/*/net/if_inet6 r, owner /proc/*/net/ipv6_route r, # Full write access to /opt/stalwart /opt/stalwart/** rwk, # Allow creating directories under /tmp /tmp/ r, /tmp/** rwk, # Allow binding to specific ports network inet stream bind port 25, network inet stream bind port 587, network inet stream bind port 465, network inet stream bind port 143, network inet stream bind port 993, network inet stream bind port 110, network inet stream bind port 995, network inet stream bind port 4190, network inet stream bind port 443, network inet stream bind port 8080, network inet6 stream bind port 25, network inet6 stream bind port 587, network inet6 stream bind port 465, network inet6 stream bind port 143, network inet6 stream bind port 993, network inet6 stream bind port 110, network inet6 stream bind port 995, network inet6 stream bind port 4190, network inet6 stream bind port 443, network inet6 stream bind port 8080, # Allow UDP port 7911 network inet dgram bind port 7911, network inet6 dgram bind port 7911, # Basic system access /usr/bin/stalwart rix, /etc/stalwart/** r, /var/log/stalwart/** w, # Additional permissions might be needed depending on specific requirements } ================================================ FILE: resources/config/config.toml ================================================ ############################################# # Stalwart Configuration File ############################################# [server.listener."smtp"] bind = ["[::]:25"] protocol = "smtp" [server.listener."submission"] bind = ["[::]:587"] protocol = "smtp" [server.listener."submissions"] bind = ["[::]:465"] protocol = "smtp" tls.implicit = true [server.listener."imap"] bind = ["[::]:143"] protocol = "imap" [server.listener."imaptls"] bind = ["[::]:993"] protocol = "imap" tls.implicit = true [server.listener.pop3] bind = "[::]:110" protocol = "pop3" [server.listener.pop3s] bind = "[::]:995" protocol = "pop3" tls.implicit = true [server.listener."sieve"] bind = ["[::]:4190"] protocol = "managesieve" [server.listener."https"] protocol = "http" bind = ["[::]:443"] tls.implicit = true [storage] data = "rocksdb" fts = "rocksdb" blob = "rocksdb" lookup = "rocksdb" directory = "internal" [store."rocksdb"] type = "rocksdb" path = "%{env:STALWART_PATH}%/data" compression = "lz4" [directory."internal"] type = "internal" store = "rocksdb" [tracer."stdout"] type = "stdout" level = "info" ansi = false enable = true #[server.run-as] #user = "stalwart" #group = "stalwart" [authentication.fallback-admin] user = "admin" secret = "%{env:ADMIN_SECRET}%" ================================================ FILE: resources/docker/Dockerfile.fdb ================================================ FROM debian:trixie-slim AS chef RUN apt-get update && \ export DEBIAN_FRONTEND=noninteractive && \ apt-get install -yq \ build-essential \ cmake \ clang \ curl \ protobuf-compiler \ adduser ENV RUSTUP_HOME=/opt/rust/rustup \ PATH=/home/root/.cargo/bin:/opt/rust/cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN curl https://sh.rustup.rs -sSf | \ env CARGO_HOME=/opt/rust/cargo \ sh -s -- -y --default-toolchain stable --profile minimal --no-modify-path && \ env CARGO_HOME=/opt/rust/cargo \ rustup component add rustfmt RUN curl -LO https://github.com/apple/foundationdb/releases/download/7.3.69/foundationdb-clients_7.3.69-1_amd64.deb && \ dpkg -i foundationdb-clients_7.3.69-1_amd64.deb RUN env CARGO_HOME=/opt/rust/cargo cargo install cargo-chef && \ rm -rf /opt/rust/cargo/registry/ WORKDIR /app FROM chef AS planner COPY Cargo.toml . COPY Cargo.lock . COPY crates/ crates/ COPY resources/ resources/ COPY tests/ tests/ RUN cargo chef prepare --recipe-path recipe.json FROM chef AS builder COPY --from=planner /app/recipe.json recipe.json RUN cargo chef cook --release --recipe-path recipe.json COPY Cargo.toml . COPY Cargo.lock . COPY crates/ crates/ COPY resources/ resources/ COPY tests/ tests/ RUN cargo build -p stalwart --no-default-features --features "foundationdb elastic s3 redis azure nats enterprise" --release FROM debian:trixie-slim AS runtime COPY --from=builder /app/target/release/stalwart /usr/local/bin/stalwart RUN apt-get update -y && apt-get install -yq --no-install-recommends ca-certificates curl adduser RUN curl -LO https://github.com/apple/foundationdb/releases/download/7.3.69/foundationdb-clients_7.3.69-1_amd64.deb && \ dpkg -i foundationdb-clients_7.3.69-1_amd64.deb RUN useradd stalwart -s /sbin/nologin -M RUN mkdir -p /opt/stalwart RUN chown stalwart:stalwart /opt/stalwart ENTRYPOINT ["/usr/local/bin/stalwart", "--config", "/opt/stalwart/etc/config.toml"] ================================================ FILE: resources/docker/download.sh ================================================ #!/usr/bin/env sh # shellcheck shell=dash # Stalwart install script -- based on the rustup installation script. set -e set -u readonly BASE_URL="https://github.com/stalwartlabs/stalwart/releases/latest/download" main() { downloader --check need_cmd uname need_cmd mktemp need_cmd chmod need_cmd mkdir need_cmd rm need_cmd rmdir need_cmd tar # Make sure we are running as root if [ "$(id -u)" -ne 0 ] ; then err "❌ Install failed: This program needs to run as root." fi # Detect OS local _os="unknown" local _uname="$(uname)" _account="stalwart" if [ "${_uname}" = "Linux" ]; then _os="linux" elif [ "${_uname}" = "Darwin" ]; then _os="macos" _account="_stalwart" fi # Default component setting local _component="stalwart" local _dir="/usr/local/bin" # Detect platform architecture get_architecture || return 1 local _arch="$RETVAL" assert_nz "$_arch" "arch" # Download latest binary say "⏳ Downloading ${_component} for ${_arch}..." local _file="${_dir}/stalwart.tar.gz" local _url="${BASE_URL}/${_component}-${_arch}.tar.gz" ensure downloader "$_url" "$_file" "$_arch" ensure tar zxvf "$_file" -C "$_dir" ignore chmod +x "$_dir/stalwart" ignore rm "$_file" return 0 } get_architecture() { local _ostype _cputype _bitness _arch _clibtype _ostype="$(uname -s)" _cputype="$(uname -m)" _clibtype="gnu" if [ "$_ostype" = Linux ]; then if [ "$(uname -o)" = Android ]; then _ostype=Android fi if ldd --version 2>&1 | grep -q 'musl'; then _clibtype="musl" fi fi if [ "$_ostype" = Darwin ] && [ "$_cputype" = i386 ]; then # Darwin `uname -m` lies if sysctl hw.optional.x86_64 | grep -q ': 1'; then _cputype=x86_64 fi fi if [ "$_ostype" = SunOS ]; then # Both Solaris and illumos presently announce as "SunOS" in "uname -s" # so use "uname -o" to disambiguate. We use the full path to the # system uname in case the user has coreutils uname first in PATH, # which has historically sometimes printed the wrong value here. if [ "$(/usr/bin/uname -o)" = illumos ]; then _ostype=illumos fi # illumos systems have multi-arch userlands, and "uname -m" reports the # machine hardware name; e.g., "i86pc" on both 32- and 64-bit x86 # systems. Check for the native (widest) instruction set on the # running kernel: if [ "$_cputype" = i86pc ]; then _cputype="$(isainfo -n)" fi fi case "$_ostype" in Android) _ostype=linux-android ;; Linux) check_proc _ostype=unknown-linux-$_clibtype _bitness=$(get_bitness) ;; FreeBSD) _ostype=unknown-freebsd ;; NetBSD) _ostype=unknown-netbsd ;; DragonFly) _ostype=unknown-dragonfly ;; Darwin) _ostype=apple-darwin ;; illumos) _ostype=unknown-illumos ;; MINGW* | MSYS* | CYGWIN* | Windows_NT) _ostype=pc-windows-gnu ;; *) err "unrecognized OS type: $_ostype" ;; esac case "$_cputype" in i386 | i486 | i686 | i786 | x86) _cputype=i686 ;; xscale | arm) _cputype=arm if [ "$_ostype" = "linux-android" ]; then _ostype=linux-androideabi fi ;; armv6l) _cputype=arm if [ "$_ostype" = "linux-android" ]; then _ostype=linux-androideabi else _ostype="${_ostype}eabihf" fi ;; armv7l | armv8l) _cputype=armv7 if [ "$_ostype" = "linux-android" ]; then _ostype=linux-androideabi else _ostype="${_ostype}eabihf" fi ;; aarch64 | arm64) _cputype=aarch64 ;; x86_64 | x86-64 | x64 | amd64) _cputype=x86_64 ;; mips) _cputype=$(get_endianness mips '' el) ;; mips64) if [ "$_bitness" -eq 64 ]; then # only n64 ABI is supported for now _ostype="${_ostype}abi64" _cputype=$(get_endianness mips64 '' el) fi ;; ppc) _cputype=powerpc ;; ppc64) _cputype=powerpc64 ;; ppc64le) _cputype=powerpc64le ;; s390x) _cputype=s390x ;; riscv64) _cputype=riscv64gc ;; *) err "unknown CPU type: $_cputype" esac # Detect 64-bit linux with 32-bit userland if [ "${_ostype}" = unknown-linux-gnu ] && [ "${_bitness}" -eq 32 ]; then case $_cputype in x86_64) if [ -n "${RUSTUP_CPUTYPE:-}" ]; then _cputype="$RUSTUP_CPUTYPE" else { # 32-bit executable for amd64 = x32 if is_host_amd64_elf; then { echo "This host is running an x32 userland; as it stands, x32 support is poor," 1>&2 echo "and there isn't a native toolchain -- you will have to install" 1>&2 echo "multiarch compatibility with i686 and/or amd64, then select one" 1>&2 echo "by re-running this script with the RUSTUP_CPUTYPE environment variable" 1>&2 echo "set to i686 or x86_64, respectively." 1>&2 echo 1>&2 echo "You will be able to add an x32 target after installation by running" 1>&2 echo " rustup target add x86_64-unknown-linux-gnux32" 1>&2 exit 1 }; else _cputype=i686 fi }; fi ;; mips64) _cputype=$(get_endianness mips '' el) ;; powerpc64) _cputype=powerpc ;; aarch64) _cputype=armv7 if [ "$_ostype" = "linux-android" ]; then _ostype=linux-androideabi else _ostype="${_ostype}eabihf" fi ;; riscv64gc) err "riscv64 with 32-bit userland unsupported" ;; esac fi # Detect armv7 but without the CPU features Rust needs in that build, # and fall back to arm. # See https://github.com/rust-lang/rustup.rs/issues/587. if [ "$_ostype" = "unknown-linux-gnueabihf" ] && [ "$_cputype" = armv7 ]; then if ensure grep '^Features' /proc/cpuinfo | grep -q -v neon; then # At least one processor does not have NEON. _cputype=arm fi fi _arch="${_cputype}-${_ostype}" RETVAL="$_arch" } check_proc() { # Check for /proc by looking for the /proc/self/exe link # This is only run on Linux if ! test -L /proc/self/exe ; then err "fatal: Unable to find /proc/self/exe. Is /proc mounted? Installation cannot proceed without /proc." fi } get_bitness() { need_cmd head # Architecture detection without dependencies beyond coreutils. # ELF files start out "\x7fELF", and the following byte is # 0x01 for 32-bit and # 0x02 for 64-bit. # The printf builtin on some shells like dash only supports octal # escape sequences, so we use those. local _current_exe_head _current_exe_head=$(head -c 5 /proc/self/exe ) if [ "$_current_exe_head" = "$(printf '\177ELF\001')" ]; then echo 32 elif [ "$_current_exe_head" = "$(printf '\177ELF\002')" ]; then echo 64 else err "unknown platform bitness" fi } is_host_amd64_elf() { need_cmd head need_cmd tail # ELF e_machine detection without dependencies beyond coreutils. # Two-byte field at offset 0x12 indicates the CPU, # but we're interested in it being 0x3E to indicate amd64, or not that. local _current_exe_machine _current_exe_machine=$(head -c 19 /proc/self/exe | tail -c 1) [ "$_current_exe_machine" = "$(printf '\076')" ] } get_endianness() { local cputype=$1 local suffix_eb=$2 local suffix_el=$3 # detect endianness without od/hexdump, like get_bitness() does. need_cmd head need_cmd tail local _current_exe_endianness _current_exe_endianness="$(head -c 6 /proc/self/exe | tail -c 1)" if [ "$_current_exe_endianness" = "$(printf '\001')" ]; then echo "${cputype}${suffix_el}" elif [ "$_current_exe_endianness" = "$(printf '\002')" ]; then echo "${cputype}${suffix_eb}" else err "unknown platform endianness" fi } say() { printf '%s\n' "$1" } err() { say "$1" >&2 exit 1 } need_cmd() { if ! check_cmd "$1"; then err "need '$1' (command not found)" fi } check_cmd() { command -v "$1" > /dev/null 2>&1 } assert_nz() { if [ -z "$1" ]; then err "assert_nz $2"; fi } # Run a command that should never fail. If the command fails execution # will immediately terminate with an error showing the failing # command. ensure() { if ! "$@"; then err "command failed: $*"; fi } # This wraps curl or wget. Try curl first, if not installed, # use wget instead. downloader() { local _dld local _ciphersuites local _err local _status local _retry if check_cmd curl; then _dld=curl elif check_cmd wget; then _dld=wget else _dld='curl or wget' # to be used in error message of need_cmd fi if [ "$1" = --check ]; then need_cmd "$_dld" elif [ "$_dld" = curl ]; then check_curl_for_retry_support _retry="$RETVAL" get_ciphersuites_for_curl _ciphersuites="$RETVAL" if [ -n "$_ciphersuites" ]; then _err=$(curl $_retry --proto '=https' --tlsv1.2 --ciphers "$_ciphersuites" --silent --show-error --fail --location "$1" --output "$2" 2>&1) _status=$? else echo "Warning: Not enforcing strong cipher suites for TLS, this is potentially less secure" if ! check_help_for "$3" curl --proto --tlsv1.2; then echo "Warning: Not enforcing TLS v1.2, this is potentially less secure" _err=$(curl $_retry --silent --show-error --fail --location "$1" --output "$2" 2>&1) _status=$? else _err=$(curl $_retry --proto '=https' --tlsv1.2 --silent --show-error --fail --location "$1" --output "$2" 2>&1) _status=$? fi fi if [ -n "$_err" ]; then if echo "$_err" | grep -q 404; then err "❌ Binary for platform '$3' not found, this platform may be unsupported." else echo "$_err" >&2 fi fi return $_status elif [ "$_dld" = wget ]; then if [ "$(wget -V 2>&1|head -2|tail -1|cut -f1 -d" ")" = "BusyBox" ]; then echo "Warning: using the BusyBox version of wget. Not enforcing strong cipher suites for TLS or TLS v1.2, this is potentially less secure" _err=$(wget "$1" -O "$2" 2>&1) _status=$? else get_ciphersuites_for_wget _ciphersuites="$RETVAL" if [ -n "$_ciphersuites" ]; then _err=$(wget --https-only --secure-protocol=TLSv1_2 --ciphers "$_ciphersuites" "$1" -O "$2" 2>&1) _status=$? else echo "Warning: Not enforcing strong cipher suites for TLS, this is potentially less secure" if ! check_help_for "$3" wget --https-only --secure-protocol; then echo "Warning: Not enforcing TLS v1.2, this is potentially less secure" _err=$(wget "$1" -O "$2" 2>&1) _status=$? else _err=$(wget --https-only --secure-protocol=TLSv1_2 "$1" -O "$2" 2>&1) _status=$? fi fi fi if [ -n "$_err" ]; then if echo "$_err" | grep -q ' 404 Not Found'; then err "❌ Binary for platform '$3' not found, this platform may be unsupported." else echo "$_err" >&2 fi fi return $_status else err "Unknown downloader" # should not reach here fi } # Check if curl supports the --retry flag, then pass it to the curl invocation. check_curl_for_retry_support() { local _retry_supported="" # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. if check_help_for "notspecified" "curl" "--retry"; then _retry_supported="--retry 3" fi RETVAL="$_retry_supported" } check_help_for() { local _arch local _cmd local _arg _arch="$1" shift _cmd="$1" shift local _category if "$_cmd" --help | grep -q 'For all options use the manual or "--help all".'; then _category="all" else _category="" fi case "$_arch" in *darwin*) if check_cmd sw_vers; then case $(sw_vers -productVersion) in 10.*) # If we're running on macOS, older than 10.13, then we always # fail to find these options to force fallback if [ "$(sw_vers -productVersion | cut -d. -f2)" -lt 13 ]; then # Older than 10.13 echo "Warning: Detected macOS platform older than 10.13" return 1 fi ;; 11.*) # We assume Big Sur will be OK for now ;; *) # Unknown product version, warn and continue echo "Warning: Detected unknown macOS major version: $(sw_vers -productVersion)" echo "Warning TLS capabilities detection may fail" ;; esac fi ;; esac for _arg in "$@"; do if ! "$_cmd" --help $_category | grep -q -- "$_arg"; then return 1 fi done true # not strictly needed } # Return cipher suite string specified by user, otherwise return strong TLS 1.2-1.3 cipher suites # if support by local tools is detected. Detection currently supports these curl backends: # GnuTLS and OpenSSL (possibly also LibreSSL and BoringSSL). Return value can be empty. get_ciphersuites_for_curl() { if [ -n "${RUSTUP_TLS_CIPHERSUITES-}" ]; then # user specified custom cipher suites, assume they know what they're doing RETVAL="$RUSTUP_TLS_CIPHERSUITES" return fi local _openssl_syntax="no" local _gnutls_syntax="no" local _backend_supported="yes" if curl -V | grep -q ' OpenSSL/'; then _openssl_syntax="yes" elif curl -V | grep -iq ' LibreSSL/'; then _openssl_syntax="yes" elif curl -V | grep -iq ' BoringSSL/'; then _openssl_syntax="yes" elif curl -V | grep -iq ' GnuTLS/'; then _gnutls_syntax="yes" else _backend_supported="no" fi local _args_supported="no" if [ "$_backend_supported" = "yes" ]; then # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. if check_help_for "notspecified" "curl" "--tlsv1.2" "--ciphers" "--proto"; then _args_supported="yes" fi fi local _cs="" if [ "$_args_supported" = "yes" ]; then if [ "$_openssl_syntax" = "yes" ]; then _cs=$(get_strong_ciphersuites_for "openssl") elif [ "$_gnutls_syntax" = "yes" ]; then _cs=$(get_strong_ciphersuites_for "gnutls") fi fi RETVAL="$_cs" } # Return cipher suite string specified by user, otherwise return strong TLS 1.2-1.3 cipher suites # if support by local tools is detected. Detection currently supports these wget backends: # GnuTLS and OpenSSL (possibly also LibreSSL and BoringSSL). Return value can be empty. get_ciphersuites_for_wget() { if [ -n "${RUSTUP_TLS_CIPHERSUITES-}" ]; then # user specified custom cipher suites, assume they know what they're doing RETVAL="$RUSTUP_TLS_CIPHERSUITES" return fi local _cs="" if wget -V | grep -q '\-DHAVE_LIBSSL'; then # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. if check_help_for "notspecified" "wget" "TLSv1_2" "--ciphers" "--https-only" "--secure-protocol"; then _cs=$(get_strong_ciphersuites_for "openssl") fi elif wget -V | grep -q '\-DHAVE_LIBGNUTLS'; then # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. if check_help_for "notspecified" "wget" "TLSv1_2" "--ciphers" "--https-only" "--secure-protocol"; then _cs=$(get_strong_ciphersuites_for "gnutls") fi fi RETVAL="$_cs" } # Return strong TLS 1.2-1.3 cipher suites in OpenSSL or GnuTLS syntax. TLS 1.2 # excludes non-ECDHE and non-AEAD cipher suites. DHE is excluded due to bad # DH params often found on servers (see RFC 7919). Sequence matches or is # similar to Firefox 68 ESR with weak cipher suites disabled via about:config. # $1 must be openssl or gnutls. get_strong_ciphersuites_for() { if [ "$1" = "openssl" ]; then # OpenSSL is forgiving of unknown values, no problems with TLS 1.3 values on versions that don't support it yet. echo "TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_256_GCM_SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384" elif [ "$1" = "gnutls" ]; then # GnuTLS isn't forgiving of unknown values, so this may require a GnuTLS version that supports TLS 1.3 even if wget doesn't. # Begin with SECURE128 (and higher) then remove/add to build cipher suites. Produces same 9 cipher suites as OpenSSL but in slightly different order. echo "SECURE128:-VERS-SSL3.0:-VERS-TLS1.0:-VERS-TLS1.1:-VERS-DTLS-ALL:-CIPHER-ALL:-MAC-ALL:-KX-ALL:+AEAD:+ECDHE-ECDSA:+ECDHE-RSA:+AES-128-GCM:+CHACHA20-POLY1305:+AES-256-GCM" fi } # This is just for indicating that commands' results are being # intentionally ignored. Usually, because it's being executed # as part of error handling. ignore() { "$@" } main "$@" || exit 1 ================================================ FILE: resources/docker/entrypoint.sh ================================================ #!/usr/bin/env sh # shellcheck shell=dash # If the configuration file does not exist initialize it. if [ ! -f /opt/stalwart/etc/config.toml ]; then /usr/local/bin/stalwart --init /opt/stalwart fi # If the configuration file exists, start the server. exec /usr/local/bin/stalwart --config /opt/stalwart/etc/config.toml ================================================ FILE: resources/html-templates/calendar-alarm.html ================================================ {{page_title}}

================================================ FILE: resources/html-templates/calendar-alarm.html.min ================================================ {{page_title}}
Logo
{{#if event_description}}{{/if event_description}} {{#each event_details}}{{/each event_details}} {{#if attendees}}{{/if attendees}}
{{header}}
{{event_title}}
{{event_description}}
{{key}}: {{value}}
{{attendees_title}}:
{{#each attendees}}• {{key}} <{{value}}>
{{/each attendees}}
{{action_name}}

{{footer}}
================================================ FILE: resources/html-templates/calendar-alarm.mjml ================================================ {{title}} :root { color-scheme: light only; } .event-detail { font-weight: bold; color: #2c5aa0; } .guest-list { background-color: #f8f9fa; padding: 10px; border-radius: 4px; margin-top: 5px; } {{upcoming_event}} {{event_title}} {{event_description}} {{field_name}}: {{field_value}} {{attendees}}:
• {{guest_name}} ({{guest_email}})
 {{open_button}} {{footer}} ================================================ FILE: resources/html-templates/calendar-invite.html ================================================ {{page_title}}
{{#if header}}
{{/if header}}
{{#each event_details}} {{/each event_details}} {{#if attendees}} {{/if attendees}} {{#if rsvp}} {{/if rsvp}}
{{key}} {{#if changed}}{{changed}} {{/if changed}}
{{#if old_value}}
{{old_value}}
{{/if old_value}}
{{value}}
{{attendees_title}}
{{#each attendees}}
• {{key}} ({{value}})
{{/each attendees}}

{{rsvp}}
{{#if rsvp}}
{{#each actions}}
{{/each actions}}
{{/if rsvp}} {{#if footer}}
{{#each footer}} {{/each footer}}
{{key}}
{{/if footer}}
================================================ FILE: resources/html-templates/calendar-invite.html.min ================================================ {{page_title}}
{{#if header}}
{{/if header}}
{{#each event_details}}{{/each event_details}} {{#if attendees}}{{/if attendees}} {{#if rsvp}}{{/if rsvp}}
{{key}} {{#if changed}}{{changed}}{{/if changed}}
{{#if old_value}}
{{old_value}}
{{/if old_value}}
{{value}}
{{attendees_title}}
{{#each attendees}}
• {{key}} ({{value}})
{{/each attendees}}

{{rsvp}}
{{#if rsvp}}
{{#each actions}}
{{/each actions}}
{{/if rsvp}} {{#if footer}}
{{#each footer}}{{/each footer}}
{{key}}
{{/if footer}}
================================================ FILE: resources/html-templates/calendar-invite.mjml ================================================ Event Changed An event you're attending has been updated .changed-pill { background-color: #4CAF50; color: white; padding: 4px 8px; border-radius: 12px; font-size: 8px; font-weight: bold; margin-left: 8px; display: inline-block; } .event-detail { margin-bottom: 16px; } .event-detail-label { font-weight: bold; color: #666666; margin-bottom: 4px; } .guest-item { margin-bottom: 4px; } .strikethrough { text-decoration: line-through; color: #999999; } This event has been updated
Title
Annual Team Building Workshop 2025
Description
Join us for our annual team building workshop featuring interactive activities, networking sessions, and strategic planning for the upcoming year. This year's theme focuses on collaboration and innovation.
When CHANGED
Friday, March 15, 2025 9:00 AM - 5:00 PM (PST)
Friday, March 15, 2025 9:00 AM - 5:00 PM (PST)
Location
Grand Conference Center
123 Business Park Drive
San Francisco, CA 94105
Guest List
• John Smith (john.smith@company.com)
+ 12 more attendees
 Reply as email@domain.com for this event series: YES NO MAYBE You’re receiving this e-mail as you're listed as a participant for this event. Forwarding this e-mail could allow any recipient to reply to the organizer, join the guest list, extend the invitation to others, or alter your RSVP. ================================================ FILE: resources/locales/i18n.yml ================================================ # SPDX-FileCopyrightText: 2020 Stalwart Labs LLC # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL calendar.alarm_subject_prefix: en: Notification es: Notificación fr: Notification de: Benachrichtigung it: Notifica pt: Notificação nl: Melding da: Notifikation ca: Notificació el: Ειδοποίηση sv: Notifikation pl: Powiadomienie calendar.alarm_header: en: You have an upcoming event es: Tienes un evento próximo fr: Vous avez un événement à venir de: Sie haben einen bevorstehenden Termin it: Hai un evento in programma pt: Você tem um evento próximo nl: U heeft een aankomende gebeurtenis da: Du har en kommende begivenhed ca: Tens un esdeveniment d'aquí poc el: Έχετε μία επερχόμενη εκδήλωση sv: Du har en kommande händelse pl: Masz nadchodzące wydarzenie calendar.alarm_footer: en: You are receiving this email because you have enabled calendar notifications. To stop receiving these emails, login to the self-service portal and disable event notifications. es: Recibe este correo porque ha habilitado las notificaciones de calendario. Para dejar de recibir estos correos, inicie sesión en el portal de autoservicio y desactive las notificaciones de eventos. fr: Vous recevez cet e-mail car vous avez activé les notifications de calendrier. Pour arrêter de recevoir ces e-mails, connectez-vous au portail libre-service et désactivez les notifications d'événements. de: Sie erhalten diese E-Mail, weil Sie Kalender-Benachrichtigungen aktiviert haben. Um diese E-Mails nicht mehr zu erhalten, melden Sie sich im Self-Service-Portal an und deaktivieren Sie Ereignisbenachrichtigungen. it: Ricevi questa email perché hai abilitato le notifiche del calendario. Per smettere di ricevere queste email, accedi al portale self-service e disabilita le notifiche degli eventi. pt: Você está recebendo este e-mail porque habilitou as notificações do calendário. Para parar de receber estes e-mails, faça login no portal de autoatendimento e desative as notificações de eventos. nl: U ontvangt deze e-mail omdat u kalendernotificaties heeft ingeschakeld. Om deze e-mails niet meer te ontvangen, logt u in op de selfservice-portal en schakelt u gebeurtenismeldingen uit. da: Du modtager denne e-mail, fordi du har aktiveret kalendernotifikationer. Du kan deaktivere notifikationer i selvbetjeningsportalen. ca: Estàs revent aquest correu perquè tens habilitada les notificacions del calendari. Per no rebre més aquests correus, identificat al portal d'autoservei i desactiva les notificacions d'esdeveniments. el: Λαμβάνεται αυτό το e-mail γιατί έχετε ένεργοποιήσει τις ειδοποιήσεις ημερολογίου. Για διακοπή της λήψης τους, μπείτε στη πύλη αυτοεξυπηρέτησης και απενεργοποιείστε τις ειδοποιήσεις εκδηλώσεων. sv: Du får det här mejlet för att du har slagit på kalendernotifikationer. För att sluta få dessa mejl, stäng av inställningen i självbetjäningsportalen. pl: Otrzymujesz tę wiadomość, ponieważ włączyłeś powiadomienia kalendarza. Aby przestać je otrzymywać, zaloguj się do portalu samoobsługowego i wyłącz powiadomienia o wydarzeniach. calendar.alarm_open: en: View Event es: Ver Evento fr: Voir l'Événement de: Termin Anzeigen it: Visualizza Evento pt: Ver Evento nl: Gebeurtenis Bekijken da: Vis begivenhed ca: Veure esdeveniment el: Προβολή Εκδήλωσης sv: Visa händelse pl: Wyświetl wydarzenie calendar.organizer: en: Organizer es: Organizador fr: Organisateur de: Organisator it: Organizzatore pt: Organizador nl: Organisator da: Arrangør ca: Organitzador el: Διοργανωτής sv: Arrangör pl: Organizator calendar.attendees: en: Guests es: Invitados fr: Invités de: Gäste it: Ospiti pt: Convidados nl: Gasten da: Gæster ca: Invitats el: Συμμετέχωντες sv: Gäster pl: Goście calendar.start: en: Start es: Inicio fr: Début de: Beginn it: Inizio pt: Início nl: Begin da: Start ca: Inici el: Έναρξη sv: Start pl: Start calendar.end: en: End es: Fin fr: Fin de: Ende it: Fine pt: Fim nl: Einde da: Slut ca: Fi el: Λήξη sv: Slut pl: Koniec calendar.location: en: Location es: Ubicación fr: Lieu de: Ort it: Luogo pt: Local nl: Locatie da: Lokation ca: Lloc el: Τοποθεσία sv: Plats pl: Lokalizacja calendar.date_template: # English: "Sun May 25, 2025 9:30am" en: "%a %b %-d, %Y %-I:%M%P" # Spanish: "dom 25 may 2025 9:30h" (day month year hour and minute) es: "%a %-d %b %Y %-H:%Mh" # French: "dim 25 mai 2025 9:30h" (day month year hour and minute) fr: "%a %-d %b %Y %-H:%Mh" # German: "So 25. Mai 2025 9:30 Uhr" (day date month year hour and minute) de: "%a %-d. %b %Y %-H:%M Uhr" # Italian: "dom 25 mag 2025 ore 9:30" (day date month year hour and minute) it: "%a %-d %b %Y ore %-H:%M" # Portuguese: "dom 25 mai 2025 9:30h" (day date month year hour and minute) pt: "%a %-d %b %Y %-H:%Mh" # Dutch: "zo 25 mei 2025 9:30u" (weekday date month year hour and minute) nl: "%a %-d %b %Y %-H:%Mu" # Danish: "søn 25. maj 2025 kl. 9:30" (weekday date month year hour and minute) da: "%a %-d. %b %Y kl. %-H:%M" # Catalan: "diu 25 mai 2025 9:30h" (day month year hour and minute) ca: "%a %-d %b %Y %-H:%Mh" # Greek: "Κυρ 25 Μαϊ 2025 9:30 η ώρα" (day month year hour and minute) el: "%a %-d %b %Y %-H:%M η ώρα" # Svenska: "sön 25 maj 2025 kl. 9:30" (weekday date month year hour and minute) sv: "%a %-d %b %Y kl. %-H:%M" # Polish: "nie 25 maj 2025 09:00" (weekday day month year hour and minute) pl: "%a %d %b %Y %H:%M" calendar.date_template_long: # English: "Sunday May 25, 2025 9:30am" en: "%A %B %-d, %Y %-I:%M%P" # Spanish: "domingo 25 mayo 2025 9:30h" (day month year hour and minute) es: "%A %-d %B %Y %-H:%Mh" # French: "dimanche 25 mai 2025 9:30h" (day month year hour and minute) fr: "%A %-d %B %Y %-H:%Mh" # German: "Sonntag 25. Mai 2025 9:30 Uhr" (day date month year hour and minute) de: "%A %-d. %B %Y %-H:%M Uhr" # Italian: "domenica 25 maggio 2025 ore 9:30" (day date month year hour and minute) it: "%A %-d %B %Y ore %-H:%M" # Portuguese: "domingo 25 maio 2025 9:30h" (day date month year hour and minute) pt: "%A %-d %B %Y %-H:%Mh" # Dutch: "zondag 25 mei 2025 9:30u" (weekday date month year hour and minute) nl: "%A %-d %B %Y %-H:%Mu" # Danish: "søndag 25. maj 2025 kl. 9:30" (weekday date month year hour and minute) da: "%A %-d. %B %Y kl. %-H:%M" # Catalan: "diumenge 25 maig 2025 9:30h" (day month year hour and minute) ca: "%A %-d %B %Y %-H:%Mh" # Greek: "Κυριακή 25 Μαΐου 2025 9:30 η ώρα" (day month year hour and minute) el: "%A %-d %B %Y %-H:%M η ώρα" # Svenska: "söndag 25 maj 2025 kl. 9:30" (weekday date month year hour and minute) sv: "%A %-d. %B %Y kl. %-H:%M" # Polish: "niedziela 25 maj 2025 09:00" (weekday day month year hour and minute) pl: "%A %d %b %Y %H:%M" calendar.invitation: en: Invitation es: Invitación fr: Invitation de: Einladung it: Invito pt: Convite nl: Uitnodiging da: Invitation ca: Invitació el: Πρόσκληση sv: Inbjudan pl: Zaproszenie calendar.updated_invitation: en: Updated invitation es: Invitación actualizada fr: Invitation mise à jour de: Aktualisierte Einladung it: Invito aggiornato pt: Convite atualizado nl: Bijgewerkte uitnodiging da: Opdateret invitation ca: Invitació actualitzada el: Ενημερωμένη πρόσκληση sv: Uppdaterad inbjudan pl: Zaktualizowane zaproszenie calendar.event_updated: en: This event has been updated es: Este evento ha sido actualizado fr: Cet événement a été mis à jour de: Dieses Ereignis wurde aktualisiert it: Questo evento è stato aggiornato pt: Este evento foi atualizado nl: Dit evenement is bijgewerkt da: Denne begivenhed er blevet opdateret ca: Aquest esdeveniment s'ha actualitzat el: Αυτή η εκδήλωση ενημερώθηκε sv: Den här händelsen har blivit uppdaterad pl: To wydarzenie zostało zaktualizowane calendar.cancelled: en: Cancelled es: Cancelado fr: Annulé de: Abgesagt it: Annullato pt: Cancelado nl: Geannuleerd da: Aflyst ca: Cancel·lat el: Ακυρώθηκε sv: Inställd pl: Anulowane calendar.event_cancelled: en: This event has been canceled es: Este evento ha sido cancelado fr: Cet événement a été annulé de: Dieses Ereignis wurde abgesagt it: Questo evento è stato annullato pt: Este evento foi cancelado nl: Dit evenement is geannuleerd da: Denne begivenhed er blevet aflyst ca: Aquest esdeveniment s'ha cancel·lat el: Αυτή η εκδήλωση ακυρώθηκε sv: Den här händelsen har blivit inställd pl: To wydarzenie zostało anulowane calendar.accepted: en: Accepted es: Aceptado fr: Accepté de: Angenommen it: Accettato pt: Aceito nl: Geaccepteerd da: Accepteret ca: Acceptat el: Αποδοχή sv: Accepterat pl: Zaakceptowano calendar.participant_accepted: en: Participant $name accepted the invitation es: El participante $name aceptó la invitación fr: Le participant $name a accepté l'invitation de: Teilnehmer $name hat die Einladung angenommen it: Il partecipante $name ha accettato l'invito pt: O participante $name aceitou o convite nl: Deelnemer $name heeft de uitnodiging geaccepteerd da: Deltager $name accepterede invitationen ca: El participatn $name a acceptat la invitació el: Το συμμετέχων πρόσωπο $name αποδέχτηκε τη πρόσκληση sv: Deltagaren $name har tackat ja till inbjudan pl: Uczestnik $name zaakceptował zaproszenie calendar.declined: en: Declined es: Rechazado fr: Refusé de: Abgelehnt it: Rifiutato pt: Recusado nl: Afgewezen da: Afvist ca: Rebutjat el: Απορρίφθηκε sv: Avböjt pl: Odrzucono calendar.participant_declined: en: Participant $name declined the invitation es: El participante $name rechazó la invitación fr: Le participant $name a refusé l'invitation de: Teilnehmer $name hat die Einladung abgelehnt it: Il partecipante $name ha rifiutato l'invito pt: O participante $name recusou o convite nl: Deelnemer $name heeft de uitnodiging afgewezen da: Deltager $name afslog invitationen ca: El participatn $name ha rebutjat la invitació el: Το συμμετέχων πρόσωπο $name απέρριψε τη πρόσκληση sv: Deltagaren $name har tackat nej till inbjudan pl: Uczestnik $name odrzucił zaproszenie calendar.tentative: en: Tentative es: Provisional fr: Provisoire de: Vorläufig it: Provvisorio pt: Provisório nl: Voorlopig da: Foreløbig ca: Provisional el: Μη δεσμευτικά sv: Möjligen pl: Wstępnie calendar.participant_tentative: en: Participant $name tentatively accepted the invitation es: El participante $name aceptó provisionalmente la invitación fr: Le participant $name a accepté provisoirement l'invitation de: Teilnehmer $name hat die Einladung vorläufig angenommen it: Il partecipante $name ha accettato provvisoriamente l'invito pt: O participante $name aceitou provisoriamente o convite nl: Deelnemer $name heeft de uitnodiging voorlopig geaccepteerd da: Deltager $name accepterede foreløbigt invitationen ca: El participan $name accepta provisionalment la invitació a l'esdeveniment el: Το συμμετέχων πρόσωπο $name αποδέχτηκε μη δεσμευτικά τη πρόσκληση sv: Deltagaren $name har preliminärt tackat ja till inbjudan pl: Uczestnik $name wstępnie zaakceptował zaproszenie calendar.delegated: en: Delegated es: Delegado fr: Délégué de: Delegiert it: Delegato pt: Delegado nl: Gedelegeerd da: Delegeret ca: Delegat el: Ανατέθηκε sv: Delegerat pl: Delegowane calendar.participant_delegated: en: Participant $name delegated the invitation es: El participante $name delegó la invitación fr: Le participant $name a délégué l'invitation de: Teilnehmer $name hat die Einladung delegiert it: Il partecipante $name ha delegato l'invito pt: O participante $name delegou o convite nl: Deelnemer $name heeft de uitnodiging gedelegeerd da: Deltager $name delegerede invitationen ca: El participant $name ha delegat la invitació el: Το συμμετέχων πρόσωπο $name ανέθεσε τη πρόσκληση sv: Deltagaren $name delegerade inbjudan pl: Uczestnik $name przekazał zaproszenie calendar.reply: en: Reply es: Respuesta fr: Réponse de: Antwort it: Risposta pt: Resposta nl: Antwoord da: Svar ca: Resposta el: Απάντηση sv: Svar pl: Odpowiedź calendar.participant_reply: en: Participant $name replied to the invitation es: El participante $name respondió a la invitación fr: Le participant $name a répondu à l'invitation de: Teilnehmer $name hat auf die Einladung geantwortet it: Il partecipante $name ha risposto all'invito pt: O participante $name respondeu ao convite nl: Deelnemer $name heeft gereageerd op de uitnodiging da: Deltager $name svarede på invitationen ca: El participant $name ha respost a la invitació el: Το συμμετέχων πρόσωπο $name απάντησε στη πρόσκληση sv: Deltagaren $name har svarat på inbjudan pl: Uczestnik $name odpowiedział na zaproszenie calendar.summary: en: Summary es: Resumen fr: Résumé de: Zusammenfassung it: Riepilogo pt: Resumo nl: Samenvatting da: Resumé ca: Resum el: Περίληψη sv: Sammanfattning pl: Podsumowanie calendar.description: en: Description es: Descripción fr: Description de: Beschreibung it: Descrizione pt: Descrição nl: Beschrijving da: Beskrivelse ca: Descripció el: Περιγραφή sv: Beskrivning pl: Opis calendar.when: en: When es: Cuándo fr: Quand de: Wann it: Quando pt: Quando nl: Wanneer da: Hvornår ca: Quan el: Πότε sv: När pl: Kiedy calendar.changed: en: Changed es: Cambiado fr: Modifié de: Geändert it: Modificato pt: Alterado nl: Gewijzigd da: Ændret ca: Canviat el: Μεταβλήθηκε sv: Ändrat pl: Zmieniono calendar.reply_as: en: Reply as $name for this event series es: Responder como $name para esta serie de eventos fr: Répondre en tant que $name pour cette série d'événements de: Als $name für diese Ereignisserie antworten it: Rispondi come $name per questa serie di eventi pt: Responder como $name para esta série de eventos nl: Antwoord als $name voor deze evenementenreeks da: Svar som $name for denne begivenhedsserie ca: Respondre com a $name per aquesta serie d'esdeveniments el: Απάντηση ως $name για αυτή τη σειρά εκδηλώσεων sv: Svara som $name på den här inbjudan pl: Odpowiedz jako $name dla tej serii wydarzeń calendar.yes: en: Yes es: Sí fr: Oui de: Ja it: Sì pt: Sim nl: Ja da: Ja ca: Sí el: Ναι sv: Ja pl: Tak calendar.no: en: No es: No fr: Non de: Nein it: No pt: Não nl: Nee da: Nej ca: No el: Όχι sv: Nej pl: Nie calendar.maybe: en: Maybe es: Quizás fr: Peut-être de: Vielleicht it: Forse pt: Talvez nl: Misschien da: Måske ca: Potser el: Ίσως sv: Kanske pl: Może calendar.imip_footer_1: en: You're receiving this e-mail as you're listed as a participant for this event. es: Recibes este correo electrónico porque estás registrado como participante de este evento. fr: Vous recevez cet e-mail car vous êtes inscrit comme participant à cet événement. de: Sie erhalten diese E-Mail, weil Sie als Teilnehmer für dieses Ereignis aufgeführt sind. it: Ricevi questa e-mail perché sei elencato come partecipante a questo evento. pt: Você está recebendo este e-mail porque está listado como participante deste evento. nl: U ontvangt deze e-mail omdat u staat vermeld als deelnemer aan dit evenement. da: Du modtager denne e-mail, fordi du er opført som deltager i denne begivenhed. ca: Reps aquest correu electrònic perquè ets registrat com a participant d'aquest esdeveniment. el: Λαμβάνετε αυτό το e-mail καθώς είστε στη λίστα συμμετεχόντων αυτής της εκδήλωσης. sv: Du får det här mejlet för att du är uppskriven som deltagare i den här händelsen pl: Otrzymujesz tego e-maila, ponieważ jesteś uczestnikiem tego wydarzenia. calendar.imip_footer_2: en: Forwarding this e-mail could allow any recipient to reply to the organizer, join the guest list, extend the invitation to others, or alter your RSVP. es: Reenviar este correo electrónico podría permitir que cualquier destinatario responda al organizador, se una a la lista de invitados, extienda la invitación a otros o modifique tu confirmación de asistencia. fr: Le transfert de cet e-mail pourrait permettre à tout destinataire de répondre à l'organisateur, de rejoindre la liste des invités, d'étendre l'invitation à d'autres ou de modifier votre RSVP. de: Das Weiterleiten dieser E-Mail könnte es jedem Empfänger ermöglichen, dem Organisator zu antworten, der Gästeliste beizutreten, die Einladung an andere weiterzugeben oder Ihre Zusage zu ändern. it: L'inoltro di questa e-mail potrebbe consentire a qualsiasi destinatario di rispondere all'organizzatore, unirsi alla lista degli ospiti, estendere l'invito ad altri o modificare la tua conferma di partecipazione. pt: Encaminhar este e-mail pode permitir que qualquer destinatário responda ao organizador, se junte à lista de convidados, estenda o convite a outros ou altere sua confirmação de presença. nl: Het doorsturen van deze e-mail kan elke ontvanger in staat stellen om te reageren op de organisator, deel te nemen aan de gastenlijst, de uitnodiging uit te breiden naar anderen, of uw RSVP te wijzigen. da: Videresendelse af denne e-mail kan give andre mulighed for at svare arrangøren, tilslutte sig gæstelisten, videreformidle invitationen eller ændre din tilmelding. ca: Reenviar aquest correu electrònic podria fer que qualsevol destinatari respongues a l'organitzador, afegir-se a la llista de convidats, estengui la invitació a d'altres o modificar la teva confirmació d'assistència. el: Προωθόντας αυτό το e-mail μπορεί να επιτρέψει σε οποιοδήποτε παραλήπτη να απαντήση στον οργανωτή, να μπει στη λίστα επισκεπτών, να επεκτείνει τη πρόσκληση σε άλλους ή να αλλάξει την παρουσία σας. sv: Att vidarebefordra det här mejlet kan göra det möjligt för vilken mottagare som helst att skicka svar till arrangören, lägga till sig själv på gästlistan, skicka inbjudan vidare till andra, samt ändra ditt svar. pl: Przekazanie tej wiadomości e-mail może umożliwić każdemu odbiorcy odpowiedź do organizatora, dołączenie do listy gości, przesłanie zaproszenia innym osobom lub zmianę Twojej odpowiedzi RSVP. calendar.rsvp_recorded: en: Your RSVP has been recorded. es: Tu confirmación de asistencia ha sido registrada. fr: Votre RSVP a été enregistré. de: Ihre Zusage wurde aufgezeichnet. it: La tua conferma di partecipazione è stata registrata. pt: Sua confirmação de presença foi registrada. nl: Uw RSVP is geregistreerd. da: Din tilmelding er blevet registreret. ca: La teva confirmació d'assistència ha sigut registrada. el: Η κοινοποίηση της παρουσίας σας καταγράφηκε. sv: Ditt svar har registrerats. pl: Twoja odpowiedź RSVP została zarejestrowana. calendar.rsvp_failed: en: Failed to record your RSVP. es: No se pudo registrar tu confirmación de asistencia. fr: Impossible d'enregistrer votre RSVP. de: Ihre Zusage konnte nicht aufgezeichnet werden. it: Impossibile registrare la tua conferma di partecipazione. pt: Falha ao registrar sua confirmação de presença. nl: Kan uw RSVP niet registreren. da: Kunne ikke registrere din tilmelding. ca: No s'ha pogut registrar la teva confirmació d'assistència. el: Αποτυχία κατα τη καταγραφή της παρουσίας σας sv: Det gick inte att registrera ditt svar. pl: Nie udało się zarejestrować Twojej odpowiedzi RSVP. calendar.event_not_found: en: The event you are trying to RSVP to was not found. es: No se encontró el evento al que intentas confirmar asistencia. fr: L'événement auquel vous essayez de répondre n'a pas été trouvé. de: Das Ereignis, für das Sie eine Zusage geben möchten, wurde nicht gefunden. it: L'evento a cui stai cercando di confermare la partecipazione non è stato trovato. pt: O evento para o qual você está tentando confirmar presença não foi encontrado. nl: Het evenement waarvoor u probeert te reageren is niet gevonden. da: Begivenheden du forsøger at tilmelde dig blev ikke fundet. ca: L'esdeveniment al que intentes confirmar l'assistència no es troba. el: Η εκδήλωση που θέλετε να κοινοποιήσετε την παρουσία σας δεν υπάρχει. sv: Händelsen du försöker OSA till kunde inte hittas. pl: Wydarzenie, na które próbujesz odpowiedzieć RSVP, nie zostało znalezione. calendar.invalid_rsvp: en: The RSVP request was invalid or malformed. es: La solicitud de confirmación de asistencia era inválida o estaba mal formada. fr: La demande de RSVP était invalide ou mal formée. de: Die Zusage-Anfrage war ungültig oder fehlerhaft. it: La richiesta di conferma di partecipazione era non valida o mal formata. pt: A solicitação de confirmação de presença era inválida ou mal formada. nl: Het RSVP-verzoek was ongeldig of onjuist gevormd. da: Tilmeldingsanmodningen var ugyldig eller forkert udformet. ca: La sol·licitud de confirmació d'assistència no era vàlida o estava mal formada. el: Η αίτητη για κοινοποίηση παρουσίας είναι άκυρη ή έχει πρόβλημα. sv: Svaret på inbjudan var ogiltigt eller i fel format. pl: Żądanie RSVP było nieprawidłowe lub źle sformułowane. calendar.not_participant: en: You are no longer a participant in this event. es: Ya no eres participante de este evento. fr: Vous n'êtes plus participant à cet événement. de: Sie sind kein Teilnehmer dieses Ereignisses mehr. it: Non sei più un partecipante a questo evento. pt: Você não é mais um participante deste evento. nl: U bent geen deelnemer meer aan dit evenement. da: Du deltager ikke længere i denne begivenhed. ca: Ja no ets un participant d'aquest esdeveniment. el: Δε συμμετέχετε πια σε αυτή την εκδήλωση. sv: Du är inte längre en deltagare i den här händelse. pl: Nie jesteś już uczestnikiem tego wydarzenia. ================================================ FILE: resources/scripts/ossify.py ================================================ #!/usr/bin/env python3 """ Stalwart SEL code remover This script removes SEL code from the Stalwart codebase by: 1. Removing entire .rs files that contain "SPDX-License-Identifier: LicenseRef-SEL" in their first comment 2. Removing SEL snippets marked with SPDX-SnippetBegin/End from mixed files Usage: python ossify.py /crates """ import os import sys import re import argparse from pathlib import Path from typing import List, Tuple, Optional def find_first_comment_block(content: str) -> Optional[str]: """ Find the first comment block in a Rust file. Returns the comment content or None if no comment block is found. """ # Remove leading whitespace and find the first comment lines = content.strip().split('\n') if not lines: return None first_line = lines[0].strip() # Check for block comment starting with /* if first_line.startswith('/*'): comment_lines = [] in_comment = True for line in lines: if in_comment: comment_lines.append(line) if '*/' in line: break return '\n'.join(comment_lines) # Check for line comments starting with // elif first_line.startswith('//'): comment_lines = [] for line in lines: stripped = line.strip() if stripped.startswith('//'): comment_lines.append(line) elif stripped == '': comment_lines.append(line) # Keep empty lines within comment block else: break # Stop at first non-comment, non-empty line return '\n'.join(comment_lines) return None def should_remove_file(file_path: str) -> bool: """ Check if a .rs file should be completely removed based on its first comment. Returns True if the file contains "SPDX-License-Identifier: LicenseRef-SEL" in the first comment. """ try: with open(file_path, 'r', encoding='utf-8') as f: content = f.read() first_comment = find_first_comment_block(content) if first_comment and 'SPDX-License-Identifier: LicenseRef-SEL' in first_comment: return True except Exception as e: print(f"Error reading file {file_path}: {e}") return False def remove_proprietary_snippets(content: str) -> Tuple[str, int]: """ Remove proprietary snippets from file content. Returns tuple of (modified_content, number_of_snippets_removed) """ snippets_removed = 0 # Pattern to match SPDX snippets that contain LicenseRef-SEL # We look for SPDX-SnippetBegin, then check if the snippet contains LicenseRef-SEL, # and if so, remove everything until SPDX-SnippetEnd lines = content.split('\n') result_lines = [] i = 0 while i < len(lines): line = lines[i] # Check if this line starts a snippet if '// SPDX-SnippetBegin' in line: # Look ahead to see if this snippet contains LicenseRef-SEL snippet_start = i snippet_lines = [] j = i # Collect the snippet lines until we find SnippetEnd or reach end of file while j < len(lines): snippet_lines.append(lines[j]) if '// SPDX-SnippetEnd' in lines[j]: break j += 1 # Check if this snippet contains LicenseRef-SEL snippet_content = '\n'.join(snippet_lines) if 'SPDX-License-Identifier: LicenseRef-SEL' in snippet_content: # Remove this snippet snippets_removed += 1 i = j + 1 # Skip past the SnippetEnd line continue else: # Keep this snippet as it's not proprietary result_lines.append(line) i += 1 else: result_lines.append(line) i += 1 return '\n'.join(result_lines), snippets_removed def process_rust_file(file_path: str, dry_run: bool = False) -> dict: """ Process a single Rust file, removing proprietary content. Returns a dictionary with processing results. """ result = { 'file': file_path, 'action': 'none', 'snippets_removed': 0, 'error': None } try: # Check if the entire file should be removed if should_remove_file(file_path): result['action'] = 'file_removed' if not dry_run: os.remove(file_path) return result # Process snippets in the file with open(file_path, 'r', encoding='utf-8') as f: original_content = f.read() modified_content, snippets_removed = remove_proprietary_snippets(original_content) if snippets_removed > 0: result['action'] = 'snippets_removed' result['snippets_removed'] = snippets_removed if not dry_run: with open(file_path, 'w', encoding='utf-8') as f: f.write(modified_content) except Exception as e: result['error'] = str(e) return result def find_rust_files(directory: str) -> List[str]: """Find all .rs files in the given directory recursively.""" rust_files = [] for root, dirs, files in os.walk(directory): for file in files: if file.endswith('.rs'): rust_files.append(os.path.join(root, file)) return rust_files def main(): parser = argparse.ArgumentParser( description='Remove Enterprise licensed code from Stalwart codebase' ) parser.add_argument( 'directory', help='Directory containing Stalwart code to process' ) parser.add_argument( '--dry-run', action='store_true', help='Show what would be done without making changes' ) parser.add_argument( '--verbose', action='store_true', help='Show detailed output for each file' ) args = parser.parse_args() if not os.path.isdir(args.directory): print(f"Error: {args.directory} is not a valid directory") sys.exit(1) print(f"Processing Rust files in: {args.directory}") if args.dry_run: print("DRY RUN MODE - No changes will be made") print() rust_files = find_rust_files(args.directory) if not rust_files: print("No .rs files found in the specified directory") return print(f"Found {len(rust_files)} Rust files") print() files_removed = 0 files_with_snippets_removed = 0 total_snippets_removed = 0 errors = [] for file_path in rust_files: result = process_rust_file(file_path, args.dry_run) if result['error']: errors.append(f"{file_path}: {result['error']}") continue if result['action'] == 'file_removed': files_removed += 1 if args.verbose or args.dry_run: action_text = "Would remove" if args.dry_run else "Removed" print(f"{action_text} file: {file_path}") elif result['action'] == 'snippets_removed': files_with_snippets_removed += 1 total_snippets_removed += result['snippets_removed'] if args.verbose or args.dry_run: action_text = "Would remove" if args.dry_run else "Removed" print(f"{action_text} {result['snippets_removed']} snippet(s) from: {file_path}") # Summary print("\nSummary:") action_text = "Would be" if args.dry_run else "Were" print(f"- {files_removed} files {action_text.lower()} completely removed") print(f"- {total_snippets_removed} proprietary snippets {action_text.lower()} removed from {files_with_snippets_removed} files") if errors: print(f"- {len(errors)} errors occurred:") for error in errors: print(f" {error}") if args.dry_run: print("\nRun without --dry-run to apply changes") if __name__ == '__main__': main() ================================================ FILE: resources/systemd/stalwart-mail.service ================================================ [Unit] Description=Stalwart Server Conflicts=postfix.service sendmail.service exim4.service ConditionPathExists=__PATH__/etc/config.toml After=network-online.target [Service] Type=simple LimitNOFILE=65536 KillMode=process KillSignal=SIGINT Restart=on-failure RestartSec=5 ExecStart=__PATH__/bin/stalwart --config=__PATH__/etc/config.toml SyslogIdentifier=stalwart User=stalwart Group=stalwart AmbientCapabilities=CAP_NET_BIND_SERVICE [Install] WantedBy=multi-user.target ================================================ FILE: resources/systemd/stalwart.mail.plist ================================================ Label stalwart.mail ServiceDescription Stalwart ProgramArguments __PATH__/bin/stalwart --config=__PATH__/etc/config.toml RunAtLoad KeepAlive ================================================ FILE: tests/Cargo.toml ================================================ [package] name = "tests" version = "0.15.5" edition = "2024" [features] #default = ["sqlite", "postgres", "mysql", "rocks", "s3", "redis", "nats", "azure", "foundationdb"] default = ["sqlite", "postgres", "mysql", "rocks", "s3", "redis", "foundationdb"] #default = ["rocks", "foundationdb"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation", "common/foundation"] postgres = ["store/postgres"] mysql = ["store/mysql"] rocks = ["store/rocks"] s3 = ["store/s3"] redis = ["store/redis"] nats = ["store/nats"] azure = ["store/azure"] [dev-dependencies] store = { path = "../crates/store", features = ["test_mode", "enterprise"] } nlp = { path = "../crates/nlp" } directory = { path = "../crates/directory", features = ["test_mode", "enterprise"] } jmap = { path = "../crates/jmap", features = ["test_mode", "enterprise"] } jmap_proto = { path = "../crates/jmap-proto" } imap = { path = "../crates/imap", features = ["test_mode"] } imap_proto = { path = "../crates/imap-proto" } types = { path = "../crates/types" } dav = { path = "../crates/dav", features = ["test_mode"] } dav-proto = { path = "../crates/dav-proto", features = ["test_mode"] } calcard = { version = "0.3", features = ["rkyv"] } groupware = { path = "../crates/groupware", features = ["test_mode"] } http = { path = "../crates/http", features = ["test_mode", "enterprise"] } http_proto = { path = "../crates/http-proto" } services = { path = "../crates/services", features = ["test_mode", "enterprise"] } pop3 = { path = "../crates/pop3", features = ["test_mode"] } smtp = { path = "../crates/smtp", features = ["test_mode", "enterprise"] } common = { path = "../crates/common", features = ["test_mode", "enterprise"] } email = { path = "../crates/email", features = ["test_mode", "enterprise"] } spam-filter = { path = "../crates/spam-filter", features = ["test_mode", "enterprise"] } migration = { path = "../crates/migration", features = ["test_mode", "enterprise"] } trc = { path = "../crates/trc", features = ["enterprise"] } managesieve = { path = "../crates/managesieve", features = ["test_mode", "enterprise"] } smtp-proto = { version = "0.2" } mail-send = { version = "0.5", default-features = false, features = ["cram-md5", "ring", "tls12"] } mail-auth = { version = "0.7.1", features = ["test"] } mail-parser = { version = "0.11", features = ["full_encoding", "rkyv"] } mail-builder = "0.4.4" sieve-rs = { version = "0.7", features = ["rkyv"] } utils = { path = "../crates/utils", features = ["test_mode"] } jmap-client = { version = "0.4", features = ["websockets", "debug", "async"] } tokio = { version = "1.47", features = ["full"] } tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } rustls = { version = "0.23.5", default-features = false, features = ["std", "ring", "tls12"] } rustls-pemfile = "2.0" rustls-pki-types = { version = "1" } csv = "1.1" rayon = { version = "1.5.1" } flate2 = { version = "1.0.17", features = ["zlib"], default-features = false } serde = { version = "1.0", features = ["derive"]} serde_json = "1.0" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-webpki-roots", "multipart", "http2"]} bytes = "1.4.0" futures = "0.3" ece = "2.2" hyper = { version = "1.0.1", features = ["server", "http1", "http2"] } hyper-util = { version = "0.1.1", features = ["tokio"] } http-body-util = "0.1.0" base64 = "0.22" ahash = { version = "0.8" } serial_test = "3.0.0" num_cpus = "1.15.0" async-trait = "0.1.68" chrono = "0.4" ring = { version = "0.17" } biscuit = "0.7.0" form_urlencoded = "1.1.0" rkyv = { version = "0.8.10", features = ["little_endian"] } compact_str = "0.9.0" quick-xml = "0.38" [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.0" ================================================ FILE: tests/resources/acme/Docker.pebble ================================================ FROM golang:1.18-alpine as builder ENV CGO_ENABLED=0 WORKDIR /pebble-src RUN apk update && apk add --no-cache git RUN git clone https://github.com/letsencrypt/pebble/ /pebble-src RUN go build -o /go/bin/pebble ./cmd/pebble ## main FROM alpine:3.15.4 COPY --from=builder /go/bin/pebble /usr/bin/pebble COPY --from=builder /pebble-src/test/ /test/ CMD [ "/usr/bin/pebble" ] EXPOSE 14000 EXPOSE 15000 # Build: # docker build -f Docker.pebble -t pebble # Run: # docker run -d -p 14000:14000 -p 15000:15000 pebble # docker run -d --name pebble -p 14000:14000 -p 15000:15000 pebble pebble -config /test/config/pebble-config.json -strict ================================================ FILE: tests/resources/acme/config.toml ================================================ acme.pebble.contact = "postmaster@example.org" acme.pebble.directory = "https://localhost:14000/dir" #acme.pebble.domains = "mail.example.org" acme.pebble.renew-before = "30d" acme.pebble.challenge = "tls-alpn-01" #acme.pebble.challenge = "http-01" #acme.pebble.challenge = "dns-01" acme.pebble.domains = "*.example.org" acme.pebble.provider = "cloudflare" acme.pebble.secret = "" authentication.fallback-admin.secret = "secret" authentication.fallback-admin.user = "admin" config.local-keys.0 = "*" directory.internal.store = "rocksdb" directory.internal.type = "internal" lookup.default.hostname = "mail.example.org" lookup.default.domain = "example.org" oauth.key = "0Wn7rO4UdmBoE8mp3cDcD9Qlpz3na74z7fGRoSuq8fVsGPelLl3KrHomBN8h2biA" queue.quota.size.enable = true queue.quota.size.messages = 100000 queue.quota.size.size = 10737418240 report.analysis.addresses = "postmaster@*" server.http.permissive-cors = true server.listener.http.bind = "[::]:5002" server.listener.http.protocol = "http" server.listener.https.bind = "[::]:5001" server.listener.https.protocol = "http" server.listener.https.tls.implicit = true server.listener.imap.bind = "[::]:143" server.listener.imap.protocol = "imap" server.listener.imaptls.bind = "[::]:993" server.listener.imaptls.protocol = "imap" server.listener.imaptls.tls.implicit = true server.listener.sieve.bind = "[::]:4190" server.listener.sieve.protocol = "managesieve" server.listener.smtp.bind = "[::]:25" server.listener.smtp.protocol = "smtp" server.listener.submission.bind = "[::]:587" server.listener.submission.protocol = "smtp" server.listener.submissions.bind = "[::]:465" server.listener.submissions.protocol = "smtp" server.listener.submissions.tls.implicit = true storage.blob = "rocksdb" storage.data = "rocksdb" storage.directory = "internal" storage.fts = "rocksdb" storage.lookup = "rocksdb" store.rocksdb.compression = "lz4" store.rocksdb.path = "/tmp/stalwart-temp-data" store.rocksdb.type = "rocksdb" tracer.stdout.ansi = true tracer.stdout.enable = true tracer.stdout.level = "trace" tracer.stdout.type = "stdout" version.spam-filter = 1.0 ================================================ FILE: tests/resources/acme/docker-compose-pebble.yaml ================================================ # docker-compose -f docker-compose-pebble.yaml up # curl --request POST --data '{"ip":"192.168.5.2"}' http://localhost:8055/set-default-ipv4 # HTTPS port should be 5001 # HTTP port should be 5002 # Directory https://localhost:14000/dir version: '3' services: pebble: image: letsencrypt/pebble:latest command: pebble -config /test/config/pebble-config.json -strict -dnsserver 10.30.50.3:8053 #-dnsserver 8.8.8.8:53 ports: - 14000:14000 # HTTPS ACME API - 15000:15000 # HTTPS Management API networks: acmenet: ipv4_address: 10.30.50.2 challtestsrv: image: letsencrypt/pebble-challtestsrv:latest command: pebble-challtestsrv -defaultIPv6 "" -defaultIPv4 10.30.50.3 ports: - 8055:8055 # HTTP Management API networks: acmenet: ipv4_address: 10.30.50.3 networks: acmenet: driver: bridge ipam: driver: default config: - subnet: 10.30.50.0/24 ================================================ FILE: tests/resources/acme/test_acme.sh ================================================ #!/bin/sh rm -Rf /tmp/stalwart-temp-data mkdir -p /tmp/stalwart-temp-data cp ./tests/resources/acme/config.toml /tmp/stalwart-temp-data/config.toml curl --request POST --data '{"ip":"192.168.5.2"}' http://localhost:8055/set-default-ipv4 cargo run -p stalwart --no-default-features --features "sqlite foundationdb postgres mysql rocks elastic s3 redis" -- --config=/tmp/stalwart-temp-data/config.toml ================================================ FILE: tests/resources/crypto/cert_mixed.pem ================================================ -----BEGIN PGP PUBLIC KEY BLOCK----- xjMEZMYfNhYJKwYBBAHaRw8BAQdAYyTN1HzqapLw8xwkCGwa0OjsgT/JqhcB/+Dy Ga1fsBrNG0pvaG4gRG9lIDxqb2huQGV4YW1wbGUub3JnPsKJBBMWCAAxFiEEg836 pwbXpuQ/THMtpJwd4oBfIrUFAmTGHzYCGwMECwkIBwUVCAkKCwUWAgMBAAAKCRCk nB3igF8itYhyAQD2jEdeYa3gyQ47X9YWZTK1wEJkN8W9//V1fYl2XQwqlQEA0qBv Ai6nUh99oDw+/zQ8DFIKdeb5Ti4tu/X58PdpiQ7OOARkxh82EgorBgEEAZdVAQUB AQdAvXz2FbFN0DovQF/ACnZyczTsSIQp0mvmF1PE+aijbC8DAQgHwngEGBYIACAW IQSDzfqnBtem5D9Mcy2knB3igF8itQUCZMYfNgIbDAAKCRCknB3igF8itRnoAQC3 GzPmgx7TnB+SexPuJV/DoKSMJ0/X+hbEFcZkulxaDQEAh+xiJCvf+ZNAKw6kFhsL UuZhEDktxnY6Ehz3aB7FawA= =KGrr -----END PGP PUBLIC KEY BLOCK----- -----BEGIN CERTIFICATE----- MIIDbjCCAlagAwIBAgIUZ4K0WXNSS8H0cUcZavD9EYqqTAswDQYJKoZIhvcNAQEN BQAwLTErMCkGA1UEAxMiU2FtcGxlIExBTVBTIENlcnRpZmljYXRlIEF1dGhvcml0 eTAgFw0xOTExMjAwNjU0MThaGA8yMDUyMDkyNzA2NTQxOFowGTEXMBUGA1UEAxMO QWxpY2UgTG92ZWxhY2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDD 7q35ZdG2JAzzJGNZDZ9sV7AKh0hlRfoFjTZN5m4RegQAYSyag43ouWi1xRN0avf0 UTYrwjK04qRdV7GzCACoEKq/xiNUOsjfJXzbCublN3fZMOXDshKKBqThlK75SjA9 Czxg7ejGoiY/iidk0e91neK30SCCaBTJlfR2ZDrPk73IPMeksxoTatfF9hw9dDA+ /Hi1yptN/aG0Q/s9icFrxr6y2zQXsjuQPmjMZgj10aD9cazWVgRYCgflhmA0V1uQ l1wobYU8DAVxVn+GgabqyjGQMoythIK0Gn5+ofwxXXUM/zbU+g6+1ISdoXxRRFtq 2GzbIqkAHZZQm+BbnFrhAgMBAAGjgZcwgZQwDAYDVR0TAQH/BAIwADAeBgNVHREE FzAVgRNhbGljZUBzbWltZS5leGFtcGxlMBMGA1UdJQQMMAoGCCsGAQUFBwMEMA8G A1UdDwEB/wQFAwMHoAAwHQYDVR0OBBYEFKwuVFqk/VUYry7oZkQ40SXR1wB5MB8G A1UdIwQYMBaAFLdSTXPAiD2yw3paDPOU9/eAonfbMA0GCSqGSIb3DQEBDQUAA4IB AQB76o4Yz7yrVSFcpXqLrcGtdI4q93aKCXECCCzNQLp4yesh6brqaZHNJtwYcJ5T qbUym9hJ70iJE4jGNN+yAZR1ltte0HFKYIBKM4EJumG++2hqbUaLz4tl06BHaQPC v/9NiNY7q9R9c/B6s1YzHhwqkWht2a+AtgJ4BkpG+g+MmZMQV/Ao7RwLFKJ9OlMW LBmEXFcpIJN0HpPasT0nEl/MmotSu+8RnClAi3yFfyTKb+8rD7VxuyXetqDZ6dU/ 9/iqD/SZS7OQIjywtd343mACz3B1RlFxMHSA6dQAf2btGumqR0KiAp3KkYRAePoa JqYkB7Zad06ngFl0G0FHON+7 -----END CERTIFICATE----- ================================================ FILE: tests/resources/crypto/cert_pgp.pem ================================================ -----BEGIN PGP PUBLIC KEY BLOCK----- xsFNBGTGHwkBEADRB5EEtfsnUwgF2ZRg6h1fp2E8LNhv4lb9AWersI8KNFoWM6qx Bk/MfEpgILSPdW3g7PWHOxPV/hxjtStFHfbU/Ye5VvfbkU49faIPiw1V3MQJJ171 cN6kgMnABfdixNiutDkHP4f34ABrEqexX2myOP+btxL24gI/N9UpOD5PiKTyKR7i GwNpi+O022rs/KvjlWR7iSJ4vk7bGFfTNHvWI6dZworey1tZoTIZ0CgvgMeB/F1q OOa0FvrJdNYR227RpHmICqFqTptNZ2EfdkJ6QUXW7bZ9dWgL36ds9QPJOGcG3c5i JebeX5YdJnniBefiWjfZElcqh/N6SqVuEwoTLyMCnMZ6gjNMn6tddwPH24kavZhT p6+vhTHmyq8XBqK/XEt9r+clSfg2hi5s7GO7hQV+W26xRjX7sQJY41PfzkgYJ0BM 6+w09X1ZO/iMjEp44t2rd3xSudwGYhlbazXbdB+OJaa3RtyjOAeFgY8OyNlODx3V xXLtF+104HGSL7nkpBsu6LLighSgEEF2Vok43grr0omyb1NPhWoAZhM8sT5iv5gW fKvB1O13c+hDc/iGTAvcrtdLLnF2Cs+6HD7r7zPPM4L6DrD1+oQt510H/oOEE5NZ wIS9CmBf0txqwk7n1U5V95lonaCK9nfoKeQ1fKl/tu01dCeERRbMXG2nCQARAQAB zRtKb2huIERvZSA8am9obkBleGFtcGxlLm9yZz7CwYcEEwEIADEWIQQWwx1eM+Aa o8okGzL45grMTSggxQUCZMYfCQIbAwQLCQgHBRUICQoLBRYCAwEAAAoJEPjmCsxN KCDFWP4QAI3eS5nPxmU0AC9/h8jeKNgjgpENroNQZKeWZQ8x4PfncDRkcbsJfT7Y IVZl4zw6gFKY5EoB1s1KkYJxPgYsqicmKNiR7Tnzabb3mzomU48FKaIyVCBzFUnJ YMroL/rm7QhoW2WWLvT+CPCPway/tA3By8Be/YOjhavJ8mf1W3rPzt87/4Vo6erf yzL0lN+FQmmhKfT4j42jF4SMSyyC2yzvfC7PT49u+KUKQm/LpQsfKHpwXZ/VI6+X GtZjTqsc+uglJYRo69oosImLzieA/ST1ltjmUutZQOSvlQFpDUEFrMej8XZ0qsrf 0gP2iwxyl0vkhV8c6wO6CacDHPivvQEHed9H1PNGn3DBfKb7Mq/jado2DapRtJg3 2OH0F0HTvQ0uNKl30xMUcwGQB0cKOlaFtksZT1LsosQPhtPLpFy1TuWaXOInpQLq JmNVcTbydOsCKq0mb6bgGcvhElC1q39tclKP3rOEDOnJ8hE6wYNaMGrt6WSKr3Tt h52M6KwTXOuMAecMvpDBSS3UFEVQ+T5puzInDTkjINxmj23ip+swA1x3HH2IgNrO VJ7O20oEf0+qC47R5rTRUxrvh/U0U3DRE5xt2J2T3xetFDT2mnQv0jcyMg/UlXXv GpGVfwNkvN0Cxmb1tFiBNLKCcPVizxq4MLrwx+MVfQBaRCwjJrUszsFNBGTGHwoB EACr5lA+j5pH0Er6Q76btbS4q9JgNjDNrjKJwX9brdBY1oXIUeBqCW9ekoqDTFpn xA5EFGJvPO++/0ZCa+zXE4IAcXS9+I9HVBouenPYBLETnXK0Phws+OCLoe0cAIvG e9Xo9VrHcGXCs9tJruVSAW3NF04YejHmnHNfEuD8mbaUdxVn5zc23w/2gLaY/ABL ZfNV8XZw0jBVBm3YXS3Ob3uIO+RvsNqBgnhGYN/C51QI9hdxXWUDlD1vdRacXmcI LDCYC3w6u8caxL0ktXTS4zwN+hEu7jHxBNiKcovCeIF5VZ5NcPpp6+6Y+vNdmmXw +lWNwAzj3ah6iu+y25LKSsz+7IkCh5liOwwYohO+YI7SjtTD+gL9HiHYAIO+PtBh 7GudmUwFoARu/q54hE4ThpzkeOzJzPqGkM/CzmwdKKM3u81ze+72ptJOqVKbFEsQ 3+RURrIAfyYyeJj4VVCfHNzrRRVpARZc9hJm1AXefxPnDN9dxbikjQgbg5UxrKaJ cjVU+go5CH5lg2D1LRGfKqTJtfiWFPjtztNgMp/SeslkhhFXsyJ0RJDcU8VfRBrO DBnZvPnZi4nLaWCL1LdHA8Y9EJgSwVOsfdRqL/Xk9qxqgl5R8m8lsNKZN2EYkfMN 4Vd+/8UBbmibHYoGIQi7UlNSPthc0XQcRzFen+3H4sg5kQARAQABwsF2BBgBCAAg FiEEFsMdXjPgGqPKJBsy+OYKzE0oIMUFAmTGHwsCGwwACgkQ+OYKzE0oIMXn4hAA lUWeF7tDdyENsOYyhsbtLIuLipYe6orHFY5m68NNOoLWwqEeTvutJgFeDT4WxYi0 PJaNQYFPyGVyg7N0hCx5cGwajdnwGpb5zpSNyvG2Yes9I1O/u7+FFrbSwOuo61t1 scGa8YlgTKoyGc9cwxl5U8krrlEwXTWQ/qF1Gq2wHG23wm1D2d2PXFDRvw3gPxJn yWkrx5k26ru1kguM7XFVyRi7B+uG4vdvMlxMBXM3jpH1CJRr82VvzYPv7f05Z5To C7XDqHpWKx3+AQvh/ZsSBpBhzK8qaixysMwnawe05rOPydWvsLlnMCGManKVnq9Y Wek1P2dwYT9zuroBR5nmrECY+xVWk7vhsDasKsYlQ/LdDyzSL7qh0Vq3DjcoHxLI uL7qQ3O0YRcKGfmQibpKdDzvIqA+48Nfh2nDnTxvfuwOxb41zdLTZQftaSXc0Xwd HgquBAFbRDr5TyWlUUc8iACowKkk01pEPc8coxPCp6F/hz6kgmebRevzs7sxwrS7 aUWycSls783JC7WO267DRD30FNx+9S7SY4ECzhDGjLdne6wIoib1L9SFkk1AAKb3 m2+6BB/HxCXtMqi95pFeCjV99bp+PBqoifx9SlFYZq9qcGDr/jyrdG8V2Wf/HF4n K8RIPxB+daAPMLTpj4WBhNquSE6mRQvABEf0GPi2eLA= =0TDv -----END PGP PUBLIC KEY BLOCK----- ================================================ FILE: tests/resources/crypto/cert_smime.pem ================================================ -----BEGIN CERTIFICATE----- MIIHbTCCBVWgAwIBAgIQFxA+3j2KHLXKBlGT58pDazANBgkqhkiG9w0BAQsFADBr MQswCQYDVQQGEwJJVDEOMAwGA1UEBwwFTWlsYW4xIzAhBgNVBAoMGkFjdGFsaXMg Uy5wLkEuLzAzMzU4NTIwOTY3MScwJQYDVQQDDB5BY3RhbGlzIEF1dGhlbnRpY2F0 aW9uIFJvb3QgQ0EwHhcNMjAwNzA2MDg0NTQ3WhcNMzAwOTIyMTEyMjAyWjCBgTEL MAkGA1UEBhMCSVQxEDAOBgNVBAgMB0JlcmdhbW8xGTAXBgNVBAcMEFBvbnRlIFNh biBQaWV0cm8xFzAVBgNVBAoMDkFjdGFsaXMgUy5wLkEuMSwwKgYDVQQDDCNBY3Rh bGlzIENsaWVudCBBdXRoZW50aWNhdGlvbiBDQSBHMzCCAiIwDQYJKoZIhvcNAQEB BQADggIPADCCAgoCggIBAO3mh5ahwaS27cJCVfc/Dw8iYF8T4KZDiIZJkXkcGy8a UA/cRgHu9ro6hsxRYe/ED4AIcSlarRh82HqtFSVQs4ZwikQW1V/icCIS91C2IVAG a1YlKfedqgweqky+bBniUvRevVT0keZOqRTcO5hw007dL6FhYNmlZBt5IaJs1V6I niRjokOHR++qWgrUGy5LefY6ACs9gZ8Bi0OMK9PZ37pibeQCsdmMRytl4Ej7JVWe M/BtNIIprHwO1LY0/8InpGOmdG+5LC6xHLzg53B0HvVUqzUQNePUhNwJZFmmTP46 FXovxmH4/SuY5IkXop0eJqjN+dxRHHizngYUk1EaTHUOcLFy4vQ0kxgbjb+GsNg6 M2/6gZZIRk78JPdpotIwHnBNtkp9wPVH61NqdcP7kbPkyLXkNMTtAfydpmNnGqqH LEvUrK4iBpUPG9C09KOjm9OyhrT2uf5SLzJsee9g79r/rw4hAgcsZtR3YI6fCbRO JncmD+hgbHCck+9TWcNc1x5xZMgm8UXmoPamkkfceAlVV49QQ5jUTgqneTQHyF1F 2ExXmf47pEIoJMVxloRIXywQuB2uqcIs8/X6tfsMDynFmhfT/0mTrgQ6xt9DIsgm WuuhvZhLReWS7oeKxnyqscuGeTMXnLs7fjGZq0inyhnlznhA/4rl+WdNjNaO4jEv AgMBAAGjggH0MIIB8DAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFFLYiDrI n3hm7YnzezhwlMkCAjbQMEEGCCsGAQUFBwEBBDUwMzAxBggrBgEFBQcwAYYlaHR0 cDovL29jc3AwNS5hY3RhbGlzLml0L1ZBL0FVVEgtUk9PVDBFBgNVHSAEPjA8MDoG BFUdIAAwMjAwBggrBgEFBQcCARYkaHR0cHM6Ly93d3cuYWN0YWxpcy5pdC9hcmVh LWRvd25sb2FkMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDBDCB4wYDVR0f BIHbMIHYMIGWoIGToIGQhoGNbGRhcDovL2xkYXAwNS5hY3RhbGlzLml0L2NuJTNk QWN0YWxpcyUyMEF1dGhlbnRpY2F0aW9uJTIwUm9vdCUyMENBLG8lM2RBY3RhbGlz JTIwUy5wLkEuJTJmMDMzNTg1MjA5NjcsYyUzZElUP2NlcnRpZmljYXRlUmV2b2Nh dGlvbkxpc3Q7YmluYXJ5MD2gO6A5hjdodHRwOi8vY3JsMDUuYWN0YWxpcy5pdC9S ZXBvc2l0b3J5L0FVVEgtUk9PVC9nZXRMYXN0Q1JMMB0GA1UdDgQWBBS+l6mqhL+A vxBTfQky+eEuMhvPdzAOBgNVHQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQELBQADggIB ACab5xtZDXSzEgPp51X3hICFzULDO2EcV8em5hLfSCKxZR9amCnjcODVfMbaKfdU ZXtevMIIZmHgkz9dBan7ijGbJXjZCPP29zwZGSyCjpfadg5s9hnNCN1r3DGwIHfy LgbcfffDyV/2wW+XTGbhldnazZsX892q+srRmC8XnX4ygg+eWL/AkHDenvbFuTlJ vUyd5I7e1nb3dYXMObPu24ZTQ9/K1hSQbs7pqecaptTUjoIDpBUpSp4Us+h1I4MA WonemKYoPS9f0y65JrRCKcfsKSI+1kwPSanDDMiydKzeo46XrS0hlA5NzQjqUJ7U suGvPtDvknqc0v03nNXBnUjejYtvwO3sEDXdUW5m9kjNqlQZXzdHumZJVqPUGKTW cn9Hf3d7qbCmmxPXjQoNUuHg56fLCanZWkEO4SP1GAgIA7SyJu/yffv0ts7sBFrS TD3L2mCAXM3Y8BfblvvDSf2bvySm/fPe9brmuzrCXsTxUQc1+/z5ydvzV3E3cLnU oSXP6XfXNyEVO6sPkcUSnISHM798xLkCTB5EkjPCjPE2zs4v9L9JVOkkskvW6RnW WccdfR3fELNHL/kep8re6IbbYs8Hn5GM0Ohs8CMDPYEox+QX/6/SnOfyaqqSilBo nMQBstsymBBgdEKO+tTHHCMnJQVvZn7jRQ20wXgxMrvN -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- MIIFuzCCA6OgAwIBAgIIVwoRl0LE48wwDQYJKoZIhvcNAQELBQAwazELMAkGA1UE BhMCSVQxDjAMBgNVBAcMBU1pbGFuMSMwIQYDVQQKDBpBY3RhbGlzIFMucC5BLi8w MzM1ODUyMDk2NzEnMCUGA1UEAwweQWN0YWxpcyBBdXRoZW50aWNhdGlvbiBSb290 IENBMB4XDTExMDkyMjExMjIwMloXDTMwMDkyMjExMjIwMlowazELMAkGA1UEBhMC SVQxDjAMBgNVBAcMBU1pbGFuMSMwIQYDVQQKDBpBY3RhbGlzIFMucC5BLi8wMzM1 ODUyMDk2NzEnMCUGA1UEAwweQWN0YWxpcyBBdXRoZW50aWNhdGlvbiBSb290IENB MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAp8bEpSmkLO/lGMWwUKNv UTufClrJwkg4CsIcoBh/kbWHuUA/3R1oHwiD1S0eiKD4j1aPbZkCkpAW1V8IbInX 4ay8IMKx4INRimlNAJZaby/ARH6jDuSRzVju3PvHHkVH3Se5CAGfpiEd9UEtL0z9 KK3giq0itFZljoZUj5NDKd45RnijMCO6zfB9E1fAXdKDa0hMxKufgFpbOr3JpyI/ gCczWw63igxdBzcIy2zSekciRDXFzMwujt0q7bd9Zg1fYVEiVRvjRuPjPdA1Yprb rxTIW6HMiRvhMCb8oJsfgadHHwTrozmSBp+Z07/T6k9QnBn+locePGX2oxgkg4YQ 51Q+qDp2JE+BIcXjDwL4k5RHILv+1A7TaLndxHqEguNTVHnd25zS8gebLra8Pu2F be8lEfKXGkJh90qX6IuxEAf6ZYGyojnP9zz/GPvG8VqLWeICrHuS0E4UT1lF9gxe KF+w6D9Fz8+vm2/7hNN3WpVvrJSEnu68wEqPSpP4RCHiMUVhUE4Q2OM1fEwZtN4F v6MGn8i1zeQf1xcGDXqVdFUNaBr8EBtiZJ1t4JWgw5QHVw0U5r0F+7if5t+L4sbn fpb2U8WANFAoWPASUHEXMLrmeGO89LKtmyuy/uE5jF66CyCU3nuDuP/jVo23Eek7 jPKxwV2dpAtMK9myGPW1n0sCAwEAAaNjMGEwHQYDVR0OBBYEFFLYiDrIn3hm7Ynz ezhwlMkCAjbQMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUUtiIOsifeGbt ifN7OHCUyQICNtAwDgYDVR0PAQH/BAQDAgEGMA0GCSqGSIb3DQEBCwUAA4ICAQAL e3KHwGCmSUyIWOYdiPcUZEim2FgKDk8TNd81HdTtBjHIgT5q1d07GjLukD0R0i70 jsNjLiNmsGe+b7bAEzlgqqI0JZN1Ut6nna0Oh4lScWoWPBkdg/iaKWW+9D+a2fDz WochcYBNy+A4mz+7+uAwTc+G02UQGRjRlwKxK3JCaKygvU5a2hi/a5iB0P2avl4V SM0RFbnAKVy06Ij3Pjaut2L9HmLecHgQHEhb2rykOLpn7VU+Xlff1ANATIGk0k9j pwlCCRT8AKnCgHNPLsBA2RF7SOp6AsDT6ygBJlh0wcBzIm2Tlf05fbsq4/aC4yyX X04fkZT6/iyj2HYauE2yOE+b+h1IYHkm4vP9qdCa6HCPSXrW5b0KDtst842/6+Ok fcvHlXHo2qN8xcL4dJIEG4aspCJTQLas/kx2z/uUMsA1n3Y/buWQbqCmJqK4LL7R K4X9p2jIugErsWx0Hbhzlefut8cl8ABMALJ+tguLHPPAUJ4lueAI3jZm/zel0btU ZCzJJ7VLkn5l/9Mt4blOvH+kQSGQQXemOR/qnuOf0GZvBeyqdn6/axag67XH/JJU LysRJyU3eExRarDzzFhdFPFqSBX/wge2sY0PjlxQRrM9vwGYT7JZVEc+NHt4bVaT LnPqZih4zR0Uv6CPLy64Lo7yFIrM6bV8+2ydDKXhlg== -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- MIIDbjCCAlagAwIBAgIUZ4K0WXNSS8H0cUcZavD9EYqqTAswDQYJKoZIhvcNAQEN BQAwLTErMCkGA1UEAxMiU2FtcGxlIExBTVBTIENlcnRpZmljYXRlIEF1dGhvcml0 eTAgFw0xOTExMjAwNjU0MThaGA8yMDUyMDkyNzA2NTQxOFowGTEXMBUGA1UEAxMO QWxpY2UgTG92ZWxhY2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDD 7q35ZdG2JAzzJGNZDZ9sV7AKh0hlRfoFjTZN5m4RegQAYSyag43ouWi1xRN0avf0 UTYrwjK04qRdV7GzCACoEKq/xiNUOsjfJXzbCublN3fZMOXDshKKBqThlK75SjA9 Czxg7ejGoiY/iidk0e91neK30SCCaBTJlfR2ZDrPk73IPMeksxoTatfF9hw9dDA+ /Hi1yptN/aG0Q/s9icFrxr6y2zQXsjuQPmjMZgj10aD9cazWVgRYCgflhmA0V1uQ l1wobYU8DAVxVn+GgabqyjGQMoythIK0Gn5+ofwxXXUM/zbU+g6+1ISdoXxRRFtq 2GzbIqkAHZZQm+BbnFrhAgMBAAGjgZcwgZQwDAYDVR0TAQH/BAIwADAeBgNVHREE FzAVgRNhbGljZUBzbWltZS5leGFtcGxlMBMGA1UdJQQMMAoGCCsGAQUFBwMEMA8G A1UdDwEB/wQFAwMHoAAwHQYDVR0OBBYEFKwuVFqk/VUYry7oZkQ40SXR1wB5MB8G A1UdIwQYMBaAFLdSTXPAiD2yw3paDPOU9/eAonfbMA0GCSqGSIb3DQEBDQUAA4IB AQB76o4Yz7yrVSFcpXqLrcGtdI4q93aKCXECCCzNQLp4yesh6brqaZHNJtwYcJ5T qbUym9hJ70iJE4jGNN+yAZR1ltte0HFKYIBKM4EJumG++2hqbUaLz4tl06BHaQPC v/9NiNY7q9R9c/B6s1YzHhwqkWht2a+AtgJ4BkpG+g+MmZMQV/Ao7RwLFKJ9OlMW LBmEXFcpIJN0HpPasT0nEl/MmotSu+8RnClAi3yFfyTKb+8rD7VxuyXetqDZ6dU/ 9/iqD/SZS7OQIjywtd343mACz3B1RlFxMHSA6dQAf2btGumqR0KiAp3KkYRAePoa JqYkB7Zad06ngFl0G0FHON+7 -----END CERTIFICATE----- ================================================ FILE: tests/resources/crypto/is_encrypted.txt ================================================ Subject: TRUE Content-Type: multipart/encrypted; protocol="application/pgp-encrypted"; boundary="17778885806d6b28_46555a54adda6fa5_d83705f326a6b951" body !!! Subject: FALSE Content-Type: multipart/signed; protocol="application/pkcs7-signature"; boundary="17778885806d6b28_46555a54adda6fa5_d83705f326a6b951" body !!! Subject: TRUE Content-Type: application/pkcs7-mime; name="smime.p7m"; smime-type=enveloped-data Content-Disposition: attachment; filename="smime.p7m" Content-Transfer-Encoding: base64 body !!! Subject: TRUE Content-Type: application/pkcs7-signature; name="smime.p7s"; smime-type="signed-data" Content-Disposition: attachment; filename="smime.p7s" Content-Transfer-Encoding: base64 body !!! Subject: TRUE Content-Type: application/octet-stream; name="smime.p7m" Content-Disposition: attachment; filename="smime.p7m" Content-Transfer-Encoding: base64 body !!! Subject: TRUE Content-Type: application/octet-stream; name="smime.p7m" Content-Transfer-Encoding: base64 body !!! Subject: TRUE Content-Type: application/octet-stream Content-Disposition: attachment; filename="smime.p7m" Content-Transfer-Encoding: base64 body !!! Subject: TRUE Content-Type: application/octet-stream Content-Disposition: attachment; filename="smime.p7s" Content-Transfer-Encoding: base64 body !!! Subject: TRUE Content-Type: application/octet-stream Content-Disposition: attachment; filename="smime.p7c" Content-Transfer-Encoding: base64 body !!! Subject: TRUE Content-Type: application/octet-stream Content-Disposition: attachment; filename="smime.p7z" Content-Transfer-Encoding: base64 body !!! Subject: FALSE Content-Type: application/octet-stream Content-Disposition: attachment; filename="file.gz" Content-Transfer-Encoding: base64 body !!! Subject: FALSE Content-Type: application/octet-stream Content-Disposition: attachment Content-Transfer-Encoding: base64 body !!! Subject: FALSE Content-Type: multipart/mixed; boundary="17778885806d6b28_46555a54adda6fa5_d83705f326a6b951" body !!! Subject: TRUE Content-Type: multipart/mixed; boundary="----sinikael-?=_1-17364333937490.28304631087789" ------sinikael-?=_1-17364333937490.28304631087789 Content-Type: text/plain Content-Transfer-Encoding: quoted-printable -----BEGIN PGP MESSAGE----- Version: Mailvelope v6.0.1 Comment: https://mailvelope.com wcFMAy3n9oegk2NSARAApgl4nuztm5uxuCe+7ISW9Ox+315/pCHZmOUibORX xGEbUs+ZgTqKIdYMet3+zSS70ykgjZDvh+cLm1oGmD5ZzLxSMjhgLpqrA0tG 3GB3gus844irLGBjVwgTuwAKhRz/CYaqlKGOloQkvcwRwy3N8UY8KG/WqD63 g9WQKYc+8+cR4wfQ0PSzZL6HexdT2LE2mWgKxStwDbbkwZumR0qZy4OvyI6O zFCZSaz3F3VnDGbQgo5hK8Iynf25sC54P/E0ykdG6QpJZPo/ASoWePv0XyT6 wRiqq2cx366P6lxB3CJyjn+qRCfcXz0XXYejHmuI++S6ucNS4JcIhpuoi+tv 8b7pSy/6yQEKLyUhPfUmvFLxBm10bu8E9uhsmZnpiSnBsEGkLm/l9TFbOOSd 9nK0NeEVfqfIi25bxVJtRcYryf7BpjmgEdKlmF8Efeb1RpaI8IwVc6E7AhYB c1vz1gN6BQ5ePtlHDFKxFV3jIqBKck8h2GSFGEYOXyTT+3tgFkqv/SuAUed9 X0bCjy7v578nsNieoyy/133X1+mu7L8rdADEAHhU/WLQ9VY/xu6GhNA0vzU8 yVQfSEJ1DJ4awU9Cn1mrbVg3vD/jpQzs67Sd33gC8xMqA9ZFOmXrnR4V1wkz Y8XobC+UY6o9qn/Oz5mEtZG8rItYJ/bPOSSlFP6kXVrBwUwD8GquMWN8AUEB D/9IVPN/qCtuhF+9jIjrxiw+agmeqUPWBEmLLZYL6xivP7Bn5bOJb6wwgU4e IRIoh3Q5AKLvqN26BPePM16yZ6Z6qxTP/CFT5mOQQbj+3TH6fYD1ui74FoyR +szTfntIdUOShKqogSKoRytMXheR+4TW7/D525ohRUsz71pfdCxqYC+k/vXr ffl7lkv/V7GkPJ5fT+CantgLqXVevCIKOMVDVBT6G92t8Zn0klqv+MfVsZ3V 6FbwcqS+GxsCNezSdT1+K2yPja0fZffQOHD6nLDyIZmcr8KrCmP1MPY5LKJw bMVCbYz0CrRl9d6JJGZOq+yapVOc0CDxWmxhGJP3M9OiCvGoPz3l+fDdNJ0f dmxEv8c/I5t+AELxLMBa4aPuGE3pZQj4zyzhC3gnK5n9bqFecHl2+Kz+aVsi 4B1xZXIPT+XmlXM4rgCtuMlKZ6YkGRWiAnC8jok/k4iPaN7mfKbos0xwa187 haMjKKFVP8YvXlYpGeuvP3C2/oBxwsw9WihqL2tLTEE60ZgDEKWcZ3zwVKjl bSNeDsWXlj+vIletdDyH/AMhyDOnUDc4bTkRW7W8ntq3fi8XmpQjHzW02BEd Wq1Y/bV7nlh97Y7d3z5zlLhQnKrrP5IyQdOkd2WGpU39cDZe6XE6arQZKGVT upnuX3Q48Hm6oSga+cydngjShtLCFQF/MFCZVYydjhe+KtzZCYBZ8nCMRxcV PTuACJtUqOR/z5aPgRjM14accCYDwJ0Dm51h5LuEzYFDJjcdFbVSVfvnCy5a b9h6VBCpWieI6ubAzYKKJxGwp/OcoaQyYPjwlNQyTSb7omZPsAtFpTALNXbU CIM2cFnD/MMvgEHCuqPZoKh0YdQxYV50p36Wa/rMvTfW+S1T2neWGD6kMDWZ I651K+HBCQV6INrJk5ugvCt26/l8+Y75cnGEKbkyJgt+NGjULX3jjPawhvwN 0EKx8hYwASuJ3E71iAXga7oSMRemDk4ZXoz+PDngoWtcoXcdQNMiVymOt6iZ P5ImSQXkId1XZYjS3+1t+Sk8WgUi5kvu4WP3hm1Ovjd1kXTy3lqXpfci23si eZofjs9DLldkV5xtOwWVx/B1UkqvtD2BuwcDWgvGEm2LEn4/eXkHUTzHLs4j T9s454HasmVfd+4/koBdD6ASDqFf5Ue14OonsXvIvLop2CgUHxDVffr1onSk K9r0kKxe9iAFZ8KYpf2Do/FPTL8BIXjHC2vfJ05wl262xs7uNMdgWm4+NP/q uVNOoXByZIDEPixx7jimKhlEIZx36AbsI+OB5cV1Pr5SSbO3w205ALol+xKN TPJiaQDH8qNXX+9j4JfO8uWvDmhPUe83bm/mHg4wf/39+PNQptwmhLH9iXtC Czqgmr1vUNd/8dLgQuqVn/R3uUbGeWmKD9cqURWcvumbq+rFWPl1N7T7VRfc XGDnMBNBmLke2YO8XUY561lqKekfS/2fekSdz005sI2cnGmy5LLh+kPN6opz 0pA2kSskbfSQqJmEdjKQU3oGEzEgwvOXjFwT7cChevgLEmDBkQXTDuEeo3Kb KnwaNPnodzaNXAfZh/NylEgcUFjEx6crZMN3ztl8zsvAvP4P0BPmixZ89G80 5fkw+PwLe2vPtuObvuY+ezbJGb1jV0tWZFXF =3DaQyM -----END PGP MESSAGE----- ------sinikael-?=_1-17364333937490.28304631087789-- !!! Subject: TRUE Content-Type: text/plain Content-Transfer-Encoding: quoted-printable -----BEGIN PGP MESSAGE----- Version: Mailvelope v6.0.1 Comment: https://mailvelope.com wcFMAy3n9oegk2NSARAApgl4nuztm5uxuCe+7ISW9Ox+315/pCHZmOUibORX xGEbUs+ZgTqKIdYMet3+zSS70ykgjZDvh+cLm1oGmD5ZzLxSMjhgLpqrA0tG 3GB3gus844irLGBjVwgTuwAKhRz/CYaqlKGOloQkvcwRwy3N8UY8KG/WqD63 g9WQKYc+8+cR4wfQ0PSzZL6HexdT2LE2mWgKxStwDbbkwZumR0qZy4OvyI6O zFCZSaz3F3VnDGbQgo5hK8Iynf25sC54P/E0ykdG6QpJZPo/ASoWePv0XyT6 wRiqq2cx366P6lxB3CJyjn+qRCfcXz0XXYejHmuI++S6ucNS4JcIhpuoi+tv 8b7pSy/6yQEKLyUhPfUmvFLxBm10bu8E9uhsmZnpiSnBsEGkLm/l9TFbOOSd 9nK0NeEVfqfIi25bxVJtRcYryf7BpjmgEdKlmF8Efeb1RpaI8IwVc6E7AhYB c1vz1gN6BQ5ePtlHDFKxFV3jIqBKck8h2GSFGEYOXyTT+3tgFkqv/SuAUed9 X0bCjy7v578nsNieoyy/133X1+mu7L8rdADEAHhU/WLQ9VY/xu6GhNA0vzU8 yVQfSEJ1DJ4awU9Cn1mrbVg3vD/jpQzs67Sd33gC8xMqA9ZFOmXrnR4V1wkz Y8XobC+UY6o9qn/Oz5mEtZG8rItYJ/bPOSSlFP6kXVrBwUwD8GquMWN8AUEB D/9IVPN/qCtuhF+9jIjrxiw+agmeqUPWBEmLLZYL6xivP7Bn5bOJb6wwgU4e IRIoh3Q5AKLvqN26BPePM16yZ6Z6qxTP/CFT5mOQQbj+3TH6fYD1ui74FoyR +szTfntIdUOShKqogSKoRytMXheR+4TW7/D525ohRUsz71pfdCxqYC+k/vXr ffl7lkv/V7GkPJ5fT+CantgLqXVevCIKOMVDVBT6G92t8Zn0klqv+MfVsZ3V 6FbwcqS+GxsCNezSdT1+K2yPja0fZffQOHD6nLDyIZmcr8KrCmP1MPY5LKJw bMVCbYz0CrRl9d6JJGZOq+yapVOc0CDxWmxhGJP3M9OiCvGoPz3l+fDdNJ0f dmxEv8c/I5t+AELxLMBa4aPuGE3pZQj4zyzhC3gnK5n9bqFecHl2+Kz+aVsi 4B1xZXIPT+XmlXM4rgCtuMlKZ6YkGRWiAnC8jok/k4iPaN7mfKbos0xwa187 haMjKKFVP8YvXlYpGeuvP3C2/oBxwsw9WihqL2tLTEE60ZgDEKWcZ3zwVKjl bSNeDsWXlj+vIletdDyH/AMhyDOnUDc4bTkRW7W8ntq3fi8XmpQjHzW02BEd Wq1Y/bV7nlh97Y7d3z5zlLhQnKrrP5IyQdOkd2WGpU39cDZe6XE6arQZKGVT upnuX3Q48Hm6oSga+cydngjShtLCFQF/MFCZVYydjhe+KtzZCYBZ8nCMRxcV PTuACJtUqOR/z5aPgRjM14accCYDwJ0Dm51h5LuEzYFDJjcdFbVSVfvnCy5a b9h6VBCpWieI6ubAzYKKJxGwp/OcoaQyYPjwlNQyTSb7omZPsAtFpTALNXbU CIM2cFnD/MMvgEHCuqPZoKh0YdQxYV50p36Wa/rMvTfW+S1T2neWGD6kMDWZ I651K+HBCQV6INrJk5ugvCt26/l8+Y75cnGEKbkyJgt+NGjULX3jjPawhvwN 0EKx8hYwASuJ3E71iAXga7oSMRemDk4ZXoz+PDngoWtcoXcdQNMiVymOt6iZ P5ImSQXkId1XZYjS3+1t+Sk8WgUi5kvu4WP3hm1Ovjd1kXTy3lqXpfci23si eZofjs9DLldkV5xtOwWVx/B1UkqvtD2BuwcDWgvGEm2LEn4/eXkHUTzHLs4j T9s454HasmVfd+4/koBdD6ASDqFf5Ue14OonsXvIvLop2CgUHxDVffr1onSk K9r0kKxe9iAFZ8KYpf2Do/FPTL8BIXjHC2vfJ05wl262xs7uNMdgWm4+NP/q uVNOoXByZIDEPixx7jimKhlEIZx36AbsI+OB5cV1Pr5SSbO3w205ALol+xKN TPJiaQDH8qNXX+9j4JfO8uWvDmhPUe83bm/mHg4wf/39+PNQptwmhLH9iXtC Czqgmr1vUNd/8dLgQuqVn/R3uUbGeWmKD9cqURWcvumbq+rFWPl1N7T7VRfc XGDnMBNBmLke2YO8XUY561lqKekfS/2fekSdz005sI2cnGmy5LLh+kPN6opz 0pA2kSskbfSQqJmEdjKQU3oGEzEgwvOXjFwT7cChevgLEmDBkQXTDuEeo3Kb KnwaNPnodzaNXAfZh/NylEgcUFjEx6crZMN3ztl8zsvAvP4P0BPmixZ89G80 5fkw+PwLe2vPtuObvuY+ezbJGb1jV0tWZFXF =3DaQyM -----END PGP MESSAGE----- ================================================ FILE: tests/resources/imap/000.imap ================================================ BODY ( ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 262 7 )( "text" "plain" ( "charset" "US-ASCII" ) NIL NIL "7bit" 111 3 )( ( "text" "basic" ( "charset" "us-ascii" ) NIL NIL "base64" 85 2 )( "text" "jpeg" ( "charset" "us-ascii" ) NIL NIL "base64" 44 1 ) "parallel" )( "text" "enriched" ( "charset" "us-ascii" ) NIL NIL "7bit" 140 5 )( "message" "rfc822" NIL NIL NIL NIL 223 ( NIL "(subject in US-ASCII)" ( ( NIL NIL "unknown" "localhost" ) ) ( ( NIL NIL "unknown" "localhost" ) ) ( ( NIL NIL "unknown" "localhost" ) ) NIL NIL NIL NIL NIL ) ( "text" "plain" ( "charset" "ISO-8859-1" ) NIL NIL "Quoted-printable" 48 1 ) 0 ) "mixed" ) BODYSTRUCTURE ( ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 262 7 "1ba9f4410aca245ffe18870d1767eaf1" NIL NIL NIL )( "text" "plain" ( "charset" "US-ASCII" ) NIL NIL "7bit" 111 3 "457a84e4830817334703ecead7a71bdf" NIL NIL NIL )( ( "text" "basic" ( "charset" "us-ascii" ) NIL NIL "base64" 85 2 "c6e3404e683fdeb88e96dfed0abbddad" NIL NIL NIL )( "text" "jpeg" ( "charset" "us-ascii" ) NIL NIL "base64" 44 1 "88217c7613d6252c87ba0cd279f1e93c" NIL NIL NIL ) "parallel" ( "boundary" "unique-boundary-2" ) NIL NIL NIL )( "text" "enriched" ( "charset" "us-ascii" ) NIL NIL "7bit" 140 5 "4a5cd280a875e791bd6d767dcf60c43d" NIL NIL NIL )( "message" "rfc822" NIL NIL NIL NIL 223 ( NIL "(subject in US-ASCII)" ( ( NIL NIL "unknown" "localhost" ) ) ( ( NIL NIL "unknown" "localhost" ) ) ( ( NIL NIL "unknown" "localhost" ) ) NIL NIL NIL NIL NIL ) ( "text" "plain" ( "charset" "ISO-8859-1" ) NIL NIL "Quoted-printable" 48 1 "632b0aae19d3c55d420c0dc8cebaf049" NIL NIL NIL ) 0 "40ceb8762dcb7270c37f1395e91aa893" NIL NIL NIL ) "mixed" ( "boundary" "unique-boundary-1" ) NIL NIL NIL ) BODY[] {1854} MIME-Version: 1.0 From: Nathaniel Borenstein To: Ned Freed Date: Fri, 07 Oct 1994 16:15:05 -0700 (PDT) Subject: A multipart example Content-Type: multipart/mixed; boundary=unique-boundary-1 This is the preamble area of a multipart message. Mail readers that understand multipart format should ignore this preamble. If you are reading this text, you might want to consider changing to a mail reader that understands how to properly display multipart messages. --unique-boundary-1 ... Some text appears here ... [Note that the blank between the boundary and the start of the text in this part means no header fields were given and this is text in the US-ASCII character set. It could have been done with explicit typing as in the next part.] --unique-boundary-1 Content-type: text/plain; charset=US-ASCII This could have been part of the previous part, but illustrates explicit versus implicit typing of body parts. --unique-boundary-1 Content-Type: multipart/parallel; boundary=unique-boundary-2 --unique-boundary-2 Content-Type: audio/basic Content-Transfer-Encoding: base64 ... base64-encoded 8000 Hz single-channel mu-law-format audio data goes here ... --unique-boundary-2 Content-Type: image/jpeg Content-Transfer-Encoding: base64 ... base64-encoded image data goes here ... --unique-boundary-2-- --unique-boundary-1 Content-type: text/enriched This is enriched. as defined in RFC 1896 Isn't it cool? --unique-boundary-1 Content-Type: message/rfc822 From: (mailbox in US-ASCII) To: (address in US-ASCII) Subject: (subject in US-ASCII) Content-Type: Text/plain; charset=ISO-8859-1 Content-Transfer-Encoding: Quoted-printable ... Additional text in ISO-8859-1 goes here ... --unique-boundary-1-- BINARY[] {16} [binary content] BINARY.SIZE[] 1854 ---------------------------------- BODY[HEADER] {239} MIME-Version: 1.0 From: Nathaniel Borenstein To: Ned Freed Date: Fri, 07 Oct 1994 16:15:05 -0700 (PDT) Subject: A multipart example Content-Type: multipart/mixed; boundary=unique-boundary-1 ---------------------------------- BODY[TEXT] {1615} This is the preamble area of a multipart message. Mail readers that understand multipart format should ignore this preamble. If you are reading this text, you might want to consider changing to a mail reader that understands how to properly display multipart messages. --unique-boundary-1 ... Some text appears here ... [Note that the blank between the boundary and the start of the text in this part means no header fields were given and this is text in the US-ASCII character set. It could have been done with explicit typing as in the next part.] --unique-boundary-1 Content-type: text/plain; charset=US-ASCII This could have been part of the previous part, but illustrates explicit versus implicit typing of body parts. --unique-boundary-1 Content-Type: multipart/parallel; boundary=unique-boundary-2 --unique-boundary-2 Content-Type: audio/basic Content-Transfer-Encoding: base64 ... base64-encoded 8000 Hz single-channel mu-law-format audio data goes here ... --unique-boundary-2 Content-Type: image/jpeg Content-Transfer-Encoding: base64 ... base64-encoded image data goes here ... --unique-boundary-2-- --unique-boundary-1 Content-type: text/enriched This is enriched. as defined in RFC 1896 Isn't it cool? --unique-boundary-1 Content-Type: message/rfc822 From: (mailbox in US-ASCII) To: (address in US-ASCII) Subject: (subject in US-ASCII) Content-Type: Text/plain; charset=ISO-8859-1 Content-Transfer-Encoding: Quoted-printable ... Additional text in ISO-8859-1 goes here ... --unique-boundary-1-- ---------------------------------- BODY[MIME] {72} Content-Type: multipart/mixed; boundary=unique-boundary-1 ---------------------------------- BODY[1] {262} ... Some text appears here ... [Note that the blank between the boundary and the start of the text in this part means no header fields were given and this is text in the US-ASCII character set. It could have been done with explicit typing as in the next part.] BINARY[1] {262} ... Some text appears here ... [Note that the blank between the boundary and the start of the text in this part means no header fields were given and this is text in the US-ASCII character set. It could have been done with explicit typing as in the next part.] BINARY.SIZE[1] 262 ---------------------------------- BODY[1.HEADER] {1} ---------------------------------- BODY[1.TEXT] {262} ... Some text appears here ... [Note that the blank between the boundary and the start of the text in this part means no header fields were given and this is text in the US-ASCII character set. It could have been done with explicit typing as in the next part.] ---------------------------------- BODY[1.MIME] {2} ---------------------------------- BODY[1.1] {262} ... Some text appears here ... [Note that the blank between the boundary and the start of the text in this part means no header fields were given and this is text in the US-ASCII character set. It could have been done with explicit typing as in the next part.] BINARY[1.1] {262} ... Some text appears here ... [Note that the blank between the boundary and the start of the text in this part means no header fields were given and this is text in the US-ASCII character set. It could have been done with explicit typing as in the next part.] BINARY.SIZE[1.1] 262 ---------------------------------- BODY[2] {111} This could have been part of the previous part, but illustrates explicit versus implicit typing of body parts. BINARY[2] {111} This could have been part of the previous part, but illustrates explicit versus implicit typing of body parts. BINARY.SIZE[2] 111 ---------------------------------- BODY[2.HEADER] {44} Content-type: text/plain; charset=US-ASCII ---------------------------------- BODY[2.TEXT] {111} This could have been part of the previous part, but illustrates explicit versus implicit typing of body parts. ---------------------------------- BODY[2.MIME] {45} Content-Type: text/plain; charset=US-ASCII ---------------------------------- BODY[2.1] {111} This could have been part of the previous part, but illustrates explicit versus implicit typing of body parts. BINARY[2.1] {111} This could have been part of the previous part, but illustrates explicit versus implicit typing of body parts. BINARY.SIZE[2.1] 111 ---------------------------------- BODY[3] {314} --unique-boundary-2 Content-Type: audio/basic Content-Transfer-Encoding: base64 ... base64-encoded 8000 Hz single-channel mu-law-format audio data goes here ... --unique-boundary-2 Content-Type: image/jpeg Content-Transfer-Encoding: base64 ... base64-encoded image data goes here ... --unique-boundary-2-- BINARY[3] {16} [binary content] BINARY.SIZE[3] 376 ---------------------------------- BODY[3.HEADER] {62} Content-Type: multipart/parallel; boundary=unique-boundary-2 ---------------------------------- BODY[3.TEXT] {314} --unique-boundary-2 Content-Type: audio/basic Content-Transfer-Encoding: base64 ... base64-encoded 8000 Hz single-channel mu-law-format audio data goes here ... --unique-boundary-2 Content-Type: image/jpeg Content-Transfer-Encoding: base64 ... base64-encoded image data goes here ... --unique-boundary-2-- ---------------------------------- BODY[3.MIME] {63} Content-Type: multipart/parallel; boundary=unique-boundary-2 ---------------------------------- BODY[3.1] {85} ... base64-encoded 8000 Hz single-channel mu-law-format audio data goes here ... * NO [UNKNOWN-CTE] Failed to decode part 3.1 of message 0. BINARY.SIZE[3.1] 85 ---------------------------------- BODY[3.1.HEADER] {61} Content-Type: audio/basic Content-Transfer-Encoding: base64 ---------------------------------- BODY[3.1.TEXT] {85} ... base64-encoded 8000 Hz single-channel mu-law-format audio data goes here ... ---------------------------------- BODY[3.1.MIME] {62} Content-Type: audio/basic Content-Transfer-Encoding: base64 ---------------------------------- BODY[3.1.1] {85} ... base64-encoded 8000 Hz single-channel mu-law-format audio data goes here ... * NO [UNKNOWN-CTE] Failed to decode part 3.1.1 of message 0. BINARY.SIZE[3.1.1] 85 ---------------------------------- BODY[3.2] {44} ... base64-encoded image data goes here ... * NO [UNKNOWN-CTE] Failed to decode part 3.2 of message 0. BINARY.SIZE[3.2] 44 ---------------------------------- BODY[3.2.HEADER] {60} Content-Type: image/jpeg Content-Transfer-Encoding: base64 ---------------------------------- BODY[3.2.TEXT] {44} ... base64-encoded image data goes here ... ---------------------------------- BODY[3.2.MIME] {61} Content-Type: image/jpeg Content-Transfer-Encoding: base64 ---------------------------------- BODY[3.2.1] {44} ... base64-encoded image data goes here ... * NO [UNKNOWN-CTE] Failed to decode part 3.2.1 of message 0. BINARY.SIZE[3.2.1] 44 ---------------------------------- BODY[4] {140} This is enriched. as defined in RFC 1896 Isn't it cool? BINARY[4] {140} This is enriched. as defined in RFC 1896 Isn't it cool? BINARY.SIZE[4] 140 ---------------------------------- BODY[4.HEADER] {29} Content-type: text/enriched ---------------------------------- BODY[4.TEXT] {140} This is enriched. as defined in RFC 1896 Isn't it cool? ---------------------------------- BODY[4.MIME] {30} Content-Type: text/enriched ---------------------------------- BODY[4.1] {140} This is enriched. as defined in RFC 1896 Isn't it cool? BINARY[4.1] {140} This is enriched. as defined in RFC 1896 Isn't it cool? BINARY.SIZE[4.1] 140 ---------------------------------- BODY[5] {223} From: (mailbox in US-ASCII) To: (address in US-ASCII) Subject: (subject in US-ASCII) Content-Type: Text/plain; charset=ISO-8859-1 Content-Transfer-Encoding: Quoted-printable ... Additional text in ISO-8859-1 goes here ... BINARY[5] {16} [binary content] BINARY.SIZE[5] 223 ---------------------------------- BODY[5.HEADER] {175} From: (mailbox in US-ASCII) To: (address in US-ASCII) Subject: (subject in US-ASCII) Content-Type: Text/plain; charset=ISO-8859-1 Content-Transfer-Encoding: Quoted-printable ---------------------------------- BODY[5.TEXT] {48} ... Additional text in ISO-8859-1 goes here ... ---------------------------------- BODY[5.MIME] {31} Content-Type: message/rfc822 ---------------------------------- BODY[5.1] {48} ... Additional text in ISO-8859-1 goes here ... BINARY[5.1] {48} ... Additional text in ISO-8859-1 goes here ... BINARY.SIZE[5.1] 48 ---------------------------------- BODY[HEADER.FIELDS (FROM TO)] {79} From: Nathaniel Borenstein To: Ned Freed ---------------------------------- BODY[HEADER.FIELDS (FROM TO)]<10> {25} aniel Borenstein To: Ned Freed Date: Fri, 07 Oct 1994 16:15:05 -0700 (PDT) Content-Type: multipart/mixed; boundary=unique-boundary-1 ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)]<10> {25} on: 1.0 From: Nathaniel B ---------------------------------- ================================================ FILE: tests/resources/imap/000.txt ================================================ MIME-Version: 1.0 From: Nathaniel Borenstein To: Ned Freed Date: Fri, 07 Oct 1994 16:15:05 -0700 (PDT) Subject: A multipart example Content-Type: multipart/mixed; boundary=unique-boundary-1 This is the preamble area of a multipart message. Mail readers that understand multipart format should ignore this preamble. If you are reading this text, you might want to consider changing to a mail reader that understands how to properly display multipart messages. --unique-boundary-1 ... Some text appears here ... [Note that the blank between the boundary and the start of the text in this part means no header fields were given and this is text in the US-ASCII character set. It could have been done with explicit typing as in the next part.] --unique-boundary-1 Content-type: text/plain; charset=US-ASCII This could have been part of the previous part, but illustrates explicit versus implicit typing of body parts. --unique-boundary-1 Content-Type: multipart/parallel; boundary=unique-boundary-2 --unique-boundary-2 Content-Type: audio/basic Content-Transfer-Encoding: base64 ... base64-encoded 8000 Hz single-channel mu-law-format audio data goes here ... --unique-boundary-2 Content-Type: image/jpeg Content-Transfer-Encoding: base64 ... base64-encoded image data goes here ... --unique-boundary-2-- --unique-boundary-1 Content-type: text/enriched This is enriched. as defined in RFC 1896 Isn't it cool? --unique-boundary-1 Content-Type: message/rfc822 From: (mailbox in US-ASCII) To: (address in US-ASCII) Subject: (subject in US-ASCII) Content-Type: Text/plain; charset=ISO-8859-1 Content-Transfer-Encoding: Quoted-printable ... Additional text in ISO-8859-1 goes here ... --unique-boundary-1-- ================================================ FILE: tests/resources/imap/001.imap ================================================ BODY ( ( "message" "external-body" ( "name" "BodyFormats.ps" "site" "thumper.bellcore.com" "mode" "image" "access-type" "ANON-FTP" "directory" "pub" "expiration" "Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" ) NIL NIL NIL 79 )( "message" "external-body" ( "access-type" "local-file" "name" "/u/nsb/writing/rfcs/RFC-MIME.ps" "site" "thumper.bellcore.com" "expiration" "Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" ) NIL NIL NIL 79 )( "message" "external-body" ( "access-type" "mail-server" "server" "listserv@bogus.bitnet" "expiration" "Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" ) NIL NIL NIL 97 ) "alternative" ) BODYSTRUCTURE ( ( "message" "external-body" ( "name" "BodyFormats.ps" "site" "thumper.bellcore.com" "mode" "image" "access-type" "ANON-FTP" "directory" "pub" "expiration" "Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" ) NIL NIL NIL 79 "13a120642a010037cbc238c999349116" NIL NIL NIL )( "message" "external-body" ( "access-type" "local-file" "name" "/u/nsb/writing/rfcs/RFC-MIME.ps" "site" "thumper.bellcore.com" "expiration" "Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" ) NIL NIL NIL 79 "13a120642a010037cbc238c999349116" NIL NIL NIL )( "message" "external-body" ( "access-type" "mail-server" "server" "listserv@bogus.bitnet" "expiration" "Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" ) NIL NIL NIL 97 "6f6c2486f2516ed45542404973c1cebb" NIL NIL NIL ) "alternative" ( "boundary" "42" ) NIL NIL NIL ) BODY[] {1109} From: Whomever To: Someone Date: Whenever Subject: whatever MIME-Version: 1.0 Message-ID: Content-Type: multipart/alternative; boundary=42 Content-ID: --42 Content-Type: message/external-body; name="BodyFormats.ps"; site="thumper.bellcore.com"; mode="image"; access-type=ANON-FTP; directory="pub"; expiration="Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" Content-type: application/postscript Content-ID: --42 Content-Type: message/external-body; access-type=local-file; name="/u/nsb/writing/rfcs/RFC-MIME.ps"; site="thumper.bellcore.com"; expiration="Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" Content-type: application/postscript Content-ID: --42 Content-Type: message/external-body; access-type=mail-server server="listserv@bogus.bitnet"; expiration="Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" Content-type: application/postscript Content-ID: get RFC-MIME.DOC --42-- BINARY[] {16} [binary content] BINARY.SIZE[] 1109 ---------------------------------- BODY[HEADER] {198} From: Whomever To: Someone Date: Whenever Subject: whatever MIME-Version: 1.0 Message-ID: Content-Type: multipart/alternative; boundary=42 Content-ID: ---------------------------------- BODY[TEXT] {911} --42 Content-Type: message/external-body; name="BodyFormats.ps"; site="thumper.bellcore.com"; mode="image"; access-type=ANON-FTP; directory="pub"; expiration="Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" Content-type: application/postscript Content-ID: --42 Content-Type: message/external-body; access-type=local-file; name="/u/nsb/writing/rfcs/RFC-MIME.ps"; site="thumper.bellcore.com"; expiration="Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" Content-type: application/postscript Content-ID: --42 Content-Type: message/external-body; access-type=mail-server server="listserv@bogus.bitnet"; expiration="Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" Content-type: application/postscript Content-ID: get RFC-MIME.DOC --42-- ---------------------------------- BODY[MIME] {94} Content-Type: multipart/alternative; boundary=42 Content-ID: ---------------------------------- BODY[1] {79} Content-type: application/postscript Content-ID: BINARY[1] {16} [binary content] BINARY.SIZE[1] 79 ---------------------------------- BODY[1.HEADER] {230} Content-Type: message/external-body; name="BodyFormats.ps"; site="thumper.bellcore.com"; mode="image"; access-type=ANON-FTP; directory="pub"; expiration="Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" ---------------------------------- BODY[1.TEXT] {79} Content-type: application/postscript Content-ID: ---------------------------------- BODY[1.MIME] {231} Content-Type: message/external-body; name="BodyFormats.ps"; site="thumper.bellcore.com"; mode="image"; access-type=ANON-FTP; directory="pub"; expiration="Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" ---------------------------------- BODY[1.1] {79} Content-type: application/postscript Content-ID: BINARY[1.1] {16} [binary content] BINARY.SIZE[1.1] 79 ---------------------------------- BODY[2] {79} Content-type: application/postscript Content-ID: BINARY[2] {16} [binary content] BINARY.SIZE[2] 79 ---------------------------------- BODY[2.HEADER] {218} Content-Type: message/external-body; access-type=local-file; name="/u/nsb/writing/rfcs/RFC-MIME.ps"; site="thumper.bellcore.com"; expiration="Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" ---------------------------------- BODY[2.TEXT] {79} Content-type: application/postscript Content-ID: ---------------------------------- BODY[2.MIME] {219} Content-Type: message/external-body; access-type=local-file; name="/u/nsb/writing/rfcs/RFC-MIME.ps"; site="thumper.bellcore.com"; expiration="Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" ---------------------------------- BODY[2.1] {79} Content-type: application/postscript Content-ID: BINARY[2.1] {16} [binary content] BINARY.SIZE[2.1] 79 ---------------------------------- BODY[3] {97} Content-type: application/postscript Content-ID: get RFC-MIME.DOC BINARY[3] {16} [binary content] BINARY.SIZE[3] 97 ---------------------------------- BODY[3.HEADER] {181} Content-Type: message/external-body; access-type=mail-server server="listserv@bogus.bitnet"; expiration="Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" ---------------------------------- BODY[3.TEXT] {97} Content-type: application/postscript Content-ID: get RFC-MIME.DOC ---------------------------------- BODY[3.MIME] {182} Content-Type: message/external-body; access-type=mail-server server="listserv@bogus.bitnet"; expiration="Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" ---------------------------------- BODY[3.1] {97} Content-type: application/postscript Content-ID: get RFC-MIME.DOC BINARY[3.1] {16} [binary content] BINARY.SIZE[3.1] 97 ---------------------------------- BODY[HEADER.FIELDS (FROM TO)] {29} From: Whomever To: Someone ---------------------------------- BODY[HEADER.FIELDS (FROM TO)]<10> {19} ever To: Someone ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)] {181} From: Whomever To: Someone Date: Whenever MIME-Version: 1.0 Message-ID: Content-Type: multipart/alternative; boundary=42 Content-ID: ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)]<10> {25} ever To: Someone Date: Wh ---------------------------------- ================================================ FILE: tests/resources/imap/001.txt ================================================ From: Whomever To: Someone Date: Whenever Subject: whatever MIME-Version: 1.0 Message-ID: Content-Type: multipart/alternative; boundary=42 Content-ID: --42 Content-Type: message/external-body; name="BodyFormats.ps"; site="thumper.bellcore.com"; mode="image"; access-type=ANON-FTP; directory="pub"; expiration="Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" Content-type: application/postscript Content-ID: --42 Content-Type: message/external-body; access-type=local-file; name="/u/nsb/writing/rfcs/RFC-MIME.ps"; site="thumper.bellcore.com"; expiration="Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" Content-type: application/postscript Content-ID: --42 Content-Type: message/external-body; access-type=mail-server server="listserv@bogus.bitnet"; expiration="Fri, 14 Jun 1991 19:13:14 -0400 (EDT)" Content-type: application/postscript Content-ID: get RFC-MIME.DOC --42-- ================================================ FILE: tests/resources/imap/002.imap ================================================ BODY ( ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 61 5 )( "message" "rfc822" NIL NIL NIL "7bit" 1979 ( "Thu, 13 Aug 1998 17:42:41 +1000" "Map of Argentina with Description" ( ( "Bill Clinton" NIL "president" "whitehouse.gov" ) ) ( ( "Bill Clinton" NIL "president" "whitehouse.gov" ) ) ( ( "Bill Clinton" NIL "president" "whitehouse.gov" ) ) ( ( "A1 Gore (The Enforcer)" NIL "vice-president" "whitehouse.gov" ) ) NIL NIL NIL "<199804130742.RAA20366@mai1host.whitehouse.gov>" ) ( ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 355 12 )( "image" "gif" ( "name" "map_of_Argentina.gif" ) NIL NIL "base64" 389 ) "mixed" ) 0 ) "mixed" ) BODYSTRUCTURE ( ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 61 5 "63aa55d7eacd3e7d4af551d3d7f8fe46" NIL NIL NIL )( "message" "rfc822" NIL NIL NIL "7bit" 1979 ( "Thu, 13 Aug 1998 17:42:41 +1000" "Map of Argentina with Description" ( ( "Bill Clinton" NIL "president" "whitehouse.gov" ) ) ( ( "Bill Clinton" NIL "president" "whitehouse.gov" ) ) ( ( "Bill Clinton" NIL "president" "whitehouse.gov" ) ) ( ( "A1 Gore (The Enforcer)" NIL "vice-president" "whitehouse.gov" ) ) NIL NIL NIL "<199804130742.RAA20366@mai1host.whitehouse.gov>" ) ( ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 355 12 "47027313db254547987c7eb4be377593" NIL NIL NIL )( "image" "gif" ( "name" "map_of_Argentina.gif" ) NIL NIL "base64" 389 "5359a63540c1a8eae06a27e8760e1501" ( "inline" ( "fi1ename" "map_of_Argentina.gif" ) ) NIL NIL ) "mixed" ( "boundary" "DC8------------DC8638F443D87A7F0726DEF7" ) NIL NIL NIL ) 0 "1522079b4114146b37ea1af171484a1a" ( "inline" NIL ) NIL NIL ) "mixed" ( "boundary" "D7F------------D7FD5A0B8AB9C65CCDBFA872" ) NIL NIL NIL ) BODY[] {2652} From: Al Gore To: White House Transportation Coordinator Subject: [Fwd: Map of Argentina with Description] Content-Type: multipart/mixed; boundary="D7F------------D7FD5A0B8AB9C65CCDBFA872" This is a multi-part message in MIME format. --D7F------------D7FD5A0B8AB9C65CCDBFA872 Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit Fred, Fire up Air Force One! We're going South! Thanks, Al --D7F------------D7FD5A0B8AB9C65CCDBFA872 Content-Type: message/rfc822 Content-Transfer-Encoding: 7bit Content-Disposition: inline Return-Path: Received: from mailhost.whitehouse.gov ([192.168.51.200]) by heartbeat.whitehouse.gov (8.8.8/8.8.8) with ESMTP id SAA22453 for ; Mon, 13 Aug 1998 l8:14:23 +1000 Received: from the_big_box.whitehouse.gov ([192.168.51.50]) by mailhost.whitehouse.gov (8.8.8/8.8.7) with ESMTP id RAA20366 for vice-president@whitehouse.gov; Mon, 13 Aug 1998 17:42:41 +1000 Date: Mon, 13 Aug 1998 17:42:41 +1000 Message-Id: <199804130742.RAA20366@mai1host.whitehouse.gov> From: Bill Clinton To: A1 (The Enforcer) Gore Subject: Map of Argentina with Description MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="DC8------------DC8638F443D87A7F0726DEF7" This is a multi-part message in MIME format. --DC8------------DC8638F443D87A7F0726DEF7 Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit Hi A1, I finally figured out this MIME thing. Pretty cool. I'll send you some sax music in .au files next week! Anyway, the attached image is really too small to get a good look at Argentina. Try this for a much better map: http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm Then again, shouldn't the CIA have something like that? Bill --DC8------------DC8638F443D87A7F0726DEF7 Content-Type: image/gif; name="map_of_Argentina.gif" Content-Transfer-Encoding: base64 Content-Disposition: inline; fi1ename="map_of_Argentina.gif" R01GOD1hJQA1AKIAAP/////78P/omn19fQAAAAAAAAAAAAAAACwAAAAAJQA1AAAD7Qi63P5w wEmjBCLrnQnhYCgM1wh+pkgqqeC9XrutmBm7hAK3tP31gFcAiFKVQrGFR6kscnonTe7FAAad GugmRu3CmiBt57fsVq3Y0VFKnpYdxPC6M7Ze4crnnHum4oN6LFJ1bn5NXTN7OF5fQkN5WYow BEN2dkGQGWJtSzqGTICJgnQuTJN/WJsojad9qXMuhIWdjXKjY4tenjo6tjVssk2gaWq3uGNX U6ZGxseyk8SasGw3J9GRzdTQky1iHNvcPNNI4TLeKdfMvy0vMqLrItvuxfDW8ubjueDtJufz 7itICBxISKDBgwgTKjyYAAA7 --DC8------------DC8638F443D87A7F0726DEF7-- --D7F------------D7FD5A0B8AB9C65CCDBFA872-- BINARY[] {16} [binary content] BINARY.SIZE[] 2652 ---------------------------------- BODY[HEADER] {270} From: Al Gore To: White House Transportation Coordinator Subject: [Fwd: Map of Argentina with Description] Content-Type: multipart/mixed; boundary="D7F------------D7FD5A0B8AB9C65CCDBFA872" ---------------------------------- BODY[TEXT] {2382} This is a multi-part message in MIME format. --D7F------------D7FD5A0B8AB9C65CCDBFA872 Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit Fred, Fire up Air Force One! We're going South! Thanks, Al --D7F------------D7FD5A0B8AB9C65CCDBFA872 Content-Type: message/rfc822 Content-Transfer-Encoding: 7bit Content-Disposition: inline Return-Path: Received: from mailhost.whitehouse.gov ([192.168.51.200]) by heartbeat.whitehouse.gov (8.8.8/8.8.8) with ESMTP id SAA22453 for ; Mon, 13 Aug 1998 l8:14:23 +1000 Received: from the_big_box.whitehouse.gov ([192.168.51.50]) by mailhost.whitehouse.gov (8.8.8/8.8.7) with ESMTP id RAA20366 for vice-president@whitehouse.gov; Mon, 13 Aug 1998 17:42:41 +1000 Date: Mon, 13 Aug 1998 17:42:41 +1000 Message-Id: <199804130742.RAA20366@mai1host.whitehouse.gov> From: Bill Clinton To: A1 (The Enforcer) Gore Subject: Map of Argentina with Description MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="DC8------------DC8638F443D87A7F0726DEF7" This is a multi-part message in MIME format. --DC8------------DC8638F443D87A7F0726DEF7 Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit Hi A1, I finally figured out this MIME thing. Pretty cool. I'll send you some sax music in .au files next week! Anyway, the attached image is really too small to get a good look at Argentina. Try this for a much better map: http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm Then again, shouldn't the CIA have something like that? Bill --DC8------------DC8638F443D87A7F0726DEF7 Content-Type: image/gif; name="map_of_Argentina.gif" Content-Transfer-Encoding: base64 Content-Disposition: inline; fi1ename="map_of_Argentina.gif" R01GOD1hJQA1AKIAAP/////78P/omn19fQAAAAAAAAAAAAAAACwAAAAAJQA1AAAD7Qi63P5w wEmjBCLrnQnhYCgM1wh+pkgqqeC9XrutmBm7hAK3tP31gFcAiFKVQrGFR6kscnonTe7FAAad GugmRu3CmiBt57fsVq3Y0VFKnpYdxPC6M7Ze4crnnHum4oN6LFJ1bn5NXTN7OF5fQkN5WYow BEN2dkGQGWJtSzqGTICJgnQuTJN/WJsojad9qXMuhIWdjXKjY4tenjo6tjVssk2gaWq3uGNX U6ZGxseyk8SasGw3J9GRzdTQky1iHNvcPNNI4TLeKdfMvy0vMqLrItvuxfDW8ubjueDtJufz 7itICBxISKDBgwgTKjyYAAA7 --DC8------------DC8638F443D87A7F0726DEF7-- --D7F------------D7FD5A0B8AB9C65CCDBFA872-- ---------------------------------- BODY[MIME] {98} Content-Type: multipart/mixed; boundary="D7F------------D7FD5A0B8AB9C65CCDBFA872" ---------------------------------- BODY[1] {61} Fred, Fire up Air Force One! We're going South! Thanks, Al BINARY[1] {61} Fred, Fire up Air Force One! We're going South! Thanks, Al BINARY.SIZE[1] 61 ---------------------------------- BODY[1.HEADER] {76} Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit ---------------------------------- BODY[1.TEXT] {61} Fred, Fire up Air Force One! We're going South! Thanks, Al ---------------------------------- BODY[1.MIME] {77} Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit ---------------------------------- BODY[1.1] {61} Fred, Fire up Air Force One! We're going South! Thanks, Al BINARY[1.1] {61} Fred, Fire up Air Force One! We're going South! Thanks, Al BINARY.SIZE[1.1] 61 ---------------------------------- BODY[2] {1979} Return-Path: Received: from mailhost.whitehouse.gov ([192.168.51.200]) by heartbeat.whitehouse.gov (8.8.8/8.8.8) with ESMTP id SAA22453 for ; Mon, 13 Aug 1998 l8:14:23 +1000 Received: from the_big_box.whitehouse.gov ([192.168.51.50]) by mailhost.whitehouse.gov (8.8.8/8.8.7) with ESMTP id RAA20366 for vice-president@whitehouse.gov; Mon, 13 Aug 1998 17:42:41 +1000 Date: Mon, 13 Aug 1998 17:42:41 +1000 Message-Id: <199804130742.RAA20366@mai1host.whitehouse.gov> From: Bill Clinton To: A1 (The Enforcer) Gore Subject: Map of Argentina with Description MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="DC8------------DC8638F443D87A7F0726DEF7" This is a multi-part message in MIME format. --DC8------------DC8638F443D87A7F0726DEF7 Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit Hi A1, I finally figured out this MIME thing. Pretty cool. I'll send you some sax music in .au files next week! Anyway, the attached image is really too small to get a good look at Argentina. Try this for a much better map: http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm Then again, shouldn't the CIA have something like that? Bill --DC8------------DC8638F443D87A7F0726DEF7 Content-Type: image/gif; name="map_of_Argentina.gif" Content-Transfer-Encoding: base64 Content-Disposition: inline; fi1ename="map_of_Argentina.gif" R01GOD1hJQA1AKIAAP/////78P/omn19fQAAAAAAAAAAAAAAACwAAAAAJQA1AAAD7Qi63P5w wEmjBCLrnQnhYCgM1wh+pkgqqeC9XrutmBm7hAK3tP31gFcAiFKVQrGFR6kscnonTe7FAAad GugmRu3CmiBt57fsVq3Y0VFKnpYdxPC6M7Ze4crnnHum4oN6LFJ1bn5NXTN7OF5fQkN5WYow BEN2dkGQGWJtSzqGTICJgnQuTJN/WJsojad9qXMuhIWdjXKjY4tenjo6tjVssk2gaWq3uGNX U6ZGxseyk8SasGw3J9GRzdTQky1iHNvcPNNI4TLeKdfMvy0vMqLrItvuxfDW8ubjueDtJufz 7itICBxISKDBgwgTKjyYAAA7 --DC8------------DC8638F443D87A7F0726DEF7-- BINARY[2] {16} [binary content] BINARY.SIZE[2] 1979 ---------------------------------- BODY[2.HEADER] {835} Return-Path: Received: from mailhost.whitehouse.gov ([192.168.51.200]) by heartbeat.whitehouse.gov (8.8.8/8.8.8) with ESMTP id SAA22453 for ; Mon, 13 Aug 1998 l8:14:23 +1000 Received: from the_big_box.whitehouse.gov ([192.168.51.50]) by mailhost.whitehouse.gov (8.8.8/8.8.7) with ESMTP id RAA20366 for vice-president@whitehouse.gov; Mon, 13 Aug 1998 17:42:41 +1000 Date: Mon, 13 Aug 1998 17:42:41 +1000 Message-Id: <199804130742.RAA20366@mai1host.whitehouse.gov> From: Bill Clinton To: A1 (The Enforcer) Gore Subject: Map of Argentina with Description MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="DC8------------DC8638F443D87A7F0726DEF7" ---------------------------------- BODY[2.TEXT] {1144} This is a multi-part message in MIME format. --DC8------------DC8638F443D87A7F0726DEF7 Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit Hi A1, I finally figured out this MIME thing. Pretty cool. I'll send you some sax music in .au files next week! Anyway, the attached image is really too small to get a good look at Argentina. Try this for a much better map: http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm Then again, shouldn't the CIA have something like that? Bill --DC8------------DC8638F443D87A7F0726DEF7 Content-Type: image/gif; name="map_of_Argentina.gif" Content-Transfer-Encoding: base64 Content-Disposition: inline; fi1ename="map_of_Argentina.gif" R01GOD1hJQA1AKIAAP/////78P/omn19fQAAAAAAAAAAAAAAACwAAAAAJQA1AAAD7Qi63P5w wEmjBCLrnQnhYCgM1wh+pkgqqeC9XrutmBm7hAK3tP31gFcAiFKVQrGFR6kscnonTe7FAAad GugmRu3CmiBt57fsVq3Y0VFKnpYdxPC6M7Ze4crnnHum4oN6LFJ1bn5NXTN7OF5fQkN5WYow BEN2dkGQGWJtSzqGTICJgnQuTJN/WJsojad9qXMuhIWdjXKjY4tenjo6tjVssk2gaWq3uGNX U6ZGxseyk8SasGw3J9GRzdTQky1iHNvcPNNI4TLeKdfMvy0vMqLrItvuxfDW8ubjueDtJufz 7itICBxISKDBgwgTKjyYAAA7 --DC8------------DC8638F443D87A7F0726DEF7-- ---------------------------------- BODY[2.MIME] {91} Content-Type: message/rfc822 Content-Transfer-Encoding: 7bit Content-Disposition: inline ---------------------------------- BODY[2.1] {355} Hi A1, I finally figured out this MIME thing. Pretty cool. I'll send you some sax music in .au files next week! Anyway, the attached image is really too small to get a good look at Argentina. Try this for a much better map: http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm Then again, shouldn't the CIA have something like that? Bill BINARY[2.1] {355} Hi A1, I finally figured out this MIME thing. Pretty cool. I'll send you some sax music in .au files next week! Anyway, the attached image is really too small to get a good look at Argentina. Try this for a much better map: http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm Then again, shouldn't the CIA have something like that? Bill BINARY.SIZE[2.1] 355 ---------------------------------- BODY[2.1.HEADER] {76} Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit ---------------------------------- BODY[2.1.TEXT] {355} Hi A1, I finally figured out this MIME thing. Pretty cool. I'll send you some sax music in .au files next week! Anyway, the attached image is really too small to get a good look at Argentina. Try this for a much better map: http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm Then again, shouldn't the CIA have something like that? Bill ---------------------------------- BODY[2.1.MIME] {77} Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit ---------------------------------- BODY[2.1.1] {355} Hi A1, I finally figured out this MIME thing. Pretty cool. I'll send you some sax music in .au files next week! Anyway, the attached image is really too small to get a good look at Argentina. Try this for a much better map: http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm Then again, shouldn't the CIA have something like that? Bill BINARY[2.1.1] {355} Hi A1, I finally figured out this MIME thing. Pretty cool. I'll send you some sax music in .au files next week! Anyway, the attached image is really too small to get a good look at Argentina. Try this for a much better map: http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm Then again, shouldn't the CIA have something like that? Bill BINARY.SIZE[2.1.1] 355 ---------------------------------- BODY[2.2] {389} R01GOD1hJQA1AKIAAP/////78P/omn19fQAAAAAAAAAAAAAAACwAAAAAJQA1AAAD7Qi63P5w wEmjBCLrnQnhYCgM1wh+pkgqqeC9XrutmBm7hAK3tP31gFcAiFKVQrGFR6kscnonTe7FAAad GugmRu3CmiBt57fsVq3Y0VFKnpYdxPC6M7Ze4crnnHum4oN6LFJ1bn5NXTN7OF5fQkN5WYow BEN2dkGQGWJtSzqGTICJgnQuTJN/WJsojad9qXMuhIWdjXKjY4tenjo6tjVssk2gaWq3uGNX U6ZGxseyk8SasGw3J9GRzdTQky1iHNvcPNNI4TLeKdfMvy0vMqLrItvuxfDW8ubjueDtJufz 7itICBxISKDBgwgTKjyYAAA7 BINARY[2.2] {16} [binary content] BINARY.SIZE[2.2] 288 ---------------------------------- BODY[2.2.HEADER] {149} Content-Type: image/gif; name="map_of_Argentina.gif" Content-Transfer-Encoding: base64 Content-Disposition: inline; fi1ename="map_of_Argentina.gif" ---------------------------------- BODY[2.2.TEXT] {389} R01GOD1hJQA1AKIAAP/////78P/omn19fQAAAAAAAAAAAAAAACwAAAAAJQA1AAAD7Qi63P5w wEmjBCLrnQnhYCgM1wh+pkgqqeC9XrutmBm7hAK3tP31gFcAiFKVQrGFR6kscnonTe7FAAad GugmRu3CmiBt57fsVq3Y0VFKnpYdxPC6M7Ze4crnnHum4oN6LFJ1bn5NXTN7OF5fQkN5WYow BEN2dkGQGWJtSzqGTICJgnQuTJN/WJsojad9qXMuhIWdjXKjY4tenjo6tjVssk2gaWq3uGNX U6ZGxseyk8SasGw3J9GRzdTQky1iHNvcPNNI4TLeKdfMvy0vMqLrItvuxfDW8ubjueDtJufz 7itICBxISKDBgwgTKjyYAAA7 ---------------------------------- BODY[2.2.MIME] {150} Content-Type: image/gif; name="map_of_Argentina.gif" Content-Transfer-Encoding: base64 Content-Disposition: inline; fi1ename="map_of_Argentina.gif" ---------------------------------- BODY[2.2.1] {389} R01GOD1hJQA1AKIAAP/////78P/omn19fQAAAAAAAAAAAAAAACwAAAAAJQA1AAAD7Qi63P5w wEmjBCLrnQnhYCgM1wh+pkgqqeC9XrutmBm7hAK3tP31gFcAiFKVQrGFR6kscnonTe7FAAad GugmRu3CmiBt57fsVq3Y0VFKnpYdxPC6M7Ze4crnnHum4oN6LFJ1bn5NXTN7OF5fQkN5WYow BEN2dkGQGWJtSzqGTICJgnQuTJN/WJsojad9qXMuhIWdjXKjY4tenjo6tjVssk2gaWq3uGNX U6ZGxseyk8SasGw3J9GRzdTQky1iHNvcPNNI4TLeKdfMvy0vMqLrItvuxfDW8ubjueDtJufz 7itICBxISKDBgwgTKjyYAAA7 BINARY[2.2.1] {16} [binary content] BINARY.SIZE[2.2.1] 288 ---------------------------------- BODY[HEADER.FIELDS (FROM TO)] {125} From: Al Gore To: White House Transportation Coordinator ---------------------------------- BODY[HEADER.FIELDS (FROM TO)]<10> {25} Gore To: White House Transportation Coordinator Content-Type: multipart/mixed; boundary="D7F------------D7FD5A0B8AB9C65CCDBFA872" ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)]<10> {25} Gore To: White House Transportation Coordinator Subject: [Fwd: Map of Argentina with Description] Content-Type: multipart/mixed; boundary="D7F------------D7FD5A0B8AB9C65CCDBFA872" This is a multi-part message in MIME format. --D7F------------D7FD5A0B8AB9C65CCDBFA872 Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit Fred, Fire up Air Force One! We're going South! Thanks, Al --D7F------------D7FD5A0B8AB9C65CCDBFA872 Content-Type: message/rfc822 Content-Transfer-Encoding: 7bit Content-Disposition: inline Return-Path: Received: from mailhost.whitehouse.gov ([192.168.51.200]) by heartbeat.whitehouse.gov (8.8.8/8.8.8) with ESMTP id SAA22453 for ; Mon, 13 Aug 1998 l8:14:23 +1000 Received: from the_big_box.whitehouse.gov ([192.168.51.50]) by mailhost.whitehouse.gov (8.8.8/8.8.7) with ESMTP id RAA20366 for vice-president@whitehouse.gov; Mon, 13 Aug 1998 17:42:41 +1000 Date: Mon, 13 Aug 1998 17:42:41 +1000 Message-Id: <199804130742.RAA20366@mai1host.whitehouse.gov> From: Bill Clinton To: A1 (The Enforcer) Gore Subject: Map of Argentina with Description MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="DC8------------DC8638F443D87A7F0726DEF7" This is a multi-part message in MIME format. --DC8------------DC8638F443D87A7F0726DEF7 Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit Hi A1, I finally figured out this MIME thing. Pretty cool. I'll send you some sax music in .au files next week! Anyway, the attached image is really too small to get a good look at Argentina. Try this for a much better map: http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm Then again, shouldn't the CIA have something like that? Bill --DC8------------DC8638F443D87A7F0726DEF7 Content-Type: image/gif; name="map_of_Argentina.gif" Content-Transfer-Encoding: base64 Content-Disposition: inline; fi1ename="map_of_Argentina.gif" R01GOD1hJQA1AKIAAP/////78P/omn19fQAAAAAAAAAAAAAAACwAAAAAJQA1AAAD7Qi63P5w wEmjBCLrnQnhYCgM1wh+pkgqqeC9XrutmBm7hAK3tP31gFcAiFKVQrGFR6kscnonTe7FAAad GugmRu3CmiBt57fsVq3Y0VFKnpYdxPC6M7Ze4crnnHum4oN6LFJ1bn5NXTN7OF5fQkN5WYow BEN2dkGQGWJtSzqGTICJgnQuTJN/WJsojad9qXMuhIWdjXKjY4tenjo6tjVssk2gaWq3uGNX U6ZGxseyk8SasGw3J9GRzdTQky1iHNvcPNNI4TLeKdfMvy0vMqLrItvuxfDW8ubjueDtJufz 7itICBxISKDBgwgTKjyYAAA7 --DC8------------DC8638F443D87A7F0726DEF7-- --D7F------------D7FD5A0B8AB9C65CCDBFA872-- ================================================ FILE: tests/resources/imap/003.imap ================================================ BODY ( ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 48 1 )( "text" "enriched" ( "charset" "us-ascii" ) NIL NIL "7bit" 69 2 )( "application" "x-whatever" NIL NIL NIL NIL 51 ) "alternative" ) BODYSTRUCTURE ( ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 48 1 "2229d79e5de40eae37c43fe934adbd24" NIL NIL NIL )( "text" "enriched" ( "charset" "us-ascii" ) NIL NIL "7bit" 69 2 "1aba0fd91c6544b8626008f78b83c3f9" NIL NIL NIL )( "application" "x-whatever" NIL NIL NIL NIL 51 "7f7ef645554f637854b8acdd853aa612" NIL NIL NIL ) "alternative" ( "boundary" "boundary42" ) NIL NIL NIL ) BODY[] {565} From: Nathaniel Borenstein To: Ned Freed Date: Mon, 22 Mar 1993 09:41:09 -0800 (PST) Subject: Formatted text mail MIME-Version: 1.0 Content-Type: multipart/alternative; boundary=boundary42 --boundary42 Content-Type: text/plain; charset=us-ascii ... plain text version of message goes here ... --boundary42 Content-Type: text/enriched ... RFC 1896 text/enriched version of same message goes here ... --boundary42 Content-Type: application/x-whatever ... fanciest version of same message goes here ... --boundary42-- BINARY[] {16} [binary content] BINARY.SIZE[] 565 ---------------------------------- BODY[HEADER] {228} From: Nathaniel Borenstein To: Ned Freed Date: Mon, 22 Mar 1993 09:41:09 -0800 (PST) Subject: Formatted text mail MIME-Version: 1.0 Content-Type: multipart/alternative; boundary=boundary42 ---------------------------------- BODY[TEXT] {337} --boundary42 Content-Type: text/plain; charset=us-ascii ... plain text version of message goes here ... --boundary42 Content-Type: text/enriched ... RFC 1896 text/enriched version of same message goes here ... --boundary42 Content-Type: application/x-whatever ... fanciest version of same message goes here ... --boundary42-- ---------------------------------- BODY[MIME] {59} Content-Type: multipart/alternative; boundary=boundary42 ---------------------------------- BODY[1] {48} ... plain text version of message goes here ... BINARY[1] {48} ... plain text version of message goes here ... BINARY.SIZE[1] 48 ---------------------------------- BODY[1.HEADER] {44} Content-Type: text/plain; charset=us-ascii ---------------------------------- BODY[1.TEXT] {48} ... plain text version of message goes here ... ---------------------------------- BODY[1.MIME] {45} Content-Type: text/plain; charset=us-ascii ---------------------------------- BODY[1.1] {48} ... plain text version of message goes here ... BINARY[1.1] {48} ... plain text version of message goes here ... BINARY.SIZE[1.1] 48 ---------------------------------- BODY[2] {69} ... RFC 1896 text/enriched version of same message goes here ... BINARY[2] {69} ... RFC 1896 text/enriched version of same message goes here ... BINARY.SIZE[2] 69 ---------------------------------- BODY[2.HEADER] {29} Content-Type: text/enriched ---------------------------------- BODY[2.TEXT] {69} ... RFC 1896 text/enriched version of same message goes here ... ---------------------------------- BODY[2.MIME] {30} Content-Type: text/enriched ---------------------------------- BODY[2.1] {69} ... RFC 1896 text/enriched version of same message goes here ... BINARY[2.1] {69} ... RFC 1896 text/enriched version of same message goes here ... BINARY.SIZE[2.1] 69 ---------------------------------- BODY[3] {51} ... fanciest version of same message goes here ... BINARY[3] {16} [binary content] BINARY.SIZE[3] 51 ---------------------------------- BODY[3.HEADER] {38} Content-Type: application/x-whatever ---------------------------------- BODY[3.TEXT] {51} ... fanciest version of same message goes here ... ---------------------------------- BODY[3.MIME] {39} Content-Type: application/x-whatever ---------------------------------- BODY[3.1] {51} ... fanciest version of same message goes here ... BINARY[3.1] {16} [binary content] BINARY.SIZE[3.1] 51 ---------------------------------- BODY[HEADER.FIELDS (FROM TO)] {81} From: Nathaniel Borenstein To: Ned Freed ---------------------------------- BODY[HEADER.FIELDS (FROM TO)]<10> {25} aniel Borenstein To: Ned Freed Date: Mon, 22 Mar 1993 09:41:09 -0800 (PST) MIME-Version: 1.0 Content-Type: multipart/alternative; boundary=boundary42 ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)]<10> {25} aniel Borenstein To: Ned Freed Date: Mon, 22 Mar 1993 09:41:09 -0800 (PST) Subject: Formatted text mail MIME-Version: 1.0 Content-Type: multipart/alternative; boundary=boundary42 --boundary42 Content-Type: text/plain; charset=us-ascii ... plain text version of message goes here ... --boundary42 Content-Type: text/enriched ... RFC 1896 text/enriched version of same message goes here ... --boundary42 Content-Type: application/x-whatever ... fanciest version of same message goes here ... --boundary42-- ================================================ FILE: tests/resources/imap/004.imap ================================================ BODY ( ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 45 1 )( ( "message" NIL NIL NIL NIL NIL 100 ( "Fri, 26 Mar 1993 11:13:32 +0200" "my opinion" ( ( NIL NIL "unknown" "localhost" ) ) ( ( NIL NIL "unknown" "localhost" ) ) ( ( NIL NIL "unknown" "localhost" ) ) NIL NIL NIL NIL NIL ) ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 22 1 ) 0 )( "message" NIL NIL NIL NIL NIL 125 ( "Fri, 26 Mar 1993 10:07:13 -0500" "my different opinion" ( ( NIL NIL "unknown" "localhost" ) ) ( ( NIL NIL "unknown" "localhost" ) ) ( ( NIL NIL "unknown" "localhost" ) ) NIL NIL NIL NIL NIL ) ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 31 1 ) 0 ) "digest" ) "mixed" ) BODYSTRUCTURE ( ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 45 1 "f7f7e917e54763330bc648984459b38e" NIL NIL NIL )( ( "message" NIL NIL NIL NIL NIL 100 ( "Fri, 26 Mar 1993 11:13:32 +0200" "my opinion" ( ( NIL NIL "unknown" "localhost" ) ) ( ( NIL NIL "unknown" "localhost" ) ) ( ( NIL NIL "unknown" "localhost" ) ) NIL NIL NIL NIL NIL ) ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 22 1 "1ec4e1f0175dc661a12854b33879ce9b" NIL NIL NIL ) 0 "ef1527984c599523c90fb184a8fdfd61" NIL NIL NIL )( "message" NIL NIL NIL NIL NIL 125 ( "Fri, 26 Mar 1993 10:07:13 -0500" "my different opinion" ( ( NIL NIL "unknown" "localhost" ) ) ( ( NIL NIL "unknown" "localhost" ) ) ( ( NIL NIL "unknown" "localhost" ) ) NIL NIL NIL NIL NIL ) ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 31 1 "7a0001277b1197e7029ad48b54919b62" NIL NIL NIL ) 0 "9609bccef55fba1208046885cdcc9db8" NIL NIL NIL ) "digest" ( "boundary" "---- next message ----" ) NIL NIL NIL ) "mixed" ( "boundary" "---- main boundary ----" ) NIL NIL NIL ) BODY[] {728} From: Moderator-Address To: Recipient-List Date: Mon, 22 Mar 1994 13:34:51 +0000 Subject: Internet Digest, volume 42 MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="---- main boundary ----" ------ main boundary ---- ...Introductory text or table of contents... ------ main boundary ---- Content-Type: multipart/digest; boundary="---- next message ----" ------ next message ---- From: someone-else Date: Fri, 26 Mar 1993 11:13:32 +0200 Subject: my opinion ...body goes here ... ------ next message ---- From: someone-else-again Date: Fri, 26 Mar 1993 10:07:13 -0500 Subject: my different opinion ... another body goes here ... ------ next message ------ ------ main boundary ------ BINARY[] {16} [binary content] BINARY.SIZE[] 728 ---------------------------------- BODY[HEADER] {214} From: Moderator-Address To: Recipient-List Date: Mon, 22 Mar 1994 13:34:51 +0000 Subject: Internet Digest, volume 42 MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="---- main boundary ----" ---------------------------------- BODY[TEXT] {514} ------ main boundary ---- ...Introductory text or table of contents... ------ main boundary ---- Content-Type: multipart/digest; boundary="---- next message ----" ------ next message ---- From: someone-else Date: Fri, 26 Mar 1993 11:13:32 +0200 Subject: my opinion ...body goes here ... ------ next message ---- From: someone-else-again Date: Fri, 26 Mar 1993 10:07:13 -0500 Subject: my different opinion ... another body goes here ... ------ next message ------ ------ main boundary ------ ---------------------------------- BODY[MIME] {80} Content-Type: multipart/mixed; boundary="---- main boundary ----" ---------------------------------- BODY[1] {45} ...Introductory text or table of contents... BINARY[1] {45} ...Introductory text or table of contents... BINARY.SIZE[1] 45 ---------------------------------- BODY[1.HEADER] {1} ---------------------------------- BODY[1.TEXT] {45} ...Introductory text or table of contents... ---------------------------------- BODY[1.MIME] {2} ---------------------------------- BODY[1.1] {45} ...Introductory text or table of contents... BINARY[1.1] {45} ...Introductory text or table of contents... BINARY.SIZE[1.1] 45 ---------------------------------- BODY[2] {306} ------ next message ---- From: someone-else Date: Fri, 26 Mar 1993 11:13:32 +0200 Subject: my opinion ...body goes here ... ------ next message ---- From: someone-else-again Date: Fri, 26 Mar 1993 10:07:13 -0500 Subject: my different opinion ... another body goes here ... ------ next message ------ BINARY[2] {16} [binary content] BINARY.SIZE[2] 385 ---------------------------------- BODY[2.HEADER] {79} Content-Type: multipart/digest; boundary="---- next message ----" ---------------------------------- BODY[2.TEXT] {306} ------ next message ---- From: someone-else Date: Fri, 26 Mar 1993 11:13:32 +0200 Subject: my opinion ...body goes here ... ------ next message ---- From: someone-else-again Date: Fri, 26 Mar 1993 10:07:13 -0500 Subject: my different opinion ... another body goes here ... ------ next message ------ ---------------------------------- BODY[2.MIME] {80} Content-Type: multipart/digest; boundary="---- next message ----" ---------------------------------- BODY[2.1] {100} From: someone-else Date: Fri, 26 Mar 1993 11:13:32 +0200 Subject: my opinion ...body goes here ... BINARY[2.1] {16} [binary content] BINARY.SIZE[2.1] 100 ---------------------------------- BODY[2.1.HEADER] {78} From: someone-else Date: Fri, 26 Mar 1993 11:13:32 +0200 Subject: my opinion ---------------------------------- BODY[2.1.TEXT] {22} ...body goes here ... ---------------------------------- BODY[2.1.MIME] {2} ---------------------------------- BODY[2.1.1] {22} ...body goes here ... BINARY[2.1.1] {22} ...body goes here ... BINARY.SIZE[2.1.1] 22 ---------------------------------- BODY[2.2] {125} From: someone-else-again Date: Fri, 26 Mar 1993 10:07:13 -0500 Subject: my different opinion ... another body goes here ... BINARY[2.2] {16} [binary content] BINARY.SIZE[2.2] 125 ---------------------------------- BODY[2.2.HEADER] {94} From: someone-else-again Date: Fri, 26 Mar 1993 10:07:13 -0500 Subject: my different opinion ---------------------------------- BODY[2.2.TEXT] {31} ... another body goes here ... ---------------------------------- BODY[2.2.MIME] {2} ---------------------------------- BODY[2.2.1] {31} ... another body goes here ... BINARY[2.2.1] {31} ... another body goes here ... BINARY.SIZE[2.2.1] 31 ---------------------------------- BODY[HEADER.FIELDS (FROM TO)] {45} From: Moderator-Address To: Recipient-List ---------------------------------- BODY[HEADER.FIELDS (FROM TO)]<10> {25} rator-Address To: Recipie ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)] {179} From: Moderator-Address To: Recipient-List Date: Mon, 22 Mar 1994 13:34:51 +0000 MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="---- main boundary ----" ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)]<10> {25} rator-Address To: Recipie ---------------------------------- ================================================ FILE: tests/resources/imap/004.txt ================================================ From: Moderator-Address To: Recipient-List Date: Mon, 22 Mar 1994 13:34:51 +0000 Subject: Internet Digest, volume 42 MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="---- main boundary ----" ------ main boundary ---- ...Introductory text or table of contents... ------ main boundary ---- Content-Type: multipart/digest; boundary="---- next message ----" ------ next message ---- From: someone-else Date: Fri, 26 Mar 1993 11:13:32 +0200 Subject: my opinion ...body goes here ... ------ next message ---- From: someone-else-again Date: Fri, 26 Mar 1993 10:07:13 -0500 Subject: my different opinion ... another body goes here ... ------ next message ------ ------ main boundary ------ ================================================ FILE: tests/resources/imap/005.imap ================================================ BODY ( ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 79 1 )( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 76 2 ) "mixed" ) BODYSTRUCTURE ( ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 79 1 "b35878dedb7cd0aa6934f90df9d517b0" NIL NIL NIL )( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 76 2 "d3905bfd2a53ad0a4438d01bf271677a" NIL NIL NIL ) "mixed" ( "boundary" "simple boundary" ) NIL NIL NIL ) BODY[] {691} From: Nathaniel Borenstein To: Ned Freed Date: Sun, 21 Mar 1993 23:56:48 -0800 (PST) Subject: Sample message MIME-Version: 1.0 Content-type: multipart/mixed; boundary="simple boundary" This is the preamble. It is to be ignored, though it is a handy place for composition agents to include an explanatory note to non-MIME conformant readers. --simple boundary This is implicitly typed plain US-ASCII text. It does NOT end with a linebreak. --simple boundary Content-type: text/plain; charset=us-ascii This is explicitly typed plain US-ASCII text. It DOES end with a linebreak. --simple boundary-- This is the epilogue. It is also to be ignored. BINARY[] {16} [binary content] BINARY.SIZE[] 691 ---------------------------------- BODY[HEADER] {224} From: Nathaniel Borenstein To: Ned Freed Date: Sun, 21 Mar 1993 23:56:48 -0800 (PST) Subject: Sample message MIME-Version: 1.0 Content-type: multipart/mixed; boundary="simple boundary" ---------------------------------- BODY[TEXT] {467} This is the preamble. It is to be ignored, though it is a handy place for composition agents to include an explanatory note to non-MIME conformant readers. --simple boundary This is implicitly typed plain US-ASCII text. It does NOT end with a linebreak. --simple boundary Content-type: text/plain; charset=us-ascii This is explicitly typed plain US-ASCII text. It DOES end with a linebreak. --simple boundary-- This is the epilogue. It is also to be ignored. ---------------------------------- BODY[MIME] {60} Content-Type: multipart/mixed; boundary="simple boundary" ---------------------------------- BODY[1] {79} This is implicitly typed plain US-ASCII text. It does NOT end with a linebreak. BINARY[1] {79} This is implicitly typed plain US-ASCII text. It does NOT end with a linebreak. BINARY.SIZE[1] 79 ---------------------------------- BODY[1.HEADER] {1} ---------------------------------- BODY[1.TEXT] {79} This is implicitly typed plain US-ASCII text. It does NOT end with a linebreak. ---------------------------------- BODY[1.MIME] {2} ---------------------------------- BODY[1.1] {79} This is implicitly typed plain US-ASCII text. It does NOT end with a linebreak. BINARY[1.1] {79} This is implicitly typed plain US-ASCII text. It does NOT end with a linebreak. BINARY.SIZE[1.1] 79 ---------------------------------- BODY[2] {76} This is explicitly typed plain US-ASCII text. It DOES end with a linebreak. BINARY[2] {76} This is explicitly typed plain US-ASCII text. It DOES end with a linebreak. BINARY.SIZE[2] 76 ---------------------------------- BODY[2.HEADER] {44} Content-type: text/plain; charset=us-ascii ---------------------------------- BODY[2.TEXT] {76} This is explicitly typed plain US-ASCII text. It DOES end with a linebreak. ---------------------------------- BODY[2.MIME] {45} Content-Type: text/plain; charset=us-ascii ---------------------------------- BODY[2.1] {76} This is explicitly typed plain US-ASCII text. It DOES end with a linebreak. BINARY[2.1] {76} This is explicitly typed plain US-ASCII text. It DOES end with a linebreak. BINARY.SIZE[2.1] 76 ---------------------------------- BODY[HEADER.FIELDS (FROM TO)] {81} From: Nathaniel Borenstein To: Ned Freed ---------------------------------- BODY[HEADER.FIELDS (FROM TO)]<10> {25} aniel Borenstein To: Ned Freed Date: Sun, 21 Mar 1993 23:56:48 -0800 (PST) MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="simple boundary" ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)]<10> {25} aniel Borenstein To: Ned Freed Date: Sun, 21 Mar 1993 23:56:48 -0800 (PST) Subject: Sample message MIME-Version: 1.0 Content-type: multipart/mixed; boundary="simple boundary" This is the preamble. It is to be ignored, though it is a handy place for composition agents to include an explanatory note to non-MIME conformant readers. --simple boundary This is implicitly typed plain US-ASCII text. It does NOT end with a linebreak. --simple boundary Content-type: text/plain; charset=us-ascii This is explicitly typed plain US-ASCII text. It DOES end with a linebreak. --simple boundary-- This is the epilogue. It is also to be ignored. ================================================ FILE: tests/resources/imap/006.imap ================================================ BODY ( ( "text" "plain" ( "charset" "utf-8" ) NIL NIL "quoted-printable" 87 2 )( "text" "html" ( "charset" "utf-8" ) NIL NIL "quoted-printable" 93 2 ) "alternative" ) BODYSTRUCTURE ( ( "text" "plain" ( "charset" "utf-8" ) NIL NIL "quoted-printable" 87 2 "ba404f7e85d80b41eddae75d5087186d" ( "inline" NIL ) NIL NIL )( "text" "html" ( "charset" "utf-8" ) NIL NIL "quoted-printable" 93 2 "bb2724458c0a6183195d922d711523c0" ( "inline" NIL ) NIL NIL ) "alternative" ( "boundary" "boundary-string" ) NIL NIL NIL ) BODY[] {617} From: sender@example.com To: recipient@example.com Subject: Multipart Email Example Content-Type: multipart/alternative; boundary="boundary-string" --boundary-string Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Content-Disposition: inline Plain text email goes here! This is the fallback if email client does not support HTML --boundary-string Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: quoted-printable Content-Disposition: inline

This is the HTML Section!

This is what displays in most modern email clients

--boundary-string-- BINARY[] {16} [binary content] BINARY.SIZE[] 617 ---------------------------------- BODY[HEADER] {149} From: sender@example.com To: recipient@example.com Subject: Multipart Email Example Content-Type: multipart/alternative; boundary="boundary-string" ---------------------------------- BODY[TEXT] {468} --boundary-string Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Content-Disposition: inline Plain text email goes here! This is the fallback if email client does not support HTML --boundary-string Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: quoted-printable Content-Disposition: inline

This is the HTML Section!

This is what displays in most modern email clients

--boundary-string-- ---------------------------------- BODY[MIME] {66} Content-Type: multipart/alternative; boundary="boundary-string" ---------------------------------- BODY[1] {87} Plain text email goes here! This is the fallback if email client does not support HTML BINARY[1] {87} Plain text email goes here! This is the fallback if email client does not support HTML BINARY.SIZE[1] 87 ---------------------------------- BODY[1.HEADER] {115} Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Content-Disposition: inline ---------------------------------- BODY[1.TEXT] {87} Plain text email goes here! This is the fallback if email client does not support HTML ---------------------------------- BODY[1.MIME] {116} Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Content-Disposition: inline ---------------------------------- BODY[1.1] {87} Plain text email goes here! This is the fallback if email client does not support HTML BINARY[1.1] {87} Plain text email goes here! This is the fallback if email client does not support HTML BINARY.SIZE[1.1] 87 ---------------------------------- BODY[2] {93}

This is the HTML Section!

This is what displays in most modern email clients

BINARY[2] {93}

This is the HTML Section!

This is what displays in most modern email clients

BINARY.SIZE[2] 93 ---------------------------------- BODY[2.HEADER] {114} Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: quoted-printable Content-Disposition: inline ---------------------------------- BODY[2.TEXT] {93}

This is the HTML Section!

This is what displays in most modern email clients

---------------------------------- BODY[2.MIME] {115} Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: quoted-printable Content-Disposition: inline ---------------------------------- BODY[2.1] {93}

This is the HTML Section!

This is what displays in most modern email clients

BINARY[2.1] {93}

This is the HTML Section!

This is what displays in most modern email clients

BINARY.SIZE[2.1] 93 ---------------------------------- BODY[HEADER.FIELDS (FROM TO)] {53} From: sender@example.com To: recipient@example.com ---------------------------------- BODY[HEADER.FIELDS (FROM TO)]<10> {25} er@example.com To: recipi ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)] {117} From: sender@example.com To: recipient@example.com Content-Type: multipart/alternative; boundary="boundary-string" ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)]<10> {25} er@example.com To: recipi ---------------------------------- ================================================ FILE: tests/resources/imap/006.txt ================================================ From: sender@example.com To: recipient@example.com Subject: Multipart Email Example Content-Type: multipart/alternative; boundary="boundary-string" --boundary-string Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Content-Disposition: inline Plain text email goes here! This is the fallback if email client does not support HTML --boundary-string Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: quoted-printable Content-Disposition: inline

This is the HTML Section!

This is what displays in most modern email clients

--boundary-string-- ================================================ FILE: tests/resources/imap/007.imap ================================================ BODY ( ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 1 0 )( ( ( ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 1 0 )( "image" "jpeg" NIL NIL NIL NIL 1 )( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 1 0 ) "mixed" )( ( "text" "html" ( "charset" "us-ascii" ) NIL NIL "7bit" 14 0 )( "image" "jpeg" NIL NIL NIL NIL 1 ) "related" ) "alternative" )( "image" "jpeg" NIL NIL NIL NIL 1 )( "application" "x-excel" NIL NIL NIL NIL 1 )( "message" "rfc822" NIL NIL NIL NIL 13 ( NIL "J" ( ( NIL NIL "unknown" "localhost" ) ) ( ( NIL NIL "unknown" "localhost" ) ) ( ( NIL NIL "unknown" "localhost" ) ) NIL NIL NIL NIL NIL ) ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 1 0 ) 0 ) "mixed" )( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 1 0 ) "mixed" ) BODYSTRUCTURE ( ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 1 0 "7fc56270e7a70fa81a5935b72eacbe29" ( "inline" NIL ) NIL NIL )( ( ( ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 1 0 "9d5ed678fe57bcca610140957afab571" ( "inline" NIL ) NIL NIL )( "image" "jpeg" NIL NIL NIL NIL 1 "0d61f8370cad1d412f80b84d143e1257" ( "inline" NIL ) NIL NIL )( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 1 0 "f623e75af30e62bbd73d6df5b50bb7b5" ( "inline" NIL ) NIL NIL ) "mixed" ( "boundary" "4" ) NIL NIL NIL )( ( "text" "html" ( "charset" "us-ascii" ) NIL NIL "7bit" 14 0 "ece7293a99c7e12f8d044b683e5c2f33" NIL NIL NIL )( "image" "jpeg" NIL NIL NIL NIL 1 "800618943025315f869e4e1f09471012" NIL NIL NIL ) "related" ( "boundary" "5" ) NIL NIL NIL ) "alternative" ( "boundary" "3" ) NIL NIL NIL )( "image" "jpeg" NIL NIL NIL NIL 1 "dfcf28d0734569a6a693bc8194de62bf" ( "attachment" NIL ) NIL NIL )( "application" "x-excel" NIL NIL NIL NIL 1 "c1d9f50f86825a1a2302ec2449c17196" NIL NIL NIL )( "message" "rfc822" NIL NIL NIL NIL 13 ( NIL "J" ( ( NIL NIL "unknown" "localhost" ) ) ( ( NIL NIL "unknown" "localhost" ) ) ( ( NIL NIL "unknown" "localhost" ) ) NIL NIL NIL NIL NIL ) ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 1 0 "ff44570aca8241914870afbc310cdb85" NIL NIL NIL ) 0 "9c9888d1b2c167dd33f7542df5a65aa7" NIL NIL NIL ) "mixed" ( "boundary" "2" ) NIL NIL NIL )( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 1 0 "a5f3c6a11b03839d46af9fb43c97c188" ( "inline" NIL ) NIL NIL ) "mixed" ( "boundary" "1" ) NIL NIL NIL ) BODY[] {871} Subject: RFC 8621 Section 4.1.4 test Content-Type: multipart/mixed; boundary="1" --1 Content-Type: text/plain Content-Disposition: inline A --1 Content-Type: multipart/mixed; boundary="2" --2 Content-Type: multipart/alternative; boundary="3" --3 Content-Type: multipart/mixed; boundary="4" --4 Content-Type: text/plain Content-Disposition: inline B --4 Content-Type: image/jpeg Content-Disposition: inline C --4 Content-Type: text/plain Content-Disposition: inline D --4-- --3 Content-Type: multipart/related; boundary="5" --5 Content-Type: text/html E --5 Content-Type: image/jpeg F --5-- --3-- --2 Content-Type: image/jpeg Content-Disposition: attachment G --2 Content-Type: application/x-excel H --2 Content-Type: message/rfc822 Subject: J J --2-- --1 Content-Type: text/plain Content-Disposition: inline K --1-- BINARY[] {16} [binary content] BINARY.SIZE[] 871 ---------------------------------- BODY[HEADER] {82} Subject: RFC 8621 Section 4.1.4 test Content-Type: multipart/mixed; boundary="1" ---------------------------------- BODY[TEXT] {789} --1 Content-Type: text/plain Content-Disposition: inline A --1 Content-Type: multipart/mixed; boundary="2" --2 Content-Type: multipart/alternative; boundary="3" --3 Content-Type: multipart/mixed; boundary="4" --4 Content-Type: text/plain Content-Disposition: inline B --4 Content-Type: image/jpeg Content-Disposition: inline C --4 Content-Type: text/plain Content-Disposition: inline D --4-- --3 Content-Type: multipart/related; boundary="5" --5 Content-Type: text/html E --5 Content-Type: image/jpeg F --5-- --3-- --2 Content-Type: image/jpeg Content-Disposition: attachment G --2 Content-Type: application/x-excel H --2 Content-Type: message/rfc822 Subject: J J --2-- --1 Content-Type: text/plain Content-Disposition: inline K --1-- ---------------------------------- BODY[MIME] {46} Content-Type: multipart/mixed; boundary="1" ---------------------------------- BODY[1] {1} A BINARY[1] {1} A BINARY.SIZE[1] 1 ---------------------------------- BODY[1.HEADER] {54} Content-Type: text/plain Content-Disposition: inline ---------------------------------- BODY[1.TEXT] {1} A ---------------------------------- BODY[1.MIME] {55} Content-Type: text/plain Content-Disposition: inline ---------------------------------- BODY[1.1] {1} A BINARY[1.1] {1} A BINARY.SIZE[1.1] 1 ---------------------------------- BODY[2] {608} --2 Content-Type: multipart/alternative; boundary="3" --3 Content-Type: multipart/mixed; boundary="4" --4 Content-Type: text/plain Content-Disposition: inline B --4 Content-Type: image/jpeg Content-Disposition: inline C --4 Content-Type: text/plain Content-Disposition: inline D --4-- --3 Content-Type: multipart/related; boundary="5" --5 Content-Type: text/html E --5 Content-Type: image/jpeg F --5-- --3-- --2 Content-Type: image/jpeg Content-Disposition: attachment G --2 Content-Type: application/x-excel H --2 Content-Type: message/rfc822 Subject: J J --2-- BINARY[2] {16} [binary content] BINARY.SIZE[2] 653 ---------------------------------- BODY[2.HEADER] {45} Content-Type: multipart/mixed; boundary="2" ---------------------------------- BODY[2.TEXT] {608} --2 Content-Type: multipart/alternative; boundary="3" --3 Content-Type: multipart/mixed; boundary="4" --4 Content-Type: text/plain Content-Disposition: inline B --4 Content-Type: image/jpeg Content-Disposition: inline C --4 Content-Type: text/plain Content-Disposition: inline D --4-- --3 Content-Type: multipart/related; boundary="5" --5 Content-Type: text/html E --5 Content-Type: image/jpeg F --5-- --3-- --2 Content-Type: image/jpeg Content-Disposition: attachment G --2 Content-Type: application/x-excel H --2 Content-Type: message/rfc822 Subject: J J --2-- ---------------------------------- BODY[2.MIME] {46} Content-Type: multipart/mixed; boundary="2" ---------------------------------- BODY[2.1] {386} --3 Content-Type: multipart/mixed; boundary="4" --4 Content-Type: text/plain Content-Disposition: inline B --4 Content-Type: image/jpeg Content-Disposition: inline C --4 Content-Type: text/plain Content-Disposition: inline D --4-- --3 Content-Type: multipart/related; boundary="5" --5 Content-Type: text/html E --5 Content-Type: image/jpeg F --5-- --3-- BINARY[2.1] {16} [binary content] BINARY.SIZE[2.1] 437 ---------------------------------- BODY[2.1.HEADER] {51} Content-Type: multipart/alternative; boundary="3" ---------------------------------- BODY[2.1.TEXT] {386} --3 Content-Type: multipart/mixed; boundary="4" --4 Content-Type: text/plain Content-Disposition: inline B --4 Content-Type: image/jpeg Content-Disposition: inline C --4 Content-Type: text/plain Content-Disposition: inline D --4-- --3 Content-Type: multipart/related; boundary="5" --5 Content-Type: text/html E --5 Content-Type: image/jpeg F --5-- --3-- ---------------------------------- BODY[2.1.MIME] {52} Content-Type: multipart/alternative; boundary="3" ---------------------------------- BODY[2.1.1] {190} --4 Content-Type: text/plain Content-Disposition: inline B --4 Content-Type: image/jpeg Content-Disposition: inline C --4 Content-Type: text/plain Content-Disposition: inline D --4-- BINARY[2.1.1] {16} [binary content] BINARY.SIZE[2.1.1] 235 ---------------------------------- BODY[2.1.1.HEADER] {45} Content-Type: multipart/mixed; boundary="4" ---------------------------------- BODY[2.1.1.TEXT] {190} --4 Content-Type: text/plain Content-Disposition: inline B --4 Content-Type: image/jpeg Content-Disposition: inline C --4 Content-Type: text/plain Content-Disposition: inline D --4-- ---------------------------------- BODY[2.1.1.MIME] {46} Content-Type: multipart/mixed; boundary="4" ---------------------------------- BODY[2.1.1.1] {1} B BINARY[2.1.1.1] {1} B BINARY.SIZE[2.1.1.1] 1 ---------------------------------- BODY[2.1.1.1.HEADER] {54} Content-Type: text/plain Content-Disposition: inline ---------------------------------- BODY[2.1.1.1.TEXT] {1} B ---------------------------------- BODY[2.1.1.1.MIME] {55} Content-Type: text/plain Content-Disposition: inline ---------------------------------- BODY[2.1.1.1.1] {1} B BINARY[2.1.1.1.1] {1} B BINARY.SIZE[2.1.1.1.1] 1 ---------------------------------- BODY[2.1.1.2] {1} C BINARY[2.1.1.2] {16} [binary content] BINARY.SIZE[2.1.1.2] 1 ---------------------------------- BODY[2.1.1.2.HEADER] {54} Content-Type: image/jpeg Content-Disposition: inline ---------------------------------- BODY[2.1.1.2.TEXT] {1} C ---------------------------------- BODY[2.1.1.2.MIME] {55} Content-Type: image/jpeg Content-Disposition: inline ---------------------------------- BODY[2.1.1.2.1] {1} C BINARY[2.1.1.2.1] {16} [binary content] BINARY.SIZE[2.1.1.2.1] 1 ---------------------------------- BODY[2.1.1.3] {1} D BINARY[2.1.1.3] {1} D BINARY.SIZE[2.1.1.3] 1 ---------------------------------- BODY[2.1.1.3.HEADER] {54} Content-Type: text/plain Content-Disposition: inline ---------------------------------- BODY[2.1.1.3.TEXT] {1} D ---------------------------------- BODY[2.1.1.3.MIME] {55} Content-Type: text/plain Content-Disposition: inline ---------------------------------- BODY[2.1.1.3.1] {1} D BINARY[2.1.1.3.1] {1} D BINARY.SIZE[2.1.1.3.1] 1 ---------------------------------- BODY[2.1.2] {86} --5 Content-Type: text/html E --5 Content-Type: image/jpeg F --5-- BINARY[2.1.2] {16} [binary content] BINARY.SIZE[2.1.2] 133 ---------------------------------- BODY[2.1.2.HEADER] {47} Content-Type: multipart/related; boundary="5" ---------------------------------- BODY[2.1.2.TEXT] {86} --5 Content-Type: text/html E --5 Content-Type: image/jpeg F --5-- ---------------------------------- BODY[2.1.2.MIME] {48} Content-Type: multipart/related; boundary="5" ---------------------------------- BODY[2.1.2.1] {14} E BINARY[2.1.2.1] {14} E BINARY.SIZE[2.1.2.1] 14 ---------------------------------- BODY[2.1.2.1.HEADER] {25} Content-Type: text/html ---------------------------------- BODY[2.1.2.1.TEXT] {14} E ---------------------------------- BODY[2.1.2.1.MIME] {26} Content-Type: text/html ---------------------------------- BODY[2.1.2.1.1] {14} E BINARY[2.1.2.1.1] {14} E BINARY.SIZE[2.1.2.1.1] 14 ---------------------------------- BODY[2.1.2.2] {1} F BINARY[2.1.2.2] {16} [binary content] BINARY.SIZE[2.1.2.2] 1 ---------------------------------- BODY[2.1.2.2.HEADER] {26} Content-Type: image/jpeg ---------------------------------- BODY[2.1.2.2.TEXT] {1} F ---------------------------------- BODY[2.1.2.2.MIME] {27} Content-Type: image/jpeg ---------------------------------- BODY[2.1.2.2.1] {1} F BINARY[2.1.2.2.1] {16} [binary content] BINARY.SIZE[2.1.2.2.1] 1 ---------------------------------- BODY[2.2] {1} G BINARY[2.2] {16} [binary content] BINARY.SIZE[2.2] 1 ---------------------------------- BODY[2.2.HEADER] {58} Content-Type: image/jpeg Content-Disposition: attachment ---------------------------------- BODY[2.2.TEXT] {1} G ---------------------------------- BODY[2.2.MIME] {59} Content-Type: image/jpeg Content-Disposition: attachment ---------------------------------- BODY[2.2.1] {1} G BINARY[2.2.1] {16} [binary content] BINARY.SIZE[2.2.1] 1 ---------------------------------- BODY[2.3] {1} H BINARY[2.3] {16} [binary content] BINARY.SIZE[2.3] 1 ---------------------------------- BODY[2.3.HEADER] {35} Content-Type: application/x-excel ---------------------------------- BODY[2.3.TEXT] {1} H ---------------------------------- BODY[2.3.MIME] {36} Content-Type: application/x-excel ---------------------------------- BODY[2.3.1] {1} H BINARY[2.3.1] {16} [binary content] BINARY.SIZE[2.3.1] 1 ---------------------------------- BODY[2.4] {13} Subject: J J BINARY[2.4] {16} [binary content] BINARY.SIZE[2.4] 13 ---------------------------------- BODY[2.4.HEADER] {12} Subject: J ---------------------------------- BODY[2.4.TEXT] {1} J ---------------------------------- BODY[2.4.MIME] {31} Content-Type: message/rfc822 ---------------------------------- BODY[2.4.1] {1} J BINARY[2.4.1] {1} J BINARY.SIZE[2.4.1] 1 ---------------------------------- BODY[3] {1} K BINARY[3] {1} K BINARY.SIZE[3] 1 ---------------------------------- BODY[3.HEADER] {54} Content-Type: text/plain Content-Disposition: inline ---------------------------------- BODY[3.TEXT] {1} K ---------------------------------- BODY[3.MIME] {55} Content-Type: text/plain Content-Disposition: inline ---------------------------------- BODY[3.1] {1} K BINARY[3.1] {1} K BINARY.SIZE[3.1] 1 ---------------------------------- BODY[HEADER.FIELDS (FROM TO)] {2} ---------------------------------- BODY[HEADER.FIELDS (FROM TO)]<10> {0} ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)] {46} Content-Type: multipart/mixed; boundary="1" ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)]<10> {25} pe: multipart/mixed; boun ---------------------------------- ================================================ FILE: tests/resources/imap/007.txt ================================================ Subject: RFC 8621 Section 4.1.4 test Content-Type: multipart/mixed; boundary="1" --1 Content-Type: text/plain Content-Disposition: inline A --1 Content-Type: multipart/mixed; boundary="2" --2 Content-Type: multipart/alternative; boundary="3" --3 Content-Type: multipart/mixed; boundary="4" --4 Content-Type: text/plain Content-Disposition: inline B --4 Content-Type: image/jpeg Content-Disposition: inline C --4 Content-Type: text/plain Content-Disposition: inline D --4-- --3 Content-Type: multipart/related; boundary="5" --5 Content-Type: text/html E --5 Content-Type: image/jpeg F --5-- --3-- --2 Content-Type: image/jpeg Content-Disposition: attachment G --2 Content-Type: application/x-excel H --2 Content-Type: message/rfc822 Subject: J J --2-- --1 Content-Type: text/plain Content-Disposition: inline K --1-- ================================================ FILE: tests/resources/imap/008.imap ================================================ BODY ( ( "text" "plain" ( "charset" "utf-8" ) NIL NIL "7bit" 54 0 )( "message" "rfc822" NIL NIL NIL "base64" 1179 ( "Tue, 14 Dec 2021 11:48:25 +0100" "HTML test" ( ( "Name" NIL "email" "example.com" ) ) ( ( "Name" NIL "email" "example.com" ) ) ( ( "Name" NIL "email" "example.com" ) ) ( ( "email@example.com" NIL "email" "example.com" ) ) NIL NIL NIL "" ) ( ( "text" "plain" ( "charset" "utf-8" "format" "flowed" ) NIL NIL "7bit" 30 0 )( "text" "html" ( "charset" "utf-8" ) NIL NIL "7bit" 173 8 ) "alternative" ) 0 ) "mixed" ) BODYSTRUCTURE ( ( "text" "plain" ( "charset" "utf-8" ) NIL NIL "7bit" 54 0 "e377afc895a2c4c0d17b378f355de59e" NIL NIL NIL )( "message" "rfc822" NIL NIL NIL "base64" 1179 ( "Tue, 14 Dec 2021 11:48:25 +0100" "HTML test" ( ( "Name" NIL "email" "example.com" ) ) ( ( "Name" NIL "email" "example.com" ) ) ( ( "Name" NIL "email" "example.com" ) ) ( ( "email@example.com" NIL "email" "example.com" ) ) NIL NIL NIL "" ) ( ( "text" "plain" ( "charset" "utf-8" "format" "flowed" ) NIL NIL "7bit" 30 0 "6891396510cbadf4e2cfe31aee5bd25f" NIL NIL NIL )( "text" "html" ( "charset" "utf-8" ) NIL NIL "7bit" 173 8 "1a04bd2ec90f44a42792eacdc14fe8ea" NIL NIL NIL ) "alternative" ( "boundary" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" ) NIL "en-US" NIL ) 0 "eb1ac6e049a544c2bda1b7787e03db50" ( "attachment" ( "filename" "attached_email.eml" ) ) NIL NIL ) "mixed" ( "boundary" "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ) NIL NIL NIL ) BODY[] {1649} Content-Type: multipart/mixed; boundary=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb --bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 7bit This is a message with a base64 encoded attached email --bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb Content-Disposition: attachment; filename="attached_email.eml" Content-Type: message/rfc822 Content-Transfer-Encoding: base64 VG86ICJlbWFpbEBleGFtcGxlLmNvbSIgPGVtYWlsQGV4YW1wbGUuY29tPg0KRnJvbTogTmFtZSA8 ZW1haWxAZXhhbXBsZS5jb20+DQpTdWJqZWN0OiBIVE1MIHRlc3QNCk1lc3NhZ2UtSUQ6IDxyYW5k b20tbWVzc2FnZS1pZEBleGFtcGxlLmNvbT4NCkRhdGU6IFR1ZSwgMTQgRGVjIDIwMjEgMTE6NDg6 MjUgKzAxMDANCk1JTUUtVmVyc2lvbjogMS4wDQpDb250ZW50LVR5cGU6IG11bHRpcGFydC9hbHRl cm5hdGl2ZTsNCiBib3VuZGFyeT0iYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh YWFhYSINCkNvbnRlbnQtTGFuZ3VhZ2U6IGVuLVVTDQoNClRoaXMgaXMgYSBtdWx0aS1wYXJ0IG1l c3NhZ2UgaW4gTUlNRSBmb3JtYXQuDQotLWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh YWFhYWFhYWENCkNvbnRlbnQtVHlwZTogdGV4dC9wbGFpbjsgY2hhcnNldD11dGYtODsgZm9ybWF0 PWZsb3dlZA0KQ29udGVudC1UcmFuc2Zlci1FbmNvZGluZzogN2JpdA0KDQpUaGlzIGlzIGFuICpI VE1MKiB0ZXN0IG1lc3NhZ2UNCi0tYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh YWFhYQ0KQ29udGVudC1UeXBlOiB0ZXh0L2h0bWw7IGNoYXJzZXQ9dXRmLTgNCkNvbnRlbnQtVHJh bnNmZXItRW5jb2Rpbmc6IDdiaXQNCg0KPGh0bWw+DQogIDxoZWFkPg0KICAgIDxtZXRhIGh0dHAt ZXF1aXY9ImNvbnRlbnQtdHlwZSIgY29udGVudD0idGV4dC9odG1sOyBjaGFyc2V0PVVURi04Ij4N CiAgPC9oZWFkPg0KICA8Ym9keT4NCiAgICBUaGlzIGlzIGFuIDxiPkhUTUw8L2I+IHRlc3QgbWVz c2FnZQ0KICA8L2JvZHk+DQo8L2h0bWw+DQoNCi0tYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh YWFhYWFhYWFhYWFhYS0tDQo= --bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-- BINARY[] {16} [binary content] BINARY.SIZE[] 1649 ---------------------------------- BODY[HEADER] {83} Content-Type: multipart/mixed; boundary=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb ---------------------------------- BODY[TEXT] {1566} --bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 7bit This is a message with a base64 encoded attached email --bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb Content-Disposition: attachment; filename="attached_email.eml" Content-Type: message/rfc822 Content-Transfer-Encoding: base64 VG86ICJlbWFpbEBleGFtcGxlLmNvbSIgPGVtYWlsQGV4YW1wbGUuY29tPg0KRnJvbTogTmFtZSA8 ZW1haWxAZXhhbXBsZS5jb20+DQpTdWJqZWN0OiBIVE1MIHRlc3QNCk1lc3NhZ2UtSUQ6IDxyYW5k b20tbWVzc2FnZS1pZEBleGFtcGxlLmNvbT4NCkRhdGU6IFR1ZSwgMTQgRGVjIDIwMjEgMTE6NDg6 MjUgKzAxMDANCk1JTUUtVmVyc2lvbjogMS4wDQpDb250ZW50LVR5cGU6IG11bHRpcGFydC9hbHRl cm5hdGl2ZTsNCiBib3VuZGFyeT0iYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh YWFhYSINCkNvbnRlbnQtTGFuZ3VhZ2U6IGVuLVVTDQoNClRoaXMgaXMgYSBtdWx0aS1wYXJ0IG1l c3NhZ2UgaW4gTUlNRSBmb3JtYXQuDQotLWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh YWFhYWFhYWENCkNvbnRlbnQtVHlwZTogdGV4dC9wbGFpbjsgY2hhcnNldD11dGYtODsgZm9ybWF0 PWZsb3dlZA0KQ29udGVudC1UcmFuc2Zlci1FbmNvZGluZzogN2JpdA0KDQpUaGlzIGlzIGFuICpI VE1MKiB0ZXN0IG1lc3NhZ2UNCi0tYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh YWFhYQ0KQ29udGVudC1UeXBlOiB0ZXh0L2h0bWw7IGNoYXJzZXQ9dXRmLTgNCkNvbnRlbnQtVHJh bnNmZXItRW5jb2Rpbmc6IDdiaXQNCg0KPGh0bWw+DQogIDxoZWFkPg0KICAgIDxtZXRhIGh0dHAt ZXF1aXY9ImNvbnRlbnQtdHlwZSIgY29udGVudD0idGV4dC9odG1sOyBjaGFyc2V0PVVURi04Ij4N CiAgPC9oZWFkPg0KICA8Ym9keT4NCiAgICBUaGlzIGlzIGFuIDxiPkhUTUw8L2I+IHRlc3QgbWVz c2FnZQ0KICA8L2JvZHk+DQo8L2h0bWw+DQoNCi0tYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh YWFhYWFhYWFhYWFhYS0tDQo= --bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-- ---------------------------------- BODY[MIME] {84} Content-Type: multipart/mixed; boundary=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb ---------------------------------- BODY[1] {54} This is a message with a base64 encoded attached email BINARY[1] {54} This is a message with a base64 encoded attached email BINARY.SIZE[1] 54 ---------------------------------- BODY[1.HEADER] {73} Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 7bit ---------------------------------- BODY[1.TEXT] {54} This is a message with a base64 encoded attached email ---------------------------------- BODY[1.MIME] {74} Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 7bit ---------------------------------- BODY[1.1] {54} This is a message with a base64 encoded attached email BINARY[1.1] {54} This is a message with a base64 encoded attached email BINARY.SIZE[1.1] 54 ---------------------------------- BODY[2] {1179} VG86ICJlbWFpbEBleGFtcGxlLmNvbSIgPGVtYWlsQGV4YW1wbGUuY29tPg0KRnJvbTogTmFtZSA8 ZW1haWxAZXhhbXBsZS5jb20+DQpTdWJqZWN0OiBIVE1MIHRlc3QNCk1lc3NhZ2UtSUQ6IDxyYW5k b20tbWVzc2FnZS1pZEBleGFtcGxlLmNvbT4NCkRhdGU6IFR1ZSwgMTQgRGVjIDIwMjEgMTE6NDg6 MjUgKzAxMDANCk1JTUUtVmVyc2lvbjogMS4wDQpDb250ZW50LVR5cGU6IG11bHRpcGFydC9hbHRl cm5hdGl2ZTsNCiBib3VuZGFyeT0iYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh YWFhYSINCkNvbnRlbnQtTGFuZ3VhZ2U6IGVuLVVTDQoNClRoaXMgaXMgYSBtdWx0aS1wYXJ0IG1l c3NhZ2UgaW4gTUlNRSBmb3JtYXQuDQotLWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh YWFhYWFhYWENCkNvbnRlbnQtVHlwZTogdGV4dC9wbGFpbjsgY2hhcnNldD11dGYtODsgZm9ybWF0 PWZsb3dlZA0KQ29udGVudC1UcmFuc2Zlci1FbmNvZGluZzogN2JpdA0KDQpUaGlzIGlzIGFuICpI VE1MKiB0ZXN0IG1lc3NhZ2UNCi0tYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh YWFhYQ0KQ29udGVudC1UeXBlOiB0ZXh0L2h0bWw7IGNoYXJzZXQ9dXRmLTgNCkNvbnRlbnQtVHJh bnNmZXItRW5jb2Rpbmc6IDdiaXQNCg0KPGh0bWw+DQogIDxoZWFkPg0KICAgIDxtZXRhIGh0dHAt ZXF1aXY9ImNvbnRlbnQtdHlwZSIgY29udGVudD0idGV4dC9odG1sOyBjaGFyc2V0PVVURi04Ij4N CiAgPC9oZWFkPg0KICA8Ym9keT4NCiAgICBUaGlzIGlzIGFuIDxiPkhUTUw8L2I+IHRlc3QgbWVz c2FnZQ0KICA8L2JvZHk+DQo8L2h0bWw+DQoNCi0tYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh YWFhYWFhYWFhYWFhYS0tDQo= BINARY[2] {16} [binary content] BINARY.SIZE[2] 872 ---------------------------------- BODY[2.HEADER] {319} To: "email@example.com" From: Name Subject: HTML test Message-ID: Date: Tue, 14 Dec 2021 11:48:25 +0100 MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" Content-Language: en-US ---------------------------------- BODY[2.TEXT] {553} This is a multi-part message in MIME format. --aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa Content-Type: text/plain; charset=utf-8; format=flowed Content-Transfer-Encoding: 7bit This is an *HTML* test message --aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa Content-Type: text/html; charset=utf-8 Content-Transfer-Encoding: 7bit This is an HTML test message --aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-- ---------------------------------- BODY[2.MIME] {128} Content-Disposition: attachment; filename="attached_email.eml" Content-Type: message/rfc822 Content-Transfer-Encoding: base64 ---------------------------------- BODY[2.1] {30} This is an *HTML* test message BINARY[2.1] {30} This is an *HTML* test message BINARY.SIZE[2.1] 30 ---------------------------------- BODY[2.1.HEADER] {91} Content-Type: text/plain; charset=utf-8; format=flowed Content-Transfer-Encoding: 7bit ---------------------------------- BODY[2.1.TEXT] {30} This is an *HTML* test message ---------------------------------- BODY[2.1.MIME] {91} Content-Type: text/plain; charset=utf-8; format=flowed Content-Transfer-Encoding: 7bit ---------------------------------- BODY[2.1.1] {30} This is an *HTML* test message BINARY[2.1.1] {30} This is an *HTML* test message BINARY.SIZE[2.1.1] 30 ---------------------------------- BODY[2.2] {173} This is an HTML test message BINARY[2.2] {173} This is an HTML test message BINARY.SIZE[2.2] 173 ---------------------------------- BODY[2.2.HEADER] {75} Content-Type: text/html; charset=utf-8 Content-Transfer-Encoding: 7bit ---------------------------------- BODY[2.2.TEXT] {173} This is an HTML test message ---------------------------------- BODY[2.2.MIME] {75} Content-Type: text/html; charset=utf-8 Content-Transfer-Encoding: 7bit ---------------------------------- BODY[2.2.1] {173} This is an HTML test message BINARY[2.2.1] {173} This is an HTML test message BINARY.SIZE[2.2.1] 173 ---------------------------------- BODY[HEADER.FIELDS (FROM TO)] {2} ---------------------------------- BODY[HEADER.FIELDS (FROM TO)]<10> {0} ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)] {84} Content-Type: multipart/mixed; boundary=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)]<10> {25} pe: multipart/mixed; bou ---------------------------------- ================================================ FILE: tests/resources/imap/008.txt ================================================ Content-Type: multipart/mixed; boundary=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb --bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 7bit This is a message with a base64 encoded attached email --bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb Content-Disposition: attachment; filename="attached_email.eml" Content-Type: message/rfc822 Content-Transfer-Encoding: base64 VG86ICJlbWFpbEBleGFtcGxlLmNvbSIgPGVtYWlsQGV4YW1wbGUuY29tPg0KRnJvbTogTmFtZSA8 ZW1haWxAZXhhbXBsZS5jb20+DQpTdWJqZWN0OiBIVE1MIHRlc3QNCk1lc3NhZ2UtSUQ6IDxyYW5k b20tbWVzc2FnZS1pZEBleGFtcGxlLmNvbT4NCkRhdGU6IFR1ZSwgMTQgRGVjIDIwMjEgMTE6NDg6 MjUgKzAxMDANCk1JTUUtVmVyc2lvbjogMS4wDQpDb250ZW50LVR5cGU6IG11bHRpcGFydC9hbHRl cm5hdGl2ZTsNCiBib3VuZGFyeT0iYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh YWFhYSINCkNvbnRlbnQtTGFuZ3VhZ2U6IGVuLVVTDQoNClRoaXMgaXMgYSBtdWx0aS1wYXJ0IG1l c3NhZ2UgaW4gTUlNRSBmb3JtYXQuDQotLWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh YWFhYWFhYWENCkNvbnRlbnQtVHlwZTogdGV4dC9wbGFpbjsgY2hhcnNldD11dGYtODsgZm9ybWF0 PWZsb3dlZA0KQ29udGVudC1UcmFuc2Zlci1FbmNvZGluZzogN2JpdA0KDQpUaGlzIGlzIGFuICpI VE1MKiB0ZXN0IG1lc3NhZ2UNCi0tYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh YWFhYQ0KQ29udGVudC1UeXBlOiB0ZXh0L2h0bWw7IGNoYXJzZXQ9dXRmLTgNCkNvbnRlbnQtVHJh bnNmZXItRW5jb2Rpbmc6IDdiaXQNCg0KPGh0bWw+DQogIDxoZWFkPg0KICAgIDxtZXRhIGh0dHAt ZXF1aXY9ImNvbnRlbnQtdHlwZSIgY29udGVudD0idGV4dC9odG1sOyBjaGFyc2V0PVVURi04Ij4N CiAgPC9oZWFkPg0KICA8Ym9keT4NCiAgICBUaGlzIGlzIGFuIDxiPkhUTUw8L2I+IHRlc3QgbWVz c2FnZQ0KICA8L2JvZHk+DQo8L2h0bWw+DQoNCi0tYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh YWFhYWFhYWFhYWFhYS0tDQo= --bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-- ================================================ FILE: tests/resources/imap/009.imap ================================================ BODY ( ( "text" "html" ( "charset" "us-ascii" ) NIL NIL "base64" 239 3 )( "message" "rfc822" NIL NIL NIL NIL 723 ( NIL "Exporting my book about coffee tables" ( ( "Cosmo Kramer" NIL "kramer" "kramerica.com" ) ) ( ( "Cosmo Kramer" NIL "kramer" "kramerica.com" ) ) ( ( "Cosmo Kramer" NIL "kramer" "kramerica.com" ) ) NIL NIL NIL NIL NIL ) ( ( "text" "plain" ( "charset" "utf-16" ) NIL NIL "quoted-printable" 228 3 )( "image" "gif" ( "name" "Book about ☕ tables.gif" ) NIL NIL "Base64" 56 ) "mixed" ) 0 ) "mixed" ) BODYSTRUCTURE ( ( "text" "html" ( "charset" "us-ascii" ) NIL NIL "base64" 239 3 "07aab44e51c5f1833a5d19f2e1804c4b" NIL NIL NIL )( "message" "rfc822" NIL NIL NIL NIL 723 ( NIL "Exporting my book about coffee tables" ( ( "Cosmo Kramer" NIL "kramer" "kramerica.com" ) ) ( ( "Cosmo Kramer" NIL "kramer" "kramerica.com" ) ) ( ( "Cosmo Kramer" NIL "kramer" "kramerica.com" ) ) NIL NIL NIL NIL NIL ) ( ( "text" "plain" ( "charset" "utf-16" ) NIL NIL "quoted-printable" 228 3 "3a942a99cdd8a099ae107d3867ec20fb" NIL NIL NIL )( "image" "gif" ( "name" "Book about ☕ tables.gif" ) NIL NIL "Base64" 56 "d40fa7f401e9dc2df56cbb740d65ff52" ( "attachment" NIL ) NIL NIL ) "mixed" ( "boundary" "giddyup" ) NIL NIL NIL ) 0 "cdb0382a03a15601fb1b3c7422521620" NIL NIL NIL ) "mixed" ( "boundary" "festivus" ) NIL NIL NIL ) BODY[] {1457} From: Art Vandelay (Vandelay Industries) To: "Colleagues": "James Smythe" ; Friends: jane@example.com, =?UTF-8?Q?John_Sm=C3=AEth?= ; Date: Sat, 20 Nov 2021 14:22:01 -0800 Subject: Why not both importing AND exporting? =?utf-8?b?4pi6?= Content-Type: multipart/mixed; boundary="festivus"; --festivus Content-Type: text/html; charset="us-ascii" Content-Transfer-Encoding: base64 PGh0bWw+PHA+SSB3YXMgdGhpbmtpbmcgYWJvdXQgcXVpdHRpbmcgdGhlICZsZHF1bztle HBvcnRpbmcmcmRxdW87IHRvIGZvY3VzIGp1c3Qgb24gdGhlICZsZHF1bztpbXBvcnRpbm cmcmRxdW87LDwvcD48cD5idXQgdGhlbiBJIHRob3VnaHQsIHdoeSBub3QgZG8gYm90aD8 gJiN4MjYzQTs8L3A+PC9odG1sPg== --festivus Content-Type: message/rfc822 From: "Cosmo Kramer" Subject: Exporting my book about coffee tables Content-Type: multipart/mixed; boundary="giddyup"; --giddyup Content-Type: text/plain; charset="utf-16" Content-Transfer-Encoding: quoted-printable =FF=FE=0C!5=D8"=DD5=D8)=DD5=D8-=DD =005=D8*=DD5=D8"=DD =005=D8"= =DD5=D85=DD5=D8-=DD5=D8,=DD5=D8/=DD5=D81=DD =005=D8*=DD5=D86=DD = =005=D8=1F=DD5=D8,=DD5=D8,=DD5=D8(=DD =005=D8-=DD5=D8)=DD5=D8"= =DD5=D8=1E=DD5=D80=DD5=D8"=DD!=00 --giddyup Content-Type: image/gif; name*1="about "; name*0="Book "; name*2*=utf-8''%e2%98%95 tables.gif Content-Transfer-Encoding: Base64 Content-Disposition: attachment R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7 --giddyup-- --festivus-- BINARY[] {16} [binary content] BINARY.SIZE[] 1457 ---------------------------------- BODY[HEADER] {349} From: Art Vandelay (Vandelay Industries) To: "Colleagues": "James Smythe" ; Friends: jane@example.com, =?UTF-8?Q?John_Sm=C3=AEth?= ; Date: Sat, 20 Nov 2021 14:22:01 -0800 Subject: Why not both importing AND exporting? =?utf-8?b?4pi6?= Content-Type: multipart/mixed; boundary="festivus"; ---------------------------------- BODY[TEXT] {1108} --festivus Content-Type: text/html; charset="us-ascii" Content-Transfer-Encoding: base64 PGh0bWw+PHA+SSB3YXMgdGhpbmtpbmcgYWJvdXQgcXVpdHRpbmcgdGhlICZsZHF1bztle HBvcnRpbmcmcmRxdW87IHRvIGZvY3VzIGp1c3Qgb24gdGhlICZsZHF1bztpbXBvcnRpbm cmcmRxdW87LDwvcD48cD5idXQgdGhlbiBJIHRob3VnaHQsIHdoeSBub3QgZG8gYm90aD8 gJiN4MjYzQTs8L3A+PC9odG1sPg== --festivus Content-Type: message/rfc822 From: "Cosmo Kramer" Subject: Exporting my book about coffee tables Content-Type: multipart/mixed; boundary="giddyup"; --giddyup Content-Type: text/plain; charset="utf-16" Content-Transfer-Encoding: quoted-printable =FF=FE=0C!5=D8"=DD5=D8)=DD5=D8-=DD =005=D8*=DD5=D8"=DD =005=D8"= =DD5=D85=DD5=D8-=DD5=D8,=DD5=D8/=DD5=D81=DD =005=D8*=DD5=D86=DD = =005=D8=1F=DD5=D8,=DD5=D8,=DD5=D8(=DD =005=D8-=DD5=D8)=DD5=D8"= =DD5=D8=1E=DD5=D80=DD5=D8"=DD!=00 --giddyup Content-Type: image/gif; name*1="about "; name*0="Book "; name*2*=utf-8''%e2%98%95 tables.gif Content-Transfer-Encoding: Base64 Content-Disposition: attachment R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7 --giddyup-- --festivus-- ---------------------------------- BODY[MIME] {54} Content-Type: multipart/mixed; boundary="festivus"; ---------------------------------- BODY[1] {239} PGh0bWw+PHA+SSB3YXMgdGhpbmtpbmcgYWJvdXQgcXVpdHRpbmcgdGhlICZsZHF1bztle HBvcnRpbmcmcmRxdW87IHRvIGZvY3VzIGp1c3Qgb24gdGhlICZsZHF1bztpbXBvcnRpbm cmcmRxdW87LDwvcD48cD5idXQgdGhlbiBJIHRob3VnaHQsIHdoeSBub3QgZG8gYm90aD8 gJiN4MjYzQTs8L3A+PC9odG1sPg== BINARY[1] {175}

I was thinking about quitting the “exporting” to focus just on the “importing”,

but then I thought, why not do both? ☺

BINARY.SIZE[1] 175 ---------------------------------- BODY[1.HEADER] {79} Content-Type: text/html; charset="us-ascii" Content-Transfer-Encoding: base64 ---------------------------------- BODY[1.TEXT] {239} PGh0bWw+PHA+SSB3YXMgdGhpbmtpbmcgYWJvdXQgcXVpdHRpbmcgdGhlICZsZHF1bztle HBvcnRpbmcmcmRxdW87IHRvIGZvY3VzIGp1c3Qgb24gdGhlICZsZHF1bztpbXBvcnRpbm cmcmRxdW87LDwvcD48cD5idXQgdGhlbiBJIHRob3VnaHQsIHdoeSBub3QgZG8gYm90aD8 gJiN4MjYzQTs8L3A+PC9odG1sPg== ---------------------------------- BODY[1.MIME] {80} Content-Type: text/html; charset="us-ascii" Content-Transfer-Encoding: base64 ---------------------------------- BODY[1.1] {239} PGh0bWw+PHA+SSB3YXMgdGhpbmtpbmcgYWJvdXQgcXVpdHRpbmcgdGhlICZsZHF1bztle HBvcnRpbmcmcmRxdW87IHRvIGZvY3VzIGp1c3Qgb24gdGhlICZsZHF1bztpbXBvcnRpbm cmcmRxdW87LDwvcD48cD5idXQgdGhlbiBJIHRob3VnaHQsIHdoeSBub3QgZG8gYm90aD8 gJiN4MjYzQTs8L3A+PC9odG1sPg== BINARY[1.1] {175}

I was thinking about quitting the “exporting” to focus just on the “importing”,

but then I thought, why not do both? ☺

BINARY.SIZE[1.1] 175 ---------------------------------- BODY[2] {723} From: "Cosmo Kramer" Subject: Exporting my book about coffee tables Content-Type: multipart/mixed; boundary="giddyup"; --giddyup Content-Type: text/plain; charset="utf-16" Content-Transfer-Encoding: quoted-printable =FF=FE=0C!5=D8"=DD5=D8)=DD5=D8-=DD =005=D8*=DD5=D8"=DD =005=D8"= =DD5=D85=DD5=D8-=DD5=D8,=DD5=D8/=DD5=D81=DD =005=D8*=DD5=D86=DD = =005=D8=1F=DD5=D8,=DD5=D8,=DD5=D8(=DD =005=D8-=DD5=D8)=DD5=D8"= =DD5=D8=1E=DD5=D80=DD5=D8"=DD!=00 --giddyup Content-Type: image/gif; name*1="about "; name*0="Book "; name*2*=utf-8''%e2%98%95 tables.gif Content-Transfer-Encoding: Base64 Content-Disposition: attachment R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7 --giddyup-- BINARY[2] {16} [binary content] BINARY.SIZE[2] 723 ---------------------------------- BODY[2.HEADER] {143} From: "Cosmo Kramer" Subject: Exporting my book about coffee tables Content-Type: multipart/mixed; boundary="giddyup"; ---------------------------------- BODY[2.TEXT] {580} --giddyup Content-Type: text/plain; charset="utf-16" Content-Transfer-Encoding: quoted-printable =FF=FE=0C!5=D8"=DD5=D8)=DD5=D8-=DD =005=D8*=DD5=D8"=DD =005=D8"= =DD5=D85=DD5=D8-=DD5=D8,=DD5=D8/=DD5=D81=DD =005=D8*=DD5=D86=DD = =005=D8=1F=DD5=D8,=DD5=D8,=DD5=D8(=DD =005=D8-=DD5=D8)=DD5=D8"= =DD5=D8=1E=DD5=D80=DD5=D8"=DD!=00 --giddyup Content-Type: image/gif; name*1="about "; name*0="Book "; name*2*=utf-8''%e2%98%95 tables.gif Content-Transfer-Encoding: Base64 Content-Disposition: attachment R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7 --giddyup-- ---------------------------------- BODY[2.MIME] {31} Content-Type: message/rfc822 ---------------------------------- BODY[2.1] {228} =FF=FE=0C!5=D8"=DD5=D8)=DD5=D8-=DD =005=D8*=DD5=D8"=DD =005=D8"= =DD5=D85=DD5=D8-=DD5=D8,=DD5=D8/=DD5=D81=DD =005=D8*=DD5=D86=DD = =005=D8=1F=DD5=D8,=DD5=D8,=DD5=D8(=DD =005=D8-=DD5=D8)=DD5=D8"= =DD5=D8=1E=DD5=D80=DD5=D8"=DD!=00 BINARY[2.1] {101} ℌ𝔢𝔩𝔭 𝔪𝔢 𝔢𝔵𝔭𝔬𝔯𝔱 𝔪𝔶 𝔟𝔬𝔬𝔨 𝔭𝔩𝔢𝔞𝔰𝔢! BINARY.SIZE[2.1] 101 ---------------------------------- BODY[2.1.HEADER] {88} Content-Type: text/plain; charset="utf-16" Content-Transfer-Encoding: quoted-printable ---------------------------------- BODY[2.1.TEXT] {228} =FF=FE=0C!5=D8"=DD5=D8)=DD5=D8-=DD =005=D8*=DD5=D8"=DD =005=D8"= =DD5=D85=DD5=D8-=DD5=D8,=DD5=D8/=DD5=D81=DD =005=D8*=DD5=D86=DD = =005=D8=1F=DD5=D8,=DD5=D8,=DD5=D8(=DD =005=D8-=DD5=D8)=DD5=D8"= =DD5=D8=1E=DD5=D80=DD5=D8"=DD!=00 ---------------------------------- BODY[2.1.MIME] {89} Content-Type: text/plain; charset="utf-16" Content-Transfer-Encoding: quoted-printable ---------------------------------- BODY[2.1.1] {228} =FF=FE=0C!5=D8"=DD5=D8)=DD5=D8-=DD =005=D8*=DD5=D8"=DD =005=D8"= =DD5=D85=DD5=D8-=DD5=D8,=DD5=D8/=DD5=D81=DD =005=D8*=DD5=D86=DD = =005=D8=1F=DD5=D8,=DD5=D8,=DD5=D8(=DD =005=D8-=DD5=D8)=DD5=D8"= =DD5=D8=1E=DD5=D80=DD5=D8"=DD!=00 BINARY[2.1.1] {101} ℌ𝔢𝔩𝔭 𝔪𝔢 𝔢𝔵𝔭𝔬𝔯𝔱 𝔪𝔶 𝔟𝔬𝔬𝔨 𝔭𝔩𝔢𝔞𝔰𝔢! BINARY.SIZE[2.1.1] 101 ---------------------------------- BODY[2.2] {56} R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7 BINARY[2.2] {16} [binary content] BINARY.SIZE[2.2] 42 ---------------------------------- BODY[2.2.HEADER] {175} Content-Type: image/gif; name*1="about "; name*0="Book "; name*2*=utf-8''%e2%98%95 tables.gif Content-Transfer-Encoding: Base64 Content-Disposition: attachment ---------------------------------- BODY[2.2.TEXT] {56} R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7 ---------------------------------- BODY[2.2.MIME] {176} Content-Type: image/gif; name*1="about "; name*0="Book "; name*2*=utf-8''%e2%98%95 tables.gif Content-Transfer-Encoding: Base64 Content-Disposition: attachment ---------------------------------- BODY[2.2.1] {56} R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7 BINARY[2.2.1] {16} [binary content] BINARY.SIZE[2.2.1] 42 ---------------------------------- BODY[HEADER.FIELDS (FROM TO)] {196} From: Art Vandelay (Vandelay Industries) To: "Colleagues": "James Smythe" ; Friends: jane@example.com, =?UTF-8?Q?John_Sm=C3=AEth?= ; ---------------------------------- BODY[HEADER.FIELDS (FROM TO)]<10> {25} Vandelay (Vandelay Industries) To: "Colleagues": "James Smythe" ; Friends: jane@example.com, =?UTF-8?Q?John_Sm=C3=AEth?= ; Date: Sat, 20 Nov 2021 14:22:01 -0800 Content-Type: multipart/mixed; boundary="festivus"; ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)]<10> {25} Vandelay (Vandelay Industries) To: "Colleagues": "James Smythe" ; Friends: jane@example.com, =?UTF-8?Q?John_Sm=C3=AEth?= ; Date: Sat, 20 Nov 2021 14:22:01 -0800 Subject: Why not both importing AND exporting? =?utf-8?b?4pi6?= Content-Type: multipart/mixed; boundary="festivus"; --festivus Content-Type: text/html; charset="us-ascii" Content-Transfer-Encoding: base64 PGh0bWw+PHA+SSB3YXMgdGhpbmtpbmcgYWJvdXQgcXVpdHRpbmcgdGhlICZsZHF1bztle HBvcnRpbmcmcmRxdW87IHRvIGZvY3VzIGp1c3Qgb24gdGhlICZsZHF1bztpbXBvcnRpbm cmcmRxdW87LDwvcD48cD5idXQgdGhlbiBJIHRob3VnaHQsIHdoeSBub3QgZG8gYm90aD8 gJiN4MjYzQTs8L3A+PC9odG1sPg== --festivus Content-Type: message/rfc822 From: "Cosmo Kramer" Subject: Exporting my book about coffee tables Content-Type: multipart/mixed; boundary="giddyup"; --giddyup Content-Type: text/plain; charset="utf-16" Content-Transfer-Encoding: quoted-printable =FF=FE=0C!5=D8"=DD5=D8)=DD5=D8-=DD =005=D8*=DD5=D8"=DD =005=D8"= =DD5=D85=DD5=D8-=DD5=D8,=DD5=D8/=DD5=D81=DD =005=D8*=DD5=D86=DD = =005=D8=1F=DD5=D8,=DD5=D8,=DD5=D8(=DD =005=D8-=DD5=D8)=DD5=D8"= =DD5=D8=1E=DD5=D80=DD5=D8"=DD!=00 --giddyup Content-Type: image/gif; name*1="about "; name*0="Book "; name*2*=utf-8''%e2%98%95 tables.gif Content-Transfer-Encoding: Base64 Content-Disposition: attachment R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7 --giddyup-- --festivus-- ================================================ FILE: tests/resources/imap/010.imap ================================================ BODY ( "message" "rfc822" NIL NIL NIL NIL 88 ( "Sun, 12 Aug 2012 12:34:56 +0300" "submsg" ( ( NIL NIL "sub" "domain.org" ) ) ( ( NIL NIL "sub" "domain.org" ) ) ( ( NIL NIL "sub" "domain.org" ) ) NIL NIL NIL NIL NIL ) ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 12 1 ) 0 ) BODYSTRUCTURE ( "message" "rfc822" NIL NIL NIL NIL 88 ( "Sun, 12 Aug 2012 12:34:56 +0300" "submsg" ( ( NIL NIL "sub" "domain.org" ) ) ( ( NIL NIL "sub" "domain.org" ) ) ( ( NIL NIL "sub" "domain.org" ) ) NIL NIL NIL NIL NIL ) ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 12 1 "f0ef7081e1539ac00ef5b761b4fb01b3" NIL NIL NIL ) 0 "dfea5a78f321b331e6d7983a1f9cc6b7" NIL NIL NIL ) BODY[] {196} From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Hello world BINARY[] {16} [binary content] BINARY.SIZE[] 88 ---------------------------------- BODY[HEADER] {108} From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: message/rfc822 ---------------------------------- BODY[TEXT] {88} From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Hello world ---------------------------------- BODY[MIME] {31} Content-Type: message/rfc822 ---------------------------------- BODY[1] {88} From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Hello world BINARY[1] {16} [binary content] BINARY.SIZE[1] 88 ---------------------------------- BODY[1.HEADER] {76} From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg ---------------------------------- BODY[1.TEXT] {12} Hello world ---------------------------------- BODY[1.MIME] {31} Content-Type: message/rfc822 ---------------------------------- BODY[1.1] {12} Hello world BINARY[1.1] {12} Hello world BINARY.SIZE[1.1] 12 ---------------------------------- BODY[HEADER.FIELDS (FROM TO)] {24} From: user@domain.org ---------------------------------- BODY[HEADER.FIELDS (FROM TO)]<10> {14} @domain.org ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)] {109} From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 MIME-Version: 1.0 Content-Type: message/rfc822 ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)]<10> {25} @domain.org Date: Sat, 24 ---------------------------------- ================================================ FILE: tests/resources/imap/010.txt ================================================ From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Hello world ================================================ FILE: tests/resources/imap/011.imap ================================================ BODY ( "message" "rfc822" NIL NIL NIL NIL 271 ( "Sun, 12 Aug 2012 12:34:56 +0300" "submsg" ( ( NIL NIL "sub" "domain.org" ) ) ( ( NIL NIL "sub" "domain.org" ) ) ( ( NIL NIL "sub" "domain.org" ) ) NIL NIL NIL NIL NIL ) ( ( "message" NIL NIL NIL NIL NIL 42 ( NIL "m1" ( ( NIL NIL "m1" "example.com" ) ) ( ( NIL NIL "m1" "example.com" ) ) ( ( NIL NIL "m1" "example.com" ) ) NIL NIL NIL NIL NIL ) ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 8 1 ) 0 )( "message" NIL NIL NIL NIL NIL 42 ( NIL "m2" ( ( NIL NIL "m2" "example.com" ) ) ( ( NIL NIL "m2" "example.com" ) ) ( ( NIL NIL "m2" "example.com" ) ) NIL NIL NIL NIL NIL ) ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 8 1 ) 0 ) "digest" ) 0 ) BODYSTRUCTURE ( "message" "rfc822" NIL NIL NIL NIL 271 ( "Sun, 12 Aug 2012 12:34:56 +0300" "submsg" ( ( NIL NIL "sub" "domain.org" ) ) ( ( NIL NIL "sub" "domain.org" ) ) ( ( NIL NIL "sub" "domain.org" ) ) NIL NIL NIL NIL NIL ) ( ( "message" NIL NIL NIL NIL NIL 42 ( NIL "m1" ( ( NIL NIL "m1" "example.com" ) ) ( ( NIL NIL "m1" "example.com" ) ) ( ( NIL NIL "m1" "example.com" ) ) NIL NIL NIL NIL NIL ) ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 8 1 "8dc313ad8cf1d82dbe8d46f5f0d3d79c" NIL NIL NIL ) 0 "702907ad1c165219425153a8f0a5f578" NIL NIL NIL )( "message" NIL NIL NIL NIL NIL 42 ( NIL "m2" ( ( NIL NIL "m2" "example.com" ) ) ( ( NIL NIL "m2" "example.com" ) ) ( ( NIL NIL "m2" "example.com" ) ) NIL NIL NIL NIL NIL ) ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 8 1 "f344a10ee7adfdcfc29650b6e31601d8" NIL NIL NIL ) 0 "0c79449f982ccecbc258d902cd989f69" NIL NIL NIL ) "digest" ( "boundary" "foo" ) NIL NIL NIL ) 0 "4935800d6cfad87d931093820097206a" NIL NIL NIL ) BODY[] {379} From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/digest; boundary="foo" prologue --foo From: m1@example.com Subject: m1 m1 body --foo X-Mime: m2 header From: m2@example.com Subject: m2 m2 body --foo-- epilogue BINARY[] {16} [binary content] BINARY.SIZE[] 260 ---------------------------------- BODY[HEADER] {108} From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: message/rfc822 ---------------------------------- BODY[TEXT] {271} From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/digest; boundary="foo" prologue --foo From: m1@example.com Subject: m1 m1 body --foo X-Mime: m2 header From: m2@example.com Subject: m2 m2 body --foo-- epilogue ---------------------------------- BODY[MIME] {31} Content-Type: message/rfc822 ---------------------------------- BODY[1] {271} From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/digest; boundary="foo" prologue --foo From: m1@example.com Subject: m1 m1 body --foo X-Mime: m2 header From: m2@example.com Subject: m2 m2 body --foo-- epilogue BINARY[1] {16} [binary content] BINARY.SIZE[1] 260 ---------------------------------- BODY[1.HEADER] {123} From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/digest; boundary="foo" ---------------------------------- BODY[1.TEXT] {137} prologue --foo From: m1@example.com Subject: m1 m1 body --foo X-Mime: m2 header From: m2@example.com Subject: m2 m2 body --foo-- ---------------------------------- BODY[1.MIME] {31} Content-Type: message/rfc822 ---------------------------------- BODY[1.1] {42} From: m1@example.com Subject: m1 m1 body BINARY[1.1] {16} [binary content] BINARY.SIZE[1.1] 42 ---------------------------------- BODY[1.1.HEADER] {34} From: m1@example.com Subject: m1 ---------------------------------- BODY[1.1.TEXT] {8} m1 body ---------------------------------- BODY[1.1.MIME] {2} ---------------------------------- BODY[1.1.1] {8} m1 body BINARY[1.1.1] {8} m1 body BINARY.SIZE[1.1.1] 8 ---------------------------------- BODY[1.2] {42} From: m2@example.com Subject: m2 m2 body BINARY[1.2] {16} [binary content] BINARY.SIZE[1.2] 42 ---------------------------------- BODY[1.2.HEADER] {34} From: m2@example.com Subject: m2 ---------------------------------- BODY[1.2.TEXT] {8} m2 body ---------------------------------- BODY[1.2.MIME] {2} ---------------------------------- BODY[1.2.1] {8} m2 body BINARY[1.2.1] {8} m2 body BINARY.SIZE[1.2.1] 8 ---------------------------------- BODY[HEADER.FIELDS (FROM TO)] {24} From: user@domain.org ---------------------------------- BODY[HEADER.FIELDS (FROM TO)]<10> {14} @domain.org ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)] {109} From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 MIME-Version: 1.0 Content-Type: message/rfc822 ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)]<10> {25} @domain.org Date: Sat, 24 ---------------------------------- ================================================ FILE: tests/resources/imap/011.txt ================================================ From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/digest; boundary="foo" prologue --foo From: m1@example.com Subject: m1 m1 body --foo X-Mime: m2 header From: m2@example.com Subject: m2 m2 body --foo-- epilogue ================================================ FILE: tests/resources/imap/012.imap ================================================ BODY ( ( "text" "x-myown" ( "charset" "us-ascii" ) NIL NIL "7bit" 6 1 )( "message" "rfc822" NIL NIL NIL NIL 280 ( "Sun, 12 Aug 2012 12:34:56 +0300" "submsg" ( ( NIL NIL "sub" "domain.org" ) ) ( ( NIL NIL "sub" "domain.org" ) ) ( ( NIL NIL "sub" "domain.org" ) ) NIL NIL NIL NIL NIL ) ( ( "text" "html" ( "charset" "us-ascii" ) NIL NIL "7bit" 19 1 )( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 20 1 ) "alternative" ) 0 ) "mixed" ) BODYSTRUCTURE ( ( "text" "x-myown" ( "charset" "us-ascii" ) NIL NIL "7bit" 6 1 "b1946ac92492d2347c6235b4d2611184" NIL NIL NIL )( "message" "rfc822" NIL NIL NIL NIL 280 ( "Sun, 12 Aug 2012 12:34:56 +0300" "submsg" ( ( NIL NIL "sub" "domain.org" ) ) ( ( NIL NIL "sub" "domain.org" ) ) ( ( NIL NIL "sub" "domain.org" ) ) NIL NIL NIL NIL NIL ) ( ( "text" "html" ( "charset" "us-ascii" ) NIL NIL "7bit" 19 1 "35c5b687e359e8ce7be1f1ecafd9b475" NIL NIL NIL )( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 20 1 "deeff770fadb17c664d431b97bcd05c5" NIL NIL NIL ) "alternative" ( "boundary" "sub1" ) NIL NIL NIL ) 0 "735ed696bc05fdf6840de404781d5d77" NIL NIL NIL ) "mixed" ( "boundary" "foo bar" ) NIL NIL NIL ) BODY[] {565} From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="foo bar" Root MIME prologue --foo bar Content-Type: text/x-myown; charset=us-ascii hello --foo bar Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/alternative; boundary="sub1" Sub MIME prologue --sub1 Content-Type: text/html

Hello world

--sub1 Content-Type: text/plain Hello another world --sub1-- Sub MIME epilogue --foo bar-- Root MIME epilogue BINARY[] {16} [binary content] BINARY.SIZE[] 565 ---------------------------------- BODY[HEADER] {130} From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="foo bar" ---------------------------------- BODY[TEXT] {435} Root MIME prologue --foo bar Content-Type: text/x-myown; charset=us-ascii hello --foo bar Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/alternative; boundary="sub1" Sub MIME prologue --sub1 Content-Type: text/html

Hello world

--sub1 Content-Type: text/plain Hello another world --sub1-- Sub MIME epilogue --foo bar-- Root MIME epilogue ---------------------------------- BODY[MIME] {53} Content-Type: multipart/mixed; boundary="foo bar" ---------------------------------- BODY[1] {6} hello BINARY[1] {6} hello BINARY.SIZE[1] 6 ---------------------------------- BODY[1.HEADER] {46} Content-Type: text/x-myown; charset=us-ascii ---------------------------------- BODY[1.TEXT] {6} hello ---------------------------------- BODY[1.MIME] {47} Content-Type: text/x-myown; charset=us-ascii ---------------------------------- BODY[1.1] {6} hello BINARY[1.1] {6} hello BINARY.SIZE[1.1] 6 ---------------------------------- BODY[2] {280} From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/alternative; boundary="sub1" Sub MIME prologue --sub1 Content-Type: text/html

Hello world

--sub1 Content-Type: text/plain Hello another world --sub1-- Sub MIME epilogue BINARY[2] {16} [binary content] BINARY.SIZE[2] 280 ---------------------------------- BODY[2.HEADER] {129} From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/alternative; boundary="sub1" ---------------------------------- BODY[2.TEXT] {151} Sub MIME prologue --sub1 Content-Type: text/html

Hello world

--sub1 Content-Type: text/plain Hello another world --sub1-- Sub MIME epilogue ---------------------------------- BODY[2.MIME] {31} Content-Type: message/rfc822 ---------------------------------- BODY[2.1] {19}

Hello world

BINARY[2.1] {19}

Hello world

BINARY.SIZE[2.1] 19 ---------------------------------- BODY[2.1.HEADER] {25} Content-Type: text/html ---------------------------------- BODY[2.1.TEXT] {19}

Hello world

---------------------------------- BODY[2.1.MIME] {26} Content-Type: text/html ---------------------------------- BODY[2.1.1] {19}

Hello world

BINARY[2.1.1] {19}

Hello world

BINARY.SIZE[2.1.1] 19 ---------------------------------- BODY[2.2] {20} Hello another world BINARY[2.2] {20} Hello another world BINARY.SIZE[2.2] 20 ---------------------------------- BODY[2.2.HEADER] {26} Content-Type: text/plain ---------------------------------- BODY[2.2.TEXT] {20} Hello another world ---------------------------------- BODY[2.2.MIME] {27} Content-Type: text/plain ---------------------------------- BODY[2.2.1] {20} Hello another world BINARY[2.2.1] {20} Hello another world BINARY.SIZE[2.2.1] 20 ---------------------------------- BODY[HEADER.FIELDS (FROM TO)] {24} From: user@domain.org ---------------------------------- BODY[HEADER.FIELDS (FROM TO)]<10> {14} @domain.org ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)] {131} From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="foo bar" ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)]<10> {25} @domain.org Date: Sat, 24 ---------------------------------- ================================================ FILE: tests/resources/imap/012.txt ================================================ From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="foo bar" Root MIME prologue --foo bar Content-Type: text/x-myown; charset=us-ascii hello --foo bar Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/alternative; boundary="sub1" Sub MIME prologue --sub1 Content-Type: text/html

Hello world

--sub1 Content-Type: text/plain Hello another world --sub1-- Sub MIME epilogue --foo bar-- Root MIME epilogue ================================================ FILE: tests/resources/imap/013.imap ================================================ BODY ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 356 13 ) BODYSTRUCTURE ( "text" "plain" ( "charset" "us-ascii" ) NIL NIL "7bit" 356 13 "77beb490b61fa7ed17f997c4124a57bf" NIL NIL NIL ) BODY[] {697} Date: Mon, 13 Aug 1998 17:42:41 +1000 Message-Id: <199804130742.RAA20366@mai1host.whitehouse.gov> From: Bill Clinton To: A1 (The Enforcer) Gore Subject: Map of Argentina with Description MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit Hi A1, I finally figured out this MIME thing. Pretty cool. I'll send you some sax music in .au files next week! Anyway, the attached image is really too small to get a good look at Argentina. Try this for a much better map: http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm Then again, shouldn't the CIA have something like that? Bill BINARY[] {356} Hi A1, I finally figured out this MIME thing. Pretty cool. I'll send you some sax music in .au files next week! Anyway, the attached image is really too small to get a good look at Argentina. Try this for a much better map: http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm Then again, shouldn't the CIA have something like that? Bill BINARY.SIZE[] 356 ---------------------------------- BODY[HEADER] {341} Date: Mon, 13 Aug 1998 17:42:41 +1000 Message-Id: <199804130742.RAA20366@mai1host.whitehouse.gov> From: Bill Clinton To: A1 (The Enforcer) Gore Subject: Map of Argentina with Description MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit ---------------------------------- BODY[TEXT] {356} Hi A1, I finally figured out this MIME thing. Pretty cool. I'll send you some sax music in .au files next week! Anyway, the attached image is really too small to get a good look at Argentina. Try this for a much better map: http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm Then again, shouldn't the CIA have something like that? Bill ---------------------------------- BODY[MIME] {77} Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit ---------------------------------- BODY[1] {356} Hi A1, I finally figured out this MIME thing. Pretty cool. I'll send you some sax music in .au files next week! Anyway, the attached image is really too small to get a good look at Argentina. Try this for a much better map: http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm Then again, shouldn't the CIA have something like that? Bill BINARY[1] {356} Hi A1, I finally figured out this MIME thing. Pretty cool. I'll send you some sax music in .au files next week! Anyway, the attached image is really too small to get a good look at Argentina. Try this for a much better map: http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm Then again, shouldn't the CIA have something like that? Bill BINARY.SIZE[1] 356 ---------------------------------- BODY[HEADER.FIELDS (FROM TO)] {107} From: Bill Clinton To: A1 (The Enforcer) Gore ---------------------------------- BODY[HEADER.FIELDS (FROM TO)]<10> {25} Clinton From: Bill Clinton To: A1 (The Enforcer) Gore MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)]<10> {25} 13 Aug 1998 17:42:41 +10 ---------------------------------- ================================================ FILE: tests/resources/imap/013.txt ================================================ Date: Mon, 13 Aug 1998 17:42:41 +1000 Message-Id: <199804130742.RAA20366@mai1host.whitehouse.gov> From: Bill Clinton To: A1 (The Enforcer) Gore Subject: Map of Argentina with Description MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit Hi A1, I finally figured out this MIME thing. Pretty cool. I'll send you some sax music in .au files next week! Anyway, the attached image is really too small to get a good look at Argentina. Try this for a much better map: http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm Then again, shouldn't the CIA have something like that? Bill ================================================ FILE: tests/resources/imap/014.imap ================================================ BODY ( ( "text" "x-myown" ( "charset" "us-ascii" ) NIL NIL "quoted-printable" 79 15 ) "mixed" ) BODYSTRUCTURE ( ( "text" "x-myown" ( "charset" "us-ascii" ) NIL NIL "quoted-printable" 79 15 "22839bb2efefde05dda98625a9ed8875" NIL NIL NIL ) "mixed" ( "boundary" "foo bar" ) NIL NIL NIL ) BODY[] {404} From user@domain Fri Feb 22 17:06:23 2008 From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="foo bar" Root MIME prologue --foo bar Content-Type: text/x-myown; charset=us-ascii Content-Transfer-Encoding: quoted-printable hello bar= foo = bar foo = =62 foo = bar foo = =62 foo bar= foo_bar --foo bar-- Root MIME epilogue BINARY[] {16} [binary content] BINARY.SIZE[] 404 ---------------------------------- BODY[HEADER] {173} From user@domain Fri Feb 22 17:06:23 2008 From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="foo bar" ---------------------------------- BODY[TEXT] {231} Root MIME prologue --foo bar Content-Type: text/x-myown; charset=us-ascii Content-Transfer-Encoding: quoted-printable hello bar= foo = bar foo = =62 foo = bar foo = =62 foo bar= foo_bar --foo bar-- Root MIME epilogue ---------------------------------- BODY[MIME] {53} Content-Type: multipart/mixed; boundary="foo bar" ---------------------------------- BODY[1] {79} hello bar= foo = bar foo = =62 foo = bar foo = =62 foo bar= foo_bar BINARY[1] {56} hello bar foo bar foo b foo bar foo b foo bar foo_bar BINARY.SIZE[1] 56 ---------------------------------- BODY[1.HEADER] {90} Content-Type: text/x-myown; charset=us-ascii Content-Transfer-Encoding: quoted-printable ---------------------------------- BODY[1.TEXT] {79} hello bar= foo = bar foo = =62 foo = bar foo = =62 foo bar= foo_bar ---------------------------------- BODY[1.MIME] {91} Content-Type: text/x-myown; charset=us-ascii Content-Transfer-Encoding: quoted-printable ---------------------------------- BODY[1.1] {79} hello bar= foo = bar foo = =62 foo = bar foo = =62 foo bar= foo_bar BINARY[1.1] {56} hello bar foo bar foo b foo bar foo b foo bar foo_bar BINARY.SIZE[1.1] 56 ---------------------------------- BODY[HEADER.FIELDS (FROM TO)] {24} From: user@domain.org ---------------------------------- BODY[HEADER.FIELDS (FROM TO)]<10> {14} @domain.org ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)] {174} From user@domain Fri Feb 22 17:06:23 2008 From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="foo bar" ---------------------------------- BODY[HEADER.FIELDS.NOT (SUBJECT CC)]<10> {25} domain Fri Feb 22 17:06: ---------------------------------- ================================================ FILE: tests/resources/imap/014.txt ================================================ From user@domain Fri Feb 22 17:06:23 2008 From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="foo bar" Root MIME prologue --foo bar Content-Type: text/x-myown; charset=us-ascii Content-Transfer-Encoding: quoted-printable hello bar= foo = bar foo = =62 foo = bar foo = =62 foo bar= foo_bar --foo bar-- Root MIME epilogue ================================================ FILE: tests/resources/imap-test/append ================================================ connections: 3 state: created 1 ok select $mailbox 2 ok select $mailbox # Two connections have mailbox SELECTed, one doesn't. # The \recent flags can be given to either one of the SELECTed connections, # but never for the 3rd. We rely on mailbox state tracking to catch duplicate # \recent flags (which is why there are two FETCH FLAGS commands). 1 ok append $mailbox (\seen \flagged) * 1 exists 2 ok noop * 1 exists 3 ok status $mailbox (messages unseen recent) * status $mailbox (messages 1 unseen 0 recent 0) 1 ok fetch 1 (uid flags) * 1 fetch (uid $uid1 flags (\seen \flagged)) 2 ok fetch 1 (uid flags) * 1 fetch (uid $uid1 flags (\seen \flagged)) 2 ok append * 2 exists 1 ok noop * 2 exists 3 ok status $mailbox (messages unseen recent) * status $mailbox (messages 2 unseen 1 recent 0) 1 ok fetch 2 (uid flags) 2 ok fetch 2 (uid flags) 3 ok append 3 ok status $mailbox (messages unseen) * status $mailbox (messages 3 unseen 2) 2 ok noop * 3 exists 1 ok noop * 3 exists 1 ok fetch 3 (uid flags) 2 ok fetch 3 (uid flags) 1 append ${mailbox}nonexistent no [trycreate] ================================================ FILE: tests/resources/imap-test/append-binary ================================================ capabilities: BINARY state: created ok append $mailbox ~{{{ From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: application/octet-stream Content-Transfer-Encoding: binary nil }}} ok select $mailbox # should have been converted to base64, or something ok fetch 1 (body.peek[1]) ! 1 fetch (body[1] {{{ nil }}}) ! 1 fetch (body[1] {{{ nil }}}) ! 1 fetch (body[1] ~{{{ nil }}}) ok fetch 1 (binary.size[1] binary.peek[1]) ! 1 fetch (binary.size[1] 6 binary[1] ~{{{ nil }}}) ================================================ FILE: tests/resources/imap-test/atoms ================================================ state: auth # Don't confuse with ~{literal8} ok list "" ~foo no select ~foo # atom-specials = "(" / ")" / "{" / SP / CTL / list-wildcards / # quoted-specials / resp-specials # quoted-specials = DQUOTE / "\" # resp-specials = "]" # list-wildcards = "%" / "*" no select !#$$&'+,-.0123456789:;<=>?@^_`|[} ok list "" !#$$&'+,-.0123456789:;<=>?@^_`|[}%* ================================================ FILE: tests/resources/imap-test/broken/search-intdate ================================================ # this test assumes that server stores INTERNALDATE timestamps in # EET/EEST timezones (or stores/fetches them using the same timezones as # they were APPENDed with) # 1) Timezone changes from EET +0200 -> EEST +0300 # 1a) BEFORE ok search before 24-mar-2007 * search ok search before 25-mar-2007 * search 1 ok search before 26-mar-2007 * search 1 2 3 4 5 6 ok search before 27-mar-2007 * search 1 2 3 4 5 6 7 # 1b) ON ok search on 23-mar-2007 * search ok search on 24-mar-2007 * search 1 ok search on 25-mar-2007 * search 2 3 4 5 6 ok search on 26-mar-2007 * search 7 # 1c) SINCE ok search 1:7 since 24-mar-2007 * search 1 2 3 4 5 6 7 ok search 1:7 since 25-mar-2007 * search 2 3 4 5 6 7 ok search 1:7 since 26-mar-2007 * search 7 ok search 1:7 since 27-mar-2007 * search # 2) Timezone changes from EEST +0300 -> EET +0200 # 2a) BEFORE ok search 8:* before 27-oct-2007 * search ok search 8:* before 28-oct-2007 * search 8 ok search 8:* before 29-oct-2007 * search 8 9 10 11 12 13 14 15 ok search 8:* before 30-oct-2007 * search 8 9 10 11 12 13 14 15 16 # 2b) ON ok search 8:* on 26-oct-2007 * search ok search 8:* on 27-oct-2007 * search 8 ok search 8:* on 28-oct-2007 * search 9 10 11 12 13 14 15 ok search 8:* on 29-oct-2007 * search 16 # 2c) SINCE ok search 8:* since 27-oct-2007 * search 8 9 10 11 12 13 14 15 16 ok search 8:* since 28-oct-2007 * search 9 10 11 12 13 14 15 16 ok search 8:* since 29-oct-2007 * search 16 ok search 8:* since 30-oct-2007 * search # 3) Try a couple of NOTs ok search 1:7 not before 26-mar-2007 * search 7 ok search 1:7 not on 25-mar-2007 * search 1 7 ok search 8:* not since 28-oct-2007 * search 8 ok search 8:* not on 28-oct-2007 * search 8 16 ================================================ FILE: tests/resources/imap-test/broken/search-intdate.mbox ================================================ From user@domain Sat Mar 24 23:00:00 2007 +0200 Date: Sat, 24 Mar 2007 23:00:00 +0200 body From user@domain Sun Mar 25 00:00:00 2007 +0200 Date: Sat, 24 Mar 2007 23:00:00 +0200 body From user@domain Sun Mar 25 01:00:00 2007 +0200 Date: Sat, 24 Mar 2007 23:00:00 +0200 body From user@domain Sun Mar 25 02:00:00 2007 +0200 Date: Sat, 24 Mar 2007 23:00:00 +0200 body From user@domain Sun Mar 25 04:00:00 2007 +0300 Date: Sat, 24 Mar 2007 23:00:00 +0200 body From user@domain Sun Mar 25 23:00:00 2007 +0300 Date: Sat, 24 Mar 2007 23:00:00 +0200 body From user@domain Mon Mar 26 00:00:00 2007 +0300 Date: Sat, 24 Mar 2007 23:00:00 +0200 body From user@domain Sat Oct 27 23:00:00 2007 +0300 Date: Sat, 24 Mar 2007 23:00:00 +0200 body From user@domain Sun Oct 28 00:00:00 2007 +0300 Date: Sat, 24 Mar 2007 23:00:00 +0200 body From user@domain Sun Oct 28 01:00:00 2007 +0300 Date: Sat, 24 Mar 2007 23:00:00 +0200 body From user@domain Sun Oct 28 02:00:00 2007 +0300 Date: Sat, 24 Mar 2007 23:00:00 +0200 body From user@domain Sun Oct 28 03:00:00 2007 +0300 Date: Sat, 24 Mar 2007 23:00:00 +0200 body From user@domain Sun Oct 28 03:00:00 2007 +0200 Date: Sat, 24 Mar 2007 23:00:00 +0200 body From user@domain Sun Oct 28 04:00:00 2007 +0200 Date: Sat, 24 Mar 2007 23:00:00 +0200 body From user@domain Sun Oct 28 23:00:00 2007 +0200 Date: Sat, 24 Mar 2007 23:00:00 +0200 body From user@domain Mon Oct 29 00:00:00 2007 +0200 Date: Sat, 24 Mar 2007 23:00:00 +0200 body ================================================ FILE: tests/resources/imap-test/catenate ================================================ state: created capabilities: CATENATE "" delete ${mailbox}2 ok append $mailbox (\seen \flagged) catenate (text {{{ From: foo@example.com Hello world body }}}) ok append $mailbox (\seen \flagged) catenate (text {{{ From: foo2@example.com Lookslike: header Another body }}}) ok select $mailbox * ok [uidvalidity $uidv] ok fetch 1:2 (uid body.peek[]) * 1 fetch (uid $uid body[] {{{ From: foo@example.com Hello world body }}}) * 2 fetch (uid $uid2 body[] {{{ From: foo2@example.com Lookslike: header Another body }}}) ok create ${mailbox}2 ok append ${mailbox}2 (\seen \flagged) catenate (url "/$mailbox_url/;uid=$uid/;section=header" text {{{ body1 }}}) ok append ${mailbox}2 (\seen \flagged) catenate (url "/$mailbox_url;uidvalidity=$uidv/;uid=$uid/;section=header" text {{{ body2 }}}) ok append ${mailbox}2 (\seen \flagged) catenate (url "/$mailbox_url/;uid=$uid2/;section=text" text {{{ body3 }}}) ok append ${mailbox}2 (\seen \flagged) catenate (url "/$mailbox_url/;uid=$uid2/;section=1" text {{{ body4 }}}) ok append ${mailbox}2 (\seen \flagged) catenate (text {{{ From: new@example.com Subject: test header }}} url "/$mailbox_url/;uid=$uid/;section=1" text {{{ suffix }}}) ok append ${mailbox}2 (\seen \flagged) catenate (text {{{ From: hdr1@example.com }}} text {{{ Subject: hdr2 }}} text {{{ body6 }}}) ok append ${mailbox}2 (\seen \flagged) catenate (url "/$mailbox_url/;uid=$uid2/;partial=100000.1" text {{{ Hdr: foo body7 }}}) # # Try invalid URLs # no append ${mailbox}2 (\seen \flagged) catenate (url "/$mailbox_url;uidvalidity=12345/;uid=$uid/;section=header" text {{{ body }}}) no append ${mailbox}2 (\seen \flagged) catenate (text {{{ hdr: 1 }}} url "/$mailbox_url;uidvalidity=12345/;uid=$uid/;section=header") no append ${mailbox}2 (\seen \flagged) catenate (text {{{ hdr: 1 }}} url "/$mailbox_url;uidvalidity=12345/;uid=$uid/;section=header" text {{{ hdr: 2 }}}) no append ${mailbox}2 (\seen \flagged) catenate (text {{{ hdr: 1 }}} url "/$mailbox_url;uidvalidity=12345/;uid=$uid/;section=header" text {{{ hdr: 2 }}} url "/$mailbox_url;uidvalidity=12345/;uid=$uid/;section=header" text {{{ hdr: 3 }}}) no append ${mailbox}2 (\seen \flagged) catenate (text {{{ hdr: 1 }}} url "/$mailbox_url/;uid=$uid/;section=header" text {{{ hdr: 2 }}} url "/$mailbox_url;uidvalidity=12345/;uid=$uid/;section=header" text {{{ hdr: 3 }}}) no append ${mailbox}2 (\seen \flagged) catenate (text {{{ hdr: 1 }}} url "/$mailbox_url;uidvalidity=12345/;uid=$uid/;section=header" text {{{ hdr: 2 }}} url "/$mailbox_url/;uid=$uid/;section=header" text {{{ hdr: 3 }}}) no append ${mailbox}2 (\seen \flagged) catenate (url "/$mailbox_url;uidvalidity=12345/;uid=$uid/;section=header" url "/$mailbox_url/;uid=$uid/;section=header") no append ${mailbox}2 (\seen \flagged) catenate (url "/$mailbox_url/;uid=$uid/;section=header" url "/$mailbox_url;uidvalidity=12345/;uid=$uid/;section=header") # # verify previous appends # ok select ${mailbox}2 ok fetch 1:* body.peek[] * 1 fetch (body[] {{{ From: foo@example.com body1 }}}) * 2 fetch (body[] {{{ From: foo@example.com body2 }}}) * 3 fetch (body[] {{{ Lookslike: header Another body body3 }}}) * 4 fetch (body[] {{{ Lookslike: header Another body body4 }}}) * 5 fetch (body[] {{{ From: new@example.com Subject: test header Hello world body suffix }}}) * 6 fetch (body[] {{{ From: hdr1@example.com Subject: hdr2 body6 }}}) * 7 fetch (body[] {{{ Hdr: foo body7 }}}) # # Try appending to nonexistent mailbox # append ${mailbox}nonexistent catenate (url "/$mailbox_url/;uid=$uid/;section=1") no [trycreate] append ${mailbox}nonexistent catenate (url "/$mailbox_url/;uid=$uid/;section=1" text {{{ hello }}}) no [trycreate] append ${mailbox}nonexistent catenate (text {{{ hello }}} url "/$mailbox_url/;uid=$uid/;section=1") no [trycreate] append ${mailbox}nonexistent (\seen \flagged) catenate (text {{{ hdr1: 1 }}} url "/$mailbox_url;uidvalidity=12345/;uid=$uid/;section=header" text {{{ hdr2: 2 }}} url "/$mailbox_url/;uid=$uid/;section=header" text {{{ hdr3: 3 }}}) no [trycreate] ok noop ================================================ FILE: tests/resources/imap-test/catenate-multiappend ================================================ state: created capabilities: CATENATE MULTIAPPEND ok append $mailbox catenate (text {{{ From: foo@example.com Hello world body }}}) catenate (text {{{ From: bar@example.com Second body }}}) ok select $mailbox * ok [uidvalidity $uidv] ok fetch 1:2 (uid body.peek[]) * 1 fetch (uid $uid body[] {{{ From: foo@example.com Hello world body }}}) * 2 fetch (uid $uid2 body[] {{{ From: bar@example.com Second body }}}) ok append ${mailbox} catenate (url "/$mailbox_url/;uid=$uid/;section=header" text {{{ body1 }}}) catenate (url "/$mailbox_url/;uid=$uid2/;section=header" text {{{ body2 }}}) ok append ${mailbox} catenate (url "/$mailbox_url/;uid=$uid") catenate (url "/$mailbox_url/;uid=$uid2/;section=header" text {{{ body3 }}}) ok append ${mailbox} catenate (url "/$mailbox_url/;uid=$uid") {{{ New: Message body4 }}} ok append ${mailbox} catenate (url "/$mailbox_url/;uid=$uid") (\answered) {{{ New: Message body5 }}} ok noop ok fetch 3:* body.peek[] * 3 fetch (body[] {{{ From: foo@example.com body1 }}}) * 4 fetch (body[] {{{ From: bar@example.com body2 }}}) * 5 fetch (body[] {{{ From: foo@example.com Hello world body }}}) * 6 fetch (body[] {{{ From: bar@example.com body3 }}}) * 7 fetch (body[] {{{ From: foo@example.com Hello world body }}}) * 8 fetch (body[] {{{ New: Message body4 }}}) * 9 fetch (body[] {{{ From: foo@example.com Hello world body }}}) * 10 fetch (body[] {{{ New: Message body5 }}}) # # Try invalid URLs # no append ${mailbox} (\seen \flagged) catenate (url "/$mailbox_url;uidvalidity=12345/;uid=$uid/;section=header") catenate (url "/$mailbox_url/;uid=$uid") no append ${mailbox} (\seen \flagged) catenate (url "/$mailbox_url;uidvalidity=12345/;uid=$uid/;section=header") catenate (url "/$mailbox_url/;uid=$uid" text {{{ hello }}}) no append ${mailbox} (\seen \flagged) catenate (url "/$mailbox_url;uidvalidity=12345/;uid=$uid/;section=header") catenate (text {{{ hello }}}) no append ${mailbox} (\seen \flagged) catenate (url "/$mailbox_url;uidvalidity=12345/;uid=$uid/;section=header") {{{ hello }}} ok noop ================================================ FILE: tests/resources/imap-test/close ================================================ connections: 2 messages: 3 1 ok store 1,3 +flags \deleted 1 ok close 2 ok noop * $1 expunge * $3 expunge ================================================ FILE: tests/resources/imap-test/copy ================================================ connections: 2 state: created 1 ok select ${mailbox} 1 ok append 1 ok append 1 ok append 1 ok append 1 ok append # make sure the server sees the appended messages 1 ok check 2 "" delete ${mailbox}2 2 ok create ${mailbox}2 2 ok select ${mailbox}2 1 ok store 1 flags (\seen) 1 ok store 2 flags (\answered \flagged) 1 ok store 5 flags (\flagged $$keyword1 $$keyword2) 1 ok fetch 1:5 (internaldate) * 1 fetch (internaldate $date1) * 2 fetch (internaldate $date2) * 4 fetch (internaldate $date4) * 5 fetch (internaldate $date5) 1 ok copy 1:2,4 ${mailbox}2 2 ok noop * 3 exists #* 3 recent 2 ok fetch 1:3 (flags internaldate) ? 1 fetch (flags (\seen) internaldate $date1) ? 2 fetch (flags (\answered \flagged) internaldate $date2) ? 3 fetch (flags () internaldate $date4) # keywords aren't required to be created on COPY, so help the server here 2 ok store 3 +flags ($$keyword1 $$keyword2) 1 ok copy 5 ${mailbox}2 2 ok noop * 4 exists 2 ok fetch 4 (flags internaldate) * 4 fetch (flags (\flagged $$keyword1 $$keyword2) internaldate $date5) 2 ok close 2 "" delete ${mailbox}2 ================================================ FILE: tests/resources/imap-test/default.mbox ================================================ From user@domain Fri Feb 22 17:06:23 2008 From: user1@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Subject: s1 body1 From user@domain Fri Feb 22 17:06:23 2008 From: user2@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Subject: s22 body22 From user@domain Fri Feb 22 17:06:23 2008 From: user3@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Subject: s333 body33 From user@domain Fri Feb 22 17:06:23 2008 From: user4@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Subject: s4444 body4444 From user@domain Fri Feb 22 17:06:23 2008 From: user5@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Subject: s55555 body55555 From user@domain Fri Feb 22 17:06:23 2008 From: user6@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Subject: s666666 body666666 ================================================ FILE: tests/resources/imap-test/esearch ================================================ capabilities: ESEARCH messages: all ok store 4 +flags \deleted ok expunge # SEARCH ALL ok search return (all) all * esearch (tag $tag) all 1:6 ok search return () all * esearch (tag $tag) all 1:6 ok search return (min) all * esearch (tag $tag) min 1 ok search return (max) all * esearch (tag $tag) max 6 ok search return (count) all * esearch (tag $tag) count 6 # UID SEARCH ALL ok fetch 1:* UID * 1 fetch (uid $uid1) * 2 fetch (uid $uid2) * 3 fetch (uid $uid3) * 4 fetch (uid $uid4) * 5 fetch (uid $uid5) * 6 fetch (uid $uid6) ok uid search return (all) all * esearch (tag $tag) uid all $uid1:$uid3,$uid4:$uid6 ok uid search return () all * esearch (tag $tag) uid all $uid1:$uid3,$uid4:$uid6 ok uid search return (min) all * esearch (tag $tag) uid min $uid1 ok uid search return (max) all * esearch (tag $tag) uid max $uid6 ok uid search return (count) all * esearch (tag $tag) uid count 6 # \Seen flag test ok store 2,4 +flags \seen ok uid search return (all) seen * esearch (tag $tag) uid all $uid2,$uid4 ok uid search return () seen * esearch (tag $tag) uid all $uid2,$uid4 ok uid search return (min) seen * esearch (tag $tag) uid min $uid2 ok uid search return (max) seen * esearch (tag $tag) uid max $uid4 ok uid search return (count) seen * esearch (tag $tag) uid count 2 # nonexistent ok search return () 1000 * esearch (tag $tag) ok search return (min) 1000 * esearch (tag $tag) ok search return (max) 1000 * esearch (tag $tag) ok search return (count) 1000 * esearch (tag $tag) count 0 # UID nonexistent ok uid search return () 1000 * esearch (tag $tag) uid ok uid search return (min) 1000 * esearch (tag $tag) uid ok uid search return (max) 1000 * esearch (tag $tag) uid ok uid search return (count) 1000 * esearch (tag $tag) uid count 0 ================================================ FILE: tests/resources/imap-test/esearch-condstore ================================================ capabilities: ESEARCH CONDSTORE messages: 4 # ENABLE is valid only in authenticated state. we could do enable+select # manually here also, but lets just make sure that switching it on via # condstore-enabling command works as well (since that is valid) ok fetch 1 modseq * ok [highestmodseq $highestmodseq] ok store 1 +flags \seen * 1 fetch (modseq ($modseq1)) ok store 3 +flags \seen * 3 fetch (modseq ($modseq3)) ok store 2 +flags \seen * 2 fetch (modseq ($modseq2)) ok store 4 +flags \seen * 4 fetch (modseq ($modseq4)) ok search return (min) 1:3 modseq "/flags/\\seen" all $highestmodseq * esearch (tag $tag) min 1 modseq $modseq4 ok search return (max) 1:3 modseq "/flags/\\seen" all $highestmodseq * esearch (tag $tag) max 3 modseq $modseq4 ok search return () 1:3 modseq "/flags/\\seen" all $highestmodseq * esearch (tag $tag) all 1:3 modseq $modseq4 ok search return (min max) 2:3 modseq "/flags/\\seen" all $highestmodseq * esearch (tag $tag) min 2 max 3 modseq $modseq4 ok search return (all) 2:4 modseq "/flags/\\seen" all $modseq3 * esearch (tag $tag) all 2,4 modseq $modseq4 ok search return (all) 2:4 modseq "/flags/\\seen" all $modseq2 * esearch (tag $tag) all 4 modseq $modseq4 ok search return (all) 2:4 modseq "/flags/\\seen" all $modseq4 * esearch (tag $tag) modseq $modseq4 ================================================ FILE: tests/resources/imap-test/esearch.mbox ================================================ From user@domain Fri Feb 22 17:06:21 2008 1 From user@domain Fri Feb 22 17:06:22 2008 2 From user@domain Fri Feb 22 17:06:23 2008 3 From user@domain Fri Feb 22 17:06:23 2008 4 From user@domain Fri Feb 22 17:06:25 2008 5 From user@domain Fri Feb 22 17:06:25 2008 6 From user@domain Fri Feb 22 17:06:25 2008 7 ================================================ FILE: tests/resources/imap-test/esort ================================================ capabilities: ESORT messages: 5 ok sort return (min) (arrival) us-ascii 1:5 * esearch (tag $tag) min 1 ok sort return (max) (arrival) us-ascii 1:5 * esearch (tag $tag) max 5 ok sort return (all) (arrival) us-ascii 1:5 * esearch (tag $tag) all 1:5 ok sort return (count) (arrival) us-ascii 1:5 * esearch (tag $tag) count 5 ================================================ FILE: tests/resources/imap-test/expunge ================================================ connections: 2 messages: 8 # get UIDs 1 ok fetch 1:4 uid * 1 fetch (uid $uid1) * 2 fetch (uid $uid2) * 3 fetch (uid $uid3) * 4 fetch (uid $uid4) # 1) test that expunges work ok and session 2 fetch sees 1's flag changes. 1 ok store 1,3 flags \deleted * 1 fetch (flags (\deleted)) * 3 fetch (flags (\deleted)) 1 ok store 2,4 flags \seen * 2 fetch (flags (\seen)) * 4 fetch (flags (\seen)) 1 ok expunge * $1 expunge * $3 expunge 2 ok fetch 2,4 (uid) * 2 fetch (uid $uid2) * 4 fetch (uid $uid4) 2 ok fetch 2,4 (uid) * 2 fetch (uid $uid2) * 4 fetch (uid $uid4) 2 ok check * $1 expunge * $3 expunge 2 ok fetch 1:2 (uid flags) * 1 fetch (uid $uid2 flags (\seen)) * 2 fetch (uid $uid4 flags (\seen)) # 2) test that session 2 can update flags while some messages are expunged 1 ok store 2 +flags \deleted 1 ok expunge 2 ok store 1 flags \answered * 1 fetch (flags (\answered)) 1 ok check * 1 fetch (flags (\answered)) 2 ok noop * 2 expunge # 3) check notices flag changes correctly with expunges 1 ok store 1,3 +flags \deleted 1 ok store 2,4 flags \flagged 1 ok expunge 2 ok check * $1 expunge * $2 fetch (flags (\flagged)) * $3 expunge * $4 fetch (flags (\flagged)) # 4) expunging while message is already expunged 1 ok store 1 +flags \deleted 2 ok store 1 +flags \deleted 1 ok expunge * 1 expunge 2 ok expunge * 1 expunge ================================================ FILE: tests/resources/imap-test/expunge2 ================================================ connections: 2 messages: 6 1 ok fetch 2,4 uid * 2 fetch (uid $uid2) * 4 fetch (uid $uid4) # UID FETCH 1 ok store 1,3 +flags \deleted 1 ok expunge * $1 expunge * $3 expunge 2 ok uid fetch $uid2,$uid4 uid * $2 fetch (uid $uid2) * $4 fetch (uid $uid4) 2 ok noop # UID STORE 1 ok store 1 +flags \deleted 1 ok expunge * 1 expunge 2 ok uid store $uid4 flags \seen * $2 fetch (uid $uid4 flags (\seen)) 2 ok noop # Make sure CHECK works just as well as NOOP 1 ok store 1 +flags \deleted 1 ok expunge * 1 expunge 2 ok check * 1 expunge # Make sure FETCH, STORE and SEARCH don't trigger EXPUNGE 1 ok store 1 +flags \deleted 1 ok expunge 2 ok fetch 2 flags ! $1 expunge 2 ok store 2 flags (\seen) ! $1 expunge 2 ok search all ! $1 expunge ================================================ FILE: tests/resources/imap-test/fetch ================================================ messages: 3 ok status $mailbox (uidnext) * status $mailbox (uidnext $uidnext) ok fetch 1:3 uid * 1 fetch (uid $uid1) * 2 fetch (uid $uid2) * 3 fetch (uid $uid3) ok fetch 3:1 uid * 1 fetch (uid $uid1) * 2 fetch (uid $uid2) * 3 fetch (uid $uid3) ok fetch 1,* uid * 1 fetch (uid $uid1) * 3 fetch (uid $uid3) ok fetch * uid * 3 fetch (uid $uid3) ok fetch *:1 uid * 1 fetch (uid $uid1) * 2 fetch (uid $uid2) * 3 fetch (uid $uid3) ok uid fetch $uidnext:* uid * 3 fetch (uid $uid3) # break seq=uid map ok store 2 flags \deleted ok expunge * 2 expunge ok uid fetch $uidnext:* uid * 2 fetch (uid $uid3) ok uid fetch 1:* uid * 1 fetch (uid $uid1) * 2 fetch (uid $uid3) ok uid fetch $uid2 uid ! 1 fetch (uid $uid1) ! 2 fetch (uid $uid3) ok uid fetch $uid1,* uid * 1 fetch (uid $uid1) * 2 fetch (uid $uid3) # test macros ok fetch 1 full * 1 fetch (flags () internaldate $intdate1 rfc822.size $size1 envelope ($!unordered) body ($!unordered)) ok fetch 1 all * 1 fetch (flags () internaldate $intdate1 rfc822.size $size1 envelope ($!unordered)) ok fetch 1 fast * 1 fetch (flags () internaldate $intdate1 rfc822.size $size1) ================================================ FILE: tests/resources/imap-test/fetch-binary-mime ================================================ capabilities: BINARY messages: all # This is the fetch-body-mime test, except with body[] changed to binary[]. # The idea is to verify that binary[] works correctly when it doesn't actually # have to convert anything. ok fetch 1 (binary.peek[]) * 1 fetch (binary[] {{{ From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="foo bar" Root MIME prologue --foo bar Content-Type: text/x-myown; charset=us-ascii hello --foo bar Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/alternative; boundary="sub1" Sub MIME prologue --sub1 Content-Type: text/html

Hello world

--sub1 Content-Type: text/plain Hello another world --sub1-- Sub MIME epilogue --foo bar-- Root MIME epilogue }}}) ok fetch 1 (binary.size[]) * 1 fetch (binary.size[] 602) ok fetch 1 (binary.size[1]) * 1 fetch (binary.size[1] 7) ok fetch 1 (binary.peek[1]) * 1 fetch (binary[1] {{{ hello }}}) ok fetch 1 (binary.peek[2]) * 1 fetch (binary[2] {{{ From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/alternative; boundary="sub1" Sub MIME prologue --sub1 Content-Type: text/html

Hello world

--sub1 Content-Type: text/plain Hello another world --sub1-- Sub MIME epilogue }}}) ok fetch 1 (binary.size[2]) * 1 fetch (binary.size[2] 298) ok fetch 1 (binary.size[2.1]) * 1 fetch (binary.size[2.1] 20) ok fetch 1 (binary.peek[2.1]) * 1 fetch (binary[2.1] {{{

Hello world

}}}) ok fetch 1 (binary.peek[2.2]) * 1 fetch (binary[2.2] {{{ Hello another world }}}) ok fetch 1 (binary.size[2.2]) * 1 fetch (binary.size[2.2] 21) ================================================ FILE: tests/resources/imap-test/fetch-binary-mime-base64 ================================================ capabilities: BINARY messages: all ok fetch 1 (binary.size[1] binary.size[5]) * 1 fetch (binary.size[1] 11 binary.size[5] 30) ok fetch 1 (binary.peek[1] binary.peek[2] binary.peek[3] binary.peek[4] binary.peek[5]) * 1 fetch (binary[1] {{{ hello world }}} binary[2] {{{ hello world }}} binary[3] {{{ hello to you too }}} binary[4] {{{ hello to everyone! }}} binary[5] ~{{{ abcdefg hijkl mno pqrstuvqxyz }}}) ok fetch 1 (binary.size[1] binary.peek[1]) * 1 fetch (binary.size[1] 11 binary[1] {{{ hello world }}}) ok fetch 1 (binary.size[3] binary.peek[2]) * 1 fetch (binary.size[3] 16 binary[2] {{{ hello world }}}) ok fetch 1 (binary.size[2] binary.size[3] binary.size[4]) * 1 fetch (binary.size[2] 11 binary.size[3] 16 binary.size[4] 18) ok fetch 1 (binary.peek[5]<0.7>) * 1 fetch (binary[5]<0> ~{{{ abcdefg }}}) ok fetch 1 (binary.peek[5]<0.8>) * 1 fetch (binary[5]<0> ~{{{ abcdefg }}}) ok fetch 1 (binary.peek[5]<0.10>) * 1 fetch (binary[5]<0> ~{{{ abcdefg hi }}}) ok fetch 1 (binary.peek[5]<10.10>) * 1 fetch (binary[5]<10> ~{{{ jkl mno p }}}) ok fetch 1 (binary.peek[5]<5.10>) * 1 fetch (binary[5]<5> ~{{{ fg hijkl }}}) ok fetch 1 (binary.peek[5]<15.100>) * 1 fetch (binary[5]<15> ~{{{ mno pqrstuvqxyz }}}) ================================================ FILE: tests/resources/imap-test/fetch-binary-mime-base64.mbox ================================================ From user@domain Fri Feb 22 17:06:23 2008 From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="foo bar" Root MIME prologue --foo bar Content-Type: text/x-myown; charset=us-ascii Content-Transfer-Encoding: base64 aGVs bG8gd29y bGQ= --foo bar Content-Type: text/x-myown; charset=us-ascii Content-Transfer-Encoding: base64 aGVs bG8g d29y bGQ= --foo bar Content-Type: text/x-myown; charset=us-ascii Content-Transfer-Encoding: base64 aGVsbG8g dG8geW91IHRv bw== --foo bar Content-Type: text/x-myown; charset=us-ascii Content-Transfer-Encoding: base64 aGVsbG8gdG8gZXZlcnlvbmUh --foo bar Content-Type: text/x-myown; charset=us-ascii Content-Transfer-Encoding: base64 YWJjZGVmZw1oaWprbA0KbW5vCnBxcnN0dXZxeHl6 --foo bar-- Root MIME epilogue ================================================ FILE: tests/resources/imap-test/fetch-binary-mime-qp ================================================ capabilities: BINARY messages: all ok fetch 1 (binary.size[1]) * 1 fetch (binary.size[1] 65) ok fetch 1 (binary.peek[1]) * 1 fetch (binary[1] {{{ hello bar foo bar foo b foo bar foo b foo bar foo_bar }}}) ok fetch 1 (binary.peek[1]<0.10>) * 1 fetch (binary[1]<0> {{{ hello bar }}}) ok fetch 1 (binary.peek[1]<10.10>) * 1 fetch (binary[1]<10> ~{{{ foo bar }}}) ok fetch 1 (binary.peek[1]<20.10>) * 1 fetch (binary[1]<20> ~{{{ foo b }}}) ok fetch 1 (binary.peek[1]<15.10>) * 1 fetch (binary[1]<15> ~{{{ bar foo }}}) ok fetch 1 (binary.peek[1]<40.100>) * 1 fetch (binary[1]<40> {{{ oo b foo bar foo_bar }}}) ================================================ FILE: tests/resources/imap-test/fetch-binary-mime-qp.mbox ================================================ From user@domain Fri Feb 22 17:06:23 2008 From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="foo bar" Root MIME prologue --foo bar Content-Type: text/x-myown; charset=us-ascii Content-Transfer-Encoding: quoted-printable hello bar= foo = bar foo = =62 foo = bar foo = =62 foo bar= foo_bar --foo bar-- Root MIME epilogue ================================================ FILE: tests/resources/imap-test/fetch-binary-mime.mbox ================================================ From user@domain Fri Feb 22 17:06:23 2008 From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="foo bar" Root MIME prologue --foo bar Content-Type: text/x-myown; charset=us-ascii hello --foo bar Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/alternative; boundary="sub1" Sub MIME prologue --sub1 Content-Type: text/html

Hello world

--sub1 Content-Type: text/plain Hello another world --sub1-- Sub MIME epilogue --foo bar-- Root MIME epilogue ================================================ FILE: tests/resources/imap-test/fetch-body ================================================ messages: all # header and body fetches ok fetch 1 rfc822.header * 1 fetch (rfc822.header $hdr1) ok fetch 1 body.peek[header] * 1 fetch (body[header] $hdr1) ok fetch 1 (flags body.peek[text]) * 1 fetch (flags () body[text] {{{ body1 }}}) ok fetch 1 rfc822.text * 1 fetch (rfc822.text {{{ body1 }}}) * 1 fetch (flags (\seen)) ok fetch 2 (flags body.peek[]) * 2 fetch (flags () body[] $full2) ok fetch 2 rfc822 * 2 fetch (rfc822 $full2) * 2 fetch (flags (\seen)) ok fetch 3 (body[]) * 3 fetch (body[] $full3) * 3 fetch (flags (\seen)) ok fetch 4 (body[header]) * 4 fetch (body[header] $hdr4) * 4 fetch (flags (\seen)) # partial fetches ok fetch 2 body.peek[text]<0.3> * 2 fetch (body[text]<0> "bod") ok fetch 2 body.peek[text]<3.3> * 2 fetch (body[text]<3> "y22") ok fetch 3 body.peek[text]<0.1> * 3 fetch (body[text]<0> "b") ok fetch 3 body.peek[text]<5.1> * 3 fetch (body[text]<5> "3") ok fetch 3 body.peek[text]<5.2> * 3 fetch (body[text]<5> ~{{{ 3 }}}) ok fetch 3 body.peek[text]<5.3> * 3 fetch (body[text]<5> ~{{{ 3 }}}) ok fetch 3 body.peek[text]<6.1> * 3 fetch (body[text]<6> ~{{{ }}}) ok fetch 3 body.peek[text]<6.2> * 3 fetch (body[text]<6> ~{{{ }}}) ok fetch 3 body.peek[text]<7.1> * 3 fetch (body[text]<7> ~{{{ }}}) # header fields ok fetch 1 body.peek[header.fields (from)] * 1 fetch (body[header.fields (from)] {{{ From: User1 }}}) ok fetch 1 (body.peek[header.fields (from)]) * 1 fetch (body[header.fields (from)] {{{ From: User1 }}}) ok fetch 1 (body.peek[header.fields (from from)]) * 1 fetch (body[header.fields (from from)] {{{ From: User1 }}}) ok fetch 1 body.peek[header.fields (from subject)] * 1 fetch (body[header.fields (from subject)] {{{ From: User1 Subject: s1 }}}) ok fetch 1 body.peek[header.fields.not (date)] ! 1 fetch (body[header.fields.not (date)] {{{ From: User1 Date: Sat, 24 Mar 2007 23:00:00 +0200 Subject: s1 }}}) ok fetch 1 body.peek[header.fields.not (date date)] ! 1 fetch (body[header.fields.not (date date)] {{{ From: User1 Date: Sat, 24 Mar 2007 23:00:00 +0200 Subject: s1 }}}) ok fetch 1 body.peek[header.fields (xyz)] * 1 fetch (body[header.fields (xyz)] {{{ }}}) ================================================ FILE: tests/resources/imap-test/fetch-body-message-rfc822 ================================================ messages: all ok fetch 1 (body.peek[]) * 1 fetch (body[] {{{ From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Hello world }}}) ok fetch 1 (body.peek[text]) * 1 fetch (body[text] {{{ From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Hello world }}}) ok fetch 1 (body.peek[1]) * 1 fetch (body[1] {{{ From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Hello world }}}) ok fetch 1 (body.peek[1.1]) * 1 fetch (body[1.1] {{{ Hello world }}}) ok fetch 1 (body.peek[1.header]) * 1 fetch (body[1.header] {{{ From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg }}}) ok fetch 1 (body.peek[1.header.fields (from subject X-foo)]) * 1 fetch (body[1.header.fields (from subject X-foo)] {{{ From: sub@domain.org Subject: submsg }}}) ok fetch 1 (body.peek[1.header.fields.not (from subject X-foo)]) * 1 fetch (body[1.header.fields.not (from subject X-foo)] {{{ Date: Sun, 12 Aug 2012 12:34:56 +0300 }}}) ok fetch 1 (body.peek[1.text]) * 1 fetch (body[1.text] {{{ Hello world }}}) ================================================ FILE: tests/resources/imap-test/fetch-body-message-rfc822-mime ================================================ messages: all ok fetch 1 (body.peek[]) * 1 fetch (body[] {{{ From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/digest; boundary="foo" prologue --foo From: m1@example.com Subject: m1 m1 body --foo Content-Custom: m2 header From: m2@example.com Subject: m2 m2 body --foo-- }}}) ok fetch 1 (body.peek[text]) * 1 fetch (body[text] {{{ From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/digest; boundary="foo" prologue --foo From: m1@example.com Subject: m1 m1 body --foo Content-Custom: m2 header From: m2@example.com Subject: m2 m2 body --foo-- }}}) ok fetch 1 (body.peek[1]) * 1 fetch (body[1] {{{ From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/digest; boundary="foo" prologue --foo From: m1@example.com Subject: m1 m1 body --foo Content-Custom: m2 header From: m2@example.com Subject: m2 m2 body --foo-- }}}) ok fetch 1 (body.peek[1.header]) * 1 fetch (body[1.header] {{{ From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/digest; boundary="foo" }}}) ok fetch 1 (body.peek[1.text]) * 1 fetch (body[1.text] {{{ prologue --foo From: m1@example.com Subject: m1 m1 body --foo Content-Custom: m2 header From: m2@example.com Subject: m2 m2 body --foo-- }}}) ok fetch 1 (body.peek[1.1]) * 1 fetch (body[1.1] {{{ From: m1@example.com Subject: m1 m1 body }}}) ok fetch 1 (body.peek[1.1.MIME]) * 1 fetch (body[1.1.MIME] {{{ }}}) ok fetch 1 (body.peek[1.1.HEADER]) * 1 fetch (body[1.1.HEADER] {{{ From: m1@example.com Subject: m1 }}}) ok fetch 1 (body.peek[1.1.TEXT]) * 1 fetch (body[1.1.TEXT] {{{ m1 body }}}) ok fetch 1 (body.peek[1.2]) * 1 fetch (body[1.2] {{{ From: m2@example.com Subject: m2 m2 body }}}) ok fetch 1 (body.peek[1.2.MIME]) * 1 fetch (body[1.2.MIME] {{{ Content-Custom: m2 header }}}) ok fetch 1 (body.peek[1.2.HEADER]) * 1 fetch (body[1.2.HEADER] {{{ From: m2@example.com Subject: m2 }}}) ok fetch 1 (body.peek[1.2.TEXT]) * 1 fetch (body[1.2.TEXT] {{{ m2 body }}}) ================================================ FILE: tests/resources/imap-test/fetch-body-message-rfc822-mime.mbox ================================================ From user@domain Fri Feb 22 17:06:23 2008 From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/digest; boundary="foo" prologue --foo From: m1@example.com Subject: m1 m1 body --foo Content-Custom: m2 header From: m2@example.com Subject: m2 m2 body --foo-- ================================================ FILE: tests/resources/imap-test/fetch-body-message-rfc822-x2 ================================================ messages: all ok fetch 1 (body.peek[]) * 1 fetch (body[] {{{ From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: message/rfc822 From: user2@domain.org Date: Fri, 23 Mar 2007 11:22:33 +0200 Mime-Version: 1.0 Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Hello world }}}) ok fetch 1 (body.peek[text]) * 1 fetch (body[text] {{{ From: user2@domain.org Date: Fri, 23 Mar 2007 11:22:33 +0200 Mime-Version: 1.0 Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Hello world }}}) ok fetch 1 (body.peek[1]) * 1 fetch (body[1] {{{ From: user2@domain.org Date: Fri, 23 Mar 2007 11:22:33 +0200 Mime-Version: 1.0 Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Hello world }}}) ok fetch 1 (body.peek[1.1]) * 1 fetch (body[1.1] {{{ From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Hello world }}}) ok fetch 1 (body.peek[1.header]) * 1 fetch (body[1.header] {{{ From: user2@domain.org Date: Fri, 23 Mar 2007 11:22:33 +0200 Mime-Version: 1.0 Content-Type: message/rfc822 }}}) ok fetch 1 (body.peek[1.text]) * 1 fetch (body[1.text] {{{ From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Hello world }}}) ok fetch 1 (body.peek[1.1.1]) * 1 fetch (body[1.1.1] {{{ Hello world }}}) ok fetch 1 (body.peek[1.1.header]) * 1 fetch (body[1.1.header] {{{ From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg }}}) ok fetch 1 (body.peek[1.1.text]) * 1 fetch (body[1.1.text] {{{ Hello world }}}) ================================================ FILE: tests/resources/imap-test/fetch-body-message-rfc822-x2.mbox ================================================ From user@domain Fri Feb 22 17:06:23 2008 From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: message/rfc822 From: user2@domain.org Date: Fri, 23 Mar 2007 11:22:33 +0200 Mime-Version: 1.0 Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Hello world ================================================ FILE: tests/resources/imap-test/fetch-body-message-rfc822.mbox ================================================ From user@domain Fri Feb 22 17:06:23 2008 From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Hello world ================================================ FILE: tests/resources/imap-test/fetch-body-mime ================================================ messages: all ok fetch 1 (body.peek[]) * 1 fetch (body[] {{{ From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="foo bar" Root MIME prologue --foo bar Content-Type: text/x-myown; charset=us-ascii hello --foo bar Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/alternative; boundary="sub1" Sub MIME prologue --sub1 Content-Type: text/html

Hello world

--sub1 Content-Type: text/plain Hello another world --sub1-- Sub MIME epilogue --foo bar-- Root MIME epilogue }}}) ok fetch 1 (body.peek[text]) * 1 fetch (body[text] {{{ Root MIME prologue --foo bar Content-Type: text/x-myown; charset=us-ascii hello --foo bar Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/alternative; boundary="sub1" Sub MIME prologue --sub1 Content-Type: text/html

Hello world

--sub1 Content-Type: text/plain Hello another world --sub1-- Sub MIME epilogue --foo bar-- Root MIME epilogue }}}) ok fetch 1 (body.peek[1]) * 1 fetch (body[1] {{{ hello }}}) ok fetch 1 (body.peek[1.mime]) * 1 fetch (body[1.mime] {{{ Content-Type: text/x-myown; charset=us-ascii }}}) ok fetch 1 (body.peek[2]) * 1 fetch (body[2] {{{ From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/alternative; boundary="sub1" Sub MIME prologue --sub1 Content-Type: text/html

Hello world

--sub1 Content-Type: text/plain Hello another world --sub1-- Sub MIME epilogue }}}) ok fetch 1 (body.peek[2.mime]) * 1 fetch (body[2.mime] {{{ Content-Type: message/rfc822 }}}) ok fetch 1 (body.peek[2.header]) * 1 fetch (body[2.header] {{{ From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/alternative; boundary="sub1" }}}) ok fetch 1 (body.peek[2.header.fields (from subject X-foo)]) * 1 fetch (body[2.header.fields (from subject X-foo)] {{{ From: sub@domain.org Subject: submsg }}}) ok fetch 1 (body.peek[2.header.fields.not (from subject X-foo)]) * 1 fetch (body[2.header.fields.not (from subject X-foo)] {{{ Date: Sun, 12 Aug 2012 12:34:56 +0300 Content-Type: multipart/alternative; boundary="sub1" }}}) ok fetch 1 (body.peek[2.text]) * 1 fetch (body[2.text] {{{ Sub MIME prologue --sub1 Content-Type: text/html

Hello world

--sub1 Content-Type: text/plain Hello another world --sub1-- Sub MIME epilogue }}}) ok fetch 1 (body.peek[2.1.mime]) * 1 fetch (body[2.1.mime] {{{ Content-Type: text/html }}}) ok fetch 1 (body.peek[2.1]) * 1 fetch (body[2.1] {{{

Hello world

}}}) ok fetch 1 (body.peek[2.2.mime]) * 1 fetch (body[2.2.mime] {{{ Content-Type: text/plain }}}) ok fetch 1 (body.peek[2.2]) * 1 fetch (body[2.2] {{{ Hello another world }}}) ================================================ FILE: tests/resources/imap-test/fetch-body-mime.mbox ================================================ From user@domain Fri Feb 22 17:06:23 2008 From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="foo bar" Root MIME prologue --foo bar Content-Type: text/x-myown; charset=us-ascii hello --foo bar Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/alternative; boundary="sub1" Sub MIME prologue --sub1 Content-Type: text/html

Hello world

--sub1 Content-Type: text/plain Hello another world --sub1-- Sub MIME epilogue --foo bar-- Root MIME epilogue ================================================ FILE: tests/resources/imap-test/fetch-body.mbox ================================================ From user@domain Fri Feb 22 17:06:23 2008 From: User1 Date: Sat, 24 Mar 2007 23:00:00 +0200 Subject: s1 body1 From user@domain Fri Feb 22 17:06:23 2008 From: User2 Date: Sat, 24 Mar 2007 23:00:00 +0200 Subject: s22 body22 From user@domain Fri Feb 22 17:06:23 2008 From: User3 Date: Sat, 24 Mar 2007 23:00:00 +0200 Subject: s333 body33 From user@domain Fri Feb 22 17:06:23 2008 From: User4 Date: Sat, 24 Mar 2007 23:00:00 +0200 Subject: s4444 body4444 ================================================ FILE: tests/resources/imap-test/fetch-bodystructure ================================================ messages: all ok fetch 1:* body * 1 FETCH (BODY (("TEXT" "x-MYOWN" ("CHARSET" "us-ascii") NIL NIL "7BIT" 7 1) "MIXED")) ================================================ FILE: tests/resources/imap-test/fetch-bodystructure.mbox ================================================ From user@domain Fri Feb 22 17:06:23 2008 From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="foo bar" --foo bar Content-Type: text/x-myown; charset=us-ascii hello --foo bar-- ================================================ FILE: tests/resources/imap-test/fetch-envelope ================================================ messages: all # date, subject, from, sender, reply-to, to, cc, bcc, in-reply-to, message-id ok fetch 1:* envelope * 1 FETCH (ENVELOPE ("Thu, 15 Feb 2007 01:02:03 +0200" "subject header" (("From Real" NIL "fromuser" "fromdomain.org")) (("Sender Real" NIL "senderuser" "senderdomain.org")) (("ReplyTo Real" NIL "replytouser" "replytodomain.org")) (("To Real" NIL "touser" "todomain.org")) (("Cc Real" NIL "ccuser" "ccdomain.org")) (("Bcc Real" NIL "bccuser" "bccdomain.org")) "" "")) * 2 FETCH (ENVELOPE ("Thu, 15 Feb 2007 01:02:03 +0200" NIL ((NIL NIL "user" "domain")) ((NIL NIL "user" "domain")) ((NIL NIL "user" "domain")) NIL NIL NIL NIL NIL)) * 3 FETCH (ENVELOPE ("Thu, 15 Feb 2007 01:02:03 +0200" NIL ((NIL NIL "user" "domain")) ((NIL NIL "user" "domain")) ((NIL NIL "user" "domain")) NIL NIL NIL NIL NIL)) * 4 FETCH (ENVELOPE ("Thu, 15 Feb 2007 01:02:03 +0200" NIL (("Real Name" NIL "user" "domain")) (("Real Name" NIL "user" "domain")) (("Real Name" NIL "user" "domain")) ((NIL NIL "group" NIL)(NIL NIL "g1" "d1.org")(NIL NIL "g2" "d2.org")(NIL NIL NIL NIL)(NIL NIL "group2" NIL)(NIL NIL "g3" "d3.org")(NIL NIL NIL NIL)) ((NIL NIL "group" NIL)(NIL NIL NIL NIL)(NIL NIL "group2" NIL)(NIL NIL NIL NIL)) NIL NIL NIL)) * 5 FETCH (ENVELOPE ("Thu, 15 Feb 2007 01:02:03 +0200" NIL (("Real Name" NIL "user" "domain")) (("Real Name" NIL "user" "domain")) (("Real Name" NIL "user" "domain")) NIL NIL NIL NIL NIL)) * 6 FETCH (ENVELOPE ("Thu, 15 Feb 2007 01:02:03 +0200" NIL ((NIL "@route" "user" "domain")) ((NIL "@route" "user" "domain")) ((NIL "@route" "user" "domain")) NIL NIL NIL NIL NIL)) ================================================ FILE: tests/resources/imap-test/fetch-envelope.mbox ================================================ From user@domain Fri Feb 22 17:06:23 2008 Message-ID: In-Reply-To: Date: Thu, 15 Feb 2007 01:02:03 +0200 Subject: subject header From: From Real To: To Real Cc: Cc Real Bcc: Bcc Real Sender: Sender Real Reply-To: ReplyTo Real body From user@domain Fri Feb 22 17:06:23 2008 Date: Thu, 15 Feb 2007 01:02:03 +0200 From: user@domain body From user@domain Fri Feb 22 17:06:23 2008 Date: Thu, 15 Feb 2007 01:02:03 +0200 From: user@domain body From user@domain Fri Feb 22 17:06:23 2008 Date: Thu, 15 Feb 2007 01:02:03 +0200 From: user@domain (Real Name) To: group: g1@d1.org, g2@d2.org;, group2: g3@d3.org; Cc: group:;, group2: (foo) ; body From user@domain Fri Feb 22 17:06:23 2008 Date: Thu, 15 Feb 2007 01:02:03 +0200 From: user@domain (Real Name) Sender: Reply-To: body From user@domain Fri Feb 22 17:06:23 2008 Date: Thu, 15 Feb 2007 01:02:03 +0200 From: <@route:user@domain> body ================================================ FILE: tests/resources/imap-test/id ================================================ capabilities: id state: nonauth ok ID ("a23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "b23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "c23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "d23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "e23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "f23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "g23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "h23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "i23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "j23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "k23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "l23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "m23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "n23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "o23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "p23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "q23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "r23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "s23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "t23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "u23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "v23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "w23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "x23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "y23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "z23456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "aa3456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "ab3456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "ac3456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234" "ad3456789012345678901234567890" "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234") ================================================ FILE: tests/resources/imap-test/list ================================================ connections: 3 state: auth # get the separator 1 ok list "" "" * list () $sep $root # it should be \noselect, but don't fail everything if it doesn't exist * list (\noselect) $sep $root 1 ok create $mailbox${sep} 1 ok create $mailbox${sep}test 2 ok list "" $mailbox${sep}% * list () $sep $mailbox${sep}test 2 ok create $mailbox${sep}test2 1 ok list "" $mailbox${sep}% * list () $sep $mailbox${sep}test * list () $sep $mailbox${sep}test2 3 ok create $mailbox${sep}test3${sep} 2 ok create $mailbox${sep}test3${sep}test4${sep} 2 ok create $mailbox${sep}test3${sep}test4${sep}test5 2 ok list "" $mailbox${sep}% * list () $sep $mailbox${sep}test * list () $sep $mailbox${sep}test2 * list () $sep $mailbox${sep}test3 ! list () $sep $mailbox${sep}test3${sep}test4 ! list () $sep $mailbox${sep}test3${sep}test4${sep}test5 3 ok list "" $mailbox${sep}% * list () $sep $mailbox${sep}test * list () $sep $mailbox${sep}test2 * list () $sep $mailbox${sep}test3 2 ok list "" $mailbox${sep}%${sep}% ! list () $sep $mailbox${sep}test ! list () $sep $mailbox${sep}test2 ! list () $sep $mailbox${sep}test3 * list () $sep $mailbox${sep}test3${sep}test4 ! list () $sep $mailbox${sep}test3${sep}test4${sep}test5 3 ok list "" $mailbox${sep}* * list () $sep $mailbox${sep}test * list () $sep $mailbox${sep}test2 * list () $sep $mailbox${sep}test3 * list () $sep $mailbox${sep}test3${sep}test4${sep}test5 3 ok list $mailbox${sep} * * list () $sep $mailbox${sep}test * list () $sep $mailbox${sep}test2 * list () $sep $mailbox${sep}test3 * list () $sep $mailbox${sep}test3${sep}test4${sep}test5 2 ok list "" $mailbox${sep}*test4 ! list () $sep $mailbox${sep}test ! list () $sep $mailbox${sep}test2 ! list () $sep $mailbox${sep}test3 * list () $sep $mailbox${sep}test3${sep}test4 ! list () $sep $mailbox${sep}test3${sep}test4${sep}test5 2 ok list "" $mailbox${sep}*test* * list () $sep $mailbox${sep}test * list () $sep $mailbox${sep}test2 * list () $sep $mailbox${sep}test3 * list () $sep $mailbox${sep}test3${sep}test4${sep}test5 2 ok list "" $mailbox${sep}%3${sep}% * list () $sep $mailbox${sep}test3${sep}test4 2 ok list "" $mailbox${sep}%3${sep}%4 * list () $sep $mailbox${sep}test3${sep}test4 2 ok list "" $mailbox${sep}%t*4 * list () $sep $mailbox${sep}test3${sep}test4 2 ok delete $mailbox${sep}test2 1 ok list "" $mailbox${sep}* ! list () $sep $mailbox${sep}test2 1 ok list "" INBOX * list () $inboxsep INBOX ================================================ FILE: tests/resources/imap-test/listext ================================================ state: auth capabilities: LIST-EXTENDED # get the separator ok list "" "" * list () $sep "" ok create $mailbox${sep} ok create $mailbox${sep}test1 ok create $mailbox${sep}test2${sep} ok create $mailbox${sep}test2${sep}test1 ok create $mailbox${sep}test3 ok create $mailbox${sep}test4${sep}test5 ok create $mailbox${sep}test4${sep}test52 ok create $mailbox${sep}test6 ok create $mailbox${sep}test7${sep}test7 ok create $mailbox${sep}test8${sep}test8 ok create $mailbox${sep}test8${sep}test9 ok subscribe $mailbox${sep}test1 ok subscribe $mailbox${sep}test2${sep}test1 ok subscribe $mailbox${sep}test6 ok subscribe $mailbox${sep}test7${sep}test7 ok subscribe $mailbox${sep}test8${sep}test8 ok subscribe $mailbox${sep}test8${sep}test9 ok delete $mailbox${sep}test6 # "" isn't a special case with LIST-EXTENDED #ok list () "" "" #! list () $sep "" #ok list "" ("") #! list () $sep "" #ok list "" "" return () #! list () $sep "" # test multiple patterns ok list "" ($mailbox${sep}*2 $mailbox${sep}test3) ! list () $sep $mailbox${sep}test1 * list () $sep $mailbox${sep}test2 ! list () $sep $mailbox${sep}test2${sep}test1 * list () $sep $mailbox${sep}test3 ! list () $sep $mailbox${sep}test4 ! list () $sep $mailbox${sep}test4${sep}test5 * list () $sep $mailbox${sep}test4${sep}test52 ! list () $sep $mailbox${sep}test6 # test errors bad list (imaptest) "" "" bad list (recursivematch) "" "" bad list (recursivematch remote) "" "" bad list "" "" return (imaptest) ok list (remote) "" % ok list (subscribed) "" $mailbox${sep}* * list (\subscribed) $sep $mailbox${sep}test1 * list (\subscribed) $sep $mailbox${sep}test2${sep}test1 #* list ($!unordered $!ban=\noselect \subscribed \nonexistent) $sep $mailbox${sep}test6 * list (\subscribed) $sep $mailbox${sep}test7${sep}test7 * list (\subscribed) $sep $mailbox${sep}test8${sep}test9 ok list (subscribed recursivematch) "" $mailbox${sep}test2% return (children) * list (\haschildren) $sep $mailbox${sep}test2 (childinfo (subscribed)) ! list (\subscribed) $sep $mailbox${sep}test2 ! list (\subscribed) $sep $mailbox${sep}test2${sep}test1 # don't test for \hasnochildren, because it may be \noinferiors instead ok list "" $mailbox${sep}% return (children subscribed) * list ($!unordered $!ban=\haschildren \subscribed) $sep $mailbox${sep}test1 * list ($!unordered $!ban=\subscribed \haschildren) $sep $mailbox${sep}test2 * list ($!unordered $!ban=\subscribed $!ban=\haschildren) $sep $mailbox${sep}test3 * list ($!unordered $!ban=\subscribed \haschildren) $sep $mailbox${sep}test4 * list ($!unordered $!ban=\subscribed \haschildren) $sep $mailbox${sep}test7 * list ($!unordered $!ban=\subscribed \haschildren) $sep $mailbox${sep}test8 ok list (subscribed recursivematch) "" $mailbox*test7 ! list () $sep $mailbox${sep}test7 * list (\subscribed) $sep $mailbox${sep}test7${sep}test7 ok list (subscribed recursivematch) "" $mailbox*test8 * list ($!unordered $!ban=\subscribed) $sep $mailbox${sep}test8 (childinfo ("subscribed")) * list (\subscribed) $sep $mailbox${sep}test8${sep}test8 ok list (subscribed recursivematch) "" ($mailbox${sep}test2) * list () $sep $mailbox${sep}test2 (childinfo ("subscribed")) ! list () $sep $mailbox${sep}test2${sep}test1 ok list (subscribed recursivematch) "" ($mailbox${sep}test2 $mailbox${sep}test2${sep}test1) * list () $sep $mailbox${sep}test2 (childinfo ("subscribed")) * list (\subscribed) $sep $mailbox${sep}test2${sep}test1 ok list "" ($mailbox${sep}test7 $mailbox${sep}test7${sep}test7) #This isn't really an error, although it's non-optimal: ! list (\nonexistent) $sep $mailbox${sep}test7 * list () $sep $mailbox${sep}test7${sep}test7 ================================================ FILE: tests/resources/imap-test/logout ================================================ state: nonauth connections: 2 1 ok noop ! bye 2 ok noop ! bye 1 ok logout * bye 2 ok noop 2 ok logout * bye ================================================ FILE: tests/resources/imap-test/move ================================================ capabilities: MOVE state: created # - assumes COPYUID is sent untagged before expunges. # - assumes MOVE to mailbox itself changes message UID ok append ok append ok append ok create ${mailbox}2 ok select ${mailbox}2 * 0 exists * ok [uidvalidity $uidvalidity_dest] * ok [uidnext $uidnext_dest1] # MOVE: ok select $mailbox ok fetch 1:* uid * 1 fetch (uid $uid1) * 2 fetch (uid $uid2) * 3 fetch (uid $uid3) ok move 1 ${mailbox}2 * ok [copyuid $uidvalidity_dest $uid1 $uidnext_dest1] * 1 expunge # UID MOVE: ok select ${mailbox}2 * 1 exists * ok [uidvalidity $uidvalidity_dest] * ok [uidnext $uidnext_dest2] ok select $mailbox * ok [uidvalidity $uidvalidity] * ok [uidnext $uidnext1] ok uid move $uid2 ${mailbox}2 * ok [copyuid $uidvalidity_dest $uid2 $uidnext_dest2] * 1 expunge # MOVE to same mailbox: #ok move 1 $mailbox #* ok [copyuid $uidvalidity $uid3 $uidnext1] #* 1 expunge #* 1 exists ================================================ FILE: tests/resources/imap-test/multiappend ================================================ capabilities: MULTIAPPEND state: created ok append $mailbox {{{ From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Subject: mail1 body1 }}} {{{ From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Subject: mail2 body2 }}} ok select $mailbox ok fetch 1:* body.peek[header.fields (subject)] * 1 fetch (body[header.fields (subject)] {{{ Subject: mail1 }}}) * 2 fetch (body[header.fields (subject)] {{{ Subject: mail2 }}}) ================================================ FILE: tests/resources/imap-test/mutf7 ================================================ state: auth # get the separator ok list "" "" * list () $sep $root ok create "$mailbox${sep}p&AOQA5A-" ok list "" "$mailbox${sep}p&AOQA5A-" * list () $sep "$mailbox${sep}p&AOQA5A-" ok status "$mailbox${sep}p&AOQA5A-" (messages) * status "$mailbox${sep}p&AOQA5A-" (messages 0) ================================================ FILE: tests/resources/imap-test/nil ================================================ messages: 1 ok search subject NIL * search 1 ok search body nil * search 1 ok list "" NIL ok store 1 +flags NIL ok fetch 1 flags * 1 fetch (flags (NIL)) ================================================ FILE: tests/resources/imap-test/nil.mbox ================================================ From user@domain Fri Feb 22 17:06:23 2008 Date: Sat, 24 Mar 2007 23:00:00 +0200 Subject: NiL nil ================================================ FILE: tests/resources/imap-test/notify ================================================ capabilities: notify qresync ok create ${mailbox}2 ok append ${mailbox}2 # # Check that initial STATUS notifications are sent when needed # ok notify set status (mailboxes ${mailbox}2 (MessageExpunge)) * status ${mailbox}2 (messages 1) ok notify set status (mailboxes ${mailbox}2 (MessageNew)) * status ${mailbox}2 (messages 1 uidnext $uidnext uidvalidity $uidvalidity) ok notify set status (mailboxes ${mailbox}2 (FlagChange)) * status ${mailbox}2 (uidvalidity $uidvalidity highestmodseq $hmodseq) ok notify set (mailboxes ${mailbox}2 (MessageNew MessageExpunge FlagChange)) ! status ${mailbox}2 () ================================================ FILE: tests/resources/imap-test/pipeline ================================================ state: auth tag1 create ${mailbox} tag2 create ${mailbox}2 tag3 create ${mailbox}3 tag4 create ${mailbox}4 tag1 ok tag2 ok tag3 ok tag4 ok tag1 status ${mailbox} (messages) tag2 status ${mailbox}2 (messages) tag3 status ${mailbox}3 (messages) tag4 status ${mailbox}4 (messages) * status ${mailbox} (messages 0) * status ${mailbox}2 (messages 0) * status ${mailbox}3 (messages 0) * status ${mailbox}4 (messages 0) tag1 ok tag2 ok tag3 ok tag4 ok ================================================ FILE: tests/resources/imap-test/search-addresses ================================================ messages: all # full address searching ok search from user-from@domain.org * search 1 2 3 4 6 7 ok search to user-to@domain.org * search 1 2 3 4 ok search cc user-cc@domain.org * search 1 2 3 4 5 ok search bcc user-bcc@domain.org * search 1 2 3 4 # realname searching ok search from realfrom * search 2 4 6 7 ok search to realto * search 2 4 ok search cc realcc * search 2 4 ok search bcc realbcc * search 2 4 # existence searches ok search header from "" * search 1 2 3 4 5 6 7 ok search header to "" * search 1 2 3 4 6 7 ok search header cc "" * search 1 2 3 4 5 ok search header bcc "" * search 1 2 3 4 # substring address searches #ok search from ser-fro #* search 1 2 3 4 5 6 7 #ok search to ser-t #* search 1 2 3 4 #ok search cc ser-c #* search 1 2 3 4 5 #ok search bcc ser-bc #* search 1 2 3 4 # substring realname searches #ok search from ealfro #* search 2 4 6 7 #ok search to ealt #* search 2 4 #ok search cc ealc #* search 2 4 #ok search bcc ealbc #* search 2 4 # multiple addresses ok search from user-from1 * search 5 ok search from user-from2 * search 5 # groups ok search to groupname * search 6 7 ok search to groupname2 * search 6 ok search to groupuser1 * search 6 ok search to groupuser2 * search 6 ok search to groupuser3 * search 6 ok search to groupuser4 * search ================================================ FILE: tests/resources/imap-test/search-addresses.mbox ================================================ From user@domain Fri Feb 22 17:06:23 2008 From: user-from@domain.org To: user-to@domain.org Cc: user-cc@domain.org Bcc: user-bcc@domain.org body From user@domain Fri Feb 22 17:06:23 2008 From: RealFrom To: RealTo Cc: RealCc Bcc: RealBcc body2 From user@domain Fri Feb 22 17:06:23 2008 From: To: Cc: Bcc: body3 From user@domain Fri Feb 22 17:06:23 2008 From: user-from@domain.org (RealFrom) To: user-to@domain.org (RealTo) Cc: user-cc@domain.org (RealCc) Bcc: user-bcc@domain.org (RealBcc) body4 From user@domain Fri Feb 22 17:06:23 2008 From: user-from1@domain.org, user-from2@domain.org Cc: user-cc@domain.org body5 From user@domain Fri Feb 22 17:06:23 2008 From: RealFrom To: groupname: groupuser1@domain.org, groupuser2@domain.org;, groupname2: groupuser3@domain.org; body6 From user@domain Fri Feb 22 17:06:23 2008 From: RealFrom To: groupname:; body7 ================================================ FILE: tests/resources/imap-test/search-body ================================================ messages: all # search full words first ok search text asdfghjkl * search 1 2 ok search text zxcvbnm * search 1 3 4 ok search text qwertyuiop * search 2 4 ok search body asdfghjkl * search 1 2 ok search body zxcvbnm * search 1 3 4 ok search body qwertyuiop * search 4 # search substrings #ok search text sdfghjk #* search 1 2 #ok search text xcvbn #* search 1 3 4 #ok search text wertyuio #* search 2 3 4 #ok search body sdfghjk #* search 1 2 #ok search body xcvbn #* search 1 3 4 #ok search body wertyuio #* search 4 ================================================ FILE: tests/resources/imap-test/search-body.mbox ================================================ From user@domain Fri Feb 22 17:06:23 2008 Date: Sat, 24 Mar 2007 23:00:00 +0200 asdfghjkl zxcvbnm From user@domain Fri Feb 22 17:06:23 2008 Date: Sat, 24 Mar 2007 23:00:00 +0200 Subject: qwertyuiop asdfghjkl From user@domain Fri Feb 22 17:06:23 2008 Date: Sat, 24 Mar 2007 23:00:00 +0200 X-Header: qwertyuiop zxcvbnm From user@domain Fri Feb 22 17:06:23 2008 Date: Sat, 24 Mar 2007 23:00:00 +0200 zxcvbnm qwertyuiop ================================================ FILE: tests/resources/imap-test/search-context-update ================================================ capabilities: CONTEXT=SEARCH state: created ok append ok append ok append ok append ok append ok select $mailbox ok search return (update) body body seen * esearch (tag $searchtag) ok store 1,2,4 +flags \seen * esearch (tag $searchtag) addto ($pos 1:2,4) ok store 1,2 -flags \seen * esearch (tag $searchtag) removefrom ($pos2 1:2) ok store 3:5 +flags \deleted ok expunge * esearch (tag $searchtag) removefrom ($pos3 $4) * $3 expunge * $4 expunge * $5 expunge ok append $mailbox (\seen) * 3 exists * esearch (tag $searchtag) addto ($pos4 3) ok cancelupdate "$searchtag" ok store 1 +flags \seen ! esearch (tag $searchtag) addto ($pos5 1) ================================================ FILE: tests/resources/imap-test/search-context-update2 ================================================ capabilities: CONTEXT=SEARCH messages: 5 ok store 1,2 +flags \seen ok search return (update all) or seen subject s22 * esearch (tag $searchtag) all 1:2 ok store 3:5 +flags \seen * esearch (tag $searchtag) addto ($pos 3:5) ok store 1:5 -flags \seen * esearch (tag $searchtag) removefrom ($pos2 1,3:5) ok store 3,4 +flags \seen * esearch (tag $searchtag) addto ($pos3 3:4) ok store 3 +flags \deleted ok expunge * 3 expunge * esearch (tag $searchtag) removefrom ($pos4 3) ================================================ FILE: tests/resources/imap-test/search-context-update3 ================================================ capabilities: CONTEXT=SEARCH messages: 6 ok store 1 +flags \deleted ok expunge ok fetch 1:5 uid * 1 fetch (uid $uid1) * 2 fetch (uid $uid2) * 3 fetch (uid $uid3) * 4 fetch (uid $uid4) * 5 fetch (uid $uid5) ok store 1,2 +flags \seen ok uid search return (update all) or seen subject s33 * esearch (tag $searchtag) uid all $uid1:$uid2 ok store 3:5 +flags \seen * esearch (tag $searchtag) uid addto ($pos $uid3:$uid5) ok store 1:5 -flags \seen * esearch (tag $searchtag) uid removefrom ($pos2 $uid1,$uid3:$uid5) ok uid store $uid3,$uid4 +flags \seen * esearch (tag $searchtag) uid addto ($pos3 $uid3:$uid4) ok store 3 +flags \deleted ok expunge * 3 expunge * esearch (tag $searchtag) uid removefrom ($pos4 $uid3) ================================================ FILE: tests/resources/imap-test/search-date ================================================ messages: all # 1) Timezone changes from EET +0200 -> EEST +0300 # 1a) SENTBEFORE ok search sentbefore 24-mar-2007 * search ok search sentbefore 25-mar-2007 * search 1 ok search sentbefore 27-mar-2007 * search 1 2 3 4 5 6 7 # 1b) SENTON ok search senton 23-mar-2007 * search ok search senton 24-mar-2007 * search 1 ok search senton 25-mar-2007 * search 2 3 4 5 6 ok search senton 26-mar-2007 * search 7 # 1c) SENTSINCE ok search 1:7 sentsince 24-mar-2007 * search 1 2 3 4 5 6 7 ok search 1:7 sentsince 25-mar-2007 * search 2 3 4 5 6 7 ok search 1:7 sentsince 26-mar-2007 * search 7 ok search 1:7 sentsince 27-mar-2007 * search # 2) Timezone changes from EEST +0300 -> EET +0200 # 2a) SENTBEFORE ok search 8:* sentbefore 27-oct-2007 * search ok search 8:* sentbefore 28-oct-2007 * search 8 ok search 8:* sentbefore 29-oct-2007 * search 8 9 10 11 12 13 14 15 ok search 8:* sentbefore 30-oct-2007 * search 8 9 10 11 12 13 14 15 16 # 2b) SENTON ok search 8:* senton 26-oct-2007 * search ok search 8:* senton 27-oct-2007 * search 8 ok search 8:* senton 28-oct-2007 * search 9 10 11 12 13 14 15 ok search 8:* senton 29-oct-2007 * search 16 # 2c) SENTSINCE ok search 8:* sentsince 27-oct-2007 * search 8 9 10 11 12 13 14 15 16 ok search 8:* sentsince 28-oct-2007 * search 9 10 11 12 13 14 15 16 ok search 8:* sentsince 29-oct-2007 * search 16 ok search 8:* sentsince 30-oct-2007 * search # 3) Try a couple of NOTs ok search 1:7 not sentbefore 26-mar-2007 * search 7 ok search 1:7 not senton 25-mar-2007 * search 1 7 ok search 8:* not sentsince 28-oct-2007 * search 8 ok search 8:* not senton 28-oct-2007 * search 8 16 ================================================ FILE: tests/resources/imap-test/search-date.mbox ================================================ From user@domain Sat Mar 24 23:00:00 2007 +0000 Date: Sat, 24 Mar 2007 23:00:00 +0000 body From user@domain Sat Mar 24 23:00:00 2007 +0000 Date: Sun, 25 Mar 2007 00:00:00 +0000 body From user@domain Sat Mar 24 23:00:00 2007 +0000 Date: Sun, 25 Mar 2007 01:00:00 +0000 body From user@domain Sat Mar 24 23:00:00 2007 +0000 Date: Sun, 25 Mar 2007 02:00:00 +0000 body From user@domain Sat Mar 24 23:00:00 2007 +0000 Date: Sun, 25 Mar 2007 04:00:00 +0000 body From user@domain Sat Mar 24 23:00:00 2007 +0000 Date: Sun, 25 Mar 2007 23:00:00 +0000 body From user@domain Sat Mar 24 23:00:00 2007 +0000 Date: Mon, 26 Mar 2007 00:00:00 +0000 body From user@domain Sat Mar 24 23:00:00 2007 +0000 Date: Sat, 27 Oct 2007 03:00:00 +0000 body From user@domain Sat Mar 24 23:00:00 2007 +0000 Date: Sun, 28 Oct 2007 00:00:00 +0000 body From user@domain Sat Mar 24 23:00:00 2007 +0000 Date: Sun, 28 Oct 2007 01:00:00 +0000 body From user@domain Sat Mar 24 23:00:00 2007 +0000 Date: Sun, 28 Oct 2007 02:00:00 +0000 body From user@domain Sat Mar 24 23:00:00 2007 +0000 Date: Sun, 28 Oct 2007 03:00:00 +0000 body From user@domain Sat Mar 24 23:00:00 2007 +0000 Date: Sun, 28 Oct 2007 03:00:00 +0000 body From user@domain Sat Mar 24 23:00:00 2007 +0000 Date: Sun, 28 Oct 2007 04:00:00 +0000 body From user@domain Sat Mar 24 23:00:00 2007 +0000 Date: Sun, 28 Oct 2007 23:00:00 +0000 body From user@domain Sat Mar 24 23:00:00 2007 +0000 Date: Mon, 29 Oct 2007 00:00:00 +0000 body ================================================ FILE: tests/resources/imap-test/search-flags ================================================ messages: 5 ok store 1 flags ($$hello) ok store 2 flags (\seen \flagged) ok store 3 flags (\answered $$hello) ok store 4 flags (\flagged \draft) ok store 5 flags (\deleted \answered) ok search answered * search 3 5 ok search unanswered * search 1 2 4 ok search deleted * search 5 ok search undeleted * search 1 2 3 4 ok search draft * search 4 ok search undraft * search 1 2 3 5 ok search flagged * search 2 4 ok search unflagged * search 1 3 5 ok search seen * search 2 ok search unseen * search 1 3 4 5 ok search keyword $$hello * search 1 3 ok search unkeyword $$hello * search 2 4 5 #ok search new #* search 1 3 4 5 #ok search old #* search #ok search recent #* search 1 2 3 4 5 ok store 1:* flags (\seen) ok store 2 +flags (\flagged) ok search seen flagged * search 2 ok search seen not flagged * search 1 3 4 5 ok search not seen flagged * search ok search not seen not flagged * search ok store 1:* flags (\deleted) ok store 2 +flags (\flagged) ok search deleted flagged * search 2 ok search deleted not flagged * search 1 3 4 5 ok search not deleted flagged * search ok search not deleted not flagged * search ok store 1:* flags (\seen \deleted) ok store 2 +flags (\flagged) ok search seen flagged * search 2 ok search seen not flagged * search 1 3 4 5 ok search not seen flagged * search ok search not seen not flagged * search ok search seen deleted flagged * search 2 ok search seen deleted not flagged * search 1 3 4 5 ok search not seen deleted flagged * search ok search not seen deleted not flagged * search ok search seen not deleted flagged * search ok search seen not deleted not flagged * search ok search not seen not deleted flagged * search ok search not seen not deleted not flagged * search ================================================ FILE: tests/resources/imap-test/search-header ================================================ messages: all # just check that this returns ok. it's not really specified in RFC, so # don't verify the result. ok search subject "" # subject ok search subject hello * search 1 ok search subject beautiful * search 1 ok search subject world * search 1 ok search subject "hello beautiful" * search 1 ok search subject "hello beautiful world" * search 1 #ok search subject "eautiful worl" #* search 1 # header ok search header subject "" * search 1 ok search not header subject "" * search 2 #ok search header x-extra "" #* search 2 #ok search not header x-extra "" #* search 1 #ok search header x-extra hello #* search 2 #ok search header x-extra "hello beautiful" #* search 2 #ok search header x-extra "eautiful head" #* search 2 #ok search header x-extra "another" #* search 2 ================================================ FILE: tests/resources/imap-test/search-header.mbox ================================================ From user@domain Fri Feb 22 17:06:23 2008 Subject: hello beautiful world body From user@domain Fri Feb 22 17:06:23 2008 X-Extra: hello beautiful header X-Extra: another one body ================================================ FILE: tests/resources/imap-test/search-partial ================================================ capabilities: CONTEXT=SEARCH messages: all ok search return (partial 1:1) all * esearch (tag $tag) partial (1:1 1) ok search return (partial 2:4) all * esearch (tag $tag) partial (2:4 2:4) ok search return (partial 4:2) all * esearch (tag $tag) partial (2:4 2:4) ok search return (partial 1:6) all * esearch (tag $tag) partial (1:6 1:5) ok search return (partial 6:6) all * esearch (tag $tag) partial (6:6 nil) ok search return (partial 1:3) 1:2,4:5 * esearch (tag $tag) partial (1:3 1:2,4) ok search return (partial 2:3) 1:2,4:5 * esearch (tag $tag) partial (2:3 2,4) ok search return (partial 2:4) 1:2,4:5 * esearch (tag $tag) partial (2:4 2,4:5) ok search return (partial 2:10) 1:2,4:5 * esearch (tag $tag) partial (2:10 2,4:5) # UID partials ok fetch 1 uid * 1 fetch (uid $uid1) ok fetch 2 uid * 2 fetch (uid $uid2) ok fetch 3 uid * 3 fetch (uid $uid3) ok fetch 5 uid * 5 fetch (uid $uid5) ok uid search return (partial 1:1) all * esearch (tag $tag) uid partial (1:1 $uid1) ok uid search return (partial 2:2) all * esearch (tag $tag) uid partial (2:2 $uid2) ok store 2 +flags \deleted ok expunge ok uid search return (partial 2:2) all * esearch (tag $tag) uid partial (2:2 $uid3) ok uid search return (partial 5:10) all * esearch (tag $tag) uid partial (5:10 nil) # broken results bad search return (partial 1) all bad search return (partial 1:*) all bad search return (partial *:1) all ================================================ FILE: tests/resources/imap-test/search-partial.mbox ================================================ From user@domain Fri Feb 22 17:06:21 2008 1 From user@domain Fri Feb 22 17:06:22 2008 2 From user@domain Fri Feb 22 17:06:23 2008 3 From user@domain Fri Feb 22 17:06:23 2008 4 From user@domain Fri Feb 22 17:06:25 2008 5 ================================================ FILE: tests/resources/imap-test/search-sets ================================================ messages: 6 ok uid search all * search $uid1 $uid2 $uid3 $uid4 $uid5 $uid6 ok status $mailbox (uidnext) * status $mailbox (uidnext $uidnext) # break seq=uid mapping ok store 2 +flags \deleted ok expunge * 2 expunge ok search all * search 1 2 3 4 5 ok uid search all * search $uid1 $uid3 $uid4 $uid5 $uid6 ok search 1:3,5 * search 1 2 3 5 ok search 4:2 * search 2 3 4 ok search uid $uid1:$uid3,$uid5 * search 1 2 4 ok search uid $uid4:$uid2 * search 2 3 ok search 1:3 not uid $uid3 * search 1 3 ok search not 2,4 * search 1 3 5 ok search or 1 uid $uid3 * search 1 2 ok search * * search 5 ok search uid * * search 5 ok search uid $uidnext:* * search 5 ok search *:3 * search 3 4 5 ok search 1,4,* * search 1 4 5 # These are in a bit of a grey area. Most servers allow them, but it's not # explicitly defined in the RFC that they're legal: #ok search 6:* #* search 5 #ok search *:6 #* search 5 ok search (3) uid $uid4 * search 3 ok search (uid $uid4) 3 * search 3 ok search 3 (uid $uid4) * search 3 ok uid search uid 1:4294967295 * search $uid1 $uid3 $uid4 $uid5 $uid6 ok uid search uid $uidnext:4294967295 * search ================================================ FILE: tests/resources/imap-test/search-size ================================================ messages: all # get the middle size. don't trust precalculated values in case server # modifies the message while APPENDing it. Messages 3 and 4 have different # RFC822.SIZE, but with servers that store linefeeds as LFs they have the # same file size. This test catches if they search using file size. ok fetch 1:4 rfc822.size * 3 fetch (rfc822.size $size) ok search smaller $size * search 1 2 ok search larger $size * search 4 ok search not smaller $size * search 3 4 ok search not larger $size * search 1 2 3 ok search not smaller $size not larger $size * search 3 ok search or smaller $size larger $size * search 1 2 4 ================================================ FILE: tests/resources/imap-test/search-size.mbox ================================================ From user@domain Fri Feb 22 17:06:23 2008 Date: Sat, 24 Mar 2007 23:00:00 +0200 1 From user@domain Fri Feb 22 17:06:23 2008 Date: Sat, 24 Mar 2007 23:00:00 +0200 22 From user@domain Fri Feb 22 17:06:23 2008 Date: Sat, 24 Mar 2007 23:00:00 +0200 333 From user@domain Fri Feb 22 17:06:23 2008 Date: Sat, 24 Mar 2007 23:00:00 +0200 4 4 ================================================ FILE: tests/resources/imap-test/select ================================================ state: created ok append ok append # first open read-only so recent flags don't get lost examine $mailbox * 2 exists #* 2 recent #* ok [unseen 1] * ok [uidvalidity $uidvalidity] * ok [uidnext $uidnext] ok [read-only] ok close # check that STATUS replies with the same values ok status $mailbox (messages uidnext uidvalidity unseen) * status $mailbox (messages 2 uidnext $uidnext uidvalidity $uidvalidity unseen 2) # then try read-write select $mailbox * 2 exists #* 2 recent #* ok [unseen 1] * ok [uidvalidity $uidvalidity] * ok [uidnext $uidnext] ok [read-write] ok close #ok status $mailbox (recent) #* status $mailbox (recent 0) ================================================ FILE: tests/resources/imap-test/select.mbox ================================================ From user@domain Fri Feb 22 17:06:23 2008 Date: Sat, 24 Mar 2007 23:00:00 +0200 body From user@domain Fri Feb 22 17:06:23 2008 Date: Sat, 24 Mar 2007 23:00:00 +0200 body2 ================================================ FILE: tests/resources/imap-test/sort-addresses ================================================ capabilities: SORT messages: all ok sort (from) us-ascii all * sort 3 2 1 ok sort (to) us-ascii all * sort 3 2 1 ok sort (cc) us-ascii all * sort 2 1 3 ok sort (reverse from) us-ascii all * sort 1 2 3 ok sort (reverse to) us-ascii all * sort 1 2 3 ok sort (reverse cc) us-ascii all * sort 3 1 2 ok sort (from reverse arrival) us-ascii all * sort 3 2 1 ok sort (to reverse arrival) us-ascii all * sort 3 2 1 ok sort (cc reverse arrival) us-ascii all * sort 2 1 3 ================================================ FILE: tests/resources/imap-test/sort-addresses.mbox ================================================ From user@domain Fri Feb 22 00:06:23 2008 From: user2@domain2.org To: user2@domain2.org Cc: user2@domain2.org (foo bar) 1 From user@domain Fri Feb 22 01:06:23 2008 From: user2@domain1.org To: user2@domain1.org Cc: blah@domain1.org 2 From user@domain Fri Feb 22 02:06:23 2008 From: user1@domain1.org To: user1@domain1.org Cc: user1@domain1.org 3 ================================================ FILE: tests/resources/imap-test/sort-arrival ================================================ capabilities: SORT messages: all ok sort (arrival) us-ascii all * sort 1 3 2 4 5 ok sort (reverse arrival) us-ascii all * sort 5 4 2 3 1 ok sort (arrival reverse size) us-ascii all * sort 1 3 5 4 2 ================================================ FILE: tests/resources/imap-test/sort-arrival.mbox ================================================ From user@domain Fri Feb 22 00:00:00 2008 1 From user@domain Fri Feb 22 02:00:00 2008 2 From user@domain Fri Feb 22 01:00:00 2008 33 From user@domain Fri Feb 22 02:00:00 2008 44 From user@domain Fri Feb 22 02:00:00 2008 5555 ================================================ FILE: tests/resources/imap-test/sort-date ================================================ capabilities: SORT messages: all ok sort (date) us-ascii all * sort 1 3 7 5 2 4 6 ok sort (reverse date) us-ascii all * sort 6 4 2 5 7 3 1 ok sort (date reverse size) us-ascii all * sort 1 7 3 5 6 4 2 ================================================ FILE: tests/resources/imap-test/sort-date.mbox ================================================ From user@domain Fri Feb 22 23:00:00 2008 +0200 Date: Fri, 22 Feb 2008 00:00:00 +0200 1 From user@domain Fri Feb 22 22:00:00 2008 +0200 Date: Fri, 22 Feb 2008 02:00:00 +0200 2 From user@domain Fri Feb 22 21:00:00 2008 +0200 Date: Fri, 22 Feb 2008 01:00:00 +0200 33 From user@domain Fri Feb 22 20:00:00 2008 +0200 Date: Fri, 22 Feb 2008 02:00:00 +0200 44 From user@domain Fri Feb 22 01:30:23 2008 +0200 Date: Fri, 22 Feb 2008 01:30:23 +0200 Subject: foo 55555 From user@domain Fri Feb 22 18:00:00 2008 +0200 Date: Fri, 22 Feb 2008 02:00:00 +0200 6666 From user@domain Fri Feb 22 01:00:00 2008 +0200 Date: Fri, 22 Feb 2008 01:00:00 +0200 Subject: foo bar foo bar foo bar fooo 777 ================================================ FILE: tests/resources/imap-test/sort-display-from ================================================ capabilities: SORT=DISPLAY messages: all ok sort (displayfrom) us-ascii all * sort 2 3 4 1 5 ok sort (reverse displayfrom) us-ascii all * sort 5 1 4 3 2 ================================================ FILE: tests/resources/imap-test/sort-display-from.mbox ================================================ From user@domain Fri Feb 22 23:00:00 2008 +0200 From: foo bar 1 From user@domain Fri Feb 22 23:00:00 2008 +0200 From: b@c.d 2 From user@domain Fri Feb 22 23:00:00 2008 +0200 From: d 3 From user@domain Fri Feb 22 23:00:00 2008 +0200 From: =?iso-8859-1?q?foo_aar?= 4 From user@domain Fri Feb 22 23:00:00 2008 +0200 From: =?iso-8859-1?q?foo_car?= 5 ================================================ FILE: tests/resources/imap-test/sort-display-to ================================================ capabilities: SORT=DISPLAY messages: all ok sort (displayto) us-ascii all * sort 8 7 2 3 4 1 5 6 ok sort (reverse displayto) us-ascii all * sort 6 5 1 4 3 2 7 8 ================================================ FILE: tests/resources/imap-test/sort-display-to.mbox ================================================ From user@domain Fri Feb 22 23:00:00 2008 +0200 To: foo bar 1 From user@domain Fri Feb 22 23:00:00 2008 +0200 To: b@c.d 2 From user@domain Fri Feb 22 23:00:00 2008 +0200 To: d 3 From user@domain Fri Feb 22 23:00:00 2008 +0200 To: =?iso-8859-1?q?foo_aar?= 4 From user@domain Fri Feb 22 23:00:00 2008 +0200 To: =?iso-8859-1?q?foo_car?= 5 From user@domain Fri Feb 22 23:00:00 2008 +0200 To: a: xx@xx.org, yy@yy.org; 6 From user@domain Fri Feb 22 23:00:00 2008 +0200 To: z: aa@aa.org, bb@bb.org; 7 From user@domain Fri Feb 22 23:00:00 2008 +0200 To: empty:; 8 ================================================ FILE: tests/resources/imap-test/sort-partial ================================================ capabilities: CONTEXT=SORT messages: all ok sort return (partial 1:1) (arrival) us-ascii all * esearch (tag $tag) partial (1:1 1) ok sort return (partial 2:4) (arrival) us-ascii all * esearch (tag $tag) partial (2:4 2:4) ok sort return (partial 4:2) (arrival) us-ascii all * esearch (tag $tag) partial (2:4 2:4) ok sort return (partial 2:4) (reverse arrival) us-ascii all * esearch (tag $tag) partial (2:4 3:4,2) ok sort return (partial 1:6) (arrival) us-ascii all * esearch (tag $tag) partial (1:6 1:5) ok sort return (partial 1:6) (reverse arrival) us-ascii all * esearch (tag $tag) partial (1:6 5,3:4,2,1) bad sort return (partial 1) (arrival) us-ascii all bad sort return (partial 1:*) (arrival) us-ascii all bad sort return (partial *:1) (arrival) us-ascii all ================================================ FILE: tests/resources/imap-test/sort-partial.mbox ================================================ From user@domain Fri Feb 22 17:06:21 2008 1 From user@domain Fri Feb 22 17:06:22 2008 2 From user@domain Fri Feb 22 17:06:23 2008 3 From user@domain Fri Feb 22 17:06:23 2008 4 From user@domain Fri Feb 22 17:06:25 2008 5 ================================================ FILE: tests/resources/imap-test/sort-size ================================================ capabilities: SORT messages: all ok sort (size) us-ascii all * sort 1 8 2 6 3 5 4 7 ok sort (reverse size) us-ascii all * sort 7 4 5 3 6 2 8 1 ok sort (size reverse arrival) us-ascii all * sort 8 1 6 2 5 3 7 4 ================================================ FILE: tests/resources/imap-test/sort-size.mbox ================================================ From user@domain Fri Feb 22 00:06:23 2008 Date: Sat, 24 Mar 2007 23:00:00 +0200 1 From user@domain Fri Feb 22 01:06:23 2008 Date: Sat, 24 Mar 2007 23:00:00 +0200 22 From user@domain Fri Feb 22 02:06:23 2008 Date: Sat, 24 Mar 2007 23:00:00 +0200 333 From user@domain Fri Feb 22 03:06:23 2008 Date: Sat, 24 Mar 2007 23:00:00 +0200 4 4 From user@domain Fri Feb 22 04:06:23 2008 Date: Sat, 24 Mar 2007 23:00:00 +0200 555 From user@domain Fri Feb 22 05:06:23 2008 Date: Sat, 24 Mar 2007 23:00:00 +0200 66 From user@domain Fri Feb 22 06:06:23 2008 Date: Sat, 24 Mar 2007 23:00:00 +0200 7777 From user@domain Fri Feb 22 07:06:23 2008 Date: Sat, 24 Mar 2007 23:00:00 +0200 8 ================================================ FILE: tests/resources/imap-test/sort-subject ================================================ capabilities: SORT messages: all ok sort (subject) us-ascii all * sort 9 10 14 2 12 15 8 4 1 3 5 11 6 7 13 ok sort (reverse subject) us-ascii all * sort 13 7 6 11 5 3 1 4 8 15 12 2 14 10 9 ok sort (subject reverse size) us-ascii all * sort 10 9 14 12 2 15 8 4 1 11 5 3 6 13 7 ================================================ FILE: tests/resources/imap-test/sort-subject.mbox ================================================ From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: a 1 From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: C 2 From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: b 3 From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: _ 4 From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: [foo] Fwd: [bar] Re: fw: b (fWd) (fwd) 55 From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: b (a) 66 From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: Re: [FWD: c] 77 From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: [xyz] 88 From user@domain Fri Feb 22 23:00:00 2008 +0200 9 From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: =?iso-8859-1?q?_?= 10 From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: Re: =?utf-8?q?b?= 11 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: =?iso-8859-1?q?RE:_C?= 12 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: =?us-ascii?b?UmU6IGM=?= 13 The subject is 'Re: c'. xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: Ad: Re: Ad: Re: Ad: x 14 From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: re: [fwd: [fwd: re: [fwd: babylon]]] 15 ================================================ FILE: tests/resources/imap-test/store ================================================ connections: 2 messages: 4 # simple tests 1 ok store 1:3 flags (\seen) * 1 fetch (flags (\seen)) * 2 fetch (flags (\seen)) * 3 fetch (flags (\seen)) 2 ok check * 1 fetch (flags (\seen)) * 2 fetch (flags (\seen)) * 3 fetch (flags (\seen)) 1 ok store 4 -flags \seen 1 ok store 3 flags (\draft) * 3 fetch (flags (\draft)) # keywords 1 ok store 2,4 +flags ($$hello $$world) * 2 fetch (flags (\seen $$hello $$world)) * 4 fetch (flags ($$hello $$world)) # check that two sessions don't overwrite each others' changes 1 ok store 1 +flags (\answered) 2 ok store 1 -flags (\seen) 1 ok check 2 ok check 1 ok fetch 1 flags * 1 fetch (flags (\answered)) 2 ok fetch 1 flags * 1 fetch (flags (\answered)) ================================================ FILE: tests/resources/imap-test/subscribe ================================================ connections: 2 state: auth # get the separator 1 ok list "" "" * list () $sep $root 1 "" unsubscribe $mailbox${sep}test 1 "" unsubscribe $mailbox${sep}test2 1 "" unsubscribe $mailbox${sep}test2${sep}test 1 "" unsubscribe $mailbox${sep}test3${sep}test3 1 ok create $mailbox${sep} 1 ok create $mailbox${sep}test 1 ok create $mailbox${sep}test2${sep} 1 ok create $mailbox${sep}test3${sep} 1 ok create $mailbox${sep}test2${sep}test 1 ok create $mailbox${sep}test3${sep}test3 # create the test2 mailbox only if server supports inferior mailboxes 1 "" create $mailbox${sep}test2 1 ok subscribe $mailbox${sep}test 1 ok subscribe $mailbox${sep}test2${sep}test 1 ok subscribe $mailbox${sep}test3${sep}test3 2 ok lsub "" $mailbox${sep}% * lsub () $sep $mailbox${sep}test #* lsub (\noselect) $sep $mailbox${sep}test2 #* lsub (\noselect) $sep $mailbox${sep}test3 ! lsub (\noselect) $sep $mailbox${sep}test2${sep}test ! lsub (\noselect) $sep $mailbox${sep}test3${sep}test3 2 ok lsub "" *test * lsub () $sep $mailbox${sep}test * lsub () $sep $mailbox${sep}test2${sep}test ! lsub () $sep $mailbox${sep}test3${sep}test3 1 ok unsubscribe $mailbox${sep}test 2 ok lsub "" $mailbox${sep}% ! lsub () $sep $mailbox${sep}test 1 ok unsubscribe $mailbox${sep}test2${sep}test 2 ok lsub "" $mailbox${sep}* ! lsub () $sep $mailbox${sep}test * lsub () $sep $mailbox${sep}test3${sep}test3 1 ok unsubscribe $mailbox${sep}test3${sep}test3 2 ok lsub "" $mailbox${sep}% ! lsub () $sep $mailbox${sep}test3 ================================================ FILE: tests/resources/imap-test/thread ================================================ capabilities: THREAD=REFERENCES messages: all ok thread references us-ascii all * thread (1 2 3) ok store 1 +flags \deleted ok expunge ok thread references us-ascii all * thread (1 2) ================================================ FILE: tests/resources/imap-test/thread-orderedsubject ================================================ capabilities: THREAD=ORDEREDSUBJECT messages: all ok thread orderedsubject us-ascii all * THREAD (1)(2 (7)(12)(13))(3 (5)(11))(4)(6)(8)(9 10)(14)(15) ================================================ FILE: tests/resources/imap-test/thread-orderedsubject.mbox ================================================ From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: a 1 From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: C 2 From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: b 3 From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: _ 4 From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: [foo] Fwd: [bar] Re: fw: b (fWd) (fwd) 55 From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: b (a) 66 From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: Re: [FWD: c] 77 From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: [xyz] 88 From user@domain Fri Feb 22 23:00:00 2008 +0200 9 From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: =?iso-8859-1?q?_?= 10 From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: Re: =?utf-8?q?b?= 11 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: =?iso-8859-1?q?RE:_C?= 12 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: =?us-ascii?b?UmU6IGM=?= 13 The subject is 'Re: c'. xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: Ad: Re: Ad: Re: Ad: x 14 From user@domain Fri Feb 22 23:00:00 2008 +0200 Subject: re: [fwd: [fwd: re: [fwd: babylon]]] 15 ================================================ FILE: tests/resources/imap-test/thread-orderedsubject2 ================================================ capabilities: THREAD=ORDEREDSUBJECT messages: all ok thread orderedsubject us-ascii all * THREAD (4 (2)(8)(6))(3 (1)(7)(5)) ================================================ FILE: tests/resources/imap-test/thread-orderedsubject2.mbox ================================================ From user@domain Fri Feb 22 23:00:01 2008 +0200 Subject: a Date: Fri, 22 Feb 2008 22:00:10 +0200 1 From user@domain Fri Feb 22 23:00:02 2008 +0200 Date: Fri, 22 Feb 2008 22:00:09 +0200 Subject: b 2 From user@domain Fri Feb 22 23:00:03 2008 +0200 Date: Fri, 22 Feb 2008 22:00:08 +0200 Subject: a 3 From user@domain Fri Feb 22 23:00:04 2008 +0200 Date: Fri, 22 Feb 2008 22:00:07 +0200 Subject: b 4 From user@domain Fri Feb 22 22:00:30 2008 +0200 Subject: a 5 From user@domain Fri Feb 22 22:00:29 2008 +0200 Subject: b 6 From user@domain Fri Feb 22 22:00:28 2008 +0200 Subject: a 7 From user@domain Fri Feb 22 22:00:27 2008 +0200 Subject: b 8 ================================================ FILE: tests/resources/imap-test/thread.mbox ================================================ From user@domain Fri Feb 22 17:06:23 2008 Message-ID: body From user@domain Fri Feb 22 15:06:23 2008 Message-Id: References: body From user@domain Fri Feb 22 16:06:25 2008 Message-Id: References: body ================================================ FILE: tests/resources/imap-test/thread2 ================================================ capabilities: THREAD=REFERENCES messages: all ok thread references us-ascii all * thread (1)(2) ================================================ FILE: tests/resources/imap-test/thread2.mbox ================================================ From user@domain Fri Feb 22 15:06:23 2008 Message-Id: body From user@domain Fri Feb 22 15:06:23 2008 Message-Id: body ================================================ FILE: tests/resources/imap-test/thread3 ================================================ capabilities: THREAD=REFERENCES messages: all ok thread references us-ascii all * THREAD (1 2) ================================================ FILE: tests/resources/imap-test/thread3.mbox ================================================ From user@domain Fri Feb 22 15:06:23 2008 Message-Id: Subject: foo body From user@domain Fri Feb 22 15:06:24 2008 Message-Id: Subject: Re: foo body ================================================ FILE: tests/resources/imap-test/thread4 ================================================ capabilities: THREAD=REFERENCES messages: all ok thread references us-ascii all * THREAD (1 2) ================================================ FILE: tests/resources/imap-test/thread4.mbox ================================================ From user@domain Fri Feb 22 15:06:23 2008 Message-Id: Subject: foo body From user@domain Fri Feb 22 15:06:24 2008 Message-Id: References: Subject: Re: foo body ================================================ FILE: tests/resources/imap-test/thread5 ================================================ capabilities: THREAD=REFERENCES state: created ok append ok append ok append ok append ok select $mailbox ok thread references us-ascii all * THREAD (1 2 3 4) ok store 1,2 +flags \deleted ok expunge ok thread references us-ascii all * THREAD (1 2) ok append ok thread references us-ascii all * THREAD (1 2 3) ================================================ FILE: tests/resources/imap-test/thread5.mbox ================================================ From user@domain Fri Feb 22 17:06:23 2008 Message-ID: <1@b> body From user@domain Fri Feb 22 15:06:23 2008 Message-Id: <2@b> In-Reply-To: <1@b> body From user@domain Fri Feb 22 15:06:23 2008 Message-Id: <3@b> In-Reply-To: <1@b> body From user@domain Fri Feb 22 15:06:23 2008 Message-Id: <4@b> In-Reply-To: <2@b> body From user@domain Fri Feb 22 15:06:23 2008 Message-Id: <5@b> In-Reply-To: <2@b> body ================================================ FILE: tests/resources/imap-test/thread6 ================================================ capabilities: THREAD=REFERENCES messages: all ok thread references us-ascii all * THREAD (1 2) ================================================ FILE: tests/resources/imap-test/thread6.mbox ================================================ From user@domain Fri Feb 22 15:06:23 2008 References: body From user@domain Fri Feb 22 15:06:24 2008 References: body ================================================ FILE: tests/resources/imap-test/thread7 ================================================ capabilities: THREAD=REFERENCES messages: all ok thread references us-ascii all * thread (1 2 3 4) ok store 2 +flags \deleted ok expunge ok thread references us-ascii all * thread (1 2 3) ================================================ FILE: tests/resources/imap-test/thread7.mbox ================================================ From user@domain Fri Feb 22 15:06:23 2008 Message-Id: body1 From user@domain Fri Feb 22 15:06:23 2008 Message-Id: References: body2 From user@domain Fri Feb 22 15:06:23 2008 Message-Id: References: body3 From user@domain Fri Feb 22 15:06:23 2008 Message-Id: References: body4 (2 duplicate) ================================================ FILE: tests/resources/imap-test/thread8 ================================================ capabilities: THREAD=REFERENCES state: created ok append ok append ok select $mailbox ok thread references us-ascii all * thread (1 2) ok store 2 +flags \deleted ok expunge ok thread references us-ascii all * thread (1) ok append ok thread references us-ascii all * thread (1 2) ================================================ FILE: tests/resources/imap-test/thread8.mbox ================================================ From user@domain Fri Feb 22 15:06:23 2008 Message-Id: body1 From user@domain Fri Feb 22 15:06:23 2008 Message-Id: References: body2 From user@domain Fri Feb 22 15:06:23 2008 Message-Id: References: body3 ================================================ FILE: tests/resources/imap-test/uidplus ================================================ capabilities: UIDPLUS state: created ok select ${mailbox} ok append "" delete ${mailbox}2 ok create ${mailbox}2 append ${mailbox}2 ok [appenduid $uidvalidity $uid] copy 1 ${mailbox}2 ok [copyuid $uidvalidity $srcuid $uid2] ok select ${mailbox}2 * ok [uidvalidity $uidvalidity] ok fetch 1:2 uid * 1 fetch (uid $uid) * 2 fetch (uid $uid2) ok close "" delete ${mailbox}2 ================================================ FILE: tests/resources/imap-test/uidvalidity ================================================ connections: 2 state: created 1 ok append 1 ok status $mailbox (uidvalidity uidnext) * status $mailbox (uidvalidity $uidvalidity uidnext $uidnext) 1 ok delete $mailbox 2 ok create $mailbox 2 ok append $mailbox #1 ok status $mailbox (uidvalidity uidnext) #! status $mailbox (uidvalidity $uidvalidity uidnext $uidnext) ================================================ FILE: tests/resources/imap-test/uidvalidity-rename ================================================ connections: 2 state: auth 1 ok create ${mailbox} 2 ok create ${mailbox}2 1 ok append 2 ok append 1 ok status $mailbox (uidvalidity uidnext) * status $mailbox (uidvalidity $uidvalidity uidnext $uidnext) 1 ok rename ${mailbox} ${mailbox}3 2 ok rename ${mailbox}2 ${mailbox} #1 ok status $mailbox (uidvalidity uidnext) #! status $mailbox (uidvalidity $uidvalidity uidnext $uidnext) 1 "" delete ${mailbox}3 ================================================ FILE: tests/resources/imap-test/urlauth ================================================ capabilities: urlauth messages: 2 ok fetch 1:2 (uid body[]) * 1 fetch (uid $uid1 body[] $body1) * 2 fetch (uid $uid2 body[] $body2) ok GENURLAUTH "imap://$username@$domain/$mailbox_url/;uid=$uid1;urlauth=user+$user" INTERNAL * GENURLAUTH $mail_url1 ok GENURLAUTH "imap://$username@$domain/$mailbox_url/;uid=$uid2;urlauth=user+$user" INTERNAL * GENURLAUTH $mail_url2 ok URLFETCH $mail_url1 * URLFETCH $mail_url1 $body1 ok URLFETCH $mail_url2 * URLFETCH $mail_url2 $body2 ok resetkey $mailbox ok URLFETCH $mail_url1 * URLFETCH $mail_url1 NIL ok URLFETCH $mail_url2 * URLFETCH $mail_url2 NIL ================================================ FILE: tests/resources/imap-test/urlauth-binary ================================================ capabilities: urlauth urlauth=binary state: created ok select ${mailbox} ok append ok append ok fetch 1:2 uid * 1 fetch (uid $uid1) * 2 fetch (uid $uid2) ok GENURLAUTH "imap://$username@$domain/$mailbox_url/;uid=$uid1/;section=1;urlauth=user+$user" INTERNAL * GENURLAUTH $mail_url1 ok GENURLAUTH "imap://$username@$domain/$mailbox_url/;uid=$uid1/;section=1.1;urlauth=user+$user" INTERNAL * GENURLAUTH $mail_url1sub ok GENURLAUTH "imap://$username@$domain/$mailbox_url/;uid=$uid2/;section=1;urlauth=user+$user" INTERNAL * GENURLAUTH $mail_url2 ok GENURLAUTH "imap://$username@$domain/$mailbox_url/;uid=$uid2/;section=1.1;urlauth=user+$user" INTERNAL * GENURLAUTH $mail_url2sub ok URLFETCH ($mail_url1 binary) * URLFETCH $mail_url1 (binary {{{ From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/alternative; boundary="sub1" Sub MIME prologue --sub1 Content-Type: text/x-myown; charset=us-ascii Content-Transfer-Encoding: binary hello world --sub1 Content-Type: text/x-myown; charset=us-ascii Content-Transfer-Encoding: binary hello world --sub1-- Sub MIME epilogue }}}) ok URLFETCH ($mail_url1sub binary) * URLFETCH $mail_url1sub (binary {{{ hello world }}}) ok URLFETCH ($mail_url1 bodypartstructure) * URLFETCH $mail_url1 (BODYPARTSTRUCTURE ("message" "rfc822" NIL NIL NIL "7bit" 437 ("Sun, 12 Aug 2012 12:34:56 +0300" "submsg" ((NIL NIL "sub" "domain.org")) ((NIL NIL "sub" "domain.org")) ((NIL NIL "sub" "domain.org")) NIL NIL NIL NIL NIL) (("text" "x-myown" ("charset" "us-ascii") NIL NIL "base64" 22 3 NIL NIL NIL NIL)("text" "x-myown" ("charset" "us-ascii") NIL NIL "base64" 47 5 NIL NIL NIL NIL) "alternative" ("boundary" "sub1") NIL NIL NIL) 26 NIL NIL NIL NIL)) ok URLFETCH ($mail_url1sub bodypartstructure) * URLFETCH $mail_url1sub (BODYPARTSTRUCTURE ("text" "x-myown" ("charset" "us-ascii") NIL NIL "base64" 22 3 NIL NIL NIL NIL)) ok URLFETCH ($mail_url2 binary bodypartstructure) * URLFETCH $mail_url2 (BODYPARTSTRUCTURE ("message" "rfc822" NIL NIL NIL "7bit" 390 ("Sun, 12 Aug 2012 12:34:56 +0300" "submsg" ((NIL NIL "sub" "domain.org")) ((NIL NIL "sub" "domain.org")) ((NIL NIL "sub" "domain.org")) NIL NIL NIL NIL NIL) (("text" "x-myown" ("charset" "us-ascii") NIL NIL "base64" 11 0 NIL NIL NIL NIL)("text" "x-myown" ("charset" "us-ascii") NIL NIL "base64" 11 0 NIL NIL NIL NIL) "alternative" ("boundary" "sub1") NIL NIL NIL) 18 NIL NIL NIL NIL) binary {{{ From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/alternative; boundary="sub1" Sub MIME prologue --sub1 Content-Type: text/x-myown; charset=us-ascii Content-Transfer-Encoding: binary hello world --sub1 Content-Type: text/x-myown; charset=us-ascii Content-Transfer-Encoding: binary hello world --sub1-- Sub MIME epilogue }}}) ok URLFETCH ($mail_url1sub binary) * URLFETCH $mail_url1sub (binary {{{ hello world }}}) ok URLFETCH ($mail_url2sub binary bodypartstructure) * URLFETCH $mail_url2sub (BODYPARTSTRUCTURE ("text" "x-myown" ("charset" "us-ascii") NIL NIL "base64" 11 0 NIL NIL NIL NIL) BINARY {{{ hello world }}}) ================================================ FILE: tests/resources/imap-test/urlauth-binary.mbox ================================================ From user@domain Fri Feb 22 17:06:23 2008 From: user@domain.org Date: Sat, 24 Mar 2007 23:00:00 +0200 Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="foo bar" Root MIME prologue --foo bar Content-Type: message/rfc822 From: sub@domain.org Date: Sun, 12 Aug 2012 12:34:56 +0300 Subject: submsg Content-Type: multipart/alternative; boundary="sub1" Sub MIME prologue --sub1 Content-Type: text/x-myown; charset=us-ascii Content-Transfer-Encoding: base64 aGVs bG8gd29y bGQ= --sub1 Content-Type: text/x-myown; charset=us-ascii Content-Transfer-Encoding: base64 aGVs bG8g d29y bGQ= --sub1-- Sub MIME epilogue --foo bar-- Root MIME epilogue ================================================ FILE: tests/resources/imap-test/urlauth2 ================================================ capabilities: urlauth connections: 2 user 2: $user2 1 ok fetch 1:2 (uid body[]) * 1 fetch (uid $uid1 body[] $body1) * 2 fetch (uid $uid2 body[] $body2) 1 ok GENURLAUTH "imap://$username@$domain/$mailbox_url/;uid=$uid1;urlauth=authuser" INTERNAL * GENURLAUTH $mail_url1 1 ok GENURLAUTH "imap://$username@$domain/$mailbox_url/;uid=$uid2;urlauth=authuser" INTERNAL * GENURLAUTH $mail_url2 2 ok URLFETCH $mail_url1 * URLFETCH $mail_url1 $body1 2 ok URLFETCH $mail_url2 * URLFETCH $mail_url2 $body2 1 ok resetkey $mailbox 2 ok URLFETCH $mail_url1 * URLFETCH $mail_url1 NIL 2 ok URLFETCH $mail_url2 * URLFETCH $mail_url2 NIL ================================================ FILE: tests/resources/itip/google_calendar.txt ================================================ # Send initial request > put a@gmail.com 1qnf39p0m9h6n1cm9qa9d8mocv@google.com BEGIN:VCALENDAR PRODID:-//Google Inc//Google Calendar 70.9054//EN VERSION:2.0 CALSCALE:GREGORIAN BEGIN:VTIMEZONE TZID:America/Los_Angeles X-LIC-LOCATION:America/Los_Angeles BEGIN:DAYLIGHT TZOFFSETFROM:-0800 TZOFFSETTO:-0700 TZNAME:PDT DTSTART:19700308T020000 RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU END:DAYLIGHT BEGIN:STANDARD TZOFFSETFROM:-0700 TZOFFSETTO:-0800 TZNAME:PST DTSTART:19701101T020000 RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU END:STANDARD END:VTIMEZONE BEGIN:VEVENT DTSTART;TZID=America/Los_Angeles:20250619T060000 DTEND;TZID=America/Los_Angeles:20250619T064500 RRULE:FREQ=WEEKLY;WKST=SU;COUNT=2;BYDAY=TH DTSTAMP:20250616T182403Z ORGANIZER;CN=John Doe:mailto:a@gmail.com UID:1qnf39p0m9h6n1cm9qa9d8mocv@google.com ATTENDEE;CUTYPE=INDIVIDUAL;ROLE=REQ-PARTICIPANT;PARTSTAT=NEEDS-ACTION;RSVP= TRUE;CN=b@gmail.com;X-NUM-GUESTS=0:mailto:b@gmail.com ATTENDEE;CUTYPE=INDIVIDUAL;ROLE=REQ-PARTICIPANT;PARTSTAT=ACCEPTED;RSVP=TRUE ;CN=John Doe;X-NUM-GUESTS=0:mailto:a@gmail.com X-MICROSOFT-CDO-OWNERAPPTID:299828133 CREATED:20250616T182358Z LAST-MODIFIED:20250616T182358Z LOCATION: SEQUENCE:0 STATUS:CONFIRMED SUMMARY:Meet me maybe DESCRIPTION:This is the event description TRANSP:OPAQUE BEGIN:VALARM ACTION:DISPLAY DESCRIPTION:This is an event reminder TRIGGER:-P0DT0H10M0S END:VALARM END:VEVENT END:VCALENDAR > expect from: a@gmail.com to: b@gmail.com summary: invite summary.attendee: Participants([ItipParticipant { email: "a@gmail.com", name: Some("John Doe"), is_organizer: true }, ItipParticipant { email: "b@gmail.com", name: Some("b@gmail.com"), is_organizer: false }]) summary.description: Text("This is the event description") summary.dtstart: Time(ItipTime { start: 1750338000, tz_id: 148 }) summary.rrule: Rrule(ICalendarRecurrenceRule { freq: Weekly, until: None, count: Some(2), interval: None, bysecond: [], byminute: [], byhour: [], byday: [ICalendarDay { ordwk: None, weekday: Thursday }], bymonthday: [], byyearday: [], byweekno: [], bymonth: [], bysetpos: [], wkst: Some(Sunday), rscale: None, skip: None }) summary.summary: Text("Meet me maybe") BEGIN:VCALENDAR METHOD:REQUEST PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT X-MICROSOFT-CDO-OWNERAPPTID:299828133 DESCRIPTION:This is the event description LOCATION: STATUS:CONFIRMED SUMMARY:Meet me maybe DTEND;TZID=America/Los_Angeles:20250619T064500 DTSTART;TZID=America/Los_Angeles:20250619T060000 TRANSP:OPAQUE ATTENDEE;CUTYPE=INDIVIDUAL;ROLE=REQ-PARTICIPANT;PARTSTAT=NEEDS-ACTION;RSVP=TRUE; CN=b@gmail.com;X-NUM-GUESTS=0:mailto:b@gmail.com ATTENDEE;CUTYPE=INDIVIDUAL;ROLE=REQ-PARTICIPANT;PARTSTAT=ACCEPTED;RSVP=TRUE; CN="John Doe";X-NUM-GUESTS=0:mailto:a@gmail.com ORGANIZER;CN="John Doe":mailto:a@gmail.com UID:1qnf39p0m9h6n1cm9qa9d8mocv@google.com RRULE:FREQ=WEEKLY;COUNT=2;BYDAY=TH;WKST=SU CREATED:20250616T182358Z DTSTAMP:0 LAST-MODIFIED:20250616T182358Z SEQUENCE:1 BEGIN:VALARM DESCRIPTION:This is an event reminder ACTION:DISPLAY TRIGGER:-PT10M END:VALARM END:VEVENT BEGIN:VTIMEZONE X-LIC-LOCATION:America/Los_Angeles TZID:America/Los_Angeles BEGIN:DAYLIGHT DTSTART:19700308T020000 TZNAME:PDT TZOFFSETFROM:-0800 TZOFFSETTO:-0700 RRULE:FREQ=YEARLY;BYDAY=2SU;BYMONTH=3 END:DAYLIGHT BEGIN:STANDARD DTSTART:19701101T020000 TZNAME:PST TZOFFSETFROM:-0700 TZOFFSETTO:-0800 RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=11 END:STANDARD END:VTIMEZONE END:VCALENDAR # Send iTIP > send # Update the event, expect no changes > put a@gmail.com 1qnf39p0m9h6n1cm9qa9d8mocv@google.com BEGIN:VCALENDAR PRODID:-//Google Inc//Google Calendar 70.9054//EN VERSION:2.0 CALSCALE:GREGORIAN BEGIN:VTIMEZONE TZID:America/Los_Angeles X-LIC-LOCATION:America/Los_Angeles BEGIN:DAYLIGHT TZOFFSETFROM:-0800 TZOFFSETTO:-0700 TZNAME:PDT DTSTART:19700308T020000 RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU END:DAYLIGHT BEGIN:STANDARD TZOFFSETFROM:-0700 TZOFFSETTO:-0800 TZNAME:PST DTSTART:19701101T020000 RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU END:STANDARD END:VTIMEZONE BEGIN:VEVENT DTSTART;TZID=America/Los_Angeles:20250619T060000 DTEND;TZID=America/Los_Angeles:20250619T064500 RRULE:FREQ=WEEKLY;WKST=SU;COUNT=2;BYDAY=TH DTSTAMP:20250616T182416Z ORGANIZER;CN=John Doe:mailto:a@gmail.com UID:1qnf39p0m9h6n1cm9qa9d8mocv@google.com ATTENDEE;CUTYPE=INDIVIDUAL;ROLE=REQ-PARTICIPANT;PARTSTAT=NEEDS-ACTION;RSVP= TRUE;CN=b@gmail.com;X-NUM-GUESTS=0:mailto:b@gmail.com ATTENDEE;CUTYPE=INDIVIDUAL;ROLE=REQ-PARTICIPANT;PARTSTAT=ACCEPTED;RSVP=TRUE ;CN=John Doe;X-NUM-GUESTS=0:mailto:a@gmail.com X-MICROSOFT-CDO-OWNERAPPTID:299828133 CREATED:20250616T182358Z LAST-MODIFIED:20250616T182415Z LOCATION: SEQUENCE:1 STATUS:CONFIRMED SUMMARY:Meet me maybe DESCRIPTION:This is the updated event description TRANSP:OPAQUE BEGIN:VALARM ACTION:DISPLAY DESCRIPTION:This is an event reminder TRIGGER:-P0DT0H10M0S END:VALARM END:VEVENT END:VCALENDAR > expect from: a@gmail.com to: b@gmail.com summary: update REQUEST summary.attendee: Participants([ItipParticipant { email: "a@gmail.com", name: Some("John Doe"), is_organizer: true }, ItipParticipant { email: "b@gmail.com", name: Some("b@gmail.com"), is_organizer: false }]) summary.description: Text("This is the updated event description") summary.dtstart: Time(ItipTime { start: 1750338000, tz_id: 148 }) summary.rrule: Rrule(ICalendarRecurrenceRule { freq: Weekly, until: None, count: Some(2), interval: None, bysecond: [], byminute: [], byhour: [], byday: [ICalendarDay { ordwk: None, weekday: Thursday }], bymonthday: [], byyearday: [], byweekno: [], bymonth: [], bysetpos: [], wkst: Some(Sunday), rscale: None, skip: None }) summary.summary: Text("Meet me maybe") ~summary.description: Text("This is the event description") BEGIN:VCALENDAR METHOD:REQUEST PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT X-MICROSOFT-CDO-OWNERAPPTID:299828133 DESCRIPTION:This is the updated event description LOCATION: STATUS:CONFIRMED SUMMARY:Meet me maybe DTEND;TZID=America/Los_Angeles:20250619T064500 DTSTART;TZID=America/Los_Angeles:20250619T060000 TRANSP:OPAQUE ATTENDEE;CUTYPE=INDIVIDUAL;ROLE=REQ-PARTICIPANT;PARTSTAT=NEEDS-ACTION;RSVP=TRUE; CN=b@gmail.com;X-NUM-GUESTS=0:mailto:b@gmail.com ATTENDEE;CUTYPE=INDIVIDUAL;ROLE=REQ-PARTICIPANT;PARTSTAT=ACCEPTED;RSVP=TRUE; CN="John Doe";X-NUM-GUESTS=0:mailto:a@gmail.com ORGANIZER;CN="John Doe":mailto:a@gmail.com UID:1qnf39p0m9h6n1cm9qa9d8mocv@google.com RRULE:FREQ=WEEKLY;COUNT=2;BYDAY=TH;WKST=SU CREATED:20250616T182358Z DTSTAMP:0 LAST-MODIFIED:20250616T182415Z SEQUENCE:1 END:VEVENT BEGIN:VTIMEZONE X-LIC-LOCATION:America/Los_Angeles TZID:America/Los_Angeles BEGIN:DAYLIGHT DTSTART:19700308T020000 TZNAME:PDT TZOFFSETFROM:-0800 TZOFFSETTO:-0700 RRULE:FREQ=YEARLY;BYDAY=2SU;BYMONTH=3 END:DAYLIGHT BEGIN:STANDARD DTSTART:19701101T020000 TZNAME:PST TZOFFSETFROM:-0700 TZOFFSETTO:-0800 RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=11 END:STANDARD END:VTIMEZONE END:VCALENDAR > send # Make sure the original alarms are preserved > get b@gmail.com 1qnf39p0m9h6n1cm9qa9d8mocv@google.com BEGIN:VCALENDAR PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT X-MICROSOFT-CDO-OWNERAPPTID:299828133 DESCRIPTION:This is the updated event description LOCATION: STATUS:CONFIRMED SUMMARY:Meet me maybe DTEND;TZID=America/Los_Angeles:20250619T064500 DTSTART;TZID=America/Los_Angeles:20250619T060000 TRANSP:OPAQUE ATTENDEE;CUTYPE=INDIVIDUAL;ROLE=REQ-PARTICIPANT;PARTSTAT=NEEDS-ACTION;RSVP=TRUE; CN=b@gmail.com;X-NUM-GUESTS=0:mailto:b@gmail.com ATTENDEE;CUTYPE=INDIVIDUAL;ROLE=REQ-PARTICIPANT;PARTSTAT=ACCEPTED;RSVP=TRUE; CN="John Doe";X-NUM-GUESTS=0:mailto:a@gmail.com ORGANIZER;CN="John Doe":mailto:a@gmail.com UID:1qnf39p0m9h6n1cm9qa9d8mocv@google.com RRULE:FREQ=WEEKLY;COUNT=2;BYDAY=TH;WKST=SU CREATED:20250616T182358Z DTSTAMP:0 LAST-MODIFIED:20250616T182358Z SEQUENCE:1 BEGIN:VALARM DESCRIPTION:This is an event reminder ACTION:DISPLAY TRIGGER:-PT10M END:VALARM END:VEVENT BEGIN:VTIMEZONE X-LIC-LOCATION:America/Los_Angeles TZID:America/Los_Angeles BEGIN:DAYLIGHT DTSTART:19700308T020000 TZNAME:PDT TZOFFSETFROM:-0800 TZOFFSETTO:-0700 RRULE:FREQ=YEARLY;BYDAY=2SU;BYMONTH=3 END:DAYLIGHT BEGIN:STANDARD DTSTART:19701101T020000 TZNAME:PST TZOFFSETFROM:-0700 TZOFFSETTO:-0800 RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=11 END:STANDARD END:VTIMEZONE END:VCALENDAR ================================================ FILE: tests/resources/itip/itip_incoming.txt ================================================ # iTIP tests # Recipient is not organizer or attendee > itip x@example.com y@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN METHOD:REQUEST VERSION:2.0 BEGIN:VEVENT ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B:mailto:b@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=C:mailto:c@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal:mailto:d@example.com ATTENDEE;RSVP=FALSE;CUTYPE=ROOM:conf_big@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE:mailto:e@example.com DTSTAMP:19970611T190000Z DTSTART:19970701T200000Z DTEND:19970701T2100000Z SUMMARY:Conference UID:calsrv.example.com-873970198738777@example.com SEQUENCE:0 STATUS:CONFIRMED END:VEVENT END:VCALENDAR > send NotOrganizerNorAttendee > reset # Refreshing an event (1) > put a@example.com 123456789@example.com BEGIN:VCALENDAR METHOD:REQUEST PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT UID:123456789@example.com SEQUENCE:0 RDATE:19980304T180000Z RDATE:19980311T180000Z RDATE:19980318T180000Z ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;RSVP=TRUE:mailto:b@example.com SUMMARY:Review Accounts DTSTART:19980304T180000Z DTEND:19980304T200000Z DTSTAMP:19980303T193000Z LOCATION:Conference Room A STATUS:CONFIRMED END:VEVENT END:VCALENDAR # Refreshing an event (2) > expect from: a@example.com to: b@example.com summary: invite summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 889034400, tz_id: 32768 }) summary.location: Text("Conference Room A") summary.summary: Text("Review Accounts") BEGIN:VCALENDAR METHOD:REQUEST PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT LOCATION:Conference Room A STATUS:CONFIRMED SUMMARY:Review Accounts DTEND:19980304T200000Z DTSTART:19980304T180000Z ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;RSVP=TRUE;PARTSTAT=NEEDS-ACTION:mailto:b@example.com ORGANIZER:mailto:a@example.com UID:123456789@example.com RDATE:19980304T180000Z RDATE:19980311T180000Z RDATE:19980318T180000Z DTSTAMP:0 SEQUENCE:1 END:VEVENT END:VCALENDAR # Refreshing an event (3) > send # Refreshing an event (4) > itip b@example.com a@example.com BEGIN:VCALENDAR METHOD:REFRESH PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT UID:123456789@example.com SEQUENCE:0 ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;RSVP=TRUE:mailto:b@example.com DTSTART:19980304T180000Z END:VEVENT END:VCALENDAR # Refreshing an event (5) > send from: a@example.com to: b@example.com summary: invite summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 889034400, tz_id: 32768 }) summary.location: Text("Conference Room A") summary.summary: Text("Review Accounts") BEGIN:VCALENDAR METHOD:REQUEST PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT LOCATION:Conference Room A STATUS:CONFIRMED SUMMARY:Review Accounts DTEND:19980304T200000Z DTSTART:19980304T180000Z ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;RSVP=TRUE;PARTSTAT=NEEDS-ACTION:mailto:b@example.com ORGANIZER:mailto:a@example.com UID:123456789@example.com RDATE:19980304T180000Z RDATE:19980311T180000Z RDATE:19980318T180000Z DTSTAMP:0 SEQUENCE:1 END:VEVENT END:VCALENDAR ================================================ FILE: tests/resources/itip/put_validation.txt ================================================ # Scheduling invalid actions # Event has no scheduling information > put x@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT DTSTAMP:19970611T190000Z DTSTART:19970701T200000Z DTEND:19970701T2100000Z SUMMARY:Conference UID:calsrv.example.com-873970198738777@example.com SEQUENCE:0 STATUS:CONFIRMED END:VEVENT END:VCALENDAR > expect NoSchedulingInfo > reset # X is not the organizer > put x@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B:mailto:b@example.com DTSTAMP:19970611T190000Z DTSTART:19970701T200000Z DTEND:19970701T2100000Z SUMMARY:Conference UID:calsrv.example.com-873970198738777@example.com SEQUENCE:0 STATUS:CONFIRMED END:VEVENT END:VCALENDAR > expect NotOrganizerNorAttendee > reset # No attendees > put a@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com DTSTAMP:19970611T190000Z DTSTART:19970701T200000Z DTEND:19970701T2100000Z SUMMARY:Conference UID:calsrv.example.com-873970198738777@example.com SEQUENCE:0 STATUS:CONFIRMED END:VEVENT END:VCALENDAR > expect NothingToSend > reset # Organizer with client scheduling agent > put a@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT ORGANIZER;SCHEDULE-AGENT=CLIENT:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B:mailto:b@example.com DTSTAMP:19970611T190000Z DTSTART:19970701T200000Z DTEND:19970701T2100000Z SUMMARY:Conference UID:calsrv.example.com-873970198738777@example.com SEQUENCE:0 STATUS:CONFIRMED END:VEVENT END:VCALENDAR > expect OtherSchedulingAgent > reset # Single participant with client scheduling agent > put a@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;SCHEDULE-AGENT=CLIENT;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B:mailto:b@example.com DTSTAMP:19970611T190000Z DTSTART:19970701T200000Z DTEND:19970701T2100000Z SUMMARY:Conference UID:calsrv.example.com-873970198738777@example.com SEQUENCE:0 STATUS:CONFIRMED END:VEVENT END:VCALENDAR > expect NothingToSend > reset # Only send updates to clients with server scheduling agent > put a@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B:mailto:d@example.com ATTENDEE;SCHEDULE-AGENT=CLIENT;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B:mailto:b@example.com DTSTAMP:19970611T190000Z DTSTART:19970701T200000Z DTEND:19970701T2100000Z SUMMARY:Conference UID:calsrv.example.com-873970198738777@example.com SEQUENCE:0 STATUS:CONFIRMED END:VEVENT END:VCALENDAR > expect from: a@example.com to: d@example.com summary: invite summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: Some("B"), is_organizer: false }, ItipParticipant { email: "d@example.com", name: Some("B"), is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 867787200, tz_id: 32768 }) summary.summary: Text("Conference") BEGIN:VCALENDAR METHOD:REQUEST PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT STATUS:CONFIRMED SUMMARY:Conference DTEND:19970701T2100000Z DTSTART:19970701T200000Z ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B;PARTSTAT=NEEDS-ACTION:mailto:b@exa mple.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B;PARTSTAT=NEEDS-ACTION:mailto:d@exa mple.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:0 SEQUENCE:1 END:VEVENT END:VCALENDAR # Writing the same event again should not send a new request > put a@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B:mailto:d@example.com ATTENDEE;SCHEDULE-AGENT=CLIENT;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B:mailto:b@example.com DTSTAMP:19970611T190000Z DTSTART:19970701T200000Z DTEND:19970701T2100000Z SUMMARY:Conference UID:calsrv.example.com-873970198738777@example.com SEQUENCE:0 STATUS:CONFIRMED END:VEVENT END:VCALENDAR > expect NothingToSend # Adding a comment and a custom property should not send a new request > put a@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B:mailto:d@example.com ATTENDEE;SCHEDULE-AGENT=CLIENT;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B:mailto:b@example.com DTSTAMP:19970611T190000Z DTSTART:19970701T200000Z DTEND:19970701T2100000Z SUMMARY:Conference COMMENT:This is a comment X-EXAMPLE:This is an example UID:calsrv.example.com-873970198738777@example.com SEQUENCE:0 STATUS:CONFIRMED END:VEVENT END:VCALENDAR > expect NothingToSend > reset # Multiple object types should be rejected > put a@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B:mailto:b@example.com DTSTAMP:19970611T190000Z DTSTART:19970701T200000Z DTEND:19970701T2100000Z SUMMARY:Conference UID:calsrv.example.com-873970198738777@example.com SEQUENCE:0 STATUS:CONFIRMED END:VEVENT BEGIN:VTODO ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B:mailto:b@example.com DTSTAMP:19970611T190000Z DTSTART:19970701T200000Z DTEND:19970701T2100000Z SUMMARY:Conference UID:calsrv.example.com-873970198738777@example.com SEQUENCE:0 STATUS:CONFIRMED END:VTODO END:VCALENDAR > expect MultipleObjectTypes > reset # Multiple organizers should be rejected > put a@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B:mailto:b@example.com DTSTAMP:19970611T190000Z DTSTART:19970701T200000Z DTEND:19970701T2100000Z SUMMARY:Conference UID:calsrv.example.com-873970198738777@example.com SEQUENCE:0 STATUS:CONFIRMED END:VEVENT BEGIN:VEVENT ORGANIZER:mailto:d@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=D:mailto:d@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B:mailto:b@example.com DTSTAMP:19970611T190000Z DTSTART:19970701T200000Z DTEND:19970701T2100000Z SUMMARY:Conference UID:calsrv.example.com-873970198738777@example.com SEQUENCE:0 STATUS:CONFIRMED END:VEVENT END:VCALENDAR > expect MultipleOrganizer > reset # Different UIDs should be rejected > put a@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B:mailto:b@example.com DTSTAMP:19970611T190000Z DTSTART:19970701T200000Z DTEND:19970701T2100000Z SUMMARY:Conference UID:calsrv.example.com-873970198738777@example.com SEQUENCE:0 STATUS:CONFIRMED END:VEVENT BEGIN:VEVENT ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B:mailto:b@example.com DTSTAMP:19970611T190000Z DTSTART:19970701T200000Z DTEND:19970701T2100000Z SUMMARY:Conference UID:other-uid@example.com SEQUENCE:0 STATUS:CONFIRMED END:VEVENT END:VCALENDAR > expect MultipleUid > reset # Multiple object instances should be rejected > put a@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B:mailto:b@example.com DTSTAMP:19970611T190000Z DTSTART:19970701T200000Z DTEND:19970701T2100000Z SUMMARY:Conference UID:calsrv.example.com-873970198738777@example.com SEQUENCE:0 STATUS:CONFIRMED END:VEVENT BEGIN:VEVENT ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B:mailto:b@example.com DTSTAMP:19970611T190000Z DTSTART:19970701T200000Z DTEND:19970701T2100000Z RECURRENCE-ID:19970701T200000Z SUMMARY:Conference UID:calsrv.example.com-873970198738777@example.com SEQUENCE:0 STATUS:CONFIRMED END:VEVENT BEGIN:VEVENT ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B:mailto:b@example.com DTSTAMP:19970611T190000Z DTSTART:19970701T200000Z DTEND:19970701T2100000Z RECURRENCE-ID:19970701T200000Z SUMMARY:Conference UID:calsrv.example.com-873970198738777@example.com SEQUENCE:0 STATUS:CONFIRMED END:VEVENT END:VCALENDAR > expect MultipleObjectInstances ================================================ FILE: tests/resources/itip/rfc5546_event_recurring.txt ================================================ # RFC5546 - Group Event Request # A sample meeting request is sent from "A" to "B", "C", and "D". > put a@example.com guid-1@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT UID:guid-1@example.com SEQUENCE:0 RRULE:FREQ=MONTHLY;BYMONTHDAY=1;UNTIL=19980901T210000Z ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE:mailto:b@example.com ATTENDEE:mailto:c@example.com ATTENDEE:mailto:d@example.com DESCRIPTION:IETF-C&S Conference Call CLASS:PUBLIC SUMMARY:IETF Calendaring Working Group Meeting DTSTART:19970601T210000Z DTEND:19970601T220000Z LOCATION:Conference Call DTSTAMP:19970526T083000Z STATUS:CONFIRMED END:VEVENT END:VCALENDAR > expect from: a@example.com to: b@example.com, c@example.com, d@example.com summary: invite summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }, ItipParticipant { email: "c@example.com", name: None, is_organizer: false }, ItipParticipant { email: "d@example.com", name: None, is_organizer: false }]) summary.description: Text("IETF-C&S Conference Call") summary.dtstart: Time(ItipTime { start: 865198800, tz_id: 32768 }) summary.location: Text("Conference Call") summary.rrule: Rrule(ICalendarRecurrenceRule { freq: Monthly, until: Some(PartialDateTime { year: Some(1998), month: Some(9), day: Some(1), hour: Some(21), minute: Some(0), second: Some(0), tz_hour: Some(0), tz_minute: Some(0), tz_minus: false }), count: None, interval: None, bysecond: [], byminute: [], byhour: [], byday: [], bymonthday: [1], byyearday: [], byweekno: [], bymonth: [], bysetpos: [], wkst: None, rscale: None, skip: None }) summary.summary: Text("IETF Calendaring Working Group Meeting") BEGIN:VCALENDAR METHOD:REQUEST PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT CLASS:PUBLIC DESCRIPTION:IETF-C&S Conference Call LOCATION:Conference Call STATUS:CONFIRMED SUMMARY:IETF Calendaring Working Group Meeting DTEND:19970601T220000Z DTSTART:19970601T210000Z ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:b@example.com ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:c@example.com ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:d@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ORGANIZER:mailto:a@example.com UID:guid-1@example.com RRULE:FREQ=MONTHLY;UNTIL=19980901T210000Z;BYMONTHDAY=1 DTSTAMP:0 SEQUENCE:1 END:VEVENT END:VCALENDAR # Send iTIP request to attendees > send # Change a recurrence instance > put a@example.com guid-1@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT UID:guid-1@example.com SEQUENCE:2 RRULE:FREQ=MONTHLY;BYMONTHDAY=1;UNTIL=19980901T210000Z ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE:mailto:b@example.com ATTENDEE:mailto:c@example.com ATTENDEE:mailto:d@example.com DESCRIPTION:IETF-C&S Conference Call CLASS:PUBLIC SUMMARY:IETF Calendaring Working Group Meeting DTSTART:19970601T210000Z DTEND:19970601T220000Z LOCATION:Conference Call DTSTAMP:19970526T083000Z STATUS:CONFIRMED END:VEVENT BEGIN:VEVENT UID:guid-1@example.com RECURRENCE-ID:19970701T210000Z SEQUENCE:1 ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE:mailto:b@example.com ATTENDEE:mailto:c@example.com ATTENDEE:mailto:d@example.com DESCRIPTION:IETF-C&S Conference Call CLASS:PUBLIC SUMMARY:IETF Calendaring Working Group Meeting DTSTART:19970703T210000Z DTEND:19970703T220000Z LOCATION:Conference Call DTSTAMP:19970626T093000Z STATUS:CONFIRMED END:VEVENT END:VCALENDAR > expect from: a@example.com to: b@example.com, c@example.com, d@example.com summary: update ADD summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }, ItipParticipant { email: "c@example.com", name: None, is_organizer: false }, ItipParticipant { email: "d@example.com", name: None, is_organizer: false }]) summary.description: Text("IETF-C&S Conference Call") summary.dtstart: Time(ItipTime { start: 865198800, tz_id: 32768 }) summary.location: Text("Conference Call") summary.rrule: Rrule(ICalendarRecurrenceRule { freq: Monthly, until: Some(PartialDateTime { year: Some(1998), month: Some(9), day: Some(1), hour: Some(21), minute: Some(0), second: Some(0), tz_hour: Some(0), tz_minute: Some(0), tz_minus: false }), count: None, interval: None, bysecond: [], byminute: [], byhour: [], byday: [], bymonthday: [1], byyearday: [], byweekno: [], bymonth: [], bysetpos: [], wkst: None, rscale: None, skip: None }) summary.summary: Text("IETF Calendaring Working Group Meeting") BEGIN:VCALENDAR METHOD:ADD PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT CLASS:PUBLIC DESCRIPTION:IETF-C&S Conference Call LOCATION:Conference Call STATUS:CONFIRMED SUMMARY:IETF Calendaring Working Group Meeting DTEND:19970703T220000Z DTSTART:19970703T210000Z ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:b@example.com ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:c@example.com ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:d@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ORGANIZER:mailto:a@example.com RECURRENCE-ID:19970701T210000Z UID:guid-1@example.com DTSTAMP:0 SEQUENCE:2 END:VEVENT END:VCALENDAR # Send iTIP update to attendees > send # Cancel a recurrence instance only for B > put a@example.com guid-1@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT UID:guid-1@example.com SEQUENCE:3 RRULE:FREQ=MONTHLY;BYMONTHDAY=1;UNTIL=19980901T210000Z ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE:mailto:b@example.com ATTENDEE:mailto:c@example.com ATTENDEE:mailto:d@example.com DESCRIPTION:IETF-C&S Conference Call CLASS:PUBLIC SUMMARY:IETF Calendaring Working Group Meeting DTSTART:19970601T210000Z DTEND:19970601T220000Z LOCATION:Conference Call DTSTAMP:19970526T083000Z STATUS:CONFIRMED END:VEVENT BEGIN:VEVENT UID:guid-1@example.com RECURRENCE-ID:19970701T210000Z SEQUENCE:2 ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE:mailto:b@example.com ATTENDEE:mailto:c@example.com ATTENDEE:mailto:d@example.com DESCRIPTION:IETF-C&S Conference Call CLASS:PUBLIC SUMMARY:IETF Calendaring Working Group Meeting DTSTART:19970703T210000Z DTEND:19970703T220000Z LOCATION:Conference Call DTSTAMP:19970626T093000Z STATUS:CONFIRMED END:VEVENT BEGIN:VEVENT UID:guid-1@example.com ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE:mailto:b@example.com RECURRENCE-ID:19970801T210000Z SEQUENCE:2 STATUS:CANCELLED DTSTAMP:19970721T093000Z END:VEVENT END:VCALENDAR > expect from: a@example.com to: b@example.com summary: cancel summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }, ItipParticipant { email: "c@example.com", name: None, is_organizer: false }, ItipParticipant { email: "d@example.com", name: None, is_organizer: false }]) summary.description: Text("IETF-C&S Conference Call") summary.dtstart: Time(ItipTime { start: 865198800, tz_id: 32768 }) summary.location: Text("Conference Call") summary.rrule: Rrule(ICalendarRecurrenceRule { freq: Monthly, until: Some(PartialDateTime { year: Some(1998), month: Some(9), day: Some(1), hour: Some(21), minute: Some(0), second: Some(0), tz_hour: Some(0), tz_minute: Some(0), tz_minus: false }), count: None, interval: None, bysecond: [], byminute: [], byhour: [], byday: [], bymonthday: [1], byyearday: [], byweekno: [], bymonth: [], bysetpos: [], wkst: None, rscale: None, skip: None }) summary.summary: Text("IETF Calendaring Working Group Meeting") BEGIN:VCALENDAR METHOD:CANCEL PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT STATUS:CANCELLED ATTENDEE:mailto:b@example.com ORGANIZER:mailto:a@example.com RECURRENCE-ID:19970801T210000Z UID:guid-1@example.com DTSTAMP:0 SEQUENCE:3 END:VEVENT END:VCALENDAR # Send iTIP cancellation to B > send # Make sure B has the cancelled event > get b@example.com guid-1@example.com BEGIN:VCALENDAR PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT CLASS:PUBLIC DESCRIPTION:IETF-C&S Conference Call LOCATION:Conference Call STATUS:CONFIRMED SUMMARY:IETF Calendaring Working Group Meeting DTEND:19970601T220000Z DTSTART:19970601T210000Z ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:b@example.com ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:c@example.com ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:d@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ORGANIZER:mailto:a@example.com UID:guid-1@example.com RRULE:FREQ=MONTHLY;UNTIL=19980901T210000Z;BYMONTHDAY=1 DTSTAMP:0 SEQUENCE:1 END:VEVENT BEGIN:VEVENT CLASS:PUBLIC DESCRIPTION:IETF-C&S Conference Call LOCATION:Conference Call STATUS:CONFIRMED SUMMARY:IETF Calendaring Working Group Meeting DTEND:19970703T220000Z DTSTART:19970703T210000Z ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:b@example.com ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:c@example.com ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:d@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ORGANIZER:mailto:a@example.com RECURRENCE-ID:19970701T210000Z UID:guid-1@example.com DTSTAMP:0 SEQUENCE:2 END:VEVENT BEGIN:VEVENT STATUS:CANCELLED ATTENDEE:mailto:b@example.com ORGANIZER:mailto:a@example.com RECURRENCE-ID:19970801T210000Z UID:guid-1@example.com DTSTAMP:0 SEQUENCE:3 END:VEVENT END:VCALENDAR # Change all future instances > put a@example.com guid-1@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT UID:guid-1@example.com SEQUENCE:4 RRULE:FREQ=MONTHLY;BYMONTHDAY=1;UNTIL=19980901T210000Z ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE:mailto:b@example.com ATTENDEE:mailto:c@example.com ATTENDEE:mailto:d@example.com DESCRIPTION:IETF-C&S Conference Call CLASS:PUBLIC SUMMARY:IETF Calendaring Working Group Meeting DTSTART:19970601T210000Z DTEND:19970601T220000Z LOCATION:Conference Call DTSTAMP:19970526T083000Z STATUS:CONFIRMED END:VEVENT BEGIN:VEVENT UID:guid-1@example.com RECURRENCE-ID:19970701T210000Z SEQUENCE:3 ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE:mailto:b@example.com ATTENDEE:mailto:c@example.com ATTENDEE:mailto:d@example.com DESCRIPTION:IETF-C&S Conference Call CLASS:PUBLIC SUMMARY:IETF Calendaring Working Group Meeting DTSTART:19970703T210000Z DTEND:19970703T220000Z LOCATION:Conference Call DTSTAMP:19970626T093000Z STATUS:CONFIRMED END:VEVENT BEGIN:VEVENT UID:guid-1@example.com ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE:mailto:b@example.com RECURRENCE-ID:19970801T210000Z SEQUENCE:3 STATUS:CANCELLED DTSTAMP:19970721T093000Z END:VEVENT BEGIN:VEVENT UID:guid-1@example.com RECURRENCE-ID;THISANDFUTURE:19970901T210000Z SEQUENCE:3 ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;RSVP=TRUE:mailto:b@example.com ATTENDEE;RSVP=TRUE:mailto:c@example.com ATTENDEE;RSVP=TRUE:mailto:d@example.com DESCRIPTION:IETF-C&S Discussion CLASS:PUBLIC SUMMARY:IETF Calendaring Working Group Meeting DTSTART:19970901T210000Z DTEND:19970901T220000Z LOCATION:Building 32, Microsoft, Seattle, WA DTSTAMP:19970526T083000Z STATUS:CONFIRMED END:VEVENT END:VCALENDAR > expect from: a@example.com to: b@example.com, c@example.com, d@example.com summary: update ADD summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }, ItipParticipant { email: "c@example.com", name: None, is_organizer: false }, ItipParticipant { email: "d@example.com", name: None, is_organizer: false }]) summary.description: Text("IETF-C&S Conference Call") summary.dtstart: Time(ItipTime { start: 865198800, tz_id: 32768 }) summary.location: Text("Conference Call") summary.rrule: Rrule(ICalendarRecurrenceRule { freq: Monthly, until: Some(PartialDateTime { year: Some(1998), month: Some(9), day: Some(1), hour: Some(21), minute: Some(0), second: Some(0), tz_hour: Some(0), tz_minute: Some(0), tz_minus: false }), count: None, interval: None, bysecond: [], byminute: [], byhour: [], byday: [], bymonthday: [1], byyearday: [], byweekno: [], bymonth: [], bysetpos: [], wkst: None, rscale: None, skip: None }) summary.summary: Text("IETF Calendaring Working Group Meeting") BEGIN:VCALENDAR METHOD:ADD PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT CLASS:PUBLIC DESCRIPTION:IETF-C&S Discussion LOCATION:Building 32\, Microsoft\, Seattle\, WA STATUS:CONFIRMED SUMMARY:IETF Calendaring Working Group Meeting DTEND:19970901T220000Z DTSTART:19970901T210000Z ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;RSVP=TRUE;PARTSTAT=NEEDS-ACTION:mailto:b@example.com ATTENDEE;RSVP=TRUE;PARTSTAT=NEEDS-ACTION:mailto:c@example.com ATTENDEE;RSVP=TRUE;PARTSTAT=NEEDS-ACTION:mailto:d@example.com ORGANIZER:mailto:a@example.com RECURRENCE-ID;THISANDFUTURE:19970901T210000Z UID:guid-1@example.com DTSTAMP:0 SEQUENCE:4 END:VEVENT END:VCALENDAR # Send iTIP update to attendees > send # Make sure B has the complete event including all updates > get b@example.com guid-1@example.com BEGIN:VCALENDAR PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT CLASS:PUBLIC DESCRIPTION:IETF-C&S Conference Call LOCATION:Conference Call STATUS:CONFIRMED SUMMARY:IETF Calendaring Working Group Meeting DTEND:19970601T220000Z DTSTART:19970601T210000Z ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:b@example.com ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:c@example.com ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:d@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ORGANIZER:mailto:a@example.com UID:guid-1@example.com RRULE:FREQ=MONTHLY;UNTIL=19980901T210000Z;BYMONTHDAY=1 DTSTAMP:0 SEQUENCE:1 END:VEVENT BEGIN:VEVENT CLASS:PUBLIC DESCRIPTION:IETF-C&S Conference Call LOCATION:Conference Call STATUS:CONFIRMED SUMMARY:IETF Calendaring Working Group Meeting DTEND:19970703T220000Z DTSTART:19970703T210000Z ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:b@example.com ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:c@example.com ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:d@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ORGANIZER:mailto:a@example.com RECURRENCE-ID:19970701T210000Z UID:guid-1@example.com DTSTAMP:0 SEQUENCE:2 END:VEVENT BEGIN:VEVENT STATUS:CANCELLED ATTENDEE:mailto:b@example.com ORGANIZER:mailto:a@example.com RECURRENCE-ID:19970801T210000Z UID:guid-1@example.com DTSTAMP:0 SEQUENCE:3 END:VEVENT BEGIN:VEVENT CLASS:PUBLIC DESCRIPTION:IETF-C&S Discussion LOCATION:Building 32\, Microsoft\, Seattle\, WA STATUS:CONFIRMED SUMMARY:IETF Calendaring Working Group Meeting DTEND:19970901T220000Z DTSTART:19970901T210000Z ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;RSVP=TRUE;PARTSTAT=NEEDS-ACTION:mailto:b@example.com ATTENDEE;RSVP=TRUE;PARTSTAT=NEEDS-ACTION:mailto:c@example.com ATTENDEE;RSVP=TRUE;PARTSTAT=NEEDS-ACTION:mailto:d@example.com ORGANIZER:mailto:a@example.com RECURRENCE-ID;THISANDFUTURE:19970901T210000Z UID:guid-1@example.com DTSTAMP:0 SEQUENCE:4 END:VEVENT END:VCALENDAR # Cancel the recurring event > put a@example.com guid-1@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT UID:guid-1@example.com SEQUENCE:4 RRULE:FREQ=MONTHLY;BYMONTHDAY=1;UNTIL=19980901T210000Z ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE:mailto:b@example.com ATTENDEE:mailto:c@example.com ATTENDEE:mailto:d@example.com DESCRIPTION:IETF-C&S Conference Call CLASS:PUBLIC SUMMARY:IETF Calendaring Working Group Meeting DTSTART:19970601T210000Z DTEND:19970601T220000Z LOCATION:Conference Call DTSTAMP:19970526T083000Z STATUS:CONFIRMED END:VEVENT BEGIN:VEVENT UID:guid-1@example.com RECURRENCE-ID:19970701T210000Z SEQUENCE:3 ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE:mailto:b@example.com ATTENDEE:mailto:c@example.com ATTENDEE:mailto:d@example.com DESCRIPTION:IETF-C&S Conference Call CLASS:PUBLIC SUMMARY:IETF Calendaring Working Group Meeting DTSTART:19970703T210000Z DTEND:19970703T220000Z LOCATION:Conference Call DTSTAMP:19970626T093000Z STATUS:CONFIRMED END:VEVENT BEGIN:VEVENT UID:guid-1@example.com ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE:mailto:b@example.com RECURRENCE-ID:19970801T210000Z SEQUENCE:3 STATUS:CANCELLED DTSTAMP:19970721T093000Z END:VEVENT BEGIN:VEVENT UID:guid-1@example.com RECURRENCE-ID;THISANDFUTURE:19970901T210000Z SEQUENCE:3 ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;RSVP=TRUE:mailto:b@example.com ATTENDEE;RSVP=TRUE:mailto:c@example.com ATTENDEE;RSVP=TRUE:mailto:d@example.com DESCRIPTION:IETF-C&S Discussion CLASS:PUBLIC SUMMARY:IETF Calendaring Working Group Meeting DTSTART:19970901T210000Z DTEND:19970901T220000Z LOCATION:Building 32, Microsoft, Seattle, WA DTSTAMP:19970526T083000Z STATUS:CONFIRMED END:VEVENT END:VCALENDAR # Cancel a recurring event > delete a@example.com guid-1@example.com > expect from: a@example.com to: b@example.com, c@example.com, d@example.com summary: cancel summary.description: Text("IETF-C&S Conference Call") summary.dtstart: Time(ItipTime { start: 865198800, tz_id: 32768 }) summary.location: Text("Conference Call") summary.rrule: Rrule(ICalendarRecurrenceRule { freq: Monthly, until: Some(PartialDateTime { year: Some(1998), month: Some(9), day: Some(1), hour: Some(21), minute: Some(0), second: Some(0), tz_hour: Some(0), tz_minute: Some(0), tz_minus: false }), count: None, interval: None, bysecond: [], byminute: [], byhour: [], byday: [], bymonthday: [1], byyearday: [], byweekno: [], bymonth: [], bysetpos: [], wkst: None, rscale: None, skip: None }) summary.summary: Text("IETF Calendaring Working Group Meeting") BEGIN:VCALENDAR METHOD:CANCEL PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT DESCRIPTION:IETF-C&S Conference Call LOCATION:Conference Call STATUS:CANCELLED SUMMARY:IETF Calendaring Working Group Meeting DTEND:19970601T220000Z DTSTART:19970601T210000Z ATTENDEE:mailto:b@example.com ATTENDEE:mailto:c@example.com ATTENDEE:mailto:d@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ORGANIZER:mailto:a@example.com UID:guid-1@example.com DTSTAMP:0 SEQUENCE:5 END:VEVENT END:VCALENDAR # Send iTIP cancellation to attendees > send # Make sure all instances in B were deleted > get b@example.com guid-1@example.com BEGIN:VCALENDAR PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT CLASS:PUBLIC DESCRIPTION:IETF-C&S Conference Call LOCATION:Conference Call STATUS:CANCELLED SUMMARY:IETF Calendaring Working Group Meeting DTEND:19970601T220000Z DTSTART:19970601T210000Z ATTENDEE:mailto:b@example.com ATTENDEE:mailto:c@example.com ATTENDEE:mailto:d@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ORGANIZER:mailto:a@example.com UID:guid-1@example.com RRULE:FREQ=MONTHLY;UNTIL=19980901T210000Z;BYMONTHDAY=1 DTSTAMP:0 END:VEVENT BEGIN:VEVENT CLASS:PUBLIC DESCRIPTION:IETF-C&S Conference Call LOCATION:Conference Call STATUS:CANCELLED SUMMARY:IETF Calendaring Working Group Meeting DTEND:19970703T220000Z DTSTART:19970703T210000Z ATTENDEE:mailto:b@example.com ATTENDEE:mailto:c@example.com ATTENDEE:mailto:d@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ORGANIZER:mailto:a@example.com RECURRENCE-ID:19970701T210000Z UID:guid-1@example.com DTSTAMP:0 SEQUENCE:2 END:VEVENT BEGIN:VEVENT STATUS:CANCELLED ATTENDEE:mailto:b@example.com ATTENDEE:mailto:c@example.com ATTENDEE:mailto:d@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ORGANIZER:mailto:a@example.com RECURRENCE-ID:19970801T210000Z UID:guid-1@example.com DTSTAMP:0 SEQUENCE:3 END:VEVENT BEGIN:VEVENT CLASS:PUBLIC DESCRIPTION:IETF-C&S Discussion LOCATION:Building 32\, Microsoft\, Seattle\, WA STATUS:CANCELLED SUMMARY:IETF Calendaring Working Group Meeting DTEND:19970901T220000Z DTSTART:19970901T210000Z ATTENDEE:mailto:b@example.com ATTENDEE:mailto:c@example.com ATTENDEE:mailto:d@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ORGANIZER:mailto:a@example.com RECURRENCE-ID;THISANDFUTURE:19970901T210000Z UID:guid-1@example.com DTSTAMP:0 SEQUENCE:4 END:VEVENT END:VCALENDAR # Add a new series of instances to the recurring event > put a@example.com 123456789@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT UID:123456789@example.com SEQUENCE:0 RRULE:WKST=SU;BYDAY=TU;FREQ=WEEKLY ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;RSVP=TRUE:mailto:b@example.com SUMMARY:Review Accounts DTSTART:19980303T210000Z DTEND:19980303T220000Z LOCATION:The White Room DTSTAMP:19980301T093000Z STATUS:CONFIRMED END:VEVENT BEGIN:VEVENT UID:123456789@example.com SEQUENCE:2 RECURRENCE-ID;THISANDFUTURE:19970901T210000Z ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;RSVP=TRUE:mailto:c@example.com SUMMARY:Review Accounts DTSTAMP:19980303T193000Z LOCATION:The Red Room STATUS:CONFIRMED END:VEVENT END:VCALENDAR > expect from: a@example.com to: b@example.com, c@example.com summary: invite summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 888958800, tz_id: 32768 }) summary.location: Text("The White Room") summary.rrule: Rrule(ICalendarRecurrenceRule { freq: Weekly, until: None, count: None, interval: None, bysecond: [], byminute: [], byhour: [], byday: [ICalendarDay { ordwk: None, weekday: Tuesday }], bymonthday: [], byyearday: [], byweekno: [], bymonth: [], bysetpos: [], wkst: Some(Sunday), rscale: None, skip: None }) summary.summary: Text("Review Accounts") BEGIN:VCALENDAR METHOD:REQUEST PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT LOCATION:The White Room STATUS:CONFIRMED SUMMARY:Review Accounts DTEND:19980303T220000Z DTSTART:19980303T210000Z ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;RSVP=TRUE;PARTSTAT=NEEDS-ACTION:mailto:b@example.com ORGANIZER:mailto:a@example.com UID:123456789@example.com RRULE:FREQ=WEEKLY;BYDAY=TU;WKST=SU DTSTAMP:0 SEQUENCE:1 END:VEVENT BEGIN:VEVENT LOCATION:The Red Room STATUS:CONFIRMED SUMMARY:Review Accounts ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;RSVP=TRUE;PARTSTAT=NEEDS-ACTION:mailto:c@example.com ORGANIZER:mailto:a@example.com RECURRENCE-ID;THISANDFUTURE:19970901T210000Z UID:123456789@example.com DTSTAMP:0 SEQUENCE:3 END:VEVENT END:VCALENDAR # Send iTIP request to B and C > send # Add a new series of instances to the recurring event (update) > put a@example.com 123456789@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT UID:123456789@example.com SEQUENCE:2 RRULE:WKST=SU;BYDAY=TU,TH;FREQ=WEEKLY ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;RSVP=TRUE:mailto:b@example.com SUMMARY:Review Accounts DTSTART:19980303T210000Z DTEND:19980303T220000Z DTSTAMP:19980303T193000Z LOCATION:The White Room STATUS:CONFIRMED END:VCALENDAR > expect from: a@example.com to: b@example.com summary: update REQUEST summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 888958800, tz_id: 32768 }) summary.location: Text("The White Room") summary.rrule: Rrule(ICalendarRecurrenceRule { freq: Weekly, until: None, count: None, interval: None, bysecond: [], byminute: [], byhour: [], byday: [ICalendarDay { ordwk: None, weekday: Tuesday }, ICalendarDay { ordwk: None, weekday: Thursday }], bymonthday: [], byyearday: [], byweekno: [], bymonth: [], bysetpos: [], wkst: Some(Sunday), rscale: None, skip: None }) summary.summary: Text("Review Accounts") ~summary.rrule: Rrule(ICalendarRecurrenceRule { freq: Weekly, until: None, count: None, interval: None, bysecond: [], byminute: [], byhour: [], byday: [ICalendarDay { ordwk: None, weekday: Tuesday }], bymonthday: [], byyearday: [], byweekno: [], bymonth: [], bysetpos: [], wkst: Some(Sunday), rscale: None, skip: None }) BEGIN:VCALENDAR METHOD:REQUEST PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT LOCATION:The White Room STATUS:CONFIRMED SUMMARY:Review Accounts DTEND:19980303T220000Z DTSTART:19980303T210000Z ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;RSVP=TRUE;PARTSTAT=NEEDS-ACTION:mailto:b@example.com ORGANIZER:mailto:a@example.com UID:123456789@example.com RRULE:FREQ=WEEKLY;BYDAY=TU,TH;WKST=SU DTSTAMP:0 SEQUENCE:3 END:VEVENT END:VCALENDAR ================================ from: a@example.com to: c@example.com summary: cancel summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 888958800, tz_id: 32768 }) summary.location: Text("The White Room") summary.rrule: Rrule(ICalendarRecurrenceRule { freq: Weekly, until: None, count: None, interval: None, bysecond: [], byminute: [], byhour: [], byday: [ICalendarDay { ordwk: None, weekday: Tuesday }], bymonthday: [], byyearday: [], byweekno: [], bymonth: [], bysetpos: [], wkst: Some(Sunday), rscale: None, skip: None }) summary.summary: Text("Review Accounts") BEGIN:VCALENDAR METHOD:CANCEL PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT LOCATION:The Red Room STATUS:CANCELLED SUMMARY:Review Accounts ATTENDEE;RSVP=TRUE:mailto:c@example.com ORGANIZER:mailto:a@example.com RECURRENCE-ID;THISANDFUTURE:19970901T210000Z UID:123456789@example.com DTSTAMP:0 SEQUENCE:4 END:VEVENT END:VCALENDAR # Send iTIP request to B and cancellation to C > send # Make sure B has the updated event > get b@example.com 123456789@example.com BEGIN:VCALENDAR PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT LOCATION:The White Room STATUS:CONFIRMED SUMMARY:Review Accounts DTEND:19980303T220000Z DTSTART:19980303T210000Z ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;RSVP=TRUE;PARTSTAT=NEEDS-ACTION:mailto:b@example.com ORGANIZER:mailto:a@example.com UID:123456789@example.com RRULE:FREQ=WEEKLY;BYDAY=TU,TH;WKST=SU DTSTAMP:0 SEQUENCE:3 END:VEVENT END:VCALENDAR # Make sure C has the updated event > get c@example.com 123456789@example.com BEGIN:VCALENDAR PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT LOCATION:The Red Room STATUS:CANCELLED SUMMARY:Review Accounts ATTENDEE;RSVP=TRUE:mailto:c@example.com ORGANIZER:mailto:a@example.com RECURRENCE-ID;THISANDFUTURE:19970901T210000Z UID:123456789@example.com DTSTAMP:0 END:VEVENT BEGIN:VEVENT LOCATION:The White Room STATUS:CONFIRMED SUMMARY:Review Accounts DTEND:19980303T220000Z DTSTART:19980303T210000Z ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;RSVP=TRUE;PARTSTAT=NEEDS-ACTION:mailto:b@example.com ORGANIZER:mailto:a@example.com UID:123456789@example.com RRULE:FREQ=WEEKLY;BYDAY=TU;WKST=SU DTSTAMP:0 SEQUENCE:1 END:VEVENT END:VCALENDAR # Delete the event from B > delete b@example.com 123456789@example.com > expect from: b@example.com to: a@example.com summary: rsvp DECLINED summary.dtstart: Time(ItipTime { start: 888958800, tz_id: 32768 }) summary.location: Text("The White Room") summary.rrule: Rrule(ICalendarRecurrenceRule { freq: Weekly, until: None, count: None, interval: None, bysecond: [], byminute: [], byhour: [], byday: [ICalendarDay { ordwk: None, weekday: Tuesday }, ICalendarDay { ordwk: None, weekday: Thursday }], bymonthday: [], byyearday: [], byweekno: [], bymonth: [], bysetpos: [], wkst: Some(Sunday), rscale: None, skip: None }) summary.summary: Text("Review Accounts") BEGIN:VCALENDAR METHOD:REPLY PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT SUMMARY:Review Accounts DTEND:19980303T220000Z DTSTART:19980303T210000Z ATTENDEE;PARTSTAT=DECLINED:mailto:b@example.com ORGANIZER:mailto:a@example.com UID:123456789@example.com DTSTAMP:0 SEQUENCE:3 END:VEVENT END:VCALENDAR > send # Make sure A has the cancellation from B > get a@example.com 123456789@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT LOCATION:The White Room STATUS:CONFIRMED SUMMARY:Review Accounts DTEND:19980303T220000Z DTSTART:19980303T210000Z ATTENDEE;PARTSTAT=DECLINED:mailto:b@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ORGANIZER:mailto:a@example.com UID:123456789@example.com RRULE:FREQ=WEEKLY;BYDAY=TU,TH;WKST=SU DTSTAMP:1 SEQUENCE:3 END:VEVENT END:VCALENDAR ================================================ FILE: tests/resources/itip/rfc5546_event_single.txt ================================================ # RFC5546 - Group Event Request # A sample meeting request is sent from "A" to "B", "C", and "D". > put a@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B:mailto:b@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=C:mailto:c@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal:mailto:d@example.com ATTENDEE;RSVP=FALSE;CUTYPE=ROOM:conf_big@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE:mailto:e@example.com DTSTAMP:19970611T190000Z DTSTART:19970701T200000Z DTEND:19970701T2100000Z SUMMARY:Conference UID:calsrv.example.com-873970198738777@example.com SEQUENCE:0 STATUS:CONFIRMED END:VEVENT END:VCALENDAR > expect from: a@example.com to: b@example.com, c@example.com, d@example.com summary: invite summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: Some("B"), is_organizer: false }, ItipParticipant { email: "c@example.com", name: Some("C"), is_organizer: false }, ItipParticipant { email: "conf_big@example.com", name: None, is_organizer: false }, ItipParticipant { email: "d@example.com", name: Some("Hal"), is_organizer: false }, ItipParticipant { email: "e@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 867787200, tz_id: 32768 }) summary.summary: Text("Conference") BEGIN:VCALENDAR METHOD:REQUEST PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT STATUS:CONFIRMED SUMMARY:Conference DTEND:19970701T2100000Z DTSTART:19970701T200000Z ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE:mailto:e@example.com ATTENDEE;RSVP=FALSE;CUTYPE=ROOM:conf_big@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B;PARTSTAT=NEEDS-ACTION:mailto:b@exa mple.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=C;PARTSTAT=NEEDS-ACTION:mailto:c@exa mple.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal;PARTSTAT=NEEDS-ACTION:mailto:d@e xample.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:0 SEQUENCE:1 END:VEVENT END:VCALENDAR # Send iTIP request to B, C, and D > send # Make sure B receives the request > get b@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT STATUS:CONFIRMED SUMMARY:Conference DTEND:19970701T2100000Z DTSTART:19970701T200000Z ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE:mailto:e@example.com ATTENDEE;RSVP=FALSE;CUTYPE=ROOM:conf_big@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B;PARTSTAT=NEEDS-ACTION:mailto:b@exa mple.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=C;PARTSTAT=NEEDS-ACTION:mailto:c@exa mple.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal;PARTSTAT=NEEDS-ACTION:mailto:d@e xample.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:0 SEQUENCE:1 END:VEVENT END:VCALENDAR # B accepts the request > put b@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Stalwart Labs LLC//Stalwart Server//EN BEGIN:VEVENT DTSTAMP:19970701T200000Z SEQUENCE:1 UID:calsrv.example.com-873970198738777@example.com ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B;PARTSTAT=ACCEPTED:mailto:b@exa mple.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=C;PARTSTAT=NEEDS-ACTION:mailto:c@exa mple.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal;PARTSTAT=NEEDS-ACTION:mailto:d@e xample.com ATTENDEE;RSVP=FALSE;CUTYPE=ROOM:conf_big@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE:mailto:e@example.com DTSTART:19970701T200000Z DTEND:19970701T2100000Z SUMMARY:Conference STATUS:CONFIRMED END:VEVENT END:VCALENDAR > expect from: b@example.com to: a@example.com summary: rsvp ACCEPTED summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: Some("B"), is_organizer: false }, ItipParticipant { email: "c@example.com", name: Some("C"), is_organizer: false }, ItipParticipant { email: "conf_big@example.com", name: None, is_organizer: false }, ItipParticipant { email: "d@example.com", name: Some("Hal"), is_organizer: false }, ItipParticipant { email: "e@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 867787200, tz_id: 32768 }) summary.summary: Text("Conference") BEGIN:VCALENDAR METHOD:REPLY PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT SUMMARY:Conference DTEND:19970701T2100000Z DTSTART:19970701T200000Z ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B;PARTSTAT=ACCEPTED:mailto:b@example .com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:0 SEQUENCE:1 REQUEST-STATUS:2.0;Success END:VEVENT END:VCALENDAR # Send iTIP reply to A > send # Make sure A receives the reply > get a@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT STATUS:CONFIRMED SUMMARY:Conference DTEND:19970701T2100000Z DTSTART:19970701T200000Z ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED;CN=A:mailto:a@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE:mailto:e@example.com ATTENDEE;RSVP=FALSE;CUTYPE=ROOM:conf_big@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=B;PARTSTAT=ACCEPTED;SCHEDULE-STATUS= 2.0:mailto:b@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=C:mailto:c@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal:mailto:d@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:1 SEQUENCE:1 END:VEVENT END:VCALENDAR # A moved the event to a new time > put a@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL:mailto:b@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL:mailto:c@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal:mailto:d@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE;CUTYPE=ROOM:mailto:conf@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE:mailto:e@example.com DTSTART:19970701T180000Z DTEND:19970701T190000Z SUMMARY:Phone Conference UID:calsrv.example.com-873970198738777@example.com SEQUENCE:1 DTSTAMP:19970613T190000Z STATUS:CONFIRMED END:VEVENT END:VCALENDAR > expect from: a@example.com to: b@example.com, c@example.com, d@example.com summary: update REQUEST summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }, ItipParticipant { email: "c@example.com", name: None, is_organizer: false }, ItipParticipant { email: "conf@example.com", name: None, is_organizer: false }, ItipParticipant { email: "d@example.com", name: Some("Hal"), is_organizer: false }, ItipParticipant { email: "e@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 867780000, tz_id: 32768 }) summary.summary: Text("Phone Conference") ~summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: Some("B"), is_organizer: false }, ItipParticipant { email: "c@example.com", name: Some("C"), is_organizer: false }, ItipParticipant { email: "conf_big@example.com", name: None, is_organizer: false }, ItipParticipant { email: "d@example.com", name: Some("Hal"), is_organizer: false }, ItipParticipant { email: "e@example.com", name: None, is_organizer: false }]) ~summary.dtstart: Time(ItipTime { start: 867787200, tz_id: 32768 }) ~summary.summary: Text("Conference") BEGIN:VCALENDAR METHOD:REQUEST PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT STATUS:CONFIRMED SUMMARY:Phone Conference DTEND:19970701T190000Z DTSTART:19970701T180000Z ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE:mailto:e@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE;CUTYPE=ROOM:mailto:conf@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal;PARTSTAT=NEEDS-ACTION:mailto:d@e xample.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION:mailto:b@example. com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION:mailto:c@example. com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:0 SEQUENCE:2 END:VEVENT END:VCALENDAR # Send iTIP request to B, C, and D > send # Make sure C receives the request > get c@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT STATUS:CONFIRMED SUMMARY:Phone Conference DTEND:19970701T190000Z DTSTART:19970701T180000Z ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE:mailto:e@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE;CUTYPE=ROOM:mailto:conf@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal;PARTSTAT=NEEDS-ACTION:mailto:d@e xample.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION:mailto:b@example. com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION:mailto:c@example. com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:0 SEQUENCE:2 END:VEVENT END:VCALENDAR # "C" delegates presence at the meeting to "E". > put c@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Stalwart Labs LLC//Stalwart Server//EN BEGIN:VEVENT DTSTAMP:19970701T180000Z UID:calsrv.example.com-873970198738777@example.com ORGANIZER:mailto:a@example.com STATUS:CONFIRMED SEQUENCE:2 ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION:mailto:b@example. com ATTENDEE;PARTSTAT=DELEGATED;DELEGATED-TO="mailto:e@example.com":mailto:c@example.com ATTENDEE;RSVP=TRUE;DELEGATED-FROM="mailto:c@example.com":mailto:e@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal;PARTSTAT=NEEDS-ACTION:mailto:d@e xample.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE;CUTYPE=ROOM:mailto:conf@example.com DTSTART:19970701T180000Z DTEND:19970701T190000Z SUMMARY:Phone Conference END:VEVENT END:VCALENDAR > expect from: c@example.com to: a@example.com summary: rsvp DELEGATED summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }, ItipParticipant { email: "c@example.com", name: None, is_organizer: false }, ItipParticipant { email: "conf@example.com", name: None, is_organizer: false }, ItipParticipant { email: "d@example.com", name: Some("Hal"), is_organizer: false }, ItipParticipant { email: "e@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 867780000, tz_id: 32768 }) summary.summary: Text("Phone Conference") BEGIN:VCALENDAR METHOD:REPLY PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT SUMMARY:Phone Conference DTEND:19970701T190000Z DTSTART:19970701T180000Z ATTENDEE;PARTSTAT=DELEGATED;DELEGATED-TO="mailto:e@example.com":mailto:c@examp le.com ATTENDEE;RSVP=TRUE;DELEGATED-FROM="mailto:c@example.com":mailto:e@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:0 SEQUENCE:2 REQUEST-STATUS:2.0;Success END:VEVENT END:VCALENDAR ================================ from: c@example.com to: e@example.com summary: invite summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }, ItipParticipant { email: "c@example.com", name: None, is_organizer: false }, ItipParticipant { email: "conf@example.com", name: None, is_organizer: false }, ItipParticipant { email: "d@example.com", name: Some("Hal"), is_organizer: false }, ItipParticipant { email: "e@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 867780000, tz_id: 32768 }) summary.summary: Text("Phone Conference") BEGIN:VCALENDAR METHOD:REQUEST PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT STATUS:CONFIRMED SUMMARY:Phone Conference DTEND:19970701T190000Z DTSTART:19970701T180000Z ATTENDEE;PARTSTAT=DELEGATED;DELEGATED-TO="mailto:e@example.com":mailto:c@examp le.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE;CUTYPE=ROOM:mailto:conf@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal;PARTSTAT=NEEDS-ACTION:mailto:d@e xample.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION:mailto:b@example. com ATTENDEE;RSVP=TRUE;DELEGATED-FROM="mailto:c@example.com";PARTSTAT=NEEDS-ACTIO N:mailto:e@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:0 SEQUENCE:2 END:VEVENT END:VCALENDAR > send # Make sure E receives the request > get e@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT STATUS:CONFIRMED SUMMARY:Phone Conference DTEND:19970701T190000Z DTSTART:19970701T180000Z ATTENDEE;PARTSTAT=DELEGATED;DELEGATED-TO="mailto:e@example.com":mailto:c@ex ample.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE;CUTYPE=ROOM:mailto:conf@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal;PARTSTAT=NEEDS-ACTION:mailto:d@e xample.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION:mailto:b@example. com ATTENDEE;RSVP=TRUE;DELEGATED-FROM="mailto:c@example.com";PARTSTAT=NEEDS-AC TION:mailto:e@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:0 SEQUENCE:2 END:VEVENT END:VCALENDAR # Make sure A receives the request > get a@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT STATUS:CONFIRMED SUMMARY:Phone Conference DTEND:19970701T190000Z DTSTART:19970701T180000Z ATTENDEE;CUTYPE=INDIVIDUAL;PARTSTAT=DELEGATED;DELEGATED-TO="mailto:e@exam ple.com";SCHEDULE-STATUS=2.0:mailto:c@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE;CUTYPE=ROOM:mailto:conf@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL:mailto:b@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal:mailto:d@example.com ATTENDEE;RSVP=TRUE;DELEGATED-FROM="mailto:c@example.com":mailto:e@example.c om ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:2 SEQUENCE:2 END:VEVENT END:VCALENDAR # Delegate E accepts the request > put e@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Stalwart Labs LLC//Stalwart Server//EN BEGIN:VEVENT DTSTAMP:19970701T180000Z SEQUENCE:2 UID:calsrv.example.com-873970198738777@example.com ORGANIZER:mailto:a@example.com STATUS:CONFIRMED ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION:mailto:b@example. com ATTENDEE;PARTSTAT=DELEGATED;DELEGATED-TO="mailto:e@example.com":mailto:c@ex ample.com ATTENDEE;RSVP=TRUE;DELEGATED-FROM="mailto:c@example.com";PARTSTAT=ACCEPTED:mailto:e@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal;PARTSTAT=NEEDS-ACTION:mailto:d@e xample.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE;CUTYPE=ROOM:mailto:conf@example.com DTSTART:19970701T180000Z DTEND:19970701T190000Z SUMMARY:Phone Conference END:VEVENT END:VCALENDAR > expect from: e@example.com to: a@example.com, c@example.com summary: rsvp ACCEPTED summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }, ItipParticipant { email: "c@example.com", name: None, is_organizer: false }, ItipParticipant { email: "conf@example.com", name: None, is_organizer: false }, ItipParticipant { email: "d@example.com", name: Some("Hal"), is_organizer: false }, ItipParticipant { email: "e@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 867780000, tz_id: 32768 }) summary.summary: Text("Phone Conference") BEGIN:VCALENDAR METHOD:REPLY PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT SUMMARY:Phone Conference DTEND:19970701T190000Z DTSTART:19970701T180000Z ATTENDEE;PARTSTAT=DELEGATED;DELEGATED-TO="mailto:e@example.com":mailto:c@examp le.com ATTENDEE;RSVP=TRUE;DELEGATED-FROM="mailto:c@example.com";PARTSTAT=ACCEPTED:mai lto:e@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:0 SEQUENCE:2 REQUEST-STATUS:2.0;Success END:VEVENT END:VCALENDAR > send # Make sure A receives the reply > get a@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT STATUS:CONFIRMED SUMMARY:Phone Conference DTEND:19970701T190000Z DTSTART:19970701T180000Z ATTENDEE;CUTYPE=INDIVIDUAL;PARTSTAT=DELEGATED;DELEGATED-TO="mailto:e@exam ple.com";SCHEDULE-STATUS=2.0:mailto:c@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE;CUTYPE=ROOM:mailto:conf@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL:mailto:b@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal:mailto:d@example.com ATTENDEE;RSVP=TRUE;DELEGATED-FROM="mailto:c@example.com";PARTSTAT=ACCEPTED; SCHEDULE-STATUS=2.0:mailto:e@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:2 SEQUENCE:2 END:VEVENT END:VCALENDAR # Make sure C receives the reply > get c@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT STATUS:CONFIRMED SUMMARY:Phone Conference DTEND:19970701T190000Z DTSTART:19970701T180000Z ATTENDEE;PARTSTAT=DELEGATED;DELEGATED-TO="mailto:e@example.com":mailto:c@ex ample.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE;CUTYPE=ROOM:mailto:conf@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal;PARTSTAT=NEEDS-ACTION:mailto:d@e xample.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION:mailto:b@example. com ATTENDEE;RSVP=TRUE;DELEGATED-FROM="mailto:c@example.com";PARTSTAT=ACCEPTED; SCHEDULE-STATUS=2.0:mailto:e@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:3 SEQUENCE:2 END:VEVENT END:VCALENDAR # Delegate E declines the request > put e@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Stalwart Labs LLC//Stalwart Server//EN BEGIN:VEVENT DTSTAMP:19970701T180000Z SEQUENCE:2 UID:calsrv.example.com-873970198738777@example.com ORGANIZER:mailto:a@example.com STATUS:CONFIRMED ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION:mailto:b@example. com ATTENDEE;PARTSTAT=DELEGATED;DELEGATED-TO="mailto:e@example.com":mailto:c@ex ample.com ATTENDEE;RSVP=TRUE;DELEGATED-FROM="mailto:c@example.com";PARTSTAT=DECLINED:mailto:e@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal;PARTSTAT=NEEDS-ACTION:mailto:d@e xample.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE;CUTYPE=ROOM:mailto:conf@example.com DTSTART:19970701T180000Z DTEND:19970701T190000Z SUMMARY:Phone Conference END:VEVENT END:VCALENDAR > expect from: e@example.com to: a@example.com, c@example.com summary: rsvp DECLINED summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }, ItipParticipant { email: "c@example.com", name: None, is_organizer: false }, ItipParticipant { email: "conf@example.com", name: None, is_organizer: false }, ItipParticipant { email: "d@example.com", name: Some("Hal"), is_organizer: false }, ItipParticipant { email: "e@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 867780000, tz_id: 32768 }) summary.summary: Text("Phone Conference") BEGIN:VCALENDAR METHOD:REPLY PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT SUMMARY:Phone Conference DTEND:19970701T190000Z DTSTART:19970701T180000Z ATTENDEE;PARTSTAT=DELEGATED;DELEGATED-TO="mailto:e@example.com":mailto:c@examp le.com ATTENDEE;RSVP=TRUE;DELEGATED-FROM="mailto:c@example.com";PARTSTAT=DECLINED:mai lto:e@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:0 SEQUENCE:2 REQUEST-STATUS:2.0;Success END:VEVENT END:VCALENDAR # Send iTIP reply to A and C > send # Make sure C receives the reply > get c@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT STATUS:CONFIRMED SUMMARY:Phone Conference DTEND:19970701T190000Z DTSTART:19970701T180000Z ATTENDEE;PARTSTAT=DELEGATED;DELEGATED-TO="mailto:e@example.com":mailto:c@ex ample.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE;CUTYPE=ROOM:mailto:conf@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal;PARTSTAT=NEEDS-ACTION:mailto:d@e xample.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION:mailto:b@example. com ATTENDEE;RSVP=TRUE;DELEGATED-FROM="mailto:c@example.com";PARTSTAT=DECLINED; SCHEDULE-STATUS=2.0:mailto:e@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:3 SEQUENCE:2 END:VEVENT END:VCALENDAR # Make sure A receives the reply > get a@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT STATUS:CONFIRMED SUMMARY:Phone Conference DTEND:19970701T190000Z DTSTART:19970701T180000Z ATTENDEE;CUTYPE=INDIVIDUAL;PARTSTAT=DELEGATED;DELEGATED-TO="mailto:e@exam ple.com";SCHEDULE-STATUS=2.0:mailto:c@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE;CUTYPE=ROOM:mailto:conf@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL:mailto:b@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal:mailto:d@example.com ATTENDEE;RSVP=TRUE;DELEGATED-FROM="mailto:c@example.com";PARTSTAT=DECLINED; SCHEDULE-STATUS=2.0:mailto:e@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:2 SEQUENCE:2 END:VEVENT END:VCALENDAR # Remove B and expect an iTIP cancelation > put a@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VEVENT ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;CUTYPE=INDIVIDUAL;PARTSTAT=DELEGATED;DELEGATED-TO="mailto:e@exam ple.com";SCHEDULE-STATUS=2.0:mailto:c@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal:mailto:d@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE;CUTYPE=ROOM:mailto:conf@example.com ATTENDEE;RSVP=TRUE;DELEGATED-FROM="mailto:c@example.com";PARTSTAT=DECLINED; SCHEDULE-STATUS=2.0:mailto:e@example.com DTSTART:19970701T180000Z DTEND:19970701T190000Z SUMMARY:Phone Conference UID:calsrv.example.com-873970198738777@example.com SEQUENCE:2 DTSTAMP:19970701T180000Z STATUS:CONFIRMED END:VEVENT END:VCALENDAR > expect from: a@example.com to: c@example.com, d@example.com summary: update REQUEST summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "c@example.com", name: None, is_organizer: false }, ItipParticipant { email: "conf@example.com", name: None, is_organizer: false }, ItipParticipant { email: "d@example.com", name: Some("Hal"), is_organizer: false }, ItipParticipant { email: "e@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 867780000, tz_id: 32768 }) summary.summary: Text("Phone Conference") ~summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }, ItipParticipant { email: "c@example.com", name: None, is_organizer: false }, ItipParticipant { email: "conf@example.com", name: None, is_organizer: false }, ItipParticipant { email: "d@example.com", name: Some("Hal"), is_organizer: false }, ItipParticipant { email: "e@example.com", name: None, is_organizer: false }]) BEGIN:VCALENDAR METHOD:REQUEST PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT STATUS:CONFIRMED SUMMARY:Phone Conference DTEND:19970701T190000Z DTSTART:19970701T180000Z ATTENDEE;CUTYPE=INDIVIDUAL;PARTSTAT=DELEGATED;DELEGATED-TO="mailto:e@example .com":mailto:c@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE;CUTYPE=ROOM:mailto:conf@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal;PARTSTAT=NEEDS-ACTION:mailto:d@e xample.com ATTENDEE;RSVP=TRUE;DELEGATED-FROM="mailto:c@example.com";PARTSTAT=DECLINED:mai lto:e@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:0 SEQUENCE:3 END:VEVENT END:VCALENDAR ================================ from: a@example.com to: b@example.com summary: cancel summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }, ItipParticipant { email: "c@example.com", name: None, is_organizer: false }, ItipParticipant { email: "conf@example.com", name: None, is_organizer: false }, ItipParticipant { email: "d@example.com", name: Some("Hal"), is_organizer: false }, ItipParticipant { email: "e@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 867780000, tz_id: 32768 }) summary.summary: Text("Phone Conference") BEGIN:VCALENDAR METHOD:CANCEL PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT STATUS:CANCELLED SUMMARY:Phone Conference DTEND:19970701T190000Z DTSTART:19970701T180000Z ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL:mailto:b@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:0 SEQUENCE:3 END:VEVENT END:VCALENDAR # Send iTIP cancelation > send # Make sure B receives the cancelation > get b@example.com calsrv.example.com-873970198738777@example.com BEGIN:VCALENDAR PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT STATUS:CANCELLED SUMMARY:Phone Conference DTEND:19970701T190000Z DTSTART:19970701T180000Z ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL:mailto:b@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:0 END:VEVENT END:VCALENDAR # Delete the event from A's calendar and expect an iTIP cancelation > delete a@example.com calsrv.example.com-873970198738777@example.com > expect from: a@example.com to: c@example.com, d@example.com summary: cancel summary.dtstart: Time(ItipTime { start: 867780000, tz_id: 32768 }) summary.summary: Text("Phone Conference") BEGIN:VCALENDAR METHOD:CANCEL PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT STATUS:CANCELLED SUMMARY:Phone Conference DTEND:19970701T190000Z DTSTART:19970701T180000Z ATTENDEE;CUTYPE=INDIVIDUAL;PARTSTAT=DELEGATED;DELEGATED-TO="mailto:e@example .com";SCHEDULE-STATUS=2.0:mailto:c@example.com ATTENDEE;ROLE=CHAIR;PARTSTAT=ACCEPTED:mailto:a@example.com ATTENDEE;ROLE=NON-PARTICIPANT;RSVP=FALSE;CUTYPE=ROOM:mailto:conf@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;CN=Hal:mailto:d@example.com ATTENDEE;RSVP=TRUE;DELEGATED-FROM="mailto:c@example.com";PARTSTAT=DECLINED; SCHEDULE-STATUS=2.0:mailto:e@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777@example.com DTSTAMP:0 SEQUENCE:4 END:VEVENT END:VCALENDAR ================================================ FILE: tests/resources/itip/rfc5546_todo.txt ================================================ # RFC5546 - Todo Request # A sample todo is sent from "A" to "B", "C", and "D". > put a@example.com calsrv.example.com-873970198738777-00@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN METHOD:REQUEST VERSION:2.0 BEGIN:VTODO ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR:mailto:a@example.com ATTENDEE;RSVP=TRUE:mailto:b@example.com ATTENDEE;RSVP=TRUE:mailto:c@example.com ATTENDEE;RSVP=TRUE:mailto:d@example.com DTSTART:19970701T170000Z DUE:19970722T170000Z PRIORITY:1 SUMMARY:Create the requirements document UID:calsrv.example.com-873970198738777-00@example.com SEQUENCE:0 DTSTAMP:19970717T200000Z STATUS:NEEDS-ACTION END:VTODO END:VCALENDAR > expect from: a@example.com to: b@example.com, c@example.com, d@example.com summary: invite summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }, ItipParticipant { email: "c@example.com", name: None, is_organizer: false }, ItipParticipant { email: "d@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 867776400, tz_id: 32768 }) summary.summary: Text("Create the requirements document") BEGIN:VCALENDAR METHOD:REQUEST PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VTODO PRIORITY:1 STATUS:NEEDS-ACTION SUMMARY:Create the requirements document DUE:19970722T170000Z DTSTART:19970701T170000Z ATTENDEE;ROLE=CHAIR;PARTSTAT=NEEDS-ACTION:mailto:a@example.com ATTENDEE;RSVP=TRUE;PARTSTAT=NEEDS-ACTION:mailto:b@example.com ATTENDEE;RSVP=TRUE;PARTSTAT=NEEDS-ACTION:mailto:c@example.com ATTENDEE;RSVP=TRUE;PARTSTAT=NEEDS-ACTION:mailto:d@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777-00@example.com DTSTAMP:0 SEQUENCE:1 END:VTODO END:VCALENDAR # Send iTIP request to the attendees > send # Make sure B received the todo > get b@example.com calsrv.example.com-873970198738777-00@example.com BEGIN:VCALENDAR PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VTODO PRIORITY:1 STATUS:NEEDS-ACTION SUMMARY:Create the requirements document DUE:19970722T170000Z DTSTART:19970701T170000Z ATTENDEE;ROLE=CHAIR;PARTSTAT=NEEDS-ACTION:mailto:a@example.com ATTENDEE;RSVP=TRUE;PARTSTAT=NEEDS-ACTION:mailto:b@example.com ATTENDEE;RSVP=TRUE;PARTSTAT=NEEDS-ACTION:mailto:c@example.com ATTENDEE;RSVP=TRUE;PARTSTAT=NEEDS-ACTION:mailto:d@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777-00@example.com DTSTAMP:0 SEQUENCE:1 END:VTODO END:VCALENDAR # "B" accepts the to-do. > put b@example.com calsrv.example.com-873970198738777-00@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VTODO ORGANIZER:mailto:a@example.com ATTENDEE;PARTSTAT=ACCEPTED:mailto:b@example.com UID:calsrv.example.com-873970198738777-00@example.com COMMENT:I'll send you my input by email SEQUENCE:1 PRIORITY:1 STATUS:IN-PROCESS DTSTART:19970701T170000Z DUE:19970722T170000Z DTSTAMP:19970717T203000Z END:VTODO END:VCALENDAR > expect from: b@example.com to: a@example.com summary: rsvp ACCEPTED summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 867776400, tz_id: 32768 }) BEGIN:VCALENDAR METHOD:REPLY PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VTODO STATUS:IN-PROCESS DUE:19970722T170000Z DTSTART:19970701T170000Z ATTENDEE;PARTSTAT=ACCEPTED:mailto:b@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777-00@example.com DTSTAMP:0 SEQUENCE:1 REQUEST-STATUS:2.0;Success END:VTODO END:VCALENDAR # Send iTIP reply to the organizer > send # Make sure "A" received the reply > get a@example.com calsrv.example.com-873970198738777-00@example.com BEGIN:VCALENDAR METHOD:REQUEST PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VTODO PRIORITY:1 STATUS:IN-PROCESS SUMMARY:Create the requirements document DUE:19970722T170000Z DTSTART:19970701T170000Z ATTENDEE;PARTSTAT=ACCEPTED;SCHEDULE-STATUS=2.0:mailto:b@example.com ATTENDEE;ROLE=CHAIR:mailto:a@example.com ATTENDEE;RSVP=TRUE:mailto:c@example.com ATTENDEE;RSVP=TRUE:mailto:d@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777-00@example.com DTSTAMP:1 SEQUENCE:1 END:VTODO END:VCALENDAR # "B" updates percent completion of the to-do. > put b@example.com calsrv.example.com-873970198738777-00@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VTODO ORGANIZER:mailto:a@example.com PERCENT-COMPLETE:75 ATTENDEE;PARTSTAT=IN-PROCESS:mailto:b@example.com UID:calsrv.example.com-873970198738777-00@example.com COMMENT:I'll send you my input by email SEQUENCE:1 PRIORITY:1 STATUS:IN-PROCESS DTSTART:19970701T170000Z DUE:19970722T170000Z DTSTAMP:19970717T203000Z END:VTODO END:VCALENDAR > expect from: b@example.com to: a@example.com summary: rsvp IN-PROCESS summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 867776400, tz_id: 32768 }) BEGIN:VCALENDAR METHOD:REPLY PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VTODO PERCENT-COMPLETE:75 STATUS:IN-PROCESS DUE:19970722T170000Z DTSTART:19970701T170000Z ATTENDEE;PARTSTAT=IN-PROCESS:mailto:b@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777-00@example.com DTSTAMP:0 SEQUENCE:1 REQUEST-STATUS:2.0;Success END:VTODO END:VCALENDAR # Send iTIP reply to the organizer > send # Make sure "A" received the reply > get a@example.com calsrv.example.com-873970198738777-00@example.com BEGIN:VCALENDAR METHOD:REQUEST PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VTODO PERCENT-COMPLETE:75 PRIORITY:1 STATUS:IN-PROCESS SUMMARY:Create the requirements document DUE:19970722T170000Z DTSTART:19970701T170000Z ATTENDEE;PARTSTAT=IN-PROCESS;SCHEDULE-STATUS=2.0:mailto:b@example.com ATTENDEE;ROLE=CHAIR:mailto:a@example.com ATTENDEE;RSVP=TRUE:mailto:c@example.com ATTENDEE;RSVP=TRUE:mailto:d@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777-00@example.com DTSTAMP:1 SEQUENCE:1 END:VTODO END:VCALENDAR # "D" completed the to-do. > put d@example.com calsrv.example.com-873970198738777-00@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VTODO ORGANIZER:mailto:a@example.com ATTENDEE;PARTSTAT=COMPLETED:mailto:d@example.com UID:calsrv.example.com-873970198738777-00@example.com COMMENT:I'll send you my input by email SEQUENCE:1 PRIORITY:1 DTSTART:19970701T170000Z DUE:19970722T170000Z DTSTAMP:19970717T203000Z END:VTODO END:VCALENDAR > expect from: d@example.com to: a@example.com summary: rsvp COMPLETED summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "d@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 867776400, tz_id: 32768 }) BEGIN:VCALENDAR METHOD:REPLY PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VTODO DUE:19970722T170000Z DTSTART:19970701T170000Z ATTENDEE;PARTSTAT=COMPLETED:mailto:d@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777-00@example.com DTSTAMP:0 SEQUENCE:1 REQUEST-STATUS:2.0;Success END:VTODO END:VCALENDAR # Send iTIP reply to the organizer > send # Make sure "A" received the reply > get a@example.com calsrv.example.com-873970198738777-00@example.com BEGIN:VCALENDAR METHOD:REQUEST PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VTODO PERCENT-COMPLETE:75 PRIORITY:1 STATUS:IN-PROCESS SUMMARY:Create the requirements document DUE:19970722T170000Z DTSTART:19970701T170000Z ATTENDEE;PARTSTAT=COMPLETED;SCHEDULE-STATUS=2.0:mailto:d@example.com ATTENDEE;PARTSTAT=IN-PROCESS;SCHEDULE-STATUS=2.0:mailto:b@example.com ATTENDEE;ROLE=CHAIR:mailto:a@example.com ATTENDEE;RSVP=TRUE:mailto:c@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777-00@example.com DTSTAMP:1 SEQUENCE:1 END:VTODO END:VCALENDAR # Recurring to-do request > put a@example.com calsrv.example.com-873970198738777-00@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VTODO ORGANIZER:mailto:a@example.com ATTENDEE;ROLE=CHAIR:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL:mailto:b@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL:mailto:d@example.com RRULE:FREQ=MONTHLY;COUNT=10;BYDAY=1FR DTSTART:19980101T100000Z DUE:19980103T100000Z SUMMARY:Send Status Reports to Area Managers UID:calsrv.example.com-873970198738777-00@example.com SEQUENCE:0 DTSTAMP:19970717T200000Z STATUS:NEEDS-ACTION PRIORITY:1 END:VTODO END:VCALENDAR > expect from: a@example.com to: b@example.com, d@example.com summary: update REQUEST summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }, ItipParticipant { email: "d@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 883648800, tz_id: 32768 }) summary.rrule: Rrule(ICalendarRecurrenceRule { freq: Monthly, until: None, count: Some(10), interval: None, bysecond: [], byminute: [], byhour: [], byday: [ICalendarDay { ordwk: Some(1), weekday: Friday }], bymonthday: [], byyearday: [], byweekno: [], bymonth: [], bysetpos: [], wkst: None, rscale: None, skip: None }) summary.summary: Text("Send Status Reports to Area Managers") ~summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }, ItipParticipant { email: "c@example.com", name: None, is_organizer: false }, ItipParticipant { email: "d@example.com", name: None, is_organizer: false }]) ~summary.dtstart: Time(ItipTime { start: 867776400, tz_id: 32768 }) ~summary.summary: Text("Create the requirements document") BEGIN:VCALENDAR METHOD:REQUEST PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VTODO PRIORITY:1 STATUS:NEEDS-ACTION SUMMARY:Send Status Reports to Area Managers DUE:19980103T100000Z DTSTART:19980101T100000Z ATTENDEE;ROLE=CHAIR;PARTSTAT=NEEDS-ACTION:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION:mailto:b@example. com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION:mailto:d@example. com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777-00@example.com RRULE:FREQ=MONTHLY;COUNT=10;BYDAY=1FR DTSTAMP:0 SEQUENCE:1 END:VTODO END:VCALENDAR ================================ from: a@example.com to: c@example.com summary: cancel summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }, ItipParticipant { email: "c@example.com", name: None, is_organizer: false }, ItipParticipant { email: "d@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 867776400, tz_id: 32768 }) summary.rrule: Rrule(ICalendarRecurrenceRule { freq: Monthly, until: None, count: Some(10), interval: None, bysecond: [], byminute: [], byhour: [], byday: [ICalendarDay { ordwk: Some(1), weekday: Friday }], bymonthday: [], byyearday: [], byweekno: [], bymonth: [], bysetpos: [], wkst: None, rscale: None, skip: None }) summary.summary: Text("Create the requirements document") BEGIN:VCALENDAR METHOD:CANCEL PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VTODO STATUS:CANCELLED SUMMARY:Create the requirements document DUE:19970722T170000Z DTSTART:19970701T170000Z ATTENDEE;RSVP=TRUE:mailto:c@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777-00@example.com DTSTAMP:0 SEQUENCE:2 END:VTODO END:VCALENDAR > send # Make sure "C" received the cancel request > get c@example.com calsrv.example.com-873970198738777-00@example.com BEGIN:VCALENDAR PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VTODO PRIORITY:1 STATUS:CANCELLED SUMMARY:Create the requirements document DUE:19970722T170000Z DTSTART:19970701T170000Z ATTENDEE;RSVP=TRUE:mailto:c@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777-00@example.com DTSTAMP:0 END:VTODO END:VCALENDAR # Make sure "B" received the updated to-do > get b@example.com calsrv.example.com-873970198738777-00@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VTODO COMMENT:I'll send you my input by email PRIORITY:1 STATUS:NEEDS-ACTION SUMMARY:Send Status Reports to Area Managers DUE:19980103T100000Z DTSTART:19980101T100000Z ATTENDEE;ROLE=CHAIR;PARTSTAT=NEEDS-ACTION:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION:mailto:b@example. com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION:mailto:d@example. com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777-00@example.com RRULE:FREQ=MONTHLY;COUNT=10;BYDAY=1FR DTSTAMP:0 SEQUENCE:1 END:VTODO END:VCALENDAR # Reply to an instance of a recurring to-do > put b@example.com calsrv.example.com-873970198738777-00@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VTODO COMMENT:I'll send you my input by email PRIORITY:1 STATUS:NEEDS-ACTION SUMMARY:Send Status Reports to Area Managers DUE:19980103T100000Z DTSTART:19980101T100000Z ATTENDEE;ROLE=CHAIR;PARTSTAT=NEEDS-ACTION:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION:mailto:b@example. com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION:mailto:d@example. com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777-00@example.com RRULE:FREQ=MONTHLY;COUNT=10;BYDAY=1FR DTSTAMP:19970717T233000Z SEQUENCE:1 END:VTODO BEGIN:VTODO ORGANIZER:mailto:a@example.com ATTENDEE;PARTSTAT=IN-PROCESS:mailto:b@example.com PERCENT-COMPLETE:75 UID:calsrv.example.com-873970198738777-00@example.com DTSTAMP:19970717T233000Z RECURRENCE-ID:19980101T170000Z SEQUENCE:1 END:VTODO END:VCALENDAR > expect from: b@example.com to: a@example.com summary: rsvp NEEDS-ACTION summary.attendee: Participants([ItipParticipant { email: "a@example.com", name: None, is_organizer: true }, ItipParticipant { email: "b@example.com", name: None, is_organizer: false }, ItipParticipant { email: "d@example.com", name: None, is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 883648800, tz_id: 32768 }) summary.rrule: Rrule(ICalendarRecurrenceRule { freq: Monthly, until: None, count: Some(10), interval: None, bysecond: [], byminute: [], byhour: [], byday: [ICalendarDay { ordwk: Some(1), weekday: Friday }], bymonthday: [], byyearday: [], byweekno: [], bymonth: [], bysetpos: [], wkst: None, rscale: None, skip: None }) summary.summary: Text("Send Status Reports to Area Managers") BEGIN:VCALENDAR METHOD:REPLY PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VTODO PERCENT-COMPLETE:75 ATTENDEE;PARTSTAT=IN-PROCESS:mailto:b@example.com ORGANIZER:mailto:a@example.com RECURRENCE-ID:19980101T170000Z UID:calsrv.example.com-873970198738777-00@example.com DTSTAMP:0 SEQUENCE:1 REQUEST-STATUS:2.0;Success END:VTODO END:VCALENDAR > send # Make sure "A" received the reply > get a@example.com calsrv.example.com-873970198738777-00@example.com BEGIN:VCALENDAR PRODID:-//Example/ExampleCalendarClient//EN VERSION:2.0 BEGIN:VTODO PRIORITY:1 STATUS:NEEDS-ACTION SUMMARY:Send Status Reports to Area Managers DUE:19980103T100000Z DTSTART:19980101T100000Z ATTENDEE;ROLE=CHAIR:mailto:a@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL:mailto:b@example.com ATTENDEE;RSVP=TRUE;CUTYPE=INDIVIDUAL:mailto:d@example.com ORGANIZER:mailto:a@example.com UID:calsrv.example.com-873970198738777-00@example.com RRULE:FREQ=MONTHLY;COUNT=10;BYDAY=1FR DTSTAMP:1 SEQUENCE:1 END:VTODO BEGIN:VTODO PERCENT-COMPLETE:75 ATTENDEE;PARTSTAT=IN-PROCESS:mailto:b@example.com ORGANIZER:mailto:a@example.com RECURRENCE-ID:19980101T170000Z UID:calsrv.example.com-873970198738777-00@example.com DTSTAMP:0 SEQUENCE:1 END:VTODO END:VCALENDAR ================================================ FILE: tests/resources/itip/rfc6638_recurring.txt ================================================ # RFC6638 - Recurring Event Scheduling # Organizer invites participant to a recurring event > put cyrus@example.com 9263504FD3AD BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VTIMEZONE TZID:America/Montreal BEGIN:STANDARD DTSTART:20071104T020000 RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU TZNAME:EST TZOFFSETFROM:-0400 TZOFFSETTO:-0500 END:STANDARD BEGIN:DAYLIGHT DTSTART:20070311T020000 RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU TZNAME:EDT TZOFFSETFROM:-0500 TZOFFSETTO:-0400 END:DAYLIGHT END:VTIMEZONE BEGIN:VEVENT UID:9263504FD3AD SEQUENCE:0 DTSTAMP:20090602T185254Z DTSTART;TZID=America/Montreal:20090601T150000 DTEND;TZID=America/Montreal:20090601T160000 RRULE:FREQ=DAILY;INTERVAL=1;COUNT=5 TRANSP:OPAQUE SUMMARY:Review Internet-Draft ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com ATTENDEE;CN="Cyrus Daboo";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED:mailto:cyrus@example.com ATTENDEE;CN="Bernard Desruisseaux";CUTYPE=INDIVIDUAL;ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:bernard@example.net END:VEVENT END:VCALENDAR > expect from: cyrus@example.com to: bernard@example.net summary: invite summary.attendee: Participants([ItipParticipant { email: "cyrus@example.com", name: Some("Cyrus Daboo"), is_organizer: true }, ItipParticipant { email: "bernard@example.net", name: Some("Bernard Desruisseaux"), is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 1243882800, tz_id: 167 }) summary.rrule: Rrule(ICalendarRecurrenceRule { freq: Daily, until: None, count: Some(5), interval: Some(1), bysecond: [], byminute: [], byhour: [], byday: [], bymonthday: [], byyearday: [], byweekno: [], bymonth: [], bysetpos: [], wkst: None, rscale: None, skip: None }) summary.summary: Text("Review Internet-Draft") BEGIN:VCALENDAR METHOD:REQUEST PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT SUMMARY:Review Internet-Draft DTEND;TZID=America/Montreal:20090601T160000 DTSTART;TZID=America/Montreal:20090601T150000 TRANSP:OPAQUE ATTENDEE;CN="Bernard Desruisseaux";CUTYPE=INDIVIDUAL;ROLE=REQ-PARTICIPANT; RSVP=TRUE;PARTSTAT=NEEDS-ACTION:mailto:bernard@example.net ATTENDEE;CN="Cyrus Daboo";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED:mailto:cyrus@e xample.com ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com UID:9263504FD3AD RRULE:FREQ=DAILY;COUNT=5;INTERVAL=1 DTSTAMP:0 SEQUENCE:1 END:VEVENT BEGIN:VTIMEZONE TZID:America/Montreal BEGIN:STANDARD DTSTART:20071104T020000 TZNAME:EST TZOFFSETFROM:-0400 TZOFFSETTO:-0500 RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=11 END:STANDARD BEGIN:DAYLIGHT DTSTART:20070311T020000 TZNAME:EDT TZOFFSETFROM:-0500 TZOFFSETTO:-0400 RRULE:FREQ=YEARLY;BYDAY=2SU;BYMONTH=3 END:DAYLIGHT END:VTIMEZONE END:VCALENDAR # Send iTIP message to participant > send # Make sure the participant receives the event > get bernard@example.net 9263504FD3AD BEGIN:VCALENDAR PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT SUMMARY:Review Internet-Draft DTEND;TZID=America/Montreal:20090601T160000 DTSTART;TZID=America/Montreal:20090601T150000 TRANSP:OPAQUE ATTENDEE;CN="Bernard Desruisseaux";CUTYPE=INDIVIDUAL;ROLE=REQ-PARTICIPANT;RSVP= TRUE;PARTSTAT=NEEDS-ACTION:mailto:bernard@example.net ATTENDEE;CN="Cyrus Daboo";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED:mailto:cyrus@e xample.com ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com UID:9263504FD3AD RRULE:FREQ=DAILY;COUNT=5;INTERVAL=1 DTSTAMP:0 SEQUENCE:1 END:VEVENT BEGIN:VTIMEZONE TZID:America/Montreal BEGIN:STANDARD DTSTART:20071104T020000 TZNAME:EST TZOFFSETFROM:-0400 TZOFFSETTO:-0500 RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=11 END:STANDARD BEGIN:DAYLIGHT DTSTART:20070311T020000 TZNAME:EDT TZOFFSETFROM:-0500 TZOFFSETTO:-0400 RRULE:FREQ=YEARLY;BYDAY=2SU;BYMONTH=3 END:DAYLIGHT END:VTIMEZONE END:VCALENDAR # Participant declines an instance of the recurring event > put bernard@example.net 9263504FD3AD BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VTIMEZONE TZID:America/Montreal BEGIN:STANDARD DTSTART:20071104T020000 RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU TZNAME:EST TZOFFSETFROM:-0400 TZOFFSETTO:-0500 END:STANDARD BEGIN:DAYLIGHT DTSTART:20070311T020000 RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU TZNAME:EDT TZOFFSETFROM:-0500 TZOFFSETTO:-0400 END:DAYLIGHT END:VTIMEZONE BEGIN:VEVENT UID:9263504FD3AD SEQUENCE:1 DTSTAMP:20090602T185254Z DTSTART;TZID=America/Montreal:20090601T150000 DTEND;TZID=America/Montreal:20090601T160000 RRULE:FREQ=DAILY;INTERVAL=1;COUNT=5 TRANSP:OPAQUE SUMMARY:Review Internet-Draft ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com ATTENDEE;CN="Cyrus Daboo";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED:mailto:cyrus@example.com ATTENDEE;CN="Bernard Desruisseaux";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED;ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:bernard@example.net END:VEVENT BEGIN:VEVENT UID:9263504FD3AD SEQUENCE:1 DTSTAMP:20090603T183823Z RECURRENCE-ID;TZID=America/Montreal:20090602T150000 DTSTART;TZID=America/Montreal:20090602T150000 DTEND;TZID=America/Montreal:20090602T160000 TRANSP:TRANSPARENT SUMMARY:Review Internet-Draft ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com ATTENDEE;CN="Cyrus Daboo";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED:mailto:cyrus@example.com ATTENDEE;CN="Bernard Desruisseaux";CUTYPE=INDIVIDUAL;PARTSTAT=DECLINED;ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:bernard@example.net END:VEVENT END:VCALENDAR > expect from: bernard@example.net to: cyrus@example.com summary: rsvp ACCEPTED summary.attendee: Participants([ItipParticipant { email: "cyrus@example.com", name: Some("Cyrus Daboo"), is_organizer: true }, ItipParticipant { email: "bernard@example.net", name: Some("Bernard Desruisseaux"), is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 1243882800, tz_id: 167 }) summary.rrule: Rrule(ICalendarRecurrenceRule { freq: Daily, until: None, count: Some(5), interval: Some(1), bysecond: [], byminute: [], byhour: [], byday: [], bymonthday: [], byyearday: [], byweekno: [], bymonth: [], bysetpos: [], wkst: None, rscale: None, skip: None }) summary.summary: Text("Review Internet-Draft") BEGIN:VCALENDAR METHOD:REPLY PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT SUMMARY:Review Internet-Draft DTEND;TZID=America/Montreal:20090601T160000 DTSTART;TZID=America/Montreal:20090601T150000 ATTENDEE;CN="Bernard Desruisseaux";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED; ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:bernard@example.net ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com UID:9263504FD3AD DTSTAMP:0 SEQUENCE:1 REQUEST-STATUS:2.0;Success END:VEVENT BEGIN:VEVENT SUMMARY:Review Internet-Draft DTEND;TZID=America/Montreal:20090602T160000 DTSTART;TZID=America/Montreal:20090602T150000 ATTENDEE;CN="Bernard Desruisseaux";CUTYPE=INDIVIDUAL;PARTSTAT=DECLINED; ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:bernard@example.net ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com RECURRENCE-ID;TZID=America/Montreal:20090602T150000 UID:9263504FD3AD DTSTAMP:0 SEQUENCE:1 REQUEST-STATUS:2.0;Success END:VEVENT BEGIN:VTIMEZONE TZID:America/Montreal BEGIN:STANDARD DTSTART:20071104T020000 TZNAME:EST TZOFFSETFROM:-0400 TZOFFSETTO:-0500 RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=11 END:STANDARD BEGIN:DAYLIGHT DTSTART:20070311T020000 TZNAME:EDT TZOFFSETFROM:-0500 TZOFFSETTO:-0400 RRULE:FREQ=YEARLY;BYDAY=2SU;BYMONTH=3 END:DAYLIGHT END:VTIMEZONE END:VCALENDAR # Send iTIP message to organizer > send # Organizer receives the response > get cyrus@example.com 9263504FD3AD BEGIN:VCALENDAR PRODID:-//Example Corp.//CalDAV Client//EN VERSION:2.0 BEGIN:VEVENT SUMMARY:Review Internet-Draft DTEND;TZID=America/Montreal:20090601T160000 DTSTART;TZID=America/Montreal:20090601T150000 TRANSP:OPAQUE ATTENDEE;CN="Bernard Desruisseaux";CUTYPE=INDIVIDUAL;ROLE=REQ-PARTICIPANT;RSVP= TRUE;PARTSTAT=ACCEPTED;SCHEDULE-STATUS=2.0:mailto:bernard@example.net ATTENDEE;CN="Cyrus Daboo";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED:mailto:cyrus@e xample.com ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com UID:9263504FD3AD RRULE:FREQ=DAILY;COUNT=5;INTERVAL=1 DTSTAMP:1 SEQUENCE:1 END:VEVENT BEGIN:VEVENT ATTENDEE;CN="Bernard Desruisseaux";CUTYPE=INDIVIDUAL;PARTSTAT=DECLINED;ROLE= REQ-PARTICIPANT;RSVP=TRUE:mailto:bernard@example.net ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com RECURRENCE-ID;TZID=America/Montreal:20090602T150000 UID:9263504FD3AD DTSTAMP:0 SEQUENCE:1 END:VEVENT BEGIN:VTIMEZONE TZID:America/Montreal BEGIN:STANDARD DTSTART:20071104T020000 TZNAME:EST TZOFFSETFROM:-0400 TZOFFSETTO:-0500 RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=11 END:STANDARD BEGIN:DAYLIGHT DTSTART:20070311T020000 TZNAME:EDT TZOFFSETFROM:-0500 TZOFFSETTO:-0400 RRULE:FREQ=YEARLY;BYDAY=2SU;BYMONTH=3 END:DAYLIGHT END:VTIMEZONE END:VCALENDAR # Participant removes an instance of the recurring event > put bernard@example.net 9263504FD3AD BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VTIMEZONE TZID:America/Montreal BEGIN:STANDARD DTSTART:20071104T020000 RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU TZNAME:EST TZOFFSETFROM:-0400 TZOFFSETTO:-0500 END:STANDARD BEGIN:DAYLIGHT DTSTART:20070311T020000 RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU TZNAME:EDT TZOFFSETFROM:-0500 TZOFFSETTO:-0400 END:DAYLIGHT END:VTIMEZONE BEGIN:VEVENT UID:9263504FD3AD SEQUENCE:1 DTSTAMP:20090602T185254Z DTSTART;TZID=America/Montreal:20090601T150000 DTEND;TZID=America/Montreal:20090601T160000 RRULE:FREQ=DAILY;INTERVAL=1;COUNT=5 EXDATE;TZID=America/Montreal:20090603T150000 TRANSP:OPAQUE SUMMARY:Review Internet-Draft ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com ATTENDEE;CN="Cyrus Daboo";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED:mailto:cyrus@example.com ATTENDEE;CN="Bernard Desruisseaux";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED;ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:bernard@example.net END:VEVENT BEGIN:VEVENT UID:9263504FD3AD SEQUENCE:1 DTSTAMP:20090603T183823Z RECURRENCE-ID;TZID=America/Montreal:20090602T150000 DTSTART;TZID=America/Montreal:20090602T150000 DTEND;TZID=America/Montreal:20090602T160000 TRANSP:TRANSPARENT SUMMARY:Review Internet-Draft ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com ATTENDEE;CN="Cyrus Daboo";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED:mailto:cyrus@example.com ATTENDEE;CN="Bernard Desruisseaux";CUTYPE=INDIVIDUAL;PARTSTAT=DECLINED;ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:bernard@example.net END:VEVENT END:VCALENDAR > expect from: bernard@example.net to: cyrus@example.com summary: rsvp DECLINED summary.attendee: Participants([ItipParticipant { email: "cyrus@example.com", name: Some("Cyrus Daboo"), is_organizer: true }, ItipParticipant { email: "bernard@example.net", name: Some("Bernard Desruisseaux"), is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 1243882800, tz_id: 167 }) summary.rrule: Rrule(ICalendarRecurrenceRule { freq: Daily, until: None, count: Some(5), interval: Some(1), bysecond: [], byminute: [], byhour: [], byday: [], bymonthday: [], byyearday: [], byweekno: [], bymonth: [], bysetpos: [], wkst: None, rscale: None, skip: None }) summary.summary: Text("Review Internet-Draft") BEGIN:VCALENDAR METHOD:REPLY PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT SUMMARY:Review Internet-Draft DTEND;TZID=America/Montreal:20090601T160000 DTSTART;TZID=America/Montreal:20090601T150000 ATTENDEE;PARTSTAT=DECLINED:mailto:bernard@example.net ORGANIZER:mailto:cyrus@example.com RECURRENCE-ID;TZID=America/Montreal:20090603T150000 UID:9263504FD3AD DTSTAMP:0 SEQUENCE:1 END:VEVENT BEGIN:VTIMEZONE TZID:America/Montreal BEGIN:STANDARD DTSTART:20071104T020000 TZNAME:EST TZOFFSETFROM:-0400 TZOFFSETTO:-0500 RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=11 END:STANDARD BEGIN:DAYLIGHT DTSTART:20070311T020000 TZNAME:EDT TZOFFSETFROM:-0500 TZOFFSETTO:-0400 RRULE:FREQ=YEARLY;BYDAY=2SU;BYMONTH=3 END:DAYLIGHT END:VTIMEZONE END:VCALENDAR # Send iTIP message to organizer > send # Organizer receives the response > get cyrus@example.com 9263504FD3AD BEGIN:VCALENDAR PRODID:-//Example Corp.//CalDAV Client//EN VERSION:2.0 BEGIN:VEVENT ATTENDEE;PARTSTAT=DECLINED:mailto:bernard@example.net ORGANIZER:mailto:cyrus@example.com RECURRENCE-ID;TZID=America/Montreal:20090603T150000 UID:9263504FD3AD DTSTAMP:0 SEQUENCE:1 END:VEVENT BEGIN:VEVENT SUMMARY:Review Internet-Draft DTEND;TZID=America/Montreal:20090601T160000 DTSTART;TZID=America/Montreal:20090601T150000 TRANSP:OPAQUE ATTENDEE;CN="Bernard Desruisseaux";CUTYPE=INDIVIDUAL;ROLE=REQ-PARTICIPANT;RSVP= TRUE;PARTSTAT=ACCEPTED;SCHEDULE-STATUS=2.0:mailto:bernard@example.net ATTENDEE;CN="Cyrus Daboo";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED:mailto:cyrus@e xample.com ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com UID:9263504FD3AD RRULE:FREQ=DAILY;COUNT=5;INTERVAL=1 DTSTAMP:1 SEQUENCE:1 END:VEVENT BEGIN:VEVENT ATTENDEE;CN="Bernard Desruisseaux";CUTYPE=INDIVIDUAL;PARTSTAT=DECLINED;ROLE= REQ-PARTICIPANT;RSVP=TRUE:mailto:bernard@example.net ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com RECURRENCE-ID;TZID=America/Montreal:20090602T150000 UID:9263504FD3AD DTSTAMP:0 SEQUENCE:1 END:VEVENT BEGIN:VTIMEZONE TZID:America/Montreal BEGIN:STANDARD DTSTART:20071104T020000 TZNAME:EST TZOFFSETFROM:-0400 TZOFFSETTO:-0500 RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=11 END:STANDARD BEGIN:DAYLIGHT DTSTART:20070311T020000 TZNAME:EDT TZOFFSETFROM:-0500 TZOFFSETTO:-0400 RRULE:FREQ=YEARLY;BYDAY=2SU;BYMONTH=3 END:DAYLIGHT END:VTIMEZONE END:VCALENDAR ================================================ FILE: tests/resources/itip/rfc6638_single.txt ================================================ # RFC6638 - Simple Event Scheduling # Organizer Inviting Multiple Attendees > put cyrus@example.com 9263504FD3AD BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VEVENT UID:9263504FD3AD SEQUENCE:0 DTSTAMP:20090602T185254Z DTSTART:20090602T160000Z DTEND:20090602T170000Z TRANSP:OPAQUE SUMMARY:Lunch ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com ATTENDEE;CN="Cyrus Daboo";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED:mailto:cyrus@example.com ATTENDEE;CN="Wilfredo Sanchez Vega";CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION;ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:wilfredo@example.com ATTENDEE;CN="Bernard Desruisseaux";CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION;ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:bernard@example.net ATTENDEE;CN="Mike Douglass";CUTYPE=INDIVIDUAL;RSVP=TRUE:mailto:mike@example.org END:VEVENT END:VCALENDAR > expect from: cyrus@example.com to: bernard@example.net, mike@example.org, wilfredo@example.com summary: invite summary.attendee: Participants([ItipParticipant { email: "cyrus@example.com", name: Some("Cyrus Daboo"), is_organizer: true }, ItipParticipant { email: "bernard@example.net", name: Some("Bernard Desruisseaux"), is_organizer: false }, ItipParticipant { email: "mike@example.org", name: Some("Mike Douglass"), is_organizer: false }, ItipParticipant { email: "wilfredo@example.com", name: Some("Wilfredo Sanchez Vega"), is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 1243958400, tz_id: 32768 }) summary.summary: Text("Lunch") BEGIN:VCALENDAR METHOD:REQUEST PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT SUMMARY:Lunch DTEND:20090602T170000Z DTSTART:20090602T160000Z TRANSP:OPAQUE ATTENDEE;CN="Bernard Desruisseaux";CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION; ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:bernard@example.net ATTENDEE;CN="Cyrus Daboo";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED:mailto:cyrus@e xample.com ATTENDEE;CN="Mike Douglass";CUTYPE=INDIVIDUAL;RSVP=TRUE;PARTSTAT=NEEDS-ACTI ON:mailto:mike@example.org ATTENDEE;CN="Wilfredo Sanchez Vega";CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION; ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:wilfredo@example.com ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com UID:9263504FD3AD DTSTAMP:0 SEQUENCE:1 END:VEVENT END:VCALENDAR # Make sure the sequence number is updated > get cyrus@example.com 9263504FD3AD BEGIN:VCALENDAR PRODID:-//Example Corp.//CalDAV Client//EN VERSION:2.0 BEGIN:VEVENT SUMMARY:Lunch DTEND:20090602T170000Z DTSTART:20090602T160000Z TRANSP:OPAQUE ATTENDEE;CN="Bernard Desruisseaux";CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION; ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:bernard@example.net ATTENDEE;CN="Cyrus Daboo";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED:mailto:cyrus@e xample.com ATTENDEE;CN="Mike Douglass";CUTYPE=INDIVIDUAL;RSVP=TRUE:mailto:mike@example. org ATTENDEE;CN="Wilfredo Sanchez Vega";CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION; ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:wilfredo@example.com ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com UID:9263504FD3AD DTSTAMP:1 SEQUENCE:1 END:VEVENT END:VCALENDAR # Send iTIP message to attendees > send # Make sure the message was received by the attendees > get wilfredo@example.com 9263504FD3AD BEGIN:VCALENDAR PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT SUMMARY:Lunch DTEND:20090602T170000Z DTSTART:20090602T160000Z TRANSP:OPAQUE ATTENDEE;CN="Bernard Desruisseaux";CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION; ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:bernard@example.net ATTENDEE;CN="Cyrus Daboo";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED:mailto:cyrus@e xample.com ATTENDEE;CN="Mike Douglass";CUTYPE=INDIVIDUAL;RSVP=TRUE;PARTSTAT=NEEDS-ACTI ON:mailto:mike@example.org ATTENDEE;CN="Wilfredo Sanchez Vega";CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION; ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:wilfredo@example.com ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com UID:9263504FD3AD DTSTAMP:0 SEQUENCE:1 END:VEVENT END:VCALENDAR # Wilfredo accepts the invitation > put wilfredo@example.com 9263504FD3AD BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VEVENT UID:9263504FD3AD SEQUENCE:0 DTSTAMP:20090602T185254Z DTSTART:20090602T160000Z DTEND:20090602T170000Z TRANSP:OPAQUE SUMMARY:Lunch ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com ATTENDEE;CN="Cyrus Daboo";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED:mailto:cyrus@example.com ATTENDEE;CN="Wilfredo Sanchez Vega";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED;ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:wilfredo@example.com ATTENDEE;CN="Bernard Desruisseaux";CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION;ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:bernard@example.net ATTENDEE;CN="Mike Douglass";CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION;RSVP=TRUE:mailto:mike@example.org BEGIN:VALARM TRIGGER:-PT15M ACTION:DISPLAY DESCRIPTION:Reminder END:VALARM END:VEVENT END:VCALENDAR # Make sure the sequence number is not updated > get wilfredo@example.com 9263504FD3AD BEGIN:VCALENDAR PRODID:-//Example Corp.//CalDAV Client//EN VERSION:2.0 BEGIN:VEVENT SUMMARY:Lunch DTEND:20090602T170000Z DTSTART:20090602T160000Z TRANSP:OPAQUE ATTENDEE;CN="Bernard Desruisseaux";CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION; ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:bernard@example.net ATTENDEE;CN="Cyrus Daboo";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED:mailto:cyrus@e xample.com ATTENDEE;CN="Mike Douglass";CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION;RSVP=TR UE:mailto:mike@example.org ATTENDEE;CN="Wilfredo Sanchez Vega";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED;ROLE= REQ-PARTICIPANT;RSVP=TRUE:mailto:wilfredo@example.com ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com UID:9263504FD3AD DTSTAMP:1 SEQUENCE:0 BEGIN:VALARM DESCRIPTION:Reminder ACTION:DISPLAY TRIGGER:-PT15M END:VALARM END:VEVENT END:VCALENDAR > expect from: wilfredo@example.com to: cyrus@example.com summary: rsvp ACCEPTED summary.attendee: Participants([ItipParticipant { email: "cyrus@example.com", name: Some("Cyrus Daboo"), is_organizer: true }, ItipParticipant { email: "bernard@example.net", name: Some("Bernard Desruisseaux"), is_organizer: false }, ItipParticipant { email: "mike@example.org", name: Some("Mike Douglass"), is_organizer: false }, ItipParticipant { email: "wilfredo@example.com", name: Some("Wilfredo Sanchez Vega"), is_organizer: false }]) summary.dtstart: Time(ItipTime { start: 1243958400, tz_id: 32768 }) summary.summary: Text("Lunch") BEGIN:VCALENDAR METHOD:REPLY PRODID:-//Stalwart Labs LLC//Stalwart Server//EN VERSION:2.0 BEGIN:VEVENT SUMMARY:Lunch DTEND:20090602T170000Z DTSTART:20090602T160000Z ATTENDEE;CN="Wilfredo Sanchez Vega";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED; ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:wilfredo@example.com ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com UID:9263504FD3AD DTSTAMP:0 SEQUENCE:0 REQUEST-STATUS:2.0;Success END:VEVENT END:VCALENDAR # Send ITIP message to organizer > send # Make sure the message was received by the organizer > get cyrus@example.com 9263504FD3AD BEGIN:VCALENDAR PRODID:-//Example Corp.//CalDAV Client//EN VERSION:2.0 BEGIN:VEVENT SUMMARY:Lunch DTEND:20090602T170000Z DTSTART:20090602T160000Z TRANSP:OPAQUE ATTENDEE;CN="Bernard Desruisseaux";CUTYPE=INDIVIDUAL;PARTSTAT=NEEDS-ACTION; ROLE=REQ-PARTICIPANT;RSVP=TRUE:mailto:bernard@example.net ATTENDEE;CN="Cyrus Daboo";CUTYPE=INDIVIDUAL;PARTSTAT=ACCEPTED:mailto:cyrus@e xample.com ATTENDEE;CN="Mike Douglass";CUTYPE=INDIVIDUAL;RSVP=TRUE:mailto:mike@example. org ATTENDEE;CN="Wilfredo Sanchez Vega";CUTYPE=INDIVIDUAL;ROLE=REQ-PARTICIPANT; RSVP=TRUE;PARTSTAT=ACCEPTED;SCHEDULE-STATUS=2.0:mailto:wilfredo@example.com ORGANIZER;CN="Cyrus Daboo":mailto:cyrus@example.com UID:9263504FD3AD DTSTAMP:1 SEQUENCE:1 END:VEVENT END:VCALENDAR ================================================ FILE: tests/resources/jmap/email_get/headers.eml ================================================ From: Art Vandelay (Vandelay Industries) To: " James Smythe" , Friends: jane@example.com, =?UTF-8?Q?John_Sm=C3=AEth?= ; Cc: List 1: addr1@test.com, addr2@test.com; List 2: addr3@test.com, addr4@test.com; addr5@test.com, addr6@test.com Cc: =?utf-8?b?VGjDrXMgw61zIHbDoWzDrWQgw5pURjg=?=: addr1@test.com, addr2@test.com; =?utf-8?b?VGjDrXMgw61zIHbDoWzDrWQgw5pURjg=?=: addr3@test.com, addr4@test.com; addr5@test.com, addr6@test.com Bcc: Greg Vaudreuil , Ned Freed , Keith Moore Bcc: ietf-822@dimacs.rutgers.edu, ojarnef@admin.kth.se Date: Tue, 1 Jul 2003 10:52:37 +0200 Resent-Date: Tue, 2 Jul 2005 11:52:37 +0200 Resent-Date: Tue, 3 Jul 2005 12:52:37 +0300 Resent-Date: Tue, 4 Jul 2005 13:52:37 +0400 Message-ID: <5678.21-Nov-1997@example.com> References: <1234@local.machine.example> <3456@example.net> References: <789@local.machine.example> Keywords: multipart, alternative, example List-Post: (Postings are Moderated) List-Subscribe: (Use this command to join the list) List-Subscribe: (FTP), List-Owner: , List-Unsubscribe: (Use this command to get off the list) Subject: Why not both importing AND exporting? =?utf-8?b?4pi6?= X-Address-Single: =?ISO-8859-1?Q?Andr=E9?= Pirard X-Address: Mary Smith X-Address: John Doe X-AddressList-Single: Mary Smith , jdoe@example.org, Who? X-AddressList: =?US-ASCII*EN?Q?Keith_Moore?= , John =?US-ASCII*EN?Q?Doe?= X-AddressList: =?ISO-8859-1?Q?Keld_J=F8rn_Simonsen?= , =?ISO-8859-1?Q?Olle_J=E4rnefors?= X-AddressesGroup-Single: A Group(Some people) :Chris Jones , joe@example.org, John (my dear friend); (the end of the group) X-AddressesGroup: A Group:Ed Jones ,joe@where.test,John ; X-AddressesGroup: "List 1": addr1@test.com, addr2@test.com; "List 2": addr3@test.com, addr4@test.com; addr5@test.com, addr6@test.com X-List-Single: (X-Postings are Moderated) X-List: , X-List: , X-Text-Single: =?ISO-8859-1?B?SWYgeW91IGNhbiByZWFkIHRoaXMgeW8=?= =?ISO-8859-2?B?dSB1bmRlcnN0YW5kIHRoZSBleGFtcGxlLg==?= X-Text: =?iso-8859-1?q?this=20is=20some=20text?= X-Text: =?ISO-8859-1?Q?a?= =?ISO-8859-2?Q?_b?= X-Date-Single: Tue, 5 Jul 2006 13:52:37 -0500 X-Date: Sat, 20 Nov 2021 14:22:01 -0800 X-Date: Sun, 21 Nov 2021 15:23:02 -0900 X-Id-Single: X-Id: X-Id: Content-Type: multipart/mixed; boundary="festivus"; --festivus Content-Type: text/html; charset="us-ascii" Content-Transfer-Encoding: base64 X-Custom-Header: 123 PGh0bWw+PHA+SSB3YXMgdGhpbmtpbmcgYWJvdXQgcXVpdHRpbmcgdGhlICZsZHF1bztle HBvcnRpbmcmcmRxdW87IHRvIGZvY3VzIGp1c3Qgb24gdGhlICZsZHF1bztpbXBvcnRpbm cmcmRxdW87LDwvcD48cD5idXQgdGhlbiBJIHRob3VnaHQsIHdoeSBub3QgZG8gYm90aD8 gJiN4MjYzQTs8L3A+PC9odG1sPg== --festivus Content-Type: message/rfc822 X-Custom-Header-2: 345 From: "Cosmo Kramer" Subject: Exporting my book about coffee tables Content-Type: multipart/mixed; boundary="giddyup"; --giddyup Content-Type: text/plain; charset="utf-16" Content-Transfer-Encoding: quoted-printable =FF=FE=0C!5=D8"=DD5=D8)=DD5=D8-=DD =005=D8*=DD5=D8"=DD =005=D8"= =DD5=D85=DD5=D8-=DD5=D8,=DD5=D8/=DD5=D81=DD =005=D8*=DD5=D86=DD = =005=D8=1F=DD5=D8,=DD5=D8,=DD5=D8(=DD =005=D8-=DD5=D8)=DD5=D8"= =DD5=D8=1E=DD5=D80=DD5=D8"=DD!=00 --giddyup Content-Type: image/gif; name*1="about "; name*0="Book "; name*2*=utf-8''%e2%98%95 tables.gif Content-Transfer-Encoding: Base64 Content-Disposition: attachment R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7 --giddyup-- --festivus-- ================================================ FILE: tests/resources/jmap/email_get/headers.json ================================================ { "mailboxIds": { "a": true }, "keywords": { "tag": true }, "size": 4535, "receivedAt": "2113-09-16T10:13:20Z", "messageId": [ "5678.21-Nov-1997@example.com" ], "references": [ "789@local.machine.example", "abcd@example.net" ], "from": [ { "name": "Art Vandelay (Vandelay Industries)", "email": "art@vandelay.com" } ], "to": [ { "name": " James Smythe", "email": "james@example.com" }, { "name": null, "email": "jane@example.com" }, { "name": "John Smîth", "email": "john@example.com" } ], "cc": [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" }, { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" }, { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ], "bcc": [ { "name": null, "email": "ietf-822@dimacs.rutgers.edu" }, { "name": null, "email": "ojarnef@admin.kth.se" } ], "subject": "Why not both importing AND exporting? ☺", "sentAt": "2003-07-01T08:52:37Z", "bodyStructure": { "type": "multipart/mixed", "subParts": [ { "partId": "1", "size": 175, "type": "text/html", "charset": "us-ascii" }, { "partId": "2", "size": 723, "type": "message/rfc822" } ] }, "bodyValues": { "1": { "value": "

I was thinking about quitting the “exporting” to focus just on the “im...", "isEncodingProblem": false, "isTruncated": true } }, "textBody": [ { "partId": "1", "size": 175, "type": "text/html", "charset": "us-ascii" } ], "htmlBody": [ { "partId": "1", "size": 175, "type": "text/html", "charset": "us-ascii" } ], "attachments": [ { "partId": "2", "size": 723, "type": "message/rfc822" } ], "hasAttachment": true, "preview": "I was thinking about quitting the “exporting” to focus just on the “importing”,\nbut then I thought, why not do both? ☺\n", "header:Bcc": " ietf-822@dimacs.rutgers.edu, ojarnef@admin.kth.se", "header:Bcc:all": [ " Greg Vaudreuil , Ned Freed\n , Keith Moore ", " ietf-822@dimacs.rutgers.edu, ojarnef@admin.kth.se" ], "header:Bcc:asAddresses": [ { "name": null, "email": "ietf-822@dimacs.rutgers.edu" }, { "name": null, "email": "ojarnef@admin.kth.se" } ], "header:Bcc:asAddresses:all": [ [ { "name": "Greg Vaudreuil", "email": "gvaudre@NRI.Reston.VA.US" }, { "name": "Ned Freed", "email": "ned@innosoft.com" }, { "name": "Keith Moore", "email": "moore@cs.utk.edu" } ], [ { "name": null, "email": "ietf-822@dimacs.rutgers.edu" }, { "name": null, "email": "ojarnef@admin.kth.se" } ] ], "header:Bcc:asGroupedAddresses": [ { "name": null, "addresses": [ { "name": null, "email": "ietf-822@dimacs.rutgers.edu" }, { "name": null, "email": "ojarnef@admin.kth.se" } ] } ], "header:Bcc:asGroupedAddresses:all": [ [ { "name": null, "addresses": [ { "name": "Greg Vaudreuil", "email": "gvaudre@NRI.Reston.VA.US" }, { "name": "Ned Freed", "email": "ned@innosoft.com" }, { "name": "Keith Moore", "email": "moore@cs.utk.edu" } ] } ], [ { "name": null, "addresses": [ { "name": null, "email": "ietf-822@dimacs.rutgers.edu" }, { "name": null, "email": "ojarnef@admin.kth.se" } ] } ] ], "header:Cc": " =?utf-8?b?VGjDrXMgw61zIHbDoWzDrWQgw5pURjg=?=: addr1@test.com, \n addr2@test.com; =?utf-8?b?VGjDrXMgw61zIHbDoWzDrWQgw5pURjg=?=: \n addr3@test.com, addr4@test.com; addr5@test.com, addr6@test.com", "header:Cc:all": [ " List 1: addr1@test.com, addr2@test.com; List 2: addr3@test.com, \n addr4@test.com; addr5@test.com, addr6@test.com", " =?utf-8?b?VGjDrXMgw61zIHbDoWzDrWQgw5pURjg=?=: addr1@test.com, \n addr2@test.com; =?utf-8?b?VGjDrXMgw61zIHbDoWzDrWQgw5pURjg=?=: \n addr3@test.com, addr4@test.com; addr5@test.com, addr6@test.com" ], "header:Cc:asAddresses": [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" }, { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" }, { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ], "header:Cc:asAddresses:all": [ [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" }, { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" }, { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ], [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" }, { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" }, { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ] ], "header:Cc:asGroupedAddresses": [ { "name": "Thís ís válíd ÚTF8", "addresses": [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" } ] }, { "name": "Thís ís válíd ÚTF8", "addresses": [ { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" } ] }, { "name": null, "addresses": [ { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ] } ], "header:Cc:asGroupedAddresses:all": [ [ { "name": "List 1", "addresses": [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" } ] }, { "name": "List 2", "addresses": [ { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" } ] }, { "name": null, "addresses": [ { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ] } ], [ { "name": "Thís ís válíd ÚTF8", "addresses": [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" } ] }, { "name": "Thís ís válíd ÚTF8", "addresses": [ { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" } ] }, { "name": null, "addresses": [ { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ] } ] ], "header:Date": " Tue, 1 Jul 2003 10:52:37 +0200", "header:Date:all": [ " Tue, 1 Jul 2003 10:52:37 +0200" ], "header:Date:asDate": "2003-07-01T08:52:37Z", "header:Date:asDate:all": [ "2003-07-01T08:52:37Z" ], "header:From": " Art Vandelay (Vandelay Industries)", "header:From:all": [ " Art Vandelay (Vandelay Industries)" ], "header:From:asAddresses": [ { "name": "Art Vandelay (Vandelay Industries)", "email": "art@vandelay.com" } ], "header:From:asAddresses:all": [ [ { "name": "Art Vandelay (Vandelay Industries)", "email": "art@vandelay.com" } ] ], "header:From:asGroupedAddresses": [ { "name": null, "addresses": [ { "name": "Art Vandelay (Vandelay Industries)", "email": "art@vandelay.com" } ] } ], "header:From:asGroupedAddresses:all": [ [ { "name": null, "addresses": [ { "name": "Art Vandelay (Vandelay Industries)", "email": "art@vandelay.com" } ] } ] ], "header:Keywords": " multipart, alternative, example", "header:Keywords:all": [ " multipart, alternative, example" ], "header:Keywords:asText": "multipart, alternative, example", "header:Keywords:asText:all": [ "multipart, alternative, example" ], "header:List-Owner": " ,\n ", "header:List-Owner:all": [ " ,\n " ], "header:List-Owner:asURLs": [ "http://www.host.com/list.cgi?cmd=sub&lst=list", "mailto:list-manager@host.com?body=subscribe%20list" ], "header:List-Owner:asURLs:all": [ [ "http://www.host.com/list.cgi?cmd=sub&lst=list", "mailto:list-manager@host.com?body=subscribe%20list" ] ], "header:List-Post": " (Postings are Moderated)", "header:List-Post:all": [ " (Postings are Moderated)" ], "header:List-Post:asURLs": [ "mailto:moderator@host.com" ], "header:List-Post:asURLs:all": [ [ "mailto:moderator@host.com" ] ], "header:List-Subscribe": " (FTP), \n ", "header:List-Subscribe:all": [ " (Use this command to join the list)\n ", " (FTP), \n " ], "header:List-Subscribe:asURLs": [ "ftp://ftp.host.com/list.txt", "mailto:list@host.com?subject=subscribe" ], "header:List-Subscribe:asURLs:all": [ [ "mailto:list-manager@host.com?body=subscribe%20list" ], [ "ftp://ftp.host.com/list.txt", "mailto:list@host.com?subject=subscribe" ] ], "header:List-Unsubscribe": " (Use this command to get off the list)\n ", "header:List-Unsubscribe:all": [ " (Use this command to get off the list)\n " ], "header:List-Unsubscribe:asURLs": [ "mailto:list-manager@host.com?body=unsubscribe%20list" ], "header:List-Unsubscribe:asURLs:all": [ [ "mailto:list-manager@host.com?body=unsubscribe%20list" ] ], "header:Message-ID": " <5678.21-Nov-1997@example.com>", "header:Message-ID:all": [ " <5678.21-Nov-1997@example.com>" ], "header:Message-ID:asMessageIds": [ "5678.21-Nov-1997@example.com" ], "header:Message-ID:asMessageIds:all": [ [ "5678.21-Nov-1997@example.com" ] ], "header:References": " <789@local.machine.example>\n ", "header:References:all": [ " <1234@local.machine.example>\n <3456@example.net>", " <789@local.machine.example>\n " ], "header:References:asMessageIds": [ "789@local.machine.example", "abcd@example.net" ], "header:References:asMessageIds:all": [ [ "1234@local.machine.example", "3456@example.net" ], [ "789@local.machine.example", "abcd@example.net" ] ], "header:Resent-Date": " Tue, 4 Jul 2005 13:52:37 +0400", "header:Resent-Date:all": [ " Tue, 2 Jul 2005 11:52:37 +0200", " Tue, 3 Jul 2005 12:52:37 +0300", " Tue, 4 Jul 2005 13:52:37 +0400" ], "header:Resent-Date:asDate": "2005-07-04T09:52:37Z", "header:Resent-Date:asDate:all": [ "2005-07-02T09:52:37Z", "2005-07-03T09:52:37Z", "2005-07-04T09:52:37Z" ], "header:Subject": " Why not both importing AND exporting? =?utf-8?b?4pi6?=", "header:Subject:all": [ " Why not both importing AND exporting? =?utf-8?b?4pi6?=" ], "header:Subject:asText": "Why not both importing AND exporting? ☺", "header:Subject:asText:all": [ "Why not both importing AND exporting? ☺" ], "header:To": " \" James Smythe\" , Friends:\n jane@example.com, =?UTF-8?Q?John_Sm=C3=AEth?=\n ;", "header:To:all": [ " \" James Smythe\" , Friends:\n jane@example.com, =?UTF-8?Q?John_Sm=C3=AEth?=\n ;" ], "header:To:asAddresses": [ { "name": " James Smythe", "email": "james@example.com" }, { "name": null, "email": "jane@example.com" }, { "name": "John Smîth", "email": "john@example.com" } ], "header:To:asAddresses:all": [ [ { "name": " James Smythe", "email": "james@example.com" }, { "name": null, "email": "jane@example.com" }, { "name": "John Smîth", "email": "john@example.com" } ] ], "header:To:asGroupedAddresses": [ { "name": null, "addresses": [ { "name": " James Smythe", "email": "james@example.com" } ] }, { "name": "Friends", "addresses": [ { "name": null, "email": "jane@example.com" }, { "name": "John Smîth", "email": "john@example.com" } ] } ], "header:To:asGroupedAddresses:all": [ [ { "name": null, "addresses": [ { "name": " James Smythe", "email": "james@example.com" } ] }, { "name": "Friends", "addresses": [ { "name": null, "email": "jane@example.com" }, { "name": "John Smîth", "email": "john@example.com" } ] } ] ], "header:X-Address": " John Doe ", "header:X-Address:all": [ " Mary Smith ", " John Doe " ], "header:X-Address:asAddresses": [ { "name": "John Doe", "email": "jdoe@machine.example" } ], "header:X-Address:asAddresses:all": [ [ { "name": "Mary Smith", "email": "mary@example.net" } ], [ { "name": "John Doe", "email": "jdoe@machine.example" } ] ], "header:X-Address:asGroupedAddresses": [ { "name": null, "addresses": [ { "name": "John Doe", "email": "jdoe@machine.example" } ] } ], "header:X-Address:asGroupedAddresses:all": [ [ { "name": null, "addresses": [ { "name": "Mary Smith", "email": "mary@example.net" } ] } ], [ { "name": null, "addresses": [ { "name": "John Doe", "email": "jdoe@machine.example" } ] } ] ], "header:X-Address-Single": " =?ISO-8859-1?Q?Andr=E9?= Pirard ", "header:X-Address-Single:all": [ " =?ISO-8859-1?Q?Andr=E9?= Pirard " ], "header:X-Address-Single:asAddresses": [ { "name": "André Pirard", "email": "PIRARD@vm1.ulg.ac.be" } ], "header:X-Address-Single:asAddresses:all": [ [ { "name": "André Pirard", "email": "PIRARD@vm1.ulg.ac.be" } ] ], "header:X-Address-Single:asGroupedAddresses": [ { "name": null, "addresses": [ { "name": "André Pirard", "email": "PIRARD@vm1.ulg.ac.be" } ] } ], "header:X-Address-Single:asGroupedAddresses:all": [ [ { "name": null, "addresses": [ { "name": "André Pirard", "email": "PIRARD@vm1.ulg.ac.be" } ] } ] ], "header:X-AddressList": " =?ISO-8859-1?Q?Keld_J=F8rn_Simonsen?= ,\n =?ISO-8859-1?Q?Olle_J=E4rnefors?= ", "header:X-AddressList:all": [ " =?US-ASCII*EN?Q?Keith_Moore?= , \n John =?US-ASCII*EN?Q?Doe?= ", " =?ISO-8859-1?Q?Keld_J=F8rn_Simonsen?= ,\n =?ISO-8859-1?Q?Olle_J=E4rnefors?= " ], "header:X-AddressList:asAddresses": [ { "name": "Keld Jørn Simonsen", "email": "keld@dkuug.dk" }, { "name": "Olle Järnefors", "email": "ojarnef@admin.kth.se" } ], "header:X-AddressList:asAddresses:all": [ [ { "name": "Keith Moore", "email": "moore@cs.utk.edu" }, { "name": "John Doe", "email": "moore@cs.utk.edu" } ], [ { "name": "Keld Jørn Simonsen", "email": "keld@dkuug.dk" }, { "name": "Olle Järnefors", "email": "ojarnef@admin.kth.se" } ] ], "header:X-AddressList:asGroupedAddresses": [ { "name": null, "addresses": [ { "name": "Keld Jørn Simonsen", "email": "keld@dkuug.dk" }, { "name": "Olle Järnefors", "email": "ojarnef@admin.kth.se" } ] } ], "header:X-AddressList:asGroupedAddresses:all": [ [ { "name": null, "addresses": [ { "name": "Keith Moore", "email": "moore@cs.utk.edu" }, { "name": "John Doe", "email": "moore@cs.utk.edu" } ] } ], [ { "name": null, "addresses": [ { "name": "Keld Jørn Simonsen", "email": "keld@dkuug.dk" }, { "name": "Olle Järnefors", "email": "ojarnef@admin.kth.se" } ] } ] ], "header:X-AddressList-Single": " Mary Smith , jdoe@example.org, Who? ", "header:X-AddressList-Single:all": [ " Mary Smith , jdoe@example.org, Who? " ], "header:X-AddressList-Single:asAddresses": [ { "name": "Mary Smith", "email": "mary@x.test" }, { "name": null, "email": "jdoe@example.org" }, { "name": "Who?", "email": "one@y.test" } ], "header:X-AddressList-Single:asAddresses:all": [ [ { "name": "Mary Smith", "email": "mary@x.test" }, { "name": null, "email": "jdoe@example.org" }, { "name": "Who?", "email": "one@y.test" } ] ], "header:X-AddressList-Single:asGroupedAddresses": [ { "name": null, "addresses": [ { "name": "Mary Smith", "email": "mary@x.test" }, { "name": null, "email": "jdoe@example.org" }, { "name": "Who?", "email": "one@y.test" } ] } ], "header:X-AddressList-Single:asGroupedAddresses:all": [ [ { "name": null, "addresses": [ { "name": "Mary Smith", "email": "mary@x.test" }, { "name": null, "email": "jdoe@example.org" }, { "name": "Who?", "email": "one@y.test" } ] } ] ], "header:X-AddressesGroup": " \"List 1\": addr1@test.com, addr2@test.com; \"List 2\": \n addr3@test.com, addr4@test.com; addr5@test.com, addr6@test.com", "header:X-AddressesGroup:all": [ " A Group:Ed Jones ,joe@where.test,John ;", " \"List 1\": addr1@test.com, addr2@test.com; \"List 2\": \n addr3@test.com, addr4@test.com; addr5@test.com, addr6@test.com" ], "header:X-AddressesGroup:asAddresses": [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" }, { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" }, { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ], "header:X-AddressesGroup:asAddresses:all": [ [ { "name": "Ed Jones", "email": "c@a.test" }, { "name": null, "email": "joe@where.test" }, { "name": "John", "email": "jdoe@one.test" } ], [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" }, { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" }, { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ] ], "header:X-AddressesGroup:asGroupedAddresses": [ { "name": "List 1", "addresses": [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" } ] }, { "name": "List 2", "addresses": [ { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" } ] }, { "name": null, "addresses": [ { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ] } ], "header:X-AddressesGroup:asGroupedAddresses:all": [ [ { "name": "A Group", "addresses": [ { "name": "Ed Jones", "email": "c@a.test" }, { "name": null, "email": "joe@where.test" }, { "name": "John", "email": "jdoe@one.test" } ] } ], [ { "name": "List 1", "addresses": [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" } ] }, { "name": "List 2", "addresses": [ { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" } ] }, { "name": null, "addresses": [ { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ] } ] ], "header:X-AddressesGroup-Single": " A Group(Some people)\n :Chris Jones ,\n joe@example.org, John (my dear\n friend); (the end of the group)", "header:X-AddressesGroup-Single:all": [ " A Group(Some people)\n :Chris Jones ,\n joe@example.org, John (my dear\n friend); (the end of the group)" ], "header:X-AddressesGroup-Single:asAddresses": [ { "name": "Chris Jones (Chris's host.)", "email": "c@public.example" }, { "name": null, "email": "joe@example.org" }, { "name": "John (my dear friend)", "email": "jdoe@one.test" }, { "name": "the end of the group", "email": "" } ], "header:X-AddressesGroup-Single:asAddresses:all": [ [ { "name": "Chris Jones (Chris's host.)", "email": "c@public.example" }, { "name": null, "email": "joe@example.org" }, { "name": "John (my dear friend)", "email": "jdoe@one.test" }, { "name": "the end of the group", "email": "" } ] ], "header:X-AddressesGroup-Single:asGroupedAddresses": [ { "name": "A Group (Some people)", "addresses": [ { "name": "Chris Jones (Chris's host.)", "email": "c@public.example" }, { "name": null, "email": "joe@example.org" }, { "name": "John (my dear friend)", "email": "jdoe@one.test" } ] }, { "name": null, "addresses": [ { "name": "the end of the group", "email": "" } ] } ], "header:X-AddressesGroup-Single:asGroupedAddresses:all": [ [ { "name": "A Group (Some people)", "addresses": [ { "name": "Chris Jones (Chris's host.)", "email": "c@public.example" }, { "name": null, "email": "joe@example.org" }, { "name": "John (my dear friend)", "email": "jdoe@one.test" } ] }, { "name": null, "addresses": [ { "name": "the end of the group", "email": "" } ] } ] ], "header:X-Date": " Sun, 21 Nov 2021 15:23:02 -0900", "header:X-Date:all": [ " Sat, 20 Nov 2021 14:22:01 -0800", " Sun, 21 Nov 2021 15:23:02 -0900" ], "header:X-Date:asDate": "2021-11-22T00:23:02Z", "header:X-Date:asDate:all": [ "2021-11-20T22:22:01Z", "2021-11-22T00:23:02Z" ], "header:X-Date-Single": " Tue, 5 Jul 2006 13:52:37 -0500", "header:X-Date-Single:all": [ " Tue, 5 Jul 2006 13:52:37 -0500" ], "header:X-Date-Single:asDate": "2006-07-05T18:52:37Z", "header:X-Date-Single:asDate:all": [ "2006-07-05T18:52:37Z" ], "header:X-Id": " ", "header:X-Id:all": [ " ", " " ], "header:X-Id:asMessageIds": [ "myid4@example.com", "myid5@example.com" ], "header:X-Id:asMessageIds:all": [ [ "myid3@example.com" ], [ "myid4@example.com", "myid5@example.com" ] ], "header:X-Id-Single": " ", "header:X-Id-Single:all": [ " " ], "header:X-Id-Single:asMessageIds": [ "myid@example.com", "myid2@example.com" ], "header:X-Id-Single:asMessageIds:all": [ [ "myid@example.com", "myid2@example.com" ] ], "header:X-List": " ,\n ", "header:X-List:all": [ " ,\n ", " ,\n " ], "header:X-List:asURLs": [ "http://www.mylist2.com/list2", "mailto:list2@mylist2.com" ], "header:X-List:asURLs:all": [ [ "http://www.mylist.com/list", "mailto:list@mylist.com" ], [ "http://www.mylist2.com/list2", "mailto:list2@mylist2.com" ] ], "header:X-List-Single": " (X-Postings are Moderated)", "header:X-List-Single:all": [ " (X-Postings are Moderated)" ], "header:X-List-Single:asURLs": [ "mailto:x-moderator@host.com" ], "header:X-List-Single:asURLs:all": [ [ "mailto:x-moderator@host.com" ] ], "header:X-Text": " =?ISO-8859-1?Q?a?= =?ISO-8859-2?Q?_b?=", "header:X-Text:all": [ " =?iso-8859-1?q?this=20is=20some=20text?=", " =?ISO-8859-1?Q?a?= =?ISO-8859-2?Q?_b?=" ], "header:X-Text:asText": "a b", "header:X-Text:asText:all": [ "this is some text", "a b" ], "header:X-Text-Single": " =?ISO-8859-1?B?SWYgeW91IGNhbiByZWFkIHRoaXMgeW8=?=\n =?ISO-8859-2?B?dSB1bmRlcnN0YW5kIHRoZSBleGFtcGxlLg==?=", "header:X-Text-Single:all": [ " =?ISO-8859-1?B?SWYgeW91IGNhbiByZWFkIHRoaXMgeW8=?=\n =?ISO-8859-2?B?dSB1bmRlcnN0YW5kIHRoZSBleGFtcGxlLg==?=" ], "header:X-Text-Single:asText": "If you can read this you understand the example.", "header:X-Text-Single:asText:all": [ "If you can read this you understand the example." ] } ================================================ FILE: tests/resources/jmap/email_get/message_attachment.eml ================================================ From: Al Gore To: White House Transportation Coordinator Subject: [Fwd: Map of Argentina with Description] Content-Type: multipart/mixed; boundary="D7F------------D7FD5A0B8AB9C65CCDBFA872" This is a multi-part message in MIME format. --D7F------------D7FD5A0B8AB9C65CCDBFA872 Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit Fred, Fire up Air Force One! We're going South! Thanks, Al --D7F------------D7FD5A0B8AB9C65CCDBFA872 Content-Type: message/rfc822 Content-Transfer-Encoding: 7bit Content-Disposition: inline Return-Path: Received: from mailhost.whitehouse.gov ([192.168.51.200]) by heartbeat.whitehouse.gov (8.8.8/8.8.8) with ESMTP id SAA22453 for ; Mon, 13 Aug 1998 l8:14:23 +1000 Received: from the_big_box.whitehouse.gov ([192.168.51.50]) by mailhost.whitehouse.gov (8.8.8/8.8.7) with ESMTP id RAA20366 for vice-president@whitehouse.gov; Mon, 13 Aug 1998 17:42:41 +1000 Date: Mon, 13 Aug 1998 17:42:41 +1000 Message-Id: <199804130742.RAA20366@mai1host.whitehouse.gov> From: Bill Clinton To: A1 (The Enforcer) Gore Subject: Map of Argentina with Description MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="DC8------------DC8638F443D87A7F0726DEF7" This is a multi-part message in MIME format. --DC8------------DC8638F443D87A7F0726DEF7 Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit Hi A1, I finally figured out this MIME thing. Pretty cool. I'll send you some sax music in .au files next week! Anyway, the attached image is really too small to get a good look at Argentina. Try this for a much better map: http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm Then again, shouldn't the CIA have something like that? Bill --DC8------------DC8638F443D87A7F0726DEF7 Content-Type: image/gif; name="map_of_Argentina.gif" Content-Transfer-Encoding: base64 Content-Disposition: inline; fi1ename="map_of_Argentina.gif" R01GOD1hJQA1AKIAAP/////78P/omn19fQAAAAAAAAAAAAAAACwAAAAAJQA1AAAD7Qi63P5w wEmjBCLrnQnhYCgM1wh+pkgqqeC9XrutmBm7hAK3tP31gFcAiFKVQrGFR6kscnonTe7FAAad GugmRu3CmiBt57fsVq3Y0VFKnpYdxPC6M7Ze4crnnHum4oN6LFJ1bn5NXTN7OF5fQkN5WYow BEN2dkGQGWJtSzqGTICJgnQuTJN/WJsojad9qXMuhIWdjXKjY4tenjo6tjVssk2gaWq3uGNX U6ZGxseyk8SasGw3J9GRzdTQky1iHNvcPNNI4TLeKdfMvy0vMqLrItvuxfDW8ubjueDtJufz 7itICBxISKDBgwgTKjyYAAA7 --DC8------------DC8638F443D87A7F0726DEF7-- --D7F------------D7FD5A0B8AB9C65CCDBFA872-- ================================================ FILE: tests/resources/jmap/email_get/message_attachment.json ================================================ { "mailboxIds": { "a": true }, "keywords": { "tag": true }, "size": 2651, "receivedAt": "2054-01-02T20:53:20Z", "from": [ { "name": "Al Gore", "email": "vice-president@whitehouse.gov" } ], "to": [ { "name": "White House Transportation Coordinator", "email": "transport@whitehouse.gov" } ], "subject": "[Fwd: Map of Argentina with Description]", "bodyStructure": { "headers": [ { "name": "From", "value": " Al Gore " }, { "name": "To", "value": " White House Transportation Coordinator\n " }, { "name": "Subject", "value": " [Fwd: Map of Argentina with Description]" }, { "name": "Content-Type", "value": " multipart/mixed;\n boundary=\"D7F------------D7FD5A0B8AB9C65CCDBFA872\"" } ], "type": "multipart/mixed", "subParts": [ { "partId": "1", "blobId": "blob_0", "size": 61, "headers": [ { "name": "Content-Type", "value": " text/plain; charset=us-ascii" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "us-ascii" }, { "partId": "2", "blobId": "blob_1", "size": 1979, "headers": [ { "name": "Content-Type", "value": " message/rfc822" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" }, { "name": "Content-Disposition", "value": " inline" } ], "type": "message/rfc822", "disposition": "inline" } ] }, "bodyValues": { "1": { "value": "Fred,\n\nFire up Air Force One! We're going South!\n\nThanks,\nAl", "isEncodingProblem": false, "isTruncated": false } }, "textBody": [ { "partId": "1", "blobId": "blob_0", "size": 61, "headers": [ { "name": "Content-Type", "value": " text/plain; charset=us-ascii" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "us-ascii" } ], "htmlBody": [ { "partId": "1", "blobId": "blob_0", "size": 61, "headers": [ { "name": "Content-Type", "value": " text/plain; charset=us-ascii" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "us-ascii" } ], "attachments": [ { "partId": "2", "blobId": "blob_1", "size": 1979, "headers": [ { "name": "Content-Type", "value": " message/rfc822" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" }, { "name": "Content-Disposition", "value": " inline" } ], "type": "message/rfc822", "disposition": "inline" } ], "hasAttachment": true, "preview": "Fred,\n\nFire up Air Force One! We're going South!\n\nThanks,\nAl" } ================================================ FILE: tests/resources/jmap/email_get/multipart_alternative.eml ================================================ From: sender@example.com To: recipient@example.com Subject: Multipart Email Example Content-Type: multipart/alternative; boundary="boundary-string" --boundary-string Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Content-Disposition: inline Plain text email goes here! This is the fallback if email client does not support HTML --boundary-string Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: quoted-printable Content-Disposition: inline

This is the HTML Section!

This is what displays in most modern email clients

--boundary-string-- ================================================ FILE: tests/resources/jmap/email_get/multipart_alternative.json ================================================ { "mailboxIds": { "a": true }, "keywords": { "tag": true }, "size": 616, "receivedAt": "1989-07-09T15:06:40Z", "from": [ { "name": null, "email": "sender@example.com" } ], "to": [ { "name": null, "email": "recipient@example.com" } ], "subject": "Multipart Email Example", "bodyStructure": { "headers": [ { "name": "From", "value": " sender@example.com" }, { "name": "To", "value": " recipient@example.com" }, { "name": "Subject", "value": " Multipart Email Example" }, { "name": "Content-Type", "value": " multipart/alternative; boundary=\"boundary-string\"" } ], "type": "multipart/alternative", "subParts": [ { "partId": "1", "blobId": "blob_0", "size": 87, "headers": [ { "name": "Content-Type", "value": " text/plain; charset=\"utf-8\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" }, { "name": "Content-Disposition", "value": " inline" } ], "type": "text/plain", "charset": "utf-8", "disposition": "inline" }, { "partId": "2", "blobId": "blob_1", "size": 93, "headers": [ { "name": "Content-Type", "value": " text/html; charset=\"utf-8\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" }, { "name": "Content-Disposition", "value": " inline" } ], "type": "text/html", "charset": "utf-8", "disposition": "inline" } ] }, "bodyValues": { "1": { "value": "Plain text email goes here!\nThis is the fallback if email client does not support HTML\n", "isEncodingProblem": false, "isTruncated": false }, "2": { "value": "

This is the HTML Section!

\n

This is what displays in most modern email clients

\n", "isEncodingProblem": false, "isTruncated": false } }, "textBody": [ { "partId": "1", "blobId": "blob_0", "size": 87, "headers": [ { "name": "Content-Type", "value": " text/plain; charset=\"utf-8\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" }, { "name": "Content-Disposition", "value": " inline" } ], "type": "text/plain", "charset": "utf-8", "disposition": "inline" } ], "htmlBody": [ { "partId": "2", "blobId": "blob_1", "size": 93, "headers": [ { "name": "Content-Type", "value": " text/html; charset=\"utf-8\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" }, { "name": "Content-Disposition", "value": " inline" } ], "type": "text/html", "charset": "utf-8", "disposition": "inline" } ], "attachments": [], "hasAttachment": false, "preview": "Plain text email goes here!\nThis is the fallback if email client does not support HTML\n" } ================================================ FILE: tests/resources/jmap/email_get/multipart_cid.eml ================================================ From: "Doug Sauder" To: "Joe Blow" Subject: Test message from Microsoft Outlook 00 Date: Wed, 17 May 2000 19:44:45 -0400 Message-ID: MIME-Version: 1.0 Content-Type: multipart/related; boundary="----=_NextPart_000_000C_01BFC038.5A5C8E60" X-Priority: 3 (Normal) X-MSMail-Priority: Normal X-Mailer: Microsoft Outlook IMO, Build 9.0.2416 (9.0.2910.0) Importance: Normal X-MimeOLE: Produced By Microsoft MimeOLE V5.00.2314.1300 This is a multi-part message in MIME format. ------=_NextPart_000_000C_01BFC038.5A5C8E60 Content-Type: multipart/alternative; boundary="----=_NextPart_001_000D_01BFC038.5A5C8E60" ------=_NextPart_001_000D_01BFC038.5A5C8E60 Content-Type: text/plain; charset="iso-8859-1" Content-Transfer-Encoding: quoted-printable The Hare and the Tortoise=20 =20 A HARE one day ridiculed the short feet and slow pace of the Tortoise, = who replied, laughing: "Though you be swift as the wind, I will beat = you in a race." The Hare, believing her assertion to be simply = impossible, assented to the proposal; and they agreed that the Fox = should choose the course and fix the goal. On the day appointed for the = race the two started together. The Tortoise never for a moment stopped, = but went on with a slow but steady pace straight to the end of the = course. The Hare, lying down by the wayside, fell fast asleep. At last = waking up, and moving as fast as he could, he saw the Tortoise had = reached the goal, and was comfortably dozing after her fatigue. =20 =20 Slow but steady wins the race. =20 ------=_NextPart_001_000D_01BFC038.5A5C8E60 Content-Type: text/html; charset="iso-8859-1" Content-Transfer-Encoding: quoted-printable

The Hare and the Tortoise =
 
A HARE=20 one day ridiculed the short feet and slow pace of the Tortoise, who = replied,=20 laughing:  "Though you be swift as the wind, I will beat you in a=20 race."  The Hare, believing her assertion to be simply impossible, = assented=20 to the proposal; and they agreed that the Fox should choose the course = and fix=20 the goal.  On the day appointed for the race the two started=20 together.  The Tortoise never for a moment stopped, but went on = with a slow=20 but steady pace straight to the end of the course.  The Hare, lying = down by=20 the wayside, fell fast asleep.  At last waking up, and moving as = fast as he=20 could, he saw the Tortoise had reached the goal, and was comfortably = dozing=20 after her fatigue.  
 
3D"blue
 
Slow but = steady wins=20 the race. 
 
3D"red
------=_NextPart_001_000D_01BFC038.5A5C8E60-- ------=_NextPart_000_000C_01BFC038.5A5C8E60 Content-Type: image/png; name="blueball.png" Content-Transfer-Encoding: base64 Content-ID: <823504223@17052000-0f8d> iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8AAAgAABAAABgAAAAA CCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkIIWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQ MYwpUrU5Y8Y5Y84pWs4YSs4YQs4YQr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO 5+/n7++cxu9ShO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaMvee1 5++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAADBMg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/bfPn/vyh70lbssceb L5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEoQdvock4ne0IKMVUpKZLQDeqSTIsv+18P yqqWUw2IBsRM7307PPp+fDJrWtnpLDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XC UpaDeQwiMpHXP/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/MjRxm T6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8+VZmYqKmdd1CSYoOiMOS GwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B 1iuuzCGTxwXjnDO4d7NpbX42YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD /wEUVMzYmWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBpZ3VldDZz O7wAAAAASUVORK5CYII= ------=_NextPart_000_000C_01BFC038.5A5C8E60 Content-Type: image/png; name="redball.png" Content-Transfer-Encoding: base64 Content-ID: <823504223@17052000-0f94> iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8AAAABAAALAAAVAAAa AAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAjAAAWAAAmAABhAAB7AACGAACHAAB9AAB0 AABgAAA5AAAUAAAGAAAnAABLAABvAACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABM AAB3AACZAAC0GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHfhITm f3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5PlrKzpmZntZWXvJSXXAADB AACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADLICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2 AAB4AABeAABAAAAiAABXAACSAADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABH AAArAAAPAACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABPAAASAAAC AABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADIAADTAADNAACzAACDAABuAAAe AAB+AADAAACkAACNAAB/AABpAABQAAAwAACRAACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACs AACvAACtAACmAACJAAB6AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABV AACOAACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8AAA6AAAfAAAM AAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAD8LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkFBDlQJf8zC/EIi4iK iUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ 29ja2Ts4Ojkr6Li4urFDNf53N/Ow8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFW SE1LF4A69n9GZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2YnOAj+ d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1a/acUG5piNz/uXLzVJ2q m6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2TVjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqV tWXrtu07BJihcsw71+zanRW8Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZw HBqL//8flz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/joOyYed5 QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms1y9evXid7QZacgOxmSxktNzd tSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAAJXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5 IFl2ZXMgUGlndWV0NnM7vAAAAABJRU5ErkJggg== ------=_NextPart_000_000C_01BFC038.5A5C8E60-- ================================================ FILE: tests/resources/jmap/email_get/multipart_cid.json ================================================ { "mailboxIds": { "a": true }, "keywords": { "tag": true }, "size": 7477, "receivedAt": "2206-12-09T08:26:40Z", "messageId": [ "NDBBIAKOPKHFGPLCODIGKEKFCHAA.doug@example.com" ], "from": [ { "name": "Doug Sauder", "email": "doug@example.com" } ], "to": [ { "name": "Joe Blow", "email": "jblow@example.com" } ], "subject": "Test message from Microsoft Outlook 00", "sentAt": "2000-05-17T23:44:45Z", "bodyStructure": { "headers": [ { "name": "From", "value": " \"Doug Sauder\" " }, { "name": "To", "value": " \"Joe Blow\" " }, { "name": "Subject", "value": " Test message from Microsoft Outlook 00" }, { "name": "Date", "value": " Wed, 17 May 2000 19:44:45 -0400" }, { "name": "Message-ID", "value": " " }, { "name": "MIME-Version", "value": " 1.0" }, { "name": "Content-Type", "value": " multipart/related;\n\tboundary=\"----=_NextPart_000_000C_01BFC038.5A5C8E60\"" }, { "name": "X-Priority", "value": " 3 (Normal)" }, { "name": "X-MSMail-Priority", "value": " Normal" }, { "name": "X-Mailer", "value": " Microsoft Outlook IMO, Build 9.0.2416 (9.0.2910.0)" }, { "name": "Importance", "value": " Normal" }, { "name": "X-MimeOLE", "value": " Produced By Microsoft MimeOLE V5.00.2314.1300" } ], "type": "multipart/related", "subParts": [ { "headers": [ { "name": "Content-Type", "value": " multipart/alternative;\n\tboundary=\"----=_NextPart_001_000D_01BFC038.5A5C8E60\"" } ], "type": "multipart/alternative", "subParts": [ { "partId": "2", "blobId": "blob_0", "size": 761, "headers": [ { "name": "Content-Type", "value": " text/plain;\n\tcharset=\"iso-8859-1\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/plain", "charset": "iso-8859-1" }, { "partId": "3", "blobId": "blob_1", "size": 1442, "headers": [ { "name": "Content-Type", "value": " text/html;\n\tcharset=\"iso-8859-1\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/html", "charset": "iso-8859-1" } ] }, { "partId": "4", "blobId": "blob_2", "size": 1325, "headers": [ { "name": "Content-Type", "value": " image/png;\n\tname=\"blueball.png\"" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-ID", "value": " <823504223@17052000-0f8d>" } ], "name": "blueball.png", "type": "image/png", "cid": "823504223@17052000-0f8d" }, { "partId": "5", "blobId": "blob_3", "size": 1453, "headers": [ { "name": "Content-Type", "value": " image/png;\n\tname=\"redball.png\"" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-ID", "value": " <823504223@17052000-0f94>" } ], "name": "redball.png", "type": "image/png", "cid": "823504223@17052000-0f94" } ] }, "bodyValues": { "2": { "value": "\nThe Hare and the Tortoise \n \nA HARE one day ridiculed the short feet and slow pace of the Tortoi...", "isEncodingProblem": false, "isTruncated": true }, "3": { "value": "\n\n...", "isEncodingProblem": false, "isTruncated": true } }, "textBody": [ { "partId": "2", "blobId": "blob_0", "size": 761, "headers": [ { "name": "Content-Type", "value": " text/plain;\n\tcharset=\"iso-8859-1\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/plain", "charset": "iso-8859-1" } ], "htmlBody": [ { "partId": "3", "blobId": "blob_1", "size": 1442, "headers": [ { "name": "Content-Type", "value": " text/html;\n\tcharset=\"iso-8859-1\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/html", "charset": "iso-8859-1" } ], "attachments": [ { "partId": "4", "blobId": "blob_2", "size": 1325, "headers": [ { "name": "Content-Type", "value": " image/png;\n\tname=\"blueball.png\"" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-ID", "value": " <823504223@17052000-0f8d>" } ], "name": "blueball.png", "type": "image/png", "cid": "823504223@17052000-0f8d" }, { "partId": "5", "blobId": "blob_3", "size": 1453, "headers": [ { "name": "Content-Type", "value": " image/png;\n\tname=\"redball.png\"" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-ID", "value": " <823504223@17052000-0f94>" } ], "name": "redball.png", "type": "image/png", "cid": "823504223@17052000-0f94" } ], "hasAttachment": true, "preview": "\nThe Hare and the Tortoise \n \nA HARE one day ridiculed the short feet and slow pace of the Tortoise, who replied, laughing: \"Though you be swift as the wind, I will beat you in a race.\" The Hare, believing her assertion to be simply impossible, assent..." } ================================================ FILE: tests/resources/jmap/email_get/multipart_mixed.eml ================================================ MIME-Version: 1.0 From: Nathaniel Borenstein To: Ned Freed Date: Fri, 07 Oct 1994 16:15:05 -0700 (PDT) Subject: A multipart example Content-Type: multipart/mixed; boundary=unique-boundary-1 This is the preamble area of a multipart message. Mail readers that understand multipart format should ignore this preamble. If you are reading this text, you might want to consider changing to a mail reader that understands how to properly display multipart messages. --unique-boundary-1 ... Some text appears here ... [Note that the blank between the boundary and the start of the text in this part means no header fields were given and this is text in the US-ASCII character set. It could have been done with explicit typing as in the next part.] --unique-boundary-1 Content-type: text/plain; charset=US-ASCII This could have been part of the previous part, but illustrates explicit versus implicit typing of body parts. --unique-boundary-1 Content-Type: multipart/parallel; boundary=unique-boundary-2 --unique-boundary-2 Content-Type: audio/basic Content-Transfer-Encoding: base64 ... base64-encoded 8000 Hz single-channel mu-law-format audio data goes here ... --unique-boundary-2 Content-Type: image/jpeg Content-Transfer-Encoding: base64 ... base64-encoded image data goes here ... --unique-boundary-2-- --unique-boundary-1 Content-type: text/enriched This is enriched. as defined in RFC 1896 Isn't it cool? --unique-boundary-1 Content-Type: message/rfc822 From: (mailbox in US-ASCII) To: (address in US-ASCII) Subject: (subject in US-ASCII) Content-Type: Text/plain; charset=ISO-8859-1 Content-Transfer-Encoding: Quoted-printable ... Additional text in ISO-8859-1 goes here ... --unique-boundary-1-- ================================================ FILE: tests/resources/jmap/email_get/multipart_mixed.json ================================================ { "mailboxIds": { "a": true }, "keywords": { "tag": true }, "size": 1853, "receivedAt": "2028-09-19T18:13:20Z", "from": [ { "name": "Nathaniel Borenstein", "email": "nsb@nsb.fv.com" } ], "to": [ { "name": "Ned Freed", "email": "ned@innosoft.com" } ], "subject": "A multipart example", "sentAt": "1994-10-07T23:15:05Z", "bodyStructure": { "headers": [ { "name": "MIME-Version", "value": " 1.0" }, { "name": "From", "value": " Nathaniel Borenstein " }, { "name": "To", "value": " Ned Freed " }, { "name": "Date", "value": " Fri, 07 Oct 1994 16:15:05 -0700 (PDT)" }, { "name": "Subject", "value": " A multipart example" }, { "name": "Content-Type", "value": " multipart/mixed;\n boundary=unique-boundary-1" } ], "type": "multipart/mixed", "subParts": [ { "partId": "1", "blobId": "blob_0", "size": 262, "headers": [], "type": "text/plain", "charset": "us-ascii" }, { "partId": "2", "blobId": "blob_1", "size": 111, "headers": [ { "name": "Content-Type", "value": " text/plain; charset=US-ASCII" } ], "type": "text/plain", "charset": "US-ASCII" }, { "headers": [ { "name": "Content-Type", "value": " multipart/parallel; boundary=unique-boundary-2" } ], "type": "multipart/parallel", "subParts": [ { "partId": "4", "blobId": "blob_2", "size": 85, "headers": [ { "name": "Content-Type", "value": " audio/basic" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "type": "audio/basic", "charset": "us-ascii" }, { "partId": "5", "blobId": "blob_3", "size": 44, "headers": [ { "name": "Content-Type", "value": " image/jpeg" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "type": "image/jpeg", "charset": "us-ascii" } ] }, { "partId": "6", "blobId": "blob_4", "size": 140, "headers": [ { "name": "Content-Type", "value": " text/enriched" } ], "type": "text/enriched", "charset": "us-ascii" }, { "partId": "7", "blobId": "blob_5", "size": 223, "headers": [ { "name": "Content-Type", "value": " message/rfc822" } ], "type": "message/rfc822" } ] }, "bodyValues": { "1": { "value": "... Some text appears here ...\n\n[Note that the blank between the boundary and the start\nof the te...", "isEncodingProblem": false, "isTruncated": true }, "2": { "value": "This could have been part of the previous part, but\nillustrates explicit versus implicit typing o...", "isEncodingProblem": false, "isTruncated": true } }, "textBody": [ { "partId": "1", "blobId": "blob_0", "size": 262, "headers": [], "type": "text/plain", "charset": "us-ascii" }, { "partId": "2", "blobId": "blob_1", "size": 111, "headers": [ { "name": "Content-Type", "value": " text/plain; charset=US-ASCII" } ], "type": "text/plain", "charset": "US-ASCII" } ], "htmlBody": [ { "partId": "1", "blobId": "blob_0", "size": 262, "headers": [], "type": "text/plain", "charset": "us-ascii" }, { "partId": "2", "blobId": "blob_1", "size": 111, "headers": [ { "name": "Content-Type", "value": " text/plain; charset=US-ASCII" } ], "type": "text/plain", "charset": "US-ASCII" } ], "attachments": [ { "partId": "4", "blobId": "blob_2", "size": 85, "headers": [ { "name": "Content-Type", "value": " audio/basic" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "type": "audio/basic", "charset": "us-ascii" }, { "partId": "5", "blobId": "blob_3", "size": 44, "headers": [ { "name": "Content-Type", "value": " image/jpeg" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "type": "image/jpeg", "charset": "us-ascii" }, { "partId": "6", "blobId": "blob_4", "size": 140, "headers": [ { "name": "Content-Type", "value": " text/enriched" } ], "type": "text/enriched", "charset": "us-ascii" }, { "partId": "7", "blobId": "blob_5", "size": 223, "headers": [ { "name": "Content-Type", "value": " message/rfc822" } ], "type": "message/rfc822" } ], "hasAttachment": true, "preview": "... Some text appears here ...\n\n[Note that the blank between the boundary and the start\nof the text in this part means no header fields were\ngiven and this is text in the US-ASCII character set.\nIt could have been done with explicit typing as in the\nnex..." } ================================================ FILE: tests/resources/jmap/email_get/multipart_related.eml ================================================ Message-Id: <4.2.0.58.20000519003556.00a918e0@pop.example.com> X-Sender: dwsauder@pop.example.com (Unverified) X-Mailer: QUALCOMM Windows Eudora Pro Version 4.2.0.58 X-Priority: 2 (High) Date: Fri, 19 May 2000 00:36:58 -0400 To: Heinz =?iso-8859-1?Q?Muller?= From: Doug Sauder Subject: =?iso-8859-1?Q?Die_Hasen_und_die_Frosche?= Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="=====================_715392540==_" --=====================_715392540==_ Content-Type: multipart/related; type="multipart/alternative"; boundary="=====================_715392540==_.REL" --=====================_715392540==_.REL Content-Type: multipart/alternative; boundary="=====================_715392550==_.ALT" --=====================_715392550==_.ALT Content-Type: text/plain; charset="iso-8859-1" Content-Transfer-Encoding: quoted-printable Die Hasen und die Fr=F6sche Die Hasen klagten einst =FCber ihre mi=DFliche Lage; "wir leben", sprach ein= Redner, "in steter Furcht vor Menschen und Tieren, eine Beute der Hunde,= der Adler, ja fast aller Raubtiere! Unsere stete Angst ist =E4rger als der= Tod selbst. Auf, la=DFt uns ein f=FCr allemal sterben."=20 In einem nahen Teich wollten sie sich nun ers=E4ufen; sie eilten ihm zu;= allein das au=DFerordentliche Get=F6se und ihre wunderbare Gestalt= erschreckte eine Menge Fr=F6sche, die am Ufer sa=DFen, so sehr, da=DF sie= aufs schnellste untertauchten.=20 "Halt", rief nun eben dieser Sprecher, "wir wollen das Ers=E4ufen noch ein= wenig aufschieben, denn auch uns f=FCrchten, wie ihr seht, einige Tiere,= welche also wohl noch ungl=FCcklicher sein m=FCssen als wir."=20 2aa3ed95.png2aa3edd1.png --=====================_715392550==_.ALT Content-Type: text/html; charset="iso-8859-1" Content-Transfer-Encoding: quoted-printable Die Hasen und = die Fr=F6sche

Die Hasen klagten einst =FCber ihre mi=DFliche Lage; "wir leben", sprach ein Redner, "in steter Furcht vor Menschen und Tieren, eine Beute der Hunde, der Adler, ja fast aller Raubtiere! Unsere stete Angst ist =E4rger als der Tod selbst. Auf, la=DFt uns ein f=FCr allemal sterben."

In einem nahen Teich wollten sie sich nun ers=E4ufen; sie eilten ihm zu; allein das au=DFerordentliche Get=F6se und ihre wunderbare Gestalt erschreckte eine Menge Fr=F6sche, die am Ufer sa=DFen, so sehr, da=DF sie au= fs schnellste untertauchten.

"Halt", rief nun eben dieser Sprecher, "wir wollen das Ers=E4ufen noch ein wenig aufschieben, denn auch uns f=FCrchten, wie ihr seht, einige Tiere, welche also wohl noch ungl=FCcklicher sein m=FCssen als wir."

3D"2aa3ed95.png" --=====================_715392550==_.ALT-- --=====================_715392540==_.REL Content-Type: image/png; name="2aa3ed95.png" Content-ID: <4.2.0.58.20000519003556.00a918e0@pop.example.com.2> Content-Transfer-Encoding: base64 Content-Disposition: inline; filename="2aa3ed95.png" iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8AAAgAABAAABgAAAAA CCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkIIWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQ MYwpUrU5Y8Y5Y84pWs4YSs4YQs4YQr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO 5+/n7++cxu9ShO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaMvee1 5++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAADBMg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/bfPn/vyh70lbssceb L5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEoQdvock4ne0IKMVUpKZLQDeqSTIsv+18P yqqWUw2IBsRM7307PPp+fDJrWtnpLDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XC UpaDeQwiMpHXP/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/MjRxm T6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8+VZmYqKmdd1CSYoOiMOS GwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B 1iuuzCGTxwXjnDO4d7NpbX42YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD /wEUVMzYmWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBpZ3VldDZz O7wAAAAASUVORK5CYII= --=====================_715392540==_.REL Content-Type: image/png; name="2aa3edd1.png" Content-ID: <4.2.0.58.20000519003556.00a918e0@pop.example.com.3> Content-Transfer-Encoding: base64 Content-Disposition: inline; filename="2aa3edd1.png" iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8AAAABAAALAAAVAAAa AAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAjAAAWAAAmAABhAAB7AACGAACHAAB9AAB0 AABgAAA5AAAUAAAGAAAnAABLAABvAACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABM AAB3AACZAAC0GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHfhITm f3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5PlrKzpmZntZWXvJSXXAADB AACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADLICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2 AAB4AABeAABAAAAiAABXAACSAADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABH AAArAAAPAACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABPAAASAAAC AABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADIAADTAADNAACzAACDAABuAAAe AAB+AADAAACkAACNAAB/AABpAABQAAAwAACRAACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACs AACvAACtAACmAACJAAB6AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABV AACOAACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8AAA6AAAfAAAM AAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAD8LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkFBDlQJf8zC/EIi4iK iUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ 29ja2Ts4Ojkr6Li4urFDNf53N/Ow8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFW SE1LF4A69n9GZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2YnOAj+ d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1a/acUG5piNz/uXLzVJ2q m6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2TVjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqV tWXrtu07BJihcsw71+zanRW8Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZw HBqL//8flz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/joOyYed5 QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms1y9evXid7QZacgOxmSxktNzd tSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAAJXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5 IFl2ZXMgUGlndWV0NnM7vAAAAABJRU5ErkJggg== --=====================_715392540==_.REL-- --=====================_715392540==_ Content-Type: image/png; name="blueball.png" Content-Transfer-Encoding: base64 Content-Disposition: attachment; filename="blueball.png" iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8AAAgAABAAABgAAAAA CCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkIIWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQ MYwpUrU5Y8Y5Y84pWs4YSs4YQs4YQr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO 5+/n7++cxu9ShO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaMvee1 5++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAADBMg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/bfPn/vyh70lbssceb L5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEoQdvock4ne0IKMVUpKZLQDeqSTIsv+18P yqqWUw2IBsRM7307PPp+fDJrWtnpLDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XC UpaDeQwiMpHXP/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/MjRxm T6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8+VZmYqKmdd1CSYoOiMOS GwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B 1iuuzCGTxwXjnDO4d7NpbX42YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD /wEUVMzYmWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBpZ3VldDZz O7wAAAAASUVORK5CYII= --=====================_715392540==_ Content-Type: image/png; name="greenball.png" Content-Transfer-Encoding: base64 Content-Disposition: attachment; filename="greenball.png" iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8AAAAAEAAAGAAAIQAA CAAAMQAAQgAAUgAAWgAASgAIYwAIcwAIewAQjAAIawAAOQAAYwAQlAAQnAAhpQAQpQAhrQBCvRhj xjFjxjlSxiEpzgAYvQAQrQAYrQAhvQCU1mOt1nuE1lJK3hgh1gAYxgAYtQAAKQBCzhDO55Te563G 55SU52NS5yEh3gAYzgBS3iGc52vW75y974yE71JC7xCt73ul3nNa7ykh5wAY1gAx5wBS7yFr7zlK 7xgp5wAp7wAx7wAIhAAQtQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAp1fnZAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu MT1evmgAAAFtSURBVHicddJtV8IgFAdwD2zIgMEE1+NcqdsoK+m5tCyz7/+ZiLmHsyzvq53zO/cy +N9ery1bVe9PWQA9z4MQ+H8Yoj7GASZ95IHfaBGmLOSchyIgyOu22mgQSjUcDuNYcoGjLiLK1cHh 0fHJaTKKOcMItgYxT89OzsfjyTTLC8UF0c2ZNmKquJhczq6ub+YmSVUYRF59GeDastu7+9nD41Nm kiJ2jc2J3kAWZ9Pr55fH18XSmRuKUTXUaqHy7O19tfr4NFle/w3YDrWRUIlZrL/W86XJkyJVG9Ea EjIx2XyZmZJGioeUaL+2AY8TY8omR6nkLKhu70zjUKVJXsp3quS2DVSJWNh3zzJKCyexI0ZxBP3a fE0ElyqOlZJyw8r3BE2SFiJCyxA434SCkg65RhdeQBljQtCg39LWrA90RDDG1EWrYUO23hMANUKR Rl61E529cR++D2G5LK002dr/qrcfu9u0V3bxn/XdhR/NYeeN0ggsLAAAACV0RVh0Q29tbWVudABj bGlwMmdpZiB2LjAuNiBieSBZdmVzIFBpZ3VldDZzO7wAAAAASUVORK5CYII= --=====================_715392540==_-- ================================================ FILE: tests/resources/jmap/email_get/multipart_related.json ================================================ { "mailboxIds": { "a": true }, "keywords": { "tag": true }, "size": 11304, "receivedAt": "2328-03-18T08:00:00Z", "messageId": [ "4.2.0.58.20000519003556.00a918e0@pop.example.com" ], "from": [ { "name": "Doug Sauder", "email": "dwsauder@example.com" } ], "to": [ { "name": "Heinz Muller", "email": "mueller@example.com" } ], "subject": "Die Hasen und die Frosche", "sentAt": "2000-05-19T04:36:58Z", "bodyStructure": { "headers": [ { "name": "Message-ID", "value": " <4.2.0.58.20000519003556.00a918e0@pop.example.com>" }, { "name": "X-Sender", "value": " dwsauder@pop.example.com (Unverified)" }, { "name": "X-Mailer", "value": " QUALCOMM Windows Eudora Pro Version 4.2.0.58" }, { "name": "X-Priority", "value": " 2 (High)" }, { "name": "Date", "value": " Fri, 19 May 2000 00:36:58 -0400" }, { "name": "To", "value": " Heinz =?iso-8859-1?Q?Muller?= " }, { "name": "From", "value": " Doug Sauder " }, { "name": "Subject", "value": " =?iso-8859-1?Q?Die_Hasen_und_die_Frosche?=" }, { "name": "MIME-Version", "value": " 1.0" }, { "name": "Content-Type", "value": " multipart/mixed;\n\tboundary=\"=====================_715392540==_\"" } ], "type": "multipart/mixed", "subParts": [ { "headers": [ { "name": "Content-Type", "value": " multipart/related;\n\ttype=\"multipart/alternative\";\n\tboundary=\"=====================_715392540==_.REL\"" } ], "type": "multipart/related", "subParts": [ { "headers": [ { "name": "Content-Type", "value": " multipart/alternative;\n\tboundary=\"=====================_715392550==_.ALT\"" } ], "type": "multipart/alternative", "subParts": [ { "partId": "3", "blobId": "blob_0", "size": 779, "headers": [ { "name": "Content-Type", "value": " text/plain; charset=\"iso-8859-1\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/plain", "charset": "iso-8859-1" }, { "partId": "4", "blobId": "blob_1", "size": 1154, "headers": [ { "name": "Content-Type", "value": " text/html; charset=\"iso-8859-1\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/html", "charset": "iso-8859-1" } ] }, { "partId": "5", "blobId": "blob_2", "size": 1325, "headers": [ { "name": "Content-Type", "value": " image/png; name=\"2aa3ed95.png\"" }, { "name": "Content-ID", "value": " <4.2.0.58.20000519003556.00a918e0@pop.example.com.2>" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-Disposition", "value": " inline; filename=\"2aa3ed95.png\"" } ], "name": "2aa3ed95.png", "type": "image/png", "disposition": "inline", "cid": "4.2.0.58.20000519003556.00a918e0@pop.example.com.2" }, { "partId": "6", "blobId": "blob_3", "size": 1453, "headers": [ { "name": "Content-Type", "value": " image/png; name=\"2aa3edd1.png\"" }, { "name": "Content-ID", "value": " <4.2.0.58.20000519003556.00a918e0@pop.example.com.3>" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-Disposition", "value": " inline; filename=\"2aa3edd1.png\"" } ], "name": "2aa3edd1.png", "type": "image/png", "disposition": "inline", "cid": "4.2.0.58.20000519003556.00a918e0@pop.example.com.3" } ] }, { "partId": "7", "blobId": "blob_4", "size": 1325, "headers": [ { "name": "Content-Type", "value": " image/png; name=\"blueball.png\"" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-Disposition", "value": " attachment; filename=\"blueball.png\"" } ], "name": "blueball.png", "type": "image/png", "disposition": "attachment" }, { "partId": "8", "blobId": "blob_5", "size": 1298, "headers": [ { "name": "Content-Type", "value": " image/png; name=\"greenball.png\"" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-Disposition", "value": " attachment; filename=\"greenball.png\"" } ], "name": "greenball.png", "type": "image/png", "disposition": "attachment" } ] }, "bodyValues": { "3": { "value": "Die Hasen und die Frösche\n\nDie Hasen klagten einst über ihre mißliche Lage; \"wir leben\", sprac...", "isEncodingProblem": false, "isTruncated": true }, "4": { "value": "\nDie Hasen und die\nFrösche
\n...", "isEncodingProblem": false, "isTruncated": true } }, "textBody": [ { "partId": "3", "blobId": "blob_0", "size": 779, "headers": [ { "name": "Content-Type", "value": " text/plain; charset=\"iso-8859-1\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/plain", "charset": "iso-8859-1" } ], "htmlBody": [ { "partId": "4", "blobId": "blob_1", "size": 1154, "headers": [ { "name": "Content-Type", "value": " text/html; charset=\"iso-8859-1\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/html", "charset": "iso-8859-1" } ], "attachments": [ { "partId": "5", "blobId": "blob_2", "size": 1325, "headers": [ { "name": "Content-Type", "value": " image/png; name=\"2aa3ed95.png\"" }, { "name": "Content-ID", "value": " <4.2.0.58.20000519003556.00a918e0@pop.example.com.2>" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-Disposition", "value": " inline; filename=\"2aa3ed95.png\"" } ], "name": "2aa3ed95.png", "type": "image/png", "disposition": "inline", "cid": "4.2.0.58.20000519003556.00a918e0@pop.example.com.2" }, { "partId": "6", "blobId": "blob_3", "size": 1453, "headers": [ { "name": "Content-Type", "value": " image/png; name=\"2aa3edd1.png\"" }, { "name": "Content-ID", "value": " <4.2.0.58.20000519003556.00a918e0@pop.example.com.3>" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-Disposition", "value": " inline; filename=\"2aa3edd1.png\"" } ], "name": "2aa3edd1.png", "type": "image/png", "disposition": "inline", "cid": "4.2.0.58.20000519003556.00a918e0@pop.example.com.3" }, { "partId": "7", "blobId": "blob_4", "size": 1325, "headers": [ { "name": "Content-Type", "value": " image/png; name=\"blueball.png\"" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-Disposition", "value": " attachment; filename=\"blueball.png\"" } ], "name": "blueball.png", "type": "image/png", "disposition": "attachment" }, { "partId": "8", "blobId": "blob_5", "size": 1298, "headers": [ { "name": "Content-Type", "value": " image/png; name=\"greenball.png\"" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-Disposition", "value": " attachment; filename=\"greenball.png\"" } ], "name": "greenball.png", "type": "image/png", "disposition": "attachment" } ], "hasAttachment": true, "preview": "Die Hasen und die Frösche\n\nDie Hasen klagten einst über ihre mißliche Lage; \"wir leben\", sprach ein Redner, \"in steter Furcht vor Menschen und Tieren, eine Beute der Hunde, der Adler, ja fast aller Raubtiere! Unsere stete Angst ist ärger als der Tod..." } ================================================ FILE: tests/resources/jmap/email_get/rfc8621.eml ================================================ Subject: RFC 8621 Section 4.1.4 test Content-Type: multipart/mixed; boundary="1" --1 Content-Type: text/plain Content-Disposition: inline A --1 Content-Type: multipart/mixed; boundary="2" --2 Content-Type: multipart/alternative; boundary="3" --3 Content-Type: multipart/mixed; boundary="4" --4 Content-Type: text/plain Content-Disposition: inline B --4 Content-Type: image/jpeg Content-Disposition: inline C --4 Content-Type: text/plain Content-Disposition: inline D --4-- --3 Content-Type: multipart/related; boundary="5" --5 Content-Type: text/html E --5 Content-Type: image/jpeg F --5-- --3-- --2 Content-Type: image/jpeg Content-Disposition: attachment G --2 Content-Type: application/x-excel H --2 Content-Type: message/rfc822 Subject: J J --2-- --1 Content-Type: text/plain Content-Disposition: inline K --1-- ================================================ FILE: tests/resources/jmap/email_get/rfc8621.json ================================================ { "mailboxIds": { "a": true }, "keywords": { "tag": true }, "size": 870, "receivedAt": "1997-07-27T10:40:00Z", "subject": "RFC 8621 Section 4.1.4 test", "bodyStructure": { "headers": [ { "name": "Subject", "value": " RFC 8621 Section 4.1.4 test" }, { "name": "Content-Type", "value": " multipart/mixed; boundary=\"1\"" } ], "type": "multipart/mixed", "subParts": [ { "partId": "1", "blobId": "blob_0", "size": 1, "headers": [ { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Disposition", "value": " inline" } ], "type": "text/plain", "charset": "us-ascii", "disposition": "inline" }, { "headers": [ { "name": "Content-Type", "value": " multipart/mixed; boundary=\"2\"" } ], "type": "multipart/mixed", "subParts": [ { "headers": [ { "name": "Content-Type", "value": " multipart/alternative; boundary=\"3\"" } ], "type": "multipart/alternative", "subParts": [ { "headers": [ { "name": "Content-Type", "value": " multipart/mixed; boundary=\"4\"" } ], "type": "multipart/mixed", "subParts": [ { "partId": "5", "blobId": "blob_1", "size": 1, "headers": [ { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Disposition", "value": " inline" } ], "type": "text/plain", "charset": "us-ascii", "disposition": "inline" }, { "partId": "6", "blobId": "blob_2", "size": 1, "headers": [ { "name": "Content-Type", "value": " image/jpeg" }, { "name": "Content-Disposition", "value": " inline" } ], "type": "image/jpeg", "disposition": "inline" }, { "partId": "7", "blobId": "blob_3", "size": 1, "headers": [ { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Disposition", "value": " inline" } ], "type": "text/plain", "charset": "us-ascii", "disposition": "inline" } ] }, { "headers": [ { "name": "Content-Type", "value": " multipart/related; boundary=\"5\"" } ], "type": "multipart/related", "subParts": [ { "partId": "9", "blobId": "blob_4", "size": 14, "headers": [ { "name": "Content-Type", "value": " text/html" } ], "type": "text/html", "charset": "us-ascii" }, { "partId": "10", "blobId": "blob_5", "size": 1, "headers": [ { "name": "Content-Type", "value": " image/jpeg" } ], "type": "image/jpeg" } ] } ] }, { "partId": "11", "blobId": "blob_6", "size": 1, "headers": [ { "name": "Content-Type", "value": " image/jpeg" }, { "name": "Content-Disposition", "value": " attachment" } ], "type": "image/jpeg", "disposition": "attachment" }, { "partId": "12", "blobId": "blob_7", "size": 1, "headers": [ { "name": "Content-Type", "value": " application/x-excel" } ], "type": "application/x-excel" }, { "partId": "13", "blobId": "blob_8", "size": 13, "headers": [ { "name": "Content-Type", "value": " message/rfc822" } ], "type": "message/rfc822" } ] }, { "partId": "14", "blobId": "blob_9", "size": 1, "headers": [ { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Disposition", "value": " inline" } ], "type": "text/plain", "charset": "us-ascii", "disposition": "inline" } ] }, "bodyValues": { "1": { "value": "A", "isEncodingProblem": false, "isTruncated": false }, "14": { "value": "K", "isEncodingProblem": false, "isTruncated": false }, "5": { "value": "B", "isEncodingProblem": false, "isTruncated": false }, "7": { "value": "D", "isEncodingProblem": false, "isTruncated": false }, "9": { "value": "E", "isEncodingProblem": false, "isTruncated": false } }, "textBody": [ { "partId": "1", "blobId": "blob_0", "size": 1, "headers": [ { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Disposition", "value": " inline" } ], "type": "text/plain", "charset": "us-ascii", "disposition": "inline" }, { "partId": "5", "blobId": "blob_1", "size": 1, "headers": [ { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Disposition", "value": " inline" } ], "type": "text/plain", "charset": "us-ascii", "disposition": "inline" }, { "partId": "6", "blobId": "blob_2", "size": 1, "headers": [ { "name": "Content-Type", "value": " image/jpeg" }, { "name": "Content-Disposition", "value": " inline" } ], "type": "image/jpeg", "disposition": "inline" }, { "partId": "7", "blobId": "blob_3", "size": 1, "headers": [ { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Disposition", "value": " inline" } ], "type": "text/plain", "charset": "us-ascii", "disposition": "inline" }, { "partId": "14", "blobId": "blob_9", "size": 1, "headers": [ { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Disposition", "value": " inline" } ], "type": "text/plain", "charset": "us-ascii", "disposition": "inline" } ], "htmlBody": [ { "partId": "1", "blobId": "blob_0", "size": 1, "headers": [ { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Disposition", "value": " inline" } ], "type": "text/plain", "charset": "us-ascii", "disposition": "inline" }, { "partId": "9", "blobId": "blob_4", "size": 14, "headers": [ { "name": "Content-Type", "value": " text/html" } ], "type": "text/html", "charset": "us-ascii" }, { "partId": "14", "blobId": "blob_9", "size": 1, "headers": [ { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Disposition", "value": " inline" } ], "type": "text/plain", "charset": "us-ascii", "disposition": "inline" } ], "attachments": [ { "partId": "6", "blobId": "blob_2", "size": 1, "headers": [ { "name": "Content-Type", "value": " image/jpeg" }, { "name": "Content-Disposition", "value": " inline" } ], "type": "image/jpeg", "disposition": "inline" }, { "partId": "10", "blobId": "blob_5", "size": 1, "headers": [ { "name": "Content-Type", "value": " image/jpeg" } ], "type": "image/jpeg" }, { "partId": "11", "blobId": "blob_6", "size": 1, "headers": [ { "name": "Content-Type", "value": " image/jpeg" }, { "name": "Content-Disposition", "value": " attachment" } ], "type": "image/jpeg", "disposition": "attachment" }, { "partId": "12", "blobId": "blob_7", "size": 1, "headers": [ { "name": "Content-Type", "value": " application/x-excel" } ], "type": "application/x-excel" }, { "partId": "13", "blobId": "blob_8", "size": 13, "headers": [ { "name": "Content-Type", "value": " message/rfc822" } ], "type": "message/rfc822" } ], "hasAttachment": true, "preview": "A" } ================================================ FILE: tests/resources/jmap/email_get/single_part.eml ================================================ From: "Doug Sauder" To: =?iso-8859-1?B?SvxyZ2VuIFNjaG38cmdlbg==?= Subject: =?iso-8859-1?Q?Die_Hasen_und_die_Fr=F6sche_=28Microsoft_Outlook_00=29?= Date: Wed, 17 May 2000 19:15:35 -0400 Message-ID: MIME-Version: 1.0 Content-Type: text/plain; charset="iso-8859-1" Content-Transfer-Encoding: quoted-printable X-Priority: 3 (Normal) X-MSMail-Priority: Normal X-Mailer: Microsoft Outlook IMO, Build 9.0.2416 (9.0.2910.0) Importance: Normal X-MimeOLE: Produced By Microsoft MimeOLE V5.00.2314.1300 Die Hasen und die Fr=F6sche Die Hasen klagten einst =FCber ihre mi=DFliche Lage; "wir leben", sprach = ein Redner, "in steter Furcht vor Menschen und Tieren, eine Beute der = Hunde, der Adler, ja fast aller Raubtiere! Unsere stete Angst ist = =E4rger als der Tod selbst. Auf, la=DFt uns ein f=FCr allemal sterben."=20 In einem nahen Teich wollten sie sich nun ers=E4ufen; sie eilten ihm zu; = allein das au=DFerordentliche Get=F6se und ihre wunderbare Gestalt = erschreckte eine Menge Fr=F6sche, die am Ufer sa=DFen, so sehr, da=DF = sie aufs schnellste untertauchten.=20 "Halt", rief nun eben dieser Sprecher, "wir wollen das Ers=E4ufen noch = ein wenig aufschieben, denn auch uns f=FCrchten, wie ihr seht, einige = Tiere, welche also wohl noch ungl=FCcklicher sein m=FCssen als wir."=20 ================================================ FILE: tests/resources/jmap/email_get/single_part.json ================================================ { "mailboxIds": { "a": true }, "keywords": { "tag": true }, "size": 1378, "receivedAt": "2013-09-01T01:46:40Z", "messageId": [ "NDBBIAKOPKHFGPLCODIGIEKCCHAA.doug@example.com" ], "from": [ { "name": "Doug Sauder", "email": "doug@example.com" } ], "to": [ { "name": "Jürgen Schmürgen", "email": "schmuergen@example.com" } ], "subject": "Die Hasen und die Frösche (Microsoft Outlook 00)", "sentAt": "2000-05-17T23:15:35Z", "bodyStructure": { "partId": "0", "blobId": "blob_0", "size": 754, "headers": [ { "name": "From", "value": " \"Doug Sauder\" " }, { "name": "To", "value": " =?iso-8859-1?B?SvxyZ2VuIFNjaG38cmdlbg==?= " }, { "name": "Subject", "value": " =?iso-8859-1?Q?Die_Hasen_und_die_Fr=F6sche_=28Microsoft_Outlook_00=29?=" }, { "name": "Date", "value": " Wed, 17 May 2000 19:15:35 -0400" }, { "name": "Message-ID", "value": " " }, { "name": "MIME-Version", "value": " 1.0" }, { "name": "Content-Type", "value": " text/plain;\n\tcharset=\"iso-8859-1\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" }, { "name": "X-Priority", "value": " 3 (Normal)" }, { "name": "X-MSMail-Priority", "value": " Normal" }, { "name": "X-Mailer", "value": " Microsoft Outlook IMO, Build 9.0.2416 (9.0.2910.0)" }, { "name": "Importance", "value": " Normal" }, { "name": "X-MimeOLE", "value": " Produced By Microsoft MimeOLE V5.00.2314.1300" } ], "type": "text/plain", "charset": "iso-8859-1" }, "bodyValues": { "0": { "value": "Die Hasen und die Frösche\n\nDie Hasen klagten einst über ihre mißliche Lage; \"wir leben\", sprac...", "isEncodingProblem": false, "isTruncated": true } }, "textBody": [ { "partId": "0", "blobId": "blob_0", "size": 754, "headers": [ { "name": "From", "value": " \"Doug Sauder\" " }, { "name": "To", "value": " =?iso-8859-1?B?SvxyZ2VuIFNjaG38cmdlbg==?= " }, { "name": "Subject", "value": " =?iso-8859-1?Q?Die_Hasen_und_die_Fr=F6sche_=28Microsoft_Outlook_00=29?=" }, { "name": "Date", "value": " Wed, 17 May 2000 19:15:35 -0400" }, { "name": "Message-ID", "value": " " }, { "name": "MIME-Version", "value": " 1.0" }, { "name": "Content-Type", "value": " text/plain;\n\tcharset=\"iso-8859-1\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" }, { "name": "X-Priority", "value": " 3 (Normal)" }, { "name": "X-MSMail-Priority", "value": " Normal" }, { "name": "X-Mailer", "value": " Microsoft Outlook IMO, Build 9.0.2416 (9.0.2910.0)" }, { "name": "Importance", "value": " Normal" }, { "name": "X-MimeOLE", "value": " Produced By Microsoft MimeOLE V5.00.2314.1300" } ], "type": "text/plain", "charset": "iso-8859-1" } ], "htmlBody": [ { "partId": "0", "blobId": "blob_0", "size": 754, "headers": [ { "name": "From", "value": " \"Doug Sauder\" " }, { "name": "To", "value": " =?iso-8859-1?B?SvxyZ2VuIFNjaG38cmdlbg==?= " }, { "name": "Subject", "value": " =?iso-8859-1?Q?Die_Hasen_und_die_Fr=F6sche_=28Microsoft_Outlook_00=29?=" }, { "name": "Date", "value": " Wed, 17 May 2000 19:15:35 -0400" }, { "name": "Message-ID", "value": " " }, { "name": "MIME-Version", "value": " 1.0" }, { "name": "Content-Type", "value": " text/plain;\n\tcharset=\"iso-8859-1\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" }, { "name": "X-Priority", "value": " 3 (Normal)" }, { "name": "X-MSMail-Priority", "value": " Normal" }, { "name": "X-Mailer", "value": " Microsoft Outlook IMO, Build 9.0.2416 (9.0.2910.0)" }, { "name": "Importance", "value": " Normal" }, { "name": "X-MimeOLE", "value": " Produced By Microsoft MimeOLE V5.00.2314.1300" } ], "type": "text/plain", "charset": "iso-8859-1" } ], "attachments": [], "hasAttachment": false, "preview": "Die Hasen und die Frösche\n\nDie Hasen klagten einst über ihre mißliche Lage; \"wir leben\", sprach ein Redner, \"in steter Furcht vor Menschen und Tieren, eine Beute der Hunde, der Adler, ja fast aller Raubtiere! Unsere stete Angst ist ärger als der Tod..." } ================================================ FILE: tests/resources/jmap/email_get/text_body_missing.eml ================================================ From: "Doug Sauder" To: "Joe Blow" Subject: Test message from Microsoft Outlook 00 Date: Wed, 17 May 2000 19:35:05 -0400 Message-ID: MIME-Version: 1.0 Content-Type: image/png; name="redball.png" Content-Transfer-Encoding: base64 Content-Disposition: attachment; filename="redball.png" X-Priority: 3 (Normal) X-MSMail-Priority: Normal X-Mailer: Microsoft Outlook IMO, Build 9.0.2416 (9.0.2910.0) Importance: Normal X-MimeOLE: Produced By Microsoft MimeOLE V5.00.2314.1300 iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8AAAABAAALAAAVAAAa AAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAjAAAWAAAmAABhAAB7AACGAACHAAB9AAB0 AABgAAA5AAAUAAAGAAAnAABLAABvAACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABM AAB3AACZAAC0GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHfhITm f3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5PlrKzpmZntZWXvJSXXAADB AACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADLICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2 AAB4AABeAABAAAAiAABXAACSAADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABH AAArAAAPAACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABPAAASAAAC AABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADIAADTAADNAACzAACDAABuAAAe AAB+AADAAACkAACNAAB/AABpAABQAAAwAACRAACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACs AACvAACtAACmAACJAAB6AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABV AACOAACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8AAA6AAAfAAAM AAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAD8LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkFBDlQJf8zC/EIi4iK iUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ 29ja2Ts4Ojkr6Li4urFDNf53N/Ow8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFW SE1LF4A69n9GZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2YnOAj+ d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1a/acUG5piNz/uXLzVJ2q m6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2TVjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqV tWXrtu07BJihcsw71+zanRW8Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZw HBqL//8flz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/joOyYed5 QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms1y9evXid7QZacgOxmSxktNzd tSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAAJXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5 IFl2ZXMgUGlndWV0NnM7vAAAAABJRU5ErkJggg== ================================================ FILE: tests/resources/jmap/email_get/text_body_missing.json ================================================ { "mailboxIds": { "a": true }, "keywords": { "tag": true }, "size": 2527, "receivedAt": "2050-01-28T16:26:40Z", "messageId": [ "NDBBIAKOPKHFGPLCODIGKEKECHAA.doug@example.com" ], "from": [ { "name": "Doug Sauder", "email": "doug@example.com" } ], "to": [ { "name": "Joe Blow", "email": "jblow@example.com" } ], "subject": "Test message from Microsoft Outlook 00", "sentAt": "2000-05-17T23:35:05Z", "bodyStructure": { "partId": "0", "blobId": "blob_0", "size": 1453, "headers": [ { "name": "From", "value": " \"Doug Sauder\" " }, { "name": "To", "value": " \"Joe Blow\" " }, { "name": "Subject", "value": " Test message from Microsoft Outlook 00" }, { "name": "Date", "value": " Wed, 17 May 2000 19:35:05 -0400" }, { "name": "Message-ID", "value": " " }, { "name": "MIME-Version", "value": " 1.0" }, { "name": "Content-Type", "value": " image/png;\n\tname=\"redball.png\"" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-Disposition", "value": " attachment;\n\tfilename=\"redball.png\"" }, { "name": "X-Priority", "value": " 3 (Normal)" }, { "name": "X-MSMail-Priority", "value": " Normal" }, { "name": "X-Mailer", "value": " Microsoft Outlook IMO, Build 9.0.2416 (9.0.2910.0)" }, { "name": "Importance", "value": " Normal" }, { "name": "X-MimeOLE", "value": " Produced By Microsoft MimeOLE V5.00.2314.1300" } ], "name": "redball.png", "type": "image/png", "disposition": "attachment" }, "bodyValues": {}, "textBody": [], "htmlBody": [], "attachments": [ { "partId": "0", "blobId": "blob_0", "size": 1453, "headers": [ { "name": "From", "value": " \"Doug Sauder\" " }, { "name": "To", "value": " \"Joe Blow\" " }, { "name": "Subject", "value": " Test message from Microsoft Outlook 00" }, { "name": "Date", "value": " Wed, 17 May 2000 19:35:05 -0400" }, { "name": "Message-ID", "value": " " }, { "name": "MIME-Version", "value": " 1.0" }, { "name": "Content-Type", "value": " image/png;\n\tname=\"redball.png\"" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-Disposition", "value": " attachment;\n\tfilename=\"redball.png\"" }, { "name": "X-Priority", "value": " 3 (Normal)" }, { "name": "X-MSMail-Priority", "value": " Normal" }, { "name": "X-Mailer", "value": " Microsoft Outlook IMO, Build 9.0.2416 (9.0.2910.0)" }, { "name": "Importance", "value": " Normal" }, { "name": "X-MimeOLE", "value": " Produced By Microsoft MimeOLE V5.00.2314.1300" } ], "name": "redball.png", "type": "image/png", "disposition": "attachment" } ], "hasAttachment": true } ================================================ FILE: tests/resources/jmap/email_get/text_body_missing_multipart.eml ================================================ From: "Doug Sauder" To: "Joe Blow" Subject: Test message from Microsoft Outlook 00 Date: Wed, 17 May 2000 19:36:13 -0400 Message-ID: MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="----=_NextPart_000_0004_01BFC037.28F2FA90" X-Priority: 3 (Normal) X-MSMail-Priority: Normal X-Mailer: Microsoft Outlook IMO, Build 9.0.2416 (9.0.2910.0) Importance: Normal X-MimeOLE: Produced By Microsoft MimeOLE V5.00.2314.1300 This is a multi-part message in MIME format. ------=_NextPart_000_0004_01BFC037.28F2FA90 Content-Type: image/png; name="blueball.png" Content-Transfer-Encoding: base64 Content-Disposition: attachment; filename="blueball.png" iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8AAAgAABAAABgAAAAA CCkAEEIAEEoACDEAEFIIIXMIKXsIKYQIIWsAGFoACDkIIWMQOZwYQqUYQq0YQrUQOaUQMZQAGFIQ MYwpUrU5Y8Y5Y84pWs4YSs4YQs4YQr1Ca8Z7nNacvd6Mtd5jlOcxa94hUt4YStYYQsYQMaUAACHO 5+/n7++cxu9ShO8pWucQOa1Ke86tzt6lzu9ajO8QMZxahNat1ufO7++Mve9Ke+8YOaUYSsaMvee1 5++Uve8AAClajOdzpe9rnO8IKYwxY+8pWu8IIXsAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAADBMg1VAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu MT1evmgAAAGISURBVHicddJtV5swGAbgEk6AJhBSk4bMCUynBSLaqovbrG/bfPn/vyh70lbssceb L5xznTsh5BmNhgQoRChwo50EOIohUYLDj4zHhKYQkrEoQdvock4ne0IKMVUpKZLQDeqSTIsv+18P yqqWUw2IBsRM7307PPp+fDJrWtnpLDJvewYxnewfnvanZ+fzpmwXijC8KbqEa3Fx2ff91Y95U9XC UpaDeQwiMpHXP/v+1++bWVPWQoGFawtjury9vru/f/C1Vi7ezT0WWpQHf/7+u/G71aLThK/MjRxm T6KdzZ9fGk9yatMsTgZLl3XVgFRAC6spj/13enssqJVtWVa3NdBSacL8+VZmYqKmdd1CSYoOiMOS GwtzlqqlFFIuOqv0a1ZEZrUkWICLLFW266y1KvWE1zV/iDAH1EopnVLCiygZCIomH3NCKX0lnI+B 1iuuzCGTxwXjnDO4d7NpbX42YJJHkBwmAm2TxwAZg40J3+Xtbv1rgOAZwG0NxW62p+lT+Yi747sD /wEUVMzYmWkOvwAAACV0RVh0Q29tbWVudABjbGlwMmdpZiB2LjAuNiBieSBZdmVzIFBpZ3VldDZz O7wAAAAASUVORK5CYII= ------=_NextPart_000_0004_01BFC037.28F2FA90 Content-Type: image/png; name="redball.png" Content-Transfer-Encoding: base64 Content-Disposition: attachment; filename="redball.png" iVBORw0KGgoAAAANSUhEUgAAABsAAAAbCAMAAAC6CgRnAAADAFBMVEX///8AAAABAAALAAAVAAAa AAAXAAARAAAKAAADAAAcAAAyAABEAABNAABIAAA9AAAjAAAWAAAmAABhAAB7AACGAACHAAB9AAB0 AABgAAA5AAAUAAAGAAAnAABLAABvAACQAAClAAC7AAC/AACrAAChAACMAABzAABbAAAuAAAIAABM AAB3AACZAAC0GRnKODjVPT3bKSndBQW4AACoAAB5AAAxAAAYAAAEAABFAACaAAC7JCTRYWHfhITm f3/mVlbqHx/SAAC5AACjAABdAABCAAAoAAAJAABnAAC6Dw/QVFTek5PlrKzpmZntZWXvJSXXAADB AACxAACcAABtAABTAAA2AAAbAAAFAABKAACBAADLICDdZ2fonJzrpqbtiorvUVHvFBTRAADDAAC2 AAB4AABeAABAAAAiAABXAACSAADCAADaGxvoVVXseHjveHjvV1fvJibhAADOAAC3AACnAACVAABH AAArAAAPAACdAADFAADhBQXrKCjvPDzvNTXvGxvjAADQAADJAAC1AACXAACEAABsAABPAAASAAAC AABiAADpAADvAgLnAADYAADLAAC6AACwAABwAAATAAAkAABYAADIAADTAADNAACzAACDAABuAAAe AAB+AADAAACkAACNAAB/AABpAABQAAAwAACRAACpAAC8AACqAACbAABlAABJAAAqAAAOAAA0AACs AACvAACtAACmAACJAAB6AABrAABaAAA+AAApAABqAACCAACfAACeAACWAACPAAB8AAAZAAAHAABV AACOAACKAAA4AAAQAAA/AAByAACAAABcAAA3AAAsAABmAABDAABWAAAgAAAzAAA8AAA6AAAfAAAM AAAdAAANAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAD8LtlFAAAAAXRSTlMAQObYZgAAABZ0RVh0U29mdHdhcmUAZ2lmMnBuZyAyLjAu MT1evmgAAAIISURBVHicY2CAg/8QwIABmJhZWFnZ2Dk4MaU5uLh5eHn5+LkFBDlQJf8zC/EIi4iK iUtI8koJScsgyf5nlpWTV1BUUlZRVVPX4NFk1UJIyghp6+jq6RsYGhmbKJgK85mZW8Dk/rNaSlhZ 29ja2Ts4Ojkr6Li4urFDNf53N/Ow8vTy9vH18w8IDAoWDQkNC4+ASP5ni4wKio6JjYtPSExKTnFW SE1LF4A69n9GZlZ2Tm5efkFhUXFySWlZlEd5RSVY7j+TkGRVdU1tXX1DY1Ozcktpa1t7h2YnOAj+ d7l1tyo79vT29SdNSJ44SbFVdHIo9xSIHNPUaWqTpifNSJrZnK00S0U1a/acUG5piNz/uXLzVJ2q m6dXz584S2WB1cJFi5cshZr539xVftnyFKUVTi2TVjqvyhJLXb1m7TqoHPt6F/HW0g0bN63crGqV tWXrtu07BJihcsw71+zanRW8Z89eq337RQ/Ip60xO3gIElX/LbikDm8T36KwbNmRo7O3zpHkPSZw HBqL//8flz1x2OOkyKJTi7aqbzutfUZI2gIuF8F2lr/D5dw2+fZdwpl8YVOlI+CJ4/9/joOyYed5 QzMvhGqnm2V0WiClm///D0lfXHtJ6vLlK9w7rx7vQk5SQJbFtSms1y9evXid7QZacgOxmSxktNzd tSwwU+J/VICaCPFIYU3XAJhIOtjf5sfyAAAAJXRFWHRDb21tZW50AGNsaXAyZ2lmIHYuMC42IGJ5 IFl2ZXMgUGlndWV0NnM7vAAAAABJRU5ErkJggg== ------=_NextPart_000_0004_01BFC037.28F2FA90-- ================================================ FILE: tests/resources/jmap/email_get/text_body_missing_multipart.json ================================================ { "mailboxIds": { "a": true }, "keywords": { "tag": true }, "size": 4726, "receivedAt": "2119-10-06T01:46:40Z", "messageId": [ "NDBBIAKOPKHFGPLCODIGOEKECHAA.doug@example.com" ], "from": [ { "name": "Doug Sauder", "email": "doug@example.com" } ], "to": [ { "name": "Joe Blow", "email": "jblow@example.com" } ], "subject": "Test message from Microsoft Outlook 00", "sentAt": "2000-05-17T23:36:13Z", "bodyStructure": { "headers": [ { "name": "From", "value": " \"Doug Sauder\" " }, { "name": "To", "value": " \"Joe Blow\" " }, { "name": "Subject", "value": " Test message from Microsoft Outlook 00" }, { "name": "Date", "value": " Wed, 17 May 2000 19:36:13 -0400" }, { "name": "Message-ID", "value": " " }, { "name": "MIME-Version", "value": " 1.0" }, { "name": "Content-Type", "value": " multipart/mixed;\n\tboundary=\"----=_NextPart_000_0004_01BFC037.28F2FA90\"" }, { "name": "X-Priority", "value": " 3 (Normal)" }, { "name": "X-MSMail-Priority", "value": " Normal" }, { "name": "X-Mailer", "value": " Microsoft Outlook IMO, Build 9.0.2416 (9.0.2910.0)" }, { "name": "Importance", "value": " Normal" }, { "name": "X-MimeOLE", "value": " Produced By Microsoft MimeOLE V5.00.2314.1300" } ], "type": "multipart/mixed", "subParts": [ { "partId": "1", "blobId": "blob_0", "size": 1325, "headers": [ { "name": "Content-Type", "value": " image/png;\n\tname=\"blueball.png\"" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-Disposition", "value": " attachment;\n\tfilename=\"blueball.png\"" } ], "name": "blueball.png", "type": "image/png", "disposition": "attachment" }, { "partId": "2", "blobId": "blob_1", "size": 1453, "headers": [ { "name": "Content-Type", "value": " image/png;\n\tname=\"redball.png\"" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-Disposition", "value": " attachment;\n\tfilename=\"redball.png\"" } ], "name": "redball.png", "type": "image/png", "disposition": "attachment" } ] }, "bodyValues": {}, "textBody": [], "htmlBody": [], "attachments": [ { "partId": "1", "blobId": "blob_0", "size": 1325, "headers": [ { "name": "Content-Type", "value": " image/png;\n\tname=\"blueball.png\"" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-Disposition", "value": " attachment;\n\tfilename=\"blueball.png\"" } ], "name": "blueball.png", "type": "image/png", "disposition": "attachment" }, { "partId": "2", "blobId": "blob_1", "size": 1453, "headers": [ { "name": "Content-Type", "value": " image/png;\n\tname=\"redball.png\"" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-Disposition", "value": " attachment;\n\tfilename=\"redball.png\"" } ], "name": "redball.png", "type": "image/png", "disposition": "attachment" } ], "hasAttachment": true } ================================================ FILE: tests/resources/jmap/email_parse/attachment.eml ================================================ From: Al Gore To: White House Transportation Coordinator Subject: [Fwd: Map of Argentina with Description] Content-Type: multipart/mixed; boundary="D7F------------D7FD5A0B8AB9C65CCDBFA872" This is a multi-part message in MIME format. --D7F------------D7FD5A0B8AB9C65CCDBFA872 Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit Fred, Fire up Air Force One! We're going South! Thanks, Al --D7F------------D7FD5A0B8AB9C65CCDBFA872 Content-Type: message/rfc822 Content-Transfer-Encoding: 7bit Content-Disposition: inline Return-Path: Received: from mailhost.whitehouse.gov ([192.168.51.200]) by heartbeat.whitehouse.gov (8.8.8/8.8.8) with ESMTP id SAA22453 for ; Mon, 13 Aug 1998 l8:14:23 +1000 Received: from the_big_box.whitehouse.gov ([192.168.51.50]) by mailhost.whitehouse.gov (8.8.8/8.8.7) with ESMTP id RAA20366 for vice-president@whitehouse.gov; Mon, 13 Aug 1998 17:42:41 +1000 Date: Mon, 13 Aug 1998 17:42:41 +1000 Message-Id: <199804130742.RAA20366@mai1host.whitehouse.gov> From: Bill Clinton To: A1 (The Enforcer) Gore Subject: Map of Argentina with Description MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="DC8------------DC8638F443D87A7F0726DEF7" This is a multi-part message in MIME format. --DC8------------DC8638F443D87A7F0726DEF7 Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit Hi A1, I finally figured out this MIME thing. Pretty cool. I'll send you some sax music in .au files next week! Anyway, the attached image is really too small to get a good look at Argentina. Try this for a much better map: http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm Then again, shouldn't the CIA have something like that? Bill --DC8------------DC8638F443D87A7F0726DEF7 Content-Type: image/gif; name="map_of_Argentina.gif" Content-Transfer-Encoding: base64 Content-Disposition: inline; fi1ename="map_of_Argentina.gif" R01GOD1hJQA1AKIAAP/////78P/omn19fQAAAAAAAAAAAAAAACwAAAAAJQA1AAAD7Qi63P5w wEmjBCLrnQnhYCgM1wh+pkgqqeC9XrutmBm7hAK3tP31gFcAiFKVQrGFR6kscnonTe7FAAad GugmRu3CmiBt57fsVq3Y0VFKnpYdxPC6M7Ze4crnnHum4oN6LFJ1bn5NXTN7OF5fQkN5WYow BEN2dkGQGWJtSzqGTICJgnQuTJN/WJsojad9qXMuhIWdjXKjY4tenjo6tjVssk2gaWq3uGNX U6ZGxseyk8SasGw3J9GRzdTQky1iHNvcPNNI4TLeKdfMvy0vMqLrItvuxfDW8ubjueDtJufz 7itICBxISKDBgwgTKjyYAAA7 --DC8------------DC8638F443D87A7F0726DEF7-- --D7F------------D7FD5A0B8AB9C65CCDBFA872-- ================================================ FILE: tests/resources/jmap/email_parse/attachment.json ================================================ { "mailboxIds": null, "size": 1979, "messageId": [ "199804130742.RAA20366@mai1host.whitehouse.gov" ], "from": [ { "name": "Bill Clinton", "email": "president@whitehouse.gov" } ], "to": [ { "name": "A1 Gore (The Enforcer)", "email": "vice-president@whitehouse.gov" } ], "subject": "Map of Argentina with Description", "sentAt": "1998-08-13T07:42:41Z", "bodyStructure": { "headers": [ { "name": "Return-Path", "value": " " }, { "name": "Received", "value": " from mailhost.whitehouse.gov ([192.168.51.200])\n by heartbeat.whitehouse.gov (8.8.8/8.8.8) with ESMTP id SAA22453\n for ;\n Mon, 13 Aug 1998 l8:14:23 +1000" }, { "name": "Received", "value": " from the_big_box.whitehouse.gov ([192.168.51.50])\n by mailhost.whitehouse.gov (8.8.8/8.8.7) with ESMTP id RAA20366\n for vice-president@whitehouse.gov; Mon, 13 Aug 1998 17:42:41 +1000" }, { "name": "Date", "value": " Mon, 13 Aug 1998 17:42:41 +1000" }, { "name": "Message-ID", "value": " <199804130742.RAA20366@mai1host.whitehouse.gov>" }, { "name": "From", "value": " Bill Clinton " }, { "name": "To", "value": " A1 (The Enforcer) Gore " }, { "name": "Subject", "value": " Map of Argentina with Description" }, { "name": "MIME-Version", "value": " 1.0" }, { "name": "Content-Type", "value": " multipart/mixed;\n boundary=\"DC8------------DC8638F443D87A7F0726DEF7\"" } ], "type": "multipart/mixed", "subParts": [ { "partId": "1", "blobId": "blob_0", "size": 355, "headers": [ { "name": "Content-Type", "value": " text/plain; charset=us-ascii" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "us-ascii" }, { "partId": "2", "blobId": "blob_1", "size": 288, "headers": [ { "name": "Content-Type", "value": " image/gif; name=\"map_of_Argentina.gif\"" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-Disposition", "value": " inline; fi1ename=\"map_of_Argentina.gif\"" } ], "name": "map_of_Argentina.gif", "type": "image/gif", "disposition": "inline" } ] }, "bodyValues": { "1": { "value": "Hi A1,\n\nI finally figured out this MIME thing. Pretty cool. I'll send you\nsome sax music in .au...", "isEncodingProblem": false, "isTruncated": true } }, "textBody": [ { "partId": "1", "blobId": "blob_0", "size": 355, "headers": [ { "name": "Content-Type", "value": " text/plain; charset=us-ascii" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "us-ascii" }, { "partId": "2", "blobId": "blob_1", "size": 288, "headers": [ { "name": "Content-Type", "value": " image/gif; name=\"map_of_Argentina.gif\"" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-Disposition", "value": " inline; fi1ename=\"map_of_Argentina.gif\"" } ], "name": "map_of_Argentina.gif", "type": "image/gif", "disposition": "inline" } ], "htmlBody": [ { "partId": "1", "blobId": "blob_0", "size": 355, "headers": [ { "name": "Content-Type", "value": " text/plain; charset=us-ascii" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "us-ascii" }, { "partId": "2", "blobId": "blob_1", "size": 288, "headers": [ { "name": "Content-Type", "value": " image/gif; name=\"map_of_Argentina.gif\"" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-Disposition", "value": " inline; fi1ename=\"map_of_Argentina.gif\"" } ], "name": "map_of_Argentina.gif", "type": "image/gif", "disposition": "inline" } ], "attachments": [ { "partId": "2", "blobId": "blob_1", "size": 288, "headers": [ { "name": "Content-Type", "value": " image/gif; name=\"map_of_Argentina.gif\"" }, { "name": "Content-Transfer-Encoding", "value": " base64" }, { "name": "Content-Disposition", "value": " inline; fi1ename=\"map_of_Argentina.gif\"" } ], "name": "map_of_Argentina.gif", "type": "image/gif", "disposition": "inline" } ], "hasAttachment": false, "preview": "Hi A1,\n\nI finally figured out this MIME thing. Pretty cool. I'll send you\nsome sax music in .au files next week!\n\nAnyway, the attached image is really too small to get a good look at\nArgentina. Try this for a much better map:\n\n http://www.1one1yp..." } ================================================ FILE: tests/resources/jmap/email_parse/attachment.part1 ================================================ Hi A1, I finally figured out this MIME thing. Pretty cool. I'll send you some sax music in .au files next week! Anyway, the attached image is really too small to get a good look at Argentina. Try this for a much better map: http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm Then again, shouldn't the CIA have something like that? Bill ================================================ FILE: tests/resources/jmap/email_parse/attachment.part3 ================================================ Hi A1,

I finally figured out this MIME thing. Pretty cool. I'll send you
some sax music in .au files next week!

Anyway, the attached image is really too small to get a good look at
Argentina. Try this for a much better map:

http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm

Then again, shouldn't the CIA have something like that?

Bill ================================================ FILE: tests/resources/jmap/email_parse/attachment_b64.eml ================================================ Content-Type: multipart/mixed; boundary=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb --bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 7bit This is a message with a base64 encoded attached email --bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb Content-Disposition: attachment; filename="attached_email.eml" Content-Type: message/rfc822 Content-Transfer-Encoding: base64 VG86ICJlbWFpbEBleGFtcGxlLmNvbSIgPGVtYWlsQGV4YW1wbGUuY29tPg0KRnJvbTogTmFtZSA8 ZW1haWxAZXhhbXBsZS5jb20+DQpTdWJqZWN0OiBIVE1MIHRlc3QNCk1lc3NhZ2UtSUQ6IDxyYW5k b20tbWVzc2FnZS1pZEBleGFtcGxlLmNvbT4NCkRhdGU6IFR1ZSwgMTQgRGVjIDIwMjEgMTE6NDg6 MjUgKzAxMDANCk1JTUUtVmVyc2lvbjogMS4wDQpDb250ZW50LVR5cGU6IG11bHRpcGFydC9hbHRl cm5hdGl2ZTsNCiBib3VuZGFyeT0iYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh YWFhYSINCkNvbnRlbnQtTGFuZ3VhZ2U6IGVuLVVTDQoNClRoaXMgaXMgYSBtdWx0aS1wYXJ0IG1l c3NhZ2UgaW4gTUlNRSBmb3JtYXQuDQotLWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh YWFhYWFhYWENCkNvbnRlbnQtVHlwZTogdGV4dC9wbGFpbjsgY2hhcnNldD11dGYtODsgZm9ybWF0 PWZsb3dlZA0KQ29udGVudC1UcmFuc2Zlci1FbmNvZGluZzogN2JpdA0KDQpUaGlzIGlzIGFuICpI VE1MKiB0ZXN0IG1lc3NhZ2UNCi0tYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh YWFhYQ0KQ29udGVudC1UeXBlOiB0ZXh0L2h0bWw7IGNoYXJzZXQ9dXRmLTgNCkNvbnRlbnQtVHJh bnNmZXItRW5jb2Rpbmc6IDdiaXQNCg0KPGh0bWw+DQogIDxoZWFkPg0KICAgIDxtZXRhIGh0dHAt ZXF1aXY9ImNvbnRlbnQtdHlwZSIgY29udGVudD0idGV4dC9odG1sOyBjaGFyc2V0PVVURi04Ij4N CiAgPC9oZWFkPg0KICA8Ym9keT4NCiAgICBUaGlzIGlzIGFuIDxiPkhUTUw8L2I+IHRlc3QgbWVz c2FnZQ0KICA8L2JvZHk+DQo8L2h0bWw+DQoNCi0tYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh YWFhYWFhYWFhYWFhYS0tDQo= --bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-- ================================================ FILE: tests/resources/jmap/email_parse/attachment_b64.json ================================================ { "mailboxIds": null, "size": 872, "messageId": [ "random-message-id@example.com" ], "from": [ { "name": "Name", "email": "email@example.com" } ], "to": [ { "name": "email@example.com", "email": "email@example.com" } ], "subject": "HTML test", "sentAt": "2021-12-14T10:48:25Z", "bodyStructure": { "headers": [ { "name": "To", "value": " \"email@example.com\" " }, { "name": "From", "value": " Name " }, { "name": "Subject", "value": " HTML test" }, { "name": "Message-ID", "value": " " }, { "name": "Date", "value": " Tue, 14 Dec 2021 11:48:25 +0100" }, { "name": "MIME-Version", "value": " 1.0" }, { "name": "Content-Type", "value": " multipart/alternative;\r\n boundary=\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"" }, { "name": "Content-Language", "value": " en-US" } ], "type": "multipart/alternative", "language": [ "en-US" ], "subParts": [ { "partId": "1", "blobId": "blob_0", "size": 30, "headers": [ { "name": "Content-Type", "value": " text/plain; charset=utf-8; format=flowed" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "utf-8" }, { "partId": "2", "blobId": "blob_1", "size": 173, "headers": [ { "name": "Content-Type", "value": " text/html; charset=utf-8" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/html", "charset": "utf-8" } ] }, "bodyValues": { "1": { "value": "This is an *HTML* test message", "isEncodingProblem": false, "isTruncated": false }, "2": { "value": "\n \n \n ...", "isEncodingProblem": false, "isTruncated": true } }, "textBody": [ { "partId": "1", "blobId": "blob_0", "size": 30, "headers": [ { "name": "Content-Type", "value": " text/plain; charset=utf-8; format=flowed" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "utf-8" } ], "htmlBody": [ { "partId": "2", "blobId": "blob_1", "size": 173, "headers": [ { "name": "Content-Type", "value": " text/html; charset=utf-8" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/html", "charset": "utf-8" } ], "attachments": [], "hasAttachment": false, "preview": "This is an *HTML* test message" } ================================================ FILE: tests/resources/jmap/email_parse/attachment_b64.part1 ================================================ This is an *HTML* test message ================================================ FILE: tests/resources/jmap/email_parse/attachment_b64.part2 ================================================ This is an HTML test message ================================================ FILE: tests/resources/jmap/email_parse/headers.eml ================================================ From: Art Vandelay (Vandelay Industries) To: " James Smythe" , Friends: jane@example.com, =?UTF-8?Q?John_Sm=C3=AEth?= ; Cc: List 1: addr1@test.com, addr2@test.com; List 2: addr3@test.com, addr4@test.com; addr5@test.com, addr6@test.com Cc: =?utf-8?b?VGjDrXMgw61zIHbDoWzDrWQgw5pURjg=?=: addr1@test.com, addr2@test.com; =?utf-8?b?VGjDrXMgw61zIHbDoWzDrWQgw5pURjg=?=: addr3@test.com, addr4@test.com; addr5@test.com, addr6@test.com Bcc: Greg Vaudreuil , Ned Freed , Keith Moore Bcc: ietf-822@dimacs.rutgers.edu, ojarnef@admin.kth.se Date: Tue, 1 Jul 2003 10:52:37 +0200 Resent-Date: Tue, 2 Jul 2005 11:52:37 +0200 Resent-Date: Tue, 3 Jul 2005 12:52:37 +0300 Resent-Date: Tue, 4 Jul 2005 13:52:37 +0400 Message-ID: <5678.21-Nov-1997@example.com> References: <1234@local.machine.example> <3456@example.net> References: <789@local.machine.example> Keywords: multipart, alternative, example List-Post: (Postings are Moderated) List-Subscribe: (Use this command to join the list) List-Subscribe: (FTP), List-Owner: , List-Unsubscribe: (Use this command to get off the list) Subject: Why not both importing AND exporting? =?utf-8?b?4pi6?= X-Address-Single: =?ISO-8859-1?Q?Andr=E9?= Pirard X-Address: Mary Smith X-Address: John Doe X-AddressList-Single: Mary Smith , jdoe@example.org, Who? X-AddressList: =?US-ASCII*EN?Q?Keith_Moore?= , John =?US-ASCII*EN?Q?Doe?= X-AddressList: =?ISO-8859-1?Q?Keld_J=F8rn_Simonsen?= , =?ISO-8859-1?Q?Olle_J=E4rnefors?= X-AddressesGroup-Single: A Group(Some people) :Chris Jones , joe@example.org, John (my dear friend); (the end of the group) X-AddressesGroup: A Group:Ed Jones ,joe@where.test,John ; X-AddressesGroup: "List 1": addr1@test.com, addr2@test.com; "List 2": addr3@test.com, addr4@test.com; addr5@test.com, addr6@test.com X-List-Single: (X-Postings are Moderated) X-List: , X-List: , X-Text-Single: =?ISO-8859-1?B?SWYgeW91IGNhbiByZWFkIHRoaXMgeW8=?= =?ISO-8859-2?B?dSB1bmRlcnN0YW5kIHRoZSBleGFtcGxlLg==?= X-Text: =?iso-8859-1?q?this=20is=20some=20text?= X-Text: =?ISO-8859-1?Q?a?= =?ISO-8859-2?Q?_b?= X-Date-Single: Tue, 5 Jul 2006 13:52:37 -0500 X-Date: Sat, 20 Nov 2021 14:22:01 -0800 X-Date: Sun, 21 Nov 2021 15:23:02 -0900 X-Id-Single: X-Id: X-Id: Content-Type: multipart/mixed; boundary="festivus"; --festivus Content-Type: text/html; charset="us-ascii" Content-Transfer-Encoding: base64 X-Custom-Header: 123 PGh0bWw+PHA+SSB3YXMgdGhpbmtpbmcgYWJvdXQgcXVpdHRpbmcgdGhlICZsZHF1bztle HBvcnRpbmcmcmRxdW87IHRvIGZvY3VzIGp1c3Qgb24gdGhlICZsZHF1bztpbXBvcnRpbm cmcmRxdW87LDwvcD48cD5idXQgdGhlbiBJIHRob3VnaHQsIHdoeSBub3QgZG8gYm90aD8 gJiN4MjYzQTs8L3A+PC9odG1sPg== --festivus Content-Type: message/rfc822 X-Custom-Header-2: 345 From: "Cosmo Kramer" Subject: Exporting my book about coffee tables Content-Type: multipart/mixed; boundary="giddyup"; --giddyup Content-Type: text/plain; charset="utf-16" Content-Transfer-Encoding: quoted-printable =FF=FE=0C!5=D8"=DD5=D8)=DD5=D8-=DD =005=D8*=DD5=D8"=DD =005=D8"= =DD5=D85=DD5=D8-=DD5=D8,=DD5=D8/=DD5=D81=DD =005=D8*=DD5=D86=DD = =005=D8=1F=DD5=D8,=DD5=D8,=DD5=D8(=DD =005=D8-=DD5=D8)=DD5=D8"= =DD5=D8=1E=DD5=D80=DD5=D8"=DD!=00 --giddyup Content-Type: image/gif; name*1="about "; name*0="Book "; name*2*=utf-8''%e2%98%95 tables.gif Content-Transfer-Encoding: Base64 Content-Disposition: attachment R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7 --giddyup-- --festivus-- ================================================ FILE: tests/resources/jmap/email_parse/headers.json ================================================ { "mailboxIds": null, "messageId": [ "5678.21-Nov-1997@example.com" ], "references": [ "789@local.machine.example", "abcd@example.net" ], "from": [ { "name": "Art Vandelay (Vandelay Industries)", "email": "art@vandelay.com" } ], "to": [ { "name": " James Smythe", "email": "james@example.com" }, { "name": null, "email": "jane@example.com" }, { "name": "John Smîth", "email": "john@example.com" } ], "cc": [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" }, { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" }, { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ], "bcc": [ { "name": null, "email": "ietf-822@dimacs.rutgers.edu" }, { "name": null, "email": "ojarnef@admin.kth.se" } ], "subject": "Why not both importing AND exporting? ☺", "sentAt": "2003-07-01T08:52:37Z", "textBody": [ { "size": 175, "type": "text/html", "charset": "us-ascii" } ], "htmlBody": [ { "size": 175, "type": "text/html", "charset": "us-ascii" } ], "attachments": [ { "size": 723, "type": "message/rfc822" } ], "preview": "I was thinking about quitting the “exporting” to focus just on the “importing”,\nbut then I thought, why not do both? ☺\n", "header:Bcc": " ietf-822@dimacs.rutgers.edu, ojarnef@admin.kth.se", "header:Bcc:all": [ " Greg Vaudreuil , Ned Freed\n , Keith Moore ", " ietf-822@dimacs.rutgers.edu, ojarnef@admin.kth.se" ], "header:Bcc:asAddresses": [ { "name": null, "email": "ietf-822@dimacs.rutgers.edu" }, { "name": null, "email": "ojarnef@admin.kth.se" } ], "header:Bcc:asAddresses:all": [ [ { "name": "Greg Vaudreuil", "email": "gvaudre@NRI.Reston.VA.US" }, { "name": "Ned Freed", "email": "ned@innosoft.com" }, { "name": "Keith Moore", "email": "moore@cs.utk.edu" } ], [ { "name": null, "email": "ietf-822@dimacs.rutgers.edu" }, { "name": null, "email": "ojarnef@admin.kth.se" } ] ], "header:Bcc:asGroupedAddresses": [ { "name": null, "addresses": [ { "name": null, "email": "ietf-822@dimacs.rutgers.edu" }, { "name": null, "email": "ojarnef@admin.kth.se" } ] } ], "header:Bcc:asGroupedAddresses:all": [ [ { "name": null, "addresses": [ { "name": "Greg Vaudreuil", "email": "gvaudre@NRI.Reston.VA.US" }, { "name": "Ned Freed", "email": "ned@innosoft.com" }, { "name": "Keith Moore", "email": "moore@cs.utk.edu" } ] } ], [ { "name": null, "addresses": [ { "name": null, "email": "ietf-822@dimacs.rutgers.edu" }, { "name": null, "email": "ojarnef@admin.kth.se" } ] } ] ], "header:Cc": " =?utf-8?b?VGjDrXMgw61zIHbDoWzDrWQgw5pURjg=?=: addr1@test.com, \n addr2@test.com; =?utf-8?b?VGjDrXMgw61zIHbDoWzDrWQgw5pURjg=?=: \n addr3@test.com, addr4@test.com; addr5@test.com, addr6@test.com", "header:Cc:all": [ " List 1: addr1@test.com, addr2@test.com; List 2: addr3@test.com, \n addr4@test.com; addr5@test.com, addr6@test.com", " =?utf-8?b?VGjDrXMgw61zIHbDoWzDrWQgw5pURjg=?=: addr1@test.com, \n addr2@test.com; =?utf-8?b?VGjDrXMgw61zIHbDoWzDrWQgw5pURjg=?=: \n addr3@test.com, addr4@test.com; addr5@test.com, addr6@test.com" ], "header:Cc:asAddresses": [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" }, { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" }, { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ], "header:Cc:asAddresses:all": [ [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" }, { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" }, { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ], [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" }, { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" }, { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ] ], "header:Cc:asGroupedAddresses": [ { "name": "Thís ís válíd ÚTF8", "addresses": [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" } ] }, { "name": "Thís ís válíd ÚTF8", "addresses": [ { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" } ] }, { "name": null, "addresses": [ { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ] } ], "header:Cc:asGroupedAddresses:all": [ [ { "name": "List 1", "addresses": [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" } ] }, { "name": "List 2", "addresses": [ { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" } ] }, { "name": null, "addresses": [ { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ] } ], [ { "name": "Thís ís válíd ÚTF8", "addresses": [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" } ] }, { "name": "Thís ís válíd ÚTF8", "addresses": [ { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" } ] }, { "name": null, "addresses": [ { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ] } ] ], "header:Date": " Tue, 1 Jul 2003 10:52:37 +0200", "header:Date:all": [ " Tue, 1 Jul 2003 10:52:37 +0200" ], "header:Date:asDate": "2003-07-01T08:52:37Z", "header:Date:asDate:all": [ "2003-07-01T08:52:37Z" ], "header:From": " Art Vandelay (Vandelay Industries)", "header:From:all": [ " Art Vandelay (Vandelay Industries)" ], "header:From:asAddresses": [ { "name": "Art Vandelay (Vandelay Industries)", "email": "art@vandelay.com" } ], "header:From:asAddresses:all": [ [ { "name": "Art Vandelay (Vandelay Industries)", "email": "art@vandelay.com" } ] ], "header:From:asGroupedAddresses": [ { "name": null, "addresses": [ { "name": "Art Vandelay (Vandelay Industries)", "email": "art@vandelay.com" } ] } ], "header:From:asGroupedAddresses:all": [ [ { "name": null, "addresses": [ { "name": "Art Vandelay (Vandelay Industries)", "email": "art@vandelay.com" } ] } ] ], "header:Keywords": " multipart, alternative, example", "header:Keywords:all": [ " multipart, alternative, example" ], "header:Keywords:asText": "multipart, alternative, example", "header:Keywords:asText:all": [ "multipart, alternative, example" ], "header:List-Owner": " ,\n ", "header:List-Owner:all": [ " ,\n " ], "header:List-Owner:asURLs": [ "http://www.host.com/list.cgi?cmd=sub&lst=list", "mailto:list-manager@host.com?body=subscribe%20list" ], "header:List-Owner:asURLs:all": [ [ "http://www.host.com/list.cgi?cmd=sub&lst=list", "mailto:list-manager@host.com?body=subscribe%20list" ] ], "header:List-Post": " (Postings are Moderated)", "header:List-Post:all": [ " (Postings are Moderated)" ], "header:List-Post:asURLs": [ "mailto:moderator@host.com" ], "header:List-Post:asURLs:all": [ [ "mailto:moderator@host.com" ] ], "header:List-Subscribe": " (FTP), \n ", "header:List-Subscribe:all": [ " (Use this command to join the list)\n ", " (FTP), \n " ], "header:List-Subscribe:asURLs": [ "ftp://ftp.host.com/list.txt", "mailto:list@host.com?subject=subscribe" ], "header:List-Subscribe:asURLs:all": [ [ "mailto:list-manager@host.com?body=subscribe%20list" ], [ "ftp://ftp.host.com/list.txt", "mailto:list@host.com?subject=subscribe" ] ], "header:List-Unsubscribe": " (Use this command to get off the list)\n ", "header:List-Unsubscribe:all": [ " (Use this command to get off the list)\n " ], "header:List-Unsubscribe:asURLs": [ "mailto:list-manager@host.com?body=unsubscribe%20list" ], "header:List-Unsubscribe:asURLs:all": [ [ "mailto:list-manager@host.com?body=unsubscribe%20list" ] ], "header:Message-ID": " <5678.21-Nov-1997@example.com>", "header:Message-ID:all": [ " <5678.21-Nov-1997@example.com>" ], "header:Message-ID:asMessageIds": [ "5678.21-Nov-1997@example.com" ], "header:Message-ID:asMessageIds:all": [ [ "5678.21-Nov-1997@example.com" ] ], "header:References": " <789@local.machine.example>\n ", "header:References:all": [ " <1234@local.machine.example>\n <3456@example.net>", " <789@local.machine.example>\n " ], "header:References:asMessageIds": [ "789@local.machine.example", "abcd@example.net" ], "header:References:asMessageIds:all": [ [ "1234@local.machine.example", "3456@example.net" ], [ "789@local.machine.example", "abcd@example.net" ] ], "header:Resent-Date": " Tue, 4 Jul 2005 13:52:37 +0400", "header:Resent-Date:all": [ " Tue, 2 Jul 2005 11:52:37 +0200", " Tue, 3 Jul 2005 12:52:37 +0300", " Tue, 4 Jul 2005 13:52:37 +0400" ], "header:Resent-Date:asDate": "2005-07-04T09:52:37Z", "header:Resent-Date:asDate:all": [ "2005-07-02T09:52:37Z", "2005-07-03T09:52:37Z", "2005-07-04T09:52:37Z" ], "header:Subject": " Why not both importing AND exporting? =?utf-8?b?4pi6?=", "header:Subject:all": [ " Why not both importing AND exporting? =?utf-8?b?4pi6?=" ], "header:Subject:asText": "Why not both importing AND exporting? ☺", "header:Subject:asText:all": [ "Why not both importing AND exporting? ☺" ], "header:To": " \" James Smythe\" , Friends:\n jane@example.com, =?UTF-8?Q?John_Sm=C3=AEth?=\n ;", "header:To:all": [ " \" James Smythe\" , Friends:\n jane@example.com, =?UTF-8?Q?John_Sm=C3=AEth?=\n ;" ], "header:To:asAddresses": [ { "name": " James Smythe", "email": "james@example.com" }, { "name": null, "email": "jane@example.com" }, { "name": "John Smîth", "email": "john@example.com" } ], "header:To:asAddresses:all": [ [ { "name": " James Smythe", "email": "james@example.com" }, { "name": null, "email": "jane@example.com" }, { "name": "John Smîth", "email": "john@example.com" } ] ], "header:To:asGroupedAddresses": [ { "name": null, "addresses": [ { "name": " James Smythe", "email": "james@example.com" } ] }, { "name": "Friends", "addresses": [ { "name": null, "email": "jane@example.com" }, { "name": "John Smîth", "email": "john@example.com" } ] } ], "header:To:asGroupedAddresses:all": [ [ { "name": null, "addresses": [ { "name": " James Smythe", "email": "james@example.com" } ] }, { "name": "Friends", "addresses": [ { "name": null, "email": "jane@example.com" }, { "name": "John Smîth", "email": "john@example.com" } ] } ] ], "header:X-Address": " John Doe ", "header:X-Address:all": [ " Mary Smith ", " John Doe " ], "header:X-Address:asAddresses": [ { "name": "John Doe", "email": "jdoe@machine.example" } ], "header:X-Address:asAddresses:all": [ [ { "name": "Mary Smith", "email": "mary@example.net" } ], [ { "name": "John Doe", "email": "jdoe@machine.example" } ] ], "header:X-Address:asGroupedAddresses": [ { "name": null, "addresses": [ { "name": "John Doe", "email": "jdoe@machine.example" } ] } ], "header:X-Address:asGroupedAddresses:all": [ [ { "name": null, "addresses": [ { "name": "Mary Smith", "email": "mary@example.net" } ] } ], [ { "name": null, "addresses": [ { "name": "John Doe", "email": "jdoe@machine.example" } ] } ] ], "header:X-Address-Single": " =?ISO-8859-1?Q?Andr=E9?= Pirard ", "header:X-Address-Single:all": [ " =?ISO-8859-1?Q?Andr=E9?= Pirard " ], "header:X-Address-Single:asAddresses": [ { "name": "André Pirard", "email": "PIRARD@vm1.ulg.ac.be" } ], "header:X-Address-Single:asAddresses:all": [ [ { "name": "André Pirard", "email": "PIRARD@vm1.ulg.ac.be" } ] ], "header:X-Address-Single:asGroupedAddresses": [ { "name": null, "addresses": [ { "name": "André Pirard", "email": "PIRARD@vm1.ulg.ac.be" } ] } ], "header:X-Address-Single:asGroupedAddresses:all": [ [ { "name": null, "addresses": [ { "name": "André Pirard", "email": "PIRARD@vm1.ulg.ac.be" } ] } ] ], "header:X-AddressList": " =?ISO-8859-1?Q?Keld_J=F8rn_Simonsen?= ,\n =?ISO-8859-1?Q?Olle_J=E4rnefors?= ", "header:X-AddressList:all": [ " =?US-ASCII*EN?Q?Keith_Moore?= , \n John =?US-ASCII*EN?Q?Doe?= ", " =?ISO-8859-1?Q?Keld_J=F8rn_Simonsen?= ,\n =?ISO-8859-1?Q?Olle_J=E4rnefors?= " ], "header:X-AddressList:asAddresses": [ { "name": "Keld Jørn Simonsen", "email": "keld@dkuug.dk" }, { "name": "Olle Järnefors", "email": "ojarnef@admin.kth.se" } ], "header:X-AddressList:asAddresses:all": [ [ { "name": "Keith Moore", "email": "moore@cs.utk.edu" }, { "name": "John Doe", "email": "moore@cs.utk.edu" } ], [ { "name": "Keld Jørn Simonsen", "email": "keld@dkuug.dk" }, { "name": "Olle Järnefors", "email": "ojarnef@admin.kth.se" } ] ], "header:X-AddressList:asGroupedAddresses": [ { "name": null, "addresses": [ { "name": "Keld Jørn Simonsen", "email": "keld@dkuug.dk" }, { "name": "Olle Järnefors", "email": "ojarnef@admin.kth.se" } ] } ], "header:X-AddressList:asGroupedAddresses:all": [ [ { "name": null, "addresses": [ { "name": "Keith Moore", "email": "moore@cs.utk.edu" }, { "name": "John Doe", "email": "moore@cs.utk.edu" } ] } ], [ { "name": null, "addresses": [ { "name": "Keld Jørn Simonsen", "email": "keld@dkuug.dk" }, { "name": "Olle Järnefors", "email": "ojarnef@admin.kth.se" } ] } ] ], "header:X-AddressList-Single": " Mary Smith , jdoe@example.org, Who? ", "header:X-AddressList-Single:all": [ " Mary Smith , jdoe@example.org, Who? " ], "header:X-AddressList-Single:asAddresses": [ { "name": "Mary Smith", "email": "mary@x.test" }, { "name": null, "email": "jdoe@example.org" }, { "name": "Who?", "email": "one@y.test" } ], "header:X-AddressList-Single:asAddresses:all": [ [ { "name": "Mary Smith", "email": "mary@x.test" }, { "name": null, "email": "jdoe@example.org" }, { "name": "Who?", "email": "one@y.test" } ] ], "header:X-AddressList-Single:asGroupedAddresses": [ { "name": null, "addresses": [ { "name": "Mary Smith", "email": "mary@x.test" }, { "name": null, "email": "jdoe@example.org" }, { "name": "Who?", "email": "one@y.test" } ] } ], "header:X-AddressList-Single:asGroupedAddresses:all": [ [ { "name": null, "addresses": [ { "name": "Mary Smith", "email": "mary@x.test" }, { "name": null, "email": "jdoe@example.org" }, { "name": "Who?", "email": "one@y.test" } ] } ] ], "header:X-AddressesGroup": " \"List 1\": addr1@test.com, addr2@test.com; \"List 2\": \n addr3@test.com, addr4@test.com; addr5@test.com, addr6@test.com", "header:X-AddressesGroup:all": [ " A Group:Ed Jones ,joe@where.test,John ;", " \"List 1\": addr1@test.com, addr2@test.com; \"List 2\": \n addr3@test.com, addr4@test.com; addr5@test.com, addr6@test.com" ], "header:X-AddressesGroup:asAddresses": [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" }, { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" }, { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ], "header:X-AddressesGroup:asAddresses:all": [ [ { "name": "Ed Jones", "email": "c@a.test" }, { "name": null, "email": "joe@where.test" }, { "name": "John", "email": "jdoe@one.test" } ], [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" }, { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" }, { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ] ], "header:X-AddressesGroup:asGroupedAddresses": [ { "name": "List 1", "addresses": [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" } ] }, { "name": "List 2", "addresses": [ { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" } ] }, { "name": null, "addresses": [ { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ] } ], "header:X-AddressesGroup:asGroupedAddresses:all": [ [ { "name": "A Group", "addresses": [ { "name": "Ed Jones", "email": "c@a.test" }, { "name": null, "email": "joe@where.test" }, { "name": "John", "email": "jdoe@one.test" } ] } ], [ { "name": "List 1", "addresses": [ { "name": null, "email": "addr1@test.com" }, { "name": null, "email": "addr2@test.com" } ] }, { "name": "List 2", "addresses": [ { "name": null, "email": "addr3@test.com" }, { "name": null, "email": "addr4@test.com" } ] }, { "name": null, "addresses": [ { "name": null, "email": "addr5@test.com" }, { "name": null, "email": "addr6@test.com" } ] } ] ], "header:X-AddressesGroup-Single": " A Group(Some people)\n :Chris Jones ,\n joe@example.org, John (my dear\n friend); (the end of the group)", "header:X-AddressesGroup-Single:all": [ " A Group(Some people)\n :Chris Jones ,\n joe@example.org, John (my dear\n friend); (the end of the group)" ], "header:X-AddressesGroup-Single:asAddresses": [ { "name": "Chris Jones (Chris's host.)", "email": "c@public.example" }, { "name": null, "email": "joe@example.org" }, { "name": "John (my dear friend)", "email": "jdoe@one.test" }, { "name": "the end of the group", "email": "" } ], "header:X-AddressesGroup-Single:asAddresses:all": [ [ { "name": "Chris Jones (Chris's host.)", "email": "c@public.example" }, { "name": null, "email": "joe@example.org" }, { "name": "John (my dear friend)", "email": "jdoe@one.test" }, { "name": "the end of the group", "email": "" } ] ], "header:X-AddressesGroup-Single:asGroupedAddresses": [ { "name": "A Group (Some people)", "addresses": [ { "name": "Chris Jones (Chris's host.)", "email": "c@public.example" }, { "name": null, "email": "joe@example.org" }, { "name": "John (my dear friend)", "email": "jdoe@one.test" } ] }, { "name": null, "addresses": [ { "name": "the end of the group", "email": "" } ] } ], "header:X-AddressesGroup-Single:asGroupedAddresses:all": [ [ { "name": "A Group (Some people)", "addresses": [ { "name": "Chris Jones (Chris's host.)", "email": "c@public.example" }, { "name": null, "email": "joe@example.org" }, { "name": "John (my dear friend)", "email": "jdoe@one.test" } ] }, { "name": null, "addresses": [ { "name": "the end of the group", "email": "" } ] } ] ], "header:X-Date": " Sun, 21 Nov 2021 15:23:02 -0900", "header:X-Date:all": [ " Sat, 20 Nov 2021 14:22:01 -0800", " Sun, 21 Nov 2021 15:23:02 -0900" ], "header:X-Date:asDate": "2021-11-22T00:23:02Z", "header:X-Date:asDate:all": [ "2021-11-20T22:22:01Z", "2021-11-22T00:23:02Z" ], "header:X-Date-Single": " Tue, 5 Jul 2006 13:52:37 -0500", "header:X-Date-Single:all": [ " Tue, 5 Jul 2006 13:52:37 -0500" ], "header:X-Date-Single:asDate": "2006-07-05T18:52:37Z", "header:X-Date-Single:asDate:all": [ "2006-07-05T18:52:37Z" ], "header:X-Id": " ", "header:X-Id:all": [ " ", " " ], "header:X-Id:asMessageIds": [ "myid4@example.com", "myid5@example.com" ], "header:X-Id:asMessageIds:all": [ [ "myid3@example.com" ], [ "myid4@example.com", "myid5@example.com" ] ], "header:X-Id-Single": " ", "header:X-Id-Single:all": [ " " ], "header:X-Id-Single:asMessageIds": [ "myid@example.com", "myid2@example.com" ], "header:X-Id-Single:asMessageIds:all": [ [ "myid@example.com", "myid2@example.com" ] ], "header:X-List": " ,\n ", "header:X-List:all": [ " ,\n ", " ,\n " ], "header:X-List:asURLs": [ "http://www.mylist2.com/list2", "mailto:list2@mylist2.com" ], "header:X-List:asURLs:all": [ [ "http://www.mylist.com/list", "mailto:list@mylist.com" ], [ "http://www.mylist2.com/list2", "mailto:list2@mylist2.com" ] ], "header:X-List-Single": " (X-Postings are Moderated)", "header:X-List-Single:all": [ " (X-Postings are Moderated)" ], "header:X-List-Single:asURLs": [ "mailto:x-moderator@host.com" ], "header:X-List-Single:asURLs:all": [ [ "mailto:x-moderator@host.com" ] ], "header:X-Text": " =?ISO-8859-1?Q?a?= =?ISO-8859-2?Q?_b?=", "header:X-Text:all": [ " =?iso-8859-1?q?this=20is=20some=20text?=", " =?ISO-8859-1?Q?a?= =?ISO-8859-2?Q?_b?=" ], "header:X-Text:asText": "a b", "header:X-Text:asText:all": [ "this is some text", "a b" ], "header:X-Text-Single": " =?ISO-8859-1?B?SWYgeW91IGNhbiByZWFkIHRoaXMgeW8=?=\n =?ISO-8859-2?B?dSB1bmRlcnN0YW5kIHRoZSBleGFtcGxlLg==?=", "header:X-Text-Single:all": [ " =?ISO-8859-1?B?SWYgeW91IGNhbiByZWFkIHRoaXMgeW8=?=\n =?ISO-8859-2?B?dSB1bmRlcnN0YW5kIHRoZSBleGFtcGxlLg==?=" ], "header:X-Text-Single:asText": "If you can read this you understand the example.", "header:X-Text-Single:asText:all": [ "If you can read this you understand the example." ] } ================================================ FILE: tests/resources/jmap/email_set/headers.eml ================================================ Bcc: "=?utf-8?B?wqFFbCDDsWFuZMO6IGNvbWnDsyDDsW9xdWlzIQ==?=" Cc: "=?utf-8?B?0J/RgNC40LLQtdGCLCDQvNC40YA=?=" Date: Tue, 10 Jul 2018 01:03:11 +0000 From: "Joe Bloggs" In-Reply-To: List-Owner: , List-Subscribe: , List-Subscribe: Message-ID: References: Reply-To: "=?utf-8?B?7JWI64WV7ZWY7IS47JqUIOyEuOqzhA==?=" , "=?utf-8?Q?Antoine_de_Saint-Exup=C3=A9ry?=" Resent-Date: Sat, 2 Jul 2005 09:52:37 +0000 Resent-Date: Sun, 3 Jul 2005 09:52:37 +0000 Resent-Date: Mon, 4 Jul 2005 09:52:37 +0000 Sender: "=?utf-8?B?44OP44Ot44O844O744Ov44O844Or44OJ?=" Subject: Headers test To: "Greg Vaudreuil" , "Ned Freed" , "Keith Moore" X-AddressesGroup: "A Group": "Ed Jones" , , "John" ; X-AddressesGroup: "List 1": , ; "List 2": , ; , ; X-References: <1234@local.machine.example> <3456@example.net> X-References: <789@local.machine.example> X-Text: a b X-Text: this is some text MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="boundary_0" --boundary_0 Content-Language: en Content-Type: text/plain; charset="us-ascii" X-Header: just a value X-Text: more text Content-Transfer-Encoding: quoted-printable I have the most brilliant plan. Let me tell you all about it. What we do i= s, we --boundary_0 Content-Location: https://example.com/html-body.html Content-Type: text/html; charset="utf-8"; name="html-body.html" Content-Transfer-Encoding: quoted-printable
I have the most brilliant plan. = Let me tell you all about it. What we do is, we
--boundary_0-- ================================================ FILE: tests/resources/jmap/email_set/headers.jmap ================================================ { "mailboxIds": { "a": true }, "keywords": { "$draft": true, "$seen": true }, "receivedAt": "2018-07-10T01:03:11Z", "messageId": [ "my-message-id" ], "inReplyTo": [ "other-message-id", "yet-another-message-id" ], "references": [ "first-message-id", "second-message-id" ], "sender": [ { "name": "ハロー・ワールド", "email": "joe@example.com" } ], "from": [ { "name": "Joe Bloggs", "email": "joe@example.com" } ], "to": [ { "name": "Greg Vaudreuil", "email": "gvaudre@NRI.Reston.VA.US" }, { "name": "Ned Freed", "email": "ned@innosoft.com" }, { "name": "Keith Moore", "email": "moore@cs.utk.edu" } ], "cc": [ { "name": "Привет, мир", "email": "addr0@example.com" } ], "bcc": [ { "name": "¡El ñandú comió ñoquis!", "email": "addr1@example.com" } ], "replyTo": [ { "name": "안녕하세요 세계", "email": "addr2@example.com" }, { "name": "Antoine de Saint-Exupéry", "email": "addr3@example.com" } ], "subject": "Headers test", "sentAt": "2018-07-10T01:03:11Z", "bodyStructure": { "headers": [ { "name": "Bcc", "value": " \"=?utf-8?B?wqFFbCDDsWFuZMO6IGNvbWnDsyDDsW9xdWlzIQ==?=\"\r\n\t" }, { "name": "Cc", "value": " \"=?utf-8?B?0J/RgNC40LLQtdGCLCDQvNC40YA=?=\" " }, { "name": "Date", "value": " Tue, 10 Jul 2018 01:03:11 +0000" }, { "name": "From", "value": " \"Joe Bloggs\" " }, { "name": "In-Reply-To", "value": " " }, { "name": "List-Owner", "value": " ,\r\n\t" }, { "name": "List-Subscribe", "value": " ,\r\n\t" }, { "name": "List-Subscribe", "value": " " }, { "name": "Message-ID", "value": " " }, { "name": "References", "value": " " }, { "name": "Reply-To", "value": " \"=?utf-8?B?7JWI64WV7ZWY7IS47JqUIOyEuOqzhA==?=\" , \r\n\t\"=?utf-8?Q?Antoine_de_Saint-Exup=C3=A9ry?=\" " }, { "name": "Resent-Date", "value": " Sat, 2 Jul 2005 09:52:37 +0000" }, { "name": "Resent-Date", "value": " Sun, 3 Jul 2005 09:52:37 +0000" }, { "name": "Resent-Date", "value": " Mon, 4 Jul 2005 09:52:37 +0000" }, { "name": "Sender", "value": " \"=?utf-8?B?44OP44Ot44O844O744Ov44O844Or44OJ?=\" " }, { "name": "Subject", "value": " Headers test" }, { "name": "To", "value": " \"Greg Vaudreuil\" , \r\n\t\"Ned Freed\" , \"Keith Moore\" " }, { "name": "X-AddressesGroup", "value": " \"A Group\": \"Ed Jones\" , \r\n\t, \"John\" ;" }, { "name": "X-AddressesGroup", "value": " \"List 1\": , \r\n\t; \"List 2\": , \r\n\t; , \r\n\t;" }, { "name": "X-References", "value": " <1234@local.machine.example> <3456@example.net>" }, { "name": "X-References", "value": " <789@local.machine.example> " }, { "name": "X-Text", "value": " a b" }, { "name": "X-Text", "value": " this is some text" }, { "name": "MIME-Version", "value": " 1.0" }, { "name": "Content-Type", "value": " multipart/alternative; \r\n\tboundary=\"boundary_0\"" } ], "type": "multipart/alternative", "subParts": [ { "partId": "1", "blobId": "blob_0", "size": 81, "headers": [ { "name": "Content-Language", "value": " en" }, { "name": "Content-Type", "value": " text/plain; charset=\"us-ascii\"" }, { "name": "X-Header", "value": " just a value" }, { "name": "X-Text", "value": " more text" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/plain", "charset": "us-ascii", "language": [ "en" ] }, { "partId": "2", "blobId": "blob_1", "size": 218, "headers": [ { "name": "Content-Location", "value": " https://example.com/html-body.html" }, { "name": "Content-Type", "value": " text/html; charset=\"utf-8\"; name=\"html-body.html\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "name": "html-body.html", "type": "text/html", "charset": "utf-8", "location": "https://example.com/html-body.html" } ] }, "bodyValues": { "1": { "value": "I have the most brilliant plan. Let me tell you all about it. What we do is, we", "isEncodingProblem": false, "isTruncated": false }, "2": { "value": "...", "isEncodingProblem": false, "isTruncated": true } }, "textBody": [ { "partId": "1", "blobId": "blob_0", "size": 81, "headers": [ { "name": "Content-Language", "value": " en" }, { "name": "Content-Type", "value": " text/plain; charset=\"us-ascii\"" }, { "name": "X-Header", "value": " just a value" }, { "name": "X-Text", "value": " more text" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/plain", "charset": "us-ascii", "language": [ "en" ] } ], "htmlBody": [ { "partId": "2", "blobId": "blob_1", "size": 218, "headers": [ { "name": "Content-Location", "value": " https://example.com/html-body.html" }, { "name": "Content-Type", "value": " text/html; charset=\"utf-8\"; name=\"html-body.html\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "name": "html-body.html", "type": "text/html", "charset": "utf-8", "location": "https://example.com/html-body.html" } ], "attachments": [], "hasAttachment": false, "preview": "I have the most brilliant plan. Let me tell you all about it. What we do is, we" } ================================================ FILE: tests/resources/jmap/email_set/headers.json ================================================ { "keywords": { "$seen": true, "$draft": true }, "from": [ { "name": "Joe Bloggs", "email": "joe@example.com" } ], "subject": "Headers test", "receivedAt": "2018-07-10T01:03:11Z", "sentAt": "2018-07-10T11:03:11+10:00", "messageId": [ "my-message-id" ], "inReplyTo": [ "other-message-id", "yet-another-message-id" ], "references": [ "first-message-id", "second-message-id" ], "sender": [ { "name": "ハロー・ワールド", "email": "joe@example.com" } ], "to": [ { "email": "gvaudre@NRI.Reston.VA.US", "name": "Greg Vaudreuil" }, { "email": "ned@innosoft.com", "name": "Ned Freed" }, { "email": "moore@cs.utk.edu", "name": "Keith Moore" } ], "cc": [ { "name": "Привет, мир", "email": "addr0@example.com" } ], "bcc": [ { "name": "¡El ñandú comió ñoquis!", "email": "addr1@example.com" } ], "replyTo": [ { "name": "안녕하세요 세계", "email": "addr2@example.com" }, { "name": "Antoine de Saint-Exupéry", "email": "addr3@example.com" } ], "header:Resent-Date:asDate:all": [ "2005-07-02T11:52:37+02:00", "2005-07-03T12:52:37+03:00", "2005-07-04T13:52:37+04:00" ], "header:X-Text:asText:all": [ "this is some text", "a b" ], "header:List-Owner:asURLs": [ "http://www.host.com/list.cgi?cmd=sub&lst=list", "mailto:list-manager@host.com?body=subscribe%20list" ], "header:List-Subscribe:asURLs:all": [ [ "mailto:list-manager@host.com?body=subscribe%20list" ], [ "ftp://ftp.host.com/list.txt", "mailto:list@host.com?subject=subscribe" ] ], "header:X-References:asMessageIds:all": [ [ "1234@local.machine.example", "3456@example.net" ], [ "789@local.machine.example", "abcd@example.net" ] ], "header:X-AddressesGroup:asGroupedAddresses:all": [ [ { "addresses": [ { "email": "c@a.test", "name": "Ed Jones" }, { "email": "joe@where.test", "name": null }, { "email": "jdoe@one.test", "name": "John" } ], "name": "A Group" } ], [ { "addresses": [ { "email": "addr1@test.com", "name": null }, { "email": "addr2@test.com", "name": null } ], "name": "List 1" }, { "addresses": [ { "email": "addr3@test.com", "name": null }, { "email": "addr4@test.com", "name": null } ], "name": "List 2" }, { "addresses": [ { "email": "addr5@test.com", "name": null }, { "email": "addr6@test.com", "name": null } ], "name": null } ] ], "textBody": [ { "type": "text/plain", "blobId": "I have the most brilliant plan. Let me tell you all about it. What we do is, we", "charset": "us-ascii", "header:Content-Language": "en", "header:X-Header": "just a value", "header:X-Text": "more text" } ], "htmlBody": [ { "type": "text/html", "partId": "a49d", "location": "https://example.com/html-body.html", "name": "html-body.html" } ], "bodyValues": { "a49d": { "value": "
I have the most brilliant plan. Let me tell you all about it. What we do is, we
", "isTruncated": false } } } ================================================ FILE: tests/resources/jmap/email_set/minimal.eml ================================================ Date: Tue, 10 Jul 2018 01:03:11 +0000 Message-ID: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit ================================================ FILE: tests/resources/jmap/email_set/minimal.jmap ================================================ { "mailboxIds": { "a": true }, "keywords": {}, "receivedAt": "2018-07-10T01:03:11Z", "messageId": [ "my-message-id" ], "sentAt": "2018-07-10T01:03:11Z", "bodyStructure": { "partId": "0", "blobId": "blob_0", "size": 2, "headers": [ { "name": "Date", "value": " Tue, 10 Jul 2018 01:03:11 +0000" }, { "name": "Message-ID", "value": " " }, { "name": "MIME-Version", "value": " 1.0" }, { "name": "Content-Type", "value": " text/plain; charset=\"utf-8\"" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "utf-8" }, "bodyValues": { "0": { "value": "\n", "isEncodingProblem": false, "isTruncated": false } }, "textBody": [ { "partId": "0", "blobId": "blob_0", "size": 2, "headers": [ { "name": "Date", "value": " Tue, 10 Jul 2018 01:03:11 +0000" }, { "name": "Message-ID", "value": " " }, { "name": "MIME-Version", "value": " 1.0" }, { "name": "Content-Type", "value": " text/plain; charset=\"utf-8\"" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "utf-8" } ], "htmlBody": [ { "partId": "0", "blobId": "blob_0", "size": 2, "headers": [ { "name": "Date", "value": " Tue, 10 Jul 2018 01:03:11 +0000" }, { "name": "Message-ID", "value": " " }, { "name": "MIME-Version", "value": " 1.0" }, { "name": "Content-Type", "value": " text/plain; charset=\"utf-8\"" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "utf-8" } ], "attachments": [], "hasAttachment": false, "preview": "\n" } ================================================ FILE: tests/resources/jmap/email_set/minimal.json ================================================ { "receivedAt": "2018-07-10T01:03:11Z", "sentAt": "2018-07-10T11:03:11+10:00", "messageId": [ "my-message-id" ] } ================================================ FILE: tests/resources/jmap/email_set/mixed.eml ================================================ Date: Sat, 20 Nov 2021 22:22:01 +0000 From: "Art Vandelay (Vandelay Industries)" Message-ID: Subject: =?utf-8?Q?Why_not_both_importing_AND_exporting=3F_=E2=98=BA?= To: "Colleagues": "James Smythe" ; "Friends": , "=?utf-8?Q?John_Sm=C3=AEth?=" ; MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="boundary_0" --boundary_0 Content-Type: multipart/alternative; boundary="boundary_1" --boundary_1 Content-Language: en Content-Type: text/plain Content-Transfer-Encoding: quoted-printable I was thinking about quitting the =E2=80=9Cexporting=E2=80=9D to focus just = on the =E2=80=9Cimporting=E2=80=9D, but then I thought, why not do both? =E2=98=BA --boundary_1 Content-Language: en_US Content-Type: text/html Content-Transfer-Encoding: quoted-printable

I was thinking about quitting the “exporting” to focus = just on the “importing”,

but then I thought, why not do bo= th? ☺

--boundary_1-- --boundary_0 Content-ID: Content-Type: image/png Content-Transfer-Encoding: base64 aGVyZSBhcmUgdGhlIGVtYmVkZGVkIGltYWdlIGNvbnRlbnRzIQ== --boundary_0 Content-Disposition: attachment; filename="=?utf-8?Q?Book_about_=E2=98=95_tables.pdf?=" Content-Type: x-document/pdf Content-Transfer-Encoding: base64 PGh0bWw+PGJvZHk+4oSM8J2UovCdlKnwnZStIPCdlKrwnZSiIPCdlKLwnZS18J2UrfCdlKzwnZSv 8J2UsSDwnZSq8J2UtiDwnZSf8J2UrPCdlKzwnZSoIPCdlK3wnZSp8J2UovCdlJ7wnZSw8J2UoiE8 L2JvZHk+PC9odG1sPg== --boundary_0-- ================================================ FILE: tests/resources/jmap/email_set/mixed.jmap ================================================ { "mailboxIds": { "a": true }, "keywords": { "$draft": true, "$seen": true, "my-tag": true }, "receivedAt": "2021-11-20T22:22:01Z", "messageId": [ "my-message-id" ], "from": [ { "name": "Art Vandelay (Vandelay Industries)", "email": "art@vandelay.com" } ], "to": [ { "name": "James Smythe", "email": "james@vandelay.com" }, { "name": null, "email": "jane@example.com" }, { "name": "John Smîth", "email": "john@example.com" } ], "subject": "Why not both importing AND exporting? ☺", "sentAt": "2021-11-20T22:22:01Z", "bodyStructure": { "headers": [ { "name": "Date", "value": " Sat, 20 Nov 2021 22:22:01 +0000" }, { "name": "From", "value": " \"Art Vandelay (Vandelay Industries)\" " }, { "name": "Message-ID", "value": " " }, { "name": "Subject", "value": " =?utf-8?Q?Why_not_both_importing_AND_exporting=3F_=E2=98=BA?=" }, { "name": "To", "value": " \"Colleagues\": \"James Smythe\" ; \r\n\t\"Friends\": , \r\n\t\"=?utf-8?Q?John_Sm=C3=AEth?=\" ;" }, { "name": "MIME-Version", "value": " 1.0" }, { "name": "Content-Type", "value": " multipart/mixed; \r\n\tboundary=\"boundary_0\"" } ], "type": "multipart/mixed", "subParts": [ { "headers": [ { "name": "Content-Type", "value": " multipart/alternative; \r\n\tboundary=\"boundary_1\"" } ], "type": "multipart/alternative", "subParts": [ { "partId": "2", "blobId": "blob_0", "size": 131, "headers": [ { "name": "Content-Language", "value": " en" }, { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/plain", "charset": "us-ascii", "language": [ "en" ] }, { "partId": "3", "blobId": "blob_1", "size": 175, "headers": [ { "name": "Content-Language", "value": " en_US" }, { "name": "Content-Type", "value": " text/html" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/html", "charset": "us-ascii", "language": [ "en_US" ] } ] }, { "partId": "4", "blobId": "blob_2", "size": 37, "headers": [ { "name": "Content-ID", "value": " " }, { "name": "Content-Type", "value": " image/png" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "type": "image/png", "cid": "cid:1234-5678-9012-3456" }, { "partId": "5", "blobId": "blob_3", "size": 127, "headers": [ { "name": "Content-Disposition", "value": " attachment; filename=\"=?utf-8?Q?Book_about_=E2=98=95_tables.pdf?=\"" }, { "name": "Content-Type", "value": " x-document/pdf" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "name": "Book about ☕ tables.pdf", "type": "x-document/pdf", "disposition": "attachment" } ] }, "bodyValues": { "2": { "value": "I was thinking about quitting the “exporting” to focus just on the “importing”,\nbut then ...", "isEncodingProblem": false, "isTruncated": true }, "3": { "value": "

I was thinking about quitting the “exporting” to focus just on the “im...", "isEncodingProblem": false, "isTruncated": true } }, "textBody": [ { "partId": "2", "blobId": "blob_0", "size": 131, "headers": [ { "name": "Content-Language", "value": " en" }, { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/plain", "charset": "us-ascii", "language": [ "en" ] }, { "partId": "4", "blobId": "blob_2", "size": 37, "headers": [ { "name": "Content-ID", "value": " " }, { "name": "Content-Type", "value": " image/png" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "type": "image/png", "cid": "cid:1234-5678-9012-3456" } ], "htmlBody": [ { "partId": "3", "blobId": "blob_1", "size": 175, "headers": [ { "name": "Content-Language", "value": " en_US" }, { "name": "Content-Type", "value": " text/html" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/html", "charset": "us-ascii", "language": [ "en_US" ] }, { "partId": "4", "blobId": "blob_2", "size": 37, "headers": [ { "name": "Content-ID", "value": " " }, { "name": "Content-Type", "value": " image/png" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "type": "image/png", "cid": "cid:1234-5678-9012-3456" } ], "attachments": [ { "partId": "4", "blobId": "blob_2", "size": 37, "headers": [ { "name": "Content-ID", "value": " " }, { "name": "Content-Type", "value": " image/png" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "type": "image/png", "cid": "cid:1234-5678-9012-3456" }, { "partId": "5", "blobId": "blob_3", "size": 127, "headers": [ { "name": "Content-Disposition", "value": " attachment; filename=\"=?utf-8?Q?Book_about_=E2=98=95_tables.pdf?=\"" }, { "name": "Content-Type", "value": " x-document/pdf" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "name": "Book about ☕ tables.pdf", "type": "x-document/pdf", "disposition": "attachment" } ], "hasAttachment": true, "preview": "I was thinking about quitting the “exporting” to focus just on the “importing”,\nbut then I thought, why not do both? ☺\n" } ================================================ FILE: tests/resources/jmap/email_set/mixed.json ================================================ { "keywords": { "$seen": true, "$draft": true, "my-tag": true, "ignore-me": false }, "from": [ { "name": "Art Vandelay (Vandelay Industries)", "email": "art@vandelay.com" } ], "header:To:asGroupedAddresses": [ { "name": "Colleagues", "addresses": [ { "name": "James Smythe", "email": "james@vandelay.com" } ] }, { "name": "Friends", "addresses": [ { "email": "jane@example.com" }, { "name": "John Smîth", "email": "john@example.com" } ] } ], "subject": "Why not both importing AND exporting? ☺", "receivedAt": "2021-11-20T14:22:01-08:00", "sentAt": "2021-11-20T14:22:01-08:00", "messageId": [ "my-message-id" ], "textBody": [ { "type": "text/plain", "blobId": "I was thinking about quitting the “exporting” to focus just on the “importing”,\nbut then I thought, why not do both? ☺\n", "header:Content-Language": "en" } ], "htmlBody": [ { "type": "text/html", "blobId": "

I was thinking about quitting the “exporting” to focus just on the “importing”,

but then I thought, why not do both? ☺

", "header:Content-Language": "en_US" } ], "attachments": [ { "type": "image/png", "blobId": "here are the embedded image contents!", "cid": "cid:1234-5678-9012-3456" }, { "type": "x-document/pdf", "blobId": "ℌ𝔢𝔩𝔭 𝔪𝔢 𝔢𝔵𝔭𝔬𝔯𝔱 𝔪𝔶 𝔟𝔬𝔬𝔨 𝔭𝔩𝔢𝔞𝔰𝔢!", "disposition": "attachment", "name": "Book about ☕ tables.pdf" } ] } ================================================ FILE: tests/resources/jmap/email_set/nested_body.eml ================================================ Date: Tue, 10 Jul 2018 01:03:11 +0000 From: "Joe Bloggs" Message-ID: Subject: RFC 8621 Section 4.1.4 test MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="boundary_0" --boundary_0 Content-Disposition: inline Content-Type: text/plain Content-Transfer-Encoding: 7bit Part A --boundary_0 Content-Type: multipart/mixed; boundary="boundary_1" --boundary_1 Content-Type: multipart/alternative; boundary="boundary_2" --boundary_2 Content-Type: multipart/mixed; boundary="boundary_3" --boundary_3 Content-Disposition: inline Content-Type: text/plain Content-Transfer-Encoding: 7bit Part B --boundary_3 Content-Disposition: inline Content-Type: image/jpeg Content-Transfer-Encoding: base64 UGFydCBD --boundary_3 Content-Disposition: inline Content-Type: text/plain Content-Transfer-Encoding: 7bit Part D --boundary_3-- --boundary_2 Content-Type: multipart/related; boundary="boundary_4" --boundary_4 Content-Type: text/html Content-Transfer-Encoding: 7bit Part E --boundary_4 Content-Type: image/jpeg Content-Transfer-Encoding: base64 UGFydCBG --boundary_4-- --boundary_2-- --boundary_1 Content-Disposition: attachment Content-Type: image/jpeg Content-Transfer-Encoding: base64 UGFydCBH --boundary_1 Content-Type: application/x-excel Content-Transfer-Encoding: base64 UGFydCBI --boundary_1 Content-Type: x-message/rfc822 Content-Transfer-Encoding: base64 UGFydCBK --boundary_1-- --boundary_0 Content-Disposition: inline Content-Type: text/plain Content-Transfer-Encoding: 7bit Part K --boundary_0-- ================================================ FILE: tests/resources/jmap/email_set/nested_body.jmap ================================================ { "mailboxIds": { "a": true }, "keywords": { "$draft": true, "$seen": true }, "receivedAt": "2018-07-10T01:03:11Z", "messageId": [ "my-message-id" ], "from": [ { "name": "Joe Bloggs", "email": "joe@example.com" } ], "subject": "RFC 8621 Section 4.1.4 test", "sentAt": "2018-07-10T01:03:11Z", "bodyStructure": { "headers": [ { "name": "Date", "value": " Tue, 10 Jul 2018 01:03:11 +0000" }, { "name": "From", "value": " \"Joe Bloggs\" " }, { "name": "Message-ID", "value": " " }, { "name": "Subject", "value": " RFC 8621 Section 4.1.4 test" }, { "name": "MIME-Version", "value": " 1.0" }, { "name": "Content-Type", "value": " multipart/mixed; \r\n\tboundary=\"boundary_0\"" } ], "type": "multipart/mixed", "subParts": [ { "partId": "1", "blobId": "blob_0", "size": 6, "headers": [ { "name": "Content-Disposition", "value": " inline" }, { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "us-ascii", "disposition": "inline" }, { "headers": [ { "name": "Content-Type", "value": " multipart/mixed; \r\n\tboundary=\"boundary_1\"" } ], "type": "multipart/mixed", "subParts": [ { "headers": [ { "name": "Content-Type", "value": " multipart/alternative; \r\n\tboundary=\"boundary_2\"" } ], "type": "multipart/alternative", "subParts": [ { "headers": [ { "name": "Content-Type", "value": " multipart/mixed; \r\n\tboundary=\"boundary_3\"" } ], "type": "multipart/mixed", "subParts": [ { "partId": "5", "blobId": "blob_1", "size": 6, "headers": [ { "name": "Content-Disposition", "value": " inline" }, { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "us-ascii", "disposition": "inline" }, { "partId": "6", "blobId": "blob_2", "size": 6, "headers": [ { "name": "Content-Disposition", "value": " inline" }, { "name": "Content-Type", "value": " image/jpeg" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "type": "image/jpeg", "disposition": "inline" }, { "partId": "7", "blobId": "blob_3", "size": 6, "headers": [ { "name": "Content-Disposition", "value": " inline" }, { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "us-ascii", "disposition": "inline" } ] }, { "headers": [ { "name": "Content-Type", "value": " multipart/related; \r\n\tboundary=\"boundary_4\"" } ], "type": "multipart/related", "subParts": [ { "partId": "9", "blobId": "blob_4", "size": 6, "headers": [ { "name": "Content-Type", "value": " text/html" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/html", "charset": "us-ascii" }, { "partId": "10", "blobId": "blob_5", "size": 6, "headers": [ { "name": "Content-Type", "value": " image/jpeg" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "type": "image/jpeg" } ] } ] }, { "partId": "11", "blobId": "blob_6", "size": 6, "headers": [ { "name": "Content-Disposition", "value": " attachment" }, { "name": "Content-Type", "value": " image/jpeg" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "type": "image/jpeg", "disposition": "attachment" }, { "partId": "12", "blobId": "blob_7", "size": 6, "headers": [ { "name": "Content-Type", "value": " application/x-excel" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "type": "application/x-excel" }, { "partId": "13", "blobId": "blob_8", "size": 6, "headers": [ { "name": "Content-Type", "value": " x-message/rfc822" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "type": "x-message/rfc822" } ] }, { "partId": "14", "blobId": "blob_9", "size": 6, "headers": [ { "name": "Content-Disposition", "value": " inline" }, { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "us-ascii", "disposition": "inline" } ] }, "bodyValues": { "1": { "value": "Part A", "isEncodingProblem": false, "isTruncated": false }, "14": { "value": "Part K", "isEncodingProblem": false, "isTruncated": false }, "5": { "value": "Part B", "isEncodingProblem": false, "isTruncated": false }, "7": { "value": "Part D", "isEncodingProblem": false, "isTruncated": false }, "9": { "value": "Part E", "isEncodingProblem": false, "isTruncated": false } }, "textBody": [ { "partId": "1", "blobId": "blob_0", "size": 6, "headers": [ { "name": "Content-Disposition", "value": " inline" }, { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "us-ascii", "disposition": "inline" }, { "partId": "5", "blobId": "blob_1", "size": 6, "headers": [ { "name": "Content-Disposition", "value": " inline" }, { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "us-ascii", "disposition": "inline" }, { "partId": "6", "blobId": "blob_2", "size": 6, "headers": [ { "name": "Content-Disposition", "value": " inline" }, { "name": "Content-Type", "value": " image/jpeg" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "type": "image/jpeg", "disposition": "inline" }, { "partId": "7", "blobId": "blob_3", "size": 6, "headers": [ { "name": "Content-Disposition", "value": " inline" }, { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "us-ascii", "disposition": "inline" }, { "partId": "14", "blobId": "blob_9", "size": 6, "headers": [ { "name": "Content-Disposition", "value": " inline" }, { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "us-ascii", "disposition": "inline" } ], "htmlBody": [ { "partId": "1", "blobId": "blob_0", "size": 6, "headers": [ { "name": "Content-Disposition", "value": " inline" }, { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "us-ascii", "disposition": "inline" }, { "partId": "9", "blobId": "blob_4", "size": 6, "headers": [ { "name": "Content-Type", "value": " text/html" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/html", "charset": "us-ascii" }, { "partId": "14", "blobId": "blob_9", "size": 6, "headers": [ { "name": "Content-Disposition", "value": " inline" }, { "name": "Content-Type", "value": " text/plain" }, { "name": "Content-Transfer-Encoding", "value": " 7bit" } ], "type": "text/plain", "charset": "us-ascii", "disposition": "inline" } ], "attachments": [ { "partId": "6", "blobId": "blob_2", "size": 6, "headers": [ { "name": "Content-Disposition", "value": " inline" }, { "name": "Content-Type", "value": " image/jpeg" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "type": "image/jpeg", "disposition": "inline" }, { "partId": "10", "blobId": "blob_5", "size": 6, "headers": [ { "name": "Content-Type", "value": " image/jpeg" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "type": "image/jpeg" }, { "partId": "11", "blobId": "blob_6", "size": 6, "headers": [ { "name": "Content-Disposition", "value": " attachment" }, { "name": "Content-Type", "value": " image/jpeg" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "type": "image/jpeg", "disposition": "attachment" }, { "partId": "12", "blobId": "blob_7", "size": 6, "headers": [ { "name": "Content-Type", "value": " application/x-excel" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "type": "application/x-excel" }, { "partId": "13", "blobId": "blob_8", "size": 6, "headers": [ { "name": "Content-Type", "value": " x-message/rfc822" }, { "name": "Content-Transfer-Encoding", "value": " base64" } ], "type": "x-message/rfc822" } ], "hasAttachment": true, "preview": "Part A" } ================================================ FILE: tests/resources/jmap/email_set/nested_body.json ================================================ { "keywords": { "$seen": true, "$draft": true }, "from": [ { "name": "Joe Bloggs", "email": "joe@example.com" } ], "subject": "RFC 8621 Section 4.1.4 test", "receivedAt": "2018-07-10T01:03:11Z", "sentAt": "2018-07-10T11:03:11+10:00", "messageId": [ "my-message-id" ], "bodyStructure": { "subParts": [ { "blobId": "Part A", "disposition": "inline", "type": "text/plain" }, { "subParts": [ { "subParts": [ { "subParts": [ { "blobId": "Part B", "disposition": "inline", "type": "text/plain" }, { "blobId": "Part C", "disposition": "inline", "type": "image/jpeg" }, { "blobId": "Part D", "disposition": "inline", "type": "text/plain" } ], "type": "multipart/mixed" }, { "subParts": [ { "blobId": "Part E", "type": "text/html" }, { "blobId": "Part F", "type": "image/jpeg" } ], "type": "multipart/related" } ], "type": "multipart/alternative" }, { "blobId": "Part G", "disposition": "attachment", "type": "image/jpeg" }, { "blobId": "Part H", "type": "application/x-excel" }, { "blobId": "Part J", "type": "x-message/rfc822" } ], "type": "multipart/mixed" }, { "blobId": "Part K", "disposition": "inline", "type": "text/plain" } ], "type": "multipart/mixed" } } ================================================ FILE: tests/resources/jmap/email_set/rfc8621_1.eml ================================================ Date: Tue, 10 Jul 2018 01:03:11 +0000 From: "Joe Bloggs" Message-ID: Subject: World domination MIME-Version: 1.0 Content-Language: en Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable I have the most brilliant plan. Let me tell you all about it. What we do i= s, we ================================================ FILE: tests/resources/jmap/email_set/rfc8621_1.jmap ================================================ { "mailboxIds": { "a": true }, "keywords": { "$draft": true, "$seen": true }, "receivedAt": "2018-07-10T01:03:11Z", "messageId": [ "my-message-id" ], "from": [ { "name": "Joe Bloggs", "email": "joe@example.com" } ], "subject": "World domination", "sentAt": "2018-07-10T01:03:11Z", "bodyStructure": { "partId": "0", "blobId": "blob_0", "size": 81, "headers": [ { "name": "Date", "value": " Tue, 10 Jul 2018 01:03:11 +0000" }, { "name": "From", "value": " \"Joe Bloggs\" " }, { "name": "Message-ID", "value": " " }, { "name": "Subject", "value": " World domination" }, { "name": "MIME-Version", "value": " 1.0" }, { "name": "Content-Language", "value": " en" }, { "name": "Content-Type", "value": " text/plain; charset=\"utf-8\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/plain", "charset": "utf-8", "language": [ "en" ] }, "bodyValues": { "0": { "value": "I have the most brilliant plan. Let me tell you all about it. What we do is, we", "isEncodingProblem": false, "isTruncated": false } }, "textBody": [ { "partId": "0", "blobId": "blob_0", "size": 81, "headers": [ { "name": "Date", "value": " Tue, 10 Jul 2018 01:03:11 +0000" }, { "name": "From", "value": " \"Joe Bloggs\" " }, { "name": "Message-ID", "value": " " }, { "name": "Subject", "value": " World domination" }, { "name": "MIME-Version", "value": " 1.0" }, { "name": "Content-Language", "value": " en" }, { "name": "Content-Type", "value": " text/plain; charset=\"utf-8\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/plain", "charset": "utf-8", "language": [ "en" ] } ], "htmlBody": [ { "partId": "0", "blobId": "blob_0", "size": 81, "headers": [ { "name": "Date", "value": " Tue, 10 Jul 2018 01:03:11 +0000" }, { "name": "From", "value": " \"Joe Bloggs\" " }, { "name": "Message-ID", "value": " " }, { "name": "Subject", "value": " World domination" }, { "name": "MIME-Version", "value": " 1.0" }, { "name": "Content-Language", "value": " en" }, { "name": "Content-Type", "value": " text/plain; charset=\"utf-8\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/plain", "charset": "utf-8", "language": [ "en" ] } ], "attachments": [], "hasAttachment": false, "preview": "I have the most brilliant plan. Let me tell you all about it. What we do is, we" } ================================================ FILE: tests/resources/jmap/email_set/rfc8621_1.json ================================================ { "keywords": { "$seen": true, "$draft": true }, "from": [ { "name": "Joe Bloggs", "email": "joe@example.com" } ], "subject": "World domination", "receivedAt": "2018-07-10T01:03:11Z", "sentAt": "2018-07-10T11:03:11+10:00", "messageId": [ "my-message-id" ], "bodyStructure": { "type": "text/plain", "partId": "bd48", "header:Content-Language": "en" }, "bodyValues": { "bd48": { "value": "I have the most brilliant plan. Let me tell you all about it. What we do is, we", "isTruncated": false } } } ================================================ FILE: tests/resources/jmap/email_set/rfc8621_2.eml ================================================ Date: Tue, 10 Jul 2018 01:05:08 +0000 From: "Joe Bloggs" Message-ID: Subject: World domination To: "John" MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="boundary_0" --boundary_0 Content-Language: en Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: quoted-printable
I have the most brilliant plan. = Let me tell you all about it. What we do is, we
--boundary_0 Content-Language: en Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable I have the most brilliant plan. Let me tell you all about it. What we do i= s, we --boundary_0-- ================================================ FILE: tests/resources/jmap/email_set/rfc8621_2.jmap ================================================ { "mailboxIds": { "a": true }, "keywords": { "$draft": true, "$seen": true }, "receivedAt": "2018-07-10T01:05:08Z", "messageId": [ "my-message-id" ], "from": [ { "name": "Joe Bloggs", "email": "joe@example.com" } ], "to": [ { "name": "John", "email": "john@example.com" } ], "subject": "World domination", "sentAt": "2018-07-10T01:05:08Z", "bodyStructure": { "headers": [ { "name": "Date", "value": " Tue, 10 Jul 2018 01:05:08 +0000" }, { "name": "From", "value": " \"Joe Bloggs\" " }, { "name": "Message-ID", "value": " " }, { "name": "Subject", "value": " World domination" }, { "name": "To", "value": " \"John\" " }, { "name": "MIME-Version", "value": " 1.0" }, { "name": "Content-Type", "value": " multipart/alternative; \r\n\tboundary=\"boundary_0\"" } ], "type": "multipart/alternative", "subParts": [ { "partId": "1", "blobId": "blob_0", "size": 218, "headers": [ { "name": "Content-Language", "value": " en" }, { "name": "Content-Type", "value": " text/html; charset=\"utf-8\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/html", "charset": "utf-8", "language": [ "en" ] }, { "partId": "2", "blobId": "blob_1", "size": 81, "headers": [ { "name": "Content-Language", "value": " en" }, { "name": "Content-Type", "value": " text/plain; charset=\"utf-8\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/plain", "charset": "utf-8", "language": [ "en" ] } ] }, "bodyValues": { "1": { "value": "...", "isEncodingProblem": false, "isTruncated": true }, "2": { "value": "I have the most brilliant plan. Let me tell you all about it. What we do is, we", "isEncodingProblem": false, "isTruncated": false } }, "textBody": [ { "partId": "2", "blobId": "blob_1", "size": 81, "headers": [ { "name": "Content-Language", "value": " en" }, { "name": "Content-Type", "value": " text/plain; charset=\"utf-8\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/plain", "charset": "utf-8", "language": [ "en" ] } ], "htmlBody": [ { "partId": "1", "blobId": "blob_0", "size": 218, "headers": [ { "name": "Content-Language", "value": " en" }, { "name": "Content-Type", "value": " text/html; charset=\"utf-8\"" }, { "name": "Content-Transfer-Encoding", "value": " quoted-printable" } ], "type": "text/html", "charset": "utf-8", "language": [ "en" ] } ], "attachments": [], "hasAttachment": false, "preview": "I have the most brilliant plan. Let me tell you all about it. What we do is, we" } ================================================ FILE: tests/resources/jmap/email_set/rfc8621_2.json ================================================ { "keywords": { "$seen": true, "$draft": true }, "from": [ { "name": "Joe Bloggs", "email": "joe@example.com" } ], "to": [ { "name": "John", "email": "john@example.com" } ], "messageId": [ "my-message-id" ], "subject": "World domination", "receivedAt": "2018-07-10T01:05:08Z", "sentAt": "2018-07-10T11:05:08+10:00", "bodyStructure": { "type": "multipart/alternative", "subParts": [ { "partId": "a49d", "type": "text/html", "header:Content-Language": "en" }, { "partId": "bd48", "type": "text/plain", "header:Content-Language": "en" } ] }, "bodyValues": { "bd48": { "value": "I have the most brilliant plan. Let me tell you all about it. What we do is, we", "isTruncated": false }, "a49d": { "value": "
I have the most brilliant plan. Let me tell you all about it. What we do is, we
", "isTruncated": false } } } ================================================ FILE: tests/resources/jmap/email_snippet/html.eml ================================================ Message-Id: <4.2.0.58.20000519003052.00a89c40@pop.example.com> X-Sender: dwsauder@pop.example.com (Unverified) X-Mailer: QUALCOMM Windows Eudora Pro Version 4.2.0.58 X-Priority: 2 (High) Date: Fri, 19 May 2000 00:31:00 -0400 To: Heinz =?iso-8859-1?Q?M=FCller?= From: Doug Sauder Subject: =?iso-8859-1?Q?Die_Hasen_und_die_Fr=F6sche?= Mime-Version: 1.0 Content-Language: de Content-Type: text/html; charset="iso-8859-1" Content-Transfer-Encoding: quoted-printable Die Hasen und = die Fr=F6sche

Die Hasen klagten einst =FCber ihre mi=DFliche Lage; "wir leben", sprach ein Redner, "in steter Furcht vor Menschen und Tieren, eine Beute der Hunde, der Adler, ja fast aller Raubtiere! Unsere stete Angst ist =E4rger als der Tod selbst. Auf, la=DFt uns ein f=FCr allemal sterben."

In einem nahen Teich wollten sie sich nun ers=E4ufen; sie eilten ihm zu; allein das au=DFerordentliche Get=F6se und ihre wunderbare Gestalt erschreckte eine Menge Fr=F6sche, die am Ufer sa=DFen, so sehr, da=DF sie au= fs schnellste untertauchten.

"Halt", rief nun eben dieser Sprecher, "wir wollen das Ers=E4ufen noch ein wenig aufschieben, denn auch uns f=FCrchten, wie ihr seht, einige Tiere, welche also wohl noch ungl=FCcklicher sein m=FCssen als wir."

================================================ FILE: tests/resources/jmap/email_snippet/mixed.eml ================================================ MIME-Version: 1.0 Date: Sat, 13 Aug 2021 15:51:01 +0200 Message-ID: Subject: Biblioteca de Babel From: Jorge Luis Borges To: Julio Cortázar Content-Language: es Content-Type: multipart/alternative; boundary="0000000000006adf7205e61fb0a1" --0000000000006adf7205e61fb0a1 Content-Type: text/plain; charset="UTF-8" Content-Transfer-Encoding: quoted-printable El universo (que otros llaman la *Biblioteca*) se compone de un n=C3=BAmero indefinido, y tal vez infinito, de galer=C3=ADas hexagonales, con vastos pozos de ventila= ci=C3=B3n en el medio, cercados por barandas baj=C3=ADsimas. Desde cualquier hex=C3=A1gono se ven = los pisos inferiores y superiores: interminablemente. La distribuci=C3=B3n de las galer=C3=ADas es invariable. Veinte anaqueles, a cinco largos anaqueles por lado, cubren todos los lados menos dos; su altura, que es la de los pisos, excede apenas la de un bibliotecario normal. Una de las caras libres da a un angosto zagu=C3=A1n, que desemboca en otra galer=C3=ADa, id=C3=A9nt= ica a la primera y a todas. A izquierda y a derecha del zagu=C3=A1n hay dos gabinetes min=C3=BAsculos. Un= o permite dormir de pie; otro, satisfacer las necesidades finales. Por ah=C3=AD pasa la escaler= a espiral, que se abisma y se eleva hacia lo remoto. En el zagu=C3=A1n hay un espejo, que fielmente duplica las apariencias. Los hombres suelen inferir de ese espejo que la *Biblioteca* no es infinita (si lo fuera realmente =C2=BFa qu=C3=A9 esa duplicaci=C3=B3n ilusoria?); yo pre= fiero so=C3=B1ar que las superficies bru=C3=B1idas figuran y prometen el infinito... La luz procede de unas frut= as esf=C3=A9ricas que llevan el nombre de l=C3=A1mparas. Hay dos en cada hex=C3=A1gono: transvers= ales. La luz que emiten es insuficiente, incesante. --0000000000006adf7205e61fb0a1 Content-Type: text/html; charset="UTF-8" Content-Transfer-Encoding: quoted-printable
El universo (que otros llaman la Biblioteca) se com= pone de un n=C3=BAmero indefinido, y
tal vez infinito, de galer=C3=ADas = hexagonales, con vastos pozos de ventilaci=C3=B3n en el medio,
cercados = por barandas baj=C3=ADsimas. Desde cualquier hex=C3=A1gono se ven los pisos= inferiores y
superiores: interminablemente. La distribuci=C3=B3n de las= galer=C3=ADas es invariable. Veinte
anaqueles, a cinco largos anaqueles= por lado, cubren todos los lados menos dos; su altura,
que es la de los= pisos, excede apenas la de un bibliotecario normal. Una de las caras libre= s
da a un angosto zagu=C3=A1n, que desemboca en otra galer=C3=ADa, id=C3= =A9ntica a la primera y a todas. A
izquierda y a derecha del zagu=C3=A1n= hay dos gabinetes min=C3=BAsculos. Uno permite dormir de
pie; otro, sat= isfacer las necesidades finales. Por ah=C3=AD pasa la escalera espiral, que= se abisma
y se eleva hacia lo remoto. En el zagu=C3=A1n hay un espejo, = que fielmente duplica las
apariencias. Los hombres suelen inferir de ese= espejo que la Biblioteca no es infinita (si
lo fuera realmente = =C2=BFa qu=C3=A9 esa duplicaci=C3=B3n ilusoria?); yo prefiero so=C3=B1ar qu= e las superficies
bru=C3=B1idas figuran y prometen el infinito... La luz= procede de unas frutas esf=C3=A9ricas que
llevan el nombre de l=C3=A1mp= aras. Hay dos en cada hex=C3=A1gono: transversales. La luz que
emit= en es insuficiente, incesante.

--0000000000006adf7205e61fb0a1-- ================================================ FILE: tests/resources/jmap/email_snippet/subpart.eml ================================================ From: Al Gore To: White House Transportation Coordinator Subject: [Fwd: Map of Argentina with Description] Content-Language: en Content-Type: multipart/mixed; boundary="D7F------------D7FD5A0B8AB9C65CCDBFA872" This is a multi-part message in MIME format. --D7F------------D7FD5A0B8AB9C65CCDBFA872 Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit Fred, Fire up Air Force One! We're going South! Thanks, Al --D7F------------D7FD5A0B8AB9C65CCDBFA872 Content-Type: message/rfc822 Content-Transfer-Encoding: 7bit Content-Disposition: inline Return-Path: Received: from mailhost.whitehouse.gov ([192.168.51.200]) by heartbeat.whitehouse.gov (8.8.8/8.8.8) with ESMTP id SAA22453 for ; Mon, 13 Aug 1998 l8:14:23 +1000 Received: from the_big_box.whitehouse.gov ([192.168.51.50]) by mailhost.whitehouse.gov (8.8.8/8.8.7) with ESMTP id RAA20366 for vice-president@whitehouse.gov; Mon, 13 Aug 1998 17:42:41 +1000 Date: Mon, 13 Aug 1998 17:42:41 +1000 Message-Id: <199804130742.RAA20366@mai1host.whitehouse.gov> From: Bill Clinton To: A1 (The Enforcer) Gore Subject: Map of Argentina with Description MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="DC8------------DC8638F443D87A7F0726DEF7" This is a multi-part message in MIME format. --DC8------------DC8638F443D87A7F0726DEF7 Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit Hi A1, I finally figured out this MIME thing. Pretty cool. I'll send you some sax music in .au files next week! Anyway, the attached image is really too small to get a good look at Argentina. Try this for a much better map: http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm Then again, shouldn't the CIA have something like that? Bill --DC8------------DC8638F443D87A7F0726DEF7 Content-Type: image/gif; name="map_of_Argentina.gif" Content-Transfer-Encoding: base64 Content-Disposition: inline; fi1ename="map_of_Argentina.gif" R01GOD1hJQA1AKIAAP/////78P/omn19fQAAAAAAAAAAAAAAACwAAAAAJQA1AAAD7Qi63P5w wEmjBCLrnQnhYCgM1wh+pkgqqeC9XrutmBm7hAK3tP31gFcAiFKVQrGFR6kscnonTe7FAAad GugmRu3CmiBt57fsVq3Y0VFKnpYdxPC6M7Ze4crnnHum4oN6LFJ1bn5NXTN7OF5fQkN5WYow BEN2dkGQGWJtSzqGTICJgnQuTJN/WJsojad9qXMuhIWdjXKjY4tenjo6tjVssk2gaWq3uGNX U6ZGxseyk8SasGw3J9GRzdTQky1iHNvcPNNI4TLeKdfMvy0vMqLrItvuxfDW8ubjueDtJufz 7itICBxISKDBgwgTKjyYAAA7 --DC8------------DC8638F443D87A7F0726DEF7-- --D7F------------D7FD5A0B8AB9C65CCDBFA872-- ================================================ FILE: tests/resources/jmap/email_snippet/text_plain.eml ================================================ From: Abidjan Prince To: Bill Foobar Content-Language: en Subject: Help a friend from Abidjan Côte d'Ivoire When my mother died when she was given birth to me, my father took me so special because I am motherless. Before the death of my late father on 22nd June 2013 in a private hospital here in Abidjan Côte d'Ivoire. He secretly called me on his bedside and told me that he has a sum of $7.5M (Seven Million five Hundred Thousand Dollars) left in a suspense account in a local bank here in Abidjan Côte d'Ivoire, that he used my name as his only daughter for the next of kin in deposit of the fund. I am 24year old. Dear I am honorably seeking your assistance in the following ways. 1) To provide any bank account where this money would be transferred into. 2) To serve as the guardian of this fund. 3) To make arrangement for me to come over to your country to further my education and to secure a residential permit for me in your country. Moreover, I am willing to offer you 30 percent of the total sum as compensation for your effort input after the successful transfer of this fund to your nominated account overseas. ================================================ FILE: tests/resources/jmap/email_snippet/text_plain_chinese.eml ================================================ From: "孫子" To: "Bill Foobar" Content-Language: zh Subject: 孫子兵法 <"孫子兵法："> 孫子曰：兵者，國之大事，死生之地，存亡之道，不可不察也。 孫子曰：凡用兵之法，馳車千駟，革車千乘，帶甲十萬；千里饋糧，則內外之費賓客之用，膠漆之材， 車甲之奉，日費千金，然後十萬之師舉矣。 孫子曰：凡用兵之法，全國為上，破國次之；全旅為上，破旅次之；全卒為上，破卒次之；全伍為上，破伍次之。 是故百戰百勝，非善之善者也；不戰而屈人之兵，善之善者也。 孫子曰：昔之善戰者，先為不可勝，以待敵之可勝，不可勝在己，可勝在敵。故善戰者，能為不可勝，不能使敵必可勝。 故曰：勝可知，而不可為。 兵者，詭道也。故能而示之不能，用而示之不用，近而示之遠，遠而示之近。利而誘之，亂而取之，實而備之，強而避之， 怒而撓之，卑而驕之，佚而勞之，親而離之。攻其無備，出其不意，此兵家之勝，不可先傳也。 夫未戰而廟算勝者，得算多也；未戰而廟算不勝者，得算少也；多算勝，少算不勝，而況於無算乎？吾以此觀之，勝負見矣。 孫子曰：凡治眾如治寡，分數是也。鬥眾如鬥寡，形名是也。三軍之眾，可使必受敵而無敗者，奇正是也。兵之所加， 如以碬投卵者，虛實是也。 ================================================ FILE: tests/resources/jmap/sieve/test_discard_reject.sieve ================================================ require ["duplicate", "ihave", "reject", "body"]; if body :contains "TPS" { if duplicate :handle "one_sec_expire" :seconds 1 { error "one_sec_expire handle should not be duplicate."; } if duplicate :uniqueid "one_sec_expire" :seconds 1 { error "one_sec_expire uniqueid should not be duplicate."; } if duplicate :handle "five_secs_expire" :seconds 5 { error "five_secs_expire handle should not be duplicate."; } if duplicate :uniqueid "five_secs_expire" :seconds 5 { error "five_secs_expire uniqueid should not be duplicate."; } discard; } elsif body :contains "T.P.S." { if duplicate :handle "one_sec_expire" :seconds 1 { error "one_sec_expire handle should have expired."; } if duplicate :uniqueid "one_sec_expire" :seconds 1 { error "one_sec_expire uniqueid should have expired."; } if not duplicate :handle "five_secs_expire" :seconds 5 { error "five_secs_expire handle should be duplicate."; } if not duplicate :uniqueid "five_secs_expire" :seconds 5 { error "five_secs_expire uniqueid should be duplicate."; } reject "No soup for you, next!"; } else { error "Unexpected body contents."; } ================================================ FILE: tests/resources/jmap/sieve/test_include.sieve ================================================ require ["include", "ihave"]; include "test_include_this"; error "'stop' within included script ignored or include failed."; ================================================ FILE: tests/resources/jmap/sieve/test_include_global.sieve ================================================ require ["include", "ihave"]; include :global "common"; error "'stop' within included script ignored or global include failed."; ================================================ FILE: tests/resources/jmap/sieve/test_include_this.sieve ================================================ require "reject"; reject "Rejected from an included script."; stop; ================================================ FILE: tests/resources/jmap/sieve/test_mailbox.sieve ================================================ require ["fileinto", "mailbox", "mailboxid", "special-use", "ihave", "imap4flags", "vnd.stalwart.expressions"]; # SpecialUse extension tests if not specialuse_exists ["inbox", "trash"] { error "Special-use mailboxes INBOX or TRASH do not exist (lowercase)."; } if not anyof(specialuse_exists "Inbox" "inbox", specialuse_exists "Deleted Items" "trash") { error "Special-use mailboxes INBOX or TRASH do not exist (mixed-case)."; } if specialuse_exists "dingleberry" { error "An invalid special-use exists."; } if specialuse_exists "archive" { error "A non-existent special-use exists."; } # MailboxId tests if not mailboxidexists "a" { error "Inbox not found by mailboxid."; } if not mailboxidexists ["a", "b"] { error "Inbox and Trash mailboxes not found by mailboxid."; } # MailboxExists tests if not mailboxexists "Inbox" { error "Inbox not found by name."; } if not mailboxexists ["Drafts", "Sent Items"] { error "Drafts and Sent Items not found by name."; } # File into new mailboxes using flags fileinto :create "INBOX / Folder "; fileinto :flags ["$important", "\\Seen"] :create "My/Nested/Mailbox/with/multiple/levels"; # Make sure all mailboxes were created if not mailboxexists "Inbox/Folder" { error "'Inbox/Folder' not found."; } if not mailboxexists "My/Nested/Mailbox/with/multiple/levels" { error "'My/Nested/Mailbox/with/multiple/levels' not found."; } if not mailboxexists "My/Nested/Mailbox/with/multiple" { error "'My/Nested/Mailbox/with/multiple' not found."; } if not mailboxexists "My/Nested" { error "'My/Nested' not found."; } if not mailboxexists "My" { error "'My' not found."; } if eval "llm_prompt('echo-test', 'hello world', 0.5) != 'hello world'" { error "llm_prompt is unavailable."; } ================================================ FILE: tests/resources/jmap/sieve/test_notify_fcc.sieve ================================================ require ["enotify", "fcc", "mailbox", "editheader", "imap4flags"]; if header :matches "Subject" "*TPS*" { notify :message "It's time to file your TPS report." :fcc "Notifications" :create "mailto:sms_gateway@remote.org?subject=It's%20TPS-o-clock"; deleteheader "Subject"; addheader "Subject" "${1}**censored**${2}"; setflag "$seen"; } keep; ================================================ FILE: tests/resources/jmap/sieve/test_redirect_enclose.sieve ================================================ require ["enclose"]; enclose :subject "Check this out" "Attached you'll find a message I just received."; redirect "jane@remote.org"; discard; ================================================ FILE: tests/resources/jmap/sieve/validate_error.sieve ================================================ keep :invalidtag; ================================================ FILE: tests/resources/jmap/sieve/validate_ok.sieve ================================================ if true { keep; } ================================================ FILE: tests/resources/ldap/ldap.cfg ================================================ ################# # LDAP test config ################# # General configuration. debug = true watchconfig = true ################# # Server configuration. [ldap] enabled = true # run on a non privileged port listen = "0.0.0.0:3893" [ldaps] # to enable ldaps genrerate a certificate, eg. with: # openssl req -x509 -newkey rsa:4096 -keyout example.key -out example.crt -days 365 -nodes -subj '/CN=`hostname`' enabled = false listen = "0.0.0.0:3894" cert = "example.crt" key = "example.key" ################# # The backend section controls the data store. [backend] datastore = "config" baseDN = "dc=example,dc=org" nameformat = "cn" groupformat = "ou" [behaviors] # Ignore all capabilities restrictions, for instance allowing every user to perform a search IgnoreCapabilities = false # Enable a "fail2ban" type backoff mechanism temporarily banning repeated failed login attempts LimitFailedBinds = true # How many failed login attempts are allowed before a ban is imposed NumberOfFailedBinds = 3 # How long (in seconds) is the window for failed login attempts PeriodOfFailedBinds = 10 # How long (in seconds) is the ban duration BlockFailedBindsFor = 60 # Clean learnt IP addresses every N seconds PruneSourceTableEvery = 600 # Clean learnt IP addresses not seen in N seconds PruneSourcesOlderThan = 600 ################# # The users section contains a hardcoded list of valid users. [[users]] name = "john" givenname = "john.doe@example.org" sn = "info@example.org" uidnumber = 2 primarygroup = 5 mail = "john@example.org" [[users.customattributes]] principalName = ["John Doe"] userPassword = ["12345"] [[users]] name = "jane" sn = "info@example.org" mail = "jane@example.org" uidnumber = 3 primarygroup = 5 [[users.customattributes]] otherGroups = ["support"] principalName = ["Jane Doe"] userPassword = ["abcde"] [[users]] name = "bill" sn = "info@example.org" mail = "bill@example.org" uidnumber = 4 passsha256 = "5e884898da28047151d0e56f8dc6292773603d0d6aabbdd62a11ef721d1542d8" [[users.customattributes]] principalName = ["Bill Foobar"] diskQuota = [500000] userPassword = ["$2y$05$bvIG6Nmid91Mu9RcmmWZfO5HJIMCT8riNW0hEp8f6/FuA2/mHZFpe"] [[users]] name = "robert" sn = "@catchall.org" mail = "robert@catchall.org" uidnumber = 7 [[users.customattributes]] principalName = ["Robect Foobar"] userPassword = ["nopass"] [[users]] name = "serviceuser" mail = "serviceuser@example.org" uidnumber = 5003 primarygroup = 5502 passsha256 = "652c7dc687d98c9889304ed2e408c74b611e86a40caa51c4b43f1dd5913c5cd0" # mysecret [[users.capabilities]] action = "search" object = "*" ################# # The groups section contains a hardcoded list of valid users. [[groups]] name = "sales" gidnumber = 5 [[groups]] name = "support" gidnumber = 6 [[groups]] name = "svcaccts" gidnumber = 5502 ################# # Enable and configure the optional REST API here. [api] enabled = false internals = true # debug application performance tls = false # enable TLS for production!! listen = "0.0.0.0:5555" cert = "cert.pem" key = "key.pem" ================================================ FILE: tests/resources/ldap/run_glauth.sh ================================================ #!/bin/sh ~/utils/glauth/glauth-darwin-arm64 -c tests/resources/ldap/ldap.cfg ================================================ FILE: tests/resources/otel/docker-compose.yaml ================================================ # docker compose up -d version: "2" services: # Jaeger jaeger-all-in-one: image: jaegertracing/all-in-one:latest restart: always network_mode: host ports: - "16686:16686" - "14268" - "14250" # Zipkin zipkin-all-in-one: image: openzipkin/zipkin:latest restart: always network_mode: host ports: - "9411:9411" # Collector otel-collector: image: otel/opentelemetry-collector:latest restart: always network_mode: host command: [ "--config=/etc/otel-collector-config.yaml", "${OTELCOL_ARGS}" ] volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml ports: - "1888:1888" # pprof extension - "8888:8888" # Prometheus metrics exposed by the collector - "8889:8889" # Prometheus exporter metrics - "13133:13133" # health_check extension - "4317:4317" # OTLP gRPC receiver - "55679:55679" # zpages extension depends_on: - jaeger-all-in-one - zipkin-all-in-one ================================================ FILE: tests/resources/otel/otel-collector-config.yaml ================================================ # docker run -p 4317:4317 --network host --rm -v $(pwd)/tests/resources/otel/otel-collector-config.yaml:/etc/otelcol/config.yaml otel/opentelemetry-collector receivers: otlp: protocols: grpc: exporters: zipkin: endpoint: "http://zipkin-all-in-one:9411/api/v2/spans" format: proto otlp: endpoint: jaeger-all-in-one:4317 tls: insecure: true debug: verbosity: detailed processors: batch: extensions: health_check: pprof: endpoint: :1888 zpages: endpoint: :55679 service: extensions: [pprof, zpages, health_check] pipelines: traces: receivers: [otlp] processors: [batch] exporters: [zipkin, otlp] logs: receivers: [otlp] processors: [batch] exporters: [debug] metrics: receivers: [otlp] processors: [batch] exporters: [debug] ================================================ FILE: tests/resources/otel/stalwart-config.toml ================================================ tracer.otel.type = "otel" tracer.otel.transport = "grpc" tracer.otel.endpoint = "http://127.0.0.1:4317" tracer.otel.level = "trace" metrics.open-telemetry.interval = "10s" metrics.open-telemetry.endpoint = "http://127.0.0.1:4317" metrics.open-telemetry.transport = "grpc" ================================================ FILE: tests/resources/proxy-protocol/Docker.haproxy ================================================ # docker build -t test-haproxy -f Docker.haproxy . # docker run -it --rm --name haproxy-syntax-check test-haproxy haproxy -c -f /usr/local/etc/haproxy/haproxy.cfg # docker run -d -p 1111:1111 --name some-haproxy --sysctl net.ipv4.ip_unprivileged_port_start=0 test-haproxy FROM haproxy:2.3 COPY haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg ================================================ FILE: tests/resources/proxy-protocol/haproxy.cfg ================================================ global log stdout format raw local0 defaults log global timeout connect 5000ms timeout client 50000ms timeout server 50000ms frontend tcp_in bind *:1111 mode tcp option tcplog default_backend tcp_out backend tcp_out mode tcp server docker_server host.docker.internal:143 send-proxy ================================================ FILE: tests/resources/scripts/create_test_cluster.sh ================================================ #!/bin/bash BASE_DIR="/Users/me/Downloads/stalwart-cluster" FEATURES="rocks" NUM_NODES=5 # Kill previous processes sudo pkill stalwart # Delete previous tests rm -rf $BASE_DIR # Build the stalwart binary cargo build -p stalwart --no-default-features --features "$FEATURES" for NUM in $(seq 1 $NUM_NODES); do sudo ifconfig en0 alias 10.0.$NUM.1 netmask 255.255.255.0 mkdir -p $BASE_DIR/data$NUM cat < $BASE_DIR/config$NUM.toml cluster.bind-addr = "10.0._N_.1" cluster.key = "the cluster key" cluster.seed-nodes = ["10.0.1.1", "10.0.2.1", "10.0.3.1"] authentication.fallback-admin.secret = "secret" authentication.fallback-admin.user = "admin" directory.internal.store = "rocksdb" directory.internal.type = "internal" lookup.default.hostname = "mail_N_.example.org" server.http.permissive-cors = true server.listener.https.bind = "10.0._N_.1:1443" server.listener.https.protocol = "http" server.listener.https.tls.implicit = true server.listener.imap.bind = "10.0._N_.1:1143" server.listener.imap.protocol = "imap" server.listener.smtp.bind = "10.0._N_.1:1125" server.listener.smtp.protocol = "smtp" storage.blob = "rocksdb" storage.data = "rocksdb" storage.directory = "internal" storage.fts = "rocksdb" storage.lookup = "rocksdb" store.rocksdb.compression = "lz4" store.rocksdb.path = "_D_/data_N_" store.rocksdb.type = "rocksdb" tracer.stdout.ansi = true tracer.stdout.enable = true tracer.stdout.level = "debug" tracer.stdout.type = "stdout" config.resource.spam-filter = "file:///dev/null" config.resource.webadmin = "file:///dev/null" EOF sudo ./target/debug/stalwart --config $BASE_DIR/config$NUM.toml & done ================================================ FILE: tests/resources/scripts/create_test_env.sh ================================================ #!/bin/bash BASE_DIR="/Users/me/Downloads/stalwart-test" FEATURES="sqlite foundationdb postgres mysql rocks elastic s3 redis" # Delete previous tests rm -rf $BASE_DIR # Create admin user cargo run -p stalwart --no-default-features --features "$FEATURES" -- --init=$BASE_DIR printf "[server.http]\npermissive-cors = true\n" >> $BASE_DIR/etc/config.toml printf "[tracer.stdout]\ntype = 'stdout'\nlevel = 'trace'\nansi = true\nenable = true\n" >> $BASE_DIR/etc/config.toml sed -i '' 's/secret =/secret = "secret"\n#secret =/g' $BASE_DIR/etc/config.toml #cargo run -p stalwart --no-default-features --features "$FEATURES" -- --config=$BASE_DIR/etc/config.toml ================================================ FILE: tests/resources/scripts/create_test_users.sh ================================================ #!/bin/bash export URL="https://127.0.0.1:443" CREDENTIALS="admin:secret" cargo run -p stalwart-cli -- domain create example.org cargo run -p stalwart-cli -- account create john 12345 -d "John Doe" -a john@example.org -a john.doe@example.org cargo run -p stalwart-cli -- account create jane abcde -d "Jane Doe" -a jane@example.org cargo run -p stalwart-cli -- account create bill xyz12 -d "Bill Foobar" -a bill@example.org cargo run -p stalwart-cli -- group create sales -d "Sales Department" cargo run -p stalwart-cli -- group create support -d "Technical Support" cargo run -p stalwart-cli -- account add-to-group john sales support cargo run -p stalwart-cli -- account remove-from-group john support cargo run -p stalwart-cli -- account add-email jane jane.doe@example.org cargo run -p stalwart-cli -- list create everyone everyone@example.org cargo run -p stalwart-cli -- list add-members everyone jane john bill cargo run -p stalwart-cli -- account list cargo run -p stalwart-cli -- import messages --format mbox john _ignore/dovecot-crlf cargo run -p stalwart-cli -- import messages --format maildir john /var/mail/john ================================================ FILE: tests/resources/scripts/imap-log-parser.py ================================================ #!/usr/bin/env python3 """ IMAP Log Parser - Extracts and groups IMAP transactions from log files """ import re import json from collections import defaultdict from datetime import datetime import argparse def unescape_imap_content(content): """ Unescape IMAP content by converting escape sequences back to their original characters """ # Remove surrounding quotes if present if content.startswith('"') and content.endswith('"'): content = content[1:-1] # Common escape sequences in IMAP logs replacements = { '\\r\\n': '\r\n', '\\n': '\n', '\\r': '\r', '\\t': '\t', '\\"': '"', '\\\\': '\\' } for escaped, unescaped in replacements.items(): content = content.replace(escaped, unescaped) return content def parse_imap_log_line(line): """ Parse a single IMAP log line and extract relevant information """ # Pattern to match the log format pattern = r'(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)\s+TRACE\s+Raw IMAP\s+(input received|output sent)\s+.*?remoteIp\s*=\s*([^,]+),\s*remotePort\s*=\s*(\d+).*?contents\s*=\s*(.+)$' match = re.search(pattern, line) if not match: return None timestamp, direction, remote_ip, remote_port, contents = match.groups() return { 'timestamp': timestamp, 'direction': direction, 'remote_ip': remote_ip.strip(), 'remote_port': int(remote_port), 'contents': unescape_imap_content(contents.strip()), 'raw_line': line.strip() } def group_by_connection(log_entries): """ Group log entries by IP and port combination """ connections = defaultdict(list) for entry in log_entries: if entry: # Skip None entries key = f"{entry['remote_ip']}:{entry['remote_port']}" connections[key].append(entry) # Sort entries within each connection by timestamp for key in connections: connections[key].sort(key=lambda x: x['timestamp']) return dict(connections) def format_imap_transaction(entries): """ Format IMAP transaction entries into a readable format """ transaction = [] for entry in entries: direction_symbol = "C: " if "input received" in entry['direction'] else "S: " timestamp = entry['timestamp'] content = entry['contents'] # Clean up the content display if content.endswith('\\r\\n') or content.endswith('\r\n'): content = content.rstrip('\\r\\n\r\n') transaction.append(f"[{timestamp}] {direction_symbol}{content}") return transaction def write_output_file(connections, output_file): """ Write the grouped transactions to an output file """ with open(output_file, 'w', encoding='utf-8') as f: f.write("IMAP Transaction Log Analysis\n") f.write("=" * 50 + "\n\n") for connection_key, entries in connections.items(): f.write(f"Connection: {connection_key}\n") f.write("-" * 30 + "\n") f.write(f"Total messages: {len(entries)}\n") f.write(f"Duration: {entries[0]['timestamp']} to {entries[-1]['timestamp']}\n\n") transaction = format_imap_transaction(entries) for line in transaction: f.write(line + "\n") f.write("\n" + "=" * 50 + "\n\n") def main(): parser = argparse.ArgumentParser(description='Parse IMAP log files and group transactions by connection') parser.add_argument('input_file', help='Input log file path') parser.add_argument('-o', '--output', default='imap_transactions.txt', help='Output file path (default: imap_transactions.txt)') parser.add_argument('-j', '--json', action='store_true', help='Also output raw data as JSON') parser.add_argument('-v', '--verbose', action='store_true', help='Enable verbose output') args = parser.parse_args() if args.verbose: print(f"Reading log file: {args.input_file}") # Parse the log file log_entries = [] imap_line_count = 0 try: with open(args.input_file, 'r', encoding='utf-8') as f: for line_num, line in enumerate(f, 1): if 'Raw IMAP' in line: imap_line_count += 1 parsed_entry = parse_imap_log_line(line) if parsed_entry: log_entries.append(parsed_entry) elif args.verbose: print(f"Warning: Could not parse line {line_num}: {line.strip()}") except FileNotFoundError: print(f"Error: File '{args.input_file}' not found") return 1 except Exception as e: print(f"Error reading file: {e}") return 1 if args.verbose: print(f"Found {imap_line_count} Raw IMAP lines") print(f"Successfully parsed {len(log_entries)} entries") # Group by connection connections = group_by_connection(log_entries) if args.verbose: print(f"Found {len(connections)} unique connections:") for conn_key, entries in connections.items(): print(f" {conn_key}: {len(entries)} messages") # Write output try: write_output_file(connections, args.output) print(f"IMAP transactions written to: {args.output}") # Optionally write JSON output if args.json: json_file = args.output.rsplit('.', 1)[0] + '.json' with open(json_file, 'w', encoding='utf-8') as f: json.dump(connections, f, indent=2, ensure_ascii=False) print(f"Raw data written to: {json_file}") except Exception as e: print(f"Error writing output: {e}") return 1 return 0 if __name__ == "__main__": exit(main()) ================================================ FILE: tests/resources/scripts/imap_import.py ================================================ import imaplib import socket import time import threading from email.message import Message from email.utils import formatdate from datetime import datetime, timedelta def append_message(thread_id, start, end): conn = imaplib.IMAP4('localhost') conn.login('john', '12345') conn.socket().setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) start_time = time.time() base_date = datetime(2000, 1, 1) for n in range(start, end): current_date = base_date + timedelta(hours=n) msg = Message() msg['From'] = 'somebody@some.where' msg['To'] = 'john@example.org' msg['Message-Id'] = f'unique.message.id.{n}@nowhere' msg['Date'] = formatdate(time.mktime(current_date.timetuple()), localtime=False, usegmt=True) msg['Subject'] = f"This is message #{n}" msg.set_payload('...nothing...') response_code, response_details = conn.append('INBOX', '', imaplib.Time2Internaldate(time.mktime(current_date.timetuple())), str(msg).encode('utf-8')) if response_code != 'OK': print(f'Thread {thread_id}: Error while appending message #{n}: {response_code} {response_details}') break if n != 0 and n % 100 == 0: elapsed_time = (time.time() - start_time) * 1000 print(f'Thread {thread_id}: Inserting batch {n} took {elapsed_time} ms.', flush=True) start_time = time.time() conn.logout() num_threads = 5 num_messages = 10000 messages_per_thread = num_messages // num_threads threads = [] for i in range(num_threads): start = i * messages_per_thread end = start + messages_per_thread thread = threading.Thread(target=append_message, args=(i, start, end)) threads.append(thread) thread.start() for thread in threads: thread.join() print("All messages appended.") ================================================ FILE: tests/resources/scripts/imap_import_single.py ================================================ import imaplib import socket import time from email.message import Message from email.utils import formatdate from datetime import datetime, timedelta conn = imaplib.IMAP4('localhost') conn.login('john', '12345') conn.socket().setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) current_date = datetime.now() timestamp = current_date.timestamp() msg = Message() msg['From'] = 'somebody@some.where' msg['To'] = 'john@example.org' msg['Message-Id'] = f'unique.message.id.{current_date}@nowhere' msg['Date'] = formatdate(time.mktime(current_date.timetuple()), localtime=False, usegmt=True) msg['Subject'] = f"This is message #{timestamp}" msg.set_payload('...nothing...') response_code, response_details = conn.append('INBOX', '', imaplib.Time2Internaldate(time.mktime(current_date.timetuple())), str(msg).encode('utf-8')) if response_code != 'OK': print(f'Error while appending message: {response_code} {response_details}') print("Message appended.") conn.logout() ================================================ FILE: tests/resources/scripts/stress_test.py ================================================ import smtplib import imaplib import ssl import threading import random import time import string from email.mime.text import MIMEText smtp_server = "127.0.0.1" smtp_port = 465 imap_server = "127.0.0.1" imap_port = 993 num_threads = 5 runs = 10 # Set to None for infinite loop def read_credentials(file_path): with open(file_path, "r") as file: credentials = [line.strip().split(':') for line in file if line.strip()] return credentials def allow_invalid_certificates(): # Create an SSL context context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE return context def generate_random_string(min_size, max_size): """Generates a random string of a size between min_size and max_size.""" size = random.randint(min_size, max_size) chars = string.ascii_letters + string.digits + ' ' return ''.join(random.choice(chars) for _ in range(size)) def generate_email(username, recipient): """Generate random subject and content for email.""" subject = generate_random_string(10, 100) # Random subject between 10 and 100 characters content_size = random.randint(100, 1048576) # Random content size between 100 bytes and ~1MB content = generate_random_string(content_size, content_size) message = MIMEText(content) message['Subject'] = subject message['From'] = username message['To'] = recipient return message.as_string() def smtp_send_message(username, password, recipient): try: with smtplib.SMTP_SSL(smtp_server, smtp_port, context=allow_invalid_certificates()) as server: server.login(username, password) start_time = time.time() server.sendmail(username, recipient, generate_email(username, recipient)) elapsed_time_ms = (time.time() - start_time) * 1000 print(f"OK {elapsed_time_ms} SMTP {username} -> {recipient}") except Exception as e: print(f"ERR SMTP {e}") def imap_append_message(username, password, recipient): try: with imaplib.IMAP4_SSL(imap_server, imap_port, ssl_context=allow_invalid_certificates()) as imap: imap.login(username, password) start_time = time.time() imap.append('INBOX', None, imaplib.Time2Internaldate(time.time()), generate_email(username, recipient).encode('utf-8')) elapsed_time_ms = (time.time() - start_time) * 1000 print(f"OK {elapsed_time_ms} IMAP APPEND {username}") except Exception as e: print(f"ERR IMAP {e}") def imap_list_fetch(username, password): try: with imaplib.IMAP4_SSL(imap_server, imap_port, ssl_context=allow_invalid_certificates()) as imap: imap.login(username, password) imap.select('INBOX') start_time = time.time() typ, data = imap.search(None, 'ALL') if data[0]: messages = data[0].split() random_msg_num = random.choice(messages) typ, msg_data = imap.fetch(random_msg_num, '(RFC822)') elapsed_time_ms = (time.time() - start_time) * 1000 print(f"OK {elapsed_time_ms} IMAP FETCH {username} {random_msg_num}") except Exception as e: print(f"ERR IMAP {e}") def imap_delete_message(username, password): try: with imaplib.IMAP4_SSL(imap_server, imap_port, ssl_context=allow_invalid_certificates()) as imap: imap.login(username, password) imap.select('INBOX') start_time = time.time() typ, data = imap.search(None, 'ALL') if data[0]: messages = data[0].split() random_msg_num = random.choice(messages) imap.store(random_msg_num, '+FLAGS', '\\Deleted') imap.expunge() elapsed_time_ms = (time.time() - start_time) * 1000 print(f"OK {elapsed_time_ms} IMAP DELETE {username} {random_msg_num}") except Exception as e: print(f"ERR IMAP {e}") def perform_random_action(credentials): username, password = random.choice(credentials) recipient, _ = random.choice(credentials) action = random.choice([smtp_send_message, imap_append_message, imap_list_fetch, imap_delete_message]) if action == smtp_send_message or action == imap_append_message: action(username, password, recipient) else: action(username, password) def thread_function(credentials): if runs: for _ in range(runs): perform_random_action(credentials) else: while True: perform_random_action(credentials) def main(): credentials = read_credentials("users.txt") threads = [] for _ in range(num_threads): thread = threading.Thread(target=thread_function, args=(credentials,)) threads.append(thread) thread.start() for thread in threads: thread.join() if __name__ == '__main__': main() ================================================ FILE: tests/resources/scripts/stress_test_prepare.py ================================================ import requests import random import string import urllib3 # Configuration Variables HOSTNAME = '127.0.0.1' # Replace with the actual hostname DOMAIN = 'test.org' # Replace with your domain name USERNAME = 'admin' # Basic auth username PASSWORD = 'secret' # Basic auth password NUM_USERS = 1000 # Number of test user accounts to create # Suppress InsecureRequestWarning urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) # Generate SHA512 password hash def generate_password(): return ''.join(random.choices(string.ascii_letters + string.digits, k=10)) # Create Domain def create_domain(): url = f"https://{HOSTNAME}/api/domain/{DOMAIN}" response = requests.post(url, auth=(USERNAME, PASSWORD), verify=False) if response.status_code == 200: print(f"Domain '{DOMAIN}' created successfully.") else: print(f"Failed to create domain '{DOMAIN}'. Status Code: {response.status_code}") print(response.text) # Create User Accounts def create_user_accounts(): with open('users.txt', 'w') as file: for i in range(1, NUM_USERS + 1): username = f"test{i}@{DOMAIN}" password = generate_password() data = { "type": "individual", "name": username, "secrets": [password], "emails": [username], "description": f"Tester {i}" } url = f"https://{HOSTNAME}/api/principal" response = requests.post(url, json=data, auth=(USERNAME, PASSWORD), verify=False) if response.status_code == 200: file.write(f"{username}:{password}\n") print(f"User account '{username}' created successfully.") else: print(f"Failed to create user account '{username}'. Status Code: {response.status_code}") print(response.text) def main(): create_domain() create_user_accounts() if __name__ == "__main__": main() ================================================ FILE: tests/resources/smtp/antispam/bounce.test ================================================ expect SUBJ_BOUNCE_WORDS SINGLE_SHORT_PART Subject: Delivery Status Notification (Failure) Test expect BOUNCE SINGLE_SHORT_PART IS_DSN MIME-Version: 1.0 Content-Type: multipart/report; report-type="delivery-status"; boundary="176e677bbd667276_87a2ed9cf1f4ecb_a49e592dab77f72e" --176e677bbd667276_87a2ed9cf1f4ecb_a49e592dab77f72e Content-Type: text/plain Content-Transfer-Encoding: 7bit Your message could not be delivered. --176e677bbd667276_87a2ed9cf1f4ecb_a49e592dab77f72e-- envelope_from spammer@domain.com expect SINGLE_SHORT_PART IS_DSN MIME-Version: 1.0 Content-Type: multipart/report; report-type="delivery-status"; boundary="176e677bbd667276_87a2ed9cf1f4ecb_a49e592dab77f72e" --176e677bbd667276_87a2ed9cf1f4ecb_a49e592dab77f72e Content-Type: text/plain Content-Transfer-Encoding: 7bit Your message could not be delivered. --176e677bbd667276_87a2ed9cf1f4ecb_a49e592dab77f72e-- expect BOUNCE SINGLE_SHORT_PART From: MDaemon X-MDDSN-Message: True Subject: Something went wrong Your message could not be delivered. expect BOUNCE SUBJ_BOUNCE_WORDS SINGLE_SHORT_PART From: Automated Subject: Delivery failure Your message could not be delivered. expect BOUNCE HAS_ATTACHMENT HAS_MESSAGE_PARTS MIME-Version: 1.0 From: Automated Subject: Something unexpected happened Content-Type: multipart/mixed; boundary="176e677bbd667276_87a2ed9cf1f4ecb_a49e592dab77f72e" --176e677bbd667276_87a2ed9cf1f4ecb_a49e592dab77f72e Content-Type: text/plain Content-Transfer-Encoding: 7bit Your message could not be delivered to the following recipients: (TLS error from 'inc.test.com': STARTTLS not advertised by host.) --176e677bbd667276_87a2ed9cf1f4ecb_a49e592dab77f72e Content-Type: message/delivery-status Content-Transfer-Encoding: 7bit Reporting-MTA: dns;mail.stalw.art Arrival-Date: Mon, 3 Jul 2023 16:11:29 +0000 Final-Recipient: rfc822;user@test.com Action: failed Status: 5.0.0 Remote-MTA: dns;inc.test.com --176e677bbd667276_87a2ed9cf1f4ecb_a49e592dab77f72e Content-Type: message/rfc822 Content-Transfer-Encoding: 7bit From: Test Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: quoted-printable Mime-Version: 1.0 (Mac OS X Mail 16.0 \(3731.600.7\)) Subject: Re: Test Date: Mon, 3 Jul 2023 18:11:18 +0200 References: <86a9efeb-1cd4-2aee-fdc4-10dc133d4c1e@test.com> To: test In-Reply-To: <86a9efeb-1cd4-2aee-fdc4-10dc133d4c1e@test.com> --176e677bbd667276_87a2ed9cf1f4ecb_a49e592dab77f72e-- ================================================ FILE: tests/resources/smtp/antispam/classifier.ham ================================================ Message-ID: Subject: i have been trying to research via sa mirrors and search engines if a canned script exists giving clients access to their user_prefs options via a web based cgi interface numerous isps provide this feature to clients but so far i can find nothing our configuration uses amavis postfix and clamav for virus filtering and procmail with spamassassin for spam filtering i would prefer not to have to write a script myself but will appreciate any suggestions this URL email is sponsored by osdn tired of that same old cell phone get a new here for free URL _______________________________________________ spamassassin talk mailing list spamassassin talk URL URL Message-ID: mid2@foobar.org Subject: hello have you seen and discussed this article and his approach thank you URL hell there are no rules here we re trying to accomplish something thomas alva edison this URL email is sponsored by osdn tired of that same old cell phone get a new here for free URL _______________________________________________ spamassassin devel mailing list spamassassin devel URL URL Message-ID: Subject: hi all apologies for the possible silly question i don t think it is but but is eircom s adsl service nat ed and what implications would that have for voip i know there are difficulties with voip or connecting to clients connected to a nat ed network from the internet wild i e machines with static real ips any help pointers would be helpful cheers rgrds bernard bernard tyers national centre for sensor research p NUMBER NUMBER NUMBER NUMBER e bernard tyers URL w URL l nNUMBER _______________________________________________ iiu mailing list iiu URL URL Message-ID: Subject: can someone explain what type of operating system solaris is as ive never seen or used it i dont know wheather to get a server from sun or from dell i would prefer a linux based server and sun seems to be the one for that but im not sure if solaris is a distro of linux or a completely different operating system can someone explain kiall mac innes irish linux users group ilug URL URL for un subscription information list maintainer listmaster URL Message-ID: Subject: folks my first time posting have a bit of unix experience but am new to linux just got a new pc at home dell box with windows xp added a second hard disk for linux partitioned the disk and have installed suse NUMBER NUMBER from cd which went fine except it didn t pick up my monitor i have a dell branded eNUMBERfpp NUMBER lcd flat panel monitor and a nvidia geforceNUMBER tiNUMBER video card both of which are probably too new to feature in suse s default set i downloaded a driver from the nvidia website and installed it using rpm then i ran saxNUMBER as was recommended in some postings i found on the net but it still doesn t feature my video card in the available list what next another problem i have a dell branded keyboard and if i hit caps lock twice the whole machine crashes in linux not windows even the on off switch is inactive leaving me to reach for the power cable instead if anyone can help me in any way with these probs i d be really grateful i ve searched the net but have run out of ideas or should i be going for a different version of linux such as redhat opinions welcome thanks a lot peter irish linux users group ilug URL URL for un subscription information list maintainer listmaster URL Message-ID: Subject: has anyone seen heard of used some package that would let a random person go to a webpage create a mailing list then administer that list also of course let ppl sign up for the lists and manage their subscriptions similar to the old URL but i d like to have it running on my server not someone elses chris URL Message-ID: Subject: hi thank you for the useful replies i have found some interesting tutorials in the ibm developer connection URL and URL registration is needed i will post the same message on the web application security list as suggested by someone for now i thing i will use mdNUMBER for password checking i will use the approach described in secure programmin fo linux and unix how to i will separate the authentication module so i can change its implementation at anytime thank you again mario torre please avoid sending me word or powerpoint attachments see URL Message-ID: Subject: hehe sorry but if you hit caps lock twice the computer crashes theres one ive never heard before have you tryed dell support yet i think dell computers prefer redhat dell provide some computers pre loaded with red hat i dont know for sure tho so get someone elses opnion as well as mine original message from ilug admin URL mailto ilug admin URL on behalf of peter staunton sent NUMBER august NUMBER NUMBER NUMBER to ilug URL subject ilug newbie seeks advice suse NUMBER NUMBER folks my first time posting have a bit of unix experience but am new to linux just got a new pc at home dell box with windows xp added a second hard disk for linux partitioned the disk and have installed suse NUMBER NUMBER from cd which went fine except it didn t pick up my monitor i have a dell branded eNUMBERfpp NUMBER lcd flat panel monitor and a nvidia geforceNUMBER tiNUMBER video card both of which are probably too new to feature in suse s default set i downloaded a driver from the nvidia website and installed it using rpm then i ran saxNUMBER as was recommended in some postings i found on the net but it still doesn t feature my video card in the available list what next another problem i have a dell branded keyboard and if i hit caps lock twice the whole machine crashes in linux not windows even the on off switch is inactive leaving me to reach for the power cable instead if anyone can help me in any way with these probs i d be really grateful i ve searched the net but have run out of ideas or should i be going for a different version of linux such as redhat opinions welcome thanks a lot peter irish linux users group ilug URL URL for un subscription information list maintainer listmaster URL irish linux users group ilug URL URL for un subscription information list maintainer listmaster URL Message-ID: Subject: it will function as a router if that is what you wish it even looks like the modem s embedded os is some kind of linux being that it has interesting interfaces like ethNUMBER i don t use it as a router though i just have it do the absolute minimum dsl stuff and do all the really fun stuff like pppoe on my linux box also the manual tells you what the default password is don t forget to run pppoe over the alcatel speedtouch NUMBERi as in my case you have to have a bridge configured in the router modem s software this lists your vci values etc also does anyone know if the high end speedtouch with NUMBER ethernet ports can act as a full router or do i still need to run a pppoe stack on the linux box regards vin irish linux users group ilug URL URL for un subscription information list maintainer listmaster URL irish linux users group ilug URL URL for un subscription information list maintainer listmaster URL Message-ID: Subject: all is it just me or has there been a massive increase in the amount of email being falsely bounced around the place i ve already received email from a number of people i don t know asking why i am sending them email these can be explained by servers from russia and elsewhere coupled with the false emails i received myself it s really starting to annoy me am i the only one seeing an increase in recent weeks martin martin whelan déise design URL tel NUMBER NUMBER our core product déiseditor allows organisations to publish information to their web site in a fast and cost effective manner there is no need for a full time web developer as the site can be easily updated by the organisations own staff instant updates to keep site information fresh sites which are updated regularly bring users back visit URL for a demonstration déiseditor managing your information _______________________________________________ iiu mailing list iiu URL URL ,0 ================================================ FILE: tests/resources/smtp/antispam/classifier.spam ================================================ Subject: save up to NUMBER on life insurance why spend more than you have to life quote savings ensuring your family s financial security is very important life quote savings makes buying life insurance simple and affordable we provide free access to the very best companies and the lowest rates life quote savings is fast easy and saves you money let us help you get started with the best values in the country on new coverage you can save hundreds or even thousands of dollars by requesting a free quote from lifequote savings our service will take you less than NUMBER minutes to complete shop and compare save up to NUMBER on all types of life insurance hyperlink click here for your free quote protecting your family is the best investment you ll ever make if you are in receipt of this email in error and or wish to be removed from our list hyperlink please click here and type remove if you reside in any state which prohibits e mail solicitations for insurance please disregard this email Subject: a powerhouse gifting program you don t want to miss get in with the founders the major players are on this one for once be where the players are this is your private invitation experts are calling this the fastest way to huge cash flow ever conceived leverage NUMBER NUMBER into NUMBER NUMBER over and over again the question here is you either want to be wealthy or you don t which one are you i am tossing you a financial lifeline and for your sake i hope you grab onto it and hold on tight for the ride of your life testimonials hear what average people are doing their first few days we ve received NUMBER NUMBER in NUMBER day and we are doing that over and over again q s in al i m a single mother in fl and i ve received NUMBER NUMBER in the last NUMBER days d s in fl i was not sure about this when i sent off my NUMBER NUMBER pledge but i got back NUMBER NUMBER the very next day l l in ky i didn t have the money so i found myself a partner to work this with we have received NUMBER NUMBER over the last NUMBER days i think i made the right decision don t you k c in fl i pick up NUMBER NUMBER my first day and i they gave me free leads and all the training you can too j w in ca announcing we will close your sales for you and help you get a fax blast immediately upon your entry you make the money free leads training don t wait call now fax back to NUMBER NUMBER NUMBER NUMBER or call NUMBER NUMBER NUMBER NUMBER name__________________________________phone___________________________________________ fax_____________________________________email____________________________________________ best time to call_________________________time zone________________________________________ this message is sent in compliance of the new e mail bill per section NUMBER paragraph a NUMBER c of s NUMBER further transmissions by the sender of this email may be stopped at no cost to you by sending a reply to this email address with the word remove in the subject line errors omissions and exceptions excluded this is not spam i have compiled this list from our replicate database relative to seattle marketing group the gigt or turbo team for the sole purpose of these communications your continued inclusion is only by your gracious permission if you wish to not receive this mail from me please send an email to tesrewinter URL with remove in the subject and you will be deleted immediately Subject: help wanted we are a NUMBER year old fortune NUMBER company that is growing at a tremendous rate we are looking for individuals who want to work from home this is an opportunity to make an excellent income no experience is required we will train you so if you are looking to be employed from home with a career that has vast opportunities then go URL we are looking for energetic and self motivated people if that is you than click on the link and fill out the form and one of our employement specialist will contact you to be removed from our link simple go to URL Subject: tired of the bull out there want to stop losing money want a real money maker receive NUMBER NUMBER NUMBER NUMBER today experts are calling this the fastest way to huge cash flow ever conceived a powerhouse gifting program you don t want to miss we work as a team this is your private invitation get in with the founders this is where the big boys play the major players are on this one for once be where the players are this is a system that will drive NUMBER NUMBER s to your doorstep in a short period of time leverage NUMBER NUMBER into NUMBER NUMBER over and over again the question here is you either want to be wealthy or you don t which one are you i am tossing you a financial lifeline and for your sake i hope you grab onto it and hold on tight for the ride of your life testimonials hear what average people are doing their first few days we ve received NUMBER NUMBER in NUMBER day and we are doing that over and over again q s in al i m a single mother in fl and i ve received NUMBER NUMBER in the last NUMBER days d s in fl i was not sure about this when i sent off my NUMBER NUMBER pledge but i got back NUMBER NUMBER the very next day l l in ky i didn t have the money so i found myself a partner to work this with we have received NUMBER NUMBER over the last NUMBER days i think i made the right decision don t you k c in fl i pick up NUMBER NUMBER my first day and i they gave me free leads and all the training you can too j w in ca this will be the most important call you make this year free leads training announcing we will close your sales for you and help you get a fax blast immediately upon your entry you make the money free leads training don t wait call now NUMBER NUMBER NUMBER NUMBER print and fax to NUMBER NUMBER NUMBER NUMBER or send an email requesting more information to successleads URL please include your name and telephone number receive NUMBER NUMBER free leads just for responding a NUMBER NUMBER value name___________________________________ phone___________________________________ fax_____________________________________ email___________________________________ this message is sent in compliance of the new e mail bill per section NUMBER paragraph a NUMBER c of s NUMBER further transmissions by the sender of this email may be stopped at no cost to you by sending a reply to this email address with the word remove in the subject line errors omissions and exceptions excluded this is not spam i have compiled this list from our replicate database relative to seattle marketing group the gigt or turbo team for the sole purpose of these communications your continued inclusion is only by your gracious permission if you wish to not receive this mail from me please send an email to tesrewinter URL with remove in the subject and you will be deleted immediately Subject: cellular phone accessories all at below wholesale prices http NUMBER NUMBER NUMBER NUMBER NUMBER sites merchant sales hands free ear buds NUMBER NUMBER phone holsters NUMBER NUMBER booster antennas only NUMBER NUMBER phone cases NUMBER NUMBER car chargers NUMBER NUMBER face plates as low as NUMBER NUMBER lithium ion batteries as low as NUMBER NUMBER http NUMBER NUMBER NUMBER NUMBER NUMBER sites merchant sales click below for accessories on all nokia motorola lg nextel samsung qualcomm ericsson audiovox phones at below wholesale prices http NUMBER NUMBER NUMBER NUMBER NUMBER sites merchant sales if you need assistance please call us NUMBER NUMBER NUMBER to be removed from future mailings please send your remove request to remove me now NUMBER URL thank you and have a super day Subject: conferencing made easy only NUMBER cents per minute including long distance no setup fees no contracts or monthly fees call anytime from anywhere to anywhere connects up to NUMBER participants simplicity in set up and administration operator help available NUMBER NUMBER the highest quality service for the lowest rate in the industry fill out the form below to find out how you can lower your phone bill every month required input field name web address company name state business phone home phone email address type of business to be removed from our distribution lists please hyperlink click here Subject: dear friend i am mrs sese seko widow of late president mobutu sese seko of zaire now known as democratic republic of congo drc i am moved to write you this letter this was in confidence considering my presentcircumstance and situation i escaped along with my husband and two of our sons george kongolo and basher out of democratic republic of congo drc to abidjan cote d ivoire where my family and i settled while we later moved to settled in morroco where my husband later died of cancer disease however due to this situation we decided to changed most of my husband s billions of dollars deposited in swiss bank and other countries into other forms of money coded for safe purpose because the new head of state of dr mr laurent kabila has made arrangement with the swiss government and other european countries to freeze all my late husband s treasures deposited in some european countries hence my children and i decided laying low in africa to study the situation till when things gets better like now that president kabila is dead and the son taking over joseph kabila one of my late husband s chateaux in southern france was confiscated by the french government and as such i had to change my identity so that my investment will not be traced and confiscated i have deposited the sum eighteen million united state dollars us NUMBER NUMBER NUMBER NUMBER with a security company for safekeeping the funds are security coded to prevent them from knowing the content what i want you to do is to indicate your interest that you will assist us by receiving the money on our behalf acknowledge this message so that i can introduce you to my son kongolo who has the out modalities for the claim of the said funds i want you to assist in investing this money but i will not want my identity revealed i will also want to buy properties and stock in multi national companies and to engage in other safe and non speculative investments may i at this point emphasise the high level of confidentiality which this business demands and hope you will not betray the trust and confidence which i repose in you in conclusion if you want to assist us my son shall put you in the picture of the business tell you where the funds are currently being maintained and also discuss other modalities including remunerationfor your services for this reason kindly furnish us your contact information that is your personal telephone and fax number for confidential URL regards mrs m sese seko Subject: lowest rates available for term life insurance take a moment and fill out our online form to see the low rate you qualify for save up to NUMBER from regular rates smokers accepted URL representing quality nationwide carriers act now to easily remove your address from the list go to URL please allow NUMBER NUMBER hours for removal Subject: central bank of nigeria foreign remittance dept tinubu square lagos nigeria email smith_j URL NUMBERth of august NUMBER attn president ceo strictly private business proposal i am mr johnson s abu the bills and exchange director at the foreignremittance department of the central bank of nigeria i am writingyou this letter to ask for your support and cooperation to carrying thisbusiness opportunity in my department we discovered abandoned the sumof us NUMBER NUMBER NUMBER NUMBER thirty seven million four hundred thousand unitedstates dollars in an account that belong to one of our foreign customers an american late engr john creek junior an oil merchant with the federal government of nigeria who died along with his entire family of a wifeand two children in kenya airbus aNUMBER NUMBER flight kqNUMBER in novemberNUMBER since we heard of his death we have been expecting his next of kin tocome over and put claims for his money as the heir because we cannotrelease the fund from his account unless someone applies for claims asthe next of kin to the deceased as indicated in our banking guidelines unfortunately neither their family member nor distant relative hasappeared to claim the said fund upon this discovery i and other officialsin my department have agreed to make business with you release the totalamount into your account as the heir of the fund since no one came forit or discovered either maintained account with our bank other wisethe fund will be returned to the bank treasury as unclaimed fund we have agreed that our ratio of sharing will be as stated thus NUMBER for you as foreign partner and NUMBER for us the officials in my department upon the successful completion of this transfer my colleague and i willcome to your country and mind our share it is from our NUMBER we intendto import computer accessories into my country as way of recycling thefund to commence this transaction we require you to immediately indicateyour interest by calling me or sending me a fax immediately on the abovetelefax and enclose your private contact telephone fax full nameand address and your designated banking co ordinates to enable us fileletter of claim to the appropriate department for necessary approvalsbefore the transfer can be made note also this transaction must be kept strictly confidential becauseof its nature nb please remember to give me your phone and fax no mr johnson smith abu irish linux users group ilug URL URL for un subscription information list maintainer listmaster URL Subject: dear stuart are you tired of searching for love in all the wrong places find love now at URL URL browse through thousands of personals in your area join for free URL search e mail chat use URL to meet cool guys and hot girls go NUMBER on NUMBER or use our private chat rooms click on the link to get started URL find love now you have received this email because you have registerd with emailrewardz or subscribed through one of our marketing partners if you have received this message in error or wish to stop receiving these great offers please click the remove link above to unsubscribe from these mailings please click here URL ================================================ FILE: tests/resources/smtp/antispam/classifier.test ================================================ envelope_to hello@world.com expect PROB_SPAM_HIGH Subject: save up to NUMBER on life insurance why spend more than you have to life quote savings ensuring your family s financial security is very important life quote savings makes buying life insurance simple and affordable we provide free access to the very best companies and the lowest rates life quote savings is fast easy and saves you money let us help you get started with the best values in the country on new coverage you can save hundreds or even thousands of dollars by requesting a free quote from lifequote savings our service will take you less than NUMBER minutes to complete shop and compare save up to NUMBER on all types of life insurance hyperlink click here for your free quote protecting your family is the best investment you ll ever make if you are in receipt of this email in error and or wish to be removed from our list hyperlink please click here and type remove if you reside in any state which prohibits e mail solicitations for insurance please disregard this email envelope_to hello@world.com expect PROB_HAM_HIGH Subject: can someone explain what type of operating system solaris is as ive never seen or used it i dont know wheather to get a server from sun or from dell i would prefer a linux based server and sun seems to be the one for that but im not sure if solaris is a distro of linux or a completely different operating system can someone explain kiall mac innes irish linux users group ilug URL URL for un subscription information list maintainer listmaster URL envelope_to hello@world.com expect PROB_SPAM_UNCERTAIN Subject: Lorem ipsum dolor sit amet, consectetur adipiscing elit sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. ================================================ FILE: tests/resources/smtp/antispam/classifier_features.test ================================================ From: bill@example.com To: jdoe@example.com Subject: TPS Report I'm going to need those TPS reports ASAP. So, if you could do that, that'd be great. [ { "type": "word", "value": "_allcaps" }, { "type": "word", "value": "asap" }, { "type": "word", "value": "could" }, { "type": "word", "value": "go" }, { "type": "word", "value": "great" }, { "type": "word", "value": "need" }, { "type": "word", "value": "report" }, { "type": "word", "value": "tps" }, { "type": "sender", "value": "bill@example.com" }, { "type": "sender", "value": "example.com" } ] From: Hendrik To: Harrie Date: Sat, 11 Oct 2010 00:31:44 +0200 Subject: One Two Three Four Content-Type: multipart/mixed; boundary=AA This is a multi-part message in MIME format. --AA Content-Type: multipart/mixed; boundary=BB This is a multi-part message in MIME format. --BB Content-Type: text/plain; charset="us-ascii" This is the first message part containing plain text. --BB Content-Type: text/plain; charset="us-ascii" This is another plain text message part. --BB-- This is the end of MIME multipart. --AA Content-Type: text/html; charset="us-ascii" This is a piece of HTML text. --AA-- This is the end of MIME multipart. [ { "type": "word", "value": "anoth" }, { "type": "word", "value": "contain" }, { "type": "word", "value": "first" }, { "type": "word", "value": "four" }, { "type": "word", "value": "html" }, { "type": "word", "value": "messag" }, { "type": "word", "value": "one" }, { "type": "word", "value": "part" }, { "type": "word", "value": "piec" }, { "type": "word", "value": "plain" }, { "type": "word", "value": "text" }, { "type": "word", "value": "three" }, { "type": "word", "value": "two" }, { "type": "sender", "value": "example.com" }, { "type": "sender", "value": "hendrik@example.com" }, { "type": "mime_type", "value": "multipart/mixed" }, { "type": "mime_type", "value": "text/html" }, { "type": "mime_type", "value": "text/plain" } ] Content-Type: text/html; charset="utf-8" Subject: IPs in HTML are not urls Das System wurde um 01.01.1970 08:28:00 für die IP-Adresse 123.123.123.123 gesperrt.

Der Besucher hat versucht, sich mit folgenden Daten anzumelden.
Partner: 12345678
Portal: IP-Sperre einsehen [ { "type": "word", "value": "adress" }, { "type": "word", "value": "anzumeld" }, { "type": "word", "value": "besuch" }, { "type": "word", "value": "dat" }, { "type": "word", "value": "einseh" }, { "type": "word", "value": "folgend" }, { "type": "word", "value": "gesperrt" }, { "type": "word", "value": "html" }, { "type": "word", "value": "ip" }, { "type": "word", "value": "partn" }, { "type": "word", "value": "portal" }, { "type": "word", "value": "sperr" }, { "type": "word", "value": "syst" }, { "type": "word", "value": "url" }, { "type": "word", "value": "versucht" }, { "type": "word", "value": "wurd" }, { "type": "number", "code": [ 105, 2 ] }, { "type": "number", "code": [ 105, 4 ] }, { "type": "number", "code": [ 105, 8 ] }, { "type": "url", "value": "!ip" }, { "type": "url", "value": "_example" }, { "type": "url", "value": "_php" }, { "type": "url", "value": "localhost.de" }, { "type": "url", "value": "www.localhost.de" }, { "type": "mime_type", "value": "text/html" }, { "type": "html_anchor", "href": "https" } ] X-Spam-Result: DMARC_POLICY_ALLOW (-0.50), TEST (0.0), SOURCE_ASN_123 (1.00) From: Client Services To: user@domain.org Subject: Tether Important Update ! Content-Type: text/html Content-Transfer-Encoding: quoted-printable 3D"If [ { "type": "word", "value": "_allcaps" }, { "type": "word", "value": "_null" }, { "type": "word", "value": "browser" }, { "type": "word", "value": "cashback" }, { "type": "word", "value": "click" }, { "type": "word", "value": "import" }, { "type": "word", "value": "messag" }, { "type": "word", "value": "open" }, { "type": "word", "value": "pleas" }, { "type": "word", "value": "read" }, { "type": "word", "value": "reward" }, { "type": "word", "value": "tether" }, { "type": "word", "value": "updat" }, { "type": "sender", "value": "noreply@tetheer.com" }, { "type": "sender", "value": "tetheer.com" }, { "type": "asn", "number": [ 0, 0, 0, 123 ] }, { "type": "url", "value": "metaskwap.online" }, { "type": "mime_type", "value": "text/html" }, { "type": "html_image", "src": "data" }, { "type": "html_anchor", "href": "https" } ] From: "BBVA" Reply-To: noreply@grupokonecta.net Content-Type: multipart/alternative; charset="UTF-8"; boundary="b1_3d217f30a568faa9ce3dd7dc73399561" Content-Transfer-Encoding: quoted-printable --b1_3d217f30a568faa9ce3dd7dc73399561 Content-Type: text/plain; format=flowed; charset="UTF-8" Content-Transfer-Encoding: quoted-printable Tarjeta de cr=C3=A9dito BBVA La tarjeta de cr=C3=A9dito para viajar con tus consumos Pedila 100% online y empez=C3=A1 a disfrutar Conocer oferta Un mundo de beneficios con las tarjetas de cr=C3=A9dito BBVA compras en cuotas Compras en cuotas Pod=C3=A9s disfrutar hoy de los productos =E2=80=A8que quer=C3=A9s y pagarl= os en cuotas descuentos y reintegros Descuentos y reintegros Entretenimiento, gastronom=C3=ADa, farmacia, ropa =E2=80=A8y m=C3=A1s rubro= s con promociones exclusivas puntos bbva Viajes con Puntos BBVA Vuelos, alojamientos y mucho m=C3=A1s canjeando Puntos BBVA que sum=C3= =A1s con tus compras Conocer oferta Descubr=C3=AD la tarjeta que mejor se adapta a vos Todas las tarjetas Black Platinum Gold Internacional Todas las tarjetas visa black Tarjeta Visa Signature L=C3=ADmites desde $600.000 15% extra en acumulaci=C3=B3n de Puntos BBVA Acceso a salas VIP en aeropuertos Asistencia en viajes con cobertura de hasta 250.000 USD Extracci=C3=B3n de efectivo en el exterior Seguro de robo en cajero y compra protegida Atenci=C3=B3n personalizada para resolver tus consultas Tarjetas adicionales sin costo Es necesario un ingreso m=C3=ADnimo mensual de $200.000 Conocer m=C3=A1s mastercard black Tarjeta Mastercard Black L=C3=ADmites desde $600.000 15% extra en acumulaci=C3=B3n de Puntos BBVA Acceso a salas VIP en aeropuertos Asistencia en viajes con cobertura de hasta 250.000 USD Extracci=C3=B3n de efectivo en el exterior Seguro de robo en cajero y compra protegida Atenci=C3=B3n personalizada para resolver tus consultas Tarjetas adicionales sin costo Es necesario un ingreso m=C3=ADnimo mensual de $200.000 Conocer m=C3=A1s tarjeta platinum visa Tarjeta Visa Platinum L=C3=ADmites desde $350.000 5% extra en acumulaci=C3=B3n de Puntos BBVA Asistencia en viajes con cobertura de hasta 170.000 USD Extracci=C3=B3n de efectivo en el exterior Atenci=C3=B3n personalizada para resolver tus consultas Tarjetas adicionales sin costo Es necesario un ingreso m=C3=ADnimo mensual de $120.000 Conocer m=C3=A1s tarjeta platinum mastercard Tarjeta Mastercard Platinum L=C3=ADmites desde $350.000 5% extra en acumulaci=C3=B3n de Puntos BBVA Asistencia en viajes con cobertura de hasta 50.000 USD y 30.000 EUR Extracci=C3=B3n de efectivo en el exterior Atenci=C3=B3n personalizada para resolver tus consultas Tarjetas adicionales sin costo Es necesario un ingreso m=C3=ADnimo mensual de $120.000 Conocer m=C3=A1s tarjeta gold visa Tarjeta Visa Gold L=C3=ADmites desde $100.000 Puntos BBVA para viajar Extracci=C3=B3n de efectivo en el exterior Tarjetas adicionales sin costo Es necesario un ingreso m=C3=ADnimo mensual de $20.000 Conocer m=C3=A1s tarjeta gold mastercard Tarjeta Mastercard Gold L=C3=ADmites desde $100.000 Puntos BBVA para viajar Extracci=C3=B3n de efectivo en el exterior Tarjetas adicionales sin costo Es necesario un ingreso m=C3=ADnimo mensual de $35.000 Conocer m=C3=A1s Tarjeta Visa Internacional L=C3=ADmites desde $10.000 Puntos BBVA para viajar Extracci=C3=B3n de efectivo en el exterior Tarjetas adicionales sin costo Es necesario un ingreso m=C3=ADnimo mensual de $20.000 Conocer m=C3=A1s Tarjeta Mastercard Internacional L=C3=ADmites desde $10.000 Puntos BBVA para viajar Extracci=C3=B3n de efectivo en el exterior Tarjetas adicionales sin costo Es necesario un ingreso m=C3=ADnimo mensual de $20.000 Conocer m=C3=A1s --b1_3d217f30a568faa9ce3dd7dc73399561 Content-Type: text/html; charset="UTF-8" Content-Transfer-Encoding: quoted-printable --b1_3d217f30a568faa9ce3dd7dc73399561-- [ { "type": "word", "value": "_allcaps" }, { "type": "word", "value": "_null" }, { "type": "word", "value": "acces" }, { "type": "word", "value": "acumul" }, { "type": "word", "value": "adapt" }, { "type": "word", "value": "adicional" }, { "type": "word", "value": "aeropuert" }, { "type": "word", "value": "aloj" }, { "type": "word", "value": "aqu" }, { "type": "word", "value": "asistent" }, { "type": "word", "value": "atencion" }, { "type": "word", "value": "bbva" }, { "type": "word", "value": "benefici" }, { "type": "word", "value": "black" }, { "type": "word", "value": "cajer" }, { "type": "word", "value": "canj" }, { "type": "word", "value": "click" }, { "type": "word", "value": "cobertur" }, { "type": "word", "value": "compr" }, { "type": "word", "value": "conoc" }, { "type": "word", "value": "consult" }, { "type": "word", "value": "consum" }, { "type": "word", "value": "cost" }, { "type": "word", "value": "credit" }, { "type": "word", "value": "cuot" }, { "type": "word", "value": "descubr" }, { "type": "word", "value": "descuent" }, { "type": "word", "value": "disfrut" }, { "type": "word", "value": "efect" }, { "type": "word", "value": "empez" }, { "type": "word", "value": "entreten" }, { "type": "word", "value": "es" }, { "type": "word", "value": "esperando" }, { "type": "word", "value": "estaba" }, { "type": "word", "value": "eur" }, { "type": "word", "value": "exclus" }, { "type": "word", "value": "exterior" }, { "type": "word", "value": "extra" }, { "type": "word", "value": "extraccion" }, { "type": "word", "value": "farmaci" }, { "type": "word", "value": "gastronom" }, { "type": "word", "value": "gold" }, { "type": "word", "value": "hoy" }, { "type": "word", "value": "iacut" }, { "type": "word", "value": "ingres" }, { "type": "word", "value": "internacional" }, { "type": "word", "value": "la" }, { "type": "word", "value": "limit" }, { "type": "word", "value": "mastercard" }, { "type": "word", "value": "mejor" }, { "type": "word", "value": "mensual" }, { "type": "word", "value": "minim" }, { "type": "word", "value": "mund" }, { "type": "word", "value": "necesari" }, { "type": "word", "value": "ofert" }, { "type": "word", "value": "onlin" }, { "type": "word", "value": "oportunidad" }, { "type": "word", "value": "pag" }, { "type": "word", "value": "pedil" }, { "type": "word", "value": "personaliz" }, { "type": "word", "value": "platinum" }, { "type": "word", "value": "podes" }, { "type": "word", "value": "product" }, { "type": "word", "value": "promocion" }, { "type": "word", "value": "proteg" }, { "type": "word", "value": "punt" }, { "type": "word", "value": "que" }, { "type": "word", "value": "queres" }, { "type": "word", "value": "reintegr" }, { "type": "word", "value": "resolv" }, { "type": "word", "value": "rob" }, { "type": "word", "value": "rop" }, { "type": "word", "value": "rubr" }, { "type": "word", "value": "sal" }, { "type": "word", "value": "segur" }, { "type": "word", "value": "signatur" }, { "type": "word", "value": "sumas" }, { "type": "word", "value": "tarjet" }, { "type": "word", "value": "tod" }, { "type": "word", "value": "usd" }, { "type": "word", "value": "viaj" }, { "type": "word", "value": "vip" }, { "type": "word", "value": "vis" }, { "type": "word", "value": "vos" }, { "type": "word", "value": "vuel" }, { "type": "number", "code": [ 105, 1 ] }, { "type": "number", "code": [ 105, 2 ] }, { "type": "number", "code": [ 105, 3 ] }, { "type": "unicode_category", "value": "Sc" }, { "type": "sender", "value": "grupokonecta.net" }, { "type": "sender", "value": "noreply@grupokonecta.net" }, { "type": "url", "value": "_aff" }, { "type": "url", "value": "_jpg" }, { "type": "url", "value": "i.imgur.com" }, { "type": "url", "value": "imgur.com" }, { "type": "url", "value": "leadsinbx.com" }, { "type": "url", "value": "track.leadsinbx.com" }, { "type": "mime_type", "value": "multipart/alternative" }, { "type": "mime_type", "value": "text/html" }, { "type": "mime_type", "value": "text/plain" }, { "type": "html_image", "src": "https" }, { "type": "html_anchor", "href": "http" } ] From: Spammer Systems Iran Subject: =?utf-8?b?2KfZgdiy2YjZhtmH4oCM2YfYp9uMINin2LPZhdin2LHYqtix2YXbjNmEIHw=?= =?utf-8?b?INiq2YjYs9i52Ycg24zYp9mB2KrZhyDYqtmI2LPYtyDYotix2qnYpw==?= Message-Id: To: spam@target.org Reply-To: Spammer Systems Iran Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable vＥⓡ𝔂 𝔽𝕌Ňℕｙ ţ乇𝕏𝓣 =D8=A7=D9=81=D8=B2=D9=88=D9=86=D9=87=E2=80=8C=D9=87=D8=A7=DB=8C SpammerMail= =D8=AA=D9=88=D8=B3=D8=B9=D9=87 =DB=8C=D8=A7=D9=81=D8=AA=D9=87 =D8=AA=D9=88= =D8=B3=D8=B7 =D8=A2=D8=B1=DA=A9=D8=A7 =D8=B1=D8=A7=DB=8C=D8=A7=D9=86 =D8=B3=D8=A7=D9=85=D8=A7=D9=86=D9=87 =D8=A2= =D8=B1=DA=A9=D8=A7 | =D8=AA=D9=85=D8=A7=D8=B3: 91300476-021 | =D8=A7=DB=8C= =D9=85=DB=8C=D9=84: info@spammy.ir [Telegram] [Instagram] [LinkedIn] [Email] [ { "type": "word", "value": "email" }, { "type": "word", "value": "funny" }, { "type": "word", "value": "instagram" }, { "type": "word", "value": "linkedin" }, { "type": "word", "value": "spammermail" }, { "type": "word", "value": "telegram" }, { "type": "word", "value": "text" }, { "type": "word", "value": "very" }, { "type": "word", "value": "آرکا" }, { "type": "word", "value": "اسمارترمی" }, { "type": "word", "value": "افزونه" }, { "type": "word", "value": "ایمیل" }, { "type": "word", "value": "تماس" }, { "type": "word", "value": "توسط" }, { "type": "word", "value": "توسعه" }, { "type": "word", "value": "رایان" }, { "type": "word", "value": "سامانه" }, { "type": "word", "value": "های" }, { "type": "word", "value": "یافته" }, { "type": "number", "code": [ 105, 3 ] }, { "type": "number", "code": [ 105, 8 ] }, { "type": "unicode_category", "value": "Cf" }, { "type": "unicode_category", "value": "Sm" }, { "type": "sender", "value": "marketing@spammer.ir" }, { "type": "sender", "value": "spammer.ir" }, { "type": "email", "value": "info@spammy.ir" }, { "type": "email", "value": "spammy.ir" }, { "type": "mime_type", "value": "text/plain" } ] Received: from localhost ([217.61.8.72]) by Consip with ESMTP id PMLhve2ETFdIAPMLyvhh0c; Sat, 29 Nov 2025 15:55:14 +0100 Received: from zspmta-mint02.ad.aruba.it ([127.0.0.1]) by localhost (zspmta-mint02.ad.aruba.it [127.0.0.1]) (amavis, port 10026) with ESMTP id UV6fqMysWqKE; Sat, 29 Nov 2025 15:55:13 +0100 (CET) Received: from zspmbx-mint11.ad.aruba.it (unknown [10.202.133.51]) by zspmta-mint02.ad.aruba.it (Postfix) with ESMTP id 3042B120F77; Sat, 29 Nov 2025 15:54:59 +0100 (CET) Date: Sat, 29 Nov 2025 15:54:59 +0100 (CET) From: gianfranco.mangini@interno.it Reply-To: "Hr. Charles Jackson Jr." Message-ID: <1933878358.10239097.1764428099117.JavaMail.zimbra@interno.it> Subject: Content-Type: multipart/alternative; boundary="=_bf54163b-f3b6-421f-bc9d-b64439167a39" --=_bf54163b-f3b6-421f-bc9d-b64439167a39 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable Hvorfor har du ikke modtaget donationen p=C3=A5 =E2=82=AC955.000,00 fra hr.= Charles Jackson Jr.? Bankdirekt=C3=B8ren informerede mig i g=C3=A5r om, at= en af =E2=80=8B=E2=80=8Bmodtagerne ikke havde gjort krav p=C3=A5 donatione= n. Efter at have gennemg=C3=A5et mine optegnelser opdagede jeg, at du var b= landt de ber=C3=B8rte, og jeg er meget ked af at h=C3=B8re dette. Bem=C3=A6= rk venligst, at der ikke kr=C3=A6ves nogen betaling; et simpelt bekr=C3=A6f= telsesstempel er alt, hvad der skal til for at pengene kan frigives og kred= iteres din bankkonto inden for 24 timer.=20 Bem=C3=A6rk: For yderligere information og for at sikre, at din donation kr= editeres inden for 24 timer, anbefaler jeg, at du sender mig dine oplysning= er med det samme via e-mail til ferassutti34@gmail.com=20 Jeg =C3=B8nsker dig en velsignet m=C3=A5ned med stor succes.=20 Hr. Charles Jackson Jr.=20 --=_bf54163b-f3b6-421f-bc9d-b64439167a39 Content-Type: text/html; charset=utf-8 Content-Transfer-Encoding: quoted-printable


Hvorfor har du ikke modtaget donationen p=C3=A5 =E2=82=AC9= 55.000,00 fra hr. Charles Jackson Jr.? Bankdirekt=C3=B8ren informerede mig = i g=C3=A5r om, at en af =E2=80=8B=E2=80=8Bmodtagerne ikke havde gjort krav = p=C3=A5 donationen. Efter at have gennemg=C3=A5et mine optegnelser opdagede= jeg, at du var blandt de ber=C3=B8rte, og jeg er meget ked af at h=C3=B8re= dette. Bem=C3=A6rk venligst, at der ikke kr=C3=A6ves nogen betaling; et si= mpelt bekr=C3=A6ftelsesstempel er alt, hvad der skal til for at pengene kan= frigives og krediteres din bankkonto inden for 24 timer.

Bem=C3=A6r= k: For yderligere information og for at sikre, at din donation krediteres i= nden for 24 timer, anbefaler jeg, at du sender mig dine oplysninger med det= samme via e-mail til ferassutti34@gmail.com

Jeg =C3=B8nsker dig en = velsignet m=C3=A5ned med stor succes.
Hr. Charles Jackson Jr.
--=_bf54163b-f3b6-421f-bc9d-b64439167a39-- [ { "type": "word", "value": "anbefal" }, { "type": "word", "value": "bankdirektør" }, { "type": "word", "value": "bankkonto" }, { "type": "word", "value": "bekræftelsesstem" }, { "type": "word", "value": "bemærk" }, { "type": "word", "value": "berørt" }, { "type": "word", "value": "betaling" }, { "type": "word", "value": "bland" }, { "type": "word", "value": "charl" }, { "type": "word", "value": "din" }, { "type": "word", "value": "donation" }, { "type": "word", "value": "e" }, { "type": "word", "value": "frigiv" }, { "type": "word", "value": "gennemgå" }, { "type": "word", "value": "gjort" }, { "type": "word", "value": "går" }, { "type": "word", "value": "hr" }, { "type": "word", "value": "hvorfor" }, { "type": "word", "value": "hør" }, { "type": "word", "value": "ind" }, { "type": "word", "value": "inform" }, { "type": "word", "value": "information" }, { "type": "word", "value": "jackson" }, { "type": "word", "value": "jr" }, { "type": "word", "value": "kan" }, { "type": "word", "value": "ked" }, { "type": "word", "value": "krav" }, { "type": "word", "value": "kredit" }, { "type": "word", "value": "kræv" }, { "type": "word", "value": "mail" }, { "type": "word", "value": "modtag" }, { "type": "word", "value": "måned" }, { "type": "word", "value": "nog" }, { "type": "word", "value": "opdaged" }, { "type": "word", "value": "oplysning" }, { "type": "word", "value": "optegn" }, { "type": "word", "value": "peng" }, { "type": "word", "value": "sam" }, { "type": "word", "value": "send" }, { "type": "word", "value": "sikr" }, { "type": "word", "value": "simpelt" }, { "type": "word", "value": "stor" }, { "type": "word", "value": "suc" }, { "type": "word", "value": "tim" }, { "type": "word", "value": "velsign" }, { "type": "word", "value": "ven" }, { "type": "word", "value": "via" }, { "type": "word", "value": "yder" }, { "type": "word", "value": "ønsk" }, { "type": "number", "code": [ 105, 2 ] }, { "type": "number", "code": [ 105, 3 ] }, { "type": "unicode_category", "value": "Cf" }, { "type": "unicode_category", "value": "Sc" }, { "type": "sender", "value": "ferassutti34@gmail.com" }, { "type": "sender", "value": "gianfranco.mangini@interno.it" }, { "type": "sender", "value": "gmail.com" }, { "type": "sender", "value": "interno.it" }, { "type": "hostname", "value": "aruba.it" }, { "type": "hostname", "value": "interno.it" }, { "type": "hostname", "value": "zspmbx-mint11.ad.aruba.it" }, { "type": "hostname", "value": "zspmta-mint02.ad.aruba.it" }, { "type": "mime_type", "value": "multipart/alternative" }, { "type": "mime_type", "value": "text/html" }, { "type": "mime_type", "value": "text/plain" } ] Delivered-To: mcfadden@domain.com Received: from gamma.stellaryx.space (unknown [85.120.227.61] (AS6718 NAV COMMUNICATIONS SRL, RO)) by mail.stalw.art (Stalwart SMTP) with ESMTP id 3D6018102E32AFD; Tue, 25 Nov 2025 14:56:37 +0000 Return-Path: <102356-235606-568806-22158-mcfadden=domain.com@mail.stellaryx.space> Content-Type: multipart/alternative; boundary="4521ddb80d67f83dc7585cae40234a09_39856_8ade6" Date: Tue, 25 Nov 2025 15:56:05 +0100 From: "ZenFluff" Reply-To: "ZenFluff" Subject: Sleep better with FluffCo To: Message-ID: --4521ddb80d67f83dc7585cae40234a09_39856_8ade6 Content-Type: text/plain; Content-Transfer-Encoding: 8bit Sleep better with FluffCo http://stellaryx.space/Bm6NYOrhicX--9bFn47T2mFlb-Soxhs-FJ8RCnM_SXJyHmebLw http://stellaryx.space/Y46ntHIiWyxTreKwOcyT4txY9f-M-eCwfHuhx0SMoyGjXy1iuA --4521ddb80d67f83dc7585cae40234a09_39856_8ade6 Content-Type: text/html; Content-Transfer-Encoding: 8bit Newsletter
--4521ddb80d67f83dc7585cae40234a09_39856_8ade6-- [ { "type": "word", "value": "better" }, { "type": "word", "value": "fluffco" }, { "type": "word", "value": "sleep" }, { "type": "sender", "value": "fluffcopartner@stellaryx.space" }, { "type": "sender", "value": "fluffcopromo@stellaryx.space" }, { "type": "sender", "value": "stellaryx.space" }, { "type": "url", "value": "_jpg" }, { "type": "url", "value": "_sxjyhmeblw" }, { "type": "url", "value": "stellaryx.space" }, { "type": "url", "value": "www.stellaryx.space" }, { "type": "hostname", "value": "gamma.stellaryx.space" }, { "type": "hostname", "value": "stellaryx.space" }, { "type": "mime_type", "value": "multipart/alternative" }, { "type": "mime_type", "value": "text/html" }, { "type": "mime_type", "value": "text/plain" }, { "type": "html_image", "src": "http" }, { "type": "html_anchor", "href": "http" } ] Delivered-To: mcfadden@domain.com Received: from cache.agelessknees.za.com (unknown [193.36.60.184] (AS210107 PLUSWEB SUNUCU INTERNET HIZMETLERI TICARET LIMITED SIRKETI, TR)) by mail.stalw.art (Stalwart SMTP) with ESMTP id 3CB95BA83AFB5B8; Sun, 9 Nov 2025 10:25:14 +0000 Return-Path: <2993-2338-35418-95-mcfadden=domain.com@mail.agelessknees.za.com> Content-Type: multipart/alternative; boundary="f2a4125f95dc25c4cd4f09657da6c1b1" Date: Sun, 9 Nov 2025 02:05:37 -0800 From: "ENLARGED PROSTATE" Reply-To: "ENLARGED PROSTATE" Subject: 90% Success Rate: Shrink Your Prostate by 68%... To: Message-ID: <8btb35y8w3xurryz-6647pok20gxdwmew-8a5a@agelessknees.za.com> --f2a4125f95dc25c4cd4f09657da6c1b1 Content-Type: text/plain; Content-Transfer-Encoding: 8bit http://agelessknees.za.com/WEIPdYsx_Fz316dLbPRtpgbW8tLjN6VeuG_xqNr-08jn http://[Unsubscribe]] --f2a4125f95dc25c4cd4f09657da6c1b1 Content-Type: text/html; Content-Transfer-Encoding: 8bit

Urologists are in complete shock after this classified 1970 study  has been
accidentally released to the public. 
 
In the study, almost 90% of the men emptied their bladders fully…
and stopped nighttime pee trips! 
 
And it’s because of a bizarre “Brazilian Jelly”... 

click here to watch video, Picture

Which not only helps you pee like a racehorse, but it also shrinks your
prostate size by 68%, almost overnight. 
 
So, as you can imagine, this prostate-shrinking method
is spreading like wildfire.. 
 
And that’s why over 45,000 men have managed to
get rid of prostate problems… 
 
Without painful medical procedures or Rapaflo, Uroxatral, and other toxic medications. So, while this video is still up…
 
So, while this video is still up… 
 
[WATCH NOW]

to see how this Brazilian Jelly can help shrink your
enlarged prostate as well.










unsubscribe
1770 Walnut Hill Drive Dayton, OH 45406 --f2a4125f95dc25c4cd4f09657da6c1b1-- [ { "type": "word", "value": "_allcaps" }, { "type": "word", "value": "accident" }, { "type": "word", "value": "almost" }, { "type": "word", "value": "also" }, { "type": "word", "value": "bizarr" }, { "type": "word", "value": "bladder" }, { "type": "word", "value": "brazilian" }, { "type": "word", "value": "classifi" }, { "type": "word", "value": "click" }, { "type": "word", "value": "complet" }, { "type": "word", "value": "dayton" }, { "type": "word", "value": "drive" }, { "type": "word", "value": "empti" }, { "type": "word", "value": "enlarg" }, { "type": "word", "value": "fulli" }, { "type": "word", "value": "get" }, { "type": "word", "value": "help" }, { "type": "word", "value": "hill" }, { "type": "word", "value": "http" }, { "type": "word", "value": "imagin" }, { "type": "word", "value": "jelli" }, { "type": "word", "value": "like" }, { "type": "word", "value": "manag" }, { "type": "word", "value": "medic" }, { "type": "word", "value": "men" }, { "type": "word", "value": "method" }, { "type": "word", "value": "nighttim" }, { "type": "word", "value": "oh" }, { "type": "word", "value": "overnight" }, { "type": "word", "value": "pain" }, { "type": "word", "value": "pee" }, { "type": "word", "value": "pictur" }, { "type": "word", "value": "problem" }, { "type": "word", "value": "procedur" }, { "type": "word", "value": "prostat" }, { "type": "word", "value": "public" }, { "type": "word", "value": "racehors" }, { "type": "word", "value": "rapaflo" }, { "type": "word", "value": "rate" }, { "type": "word", "value": "releas" }, { "type": "word", "value": "rid" }, { "type": "word", "value": "see" }, { "type": "word", "value": "shock" }, { "type": "word", "value": "shrink" }, { "type": "word", "value": "size" }, { "type": "word", "value": "spread" }, { "type": "word", "value": "still" }, { "type": "word", "value": "stop" }, { "type": "word", "value": "studi" }, { "type": "word", "value": "success" }, { "type": "word", "value": "toxic" }, { "type": "word", "value": "trip" }, { "type": "word", "value": "unsubscrib" }, { "type": "word", "value": "urologist" }, { "type": "word", "value": "uroxatr" }, { "type": "word", "value": "video" }, { "type": "word", "value": "walnut" }, { "type": "word", "value": "watch" }, { "type": "word", "value": "well" }, { "type": "word", "value": "wildfir" }, { "type": "word", "value": "without" }, { "type": "number", "code": [ 105, 2 ] }, { "type": "number", "code": [ 105, 3 ] }, { "type": "number", "code": [ 105, 4 ] }, { "type": "number", "code": [ 105, 5 ] }, { "type": "sender", "value": "agelessknees.za.com" }, { "type": "sender", "value": "prostate@agelessknees.za.com" }, { "type": "url", "value": "_png" }, { "type": "url", "value": "_weipdysx" }, { "type": "url", "value": "agelessknees.za.com" }, { "type": "hostname", "value": "agelessknees.za.com" }, { "type": "hostname", "value": "cache.agelessknees.za.com" }, { "type": "mime_type", "value": "multipart/alternative" }, { "type": "mime_type", "value": "text/html" }, { "type": "mime_type", "value": "text/plain" }, { "type": "html_image", "src": "http" }, { "type": "html_anchor", "href": "http" } ] Delivered-To: hello@stalw.art Received: from mail-wm1-x32d.google.com (mail-wm1-x32d.google.com [2a00:1450:4864:20::32d] (AS15169 Google LLC)) (using TLSv1.3 with cipher TLS13_AES_256_GCM_SHA384) by mail.stalw.art (Stalwart SMTP) with ESMTPS id 3BBB57D01CB793C; Wed, 15 Oct 2025 18:31:22 +0000 Return-Path: Received: by mail-wm1-x32d.google.com with SMTP id 5b1f17b1804b1-4710683a644so7830385e9.0 for ; Wed, 15 Oct 2025 11:31:19 -0700 (PDT) Received: from 52669349336 named unknown by gmailapi.google.com with HTTPREST; Wed, 15 Oct 2025 14:31:17 -0400 Received: from 52669349336 named unknown by gmailapi.google.com with HTTPREST; Wed, 15 Oct 2025 14:31:16 -0400 MIME-Version: 1.0 Sender: Yash from SpamTest From: Yash from SpamTest Reply-To: yashbansal@spamtest.com Date: Wed, 15 Oct 2025 14:31:17 -0400 Message-ID: Subject: SpamTest Open-Source Sponsorships for Stalwart To: Hello Content-Type: multipart/alternative; boundary="000000000000d8b207064136b41e" --000000000000d8b207064136b41e Content-Type: text/plain; charset="UTF-8" Hi Team, I'm Yash from SpamTest, a GenAI-powered quality engineering platform. We've been following the excellent work you're doing with Stalwart and would like to support your project through our Open Source Program. What we're offering: - Free SpamTest licenses for your testing infrastructure - Financial sponsorship for your project - Co-marketing initiative to amplify your project's reach In return, we'd appreciate featuring the SpamTest logo in the ReadMe file and under your sponsors section. Would you be interested in a quick call to discuss how we can support? Here's my Calendly: https://calendly.com/yashbansal-spamtest/ Regards, Yash [image: beacon] --000000000000d8b207064136b41e Content-Type: text/html; charset="UTF-8" Content-Transfer-Encoding: quoted-printable
= Hi Team,

I'= ;m Yash from SpamTest, a=C2=A0GenAI-powered quality engineering platform.= We've been following the excellent work you're doing with Stalwart= and would like to support your project through our Open Source Program.

What we're o= ffering:
  • Free SpamTest licenses for y= our testing infrastructure
  • Co-marketing initiative to amplify your project's reach=
In return, we'd app= reciate featuring the SpamTest logo in the ReadMe file and under your spo= nsors section.

Would you be interested in a quick call to discuss how we can support? Her= e's my Calendly: https:/= /calendly.com/yashbansal-spamtest/

Regards,
Yash
3D"beacon" --000000000000d8b207064136b41e-- [ { "type": "word", "value": "amplifi" }, { "type": "word", "value": "appreci" }, { "type": "word", "value": "beacon" }, { "type": "word", "value": "calend" }, { "type": "word", "value": "call" }, { "type": "word", "value": "co" }, { "type": "word", "value": "discuss" }, { "type": "word", "value": "engin" }, { "type": "word", "value": "excel" }, { "type": "word", "value": "featur" }, { "type": "word", "value": "file" }, { "type": "word", "value": "financi" }, { "type": "word", "value": "follow" }, { "type": "word", "value": "free" }, { "type": "word", "value": "genai" }, { "type": "word", "value": "hi" }, { "type": "word", "value": "imag" }, { "type": "word", "value": "infrastructur" }, { "type": "word", "value": "initi" }, { "type": "word", "value": "interest" }, { "type": "word", "value": "licens" }, { "type": "word", "value": "like" }, { "type": "word", "value": "logo" }, { "type": "word", "value": "market" }, { "type": "word", "value": "offer" }, { "type": "word", "value": "open" }, { "type": "word", "value": "platform" }, { "type": "word", "value": "power" }, { "type": "word", "value": "program" }, { "type": "word", "value": "project" }, { "type": "word", "value": "qualiti" }, { "type": "word", "value": "quick" }, { "type": "word", "value": "reach" }, { "type": "word", "value": "readm" }, { "type": "word", "value": "regard" }, { "type": "word", "value": "return" }, { "type": "word", "value": "section" }, { "type": "word", "value": "sourc" }, { "type": "word", "value": "spamtest" }, { "type": "word", "value": "sponsor" }, { "type": "word", "value": "sponsorship" }, { "type": "word", "value": "stalwart" }, { "type": "word", "value": "support" }, { "type": "word", "value": "team" }, { "type": "word", "value": "test" }, { "type": "word", "value": "work" }, { "type": "word", "value": "would" }, { "type": "word", "value": "yash" }, { "type": "unicode_category", "value": "Sm" }, { "type": "sender", "value": "spamtest.com" }, { "type": "sender", "value": "yashbansal@spamtest.com" }, { "type": "url", "value": "calendly.com" }, { "type": "url", "value": "spamtest-dot-yamm-track.appspot.com" }, { "type": "hostname", "value": "gmail.com" }, { "type": "hostname", "value": "gmailapi.google.com" }, { "type": "hostname", "value": "google.com" }, { "type": "hostname", "value": "mail-wm1-x32d.google.com" }, { "type": "hostname", "value": "mail.gmail.com" }, { "type": "mime_type", "value": "multipart/alternative" }, { "type": "mime_type", "value": "text/html" }, { "type": "mime_type", "value": "text/plain" }, { "type": "html_image", "src": "https" }, { "type": "html_anchor", "href": "https" } ] Delivered-To: hello@stalw.art Received: from Beijing--------chsi.com.cn (unknown [182.107.82.163] (AS4134 Chinanet, CN)) by mail.stalw.art (Stalwart SMTP) with ESMTP id 3D6F265DB441EFD; Thu, 27 Nov 2025 02:01:35 +0000 Received-SPF: none (mail.stalw.art: no SPF records found for hello-------锟斤拷锟斤拷------kefu@beijing--------chsi.com.cn) receiver=mail.stalw.art; client-ip=182.107.82.163; envelope-from="hello-------锟斤拷锟斤拷------kefu@beijing--------chsi.com.cn"; helo=Beijing--------chsi.com.cn; Return-Path: Message-ID: <187bbaa5967a07c4.15a3b47be71017b4.d92ade25f52781da@mail.stalw.art> From: =?GB2312?B?zOGwzr36yf2439C9x+HLyciruOO2qDEwOjAxOjM0?= Subject: =?GB2312?B?0afA+tGnzrvLq9akyKu5+rbAvNLL2bDssb6/xsu2yr+yqcq/ICC547jm?= AD hello To: hello@stalw.art Content-Type: multipart/mixed; boundary="=_NextPart_2rfkindysadvnqw3nerasdf";charset="GB2312" MIME-Version: 1.0 Date: Thu, 27 Nov 2025 10:01:37 +0800 This is a multi-part message in MIME format --=_NextPart_2rfkindysadvnqw3nerasdf Content-Type: text/plain Content-Transfer-Encoding: 7bit 10:01:34 hello AD --=_NextPart_2rfkindysadvnqw3nerasdf Content-Type: application/octet-stream; name="独家速办学历学位 学信网永久查询 本科硕博士高薪晋升职称轻松快速全搞定.txt" Content-Transfer-Encoding: base64 Content-Disposition: attachment; filename="独家速办学历学位 学信网永久查询 本科硕博士高薪晋升职称轻松快速全搞定.txt" 5YWo5Zu954us5a625p2D5aiB5Luj5Yqe77yB77yBDQoNCueLrOWutuWFqOWbvemrmOagoemZouez u+S8mOi0qOi1hOa6kOa4oOmBk++8jOWFqOWbveeLrOWutuadg+WogeS7o+WKnu+8gQ0KDQrpq5jo lqrlt6XkvZzvvIzkvJjljprogYzkvY3vvIzmj5Dmi5TmmYvljYfvvIzogYznp7Dor4TlrprvvIzm iLflj6Plip7nkIbvvIzlh7rlm73np7vmsJEg6L275p2+5YWo5pCe5a6a77yB77yBDQoNCuS9oOaY r+WQpuWboOS4uuayoeacieWkp+WtpuWtpuWOhuWSjOWtpuS9jeiAjOaJvuS4jeWIsOS4gOS7veeQ huaDs+W3peS9nO+8jOaIluiAheWwveeuoeS9oOWcqOWunumZheW3peS9nOS4reenr+e0r+S6huS4 sOWvjOe7j+mqjOWNtOWboOayoeacieWtpuWOhuWtpuS9jeivgeS5puiAjOWkseWOu+aPkOaLlOaZ i+WNh+eahOacuuS8mj8g5oul5pyJ5LiA5Liq5aSn5a2m5a2m5Y6G77yI5a2m5L2N77yJ77yM56Gu 5L+d5L2g5Zyo5bel5L2c5LqL5Lia5LiK55qE5b+r6YCf5oiQ5Yqf77yBDQoNCuS4uuWuouaIt+in o+WGs+WPkeWxleeTtumiiOacn+mXrumimO+8jOW/q+mAn+i9u+advuWunueOsOiBjOWKoeWNh+i/ ge+8jOiBjOensOivhOWumuWyl+S9jeaZi+WNh++8jOWkh+WPl+engeS8geWkluS8geeMjuWktOaO qOW0h++8jOaKlei1hOS6juWtpuWOhuWtpuS9jeS6p+eUn+eahOWbnuaKpeWSjOS7t+WAvOS5i+mr mOi/nOi2heS7u+S9leaKlei1hOWTgeenje+8jOS4lOS4gOasoeaKlei1hOe7iOi6q+WPl+ebiuWP r+aMgee7reWPkeWxle+8ge+8geaXouWPr+S7peeri+erv+ingeW9seWcqOiBjOWcuuWVhuWcuuWm gumxvOW+l+awtO+8jOS5n+WPr+S7peS4uuS7iuWQjueahOaKpeiAg+WNh+i/geaPkOS+m+WdmuWu nueahOWfuuehgO+8ge+8gQ0KDQrkvaDkuLrmsqHmnInlrabljobmib7kuI3liLDlpb3lt6XkvZzn g6bmgbzlkJc/6L+Y5Zug5Li65rKh5pyJ5a2m5Y6G5peg5rOV5aSn5bmF5o+Q6auY5b6F6YGH5pS2 5YWl6ICM5b+n6JmR5ZCX77yf6L+Y5Li65rKh5pyJ5q2j6KeE5aSn5a2m5a2m5Y6G5a2m5L2N6ICM 5peg5rOV5a6e546w6IGM56ew6K+E5a6a77yM5o+Q5ouU6YeN55So77yM5pmL57qn5Yqg6Jaq6ICM 54Om5oG85ZCX77yfIOWtpuWOhuaUueWPmOWRvei/kOWIm+mAoOS7t+WAvO+8ge+8geacrOWFrOWP uOWPr+WKqeS9oOaIkOWKn+i+ieeFjO+8ge+8gQ0KDQrmnIDmnYPlqIHni6zlrrblnoTmlq3lhajl m73pq5jmoKHpmaLns7votYTmupDmuKDpgZPvvIzlr7nmsYLogYzmmYvljYfot7Pmp73ljYfogYzm tqjolqrmj5Dmi5Tpg73og73otbfliLDmnoHlhbboh7PlhbPph43opoHnmoTkvZznlKjvvIzku6Pn kIbllYbliqDnm5/lvoXpgYfkvJjljprlm57miqXkuLDljprvvIHvvIHliJvpgKDku7flgLzov5zo v5znianotoXmiYDlgLzvvIznu4jouqvlj5fnm4rvvIwg5pys56eR5Y2H56GV5aOr77yM56GV5aOr 5Y2H5Y2a5aOr77yM6IGM56ew5b6F6YGH57qn5Yir5Lmf6YO955u45bqU5b+r6YCf5o+Q5Y2H77yB 77yBDQoNCuWtpuS/oee9kemmlumhteacgOW6leagj+S4gOagj+KAnOiBlOezu+aIkeS7rOKAneS4 iuacieWcsOWbvuaMh+W8leWcsOWdgOS7peWPiueUteivnemCrueuse+8jOWtpuS/oeWFrOWPuOWP keW4g+W5v+WRiueahOmCrueuseaYr0BjaHNpLmNvbS5jbuWfn+WQjeWQjue8gOeahOWtpuS/oee9 keacjeWKoeWZqO+8jOWbnuWkjeWNj+iuruS7peWPiuaUr+S7mOWuneaIlumTtuihjOi0puWPt+ea hOmCrueuseaYr+WtpuS/oeWFrOWPuOWvueWkluWFrOW8gOeahOS8geS4mumCrueusWtlZnVAY2hz aS5jb20uY24NCg0K5YWo5Zu954us5a625bi45bm05Yqe55CGOg0KDQrlhajml6XliLbnu5/mi5vp h43ngrnpmaLmoKE5ODUgMjExIOWPjOS4gOa1gSDlhajlm73pq5jnrYnpmaLmoKHlrabljoblrabk vY3or4HkuabvvIzmnKznp5Hlrabljoblj4zor4HlkKvlrablo6vlrabkvY3vvIjlt6Xlrablrabl o6ss55CG5a2m5a2m5aOrLOWGnOWtpuWtpuWjqyznrqHnkIblrablrablo6ss57uP5rWO5a2m5a2m 5aOrLOWMu+WtpuWtpuWjqyzmlZnogrLlrablrablo6ss5paH5a2m5a2m5aOrLOazleWtpuWtpuWj q+WtpuS9jeetie+8iQ0K5Y+M6K+B5YWo5pel5Yi257uf5oub56GV5aOrL+WcqOiBjOehleWjq+eg lOeptueUn++8iOW3peWVhueuoeeQhuehleWjq++8iE1CQe+8ieaVmeiCsuehleWjq++8iE1FQe+8 ieazleW+i+ehleWjq++8iEpN77yJ6YeR6J6N566h55CG56GV5aOrRk1CQe+8jOazleWtpuehleWj q++8jOWFrOWFseeuoeeQhuehleWjq01QQe+8jOWFrOWFseWNq+eUn+ehleWjq++8jOW3peeoi+eh leWjq++8jOS8muiuoeehleWjq01QQWNj77yM5bu6562R5a2m56GV5aOr77yM5Li05bqK5Yy75a2m 56GV5aOr77yM6Im65pyv56GV5aOrTUZB562J77yJ5YWo5pel5Yi25oiW5Zyo6IGM5Y+M6K+B5Y2a 5aOr5LiT5Lia5Z6L5Y2a5aOr77ya5bel56iL5Y2a5aOr77yIRW5nRO+8ieWMu+WtpuWNmuWjq++8 iE1E77yJ5pWZ6IKy5Y2a5aOr77yIRWRE77yJ5a2m5pyv5Z6L5Y2a5aOr77ya5aaC5ZOy5a2m5Y2a 5aOrUGhE77yI57uP5rWO5a2m5Y2a5aOr77yMIOeuoeeQhuWtpuWNmuWjq++8jOS8muiuoeWtpuWN muWjq+ivreiogOWtpuWNmuWjq++8jOS4tOW6iuWMu+WtpuWNmuWjq++8jOW3peWVhueuoeeQhuWN muWjq++8jOmHkeiejeWtpuWNmuWjq+W/g+eQhuWtpuWNmuWjq++8jOekvuS8muWtpuWNmuWjq++8 jOaWsOmXu+WtpuWNmuWjq++8jOazleWtpuWNmuWjq++8jOWFrOWFseeuoeeQhuWNmuWjq++8jOaV meiCsuWtpuWNmuWjq++8jOiuoeeul+acuuWNmuWjq++8jOaWh+WtpuWNmuWjq+etie+8iQ0K5a2m 5L2N6K+B5Lmm77yI5a2m5aOr77yM56GV5aOrLCDljZrlo6vvvIkNCg0K5Z2H5o+Q5L6b5a6M5aSH 5a2m57GN5qGj5qGI5oiQ57up5Y2V77yM5aSn5a2m6Iux6K+t5Zub5YWt57qnY2V0NCxjZXQ25ZCI 5qC85oiQ57up5Y2V6K+B5piO77yM5rS+6YGj6K+B77yM55S15a2Q5rOo5YaM5YWl5a2m5L+h572R 5pWw5o2u5bqT77yM57uI6Lqr5rC45LmF5Y+v5p+l77yM5Y+v57uP5YWs6K+B5aSE5YWs6K+B77yM 56Gu5L+d6aG65Yip6YCa6L+H5ZCE56eN5b2i5byP55qE5a6h5p+l6aqM6K+B77yM5Y+v55So5LqO 5oql6ICD5YWs5Yqh5ZGY77yM5ZCE57G76LWE5qC86ICD6K+V77yM6ICD56CU77yM5Y2H6IGM77yM 6K+E6IGM56ew562J55So6YCU77yM5omA5Yqe6K+B5Lmm55Sx5YWo5Zu95ZCE5Zyw5Zu956uL5YWs 5Yqe6Zmi5qCh5YaF6YOo5rig6YGT5YWz57O75Yqe55CG77yM5qyi6L+O6ZW/5pyf5Luj55CG5ZCI 5L2c5Zue5oql5Liw5Y6a77yB5YWo5Zu96L+R55m+5a625Luj55CG5py65p6E77yM5Lia5Yqh6YGN 5biD5YWo5Zu977ya5YyX5LqsIOS4iua1tyDmt7HlnLMg5aSp5rSlIOadreW3niAg5Y2X5LqsICDl jqbpl6ggIOW5v+W3niAg5q2m5rGJICDmiJDpg70gIOmDkeW3niDkuJzojp4gIOa1juWNlyAg56aP 5bee562J5Zyw5Yy677yB77yB6LWE5rqQ5oyB57ut5aKe6ZW/77yBDQoNCuacrOWFrOWPuOS4muWK oemAgueUqOS6juWQhOexu+mrmOerr+WuouaIt+e+pCjkuJPkuJrlrp7ot7Xog73lipvlvLog5LyB 5Lia5Li75ZKM5ZCE6KGM5Lia6YeR6aKG562JKeeahOWtpuWOhuWtpuS9jeWumuWQkeS8mOWMluaV tOWQiOWNh+e6p++8jOWbnuaKpeeOh+mrmO+8jOW5s+WPsOi1hOa6kOaWueWQkeeahOmAieaLqeWG s+WumuS6huiBjOWcuuWVhuWcuuS4iueahOmjjueUn+awtOi1t+S4gOmprOW5s+W3ne+8ge+8gemA ieaLqeavlOWKquWKm+mHjeimgSznq5nlnKjlt6jkurrogqnohoDkuIrmiY3og73po57lvpfmm7Tp q5gs5Luj55CG5ZWG5pS/562W5LyY5Y6a6L+U5Yip5Liw5Y6a77yM5qyi6L+O6L2s5Y+R5o6o6I2Q 77yM6ZW/5pyf5qyi6L+O5ZCE55WM5oul5pyJ5a6i5oi36LWE5rqQ5rig6YGT55qE5Luj55CG5Yqg 55uf77yB77yBDQoNCuWFqOWll+aho+ahiOWtpuexjeWtpuWOhuWtpuS9jeS7t+agvO+8mg0KDQrk u7fkvY3mjInkuI3lkIzmoIflh4Y5ODUgMjExIOWPjOS4gOa1gSDmma7pgJrph43ngrnlkI3niYzp maLmoKHkuInmoaPlt67liKvlkozkuJPkuJrng63luqblt67liKvvvIzlhajluKblrabljoblrabk vY3lrabnsY3moaPmoYjliqDlvIDpgJrlrabkv6HnvZHnu4jouqvmsLjkuYXmlbDmja7ms6jlhozm n6Xor6INCg0KdW5kZXJncmFkdWF0ZSDmnKznp5Hlrablo6vlrabkvY3lj4zor4Ey5LiHNei1tyAg 5qC55o2u6Zmi5qCh5LiT5Lia54Ot6Zeo56iL5bqm5bGC5qyh6LCD5pW0ICDlpoLvvJrljJfkuqzl jJfkuqznkIblt6XlpKflraYt6K6h566X5py65a2m6ZmiLeiuoeeul+acuuenkeWtpuS4juaKgOac r+acrOenkS3lt6Xlrablrablo6vlrabkvY3vvIzkuIrmtbflkIzmtY7lpKflraYt57uP5rWO566h 55CG5a2m6ZmiLeeJqea1geeuoeeQhuacrOenkS3nrqHnkIblrablrablo6vlrabkvY0gIOWMl+S6 rOWNj+WSjOWMu+WtpumZoi3kuLTluorljLvlraYt5Yy75a2m5a2m5aOr5a2m5L2NICDmuIXljY7l pKflraYgIOS4reWbveS6uuawkeWkp+WtpiAg5YyX5Lqs5biI6IyD5aSn5a2mICDlpI3ml6blpKfl raYgIOS4iua1t+S6pOmAmuWkp+WtpiDkuK3lsbHlpKflraYg5Y2O5Y2X55CG5bel5aSn5a2m5Lit 5Zu956eR5oqA5aSn5a2mICDkuK3ljZflpKflraYg5bGx5Lic5aSn5a2mIOWNl+S6rOWkp+WtpiDl jY7kuK3np5HmioDlpKflraYgIOWbvemYsuenkeaKgOWkp+WtpiDljZflvIDlpKflraYg5Lit5Zu9 5Yac5Lia5aSn5a2mIOetiSANCg0KZ3JhZHVhdGUg5YWo5pel5Yi256GV5aOr5Zyo6IGM56GV5aOr M+S4hzUtLTXkuIfotbcgIOehleWjq+eglOeptueUn+WtpuWOhuWtpuS9jeWPjOivgSAg5Zyo6IGM 56CU56m255Sf5a2m5Y6G5a2m5L2N5Y+M6K+BIOWmguW3peWVhueuoeeQhuehleWjq01CQeOAgeWF rOWFseeuoeeQhuehleWjq01QQSAgICAg5Y6f5aeL5a2m5Y6G5qC55o2u5LiN5ZCM5LiT5Lia6KaB 5rGC6ZyA6KaB5pys56eR5a2m5Y6G5oiW5a2m5aOr5a2m5L2NICDlpoLvvJrkuK3lm73kurrmsJHl pKflraYt5ZWG5a2m6ZmiLeW3peWVhueuoeeQhuWtpuehleWjqyBNQkEsRU1CQe+8iOWcqOiBjCDl hajml6XliLbvvIksIOWMl+S6rOmmlumDvee7j+a1jui0uOaYk+Wkp+Wtpi3nu4/mtY7lrabpmaIt IOS6p+S4mue7j+a1juWtpu+8iOi0uOaYk+e7j+a1ju+8ieehleWjq++8iOWcqOiBjCDlhajml6Xl iLbvvInvvIzljJfkuqzlpKflraYt5YWJ5Y2O566h55CG5a2m6ZmiLU1CQSBFTUJBICAg5LiK5rW3 5aSN5pem5aSn5a2mLeaWsOmXu+WtpumZoi3mlrDpl7vlrabkuJPkuJrnoZXlo6sgICDljY7kuJzl uIjojIPlpKflraYg5YyX5Lqs6Iiq56m66Iiq5aSp5aSn5a2mIOS4iua1t+i0oue7j+Wkp+WtpiAg 5q2m5rGJ5aSn5a2mICAg5rWZ5rGf5aSn5a2mIOetiQ0KDQpEciDljZrlo6vnoJTnqbbnlJ/lrabl joblrabkvY3lj4zor4EgNuS4hyDotbcgIOmcgOacieehleWjq+WtpuWOhuaIluWtpuS9jSAg5aaC 77ya5YyX5Lqs5aSn5a2m57uP5rWO5a2m6Zmi57uP5rWO5a2m5Y2a5aOrICAg5Lit5aSu6LSi57uP 5aSn5a2m6YeR6J6N5a2m6Zmi6YeR6J6N5bel56iL5LiT5Lia5Y2a5aOrICAg5Lit5bGx5aSn5a2m 5Yy75a2m6Zmi5Yy75a2m5Y2a5aOrICDljY7ljZfnkIblt6XlpKflraYgICDmtZnmsZ/lpKflraYt 6K6h566X5py656eR5a2m5LiO5oqA5pyv5a2m6ZmiLeeUteWtkOS/oeaBr+W3peeoi+WNmuWjqyAg 5Y2X5Lqs5aSn5a2mLeWVhuWtpumZoi3lupTnlKjnu4/mtY7lrabljZrlo6sgICDlk4jlsJTmu6jl t6XkuJrlpKflraYgICDkuK3lm73mtbfmtIvlpKflraYgICDlpKnmtKXlpKflraYgICDljqbpl6jl pKflraYgICDkuK3lm73np5HlrabmioDmnK/lpKflrabnrYkNCg0K5Yqe55CG6Z2e5bi45b+r5o23 77yMMS0z5Liq5bel5L2c5pel5Y2z5Y+v5Yqe5aW95qGj5qGI6K+B5Lmm5a2m5L2N5a2m57GN6Iux 6K+t6K+B5Lmm562J5Y6f5Lu25bm25rOo5YaM5byA6YCa5a2m5L+h572R6K6k6K+B5pWw5o2u5bqT 5p+l6K+i77yM57uI6Lqr5rC45LmF5pyJ5pWI5p+l6K+i77yM5pys5YWs5Y+45omL5py65Y+35b6u 5L+h5Y+36ZW/5pyf5a6e5ZCN6K6k6K+B77yM5LyB5Lia6YKu566x5a2m5L+h572R5Z+f5ZCN5pyN 5Yqh5ZmoQGNoc2kuY29tLmNuIOWunuWQjeWkh+ahiO+8jOWvueWFrOi0puWPt+aUtuasvu+8jOWF qOmdouaUr+aMgeaUr+S7mOWuneW+ruS/oeaJq+eggeWSjOe9keS4iumTtuihjOaJi+acuumTtuih jEFQUOaUr+S7mCzmrKLov47lhajlm73ku6PnkIbllYbliqDnm5/lkIjkvZzvvIHvvIENCg0K5Yqe 55CG5rWB56iLOg0KDQrlpIfpvZDnlLPor7fmnZDmlpnihpLlrqHmoLjpgJrov4fihpLpppbku5gz MCXlrabnsY3ms6jlhozotLnnlKjvvIjlr7nlhazotKbmiLfmlLbmrL7vvInihpLlip7lpb3lj5Hp gIHor4Hkuabmiavmj4/ku7bmn6Xor6Lpqozor4Hmu6HmhI/ihpLmlK/ku5jkvZnmrL7ihpLlj5Hp obrkuLDlv6vpgJLmlLblj5blhajlpZfor4Hkuabljp/ku7bvvIzljJfkuqzkuIrmtbfmt7HlnLPl nLDljLrpl6rpgIEgICANCg0K5YyX5Lqs5oC76YOo5Zyw5Z2A77ya5YyX5Lqs5biC6KW/5Z+O5Yy6 6KW/55u06Zeo5aSW5aSn6KGXMTjlj7fph5HotLjlpKfljqZDM+W6p+OAgA0K5LiK5rW35YWs5Y+4 5Zyw5Z2A77ya5LiK5rW35biC5rWm5Lic5paw5Yy65rWm5Lic5Y2X6LevMTA3OOWPt+S4reiejeWk p+WOpjYwOA0K5rex5Zyz5Yqe5YWs5Zyw5Z2A77ya5rex5Zyz5biC5Y2X5bGx5Yy65rex5Zyz5aSn 5a2m5Z+O5a2m6IuR5aSn6YGTMTA2OOWPt0bmoIsxODA45a6kDQoNCuWtpuS/oee9keezu+aVmeiC sumDqOaMh+WumuWUr+S4gOWtpuWOhuiupOivgeafpeivoue9keerme+8jOe9keWdgCB3d3cuY2hz aS5jb20uY24gICANCg0K5pS25qy+6LSm5Y+3IA0K5oi35ZCN77ya5YyX5Lqs5a2m5L+h5ZKo6K+i 5pyN5Yqh5pyJ6ZmQ5YWs5Y+4ICAg5oi35ZCN77ya5rex5Zyz5biC5pm65L+h5paw5L+h5oGv5oqA 5pyv5pyJ6ZmQ5YWs5Y+4ICAg5oi35ZCN77ya5LiK5rW35a2m5L+h5pWZ6IKy56eR5oqA5pyJ6ZmQ 5YWs5Y+4DQrlvIDmiLfooYzvvJrkuK3lm73msJHnlJ/pk7booYzljJfkuqzluILopb/ln47ljLrl ub/lronpl6jmlK/ooYwgIOW8gOaIt+ihjO+8muW3peWVhumTtuihjOa3seWcs+W4guWNl+WxseaU r+ihjCAg5byA5oi36KGM77ya5oub5ZWG6ZO26KGM5LiK5rW35biC5rWm5Lic5aSn6YGT5pSv6KGM DQoNCuWKnueQhuWtpuWOhuWtpuS9jeivt+iBlOezuyDljJfkuqzmgLvpg6jnlLXor506IDEzOTgz MTI1MTUx77yI5b6u5L+h5ZCM5Y+377yJIOW+ruS/oe+8mmNoc2l4dyDnjovlu7rmtpvogIHluIgg 77yI5Li75Lu76LSf6LSj5Lq6ICDlrabkv6HnvZHmlbDmja7lupPnoJTlj5Hnu7TmiqTljYfnuqcg 77yJIOW+ruS/oeWPt++8mmNoc2l4dyAgICDpgq7nrrE6IGtlZnVAY2hzaS5jb20uY24gICAgICAg IFFROjY2ODg4OCAgIA0KDQrlrqLmiLcv5Luj55CG5ZWG6YGN5biD5YWo5Zu977ya5YyX5LqsIOS4 iua1tyDmt7HlnLMg5aSp5rSlIOadreW3niDljZfkuqwg6IuP5beeIOWOpumXqCDlub/lt54g6YeN 5bqGIOatpuaxiSDmiJDpg70g6YOR5beeIOS4nOiOniDpnZLlspsg5rWO5Y2XIOetieWQhOWkp+WM ug0KDQrmt7vliqDlvq7kv6Hpobvnn6XvvJrliqDlvq7kv6Hlkqjor6Llip7nkIbliY3vvIzor7fl hYjnoa7lrprlrqLmiLflubTpvoTvvJ/mhI/lkJHlrabljobnmoTmgKfotKjvvIjlhajml6XliLbn u5/mi5sg6Ieq6ICD77yJ77yf6Zmi5qCh5Zyw5Yy65LiT5Lia77yfIOW3peS9nOS6uuWRmOS8muWF iOaKpeS7t++8jOWGs+WumuWKnueQhueahOWuouaIt+ivt+aJk+W8gOWtpuS/oee9keeZu+W9lemm lumhteW3puS4iuinkuWtpuWOhuafpeivoumhtemdouWQjuadpeeUteivne+8jOe7meaIkeS7rOWK nuWFrOS8geS4mumCrueusWtlZnVAY2hzaS5jb20uY27lj5HpgIHlpIfpvZDnmoTnlLPlip7mnZDm lpnpgq7ku7blkI7vvIzmiJHku6zlj6/ku6Xnu5nlrqLmiLfmn6XnnIvov5HmnJ/lip7lpb3nmoTl rabljobmoLfmnKwg6L6T5YWl5aeT5ZCN6K+B5Lmm57yW5Y+35Y2z5Y+v5p+l6K+i6aqM6K+B77yM 5qyi6L+O5pyJ5a6i5oi36LWE5rqQ5a6e5Yqb55qE5py65p6E5Liq5Lq65Yqg55uf5Luj55CG5aSn 5bGV5a6P5Zu+77yB77yBDQoNCuacrOWFrOWPuOaJi+acuuWPt+W+ruS/oeWPt+mVv+acn+WunuWQ jeiupOivge+8jOS8geS4mumCrueuseWtpuS/oee9keWfn+WQjeacjeWKoeWZqEBjaHNpLmNvbS5j bumVv+acn+WunuWQjeWkh+ahiO+8jOWunuWQjei0puWPt+WFqOmdouaUr+aMgeaUr+S7mOWunemT tuiBlOe9kemTtuaUr+S7mCzku6PnkIbllYbplb/mnJ/lkIjkvZzlronlhajlv6vmjbfvvIENCg0K 5pyA5aW955qE5Y+j56KR5ZKM5L+h6KqJLCDni6zlrrbpm4TljprotYTmupAs5bey5oiQ5Yqf5Li6 5aSn6YeP5rW35YaF5aSW5a6i5oi35ZyG5ruh5LqG5qKm5oOz77yMIOS4gOOAgeaVmeiCsumDqOiu pOivgee9keWSjOWtpuagoee9keWdh+WPr+S7peafpeivou+8jOWPr+S+m+eUqOS6uuWNleS9jeWS jOacieWFs+mDqOmXqOeUteivneWSqOivouWSjOS4iue9keiwg+afpSAg5LqM44CB5pyJ5a6M5pW0 6b2Q5YWo55qE5qGj5qGI44CB5a2m57GN44CB6ICD6K+V5oiQ57up5Y2V44CB5YWl5a2m55m76K6w 6KGo44CB5q+V5Lia55m76K6w6KGo562J44CC5a+55rGC6IGM44CB5bCx5Lia44CB5bqU6IGY44CB 5pmL57qn44CB5rao6Jaq44CB6IGM56ew6K+E5a6a44CB6LWE5qC85oql6ICD44CB562J57qn6K6k 6K+B44CB5Ye65Zu944CB55WZ5a2m44CB56e75rCR44CB5a2m5Y6GIOWFrOivgeetiemDveWFt+ac ieaViOWKm+OAgiDkuInjgIHkv53or4Hlv6vmjbfku7fkvJjvvJrlm6DkuLrmmK/lrabmoKHnm7Tm jqXlh7ror4Hnm7TmjqXlip7nkIbvvIzmiYDku6Xkv53or4Hkuoblh7ror4Hlv6vpgJ/vvIzku7fm oLzkvJjmg6DjgIIg5biC5Zy65peg5Y+v6ZmQ6YeP77yM5qyi6L+O5Yqg55uf5Luj55CG77yM5LiA 5qyh5om56YeP5o+Q5Lqk5Yqe55CG5a6i5oi377yM5Y+v5p2l5pys5YWs5Y+46Z2i6LCI562+57qm 77yM5Luj55CG5ZWG5Yqg55uf5b6F6YGH5LyY5Y6a5Zue5oql5Liw5Y6a77yB77yBDQoNCui/keW5 tOadpeWBh+ivgeS5puaXqeW3sue7j+W9u+W6leiiq+a3mOaxsO+8jOaXoOiuuue6uOW8oOinhOag vOi0qOWcsOmYsuS8quawtOWNsOi/mOaYr+avleS4muivgeS5pueahOe8luWPt+WtpuS9jeivgeS5 pueahOe8luWPt++8jOi/mOacieWtpuexjeWPt+aho+ahiOe8luWPt++8jOmDveaXqeW3suWFqOmD qOiBlOe9keWIsOaVmeiCsumDqOWtpuS/oee9keeahOaVsOaNruW6k+S6hu+8jOaXoOiuuuaYr+aK peiAg+i/mOaYr+W6lOiBmOmdouivleaIluaYr+WFrOivge+8jOebuOWFs+W3peS9nOS6uuWRmOmD veaYr+eZu+mZhuWtpuS/oee9keaVsOaNruW6k+W5s+WPsOadpeafpemqjOWtpuWOhuivgeS5puea hOecn+S8quOAgg0KDQrmnKzlpITni6zlrrbnmoTotYTmupDmnYPpmZDkvb/lvpflrqLmiLfkuI3n lKjlho3ovpvoi6blpIfogIPogJfotLnml7bpl7Tnsr7lipvlj4LliqDmvKvplb/nuYHnkJDnmoTl rabljobogIPor5XvvIzlj6ropoHkvaDlhbflpIfkuIDlrprnmoTkuJPkuJrln7rnoYDvvIzop4Tl iJLorr7orqHmnIDkvbPnmoTogYzkuJrmlrnlkJHvvIzkuLrkuI3lkIzlrqLmiLfmjqjojZDorqLl iLbkuI7ogYzkuJrlkozmnKrmnaXlj5HlsZXpq5jluqbljLnphY3nmoTlrabljoblrabkvY3vvIzn u4jouqvmsLjkuYXlrabkv6HnvZHmn6Xor6LvvIzmnKzlrabljoblrabkvY3kuJrliqHmnIDpgILl kIjlhbflpIfovoPlvLrlt6XkvZzog73lipvmnInovoPlpb3ku47kuJrlsaXljobnmoTpq5jnq6/l rqLmiLfvvIzljIXmi6zmjIflrprpmaLns7vkuJPkuJrnmoTlnKjogYznu5/mi5vlhajml6XliLbm nKznp5HnoZXlo6vljZrlo6vnoJTnqbbnlJ/np4HkurrlrprliLbvvIzluK7liqnlub/lpKfog73l ipvlh7rkvJfnu4/mtY7kvJjotornmoTlrqLmiLflrp7njrDkuobogYzlnLrpo57ot4PllYblnLro hb7po57ku5XpgJTlubPmraXpnZLkupHvvIHvvIHpgInmi6nmr5Tliqrlipvmm7Tph43opoHvvIzk uI7ml7bkv7Hov5vnq5nlnKjlt6jkurrnmoTogqnohoDkuIrkvaDlj6/ku6Xpo57lvpfmm7Tpq5jv vIHvvIEgDQoNCuWKnueQhuWtpuWOhuWtpuS9jeivt+iBlOezuyDljJfkuqzmgLvpg6jnlLXor506 IDEzOTgzMTI1MTUx77yI5b6u5L+h5ZCM5Y+377yJIOW+ruS/oe+8mmNoc2l4dyDnjovlu7rmtpvo gIHluIgg77yI5Li75Lu76LSf6LSj5Lq6ICDlrabkv6HnvZHmlbDmja7lupPnoJTlj5Hnu7TmiqTl jYfnuqcg77yJIOW+ruS/oeWPt++8mmNoc2l4dyAgICDpgq7nrrE6IGtlZnVAY2hzaS5jb20uY27v vIjkvIHkuJrpgq7nrrFsZDg4ODhAMTg4LmNvbe+8iSAgICAgICAgUVE6NjY4ODg4ICAgDQoNCua3 u+WKoOW+ruS/oemhu+efpe+8muWKoOW+ruS/oeWSqOivouWKnueQhuWJje+8jOivt+WFiOehruWu muWuouaIt+eahOW5tOm+hCDmiYDlip7mhI/lkJHlrabljobnmoTmgKfotKjvvIjlhajml6XliLbn u5/mi5sg6Ieq6ICD77yJIOmZouagoeWcsOWMuuS4k+S4miDlt6XkvZzkurrlkZjkvJrlhYjlm57l pI3miqXku7fvvIzlhrPlrprlip7nkIbnmoTlrqLmiLfor7fmiZPlvIDlrabkv6HnvZHnmbvlvZXp ppbpobXlt6bkuIrop5Llrabljobmn6Xor6LpobXpnaLlkI7mnaXnlLXor53vvIznu5nmiJHku6zl t6XkvZzpgq7nrrFrZWZ1QGNoc2kuY29tLmNu5Y+R6YCB5aSH6b2Q55qE55Sz5Yqe5p2Q5paZ6YKu 5Lu25ZCO77yM5oiR5Lus5Y+v5Lul57uZ5a6i5oi35p+l55yL5oiR5Lus6L+R5pyf5Yqe5aW955qE 5a2m5Y6G5qC35pys6L6T5YWl5aeT5ZCN5q+V5Lia6K+B5Lmm57yW5Y+35Y2z5Y+v5p+l6K+i6aqM 6K+B77yM5qyi6L+O5pyJ5a6i5oi36LWE5rqQ5a6e5Yqb55qE5py65p6E5Yqg55uf5Luj55CG5aSn 5bGV5a6P5Zu+ISENCg0K6ZmEOiDnlLPlip7lrabljobmiYDpnIDmnZDmlpkNCg0KMS7lrabljobm gKfotKjvvIjnu5/mi5sgIOaIkOS6uuaVmeiCsi/lnKjogYwgIOiHquWtpuiAg+ivle+8iQ0KDQrp maLmoKHlkI3np7DvvIjlkITlnLDljLrlm73nq4vlhazlip7pmaLmoKHvvIkNCg0K5a2m5Y6G5bGC 5qyh77yI5LiT56eRIOacrOenkeWtpuWjqyDnoZXlo6vnoJTnqbbnlJ8g5Y2a5aOr56CU56m255Sf IOWmgk1CQSBFTUJBIOWQhOexu+W3peeoi+ehleWjq++8iQ0KDQrmr5XkuJrml7bpl7TvvIjoh6ro gIPkuLrmr4/lubQ25pyI5bqVMTLmnIjlupXlkITmr5XkuJrnmbvorrDkuIDmrKEg57uf5oubL+aI kOaVmS/lnKjogYzkuLrmr4/lubQ35pyI77yJDQoNCjIuIOiTneiJsuW6leS4pOWvuOaVsOeggeiv geS7tuW9qeeFp++8iOWbvueJh+aWh+S7tuWPr+WOi+e8qeWQjueUqOmCruS7tumZhOS7tuS4iuS8 oOWPkeadpe+8iQ0KDQrouqvku73or4HmraPpnaLmiavmj4/ku7bvvIjnlKjpgq7ku7bpmYTku7bk uIrkvKDlj5HmnaXvvIkNCg0KMy4g5Y6f5aeL5a2m5Y6G5a2m5L2N5Y+R5p2l5LiO5ZCm6KeG5oiQ 5Lq65pWZ6IKyL+S4k+WNh+acrC/lnKjogYznoZXlo6vnrYnlrabljobnmoTkuI3lkIzlhbfkvZPo poHmsYINCg== --=_NextPart_2rfkindysadvnqw3nerasdf-- [ { "type": "word", "value": "ad" }, { "type": "word", "value": "hello" }, { "type": "word", "value": "全国" }, { "type": "word", "value": "博士" }, { "type": "word", "value": "双" }, { "type": "word", "value": "学位" }, { "type": "word", "value": "学历" }, { "type": "word", "value": "广告" }, { "type": "word", "value": "本科" }, { "type": "word", "value": "独家" }, { "type": "word", "value": "硕士" }, { "type": "word", "value": "证" }, { "type": "word", "value": "速办" }, { "type": "number", "code": [ 105, 2 ] }, { "type": "sender", "value": "beijing--------chsi.com.cn" }, { "type": "sender", "value": "hello-------北京------kefu@beijing--------chsi.com.cn" }, { "type": "hostname", "value": "beijing--------chsi.com.cn" }, { "type": "attachment", "value": "!txt" }, { "type": "attachment", "value": "_信" }, { "type": "attachment", "value": "_全" }, { "type": "attachment", "value": "_博士" }, { "type": "attachment", "value": "_学" }, { "type": "attachment", "value": "_学位" }, { "type": "attachment", "value": "_学历" }, { "type": "attachment", "value": "_快速" }, { "type": "attachment", "value": "_搞定" }, { "type": "attachment", "value": "_晋升" }, { "type": "attachment", "value": "_本科" }, { "type": "attachment", "value": "_查询" }, { "type": "attachment", "value": "_永久" }, { "type": "attachment", "value": "_独家" }, { "type": "attachment", "value": "_硕" }, { "type": "attachment", "value": "_网" }, { "type": "attachment", "value": "_职称" }, { "type": "attachment", "value": "_轻松" }, { "type": "attachment", "value": "_速办" }, { "type": "attachment", "value": "_高薪" }, { "type": "mime_type", "value": "application/octet-stream" }, { "type": "mime_type", "value": "multipart/mixed" }, { "type": "mime_type", "value": "text/plain" } ] ================================================ FILE: tests/resources/smtp/antispam/classifier_html.test ================================================ hello
world
[ { "type": "StartTag", "name": 1819112552, "attributes": [], "is_self_closing": false }, { "type": "Text", "text": "hello" }, { "type": "StartTag", "name": 29282, "attributes": [], "is_self_closing": true }, { "type": "Text", "text": "world" }, { "type": "StartTag", "name": 29282, "attributes": [], "is_self_closing": true }, { "type": "EndTag", "name": 1819112552 } ] using <>
[ { "type": "StartTag", "name": 1819112552, "attributes": [], "is_self_closing": false }, { "type": "Text", "text": "using <>" }, { "type": "StartTag", "name": 29282, "attributes": [], "is_self_closing": true }, { "type": "EndTag", "name": 1819112552 } ] test tag
[ { "type": "Text", "text": "test" }, { "type": "StartTag", "name": 7630702, "attributes": [ [ 29282, null ] ], "is_self_closing": true }, { "type": "Text", "text": " tag" }, { "type": "StartTag", "name": 29282, "attributes": [], "is_self_closing": true } ] <>< >>hello world< br /> [ { "type": "StartTag", "name": 6775156, "attributes": [], "is_self_closing": true }, { "type": "Text", "text": ">hello world" }, { "type": "StartTag", "name": 29282, "attributes": [], "is_self_closing": true } ] ignore headxyz

<body>

[ { "type": "StartTag", "name": 1684104552, "attributes": [], "is_self_closing": false }, { "type": "StartTag", "name": 435611265396, "attributes": [], "is_self_closing": false }, { "type": "Text", "text": "ignore head" }, { "type": "EndTag", "name": 435611265396 }, { "type": "StartTag", "name": 7630702, "attributes": [ [ 1684104552, null ] ], "is_self_closing": false }, { "type": "Text", "text": "xyz" }, { "type": "EndTag", "name": 7630702 }, { "type": "EndTag", "name": 1684104552 }, { "type": "StartTag", "name": 12648, "attributes": [], "is_self_closing": false }, { "type": "Text", "text": "" }, { "type": "EndTag", "name": 12648 } ]

what is ♥?

ßĂΒγ don't hurt me.

[ { "type": "StartTag", "name": 112, "attributes": [], "is_self_closing": false }, { "type": "Text", "text": "what is ♥?" }, { "type": "EndTag", "name": 112 }, { "type": "StartTag", "name": 112, "attributes": [], "is_self_closing": false }, { "type": "Text", "text": "ßĂΒγ don't hurt me." }, { "type": "EndTag", "name": 112 } ] this is the actual text [ { "type": "Comment", "text": "!--[if mso]> < < < < ignore > -> here --" }, { "type": "Text", "text": " the actual" }, { "type": "Comment", "text": "!--" }, { "type": "Text", "text": " text" } ] < p > hello < / p > < p > world < / p > !!! < br > [ { "type": "StartTag", "name": 112, "attributes": [], "is_self_closing": false }, { "type": "Text", "text": "hello" }, { "type": "EndTag", "name": 112 }, { "type": "StartTag", "name": 112, "attributes": [], "is_self_closing": false }, { "type": "Text", "text": " world" }, { "type": "EndTag", "name": 112 }, { "type": "Text", "text": " !!!" }, { "type": "StartTag", "name": 29282, "attributes": [], "is_self_closing": false } ]

please unsubscribe here.

[ { "type": "StartTag", "name": 112, "attributes": [], "is_self_closing": false }, { "type": "Text", "text": "please unsubscribe" }, { "type": "StartTag", "name": 97, "attributes": [ [ 1717924456, "#" ] ], "is_self_closing": false }, { "type": "Text", "text": " here" }, { "type": "EndTag", "name": 97 }, { "type": "Text", "text": "." }, { "type": "EndTag", "name": 112 } ] texttexttexttext< a href = "e" >texttext< anchor href = "x">text [ { "type": "StartTag", "name": 97, "attributes": [ [ 1717924456, "a" ] ], "is_self_closing": false }, { "type": "Text", "text": "text" }, { "type": "EndTag", "name": 97 }, { "type": "StartTag", "name": 97, "attributes": [ [ 1717924456, "b" ] ], "is_self_closing": false }, { "type": "Text", "text": "text" }, { "type": "EndTag", "name": 97 }, { "type": "StartTag", "name": 97, "attributes": [ [ 1717924456, "c" ] ], "is_self_closing": false }, { "type": "Text", "text": "text" }, { "type": "EndTag", "name": 97 }, { "type": "StartTag", "name": 97, "attributes": [ [ 1717924456, "d" ] ], "is_self_closing": false }, { "type": "Text", "text": "text" }, { "type": "EndTag", "name": 97 }, { "type": "StartTag", "name": 97, "attributes": [ [ 1717924456, "e" ] ], "is_self_closing": false }, { "type": "Text", "text": "text" }, { "type": "EndTag", "name": 97 }, { "type": "StartTag", "name": 97, "attributes": [ [ 125779835187816, "ignore" ] ], "is_self_closing": false }, { "type": "Text", "text": "text" }, { "type": "EndTag", "name": 97 }, { "type": "StartTag", "name": 125822818283105, "attributes": [ [ 1717924456, "x" ] ], "is_self_closing": false }, { "type": "Text", "text": "text" }, { "type": "EndTag", "name": 97 } ] texttexttexttext< a href = e >texttexttext [ { "type": "StartTag", "name": 97, "attributes": [ [ 1717924456, "a" ] ], "is_self_closing": false }, { "type": "Text", "text": "text" }, { "type": "EndTag", "name": 97 }, { "type": "StartTag", "name": 97, "attributes": [ [ 1717924456, "b" ] ], "is_self_closing": false }, { "type": "Text", "text": "text" }, { "type": "EndTag", "name": 97 }, { "type": "StartTag", "name": 97, "attributes": [ [ 1717924456, "c" ] ], "is_self_closing": false }, { "type": "Text", "text": "text" }, { "type": "EndTag", "name": 97 }, { "type": "StartTag", "name": 97, "attributes": [ [ 1717924456, "d" ] ], "is_self_closing": false }, { "type": "Text", "text": "text" }, { "type": "EndTag", "name": 97 }, { "type": "StartTag", "name": 97, "attributes": [ [ 1717924456, "e" ] ], "is_self_closing": false }, { "type": "Text", "text": "text" }, { "type": "EndTag", "name": 97 }, { "type": "StartTag", "name": 97, "attributes": [ [ 125779835187816, "ignore" ] ], "is_self_closing": false }, { "type": "Text", "text": "text" }, { "type": "EndTag", "name": 97 }, { "type": "StartTag", "name": 125822818283105, "attributes": [ [ 1717924456, "x" ] ], "is_self_closing": false }, { "type": "Text", "text": "text" }, { "type": "EndTag", "name": 97 } ] text< a href = test ignore>text< a href = fudge href ignore>text a href = "unknown" [ { "type": "Comment", "text": "!-- texttext--text--" }, { "type": "StartTag", "name": 97, "attributes": [ [ 1717924456, "hello world" ] ], "is_self_closing": false }, { "type": "Text", "text": "text" }, { "type": "EndTag", "name": 97 }, { "type": "StartTag", "name": 97, "attributes": [ [ 1717924456, "test" ], [ 111542170183529, null ] ], "is_self_closing": false }, { "type": "Text", "text": "text" }, { "type": "EndTag", "name": 97 }, { "type": "StartTag", "name": 97, "attributes": [ [ 1717924456, "fudge" ], [ 1717924456, null ], [ 111542170183529, null ] ], "is_self_closing": false }, { "type": "Text", "text": "text" }, { "type": "EndTag", "name": 97 }, { "type": "StartTag", "name": 97, "attributes": [ [ 1717924456, "foobar" ] ], "is_self_closing": false }, { "type": "Text", "text": "a href = \"unknown\"" }, { "type": "EndTag", "name": 97 } ] ================================================ FILE: tests/resources/smtp/antispam/combined.test ================================================ envelope_from noreply@tetheer.com envelope_to licensing@stalw.art helo_domain yphoo.vps.wbsprt.com iprev.result permerror spf.result none spf_ehlo.result none dmarc.result none remote_ip 195.210.29.48 expect_header X-Spam-Result: ARC_NA (0.00), DKIM_NA (0.00), FROM_EQ_ENV_FROM (0.00), FROM_HAS_DN (0.00), HAS_DATA_URI (0.00), HAS_LINK_TO_LARGE_IMG (0.00), HTML_SHORT_1 (0.00), MID_RHS_MATCH_ENV_FROM (0.00), RCPT_COUNT_ONE (0.00), SPF_NA (0.00), SUBJECT_ENDS_EXCLAIM (0.00), TO_DN_NONE (0.00), TO_MATCH_ENVRCPT_ALL (0.00), RCVD_COUNT_ZERO (0.10), RCVD_NO_TLS_LAST (0.10), MIME_HTML_ONLY (0.20), HELO_NORES_A_OR_MX (0.30), AUTH_NA (1.00), DATE_IN_PAST (1.00), DMARC_NA (1.00), MID_RHS_MATCH_FROM (1.00), FROMHOST_NORES_A_OR_MX (1.50), HTML_SHORT_LINK_IMG_1 (2.00), RDNS_NONE (2.00), PYZOR (3.50) expect_header X-Spam-Score: spam, score=13.70 From: Client Services To: licensing@stalw.art Subject: Tether Important Update ! Date: 16 Oct 2023 06:40:52 +0200 Message-ID: <20231016064052.403F7FEF5F005EFB@tetheer.com> MIME-Version: 1.0 Content-Type: text/html Content-Transfer-Encoding: quoted-printable 3D"If envelope_from l.chant@tenthrevolution.com envelope_to joe@domain.org helo_domain eu-smtp-delivery-181.mimecast.com iprev.result pass spf.result pass spf_ehlo.result pass dkim.result pass dkim.domains tenthrevolution.com dmarc.result pass remote_ip 185.58.86.181 tls.version TLSv1.3 expect_header X-Spam-Result: DMARC_POLICY_ALLOW (-0.50), DKIM_ALLOW (-0.20), SPF_ALLOW (-0.20), MIME_GOOD (-0.10), ARC_NA (0.00), DKIM_SIGNED (0.00), FROM_EQ_ENV_FROM (0.00), FROM_HAS_DN (0.00), HAS_ATTACHMENT (0.00), HTML_SHORT_2 (0.00), RCPT_COUNT_ONE (0.00), RCVD_COUNT_THREE (0.00), TO_DN_EQ_ADDR_ALL (0.00), TO_MATCH_ENVRCPT_ALL (0.00), RCVD_NO_TLS_LAST (0.10), HELO_NORES_A_OR_MX (0.30), SUBJECT_ENDS_SPACES (0.50), URI_COUNT_ODD (0.50), DATE_IN_PAST (1.00), FORGED_RCVD_TRAIL (1.00), FROMHOST_NORES_A_OR_MX (1.50) expect_header X-Spam-Score: ham, score=3.90 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=tenthrevolution.com; s=mimecast20200102; t=1669138703; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:mime-version:mime-version:content-type:content-type; bh=OiZa5h2Agb47dTPnaYIMS7L31ZUnIkX1UTz8yUhK3ME=; b=Z0erZ14U5BYcY0CGysOw3K0A7wjF9qqRlOaI4+0XGUmM5QgmgN6UVJc6J5AkypPgwEfOWx vsCbMrq14SF61IevT2cPrOwaphTL7s3Yf9YqKkk4N9bMiBVeikq1ks0kxJ8pbE8vYsiASn GEkv9T3YWfRMQR/iH+oD1dVRnCljTxQ= Received: from EUR01-DB5-obe.outbound.protection.outlook.com (mail-db5eur01lp2053.outbound.protection.outlook.com [104.47.2.53]) by relay.mimecast.com with ESMTP with STARTTLS (version=TLSv1.2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id uk-mta-233-uW3qPnzqMdi1lL-mEjQyCA-4; Tue, 22 Nov 2022 17:38:17 +0000 X-MC-Unique: uW3qPnzqMdi1lL-mEjQyCA-4 Received: from AS8PR04MB8071.eurprd04.prod.outlook.com (2603:10a6:20b:3f9::15) by DU2PR04MB8952.eurprd04.prod.outlook.com (2603:10a6:10:2e3::24) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.5834.9; Tue, 22 Nov 2022 17:37:46 +0000 Received: from AS8PR04MB8071.eurprd04.prod.outlook.com ([fe80::eb0e:b5f3:368c:ff6e]) by AS8PR04MB8071.eurprd04.prod.outlook.com ([fe80::eb0e:b5f3:368c:ff6e%5]) with mapi id 15.20.5834.015; Tue, 22 Nov 2022 17:37:46 +0000 From: Lee Chant To: "joe@domain.org" Subject: =?Windows-1252?Q?You=92re_missing_out_=96_nominate_yourself_or_your_team_?= =?Windows-1252?Q?for_a_Digital_Revolution_Award_?= Thread-Topic: =?Windows-1252?Q?You=92re_missing_out_=96_nominate_yourself_or_your_team_?= =?Windows-1252?Q?for_a_Digital_Revolution_Award_?= Thread-Index: Adj+l09eH9aJFGDfS8upOMyJ89uCyw== Date: Tue, 22 Nov 2022 17:37:46 +0000 Message-ID: Accept-Language: en-GB, en-US X-MS-Has-Attach: yes X-MS-TNEF-Correlator: MIME-Version: 1.0 X-OriginatorOrg: tenthrevolution.com X-Mimecast-Spam-Score: 0 X-Mimecast-Originator: tenthrevolution.com Content-Language: en-US Content-Type: multipart/related; boundary="_004_AS8PR04MB8071BB5854E10B6EA7161159EF0D9AS8PR04MB8071eurp_"; type="multipart/alternative" --_004_AS8PR04MB8071BB5854E10B6EA7161159EF0D9AS8PR04MB8071eurp_ Content-Type: multipart/alternative; boundary="_000_AS8PR04MB8071BB5854E10B6EA7161159EF0D9AS8PR04MB8071eurp_" --_000_AS8PR04MB8071BB5854E10B6EA7161159EF0D9AS8PR04MB8071eurp_ Content-Type: text/plain; charset=WINDOWS-1252 Content-Transfer-Encoding: quoted-printable Hi Joe My name is Lee and I=92m the Managing Director of Global Customer Solutions= at Tenth Revolution Group. I wanted to drop you a note to personally let you know that we have extende= d the nomination deadline for the Digital Revolution Awards & Fundraiser until 30th November 2022. These not-for-profit awards celebrate excellence in the cloud and bring tog= ether the tech community for a night of celebration and fundraising for som= e truly worthy causes. Founded back in 2020, we have garnered support from = IBM, AWS, Salesforce and Microsoft and it=92s a perfect opportunity to netw= ork with likeminded individuals like yourself. Our 20 specialist categories cover topics from tech for good to the climate emergency, ED&I= , allyship, outstanding leadership to name but a few. If you or your team w= ould be interested in discussing a nomination for the Digital Revolution Aw= ards, do let me know. With kind regards, Lee Lee Chant MD - Global Customer Solutions Mobile: +44 (0)7971 373432 Email: l.chant@tenthrevolution.com Website: www.tenthrevolution.com [Tenth Revolution Group] Disclaimer This email and any attachments are confidential and intended for the use of= the named recipient only. If you have received this email and any attachme= nts in error, please inform us immediately and then delete it. Any views or= opinions are solely those of the author and do not necessarily represent t= hose of Frank Recruitment Group Services Limited or its affiliates, divisio= ns or brands. Company Registration No. 08142375. Registered Office: Floor 2, The St. Nicholas Building, St. Nicholas Street,= Newcastle Upon Tyne, Tyne and Wear, NE1 1RF. Business registration information of Frank Recruitment Group Services Ltd c= ompanies and associated brands in the UK, Europe, Singapore, Australia, Jap= an and North America can be found here. Our Privacy Notice can be= found at www.tenthrevolution.com/privacy-notice --_000_AS8PR04MB8071BB5854E10B6EA7161159EF0D9AS8PR04MB8071eurp_ Content-Type: text/html; charset=WINDOWS-1252 Content-Transfer-Encoding: quoted-printable

Hi Joe

My name is Lee and I=92m the Managing Director of Gl= obal Customer Solutions at Tenth Revolution Group.

I wanted to drop you a note to personally let you kn= ow that we have extended the nomination deadline for the Digital Revolution Awa= rds & Fundraiser until 30th November 2022.  

These not-for-profit awards celebrate excellence in = the cloud and bring together the tech community for a night of celebration = and fundraising for some truly worthy causes. Founded back in 2020, we have= garnered support from IBM, AWS, Salesforce and Microsoft and it=92s a perfect opportunity to network with likeminded = individuals like yourself.

Our 20 specialist categories cover topics from tech for good to the climate emergency, ED= &I, allyship, outstanding leadership to name but a few. If you or your = team would be interested in discussing a nomination for the Digital Revolut= ion Awards, do let me know.

With kind regards,

Lee

Lee Chant
MD - Global Customer Solutio= ns

Mobile: +44 (= 0)7971 373432
Email: l.chant@tenthrevolution.com
Website: www.tenthrevolution.com

=3D"Tenth

Disclaimer
This email and any= attachments are confidential and intended for the use of the named recipie= nt only. If you have received this email and any attachments in error, plea= se inform us immediately and then delete it. Any views or opinions are solely those of the author and do not= necessarily represent those of Frank Recruitment Group Services Limited or= its affiliates, divisions or brands. Company Registration No. 08142375.
Registered Office: Floor 2, The St. Nicholas Building, St. Nicholas Street,= Newcastle Upon Tyne, Tyne and Wear, NE1 1RF.
Business registration information of Frank Recruitment Group Services Ltd c= ompanies and associated brands in the UK, Europe, Singapore, Australia, Jap= an and North America can be found here.
Our Privacy Notice= can be found at www.tenthrevolution.com/privacy-notice

 

 

 

 

 

--_000_AS8PR04MB8071BB5854E10B6EA7161159EF0D9AS8PR04MB8071eurp_-- --_004_AS8PR04MB8071BB5854E10B6EA7161159EF0D9AS8PR04MB8071eurp_ Content-Type: image/jpeg; name="image001.jpg" Content-Description: image001.jpg Content-Disposition: inline; filename="image001.jpg"; size=13809; creation-date="Tue, 22 Nov 2022 17:37:45 GMT"; modification-date="Tue, 22 Nov 2022 17:37:46 GMT" Content-ID: Content-Transfer-Encoding: base64 /9j/4AAQSkZJRgABAgAAZABkAAD/7AARRHVja3kAAQAEAAAAPAAA/+EDLGh0dHA6Ly9ucy5hZG9i ZS5jb20veGFwLzEuMC8APD94cGFja2V0IGJlZ2luPSLvu78iIGlkPSJXNU0wTXBDZWhpSHpyZVN6 TlRjemtjOWQiPz4gPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0i QWRvYmUgWE1QIENvcmUgNi4wLWMwMDIgNzkuMTY0MzUyLCAyMDIwLzAxLzMwLTE1OjUwOjM4ICAg ICAgICAiPiA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjIt cmRmLXN5bnRheC1ucyMiPiA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIiB4bWxuczp4bXBN TT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9u cy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtbG5zOnhtcD0iaHR0cDov L25zLmFkb2JlLmNvbS94YXAvMS4wLyIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDo4ODIyQUZF QTc3RkIxMUVBOUQ1REY1N0FCOTE5REQwQiIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDo4ODIy QUZFOTc3RkIxMUVBOUQ1REY1N0FCOTE5REQwQiIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90 b3Nob3AgMjEuMCAoV2luZG93cykiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJ RD0ieG1wLmlpZDozRDE0N0RCRDMzOEExMUVBQTVDMEQyQjIxRUU5OTM5QyIgc3RSZWY6ZG9jdW1l bnRJRD0ieG1wLmRpZDozRDE0N0RCRTMzOEExMUVBQTVDMEQyQjIxRUU5OTM5QyIvPiA8L3JkZjpE ZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/Pv/u AA5BZG9iZQBkwAAAAAH/2wCEAAYEBAQFBAYFBQYJBgUGCQsIBgYICwwKCgsKCgwQDAwMDAwMEAwO DxAPDgwTExQUExMcGxsbHB8fHx8fHx8fHx8BBwcHDQwNGBAQGBoVERUaHx8fHx8fHx8fHx8fHx8f Hx8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fHx8fH//AABEIAGUBwgMBEQACEQEDEQH/xAC5 AAEAAgMBAQEAAAAAAAAAAAAABAUDBgcCCAEBAQEBAQEBAAAAAAAAAAAAAAABAgMEBRAAAQMDAwID AwQKDQoGAwAAAgEDBAARBSESBhMHMUEiUWEUMiMVCHGBkUJS07R1FjehsWJysjNTs3SUVhcYgqLS c5MkNDU2duFDRFWVJtQlVxEAAgIBAwIEBQMCBQQDAAAAAAERAiExEgNBUWFxIhOBobHBBJHhMvDR QlJicoLxotIU4jND/9oADAMBAAIRAxEAPwD6poBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgF AKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCg FAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQC gFAKAUAoBQCgFAKAUB85f4y8R/ZiR/Wg/F1rac/cH+MvEf2Ykf1oPxdNo9w2/k31iMfguF8a5QeF dfa5GjxNxRfESZ6Coi7iUF3Xv7KbSu+DUP8AGXiP7MSP60H4um0nuHQO0/e+F3D+mPh8U5j/AKIb adPqPC71Orv0Swja3TqNQaraSv7Y/WP41zfPLg3YTmHnOjugo86LgPkl1JtCQQsdtUTz1+3XUlby dcItoqXsS9ZNnIuFfWIx/KcHyfKtYV2KPGoSznGifE1eRBcLYKoCbf4rx1rTqYV5NQ/xl4j+zEj+ tB+LptJ7g/xl4j+zEj+tB+LptHuG78z7+QeMZrj2Lcw7skuQRY0tt0XhBGklOKCCqKC7ttqiRXaD q1Q2c1/vsh/3vf3b/RTnxG/Z9I9Udn/CfFX6e2/h6flVYwZ3Zg6VUNGp9w+dv8Sj4pYuKPMTcxOD HRIbbwMKrrgEQ+txNv3ltbVUiNwVC8+7niikXbWXtTVduUx5Lb3Iha0gkvsXXCe4mI5UcyGEeTjM 3jVEclhp4dKSzu+SdkVUIC8iFfuXo0VOSs5P3JzWO5n+imD4w9n5wwAyTpNy2IqC0bpM/wDn2RbE KeC+dII3kwfp53R//msn/wCVx/8ApUgS+xMzncWfx7i2N5ByDAu44JM1uJlIyyGnSgtPOq2EgzbQ gMPkqqCv3yJSA2bv46pUNGvc95lD4fxmRm5LJSibJtqNCbWzj7zxoANhouqqt/DwqpEbgmZjkuLw OAPNcgeDGxWGxOUplvQCJE+bFRS5lu0RBS6+SVBJpTPc/nuUBJfHe3c6Zij1ZlT5sbGuuD5EDDu8 7F4oqqlWCbn2Ng4bz0eRSZeOl4efg8zAETlwZzSoOw1VBNp8NzTgqqLay30XTSjRUyRz3mLPEcB9 MOxSlh8RHjdECQFvIdFpCuqF8ndeiQbg2KoUreT5sMDxzKZs2lfDFxH5hMCu1TRhtXFFCVFtfbah Gz3x/LBmcDjcuDashkYrEsWVXcoI+2LiCq6XtutQqJGQlpDgSZijvSM0bygi2VUAVK1/fagNVhc6 y+V4HjOU4Hj7uTkZIQcHFDJZZMALcikrr2wF27f2asEnBreV7v8AN8VMx0PIdvZLEnLP/DY9v6Sh F1HkFS23FSQfSniVkpBnc+xsvHeWc7yGWai5bhL+Ggmhq5kHJ8OQIKIqop02iU13Em3ShpNltieW YzK8izeCioaycB8MM5wksHUlATggHmu0BRVX31IEnjmHLoXF8WE6RElz3HnRjxYUBgpD7rxoqiAo PpG+1dSVEqpBuDVF7h90QD4pztpJ+j09SqGTiHL2e34ZEuq/ud16QSX2Nn4Zzvj/AC+C7IxZuBIi n0shjpIKzLiu/wAm+0uorovuXyWjRU5KLuB3fw/B+RYbF5WI6UTKiZvZFtbjGACEN7gWVVHcaXVF 0T20SI7Qb406060DrRi404KE24KookJJdFRU0VFSoaNe5vzNnisbFPuxSlJlMnFxQiJoGwpSqiOL dFug7fCqkRuDY6hTFLlxYcV2XLeCPFYAnH33CQAABS5ERLoiIlAc4DvFls2ZlwTh8/kkECUUyjzr WNhuKK2VWXZCKpon72rBnd2JMTuTzWNMYZ5JwDIY6PIcBpJkGTHyjYKZIKE70dhACKupW0pAl9jY e4XMWeGcPyHJXopTG4CNKUYCQCLqvAzoSoVrdS/hRIrcI15Ofd0FS6dtZVl1T/8Aa4//AEqQSX2J 3Fe58PL5wuN5fFy+OckRtXmsdPQVR9sflHGebUgdQfO3v9i0gKxfcq5bgOK4dzL5yUkWG2qAOikb jhfJbbBLkZl5In7VQrcGnN9x+5M8ElYbtvLdxx6suZDIRcfIIfb8OaOEP+UtWCS+xZ8W7pY3LZn9 Hcvj5XHOTbFcDF5ARTrgnyijPAqtvClvLXx00WkBWLPuDzFnhvE5nInopTG4hMCscCQFLrvgynqV C8OpfwokVuDYqhSm5lyRvjHFsnn3GFkt41gnyjiSApoPkhKi2+5REbg9P8hba4k5yNWVVtuAuR+H 3JuUUZ62zdbx8r2oJwaRiO6XcDMYuLlcd26kvwJrQvxXvpOCO9s0uJbTUSS6e1KsE3PsT8X3Z25y Hg+WYCbxbIZEunjXJRNPxH3fJoJLKqHUXyFf27Ugbjf6ho0nvHHjj2s5SotAipjn7Kgpf5NVamba GhfVFZZPtpPU2xJfph9LqiL/AOmj+2rYnHoUX1yhEcXxURREFHplkTRPktUqTkJWE+tf2+gYaBBe xGTJ2LGZYcIW4yipNtoKql3kW10ptCujq3B+f4fnXEJWdxMZ+LF3PR1bkiAubmwRVWzZGlvX7ajR tOT4u4RwHkHI8NnM3gDNcjxpYsoYzV0eMDV1SNkh16jSsoSInjrbWyLts4JH1R2K7zMc7wJ47JuC 3yjHNf72Gg/ENJokgE/YNE8F9ypWGjrW0nNfqaohZHlSKl0ViIiov752tWM8Z0P60jDAdpJZA2Il 8XF1RERf4ypXU1fQtfq7x2C7N8cImxIlCTdVFFX/AIt6pbUtNDj31qZbUHunxmY4Kq1FhMPGIWuo tzHSVBvZL2StV0MX1N3/AMYPbn/2vMf7KL/+RU2mvcRzfhXLcfy/60sHkWOaeZhz3TJpqQgi6nTx hNLuQCMflAvgVVrBhObH17WDsc17x/8ANu3v/dET+bdqozbodKqGjlzrseV9YqN9GKhvQMA63nnG /AUcfEo7Tip9/f1Ii62q9DPUrORcwwnFu/z83Lk8LD3GGWW1jsPSS3rOMtRZEyRLCuq1ehG4ZsP9 /Pbz+VyH/wAZO/E1ILuRtXI8Jj+V8UnYiRf4PLRSbQlFUIeoNwPatrEBWJL+aVCtSa52W5FNyvCm oGUW2d4885hssCrdetDXYJLfVd7e1b+a3qslXgquR/8A27vHhuOj68Rw5tM3lk8RWc6m2C0X7oBV XU91OgeWZM/Eb5P3qxmEyCI7h+NYz6aSIWoOzn3lYZIxXQkaAVIfYtOgeWdNqGjR+9GPlSO3mTnw pLkTI4JEzEF9slGzsH56xIi2MVESTaunn5VUZtoa/wB5smmV7P4/KIOxJ8jESUD2dZ9o7f51Vakt odYrJs1fup+rLln5nn/kx1US2hn7c/q94x+aYP5MFGFoWPI/+nsp/RH/AOaKoVmrdjv1S8X/AKEP 8Iqr1M10K7ut/wBYduPz2X5OVELdDpVQ0cz7cfrW7n/0nF/kZVXoZWrMvagpGYzfMeVy3jN2VlXc VDZUi6bUPGL02kQL7UIiIyL/AMVoxU6PUNHMudRG+Pdy+IcqgIjLubl/o/mmw0SS2+2RxzNE8SaN r5XjbTwqoy9TBzWBCyHe3iUGcyEiHKxGVakMOJuAwMUQhJF8lSi0D1MfFJ8zttyRjgubeN7iuSNU 4dl3Vv0iVbrjXzXzG/zSr4+HuG6kWMEzvv8A8r4l/wB04r+GdRFsdMqGjl/ecDzOY4XwdwyDHcjy DjmUEVUVdjY5tHzYVU8jVU+5VRm3Y6JJkYzCYd2Q4gxMXjI5OOI2C7GmGAUl2gCKthAfAU+xUNGj l9YPtAKKRZ9ERPFViTUT+Zq7WZ3ojfWJdbe7I8gebXc24EMwLwuJTGFRdaV1F9DpbX8UH71P2qho 5h3Vdjv897cwIaoefbyyykENXAx4NF8UpW1EDRETXxt7qqM21Qcit8n77vNT0R3H8LxzD0KKWofH ziUviFRdFUGxsPsWypToNWdRqGjn/fHANZDgE/Ksr0MxxwFy+JnDo4y7E+dLavsMAVFTw8PZVRmy wUnejKrl/q/vZZR2LkGMVKUE8E68qM5b7W6qtSW0Ot1k2aR3t/VPyj+guftpVWpm2hln/qckf9un +QrTqOh77P8A6rOK/myN/NpR6lroa99YtyOfb4cc2qFnZ+QhN8faT+NWYkkF3Np46N7rr7/fSpL6 HUKho03vL+qrlX5uf/g1VqS2hoP1Qv1ZZD88P/k0arYzx6FB9cz/AJZxb/XTP4DVKk5C9wPdr6uM fB45iY9ASW1FZCQhYqQS9QW0Q7kkZb+rzvSGVWqbzw7uB215JByUDhclpwYbJPSWGIr0UB6iKKFZ xpkVVdvlUaKmnocb+pj/AMTy395A/bkVqxjjHfPthluFZ8O5nBt0Vpt3rZKOymkd01sTqCmisu3s 4Pgl/wAFdCYtWMoxfUz/AOZ8p/1MP+G7Sw4zov1p/wBUcv8ApcX+cqV1NX0Lb6un6meN/vJP5Y9U tqWmhyD6zwAfd/iQGKEBRookJJdFRZriKiotaroYvqfR36G8Q/8AY8f/AFVn/RrEnSEfN8SHEh/X ECNEYbjR23l6bLQiADfE3WwiiImq3rfQ5/4j6orB1OVd+Mf9InwaB8S/D+J5HGa+KiH0pDe5p1N7 Tll2knktVGLGfI9k5TsCQ1C55ypqWbZDHceyhuNo4qelTAQBSG/iiElJLt8TB2HcwuOx+R4u7jhx XMsW7/8AYmSInHZZL8iaLrikbjbqLdNbCq+xUuYqSmCEfrFzFJURP0Ua8fzgVOg6nSeq1+GP3UqG j0hIqXRbp7UoDkXIctD7cd1pGdmn0ONcugOFNP70MljG1MV9iK8x6UTxIq1qjDwy47H4manGZPKs qG3NcvknlpSL4gy5pFaRfwQaso+zdUZaknk2GyGK7iY3nUUmhxIY+RjuTk84jaNQ27ympA3+UoOI qL52X2XVAesm3YbOYfNwG8hiJrM6E6iKD7BoY6+S28F9qLqlQ1JzrujylvkgH234o+M7OZmzGWfY XqNY+CpJ8Q6+Y+lCILiIXvr9i9RmznB676Qo8DtYxCjptjxZuLZZH2A3JbEU+4lELaHT+q1+GP3U qGio5hijznEc3ho5j18jAkxGlulkN5kmxVftlREZrHZjmOLy3DcZhidGNyDBxm8dlcS6qBJadiAj Kqra+raWzcipp5eKLVaJV4JndTnGG41xae2+8LmWnMORsVi213yJEh4VbbEGkuSpuJLrb9miQs4J /bXAS+P8BwOGmJaZDhNBJFNdrqjuMbp47SJUoypYNa7rf9YduPz2X5OVES3Q6VUNHM+3H61u5/8A ScX+RlVehlash4nMR+2vN8xh88XwnF+TzjymDzDmkdqXIRFkxHjX0t+odzd9LVdSaM6i5PgtQ1mu SWghoO9ZJGKNIP4W9V22996ybNAkNJz3mXGczipUeZwvAFJlnLYcQ1eyYXYba220FpFVxD8C+5em dWYeTfr64X+bMn+0NOgepufMOJYflnH5WDyze+LJT0mOjjTg6g62X3pguqL9rwqJlak4Ry3lGdjj xvgPLiVzkmI5JinoeSsuzI49HDAJKL/KDdBcT2+3WtGG+h9IVk6HPu8GAzb8XDcp4/HWZm+JzPjm oI/KkRjHZKYD90YeH2LJraqjNkbBw7n3FuX48JeFmg6dvn4RqgyWC++B5lV3Corp7PYq0aKnJr/f yZEj9puRA++20b8bYwJkIqZq4PpBFX1L7kotSW0Krvt+oDK/0bH/AJVHqrUW0J4dmRVsV/TjlyXR PDLF7P8AV1JG017tJjoHEecZXjPJG1e5nKQn8byWU4465lIF7oIG6R7HGtvzjY+Nr62vVZK4ZZ8y ku8E7lM86faM+LZmI3i+QPtiprEeaO8aUYjdemqL019n2VRFiK8OTpmOyeOyURuZjpTUyI6l25DB i42SL7CFVSoaOad2OWNZ6K5244q8E/kecRI04mV6jUCESokh6SQ6D6PSg3vr9i9Rmz6Fn3X4dJnd nclxzCNk49EiR0gsolyNIDjbogiJ4kQs2RPbRPIssGwcI5zgOYYRjJ4qSDhGArKibk60d23radD5 QqJaa+PimlGipyaj315LBXiknhsAxm8p5Hsg4/FtKhO/OGm91wUuoNiCKu4tP2aIln0Np5PBSB2z y0FC3JEwshhC9vTikN/2KFehzvtr2oHI9v8Aj0/9MOTw/ioDDvwsTJq1Hb3Ai7Gm+mu0U8kqtmVX Bu3HO0vF8LmAzbrs7N5toVCPk8xJOY+0K6KjW+wh9lBvUkqqbpUNEXK4rH5bGycZkWUkQZbZNSWC VUQwJLKKqKotAQuL8R45xXHnjuPwQx8Jx0nzZbUyRXSERUvWpL8kBSkkSgwcr4JxLlrcZvkWNbyI RFIowuEY7FcREK2wh8dqVZDUmu/3A9n/AOzMf/aP/jKSybEXnGO2/COLHKPAYpqAU0EblK2ThbwG 6oi7yL20kqSR74p2+4bxIpJccxjeOKYgJJVsnC3o3u2X3kXhvWkhJIvpEdiSw5HkNi8w8JNutGiE JgSWISFdFRUXVKhSg4p284ZxNyS5x3Ft445iCMlWycLegKqii7yLw3L4VZIkkT+R8ZwXJcYWLzkQ Z0AyEyjmpIikC3FbiorpUDUmTBYHEYHEx8TiIwxMdFQkYjgpKIoZKZWUlJdSJV8aFSKzkPbvhfIs rFy2axbc3IwhEYsgycQgEDVwURBIU0JVXVKskaRsVQprv93nDP0r/S36Lb/SK+76R3Ob79Lo3tu2 fxfp8KskhGxVCkDK4HEZZ2C7kYwyHMbIGZBIlJOnIBFQXEsqaohL40EE+gKuRxfASOQRuQuwwXNR GiYYnCpA4jRXu2W1UQx1XQr0JBVcn7XcB5TkQyOfw7U+aDQsA84TgqjYkRIPoIU0U1qyHVMqP7ge z/8AZmP/ALR/8ZSWTYjbeOcZwXGsYOLwcQYMACIxjgpKiEa3JbkpLrUKlBj5PxHjfKceGP5BAbyE NtxHgac3IguCiihIoqK3sSp40kNSWrTTTLQMtAgNNigNgKWQRFLIiJ7EShT0qISKJJdF0VF8FSgN DyfYrtPkphy5HHmW3nVu78K6/EAr+O5uO40C3+xVlmdqNm45xPjXGoSwsDjWMdGVbmDAIKmqeZl8 o195KtSSpQZORcbwfI8W5is3EGbj3SEnI5qSIpAu4VuKiuip7aBqTUf7gez/APZmP/tH/wAZVlk2 IvOK9t+EcTkvyePYprHvyQRt82ycJSBF3Ii7yLzpJUkjFyntdwDlMhJWdwrEqYiInxY72X1RPBFd ZJtxbeV1pIdUzxxntR284xL+NwuEYjzU+TLcVyQ8N0su118nTH7S0kKqRtlQpAyWAxGTlQJc6ML8 jFvfEQHCUkVp1RUdyWVPJfOggn0BXwOP4eBksjk4cYWZ2WJs8i+ikqukyGxtSRVVE2jppQQZ8li8 blITsHJRWpsJ5LOxnwFxsk94kipQGjh2A7QBJSQnHGlJC3I2T0kmb/6knVa/zassztRvcOFDgxWo kJhuNFZHYzHZAW2wFPIRFERE+xUNEWRx/DyM1Ezb0YTysFtxmJKVS3A29/GCiIu31W80oILCgKjO 8R41npECTl8e1MkYt5JGPeO6Gy6ioSEJCqL4ii28NKEaLehRQGpcl7T9uuSylmZjBR35hLc5be+O 8S+0nWCaMvtrVkjqmQ8V2P7VYuWMuPx5h2SCoouSzemWVPBUSSbqIv2qSybUbTnuP4bkGHfw+YjD LxklAR+MSkIkjZo4OoKJaECLotQ00WCIiIiJ4JolAVWc4rx/OuwXsrDGQ/jHkk49/cbbjLqffAba iSeCXS9loRos3WmnmjaeAXGnBUXGzRCEhVLKiouiotCmhTewnaWXJOQfH22TdW7gRX5MZovd0mHW 27f5NWWZ2o2jjXEOMcYiLEwGMj45grK4jAIhGqeCma3M195KtSSpQW9Cmm8h7OdtOQTyyGTwTJTn FUnJTBuxXDJfEjKObSkvvKrJHVE7ivbfg3EzNzj+HYhPuJtOSm519UXxRXnVNy3u3UkJJF9Nhxp0 N+FKBHYsps2X2lVUQm3BUSG6WXVFqFMeKxcDE42NjMeykeDDbFmMwiqqA2CWEUUlVdE9q0BKoBQC gFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQ CgFAKAUAoBQCgFAKAUAoBQCgFAKAqT5JDDIO4/4eUUpkEdMAZIvmyIhE0VPFFUFtW/bcSed/k13O sOV4EvH5WDkEcSM4quMqgvsmJNutqqXRDbNBMb+V018qlqtanTj5a306EusnQrmMs65nJGLcjdNG mAkNv70LeJmQfJRPTqHtrbr6ZONeVvkdGukkiDNOUL6lGejdF42UR5ERTQFsjgWUrgX3q1myg3S+ 6cNQ+v1JNQ2KAUBGcmGGQahpGeIHWzcWUKJ0QUVRNhLe+4r6aVYxJh39SrD8+hJqGxQCgFAYpbr7 Uc3GGuu6KXBrcgbv8pb2qozdtKUpMGFyKZPEQskjatJMYbfRpV3KKOChWvpe16t6w2jHDyb6K3+Z STKydRQCgK5jKvHm38Y5G6aNMjIbf3oW8SNQ+SienUfbW3X0yca8rfI6NdJJzrzTIb3TFsFIQQiV ETcZIIpr5kSoie+sJHVtLU90KQ8vOegY2RMaY+JKOBOK1vQLiKKq+pUXyStVUuDnzXdKuyUwZ4kj 4iIzI27Os2Lm297bkRbX+3UahmqWlJ9zLUNCgPBPMi6DJGKOuIRA2qpuJBtuVE87bkvSCSpg90KK AUBXfSzqcgHEnG2tuRnJLUrei7ukbYEOy101e8b+Vb2+mTh7r9zZHSZ8o/uWNYO54deaZFDdMWxU hBCJURNxkgAOvmREiJ76JEbS1MMmabEqKwMZ54ZJEJPNoig1tFS3OKqoqItrJZF1qpSjNrw0obn5 eZJqGxQCgFAKAUAoBQCgFAKAUB4eeaZaJ14xbaBNxuGqCKInmqrRIjaSlkHLZZ6A9CFI3WZlvhHN 1DQemri2Rdtl3VutZk5cvK6NYlNwWNYOwoBQCgK5vLPFnXMW5G2CLHxDUjei703oCptRNNV9tb2+ mTiuV+5sjpMljWDsV0zKvRsvAgrG3MziMEk70TaYNm7bZa66B7a2qym+xxvyut61jFuvwksawdhQ CgKJj/rmb+bIn5RJrq/4Lzf2PJX/AO9/7K/WxHyzoROZ4ySGiuwZqTUHxVlhWjBV/emSon75atVN H5oxyvbz1f8AptPkoK8M3k3sEOXZdnOZNxpJTOPbhvLGLcm8Y6L0dUVPT1N/jre2lb2JWjEeZxXP d8e9O26Jja48tPnJciilzSQiKoquMZsSWunz7vtulc/8HxPV/wDu/wDYvqyrI+Ruceyxxsi+5kMf LkIwexje62yqL0lTpbLkKKiKgpqtb9O5SsNHnnlfHaLPdWz7ZjpoWHxjkmVBfiZB5YDcNJsnaLKo 6BW6O67d0V2xqu1U+T5ViITlZk7b3Z1as9u2Xpnt065/QqQzmTfwI5hl6ceTcZSUxj24bxRl3JvG Pfo+pCT09Tf46ottK6bErRiPM8657vj3p23RMbXHlp85LGYOae5QsBnIusQZENXzRBZ3tKLiBZle mvyt2qnu93tTCjbMZk7XV3y7VZqrrPTHlj6yZWkyULkmNhOZF6XHfgySdB0WURXI5xxFy7bYFcuq V7rao4dW46/3NLdXlrV2bTq+3Tb4eJHgR81MyeZgu5iQMWHIbFlxsGBfs5Hbc2qfT2bRU9LBf2rV s0knBjjre1r1d3CfhOifb7HhvNZP9FYb7sjY+cv4KXkNopsAJJRye2qmxFLZ5ptRVva1XYtz8vsR c1vaTbzuhv8A5RJJiJNTlBwwyb78KNHB91leiSI6ZECA4fT32IU3Cm5F09ipWXG2YydKbvd27m6p T0+eCZm5soJOPx0RzoPZF0xKRZCVtpptXDUEJFHetkFLoqa31tWaJQ2+h057tOtVh2evlkyxoWRj Snd0xZOPJlNoPIiug8irdUMUG4EPkvgvhRtNaZNVpareZrHXWSgw06WHGOI46I50HcjFYA5CIhE2 01F6hqCEijuXagpdFRL38q63S3Wb6f3PHw3a4uKqxuSz4Kslk47Nx+ZhQFluvxcmLzYE5sV1l5oO ohASClxIUK6Ei6onlpWITTcaHdu1Lqstq0/BoxY+TkX+KTHXJzvxTDs8AmILXU2xpTrbd02dP5La Ivpq2SVljt9DPHaz4W28p2zjpZ+EdDymQyUwMFjm5JMPT4azJssBDqbGhaQkBCQgEjcfTXbol7U2 pS+zJ7lrbKpw7Vlvyj6tnrHRn43MZDTklyUP0c0rZuoO9EV9z0qoIKF4ey9LOafEvHV15mm59K+r PXNYzz0CJslOxx+PgCQNo3YlKY1Yl3ga3FdU1t7UWpwvL8n9C/m1bqsteqvb/MvAy5H4iMcNl/Km EVRcQ7CKzJDt0UBAW2/kiO6+wL+HvqVzOP7GuSawnbH/AHN/BfREONPkzOKZtJBm6UX42ODro7HC bAVUFMUQfVtJEXRK26xZfA5V5HbhvPTchKyzrDeDxjTjrCS4qvvvx2SfdRtgWx2gIg5ZSJ0fUo6J fzVFqKsyxbla2VUqVOFOkefc9sSMm6WSiMyZgRQjg9EyD0fpOg4qmhtJ12kE0sArdRVdV18KNLDw WtrPck7REpxnyyskH4jPBxbH8gdyjhSenDeciiDQxzB1W0MSRRU1IhNfVvTXwRPCtRXc6x3OW7kX FXkds+nGIzBMykGU9zTHoGRkMCcGaYi2jFg2uRRVB3tHoV7ruuvstWatbHjqvudeWjfPX1Nem3bv XwLrNOZFvESzxodSeLRLGDRVU0TSyLZFX2X0rnSJU6Hp5nZUe3+UYKOFnYzbUyczkXpTUGI69Nxs sUbkg4CIYlsUGzFFQTRdNvhauro8KNWeWnOknZWb21bdXr9P2Pcn6bi8ePNLPNyezHWY7GVASMSC HUJlB27kG3pEt27zVV8Ki2u22MFt7lePfu9SUx07wZusMjluNeC6C9ipRivmiE9FVKkRR+a+5qZ5 qvvS31qZMCc6VEyTD0143GZj0dqVtaR0QDbbRA6fn+BS8JrHQ1wO1lZNvFmpx/aPkUcpzIZLh3HZ j054H5D2NWQrYsohm4+0u9UJstRLVLae1FrooV7KO55bO1+Hjs7OW6du68C5m/SELMYFocg+8xIe eYktOCxZxEjPOiRKDYEhIQJ4KiWTwrmoaeD033VvRbm0209OzfY/Ybk3MS8gfxjsSNDkFEjtMbEV SaEd7pqYndd5KiD4WTW9HFUsCjtyO2WknCj6lbNy2acwDr7UtWchjciEJ4gBvpPp8S23cxISIUJt xFVAJNb1tVru0w19jhflu+OU4tW0eD9SX07E8xymOzmLRzIuy2sgbrEhh0WhbEhZN4TaQAEht01G ykui6661jDq8aHZq9OSs2b3Snp2nH6EOVk5Y5KXGm5B7FTCf24pTAUhuN2HYiOKBCRFqhCRbr+Hl WlVRKU9+5ytyvc1azo59P+X+vmbTJV9IzqsKCPoBdJXL7EO3p3W1238a4rU99phxqavGyj7WSwoN ZF7IBOcOPMcVtPhjJIzjyGyaAI/LasiASpbxrs64eIg8NeVq1Is7bsPto3jHgToZzsw/PdSY7EjR ZDkSM0wgIqkzYTdcUxO6772HwsnnWXFYwdaO3I7OWknCjw6kXJryVh/j7f0ggyZTpRJwtgHQJUjP Ok8KECuIadO6Ju238qtdrnBz5fdTot2W4fb+Lc9+nkZUDJN5VjBDkn3AJp2bImOI119m8QbYBRAR RLkqqW2+mnuYjdBqLK649z0bbxPloHpuZhzMhiozizZKwDmYontqEjoqrfSNRQEId6gqKuuq3WiS aTeM5DvetrUXqe2az9PoecDkRlTEFjJPuPtsks3GTwFp9HFUdh7NgKIou5F2+nXSl6wtPiicHJut izmM1th/T9jBgMlIkSoTMvIvR8wNyymLlALaHZstyR02JuEXNqiQEvp8VWresJwsdGZ4OR2aTs1f /FV+XT49uhM50w67xuUoSHGEFB3i2jdjQjFLFvE1+5as8L9R0/Oq3xPMGLkkeWxBxrQSifkLko+y RIEFVFJVTUWhaRUT2fs1eNpt+RPyatVqpl71r+0GZhMjA5HEiuZB6bGnRpBuA+jSbHWCa2kHTBuy KjqoqfYqOHWYiDVd1OVJ2dlZPWOkdl4lcxmZOSiSJwy50d1XHhgsx4brjIi0ZAG9UZNHFPbcvVpe yWtetuiTjH6nCvM7p2my1iKuMfDJtGNkSJOOiyJLSx5DzLbjzCoqK2ZCikCouvpVbVwsoZ7+OztV NqG0al9IZR/AQckGUeayc6Syy7EbRlUFXXkB5gANs7EwCkt1uvpuWld9qVmowj5/uXfHW257rNKM d8rTp9sly02TfLm2yMnSDGbVcO24lR5E3FtQUuvuSsP+HxPSlHN/w+5HxK5bM4ZvLt5ByNImAr8N kRbVlsCurQGKipHcbb13X8du2raKuIMcW/kpv3Q3ldvD9yK9PezTXE5zJrDdmGbhEKCZApQ3VNB3 XG/iiKqL9iqq7dy/rU5vkfKuKyxu/wDFkiTkpeAyDrUmS5OgnAkzmurs6oHD2K4CEAjcTF1LXTRU qKqssYcx+pu3I+KzTe6u128fT/1IqZTKJj485p+dJyRK0bsNIbwxiEyHqNDdlNoiKrtPffS6qqaV rapjEeZz92+1WTs7YxtceWnzk3CvOfSKMsLmkzkjKtZCMJPMBGFoorhILbbhuDdUkDcvnVuv7Fdd 9dsR8/2PL7N/cd1ZZUfx8/8AV4kqBhAZlPTpbyzJ8gEaceMUERaRb9JttNBC63XVVXzVdKza8qFo b4+GG7NzZ/TsiJj8HmcfHDHxckCYxn0R0NhTkttJ8lsXVc6a7U0FSbXTxvWrXTy1k58fBei2q3pX hmO0zHyMw4jJJn3Mr8Yz0zYSMkf4c9yAJEYr1Ot43PX01Ny2xBv2re5vlaRp+56wWJm474z4mU3J SVIOSnTZJraTi3JPU47dPZUvZOIHBxWpMuZc6R92MFgGMSxIYFxXgedMm0JE+bZVV6bCfuG0VUSl 77hwfjrjTWsv5dvgR8fg8xj2Ax8TJNji2vTHQ2FOS015Ni6rmxdqaCpNrp43rVrp5ayY4+C9FtVv T5ZXhMx8jOWJyC8iHKpLaRgWFjfDKwSlsUkNV6nVtuun4FTctsQb9q3ub5URER95+x+ycTPdz8TK BKaBmK06wkdWSIiB8miP5zqil7spt9GnvqKy2wLcVnyK8qEmojvHWfDsecfiMnEm5OUcxlwsgQuC KRzFG3AaBob/ADxbh2tpdNNfNKtrJpKNCcfFetrOV6vDwjuU0/GzcbhImJcmtOOzcinRfVhQZRXH TlkL4K6e8CUVFEQkvonvrpWydm40X7Hm5OO1KKja9V+2NXbOSZFcy+KyUDHu/AORpxuCrUKOcY29 jROdVRV14SC4IK+GpJWXFk3nB1o78dq1e2LdlHTXVlnmcQuQCObL6xZsN3rxJKCh7T2qBIQKqbgI CUSS6fZvWKWjyO/Nw74hxZOUz8ai59W3VfnR1fIdrQtxiRkdUVSIVdUyW2iWNET2LRuvYiryRmyn yx9fuQYfF5DGGxkIpglMw+xIEwGVBEEGujtcbVw96GCqhWJPdZUvWnyS241OVPxWqVrOaaOPCM5J jOJkuZJrIZF8HnYwGERpltW2wVy2813E4pEqDZNUsl9POsuyiEdVxN2VrOY0IKcbygNToLOSBrFz HH3kBGLyAWSRG4Auq5s27zVf4u9ltfzrXuLDjJy/9a6Tqrelz0znxn7GVOPTQiYxW5jY5PFNqy1J RlekbRCgk2bSuKViQBXQ/lIi+6nuKXjDL/69kqw/VXrGP0n7nuLhsu3nFyr89l1XGBjvRxjkIoIG Rp0y6qqOp67t32qjutsQWnDdcm92WkafTP8Acl5vGHkYPQadRh4HmJDLqjvFHI7ovDuG43FVCy6p WaWhnXn4t9YThyn+jkhv4bMlkI+TZmxwnAwcZ9DjmbSgRoaKA9YSEktZfUqF7q0rqIjBytw33Kya 3RGmPqYQ41k24GUgt5Fsm8kThq69HUnBJ8EF2+xxoV9o2FLe+r7ilONDK/Gsq2qrfy8O+vVGRzj0 0o+OMJoN5TGCoMSxZXpG2QoJtuMq4qqJIIqtjRboipTes4wyv8e0Vz669Y+qn7ktIeaOJIB+aysl 4UBtRjkjIJ5rsV3eRKi/ylvDT25ms6HTZdpy1L8MfX7kB3jWRPi8fBJPZRWAaaWSscl3Axt2ejrJ YvRqu77VaXIt26Dk/wAaz4lx7liMx2+JLm4nIuzoOQYlMtzYrL0d1TZI2jF9WyJUBHBIVQmUVPUt RWUNdDpfis7Kya3JNad48fAnZCNIkQzZjyFjPrtUH0TdZRJC1G43RbWVL+FYq4Z15Kt1hOGV/wBB Oyp6TMo40+ox3YgtMtk2KtyFFXN6kZqV+mlk0tr41vfChHH2Ha268PDWnfUwnx7Iu4z6HenieLUE YNekqSTYRNvTJzftuo+kjQNU9i61fcUzGTL/AB7Omx29OmmY7TPzgzuYaeufj5NqUy3HjsHFGL0C Uum6TZl84jqJuuym30WT2LU3rbBp8NvcV01CURHl4+HYxN4PKx5kxYmQBmBOeWQ42rO58DIRE0ad U9iIW2/qbW1N6aUrKIuC6s9torZzpn4OfsYmeLym+Mw8OswCfx5xziyUZVB/3UxMENtXCUr7LFYk +1VfItzcamV+K1xKk5rEOO3xJEzEZiVLxkopscDx7hPEKRjVDI23Glt8/wClNjvv1S/uqKySajU3 fhvZ1cr0+Hmu/ie0xM6LLlP4yS003NPrPMPtE6IvbUBTBRNpU3IKbhXz101qbk1noX2rVbdXG7uu v6oiy+LPHhUxsWYLZuSBlypTzSuk48j4yFKwuNINzG1vIdErS5My0c7/AIr2bU+stxMuZ7ok5HEZ OXPxksJjLf0eSuECxyLqGbRtHZesO0drmia2XzWpWySajU3ycNrWq5Xp8PCO5hn4HJToUvGyJzbm PmE7vU2SJ8WnTUlbE1c2+lCsBbdNNNKtbpNOMmeTgtarq36XPTOfj+hZ5bHhkcXLx5mTYS2XGCcD 5Qo4Kiqp9i9YraGmd+Xj30de6gqHcBnXygPP5Jj4jGuo5GFuKoMrdo2SVwOqpKuxxbbTRE9i1tXq pxqed/j8j2t2U1eMY0jv49zP9C5OLOlSMVNaYZmn1pEaQwTwi8qIJONqDjKju2puFb666VN6aytD Xs3rZujSVujU58MoZDCZSS/jHW8g2hY5xX1V6OrhOOE04yqrsdZQR2urZET7dK3SnGo5OG9nVq38 c6a4a7ruSchinHpsfIxHkjzo4G0hGHUbNpxRUgMUIF+UCKKoui+66VK2xD0OnJxN2Vk4svoR1wUt w5kt6btycllI7MhkFAGGxVSRABSJVuRXJVLX3Vd6wowY9huW36monsfp4WdKmtTJ0pvqxmnm4yxm iaVFfFBI1IjcXRE0FLa+2m9JQg+G1rK1nonELv8AE8phci+7ALIy2nwxziPNq2yQOG4IE2hEROHZ PWqqiJr7baU3pTC1Hs2bruae3w/cm5rG/SeKkwep0VfCwu23bSRUUV23S9lTwvWaWhydObj30ddJ IORw+bnsxEcnRm3YsgJKqMZxRJW1uI2V9FRPbr9ytVvVTj+v0OXJw3ullYc6f/Iyy8TkH85CyQy2 gahg438OrBERI9s3+vqoiL836fTp76KyVWoNX4rPkVpWPDvHj4GGPg8tAN5rGT2mse86bwsPsK6b JOkpuI0YuNptUiVUQhW32NKrunqsma8F6SqWW191MT2yi4IuhGUjUnekFyKyKZbU1WwoiXX3JXPV nomEaXi2sxEwAZyPJxZAsdZKPPxj65Ad3NrskHkRS1sRbPHyr0Wh225/rwPmcSvXj9xOmk5Wfjaf sXkPH5SRlo+ccfbZB2KLRQCYLqAJqjiirnVtvQtL7PtVzdkltPXTju7rkmPTpHx1n7H6zgslEhnj oE8GICqSM3ZU32QNVVQbPeg+m/oUgW3nuqO6blrIXBatdtbRXyyvn+mP1PEjjT6HiRx0sIcXDiqR mVZV1VXpEym8lcG47S1S1/3SVVyaz1Jb8Z+na9qppjwjv/XczM4J16W9Lyzzct1yOUQGmm1aaBlx UV1EQjcJScUR3LfySye2O+IRpcDbdrucR4R1/U84/EZyG0zCTJg5AY2g0RMXldMfkgTquK2q2S27 p6/Z1pa1XmMk4+HkqlXd6V4Z/WY+RdVzPUKAUAoBQCgFAKAUAoBQGORGjyWSYktA8yaWNpwUMVT3 it0WqnBm1VZQ1KMMPFYyCpLChsRVOyGrLYN3RPC+1EquzerM04qV/ikvIlVk6CgFAKAUAoBQCgFA KAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgII4LBjJ+JHHRkk7t/XRltD3 fhbrXv761vtpJy9jjmdqnyJ1ZOooBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCg FAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQC gFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQCgFAKAUAoBQ CgFAKAUAoBQCgFAKA//Z --_004_AS8PR04MB8071BB5854E10B6EA7161159EF0D9AS8PR04MB8071eurp_-- envelope_from noreply@grupokonecta.net envelope_to joe@domain.org helo_domain rheology.yeloweditions.com iprev.result fail dkim.result pass dkim.domains rheology.yeloweditions.com spf.result softfail spf_ehlo.result softfail dmarc.result fail dmarc.policy reject remote_ip 51.89.165.39 tls.version TLS1_2 expect_header X-Spam-Result: DKIM_ALLOW (-0.20), HAS_LIST_UNSUB (-0.01), ARC_NA (0.00), DKIM_SIGNED (0.00), FROM_EQ_ENV_FROM (0.00), FROM_HAS_DN (0.00), HAS_EXTERNAL_IMG (0.00), HAS_LINK_TO_LARGE_IMG (0.00), HAS_REPLYTO (0.00), HTML_SHORT_1 (0.00), MID_RHS_MATCH_ENV_FROM (0.00), RCPT_COUNT_ONE (0.00), REPLYTO_ADDR_EQ_FROM (0.00), REPLYTO_EQ_FROM (0.00), SPF_SOFTFAIL (0.00), TO_DN_NONE (0.00), TO_MATCH_ENVRCPT_ALL (0.00), RCVD_COUNT_ZERO (0.10), RCVD_NO_TLS_LAST (0.10), HELO_NORES_A_OR_MX (0.30), DATE_IN_PAST (1.00), MID_RHS_MATCH_FROM (1.00), PARTS_DIFFER (1.00), FROMHOST_NORES_A_OR_MX (1.50), HTML_SHORT_LINK_IMG_1 (2.00), RDNS_NONE (2.00), VIOLATED_DIRECT_SPF (3.50), DMARC_POLICY_REJECT (4.00) expect_header X-Spam-Score: spam, score=16.29 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; s=sectionalism; d=grupokonecta.net; h=To:Subject:Message-ID:Date:From:Reply-To:MIME-Version:List-Unsubscribe: Content-Type:Content-Transfer-Encoding; i=noreply@grupokonecta.net; bh=dP5ixF9HZsWlj/p3qHOLHJE+ZBdEajVqh0aowOaVxc4=; b=nxR6ACs3hInMKEyEK9yQIEgu4RuffMNmnA6a2mC6Z1c9Qbao8AsmuWct1i7bvn5lU1mmDSXx6PwO /w2Z6TSZIW1jZkhQwDo5sJWO/0f26zabNJ3bAsZnN/Yyy0Lf4oUJuaZ97cwHiMPlDIPF++gyuY6s 30mpjo29dX3ZCSCkMkcmox55AsQSK2nJ2ZIv8dqep02kiXjldaiW5hIRzHwjeh+REy3Mb5zCuIVJ 4dcKomEGK3JsnnY1mdMVHB1ghlVMnms+HSu2AYi7186j0QSdobMxNuBzAKqzDRfunSljG1IL1im8 bnZoghgIpHHfEKY5+b5hStqgBQoPTh2CJMAGWNDkdvoFK3m606RRRiKIOP69M4v5a1UjBBoyP7QO AUNbgcjHWQCl83a1ofW2LKSDPcyfj2TVK4yjQlPAH3Q0TJgeig1beio7Ahnz581H22kkyxPtMB56 YVA2cG7FvBB40HLmuX1YKu4TAwJ8ZJdCOnt2xZ9lYLb8LuduCFrd6RZH/44lUu/DpnM1lN3Q3oXe moOQCtyUsI77tu7rzy0vgzZME4v0j3S57CYHJktZGvk/3uFHTWhStYlWNk+Ks/rCfkQjSmQV1u+T ozHF7td0rxpD0b8534vjwABr/NwpjlSLIgBvt9d0nBtipYYOumSdXpCj9mB6uZbGVq9K0mdusbY= DKIM-Signature: v=1; a=rsa-sha1; c=relaxed/relaxed; s=sectionalism; d=rheology.yeloweditions.com; h=To:Subject:Message-ID:Date:From:Reply-To:MIME-Version:List-Unsubscribe: Content-Type:Content-Transfer-Encoding; bh=BwKu50FuSdWv9iToeKQHRPlPqN0=; b=UT4zz+mBbtBTs9S/YEiEH7B3N9gklIJBZLKAyaCOvddh0wWb33fT3qyqCKo9Vfyhp+gBHlQ/pfvZ xLRSO8SjfAOLs/0Xs0sy5uVLuNXh5UfdT+IwG5KSgA+NpRvOTmHvfUzFP5DoiykHP0eutX2TxkRh dJHjLIthv+EISFEubGu1ZgFUBReFTCvhZCrYPYJlxze/tmWWChRK7r804EBR3gJiLEvgXQho5Qh5 ksdnvvqAAVTHNOWAFbxoojBmNCs/gn0BbXHeJGuR/ZRXUbC1ZzQ4hw/9xNSCHI9n24IoD+vfhldb k2hSVGQisRxn9vTISgDaNjEgFdr+0hUTqSTvrVEDdJYjCmYK/HrJrb8oeh+G/mz30pYQJlJUYIuJ VcGkqXdFQdenNJPPD1UccGCpWblaaSlqY2AVk3mjX08Du2jbuzEFnX2g5UA5okGOmUi3h5C1v3vG zLMuOT3686ZovcbxlW+K1GkOL1IZTkgXiw2X4EDZ6nE191uyVpIi3NTBkRIIWsni2lpDfg9+qq/J /viVYHHmyG7xpB+gmS9r46HZqZYcYA9g5NQuTaYPIzngwlODdoSVsDb9X7/aU2Mhvp+ayq5g5/fa 4cbe7mAJMaCifxx5YA2KqkYLqRS3Cm0D2AJGqk9DpkpncHL+g8+jOAtoWCPFa5L+yVnQIexiJqE= DomainKey-Signature: a=rsa-sha1; c=nofws; q=dns; s=sectionalism; d=grupokonecta.net; b=k7FzASi0Vd8+owLAAzg7gG2fHkmp7q+kS7fNXMORbpbqrny/kgSjEkfLUDJdmPFyY2PlDQ+/dZcs W6kMg8n/+wKKEMenIz1O0RSCbKoV5N2gchXV1nEevqyZ+Ndf4XwLhHY+Ttlh6dnlMNblWnXZcxjs 2bvwWSAVakpbSIbRkYeMHJOfQTw7efqEEhKRqXCWHnHJuC9zkf4C3bUe6WcTjBmlCe12dU+HD18K lkc7TJKwIm81vBgE+HXpVC3YuJsfsNLM4bb/bZmt3yu5SgfEWss6DgOsOa2yNzLWMx6byVhx3zvg u9k+c+seOdotD0Rphz6F2GaW3LJwOz+Uuhntwir3FZ45GEeyn2V0jiXwVpg8L2RREs3bqCUm+THD 8p0gSDNbB3SExCuvRmRnvPCogYvO9XGbwX6eh4gl9dGMrmVSxpjnIoJjoAF/AZSPV2FGBhwFWzjr P+lD8QOzpAbKXSArKcDmJU6ehafQW1fknB1colA97NVan2HCj6B1YdaKv7BtPPjh+L1hwVd3mFaD Kp4/hGJnW/UE12ZUaLIFVsioZWf6TcNvFaZQzTlL8U5fShX8iQInbAWh9+c0YGGLzcGcjqUV3BCA ZXLFzE9yrfkpMDPInBqLOXbIiWzIUopFoCPHuEyEDpalYQQx3rXbUKsNnGs8xEipZueJWmdhaUc=; To: joe@domain.org Subject: =?UTF-8?B?UGVkw60gb25saW5lIHR1IFRhcmpldGEgQkJWQSAxMDAlIEJPTklGSUNBREEu?= Message-ID: <61805635a4a8f8915503cb518cbbaeecc82ce04e861805635@grupokonecta.net> Return-Path: noreply@grupokonecta.net Date: Tue, 10 Oct 2023 11:52:03 +0000 From: "BBVA" Reply-To: noreply@grupokonecta.net MIME-Version: 1.0 List-Unsubscribe: , Content-Type: multipart/alternative; charset="UTF-8"; boundary="b1_3d217f30a568faa9ce3dd7dc73399561" Content-Transfer-Encoding: quoted-printable --b1_3d217f30a568faa9ce3dd7dc73399561 Content-Type: text/plain; format=flowed; charset="UTF-8" Content-Transfer-Encoding: quoted-printable Tarjeta de cr=C3=A9dito BBVA La tarjeta de cr=C3=A9dito para viajar con tus consumos Pedila 100% online y empez=C3=A1 a disfrutar Conocer oferta Un mundo de beneficios con las tarjetas de cr=C3=A9dito BBVA compras en cuotas Compras en cuotas Pod=C3=A9s disfrutar hoy de los productos =E2=80=A8que quer=C3=A9s y pagarl= os en cuotas descuentos y reintegros Descuentos y reintegros Entretenimiento, gastronom=C3=ADa, farmacia, ropa =E2=80=A8y m=C3=A1s rubro= s con promociones exclusivas puntos bbva Viajes con Puntos BBVA Vuelos, alojamientos y mucho m=C3=A1s canjeando Puntos BBVA que sum=C3= =A1s con tus compras Conocer oferta Descubr=C3=AD la tarjeta que mejor se adapta a vos Todas las tarjetas Black Platinum Gold Internacional Todas las tarjetas visa black Tarjeta Visa Signature L=C3=ADmites desde $600.000 15% extra en acumulaci=C3=B3n de Puntos BBVA Acceso a salas VIP en aeropuertos Asistencia en viajes con cobertura de hasta 250.000 USD Extracci=C3=B3n de efectivo en el exterior Seguro de robo en cajero y compra protegida Atenci=C3=B3n personalizada para resolver tus consultas Tarjetas adicionales sin costo Es necesario un ingreso m=C3=ADnimo mensual de $200.000 Conocer m=C3=A1s mastercard black Tarjeta Mastercard Black L=C3=ADmites desde $600.000 15% extra en acumulaci=C3=B3n de Puntos BBVA Acceso a salas VIP en aeropuertos Asistencia en viajes con cobertura de hasta 250.000 USD Extracci=C3=B3n de efectivo en el exterior Seguro de robo en cajero y compra protegida Atenci=C3=B3n personalizada para resolver tus consultas Tarjetas adicionales sin costo Es necesario un ingreso m=C3=ADnimo mensual de $200.000 Conocer m=C3=A1s tarjeta platinum visa Tarjeta Visa Platinum L=C3=ADmites desde $350.000 5% extra en acumulaci=C3=B3n de Puntos BBVA Asistencia en viajes con cobertura de hasta 170.000 USD Extracci=C3=B3n de efectivo en el exterior Atenci=C3=B3n personalizada para resolver tus consultas Tarjetas adicionales sin costo Es necesario un ingreso m=C3=ADnimo mensual de $120.000 Conocer m=C3=A1s tarjeta platinum mastercard Tarjeta Mastercard Platinum L=C3=ADmites desde $350.000 5% extra en acumulaci=C3=B3n de Puntos BBVA Asistencia en viajes con cobertura de hasta 50.000 USD y 30.000 EUR Extracci=C3=B3n de efectivo en el exterior Atenci=C3=B3n personalizada para resolver tus consultas Tarjetas adicionales sin costo Es necesario un ingreso m=C3=ADnimo mensual de $120.000 Conocer m=C3=A1s tarjeta gold visa Tarjeta Visa Gold L=C3=ADmites desde $100.000 Puntos BBVA para viajar Extracci=C3=B3n de efectivo en el exterior Tarjetas adicionales sin costo Es necesario un ingreso m=C3=ADnimo mensual de $20.000 Conocer m=C3=A1s tarjeta gold mastercard Tarjeta Mastercard Gold L=C3=ADmites desde $100.000 Puntos BBVA para viajar Extracci=C3=B3n de efectivo en el exterior Tarjetas adicionales sin costo Es necesario un ingreso m=C3=ADnimo mensual de $35.000 Conocer m=C3=A1s Tarjeta Visa Internacional L=C3=ADmites desde $10.000 Puntos BBVA para viajar Extracci=C3=B3n de efectivo en el exterior Tarjetas adicionales sin costo Es necesario un ingreso m=C3=ADnimo mensual de $20.000 Conocer m=C3=A1s Tarjeta Mastercard Internacional L=C3=ADmites desde $10.000 Puntos BBVA para viajar Extracci=C3=B3n de efectivo en el exterior Tarjetas adicionales sin costo Es necesario un ingreso m=C3=ADnimo mensual de $20.000 Conocer m=C3=A1s --b1_3d217f30a568faa9ce3dd7dc73399561 Content-Type: text/html; charset="UTF-8" Content-Transfer-Encoding: quoted-printable --b1_3d217f30a568faa9ce3dd7dc73399561-- envelope_from miah.join@outlook.com envelope_to hello@stalw.art helo_domain HK2PR02CU002.outbound.protection.outlook.com iprev.result pass dkim.result pass dkim.domains outlook.com spf.result pass spf_ehlo.result pass dmarc.result pass dmarc.policy reject remote_ip 52.103.64.5 tls.version TLS1_2 expect_header X-Spam-Result: DMARC_POLICY_ALLOW (-0.50), DKIM_ALLOW (-0.20), SPF_ALLOW (-0.20), ARC_NA (0.00), ARC_SIGNED (0.00), DKIM_SIGNED (0.00), FREEMAIL_FROM (0.00), FROM_EQ_ENV_FROM (0.00), FROM_HAS_DN (0.00), HAS_SEO_WORD (0.00), HAS_X_PRIO_ONE (0.00), HTML_SHORT_1 (0.00), MID_RHS_MATCH_ENV_FROMTLD (0.00), MID_RHS_MATCH_FROMTLD (0.00), RCPT_COUNT_ONE (0.00), RCPT_IN_BODY (0.00), RCVD_COUNT_TWO (0.00), TO_DN_EQ_ADDR_ALL (0.00), TO_MATCH_ENVRCPT_ALL (0.00), RCVD_NO_TLS_LAST (0.10), HELO_NORES_A_OR_MX (0.30), DATE_IN_PAST (1.00), FROMHOST_NORES_A_OR_MX (1.50), SEO_SPAM (5.00) expect_header X-Spam-Score: spam, score=7.00 Return-Path: ARC-Seal: i=1; a=rsa-sha256; s=arcselector10001; d=microsoft.com; cv=none; b=sTW55J00fLHM5CSFAdYk6Kpyecib7sSXQWU51a+Eo6514pesoEtpNxM3eYurQfYQY7j+MMcwJ50u9fzJPOUm0JInaQMoDrUWJ5dObEglZtxbN1fpwHLOOP5rjWm+zd9p02jLCCpvoHnu4rIZmog1MO/pCiVRMWemUMzJ2O7mk2zbmode8ryb9tT1ho8XNeCYK9zKmoHwCl2p6TjO4HFQ4SU2hYIWd3//6gfnPDN2qIOgw6Z51zgsEtUYYENIKuHswZFWjt7925Wq380r5Fi+fsaKT8xAWFTq9igFNWKDVU2k7ZL6QlCsXpTRS57rrl1dBYAod1byHHbCOqa+g+VOAA== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=microsoft.com; s=arcselector10001; h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-AntiSpam-MessageData-ChunkCount:X-MS-Exchange-AntiSpam-MessageData-0:X-MS-Exchange-AntiSpam-MessageData-1; bh=SVZW4LFWvuTy4mbM+Q+E+yH3a0DZrHFN3U3wHGbAHMQ=; b=NavrW7s0URdfDuEFuzkxV7EUwJtiynvH1o9mzF39USQEfd9l1KyQpzOxo9Po8Dar1qqa/4ECNeUSmetx+NBDArmlTpBak+BKYXAXVRlHheyxILyU/f0RX01+7aifIzLj7LWv7Sx66b9D9/DjaVDbtMvFGPFUzk0JtiATahe7ZU0iKBvsRbGJjS9r0Sq2vHY/SQEUxOxKXUhUQBepSf9k7ibBZK27OhSz9v/jjSDCL/mh5MoOgbq7S8lbxUGS356c/Rm3ZWEInIRcbVqI+P75abEvzRNhRyBDId74h9IZnv+wz9QfGnk8TaFAExRBJ5BzIKlDibTZ+Kzuc+7mvOAKLw== ARC-Authentication-Results: i=1; mx.microsoft.com 1; spf=none; dmarc=none; dkim=none; arc=none DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=outlook.com; s=selector1; h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck; bh=SVZW4LFWvuTy4mbM+Q+E+yH3a0DZrHFN3U3wHGbAHMQ=; b=Q6QpEIbyyvkSgmfTsPeVdPuTyh+6lA/+qAoEm5k5gEDuyqmLjwVELDsOJQAZzwfQfmxN02O5dpbD0mDWKLFR7Ft//121jF9EV06fbGMNXuuBpfZJ24npu+bPbHs66D7USSMEE6zvuf4bnlhVV0iTTxWwhNEawPfaFpuukvlVO1GtPOjH0SeymOnfHM3LrGSkwYpw5aeEGjrJLFQRSN+k8mD7PyoOkJFFBUyqySWdkRsQ5aw9+7f3wbHbDOb4rqkmkC6fUZSMcqpTSpFFS3fDlQQrcnwhh8ir/tq74AuVYyMoUMns81tExoILI78twEHxyGN2zgLw7K9QojJCf+IIEQ== Received: from SEYPR04MB7496.apcprd04.prod.outlook.com (2603:1096:101:1db::7) by PUZPR04MB6246.apcprd04.prod.outlook.com (2603:1096:301:ec::5) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.8293.20; Tue, 31 Dec 2024 08:24:54 +0000 Received: from SEYPR04MB7496.apcprd04.prod.outlook.com ([fe80::1aab:cf90:44d8:af4e]) by SEYPR04MB7496.apcprd04.prod.outlook.com ([fe80::1aab:cf90:44d8:af4e%6]) with mapi id 15.20.8293.000; Tue, 31 Dec 2024 08:24:54 +0000 From: Miah Join To: "hello@stalw.art" Subject: SEO-Inquiry Thread-Topic: SEO-Inquiry Thread-Index: AdtbXSTUEOZ2HAOVQCqvPiaCPhJ+xw== Importance: high X-Priority: 1 Sensitivity: private Date: Tue, 31 Dec 2024 08:24:22 +0000 Message-ID: Accept-Language: en-US Content-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: x-ms-publictraffictype: Email x-ms-traffictypediagnostic: SEYPR04MB7496:EE_|PUZPR04MB6246:EE_ x-ms-office365-filtering-correlation-id: d8480a0c-4e45-4278-ad1b-08dd29749a89 x-ms-exchange-slblob-mailprops: SoURN12IA8vYLoDX/njsxmIBKrA5xww7SXj4BirxWi/11KcMwu1z/THqGhGNTFhVj0W89DLwoSD+IqZucSTc96MoKe9ia4xsvj73oWgcR/6suK5u7GoMSUHmwhrgVtaAkaceSYHenT+iTqqsr6L31R3bTGzhR1r7TEKkTLaJO6pXyfeXy8wUEdMEUOpfggfX/BBwA6KLTGvIE9euGA810m3+uPbUX+cE3WNT2yFsiC+H/YFGbcISQcbwlqT/7fAjiNT6fore4X1HnccC1CzArvmHrHF37pjc/JTEQFUxg7woH0GKJtvsUFbDUPvXjKp9rTRqLHTb8fYVSRdeivWB4lJocoW5VqJ9YichWUGp1ELEDuto7T/ourT4i6Q/Z+D+3/Q67J0SAdD8+XAXHfYKG2rq14F+tz4KxbO6EPp7O6HpaHMSbpzps5/sOJ2gmeHucnICmy81mKE9aq2sfJPPdymPBczv4hB7QO+xQDHFNpypNedZ8cTbtW5MlVn18lgmP0BuDBEgU4WcbKu2bMzgl6HJRQctSSldfkkyfGI/URoo69SEmOZiDuuCzhKQQ09ho6xRG3kpsD9DyRAUQyyo3Wq4J1UgLuRyRVFYwPI4zL+e47La1ilj199UD/SJrbHMMTt6itlRBpnH4N7Ev1ELztRfH5VjQ5aG x-microsoft-antispam: BCL:0;ARA:14566002|8060799006|7092599003|8062599003|5062599005|15080799006|461199028|19110799003|440099028|3412199025|102099032; x-microsoft-antispam-message-info: =?us-ascii?Q?sSaF+W5EA6WGixAhyYLVcDu/rxF60TPo55Pe5lDbuGq4SQKPFczK5OPYJQHf?= =?us-ascii?Q?6w0tFaoHE+b0IsosJe0H/dJXTmVvChLHdVnaTZiPcDKrZVOcLyvm2WnEoemj?= =?us-ascii?Q?m21GgioEUF/NQdwaXt6AAYv18zt1GxKufXRYIpLkmKM8gV1XQMXPJteH53ax?= =?us-ascii?Q?vifrj3RXruNdM31oe0LrZdIbsbKuvKDSdByf1mB2PVfCgEXIjvfFLgJidnah?= =?us-ascii?Q?d30fWN6LcYhQvtVNfpiJwtroUR5oox/HxRtDbU+fkvn0LLoFX7nxdECueURA?= =?us-ascii?Q?/67CsMYT5/NKrE+oaYmPjDYkLmS+YDBmWdUyuQDJIQ0Lb7vKisWAOFrweDgc?= =?us-ascii?Q?pJUyUtAA5UliC4AO+gI+AH3cv2JaMn+Z7gCf9WpcTCk8PkOEtg4Y1661t4T3?= =?us-ascii?Q?rOHV1Q7ibkz6JYCiIVGl1macRWWcIl1GeUWLUiHduVR/ax3PLUYLHYblCHv0?= =?us-ascii?Q?XVrSCAAa8+vce6+c7ymVd8uBAvfTT/hfYKwSGR2R6V9clhEDjE5TXsIjx1qc?= =?us-ascii?Q?AsCWjvexOp/pWalj3jwJzOE/xKHehhzyIeOJ0kN8snZBJiZpD9VnhYE6Lt8I?= =?us-ascii?Q?4ttzIth0+B9mw24tVTPg2cZ3N9yFsznBvKxtuFYl3Oj3YzacJVbwk+XyRsjQ?= =?us-ascii?Q?LXmGP3d8NPm2Nhy0yfrt4/wxf2th/e+/I3siWDuHcwJiDFdhvoT/AOuUoFl9?= =?us-ascii?Q?5Vs6Amo3CmVo4TGEf+EQNPoLsXjVAMzpk/5TgnKQ2sanXVv1zPGrTT7aErrv?= =?us-ascii?Q?m2fE/VhGQOqKiizPYaMaC6W4YPdrAYOOrpZFOgzFfMoNmyDmIfIucke8DJTg?= =?us-ascii?Q?K630mHxneVrofGWREkohZiDUe/tQsjp/0Hy2x2792Pg2aumt5nc1Aw/QJrwr?= =?us-ascii?Q?VHaKD7O2hL1QLJFbojSSKlWyGQ/CE0DsyUVihC0dWqfdWa8gPlHZ3uzzMN5J?= =?us-ascii?Q?BazCEPgYBVnmiQw/ejBmBpm5iKIA0zgNKGva/8tDikzhviHpCe9lijBuNsu3?= =?us-ascii?Q?OIfW8bDcJpyKB+g5S7EVo9K+lwYq0i86di3dpDvmSBfB99PFK9kqzaAEfAIU?= =?us-ascii?Q?Duh11rVb5jqL5hm7Zhsb9T2kGOiAorAjmrAHQFSyCNO910vF+8s=3D?= x-ms-exchange-antispam-messagedata-chunkcount: 1 x-ms-exchange-antispam-messagedata-0: =?us-ascii?Q?9c/kgRHXQSmmVIMMaHuipN3QMSRVkf870g3+luQav+fyRaGeQw2TxAqSMXIQ?= =?us-ascii?Q?MqoXTcYcUTW8oy80k0fPln9Hz61fG51WDZ2vzG1wnYvfHer1zkAdp/njN/D2?= =?us-ascii?Q?jXEtZZP4w7Fi1Tf3oyRzBLQhMtORyV3RQU/uWIRpjvP/jlu3zfVeIyFtON/q?= =?us-ascii?Q?oJqPzYVyxeLv832rNiqEVsAPLOar26932L6xXlandRXwk6WZTN7J76uUF+i0?= =?us-ascii?Q?FpKfqvx2IHOT/Qc8dJLf3H/iBywKPBINfmUdTdX83EzX9DYFmCJ0mJHhgmFV?= =?us-ascii?Q?2r/JQEN6c+ziSN/sqFBrNvpWaEUGzyB7k4HT+HV9bv/8nx0jVYVdb8Jg/Lxl?= =?us-ascii?Q?nt9FxkqJPCdTjYYQwCzM+74raMHz5o3URpFYGfEYsgBBrqZPa6oNgxZBrLU6?= =?us-ascii?Q?w/m7VsQI9AbLh/jteycxh6INOvPy6SS3m7+FGFpFjwNAfSQLEyghWO0hyyk3?= =?us-ascii?Q?75mJwXHJ53u+J6jMQQPEz6h2SFHN64nrGRbPq6ruSdyEhSj9BdkuEnKb/yAR?= =?us-ascii?Q?07pJ7iBfQVTfQ04xXejXErXvrDZGCcVy3/1AOajGLmcW1iTgoRwF3KKTmKct?= =?us-ascii?Q?g+/IaxelPpT95uCPhSWREdVLoxuQW6pBltFeXod8ou3YpiOoss2h0EplB4Xa?= =?us-ascii?Q?TKPKZSDdPBeCbOwx9GQUhySu57WwarR4Q74+MQwx8zDHdlytk9bLFZrCSCnt?= =?us-ascii?Q?OixF+n8R+5U1D+Vub74mpSguv+2efQNt2lwJei3iDd1mC6WRXdGaC4+WdCEt?= =?us-ascii?Q?uPRtELZCDsLSUE+097ixYl7uMLCe8nHUFuECfu41T1yX9PPMEmpadpxazQCd?= =?us-ascii?Q?USBa0u5BAMvXiZJHfPgf3y255JPI079k+DdhGkhF2cDwLciCHELy5rM52TGT?= =?us-ascii?Q?IrCNmfxb/RmOWBVEsvbhV4glTLJlORemnfACjPfh8SzldzzIj2W8zUUrxJQt?= =?us-ascii?Q?6bdoxj9FbxsJ3wvYQTDqF9olDDPh/2z1g28uXkknEgvcdU/XvlzE+bz3ACnC?= =?us-ascii?Q?yoxmdm3P7/9aIojcU9CeOsnQBDHgBYOxXIK5vNogdrZV9Ew4G2w/gOevLmvA?= =?us-ascii?Q?HAbDE1IaI2hhX0zhg2D/QCZZNpvlDgJzESkffQNSOYsIWlHeWBzOJeq7iM0Y?= =?us-ascii?Q?pXEltktrRe3doxEsnAYKPirbULWz4u+XJXzAOddWBWiYGNi6ua5wuyv7pyRX?= =?us-ascii?Q?CMc1g/sPb+5aGiPpbK4Si7KWv6HpNO3v0ACS0qrmdaaPWSx+wL8tsxO4KBI?= =?us-ascii?Q?=3D?= Content-Type: multipart/alternative; boundary="_000_SEYPR04MB74966F6A34B2B0AC5DA8E6DDFD0A2SEYPR04MB7496apcp_" MIME-Version: 1.0 X-OriginatorOrg: outlook.com X-MS-Exchange-CrossTenant-AuthAs: Internal X-MS-Exchange-CrossTenant-AuthSource: SEYPR04MB7496.apcprd04.prod.outlook.com X-MS-Exchange-CrossTenant-RMS-PersistedConsumerOrg: 00000000-0000-0000-0000-000000000000 X-MS-Exchange-CrossTenant-Network-Message-Id: d8480a0c-4e45-4278-ad1b-08dd29749a89 X-MS-Exchange-CrossTenant-originalarrivaltime: 31 Dec 2024 08:24:22.5882 (UTC) X-MS-Exchange-CrossTenant-fromentityheader: Hosted X-MS-Exchange-CrossTenant-id: 84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa X-MS-Exchange-CrossTenant-rms-persistedconsumerorg: 00000000-0000-0000-0000-000000000000 X-MS-Exchange-Transport-CrossTenantHeadersStamped: PUZPR04MB6246 --_000_SEYPR04MB74966F6A34B2B0AC5DA8E6DDFD0A2SEYPR04MB7496apcp_ Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable Hello, hello@stalw.art I'm just checking with you to see if you're interested in (SEO) search engi= ne optimization, or if you're interested in Google 1st page for better busi= ness. If so, I'd love to tell you a little bit more about my abilities and show y= ou some of my work. I am a very Skilled SEO expert with various abilities and can (1st Page on Google). any= thing. I look forward to hearing from you. Thanks, Miah --_000_SEYPR04MB74966F6A34B2B0AC5DA8E6DDFD0A2SEYPR04MB7496apcp_ Content-Type: text/html; charset="us-ascii" Content-Transfer-Encoding: quoted-printable

Hello, hello@stalw.art  = ;    

 

I’m just checking with= you to see if you’re interested in (SEO) search engi= ne optimization, or if you’re interes= ted in Google 1st page for better business. 

 

If so, I’d love to tel= l you a little bit more about my abilities and show you some of my work. I = am a very 

Skilled SEO expert with vari= ous abilities and can (1st Page on Google). anything. 

 

I look forward to hearing fr= om you. 

 

Thanks,

Miah

--_000_SEYPR04MB74966F6A34B2B0AC5DA8E6DDFD0A2SEYPR04MB7496apcp_-- envelope_from marketing@landeray.com envelope_to hello@stalw.art helo_domain terminal4.landeray.com iprev.result pass dkim.result pass dkim.domains outlook.com spf.result pass spf_ehlo.result pass dmarc.result pass dmarc.policy reject remote_ip 173.224.123.255 tls.version TLS1_2 expect_header X-Spam-Result: DMARC_POLICY_ALLOW (-0.50), DKIM_ALLOW (-0.20), SPF_ALLOW (-0.20), ARC_NA (0.00), DKIM_SIGNED (0.00), FROM_EQ_ENV_FROM (0.00), FROM_HAS_DN (0.00), HAS_EXTERNAL_IMG (0.00), HAS_REPLYTO (0.00), HAS_X_PRIO_THREE (0.00), HTML_SHORT_1 (0.00), RCPT_COUNT_ONE (0.00), REPLYTO_DN_EQ_FROM_DN (0.00), REPLYTO_DOM_EQ_FROM_DOM (0.00), TO_DN_ALL (0.00), TO_EQ_FROM (0.00), RCVD_COUNT_ZERO (0.10), RCVD_NO_TLS_LAST (0.10), HELO_NORES_A_OR_MX (0.30), MID_RHS_NOT_FQDN (0.50), UNPARSABLE_URL (0.50), DATE_IN_PAST (1.00), FROMHOST_NORES_A_OR_MX (1.50), DIRECT_TO_MX (2.00), FORGED_RECIPIENTS (2.00), SUBJ_ALL_CAPS (3.00) expect_header X-Spam-Score: spam, score=10.10 Return-Path: DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; s=default; d=landeray.com; h=Message-ID:Reply-To:From:To:Subject:Date:MIME-Version:Content-Type; i=marketing@landeray.com; bh=nlE2QLcR2WpTkMQ4zUwkF7IBIw4eOeAhkKS/HXiMvSs=; b=EwWEML/WtTA6nI6CuagI2LRBWAZuRHk6IJiDNA3R77dTpa80gHPkfzF3NBcKcVIiCYcf6i4HCniZ bF4DT8VZrFfLCSykeL2t5FLjYwFcjXjX7qCvyaR0hLfyFRSCDO9RytMw8Q3WoODc/jMXWrdxYpqu phP9D5Q/N8vzSneReGOCV62Me1ss947uc43zYZXvVwuXeyLbU0yPzJtTTEODFXmVp2Ul/M7w963E UxzrNlTd/lEIUXVvxAOL2DKGxY8oXslevI0euO8E8TQms3SlY6LySkDia1h+N+mVpbPPeE3txZhS LJwn2MBX59b+W3aMafAt4Ae/UVOOBlMT7FhhvA== Message-ID: Reply-To: "RESERVAS Y CONSULTAS" From: "RESERVAS Y CONSULTAS" To: "Marketing Online" Subject: BRISZA ASIA TEMPORADA 2025 Date: Fri, 3 Jan 2025 14:06:03 +0000 Organization: CLIENTE-DISCOTECA MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="----=_NextPart_000_0E52_01DB5DE8.9FAFDC60" X-Priority: 3 X-MSMail-Priority: Normal X-Mailer: Microsoft Outlook Express 6.00.2900.5512 X-MimeOLE: Produced By Microsoft MimeOLE V6.00.2900.5579 This is a multi-part message in MIME format. ------=_NextPart_000_0E52_01DB5DE8.9FAFDC60 Content-Type: text/plain; charset="iso-8859-1" Content-Transfer-Encoding: quoted-printable Consulta v=EDa WhatsApp: https://api.whatsapp.com/send?phone=3D5199504167= 6&text=3DDeseo realizar una reserva en BRISZA Si no puedes visualizar la imagen haz click aqu=ED https://i.imgur.com/O1= 2POZ2.png Si esta informaci=F3n no es de su inter=E9s. Favor de escribirnos un corr= eo en blanco a eliminarcorreo2016@gmail.com con el Asunto "REMOVER". Grac= ias. ------=_NextPart_000_0E52_01DB5DE8.9FAFDC60 Content-Type: text/html; charset="iso-8859-1" Content-Transfer-Encoding: quoted-printable
Consulta v=EDa WhatsApp: https://api.whatsapp.com/send?phone=3D51995= 041676&text=3DDeseo=20 realizar una reserva en BRISZA
 
 
Si no puedes visualizar la image= n haz click=20 aqu=ED https://i.imgur.com/O12POZ2.png<= /A>
 
3D""=20
 
Si esta informaci= =F3n no es de=20 su inter=E9s. Favor de escribirnos un correo en blanco a eliminarcorreo2016@gmail.com con el Asunto "REMOVER".=20 Gracias.
------=_NextPart_000_0E52_01DB5DE8.9FAFDC60-- ================================================ FILE: tests/resources/smtp/antispam/date.test ================================================ expect MISSING_DATE X-Date: Tue, 1 Jul 2003 10:52:37 +0200 Test expect INVALID_DATE Date: blah blah blah Test expect DATE_IN_PAST Date: Tue, 1 Jul 2003 10:52:37 +0200 Test expect DATE_IN_FUTURE Date: Tue, 1 Jul 2999 10:52:37 +0200 Test ================================================ FILE: tests/resources/smtp/antispam/dmarc.test ================================================ expect DMARC_NA SPF_NA DKIM_NA ARC_NA AUTH_NA Subject: test Test spf.result pass dkim.result pass arc.result pass dmarc.result pass expect DKIM_SIGNED ARC_SIGNED DKIM_ALLOW SPF_ALLOW ARC_ALLOW DMARC_POLICY_ALLOW DKIM-Signature: abc ARC-Seal: xyz Subject: test Test spf.result fail dkim.result fail arc.result fail dmarc.result fail dmarc.policy quarantine expect SPF_FAIL ARC_REJECT DKIM_REJECT DMARC_POLICY_QUARANTINE Subject: test Test spf.result neutral dkim.result temperror arc.result permerror dmarc.result fail dmarc.policy reject expect DKIM_TEMPFAIL SPF_NEUTRAL ARC_INVALID DMARC_POLICY_REJECT Subject: test Test spf.result softfail dkim.result permerror arc.result temperror dmarc.result permerror expect ARC_DNSFAIL DMARC_BAD_POLICY DKIM_PERMFAIL SPF_SOFTFAIL Subject: test Test dkim.result pass dkim.domains spf-dkim-allow.org spf.result pass expect DKIM_ALLOW SPF_ALLOW ARC_NA DMARC_NA From: user@spf-dkim-allow.org Subject: test Test dkim.result pass spf.result pass arc.result pass expect DKIM_ALLOW SPF_ALLOW ARC_ALLOW DMARC_NA From: user@spf-dkim-allow.org Subject: test Test spf.result pass dkim.result fail expect DKIM_REJECT SPF_ALLOW ARC_NA DMARC_NA From: user@spf-dkim-allow.org Subject: test Test spf.result pass dkim.result temperror expect DKIM_TEMPFAIL SPF_ALLOW ARC_NA DMARC_NA From: user@spf-dkim-allow.org Subject: test Test dkim.result pass dkim.domains spf-dkim-allow.org spf.result fail expect DKIM_ALLOW SPF_FAIL ARC_NA DMARC_NA From: user@spf-dkim-allow.org Subject: test Test dkim.result pass dkim.domains spf-dkim-allow.org spf.result temperror expect DKIM_ALLOW SPF_DNSFAIL ARC_NA DMARC_NA From: user@spf-dkim-allow.org Subject: test Test dkim.result fail spf.result fail expect DKIM_REJECT SPF_FAIL ARC_NA DMARC_NA From: user@spf-dkim-allow.org Subject: test Test dkim.result temperror spf.result temperror expect DKIM_TEMPFAIL SPF_DNSFAIL ARC_NA DMARC_NA AUTH_NA_OR_FAIL From: user@spf-dkim-allow.org Subject: test Test spf.result pass dkim.result pass arc.result pass dmarc.result pass envelope_from hello@stalw.art expect TRUSTED_DOMAIN DMARC_POLICY_ALLOW DKIM_ALLOW SPF_ALLOW ARC_ALLOW From: Test ================================================ FILE: tests/resources/smtp/antispam/from.test ================================================ expect MISSING_FROM X-From: hello@domain.org Test envelope_from hello@domain.org expect MULTIPLE_FROM FROM_EQ_ENV_FROM FROM_NO_DN From: hello@domain.org From: hello@domain.org Test envelope_from test expect FROM_INVALID ENV_FROM_INVALID From: test Test envelope_from www-data@domain.org expect FROM_SERVICE_ACCT FROM_HAS_DN FROM_EQ_ENV_FROM From: "WWW DATA" Test envelope_from hello@domain.org expect FROM_DN_EQ_ADDR FROM_EQ_ENV_FROM From: "hello@domain.org" Test envelope_from hello@domain.org expect SPOOF_DISPLAY_NAME FROM_EQ_ENV_FROM FROM_HAS_DN From: "hello@otherdomain.org" Test envelope_from hello@domain.co.uk expect FROM_NEQ_DISPLAY_NAME FROM_EQ_ENV_FROM FROM_HAS_DN From: "hello@other.domain.co.uk" Test helo_domain mx.domain.co.uk expect FROMTLD_EQ_ENV_FROMTLD FROM_NEQ_DISPLAY_NAME FROM_HAS_DN FROM_BOUNCE From: "postmaster@mx.domain.co.uk" Test helo_domain mx.domain.co.uk expect FROMTLD_EQ_ENV_FROMTLD FROM_HAS_DN FROM_BOUNCE From: "Mailer Daemon" Test envelope_from mrspammer@domain.org expect FROM_NAME_HAS_TITLE FROM_NAME_EXCESS_SPACE FROM_EQ_ENV_FROM FROM_HAS_DN From: "Mr. Money Maker" Test envelope_from hello+world@domain.org expect TAGGED_FROM FROM_EQ_ENV_FROM FROM_NO_DN From: hello+world@domain.org Test envelope_from hello@domain.org expect TO_EQ_FROM FROM_EQ_ENV_FROM FROM_NO_DN From: hello@domain.org To: hello@domain.org Test envelope_from hello@domain.org expect FROM_EQ_ENV_FROM FROM_NO_DN From: hello@domain.org To: hello@domain.org, bye@domain.org Test envelope_from hello@domain.org expect FROM_NEEDS_ENCODING FROM_EQ_ENV_FROM FROM_HAS_DN From: "Hélló" Test param.smtputf8 1 envelope_from hello@domain.org expect FROM_EQ_ENV_FROM FROM_HAS_DN From: "Hélló" Test envelope_from hello@domain.org expect FROM_EXCESS_QP FROM_EQ_ENV_FROM FROM_HAS_DN From: =?iso-8859-1?Q?Die_Hasen_und_die_Froesche?= Test envelope_from hello@domain.org expect FROM_EXCESS_BASE64 FROM_EQ_ENV_FROM FROM_HAS_DN From: "=?iso-8859-1?B?RGllIEhhc2VuIHVuIGRpZSBGcm9lc2NoZQ==?=" Test envelope_from hello@domain.org expect FROM_EQ_ENV_FROM FROM_HAS_DN From: "=?iso-8859-1?Q?Die_Hasen_und_die_Fr=F6sche?=" Test envelope_from hello@domain.org expect NO_SPACE_IN_FROM FROM_EQ_ENV_FROM FROM_HAS_DN From: "Hello" Test envelope_from hello@domain.org expect FROM_EQ_ENV_FROM FROM_HAS_DN From: "Hello" Test envelope_from hello@domain.org expect HEADER_RCONFIRM_MISMATCH FROM_EQ_ENV_FROM FROM_HAS_DN From: "Hello" X-Confirm-Reading-To: Test envelope_from hello@domain.org expect HEADER_FORGED_MDN FROM_EQ_ENV_FROM FROM_HAS_DN From: "Hello" Disposition-Notification-To: Test envelope_from anonymous@domain.org expect FROM_SERVICE_ACCT WWW_DOT_DOMAIN FROM_EQ_ENV_FROM FROM_HAS_DN From: "Hello" Reply-to: Test envelope_from hello@custom.disposable.org expect FREEMAIL_FROM DISPOSABLE_ENV_FROM FROM_NEQ_ENV_FROM FROM_NO_DN FORGED_SENDER From: hello@gmail.com Test envelope_from hello@gmail.com expect DISPOSABLE_FROM FREEMAIL_ENV_FROM FROM_NEQ_ENV_FROM FROM_NO_DN FORGED_SENDER From: hello@custom.disposable.org Test envelope_from hello@nomx.org expect FROMHOST_NORES_A_OR_MX FROM_EQ_ENV_FROM FROM_NO_DN From: hello@nomx.org Test envelope_from baz@domain.org expect SPOOF_DISPLAY_NAME FROM_HAS_DN FROM_EQ_ENV_FROM From: "Foo (foo@bar.com)" Test envelope_from baz@domain.org expect SPOOF_DISPLAY_NAME FROM_HAS_DN FROM_EQ_ENV_FROM From: Foo (foo@bar.com) Test envelope_from baz@domain.org expect SPOOF_DISPLAY_NAME FROM_HAS_DN FROM_EQ_ENV_FROM From: "Foo foo@bar.com" Test envelope_from baz@domain.org expect SPOOF_DISPLAY_NAME FROM_HAS_DN FROM_EQ_ENV_FROM From: "Foo 'foo@bar.com'" Test ================================================ FILE: tests/resources/smtp/antispam/headers.test ================================================ expect HAS_X_PRIO_ONE X-Priority: 1 From: test@test.com To: test@test.com Test expect MULTIPLE_UNIQUE_HEADERS HAS_X_PRIO_TWO X-Mailer: my mailer 1 X-Priority: 2 From: test@test.com From: test@test.com To: test@test.com Test expect XM_CASE HAS_LIST_UNSUB PRECEDENCE_BULK MULTIPLE_UNIQUE_HEADERS X-mailer: my mailer 1 List-Unsubscribe: Precedence: bulk Subject: first subject Subject: second subject Test expect KLMS_SPAM UNITEDINTERNET_SPAM SPAM_FLAG XM_UA_NO_VERSION X-Mailer: my mailer X-KLMS-AntiSpam-Status: spam X-Spam: Yes X-UI-Filterresults: JUNK Subject: test Test expect X_PHP_EVAL HIDDEN_SOURCE_OBJ HAS_X_GMSV HAS_X_AS X-PHP-Script: sendmail.php X-PHP-Originating-Script: eval() X-Source-Args: ../script X-Authenticated-Sender: sender: test@test.org X-Get-Message-Sender-Via: authenticated_id: 123 X-AntiAbuse: 1 X-Authentication-Warning: 1 Subject: test Test expect HEADER_EMPTY_DELIMITER Subject:test Test expect Subject: test Test expect MAILLIST List-Archive: 1 List-Owner: 1 List-Help: 1 List-Post: 1 X-Loop: 1 List-Id: 1 Subject: test Test expect MAILLIST HAS_LIST_UNSUB List-Id: 1 List-Subscribe: 1 List-Unsubscribe: 1 Subject: test Test expect MISSING_ESSENTIAL_HEADERS X-Other: test Test ================================================ FILE: tests/resources/smtp/antispam/helo.test ================================================ helo_domain localhost expect HELO_NOT_FQDN Subject: test test helo_domain user expect RCVD_HELO_USER HELO_NOT_FQDN Subject: test test helo_domain 8.8.8.8 remote_ip 8.8.8.8 expect HELO_BAREIP Subject: test test helo_domain 8.8.8.8 remote_ip 1.1.1.1 expect HELO_IP_A HELO_BAREIP Subject: test test helo_domain domain.org iprev.ptr domain.org remote_ip 1.1.1.1 expect Subject: test test helo_domain domain.org iprev.ptr otherdomain.org remote_ip 1.1.1.1 expect HELO_IPREV_MISMATCH Subject: test test helo_domain otherdomain.org iprev.ptr otherdomain.org remote_ip 1.1.1.1 expect HELO_NORES_A_OR_MX Subject: test test helo_domain otherdomain.org iprev.ptr otherdomain.net remote_ip 1.1.1.1 expect HELO_NORES_A_OR_MX HELO_IPREV_MISMATCH Subject: test test ================================================ FILE: tests/resources/smtp/antispam/html.test ================================================ expect MIME_HTML_ONLY HTML_SHORT_1 Message-Id: <4.2.0.58.20000519002557.00a88870@pop.example.com> X-Sender: dwsauder@pop.example.com (Unverified) X-Mailer: QUALCOMM Windows Eudora Pro Version 4.2.0.58 X-Priority: 2 (High) Date: Fri, 19 May 2000 00:29:55 -0400 To: Heinz =?iso-8859-1?Q?M=FCller?= From: Doug Sauder Subject: =?iso-8859-1?Q?Die_Hasen_und_die_Fr=F6sche?= Mime-Version: 1.0 Content-Type: text/html; charset="iso-8859-1" Content-Transfer-Encoding: quoted-printable Die Hasen und = die Fr=F6sche

Die Hasen klagten einst =FCber ihre mi=DFliche Lage; "wir leben", sprach ein Redner, "in steter Furcht vor Menschen und Tieren, eine Beute der Hunde, der Adler, ja fast aller Raubtiere! Unsere stete Angst ist =E4rger als der Tod selbst. Auf, la=DFt uns ein f=FCr allemal sterben."

In einem nahen Teich wollten sie sich nun ers=E4ufen; sie eilten ihm zu; allein das au=DFerordentliche Get=F6se und ihre wunderbare Gestalt erschreckte eine Menge Fr=F6sche, die am Ufer sa=DFen, so sehr, da=DF sie au= fs schnellste untertauchten.

"Halt", rief nun eben dieser Sprecher, "wir wollen das Ers=E4ufen noch ein wenig aufschieben, denn auch uns f=FCrchten, wie ihr seht, einige Tiere, welche also wohl noch ungl=FCcklicher sein m=FCssen als wir."

expect HTTP_TO_HTTPS HTML_SHORT_1 Content-Type: multipart/alternative; boundary="=====================_714967308==_.ALT" --=====================_714967308==_.ALT Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable https://mydomain.com --=====================_714967308==_.ALT Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 8bit

Lorem ipsum dolor sit amet, consectetur adipiscing elit.

https://mydomain.com --=====================_714967308==_.ALT-- expect HTTP_TO_IP HTML_SHORT_1 Content-Type: multipart/alternative; boundary="=====================_714967308==_.ALT" --=====================_714967308==_.ALT Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable https://mydomain.com --=====================_714967308==_.ALT Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 8bit

some text

https://8.8.8.8 --=====================_714967308==_.ALT-- expect EXT_CSS HTML_SHORT_1 Content-Type: multipart/alternative; boundary="=====================_714967308==_.ALT" --=====================_714967308==_.ALT Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable https://mydomain.com --=====================_714967308==_.ALT Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 8bit

some text

https://mydomain.com --=====================_714967308==_.ALT-- expect EXT_CSS HTML_SHORT_1 Content-Type: multipart/alternative; boundary="=====================_714967308==_.ALT" --=====================_714967308==_.ALT Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable https://mydomain.com --=====================_714967308==_.ALT Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 8bit

some text

https://mydomain.com --=====================_714967308==_.ALT-- expect HTML_UNBALANCED_TAG HTML_SHORT_1 Content-Type: multipart/alternative; boundary="=====================_714967308==_.ALT" --=====================_714967308==_.ALT Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable https://mydomain.com --=====================_714967308==_.ALT Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 8bit hello https://mydomain.com --=====================_714967308==_.ALT-- expect HTML_UNBALANCED_TAG HTML_SHORT_1 Content-Type: multipart/alternative; boundary="=====================_714967308==_.ALT" --=====================_714967308==_.ALT Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable https://mydomain.com --=====================_714967308==_.ALT Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 8bit hello https://mydomain.com --=====================_714967308==_.ALT-- expect HTML_SHORT_LINK_IMG_1 HTML_SHORT_1 HAS_LINK_TO_LARGE_IMG Content-Type: multipart/alternative; boundary="=====================_714967308==_.ALT" --=====================_714967308==_.ALT Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Testing --=====================_714967308==_.ALT Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 8bit Test

Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nullam

--=====================_714967308==_.ALT-- expect BODY_URI_ONLY HTML_SHORT_1 Content-Type: multipart/alternative; boundary="=====================_714967308==_.ALT" --=====================_714967308==_.ALT Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Testing --=====================_714967308==_.ALT Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 8bit Test

http://myurl.com --=====================_714967308==_.ALT-- expect HTML_TEXT_IMG_RATIO HTML_SHORT_1 Content-Type: multipart/alternative; boundary="=====================_714967308==_.ALT" --=====================_714967308==_.ALT Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Testing --=====================_714967308==_.ALT Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 8bit Test --=====================_714967308==_.ALT-- expect HTML_META_REFRESH_URL MIME_HTML_ONLY HTML_SHORT_1 Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 8bit

some text

expect MIME_HTML_ONLY HTML_SHORT_1 Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 8bit

some text

expect HAS_DATA_URI DATA_URI_OBFU MIME_HTML_ONLY HTML_SHORT_1 Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 8bit

some text

Click me for a hello message expect HAS_DATA_URI MIME_HTML_ONLY HTML_SHORT_1 Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 8bit

some text and a lovely explanation to avoid the text to image ratio tag

Red dot Click me for a hello message expect PHISHING MIME_HTML_ONLY HTML_SHORT_1 Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 8bit

some text

https://domain2.com/otherquery expect MIME_HTML_ONLY HTML_SHORT_1 Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 8bit

some text

https://subdomain.domain1.co.uk/otherquery expect PHISHING MIME_HTML_ONLY HTML_SHORT_1 Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 8bit

some text

domain2.com/otherquery expect MIME_HTML_ONLY HTML_SHORT_1 Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 8bit

some text

subdomain.domain1.co.uk/otherquery expect MIME_HTML_ONLY HTML_SHORT_1 Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 8bit

some text

normal text ================================================ FILE: tests/resources/smtp/antispam/ip.test ================================================ remote_ip 8.8.8.8 iprev.result temperror expect RDNS_DNSFAIL Subject: test Test remote_ip 8.8.8.8 iprev.result fail expect RDNS_NONE Subject: test Test ================================================ FILE: tests/resources/smtp/antispam/llm.test ================================================ expect LLM_UNSOLICITED_HIGH Subject: Unsolicited,High,Test Test expect LLM_COMMERCIAL_HIGH Subject: Commercial,High,Test Test expect LLM_HARMFUL_HIGH Subject: Harmful,High,Test Test expect LLM_LEGITIMATE_HIGH Subject: Legitimate,High,Test Test expect LLM_UNSOLICITED_MEDIUM Subject: Unsolicited,Medium,Test Test expect LLM_COMMERCIAL_MEDIUM Subject: Commercial,Medium,Test Test expect LLM_HARMFUL_MEDIUM Subject: Harmful,Medium,Test Test expect LLM_LEGITIMATE_MEDIUM Subject: Legitimate,Medium,Test Test expect LLM_UNSOLICITED_LOW Subject: Unsolicited,Low,Test Test expect LLM_COMMERCIAL_LOW Subject: Commercial,Low,Test Test expect LLM_HARMFUL_LOW Subject: Harmful,Low,Test Test expect LLM_LEGITIMATE_LOW Subject: Legitimate,Low,Test Test ================================================ FILE: tests/resources/smtp/antispam/messageid.test ================================================ expect MISSING_MID X-Message-ID: Test expect MID_RHS_IP_LITERAL Message-ID: Test expect MID_BARE_IP Message-ID: Test expect MID_RHS_NOT_FQDN Message-ID: Test expect MID_RHS_WWW Message-ID: Test expect INVALID_MSGID Message-ID: <@domain.com> Test expect INVALID_MSGID Message-ID: Test expect INVALID_MSGID Message-ID: (hello world) Test expect MID_RHS_TOO_LONG Message-ID: Test expect MID_MISSING_BRACKETS Message-ID: hello@domain.com Test expect MID_CONTAINS_FROM From: Message-ID: Test expect MID_RHS_MATCH_FROM From: Message-ID: Test expect MID_RHS_MATCH_FROMTLD From: Message-ID: <1234@host.domain.co.uk> Test envelope_from hello@domain.co.uk expect MID_RHS_MATCH_ENV_FROMTLD Message-ID: <1234@host.domain.co.uk> Test expect MID_CONTAINS_TO To: User Message-ID: Test expect MID_RHS_MATCH_TO From: Myself To: User Cc: John , Jane , Bill Message-ID: Test ================================================ FILE: tests/resources/smtp/antispam/mime.test ================================================ expect MISSING_MIME_VERSION SINGLE_SHORT_PART Content-Type: text/plain; charset="us-ascii" Test expect MV_CASE SINGLE_SHORT_PART Content-Type: text/plain; charset="us-ascii" Mime-Version: 1.0 Test expect CTE_CASE CT_EXTRA_SEMI SINGLE_SHORT_PART Content-Type: text/plain; charset="us-ascii"; Content-Transfer-Encoding: 7Bit MIME-Version: 1.0 Test expect BROKEN_CONTENT_TYPE SINGLE_SHORT_PART Content-Type: ; tag=1 Content-Transfer-Encoding: 7bit MIME-Version: 1.0 Test expect MIME_HEADER_CTYPE_ONLY MISSING_MIME_VERSION SINGLE_SHORT_PART Content-Type: text/html; charset="us-ascii" Test expect BAD_CTE_7BIT SINGLE_SHORT_PART Content-Type: text/plain Content-Transfer-Encoding: 7bit MIME-Version: 1.0 Téstíng expect MISSING_CHARSET SINGLE_SHORT_PART Content-Type: text/plain Content-Transfer-Encoding: 8bit MIME-Version: 1.0 Test expect MIME_BASE64_TEXT_BOGUS SINGLE_SHORT_PART Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: base64 MIME-Version: 1.0 aGVsbG8gd29ybGQK expect MIME_BASE64_TEXT SINGLE_SHORT_PART Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: base64 MIME-Version: 1.0 aMOpbGzDsyB3w7NybGQK expect MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="boundary" --boundary Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. --boundary Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 7bit

Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

--boundary-- expect MIME_MA_MISSING_TEXT MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="boundary" --boundary Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 7bit

Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

--boundary-- expect MIME_MA_MISSING_HTML MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="boundary" --boundary Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. --boundary-- expect PARTS_DIFFER MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="boundary" --boundary Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit Lorem ipsum dolor sit Ramet, Rcnsectetur Radipiscing elit, Rsed do Reiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. --boundary Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 7bit

Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

--boundary-- expect URI_COUNT_ODD MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="boundary" --boundary Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit Find me at http://www.example.com or http://www.example.org --boundary Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 7bit

Find me at http://www.example.com or

--boundary-- expect MIME-Version: 1.0 Content-Type: multipart/alternative; boundary="boundary" --boundary Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit Find me at http://www.example.com or http://www.example.org --boundary Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 7bit

Find me at http://www.example.com or http://example.org

--boundary-- expect CTYPE_MIXED_BOGUS MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="boundary" --boundary Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit this is a test --boundary Content-Type: text/html; charset="utf-8" Content-Transfer-Encoding: 7bit

this is a test

--boundary-- expect MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="boundary" --boundary Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit this is a test --boundary Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit another test --boundary Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit last test --boundary-- expect HAS_ATTACHMENT MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="boundary" --boundary Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit this is a test --boundary Content-Type: application/octet-stream Content-Disposition: attachment Content-Transfer-Encoding: 7bit

this is a test

--boundary-- expect CTYPE_MISSING_DISPOSITION HAS_ATTACHMENT SINGLE_SHORT_PART Content-Type: application/octet-stream MIME-Version: 1.0 Test expect ENCRYPTED_PGP ENCRYPTED_SMIME SIGNED_PGP SIGNED_SMIME HAS_ATTACHMENT MIME-Version: 1.0 Content-Type: multipart/encrypted; boundary="boundary" --boundary Content-Type: application/pkcs7-mime Content-Transfer-Encoding: 7bit this is a test --boundary Content-Type: application/pkcs7-signature Content-Transfer-Encoding: 7bit this is a test --boundary Content-Type: application/pgp-encrypted Content-Transfer-Encoding: 7bit this is a test --boundary Content-Type: application/pgp-signature Content-Transfer-Encoding: 7bit this is a test --boundary Content-Type: application/octet-stream Content-Transfer-Encoding: 7bit this is a test --boundary-- expect CTYPE_MISSING_DISPOSITION HAS_ATTACHMENT SINGLE_SHORT_PART Content-Type: application/octet-stream MIME-Version: 1.0 Test expect BOGUS_ENCRYPTED_AND_TEXT ENCRYPTED_SMIME MIME-Version: 1.0 Content-Type: multipart/encrypted; boundary="boundary" --boundary Content-Type: application/pkcs7-mime Content-Transfer-Encoding: 7bit this is a test --boundary Content-Type: text/html Content-Transfer-Encoding: 7bit this is a test --boundary-- expect MIXED_CHARSET SINGLE_SHORT_PART Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 8bit MIME-Version: 1.0 Tést 孔子 expect MIME_BAD_EXTENSION MIME_GOOD HAS_ATTACHMENT MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="boundary" --boundary Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit simple text --boundary Content-Type: text/html; charset="utf-8" Content-Disposition: attachment; filename="test.html" Content-Transfer-Encoding: 8bit

hello world

--boundary-- expect MIME_BAD_ATTACHMENT HAS_ATTACHMENT MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="boundary" --boundary Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit simple text --boundary Content-Type: text/x-plain; charset="utf-8" Content-Disposition: attachment; filename="test.txt" Content-Transfer-Encoding: 8bit hello world --boundary-- expect HAS_ATTACHMENT MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="boundary" --boundary Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit simple text --boundary Content-Type: text/plain; charset="utf-8" Content-Disposition: attachment; filename="test.txt" Content-Transfer-Encoding: 8bit hello world --boundary-- expect MIME_DOUBLE_BAD_EXTENSION MIME_GOOD HAS_ATTACHMENT MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="boundary" --boundary Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit simple text --boundary Content-Type: text/html; charset="utf-8" Content-Disposition: attachment; filename="test.html.html" Content-Transfer-Encoding: 8bit

hello world

--boundary-- expect MIME_ARCHIVE_IN_ARCHIVE MIME_GOOD HAS_ATTACHMENT MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="boundary" --boundary Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit simple text --boundary Content-Type: application/zip Content-Disposition: attachment; filename="test.zip.zip" Content-Transfer-Encoding: base64 UEsDBAoAAAAAALN6RlcAAAAAAAAAAAAAAAAIABwAdGVzdC5iaW5VVAkAA+IJI GXiCSBldXgLAAEE9QEAAAQUAAAAUEsBAh4DCgAAAAAAs3pGVwAAAAAAAAAAAA AAAAgAGAAAAAAAAAAAAKSBAAAAAHRlc3QuYmluVVQFAAPiCSBldXgLAAEE9QE AAAQUAAAAUEsFBgAAAAABAAEATgAAAEIAAAAAAA== --boundary-- expect MIME_BAD HAS_ATTACHMENT MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="boundary" --boundary Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit simple text --boundary Content-Type: image/png Content-Disposition: attachment; filename="test.png" Content-Transfer-Encoding: base64 UEsDBAoAAAAAALN6RlcAAAAAAAAAAAAAAAAIABwAdGVzdC5iaW5VVAkAA+IJI GXiCSBldXgLAAEE9QEAAAQUAAAAUEsBAh4DCgAAAAAAs3pGVwAAAAAAAAAAAA AAAAgAGAAAAAAAAAAAAKSBAAAAAHRlc3QuYmluVVQFAAPiCSBldXgLAAEE9QE AAAQUAAAAUEsFBgAAAAABAAEATgAAAEIAAAAAAA== --boundary-- ================================================ FILE: tests/resources/smtp/antispam/pyzor.test ================================================ expect PYZOR Subject: test Testa Testb Testc Testd expect PYZOR Subject: test Test1 Test1 Test2 Test3 expect Subject: test TestX TestY TestZ TestW expect MIME-Version: 1.0 X-Received: by 2002:a05:6870:c7a6:b0:1e9:8f74:ce15 with SMTP id dy38-20020a056870c7a600b001e98f74ce15mr2429202oab.11.1697967052502; Sun, 22 Oct 2023 02:30:52 -0700 (PDT) Date: Sat, 21 Oct 2023 16:59:59 -0700 Message-ID: <17434871060391156945@google.com> Subject: Report domain: stalw.art Submitter: google.com Report-ID: 17434871060391156945 From: noreply-dmarc-support@google.com To: domains@stalw.art Content-Type: application/zip; name="google.com!stalw.art!1697846400!1697932799.zip" Content-Disposition: attachment; filename="google.com!stalw.art!1697846400!1697932799.zip" Content-Transfer-Encoding: base64 UEsDBAoAAAAIABdJVldhOnFiLAIAAG4JAAAuAAAAZ29vZ2xlLmNvbSFzdGFsdy5hcnQhMTY5Nzg0 NjQwMCExNjk3OTMyNzk5LnhtbO1WwXKbMBC95ys8vhshDBgYRempX9CeGRkE1hgkjSTs5O8rIgnT pMl0ptOcfEK83X27+/TwGD09j8PmQpVmgj9uYRRvN5Q3omW8f9z+/PF9V2w3T/gBdZS2R9Kc8cNm gxSVQpl6pIa0xJAZs6hQfc3JSHEvRD/QqBEjAgvocuhI2IC5sAzDy64diWp2epIz3bd1mcvzNc9G kboR3JDG1Ix3Ap+MkboCwJdGt1JAAOH6ShVI0jzPithyva93xH4N1mJ4SPdpcYBxHu9LCLO8TDME bnGXb3eltSK899tY6Eh7xjHMy0OR5mlsuzkkxClvX6PlPjmUpZ2FBzLwO9vSbS0qkmJgzUstp+PA 9IkugwgrD8fakOEaEWUsmUNcmLRnNmKFgDt4UMvuFZufDpL2IjhFQPp3HQAdENkYDOet5oODeEji 0s39pxmttI1QYVwlrosgWkyqoTWTGO6zCBYwgmUWJbHV5hYKyY2YuMF7BNwhwL4jvZBhsiK2ITAr w7QUmhnrZj/mGlnlzcJIorVNWDTyInQ+sAi12vJNT3trYTfEWsoN65j9lpayEyUtVXWnxLi+rTXs ed5VIzKZU62ongZzI3wz7OdG8CafGfxK/mW1LR1oY4TCFzqwM9NKEp4kdveALwKsO6OVNP88xUpo 6843S8/JwUl/Y6qExHEF0yyu0iJPqySuqiQ9fOwt+OXe6uwP291b/8FbTtiv9RZM87u37t76zFsI 3P46/QJQSwECCgAKAAAACAAXSVZXYTpxYiwCAABuCQAALgAAAAAAAAAAAAAAAAAAAAAAZ29vZ2xl LmNvbSFzdGFsdy5hcnQhMTY5Nzg0NjQwMCExNjk3OTMyNzk5LnhtbFBLBQYAAAAAAQABAFwAAAB4 AgAAAAA= ================================================ FILE: tests/resources/smtp/antispam/rbl.test ================================================ remote_ip 20.11.0.1 expect RCVD_IN_DNSWL_LOW RBL_SENDERSCORE_REPUT_0 Subject: test test remote_ip 20.11.0.2 expect RBL_SENDERSCORE_REPUT_0 RBL_SEM RBL_SPAMHAUS_SBL RBL_BARRACUDA RBL_BLOCKLISTDE RBL_VIRUSFREE_BOTNET RBL_SPAMCOP RCVD_IN_DNSWL_MED Subject: test test remote_ip 20.11.0.14 expect RBL_SENDERSCORE_REPUT_1 RWL_MAILSPIKE_NEUTRAL RECEIVED_SPAMHAUS_SBL RECEIVED_SPAMHAUS_XBL RECEIVED_BLOCKLISTDE RCVD_IN_DNSWL_MED Received: from Agni (localhost [20.11.0.5]) (TLS: TLSv1/SSLv3, 168bits,DES-CBC3-SHA) by agni.forevermore.net with esmtp; Mon, 28 Oct 2002 14:48:52 -0800 Received: from [20.11.0.14] (79.sub-174-252-72.myvzw.com [20.11.0.8]) by mx.google.com with ESMTPS id m16sm345129qck.28.2011.06.15.07.42.02 (version=TLSv1/SSLv3 cipher=OTHER); Wed, 15 Jun 2011 07:42:08 -0700 (PDT) Received: from user (20.11.0.2) by DB6PR07MB3384.eurprd07.prod.outlook.com ([20.11.0.2]) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.1143.11; Thu, 13 Sep 2018 14:47:44 +0000 Subject: test test envelope_from user@surbl-abuse.com expect URIBL_GREY ABUSE_SURBL DBL_MALWARE SEM_URIBL_FRESH15 SEM_URIBL From: user@uribl-grey.com Subject: check my website sh-malware.com/login.php My e-mail is spammer@sem-uribl.com And my website is https://sem-fresh15.com/offers.html Try cheating with a trusted domain user@dkimtrusted.org dkim.result pass dkim.domains dkimtrusted.org expect DWL_DNSWL_HI From: user@dkimtrusted.org Subject: test test expect MSBL_EBL MSBL_EBL_GREY From: spammer1@spamcorp.net Reply-To: User Subject: test test expect SURBL_HASHBL_ABUSE SURBL_HASHBL_MALWARE SURBL_HASHBL_PHISH URL_ONLY REDIRECTOR_URL From: spammer@spamcorp.net Reply-To: User Subject: test Content-Type: text/html; charset="utf-8" test https://lnkiy.in/other/path?query=true ================================================ FILE: tests/resources/smtp/antispam/received.test ================================================ expect RCVD_COUNT_THREE RCVD_NO_TLS_LAST Received: from BAY0-HMR08.bay0.hotmail.com (bay0-hmr08.bay0.hotmail.com [65.54.241.207]) by dogma.slashnull.org (8.11.6/8.11.6) with ESMTP id h2DBpvs24047 for ; Thu, 13 Mar 2003 11:51:57 GMT Received: from BAY0-HMR08.bay0.hotmail.com (bay0-hmr08.bay0.hotmail.com [65.54.241.207]) by dogma.slashnull.org (8.11.6/8.11.6) with ESMTP id h2DBpvs24047 for ; Thu, 13 Mar 2003 11:51:57 GMT Received: from BAY0-HMR08.bay0.hotmail.com (bay0-hmr08.bay0.hotmail.com [65.54.241.207]) by dogma.slashnull.org (8.11.6/8.11.6) with ESMTP id h2DBpvs24047 for ; Thu, 13 Mar 2003 11:51:57 GMT test authenticated_as john@doe.com tls.version TLSv1.3 expect RCVD_VIA_SMTP_AUTH RCVD_COUNT_ONE RCVD_TLS_LAST Received: from BAY0-HMR08.bay0.hotmail.com (bay0-hmr08.bay0.hotmail.com [65.54.241.207]) by dogma.slashnull.org (8.11.6/8.11.6) with ESMTP id h2DBpvs24047 for ; Thu, 13 Mar 2003 11:51:57 GMT test expect RCVD_ILLEGAL_CHARS RCVD_COUNT_ONE RCVD_NO_TLS_LAST Received: from BAY0-HMR08.bay0.hótmail.com (bay0-hmr08.bay0.hótmail.com [65.54.241.207]) by dogma.slashnull.org (8.11.6/8.11.6) with ESMTP id h2DBpvs24047 for ; Thu, 13 Mar 2003 11:51:57 GMT test tls.version TLVv1.3 expect RCVD_TLS_ALL RCVD_HELO_USER RCVD_DOUBLE_IP_SPAM FORGED_RCVD_TRAIL PREVIOUSLY_DELIVERED RCVD_COUNT_FIVE Received: from Agni (localhost [::ffff:127.0.0.1]) (TLS: TLSv1/SSLv3, 168bits,DES-CBC3-SHA) by agni.forevermore.net with esmtp; Mon, 28 Oct 2002 14:48:52 -0800 Received: from [10.231.252.223] (79.sub-174-252-72.myvzw.com [174.252.72.79]) by mx.google.com with ESMTPS id m16sm345129qck.28.2011.06.15.07.42.02 (version=TLSv1/SSLv3 cipher=OTHER); Wed, 15 Jun 2011 07:42:08 -0700 (PDT) Received: from other.myvzw.com (79.sub-174-252-72.myvzw.com [174.252.72.79]) by mx.google.com with ESMTPS id m16sm345129qck.28.2011.06.15.07.42.02 (version=TLSv1/SSLv3 cipher=OTHER); Wed, 15 Jun 2011 07:42:08 -0700 (PDT) Received: from user (10.175.233.33) by DB6PR07MB3384.eurprd07.prod.outlook.com (10.175.234.11) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.1143.11; Thu, 13 Sep 2018 14:47:44 +0000 Received: from [94.198.96.74] (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange ECDHE (P-256) server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by ietfa.amsl.com (Postfix) with ESMTPS id 10B7AC151535 for ; Mon, 28 Aug 2023 02:21:23 -0700 (PDT) To: user@domain.com Subject: test test expect DIRECT_TO_MX RCVD_COUNT_ZERO RCVD_NO_TLS_LAST To: user@domain.com X-Mailer: MUA 1.2 Subject: test test expect RCVD_UNPARSABLE RCVD_NO_TLS_LAST RCVD_COUNT_ONE To: user@domain.com Received: invalid test ================================================ FILE: tests/resources/smtp/antispam/recipient.test ================================================ expect MISSING_TO RCPT_COUNT_ZERO X-To: hello@world.com Subject: Hi Test expect RCPT_COUNT_ONE TO_DN_ALL To: "Hello World" Subject: Hi Test expect RCPT_COUNT_ONE TO_DN_NONE TAGGED_RCPT To: hello+there@world.com Subject: Hi Test envelope_from user@domain.org expect TO_DN_RECIPIENTS RCPT_COUNT_TWO TO_DN_SOME To: "recipients" Cc: other@user.org Subject: Hi Test expect RCPT_IN_SUBJECT TO_DN_NONE RCPT_COUNT_ONE To: hello@world.com Subject: Special offer for HELLO@world.com Test expect RCPT_LOCAL_IN_SUBJECT TO_DN_NONE RCPT_COUNT_ONE To: hello@world.com Subject: Special offer for hello Test envelope_from envelope_to hello@world.com envelope_to goodbye@world.com expect RCPT_BOUNCEMOREONE TO_MATCH_ENVRCPT_ALL TO_DN_NONE RCPT_COUNT_TWO To: hello@world.com Cc: goodbye@world.com Subject: Hi Test envelope_from postmaster@domain.org envelope_to hello@world.com envelope_to goodbye@world.com expect RCPT_BOUNCEMOREONE TO_MATCH_ENVRCPT_SOME TO_DN_NONE RCPT_COUNT_THREE To: hello@world.com, test@domain.com Cc: goodbye@world.com Subject: Hi Test expect RCPT_COUNT_ZERO UNDISC_RCPT To: Undisclosed recipients:; Subject: Hi Test envelope_from list@domain.org envelope_to hello@world.com expect TO_DN_ALL RCPT_COUNT_ONE List-Id: To: "Mailing List" Subject: Hi Test envelope_from spammer@domain.org envelope_to hello@world.com expect FORGED_RECIPIENTS TO_NEEDS_ENCODING TO_DN_ALL RCPT_COUNT_ONE To: "Thé Spámmer" Subject: Hi Test envelope_from user@domain.org envelope_to hello@world.com envelope_to user@domain.org expect TO_DN_ALL TO_MATCH_ENVRCPT_ALL RCPT_COUNT_ONE To: "Hello World" Subject: Hi Test envelope_from user@domain.org envelope_to hello@world.com envelope_to user@domain.org expect TO_EXCESS_QP TO_DN_ALL TO_MATCH_ENVRCPT_ALL RCPT_COUNT_ONE To: "=?iso-8859-1?Q?Die_Hasen_und_die_Froesche?=" Subject: Hi Test envelope_from user@domain.org envelope_to hello@world.com envelope_to user@domain.org expect TO_EXCESS_BASE64 TO_DN_ALL TO_MATCH_ENVRCPT_ALL RCPT_COUNT_ONE To: "=?iso-8859-1?B?RGllIEhhc2VuIHVuIGRpZSBGcm9lc2NoZQ==?=" Subject: Hi Test envelope_from test@test.com expect FREEMAIL_TO DISPOSABLE_CC RCPT_COUNT_TWO TO_DN_NONE To: user@gmail.com Cc: otheruser@guerrillamail.com Subject: Hi Test envelope_from test@test.com expect FREEMAIL_CC DISPOSABLE_TO DISPOSABLE_BCC RCPT_COUNT_THREE TO_DN_NONE To: otheruser@guerrillamail.com Cc: user@gmail.com Bcc: some@guerrillamail.com Subject: Hi Test envelope_from test@test.com expect SORTED_RECIPS RCPT_COUNT_SEVEN TO_DN_NONE To: a@domain.com, b@domain.com, c@domain.com, d@domain.com Cc: e@domain.com, f@domain.com, g@domain.com Subject: Hi Test envelope_from test@test.com expect RCPT_COUNT_SEVEN TO_DN_NONE To: tom@domain.com, mark@domain.com, bill@domain.com, peter@domain.com Cc: jane@domain.com, mary@domain.com, lucy@domain.com Subject: Hi Test envelope_from test@test.com expect SUSPICIOUS_RECIPS RCPT_COUNT_SEVEN TO_DN_NONE To: tim@domain.com, tom@domain.com, tum@domain.com, tem@domain.com Cc: tam@domain.com, tron@domain.com, tym@domain.com Subject: Hi Test envelope_from info@notalist.org envelope_to info@notalist.org expect INFO_TO_INFO_LU RCPT_COUNT_ONE TO_MATCH_ENVRCPT_ALL TO_DN_NONE From: info@notalist.org To: info@notalist.org List-Unsubscribe: Subject: Hi Test envelope_from info@notalist.org envelope_to info@notalist.org expect RCPT_COUNT_ONE TO_MATCH_ENVRCPT_ALL TO_DN_NONE From: info@notalist.org To: info@notalist.org Subject: Hi Test envelope_from hello@test.org envelope_to user@test.org envelope_to test@test.org expect TO_WRAPPED_IN_SPACES RCPT_COUNT_TWO TO_MATCH_ENVRCPT_ALL TO_DN_NONE From: hello@test.org To: < user@test.org > Cc: test@test.org Subject: Hi Test envelope_from hello@test.org envelope_to user@test.org envelope_to test@test.org expect TO_WRAPPED_IN_SPACES RCPT_COUNT_TWO TO_MATCH_ENVRCPT_ALL TO_DN_SOME From: hello@test.org To: user@test.org Cc: "Test" Subject: Hi Test expect RCPT_IN_BODY TO_DN_NONE RCPT_COUNT_ONE To: hello@world.com Subject: Special offer An offer for hello@world.com expect RCPT_DOMAIN_IN_MESSAGE RCPT_IN_BODY RCPT_DOMAIN_IN_SUBJECT RCPT_COUNT_ONE TO_DN_NONE To: hello@world.com Subject: Message for world.com An offer for hello@world.com expect RCPT_DOMAIN_IN_MESSAGE RCPT_DOMAIN_IN_BODY RCPT_DOMAIN_IN_SUBJECT RCPT_COUNT_ONE TO_DN_NONE To: hello@world.com Subject: Message for world.com An offer for world.com ================================================ FILE: tests/resources/smtp/antispam/replyto.test ================================================ expect REPLYTO_UNPARSABLE Reply-to: hello Test expect REPLYTO_EQ_FROM HAS_REPLYTO From: hello@domain.org Reply-to: hello@domain.org Test expect REPLYTO_DOM_EQ_FROM_DOM REPLYTO_DN_EQ_FROM_DN HAS_REPLYTO From: "Hello" Reply-to: "Hello" Test envelope_from hello@otherdomain.org.uk envelope_to user@somedomain.com expect REPLYTO_DOM_NEQ_FROM_DOM HAS_REPLYTO From: hello@otherdomain.org.uk To: user@somedomain.com, hello@otherdomain.org.uk Reply-to: hello@domain.org.uk Test envelope_from sender@foo.org envelope_to user@somedomain.com expect REPLYTO_EQ_TO_ADDR SPOOF_REPLYTO HAS_REPLYTO From: sender@foo.org To: user@somedomain.com Reply-to: user@somedomain.com Test envelope_from list@foo.org envelope_to user@somedomain.com expect REPLYTO_DOM_NEQ_FROM_DOM HAS_REPLYTO From: list@foo.org List-Unsubscribe: unsubcribe@foo.org To: user@somedomain.com Reply-to: user@somedomain.com Test envelope_from user@foo.org envelope_to other@foo.org expect REPLYTO_DOM_NEQ_FROM_DOM HAS_REPLYTO From: user@foo.org To: otheruser@foo.org Reply-to: user@otherdomain.org Test envelope_from user@foo.org envelope_to otheruser@domain.org expect SPOOF_REPLYTO REPLYTO_DOM_NEQ_FROM_DOM HAS_REPLYTO From: user@foo.org To: otheruser@domain.org Reply-to: user@otherdomain.org Test expect REPLYTO_EXCESS_QP REPLYTO_DOM_EQ_FROM_DOM HAS_REPLYTO From: hello@domain.org Reply-to: =?iso-8859-1?Q?Die_Hasen_und_die_Froesche?= Test expect REPLYTO_EXCESS_BASE64 REPLYTO_DOM_EQ_FROM_DOM HAS_REPLYTO From: hello@domain.org Reply-to: "=?iso-8859-1?B?RGllIEhhc2VuIHVuIGRpZSBGcm9lc2NoZQ==?=" Test expect REPLYTO_EMAIL_HAS_TITLE REPLYTO_DOM_EQ_FROM_DOM HAS_REPLYTO From: hello@domain.org Reply-to: "Mr. Hello" Test expect FREEMAIL_REPLY_TO FREEMAIL_FROM REPLYTO_DOM_EQ_FROM_DOM HAS_REPLYTO From: hello@gmail.com Reply-to: bye@gmail.com Test expect DISPOSABLE_REPLY_TO DISPOSABLE_FROM REPLYTO_DOM_EQ_FROM_DOM HAS_REPLYTO From: hello@custom.disposable.org Reply-to: bye@custom.disposable.org Test expect FREEMAIL_REPLY_TO_NEQ_FROM_DOM FREEMAIL_REPLY_TO FREEMAIL_FROM REPLYTO_DOM_NEQ_FROM_DOM SPOOF_REPLYTO HAS_REPLYTO From: hello@gmail.com Reply-to: hello@yahoomail.com Test ================================================ FILE: tests/resources/smtp/antispam/spamtrap.test ================================================ envelope_from spammer@domain.com envelope_to spamtrap@foobar.org expect SPAM_TRAP Subject: save up to NUMBER on life insurance why spend more than you have to life quote savings ensuring your family s financial security is very important life quote savings makes buying life insurance simple and affordable we provide free access to the very best companies and the lowest rates life quote savings is fast easy and saves you money let us help you get started with the best values in the country on new coverage you can save hundreds or even thousands of dollars by requesting a free quote from lifequote savings our service will take you less than NUMBER minutes to complete shop and compare save up to NUMBER on all types of life insurance hyperlink click here for your free quote protecting your family is the best investment you ll ever make if you are in receipt of this email in error and or wish to be removed from our list hyperlink please click here and type remove if you reside in any state which prohibits e mail solicitations for insurance please disregard this email ================================================ FILE: tests/resources/smtp/antispam/subject.test ================================================ expect SUBJ_ALL_CAPS Subject: HELLO WORLD Test expect LONG_SUBJ Subject: this is an extremely long subject line that should be truncated to 80 characters and folded in order to be RFC compliant and avoid the SPAM filter that is looking for long subject lines like this one which by the way, it is ridiculously long Test expect SUBJECT_NEEDS_ENCODING Subject: thís líné shóúld bé éncódéd Test param.smtputf8 1 expect Subject: thís líné shóúld bé éncódéd Test param.8bitmime 1 expect Subject: thís líné shóúld bé éncódéd Test expect Subject: =?iso-8859-1?Q?Die_Hasen_und_die_Fr=F6sche?= Test expect URL_IN_SUBJECT Subject: check out my url HTTPS://SPAMMER.COM Test expect URL_IN_SUBJECT Subject: check out my url HTTP://SPAMMER.COM Test expect MISSING_SUBJECT X-Subject: missing subject Test expect EMPTY_SUBJECT Subject: Test expect SUBJ_EXCESS_QP Subject: =?iso-8859-1?Q?Die_Hasen_und_die_Froesche?= Test expect SUBJ_EXCESS_BASE64 Subject: =?iso-8859-1?B?RGllIEhhc2VuIHVuIGRpZSBGcm9lc2NoZQ==?= Test expect FAKE_REPLY Subject: Re: about your question Test expect In-Reply-To: Subject: Re: about your question Test expect References: Subject: Re: about your question Test expect SUBJECT_ENDS_SPACES Subject: =?iso-8859-1?Q?Die_Hasen_und_die_Fr=F6sche_?= Test param.smtputf8 1 expect SUBJECT_HAS_CURRENCY SUBJECT_ENDS_EXCLAIM Subject: You have won £200! Test param.smtputf8 1 expect SUBJECT_HAS_CURRENCY SUBJECT_ENDS_QUESTION Subject: Have you won $200? Test expect RCPT_IN_SUBJECT To: hello@world.org Subject: Great offers for hello@world.org Test expect RCPT_DOMAIN_IN_SUBJECT To: hello@world.org Subject: Great offers for world.org Test expect To: hello@world.org Subject: Question about other@domain.net Test ================================================ FILE: tests/resources/smtp/antispam/url.test ================================================ expect URL_ONLY Subject: test https://url.org expect Subject: test my site is https://url.org expect SUSPICIOUS_URL Subject: test my site is https://192.168.1.1 expect HOMOGRAPH_URL Subject: test my site is https://xn--youtue-tg7b.com expect MIXED_CHARSET_URL Subject: test my site is https://www.xn--1ca81o6aa92e.com/ expect UNPARSABLE_URL Subject: test login to your account at https://bánk.com/ expect URL_REDIRECTOR_NESTED REDIRECTOR_URL Subject: nested redirect login to https://redirect.com/?https://redirect.org/?https://redirect.net/?https://redirect.io/?https://redirect.me/?https://redirect.com expect REDIRECTOR_URL HOMOGRAPH_URL Subject: redirect to omograph login to https://www.redirect.com/?https://xn--twiter-507b.com expect HAS_ONION_URI HAS_ANON_DOMAIN Subject: url in title darkweb.onion/login test expect HAS_IPFS_GATEWAY_URL HAS_WP_URI URI_HIDDEN_PATH Content-Type: text/html; charset="utf-8" Subject: html test expect HAS_GUC_PROXY_URI HAS_GOOGLE_FIREBASE_URL HAS_GOOGLE_REDIR HAS_ANON_DOMAIN URL_ONLY Content-Type: text/html; charset="utf-8" Subject: mixed urls googleusercontent.com/proxy/url google.com/url?otherurl.org expect WP_COMPROMISED Subject: plain test http://url.com/Well-known/../assetlinks.json http://wp.com/WP-content/content.pdf expect HAS_WP_URI Subject: plain test http://url.com/Well-known/../assetlinks.json http://wp.com/WP-other/content.pdf expect PHISHED_OPENPHISH PHISHED_PHISHTANK Subject: plain test https://phishing-open.org https://phishing-tank.com expect Subject: IPs are not urls 192.168.1.1 expect Content-Type: text/html; charset="utf-8" Subject: IPs in HTML are not urls Das System wurde um 01.01.1970 08:28:00 für die IP-Adresse 123.123.123.123 gesperrt.

Der Besucher hat versucht, sich mit folgenden Daten anzumelden.
Partner: 12345678
Portal: IP-Sperre einsehen expect RCPT_DOMAIN_IN_BODY To: hello@world.com Subject: Special offer An offer for world.com ================================================ FILE: tests/resources/smtp/certs/tls_cert.pem ================================================ -----BEGIN CERTIFICATE----- MIIFCTCCAvGgAwIBAgIUCgHGQYUqtelbHGVSzCVwBL3fyEUwDQYJKoZIhvcNAQEL BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIyMDUxNjExNDAzNFoXDTIzMDUx NjExNDAzNFowFDESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEF AAOCAg8AMIICCgKCAgEAtwS0Fzl3SjaCuKEXgZ/fdWbDoj/qDphyNCAKNevQ0+D0 STNkWCO04aFSH0zcL8zoD9gokNos0i7OU9//ZhZQmex4V6EFdZn8bFwUWN/scUvW HEFXVjtHldO2isZgIxH9LuwRv7KAgkISuWahqerOVDhe7SeQUV0AJGNEh3cT9PZr gSY931BxB7n+5k8eoSk8Z1gtBzQzL62kVGpHDKfw8yX8m65owF9eLUBrNzgxmXfC xpuHwj7hmVhS09PPKeN/RsFS8PsYO7bo0u8jEKalteumjRT7RyUEbioqfo6ZFOGj FHPIq/uKXS9zN1fpoyNh3ur5hMznQhrqlwBM9KlM7GdBJ0pZ3ad0YjT8IL/GnGKR 85J2WZdLqaQdUZo7nV67FhqdDlNE4MdwiykTMjfmLRXGAVhAzJHKyRKNwmkI2aqe S7aqeNgvuDBwY80Q9a2rb5py1Aw+L8yCkUBuHboToDpxSVRDNN8DrWNmmsXnxsOG wRDODy4GICKyxlP+RFSM8xWSQ6y9ktS2OfDBm+Eqcw+3pZKhdz2wgxLkUBJ8X1eh kJrCA/6LTuhy6m6mMjAfoSOFU7fu88jxaWPgvP7GKyH+LM/t9eucobz2ks5rtSjz V4Dc5DCS94/OpVRHwHdaFSPbJKBN9Ev8gnNrAyx/aBPGoHBPG/QUiU7dcUNIPt0C AwEAAaNTMFEwHQYDVR0OBBYEFI167IxBmErB11EqiPPqFLa31ZaMMB8GA1UdIwQY MBaAFI167IxBmErB11EqiPPqFLa31ZaMMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI hvcNAQELBQADggIBALU00IOiH5ubEauVCmakms5ermNTZfculnhnDfWTLMeh2+a7 G4cqADErfMhm/mmLbrw33t9s6tCAhQltvewKR40ST9uMPSyiQbYaCXd5DXnuI6Ox JtNW+UOWIaMf8abnkdLvREOvb8dVQS1i3xq14tAjY5XgpGwCPP8m54b7N3Q7soLn e5PDhPNTnhRIn2RLuYoZmQmMA5fcqEUDYff4epUww7PhrM1QckZligI3566NlGOf j1G9JrivBtY0eaJtamIFnGMBT0ThDudxVja2Nv0C2Elry0p4T/o4nc4M67BJ/y1R vjNLAgFhbxssemU3lZqSd+pykpJBwDBjFSPrZZmQcbk7H6Uz8V1xr/xuzfw6fA13 NWZ5vLgP/DQ13sM+XFlxThKfbPMPVe/UCTvfGtNW+3XyBgPntEkR+fNEawQmzbYl R+X1ymT9MZnEZqRMf7/UD/SYek1aUJefoew3upjMgxYVvh4F8dqJ+39F+xoFzIA2 1dDAEMzXtjA3zKhZ2cycZbEzpJvYA3eGLuR16Suqfi4kPvfwK0mOhCxQmpayt7/X vuEzW6dPCH8Hgbb0WvsSppGOvhdbDaZFNfFc5eNSxhyKzu3H3ACNImZRtZE+yixx 0fR8+xz9kDLf8xupV+X9heyFGHSyYU2Lveaevtr2Ij3weLRgJ6LbNALoeKXk -----END CERTIFICATE----- ================================================ FILE: tests/resources/smtp/certs/tls_privatekey.pem ================================================ -----BEGIN PRIVATE KEY----- MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC3BLQXOXdKNoK4 oReBn991ZsOiP+oOmHI0IAo169DT4PRJM2RYI7ThoVIfTNwvzOgP2CiQ2izSLs5T 3/9mFlCZ7HhXoQV1mfxsXBRY3+xxS9YcQVdWO0eV07aKxmAjEf0u7BG/soCCQhK5 ZqGp6s5UOF7tJ5BRXQAkY0SHdxP09muBJj3fUHEHuf7mTx6hKTxnWC0HNDMvraRU akcMp/DzJfybrmjAX14tQGs3ODGZd8LGm4fCPuGZWFLT088p439GwVLw+xg7tujS 7yMQpqW166aNFPtHJQRuKip+jpkU4aMUc8ir+4pdL3M3V+mjI2He6vmEzOdCGuqX AEz0qUzsZ0EnSlndp3RiNPwgv8acYpHzknZZl0uppB1RmjudXrsWGp0OU0Tgx3CL KRMyN+YtFcYBWEDMkcrJEo3CaQjZqp5Ltqp42C+4MHBjzRD1ratvmnLUDD4vzIKR QG4duhOgOnFJVEM03wOtY2aaxefGw4bBEM4PLgYgIrLGU/5EVIzzFZJDrL2S1LY5 8MGb4SpzD7elkqF3PbCDEuRQEnxfV6GQmsID/otO6HLqbqYyMB+hI4VTt+7zyPFp Y+C8/sYrIf4sz+3165yhvPaSzmu1KPNXgNzkMJL3j86lVEfAd1oVI9skoE30S/yC c2sDLH9oE8agcE8b9BSJTt1xQ0g+3QIDAQABAoICABq5oxqpF5RMtXYEgAw7rkPU h8jPkHwlIrgd3Z/WGZ53APUXfhWo0ScJiZZsgNKyF0kJBZNxaI4gq5xv3zmnFIoF j+Ur7EIqBERGheoceMhqjI9/syMycNeeHM/S/ALjA5ewfT8C7+UVhOpx5DWNxidi O+phlp9q9zRZEo69grqIqVYooWxUsMyyCljTQOPDw8BLjfe5VagmsRJqmolslLDM 4UBSjZVZ18S/3Wgo2oVQia660244BHWCAkZQbbXuNI2+eUAbSoSdxw3WQcaSrywL hzyezbqr2yPDIIVuiUgVUt0Ps0P57VCCN07jlYhvCEGnClysFzD+ATefoZ0wg7za dQu2E+d166rAjnssyhzcHMn3pxgSdtXD+dQR/xfIGbPABucCupEFqKmhLdMm9+ud lHay87qzMpIa8cITJwEQROfXqWAhNUU98pKCOx1SVXBqQC7QVqGQ5solDf0eMSVh ngQ6Dz2WUI2ty75LteiFwlyTgnU9nyPN0NXsrMEET2BHWre7ufTQqiULtQ7+9BwH AMxEKvrQHjMUjdfbXuzdyc5w5mPYJZfFVSQ1HMslx66h9yCpRIsBZvUGvoaP8Tpe nQ66FTYRbiOkkdJ7k8DtrnhsJI1oOGjnvj/rvZ8D2pvrlJcIH2AyN3MOL8Jp5Oj1 nCFt77TwpF92pgl0g9gBAoIBAQDcarmP54QboaIQ9S2gE/4gSVC5i44iDJuSRdI8 K081RQcWiNzqQXTRc5nqJ7KzLyPiGlg+6rWsBKLos5l4t+MdhhH+KUvk/OtT/g8V 0NZBNXLIbSb8j8ix4v3/f2qKHN3Co6QOlxb3gFvobKDdoKqUNiSH1zTZ8/Y/BzkM jqWKhTdaLz6eyzhKfOTA4LO8kJ3VF8HUM1N9/e8Gjorl+gZpJUXUQS0+AIi8W76C OwDrVb3BPGVnApQJfWF78h4g20RwXrx/GYUW2vOMcLjXXDV5U7+nobPUoJnLxoZC 16o88y0Ivan8dBNXsc1epyPvvEqp6MJbAyyVuNeuRJcgYA0BAoIBAQDUkGRV7fLG wCr5rNysUO+FKzVtTJnf9KEsqAqUmmVnG4oubxAJJtiB5n2+DT+CtO8Nrtz05BbR uxfWm+lbEw6lVMj63bywtp0NdULg7/2t+oq2Svv16KrZIRJttXMkdEiFFmkVAEhX l8Fyl6PJPfSMwbPdXEUPUAaNrXweVFffXczHc4W2G212ZzDB0z7QQSgEntbTDFB/ 2Cg5dvuojlM9zw0fuEyLwItZs7n16j/ONZLgBHyroMU9ZPxbnLrVyoZlqtob+RWm Ju2fSIL9QqG6O4td1TqcUBGvFQYjGvKA+q5fsG26NBJ0Ac48cNK6PS4lMkN3Av2J ccloYaMEHAXdAoIBAE8WMCy1Ok6byUXiYxOL+OPmyoM40q/e7DcovE2AkLQhZ3Cr fPDEucCphPFiexkV8f8fysgQeU0WgMmUH54UBPbD81LJyISKR3nkr875Ftdg8SV/ HL0EblN9ifuR4U1bHCrJgoUFq2T09oVH7NR44Ju7bZIcIseNZK6qzcp2qGkycXD3 gLWDX1hCxeV6+qLPFQKvuomEPRH4+jnVDXuFIaW6jPqixDP6BxXmqU2bFDJcmnBq VkwGvc1F4qORdUP+yOi05VeJdZqEx1x92aTUXg+BgEQKnjbNxUE7o1L6hQfHjUIU o5iEoagWkQTEXf2YBwY+EPaNBgNWxnSuAbfJHwECggEBALOF95ezTVWauzD/U6ic +o3n/kl/Zn4FJ5KFodn7xCSe18d7uXlhO34KYqx+l+MWWMefpbGWacdcUjfImf93 SulLgCqP12sP7/iLzp4XUpL7hOeM0NvRU2nqSpwpoUNqik0Mrlc0U+TWoGTduVCf aMjwV65e3VyfY8mIeclLxqM5n1fcM1OoOnzDjiRE+0n7nYa5eAnq3pn6v4449TZY belH03e0ucFWLtrltesBmj3YdWGJqJlzQOInRhNBfXJOh8+ZynfRmP0o54udPDQV cG3PGFd5XPTjkuvhv7sqaSGRlm/um92lWOhtFfdp+i+cuDpmByCef+7zEP19aKZx 3GkCggEAFTs7KNMfvIEaLH0yQUFeq2gLmtcMofmOmeoIECycN1rG7iJo07lJLIs0 bVODH8Z0kX8llu3cjGMAH/6R2uugJSxkmFiZKrngTzKmxDPvTCKWR4RFwXH9j8IO cPq7FtKN4SgrPy9ciAPdkcGmu3zz/sBKOaoPwvU2PdBRT+v/aoz+GCLXAvzFlKVe 9/7zdg87ilo8+AtV+71EJeR3kyBPKS9JrWYUKfiams12+uuH4/53rMFZfNCAaZ3Z 1sdXEO4o3Loc5TX4DbO9FVdBSBe6klEXx4T0QJboO6uBvTBnnRL2SQriJQQFwYT6 XzVV5pwOxkIDBWDIqMUfwJDChBKfpw== -----END PRIVATE KEY----- ================================================ FILE: tests/resources/smtp/config/if-blocks.toml ================================================ durations = [ {if = "sender = 'jdoe'", then = "5d"}, {if = "priority = -1 | starts_with(rcpt, 'jane')", then = "1h"}, {else = false} ] string-list = [ {if = "sender = 'jdoe'", then = "['From', 'To', 'Date']"}, {if = "priority = -1 | starts_with(rcpt, 'jane')", then = "'Other-ID'"}, {else = "[]"} ] string-list-bis = [ {if = "sender = 'jdoe'", then = "['From', 'To', 'Date']"}, {if = "priority = -1 | starts_with(rcpt, 'jane')", then = "[]"}, {else = "['ID-Bis']"} ] single-value = "'hello world'" bad-if-without-then = [ {if = "sender = 'jdoe'"}, {else = 1} ] bad-if-without-else = [ {if = "sender = 'jdoe'", then = 1} ] bad-multiple-else = [ {if = "sender = 'jdoe'", then = 1}, {else = 1}, {else = 2} ] ================================================ FILE: tests/resources/smtp/config/lists.toml ================================================ [list] local-domains = ["example.org", "example.net"] spammer-domains = "thatdomain.net" local-users = "file://{LIST1}" power-users = ["file://{LIST1}", "file://{LIST2}"] [remote."lmtp"] address = 192.168.0.1 port = 25 protocol = "lmtp" lookup = true [remote."lmtp".auth] username = "hello" secret = "world" [remote."lmtp".tls] implicit = true allow-invalid-certs = true ================================================ FILE: tests/resources/smtp/config/rules-dynvalue.toml ================================================ [envelope] rcpt-domain = "foo.example.org" rcpt = "user@foo.example.org" sender-domain = "foo.net" sender = "bill@foo.net" local-ip = "192.168.9.3" remote-ip = "A:B:C::D:E" mx = "mx.somedomain.com" authenticated-as = "john@foobar.org" priority = -4 listener = "smtp" helo-domain = "hi-domain.net" [eval."eq"] test = [ {if = "sender = 'bill@foo.net'", then = "sender"}, {else = false} ] expect = "bill@foo.net" [eval."starts-with"] test = [ {if = "starts_with(rcpt_domain, 'foo')", then = "'mx.' + rcpt_domain"}, {else = false} ] expect = "mx.foo.example.org" [eval."regex"] test = [ {if = "matches('^([^.]+)@([^.]+)\.(.+)$', rcpt)", then = "$1 + '+' + $2 + '@' + $3"}, {else = false} ] expect = "user+foo@example.org" [eval."regex-full"] test = [ {if = "matches('^([^.]+)@([^.]+)\.(.+)$', rcpt)", then = "rcpt"}, {else = false} ] expect = "user@foo.example.org" [eval."envelope-match"] test = [ {if = "matches('^([^.]+)@(.+)$', authenticated_as)", then = "'rcpt ' + rcpt + ' listener ' + listener + ' ip ' + local_ip + ' priority ' + priority"}, {else = false} ] expect = "rcpt user@foo.example.org listener smtp ip 192.168.9.3 priority -4" [eval."static-match"] test = [ {if = "matches('^([^.]+)@(.+)$', authenticated_as)", then = "'hello world'"}, {else = false} ] expect = "hello world" [eval."no-match"] test = [ {if = "matches('^([^.]+)@([^.]+)\.(.+)$org', authenticated_as)", then = "'test'"}, {else = false} ] expect = false ================================================ FILE: tests/resources/smtp/config/rules-eval.toml ================================================ [envelope] rcpt-domain = "example.org" rcpt = "user@example.org" sender-domain = "foo.net" sender = "bill@foo.net" local-ip = "192.168.9.3" remote-ip = "A:B:C::D:E" mx = "mx.somedomain.com" authenticated-as = "john@foobar.org" priority = -4 listener = "smtp" helo-domain = "hi-domain.net" [rule] "eq-true" = "rcpt_domain = 'example.org'" "eq-false" = "rcpt_domain = 'example.com'" "listener-eq-true" = "listener = 'smtp'" "listener-eq-false" = "listener = 'smtps'" "ip-eq-true" = "local_ip = '192.168.9.3'" "ip-eq-false" = "remote_ip = 'A:B:C::D:E'" "ne-true" = "!is_empty(authenticated_as)" "ne-false" = "authenticated_as != 'john@foobar.org'" "starts-with-true" = "starts_with(mx, 'mx.some')" "starts-with-false" = "starts_with(mx, 'enchilada')" "ends-with-true" = "ends_with(sender, '@foo.net')" "ends-with-false" = "ends_with(sender, 'chimichanga')" "regex-true" = "matches('^(.+)@(.+)$', sender)" "regex-false" = "matches('/^\\S+@\\S+\\.\\S+$/', mx)" "any-of-true" = "authenticated_as != 'john@foobar.org' | rcpt_domain = 'example.org' | starts_with(mx, 'mx.some')" "any-of-false" = "authenticated_as = 'something else' | rcpt_domain = 'something else' | starts_with(mx, 'something else')" "all-of-true" = "rcpt_domain = 'example.org' & listener = 'smtp' & starts_with(mx, 'mx.some')" "all-of-false" = "rcpt_domain = 'example.org' & listener = 'smtp' & starts_with(mx, 'something else')" "none-of-true" = "!(authenticated_as = 'something else' | rcpt_domain = 'something else' | starts_with(mx, 'something else'))" "none-of-false" = "!(rcpt_domain = 'example.org' | listener = 'smtp' | starts_with(mx, 'mx.some'))" ================================================ FILE: tests/resources/smtp/config/rules.toml ================================================ [rule] "my-nested-rule" = { any-of = [ {if = "rcpt-domain", eq = "example.org"}, {if = "remote-ip", eq = "192.168.0.0/24"}, {all-of = [ {if = "rcpt", starts-with = "no-reply@"}, {if = "sender", ends-with = "@domain.org"}, {none-of = [ {if = "priority", eq = 1}, {if = "priority", ne = -2}, ]} ]} ]} [rule."simple"] if = "listener" eq = "smtp" [rule."is-authenticated"] if = "authenticated-as" ne = "" [[rule."expanded".all-of]] if = "sender-domain" starts-with = "example" [[rule."expanded".all-of]] if = "sender" in-list = "test-list" ================================================ FILE: tests/resources/smtp/config/servers.toml ================================================ [server] hostname = "mx.example.org" greeting = "Stalwart SMTP - hi there!" [server.listener."smtp"] bind = ["127.0.0.1:9925"] protocol = "smtp" tls.implicit = false [server.listener."smtps"] bind = ["127.0.0.1:9465", "127.0.0.1:9466"] protocol = "smtp" max-connections = 1024 tls.implicit = true tls.ciphers = ["TLS13_CHACHA20_POLY1305_SHA256", "TLS13_AES_256_GCM_SHA384"] socket.ttl = 4096 [server.listener."submission"] greeting = "Stalwart SMTP submission at your service" protocol = "smtp" hostname = "submit.example.org" bind = "127.0.0.1:9991" #tls.sni = [{subject = "submit.example.org", certificate = "other"}, # {subject = "submission.example.org", certificate = "other"}] socket.backlog = 2048 [server.tls] enable = true implicit = true timeout = 300 certificate = "default" #sni = [{subject = "other.domain.org", certificate = "default"}] protocols = ["TLSv1.2", "TLSv1.3"] ciphers = [] ignore_client_order = true [server.socket] reuse-addr = true reuse-port = true backlog = 1024 ttl = 3600 send-buffer-size = 65535 recv-buffer-size = 65535 linger = 1 tos = 1 [certificate."default"] cert = "%{file:{CERT}}%" private-key = "%{file:{PK}}%" [certificate."other"] cert = "%{file:{CERT}}%" private-key = "%{file:{PK}}%" ================================================ FILE: tests/resources/smtp/config/throttle.toml ================================================ [[throttle]] match = "remote_ip == '127.0.0.1'" key = ["remote_ip", "authenticated_as"] rate = "50/30s" enable = true [[throttle]] key = "sender_domain" rate = "50/30s" enable = true ================================================ FILE: tests/resources/smtp/config/toml-parser.toml ================================================ [database] enabled = true # ignore ports = [ 8000, 8001, 8002 ] # ignore data = [ ["delta", "phi"], [3.14] ] temp_targets = { cpu = 79.5, case = 72.0 } [servers] "127.0.0.1" = "value" # ignore "character encoding" = "value" [servers.alpha] ip = "10.0.0.1" role = "frontend" [servers.beta] ip = "10.0.0.2" role = "backend" [[products]] name = "Hammer" sku = 738594937 [[products]] # empty table within the array [[products]] # ignore name = "Nail" sku = 284758393 # ignore color = "gray" [strings."my \"string\" test"] str1 = "I'm a string." str2 = "You can \"quote\" me." str3 = "Name\tTabs\nNew Line." lines = ''' The first newline is trimmed in raw strings. All other whitespace is preserved. ''' [sets] integer = { 1 } integers = { 1, 2, 3 } string = { "red" } strings = { "red", "yellow", "green" } [arrays] integers = [ 1, 2, 3 ] colors = [ "red", "yellow", "green" ] nested_arrays_of_ints = [ [ 1, 2 ], [3, 4, 5] ] nested_mixed_array = [ [ 1, 2 ], ["a", "b", "c"] ] string_array = [ "all", 'strings', """are the same""", '''type''' ] # Mixed-type arrays are allowed numbers = [ 0.1, 0.2, 0.5, 1, 2, 5 ] integers2 = [ 1, 2, 3 # this is ok ] integers3 = [ 4, # comment in the middle 5, # this is ok ] contributors = [ "Foo Bar " , { name = "Baz Qux", email = "bazqux@example.com", url = "https://example.com/bazqux" } ] [env] var1 = !CARGO_PKG_NAME var2 = !CARGO_PKG_NAME #comment ================================================ FILE: tests/resources/smtp/dane/dns.txt ================================================ _25._tcp.internet.nl 2 1 1 E1AE9C3DE848ECE1BA72E0D991AE4D0D9EC547C6BAD1DDDAB9D6BEB0A7E0E0D8 _25._tcp.internet.nl 3 1 1 D6FEA64D4E68CAEAB7CBB2E0F905D7F3CA3308B12FD88C5B469F08AD7E05C7C7 _25._tcp.mail.ietf.org 3 1 1 0C72AC70B745AC19998811B131D662C9AC69DBDBE7CB23E5B514B56664C5D3D6 ================================================ FILE: tests/resources/smtp/dsn/delay.eml ================================================ From: "Mail Delivery Subsystem" To: sender@foobar.org Auto-Submitted: auto-generated Subject: Warning: Delay in message delivery MIME-Version: 1.0 Content-Type: multipart/report; report-type="delivery-status"; boundary="mime_boundary" --mime_boundary Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable There was a temporary problem delivering your message to the following recip= ients: (connection to 'mx.domain.org' failed: Connection tim= eout) --mime_boundary Content-Type: message/delivery-status; charset="utf-8" Content-Transfer-Encoding: 7bit Reporting-MTA: dns;mx.example.org Arrival-Date: Original-Recipient: rfc822;jdoe@example.org Final-Recipient: rfc822;john.doe@example.org Action: delayed Status: 4.0.0 Remote-MTA: dns;mx.domain.org Will-Retry-Until: --mime_boundary Content-Type: message/rfc822; charset="utf-8" Content-Transfer-Encoding: 7bit Disclose-recipients: prohibited From: Message Router Submission Agent Subject: Status of: Re: Battery current sense To: owner-ups-mib@CS.UTK.EDU Message-id: <01HEGJ0WNBY28Y95LN@mr.timeplex.com> MIME-version: 1.0 Content-Type: text/plain --mime_boundary-- ================================================ FILE: tests/resources/smtp/dsn/failure.eml ================================================ From: "Mail Delivery Subsystem" To: sender@foobar.org Auto-Submitted: auto-generated Subject: Failed to deliver message MIME-Version: 1.0 Content-Type: multipart/report; report-type="delivery-status"; boundary="mime_boundary" --mime_boundary Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Your message could not be delivered to the following recipients: (host 'mx.example.org' rejected command 'RCPT TO:' with code 550 (5.1.2) 'User does not exist') --mime_boundary Content-Type: message/delivery-status; charset="utf-8" Content-Transfer-Encoding: 7bit Reporting-MTA: dns;mx.example.org Arrival-Date: Final-Recipient: rfc822;foobar@example.org Action: failed Status: 5.1.2 Diagnostic-Code: smtp;550 User does not exist Remote-MTA: dns;mx.example.org --mime_boundary Content-Type: message/rfc822; charset="utf-8" Content-Transfer-Encoding: 7bit Disclose-recipients: prohibited From: Message Router Submission Agent Subject: Status of: Re: Battery current sense To: owner-ups-mib@CS.UTK.EDU Message-id: <01HEGJ0WNBY28Y95LN@mr.timeplex.com> MIME-version: 1.0 Content-Type: text/plain --mime_boundary-- ================================================ FILE: tests/resources/smtp/dsn/mixed.eml ================================================ From: "Mail Delivery Subsystem" To: sender@foobar.org Auto-Submitted: auto-generated Subject: Partially delivered message MIME-Version: 1.0 Content-Type: multipart/report; report-type="delivery-status"; boundary="mime_boundary" --mime_boundary Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Your message has been partially delivered: ----- Delivery to the following addresses was successful ----- (delivered to 'mx2.example.org' with code 250 (2.1.5) 'Me= ssage accepted for delivery') ----- There was a temporary problem delivering to these addresses ----- (connection to 'mx.domain.org' failed: Connection tim= eout) ----- Delivery to the following addresses failed ----- (host 'mx.example.org' rejected command 'RCPT TO:' with code 550 (5.1.2) 'User does not exist') --mime_boundary Content-Type: message/delivery-status; charset="utf-8" Content-Transfer-Encoding: 7bit Reporting-MTA: dns;mx.example.org Arrival-Date: Final-Recipient: rfc822;foobar@example.org Action: failed Status: 5.1.2 Diagnostic-Code: smtp;550 User does not exist Remote-MTA: dns;mx.example.org Final-Recipient: rfc822;jane@example.org Action: delivered Status: 2.1.5 Remote-MTA: dns;mx2.example.org Original-Recipient: rfc822;jdoe@example.org Final-Recipient: rfc822;john.doe@example.org Action: delayed Status: 4.0.0 Remote-MTA: dns;mx.domain.org Will-Retry-Until: --mime_boundary Content-Type: message/rfc822; charset="utf-8" Content-Transfer-Encoding: 7bit Disclose-recipients: prohibited From: Message Router Submission Agent Subject: Status of: Re: Battery current sense To: owner-ups-mib@CS.UTK.EDU Message-id: <01HEGJ0WNBY28Y95LN@mr.timeplex.com> MIME-version: 1.0 Content-Type: text/plain --mime_boundary-- ================================================ FILE: tests/resources/smtp/dsn/original.txt ================================================ Disclose-recipients: prohibited Date: Fri, 08 Jul 1994 09:21:25 -0400 (EDT) From: Message Router Submission Agent Subject: Status of: Re: Battery current sense To: owner-ups-mib@CS.UTK.EDU Message-id: <01HEGJ0WNBY28Y95LN@mr.timeplex.com> MIME-version: 1.0 Content-Type: text/plain ================================================ FILE: tests/resources/smtp/dsn/success.eml ================================================ From: "Mail Delivery Subsystem" To: sender@foobar.org Auto-Submitted: auto-generated Subject: Successfully delivered message MIME-Version: 1.0 Content-Type: multipart/report; report-type="delivery-status"; boundary="mime_boundary" --mime_boundary Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Your message has been successfully delivered to the following recipients: (delivered to 'mx2.example.org' with code 250 (2.1.5) 'Me= ssage accepted for delivery') --mime_boundary Content-Type: message/delivery-status; charset="utf-8" Content-Transfer-Encoding: 7bit Reporting-MTA: dns;mx.example.org Arrival-Date: Final-Recipient: rfc822;jane@example.org Action: delivered Status: 2.1.5 Remote-MTA: dns;mx2.example.org --mime_boundary Content-Type: message/rfc822; charset="utf-8" Content-Transfer-Encoding: 7bit Disclose-recipients: prohibited From: Message Router Submission Agent Subject: Status of: Re: Battery current sense To: owner-ups-mib@CS.UTK.EDU Message-id: <01HEGJ0WNBY28Y95LN@mr.timeplex.com> MIME-version: 1.0 Content-Type: text/plain --mime_boundary-- ================================================ FILE: tests/resources/smtp/lists/test-list1.txt ================================================ user1@domain.org user2@domain.org ================================================ FILE: tests/resources/smtp/lists/test-list2.txt ================================================ user3@example.net user4@example.net user5@example.net ================================================ FILE: tests/resources/smtp/messages/arc.eml ================================================ ARC-Seal: i=2; a=rsa-sha256; s=rsa; d=manchego.org; cv=pass; b=wpAAy6QusmF4O8SeziNaKxXL6EleeBYxQ0HrXl2cDgzHLOvYG0N1Wpz0bpVbA8VgteD2X8XCW yrdlZ5dIPTcCvgfLGLXLRTIcYUdKyfFh5IVEciaUOUsxlSRPpekENZKzdHFkL4j1mAAvpDNJ7Ft OFIp0ku5dACn80g7D4cSEU0=; ARC-Message-Signature: i=2; a=rsa-sha256; s=rsa; d=manchego.org; c=relaxed/relaxed; h=Subject:To:From:DKIM-Signature; t=1674137914; bh=4ET7siw2kYV7jcN+fzsuYng/ sr/BmIzzEjh43dVAv40=; b=V3tMBI1RsyJJY7HUABcebHf0mDJ9odbPm++ZMY5AsCaUYNoSsAm wCf5wYlJQ26KmsluOYXoPwML0a/xvnMXPv6Rs4Z9k4IwzpzhGLsijDXymGPsW3hgq/6ivVTPkwU +pGSCC70rHNrAFFk5P67Ly0tbGYjJ0wZVHBzqL8IJBXK4=; ARC-Authentication-Results: i=2; manchego.org; dkim=pass header.d=manchego.org header.s=rsa header.b=IN4oMvqq Authentication-Results: manchego.org; dkim=pass header.d=manchego.org header.s=rsa header.b=IN4oMvqq ARC-Seal: i=1; a=ed25519-sha256; s=ed; d=scamorza.org; cv=none; b=k/MAHECtaer9v4oczoe00a6XMjrxU4QUVVPlZI8XYegbiOgDSaeR6IrwBSKVcN0ELYU+HXlNW RuUGkRuZXQODA==; ARC-Message-Signature: i=1; a=ed25519-sha256; s=ed; d=scamorza.org; c=relaxed/relaxed; h=Subject:To:From:DKIM-Signature; t=1674137914; bh=4ET7siw2kYV7jcN+fzsuYng/ sr/BmIzzEjh43dVAv40=; b=ZVPqB/5+mbOEKIgBsq+S71Sfj2JZUlGmYEA0Ygbj0S1VmTAnsVu FQSInMY4/qcIeqU23BtzMgCFVZfAg5i3zDw==; ARC-Authentication-Results: i=1; scamorza.org; dkim=pass header.d=manchego.org header.s=rsa header.b=IN4oMvqq Authentication-Results: scamorza.org; dkim=pass header.d=manchego.org header.s=rsa header.b=IN4oMvqq DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=manchego.org; c=relaxed/relaxed; h=Subject:To:From; t=1674137914; bh=4ET7siw2kYV7jcN+fzsuYng/sr/BmIzzEjh43dV Av40=; b=IN4oMvqqxWCEyC38F7fZecYJcnq+7zP3G/xjcI64M3/Dzys2lmQeLYAXipwwYvEa5a VwCcJ7XUX0kSxtr6igC8FIJEDI6UmdvJgMEj/hnEjXR8m4GPrphigjJy7hagaQymBT9WhlzsDPI QRlUVoW0y5v1aDp3KF9bLVCKTELJPM=; From: queso@manchego.org To: affumicata@scamorza.org Subject: Say cheese We need to settle which one of us is tastier. ================================================ FILE: tests/resources/smtp/messages/dkim.eml ================================================ DKIM-Signature: v=1; a=rsa-sha256; s=default; d=example.com; c=relaxed/relaxed; r=y; h=Subject:To:From; t=1674122129; bh=Xcxymouf0VhlJ7c/vHLAM3LPTUR4LKFKX7PRNni WCEs=; b=m5lYqx81xqAIo4ZBC9FDiFIBrRnnep+taSsutc5MkbBQvf9/Lb54AXHhruEdO2EGkG xUxL1c8QDH3eLz84fPTUgZue84tAsAa0q4gJFIYM5q2/GJvJ6cBvsXKZj82FjRTIz4wlLjzkW7p NdR9C5CID2PO4sW+GymS45F8hwqSj4=; DKIM-Signature: v=1; a=ed25519-sha256; s=ed; d=example.com; c=relaxed/relaxed; h=Subject:To:From; t=1674122129; bh=Xcxymouf0VhlJ7c/vHLAM3LPTUR4LKFKX7PRNni WCEs=; b=t8z1AsaxeWek+gMSVojbs2QJu+orzeR4CiHVquJYvXzv+Eb52Wq0fEmaOxoyY1teVL Odp57Vq/zTLjMMZ2hbBQ==; From: bill@example.com To: jdoe@example.com Subject: TPS Report I'm going to need those TPS reports ASAP. So, if you could do that, that'd be great. ================================================ FILE: tests/resources/smtp/messages/invalid_arc.eml ================================================ ARC-Seal: i=1; a=rsa-sha256; s=rsa; d=manchego.org; cv=fail; b=wpAAy6QusmF4O8SeziNaKxXL6EleeBYxQ0HrXl2cDgzHLOvYG0N1Wpz0bpVbA8VgteD2X8XCW yrdlZ5dIPTcCvgfLGLXLRTIcYUdKyfFh5IVEciaUOUsxlSRPpekENZKzdHFkL4j1mAAvpDNJ7Ft OFIp0ku5dACn80g7D4cSEU0=; ARC-Message-Signature: i=1; a=rsa-sha256; s=rsa; d=manchego.org; c=relaxed/relaxed; h=Subject:To:From:DKIM-Signature; t=1674137914; bh=4ET7siw2kYV7jcN+fzsuYng/ sr/BmIzzEjh43dVAv40=; b=V3tMBI1RsyJJY7HUABcebHf0mDJ9odbPm++ZMY5AsCaUYNoSsAm wCf5wYlJQ26KmsluOYXoPwML0a/xvnMXPv6Rs4Z9k4IwzpzhGLsijDXymGPsW3hgq/6ivVTPkwU +pGSCC70rHNrAFFk5P67Ly0tbGYjJ0wZVHBzqL8IJBXK4=; ARC-Authentication-Results: i=1; manchego.org; dkim=pass header.d=manchego.org header.s=rsa header.b=IN4oMvqq DKIM-Signature: v=1; a=rsa-sha256; s=default; d=example.com; c=relaxed/relaxed; r=y; h=Subject:To:From; t=1674122129; bh=Xcxymouf0VhlJ7c/vHLAM3LPTUR4LKFKX7PRNni WCEs=; b=m5lYqx81xqAIo4ZBC9FDiFIBrRnnep+taSsutc5MkbBQvf9/Lb54AXHhruEdO2EGkG xUxL1c8QDH3eLz84fPTUgZue84tAsAa0q4gJFIYM5q2/GJvJ6cBvsXKZj82FjRTIz4wlLjzkW7p NdR9C5CID2PO4sW+GymS45F8hwqSj4=; DKIM-Signature: v=1; a=ed25519-sha256; s=ed; d=example.com; c=relaxed/relaxed; h=Subject:To:From; t=1674122129; bh=Xcxymouf0VhlJ7c/vHLAM3LPTUR4LKFKX7PRNni WCEs=; b=t8z1AsaxeWek+gMSVojbs2QJu+orzeR4CiHVquJYvXzv+Eb52Wq0fEmaOxoyY1teVL Odp57Vq/zTLjMMZ2hbBQ==; From: bill@example.com To: jdoe@example.com Subject: TPS Report I'm going to need those TPS reports ASAP. So, if you could do that, that'd be great. ================================================ FILE: tests/resources/smtp/messages/invalid_dkim.eml ================================================ DKIM-Signature: v=1; a=rsa-sha256; s=default; d=example.com; c=relaxed/relaxed; r=y; h=Subject:To:From; t=1674122129; bh=Xcxymouf0VhlJ7c/vHLAM3LPTUR4LKFKX7PRNni WCEs=; b=m5lYqx81xqAIo4ZBC9FDiFIBrRnnep+taSsutc5MkbBQvf9/Lb54AXHhruEdO2EGkG xUxL1c8QDH3eLz84fPTUgZue84tAsAa0q4gJFIYM5q2/GJvJ6cBvsXKZj82FjRTIz4wlLjzkW7p NdR9C5CID2PO4sW+GymS45F8hwqSj4=; DKIM-Signature: v=1; a=ed25519-sha256; s=ed; d=example.com; c=relaxed/relaxed; h=Subject:To:From; t=1674122129; bh=Xcxymouf0VhlJ7c/vHLAM3LPTUR4LKFKX7PRNni WCEs=; b=t8z1AsaxeWek+gMSVojbs2QJu+orzeR4CiHVquJYvXzv+Eb52Wq0fEmaOxoyY1teVL Odp57Vq/zTLjMMZ2hbBQ==; From: bill@example.com To: jdoe@example.com Subject: TPS Report Body hash will not match. ================================================ FILE: tests/resources/smtp/messages/loop.eml ================================================ Received: from client1.football.example.com [192.0.2.1] by submitserver.example.com with SUBMISSION; Fri, 11 Jul 2003 21:01:54 -0700 (PDT) Received: from client1.football.example.com [192.0.2.1] by submitserver.example.com with SUBMISSION; Fri, 11 Jul 2003 21:01:54 -0700 (PDT) Received: from client1.football.example.com [192.0.2.1] by submitserver.example.com with SUBMISSION; Fri, 11 Jul 2003 21:01:54 -0700 (PDT) Received: from client1.football.example.com [192.0.2.1] by submitserver.example.com with SUBMISSION; Fri, 11 Jul 2003 21:01:54 -0700 (PDT) From: Joe SixPack To: Suzie Q Subject: Is dinner ready? Date: Fri, 11 Jul 2003 21:00:37 -0700 (PDT) Message-ID: <20030712040037.46341.5F8J@football.example.com> Hi. We lost the game. Are you hungry yet? Joe. ================================================ FILE: tests/resources/smtp/messages/multipart.eml ================================================ From: Hendrik To: Harrie Date: Sat, 11 Oct 2010 00:31:44 +0200 Subject: One Two Three Four Content-Type: multipart/mixed; boundary=AA This is a multi-part message in MIME format. --AA Content-Type: multipart/mixed; boundary=BB This is a multi-part message in MIME format. --BB Content-Type: text/plain; charset="us-ascii" This is the first message part containing plain text. --BB Content-Type: text/plain; charset="us-ascii" This is another plain text message part. --BB-- This is the end of MIME multipart. --AA Content-Type: text/html; charset="us-ascii" This is a piece of HTML text. --AA-- This is the end of MIME multipart. ================================================ FILE: tests/resources/smtp/messages/no_dkim.eml ================================================ Received: from client1.football.example.com [192.0.2.1] by submitserver.example.com with SUBMISSION; Fri, 11 Jul 2003 21:01:54 -0700 (PDT) From: Joe SixPack To: Suzie Q Subject: Is dinner ready? Date: Fri, 11 Jul 2003 21:00:37 -0700 (PDT) Message-ID: <20030712040037.46341.5F8J@football.example.com> Hi. We lost the game. Are you hungry yet? Joe. ================================================ FILE: tests/resources/smtp/messages/no_msgid.eml ================================================ From: Joe SixPack To: Suzie Q Subject: Is dinner ready? Hi. We lost the game. Are you hungry yet? Joe. ================================================ FILE: tests/resources/smtp/milter/message.eml ================================================ From: John Doe To: Mary Smith References: a References: b X-Mailer: Test X-1: 1 X-2: 2 X-3: 3 Subject: Saying Hello This is a message just to say hello. ================================================ FILE: tests/resources/smtp/milter/message.json ================================================ [ { "modifications": [ { "AddHeader": { "name": "X-Hello", "value": "World" } }, { "AddHeader": { "name": "X-CR", "value": "LF\r\n" } } ], "result": "X-Hello: World\r\nX-CR: LF\r\nFrom: John Doe \r\nTo: Mary Smith \r\nReferences: a\r\nReferences: b\r\nX-Mailer: Test\r\nX-1: 1\r\nX-2: 2\r\nX-3: 3\r\nSubject: Saying Hello\r\n\r\nThis is a message just to say hello.\r\n" }, { "modifications": [ { "ReplaceBody": { "value": [ 49, 50, 51 ] } } ], "result": "From: John Doe \r\nTo: Mary Smith \r\nReferences: a\r\nReferences: b\r\nX-Mailer: Test\r\nX-1: 1\r\nX-2: 2\r\nX-3: 3\r\nSubject: Saying Hello\r\n\r\n123" }, { "modifications": [ { "AddHeader": { "name": "X-Spam", "value": "Yes" } }, { "ReplaceBody": { "value": [ 49, 50, 51 ] } }, { "ReplaceBody": { "value": [ 52, 53, 54 ] } } ], "result": "X-Spam: Yes\r\nFrom: John Doe \r\nTo: Mary Smith \r\nReferences: a\r\nReferences: b\r\nX-Mailer: Test\r\nX-1: 1\r\nX-2: 2\r\nX-3: 3\r\nSubject: Saying Hello\r\n\r\n123456" }, { "modifications": [ { "ChangeHeader": { "index": 1, "name": "References", "value": "" } }, { "ChangeHeader": { "index": 1, "name": "References", "value": "z" } }, { "ChangeHeader": { "index": 1, "name": "Subject", "value": "[SPAM] Saying Hello" } } ], "result": "From: John Doe \r\nTo: Mary Smith \r\nReferences: z\r\nX-Mailer: Test\r\nX-1: 1\r\nX-2: 2\r\nX-3: 3\r\nSubject: [SPAM] Saying Hello\r\n\r\nThis is a message just to say hello.\r\n" }, { "modifications": [ { "ChangeHeader": { "index": 1, "name": "X-Some-Header", "value": "Some Value" } }, { "InsertHeader": { "index": 2, "name": "References", "value": "" } }, { "InsertHeader": { "index": 10, "name": "X-3", "value": "z" } }, { "ReplaceBody": { "value": [ 52, 53, 54 ] } }, { "ReplaceBody": { "value": [ 49, 50, 51 ] } } ], "result": "X-Some-Header: Some Value\r\nFrom: John Doe \r\nTo: Mary Smith \r\nReferences: a\r\nReferences: \r\nReferences: b\r\nX-Mailer: Test\r\nX-1: 1\r\nX-2: 2\r\nX-3: z\r\nX-3: 3\r\nSubject: Saying Hello\r\n\r\n456123" }, { "modifications": [ { "Quarantine": { "reason": "Virus found!" } }, { "InsertHeader": { "index": 1, "name": "References", "value": "" } } ], "result": "X-Quarantine: Virus found!\r\nFrom: John Doe \r\nTo: Mary Smith \r\nReferences: \r\nReferences: a\r\nReferences: b\r\nX-Mailer: Test\r\nX-1: 1\r\nX-2: 2\r\nX-3: 3\r\nSubject: Saying Hello\r\n\r\nThis is a message just to say hello.\r\n" } ] ================================================ FILE: tests/resources/smtp/reports/arf1.eml ================================================ From: Date: Thu, 8 Mar 2005 17:40:36 EDT Subject: FW: Earn money To: MIME-Version: 1.0 Content-Type: multipart/report; report-type=feedback-report; boundary="part1_13d.2e68ed54_boundary" --part1_13d.2e68ed54_boundary Content-Type: text/plain; charset="US-ASCII" Content-Transfer-Encoding: 7bit This is an email abuse report for an email message received from IP 192.0.2.1 on Thu, 8 Mar 2005 14:00:00 EDT. For more information about this format please see http://www.mipassoc.org/arf/. --part1_13d.2e68ed54_boundary Content-Type: message/feedback-report Feedback-Type: abuse User-Agent: SomeGenerator/1.0 Version: 1 --part1_13d.2e68ed54_boundary Content-Type: message/rfc822 Content-Disposition: inline Received: from mailserver.example.net (mailserver.example.net [192.0.2.1]) by example.com with ESMTP id M63d4137594e46; Thu, 08 Mar 2005 14:00:00 -0400 From: To: Subject: Earn money MIME-Version: 1.0 Content-type: text/plain Message-ID: 8787KJKJ3K4J3K4J3K4J3.mail@example.net Date: Thu, 02 Sep 2004 12:31:03 -0500 Spam Spam Spam Spam Spam Spam Spam Spam Spam Spam Spam Spam --part1_13d.2e68ed54_boundary-- ================================================ FILE: tests/resources/smtp/reports/arf2.eml ================================================ From: Date: Thu, 8 Mar 2005 17:40:36 EDT Subject: FW: Earn money To: MIME-Version: 1.0 Content-Type: multipart/report; report-type=feedback-report; boundary="part1_13d.2e68ed54_boundary" --part1_13d.2e68ed54_boundary Content-Type: text/plain; charset="US-ASCII" Content-Transfer-Encoding: 7bit This is an email abuse report for an email message received from IP 192.0.2.1 on Thu, 8 Mar 2005 14:00:00 EDT. For more information about this format please see http://www.mipassoc.org/arf/. --part1_13d.2e68ed54_boundary Content-Type: message/feedback-report Feedback-Type: abuse User-Agent: SomeGenerator/1.0 Version: 1 Original-Mail-From: Original-Rcpt-To: Arrival-Date: Thu, 8 Mar 2005 14:00:00 EDT Reporting-MTA: dns; mail.example.com Source-IP: 192.0.2.1 Authentication-Results: mail.example.com; spf=fail smtp.mail=somespammer@example.com Reported-Domain: example.net Reported-Uri: http://example.net/earn_money.html Reported-Uri: mailto:user@example.com Removal-Recipient: user@example.com --part1_13d.2e68ed54_boundary Content-Type: message/rfc822 Content-Disposition: inline From: Received: from mailserver.example.net (mailserver.example.net [192.0.2.1]) by example.com with ESMTP id M63d4137594e46; Thu, 08 Mar 2005 14:00:00 -0400 To: Subject: Earn money MIME-Version: 1.0 Content-type: text/plain Message-ID: 8787KJKJ3K4J3K4J3K4J3.mail@example.net Date: Thu, 02 Sep 2004 12:31:03 -0500 Spam Spam Spam Spam Spam Spam Spam Spam Spam Spam Spam Spam --part1_13d.2e68ed54_boundary-- ================================================ FILE: tests/resources/smtp/reports/arf3.eml ================================================ From: arf-daemon@example.com To: recipient@example.net Subject: This is a test Date: Wed, 14 Apr 2010 12:17:45 -0700 (PDT) MIME-Version: 1.0 Content-Type: multipart/report; report-type=feedback-report; boundary="part1_13d.2e68ed54_boundary" --part1_13d.2e68ed54_boundary Content-Type: text/plain; charset="US-ASCII" Content-Transfer-Encoding: 7bit This is an email abuse report for an email message received from IP 192.0.2.1 on Wed, 14 Apr 2010 12:15:31 PDT. For more information about this format please see http://www.mipassoc.org/arf/. --part1_13d.2e68ed54_boundary Content-Type: message/feedback-report Feedback-Type: auth-failure User-Agent: SomeDKIMFilter/1.0 Version: 1 Original-Mail-From: Original-Rcpt-To: Received-Date: Wed, 14 Apr 2010 12:15:31 -0700 (PDT) Source-IP: 192.0.2.1 Authentication-Results: mail.example.com; dkim=fail header.d=example.net Reported-Domain: example.net DKIM-Domain: example.net Auth-Failure: bodyhash --part1_13d.2e68ed54_boundary Content-Type: message/rfc822 DKIM-Signature: v=1; c=relaxed/simple; a=rsa-sha256; s=testkey; d=example.net; h=From:To:Subject:Date; bh=2jUSOH9NhtVGCQWNr9BrIAPreKQjO6Sn7XIkfJVOzv8=; b=AuUoFEfDxTDkHlLXSZEpZj79LICEps6eda7W3deTVFOk4yAUoqOB 4nujc7YopdG5dWLSdNg6xNAZpOPr+kHxt1IrE+NahM6L/LbvaHut KVdkLLkpVaVVQPzeRDI009SO2Il5Lu7rDNH6mZckBdrIx0orEtZV 4bmp/YzhwvcubU4= Received: from smtp-out.example.net by mail.example.com with SMTP id o3F52gxO029144; Wed, 14 Apr 2010 12:15:31 -0700 (PDT) Received: from internal-client-001.example.com by mail.example.com with SMTP id o3F3BwdY028431; Wed, 14 Apr 2010 12:12:09 -0700 (PDT) From: randomuser@example.net To: user@example.com Date: Wed, 14 Apr 2010 12:12:09 -0700 (PDT) Subject: This is a test Hi, just making sure DKIM is working! --part1_13d.2e68ed54_boundary-- ================================================ FILE: tests/resources/smtp/reports/arf4.eml ================================================ Return-Path: Received: by box.mydomain.name (Postfix, from userid 116) id CF8FA658E0; Tue, 5 Oct 2021 17:37:02 +1300 (NZDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=box.mydomain.name; s=mail; t=1633408622; bh=yDlkGfe4dFwlsFeoKaIHG6xiRgQs2/PqPnLtiPm5ewk=; h=From:To:Date:Subject:From; b=TwXvJJFoFwJDcb6IKMKsxp2BiRDsrjLOESyQPh/Cc4tRZltVAud/k6f0XP4l5a/T8 kh0iDOGImc0O1WZNFt0MUcwLsfW4qbYjCBtthQDnbPApvv6MJDASwau+wipu5Nrkjc flg+nMaD97pVgR0LevMoVIWoiy1f5PNC/z0xkY2wnvyoGn91WuDsdocOqyoPo4RmIT A/f3M4CjOv/QmMEAWBIsa7kAZwf+rNmzDahFOtp2vFLqHt0iZi5vs40fa6O/I0snTM fRkv2GMZAug7NMU8MN/MhuO87FV6ATZXvB0Kxvsy9z0zZYK7tM1OYHjiCYot45erG3 dlKrYiXsfd3BQ== From: OpenDMARC Filter To: postmaster@vericty.interpublication.org Date: Tue, 5 Oct 2021 17:37:02 +1300 (NZDT) Subject: FW: Wir kaufen dein Auto! MIME-Version: 1.0 Content-Type: multipart/report; report-type=feedback-report; boundary="box.mydomain.name:8BE2660E72" Message-Id: <20211005043702.CF8FA658E0@box.mydomain.name> --box.mydomain.name:8BE2660E72 Content-Type: text/plain This is an authentication failure report for an email message received from IP 148.163.85.135 on Tue, 5 Oct 2021 17:37:02 +1300 (NZDT). --box.mydomain.name:8BE2660E72 Content-Type: message/feedback-report Feedback-Type: auth-failure Version: 1 User-Agent: OpenDMARC-Filter/1.3.2 Auth-Failure: dmarc Authentication-Results: box.mydomain.name; dmarc=fail header.from=interpublication.org Original-Envelope-Id: 8BE2660E72 Original-Mail-From: info@interpublication.org Source-IP: 148.163.85.135 (sainay.interpublication.org) Reported-Domain: interpublication.org --box.mydomain.name:8BE2660E72 Content-Type: text/rfc822-headers Authentication-Results: box.mydomain.name; dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=interpublication.org header.i=@interpublication.org header.b="PrsTNnuH"; dkim-atps=neutral Received: from dslb-002-202-150-127.002.202.pools.vodafone-ip.de (dslb-188-099-080-029.188.099.pools.vodafone-ip.de [188.99.80.29]) by sainay.interpublication.org (Postfix) with ESMTPA id 6BB23A2D3 for ; Tue, 5 Oct 2021 00:36:52 -0400 (EDT) DKIM-Filter: OpenDKIM Filter v2.11.0 sainay.interpublication.org 6BB23A2D3 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=interpublication.org; s=default; t=1633408612; bh=q1/OPSn+VXteY2+DHXqOIgs5LsNCJisEcQIKVW9it6I=; h=From:Subject:To:Reply-To:Date:From; b=PrsTNnuH8D0Ch3gcWqGmXiYc2Kvu1CHGJBsqS521uYazd3G/urp7MHQvmNwK0r1gS DR3A3KwGejI5uuqzxDCqz28Mq6AkdTkOjFyXw65MLlsKTQddWTgciVnoqJempa6yzw PSM5550XqVFqqkNxEcYBUBYEwUdy1tY8rc4zhq8cIrsonQVxJJSbc3cdonICM1kLBV WASv16p3376ZBcKqFLc8UQ58YQKaFm51VZGEjtabfmWbgOQ7VikFFECDG3aRt8fZa6 D03MrzUSngwPUdcRQZuqS/sApW/a9N2YwdbR51OFzPBr4ypUEIw/qprgBG4BfQQKeS 1PhinNvVtgQpQ== From: "Rolf Bader" Subject: Wir kaufen dein Auto! To: "address" Content-Type: multipart/alternative; boundary="TD6gM3Blv=_XBZYNFT7dCsH1DHHOKUuSyA" MIME-Version: 1.0 Reply-To: "Rolf Bader" Organization: AutoTEAM24 Date: Tue, 5 Oct 2021 06:36:51 +0200 --box.mydomain.name:8BE2660E72-- ================================================ FILE: tests/resources/smtp/reports/arf5.eml ================================================ Message-ID: <433689.81121.example@mta.mail.receiver.example> From: "SomeISP Antispam Feedback" To: arf-failure@sender.example Subject: FW: You have a new bill from your bank Date: Sat, 8 Oct 2011 15:15:59 -0500 (CDT) MIME-Version: 1.0 Content-Type: multipart/report; boundary="------------Boundary-00=_3BCR4Y7kX93yP9uUPRhg"; report-type=feedback-report Content-Transfer-Encoding: 7bit --------------Boundary-00=_3BCR4Y7kX93yP9uUPRhg Content-Type: text/plain; charset="us-ascii" Content-Disposition: inline Content-Transfer-Encoding: 7bit This is an authentication failure report for an email message received from a.sender.example on 8 Oct 2011 20:15:58 +0000 (GMT). For more information about this format, please see [RFC6591]. --------------Boundary-00=_3BCR4Y7kX93yP9uUPRhg Content-Type: message/feedback-report Content-Transfer-Encoding: 7bit Feedback-Type: auth-failure User-Agent: Someisp!Mail-Feedback/1.0 Version: 1 Original-Mail-From: anexample.reply@a.sender.example Original-Envelope-Id: o3F52gxO029144 Authentication-Results: mta1011.mail.tp2.receiver.example; dkim=fail (bodyhash) header.d=sender.example Auth-Failure: bodyhash DKIM-Canonicalized-Body: VGhpcyBpcyBhIG1lc3NhZ2UgYm9keSB0 aGF0IGdvdCBtb2RpZmllZCBpbiB0cmFuc2l0LgoKQXQgdGhlIHNhbWU gdGltZSB0aGF0IHRoZSBib2R5aGFzaCBmYWlscyB0byB2ZXJpZnksIH RoZQptZXNzYWdlIGNvbnRlbnQgaXMgY2xlYXJseSBhYnVzaXZlIG9yI HBoaXNoeSwgYXMgdGhlClN1YmplY3QgYWxyZWFkeSBoaW50cy4gIElu ZGVlZCwgdGhpcyBib2R5IGFsc28gY29udGFpbnMKdGhlIGZvbGxvd2l uZyB0ZXh0OgoKICAgUGxlYXNlIGVudGVyIHlvdXIgZnVsbCBiYW5rIG NyZWRlbnRpYWxzIGF0CiAgIGh0dHA6Ly93d3cuc2VuZGVyLmV4YW1wb GUvCgpXZSBhcmUgaW1wbHlpbmcgdGhhdCwgYWx0aG91Z2ggbXVsdGlw bGUgZmFpbHVyZXMKcmVxdWlyZSBtdWx0aXBsZSByZXBvcnRzLCBhIHN pbmdsZSBmYWlsdXJlIGNhbiBiZQpyZXBvcnRlZCBhbG9uZyB3aXRoIH BoaXNoaW5nIGluIGEgc2luZ2xlIHJlcG9ydC4K DKIM-Domain: sender.example DKIM-Identity: @sender.example DKIM-Selector: testkey Arrival-Date: 8 Oct 2011 20:15:58 +0000 (GMT) Source-IP: 192.0.2.1 Reported-Domain: a.sender.example Reported-URI: http://www.sender.example/ --------------Boundary-00=_3BCR4Y7kX93yP9uUPRhg Content-Type: text/rfc822-headers Content-Transfer-Encoding: 7bit Authentication-Results: mta1011.mail.tp2.receiver.example; dkim=fail (bodyhash) header.d=sender.example; spf=pass smtp.mailfrom=anexample.reply@a.sender.example Received: from smtp-out.sender.example by mta1011.mail.tp2.receiver.example with SMTP id oB85W8xV000169; Sat, 08 Oct 2011 13:15:58 -0700 (PDT) DKIM-Signature: v=1; c=relaxed/simple; a=rsa-sha256; s=testkey; d=sender.example; h=From:To:Subject:Date; bh=2jUSOH9NhtVGCQWNr9BrIAPreKQjO6Sn7XIkfJVOzv8=; b=AuUoFEfDxTDkHlLXSZEpZj79LICEps6eda7W3deTVFOk4yAUoqOB 4nujc7YopdG5dWLSdNg6xNAZpOPr+kHxt1IrE+NahM6L/LbvaHut KVdkLLkpVaVVQPzeRDI009SO2Il5Lu7rDNH6mZckBdrIx0orEtZV 4bmp/YzhwvcubU4= Received: from mail.sender.example by smtp-out.sender.example with SMTP id o3F52gxO029144; Sat, 08 Oct 2011 13:15:31 -0700 (PDT) Received: from internal-client-001.sender.example by mail.sender.example with SMTP id o3F3BwdY028431; Sat, 08 Oct 2011 13:15:24 -0700 (PDT) Date: Sat, 8 Oct 2011 16:15:24 -0400 (EDT) Reply-To: anexample.reply@a.sender.example From: anexample@a.sender.example To: someuser@receiver.example Subject: You have a new bill from your bank Message-ID: <87913910.1318094604546@out.sender.example> --------------Boundary-00=_3BCR4Y7kX93yP9uUPRhg-- ================================================ FILE: tests/resources/smtp/reports/dmarc1.eml ================================================ Received: from mail.stalw.art ([mail.stalw.art]) by 127.0.0.1 (Stalwart JMAP) with LMTP; Mon, 28 Nov 2022 10:51:56 +0000 Received: from mail-qv1-xf4a.google.com (mail-qv1-xf4a.google.com [IPv6:2607:f8b0:4864:20::f4a]) (using TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 (128/128 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by mail.stalw.art (Postfix) with ESMTPS id 1145E7CC0B for ; Mon, 28 Nov 2022 10:51:53 +0000 (UTC) Received: by mail-qv1-xf4a.google.com with SMTP id 71-20020a0c804d000000b004b2fb260447so12985969qva.10 for ; Mon, 28 Nov 2022 02:51:52 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20210112; h=content-transfer-encoding:content-disposition:to:from:subject :message-id:date:mime-version:from:to:cc:subject:date:message-id :reply-to; bh=sMF/38UFRhmUYFRJST4vLBu/U1BXgsdCUE02HF8nXx8=; b=I7WONP7tMsULp4eKjJeeKtM+nDYqMSIYMxqNHqCP1bTsnUiW2xM278I2+F8EjtFNYf XOgusNn8kqbSnA4w1+q4G87zTF4K3tGnxNpuUMQ7GzcofBKtr7VPv9XFqvTPJ+N8YSwe 926ec7xi71BpSHAgqp5Wqocj8ruIVjcCZ37hYrG0C4s+FVBtbaU3EeyPpkESaaY2vE5y Qa2KsrMsyJXlbyW/sFJ7AGDDuXwyGkTa+btP/xIiQM2HlBKy7vNOFZKkxInOuQsXJgZy 3H7ivlpD3hMrszwU77o5jBArVwN0RIkUSosAPQf+pzgvRlkseRlDrmzKQutvYWIaTP3/ FHPA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:content-disposition:to:from:subject :message-id:date:mime-version:x-gm-message-state:from:to:cc:subject :date:message-id:reply-to; bh=sMF/38UFRhmUYFRJST4vLBu/U1BXgsdCUE02HF8nXx8=; b=D3oClvT5AKcTpEjjffHQqPPQ9j5mmtExiviSq7iBYkoq+322LtR2hGqGxtvlAwRDsQ VIfuKVExygw3c9bckjzKtJYX128HGK35gHnmsrzqvCC93JlRaC/55kcM9Bhks0xJnl7i yNFHPZ0DY/jasdUdQ1QqnI+8qiPy+/12JvD+/TGlaDuS+RWYFU4/ky46S3vMXwXmRt6D IGggXoW7snSaM4s88DzMUl0U7DH823UPQrUxnA5Oxscwn9M1ENJUWD/3EJo5ZEUMw0ll Y8AlyhjWFgVqs1Y4V/LVWeXdF10fpm78+jm8QyIZYZJjh4I33AekdsWVM71ZNNYGL0+8 +GDg== X-Gm-Message-State: ANoB5pn0zRZSWXdFXd9G0tawbSeUYuhxToVkIYoLf8OJzoBLcIc0wKcB AfU9Coz5vAuiM1mASWJhbg== X-Google-Smtp-Source: AA0mqf6WWsnqMD4cHE40jB89/zblmT7yNKeHKlsvvCmlYANKmpKLTQTaCm5qCA0mVmxR/PTQogsNndWH/qe0ug== MIME-Version: 1.0 X-Received: by 2002:ac8:5182:0:b0:39c:cb6a:300b with SMTP id c2-20020ac85182000000b0039ccb6a300bmr48409299qtn.181.1669632711968; Mon, 28 Nov 2022 02:51:51 -0800 (PST) Date: Sun, 27 Nov 2022 15:59:59 -0800 Message-ID: <5264580628977113351@google.com> Subject: Report domain: stalw.art Submitter: google.com Report-ID: 5264580628977113351 From: noreply-dmarc-support@google.com To: domains@stalw.art Content-Type: application/zip; name="google.com!stalw.art!1669507200!1669593599.zip" Content-Disposition: attachment; filename="google.com!stalw.art!1669507200!1669593599.zip" Content-Transfer-Encoding: base64 UEsDBAoAAAAIAHFUfFWAeOSU8QEAAKkEAAAuAAAAZ29vZ2xlLmNvbSFzdGFsdy5hcnQhMTY2OTUw NzIwMCExNjY5NTkzNTk5LnhtbKVUwZKjIBC9z1ekck9Qk5hoMcye9gt2zxbB1lBBoACTmb9fHNCw ma257El83f2632sUv70PYnUDY7mSr+t8m61XIJlquexf179//dyc1qs38oI7gPZM2ZW8rFbYgFbG NQM42lJHJ8yjyvSNpAOQXqlewJapAaMFDDkwUC6IVJ5BfGzagRq2saOe6H6kZSEv1rw7QxumpKPM NVx2ilyc07ZGKJZuH6WIIirtHQwq9mV5OGWe62t9II4yeEsORbn3uWVxqo7HPN/tDjlGj3BI91Kh MVT2UYyHztBzSfKyrA7Zsch8s4DMcZBtiFa7Q1X5UeRMhv5mW7qlnmKtBGcfjR7PgtsLLIMo744k 1lFx31LjPFlAQpi2Vz4Qg1E4RNDq7hObngHSfg8SMNLx3c6AnRHNHMknVdPhc8p/TeR9ZMrMwxl1 X+RbNRoGDdekoFle77uqZlme1+f9jtW1t/iRMJcwNUrfFKNwmOHYF25UjN64dg5MbnCrleXOX+A4 f4okeZMZmlrrExZfovAuBhZzEq1PPf2mZoWYtyAd77j/fJayC9AWTNMZNaQbSuHI86Ua09FdGgN2 FO5B+DTs98uP93piiJLiS6IWBDCnDLmB4FdujaayKLz2GV8MSDvjxJr/niIx2t/IJ9FTcrhPGD3+ On8AUEsBAgoACgAAAAgAcVR8VYB45JTxAQAAqQQAAC4AAAAAAAAAAAAAAAAAAAAAAGdvb2dsZS5j b20hc3RhbHcuYXJ0ITE2Njk1MDcyMDAhMTY2OTU5MzU5OS54bWxQSwUGAAAAAAEAAQBcAAAAPQIA AAAA ================================================ FILE: tests/resources/smtp/reports/dmarc2.eml ================================================ Received: from mail.stalw.art ([mail.stalw.art]) by 127.0.0.1 (Stalwart JMAP) with LMTP; Thu, 10 Nov 2022 03:27:19 +0000 Received: from mx0.backschues.net (lnxs001.backschues.net [85.183.142.13]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by mail.stalw.art (Postfix) with ESMTPS id 6DD117CC0B for ; Thu, 10 Nov 2022 03:27:16 +0000 (UTC) Received: from mx0.backschues.net (localhost [127.0.0.1]) by mx0.backschues.net with SMTP id 4N76hg4lNgz9ryP for ; Thu, 10 Nov 2022 04:27:15 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=backschues.net; s=mail-2014-01; t=1668050835; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:mime-version:mime-version:content-type:content-type; bh=LTj1tdFz9JQFL/mVJASN0b9hGcolcCtY5v0bhnChJYY=; b=AtRegYc51PTYqDOy/6fB4xETTWAbVc2ivf8AfF4ygu3+6+oqBPyloTuOnEt7xYmjLFnll/ SMZFFpRETsMlkiVg/1O0VpPRpIpiTbh4dwtUrRyo1Uw/cDJv5auz4rBMxcRNnDKypHwUKs BUahHWsVKH/TL5SzV79kqyjlYAs1HdJvS+wRINYBaptkeT6UeHGZakL21NnQUdOGt0fj4y eJvWVtCYHZ5DUJ8K8h2W1NlTAWP8nTBoQVVDQrI5Zi1AEvnUWw+H7E8d/q2cF756/IBYso rT56D3PYo2iuSt3aIBth1wL7/GJwc6N4JHcNpJ9XPV6xQbt+lm2b3+W59osL0Q== DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed; d=backschues.net; s=ed25519-mail-2018-10; t=1668050835; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:mime-version:mime-version:content-type:content-type; bh=LTj1tdFz9JQFL/mVJASN0b9hGcolcCtY5v0bhnChJYY=; b=y7d79OWWCrDX40k91FoBdGnUcrjN7xvWYyqskPfQmMoaSqFNSlTHH8gMXC/vXwiYIP3Oxp d/hVvEuIIQBlwMDQ== From: "DMARC Aggregate Report" To: domains@stalw.art Subject: Report Domain: stalw.art Submitter: backschues.net Report-ID: stalw.art.1667948400.1668034800 Date: Thu, 10 Nov 2022 03:27:02 GMT MIME-Version: 1.0 Message-ID: Content-Type: multipart/mixed; boundary="----=_NextPart_84e1fdd0-b285-4922-9fc7-88b070204303" This is a multipart message in MIME format. ------=_NextPart_84e1fdd0-b285-4922-9fc7-88b070204303 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit This is an aggregate report from backschues.net. Report domain: stalw.art Submitter: backschues.net Report ID: stalw.art.1667948400.1668034800 ------=_NextPart_84e1fdd0-b285-4922-9fc7-88b070204303 Content-Type: application/gzip Content-Transfer-Encoding: base64 Content-Disposition: attachment; filename="backschues.net!stalw.art!1667948400!1668034800.xml.gz" H4sIAAAAAAAAA5VUsXLbMAzd/RU6D9ksSo6b2heG6dKOndJZR5OQzYtEsiSVNH9fUqQoqXWHT gIfgAfgASf8/KvvijcwVij5tK3LaluAZIoLeXna/nj5tjtui2eywS0AP1P2SjZFgQ1oZVzTg6 OcOhowjypzaSTtgdz9HJR7DNGWXQew5fevLxhld4yGnoqOSOW5uo8d76lhOzvoQPxlkSrBYRR jY16qLTixjnbvJTWurB8ePp8Ox0NVBfNY3R+OVYXRHBpTfa/QGCovqQcPneEiJJnzMYrI5AfJ yZIyvCMZWrPlaktRsFadYB+NHs6dsFfIjSg/kJwH8GQRiW7KX0VPDEbRSKDV7YiFb4S0l08CR jq97QTYCdHMkTr0HYyxy7878ooyZXhcrHqfuNRgGDRCk09Vud/fl/X+VNangyfPnhjJ1CB9FY yikQrHMvBGu8HrxLOgXFitrHD+3FKzSyRHhblbv3TvzhKME7YJnlVAt2r5dcRRsOAgnWiFP/G UcAXKwTStUf1yBUt4ZPgjE9PBXRsDdujcRLVqLu1QgGtLf+zrpY6XG1KJptaGaxkf0y0Fnpts /7iRmS7KcYNukxX7zwbjWtaMicaf30qEEBaPB6P8h/gNNHLX4VQEAAA= ------=_NextPart_84e1fdd0-b285-4922-9fc7-88b070204303-- ================================================ FILE: tests/resources/smtp/reports/dmarc3.eml ================================================ Received: from mail.stalw.art ([mail.stalw.art]) by 127.0.0.1 (Stalwart JMAP) with LMTP; Tue, 08 Nov 2022 23:26:41 +0000 Received: from relay7.m.smailru.net (relay7.m.smailru.net [94.100.178.51]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.stalw.art (Postfix) with ESMTPS id DD4337CC09 for ; Tue, 8 Nov 2022 23:26:38 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=corp.mail.ru; s=mail4; h=Date:Message-ID:To:From:Subject:MIME-Version:Content-Type:From:Subject:Content-Type:Content-Transfer-Encoding:To:Cc; bh=fooa0+RBCZvyV2mP8Nx/UsLQ5RhazFg+SPGNtxZrCX0=; t=1667950001;x=1668040001; b=J9aMEkY9eVdOxjkNxaPFJ2Yk+/NCux9uOZl3iJXI0hEFaeYj9g7l+WtmXczk+YvgH3yhVhtvONUEYFsValRWWCAfmePm429N3mSuclVktk7t6RPJ4O5EcMjwrD9882vmX1xpI7ecPOzd5AD67HPt5SIA1RIa5injaOI5CWUXBBa5c0zDfmciyANAiDw0gm1axEMK4AUc61txPsX7H1qRq/FxGNITnnpYdqkkT2lR8sTl5HPwTjEsw4sYGKr5SiMpROhhbLZTM8RpojkP73bmw3UBZ9FI8iKApJUFB8i9tu0hjzHkev4uoDXgOXFYs/RAI1JkCWEp2Rjb3LpTSHT6cA==; Received: from [10.161.4.115] (port=60844 helo=60) by relay7.m.smailru.net with esmtp (envelope-from ) id 1osXzK-0007VC-BD for domains@stalw.art; Wed, 09 Nov 2022 02:26:38 +0300 Content-Type: multipart/mixed; boundary="===============5640625649776607409==" MIME-Version: 1.0 Subject: Report Domain: stalw.art; Submitter: Mail.Ru; Report-ID: 28551467700969547611667865600 From: dmarc_support@corp.mail.ru To: domains@stalw.art Message-ID: Date: Wed, 09 Nov 2022 02:26:38 +0300 Auto-Submitted: auto-generated Authentication-Results: relay7.m.smailru.net; auth=pass smtp.auth=dmarc_support@corp.mail.ru smtp.mailfrom=dmarc_support@corp.mail.ru; iprev=pass policy.iprev=10.161.4.115 --===============5640625649776607409== MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: base64 VGhpcyBpcyBhbiBhZ2dyZWdhdGUgcmVwb3J0IGZyb20gTWFpbC5SdS4= --===============5640625649776607409== Content-Type: application/gzip MIME-Version: 1.0 Content-Transfer-Encoding: base64 Content-Disposition: attachment; filename="mail.ru!stalw.art!1667865600!1667952000.xml.gz" H4sICK7lamMC/21haWwucnUhc3RhbHcuYXJ0ITE2Njc4NjU2MDAhMTY2Nzk1MjAwMC54bWwAdVNB cqMwELzvK3LzKQhYg01qouwHctkPULIYjMogqSThJL/fEQSC18kFzbRmWt0jAS/vQ/9wReeV0c+7 LEl3D6ilaZQ+P+/G0D4edy/8F7SIzUnICweH1rhQDxhEI4LgYNy51mJA/ipUn/wdga0I4EAYbwbh ZO1HGzv/SONsEvHEUe1cAfgenKil0UHIUCvdGt6FYJ8Y67Bfy1lcHyNCjfcdizbV8PxYFNm+PBzS tCqrYn8os6wsD8eyKNMU2FchkAmsndBnknvCs9J8WzgjgLqZ4KrI0wjHHNi2ld3NxZpeyY/ajqde +Q7jUYb0a+6D6N8S4QIxzAiI5qIG7oDNAQhv2ymNK1iujUZgloNfYgrAysCzKCcG9L070CENO67m jVrN6CTWyvIiTfL8d5LlVZJVe+Jad0CaURMpsDlYTOBV9CO5jSaUt8arQO/lU8oWgUl/S9dE+GQl OpjzyQu7Z2STPNWgDqpV9BQ5dCgadHXrzLAd1xYGdtMhxtDVDv3YB/+pYpm3wtAm9Ca/xu2xRxmM m7bI7JrDzMCt8D5e6ZQsTm5Iv7nEleWKvboo76zQef4N+zyO/9in6fysWBqLfIjOiXBKftA6T/l2 HGx5CGz9j/8BQWPZIPkDAAA= --===============5640625649776607409==-- ================================================ FILE: tests/resources/smtp/reports/dmarc4.eml ================================================ Received: from mail.stalw.art ([mail.stalw.art]) by 127.0.0.1 (Stalwart JMAP) with LMTP; Tue, 25 Oct 2022 04:08:22 +0000 Received: from NAM12-MW2-obe.outbound.protection.outlook.com (mail-mw2nam12on2073.outbound.protection.outlook.com [40.107.244.73]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.stalw.art (Postfix) with ESMTPS id A24107CC0A for ; Tue, 25 Oct 2022 04:08:22 +0000 (UTC) ARC-Seal: i=1; a=rsa-sha256; s=arcselector9901; d=microsoft.com; cv=none; b=SvolQ1oIEgdfCI6dbwmJ1jS0ovWmprW6kT3q9NgrbX+CMhIsdrqyS3Q1sO16KT2wCQAyNofiEZ5tKY0e1PzzMqeR29jUWvEye9T43fCfUeLFx9b45YrfkGYwqLeDIq0Ywl+ggVmsm7X83XqI6+9EC6qMukCb0cbLazu3rW/Rbyc6d5+fq6QTFZovATGRvHz71H9t7e//hYI23XjU5Q3Enw0Qq3xPSyusWDi3t7CfGXn9i2120XlNLnPxef5PCmwy4E+OTJ5qC5WtMthOskKKuFvx8onOYmc/JjJ3VrtZwALx9C+ulzix5US6H7pFvZ2jtDbMnW4U7ir/hp5xn5adFw== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=microsoft.com; s=arcselector9901; h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-AntiSpam-MessageData-ChunkCount:X-MS-Exchange-AntiSpam-MessageData-0:X-MS-Exchange-AntiSpam-MessageData-1; bh=65M39Uvc5w8zbmK6TxEdoblSMXlIyqTXJHNalJ80wk4=; b=PN7QPeXJLr6tmH2CxydbDQjHqBtFKNN9HjGimeUHaIeSr82WHf4R295QbVX7gxw6sFE7Z9lZMTrMSqbRVI7rhbx+SEkxCfAothf9207FDX6t37Zt0wd/5EwR6dzfbcNJBL+U0/iG4J03L5b1geWY+e68mHKYH4/ybGcr+SBKuv/LgfZNtOfbQ3ioiKvFcpSDqd/qGUs4U9l2tVlXgbcKkct04sCuPciqgLEuIGirPLLbDUaBRJc51ZZB6CeporySRdHp6uFXyy3VBvvLVuwDNnnPrW4BUL05AuutzK7rc8ZQEpWf7r0gUEg2ArSrvs6Znnfe97oRa01L2SeFwuZsMA== ARC-Authentication-Results: i=1; mx.microsoft.com 1; spf=none; dmarc=none action=none header.from=microsoft.com; dkim=none (message not signed); arc=none DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=notification.microsoft.com; s=selector1; h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck; bh=65M39Uvc5w8zbmK6TxEdoblSMXlIyqTXJHNalJ80wk4=; b=NjqsA7D6sEq1WgCZ/E1f5/B+XUXe5F4uv6CF2KQYVuyRnxItdox09LCWqZQ+fNQ6BbJ4Ne05Cb1BPbPP9yvb8Y6B1s2QvuxkUb69UFbAhoFgsRT6A4K76ykKQQyiPoYpxlO6FEyy+gel4y7c9XRLiWW6OxMIBcjBGB5ziP7mGFaJx4qXJ2mROfO7uZfrCu5pzOimkjPw6extWv4i0Kl3XKvBtXZnsr9eoC10mJvEAp7E2cpnaZnP46RQc9cmXzlmvhKPvCQCUWipJN9f1BTTvFjJ9ff6ehmN9RSzCckj3SZGw9XAnd0WYqh4evt6Y1RxQ4iQDSaZHNRpyMOtmkWc/w== Authentication-Results: dkim=none (message not signed) header.d=none;dmarc=none action=none header.from=microsoft.com; Received: from BN9PR03CA0046.namprd03.prod.outlook.com (2603:10b6:408:fb::21) by SJ0PR18MB3916.namprd18.prod.outlook.com (2603:10b6:a03:2c9::21) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.5746.21; Tue, 25 Oct 2022 04:08:19 +0000 Received: from BN7NAM10FT048.eop-nam10.prod.protection.outlook.com (2603:10b6:408:fb:cafe::d7) by BN9PR03CA0046.outlook.office365.com (2603:10b6:408:fb::21) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.5746.27 via Frontend Transport; Tue, 25 Oct 2022 04:08:19 +0000 Received: from nam10.map.protection.outlook.com (2a01:111:f400:fe53::30) by BN7NAM10FT048.mail.protection.outlook.com (2a01:111:e400:7e8f::199) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.5746.16 via Frontend Transport; Tue, 25 Oct 2022 04:08:19 +0000 Message-ID: <725cbfbe133940149987cfc528387235@microsoft.com> X-Sender: XATTRDIRECT=Originating XATTRORGID=xorgid:96f9e21d-a1c4-44a3-99e4-37191ac61848 MIME-Version: 1.0 From: "DMARC Aggregate Report" To: Subject: =?utf-8?B?UmVwb3J0IERvbWFpbjogc3RhbHcuYXJ0IFN1Ym1pdHRlcjogcHJvdGVjdGlvbi5vdXRsb29rLmNvbSBSZXBvcnQtSUQ6IDcyNWNiZmJlMTMzOTQwMTQ5OTg3Y2ZjNTI4Mzg3MjM1?= Content-Type: multipart/mixed; boundary="_mpm_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_" Date: Tue, 25 Oct 2022 04:08:19 +0000 X-EOPAttributedMessage: 0 X-MS-PublicTrafficType: Email X-MS-TrafficTypeDiagnostic: BN7NAM10FT048:EE_|SJ0PR18MB3916:EE_ X-MS-Office365-Filtering-Correlation-Id: 7f843e40-ccc6-4c17-1ce7-08dab63e8cd1 X-MS-Exchange-SenderADCheck: 2 X-MS-Exchange-AntiSpam-Relay: 0 X-Microsoft-Antispam: BCL:0; X-Microsoft-Antispam-Message-Info: PSiI3L4DKj/cyRBl8/bmbyQMrr1DvsYEB1+aTn/3Y39oHnyJ5HXcxu6jNUl32WcPW6Gfqmhc6P1RFE5L/9ev0cWnqh4GgIs2qmHicLexPmMjP8viPdjb1N7TSSOv1hhXMT+gVLx889X5sltd4qpfIAWhoxNonjQpVIgt4VOVnbCWTu1hyOjOVplq0rKqIF04BQGHZnBRfkcD1No+mZrvx8RLWIwInU3fpPeGz77Wn3TIvHtzypR/d22WpZ8eHk3aIxxdjwp5WLg4unpiJaieyQN7BRhD/v6b3pLFVJP8Ii2+FGjTsKASczEL4dHnIoIrHYE0wwaFFPcSNzovLhzYguDV42EGS8Fm7soiew4ch+hICM0LPNTGTZIDe7wm2eSwhN2tkJK4QCfh1DON39jXninVr88ZlzMcDXnXpgvWHHiur8az7Gvs9zHH/1tFMsPVSh7BS+8fHEcBYpdtihrP22GcjbOd98IiTAs/dVzSy0TUg6WEgJO6oUklGjqVbi99CrNZI1BtLP4vH4aSlz9JYg4et6SxiJlKyoSzqUr2NN9/pyFdQ//5d/EEjKJz8CAcQmCjjPEObGFttT3maY2+zsa2THodZgpfMyDbA3WUKxE= X-Forefront-Antispam-Report: CIP:255.255.255.255;CTRY:;LANG:en;SCL:1;SRV:;IPV:NLI;SFV:NSPM;H:nam10.map.protection.outlook.com;PTR:;CAT:NONE;SFS:(13230022)(396003)(39860400002)(346002)(34036004)(366004)(376002)(136003)(47540400005)(451199015)(2616005)(52230400001)(121820200001)(83380400001)(166002)(86362001)(41300700001)(2906002)(4001150100001)(8936002)(316002)(5660300002)(235185007)(41320700001)(508600001)(6486002)(6512007)(6506007)(24736004)(108616005)(68406010)(85236043)(8676002)(10290500003)(6916009)(36736006)(36756003)(66899015);DIR:OUT;SFP:1101; X-OriginatorOrg: dmarcrep.onmicrosoft.com X-MS-Exchange-CrossTenant-OriginalArrivalTime: 25 Oct 2022 04:08:19.1682 (UTC) X-MS-Exchange-CrossTenant-Network-Message-Id: 7f843e40-ccc6-4c17-1ce7-08dab63e8cd1 X-MS-Exchange-CrossTenant-AuthSource: BN7NAM10FT048.eop-nam10.prod.protection.outlook.com X-MS-Exchange-CrossTenant-AuthAs: Internal X-MS-Exchange-CrossTenant-Id: 96f9e21d-a1c4-44a3-99e4-37191ac61848 X-MS-Exchange-CrossTenant-FromEntityHeader: Internet X-MS-Exchange-Transport-CrossTenantHeadersStamped: SJ0PR18MB3916 This is a multi-part message in MIME format. --_mpm_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_ Content-Type: multipart/related; boundary="_rv_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_" --_rv_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_ Content-Type: multipart/alternative; boundary="_av_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_" --_av_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_ --_av_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_ Content-Type: text/html; charset=us-ascii Content-Transfer-Encoding: base64 PGRpdiBzdHlsZSA9ImZvbnQtZmFtaWx5OlNlZ29lIFVJOyBmb250LXNpemU6MTRweDsiPlRoaXMgaX MgYSBETUFSQyBhZ2dyZWdhdGUgcmVwb3J0IGZyb20gTWljcm9zb2Z0IENvcnBvcmF0aW9uLiBGb3Ig RW1haWxzIHJlY2VpdmVkIGJldHdlZW4gMjAyMi0xMC0yMyAwMDowMDowMCBVVEMgdG8gMjAyMi0xMC 0yNCAwMDowMDowMCBVVEMuPC8gZGl2PjxiciAvPjxiciAvPllvdSdyZSByZWNlaXZpbmcgdGhpcyBl bWFpbCBiZWNhdXNlIHlvdSBoYXZlIGluY2x1ZGVkIHlvdXIgZW1haWwgYWRkcmVzcyBpbiB0aGUgJ3 J1YScgdGFnIG9mIHlvdXIgRE1BUkMgcmVjb3JkIGluIEROUyBmb3Igc3RhbHcuYXJ0LiBQbGVhc2Ug cmVtb3ZlIHlvdXIgZW1haWwgYWRkcmVzcyBmcm9tIHRoZSAncnVhJyB0YWcgaWYgeW91IGRvbid0IH dhbnQgdG8gcmVjZWl2ZSB0aGlzIGVtYWlsLjxiciAvPjxiciAvPjxkaXYgc3R5bGUgPSJmb250LWZh bWlseTpTZWdvZSBVSTsgZm9udC1zaXplOjEycHg7IGNvbG9yOiM2NjY2NjY7Ij5QbGVhc2UgZG8gbm 90IHJlc3BvbmQgdG8gdGhpcyBlLW1haWwuIFRoaXMgbWFpbGJveCBpcyBub3QgbW9uaXRvcmVkIGFu ZCB5b3Ugd2lsbCBub3QgcmVjZWl2ZSBhIHJlc3BvbnNlLiBGb3IgYW55IGZlZWRiYWNrL3N1Z2dlc3 Rpb25zLCBraW5kbHkgbWFpbCB0byBkbWFyY3JlcG9ydGZlZWRiYWNrQG1pY3Jvc29mdC5jb20uPGJy IC8+PGJyIC8+TWljcm9zb2Z0IHJlc3BlY3RzIHlvdXIgcHJpdmFjeS4gUmV2aWV3IG91ciBPbmxpbm UgU2VydmljZXMgPGEgaHJlZiA9Imh0dHBzOi8vcHJpdmFjeS5taWNyb3NvZnQuY29tL2VuLXVzL3By aXZhY3lzdGF0ZW1lbnQiPlByaXZhY3kgU3RhdGVtZW50PC9hPi48YnIgLz5PbmUgTWljcm9zb2Z0IF dheSwgUmVkbW9uZCwgV0EsIFVTQSA5ODA1Mi48LyBkaXYgPg== --_av_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_-- --_rv_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_-- --_mpm_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_ Content-Type: application/gzip Content-Transfer-Encoding: base64 Content-ID: <3ff45643-7977-4f3c-a97d-14b9e7faa5e7> Content-Description: protection.outlook.com!stalw.art!1666483200!1666569600.xml.gz Content-Disposition: attachment; filename="protection.outlook.com!stalw.art!1666483200!1666569600.xml.gz"; H4sIAAAAAAAEAM1VzY7bIBi8V+o7RLnXxHZ+VyzbB2jVQy+9WQTjBMUGBDjZvn0/G4JJsu3usZcE5h vzDcNg45fXrp2dubFCyed5ni3mL+TzJ9xwXu8pO82gLO3Tq62f50fn9BNCl8slu5SZMgdULBY5+vX9 20925B2dR7J4n/xFSOuoZHwO7WYzHCQQUIDRdTJWDNfKuKrjjtbU0REEGJasJO04+dG7VqlTxlSHUU QDCzqJltQdNcyv87UTzCirGucf8ITADq1ETTbFiu2bPc/Lcrdc5MvdbrthDVsV23K7KcoVRhM3PAzi eGWoPFybA7bnBwF7Wq/Xy20JBmDkkUjgsh7Lq/VuPZSHeVgP3S0YW944gbVqBftd6X7fCnvkkxwFO5 METG4vGTUO1vNIqNP6JDpiMPKDK2p1M4LDf8A0kUpyjPQVsFfERkgzR/JhA8MgYI0iAMCvV/+mULCc KRNFG3WZvLGqN4xXQpPVIiuKMsuLXZbvltA3ViKZqV6CBIz8IOKhKz/Ttgc/61gZLBJWKyvcEDW/oR RJiYNDDQQFGJNZwYsmVCbHkt3e94VDjFvEoubSiUZA2tNEnHmrNK+cIiqNdlp4ZDdGdURw1wx3LSGP eKQfOa258WCSjBS+6nwUh2nvjpXhtm9dIvjekRCzSctN7rxpvOXMKTOS4MziPOH4PkRTa4fkj5PJ3p um/7GEv12/Ww1wVuIkrNFUFsU/tfiovaMlTeIH3WCQFdINAYD24+TDPiRvCvSQkIEfLji8CsJHhfwB wJC79XYGAAA= --_mpm_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_-- ================================================ FILE: tests/resources/smtp/reports/dmarc5.eml ================================================ Received: from mail.stalw.art ([mail.stalw.art]) by 127.0.0.1 (Stalwart JMAP) with LMTP; Tue, 20 Sep 2022 10:28:19 +0000 Received: from a14-92.smtp-out.amazonses.com (a14-92.smtp-out.amazonses.com [54.240.14.92]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-SHA256 (128/128 bits)) (No client certificate requested) by mail.stalw.art (Postfix) with ESMTPS id 1337D7E19D for ; Tue, 20 Sep 2022 10:28:18 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/simple; s=a66wkfbz3zwxdt2n5p6d7lj2ja7sdwuc; d=amazonses.com; t=1663669697; h=From:To:Message-ID:Subject:MIME-Version:Content-Type:Date; bh=h9v7dueDYfUxVokuKSLTqLuOwisdgdDRQ6TLwJOzXes=; b=dHR5EJhoY9s8g2/Y4K4rHdz44k67r7fyC4wr2AWZmemrVBoxYHJPwa295S2VJQtY kxTxppN2GEcNxhUMw8TXBrRwNKdoOLU38ZtrAN1a4hWVxmlwky1dtjXETQ/qJ257Nzg bsXkAo4S1RABFmkQQJ0zSPZGkMW+lpZTBCDzlOHU= DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/simple; s=6gbrjpgwjskckoa6a5zn6fwqkn67xbtw; d=amazonses.com; t=1663669697; h=From:To:Message-ID:Subject:MIME-Version:Content-Type:Date:Feedback-ID; bh=h9v7dueDYfUxVokuKSLTqLuOwisdgdDRQ6TLwJOzXes=; b=UDIvc6rvbihyGbzGRsmSSSzVNFgpfb3V3j0UivcNjlX2y63vjLinol463Z/+3Xh3 BmxAOiLHF/DbVnqqNg5ygdxsa7MBHXEJ5we3W8vQr37xNk5DqhV7HPBSFttWP5sy0dg rdjyfMIjqJ1J/2+aM4opFA/6EWif7TGmjo7N1KKM= From: postmaster@amazonses.com To: domains@stalw.art Message-ID: <010001835a70fc8d-a3d7eff5-7adb-41cc-87bd-a646d9776a69-000000@email.amazonses.com> Subject: Dmarc Aggregate Report Domain: {stalw.art} Submitter: {Amazon SES} Date: {2022-09-19} Report-ID: {6b06c366-0631-4ca0-8337-f5aecf137918} MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="----=_Part_42492_694130218.1663669697673" Date: Tue, 20 Sep 2022 10:28:17 +0000 Feedback-ID: 1.us-east-1.CTa/CO4t1eWkL0VlHBu5/eINCZhxZraAIsQC/FZHIgk=:AmazonSES X-SES-Outgoing: 2022.09.20-54.240.14.92 ------=_Part_42492_694130218.1663669697673 Content-Type: text/plain; charset=us-ascii Content-Transfer-Encoding: 7bit This MIME email was sent through Amazon SES. ------=_Part_42492_694130218.1663669697673 Content-Type: application/octet-stream; name=amazonses.com!stalw.art!1663545600!1663632000.xml.gz Content-Transfer-Encoding: base64 Content-Disposition: attachment; filename=amazonses.com!stalw.art!1663545600!1663632000.xml.gz H4sIAAAAAAAAAG1TwXLbIBA9O1/RyV1CWLHszlDSHHJMe8itFw1GK5uJBAwgp+3XlwXJVjK9SOzb 1b59vBV7/D0OXy7gvDL62z0tq/tHfsd6gO4o5Bu/27A5yauSMrIEEXdgjQvtCEF0IogIbZhxp1aL EfjTy9Ovnz+K1+dXRq4gVsAo1MCt8WEUPoD7Lkbx12gPvpRmZCTnsXLurzreHKtG1k1TVE1Niwcp quJQ1/ui3wmQPa33X+mBkVs9fh1HgtYJfUq0G3aEk9KcNk29e9g1VcVIRlISdJdSTb2tMIUxNiEf ulwpVpKZNYOSf1o7HQflzzCTm6hCcx/E8F4KF2KjjGBSdG9q5I6RfEiQt31C8I2A5dpoYMSmyC+h z7GVgVOcEw8I9IbHKD5xyP9MFO9SGpdnc+Y9i/ZmchJaZfm22pd0T0t6OJRb7HtLpUppJh0ZGcmH hM0scBHDFC8p9UblykdvVcAdyTOvkbkGZffR5picbyCJ7GdwvoSblA8k0YWsgKkOdFC9iiu52HiB wVhoe2dGnher1AP6uU6k2jOIDlwGVj6t4UT2iYSJKZxbB34awsy6jHu1fUV8sz0tNH7FrfAeVykF WediO/nUHcuycdHd5Zf8BxMenbqzAwAA ------=_Part_42492_694130218.1663669697673-- ================================================ FILE: tests/resources/smtp/reports/tls1.eml ================================================ From: tlsrpt@mail.sender.example.com Date: Fri, May 09 2017 16:54:30 -0800 To: mts-sts-tlsrpt@example.net Subject: Report Domain: example.net Submitter: mail.sender.example.com Report-ID: <735ff.e317+bf22029@example.net> TLS-Report-Domain: example.net TLS-Report-Submitter: mail.sender.example.com MIME-Version: 1.0 Content-Type: multipart/report; report-type="tlsrpt"; boundary="----=_NextPart_000_024E_01CC9B0A.AFE54C00" Content-Language: en-us This is a multipart message in MIME format. ------=_NextPart_000_024E_01CC9B0A.AFE54C00 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit This is an aggregate TLS report from mail.sender.example.com ------=_NextPart_000_024E_01CC9B0A.AFE54C00 Content-Type: application/tlsrpt+gzip Content-Transfer-Encoding: base64 Content-Disposition: attachment; filename="mail.sender.example!example.com!1013662812!1013749130.json.gz" H4sICCpFtWMAA3JwdDAxLmpzb24uMQCtVVtr2zAYfe+vEN7bmFzZjt3EMLbRhu1hdCUJI+soRpGU VMy2jCSHZCX/fZIvzVLHpc0WDDHSOfqOznfxwxkwP0fIFc75b6y5yGGOM+bEwLkUWYHzLZw772oU xZpBifOV3X6o1qp1pbHU0O5qXlN95EUQDSDyZgjF1XPbnFIxWE778H4QhyPz3DoVfNfEJiLXmGjI 86WwDKUVlKwQUvN89ZE0Ujcu2+CsSFkruYZATi0nRFE48C8I9AMawMEFwXARMQRHg4hhxIZksHgk FiLlhDNleD8fde/vvMdsD7x4sgf1tmCN3L/u/xSltDS3OAh1AFszqUxmYjCdTdfekYMqVCYoi4Fm ylrSC9rE4K2bYZ66rWnbJ6Z1OXiT4JU5exgNEHI6oLv+m1FhQuXWgZdEM+rgvVC634le6V1RByu7 w2COKrMMy57caaFxClVJCFNqWZpX8287g4gyt+LCwI+OqK95SyOwlKxDClDwrKSWR5k2b+qoB12x FVUyVab6sdgIM12x5MS2K9sUXDLal1plOtFUC8w0hryoWxF5MV0MY7wg1PSt58dxb8lJRhhfVwfU mWtnR7bxXllk9vqMdlzzEOrgd90jXmZMNah0qmAutMlvYWfDP3kTnOaN/0pv9ke1OgIXuZ4XuGH0 Sj/NFXoImFJu578pYTtkZVZ9DWy4e60LFZ+f18NUuZ1p2+wklgc+AE7Be9AMW0AABH4AaPAGTBv7 r4WetuaDbueenN41Tjmtv2FNM708td5o6Iaea8rNjfwT8jA8oQzgApNfZfF/GiV4Bm7HimRYVXBa RZ+HaJR8T8aTSXIz+Tb/kdx8mn1Jvo6vP5u/8fxyPL4aXx3JzcHKfsbW63dnuz+8byfQUQgAAA== ------=_NextPart_000_024E_01CC9B0A.AFE54C00-- ================================================ FILE: tests/resources/smtp/reports/tls2.eml ================================================ From: tlsrpt@mail.sender.example.com Date: Fri, May 09 2017 16:54:30 -0800 To: mts-sts-tlsrpt@example.net Subject: Report Domain: example.net Submitter: mail.sender.example.com Report-ID: <735ff.e317+bf22029@example.net> TLS-Report-Domain: example.net TLS-Report-Submitter: mail.sender.example.com MIME-Version: 1.0 Content-Type: multipart/report; report-type="tlsrpt"; boundary="----=_NextPart_000_024E_01CC9B0A.AFE54C00" Content-Language: en-us This is a multipart message in MIME format. ------=_NextPart_000_024E_01CC9B0A.AFE54C00 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit This is an aggregate TLS report from mail.sender.example.com ------=_NextPart_000_024E_01CC9B0A.AFE54C00 Content-Type: application/tlsrpt Content-Disposition: attachment; filename="mail.sender.example!example.com!1013662812!1013749130.json" { "report-id": "2020-01-01T00:00:00Z_example.com", "date-range": { "start-datetime": "2020-01-01T00:00:00Z", "end-datetime": "2020-01-07T23:59:59Z" }, "organization-name": "Google Inc.", "contact-info": "smtp-tls-reporting@google.com", "policies": [ { "policy": { "policy-type": "sts", "policy-string": [ "version: STSv1", "mode: enforce", "mx: demo.example.com", "max_age: 604800" ], "policy-domain": "example.com" }, "summary": { "total-successful-session-count": 23, "total-failure-session-count": 1 }, "failure-details": [ { "result-type": "certificate-host-mismatch", "sending-mta-ip": "123.123.123.123", "receiving-ip": "234.234.234.234", "receiving-mx-hostname": "demo.example.com", "failed-session-count": 1 } ] } ] } ------=_NextPart_000_024E_01CC9B0A.AFE54C00-- ================================================ FILE: tests/resources/smtp/sieve/awl.sieve ================================================ require ["variables", "include", "vnd.stalwart.expressions", "reject"]; global "score"; # Create AWL table if eval "!query('sql', 'CREATE TABLE awl (score FLOAT, count INT, sender TEXT NOT NULL, ip TEXT NOT NULL, PRIMARY KEY (sender, ip))' , [])" { reject "create table query failed"; stop; } set "score" "1.1"; include "awl_include"; if eval "score != 1.1" { reject "awl_include #1 set score to ${score}"; stop; } set "score" "2.2"; include "awl_include"; if eval "score != 1.6500000000000001" { reject "awl_include #2 set score to ${score}"; stop; } set "score" "9.3"; include "awl_include"; if eval "score != 5.4750000000000005" { reject "awl_include #3 set score to ${score}"; stop; } ================================================ FILE: tests/resources/smtp/sieve/awl_include.sieve ================================================ require ["variables", "include", "vnd.stalwart.expressions", "reject"]; global "score"; set "awl_factor" "0.5"; let "result" "query('sql', 'SELECT score, count FROM awl WHERE sender = ? AND ip = ?', [env.from, env.remote_ip])"; let "awl_score" "result[0]"; let "awl_count" "result[1]"; if eval "awl_count > 0" { if eval "!query('sql', 'UPDATE awl SET score = score + ?, count = count + 1 WHERE sender = ? AND ip = ?', [score, env.from, env.remote_ip])" { reject "update query failed"; stop; } let "score" "score + ((awl_score / awl_count) - score) * awl_factor"; } elsif eval "!query('sql', 'INSERT INTO awl (score, count, sender, ip) VALUES (?, 1, ?, ?)', [score, env.from, env.remote_ip])" { reject "insert query failed"; stop; } ================================================ FILE: tests/resources/smtp/sieve/stage_connect.sieve ================================================ require ["variables", "reject"]; if string "${env.remote_ip}" "10.0.0.88" { reject "Your IP '${env.remote_ip}' is not welcomed here."; } ================================================ FILE: tests/resources/smtp/sieve/stage_data.sieve ================================================ require ["envelope", "reject", "variables", "replace", "mime", "foreverypart", "editheader", "extracttext", "enotify"]; if envelope :localpart :is "to" "thomas" { deleteheader "from"; addheader "From" "no-reply@my.domain"; redirect "redirect@here.email"; discard; } if envelope :localpart :is "to" "bob" { redirect "redirect@somewhere.email"; discard; } if envelope :localpart :is "to" "bill" { reject "Bill cannot receive messages."; stop; } if envelope :localpart :is "to" "jane" { set "counter" "a"; foreverypart { if header :mime :contenttype "content-type" "text/html" { extracttext :upper "text_content"; replace "${text_content}"; } set :length "part_num" "${counter}"; addheader :last "X-Part-Number" "${part_num}"; set "counter" "${counter}a"; } } if envelope :domain :is "to" "foobar.net" { notify "mailto:john@example.net?cc=jane@example.org&subject=You%20have%20got%20mail"; } ================================================ FILE: tests/resources/smtp/sieve/stage_ehlo.sieve ================================================ require ["variables", "extlists", "reject"]; if eval "contains(['spammer.org', 'spammer.net'], env.helo_domain)" { reject "551 5.1.1 Your domain '${env.helo_domain}' has been blocklisted."; } ================================================ FILE: tests/resources/smtp/sieve/stage_mail.sieve ================================================ require ["variables", "envelope", "reject", "vnd.stalwart.expressions"]; if envelope :localpart :is "from" "spammer" { reject "450 4.1.1 Invalid address"; } eval "query('sql', 'CREATE TABLE IF NOT EXISTS blocked_senders (addr TEXT PRIMARY KEY)', [])"; eval "query('sql', 'INSERT OR IGNORE INTO blocked_senders (addr) VALUES (?)', 'marketing@spam-domain.com')"; if eval "query('sql', 'SELECT 1 FROM blocked_senders WHERE addr=? LIMIT 1', [envelope.from])" { reject "Your address has been blocked."; } if eval "!is_local_domain('', 'localdomain.org') || is_local_domain('', 'other.org')" { let "reason" "'result: ' + is_local_domain('', 'localdomain.org') + ' ' + is_local_domain('', 'other.org')"; reject "is_local_domain function failed: ${reason}"; } ================================================ FILE: tests/resources/smtp/sieve/stage_rcpt.sieve ================================================ require ["variables", "envelope", "reject", "vnd.stalwart.expressions"]; if envelope :domain :is "to" "foobar.org" { eval "query('sql', 'CREATE TABLE IF NOT EXISTS greylist (addr TEXT PRIMARY KEY)', [])"; set "triplet" "${env.remote_ip}.${envelope.from}.${envelope.to}"; if eval "!query('sql', 'SELECT 1 FROM greylist WHERE addr=? LIMIT 1', [triplet])" { eval "query('sql', 'INSERT INTO greylist (addr) VALUES (?)', [triplet])"; reject "422 4.2.2 You have been greylisted '${triplet}'."; } } ================================================ FILE: tests/resources/tls_cert.pem ================================================ -----BEGIN CERTIFICATE----- MIIFCTCCAvGgAwIBAgIUCgHGQYUqtelbHGVSzCVwBL3fyEUwDQYJKoZIhvcNAQEL BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIyMDUxNjExNDAzNFoXDTIzMDUx NjExNDAzNFowFDESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEF AAOCAg8AMIICCgKCAgEAtwS0Fzl3SjaCuKEXgZ/fdWbDoj/qDphyNCAKNevQ0+D0 STNkWCO04aFSH0zcL8zoD9gokNos0i7OU9//ZhZQmex4V6EFdZn8bFwUWN/scUvW HEFXVjtHldO2isZgIxH9LuwRv7KAgkISuWahqerOVDhe7SeQUV0AJGNEh3cT9PZr gSY931BxB7n+5k8eoSk8Z1gtBzQzL62kVGpHDKfw8yX8m65owF9eLUBrNzgxmXfC xpuHwj7hmVhS09PPKeN/RsFS8PsYO7bo0u8jEKalteumjRT7RyUEbioqfo6ZFOGj FHPIq/uKXS9zN1fpoyNh3ur5hMznQhrqlwBM9KlM7GdBJ0pZ3ad0YjT8IL/GnGKR 85J2WZdLqaQdUZo7nV67FhqdDlNE4MdwiykTMjfmLRXGAVhAzJHKyRKNwmkI2aqe S7aqeNgvuDBwY80Q9a2rb5py1Aw+L8yCkUBuHboToDpxSVRDNN8DrWNmmsXnxsOG wRDODy4GICKyxlP+RFSM8xWSQ6y9ktS2OfDBm+Eqcw+3pZKhdz2wgxLkUBJ8X1eh kJrCA/6LTuhy6m6mMjAfoSOFU7fu88jxaWPgvP7GKyH+LM/t9eucobz2ks5rtSjz V4Dc5DCS94/OpVRHwHdaFSPbJKBN9Ev8gnNrAyx/aBPGoHBPG/QUiU7dcUNIPt0C AwEAAaNTMFEwHQYDVR0OBBYEFI167IxBmErB11EqiPPqFLa31ZaMMB8GA1UdIwQY MBaAFI167IxBmErB11EqiPPqFLa31ZaMMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI hvcNAQELBQADggIBALU00IOiH5ubEauVCmakms5ermNTZfculnhnDfWTLMeh2+a7 G4cqADErfMhm/mmLbrw33t9s6tCAhQltvewKR40ST9uMPSyiQbYaCXd5DXnuI6Ox JtNW+UOWIaMf8abnkdLvREOvb8dVQS1i3xq14tAjY5XgpGwCPP8m54b7N3Q7soLn e5PDhPNTnhRIn2RLuYoZmQmMA5fcqEUDYff4epUww7PhrM1QckZligI3566NlGOf j1G9JrivBtY0eaJtamIFnGMBT0ThDudxVja2Nv0C2Elry0p4T/o4nc4M67BJ/y1R vjNLAgFhbxssemU3lZqSd+pykpJBwDBjFSPrZZmQcbk7H6Uz8V1xr/xuzfw6fA13 NWZ5vLgP/DQ13sM+XFlxThKfbPMPVe/UCTvfGtNW+3XyBgPntEkR+fNEawQmzbYl R+X1ymT9MZnEZqRMf7/UD/SYek1aUJefoew3upjMgxYVvh4F8dqJ+39F+xoFzIA2 1dDAEMzXtjA3zKhZ2cycZbEzpJvYA3eGLuR16Suqfi4kPvfwK0mOhCxQmpayt7/X vuEzW6dPCH8Hgbb0WvsSppGOvhdbDaZFNfFc5eNSxhyKzu3H3ACNImZRtZE+yixx 0fR8+xz9kDLf8xupV+X9heyFGHSyYU2Lveaevtr2Ij3weLRgJ6LbNALoeKXk -----END CERTIFICATE----- ================================================ FILE: tests/resources/tls_privatekey.pem ================================================ -----BEGIN PRIVATE KEY----- MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC3BLQXOXdKNoK4 oReBn991ZsOiP+oOmHI0IAo169DT4PRJM2RYI7ThoVIfTNwvzOgP2CiQ2izSLs5T 3/9mFlCZ7HhXoQV1mfxsXBRY3+xxS9YcQVdWO0eV07aKxmAjEf0u7BG/soCCQhK5 ZqGp6s5UOF7tJ5BRXQAkY0SHdxP09muBJj3fUHEHuf7mTx6hKTxnWC0HNDMvraRU akcMp/DzJfybrmjAX14tQGs3ODGZd8LGm4fCPuGZWFLT088p439GwVLw+xg7tujS 7yMQpqW166aNFPtHJQRuKip+jpkU4aMUc8ir+4pdL3M3V+mjI2He6vmEzOdCGuqX AEz0qUzsZ0EnSlndp3RiNPwgv8acYpHzknZZl0uppB1RmjudXrsWGp0OU0Tgx3CL KRMyN+YtFcYBWEDMkcrJEo3CaQjZqp5Ltqp42C+4MHBjzRD1ratvmnLUDD4vzIKR QG4duhOgOnFJVEM03wOtY2aaxefGw4bBEM4PLgYgIrLGU/5EVIzzFZJDrL2S1LY5 8MGb4SpzD7elkqF3PbCDEuRQEnxfV6GQmsID/otO6HLqbqYyMB+hI4VTt+7zyPFp Y+C8/sYrIf4sz+3165yhvPaSzmu1KPNXgNzkMJL3j86lVEfAd1oVI9skoE30S/yC c2sDLH9oE8agcE8b9BSJTt1xQ0g+3QIDAQABAoICABq5oxqpF5RMtXYEgAw7rkPU h8jPkHwlIrgd3Z/WGZ53APUXfhWo0ScJiZZsgNKyF0kJBZNxaI4gq5xv3zmnFIoF j+Ur7EIqBERGheoceMhqjI9/syMycNeeHM/S/ALjA5ewfT8C7+UVhOpx5DWNxidi O+phlp9q9zRZEo69grqIqVYooWxUsMyyCljTQOPDw8BLjfe5VagmsRJqmolslLDM 4UBSjZVZ18S/3Wgo2oVQia660244BHWCAkZQbbXuNI2+eUAbSoSdxw3WQcaSrywL hzyezbqr2yPDIIVuiUgVUt0Ps0P57VCCN07jlYhvCEGnClysFzD+ATefoZ0wg7za dQu2E+d166rAjnssyhzcHMn3pxgSdtXD+dQR/xfIGbPABucCupEFqKmhLdMm9+ud lHay87qzMpIa8cITJwEQROfXqWAhNUU98pKCOx1SVXBqQC7QVqGQ5solDf0eMSVh ngQ6Dz2WUI2ty75LteiFwlyTgnU9nyPN0NXsrMEET2BHWre7ufTQqiULtQ7+9BwH AMxEKvrQHjMUjdfbXuzdyc5w5mPYJZfFVSQ1HMslx66h9yCpRIsBZvUGvoaP8Tpe nQ66FTYRbiOkkdJ7k8DtrnhsJI1oOGjnvj/rvZ8D2pvrlJcIH2AyN3MOL8Jp5Oj1 nCFt77TwpF92pgl0g9gBAoIBAQDcarmP54QboaIQ9S2gE/4gSVC5i44iDJuSRdI8 K081RQcWiNzqQXTRc5nqJ7KzLyPiGlg+6rWsBKLos5l4t+MdhhH+KUvk/OtT/g8V 0NZBNXLIbSb8j8ix4v3/f2qKHN3Co6QOlxb3gFvobKDdoKqUNiSH1zTZ8/Y/BzkM jqWKhTdaLz6eyzhKfOTA4LO8kJ3VF8HUM1N9/e8Gjorl+gZpJUXUQS0+AIi8W76C OwDrVb3BPGVnApQJfWF78h4g20RwXrx/GYUW2vOMcLjXXDV5U7+nobPUoJnLxoZC 16o88y0Ivan8dBNXsc1epyPvvEqp6MJbAyyVuNeuRJcgYA0BAoIBAQDUkGRV7fLG wCr5rNysUO+FKzVtTJnf9KEsqAqUmmVnG4oubxAJJtiB5n2+DT+CtO8Nrtz05BbR uxfWm+lbEw6lVMj63bywtp0NdULg7/2t+oq2Svv16KrZIRJttXMkdEiFFmkVAEhX l8Fyl6PJPfSMwbPdXEUPUAaNrXweVFffXczHc4W2G212ZzDB0z7QQSgEntbTDFB/ 2Cg5dvuojlM9zw0fuEyLwItZs7n16j/ONZLgBHyroMU9ZPxbnLrVyoZlqtob+RWm Ju2fSIL9QqG6O4td1TqcUBGvFQYjGvKA+q5fsG26NBJ0Ac48cNK6PS4lMkN3Av2J ccloYaMEHAXdAoIBAE8WMCy1Ok6byUXiYxOL+OPmyoM40q/e7DcovE2AkLQhZ3Cr fPDEucCphPFiexkV8f8fysgQeU0WgMmUH54UBPbD81LJyISKR3nkr875Ftdg8SV/ HL0EblN9ifuR4U1bHCrJgoUFq2T09oVH7NR44Ju7bZIcIseNZK6qzcp2qGkycXD3 gLWDX1hCxeV6+qLPFQKvuomEPRH4+jnVDXuFIaW6jPqixDP6BxXmqU2bFDJcmnBq VkwGvc1F4qORdUP+yOi05VeJdZqEx1x92aTUXg+BgEQKnjbNxUE7o1L6hQfHjUIU o5iEoagWkQTEXf2YBwY+EPaNBgNWxnSuAbfJHwECggEBALOF95ezTVWauzD/U6ic +o3n/kl/Zn4FJ5KFodn7xCSe18d7uXlhO34KYqx+l+MWWMefpbGWacdcUjfImf93 SulLgCqP12sP7/iLzp4XUpL7hOeM0NvRU2nqSpwpoUNqik0Mrlc0U+TWoGTduVCf aMjwV65e3VyfY8mIeclLxqM5n1fcM1OoOnzDjiRE+0n7nYa5eAnq3pn6v4449TZY belH03e0ucFWLtrltesBmj3YdWGJqJlzQOInRhNBfXJOh8+ZynfRmP0o54udPDQV cG3PGFd5XPTjkuvhv7sqaSGRlm/um92lWOhtFfdp+i+cuDpmByCef+7zEP19aKZx 3GkCggEAFTs7KNMfvIEaLH0yQUFeq2gLmtcMofmOmeoIECycN1rG7iJo07lJLIs0 bVODH8Z0kX8llu3cjGMAH/6R2uugJSxkmFiZKrngTzKmxDPvTCKWR4RFwXH9j8IO cPq7FtKN4SgrPy9ciAPdkcGmu3zz/sBKOaoPwvU2PdBRT+v/aoz+GCLXAvzFlKVe 9/7zdg87ilo8+AtV+71EJeR3kyBPKS9JrWYUKfiams12+uuH4/53rMFZfNCAaZ3Z 1sdXEO4o3Loc5TX4DbO9FVdBSBe6klEXx4T0QJboO6uBvTBnnRL2SQriJQQFwYT6 XzVV5pwOxkIDBWDIqMUfwJDChBKfpw== -----END PRIVATE KEY----- ================================================ FILE: tests/src/cluster/broadcast.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::ClusterTest; use crate::imap::idle; use directory::backend::internal::{ PrincipalAction, PrincipalField, PrincipalUpdate, PrincipalValue, manage::{ManageDirectory, UpdatePrincipal}, }; use groupware::cache::GroupwareCache; use std::net::IpAddr; use types::collection::SyncCollection; pub async fn test(cluster: &ClusterTest) { println!("Running cluster broadcast tests..."); // Run IMAP idle tests across nodes let server1 = cluster.server(1); let server2 = cluster.server(2); let mut node1_client = cluster.imap_client("john", 1).await; let mut node2_client = cluster.imap_client("john", 2).await; idle::test(&mut node1_client, &mut node2_client, true).await; // Test event broadcast let test_ip: IpAddr = "8.8.8.8".parse().unwrap(); assert!(!server1.is_ip_blocked(&test_ip)); assert!(!server2.is_ip_blocked(&test_ip)); server1.block_ip(test_ip).await.unwrap(); tokio::time::sleep(std::time::Duration::from_millis(200)).await; assert!(server1.is_ip_blocked(&test_ip)); assert!(server2.is_ip_blocked(&test_ip)); // Change John's password and expect it to propagate let account_id = cluster.account_id("john"); assert!(server1.inner.cache.access_tokens.get(&account_id).is_some()); assert!(server2.inner.cache.access_tokens.get(&account_id).is_some()); let changes = server1 .core .storage .data .update_principal( UpdatePrincipal::by_id(account_id).with_updates(vec![PrincipalUpdate { action: PrincipalAction::AddItem, field: PrincipalField::Secrets, value: PrincipalValue::String("hello".into()), }]), ) .await .unwrap(); server1.invalidate_principal_caches(changes).await; tokio::time::sleep(std::time::Duration::from_millis(200)).await; assert!(server1.inner.cache.access_tokens.get(&account_id).is_none()); assert!(server2.inner.cache.access_tokens.get(&account_id).is_none()); // Rename John to Juan and expect DAV caches to be invalidated let access_token = server1.get_access_token(account_id).await.unwrap(); server1 .fetch_dav_resources(&access_token, account_id, SyncCollection::Calendar) .await .unwrap(); server2 .fetch_dav_resources(&access_token, account_id, SyncCollection::Calendar) .await .unwrap(); assert!(server1.inner.cache.events.get(&account_id).is_some()); assert!(server2.inner.cache.events.get(&account_id).is_some()); let changes = server1 .core .storage .data .update_principal( UpdatePrincipal::by_id(account_id).with_updates(vec![PrincipalUpdate { action: PrincipalAction::Set, field: PrincipalField::Name, value: PrincipalValue::String("juan".into()), }]), ) .await .unwrap(); server1.invalidate_principal_caches(changes).await; tokio::time::sleep(std::time::Duration::from_millis(200)).await; assert!(server1.inner.cache.events.get(&account_id).is_none()); assert!(server2.inner.cache.events.get(&account_id).is_none()); } ================================================ FILE: tests/src/cluster/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ AssertConfig, TEST_USERS, add_test_certs, directory::internal::TestInternalDirectory, imap::{ImapConnection, Type}, jmap::server::enterprise::EnterpriseCore, store::cleanup::store_destroy, }; use ahash::AHashMap; use common::{ Caches, Core, Data, Inner, Server, config::{ server::{Listeners, ServerProtocol}, telemetry::Telemetry, }, core::BuildServer, manager::{ boot::build_ipc, config::{ConfigManager, Patterns}, }, }; use http::HttpSessionManager; use imap::core::ImapSessionManager; use imap_proto::ResponseType; use jmap_client::client::{Client, Credentials}; use managesieve::core::ManageSieveSessionManager; use pop3::Pop3SessionManager; use services::{SpawnServices, broadcast::subscriber::spawn_broadcast_subscriber}; use smtp::{SpawnQueueManager, core::SmtpSessionManager}; use std::{path::PathBuf, sync::Arc, time::Duration}; use store::Stores; use tokio::sync::watch; use utils::config::Config; pub mod broadcast; pub mod stress; pub const NUM_NODES: usize = 3; #[tokio::test(flavor = "multi_thread")] pub async fn cluster_tests() { let params = init_cluster_tests(true).await; //stress::test(params.server.clone(), params.client).await; broadcast::test(¶ms).await; } #[allow(dead_code)] pub struct ClusterTest { servers: Vec, account_ids: AHashMap, shutdown_txs: Vec>, } async fn init_cluster_tests(delete_if_exists: bool) -> ClusterTest { // Load and parse config let store_id = std::env::var("STORE").expect( "Missing store type. Try running `STORE= PUBSUB= cargo test`", ); let pubsub_id = std::env::var("PUBSUB").expect( "Missing store type. Try running `STORE= PUBSUB= cargo test`", ); let mut pubsub_config = match pubsub_id.as_str() { "nats" => Config::new(SERVER_NATS).unwrap(), "redis" => Config::new(SERVER_REDIS).unwrap(), _ => panic!("Unsupported pubsub type: {}", pubsub_id), }; // Build configs let mut configs = Vec::with_capacity(NUM_NODES); for node_id in 0..NUM_NODES { let mut config = Config::new( add_test_certs(SERVER) .replace("{STORE}", &store_id) .replace("{PUBSUB}", &pubsub_id) .replace("{NODE_ID}", &node_id.to_string()) .replace( "{LEVEL}", &std::env::var("LOG").unwrap_or_else(|_| "disable".to_string()), ), ) .unwrap(); config.resolve_all_macros().await; configs.push(config); } // Build stores let stores = Stores::parse_all(configs.first_mut().unwrap(), false).await; // Build servers let mut servers = Vec::with_capacity(NUM_NODES); let mut shutdown_txs = Vec::with_capacity(NUM_NODES); for config in configs { let mut stores = stores.clone(); stores.pubsub_stores = Stores::parse(&mut pubsub_config).await.pubsub_stores; let (server, shutdown_tx) = build_server(config, stores).await; servers.push(server); shutdown_txs.push(shutdown_tx); } let store = servers.first().unwrap().store().clone(); if delete_if_exists { store_destroy(&store).await; } // Create test users let mut account_ids = AHashMap::new(); for (account, secret, name, email) in TEST_USERS { let account_id = store .create_test_user(account, secret, name, &[email]) .await; account_ids.insert(account.to_string(), account_id); } ClusterTest { servers, shutdown_txs, account_ids, } } impl ClusterTest { pub async fn jmap_client(&self, login: &str, node_id: u32) -> Client { Client::new() .credentials(Credentials::basic(login, find_account_secret(login))) .timeout(Duration::from_secs(3600)) .accept_invalid_certs(true) .connect(&format!("https://127.0.0.1:1800{node_id}")) .await .unwrap() } pub async fn imap_client(&self, login: &str, node_id: u32) -> ImapConnection { let mut conn = ImapConnection::connect_to(b"A1 ", format!("127.0.0.1:1900{node_id}")).await; conn.assert_read(Type::Untagged, ResponseType::Ok).await; conn.authenticate(login, find_account_secret(login)).await; conn } pub fn server(&self, node_id: usize) -> &Server { self.servers .get(node_id) .unwrap_or_else(|| panic!("No server found for node ID: {}", node_id)) } pub fn account_id(&self, login: &str) -> u32 { self.account_ids .get(login) .cloned() .unwrap_or_else(|| panic!("No account ID found for login: {}", login)) } } fn find_account_secret(login: &str) -> &str { TEST_USERS .iter() .find(|(account, _, _, _)| account == &login) .map(|(_, secret, _, _)| secret) .unwrap_or_else(|| panic!("No account found for login: {}", login)) } async fn build_server(mut config: Config, stores: Stores) -> (Server, watch::Sender) { // Parse servers let mut servers = Listeners::parse(&mut config); // Bind ports and drop privileges servers.bind_and_drop_priv(&mut config); // Parse core let config_manager = ConfigManager { cfg_local: Default::default(), cfg_local_path: PathBuf::new(), cfg_local_patterns: Patterns::parse(&mut config).into(), cfg_store: config .value("storage.data") .and_then(|id| stores.stores.get(id)) .cloned() .unwrap_or_default(), }; let tracers = Telemetry::parse(&mut config, &stores); let core = Core::parse(&mut config, stores, config_manager) .await .enable_enterprise(); let data = Data::parse(&mut config); let cache = Caches::parse(&mut config); let (ipc, mut ipc_rxs) = build_ipc(true); let inner = Arc::new(Inner { shared_core: core.into_shared(), data, ipc, cache, }); // Parse acceptors servers.parse_tcp_acceptors(&mut config, inner.clone()); // Enable tracing tracers.enable(true); // Start services config.assert_no_errors(); ipc_rxs.spawn_queue_manager(inner.clone()); ipc_rxs.spawn_services(inner.clone()); // Spawn servers let (shutdown_tx, shutdown_rx) = servers.spawn(|server, acceptor, shutdown_rx| { match &server.protocol { ServerProtocol::Smtp | ServerProtocol::Lmtp => server.spawn( SmtpSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Http => server.spawn( HttpSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Imap => server.spawn( ImapSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Pop3 => server.spawn( Pop3SessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::ManageSieve => server.spawn( ManageSieveSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), }; }); // Start broadcast subscriber spawn_broadcast_subscriber(inner.clone(), shutdown_rx); (inner.build_server(), shutdown_tx) } const SERVER: &str = r#" [server] hostname = "'server{NODE_ID}.example.org'" [http] url = "'https://127.0.0.1:800{NODE_ID}'" [cluster] node-id = {NODE_ID} coordinator = "{PUBSUB}" [server.listener.http] bind = ["127.0.0.1:1800{NODE_ID}"] protocol = "http" max-connections = 81920 tls.implicit = true [server.listener.imap] bind = ["127.0.0.1:1900{NODE_ID}"] protocol = "imap" max-connections = 81920 [server.listener.lmtp] bind = ['127.0.0.1:1700{NODE_ID}'] protocol = 'lmtp' tls.implicit = false [server.socket] reuse-addr = true [server.tls] enable = true implicit = false certificate = "default" [session.ehlo] reject-non-fqdn = false [session.rcpt] relay = [ { if = "!is_empty(authenticated_as)", then = true }, { else = false } ] directory = "'{STORE}'" [session.rcpt.errors] total = 5 wait = "1ms" [session.auth] mechanisms = "[plain, login, oauthbearer]" directory = "'{STORE}'" [resolver] type = "system" [queue.strategy] route = [ { if = "rcpt_domain == 'example.com'", then = "'local'" }, { else = "'mx'" } ] [store."foundationdb"] type = "foundationdb" [store."postgresql"] type = "postgresql" host = "localhost" port = 5432 database = "stalwart" user = "postgres" password = "mysecretpassword" [store."mysql"] type = "mysql" host = "localhost" port = 3307 database = "stalwart" user = "root" password = "password" [certificate.default] cert = "%{file:{CERT}}%" private-key = "%{file:{PK}}%" [storage] data = "{STORE}" fts = "{STORE}" blob = "{STORE}" lookup = "{STORE}" directory = "{STORE}" [directory."{STORE}"] type = "internal" store = "{STORE}" [imap.auth] allow-plain-text = true [oauth] key = "parerga_und_paralipomena" [spam-filter] enable = false [tracer.console] type = "console" level = "{LEVEL}" multiline = false ansi = true disabled-events = ["network.*", "telemetry.webhook-error", "http.request-body", "eval.result", "store.*", "dkim.*", "queue.*", "delivery.*", "*.raw-input", "*.raw-output" ] "#; const SERVER_NATS: &str = r#" [store."nats"] type = "nats" urls = "127.0.0.1:4444" "#; const SERVER_REDIS: &str = r#" [store."redis"] type = "redis" urls = "redis://127.0.0.1" redis-type = "single" "#; ================================================ FILE: tests/src/cluster/stress.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::{assert_is_empty, mail::mailbox::destroy_all_mailboxes_no_wait, wait_for_index}; use common::Server; use directory::backend::internal::manage::ManageDirectory; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, message::metadata::MessageData, }; use futures::future::join_all; use jmap_client::{ client::Client, core::set::{SetErrorType, SetObject}, mailbox::{self, Mailbox, Role}, }; use std::{str::FromStr, sync::Arc, time::Duration}; use store::{ ValueKey, rand::{self, Rng}, roaring::RoaringBitmap, write::{AlignedBytes, Archive}, }; use types::{collection::Collection, id::Id}; const TEST_USER_ID: u32 = 1; const NUM_PASSES: usize = 1; pub async fn test(server: Server, mut client: Client) { println!("Running cluster concurrency stress tests..."); server .core .storage .data .get_or_create_principal_id("john", directory::Type::Individual) .await .unwrap(); client.set_default_account_id(Id::from(TEST_USER_ID).to_string()); let client = Arc::new(client); email_tests(server.clone(), client.clone()).await; mailbox_tests(server.clone(), client.clone()).await; } async fn email_tests(server: Server, client: Arc) { for pass in 0..NUM_PASSES { println!( "----------------- EMAIL STRESS TEST {} -----------------", pass ); let mailboxes = Arc::new(vec![ client .mailbox_create("Stress 1", None::, Role::None) .await .unwrap() .take_id(), client .mailbox_create("Stress 2", None::, Role::None) .await .unwrap() .take_id(), client .mailbox_create("Stress 3", None::, Role::None) .await .unwrap() .take_id(), ]); let mut futures = Vec::new(); for num in 0..1000 { match rand::rng().random_range(0..3) { 0 => { let client = client.clone(); let mailboxes = mailboxes.clone(); futures.push(tokio::spawn(async move { let mailbox_num = rand::rng().random_range::(0..mailboxes.len()); let _message_id = client .email_import( format!( concat!( "From: test@test.com\n", "To: test@test.com\r\n", "Subject: test {}\r\n\r\ntest {}\r\n" ), num, num ) .into_bytes(), [&mailboxes[mailbox_num]], None::>, None, ) .await .unwrap() .take_id(); /*println!( "Inserted message {}.", Id::from_bytes(_message_id.as_bytes()) .unwrap() .document_id() );*/ })); } 1 => { let client = client.clone(); futures.push(tokio::spawn(async move { loop { let mut req = client.build(); req.query_email(); let ids = req.send_query_email().await.unwrap().take_ids(); if !ids.is_empty() { let message_id = &ids[rand::rng().random_range(0..ids.len())]; /*println!( "Deleting message {}.", Id::from_bytes(message_id.as_bytes()).unwrap().document_id() );*/ match client.email_destroy(message_id).await { Ok(_) => { break; } Err(jmap_client::Error::Set(err)) => match err.error() { SetErrorType::NotFound => { break; } SetErrorType::Forbidden => { // Concurrency issue, try again. //println!("Concurrent update, trying again."); } _ => { panic!("Unexpected error: {:?}", err); } }, Err(err) => { panic!("Unexpected error: {:?}", err); } } } else { break; } } })); } _ => { let client = client.clone(); let mailboxes = mailboxes.clone(); futures.push(tokio::spawn(async move { let mut req = client.build(); let ref_id = req.query_email().result_reference(); req.get_email() .ids_ref(ref_id) .properties([jmap_client::email::Property::MailboxIds]); let emails = req .send() .await .unwrap() .unwrap_method_responses() .pop() .unwrap() .unwrap_get_email() .unwrap() .take_list(); if !emails.is_empty() { let message = &emails[rand::rng().random_range(0..emails.len())]; let message_id = message.id().unwrap(); let mailbox_ids = message.mailbox_ids(); assert_eq!(mailbox_ids.len(), 1, "{:#?}", message); let mailbox_id = mailbox_ids.last().unwrap(); loop { let new_mailbox_id = &mailboxes[rand::rng().random_range(0..mailboxes.len())]; if new_mailbox_id != mailbox_id { /*println!( "Moving message {} from {} to {}.", Id::from_bytes(message_id.as_bytes()) .unwrap() .document_id(), Id::from_bytes(mailbox_id.as_bytes()) .unwrap() .document_id(), Id::from_bytes(new_mailbox_id.as_bytes()) .unwrap() .document_id() );*/ let mut req = client.build(); req.set_email() .update(message_id) .mailbox_ids([new_mailbox_id]); req.send_set_email().await.unwrap(); break; } } } })); } } tokio::time::sleep(Duration::from_millis(rand::rng().random_range(5..10))).await; } join_all(futures).await; let cache = server.get_cached_messages(TEST_USER_ID).await.unwrap(); let email_ids = cache .emails .items .iter() .map(|e| e.document_id) .collect::(); let mailbox_ids = cache .mailboxes .items .iter() .map(|m| m.document_id) .collect::(); assert_eq!(mailbox_ids.len(), 8); for mailbox in mailboxes.iter() { let mailbox_id = Id::from_str(mailbox).unwrap().document_id(); let email_ids_in_mailbox = RoaringBitmap::from_iter(cache.in_mailbox(mailbox_id).map(|m| m.document_id)); let mut email_ids_check = email_ids_in_mailbox.clone(); email_ids_check &= &email_ids; assert_eq!(email_ids_in_mailbox, email_ids_check); //println!("Emails {:?}", email_ids_in_mailbox); for email_id in &email_ids_in_mailbox { if let Some(mailbox_tags) = server .store() .get_value::>(ValueKey::archive( TEST_USER_ID, Collection::Email, email_id, )) .await .unwrap() { let mailbox_tags = mailbox_tags.deserialize::().unwrap().mailboxes; if mailbox_tags.len() != 1 { panic!( "Email ORM has more than one mailbox {:?}! Id {} in mailbox {} with messages {:?}", mailbox_tags, email_id, mailbox_id, email_ids_in_mailbox ); } let mailbox_tag = mailbox_tags[0]; assert!(mailbox_tag.uid != 0); if mailbox_tag.mailbox_id != mailbox_id { panic!( concat!( "Email ORM has an unexpected mailbox tag {:?}! Id {} in ", "mailbox {} with messages {:?}" ), mailbox_tag, email_id, mailbox_id, email_ids_in_mailbox, ); } } else { panic!( "Email tags not found! Id {} in mailbox {} with messages {:?}", email_id, mailbox_id, email_ids_in_mailbox ); } } } wait_for_index(&server).await; destroy_all_mailboxes_no_wait(&client).await; assert_is_empty(&server).await; } } async fn mailbox_tests(server: Server, client: Arc) { let mailboxes = Arc::new(vec![ "test/test1/test2/test3".to_string(), "test1/test2/test3".to_string(), "test2/test3/test4".to_string(), "test3/test4/test5".to_string(), "test4".to_string(), "test5".to_string(), ]); let mut futures = Vec::new(); println!("----------------- MAILBOX STRESS TEST -----------------"); for _ in 0..1000 { match rand::rng().random_range(0..=3) { 0 => { for pos in 0..mailboxes.len() { let client = client.clone(); let mailboxes = mailboxes.clone(); futures.push(tokio::spawn(async move { //println!("Creating mailbox {}.", mailboxes[pos]); create_mailbox(&client, &mailboxes[pos]).await; })); } } 1 => { let client = client.clone(); futures.push(tokio::spawn(async move { //print!("Querying mailboxes..."); query_mailboxes(&client).await; })); } 2 => { let client = client.clone(); futures.push(tokio::spawn(async move { for mailbox_id in client .mailbox_query(None::, None::>) .await .unwrap() .take_ids() { let client = client.clone(); tokio::spawn(async move { //println!("Deleting mailbox {}.", mailbox_id); delete_mailbox(&client, &mailbox_id).await; }); } })); } _ => { let client = client.clone(); futures.push(tokio::spawn(async move { let mut ids = client .mailbox_query(None::, None::>) .await .unwrap() .take_ids(); if !ids.is_empty() { let id = ids.swap_remove(rand::rng().random_range(0..ids.len())); let sort_order = rand::rng().random_range(0..100); //println!("Updating mailbox {}.", id); client.mailbox_update_sort_order(&id, sort_order).await.ok(); } })); } } tokio::time::sleep(Duration::from_millis(rand::rng().random_range(5..10))).await; } join_all(futures).await; wait_for_index(&server).await; for mailbox_id in client .mailbox_query(None::, None::>) .await .unwrap() .take_ids() { let _ = client.mailbox_move(&mailbox_id, None::).await; } for mailbox_id in client .mailbox_query(None::, None::>) .await .unwrap() .take_ids() { let _ = client.mailbox_destroy(&mailbox_id, true).await; } assert_is_empty(&server).await; } async fn create_mailbox(client: &Client, mailbox: &str) -> Vec { let mut request = client.build(); let mut create_ids: Vec = Vec::new(); let set_request = request.set_mailbox(); for path_item in mailbox.split('/') { let create_item = set_request.create().name(path_item); if let Some(create_id) = create_ids.last() { create_item.parent_id_ref(create_id); } create_ids.push(create_item.create_id().unwrap()); } let mut response = request.send_set_mailbox().await.unwrap(); let mut ids = Vec::with_capacity(create_ids.len()); for create_id in create_ids { if let Ok(mut id) = response.created(&create_id) { ids.push(id.take_id()); } } ids } async fn query_mailboxes(client: &Client) -> Vec { let mut request = client.build(); let query_result = request .query_mailbox() .calculate_total(true) .result_reference(); request.get_mailbox().ids_ref(query_result).properties([ jmap_client::mailbox::Property::Id, jmap_client::mailbox::Property::Name, jmap_client::mailbox::Property::IsSubscribed, jmap_client::mailbox::Property::ParentId, jmap_client::mailbox::Property::Role, jmap_client::mailbox::Property::TotalEmails, jmap_client::mailbox::Property::UnreadEmails, ]); request .send() .await .unwrap() .unwrap_method_responses() .pop() .unwrap() .unwrap_get_mailbox() .unwrap() .take_list() } async fn delete_mailbox(client: &Client, mailbox_id: &str) { for _ in 0..3 { match client.mailbox_destroy(mailbox_id, true).await { Ok(_) => return, Err(err) => match err { jmap_client::Error::Set(_) => break, jmap_client::Error::Transport(_) => { let backoff = rand::rng().random_range(50..=300); tokio::time::sleep(Duration::from_millis(backoff)).await; } _ => panic!("Failed: {:?}", err), }, } } /*println!( "Warning: Too many transport errors while deleting mailbox {}.", mailbox_id );*/ } ================================================ FILE: tests/src/directory/imap.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::Arc; use common::listener::limiter::{ConcurrencyLimiter, InFlight}; use directory::QueryParams; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, net::{TcpListener, TcpStream}, sync::watch, }; use tokio_rustls::TlsAcceptor; use crate::directory::{DirectoryTest, Item, LookupResult}; use super::dummy_tls_acceptor; #[tokio::test] async fn imap_directory() { // Enable logging /*tracing::subscriber::set_global_default( tracing_subscriber::FmtSubscriber::builder() .with_max_level(tracing::Level::DEBUG) .finish(), ) .unwrap();*/ // Spawn mock LMTP server let shutdown = spawn_mock_imap_server(5); tokio::time::sleep(std::time::Duration::from_millis(100)).await; // Obtain directory handle let mut config = DirectoryTest::new(None).await; let handle = config.directories.directories.remove("imap").unwrap(); // Basic lookup let tests = vec![ ( Item::Authenticate(Credentials::Plain { username: "john".to_string(), secret: "ok".to_string(), }), LookupResult::True, ), ( Item::Authenticate(Credentials::Plain { username: "john".to_string(), secret: "bad".to_string(), }), LookupResult::False, ), ]; for (item, expected) in &tests { assert_eq!( &LookupResult::from( handle .query( QueryParams::credentials(item.as_credentials()).with_return_member_of(true) ) .await .unwrap() .is_some() ), expected ); } // Concurrent requests let mut requests = Vec::new(); for n in 0..10 { let (item, expected) = &tests[n % tests.len()]; let item = item.append(n); let item_clone = item.clone(); let handle = handle.clone(); requests.push(( tokio::spawn(async move { LookupResult::from( handle .query( QueryParams::credentials(item.as_credentials()) .with_return_member_of(true), ) .await .unwrap() .is_some(), ) }), item_clone, expected.append(n), )); } for (result, item, expected_result) in requests { assert_eq!( result.await.unwrap(), expected_result, "Failed for {item:?}" ); } // Shutdown shutdown.send(false).ok(); } pub fn spawn_mock_imap_server(max_concurrency: u64) -> watch::Sender { let (tx, mut rx) = watch::channel(true); tokio::spawn(async move { let listener = TcpListener::bind("127.0.0.1:9198") .await .unwrap_or_else(|e| { panic!("Failed to bind mock IMAP server to 127.0.0.1:9198: {e}"); }); let acceptor = dummy_tls_acceptor(); let limited = ConcurrencyLimiter::new(max_concurrency); loop { tokio::select! { stream = listener.accept() => { match stream { Ok((stream, _)) => { //println!("--- Accepted connection --- "); let acceptor = acceptor.clone(); let in_flight = limited.is_allowed(); tokio::spawn(accept_imap(stream, acceptor, in_flight.into())); } Err(err) => { panic!("Something went wrong: {err}" ); } } }, _ = rx.changed() => { break; } }; } }); tx } async fn accept_imap(stream: TcpStream, acceptor: Arc, in_flight: Option) { let mut stream = acceptor.accept(stream).await.unwrap(); stream .write_all(b"* OK Clueless host service ready\r\n") .await .unwrap(); if in_flight.is_none() { eprintln!("WARNING: Concurrency exceeded!"); } let mut buf_u8 = vec![0u8; 1024]; while let Ok(br) = stream.read(&mut buf_u8).await { let buf = std::str::from_utf8(&buf_u8[0..br]).unwrap(); let (op, buf) = buf.split_once(' ').unwrap(); //print!("-> {}", buf); let response = if buf.starts_with("CAPABILITY") { format!( "* CAPABILITY IMAP4rev2 IMAP4rev1 AUTH=PLAIN\r\n{op} OK CAPABILITY completed\r\n", ) } else if buf.starts_with("NOOP") { format!("{op} OK NOOP completed\r\n") } else if buf.starts_with("AUTHENTICATE PLAIN") { let buf = base64_decode(buf.rsplit_once(' ').unwrap().1.as_bytes()).unwrap(); if String::from_utf8_lossy(&buf).contains("ok") { format!("{op} OK Great success!\r\n") } else { format!("{op} BAD No soup for you!\r\n") } } else if buf.starts_with("LOGOUT") { format!("* BYE\r\n{op} OK LOGOUT completed\r\n") } else { panic!("Unknown command: {}", buf.trim()); }; //print!("<- {}", response); for line in response.split_inclusive('\n') { stream.write_all(line.as_bytes()).await.unwrap(); stream.flush().await.unwrap(); tokio::time::sleep(std::time::Duration::from_millis(100)).await; } if buf.contains("bye") || buf.starts_with("LOGOUT") { return; } } } ================================================ FILE: tests/src/directory/internal.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::Arc; use crate::{ directory::{DirectoryTest, IntoTestPrincipal, TestPrincipal}, store::cleanup::{store_assert_is_empty, store_destroy}, }; use ahash::AHashSet; use common::{Core, Inner, Server, config::storage::Storage}; use directory::{ Permission, QueryBy, QueryParams, Type, backend::{ RcptType, internal::{ PrincipalField, PrincipalSet, PrincipalUpdate, PrincipalValue, lookup::DirectoryStore, manage::{self, ChangedPrincipals, ManageDirectory, UpdatePrincipal}, }, }, }; use http::management::stores::destroy_account_data; use mail_send::Credentials; use store::{ IterateParams, Store, ValueKey, write::{BatchBuilder, ValueClass}, }; use types::collection::Collection; #[tokio::test] async fn internal_directory() { let config = DirectoryTest::new(None).await; for (store_id, store) in config.stores.stores { println!("Testing internal directory with store {:?}", store_id); store_destroy(&store).await; // A principal without name should fail assert_eq!( store .create_principal(PrincipalSet::default(), None, None) .await, Err(manage::err_missing(PrincipalField::Name)) ); // Basic account creation let john_id = store .create_principal( TestPrincipal { name: "john".into(), description: Some("John Doe".into()), secrets: vec!["secret".into(), "$app$secret2".into()], ..Default::default() } .into(), None, None, ) .await .unwrap() .id; // Two accounts with the same name should fail assert_eq!( store .create_principal( TestPrincipal { name: "john".into(), ..Default::default() } .into(), None, None ) .await, Err(manage::err_exists(PrincipalField::Name, "john")) ); // An account using a non-existent domain should fail assert_eq!( store .create_principal( TestPrincipal { name: "jane".into(), emails: vec!["jane@example.org".into()], ..Default::default() } .into(), None, None ) .await, Err(manage::not_found("example.org")) ); // Create a domain name store .create_principal( TestPrincipal { name: "example.org".into(), typ: Type::Domain, ..Default::default() } .into(), None, None, ) .await .unwrap(); assert!(store.is_local_domain("example.org").await.unwrap()); assert!(!store.is_local_domain("otherdomain.org").await.unwrap()); // Add an email address assert!( store .update_principal(UpdatePrincipal::by_name("john").with_updates(vec![ PrincipalUpdate::add_item( PrincipalField::Emails, PrincipalValue::String("john@example.org".into()), ) ])) .await .is_ok() ); assert_eq!( store.rcpt("john@example.org").await.unwrap(), RcptType::Mailbox ); assert_eq!( store.email_to_id("john@example.org").await.unwrap(), Some(john_id) ); // Using non-existent domain should fail assert_eq!( store .update_principal(UpdatePrincipal::by_name("john").with_updates(vec![ PrincipalUpdate::add_item( PrincipalField::Emails, PrincipalValue::String("john@otherdomain.org".into()), ) ])) .await, Err(manage::not_found("otherdomain.org")) ); // Create an account with an email address let jane_id = store .create_principal( TestPrincipal { name: "jane".into(), description: Some("Jane Doe".into()), secrets: vec!["my_secret".into(), "$app$my_secret2".into()], emails: vec!["jane@example.org".into()], quota: 123, ..Default::default() } .into(), None, None, ) .await .unwrap() .id; assert_eq!( store.rcpt("jane@example.org").await.unwrap(), RcptType::Mailbox ); assert_eq!( store.rcpt("jane@otherdomain.org").await.unwrap(), RcptType::Invalid ); assert_eq!( store.email_to_id("jane@example.org").await.unwrap(), Some(jane_id) ); assert_eq!(store.vrfy("jane").await.unwrap(), vec!["jane@example.org"]); assert_eq!( store .query( QueryParams::credentials(&Credentials::new("jane".into(), "my_secret".into())) .with_return_member_of(true) ) .await .unwrap() .map(|p| p.into_test()), Some(TestPrincipal { id: jane_id, name: "jane".into(), description: Some("Jane Doe".into()), emails: vec!["jane@example.org".into()], secrets: vec!["my_secret".into(), "$app$my_secret2".into()], quota: 123, ..Default::default() }) ); assert_eq!( store .query( QueryParams::credentials(&Credentials::new( "jane".into(), "wrong_password".into() )) .with_return_member_of(true) ) .await .unwrap(), None ); // Duplicate email address should fail assert_eq!( store .create_principal( TestPrincipal { name: "janeth".into(), description: Some("Janeth Doe".into()), emails: vec!["jane@example.org".into()], ..Default::default() } .into(), None, None ) .await, Err(manage::err_exists( PrincipalField::Emails, "jane@example.org" )) ); // Create a mailing list let list_id = store .create_principal( TestPrincipal { name: "list".into(), typ: Type::List, emails: vec!["list@example.org".into()], ..Default::default() } .into(), None, None, ) .await .unwrap() .id; assert!( store .update_principal(UpdatePrincipal::by_name("list").with_updates(vec![ PrincipalUpdate::set( PrincipalField::Members, PrincipalValue::StringList(vec!["john".into(), "jane".into()]), ), PrincipalUpdate::set( PrincipalField::ExternalMembers, PrincipalValue::StringList(vec![ "mike@other.org".into(), "lucy@foobar.net".into() ]), ) ])) .await .is_ok() ); assert_list_members( &store, "list@example.org", [ "john@example.org", "mike@other.org", "lucy@foobar.net", "jane@example.org", ], ) .await; assert_eq!( store .query(QueryParams::name("list").with_return_member_of(true)) .await .unwrap() .unwrap() .into_test(), TestPrincipal { name: "list".into(), id: list_id, typ: Type::List, emails: vec!["list@example.org".into()], ..Default::default() } ); assert_eq!( store .expn("list@example.org") .await .unwrap() .into_iter() .collect::>(), [ "john@example.org", "mike@other.org", "lucy@foobar.net", "jane@example.org" ] .into_iter() .map(|s| s.into()) .collect::>() ); // Create groups store .create_principal( TestPrincipal { name: "sales".into(), description: Some("Sales Team".into()), typ: Type::Group, ..Default::default() } .into(), None, None, ) .await .unwrap(); store .create_principal( TestPrincipal { name: "support".into(), description: Some("Support Team".into()), typ: Type::Group, ..Default::default() } .into(), None, None, ) .await .unwrap(); // Add John to the Sales and Support groups assert!( store .update_principal(UpdatePrincipal::by_name("john").with_updates(vec![ PrincipalUpdate::add_item( PrincipalField::MemberOf, PrincipalValue::String("sales".into()), ), PrincipalUpdate::add_item( PrincipalField::MemberOf, PrincipalValue::String("support".into()), ) ])) .await .is_ok() ); let principal = store .query(QueryParams::name("john").with_return_member_of(true)) .await .unwrap() .unwrap(); let principal = store.map_principal(principal, &[]).await.unwrap(); assert_eq!( principal.into_test().into_sorted(), TestPrincipal { id: john_id, name: "john".into(), description: Some("John Doe".into()), secrets: vec!["secret".into(), "$app$secret2".into()], emails: vec!["john@example.org".into()], member_of: vec!["sales".into(), "support".into()], lists: vec!["list".into()], ..Default::default() } ); // Adding a non-existent user should fail assert_eq!( store .update_principal(UpdatePrincipal::by_name("john").with_updates(vec![ PrincipalUpdate::add_item( PrincipalField::MemberOf, PrincipalValue::String("accounting".into()), ) ])) .await, Err(manage::not_found("accounting")) ); // Remove a member from a group assert!( store .update_principal(UpdatePrincipal::by_name("john").with_updates(vec![ PrincipalUpdate::remove_item( PrincipalField::MemberOf, PrincipalValue::String("support".into()), ) ])) .await .is_ok() ); let principal = store .query(QueryParams::name("john").with_return_member_of(true)) .await .unwrap() .unwrap(); let principal = store.map_principal(principal, &[]).await.unwrap(); assert_eq!( principal.into_test().into_sorted(), TestPrincipal { id: john_id, name: "john".into(), description: Some("John Doe".into()), secrets: vec!["secret".into(), "$app$secret2".into()], emails: vec!["john@example.org".into()], member_of: vec!["sales".into()], lists: vec!["list".into()], ..Default::default() } ); // Update multiple fields assert!( store .update_principal(UpdatePrincipal::by_name("john").with_updates(vec![ PrincipalUpdate::set( PrincipalField::Name, PrincipalValue::String("john.doe".into()) ), PrincipalUpdate::set( PrincipalField::Description, PrincipalValue::String("Johnny Doe".into()) ), PrincipalUpdate::set( PrincipalField::Secrets, PrincipalValue::StringList(vec!["12345".into()]) ), PrincipalUpdate::set(PrincipalField::Quota, PrincipalValue::Integer(1024)), PrincipalUpdate::remove_item( PrincipalField::Emails, PrincipalValue::String("john@example.org".into()), ), PrincipalUpdate::add_item( PrincipalField::Emails, PrincipalValue::String("john.doe@example.org".into()), ) ])) .await .is_ok() ); let principal = store .query(QueryParams::name("john.doe").with_return_member_of(true)) .await .unwrap() .unwrap(); let principal = store.map_principal(principal, &[]).await.unwrap(); assert_eq!( principal.into_test().into_sorted(), TestPrincipal { id: john_id, name: "john.doe".into(), description: Some("Johnny Doe".into()), secrets: vec!["12345".into()], emails: vec!["john.doe@example.org".into()], quota: 1024, typ: Type::Individual, member_of: vec!["sales".into()], lists: vec!["list".into()], ..Default::default() } ); assert_eq!(store.get_principal_id("john").await.unwrap(), None); assert_eq!( store.rcpt("john@example.org").await.unwrap(), RcptType::Invalid ); assert_eq!( store.rcpt("john.doe@example.org").await.unwrap(), RcptType::Mailbox ); // Remove a member from a mailing list and then add it back assert!( store .update_principal(UpdatePrincipal::by_name("list").with_updates(vec![ PrincipalUpdate::remove_item( PrincipalField::Members, PrincipalValue::String("john.doe".into()), ) ])) .await .is_ok() ); assert_list_members( &store, "list@example.org", ["jane@example.org", "mike@other.org", "lucy@foobar.net"], ) .await; assert!( store .update_principal(UpdatePrincipal::by_name("list").with_updates(vec![ PrincipalUpdate::add_item( PrincipalField::Members, PrincipalValue::String("john.doe".into()), ) ])) .await .is_ok() ); assert_list_members( &store, "list@example.org", [ "john.doe@example.org", "jane@example.org", "mike@other.org", "lucy@foobar.net", ], ) .await; // Field validation assert_eq!( store .update_principal(UpdatePrincipal::by_name("john.doe").with_updates(vec![ PrincipalUpdate::set( PrincipalField::Name, PrincipalValue::String("jane".into()) ), ])) .await, Err(manage::err_exists(PrincipalField::Name, "jane")) ); assert_eq!( store .update_principal(UpdatePrincipal::by_name("john.doe").with_updates(vec![ PrincipalUpdate::add_item( PrincipalField::Emails, PrincipalValue::String("jane@example.org".into()) ), ])) .await, Err(manage::err_exists( PrincipalField::Emails, "jane@example.org" )) ); // List accounts assert_eq!( store .list_principals( None, None, &[Type::Individual, Type::Group, Type::List], true, 0, 0 ) .await .unwrap() .items .into_iter() .map(|p| p.name) .collect::>(), ["jane", "john.doe", "list", "sales", "support"] .into_iter() .map(|s| s.into()) .collect::>() ); assert_eq!( store .list_principals("john".into(), None, &[], true, 0, 0) .await .unwrap() .items .into_iter() .map(|p| p.name) .collect::>(), vec!["john.doe"] ); assert_eq!( store .list_principals(None, None, &[Type::Individual], true, 0, 0) .await .unwrap() .items .into_iter() .map(|p| p.name) .collect::>(), ["jane", "john.doe"] .into_iter() .map(|s| s.into()) .collect::>() ); assert_eq!( store .list_principals(None, None, &[Type::Group], true, 0, 0) .await .unwrap() .items .into_iter() .map(|p| p.name) .collect::>(), ["sales", "support"] .into_iter() .map(|s| s.into()) .collect::>() ); assert_eq!( store .list_principals(None, None, &[Type::List], true, 0, 0) .await .unwrap() .items .into_iter() .map(|p| p.name) .collect::>(), vec!["list"] ); assert_eq!( store .list_principals("example.org".into(), None, &[], true, 0, 0) .await .unwrap() .items .into_iter() .map(|p| p.name) .collect::>(), vec!["example.org", "jane", "john.doe", "list"] ); assert_eq!( store .list_principals("johnny doe".into(), None, &[], true, 0, 0) .await .unwrap() .items .into_iter() .map(|p| p.name) .collect::>(), vec!["john.doe"] ); // Write records on John's and Jane's accounts let mut document_id = u32::MAX; for account_id in [john_id, jane_id] { document_id = store .assign_document_ids(u32::MAX, Collection::Principal, 1) .await .unwrap(); store .write( BatchBuilder::new() .with_account_id(account_id) .with_collection(Collection::Email) .with_document(document_id) .set(ValueClass::Property(0), "hello".as_bytes()) .build_all(), ) .await .unwrap(); assert_eq!( store .get_value::(ValueKey { account_id, collection: Collection::Email.into(), document_id, class: ValueClass::Property(0) }) .await .unwrap(), Some("hello".into()) ); } // Delete John's account and make sure his records are gone let server = Server { inner: Arc::new(Inner::default()), core: Arc::new(Core { storage: Storage { data: store.clone(), blob: store.clone().into(), fts: store.clone().into(), ..Default::default() }, ..Default::default() }), }; store.delete_principal(QueryBy::Id(john_id)).await.unwrap(); destroy_account_data(&server, john_id, true).await.unwrap(); assert_eq!(store.get_principal_id("john.doe").await.unwrap(), None); assert_eq!( store.email_to_id("john.doe@example.org").await.unwrap(), None ); assert_eq!( store.rcpt("john.doe@example.org").await.unwrap(), RcptType::Invalid ); assert_eq!( store .list_principals( None, None, &[Type::Individual, Type::Group, Type::List], true, 0, 0 ) .await .unwrap() .items .into_iter() .map(|p| p.name) .collect::>(), ["jane", "list", "sales", "support"] .into_iter() .map(|s| s.into()) .collect::>() ); assert!(!account_has_emails(&store, john_id).await); assert_eq!( store .get_value::(ValueKey { account_id: john_id, collection: Collection::Email.into(), document_id: 0, class: ValueClass::Property(0) }) .await .unwrap(), None ); // Make sure Jane's records are still there assert_eq!(store.get_principal_id("jane").await.unwrap(), Some(jane_id)); assert_eq!( store.email_to_id("jane@example.org").await.unwrap(), Some(jane_id) ); assert_eq!( store.rcpt("jane@example.org").await.unwrap(), RcptType::Mailbox ); assert!(account_has_emails(&store, jane_id).await); assert_eq!( store .get_value::(ValueKey { account_id: jane_id, collection: Collection::Email.into(), document_id, class: ValueClass::Property(0) }) .await .unwrap(), Some("hello".into()) ); // Clean up destroy_account_data(&server, jane_id, true).await.unwrap(); for principal_name in ["jane", "list", "sales", "support", "example.org"] { store .delete_principal(QueryBy::Name(principal_name)) .await .unwrap(); } store_assert_is_empty(&store, store.clone().into(), true).await; } } #[allow(async_fn_in_trait)] pub trait TestInternalDirectory { async fn create_test_user(&self, login: &str, secret: &str, name: &str, emails: &[&str]) -> u32; async fn create_test_group(&self, login: &str, name: &str, emails: &[&str]) -> u32; async fn create_test_list(&self, login: &str, name: &str, emails: &[&str]) -> u32; async fn set_test_quota(&self, login: &str, quota: u32); async fn add_permissions(&self, login: &str, permissions: impl IntoIterator); async fn remove_permissions( &self, login: &str, permissions: impl IntoIterator, ); async fn add_to_group(&self, login: &str, group: &str) -> ChangedPrincipals; async fn remove_from_group(&self, login: &str, group: &str) -> ChangedPrincipals; async fn remove_test_alias(&self, login: &str, alias: &str); async fn create_test_domains(&self, domains: &[&str]); } impl TestInternalDirectory for Store { async fn create_test_user( &self, login: &str, secret: &str, name: &str, emails: &[&str], ) -> u32 { let role = if login == "admin" { "admin" } else { "user" }; self.create_test_domains(emails).await; if let Some(principal) = self .query(QueryParams::name(login).with_return_member_of(false)) .await .unwrap() { self.update_principal(UpdatePrincipal::by_id(principal.id()).with_updates(vec![ PrincipalUpdate::set( PrincipalField::Secrets, PrincipalValue::StringList(vec![secret.into()]), ), PrincipalUpdate::set( PrincipalField::Description, PrincipalValue::String(name.into()), ), PrincipalUpdate::set( PrincipalField::Emails, PrincipalValue::StringList(emails.iter().map(|s| (*s).into()).collect()), ), PrincipalUpdate::add_item( PrincipalField::Roles, PrincipalValue::String(role.into()), ), PrincipalUpdate::add_item( PrincipalField::EnabledPermissions, PrincipalValue::String(Permission::UnlimitedRequests.name().into()), ), ])) .await .unwrap(); principal.id() } else { self.create_principal( PrincipalSet::new(0, Type::Individual) .with_field(PrincipalField::Name, login) .with_field(PrincipalField::Description, name) .with_field( PrincipalField::Secrets, PrincipalValue::StringList(vec![secret.into()]), ) .with_field( PrincipalField::Emails, PrincipalValue::StringList(emails.iter().map(|s| (*s).into()).collect()), ) .with_field( PrincipalField::Roles, PrincipalValue::StringList(vec![role.into()]), ) .with_field( PrincipalField::EnabledPermissions, PrincipalValue::StringList(vec![ Permission::UnlimitedRequests.name().into(), ]), ), None, None, ) .await .unwrap() .id } } async fn create_test_group(&self, login: &str, name: &str, emails: &[&str]) -> u32 { self.create_test_domains(emails).await; if let Some(principal) = self .query(QueryParams::name(login).with_return_member_of(false)) .await .unwrap() { principal.id() } else { self.create_principal( PrincipalSet::new(0, Type::Group) .with_field(PrincipalField::Name, login) .with_field(PrincipalField::Description, name) .with_field( PrincipalField::Emails, PrincipalValue::StringList(emails.iter().map(|s| (*s).into()).collect()), ) .with_field( PrincipalField::Roles, PrincipalValue::StringList(vec!["user".into()]), ), None, None, ) .await .unwrap() .id } } async fn create_test_list(&self, login: &str, name: &str, members: &[&str]) -> u32 { if let Some(principal) = self .query(QueryParams::name(login).with_return_member_of(false)) .await .unwrap() { principal.id() } else { self.create_test_domains(&[login]).await; self.create_principal( PrincipalSet::new(0, Type::List) .with_field(PrincipalField::Name, login) .with_field(PrincipalField::Description, name) .with_field( PrincipalField::Members, PrincipalValue::StringList(members.iter().map(|s| (*s).into()).collect()), ) .with_field( PrincipalField::Emails, PrincipalValue::StringList(vec![login.into()]), ), None, None, ) .await .unwrap() .id } } async fn set_test_quota(&self, login: &str, quota: u32) { self.update_principal(UpdatePrincipal::by_name(login).with_updates(vec![ PrincipalUpdate::set(PrincipalField::Quota, PrincipalValue::Integer(quota as u64)), ])) .await .unwrap(); } async fn add_permissions( &self, login: &str, permissions: impl IntoIterator, ) { self.update_principal( UpdatePrincipal::by_name(login).with_updates( permissions .into_iter() .map(|p| { PrincipalUpdate::add_item( PrincipalField::EnabledPermissions, PrincipalValue::String(p.name().to_string()), ) }) .collect(), ), ) .await .unwrap(); } async fn remove_permissions( &self, login: &str, permissions: impl IntoIterator, ) { self.update_principal( UpdatePrincipal::by_name(login).with_updates( permissions .into_iter() .map(|p| { PrincipalUpdate::remove_item( PrincipalField::EnabledPermissions, PrincipalValue::String(p.name().to_string()), ) }) .collect(), ), ) .await .unwrap(); } async fn add_to_group(&self, login: &str, group: &str) -> ChangedPrincipals { self.update_principal(UpdatePrincipal::by_name(login).with_updates(vec![ PrincipalUpdate::add_item( PrincipalField::MemberOf, PrincipalValue::String(group.into()), ), ])) .await .unwrap() } async fn remove_from_group(&self, login: &str, group: &str) -> ChangedPrincipals { self.update_principal(UpdatePrincipal::by_name(login).with_updates(vec![ PrincipalUpdate::remove_item( PrincipalField::MemberOf, PrincipalValue::String(group.into()), ), ])) .await .unwrap() } async fn remove_test_alias(&self, login: &str, alias: &str) { self.update_principal(UpdatePrincipal::by_name(login).with_updates(vec![ PrincipalUpdate::remove_item( PrincipalField::Emails, PrincipalValue::String(alias.into()), ), ])) .await .unwrap(); } async fn create_test_domains(&self, domains: &[&str]) { for domain in domains { let domain = domain.rsplit_once('@').map_or(*domain, |(_, d)| d); if self .query(QueryParams::name(domain).with_return_member_of(false)) .await .unwrap() .is_none() { self.create_principal( PrincipalSet::new(0, Type::Domain).with_field(PrincipalField::Name, domain), None, None, ) .await .unwrap(); } } } } async fn account_has_emails(store: &Store, account_id: u32) -> bool { let mut has_emails = false; store .iterate( IterateParams::new( ValueKey { account_id, collection: Collection::Email.into(), document_id: 0, class: ValueClass::Property(0), }, ValueKey { account_id, collection: Collection::Email.into(), document_id: u32::MAX, class: ValueClass::Property(u8::MAX), }, ) .no_values(), |_, _| { has_emails = true; Ok(false) }, ) .await .unwrap(); has_emails } async fn assert_list_members( store: &Store, list_addr: &str, members: impl IntoIterator, ) { match store.rcpt(list_addr).await.unwrap() { RcptType::List(items) => { assert_eq!( items.into_iter().collect::>(), members .into_iter() .map(|s| s.into()) .collect::>() ); } other => panic!("invalid {other:?}"), } } ================================================ FILE: tests/src/directory/ldap.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::fmt::Debug; use directory::{ QueryParams, ROLE_USER, Type, backend::{RcptType, internal::manage::ManageDirectory}, }; use mail_send::Credentials; use crate::directory::{ DirectoryTest, IntoTestPrincipal, TestPrincipal, map_account_id, map_account_ids, }; #[tokio::test] async fn ldap_directory() { // Enable logging crate::enable_logging(); // Obtain directory handle let mut config = DirectoryTest::new("sqlite".into()).await; let handle = config.directories.directories.remove("ldap").unwrap(); let base_store = config.stores.stores.get("sqlite").unwrap(); let core = config.server; // Test authentication for (auth_type, handle) in [ ("Default", handle.clone()), ( "Bind template", config .directories .directories .remove("ldap-bind-template") .unwrap(), ), ( "Bind lookup", config .directories .directories .remove("ldap-bind-lookup") .unwrap(), ), ] { println!("Testing {auth_type} LDAP authentication..."); assert_eq!( handle .query( QueryParams::credentials(&Credentials::Plain { username: "john".into(), secret: "12345".into() }) .with_return_member_of(true) ) .await .unwrap() .unwrap() .into_test() .into_sorted(), TestPrincipal { id: base_store.get_principal_id("john").await.unwrap().unwrap(), name: "john".into(), description: Some("John Doe".into()), secrets: vec!["12345".into()], typ: Type::Individual, member_of: map_account_ids(base_store, vec!["sales"]) .await .into_iter() .map(|v| v.to_string()) .collect(), emails: vec!["john@example.org".into(), "john.doe@example.org".into()], roles: vec![ROLE_USER.to_string()], ..Default::default() } .into_sorted() ); assert_eq!( handle .query( QueryParams::credentials(&Credentials::Plain { username: "bill".into(), secret: "password".into() }) .with_return_member_of(true) ) .await .unwrap() .unwrap() .into_test() .into_sorted(), TestPrincipal { id: base_store.get_principal_id("bill").await.unwrap().unwrap(), name: "bill".into(), description: Some("Bill Foobar".into()), secrets: vec![ "$2y$05$bvIG6Nmid91Mu9RcmmWZfO5HJIMCT8riNW0hEp8f6/FuA2/mHZFpe".into() ], typ: Type::Individual, quota: 500000, emails: vec!["bill@example.org".into(),], roles: vec![ROLE_USER.to_string()], ..Default::default() } .into_sorted() ); assert!( handle .query( QueryParams::credentials(&Credentials::Plain { username: "bill".into(), secret: "invalid".into() }) .with_return_member_of(true) ) .await .unwrap() .is_none() ); } // Get user by name assert_eq!( handle .query(QueryParams::name("jane").with_return_member_of(true)) .await .unwrap() .unwrap() .into_test() .into_sorted(), TestPrincipal { id: base_store.get_principal_id("jane").await.unwrap().unwrap(), name: "jane".into(), description: Some("Jane Doe".into()), typ: Type::Individual, secrets: vec!["abcde".into()], member_of: map_account_ids(base_store, vec!["sales", "support"]) .await .into_iter() .map(|v| v.to_string()) .collect(), emails: vec!["jane@example.org".into(),], roles: vec![ROLE_USER.to_string()], ..Default::default() } .into_sorted() ); // Get group by name assert_eq!( handle .query(QueryParams::name("sales").with_return_member_of(true)) .await .unwrap() .unwrap() .into_test(), TestPrincipal { id: base_store.get_principal_id("sales").await.unwrap().unwrap(), name: "sales".into(), description: Some("sales".into()), typ: Type::Group, roles: vec![ROLE_USER.to_string()], ..Default::default() } ); // Ids by email assert_eq!( core.email_to_id(&handle, "jane@example.org", 0) .await .unwrap(), Some(map_account_id(base_store, "jane").await), ); assert_eq!( core.email_to_id(&handle, "jane+alias@example.org", 0) .await .unwrap(), Some(map_account_id(base_store, "jane").await), ); assert_eq!( core.email_to_id(&handle, "unknown@example.org", 0) .await .unwrap(), None, ); assert_eq!( core.email_to_id(&handle, "anything@catchall.org", 0) .await .unwrap(), Some(map_account_id(base_store, "robert").await) ); // Domain validation assert!(handle.is_local_domain("example.org").await.unwrap()); assert!(!handle.is_local_domain("other.org").await.unwrap()); // RCPT TO assert_eq!( core.rcpt(&handle, "jane@example.org", 0).await.unwrap(), RcptType::Mailbox ); assert_eq!( core.rcpt(&handle, "info@example.org", 0).await.unwrap(), RcptType::Mailbox ); assert_eq!( core.rcpt(&handle, "jane+alias@example.org", 0) .await .unwrap(), RcptType::Mailbox ); assert_eq!( core.rcpt(&handle, "info+alias@example.org", 0) .await .unwrap(), RcptType::Mailbox ); assert_eq!( core.rcpt(&handle, "random_user@catchall.org", 0) .await .unwrap(), RcptType::Mailbox ); assert_eq!( core.rcpt(&handle, "invalid@example.org", 0).await.unwrap(), RcptType::Invalid ); // VRFY compare_sorted( core.vrfy(&handle, "jane", 0).await.unwrap(), vec!["jane@example.org".into()], ); compare_sorted( core.vrfy(&handle, "john", 0).await.unwrap(), vec!["john@example.org".into(), "john.doe@example.org".into()], ); compare_sorted( core.vrfy(&handle, "jane+alias@example", 0).await.unwrap(), vec!["jane@example.org".into()], ); compare_sorted( core.vrfy(&handle, "info", 0).await.unwrap(), Vec::::new(), ); compare_sorted( core.vrfy(&handle, "invalid", 0).await.unwrap(), Vec::::new(), ); // EXPN // Now handled by the internal directory /*compare_sorted( core.expn(&handle, "info@example.org", 0).await.unwrap(), vec![ "bill@example.org".into(), "jane@example.org".into(), "john@example.org".into(), ], ); compare_sorted( core.expn(&handle, "john@example.org", 0).await.unwrap(), Vec::::new(), );*/ } fn compare_sorted(v1: Vec, v2: Vec) { for val in v1.iter() { assert!(v2.contains(val), "{v1:?} != {v2:?}"); } for val in v2.iter() { assert!(v1.contains(val), "{v1:?} != {v2:?}"); } } ================================================ FILE: tests/src/directory/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod imap; pub mod internal; pub mod ldap; pub mod oidc; pub mod smtp; pub mod sql; use common::{Core, Server, config::smtp::session::AddressMapping}; use directory::{ Directories, Principal, PrincipalData, Type, backend::internal::{PrincipalField, PrincipalSet, manage::ManageDirectory}, }; use mail_send::Credentials; use rustls::ServerConfig; use rustls_pemfile::{certs, pkcs8_private_keys}; use rustls_pki_types::PrivateKeyDer; use std::{borrow::Cow, io::BufReader, sync::Arc}; use store::{Store, Stores}; use tokio_rustls::TlsAcceptor; use crate::{AssertConfig, store::TempDir}; const CONFIG: &str = r#" [directory."rocksdb"] type = "internal" store = "rocksdb" [directory."foundationdb"] type = "internal" store = "foundationdb" [directory."sqlite"] type = "sql" store = "sqlite" [directory."sqlite".columns] name = "name" description = "description" secret = "secret" email = "address" quota = "quota" class = "type" [store."rocksdb"] type = "rocksdb" path = "{TMP}/rocksdb" [store."foundationdb"] type = "foundationdb" [store."sqlite"] type = "sqlite" path = "{TMP}/auth.db" [store."sqlite".query] name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = ? AND active = true" members = "SELECT member_of FROM group_members WHERE name = ?" recipients = "SELECT name FROM emails WHERE address = ? ORDER BY name ASC" emails = "SELECT address FROM emails WHERE name = ? AND type != 'list' ORDER BY type DESC, address ASC" verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type = 'primary' ORDER BY address LIMIT 5" expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50" domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || ? LIMIT 1" [storage] lookup = "sqlite" ############################################################################## [directory."postgresql"] type = "sql" store = "postgresql" [directory."postgresql".columns] name = "name" description = "description" secret = "secret" email = "address" quota = "quota" class = "type" [store."postgresql"] type = "postgresql" host = "localhost" port = 5432 database = "stalwart" user = "postgres" password = "mysecretpassword" [store."postgresql".query] name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = $1 AND active = true" members = "SELECT member_of FROM group_members WHERE name = $1" recipients = "SELECT name FROM emails WHERE address = $1 ORDER BY name ASC" emails = "SELECT address FROM emails WHERE name = $1 AND type != 'list' ORDER BY type DESC, address ASC" verify = "SELECT address FROM emails WHERE address LIKE '%' || $1 || '%' AND type = 'primary' ORDER BY address LIMIT 5" expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = $1 AND l.type = 'list' ORDER BY p.address LIMIT 50" domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || $1 LIMIT 1" ############################################################################## [directory."mysql"] type = "sql" store = "mysql" [directory."mysql".columns] name = "name" description = "description" secret = "secret" email = "address" quota = "quota" class = "type" [store."mysql"] type = "mysql" host = "localhost" port = 3307 database = "stalwart" user = "root" password = "password" [store."mysql".query] name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = ? AND active = true" members = "SELECT member_of FROM group_members WHERE name = ?" recipients = "SELECT name FROM emails WHERE address = ? ORDER BY name ASC" emails = "SELECT address FROM emails WHERE name = ? AND type != 'list' ORDER BY type DESC, address ASC" verify = "SELECT address FROM emails WHERE address LIKE CONCAT('%', ?, '%') AND type = 'primary' ORDER BY address LIMIT 5" expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50" domains = "SELECT 1 FROM emails WHERE address LIKE CONCAT('%@', ?) LIMIT 1" ############################################################################## [directory."ldap"] type = "ldap" url = "ldap://localhost:3893" base-dn = "dc=example,dc=org" [directory."ldap".bind] dn = "cn=serviceuser,ou=svcaccts,dc=example,dc=org" secret = "mysecret" [directory."ldap".filter] name = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(uid=?))" email = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(|(mail=?)(givenName=?)(sn=?)))" [directory."ldap".attributes] name = "uid" description = ["principalName", "description"] secret = "userPassword" groups = ["memberOf", "otherGroups"] email = "mail" email-alias = "givenName" quota = "diskQuota" class = "objectClass" [directory."ldap-bind-template"] type = "ldap" url = "ldap://localhost:3893" base-dn = "dc=example,dc=org" [directory."ldap-bind-template".bind] dn = "cn=serviceuser,ou=svcaccts,dc=example,dc=org" secret = "mysecret" [directory."ldap-bind-template".bind.auth] method = "template" template = "cn={username},ou=,dc=example,dc=org" search = false [directory."ldap-bind-template".filter] name = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(uid=?))" email = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(|(mail=?)(givenName=?)(sn=?)))" [directory."ldap-bind-template".attributes] name = "uid" description = ["principalName", "description"] secret = "userPassword" groups = ["memberOf", "otherGroups"] email = "mail" email-alias = "givenName" quota = "diskQuota" class = "objectClass" [directory."ldap-bind-lookup"] type = "ldap" url = "ldap://localhost:3893" base-dn = "dc=example,dc=org" [directory."ldap-bind-lookup".bind] dn = "cn=serviceuser,ou=svcaccts,dc=example,dc=org" secret = "mysecret" [directory."ldap-bind-lookup".bind.auth] method = "lookup" [directory."ldap-bind-lookup".filter] name = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(uid=?))" email = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(|(mail=?)(givenName=?)(sn=?)))" [directory."ldap-bind-lookup".attributes] name = "uid" description = ["principalName", "description"] secret = "userPassword" groups = ["memberOf", "otherGroups"] email = "mail" email-alias = "givenName" quota = "diskQuota" class = "objectClass" ############################################################################## [directory."imap"] type = "imap" host = "127.0.0.1" port = 9198 [directory."imap".pool] max-connections = 5 [directory."imap".tls] enable = true allow-invalid-certs = true ############################################################################## [directory."smtp"] type = "lmtp" host = "127.0.0.1" port = 9199 [directory."smtp".limits] auth-errors = 3 rcpt = 5 [directory."smtp".pool] max-connections = 5 [directory."smtp".tls] enable = true allow-invalid-certs = true [directory."smtp".cache] entries = 500 ttl = {positive = '10s', negative = '5s'} ############################################################################## [directory."local"] type = "memory" [[directory."local".principals]] name = "john" class = "individual" description = "John Doe" secret = "12345" email = ["john@example.org", "jdoe@example.org", "john.doe@example.org"] email-list = ["info@example.org"] member-of = ["sales"] [[directory."local".principals]] name = "jane" class = "individual" description = "Jane Doe" secret = "abcde" email = "jane@example.org" email-list = ["info@example.org"] member-of = ["sales", "support"] [[directory."local".principals]] name = "bill" class = "individual" description = "Bill Foobar" secret = "$2y$05$bvIG6Nmid91Mu9RcmmWZfO5HJIMCT8riNW0hEp8f6/FuA2/mHZFpe" quota = 500000 email = "bill@example.org" email-list = ["info@example.org"] [[directory."local".principals]] name = "sales" class = "group" description = "Sales Team" [[directory."local".principals]] name = "support" class = "group" description = "Support Team" ############################################################################## [directory."oidc-userinfo"] type = "oidc" store = "rocksdb" timeout = "1s" endpoint.url = "https://127.0.0.1:9090/userinfo" endpoint.method = "userinfo" fields.email = "email" fields.username = "preferred_username" fields.full-name = "name" [directory."oidc-introspect-none"] type = "oidc" store = "rocksdb" timeout = "1s" endpoint.url = "https://127.0.0.1:9090/introspect-none" endpoint.method = "introspect" auth.method = "none" fields.email = "email" fields.username = "preferred_username" fields.full-name = "name" [directory."oidc-introspect-user-token"] type = "oidc" store = "rocksdb" timeout = "1s" endpoint.url = "https://127.0.0.1:9090/introspect-user-token" endpoint.method = "introspect" auth.method = "user-token" fields.email = "email" fields.username = "preferred_username" fields.full-name = "name" [directory."oidc-introspect-token"] type = "oidc" store = "rocksdb" timeout = "1s" endpoint.url = "https://127.0.0.1:9090/introspect-token" endpoint.method = "introspect" auth.method = "token" auth.token = "token_of_gratitude" fields.email = "email" fields.username = "preferred_username" fields.full-name = "name" [directory."oidc-introspect-basic"] type = "oidc" store = "rocksdb" timeout = "1s" endpoint.url = "https://127.0.0.1:9090/introspect-basic" endpoint.method = "introspect" auth.method = "basic" auth.username = "myuser" auth.secret = "mypass" fields.email = "email" fields.username = "preferred_username" fields.full-name = "name" "#; pub struct DirectoryStore { pub store: Store, } pub struct DirectoryTest { pub directories: Directories, pub stores: Stores, pub temp_dir: TempDir, pub server: Server, } #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct TestPrincipal { pub id: u32, pub typ: Type, pub quota: u64, pub name: String, pub secrets: Vec, pub emails: Vec, pub member_of: Vec, pub roles: Vec, pub lists: Vec, pub description: Option, } impl DirectoryTest { pub async fn new(id_store: Option<&str>) -> DirectoryTest { let temp_dir = TempDir::new("directory_tests", true); let mut config_file = CONFIG.replace("{TMP}", &temp_dir.path.to_string_lossy()); if id_store.is_some() { // Disable foundationdb store for SQL tests (the fdb select api version can only be run once per process) config_file = config_file .replace( "type = \"foundationdb\"", "type = \"foundationdb\"\ndisable = true", ) .replace( "store = \"foundationdb\"", "store = \"foundationdb\"\ndisable = true", ) } else { // Disable internal store config_file = config_file.replace("type = \"memory\"", "type = \"memory\"\ndisable = true") } let mut config = utils::config::Config::new(&config_file).unwrap(); let stores = Stores::parse_all(&mut config, false).await; let directories = Directories::parse( &mut config, &stores, id_store .map(|id| stores.stores.get(id).unwrap().clone()) .unwrap_or_default(), true, ) .await; config.assert_no_errors(); // Enable catch-all and subaddressing let mut core = Core::default(); core.smtp.session.rcpt.catch_all = AddressMapping::Enable; core.smtp.session.rcpt.subaddressing = AddressMapping::Enable; DirectoryTest { directories, stores, temp_dir, server: Server { inner: Default::default(), core: core.into(), }, } } } const CERT: &str = "-----BEGIN CERTIFICATE----- MIIFCTCCAvGgAwIBAgIUCgHGQYUqtelbHGVSzCVwBL3fyEUwDQYJKoZIhvcNAQEL BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIyMDUxNjExNDAzNFoXDTIzMDUx NjExNDAzNFowFDESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEF AAOCAg8AMIICCgKCAgEAtwS0Fzl3SjaCuKEXgZ/fdWbDoj/qDphyNCAKNevQ0+D0 STNkWCO04aFSH0zcL8zoD9gokNos0i7OU9//ZhZQmex4V6EFdZn8bFwUWN/scUvW HEFXVjtHldO2isZgIxH9LuwRv7KAgkISuWahqerOVDhe7SeQUV0AJGNEh3cT9PZr gSY931BxB7n+5k8eoSk8Z1gtBzQzL62kVGpHDKfw8yX8m65owF9eLUBrNzgxmXfC xpuHwj7hmVhS09PPKeN/RsFS8PsYO7bo0u8jEKalteumjRT7RyUEbioqfo6ZFOGj FHPIq/uKXS9zN1fpoyNh3ur5hMznQhrqlwBM9KlM7GdBJ0pZ3ad0YjT8IL/GnGKR 85J2WZdLqaQdUZo7nV67FhqdDlNE4MdwiykTMjfmLRXGAVhAzJHKyRKNwmkI2aqe S7aqeNgvuDBwY80Q9a2rb5py1Aw+L8yCkUBuHboToDpxSVRDNN8DrWNmmsXnxsOG wRDODy4GICKyxlP+RFSM8xWSQ6y9ktS2OfDBm+Eqcw+3pZKhdz2wgxLkUBJ8X1eh kJrCA/6LTuhy6m6mMjAfoSOFU7fu88jxaWPgvP7GKyH+LM/t9eucobz2ks5rtSjz V4Dc5DCS94/OpVRHwHdaFSPbJKBN9Ev8gnNrAyx/aBPGoHBPG/QUiU7dcUNIPt0C AwEAAaNTMFEwHQYDVR0OBBYEFI167IxBmErB11EqiPPqFLa31ZaMMB8GA1UdIwQY MBaAFI167IxBmErB11EqiPPqFLa31ZaMMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI hvcNAQELBQADggIBALU00IOiH5ubEauVCmakms5ermNTZfculnhnDfWTLMeh2+a7 G4cqADErfMhm/mmLbrw33t9s6tCAhQltvewKR40ST9uMPSyiQbYaCXd5DXnuI6Ox JtNW+UOWIaMf8abnkdLvREOvb8dVQS1i3xq14tAjY5XgpGwCPP8m54b7N3Q7soLn e5PDhPNTnhRIn2RLuYoZmQmMA5fcqEUDYff4epUww7PhrM1QckZligI3566NlGOf j1G9JrivBtY0eaJtamIFnGMBT0ThDudxVja2Nv0C2Elry0p4T/o4nc4M67BJ/y1R vjNLAgFhbxssemU3lZqSd+pykpJBwDBjFSPrZZmQcbk7H6Uz8V1xr/xuzfw6fA13 NWZ5vLgP/DQ13sM+XFlxThKfbPMPVe/UCTvfGtNW+3XyBgPntEkR+fNEawQmzbYl R+X1ymT9MZnEZqRMf7/UD/SYek1aUJefoew3upjMgxYVvh4F8dqJ+39F+xoFzIA2 1dDAEMzXtjA3zKhZ2cycZbEzpJvYA3eGLuR16Suqfi4kPvfwK0mOhCxQmpayt7/X vuEzW6dPCH8Hgbb0WvsSppGOvhdbDaZFNfFc5eNSxhyKzu3H3ACNImZRtZE+yixx 0fR8+xz9kDLf8xupV+X9heyFGHSyYU2Lveaevtr2Ij3weLRgJ6LbNALoeKXk -----END CERTIFICATE----- "; const PK: &str = "-----BEGIN PRIVATE KEY----- MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC3BLQXOXdKNoK4 oReBn991ZsOiP+oOmHI0IAo169DT4PRJM2RYI7ThoVIfTNwvzOgP2CiQ2izSLs5T 3/9mFlCZ7HhXoQV1mfxsXBRY3+xxS9YcQVdWO0eV07aKxmAjEf0u7BG/soCCQhK5 ZqGp6s5UOF7tJ5BRXQAkY0SHdxP09muBJj3fUHEHuf7mTx6hKTxnWC0HNDMvraRU akcMp/DzJfybrmjAX14tQGs3ODGZd8LGm4fCPuGZWFLT088p439GwVLw+xg7tujS 7yMQpqW166aNFPtHJQRuKip+jpkU4aMUc8ir+4pdL3M3V+mjI2He6vmEzOdCGuqX AEz0qUzsZ0EnSlndp3RiNPwgv8acYpHzknZZl0uppB1RmjudXrsWGp0OU0Tgx3CL KRMyN+YtFcYBWEDMkcrJEo3CaQjZqp5Ltqp42C+4MHBjzRD1ratvmnLUDD4vzIKR QG4duhOgOnFJVEM03wOtY2aaxefGw4bBEM4PLgYgIrLGU/5EVIzzFZJDrL2S1LY5 8MGb4SpzD7elkqF3PbCDEuRQEnxfV6GQmsID/otO6HLqbqYyMB+hI4VTt+7zyPFp Y+C8/sYrIf4sz+3165yhvPaSzmu1KPNXgNzkMJL3j86lVEfAd1oVI9skoE30S/yC c2sDLH9oE8agcE8b9BSJTt1xQ0g+3QIDAQABAoICABq5oxqpF5RMtXYEgAw7rkPU h8jPkHwlIrgd3Z/WGZ53APUXfhWo0ScJiZZsgNKyF0kJBZNxaI4gq5xv3zmnFIoF j+Ur7EIqBERGheoceMhqjI9/syMycNeeHM/S/ALjA5ewfT8C7+UVhOpx5DWNxidi O+phlp9q9zRZEo69grqIqVYooWxUsMyyCljTQOPDw8BLjfe5VagmsRJqmolslLDM 4UBSjZVZ18S/3Wgo2oVQia660244BHWCAkZQbbXuNI2+eUAbSoSdxw3WQcaSrywL hzyezbqr2yPDIIVuiUgVUt0Ps0P57VCCN07jlYhvCEGnClysFzD+ATefoZ0wg7za dQu2E+d166rAjnssyhzcHMn3pxgSdtXD+dQR/xfIGbPABucCupEFqKmhLdMm9+ud lHay87qzMpIa8cITJwEQROfXqWAhNUU98pKCOx1SVXBqQC7QVqGQ5solDf0eMSVh ngQ6Dz2WUI2ty75LteiFwlyTgnU9nyPN0NXsrMEET2BHWre7ufTQqiULtQ7+9BwH AMxEKvrQHjMUjdfbXuzdyc5w5mPYJZfFVSQ1HMslx66h9yCpRIsBZvUGvoaP8Tpe nQ66FTYRbiOkkdJ7k8DtrnhsJI1oOGjnvj/rvZ8D2pvrlJcIH2AyN3MOL8Jp5Oj1 nCFt77TwpF92pgl0g9gBAoIBAQDcarmP54QboaIQ9S2gE/4gSVC5i44iDJuSRdI8 K081RQcWiNzqQXTRc5nqJ7KzLyPiGlg+6rWsBKLos5l4t+MdhhH+KUvk/OtT/g8V 0NZBNXLIbSb8j8ix4v3/f2qKHN3Co6QOlxb3gFvobKDdoKqUNiSH1zTZ8/Y/BzkM jqWKhTdaLz6eyzhKfOTA4LO8kJ3VF8HUM1N9/e8Gjorl+gZpJUXUQS0+AIi8W76C OwDrVb3BPGVnApQJfWF78h4g20RwXrx/GYUW2vOMcLjXXDV5U7+nobPUoJnLxoZC 16o88y0Ivan8dBNXsc1epyPvvEqp6MJbAyyVuNeuRJcgYA0BAoIBAQDUkGRV7fLG wCr5rNysUO+FKzVtTJnf9KEsqAqUmmVnG4oubxAJJtiB5n2+DT+CtO8Nrtz05BbR uxfWm+lbEw6lVMj63bywtp0NdULg7/2t+oq2Svv16KrZIRJttXMkdEiFFmkVAEhX l8Fyl6PJPfSMwbPdXEUPUAaNrXweVFffXczHc4W2G212ZzDB0z7QQSgEntbTDFB/ 2Cg5dvuojlM9zw0fuEyLwItZs7n16j/ONZLgBHyroMU9ZPxbnLrVyoZlqtob+RWm Ju2fSIL9QqG6O4td1TqcUBGvFQYjGvKA+q5fsG26NBJ0Ac48cNK6PS4lMkN3Av2J ccloYaMEHAXdAoIBAE8WMCy1Ok6byUXiYxOL+OPmyoM40q/e7DcovE2AkLQhZ3Cr fPDEucCphPFiexkV8f8fysgQeU0WgMmUH54UBPbD81LJyISKR3nkr875Ftdg8SV/ HL0EblN9ifuR4U1bHCrJgoUFq2T09oVH7NR44Ju7bZIcIseNZK6qzcp2qGkycXD3 gLWDX1hCxeV6+qLPFQKvuomEPRH4+jnVDXuFIaW6jPqixDP6BxXmqU2bFDJcmnBq VkwGvc1F4qORdUP+yOi05VeJdZqEx1x92aTUXg+BgEQKnjbNxUE7o1L6hQfHjUIU o5iEoagWkQTEXf2YBwY+EPaNBgNWxnSuAbfJHwECggEBALOF95ezTVWauzD/U6ic +o3n/kl/Zn4FJ5KFodn7xCSe18d7uXlhO34KYqx+l+MWWMefpbGWacdcUjfImf93 SulLgCqP12sP7/iLzp4XUpL7hOeM0NvRU2nqSpwpoUNqik0Mrlc0U+TWoGTduVCf aMjwV65e3VyfY8mIeclLxqM5n1fcM1OoOnzDjiRE+0n7nYa5eAnq3pn6v4449TZY belH03e0ucFWLtrltesBmj3YdWGJqJlzQOInRhNBfXJOh8+ZynfRmP0o54udPDQV cG3PGFd5XPTjkuvhv7sqaSGRlm/um92lWOhtFfdp+i+cuDpmByCef+7zEP19aKZx 3GkCggEAFTs7KNMfvIEaLH0yQUFeq2gLmtcMofmOmeoIECycN1rG7iJo07lJLIs0 bVODH8Z0kX8llu3cjGMAH/6R2uugJSxkmFiZKrngTzKmxDPvTCKWR4RFwXH9j8IO cPq7FtKN4SgrPy9ciAPdkcGmu3zz/sBKOaoPwvU2PdBRT+v/aoz+GCLXAvzFlKVe 9/7zdg87ilo8+AtV+71EJeR3kyBPKS9JrWYUKfiams12+uuH4/53rMFZfNCAaZ3Z 1sdXEO4o3Loc5TX4DbO9FVdBSBe6klEXx4T0QJboO6uBvTBnnRL2SQriJQQFwYT6 XzVV5pwOxkIDBWDIqMUfwJDChBKfpw== -----END PRIVATE KEY----- "; pub fn dummy_tls_acceptor() -> Arc { // Init server config builder with safe defaults let config = ServerConfig::builder().with_no_client_auth(); // load TLS key/cert files let cert_file = &mut BufReader::new(CERT.as_bytes()); let key_file = &mut BufReader::new(PK.as_bytes()); // convert files to key/cert objects let cert_chain = certs(cert_file).map(|r| r.unwrap()).collect(); let mut keys: Vec = pkcs8_private_keys(key_file) .map(|v| PrivateKeyDer::Pkcs8(v.unwrap())) .collect(); // exit if no keys could be parsed if keys.is_empty() { panic!("Could not locate PKCS 8 private keys."); } Arc::new(TlsAcceptor::from(Arc::new( config.with_single_cert(cert_chain, keys.remove(0)).unwrap(), ))) } trait IntoTestPrincipal { fn into_test(self) -> TestPrincipal; } impl IntoTestPrincipal for PrincipalSet { fn into_test(self) -> TestPrincipal { TestPrincipal::from(self) } } impl IntoTestPrincipal for Principal { fn into_test(self) -> TestPrincipal { TestPrincipal::from(self) } } impl TestPrincipal { pub fn into_sorted(mut self) -> Self { self.member_of.sort_unstable(); self.emails.sort_unstable(); self } } impl From for TestPrincipal { fn from(mut value: PrincipalSet) -> Self { Self { id: value.id(), typ: value.typ(), quota: value.quota(), name: value.take_str(PrincipalField::Name).unwrap_or_default(), secrets: value .take_str_array(PrincipalField::Secrets) .unwrap_or_default(), emails: value .take_str_array(PrincipalField::Emails) .unwrap_or_default(), member_of: value .take_str_array(PrincipalField::MemberOf) .unwrap_or_default(), roles: value .take_str_array(PrincipalField::Roles) .unwrap_or_default(), lists: value .take_str_array(PrincipalField::Lists) .unwrap_or_default(), description: value.take_str(PrincipalField::Description), } } } impl From for TestPrincipal { fn from(value: Principal) -> Self { Self { id: value.id(), typ: value.typ(), quota: value.quota().unwrap_or_default(), member_of: value.member_of().map(|v| v.to_string()).collect(), roles: value.roles().map(|v| v.to_string()).collect(), lists: value.lists().map(|v| v.to_string()).collect(), secrets: value .data .iter() .filter_map(|v| match v { PrincipalData::Password(s) | PrincipalData::AppPassword(s) | PrincipalData::OtpAuth(s) => Some(s.to_string()), _ => None, }) .collect(), emails: value.email_addresses().map(|v| v.to_string()).collect(), description: value.description().map(|v| v.to_string()), name: value.name, } } } impl From for PrincipalSet { fn from(value: TestPrincipal) -> Self { PrincipalSet::new(value.id, value.typ) .with_field(PrincipalField::Name, value.name) .with_field(PrincipalField::Quota, value.quota) .with_field(PrincipalField::Secrets, value.secrets) .with_field(PrincipalField::Emails, value.emails) .with_field(PrincipalField::MemberOf, value.member_of) .with_field(PrincipalField::Lists, value.lists) .with_opt_field(PrincipalField::Description, value.description) } } #[derive(Clone, PartialEq, Eq, Hash)] pub enum Item { IsAccount(String), Authenticate(Credentials), Verify(String), Expand(String), } #[derive(Debug, Clone, PartialEq, Eq)] pub enum LookupResult { True, False, Values(Vec), } impl Item { pub fn append(&self, append: usize) -> Self { match self { Item::IsAccount(str) => Item::IsAccount(format!("{append}{str}")), Item::Authenticate(str) => Item::Authenticate(match str { Credentials::Plain { username, secret } => Credentials::Plain { username: username.to_string(), secret: format!("{append}{secret}"), }, Credentials::OAuthBearer { token } => Credentials::OAuthBearer { token: format!("{append}{token}"), }, Credentials::XOauth2 { username, secret } => Credentials::XOauth2 { username: username.to_string(), secret: format!("{append}{secret}"), }, }), Item::Verify(str) => Item::Verify(format!("{append}{str}")), Item::Expand(str) => Item::Expand(format!("{append}{str}")), } } pub fn as_credentials(&self) -> &Credentials { match self { Item::Authenticate(c) => c, _ => panic!("Item is not a Credentials"), } } } impl LookupResult { fn append(&self, append: usize) -> Self { match self { LookupResult::True => LookupResult::True, LookupResult::False => LookupResult::False, LookupResult::Values(v) => { let mut r = Vec::with_capacity(v.len()); for (pos, val) in v.iter().enumerate() { r.push(if pos == 0 { format!("{append}{val}") } else { val.to_string() }); } LookupResult::Values(r) } } } } impl From for LookupResult { fn from(b: bool) -> Self { if b { LookupResult::True } else { LookupResult::False } } } impl From> for LookupResult { fn from(v: Vec) -> Self { LookupResult::Values(v) } } impl core::fmt::Debug for Item { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::IsAccount(arg0) => f.debug_tuple("Rcpt").field(arg0).finish(), Self::Authenticate(_) => f.debug_tuple("Auth").finish(), Self::Expand(arg0) => f.debug_tuple("Expn").field(arg0).finish(), Self::Verify(arg0) => f.debug_tuple("Vrfy").field(arg0).finish(), } } } #[tokio::test] async fn address_mappings() { const MAPPINGS: &str = r#" [enable] catch-all = true subaddressing = true expected-sub = "john.doe@example.org" expected-sub-nomatch = "jane@example.org" expected-catch = "@example.org" [disable] catch-all = false subaddressing = false expected-sub = "john.doe+alias@example.org" expected-sub-nomatch = "jane@example.org" expected-catch = false [custom] catch-all = [{if = "matches('(.+)@(.+)$', address)", then = "'info@' + $2"}, {else = false}] subaddressing = [{ if = "matches('^([^.]+)\\.([^.]+)@(.+)$', address)", then = "$2 + '@' + $3" }, {else = false}] expected-sub = "doe+alias@example.org" expected-sub-nomatch = "jane@example.org" expected-catch = "info@example.org" "#; let mut config = utils::config::Config::new(MAPPINGS).unwrap(); const ADDR: &str = "john.doe+alias@example.org"; const ADDR_NO_MATCH: &str = "jane@example.org"; let core = Server::default(); for test in ["enable", "disable", "custom"] { let catch_all = AddressMapping::parse(&mut config, (test, "catch-all")); let subaddressing = AddressMapping::parse(&mut config, (test, "subaddressing")); assert_eq!( subaddressing.to_subaddress(&core, ADDR, 0).await, config.value_require((test, "expected-sub")).unwrap(), "failed subaddress for {test:?}" ); assert_eq!( subaddressing.to_subaddress(&core, ADDR_NO_MATCH, 0).await, config .value_require((test, "expected-sub-nomatch")) .unwrap(), "failed subaddress no match for {test:?}" ); assert_eq!( catch_all.to_catch_all(&core, ADDR, 0).await, config .property_require::>((test, "expected-catch")) .unwrap() .map(Cow::Owned), "failed catch-all for {test:?}" ); } } async fn map_account_ids(store: &Store, names: Vec>) -> Vec { let mut ids = Vec::with_capacity(names.len()); for name in names { ids.push(map_account_id(store, name).await); } ids } async fn map_account_id(store: &Store, name: impl AsRef) -> u32 { store .get_principal_id(name.as_ref()) .await .unwrap() .unwrap() } ================================================ FILE: tests/src/directory/oidc.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: LicenseRef-SEL * * This file is subject to the Stalwart Enterprise License Agreement (SEL) and * is NOT open source software. * */ use crate::{ directory::DirectoryTest, http_server::{HttpMessage, spawn_mock_http_server}, }; use base64::{Engine, engine::general_purpose}; use directory::QueryParams; use http_proto::{JsonProblemResponse, JsonResponse, ToHttpResponse}; use hyper::{Method, StatusCode}; use mail_send::Credentials; use serde_json::json; use std::sync::Arc; use trc::{AuthEvent, EventType}; static TEST_TOKEN: &str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ"; #[tokio::test] async fn oidc_directory() { // Obtain directory handle let mut config = DirectoryTest::new("rocksdb".into()).await; // Spawn mock OIDC server let _tx = spawn_mock_http_server(Arc::new(|req: HttpMessage| { let success_response = JsonResponse::new(json!({ "email": "john@example.org", "preferred_username": "jdoe", "name": "John Doe", })) .into_http_response(); match (req.method.clone(), req.uri.path().split('/').nth(1)) { (Method::GET, Some("userinfo")) => match req.headers.get("authorization") { Some(auth) if auth == &format!("Bearer {TEST_TOKEN}") => success_response, Some(_) => JsonProblemResponse(StatusCode::UNAUTHORIZED).into_http_response(), None => panic!("Missing Authorization header: {req:#?}"), }, (Method::POST, Some("introspect-none")) => { assert!(req.headers.get("authorization").is_none()); if req.get_url_encoded("token").as_deref() == Some(TEST_TOKEN) { success_response } else { JsonProblemResponse(StatusCode::UNAUTHORIZED).into_http_response() } } (Method::POST, Some("introspect-user-token")) => match req.headers.get("authorization") { Some(auth) if auth == &format!("Bearer {TEST_TOKEN}") && req.get_url_encoded("token").as_deref() == Some(TEST_TOKEN) => { success_response } Some(_) => JsonProblemResponse(StatusCode::UNAUTHORIZED).into_http_response(), None => panic!("Missing Authorization header: {req:#?}"), }, (Method::POST, Some("introspect-token")) => match req.headers.get("authorization") { Some(auth) if auth == "Bearer token_of_gratitude" && req.get_url_encoded("token").as_deref() == Some(TEST_TOKEN) => { success_response } Some(_) => JsonProblemResponse(StatusCode::UNAUTHORIZED).into_http_response(), None => panic!("Missing Authorization header: {req:#?}"), }, (Method::POST, Some("introspect-basic")) => match req.headers.get("authorization") { Some(auth) if auth == &format!( "Basic {}", general_purpose::STANDARD.encode("myuser:mypass".as_bytes()) ) && req.get_url_encoded("token").as_deref() == Some(TEST_TOKEN) => { success_response } Some(_) => JsonProblemResponse(StatusCode::UNAUTHORIZED).into_http_response(), None => panic!("Missing Authorization header: {req:#?}"), }, _ => panic!("Unexpected request: {:?}", req), } })) .await; for test in [ "oidc-userinfo", "oidc-introspect-none", "oidc-introspect-user-token", "oidc-introspect-token", "oidc-introspect-basic", ] { println!("Running OIDC test {test:?}..."); let directory = config.directories.directories.remove(test).unwrap(); // Test an invalid token let err = directory .query( QueryParams::credentials(&Credentials::OAuthBearer { token: "invalid_or_expired_token".to_string(), }) .with_return_member_of(false), ) .await .unwrap_err(); assert!( err.matches(EventType::Auth(AuthEvent::Failed)), "Unexpected error: {:?}", err ); // Test a valid token let principal = directory .query( QueryParams::credentials(&Credentials::OAuthBearer { token: TEST_TOKEN.to_string(), }) .with_return_member_of(false), ) .await .unwrap() .unwrap(); assert_eq!(principal.name(), "jdoe"); assert_eq!(principal.email_addresses().next(), Some("john@example.org")); assert_eq!(principal.description(), Some("John Doe")); } } ================================================ FILE: tests/src/directory/smtp.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::dummy_tls_acceptor; use crate::directory::{DirectoryTest, Item, LookupResult}; use common::listener::limiter::{ConcurrencyLimiter, InFlight}; use directory::{QueryParams, backend::RcptType}; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; use std::sync::Arc; use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, net::{TcpListener, TcpStream}, sync::watch, }; use tokio_rustls::TlsAcceptor; #[tokio::test] async fn lmtp_directory() { // Spawn mock LMTP server let shutdown = spawn_mock_lmtp_server(5); tokio::time::sleep(std::time::Duration::from_millis(100)).await; // Obtain directory handle let mut config = DirectoryTest::new(None).await; let handle = config.directories.directories.remove("smtp").unwrap(); let core = config.server; // Basic lookup let tests = vec![ (Item::IsAccount("john-ok@domain".into()), LookupResult::True), ( Item::IsAccount("john-bad@domain".into()), LookupResult::False, ), ( Item::Verify("john-ok@domain".into()), LookupResult::Values(vec!["john-ok@domain".into()]), ), ( Item::Verify("doesnot@exist.org".into()), LookupResult::False, ), ( Item::Expand("sales-ok,item1,item2,item3".into()), LookupResult::Values(vec![ "sales-ok".into(), "item1".into(), "item2".into(), "item3".into(), ]), ), (Item::Expand("other".into()), LookupResult::False), ( Item::Authenticate(Credentials::Plain { username: "john".into(), secret: "ok".into(), }), LookupResult::True, ), ( Item::Authenticate(Credentials::Plain { username: "john".into(), secret: "bad".into(), }), LookupResult::False, ), ]; for (item, expected) in &tests { let result: LookupResult = match item { Item::IsAccount(v) => { (core.rcpt(&handle, v, 0).await.unwrap() == RcptType::Mailbox).into() } Item::Authenticate(v) => handle .query(QueryParams::credentials(v).with_return_member_of(true)) .await .unwrap() .is_some() .into(), Item::Verify(v) => match core.vrfy(&handle, v, 0).await { Ok(v) => v.into(), Err(e) => { if e.matches(trc::EventType::Store(trc::StoreEvent::NotSupported)) { LookupResult::False } else { panic!("Unexpected error: {e:?}") } } }, Item::Expand(v) => match core.expn(&handle, v, 0).await { Ok(v) => v.into(), Err(e) => { if e.matches(trc::EventType::Store(trc::StoreEvent::NotSupported)) { LookupResult::False } else { panic!("Unexpected error: {e:?}") } } }, }; assert_eq!(&result, expected); } // Concurrent requests let mut requests = Vec::new(); let core = Arc::new(core); for n in 0..100 { let (item, expected) = &tests[n % tests.len()]; let item = item.append(n); let item_clone = item.clone(); let handle = handle.clone(); let core = core.clone(); requests.push(( tokio::spawn(async move { let result: LookupResult = match &item { Item::IsAccount(v) => { (core.rcpt(&handle, v, 0).await.unwrap() == RcptType::Mailbox).into() } Item::Authenticate(v) => handle .query(QueryParams::credentials(v).with_return_member_of(true)) .await .unwrap() .is_some() .into(), Item::Verify(v) => match core.vrfy(&handle, v, 0).await { Ok(v) => v.into(), Err(e) => { if e.matches(trc::EventType::Store(trc::StoreEvent::NotSupported)) { LookupResult::False } else { panic!("Unexpected error: {e:?}") } } }, Item::Expand(v) => match core.expn(&handle, v, 0).await { Ok(v) => v.into(), Err(e) => { if e.matches(trc::EventType::Store(trc::StoreEvent::NotSupported)) { LookupResult::False } else { panic!("Unexpected error: {e:?}") } } }, }; result }), item_clone, expected.append(n), )); } for (result, item, expected_result) in requests { assert_eq!( result.await.unwrap(), expected_result, "Failed for {item:?}" ); } // Shutdown shutdown.send(false).ok(); // Verify that caching works TcpStream::connect("127.0.0.1:9199").await.unwrap_err(); let mut requests = Vec::new(); for n in 0..100 { let (item, expected) = &tests[n % tests.len()]; if matches!(item, Item::IsAccount(_)) { let item = item.append(n); let item_clone = item.clone(); let handle = handle.clone(); let core = core.clone(); requests.push(( tokio::spawn(async move { let result: LookupResult = match &item { Item::IsAccount(v) => { (core.rcpt(&handle, v, 0).await.unwrap() == RcptType::Mailbox).into() } _ => unreachable!(), }; result }), item_clone, expected.append(n), )); } } assert!(!requests.is_empty()); for (result, item, expected_result) in requests { assert_eq!( result.await.unwrap(), expected_result, "Failed for {item:?}" ); } } pub fn spawn_mock_lmtp_server(max_concurrency: u64) -> watch::Sender { let (tx, rx) = watch::channel(true); tokio::spawn(async move { let listener = TcpListener::bind("127.0.0.1:9199") .await .unwrap_or_else(|e| { panic!("Failed to bind mock SMTP server to 127.0.0.1:9199: {e}"); }); let acceptor = dummy_tls_acceptor(); let limited = ConcurrencyLimiter::new(max_concurrency); let mut rx_ = rx.clone(); loop { tokio::select! { stream = listener.accept() => { match stream { Ok((stream, _)) => { let acceptor = acceptor.clone(); let in_flight = limited.is_allowed(); tokio::spawn(accept_smtp(stream, rx.clone(), acceptor, in_flight.into())); } Err(err) => { panic!("Something went wrong: {err}" ); } } }, _ = rx_.changed() => { break; } }; } }); tx } async fn accept_smtp( stream: TcpStream, mut rx: watch::Receiver, acceptor: Arc, in_flight: Option, ) { let mut stream = acceptor.accept(stream).await.unwrap(); stream .write_all(b"220 [127.0.0.1] Clueless host service ready\r\n") .await .unwrap(); if in_flight.is_none() { eprintln!("WARNING: Concurrency exceeded!"); } let mut buf_u8 = vec![0u8; 1024]; loop { let br = tokio::select! { br = stream.read(&mut buf_u8) => { match br { Ok(br) => { br } Err(_) => { break; } } }, _ = rx.changed() => { break; } }; let buf = std::str::from_utf8(&buf_u8[0..br]).unwrap(); let response = if buf.starts_with("LHLO") { "250-mx.foobar.org\r\n250 AUTH PLAIN\r\n".into() } else if buf.starts_with("MAIL FROM") { if buf.contains("<>") || buf.contains("ok@") { "250 OK\r\n".into() } else { "552-I do not\r\n552 like that MAIL FROM.\r\n".into() } } else if buf.starts_with("RCPT TO") { if buf.contains("ok") { "250 OK\r\n".into() } else { "550-I refuse to\r\n550 accept that recipient.\r\n".into() } } else if buf.starts_with("VRFY") { if buf.contains("ok") { format!("250 {}\r\n", buf.split_once(' ').unwrap().1) } else { "550-I refuse to\r\n550 verify that recipient.\r\n".into() } } else if buf.starts_with("EXPN") { if buf.contains("ok") { let parts = buf .split_once(' ') .unwrap() .1 .split(',') .filter_map(|s| { if !s.is_empty() { s.to_string().into() } else { None } }) .collect::>(); let mut buf = String::with_capacity(16); for (pos, part) in parts.iter().enumerate() { buf.push_str("250"); buf.push(if pos == parts.len() - 1 { ' ' } else { '-' }); buf.push_str(part); buf.push_str("\r\n"); } buf } else { "550-I refuse to\r\n550 accept that recipient.\r\n".into() } } else if buf.starts_with("AUTH PLAIN") { let buf = base64_decode(buf.rsplit_once(' ').unwrap().1.as_bytes()).unwrap(); if String::from_utf8_lossy(&buf).contains("ok") { "235 Great success!\r\n".into() } else { "535 No soup for you\r\n".into() } } else if buf.starts_with("NOOP") { "250 Siesta time\r\n".into() } else if buf.starts_with("QUIT") { "250 Arrivederci!\r\n".into() } else if buf.starts_with("RSET") { "250 Your wish is my command.\r\n".into() } else { panic!("Unknown command: {}", buf.trim()); }; //print!("<- {}", response); stream.write_all(response.as_bytes()).await.unwrap(); if buf.contains("bye") || buf.starts_with("QUIT") { return; } } } ================================================ FILE: tests/src/directory/sql.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use directory::{ QueryParams, ROLE_ADMIN, ROLE_USER, Type, backend::{RcptType, internal::manage::ManageDirectory}, }; use mail_send::Credentials; #[allow(unused_imports)] use store::{InMemoryStore, Store}; use crate::{ directory::{DirectoryTest, IntoTestPrincipal, TestPrincipal, map_account_id, map_account_ids}, store::cleanup::store_destroy, }; use super::DirectoryStore; #[tokio::test] async fn sql_directory() { // Enable logging /*tracing::subscriber::set_global_default( tracing_subscriber::FmtSubscriber::builder() .with_max_level(tracing::Level::TRACE) .finish(), ) .unwrap();*/ // Obtain directory handle for directory_id in ["sqlite", "postgresql", "mysql"] { // Parse config let mut config = DirectoryTest::new(directory_id.into()).await; println!("Testing SQL directory {:?}", directory_id); let handle = config.directories.directories.remove(directory_id).unwrap(); let store = DirectoryStore { store: config.stores.stores.remove(directory_id).unwrap(), }; let base_store = &store.store; let core = config.server; // Create tables store_destroy(base_store).await; store.create_test_directory().await; // Create test users store .create_test_user("admin", "very_secret", "Administrator") .await; store.create_test_user("john", "12345", "John Doe").await; store.create_test_user("jane", "abcde", "Jane Doe").await; store .create_test_user( "bill", "$2y$05$bvIG6Nmid91Mu9RcmmWZfO5HJIMCT8riNW0hEp8f6/FuA2/mHZFpe", "Bill Foobar", ) .await; store.set_test_quota("bill", 500000).await; // Create test groups store.create_test_group("sales", "Sales Team").await; store.create_test_group("support", "Support Team").await; // Link users to groups store.add_to_group("john", "sales").await; store.add_to_group("jane", "sales").await; store.add_to_group("jane", "support").await; // Add email addresses store .link_test_address("john", "john@example.org", "primary") .await; store .link_test_address("jane", "jane@example.org", "primary") .await; store .link_test_address("bill", "bill@example.org", "primary") .await; // Add aliases and lists store .link_test_address("john", "john.doe@example.org", "alias") .await; store .link_test_address("john", "jdoe@example.org", "alias") .await; store .link_test_address("john", "info@example.org", "list") .await; store .link_test_address("jane", "info@example.org", "list") .await; store .link_test_address("bill", "info@example.org", "list") .await; // Add catch-all user store .create_test_user("robert", "abcde", "Robert Foobar") .await; store .link_test_address("robert", "robert@catchall.org", "primary") .await; store .link_test_address("robert", "@catchall.org", "alias") .await; // Test authentication assert_eq!( handle .query( QueryParams::credentials(&Credentials::Plain { username: "john".into(), secret: "12345".into() }) .with_return_member_of(true) ) .await .unwrap() .unwrap() .into_test(), TestPrincipal { id: base_store.get_principal_id("john").await.unwrap().unwrap(), name: "john".into(), description: Some("John Doe".into()), secrets: vec!["12345".into()], typ: Type::Individual, member_of: map_account_ids(base_store, vec!["sales"]) .await .into_iter() .map(|v| v.to_string()) .collect(), emails: vec![ "john@example.org".into(), "jdoe@example.org".into(), "john.doe@example.org".into() ], roles: vec![ROLE_USER.to_string()], ..Default::default() } ); assert_eq!( handle .query( QueryParams::credentials(&Credentials::Plain { username: "bill".into(), secret: "password".into() }) .with_return_member_of(true) ) .await .unwrap() .unwrap() .into_test(), TestPrincipal { id: base_store.get_principal_id("bill").await.unwrap().unwrap(), name: "bill".into(), description: Some("Bill Foobar".into()), secrets: vec![ "$2y$05$bvIG6Nmid91Mu9RcmmWZfO5HJIMCT8riNW0hEp8f6/FuA2/mHZFpe".into() ], typ: Type::Individual, quota: 500000, emails: vec!["bill@example.org".into(),], roles: vec![ROLE_USER.to_string()], ..Default::default() } ); assert_eq!( handle .query( QueryParams::credentials(&Credentials::Plain { username: "admin".into(), secret: "very_secret".into() }) .with_return_member_of(true) ) .await .unwrap() .unwrap() .into_test(), TestPrincipal { id: base_store.get_principal_id("admin").await.unwrap().unwrap(), name: "admin".into(), description: Some("Administrator".into()), secrets: vec!["very_secret".into()], typ: Type::Individual, roles: vec![ROLE_ADMIN.to_string()], ..Default::default() } ); assert!( handle .query( QueryParams::credentials(&Credentials::Plain { username: "bill".into(), secret: "invalid".into() }) .with_return_member_of(true) ) .await .unwrap() .is_none() ); // Get user by name let mut p = handle .query(QueryParams::name("jane").with_return_member_of(true)) .await .unwrap() .unwrap() .into_test(); p.member_of.sort(); assert_eq!( p, TestPrincipal { id: base_store.get_principal_id("jane").await.unwrap().unwrap(), name: "jane".into(), description: Some("Jane Doe".into()), typ: Type::Individual, secrets: vec!["abcde".into()], member_of: map_account_ids(base_store, vec!["sales", "support"]) .await .into_iter() .map(|v| v.to_string()) .collect(), emails: vec!["jane@example.org".into(),], roles: vec![ROLE_USER.to_string()], ..Default::default() } ); // Get group by name assert_eq!( handle .query(QueryParams::name("sales").with_return_member_of(true)) .await .unwrap() .unwrap() .into_test(), TestPrincipal { id: base_store.get_principal_id("sales").await.unwrap().unwrap(), name: "sales".into(), description: Some("Sales Team".into()), typ: Type::Group, roles: vec![ROLE_USER.to_string()], ..Default::default() } ); // Ids by email assert_eq!( core.email_to_id(&handle, "jane@example.org", 0) .await .unwrap(), Some(map_account_id(base_store, "jane").await) ); assert_eq!( core.email_to_id(&handle, "jane+alias@example.org", 0) .await .unwrap(), Some(map_account_id(base_store, "jane").await) ); assert_eq!( core.email_to_id(&handle, "unknown@example.org", 0) .await .unwrap(), None ); assert_eq!( core.email_to_id(&handle, "anything@catchall.org", 0) .await .unwrap(), Some(map_account_id(base_store, "robert").await) ); // Domain validation assert!(handle.is_local_domain("example.org").await.unwrap()); assert!(!handle.is_local_domain("other.org").await.unwrap()); // RCPT TO assert_eq!( core.rcpt(&handle, "jane@example.org", 0).await.unwrap(), RcptType::Mailbox ); assert_eq!( core.rcpt(&handle, "info@example.org", 0).await.unwrap(), RcptType::Mailbox ); assert_eq!( core.rcpt(&handle, "jane+alias@example.org", 0) .await .unwrap(), RcptType::Mailbox ); assert_eq!( core.rcpt(&handle, "info+alias@example.org", 0) .await .unwrap(), RcptType::Mailbox ); assert_eq!( core.rcpt(&handle, "random_user@catchall.org", 0) .await .unwrap(), RcptType::Mailbox ); assert_eq!( core.rcpt(&handle, "invalid@example.org", 0).await.unwrap(), RcptType::Invalid ); // VRFY assert_eq!( core.vrfy(&handle, "jane", 0).await.unwrap(), vec!["jane@example.org".to_string()] ); assert_eq!( core.vrfy(&handle, "john", 0).await.unwrap(), vec![ "john.doe@example.org".to_string(), "john@example.org".to_string(), ] ); assert_eq!( core.vrfy(&handle, "jane+alias@example", 0).await.unwrap(), vec!["jane@example.org".to_string()] ); assert_eq!( core.vrfy(&handle, "info", 0).await.unwrap(), Vec::::new() ); assert_eq!( core.vrfy(&handle, "invalid", 0).await.unwrap(), Vec::::new() ); // EXPN (now handled by the internal store) /*assert_eq!( core.expn(&handle, "info@example.org", 0).await.unwrap(), vec![ "bill@example.org".into(), "jane@example.org".into(), "john@example.org".into() ] ); assert_eq!( core.expn(&handle, "john@example.org", 0).await.unwrap(), Vec::::new() );*/ } } impl DirectoryStore { pub async fn create_test_directory(&self) { // Create tables for table in ["accounts", "group_members", "emails"] { self.store .sql_query::(&format!("DROP TABLE IF EXISTS {table}"), vec![]) .await .unwrap(); } for query in [ concat!( "CREATE TABLE accounts (name TEXT PRIMARY KEY, secret TEXT, description TEXT,", " type TEXT NOT NULL, quota INTEGER ", "DEFAULT 0, active BOOLEAN DEFAULT TRUE)" ), concat!( "CREATE TABLE group_members (name TEXT NOT NULL, member_of ", "TEXT NOT NULL, PRIMARY KEY (name, member_of))" ), concat!( "CREATE TABLE emails (name TEXT NOT NULL, address TEXT NOT", " NULL, type TEXT, PRIMARY KEY (name, address))" ), "INSERT INTO accounts (name, secret, type) VALUES ('admin', 'secret', 'admin')", ] { let query = if self.is_mysql() { query.replace("TEXT", "VARCHAR(255)") } else { query.into() }; self.store .sql_query::(&query, vec![]) .await .unwrap_or_else(|_| panic!("failed for {query}")); } } pub async fn create_test_user(&self, login: &str, secret: &str, name: &str) { let account_type = if login == "admin" { "admin" } else { "individual" }; self.store .sql_query::( if self.is_postgresql() { concat!( "INSERT INTO accounts (name, secret, description, ", "type, active) VALUES ($1, $2, $3, $4, true) ", "ON CONFLICT (name) ", "DO UPDATE SET secret = $2, description = $3, type = $4, active = true" ) } else if self.is_mysql() { concat!( "INSERT INTO accounts (name, secret, description, ", "type, active) VALUES (?, ?, ?, ?, true) ", "ON DUPLICATE KEY UPDATE ", "secret = VALUES(secret), description = VALUES(description), ", "type = VALUES(type), active = true" ) } else { concat!( "INSERT INTO accounts (name, secret, description, ", "type, active) VALUES (?, ?, ?, ?, true) ", "ON CONFLICT(name) DO UPDATE SET ", "secret = excluded.secret, description = excluded.description, ", "type = excluded.type, active = true" ) }, vec![ login.into(), secret.into(), name.into(), account_type.into(), ], ) .await .unwrap(); } pub async fn create_test_user_with_email(&self, login: &str, secret: &str, name: &str) { self.create_test_user(login, secret, name).await; self.link_test_address(login, login, "primary").await; } pub async fn create_test_group(&self, login: &str, name: &str) { self.store .sql_query::( if self.is_postgresql() { concat!( "INSERT INTO accounts (name, description, ", "type, active) VALUES ($1, $2, $3, $4) ON CONFLICT (name) DO NOTHING" ) } else if self.is_mysql() { concat!( "INSERT IGNORE INTO accounts (name, description, ", "type, active) VALUES (?, ?, ?, ?)" ) } else { concat!( "INSERT OR IGNORE INTO accounts (name, description, ", "type, active) VALUES (?, ?, ?, ?)" ) }, vec![login.into(), name.into(), "group".into(), true.into()], ) .await .unwrap(); } pub async fn create_test_group_with_email(&self, login: &str, name: &str) { self.create_test_group(login, name).await; self.link_test_address(login, login, "primary").await; } pub async fn link_test_address(&self, login: &str, address: &str, typ: &str) { self.store .sql_query::( if self.is_postgresql() { "INSERT INTO emails (name, address, type) VALUES ($1, $2, $3) ON CONFLICT (name, address) DO NOTHING" } else if self.is_mysql() { "INSERT IGNORE INTO emails (name, address, type) VALUES (?, ?, ?)" } else { "INSERT OR IGNORE INTO emails (name, address, type) VALUES (?, ?, ?)" }, vec![login.into(), address.into(), typ.into()], ) .await .unwrap(); } pub async fn set_test_quota(&self, login: &str, quota: u32) { self.store .sql_query::( if self.is_postgresql() { "UPDATE accounts SET quota = $1 where name = $2" } else { "UPDATE accounts SET quota = ? where name = ?" }, vec![quota.into(), login.into()], ) .await .unwrap(); } pub async fn add_to_group(&self, login: &str, group: &str) { self.store .sql_query::( if self.is_postgresql() { "INSERT INTO group_members (name, member_of) VALUES ($1, $2)" } else { "INSERT INTO group_members (name, member_of) VALUES (?, ?)" }, vec![login.into(), group.into()], ) .await .unwrap(); } pub async fn remove_from_group(&self, login: &str, group: &str) { self.store .sql_query::( if self.is_postgresql() { "DELETE FROM group_members WHERE name = $1 AND member_of = $2" } else { "DELETE FROM group_members WHERE name = ? AND member_of = ?" }, vec![login.into(), group.into()], ) .await .unwrap(); } pub async fn remove_test_alias(&self, login: &str, alias: &str) { self.store .sql_query::( if self.is_postgresql() { "DELETE FROM emails WHERE name = $1 AND address = $2" } else { "DELETE FROM emails WHERE name = ? AND address = ?" }, vec![login.into(), alias.into()], ) .await .unwrap(); } fn is_mysql(&self) -> bool { #[cfg(feature = "mysql")] { matches!(self.store, Store::MySQL(_)) } #[cfg(not(feature = "mysql"))] { false } } fn is_postgresql(&self) -> bool { #[cfg(feature = "postgres")] { matches!(self.store, Store::PostgreSQL(_)) } #[cfg(not(feature = "postgres"))] { false } } #[allow(dead_code)] fn is_sqlite(&self) -> bool { #[cfg(feature = "sqlite")] { matches!(self.store, Store::SQLite(_)) } #[cfg(not(feature = "sqlite"))] { false } } } ================================================ FILE: tests/src/http_server.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::Arc; use ahash::AHashMap; use common::{Caches, Core, Data, Inner, config::server::Listeners, listener::SessionData}; use http_proto::{HttpResponse, request::fetch_body}; use hyper::{Method, Uri, body, server::conn::http1, service::service_fn}; use hyper_util::rt::TokioIo; use tokio::sync::watch; use utils::config::Config; use crate::{AssertConfig, add_test_certs}; const MOCK_HTTP_SERVER: &str = r#" [server] hostname = "'oidc.example.org'" [http] url = "'https://127.0.0.1:9090'" [server.listener.jmap] bind = ['127.0.0.1:9090'] protocol = 'http' tls.implicit = true [server.socket] reuse-addr = true [certificate.default] cert = '%{file:{CERT}}%' private-key = '%{file:{PK}}%' default = true "#; #[derive(Clone)] pub struct HttpSessionManager { inner: HttpRequestHandler, } pub type HttpRequestHandler = Arc HttpResponse + Sync + Send>; #[derive(Debug)] pub struct HttpMessage { pub method: Method, pub headers: AHashMap, pub uri: Uri, pub body: Option>, } impl HttpMessage { pub fn get_url_encoded(&self, key: &str) -> Option { form_urlencoded::parse(self.body.as_ref()?.as_slice()) .find(|(k, _)| k == key) .map(|(_, v)| v.into_owned()) } } pub async fn spawn_mock_http_server( handler: HttpRequestHandler, ) -> (watch::Sender, watch::Receiver) { // Start mock push server let mut settings = Config::new(add_test_certs(MOCK_HTTP_SERVER)).unwrap(); settings.resolve_all_macros().await; let mock_inner = Arc::new(Inner { shared_core: Core::parse(&mut settings, Default::default(), Default::default()) .await .into_shared(), data: Data::parse(&mut settings), cache: Caches::parse(&mut settings), ..Default::default() }); settings.errors.clear(); settings.warnings.clear(); let mut servers = Listeners::parse(&mut settings); servers.parse_tcp_acceptors(&mut settings, mock_inner.clone()); // Start JMAP server servers.bind_and_drop_priv(&mut settings); settings.assert_no_errors(); servers.spawn(|server, acceptor, shutdown_rx| { server.spawn( HttpSessionManager { inner: handler.clone(), }, mock_inner.clone(), acceptor, shutdown_rx, ); }) } impl common::listener::SessionManager for HttpSessionManager { #[allow(clippy::manual_async_fn)] fn handle( self, session: SessionData, ) -> impl std::future::Future + Send { async move { let sender = self.inner; let _ = http1::Builder::new() .keep_alive(false) .serve_connection( TokioIo::new(session.stream), service_fn(|mut req: hyper::Request| { let sender = sender.clone(); async move { let response = sender(HttpMessage { method: req.method().clone(), uri: req.uri().clone(), headers: req .headers() .iter() .map(|(k, v)| { (k.as_str().to_lowercase(), v.to_str().unwrap().to_string()) }) .collect(), body: fetch_body(&mut req, 1024 * 1024, 0).await, }); Ok::<_, hyper::Error>(response.build()) } }), ) .await; } } #[allow(clippy::manual_async_fn)] fn shutdown(&self) -> impl std::future::Future + Send { async {} } } ================================================ FILE: tests/src/imap/acl.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::mail::delivery::SmtpConnection; use super::{AssertResult, ImapConnection, Type, append::assert_append_message}; use imap_proto::ResponseType; pub async fn test(mut imap_john: &mut ImapConnection, _imap_check: &mut ImapConnection) { // Delivery to support account println!("Running ACL tests..."); let mut lmtp = SmtpConnection::connect_port(11201).await; lmtp.ingest( "bill@example.com", &["support@example.com"], concat!( "From: bill@example.com\r\n", "To: support@example.com\r\n", "Subject: TPS Report\r\n", "\r\n", "I'm going to need those TPS reports ASAP. ", "So, if you could do that, that'd be great." ), ) .await; // Connect to all test accounts let mut imap_jane = ImapConnection::connect(b"_w ").await; let mut imap_bill = ImapConnection::connect(b"_z ").await; for (imap, secret) in [ (&mut imap_jane, "AGphbmUuc21pdGhAZXhhbXBsZS5jb20Ac2VjcmV0"), (&mut imap_bill, "AGZvb2JhckBleGFtcGxlLmNvbQBzZWNyZXQ="), ] { imap.assert_read(Type::Untagged, ResponseType::Ok).await; imap.send(&format!( "AUTHENTICATE PLAIN {{{}+}}\r\n{}", secret.len(), secret )) .await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; } // Jane should see the Support account imap_jane.send("LIST \"\" \"*\"").await; imap_jane .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("Shared Folders/support@example.com/INBOX"); imap_jane .send("SELECT \"Shared Folders/support@example.com/INBOX\"") .await; imap_jane.assert_read(Type::Tagged, ResponseType::Ok).await; imap_jane.send("FETCH 1 (PREVIEW)").await; imap_jane .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("TPS reports ASAP"); imap_jane.send("UNSELECT").await; imap_jane.assert_read(Type::Tagged, ResponseType::Ok).await; // Jane should be able to create folders under the Support account imap_jane .send("CREATE \"Shared Folders/support@example.com/inbox/Jane's Folder\"") .await; imap_jane.assert_read(Type::Tagged, ResponseType::Ok).await; imap_jane.send("LIST \"\" \"*\"").await; imap_jane .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_equals( "* LIST () \"/\" \"Shared Folders/support@example.com/INBOX/Jane's Folder\"", ); imap_jane .send("DELETE \"Shared Folders/support@example.com/INBOX/Jane's Folder\"") .await; imap_jane.assert_read(Type::Tagged, ResponseType::Ok).await; // John should have no shared folders imap_john.send("LIST \"\" \"*\"").await; imap_john .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("Shared Folders", 0); imap_john.send("NAMESPACE").await; imap_john.assert_read(Type::Tagged, ResponseType::Ok).await; // List rights imap_jane.send("LISTRIGHTS INBOX jdoe@example.com").await; imap_jane .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_equals("* LISTRIGHTS \"INBOX\" \"jdoe@example.com\" r l ws i et k x p a"); // Jane shares her Inbox to John, expect a Shared Folders item in John's list imap_jane.send("SETACL INBOX jdoe@example.com lr").await; imap_jane.assert_read(Type::Tagged, ResponseType::Ok).await; imap_john.send("LIST \"\" \"*\"").await; imap_john .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_equals("* LIST (\\NoSelect) \"/\" \"Shared Folders\"") .assert_equals("* LIST (\\NoSelect) \"/\" \"Shared Folders/jane.smith@example.com\"") .assert_equals("* LIST () \"/\" \"Shared Folders/jane.smith@example.com/INBOX\""); // Grant access to Bill and check ACLs imap_jane.send("GETACL INBOX").await; imap_jane .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("\"jdoe@example.com\" rl"); imap_jane .send("SETACL INBOX foobar@example.com lrxtws") .await; imap_jane.assert_read(Type::Tagged, ResponseType::Ok).await; imap_jane.send("GETACL INBOX").await; imap_jane .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("\"jdoe@example.com\" rl") .assert_contains("\"foobar@example.com\" tewsrxl"); imap_bill.send("LIST \"\" \"*\"").await; imap_bill .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("Shared Folders/jane.smith@example.com/INBOX"); // Namespace should now return the Shared Folders namespace imap_john.send("NAMESPACE").await; imap_john .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_equals("* NAMESPACE ((\"\" \"/\")) ((\"Shared Folders\" \"/\")) NIL"); // List John's right on Jane's Inbox imap_john .send("MYRIGHTS \"Shared Folders/jane.smith@example.com/INBOX\"") .await; imap_john .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_equals("* MYRIGHTS \"Shared Folders/jane.smith@example.com/INBOX\" rl"); // John should not be able to append messages assert_append_message( imap_john, "Shared Folders/jane.smith@example.com/INBOX", "From: john\n\ncontents", ResponseType::No, ) .await; // Grant insert access to John on Jane's Inbox, and try inserting the // message again. imap_jane.send("SETACL INBOX jdoe@example.com +i").await; imap_jane.assert_read(Type::Tagged, ResponseType::Ok).await; imap_john .send("MYRIGHTS \"Shared Folders/jane.smith@example.com/INBOX\"") .await; imap_john .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_equals("* MYRIGHTS \"Shared Folders/jane.smith@example.com/INBOX\" rli"); assert_append_message( imap_john, "Shared Folders/jane.smith@example.com/INBOX", "From: john\n\ncontents", ResponseType::Ok, ) .await; // Only Bill should be allowed to delete messages on Jane's Inbox for imap in [&mut imap_john, &mut imap_bill] { imap.send("SELECT \"Shared Folders/jane.smith@example.com/INBOX\"") .await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; } imap_john.send("UID STORE 1 +FLAGS (\\Deleted)").await; imap_john.assert_read(Type::Tagged, ResponseType::No).await; imap_bill.send("UID STORE 1 +FLAGS (\\Deleted)").await; imap_bill.assert_read(Type::Tagged, ResponseType::Ok).await; imap_john.send("UID EXPUNGE").await; imap_john.assert_read(Type::Tagged, ResponseType::No).await; imap_john.send("UID FETCH 1 (PREVIEW)").await; imap_john .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("contents"); imap_bill.send("UID EXPUNGE").await; imap_bill.assert_read(Type::Tagged, ResponseType::Ok).await; imap_bill.send("UID FETCH 1 (PREVIEW)").await; imap_bill .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("contents", 0); imap_bill .send("STATUS \"Shared Folders/jane.smith@example.com/INBOX\" (MESSAGES)") .await; imap_bill .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("(MESSAGES 0)"); // Test copying and moving between shared mailboxes let uid = assert_append_message( imap_john, "INBOX", "From: john\n\ncopy test", ResponseType::Ok, ) .await .into_append_uid(); imap_john.send("SELECT INBOX").await; imap_john.assert_read(Type::Tagged, ResponseType::Ok).await; // Copy from John's Inbox to Jane's Inbox imap_john .send(&format!( "UID COPY {} \"Shared Folders/jane.smith@example.com/INBOX\"", uid )) .await; let uid = imap_john .assert_read(Type::Tagged, ResponseType::Ok) .await .into_copy_uid(); // Check that both Bill and Jane can see the message imap_bill.send("NOOP").await; imap_bill.assert_read(Type::Tagged, ResponseType::Ok).await; imap_bill .send(&format!("UID FETCH {} (PREVIEW)", uid)) .await; imap_bill .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("copy test"); imap_jane.send("SELECT INBOX").await; imap_jane.assert_read(Type::Tagged, ResponseType::Ok).await; imap_jane .send(&format!("UID FETCH {} (PREVIEW)", uid)) .await; imap_jane .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("copy test"); // Bill now moves the message to his own Inbox imap_bill.send(&format!("UID MOVE {} INBOX", uid)).await; let uid_moved = imap_bill .assert_read(Type::Tagged, ResponseType::Ok) .await .into_copy_uid(); // Both Jane and Bill should not see the message on Jane's Inbox anymore imap_bill .send(&format!("UID FETCH {} (PREVIEW)", uid)) .await; imap_bill .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("copy test", 0); imap_jane .send(&format!("UID FETCH {} (PREVIEW)", uid)) .await; imap_jane .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("copy test", 0); // Check that the message has been moved to Bill's Inbox imap_bill.send("SELECT INBOX").await; imap_bill.assert_read(Type::Tagged, ResponseType::Ok).await; imap_bill .send(&format!("UID FETCH {} (PREVIEW)", uid_moved)) .await; imap_bill .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("copy test"); // Jane stops sharing with Bill, and removes Insert access to John imap_jane.send("DELETEACL INBOX foobar@example.com").await; imap_jane.assert_read(Type::Tagged, ResponseType::Ok).await; imap_jane.send("SETACL INBOX jdoe@example.com -i").await; imap_jane.assert_read(Type::Tagged, ResponseType::Ok).await; imap_jane.send("GETACL INBOX").await; imap_jane .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("\"jdoe@example.com\" rl") .assert_count("foobar@example.com", 0); // Bill should not have access to Jane's Inbox anymore imap_bill.send("LIST \"\" \"*\"").await; imap_bill .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("Shared Folders", 0); // And John should still have access imap_john.send("LIST \"\" \"*\"").await; imap_john .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("Shared Folders", 3); } ================================================ FILE: tests/src/imap/antispam.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{IMAPTest, ImapConnection}; use crate::{imap::Type, jmap::mail::delivery::SmtpConnection, smtp::session::VerifyResponse}; use common::{Server, manager::SPAM_TRAINER_KEY}; use imap_proto::ResponseType; use spam_filter::modules::classifier::{SpamClassifier, SpamTrainer}; use store::{ Deserialize, IterateParams, U32_LEN, U64_LEN, ValueKey, write::{AlignedBytes, Archive, BlobOp, ValueClass, key::DeserializeBigEndian}, }; use types::blob_hash::BlobHash; pub async fn test(handle: &IMAPTest) { println!("Running Spam classifier tests..."); let mut imap = ImapConnection::connect(b"_x ").await; imap.assert_read(Type::Untagged, ResponseType::Ok).await; imap.authenticate("sgd@example.com", "secret").await; let account_id = handle .server .directory() .email_to_id("sgd@example.com") .await .unwrap() .unwrap(); // Make sure there are no training samples spam_delete_samples(&handle.server).await; assert_eq!(spam_training_samples(&handle.server).await.total_count, 0); // Train the classifier via APPEND imap.append("INBOX", HAM[0]).await; imap.append("Junk Mail", SPAM[0]).await; let samples = spam_training_samples(&handle.server).await; assert_eq!(samples.ham_count, 1); assert_eq!(samples.spam_count, 1); // Append two spam samples to "Drafts", then train the classifier via STORE and MOVE imap.append("Drafts", SPAM[1]).await; imap.append("Drafts", SPAM[2]).await; imap.send_ok("SELECT Drafts").await; imap.send_ok("STORE 1 +FLAGS ($Junk)").await; imap.send_ok("MOVE 2 \"Junk Mail\"").await; let samples = spam_training_samples(&handle.server).await; assert_eq!(samples.ham_count, 1); assert_eq!(samples.spam_count, 3); // Add the remaining messages via APPEND for message in HAM.iter().skip(1) { imap.append("INBOX", message).await; } for message in SPAM.iter().skip(3) { imap.append("Junk Mail", message).await; } let samples = spam_training_samples(&handle.server).await; assert_eq!(samples.ham_count, 10); assert_eq!(samples.spam_count, 10); assert_eq!(samples.samples.len(), 20); assert!( samples .samples .iter() .all(|s| s.account_id == account_id && s.remove.is_none()) ); // Train the classifier handle.server.spam_train(false).await.unwrap(); let model = spam_classifier_model(&handle.server).await; assert_eq!(model.reservoir.ham.total_seen, 10); assert_eq!(model.reservoir.spam.total_seen, 10); assert_eq!( model.last_sample_expiry, samples.samples.iter().map(|s| s.until).max().unwrap() ); assert_eq!(spam_training_samples(&handle.server).await.total_count, 20); assert!(handle.server.inner.data.spam_classifier.load().is_active()); // Send 3 test emails for message in TEST { let mut lmtp = SmtpConnection::connect_port(11201).await; lmtp.ingest("bill@example.com", &["sgd@example.com"], message) .await; } tokio::time::sleep(std::time::Duration::from_millis(200)).await; imap.send_ok("SELECT INBOX").await; imap.send("FETCH 11 (FLAGS RFC822.TEXT)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_not_contains("FLAGS ($Junk") .assert_contains("Subject: can someone explain") .assert_contains("X-Spam-Status: No") .assert_contains("PROB_HAM_HIGH"); imap.send("FETCH 12 (FLAGS RFC822.TEXT)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_not_contains("FLAGS ($Junk") .assert_contains("Subject: classifier test") .assert_contains("X-Spam-Status: No") .assert_contains("PROB_SPAM_UNCERTAIN"); imap.send_ok("SELECT \"Junk Mail\"").await; imap.send("FETCH 10 (FLAGS RFC822.TEXT)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("FLAGS ($Junk") .assert_contains("Subject: save up to") .assert_contains("X-Spam-Status: Yes") .assert_contains("PROB_SPAM_HIGH"); imap.send_ok("MOVE 10 INBOX").await; let samples = spam_training_samples(&handle.server).await; assert_eq!(samples.ham_count, 11); assert_eq!(samples.spam_count, 10); // Make sure spam traps trigger spam classification let mut lmtp = SmtpConnection::connect_port(11201).await; lmtp.ingest("bill@example.com", &["spamtrap@example.com"], SPAM[4]) .await; tokio::time::sleep(std::time::Duration::from_millis(200)).await; let samples = spam_training_samples(&handle.server).await; assert_eq!(samples.ham_count, 11); assert_eq!(samples.spam_count, 11); } #[derive(Default, Debug)] pub struct TrainingSamples { pub samples: Vec, pub spam_count: usize, pub ham_count: usize, pub total_count: usize, } #[derive(Debug)] #[allow(dead_code)] pub struct TrainingSample { pub hash: BlobHash, pub account_id: u32, pub is_spam: bool, pub remove: Option, pub until: u64, } pub async fn spam_classifier_model(server: &Server) -> SpamTrainer { server .blob_store() .get_blob(SPAM_TRAINER_KEY, 0..usize::MAX) .await .and_then(|archive| match archive { Some(archive) => as Deserialize>::deserialize(&archive) .and_then(|archive| archive.deserialize_untrusted::()) .map(Some), None => Ok(None), }) .unwrap() .unwrap() } pub async fn spam_delete_samples(server: &Server) { let from_key = ValueKey { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Blob(BlobOp::SpamSample { hash: BlobHash::default(), until: 0, }), }; let to_key = ValueKey { account_id: u32::MAX, collection: u8::MAX, document_id: u32::MAX, class: ValueClass::Blob(BlobOp::SpamSample { hash: BlobHash::new_max(), until: u64::MAX, }), }; server.store().delete_range(from_key, to_key).await.unwrap(); } pub async fn spam_training_samples(server: &Server) -> TrainingSamples { let mut samples = TrainingSamples::default(); let from_key = ValueKey { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Blob(BlobOp::SpamSample { hash: BlobHash::default(), until: 0, }), }; let to_key = ValueKey { account_id: u32::MAX, collection: u8::MAX, document_id: u32::MAX, class: ValueClass::Blob(BlobOp::SpamSample { hash: BlobHash::new_max(), until: u64::MAX, }), }; server .store() .iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { let until = key.deserialize_be_u64(1)?; let account_id = key.deserialize_be_u32(U64_LEN + 1)?; let hash = BlobHash::try_from_hash_slice(key.get(U64_LEN + U32_LEN + 1..).ok_or_else( || trc::Error::corrupted_key(key, value.into(), trc::location!()), )?) .unwrap(); let (Some(is_spam), Some(hold)) = (value.first(), value.get(1)) else { return Err(trc::Error::corrupted_key( key, value.into(), trc::location!(), )); }; let do_remove = *hold == 0; let is_spam = *is_spam == 1; samples.samples.push(TrainingSample { hash, account_id, is_spam, remove: do_remove.then_some(until), until, }); if is_spam { samples.spam_count += 1; } else { samples.ham_count += 1; } samples.total_count += 1; Ok(true) }, ) .await .unwrap(); samples } impl ImapConnection { async fn append(&mut self, mailbox: &str, message: &str) { self.send_ok(&format!( "APPEND {:?} {{{}+}}\r\n{}", mailbox, message.len(), message )) .await; } async fn send_ok(&mut self, cmd: &str) { self.send(cmd).await; self.assert_read(Type::Tagged, ResponseType::Ok).await; } } pub const SPAM: [&str; 10] = [ concat!( "Subject: save up to = on life insurance\r\n\r\n wh", "y spend more than you have to life quote savings e", "nsuring your family s financial security is very i", "mportant life quote savings makes buying life insu", "rance simple and affordable we provide free access", " to the very best companies and the lowest rates l", "ife quote savings is fast easy and saves you money", " let us help you get started with the best values ", "in the country on new coverage you can save hundre", "ds or even thousands of dollars by requesting a fr", "ee quote from lifequote savings our service will t", "ake you less than = minutes to complete shop ", "and compare save up to = on all types of life", " insurance hyperlink click here for your free quot", "e protecting your family is the best investment yo", "u ll ever make if you are in receipt of this email", " in error and or wish to be removed from our list ", "hyperlink please click here and type remove if you", " reside in any state which prohibits e mail solici", "tations for insurance please disregard this email\r\n", " \r\n" ), concat!( "Subject: a powerhouse gifting program\r\n\r\nyou don t ", "want to miss get in with the founders the major pl", "ayers are on this one for once be where the player", "s are this is your private invitation experts are ", "calling this the fastest way to huge cash flow eve", "r conceived leverage = = into = NUM", "BER over and over again the question here is you e", "ither want to be wealthy or you don t which one ar", "e you i am tossing you a financial lifeline and fo", "r your sake i hope you grab onto it and hold on ti", "ght for the ride of your life testimonials hear wh", "at average people are doing their first few days w", "e ve received = = in = day and we a", "re doing that over and over again q s in al i m a ", "single mother in fl and i ve received = NUMBE", "R in the last = days d s in fl i was not sure", " about this when i sent off my = = pledg", "e but i got back = = the very next day l", " l in ky i didn t have the money so i found myself", " a partner to work this with we have received NUMB", "ER = over the last = days i think i made", " the right decision don t you k c in fl i pick up ", "= = my first day and i they gave me free", " leads and all the training you can too j w in ca ", "announcing we will close your sales for you and he", "lp you get a fax blast immediately upon your entry", " you make the money free leads training don t wait", " call now fax back to = = = = ", "or call = = = = name__________", "________________________phone_____________________", "______________________ fax________________________", "_____________email________________________________", "____________ best time to call____________________", "_____time zone____________________________________", "____ this message is sent in compliance of the new", " e mail bill per section = paragraph a =", " c of s = further transmissions by the sender", " of this email may be stopped at no cost to you by", " sending a reply to this email address with the wo", "rd remove in the subject line errors omissions and", " exceptions excluded this is not spam i have compi", "led this list from our replicate database relative", " to seattle marketing group the gigt or turbo team", " for the sole purpose of these communications your", " continued inclusion is only by your gracious perm", "ission if you wish to not receive this mail from m", "e please send an email to tesrewinter with rem", "ove in the subject and you will be deleted immedia", "tely\r\n\r\n" ), concat!( "Subject: help wanted \r\n\r\nwe are a = year old f", "ortune = company that is growing at a tremend", "ous rate we are looking for individuals who want t", "o work from home this is an opportunity to make an", " excellent income no experience is required we wil", "l train you so if you are looking to be employed f", "rom home with a career that has vast opportunities", " then go we are looking for energetic and self", " motivated people if that is you than click on the", " link and fill out the form and one of our employe", "ment specialist will contact you to be removed fro", "m our link simple go to \r\n\r\n" ), concat!( "Subject: tired of the bull out there\r\n\r\n want to st", "op losing money want a real money maker receive NU", "MBER = = = today experts are callin", "g this the fastest way to huge cash flow ever conc", "eived a powerhouse gifting program you don t want ", "to miss we work as a team this is your private inv", "itation get in with the founders this is where the", " big boys play the major players are on this one f", "or once be where the players are this is a system ", "that will drive = = s to your doorstep i", "n a short period of time leverage = = in", "to = = over and over again the question ", "here is you either want to be wealthy or you don t", " which one are you i am tossing you a financial li", "feline and for your sake i hope you grab onto it a", "nd hold on tight for the ride of your life testimo", "nials hear what average people are doing their fir", "st few days we ve received = = in =", " day and we are doing that over and over again q s", " in al i m a single mother in fl and i ve received", " = = in the last = days d s in fl i", " was not sure about this when i sent off my =", " = pledge but i got back = = the ve", "ry next day l l in ky i didn t have the money so i", " found myself a partner to work this with we have ", "received = = over the last = days i", " think i made the right decision don t you k c in ", "fl i pick up = = my first day and i they", " gave me free leads and all the training you can t", "oo j w in ca this will be the most important call ", "you make this year free leads training announcing ", "we will close your sales for you and help you get ", "a fax blast immediately upon your entry you make t", "he money free leads training don t wait call now N", "UMBER = = = print and fax to =", " = = = or send an email requesting ", "more information to successleads please includ", "e your name and telephone number receive = NU", "MBER free leads just for responding a = NUMBE", "R value name___________________________________ ph", "one___________________________________ fax________", "_____________________________ email_______________", "____________________ this message is sent in compl", "iance of the new e mail bill per section = pa", "ragraph a = c of s = further transmissio", "ns by the sender of this email may be stopped at n", "o cost to you by sending a reply to this email add", "ress with the word remove in the subject line erro", "rs omissions and exceptions excluded this is not s", "pam i have compiled this list from our replicate d", "atabase relative to seattle marketing group the gi", "gt or turbo team for the sole purpose of these com", "munications your continued inclusion is only by yo", "ur gracious permission if you wish to not receive ", "this mail from me please send an email to tesrewin", "ter with remove in the subject and you will be", " deleted immediately\r\n\r\n" ), concat!( "Subject: cellular phone accessories \r\n\r\n all at bel", "ow wholesale prices http = = = NUMB", "ER = sites merchant sales hands free ear buds", " = = phone holsters = = booste", "r antennas only = = phone cases = N", "UMBER car chargers = = face plates as lo", "w as = = lithium ion batteries as low as", " = = http = = = = NU", "MBER sites merchant sales click below for accessor", "ies on all nokia motorola lg nextel samsung qualco", "mm ericsson audiovox phones at below wholesale pri", "ces http = = = = = sites ", "merchant sales if you need assistance please call ", "us = = = to be removed from future ", "mailings please send your remove request to remove", " me now = thank you and have a super day\r\n", " \r\n" ), concat!( "Subject: conferencing made easy\r\n\r\n only = cen", "ts per minute including long distance no setup fee", "s no contracts or monthly fees call anytime from a", "nywhere to anywhere connects up to = particip", "ants simplicity in set up and administration opera", "tor help available = = the highest quali", "ty service for the lowest rate in the industry fil", "l out the form below to find out how you can lower", " your phone bill every month required input field ", "name web address company name state business phone", " home phone email address type of business to be r", "emoved from our distribution lists please hyperlin", "k click here\r\n\r\n" ), concat!( "Subject: dear friend\r\n\r\n i am mrs sese seko widow o", "f late president mobutu sese seko of zaire now kno", "wn as democratic republic of congo drc i am moved ", "to write you this letter this was in confidence co", "nsidering my presentcircumstance and situation i e", "scaped along with my husband and two of our sons g", "eorge kongolo and basher out of democratic republi", "c of congo drc to abidjan cote d ivoire where my f", "amily and i settled while we later moved to settle", "d in morroco where my husband later died of cancer", " disease however due to this situation we decided ", "to changed most of my husband s billions of dollar", "s deposited in swiss bank and other countries into", " other forms of money coded for safe purpose becau", "se the new head of state of dr mr laurent kabila h", "as made arrangement with the swiss government and ", "other european countries to freeze all my late hus", "band s treasures deposited in some european countr", "ies hence my children and i decided laying low in ", "africa to study the situation till when things get", "s better like now that president kabila is dead an", "d the son taking over joseph kabila one of my late", " husband s chateaux in southern france was confisc", "ated by the french government and as such i had to", " change my identity so that my investment will not", " be traced and confiscated i have deposited the su", "m eighteen million united state dollars us = ", "= = = with a security company for s", "afekeeping the funds are security coded to prevent", " them from knowing the content what i want you to ", "do is to indicate your interest that you will assi", "st us by receiving the money on our behalf acknowl", "edge this message so that i can introduce you to m", "y son kongolo who has the out modalities for the c", "laim of the said funds i want you to assist in inv", "esting this money but i will not want my identity ", "revealed i will also want to buy properties and st", "ock in multi national companies and to engage in o", "ther safe and non speculative investments may i at", " this point emphasise the high level of confidenti", "ality which this business demands and hope you wil", "l not betray the trust and confidence which i repo", "se in you in conclusion if you want to assist us m", "y son shall put you in the picture of the business", " tell you where the funds are currently being main", "tained and also discuss other modalities including", " remunerationfor your services for this reason kin", "dly furnish us your contact information that is yo", "ur personal telephone and fax number for confident", "ial regards mrs m sese seko\r\n\r\n" ), concat!( "Subject: lowest rates available for term life insu", "rance\r\n\r\n take a moment and fill out our online for", "m to see the low rate you qualify for save up to N", "UMBER from regular rates smokers accepted repr", "esenting quality nationwide carriers act now to ea", "sily remove your address from the list go to p", "lease allow = = hours for removal\r\n\r\n" ), concat!( "Subject: central bank of nigeria foreign remittanc", "e \r\n\r\n dept tinubu square lagos nigeria email smith", "_j =th of august = attn president ce", "o strictly private business proposal i am mr johns", "on s abu the bills and exchange director at the fo", "reignremittance department of the central bank of ", "nigeria i am writingyou this letter to ask for you", "r support and cooperation to carrying thisbusiness", " opportunity in my department we discovered abando", "ned the sumof us = = = = thirt", "y seven million four hundred thousand unitedstates", " dollars in an account that belong to one of our f", "oreign customers an american late engr john creek ", "junior an oil merchant with the federal government", " of nigeria who died along with his entire family ", "of a wifeand two children in kenya airbus a= ", "= flight kq= in november= since we ", "heard of his death we have been expecting his next", " of kin tocome over and put claims for his money a", "s the heir because we cannotrelease the fund from ", "his account unless someone applies for claims asth", "e next of kin to the deceased as indicated in our ", "banking guidelines unfortunately neither their fam", "ily member nor distant relative hasappeared to cla", "im the said fund upon this discovery i and other o", "fficialsin my department have agreed to make busin", "ess with you release the totalamount into your acc", "ount as the heir of the fund since no one came for", "it or discovered either maintained account with ou", "r bank other wisethe fund will be returned to the ", "bank treasury as unclaimed fund we have agreed tha", "t our ratio of sharing will be as stated thus NUMB", "ER for you as foreign partner and = for us th", "e officials in my department upon the successful c", "ompletion of this transfer my colleague and i will", "come to your country and mind our share it is from", " our = we intendto import computer accessorie", "s into my country as way of recycling thefund to c", "ommence this transaction we require you to immedia", "tely indicateyour interest by calling me or sendin", "g me a fax immediately on the abovetelefax and enc", "lose your private contact telephone fax full namea", "nd address and your designated banking co ordinate", "s to enable us fileletter of claim to the appropri", "ate department for necessary approvalsbefore the t", "ransfer can be made note also this transaction mus", "t be kept strictly confidential becauseof its natu", "re nb please remember to give me your phone and fa", "x no mr johnson smith abu irish linux users group ", "ilug for un subscription information list ", "maintainer listmaster \r\n\r\n" ), concat!( "Subject: dear stuart\r\n\r\n are you tired of searching", " for love in all the wrong places find love now at", " browse through thousands of personals in ", "your area join for free search e mail chat use", " to meet cool guys and hot girls go = on ", "= or use our private chat rooms click on the ", "link to get started find love now you have rec", "eived this email because you have registerd with e", "mailrewardz or subscribed through one of our marke", "ting partners if you have received this message in", " error or wish to stop receiving these great offer", "s please click the remove link above to unsubscrib", "e from these mailings please click here \r\n\r\n" ), ]; pub const HAM: [&str; 10] = [ concat!( "Message-ID: \r\nSubject: i have been", " trying to research via sa mirrors and search engi", "nes\r\n\r\nif a canned script exists giving clients acce", "ss to their user_prefs options via a web based cgi", " interface numerous isps provide this feature to c", "lients but so far i can find nothing our configura", "tion uses amavis postfix and clamav for virus filt", "ering and procmail with spamassassin for spam filt", "ering i would prefer not to have to write a script", " myself but will appreciate any suggestions this U", "RL email is sponsored by osdn tired of that same o", "ld cell phone get a new here for free ________", "_______________________________________ spamassass", "in talk mailing list spamassassin talk \r\n\r\n" ), concat!( "Message-ID: mid2@foobar.org\r\nSubject: hello\r\n\r\nhave y", "ou seen and discussed this article and his approac", "h thank you hell there are no rules here we re", " trying to accomplish something thomas alva edison", " this email is sponsored by osdn tired of that", " same old cell phone get a new here for free _", "______________________________________________ spa", "massassin devel mailing list spamassassin devel UR", "L \r\n\r\n" ), concat!( "Message-ID: \r\nSubject: hi all apol", "ogies for the possible silly question\r\n\r\ni don t thi", "nk it is but but is eircom s adsl service nat ed a", "nd what implications would that have for voip i kn", "ow there are difficulties with voip or connecting ", "to clients connected to a nat ed network from the ", "internet wild i e machines with static real ips an", "y help pointers would be helpful cheers rgrds bern", "ard bernard tyers national centre for sensor resea", "rch p = = = = e bernard tyers ", " w l n= ______________________________", "_________________ iiu mailing list iiu \r\n\r\n" ), concat!( "Message-ID: \r\nSubject: can someone", " explain\r\n\r\nwhat type of operating system solaris is", " as ive never seen or used it i dont know wheather", " to get a server from sun or from dell i would pre", "fer a linux based server and sun seems to be the o", "ne for that but im not sure if solaris is a distro", " of linux or a completely different operating syst", "em can someone explain kiall mac innes irish linux", " users group ilug for un subscription info", "rmation list maintainer listmaster \r\n\r\n" ), concat!( "Message-ID: \r\nSubject: folks my fi", "rst time posting\r\n\r\nhave a bit of unix experience bu", "t am new to linux just got a new pc at home dell b", "ox with windows xp added a second hard disk for li", "nux partitioned the disk and have installed suse N", "UMBER = from cd which went fine except it did", "n t pick up my monitor i have a dell branded eNUMB", "ERfpp = lcd flat panel monitor and a nvidia g", "eforce= ti= video card both of which are", " probably too new to feature in suse s default set", " i downloaded a driver from the nvidia website and", " installed it using rpm then i ran sax= as wa", "s recommended in some postings i found on the net ", "but it still doesn t feature my video card in the ", "available list what next another problem i have a ", "dell branded keyboard and if i hit caps lock twice", " the whole machine crashes in linux not windows ev", "en the on off switch is inactive leaving me to rea", "ch for the power cable instead if anyone can help ", "me in any way with these probs i d be really grate", "ful i ve searched the net but have run out of idea", "s or should i be going for a different version of ", "linux such as redhat opinions welcome thanks a lot", " peter irish linux users group ilug for un", " subscription information list maintainer listmast", "er \r\n\r\n" ), concat!( "Message-ID: \r\nSubject: has anyone\r\n", "\r\nseen heard of used some package that would let a ", "random person go to a webpage create a mailing lis", "t then administer that list also of course let ppl", " sign up for the lists and manage their subscripti", "ons similar to the old but i d like to have it", " running on my server not someone elses chris ", "\r\n\r\n" ), concat!( "Message-ID: \r\nSubject: hi thank yo", "u for the useful replies\r\n\r\ni have found some intere", "sting tutorials in the ibm developer connection UR", "L and registration is needed i will post the s", "ame message on the web application security list a", "s suggested by someone for now i thing i will use ", "md= for password checking i will use the appr", "oach described in secure programmin fo linux and u", "nix how to i will separate the authentication modu", "le so i can change its implementation at anytime t", "hank you again mario torre please avoid sending me", " word or powerpoint attachments see \r\n\r\n" ), concat!( "Message-ID: \r\nSubject: hehe sorry\r\n", "\r\nbut if you hit caps lock twice the computer crash", "es theres one ive never heard before have you trye", "d dell support yet i think dell computers prefer r", "edhat dell provide some computers pre loaded with ", "red hat i dont know for sure tho so get someone el", "ses opnion as well as mine original message from i", "lug admin mailto ilug admin on behalf of p", "eter staunton sent = august = = NUM", "BER to ilug subject ilug newbie seeks advice s", "use = = folks my first time posting have", " a bit of unix experience but am new to linux just", " got a new pc at home dell box with windows xp add", "ed a second hard disk for linux partitioned the di", "sk and have installed suse = = from cd w", "hich went fine except it didn t pick up my monitor", " i have a dell branded e=fpp = lcd flat ", "panel monitor and a nvidia geforce= ti= ", "video card both of which are probably too new to f", "eature in suse s default set i downloaded a driver", " from the nvidia website and installed it using rp", "m then i ran sax= as was recommended in some ", "postings i found on the net but it still doesn t f", "eature my video card in the available list what ne", "xt another problem i have a dell branded keyboard ", "and if i hit caps lock twice the whole machine cra", "shes in linux not windows even the on off switch i", "s inactive leaving me to reach for the power cable", " instead if anyone can help me in any way with the", "se probs i d be really grateful i ve searched the ", "net but have run out of ideas or should i be going", " for a different version of linux such as redhat o", "pinions welcome thanks a lot peter irish linux use", "rs group ilug for un subscription informat", "ion list maintainer listmaster irish linux use", "rs group ilug for un subscription informat", "ion list maintainer listmaster \r\n\r\n" ), concat!( "Message-ID: \r\nSubject: it will fun", "ction as a router\r\n\r\nif that is what you wish it eve", "n looks like the modem s embedded os is some kind ", "of linux being that it has interesting interfaces ", "like eth= i don t use it as a router though i", " just have it do the absolute minimum dsl stuff an", "d do all the really fun stuff like pppoe on my lin", "ux box also the manual tells you what the default ", "password is don t forget to run pppoe over the alc", "atel speedtouch =i as in my case you have to ", "have a bridge configured in the router modem s sof", "tware this lists your vci values etc also does any", "one know if the high end speedtouch with = et", "hernet ports can act as a full router or do i stil", "l need to run a pppoe stack on the linux box regar", "ds vin irish linux users group ilug for un", " subscription information list maintainer listmast", "er irish linux users group ilug for un", " subscription information list maintainer listmast", "er \r\n\r\n" ), concat!( "Message-ID: \r\nSubject: all is it ", "just me\r\n\r\nor has there been a massive increase in t", "he amount of email being falsely bounced around th", "e place i ve already received email from a number ", "of people i don t know asking why i am sending the", "m email these can be explained by servers from rus", "sia and elsewhere coupled with the false emails i ", "received myself it s really starting to annoy me a", "m i the only one seeing an increase in recent week", "s martin martin whelan déise design tel NUMBE", "R = our core product déiseditor allows organ", "isations to publish information to their web site ", "in a fast and cost effective manner there is no ne", "ed for a full time web developer as the site can b", "e easily updated by the organisations own staff in", "stant updates to keep site information fresh sites", " which are updated regularly bring users back visi", "t for a demonstration déiseditor managing you", "r information ____________________________________", "___________ iiu mailing list iiu ,0\r\n" ), ]; const TEST: [&str; 3] = [ concat!( "Subject: save up to = on life insurance\r\n\r\nwhy ", "spend more than you have to life quote savings ens", "uring your family s financial security is very imp", "ortant life quote savings makes buying life insura", "nce simple and affordable we provide free access t", "o the very best companies and the lowest rates lif", "e quote savings is fast easy and saves you money l", "et us help you get started with the best values in", " the country on new coverage you can save hundreds", " or even thousands of dollars by requesting a free", " quote from lifequote savings our service will tak", "e you less than = minutes to complete shop an", "d compare save up to = on all types of life i", "nsurance hyperlink click here for your free quote ", "protecting your family is the best investment you ", "ll ever make if you are in receipt of this email i", "n error and or wish to be removed from our list hy", "perlink please click here and type remove if you r", "eside in any state which prohibits e mail solicita", "tions for insurance please disregard this email\r\n" ), concat!( "Subject: can someone explain\r\n\r\nwhat type of operati", "ng system solaris is as ive never seen or used it ", "i dont know wheather to get a server from sun or f", "rom dell i would prefer a linux based server and s", "un seems to be the one for that but im not sure if", " solaris is a distro of linux or a completely diff", "erent operating system can someone explain kiall m", "ac innes irish linux users group ilug for ", "un subscription information list maintainer listma", "ster \r\n" ), concat!( "Subject: classifier test\r\n\r\nthis is a novel text tha", "t the sgd classifier has never seen before, it s", "hould be classified as ham or non-ham\r\n" ), ]; ================================================ FILE: tests/src/imap/append.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{fs, io}; use imap_proto::ResponseType; use crate::jmap::wait_for_index; use super::{AssertResult, IMAPTest, ImapConnection, Type, resources_dir}; pub async fn test(imap: &mut ImapConnection, _imap_check: &mut ImapConnection, handle: &IMAPTest) { println!("Running APPEND tests..."); // Invalid APPEND commands imap.send("APPEND \"Does not exist\" {1+}\r\na").await; imap.assert_read(Type::Tagged, ResponseType::No) .await .assert_response_code("TRYCREATE"); // Import test messages let mut entries = fs::read_dir(resources_dir()) .unwrap() .map(|res| res.map(|e| e.path())) .collect::, io::Error>>() .unwrap(); entries.sort(); let mut expected_uid = 1; for file_name in entries.into_iter().take(20) { if file_name.extension().is_none_or(|e| e != "txt") { continue; } let raw_message = fs::read(&file_name).unwrap(); imap.send(&format!( "APPEND INBOX (Flag_{}) {{{}}}", file_name .file_name() .unwrap() .to_str() .unwrap() .split_once('.') .unwrap() .0, raw_message.len() )) .await; imap.assert_read(Type::Continuation, ResponseType::Ok).await; imap.send_untagged(std::str::from_utf8(&raw_message).unwrap()) .await; let result = imap .assert_read(Type::Tagged, ResponseType::Ok) .await .into_response_code(); let mut code = result.split(' '); assert_eq!(code.next(), Some("APPENDUID")); assert_ne!(code.next(), Some("0")); assert_eq!(code.next(), Some(expected_uid.to_string().as_str())); expected_uid += 1; } wait_for_index(&handle.server).await; } pub async fn assert_append_message( imap: &mut ImapConnection, folder: &str, message: &str, expected_response: ResponseType, ) -> Vec { imap.send(&format!("APPEND \"{}\" {{{}}}", folder, message.len())) .await; imap.assert_read(Type::Continuation, ResponseType::Ok).await; imap.send_untagged(message).await; imap.assert_read(Type::Tagged, expected_response).await } fn build_message(message: usize, in_reply_to: Option, thread_num: usize) -> String { if let Some(in_reply_to) = in_reply_to { format!( "Message-ID: <{}@domain>\nReferences: <{}@domain>\nSubject: re: T{}\n\nreply\n", message, in_reply_to, thread_num ) } else { format!( "Message-ID: <{}@domain>\nSubject: T{}\n\nmsg\n", message, thread_num ) } } pub fn build_messages() -> Vec { let mut messages = Vec::new(); for parent in 0..3 { messages.push(build_message(parent, None, parent)); for child in 0..3 { messages.push(build_message( ((parent + 1) * 10) + child, parent.into(), parent, )); } } messages } ================================================ FILE: tests/src/imap/basic.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::auth::sasl::sasl_decode_challenge_oauth; use imap_proto::ResponseType; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; use super::{AssertResult, ImapConnection, Type}; pub async fn test(imap: &mut ImapConnection, _imap_check: &mut ImapConnection) { println!("Running basic tests..."); // Test OAuth Bearer decoding assert!( Credentials::OAuthBearer { token: "vF9dft4qmTc2Nvb3RlckBhbHRhdmlzdGEuY29tCg==".to_string() } == sasl_decode_challenge_oauth( &base64_decode( concat!( "bixhPXVzZXJAZXhhbXBsZS5jb20sAWhv", "c3Q9c2VydmVyLmV4YW1wbGUuY29tAXBvcnQ9MTQzAWF1dGg9QmVhcmVyI", "HZGOWRmdDRxbVRjMk52YjNSbGNrQmhiSFJoZG1semRHRXVZMjl0Q2c9PQ", "EB" ) .as_bytes(), ) .unwrap(), ) .unwrap() ); // Test CAPABILITY imap.send("CAPABILITY").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; // Test NOOP imap.send("NOOP").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; // Test ID imap.send("ID").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("* ID (\"name\" \"Stalwart\" \"version\" "); // Login should be disabled imap.send("LOGIN jdoe@example.com secret").await; imap.assert_read(Type::Tagged, ResponseType::No).await; // Try logging in with wrong password imap.send("AUTHENTICATE PLAIN {24}").await; imap.assert_read(Type::Continuation, ResponseType::Ok).await; imap.send_untagged("AGJvYXR5AG1jYm9hdGZhY2U=").await; imap.assert_read(Type::Tagged, ResponseType::No).await; } ================================================ FILE: tests/src/imap/body_structure.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::resources_dir; use email::message::metadata::{MessageMetadata, build_metadata_contents}; use imap::op::fetch::AsImapDataItem; use imap_proto::{ ResponseCode, StatusResponse, protocol::fetch::{BodyContents, DataItem, Section}, }; use mail_parser::MessageParser; use std::fs; use store::{ Deserialize, Serialize, write::{Archive, Archiver}, }; use utils::chained_bytes::ChainedBytes; pub fn test() { println!("Running BODYSTRUCTURE..."); for file_name in fs::read_dir(resources_dir()).unwrap() { let mut file_name = file_name.as_ref().unwrap().path(); if file_name.extension().is_none_or(|e| e != "txt") { continue; } let mut buf = Vec::new(); let raw_message = fs::read(&file_name).unwrap(); let message_ = MessageParser::new().parse(&raw_message).unwrap(); let metadata = MessageMetadata { preview: Default::default(), raw_headers: message_ .raw_message .as_ref() .get( message_.root_part().offset_header as usize ..message_.root_part().offset_body as usize, ) .unwrap_or_default() .into(), blob_hash: Default::default(), blob_body_offset: message_.root_part().offset_body as u32, contents: build_metadata_contents(message_), rcvd_attach: 0, }; let metadata_ = Archive::deserialize_owned(Archiver::new(metadata).serialize().unwrap()).unwrap(); let metadata = metadata_.unarchive::().unwrap(); let raw_message = ChainedBytes::new(metadata.raw_headers.as_ref()).with_last( raw_message .get(metadata.blob_body_offset.to_native() as usize..) .unwrap_or_default(), ); let decoded = metadata.decode_contents(raw_message); // Serialize body and bodystructure for is_extended in [false, true] { let mut buf_ = Vec::new(); metadata .body_structure(&decoded, is_extended) .serialize(&mut buf_, is_extended); if is_extended { buf.extend_from_slice(b"BODYSTRUCTURE "); } else { buf.extend_from_slice(b"BODY "); } // Poor man's indentation let mut indent_count = 0; let mut in_quote = false; for ch in buf_ { if ch == b'(' && !in_quote { buf.extend_from_slice(b"(\n"); indent_count += 1; for _ in 0..indent_count { buf.extend_from_slice(b" "); } } else if ch == b')' && !in_quote { buf.push(b'\n'); indent_count -= 1; for _ in 0..indent_count { buf.extend_from_slice(b" "); } buf.push(b')'); } else { if ch == b'"' { in_quote = !in_quote; } buf.push(ch); } } buf.extend_from_slice(b"\n\n"); } // Serialize body parts let mut iter = 1..9; let mut stack = Vec::new(); let mut sections = Vec::new(); loop { 'inner: while let Some(part_id) = iter.next() { if part_id == 1 { for section in [ None, Some(Section::Header), Some(Section::Text), Some(Section::Mime), ] { let mut body_sections = sections .iter() .map(|id| Section::Part { num: *id }) .collect::>(); let is_first = if let Some(section) = section { body_sections.push(section); false } else { true }; if let Some(contents) = metadata.body_section(&decoded, &body_sections, None) { DataItem::BodySection { sections: body_sections, origin_octet: None, contents, } .serialize(&mut buf); if is_first { match metadata.binary(&decoded, §ions, None) { Ok(Some(contents)) => { buf.push(b'\n'); DataItem::Binary { sections: sections.clone(), offset: None, contents: match contents { BodyContents::Bytes(_) => { BodyContents::Text("[binary content]".into()) } text => text, }, } .serialize(&mut buf); } Ok(None) => (), Err(_) => { buf.push(b'\n'); buf.extend_from_slice( &StatusResponse::no(format!( "Failed to decode part {} of message {}.", sections .iter() .map(|s| s.to_string()) .collect::>() .join("."), 0 )) .with_code(ResponseCode::UnknownCte) .serialize(Vec::new()), ); } } if let Some(size) = metadata.binary_size(&decoded, §ions) { buf.push(b'\n'); DataItem::BinarySize { sections: sections.clone(), size, } .serialize(&mut buf); } } buf.extend_from_slice(b"\n----------------------------------\n"); } else { break 'inner; } } } sections.push(part_id); stack.push(iter); iter = 1..9; } if let Some(prev_iter) = stack.pop() { sections.pop(); iter = prev_iter; } else { break; } } // Check header fields and partial sections for sections in [ vec![Section::HeaderFields { not: false, fields: vec!["From".into(), "To".into()], }], vec![Section::HeaderFields { not: true, fields: vec!["Subject".into(), "Cc".into()], }], ] { DataItem::BodySection { contents: metadata.body_section(&decoded, §ions, None).unwrap(), sections: sections.clone(), origin_octet: None, } .serialize(&mut buf); buf.extend_from_slice(b"\n----------------------------------\n"); DataItem::BodySection { contents: metadata .body_section(&decoded, §ions, (10, 25).into()) .unwrap(), sections, origin_octet: 10.into(), } .serialize(&mut buf); buf.extend_from_slice(b"\n----------------------------------\n"); } file_name.set_extension("imap"); let expected_result = fs::read(&file_name).unwrap(); if buf != expected_result { file_name.set_extension("imap_failed"); fs::write(&file_name, buf).unwrap(); panic!("Failed test, written output to {}", file_name.display()); } } } ================================================ FILE: tests/src/imap/condstore.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use imap_proto::ResponseType; use crate::imap::{ AssertResult, append::{assert_append_message, build_messages}, }; use super::{ImapConnection, Type}; pub async fn test(imap: &mut ImapConnection, imap_check: &mut ImapConnection) { println!("Running CONDSTORE..."); // Test CONDSTORE parameter imap.send("SELECT INBOX (CONDSTORE)").await; let hms = imap .assert_read(Type::Tagged, ResponseType::Ok) .await .into_highest_modseq(); // Unselect imap.send("UNSELECT").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; // Create test folders imap.send("CREATE Pecorino").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; // Enable CONDSTORE and QRESYNC imap.send("ENABLE CONDSTORE QRESYNC").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; // Make sure modseq did not change after creating a mailbox imap.send("SELECT Pecorino").await; assert_eq!( imap.assert_read(Type::Tagged, ResponseType::Ok) .await .into_highest_modseq(), hms ); imap_check.send("LIST \"\" \"*\"").await; imap_check.assert_read(Type::Tagged, ResponseType::Ok).await; imap_check.send("SELECT Pecorino (CONDSTORE)").await; imap_check.assert_read(Type::Tagged, ResponseType::Ok).await; // SEQ 0: Init let mut messages = build_messages(); let mut modseqs = vec![hms]; // SEQ 1: Append a message and make sure the modseq increased assert_append_message(imap, "Pecorino", &messages.pop().unwrap(), ResponseType::Ok).await; imap.send("STATUS Pecorino (HIGHESTMODSEQ)").await; modseqs.push( imap.assert_read(Type::Tagged, ResponseType::Ok) .await .into_highest_modseq(), ); assert_ne!(modseqs[modseqs.len() - 1], modseqs[modseqs.len() - 2]); // SEQ 2: Move out the message and make sure the modseq increased imap.send("UID MOVE 1 \"Deleted Items\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("* VANISHED 1"); imap.send("STATUS Pecorino (HIGHESTMODSEQ)").await; modseqs.push( imap.assert_read(Type::Tagged, ResponseType::Ok) .await .into_highest_modseq(), ); assert_ne!(modseqs[modseqs.len() - 1], modseqs[modseqs.len() - 2]); // SEQ 3: Insert message assert_append_message(imap, "Pecorino", &messages.pop().unwrap(), ResponseType::Ok).await; imap.send("STATUS Pecorino (HIGHESTMODSEQ)").await; modseqs.push( imap.assert_read(Type::Tagged, ResponseType::Ok) .await .into_highest_modseq(), ); // SEQ 4: Insert message assert_append_message(imap, "Pecorino", &messages.pop().unwrap(), ResponseType::Ok).await; imap.send("STATUS Pecorino (HIGHESTMODSEQ)").await; modseqs.push( imap.assert_read(Type::Tagged, ResponseType::Ok) .await .into_highest_modseq(), ); // SEQ 5: Insert message assert_append_message(imap, "Pecorino", &messages.pop().unwrap(), ResponseType::Ok).await; imap.send("STATUS Pecorino (HIGHESTMODSEQ)").await; modseqs.push( imap.assert_read(Type::Tagged, ResponseType::Ok) .await .into_highest_modseq(), ); // SEQ 6: Change a message flag imap.send("UID STORE 4 +FLAGS.SILENT (\\Answered)").await; modseqs.push( imap.assert_read(Type::Tagged, ResponseType::Ok) .await .into_modseq(), ); // SEQ 7: Insert message assert_append_message(imap, "Pecorino", &messages.pop().unwrap(), ResponseType::Ok).await; imap.send("STATUS Pecorino (HIGHESTMODSEQ)").await; modseqs.push( imap.assert_read(Type::Tagged, ResponseType::Ok) .await .into_highest_modseq(), ); // SEQ 8: Delete a message imap.send("UID STORE 2 +FLAGS.SILENT (\\Deleted)").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("EXPUNGE").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("VANISHED 2") .assert_contains("* 3 EXISTS"); imap.send("STATUS Pecorino (HIGHESTMODSEQ)").await; modseqs.push( imap.assert_read(Type::Tagged, ResponseType::Ok) .await .into_highest_modseq(), ); // Fetch changes since SEQ 0 imap.send(&format!( "UID FETCH 1:* (FLAGS) (CHANGEDSINCE {} VANISHED)", modseqs[0] )) .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("FETCH (", 3) .assert_count("VANISHED", 0); // Fetch changes since SEQ 1, UID MOVE should count as a deletion imap.send(&format!( "UID FETCH 1:* (FLAGS) (CHANGEDSINCE {} VANISHED)", modseqs[1] )) .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("VANISHED", 1) .assert_contains("VANISHED (EARLIER) 1") .assert_count("FETCH (", 3); // Fetch changes since SEQ 3 imap.send(&format!( "UID FETCH 1:* (FLAGS) (CHANGEDSINCE {} VANISHED)", modseqs[3] )) .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("VANISHED", 1) .assert_contains("VANISHED (EARLIER) 2") .assert_count("FETCH (", 3); // Fetch changes since SEQ 4 imap.send(&format!( "UID FETCH 1:* (FLAGS) (CHANGEDSINCE {} VANISHED)", modseqs[4] )) .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("VANISHED", 1) .assert_contains("VANISHED (EARLIER) 2") .assert_count("FETCH (", 2); // Fetch changes since SEQ 6 imap.send(&format!( "UID FETCH 1:* (FLAGS) (CHANGEDSINCE {} VANISHED)", modseqs[6] )) .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("VANISHED", 1) .assert_contains("VANISHED (EARLIER) 2") .assert_count("FETCH (", 1); // Fetch changes since SEQ 7 imap.send(&format!( "UID FETCH 1:* (FLAGS) (CHANGEDSINCE {} VANISHED)", modseqs[7] )) .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("VANISHED", 1) .assert_contains("VANISHED (EARLIER) 2") .assert_count("FETCH (", 0); // Fetch changes since SEQ 8 imap.send(&format!( "UID FETCH 1:* (FLAGS) (CHANGEDSINCE {} VANISHED)", modseqs[8] )) .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("VANISHED", 0) .assert_count("FETCH (", 0); // Search since MODSEQ imap.send(&format!("SEARCH RETURN (ALL) MODSEQ {}", modseqs[3])) .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("ALL 1:3 MODSEQ"); imap_check.send("NOOP").await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("3 EXISTS"); imap_check .send(&format!("SEARCH MODSEQ {}", modseqs[4])) .await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("SEARCH 2 3 (MODSEQ"); // Store unchanged since imap.send(&format!( "UID STORE 2:5 (UNCHANGEDSINCE {}) +FLAGS.SILENT (\\Junk)", modseqs[5] )) .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("* 1 FETCH") .assert_contains("(UID 3 MODSEQ") .assert_count("FETCH (", 1) .assert_contains("[MODIFIED 2,4:5]"); imap.send(&format!( "UID STORE 4,5 (UNCHANGEDSINCE {}) -FLAGS.SILENT (\\Answered)", modseqs[6] )) .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("* 2 FETCH") .assert_contains("(UID 4 MODSEQ") .assert_count("FETCH (", 1) .assert_contains("[MODIFIED 5]"); // QResync imap.send("STATUS Pecorino (UIDVALIDITY)").await; let uid_validity = imap .assert_read(Type::Tagged, ResponseType::Ok) .await .into_uid_validity(); imap.send(&format!( "SELECT Pecorino (QRESYNC ({} {} 1:5)) ", uid_validity, modseqs[6] )) .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("FETCH (", 3) .assert_contains("VANISHED (EARLIER) 2"); } ================================================ FILE: tests/src/imap/copy_move.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use imap_proto::ResponseType; use super::{AssertResult, ImapConnection, Type}; pub async fn test(_imap: &mut ImapConnection, imap_check: &mut ImapConnection) { println!("Running COPY/MOVE tests..."); // Check status imap_check .send("LIST \"\" % RETURN (STATUS (UIDNEXT MESSAGES UNSEEN SIZE RECENT))") .await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("\"INBOX\" (UIDNEXT 11 MESSAGES 10 UNSEEN 10 RECENT 0 SIZE 12193)"); // Select INBOX imap_check.send("SELECT INBOX").await; imap_check.assert_read(Type::Tagged, ResponseType::Ok).await; // Copying to the same mailbox should fail imap_check.send("COPY 1:* INBOX").await; imap_check .assert_read(Type::Tagged, ResponseType::No) .await .assert_response_code("CANNOT"); // Copying to a non-existent mailbox should fail imap_check.send("COPY 1:* \"/dev/null\"").await; imap_check .assert_read(Type::Tagged, ResponseType::No) .await .assert_response_code("TRYCREATE"); // Create test folders imap_check.send("CREATE \"Scamorza Affumicata\"").await; imap_check.assert_read(Type::Tagged, ResponseType::Ok).await; imap_check.send("CREATE \"Burrata al Tartufo\"").await; imap_check.assert_read(Type::Tagged, ResponseType::Ok).await; // Copy messages imap_check .send("COPY 1,3,5,7 \"Scamorza Affumicata\"") .await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("COPYUID") .assert_contains("1:4"); // Check status imap_check .send("STATUS \"Scamorza Affumicata\" (UIDNEXT MESSAGES UNSEEN SIZE RECENT)") .await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("MESSAGES 4") //.assert_contains("RECENT 4") .assert_contains("UNSEEN 4") .assert_contains("UIDNEXT 5") .assert_contains("SIZE 5851"); // Check \Recent flag /*imap_check.send("SELECT \"Scamorza Affumicata\"").await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("* 4 RECENT"); imap_check.send("FETCH 1:* (UID FLAGS)").await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("\\Recent", 4); imap_check.send("UNSELECT").await; imap_check.assert_read(Type::Tagged, ResponseType::Ok).await; imap_check .send("STATUS \"Scamorza Affumicata\" (UIDNEXT MESSAGES UNSEEN SIZE RECENT)") .await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("MESSAGES 4") .assert_contains("RECENT 0") .assert_contains("UNSEEN 4") .assert_contains("UIDNEXT 5") .assert_contains("SIZE 5851"); imap_check.send("SELECT \"Scamorza Affumicata\"").await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("* 0 RECENT"); imap_check.send("FETCH 1:* (UID FLAGS)").await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("\\Recent", 0);*/ // Move all messages to Burrata imap_check.send("SELECT \"Scamorza Affumicata\"").await; imap_check.assert_read(Type::Tagged, ResponseType::Ok).await; imap_check.send("MOVE 1:* \"Burrata al Tartufo\"").await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("* OK [COPYUID") .assert_contains("1:4") .assert_contains("* 1 EXPUNGE") .assert_contains("* 1 EXPUNGE") .assert_contains("* 1 EXPUNGE") .assert_contains("* 1 EXPUNGE"); // Check status imap_check .send("LIST \"\" % RETURN (STATUS (UIDNEXT MESSAGES UNSEEN SIZE))") .await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("\"Burrata al Tartufo\" (UIDNEXT 5 MESSAGES 4 UNSEEN 4 SIZE 5851)") .assert_contains("\"Scamorza Affumicata\" (UIDNEXT 5 MESSAGES 0 UNSEEN 0 SIZE 0)") .assert_contains("\"INBOX\" (UIDNEXT 11 MESSAGES 10 UNSEEN 10 SIZE 12193)"); // Move the messages back to Scamorza, UIDNEXT should increase. imap_check.send("SELECT \"Burrata al Tartufo\"").await; imap_check.assert_read(Type::Tagged, ResponseType::Ok).await; imap_check.send("MOVE 1:* \"Scamorza Affumicata\"").await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("* OK [COPYUID") .assert_contains("5:8") .assert_contains("* 1 EXPUNGE") .assert_contains("* 1 EXPUNGE") .assert_contains("* 1 EXPUNGE") .assert_contains("* 1 EXPUNGE"); // Check status imap_check .send("LIST \"\" % RETURN (STATUS (UIDNEXT MESSAGES UNSEEN SIZE))") .await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("\"Burrata al Tartufo\" (UIDNEXT 5 MESSAGES 0 UNSEEN 0 SIZE 0)") .assert_contains("\"Scamorza Affumicata\" (UIDNEXT 9 MESSAGES 4 UNSEEN 4 SIZE 5851)") .assert_contains("\"INBOX\" (UIDNEXT 11 MESSAGES 10 UNSEEN 10 SIZE 12193)"); } ================================================ FILE: tests/src/imap/fetch.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use imap_proto::ResponseType; use super::{AssertResult, ImapConnection, Type}; pub async fn test(imap: &mut ImapConnection, _imap_check: &mut ImapConnection) { println!("Running FETCH tests..."); // Examine INBOX imap.send("EXAMINE INBOX").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("10 EXISTS") .assert_contains("[UIDNEXT 11]"); // Fetch all properties available from JMAP imap.send(concat!( "FETCH 10 (FLAGS INTERNALDATE PREVIEW EMAILID THREADID ", "RFC822.SIZE UID ENVELOPE BODYSTRUCTURE)" )) .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("FLAGS (Flag_009)") .assert_contains("RFC822.SIZE 1457") .assert_contains("UID 10") .assert_contains("INTERNALDATE") .assert_contains("THREADID (") .assert_contains("EMAILID (") .assert_contains("but then I thought, why not do both?") .assert_contains(concat!( "ENVELOPE (\"Sat, 20 Nov 2021 14:22:01 -0800\" ", "\"Why not both importing AND exporting? ☺\" ", "((\"Art Vandelay (Vandelay Industries)\" NIL \"art\" \"vandelay.com\")) ", "((\"Art Vandelay (Vandelay Industries)\" NIL \"art\" \"vandelay.com\")) ", "((\"Art Vandelay (Vandelay Industries)\" NIL \"art\" \"vandelay.com\")) ", "((NIL NIL \"Colleagues\" NIL)", "(\"James Smythe\" NIL \"james\" \"vandelay.com\")", "(NIL NIL NIL NIL)(NIL NIL \"Friends\" NIL)", "(NIL NIL \"jane\" \"example.com\")", "(\"John Smîth\" NIL \"john\" \"example.com\")", "(NIL NIL NIL NIL)) NIL NIL NIL NIL)" )) .assert_contains(concat!( "BODYSTRUCTURE ((\"text\" \"html\" (\"charset\" \"us-ascii\") NIL NIL ", "\"base64\" 239 3 \"07aab44e51c5f1833a5d19f2e1804c4b\" NIL NIL NIL)", "(\"message\" \"rfc822\" NIL NIL NIL NIL 723 ", "(NIL \"Exporting my book about coffee tables\" ", "((\"Cosmo Kramer\" NIL \"kramer\" \"kramerica.com\")) ", "((\"Cosmo Kramer\" NIL \"kramer\" \"kramerica.com\")) ", "((\"Cosmo Kramer\" NIL \"kramer\" \"kramerica.com\")) ", "NIL NIL NIL NIL NIL) ", "((\"text\" \"plain\" (\"charset\" \"utf-16\") NIL NIL ", "\"quoted-printable\" 228 3 \"3a942a99cdd8a099ae107d3867ec20fb\" NIL NIL NIL)", "(\"image\" \"gif\" (\"name\" \"Book about ☕ tables.gif\") ", "NIL NIL \"Base64\" 56 \"d40fa7f401e9dc2df56cbb740d65ff52\" ", "(\"attachment\" NIL) NIL NIL) \"mixed\" (\"boundary\" \"giddyup\") NIL NIL NIL)", " 0 \"cdb0382a03a15601fb1b3c7422521620\" NIL NIL NIL) ", "\"mixed\" (\"boundary\" \"festivus\") NIL NIL NIL)" )); // Fetch bodyparts imap.send(concat!( "UID FETCH 10 (BINARY[1] BINARY.SIZE[1] BODY[1.TEXT] BODY[2.1.HEADER] ", "BINARY[2.1] BODY[MIME] BODY[HEADER.FIELDS (From)]<10.8>)" )) .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("BINARY[1] {175}") .assert_contains("BINARY.SIZE[1] 175") .assert_contains("BODY[1.TEXT] {239}") .assert_contains("BODY[2.1.HEADER] {88}") .assert_contains("BINARY[2.1] {101}") .assert_contains("BODY[MIME] {54}") .assert_contains("BODY[HEADER.FIELDS (FROM)]<10> {8}") .assert_contains("“exporting”") .assert_contains("PGh0bWw+PHA+") .assert_contains("Content-Transfer-Encoding: quoted-printable") .assert_contains("ℌ𝔢𝔩𝔭 𝔪𝔢 𝔢𝔵𝔭𝔬𝔯𝔱 𝔪𝔶 𝔟𝔬𝔬𝔨") .assert_contains("Vandelay"); // We are in EXAMINE mode, fetching body should not set \Seen imap.send("UID FETCH 10 (FLAGS)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("FLAGS (Flag_009)"); // Switch to SELECT mode imap.send("SELECT INBOX").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; // Peek bodyparts imap.send("UID FETCH 10 (BINARY.PEEK[1] BINARY.SIZE[1] BODY.PEEK[1.TEXT])") .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("BINARY[1] {175}") .assert_contains("BINARY.SIZE[1] 175") .assert_contains("BODY[1.TEXT] {239}"); // PEEK was used, \Seen should not be set imap.send("UID FETCH 10 (FLAGS)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("FLAGS (Flag_009)"); // Fetching a body section should set the \Seen flag imap.send("UID FETCH 10 (BODY[1.TEXT])").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("FLAGS") .assert_contains("\\Seen"); // Fetch a sequence imap.send("FETCH 1:5,7:10 (UID FLAGS)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("* 1 FETCH (UID 1 ") .assert_contains("* 2 FETCH (UID 2 ") .assert_contains("* 3 FETCH (UID 3 ") .assert_contains("* 4 FETCH (UID 4 ") .assert_contains("* 5 FETCH (UID 5 ") .assert_contains("* 7 FETCH (UID 7 ") .assert_contains("* 8 FETCH (UID 8 ") .assert_contains("* 9 FETCH (UID 9 ") .assert_contains("* 10 FETCH (UID 10 ") .assert_count("\\Recent", 0); imap.send("FETCH 7:* (UID FLAGS)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("* 7 FETCH (UID 7 ") .assert_contains("* 8 FETCH (UID 8 ") .assert_contains("* 9 FETCH (UID 9 ") .assert_contains("* 10 FETCH (UID 10 "); // Fetch using a saved search imap.send("UID SEARCH RETURN (SAVE) FROM \"nathaniel\"") .await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("FETCH $ (UID PREVIEW)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("* 1 FETCH (UID 1 ") .assert_contains("* 4 FETCH (UID 4 ") .assert_contains("* 6 FETCH (UID 6 ") .assert_contains("Some text appears here") .assert_contains("plain text version of message goes here") .assert_contains("This is implicitly typed plain US-ASCII text."); } ================================================ FILE: tests/src/imap/idle.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::mail::delivery::SmtpConnection; use super::{AssertResult, ImapConnection, Type}; use imap_proto::ResponseType; use std::time::Duration; const SLEEP: Duration = Duration::from_millis(200); pub async fn test( imap: &mut ImapConnection, imap_check: &mut ImapConnection, is_cluster_test: bool, ) { println!("Running IDLE tests..."); // Switch connection to IDLE mode imap_check.send("CREATE Parmeggiano").await; imap_check.assert_read(Type::Tagged, ResponseType::Ok).await; imap_check.send("SELECT Parmeggiano").await; imap_check.assert_read(Type::Tagged, ResponseType::Ok).await; imap_check.send("NOOP").await; imap_check.assert_read(Type::Tagged, ResponseType::Ok).await; imap_check.send("IDLE").await; imap_check .assert_read(Type::Continuation, ResponseType::Ok) .await; // Expect a new mailbox update imap.send("CREATE Provolone").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; if is_cluster_test { tokio::time::sleep(SLEEP).await; } imap_check .assert_read(Type::Status, ResponseType::Ok) .await .assert_contains("LIST () \"/\" \"Provolone\""); // Insert a message in the new folder and expect an update let message = "From: test@domain.com\nSubject: Test\n\nTest message\n"; imap.send(&format!("APPEND Provolone {{{}}}", message.len())) .await; imap.assert_read(Type::Continuation, ResponseType::Ok).await; imap.send_untagged(message).await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; if is_cluster_test { tokio::time::sleep(SLEEP).await; } imap_check .assert_read(Type::Status, ResponseType::Ok) .await .assert_contains("STATUS \"Provolone\"") .assert_contains("MESSAGES 1") .assert_contains("UNSEEN 1") .assert_contains("UIDNEXT 2"); // Change message to Seen and expect an update imap.send("SELECT Provolone").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("STORE 1:* +FLAGS (\\Seen)").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; if is_cluster_test { tokio::time::sleep(SLEEP).await; } imap_check .assert_read(Type::Status, ResponseType::Ok) .await .assert_contains("STATUS \"Provolone\"") .assert_contains("MESSAGES 1") .assert_contains("UNSEEN 0") .assert_contains("UIDNEXT 2"); // Delete message and expect an update imap.send("STORE 1:* +FLAGS (\\Deleted)").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("CLOSE").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; if is_cluster_test { tokio::time::sleep(SLEEP).await; } imap_check .assert_read(Type::Status, ResponseType::Ok) .await .assert_contains("STATUS \"Provolone\"") .assert_contains("MESSAGES 0") .assert_contains("UNSEEN 0") .assert_contains("UIDNEXT 2"); // Delete folder and expect an update imap.send("DELETE Provolone").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; if is_cluster_test { tokio::time::sleep(SLEEP).await; } imap_check .assert_read(Type::Status, ResponseType::Ok) .await .assert_contains("LIST (\\NonExistent) \"/\" \"Provolone\""); // Add a message to Inbox and expect an update imap.send(&format!("APPEND Parmeggiano {{{}}}", message.len())) .await; imap.assert_read(Type::Continuation, ResponseType::Ok).await; imap.send_untagged(message).await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; if is_cluster_test { tokio::time::sleep(SLEEP).await; } imap_check .assert_read(Type::Status, ResponseType::Ok) .await .assert_contains("MESSAGES 1") .assert_contains("UNSEEN 1"); imap_check .assert_read(Type::Status, ResponseType::Ok) .await .assert_contains("* 1 EXISTS"); imap_check .assert_read(Type::Status, ResponseType::Ok) .await .assert_contains("* 1 FETCH (FLAGS () UID 1)"); // Delete message and expect an update imap.send("SELECT Parmeggiano").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("STORE 1 +FLAGS (\\Deleted)").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; if is_cluster_test { tokio::time::sleep(SLEEP).await; } imap_check .assert_read(Type::Status, ResponseType::Ok) .await .assert_contains("* 1 FETCH (FLAGS (\\Deleted) UID 1)"); imap.send("UID EXPUNGE").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("* 1 EXPUNGE") .assert_contains("* 0 EXISTS"); if is_cluster_test { tokio::time::sleep(SLEEP).await; } imap_check .assert_read(Type::Status, ResponseType::Ok) .await .assert_contains("MESSAGES 0") .assert_contains("UNSEEN 0"); imap_check .assert_read(Type::Status, ResponseType::Ok) .await .assert_contains("* 1 EXPUNGE"); imap_check .assert_read(Type::Status, ResponseType::Ok) .await .assert_contains("* 0 EXISTS"); // Test SMTP delivery notifications let mut lmtp = SmtpConnection::connect_port(if is_cluster_test { 17000 } else { 11201 }).await; lmtp.ingest( "bill@example.com", &["jdoe@example.com"], concat!( "From: bill@example.com\r\n", "To: jdoe@example.com\r\n", "Subject: TPS Report\r\n", "X-Spam-Status: No\r\n", "\r\n", "I'm going to need those TPS reports ASAP. ", "So, if you could do that, that'd be great." ), ) .await; if is_cluster_test { tokio::time::sleep(SLEEP).await; } imap_check .assert_read(Type::Status, ResponseType::Ok) .await .assert_contains("STATUS \"INBOX\"") .assert_contains(if is_cluster_test { "MESSAGES 1" } else { "MESSAGES 11" }); // Stop IDLE mode imap_check.send_raw("DONE").await; imap_check.assert_read(Type::Tagged, ResponseType::Ok).await; imap_check.send("NOOP").await; imap_check.assert_read(Type::Tagged, ResponseType::Ok).await; } ================================================ FILE: tests/src/imap/mailbox.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use imap::op::list::matches_pattern; use imap_proto::ResponseType; use super::{AssertResult, ImapConnection, Type}; pub async fn test(mut imap: &mut ImapConnection, mut imap_check: &mut ImapConnection) { println!("Running mailbox tests..."); // Pattern matching tests mailbox_matches_pattern(); // Create third connection for testing let mut other_conn = ImapConnection::connect(b"_z ").await; other_conn .send("AUTHENTICATE PLAIN {32+}\r\nAGpkb2VAZXhhbXBsZS5jb20Ac2VjcmV0") .await; other_conn.assert_read(Type::Tagged, ResponseType::Ok).await; // List folders imap.send("LIST \"\" \"*\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_folders([("INBOX", [""]), ("Deleted Items", [""])], true); // Create folders imap.send("CREATE \"Tofu\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("CREATE \"Fruit\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("CREATE \"Fruit/Apple\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("CREATE \"Fruit/Apple/Green\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("CREATE \"L&APg-bende opgaver\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; // Select folder from another connection other_conn.send("SELECT \"Tofu\"").await; other_conn.assert_read(Type::Tagged, ResponseType::Ok).await; other_conn.send("SELECT \"L&APg-bende opgaver\"").await; other_conn.assert_read(Type::Tagged, ResponseType::Ok).await; // Make sure folders are visible for imap in [&mut imap, &mut imap_check] { imap.send("LIST \"\" \"*\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_folders( [ ("INBOX", [""]), ("Deleted Items", [""]), ("Fruit", [""]), ("Fruit/Apple", [""]), ("Fruit/Apple/Green", [""]), ("Tofu", [""]), ("L&APg-bende opgaver", [""]), ], true, ); } imap.send("DELETE \"L&APg-bende opgaver\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; // Special use folders that already exist should not be allowed imap.send("CREATE \"Second trash\" (USE (\\Trash))").await; imap.assert_read(Type::Tagged, ResponseType::No).await; // Enable IMAP4rev2 imap.send("ENABLE IMAP4rev2").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; // Create and delete using IMAP4rev2 imap.send("CREATE \"L&APg-bende opgaver\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("SELECT \"L&APg-bende opgaver\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("UNSELECT \"L&APg-bende opgaver\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("DELETE \"L&APg-bende opgaver\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; // Create missing parent folders imap.send("CREATE \"/Vegetable/Broccoli\" (USE (\\Important))") .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("[MAILBOXID ("); imap.send("CREATE \" Cars/Electric /4 doors/ Red/\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; for imap in [&mut imap, &mut imap_check] { imap.send("LIST \"\" \"*\" RETURN (CHILDREN SPECIAL-USE)") .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_folders( [ ("INBOX", ["HasNoChildren", ""]), ("Deleted Items", ["HasNoChildren", "Trash"]), ("Cars/Electric/4 doors/Red", ["HasNoChildren", ""]), ("Cars/Electric/4 doors", ["HasChildren", ""]), ("Cars/Electric", ["HasChildren", ""]), ("Cars", ["HasChildren", ""]), ("Fruit", ["HasChildren", ""]), ("Fruit/Apple", ["HasChildren", ""]), ("Fruit/Apple/Green", ["HasNoChildren", ""]), ("Vegetable", ["HasChildren", ""]), ("Vegetable/Broccoli", ["HasNoChildren", "\\Important"]), ("Tofu", ["HasNoChildren", ""]), ], true, ); } // Rename folders imap.send("RENAME \"Fruit/Apple/Green\" \"Fruit/Apple/Red\"") .await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("RENAME \"Cars\" \"Vehicles\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("RENAME \"Vegetable/Broccoli\" \"Veggies/Green/Broccoli\"") .await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("RENAME \"Tofu\" \"INBOX\"").await; imap.assert_read(Type::Tagged, ResponseType::No).await; imap.send("RENAME \"Tofu\" \"Inbox/Tofu\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("RENAME \"Deleted Items\" \"Recycle Bin\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; for imap in [&mut imap, &mut imap_check] { imap.send("LIST \"\" \"*\" RETURN (CHILDREN SPECIAL-USE)") .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_folders( [ ("INBOX", ["HasChildren", ""]), ("INBOX/Tofu", ["HasNoChildren", ""]), ("Recycle Bin", ["HasNoChildren", "Trash"]), ("Vehicles/Electric/4 doors/Red", ["HasNoChildren", ""]), ("Vehicles/Electric/4 doors", ["HasChildren", ""]), ("Vehicles/Electric", ["HasChildren", ""]), ("Vehicles", ["HasChildren", ""]), ("Fruit", ["HasChildren", ""]), ("Fruit/Apple", ["HasChildren", ""]), ("Fruit/Apple/Red", ["HasNoChildren", ""]), ("Vegetable", ["HasNoChildren", ""]), ("Veggies", ["HasChildren", ""]), ("Veggies/Green", ["HasChildren", ""]), ("Veggies/Green/Broccoli", ["HasNoChildren", ""]), ], true, ); } // Delete folders imap.send("DELETE \"INBOX/Tofu\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("DELETE \"Vegetable\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("DELETE \"Vehicles\"").await; imap.assert_read(Type::Tagged, ResponseType::No).await; for imap in [&mut imap, &mut imap_check] { imap.send("LIST \"\" \"*\" RETURN (CHILDREN SPECIAL-USE)") .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_folders( [ ("INBOX", ["HasNoChildren", ""]), ("Recycle Bin", ["HasNoChildren", "Trash"]), ("Vehicles/Electric/4 doors/Red", ["HasNoChildren", ""]), ("Vehicles/Electric/4 doors", ["HasChildren", ""]), ("Vehicles/Electric", ["HasChildren", ""]), ("Vehicles", ["HasChildren", ""]), ("Fruit", ["HasChildren", ""]), ("Fruit/Apple", ["HasChildren", ""]), ("Fruit/Apple/Red", ["HasNoChildren", ""]), ("Veggies", ["HasChildren", ""]), ("Veggies/Green", ["HasChildren", ""]), ("Veggies/Green/Broccoli", ["HasNoChildren", ""]), ], true, ); } // Subscribe imap.send("SUBSCRIBE \"INBOX\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("SUBSCRIBE \"Vehicles/Electric/4 doors/Red\"") .await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; for imap in [&mut imap, &mut imap_check] { imap.send("LIST \"\" \"*\" RETURN (SUBSCRIBED SPECIAL-USE)") .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_folders( [ ("INBOX", ["Subscribed", ""]), ("Recycle Bin", ["", "Trash"]), ("Vehicles/Electric/4 doors/Red", ["Subscribed", ""]), ("Vehicles/Electric/4 doors", ["", ""]), ("Vehicles/Electric", ["", ""]), ("Vehicles", ["", ""]), ("Fruit", ["", ""]), ("Fruit/Apple", ["", ""]), ("Fruit/Apple/Red", ["", ""]), ("Veggies", ["", ""]), ("Veggies/Green", ["", ""]), ("Veggies/Green/Broccoli", ["", ""]), ], true, ); } // Filter by subscribed including children imap.send("LIST (SUBSCRIBED) \"\" \"*\" RETURN (CHILDREN)") .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_folders( [ ("INBOX", ["Subscribed", "HasNoChildren"]), ( "Vehicles/Electric/4 doors/Red", ["Subscribed", "HasNoChildren"], ), ], true, ); // Recursive match including children imap.send("LIST (SUBSCRIBED RECURSIVEMATCH) \"\" \"*\" RETURN (CHILDREN)") .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_folders( [ ("INBOX", ["Subscribed", "HasNoChildren"]), ( "Vehicles/Electric/4 doors/Red", ["Subscribed", "HasNoChildren"], ), ( "Vehicles/Electric/4 doors", ["\"CHILDINFO\" (\"SUBSCRIBED\")", "HasChildren"], ), ( "Vehicles/Electric", ["\"CHILDINFO\" (\"SUBSCRIBED\")", "HasChildren"], ), ( "Vehicles", ["\"CHILDINFO\" (\"SUBSCRIBED\")", "HasChildren"], ), ], true, ); // Imap4rev1 LSUB imap.send("LSUB \"\" \"*\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_folders( [("INBOX", [""]), ("Vehicles/Electric/4 doors/Red", [""])], true, ); // Unsubscribe imap.send("UNSUBSCRIBE \"Vehicles/Electric/4 doors/Red\"") .await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; for imap in [&mut imap, &mut imap_check] { imap.send("LIST (SUBSCRIBED RECURSIVEMATCH) \"\" \"*\" RETURN (CHILDREN)") .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_folders([("INBOX", ["Subscribed", "HasNoChildren"])], true); } // LIST Filters imap.send("LIST \"\" \"%\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_folders( [ ("INBOX", [""]), ("Recycle Bin", [""]), ("Vehicles", [""]), ("Fruit", [""]), ("Veggies", [""]), ], true, ); imap.send("LIST \"\" \"*/Red\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_folders( [ ("Vehicles/Electric/4 doors/Red", [""]), ("Fruit/Apple/Red", [""]), ], true, ); imap.send("LIST \"\" \"Fruit/*\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_folders([("Fruit/Apple/Red", [""]), ("Fruit/Apple", [""])], true); imap.send("LIST \"\" \"Fruit/%\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_folders([("Fruit/Apple", [""])], true); // Restore Trash folder's original name imap.send("RENAME \"Recycle Bin\" \"Deleted Items\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; // Shared folder creation tests let mut imap_jane = ImapConnection::connect(b"_z ").await; imap_jane .authenticate("jane.smith@example.com", "secret") .await; imap_jane .send("CREATE \"Shared Folders/support@example.com/INBOX/Test\"") .await; imap_jane.assert_read(Type::Tagged, ResponseType::Ok).await; imap_jane .send("CREATE \"Shared Folders/support@example.com/Test\"") .await; imap_jane.assert_read(Type::Tagged, ResponseType::Ok).await; imap_jane .send("CREATE \"Shared Folders/support@example.com/Test/TestSubfolder\"") .await; imap_jane.assert_read(Type::Tagged, ResponseType::Ok).await; imap_jane.send("LIST \"\" \"*\"").await; imap_jane .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_folders( [ ("INBOX", [""]), ("Deleted Items", [""]), ("Drafts", [""]), ("Junk Mail", [""]), ("Sent Items", [""]), ("Shared Folders", [""]), ("Shared Folders/support@example.com", [""]), ("Shared Folders/support@example.com/Deleted Items", [""]), ("Shared Folders/support@example.com/Drafts", [""]), ("Shared Folders/support@example.com/INBOX", [""]), ("Shared Folders/support@example.com/INBOX/Test", [""]), ("Shared Folders/support@example.com/Junk Mail", [""]), ("Shared Folders/support@example.com/Sent Items", [""]), ("Shared Folders/support@example.com/Test", [""]), ( "Shared Folders/support@example.com/Test/TestSubfolder", [""], ), ], true, ); } fn mailbox_matches_pattern() { let mailboxes = [ "imaptest", "imaptest/test", "imaptest/test2", "imaptest/test3", "imaptest/test3/test4", "imaptest/test3/test4/test5", "foobar/test", "foobar/test/test", "foobar/test1/test1", ]; for (pattern, expected_match) in [ ( "imaptest/%", vec!["imaptest/test", "imaptest/test2", "imaptest/test3"], ), ("imaptest/%/%", vec!["imaptest/test3/test4"]), ( "imaptest/*", vec![ "imaptest/test", "imaptest/test2", "imaptest/test3", "imaptest/test3/test4", "imaptest/test3/test4/test5", ], ), ("imaptest/*test4", vec!["imaptest/test3/test4"]), ( "imaptest/*test*", vec![ "imaptest/test", "imaptest/test2", "imaptest/test3", "imaptest/test3/test4", "imaptest/test3/test4/test5", ], ), ("imaptest/%3/%", vec!["imaptest/test3/test4"]), ("imaptest/%3/%4", vec!["imaptest/test3/test4"]), ("imaptest/%t*4", vec!["imaptest/test3/test4"]), ("*st/%3/%4/%5", vec!["imaptest/test3/test4/test5"]), ( "*%*%*%", vec![ "imaptest", "imaptest/test", "imaptest/test2", "imaptest/test3", "imaptest/test3/test4", "imaptest/test3/test4/test5", "foobar/test", "foobar/test/test", "foobar/test1/test1", ], ), ("foobar*test", vec!["foobar/test", "foobar/test/test"]), ] { let patterns = vec![pattern.into()]; let mut matched_mailboxes = Vec::new(); for mailbox in mailboxes { if matches_pattern(&patterns, mailbox) { matched_mailboxes.push(mailbox); } } assert_eq!(matched_mailboxes, expected_match, "for pattern {}", pattern); } } ================================================ FILE: tests/src/imap/managesieve.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use imap_proto::ResponseType; use mail_send::smtp::tls::build_tls_connector; use rustls_pki_types::ServerName; use tokio::{ io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf}, net::TcpStream, }; use tokio_rustls::client::TlsStream; use super::AssertResult; pub async fn test() { println!("Running ManageSieve tests..."); // Connect to ManageSieve let mut sieve = SieveConnection::connect().await; sieve .assert_read(ResponseType::Ok) .await .assert_contains("IMPLEMENTATION"); // Authenticate sieve .send("AUTHENTICATE \"PLAIN\" \"AGpkb2VAZXhhbXBsZS5jb20Ac2VjcmV0\"") .await; sieve.assert_read(ResponseType::Ok).await; /*sieve .assert_read(ResponseType::Ok) .await .assert_contains("MAXREDIRECTS");*/ // CheckScript sieve.send("CHECKSCRIPT \"if true { keep; }\"").await; sieve.assert_read(ResponseType::Ok).await; sieve.send("CHECKSCRIPT \"keep :invalidtag;\"").await; sieve.assert_read(ResponseType::No).await; // PutScript sieve .send_literal("PUTSCRIPT \"simple script\" ", "if true { keep; }\r\n") .await; sieve.assert_read(ResponseType::Ok).await; // PutScript should overwrite existing scripts sieve.send("PUTSCRIPT \"holidays\" \"discard;\"").await; sieve.assert_read(ResponseType::Ok).await; sieve .send_literal( "PUTSCRIPT \"holidays\" ", "require \"vacation\"; vacation \"Gone fishin'\";\r\n", ) .await; sieve.assert_read(ResponseType::Ok).await; // GetScript sieve.send("GETSCRIPT \"simple script\"").await; sieve .assert_read(ResponseType::Ok) .await .assert_contains("if true"); sieve.send("GETSCRIPT \"holidays\"").await; sieve .assert_read(ResponseType::Ok) .await .assert_contains("Gone fishin'"); sieve.send("GETSCRIPT \"dummy\"").await; sieve.assert_read(ResponseType::No).await; // ListScripts sieve.send("LISTSCRIPTS").await; sieve .assert_read(ResponseType::Ok) .await .assert_contains("simple script") .assert_contains("holidays") .assert_count("ACTIVE", 0); // RenameScript sieve .send("RENAMESCRIPT \"simple script\" \"minimalist script\"") .await; sieve.assert_read(ResponseType::Ok).await; sieve .send("RENAMESCRIPT \"holidays\" \"minimalist script\"") .await; sieve .assert_read(ResponseType::No) .await .assert_contains("ALREADYEXISTS"); // SetActive sieve.send("SETACTIVE \"holidays\"").await; sieve.assert_read(ResponseType::Ok).await; sieve.send("LISTSCRIPTS").await; sieve .assert_read(ResponseType::Ok) .await .assert_contains("minimalist script") .assert_contains("holidays\" ACTIVE"); // Deleting an active script should not be allowed sieve.send("DELETESCRIPT \"holidays\"").await; sieve .assert_read(ResponseType::No) .await .assert_contains("ACTIVE"); // Deactivate all sieve.send("SETACTIVE \"\"").await; sieve.assert_read(ResponseType::Ok).await; sieve.send("LISTSCRIPTS").await; sieve .assert_read(ResponseType::Ok) .await .assert_contains("minimalist script") .assert_contains("holidays") .assert_count("ACTIVE", 0); // DeleteScript sieve.send("DELETESCRIPT \"holidays\"").await; sieve.assert_read(ResponseType::Ok).await; sieve.send("DELETESCRIPT \"minimalist script\"").await; sieve.assert_read(ResponseType::Ok).await; sieve.send("LISTSCRIPTS").await; sieve .assert_read(ResponseType::Ok) .await .assert_count("minimalist script", 0) .assert_count("holidays", 0); } pub struct SieveConnection { reader: Lines>>>, writer: WriteHalf>, } impl SieveConnection { pub async fn connect() -> Self { let (reader, writer) = tokio::io::split( build_tls_connector(true) .connect( ServerName::try_from("imap.example.org").unwrap().to_owned(), TcpStream::connect("127.0.0.1:4190").await.unwrap(), ) .await .unwrap(), ); SieveConnection { reader: BufReader::new(reader).lines(), writer, } } pub async fn assert_read(&mut self, rt: ResponseType) -> Vec { let lines = self.read().await; let mut buf = Vec::with_capacity(10); rt.serialize(&mut buf); if lines .last() .unwrap() .starts_with(&String::from_utf8(buf).unwrap()) { lines } else { panic!("Expected {:?} from server but got: {:?}", rt, lines); } } pub async fn read(&mut self) -> Vec { let mut lines = Vec::new(); loop { match tokio::time::timeout(Duration::from_millis(1500), self.reader.next_line()).await { Ok(Ok(Some(line))) => { let is_done = line.starts_with("OK") || line.starts_with("NO") || line.starts_with("BYE"); //println!("<- {:?}", line); lines.push(line); if is_done { return lines; } } Ok(Ok(None)) => { panic!("Invalid response: {:?}.", lines); } Ok(Err(err)) => { panic!("Connection broken: {} ({:?})", err, lines); } Err(_) => panic!("Timeout while waiting for server response: {:?}", lines), } } } pub async fn send(&mut self, text: &str) { //println!("-> {:?}", text); self.writer.write_all(text.as_bytes()).await.unwrap(); self.writer.write_all(b"\r\n").await.unwrap(); } pub async fn send_raw(&mut self, text: &str) { //println!("-> {:?}", text); self.writer.write_all(text.as_bytes()).await.unwrap(); } pub async fn send_literal(&mut self, text: &str, literal: &str) { self.send(&format!("{}{{{}+}}\r\n{}", text, literal.len(), literal)) .await; } } ================================================ FILE: tests/src/imap/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod acl; pub mod antispam; pub mod append; pub mod basic; pub mod body_structure; pub mod condstore; pub mod copy_move; pub mod fetch; pub mod idle; pub mod mailbox; pub mod managesieve; pub mod pop; pub mod search; pub mod store; pub mod thread; use crate::{ AssertConfig, add_test_certs, directory::internal::TestInternalDirectory, store::{ TempDir, build_store_config, cleanup::{search_store_destroy, store_destroy}, }, }; use ::managesieve::core::ManageSieveSessionManager; use ::store::Stores; use ahash::AHashSet; use base64::{Engine, engine::general_purpose}; use common::{ Caches, Core, Data, Inner, Server, config::{ server::{Listeners, ServerProtocol}, telemetry::Telemetry, }, core::BuildServer, manager::boot::build_ipc, }; use http::HttpSessionManager; use imap::core::ImapSessionManager; use imap_proto::ResponseType; use pop3::Pop3SessionManager; use services::SpawnServices; use smtp::{SpawnQueueManager, core::SmtpSessionManager}; use std::{ path::PathBuf, sync::Arc, time::{Duration, Instant}, }; use tokio::{ io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf}, net::TcpStream, sync::watch, }; use utils::config::Config; #[tokio::test] pub async fn imap_tests() { // Prepare settings let start_time = Instant::now(); let delete = true; let handle = init_imap_tests(delete).await; // Body structure tests body_structure::test(); // Connect to IMAP server let mut imap_check = ImapConnection::connect(b"_y ").await; let mut imap = ImapConnection::connect(b"_x ").await; for imap in [&mut imap, &mut imap_check] { imap.assert_read(Type::Untagged, ResponseType::Ok).await; } // Unauthenticated tests basic::test(&mut imap, &mut imap_check).await; // Login for imap in [&mut imap, &mut imap_check] { imap.send("AUTHENTICATE PLAIN {32+}\r\nAGpkb2VAZXhhbXBsZS5jb20Ac2VjcmV0") .await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; } // Delete folders for mailbox in ["Drafts", "Junk Mail", "Sent Items"] { imap.send(&format!("DELETE \"{}\"", mailbox)).await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; } mailbox::test(&mut imap, &mut imap_check).await; append::test(&mut imap, &mut imap_check, &handle).await; search::test(&mut imap, &mut imap_check, &handle).await; fetch::test(&mut imap, &mut imap_check).await; store::test(&mut imap, &mut imap_check, &handle).await; copy_move::test(&mut imap, &mut imap_check).await; thread::test(&mut imap, &mut imap_check, &handle).await; idle::test(&mut imap, &mut imap_check, false).await; condstore::test(&mut imap, &mut imap_check).await; acl::test(&mut imap, &mut imap_check).await; // Logout for imap in [&mut imap, &mut imap_check] { imap.send("UNAUTHENTICATE").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("LOGOUT").await; imap.assert_read(Type::Untagged, ResponseType::Bye).await; } // Antispam training antispam::test(&handle).await; // Run ManageSieve tests managesieve::test().await; // Run POP3 tests pop::test().await; // Print elapsed time let elapsed = start_time.elapsed(); println!( "Elapsed: {}.{:03}s", elapsed.as_secs(), elapsed.subsec_millis() ); // Remove test data if delete { handle.temp_dir.delete(); } } #[allow(dead_code)] pub struct IMAPTest { server: Server, temp_dir: TempDir, shutdown_tx: watch::Sender, } async fn init_imap_tests(delete_if_exists: bool) -> IMAPTest { // Load and parse config let temp_dir = TempDir::new("imap_tests", delete_if_exists); let mut config = Config::new( add_test_certs(&(build_store_config(&temp_dir.path.to_string_lossy()) + SERVER)) .replace("{TMP}", &temp_dir.path.display().to_string()) .replace( "{LEVEL}", &std::env::var("LOG").unwrap_or_else(|_| "disable".to_string()), ), ) .unwrap(); config.resolve_all_macros().await; // Parse servers let mut servers = Listeners::parse(&mut config); // Bind ports and drop privileges servers.bind_and_drop_priv(&mut config); // Build stores let stores = Stores::parse_all(&mut config, false).await; // Parse core let tracers = Telemetry::parse(&mut config, &stores); let core = Core::parse(&mut config, stores, Default::default()).await; let data = Data::parse(&mut config); let cache = Caches::parse(&mut config); let store = core.storage.data.clone(); let search_store = core.storage.fts.clone(); let (ipc, mut ipc_rxs) = build_ipc(false); let inner = Arc::new(Inner { shared_core: core.into_shared(), data, ipc, cache, }); // Parse acceptors servers.parse_tcp_acceptors(&mut config, inner.clone()); // Enable tracing tracers.enable(true); // Start services config.assert_no_errors(); ipc_rxs.spawn_queue_manager(inner.clone()); ipc_rxs.spawn_services(inner.clone()); // Spawn servers let (shutdown_tx, _) = servers.spawn(|server, acceptor, shutdown_rx| { match &server.protocol { ServerProtocol::Smtp | ServerProtocol::Lmtp => server.spawn( SmtpSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Http => server.spawn( HttpSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Imap => server.spawn( ImapSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Pop3 => server.spawn( Pop3SessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::ManageSieve => server.spawn( ManageSieveSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), }; }); if delete_if_exists { store_destroy(&store).await; search_store_destroy(&search_store).await; } // Create tables and test accounts store .create_test_user("admin", "secret", "Superuser", &[]) .await; store .create_test_user( "jdoe@example.com", "secret", "John Doe", &["jdoe@example.com"], ) .await; store .create_test_user( "jane.smith@example.com", "secret", "Jane Smith", &["jane.smith@example.com"], ) .await; store .create_test_user( "foobar@example.com", "secret", "Bill Foobar", &["foobar@example.com"], ) .await; store .create_test_user( "popper@example.com", "secret", "Karl Popper", &["popper@example.com"], ) .await; store .create_test_user( "sgd@example.com", "secret", "Sigmund Gudmund Dudmundsson", &["sgd@example.com"], ) .await; store .create_test_user( "spamtrap@example.com", "secret", "Spam Trap", &["spamtrap@example.com"], ) .await; store .create_test_group( "support@example.com", "Support Group", &["support@example.com"], ) .await; store .add_to_group("jane.smith@example.com", "support@example.com") .await; IMAPTest { server: inner.build_server(), temp_dir, shutdown_tx, } } pub struct ImapConnection { tag: &'static [u8], reader: Lines>>, writer: WriteHalf, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Type { Tagged, Untagged, Continuation, Status, } impl ImapConnection { pub async fn connect(tag: &'static [u8]) -> Self { Self::connect_to(tag, "127.0.0.1:9991").await } pub async fn connect_to(tag: &'static [u8], addr: impl AsRef) -> Self { let (reader, writer) = tokio::io::split(TcpStream::connect(addr.as_ref()).await.unwrap()); ImapConnection { tag, reader: BufReader::new(reader).lines(), writer, } } pub async fn assert_read(&mut self, t: Type, rt: ResponseType) -> Vec { let lines = self.read(t).await; let mut buf = Vec::with_capacity(10); buf.extend_from_slice(match t { Type::Tagged => self.tag, Type::Untagged | Type::Status => b"* ", Type::Continuation => b"+ ", }); if !matches!(t, Type::Continuation | Type::Status) { rt.serialize(&mut buf); } if lines .last() .unwrap() .starts_with(&String::from_utf8(buf).unwrap()) { lines } else { panic!("Expected {:?}/{:?} from server but got: {:?}", t, rt, lines); } } pub async fn assert_disconnect(&mut self) { match tokio::time::timeout(Duration::from_millis(1500), self.reader.next_line()).await { Ok(Ok(None)) => {} Ok(Ok(Some(line))) => { panic!("Expected connection to be closed, but got {:?}", line); } Ok(Err(err)) => { panic!("Connection broken: {:?}", err); } Err(_) => panic!("Timeout while waiting for server response."), } } pub async fn read(&mut self, t: Type) -> Vec { let mut lines = Vec::new(); loop { match tokio::time::timeout(Duration::from_millis(1500), self.reader.next_line()).await { Ok(Ok(Some(line))) => { let is_done = line.starts_with(match t { Type::Tagged => std::str::from_utf8(self.tag).unwrap(), Type::Untagged | Type::Status => "* ", Type::Continuation => "+ ", }); //let c = println!("<- {:?}", line); lines.push(line); if is_done { return lines; } } Ok(Ok(None)) => { panic!("Invalid response: {:?}.", lines); } Ok(Err(err)) => { panic!("Connection broken: {} ({:?})", err, lines); } Err(_) => panic!("Timeout while waiting for server response: {:?}", lines), } } } pub async fn authenticate(&mut self, user: &str, pass: &str) { let creds = general_purpose::STANDARD.encode(format!("\0{user}\0{pass}")); self.send(&format!( "AUTHENTICATE PLAIN {{{}+}}\r\n{creds}", creds.len() )) .await; self.assert_read(Type::Tagged, ResponseType::Ok).await; } pub async fn send(&mut self, text: &str) { //let c = println!("-> {}{:?}", std::str::from_utf8(self.tag).unwrap(), text); self.writer.write_all(self.tag).await.unwrap(); self.writer.write_all(text.as_bytes()).await.unwrap(); self.writer.write_all(b"\r\n").await.unwrap(); } pub async fn send_untagged(&mut self, text: &str) { //let c = println!("-> {:?}", text); self.writer.write_all(text.as_bytes()).await.unwrap(); self.writer.write_all(b"\r\n").await.unwrap(); } pub async fn send_raw(&mut self, text: &str) { //let c = println!("-> {:?}", text); self.writer.write_all(text.as_bytes()).await.unwrap(); } } pub trait AssertResult: Sized { fn assert_folders<'x>( self, expected: impl IntoIterator)>, match_all: bool, ) -> Self; fn assert_response_code(self, code: &str) -> Self; fn assert_contains(self, text: &str) -> Self; fn assert_count(self, text: &str, occurrences: usize) -> Self; fn assert_equals(self, text: &str) -> Self; fn into_response_code(self) -> String; fn into_highest_modseq(self) -> String; fn into_uid_validity(self) -> String; fn into_append_uid(self) -> String; fn into_copy_uid(self) -> String; fn into_modseq(self) -> String; } impl AssertResult for Vec { fn assert_folders<'x>( self, expected: impl IntoIterator)>, match_all: bool, ) -> Self { let mut match_count = 0; 'outer: for (mailbox_name, flags) in expected.into_iter() { for result in self.iter() { if result.contains(&format!("\"{}\"", mailbox_name)) { for flag in flags { if !flag.is_empty() && !result.contains(flag) { panic!("Expected mailbox {} to have flag {}", mailbox_name, flag); } } match_count += 1; continue 'outer; } } panic!("Mailbox {} is not present.", mailbox_name); } if match_all && match_count != self.len() - 1 { panic!( "Expected {} mailboxes, but got {}: {:?}", match_count, self.len() - 1, self.iter().collect::>() ); } self } fn assert_response_code(self, code: &str) -> Self { if !self.last().unwrap().contains(&format!("[{}]", code)) { panic!( "Response code {:?} not found, got {:?}", code, self.last().unwrap() ); } self } fn assert_contains(self, text: &str) -> Self { for line in &self { if line.contains(text) { return self; } } panic!("Expected response to contain {:?}, got {:?}", text, self); } fn assert_count(self, text: &str, occurrences: usize) -> Self { assert_eq!( self.iter().filter(|l| l.contains(text)).count(), occurrences, "Expected {} occurrences of {:?}, found {} in {:?}.", occurrences, text, self.iter().filter(|l| l.contains(text)).count(), self ); self } fn assert_equals(self, text: &str) -> Self { for line in &self { if line == text { return self; } } panic!("Expected response to be {:?}, got {:?}", text, self); } fn into_response_code(self) -> String { if let Some((_, code)) = self.last().unwrap().split_once('[') && let Some((code, _)) = code.split_once(']') { return code.to_string(); } panic!("No response code found in {:?}", self.last().unwrap()); } fn into_append_uid(self) -> String { if let Some((_, code)) = self.last().unwrap().split_once("[APPENDUID ") && let Some((code, _)) = code.split_once(']') && let Some((_, uid)) = code.split_once(' ') { return uid.to_string(); } panic!("No APPENDUID found in {:?}", self.last().unwrap()); } fn into_copy_uid(self) -> String { for line in &self { if let Some((_, code)) = line.split_once("[COPYUID ") && let Some((code, _)) = code.split_once(']') && let Some((_, uid)) = code.rsplit_once(' ') { return uid.to_string(); } } panic!("No COPYUID found in {:?}", self); } fn into_highest_modseq(self) -> String { for line in &self { if let Some((_, value)) = line.split_once("HIGHESTMODSEQ ") { if let Some((value, _)) = value.split_once(']') { return value.to_string(); } else if let Some((value, _)) = value.split_once(')') { return value.to_string(); } else { panic!("No HIGHESTMODSEQ delimiter found in {:?}", line); } } } panic!("No HIGHESTMODSEQ entries found in {:?}", self); } fn into_modseq(self) -> String { for line in &self { if let Some((_, value)) = line.split_once("MODSEQ (") { if let Some((value, _)) = value.split_once(')') { return value.to_string(); } else { panic!("No MODSEQ delimiter found in {:?}", line); } } } panic!("No MODSEQ entries found in {:?}", self); } fn into_uid_validity(self) -> String { for line in &self { if let Some((_, value)) = line.split_once("UIDVALIDITY ") { if let Some((value, _)) = value.split_once(']') { return value.to_string(); } else if let Some((value, _)) = value.split_once(')') { return value.to_string(); } else { panic!("No UIDVALIDITY delimiter found in {:?}", line); } } } panic!("No UIDVALIDITY entries found in {:?}", self); } } pub fn expand_uid_list(list: &str) -> AHashSet { let mut items = AHashSet::new(); for uid in list.split(',') { if let Some((start, end)) = uid.split_once(':') { let start = start.parse::().unwrap(); let end = end.parse::().unwrap(); for uid in start..=end { items.insert(uid); } } else { items.insert(uid.parse::().unwrap()); } } items } fn resources_dir() -> PathBuf { let mut resources = PathBuf::from(env!("CARGO_MANIFEST_DIR")); resources.push("resources"); resources.push("imap"); resources } const SERVER: &str = r#" [server] hostname = "imap.example.org" [server.listener.imap] bind = ["127.0.0.1:9991"] protocol = "imap" max-connections = 81920 [server.listener.imaptls] bind = ["127.0.0.1:9992"] protocol = "imap" max-connections = 81920 tls.implicit = true [server.listener.sieve] bind = ["127.0.0.1:4190"] protocol = "managesieve" max-connections = 81920 tls.implicit = true [server.listener.pop3] bind = ["127.0.0.1:4110"] protocol = "pop3" max-connections = 81920 tls.implicit = true [server.listener.lmtp-debug] bind = ['127.0.0.1:11201'] greeting = 'Test LMTP instance' protocol = 'lmtp' tls.implicit = false [server.socket] reuse-addr = true [server.tls] enable = true implicit = false certificate = "default" [session.ehlo] reject-non-fqdn = false [session.rcpt] relay = [ { if = "!is_empty(authenticated_as)", then = true }, { else = false } ] [session.rcpt.errors] total = 5 wait = "1ms" [spam-filter] enable = true [spam-filter.list] scores = {"PROB_SPAM_LOW" = "10.0", "PROB_SPAM_HIGH" = "10.0", "SPAM_TRAP" = "100.0"} [spam-filter.classifier.samples] min-ham = 10 min-spam = 10 [lookup] "spam-traps" = {"spamtrap@*"} [resolver] type = "system" [queue.strategy] route = [ { if = "rcpt_domain == 'example.com'", then = "'local'" }, { if = "contains(['remote.org', 'foobar.com', 'test.com', 'other_domain.com'], rcpt_domain)", then = "'mock-smtp'" }, { else = "'mx'" } ] [queue.route."mock-smtp"] type = "relay" address = "localhost" port = 9999 protocol = "smtp" [queue.route."mock-smtp".tls] enable = false allow-invalid-certs = true [session.data] spam-filter = "recipients[0] != 'popper@example.com'" [session.data.add-headers] delivered-to = false [session.extensions] future-release = [ { if = "!is_empty(authenticated_as)", then = "99999999d"}, { else = false } ] [certificate.default] cert = "%{file:{CERT}}%" private-key = "%{file:{PK}}%" [imap.protocol] uidplus = true [jmap.protocol] set.max-objects = 100000 [jmap.protocol.request] max-concurrent = 8 [jmap.protocol.upload] max-size = 5000000 max-concurrent = 4 ttl = "1m" [jmap.protocol.upload.quota] files = 3 size = 50000 [jmap.rate-limit] account = "1000/1m" authentication = "100/2s" anonymous = "100/1m" [jmap.event-source] throttle = "500ms" [jmap.web-sockets] throttle = "500ms" [jmap.push] throttle = "500ms" attempts.interval = "500ms" [email.folders.inbox] name = "Inbox" subscribe = false [email.folders.sent] name = "Sent Items" subscribe = false [email.folders.trash] name = "Deleted Items" subscribe = false [email.folders.junk] name = "Junk Mail" subscribe = false [email.folders.drafts] name = "Drafts" subscribe = false [store."auth"] type = "sqlite" path = "{TMP}/auth.db" [store."auth".query] name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = ? AND active = true" members = "SELECT member_of FROM group_members WHERE name = ?" recipients = "SELECT name FROM emails WHERE address = ?" emails = "SELECT address FROM emails WHERE name = ? AND type != 'list' ORDER BY type DESC, address ASC" verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type = 'primary' ORDER BY address LIMIT 5" expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50" domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || ? LIMIT 1" [oauth] key = "parerga_und_paralipomena" [oauth.auth] max-attempts = 1 [oauth.expiry] user-code = "1s" token = "1s" refresh-token = "3s" refresh-token-renew = "2s" [tracer.console] type = "console" level = "{LEVEL}" multiline = false ansi = true disabled-events = ["network.*"] "#; ================================================ FILE: tests/src/imap/pop.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{jmap::mail::delivery::SmtpConnection, smtp::session::VerifyResponse}; use mail_send::smtp::tls::build_tls_connector; use rustls_pki_types::ServerName; use std::time::Duration; use tokio::{ io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf}, net::TcpStream, }; use tokio_rustls::client::TlsStream; pub async fn test() { println!("Running POP3 tests..."); // Send 3 test emails for i in 0..3 { let mut lmtp = SmtpConnection::connect_port(11201).await; lmtp.ingest( "bill@example.com", &["popper@example.com"], &format!( concat!( "From: bill@example.com\r\n", "To: popper@example.com\r\n", "Subject: TPS Report {}\r\n", "X-Spam-Status: No\r\n", "\r\n", "I'm going to need those TPS {} reports ASAP.\r\n", "..\r\n", "So, if you could do that, that'd be great." ), i, i ), ) .await; } // Connect to POP3 let mut pop3 = Pop3Connection::connect().await; pop3.assert_read(ResponseType::Ok).await; // Capabilities pop3.send("CAPA").await; pop3.assert_read(ResponseType::Multiline) .await .assert_contains("SASL PLAIN") .assert_contains("IMPLEMENTATION"); // Noop pop3.send("NOOP").await; pop3.assert_read(ResponseType::Ok).await; // Authenticate user/pass pop3.send("PASS secret").await; pop3.assert_read(ResponseType::Err).await; pop3.send("USER popper@example.com").await; pop3.assert_read(ResponseType::Ok).await; pop3.send("PASS wrong_secret").await; pop3.assert_read(ResponseType::Err).await; pop3.send("USER popper@example.com").await; pop3.assert_read(ResponseType::Ok).await; pop3.send("PASS secret").await; pop3.assert_read(ResponseType::Ok).await; pop3.send("QUIT").await; // Authenticate using AUTH PLAIN let mut pop3 = Pop3Connection::connect().await; pop3.assert_read(ResponseType::Ok).await; pop3.send("AUTH PLAIN AHBvcHBlckBleGFtcGxlLmNvbQBzZWNyZXQ=") .await; pop3.assert_read(ResponseType::Ok).await; // STAT pop3.send("STAT").await; pop3.assert_read(ResponseType::Ok) .await .assert_contains("+OK 3 603"); // UTF8 pop3.send("UTF8").await; pop3.assert_read(ResponseType::Ok).await; // LIST pop3.send("LIST").await; pop3.assert_read(ResponseType::Multiline) .await .assert_contains("+OK 3 messages") .assert_contains("1 201") .assert_contains("2 201") .assert_contains("3 201"); pop3.send("LIST 2").await; pop3.assert_read(ResponseType::Ok) .await .assert_contains("+OK 2 201"); // UIDL pop3.send("UIDL").await; pop3.assert_read(ResponseType::Multiline) .await .assert_contains("+OK 3 messages") .assert_contains("1 ") .assert_contains("2 ") .assert_contains("3 "); pop3.send("UIDL 2").await; pop3.assert_read(ResponseType::Ok) .await .assert_contains("+OK 2 "); // RETR pop3.send("RETR 1").await; pop3.assert_read(ResponseType::Multiline) .await .assert_contains("+OK 201 octets") .assert_contains("I'm going to need those TPS 0 reports ASAP.") .assert_contains("So, if you could do that, that'd be great."); pop3.send("RETR 3").await; pop3.assert_read(ResponseType::Multiline) .await .assert_contains("+OK 201 octets") .assert_contains("I'm going to need those TPS 2 reports ASAP.") .assert_contains("So, if you could do that, that'd be great."); pop3.send("RETR 4").await; pop3.assert_read(ResponseType::Err).await; // TOP pop3.send("TOP 1 4").await; pop3.assert_read(ResponseType::Multiline) .await .assert_contains("+OK 201 octets") .assert_contains("Subject: TPS Report 0") .assert_not_contains("I'm going to need those TPS 0 reports ASAP."); pop3.send("TOP 3 4").await; pop3.assert_read(ResponseType::Multiline) .await .assert_contains("+OK 201 octets") .assert_contains("Subject: TPS Report 2") .assert_not_contains("I'm going to need those TPS 2 reports ASAP."); // DELE + RSET + QUIT (should not delete messages) pop3.send("DELE 1").await; pop3.assert_read(ResponseType::Ok).await; pop3.send("DELE 4").await; pop3.assert_read(ResponseType::Err).await; pop3.send("RSET").await; pop3.assert_read(ResponseType::Ok).await; pop3.send("QUIT").await; let mut pop3 = Pop3Connection::connect_and_login().await; pop3.send("STAT").await; pop3.assert_read(ResponseType::Ok) .await .assert_contains("+OK 3 603"); // DELE + QUIT (should delete messages) pop3.send("DELE 2").await; pop3.assert_read(ResponseType::Ok).await; pop3.send("QUIT").await; pop3.assert_read(ResponseType::Ok).await; let mut pop3 = Pop3Connection::connect_and_login().await; pop3.send("STAT").await; pop3.assert_read(ResponseType::Ok) .await .assert_contains("+OK 2 402"); pop3.send("TOP 1 4").await; pop3.assert_read(ResponseType::Multiline) .await .assert_contains("TPS Report 0"); pop3.send("TOP 2 4").await; pop3.assert_read(ResponseType::Multiline) .await .assert_contains("TPS Report 2"); // DELE using pipelining pop3.send("DELE 1\r\nDELE 2").await; pop3.assert_read(ResponseType::Ok).await; pop3.assert_read(ResponseType::Ok).await; pop3.send("QUIT").await; pop3.assert_read(ResponseType::Ok).await; let mut pop3 = Pop3Connection::connect_and_login().await; pop3.send("STAT").await; pop3.assert_read(ResponseType::Ok) .await .assert_contains("+OK 0 0"); pop3.send("QUIT").await; } #[derive(Debug, Clone, PartialEq, Eq)] pub enum ResponseType { Ok, Multiline, Err, } pub struct Pop3Connection { reader: Lines>>>, writer: WriteHalf>, } impl Pop3Connection { pub async fn connect() -> Self { let (reader, writer) = tokio::io::split( build_tls_connector(true) .connect( ServerName::try_from("pop3.example.org").unwrap().to_owned(), TcpStream::connect("127.0.0.1:4110").await.unwrap(), ) .await .unwrap(), ); Pop3Connection { reader: BufReader::new(reader).lines(), writer, } } pub async fn connect_and_login() -> Self { let mut pop3 = Self::connect().await; pop3.assert_read(ResponseType::Ok).await; pop3.send("AUTH PLAIN AHBvcHBlckBleGFtcGxlLmNvbQBzZWNyZXQ=") .await; pop3.assert_read(ResponseType::Ok).await; pop3 } pub async fn assert_read(&mut self, rt: ResponseType) -> Vec { let lines = self.read(matches!(rt, ResponseType::Multiline)).await; if lines.last().unwrap().starts_with(match rt { ResponseType::Ok => "+OK", ResponseType::Multiline => ".", ResponseType::Err => "-ERR", }) { lines } else { panic!("Expected {:?} from server but got: {:?}", rt, lines); } } pub async fn read(&mut self, is_multiline: bool) -> Vec { let mut lines = Vec::new(); loop { match tokio::time::timeout(Duration::from_millis(1500), self.reader.next_line()).await { Ok(Ok(Some(line))) => { let is_done = (!is_multiline && line.starts_with("+OK")) || (is_multiline && line == ".") || line.starts_with("-ERR"); //let c = println!("<- {:?}", line); lines.push(line); if is_done { return lines; } } Ok(Ok(None)) => { panic!("Invalid response: {:?}.", lines); } Ok(Err(err)) => { panic!("Connection broken: {} ({:?})", err, lines); } Err(_) => panic!("Timeout while waiting for server response: {:?}", lines), } } } pub async fn send(&mut self, text: &str) { //let c = println!("-> {:?}", text); self.writer.write_all(text.as_bytes()).await.unwrap(); self.writer.write_all(b"\r\n").await.unwrap(); } pub async fn send_raw(&mut self, text: &str) { //let c = println!("-> {:?}", text); self.writer.write_all(text.as_bytes()).await.unwrap(); } } ================================================ FILE: tests/src/imap/search.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{AssertResult, ImapConnection, Type}; use crate::imap::IMAPTest; use imap_proto::ResponseType; pub async fn test(imap: &mut ImapConnection, imap_check: &mut ImapConnection, handle: &IMAPTest) { println!("Running SEARCH tests..."); // Searches without selecting a mailbox should fail. imap.send("SEARCH RETURN (MIN MAX COUNT ALL) ALL").await; imap.assert_read(Type::Tagged, ResponseType::Bad).await; // Select INBOX imap.send("SELECT INBOX").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("10 EXISTS") .assert_contains("[UIDNEXT 11]"); imap_check.send("SELECT INBOX").await; imap_check.assert_read(Type::Tagged, ResponseType::Ok).await; // Min, Max and Count imap.send("SEARCH RETURN (MIN MAX COUNT ALL) ALL").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("COUNT 10 MIN 1 MAX 10 ALL 1,10"); imap_check.send("UID SEARCH ALL").await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_equals("* SEARCH 1 2 3 4 5 6 7 8 9 10"); // Filters imap_check .send("UID SEARCH OR FROM nathaniel SUBJECT argentina") .await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_equals("* SEARCH 1 3 4 6"); imap_check .send("UID SEARCH UNSEEN OR KEYWORD Flag_007 KEYWORD Flag_004") .await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_equals("* SEARCH 5 8"); imap_check .send("UID SEARCH TEXT coffee FROM vandelay SUBJECT exporting SENTON 20-Nov-2021") .await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_equals("* SEARCH 10"); imap_check .send("UID SEARCH NOT (FROM nathaniel ANSWERED)") .await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_equals("* SEARCH 2 3 5 7 8 9 10"); imap_check .send("UID SEARCH UID 0:6 LARGER 1000 SMALLER 2000") .await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_equals("* SEARCH 1 2"); // Saved search imap_check.send( "UID SEARCH RETURN (SAVE ALL) OR OR FROM nathaniel FROM vandelay OR SUBJECT rfc FROM gore", ) .await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("1,3:4,6,8,10"); imap_check.send("UID SEARCH NOT $").await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_equals("* SEARCH 2 5 7 9"); imap_check .send("UID SEARCH $ SMALLER 1000 SUBJECT section") .await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_equals("* SEARCH 8"); imap_check.send("UID SEARCH RETURN (MIN MAX) NOT $").await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("MIN 2 MAX 9"); // Sort imap_check .send("UID SORT (REVERSE SUBJECT REVERSE DATE) UTF-8 FROM Nathaniel") .await; imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await .assert_equals("* SORT 6 4 1"); imap.send("UID SORT RETURN (COUNT ALL) (DATE SUBJECT) UTF-8 ALL") .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains(if !handle.server.search_store().is_mysql() { "COUNT 10 ALL 6,4:5,1,10,3,7:8,2,9" } else { "COUNT 10 ALL 9,3,7:8,2,6,4:5,1,10" }); //6,4:5,1,10,9,3,7:8,2"); } ================================================ FILE: tests/src/imap/store.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use imap_proto::ResponseType; use crate::jmap::wait_for_index; use super::{AssertResult, IMAPTest, ImapConnection, Type}; pub async fn test(imap: &mut ImapConnection, _imap_check: &mut ImapConnection, handle: &IMAPTest) { println!("Running STORE tests..."); // Select INBOX imap.send("SELECT INBOX").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("10 EXISTS") .assert_contains("[UIDNEXT 11]"); // Set all messages to flag "Seen" imap.send("UID STORE 1:10 +FLAGS.SILENT (\\Seen)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("FLAGS", 0); // Check that the flags were set imap.send("UID FETCH 1:* (Flags)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("\\Seen", 10); // Check status imap.send("STATUS INBOX (UIDNEXT MESSAGES UNSEEN)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("MESSAGES 10") .assert_contains("UNSEEN 0") .assert_contains("UIDNEXT 11"); // Remove Seen flag from all messages imap.send("UID STORE 1:10 -FLAGS (\\Seen)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("FLAGS", 10) .assert_count("Seen", 0); // Check that the flags were removed imap.send("UID FETCH 1:* (Flags)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("\\Seen", 0); imap.send("STATUS INBOX (UIDNEXT MESSAGES UNSEEN)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("MESSAGES 10") .assert_contains("UNSEEN 10") .assert_contains("UIDNEXT 11"); // Store using saved searches wait_for_index(&handle.server).await; imap.send("SEARCH RETURN (SAVE) FROM nathaniel").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("UID STORE $ +FLAGS (\\Answered)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("FLAGS", 3); // Remove Answered flag imap.send("UID STORE 1:* -FLAGS (\\Answered)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("FLAGS", 3) .assert_count("Answered", 0); } ================================================ FILE: tests/src/imap/thread.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use imap_proto::ResponseType; use crate::imap::{AssertResult, IMAPTest, expand_uid_list}; use super::{ImapConnection, Type, append::build_messages}; pub async fn test(imap: &mut ImapConnection, _imap_check: &mut ImapConnection, handle: &IMAPTest) { println!("Running THREAD tests..."); // Create test messages let messages = build_messages(); // Insert messages using Multiappend imap.send("CREATE Manchego").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; for (pos, message) in messages.iter().enumerate() { if pos == 0 { imap.send(&format!("APPEND Manchego {{{}}}", message.len())) .await; } else { imap.send_untagged(&format!(" {{{}}}", message.len())).await; } imap.assert_read(Type::Continuation, ResponseType::Ok).await; if pos < messages.len() - 1 { imap.send_raw(message).await; } else { imap.send_untagged(message).await; assert_eq!( expand_uid_list( &imap .assert_read(Type::Tagged, ResponseType::Ok) .await .into_append_uid() ) .len(), messages.len(), ); } } // Obtain ThreadId and MessageId of the first message imap.send("SELECT Manchego").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; let mut email_id = None; let mut thread_id = None; imap.send("UID FETCH 1 (EMAILID THREADID)").await; for line in imap.assert_read(Type::Tagged, ResponseType::Ok).await { if let Some((_, value)) = line.split_once("EMAILID (") { email_id = value .split_once(')') .expect("Missing delimiter") .0 .to_string() .into(); } if let Some((_, value)) = line.split_once("THREADID (") { thread_id = value .split_once(')') .expect("Missing delimiter") .0 .to_string() .into(); } } let email_id = email_id.expect("Missing EMAILID"); let thread_id = thread_id.expect("Missing THREADID"); // 4 different threads are expected imap.send("THREAD REFERENCES UTF-8 1:*").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("(1 2 3 4)") .assert_contains("(5 6 7 8)") .assert_contains("(9 10 11 12)"); // Filter by subject (mySQL does not support searching for short keywords) if !handle.server.search_store().is_mysql() { imap.send("THREAD REFERENCES UTF-8 SUBJECT T1").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("(5 6 7 8)") .assert_count("(1 2 3 4)", 0) .assert_count("(9 10 11 12)", 0); } // Filter by threadId and messageId imap.send(&format!( "UID THREAD REFERENCES UTF-8 THREADID {}", thread_id )) .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("(1 2 3 4)") .assert_count("(", 1); imap.send(&format!("UID THREAD REFERENCES UTF-8 EMAILID {}", email_id)) .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("(1)") .assert_count("(", 1); // Delete all messages imap.send("STORE 1:* +FLAGS.SILENT (\\Deleted)").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("EXPUNGE").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("EXPUNGE", 13); } ================================================ FILE: tests/src/jmap/auth/limits.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ directory::internal::TestInternalDirectory, imap::{ImapConnection, Type}, jmap::{JMAPTest}, }; use common::listener::blocked::BLOCKED_IP_KEY; use directory::Permission; use imap_proto::ResponseType; use jmap_client::{ client::{Client, Credentials}, core::set::{SetError, SetErrorType}, mailbox::{self}, }; use std::{ net::{IpAddr, Ipv4Addr}, sync::Arc, time::Duration, }; use store::write::now; pub async fn test(params: &mut JMAPTest) { println!("Running Authorization tests..."); // Create test account let server = params.server.clone(); let account = params.account("jdoe@example.com"); // Remove unlimited requests permission params .server .store() .remove_permissions(account.name(), [Permission::UnlimitedRequests]) .await; params.server.inner.cache.access_tokens.clear(); // Reset rate limiters params.webhook.clear(); // Incorrect passwords should be rejected with a 401 error assert!(matches!( Client::new() .credentials(Credentials::basic("jdoe@example.com", "abcde")) .accept_invalid_certs(true) .follow_redirects(["127.0.0.1"]) .connect("https://127.0.0.1:8899") .await, Err(jmap_client::Error::Problem(err)) if err.status() == Some(401))); // Wait until the beginning of the 5 seconds bucket const LIMIT: u64 = 5; let now = now(); let range_start = now / LIMIT; let range_end = (range_start * LIMIT) + LIMIT; tokio::time::sleep(Duration::from_secs(range_end - now)).await; // Test fail2ban assert_eq!( server .core .storage .config .get(format!("{BLOCKED_IP_KEY}.127.0.0.1")) .await .unwrap(), None ); for n in 0..98 { match Client::new() .credentials(Credentials::basic( "not_an_account@example.com", &format!("brute_force{}", n), )) .accept_invalid_certs(true) .follow_redirects(["127.0.0.1"]) .connect("https://127.0.0.1:8899") .await { Err(jmap_client::Error::Problem(_)) => {} Err(err) => { panic!("Unexpected response: {:?}", err); } Ok(_) => { panic!("Unexpected success"); } } } let mut imap = ImapConnection::connect(b"_x ").await; imap.send("AUTHENTICATE PLAIN AGpvaG4AY2hpbWljaGFuZ2Fz") .await; imap.assert_read(Type::Tagged, ResponseType::No).await; // There are already 100 failed login attempts for this IP address // so the next one should be rejected, even if done over IMAP imap.send("AUTHENTICATE PLAIN AGpvaG4AY2hpbWljaGFuZ2Fz") .await; imap.assert_disconnect().await; // Make sure the IP address is blocked assert_eq!( server .core .storage .config .get(format!("{BLOCKED_IP_KEY}.127.0.0.1")) .await .unwrap(), Some(String::new()) ); ImapConnection::connect(b"_y ") .await .assert_disconnect() .await; // Lift ban server .core .storage .config .clear(format!("{BLOCKED_IP_KEY}.127.0.0.1")) .await .unwrap(); server .inner .data .blocked_ips .write() .remove(&IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))); // Valid authentication requests should not be rate limited for _ in 0..110 { Client::new() .credentials(Credentials::basic(account.name(), account.secret())) .accept_invalid_certs(true) .follow_redirects(["127.0.0.1"]) .connect("https://127.0.0.1:8899") .await .unwrap(); } // Login with the correct credentials let client = Client::new() .credentials(Credentials::basic(account.name(), account.secret())) .accept_invalid_certs(true) .follow_redirects(["127.0.0.1"]) .connect("https://127.0.0.1:8899") .await .unwrap(); assert_eq!(client.session().username(), account.name()); assert_eq!( client .session() .account(account.id_string()) .unwrap() .name(), account.name() ); assert!( client .session() .account(account.id_string()) .unwrap() .is_personal() ); // Uploads up to 5000000 bytes should be allowed assert_eq!( client .upload(None, vec![b'A'; 5000000], None) .await .unwrap() .size(), 5000000 ); assert!( client .upload(None, vec![b'A'; 5000001], None) .await .is_err() ); // Users should be allowed to create identities only // using email addresses associated to their principal let iid1 = client .identity_create("John Doe", "jdoe@example.com") .await .unwrap() .take_id(); let iid2 = client .identity_create("John Doe (secondary)", "john.doe@example.com") .await .unwrap() .take_id(); assert!(matches!( client .identity_create("John the Spammer", "spammy@mcspamface.com") .await, Err(jmap_client::Error::Set(SetError { type_: SetErrorType::InvalidProperties, .. })) )); client.identity_destroy(&iid1).await.unwrap(); client.identity_destroy(&iid2).await.unwrap(); // Concurrent requests check let client = Arc::new(client); for _ in 0..8 { let client_ = client.clone(); tokio::spawn(async move { let _ = client_ .mailbox_query( mailbox::query::Filter::name("__sleep").into(), [mailbox::query::Comparator::name()].into(), ) .await; }); } tokio::time::sleep(Duration::from_millis(500)).await; assert!(matches!( client .mailbox_query( mailbox::query::Filter::name("__sleep").into(), [mailbox::query::Comparator::name()].into(), ) .await, Err(jmap_client::Error::Problem(err)) if err.status() == Some(400))); // Wait for sleep to be done tokio::time::sleep(Duration::from_millis(1000)).await; // Concurrent upload test for _ in 0..4 { let client_ = client.clone(); tokio::spawn(async move { client_.upload(None, b"sleep".to_vec(), None).await.unwrap(); }); } tokio::time::sleep(Duration::from_millis(500)).await; assert!(matches!( client.upload(None, b"sleep".to_vec(), None).await, Err(jmap_client::Error::Problem(err)) if err.status() == Some(400))); // Add unlimited requests permission params .server .store() .add_permissions(account.name(), [Permission::UnlimitedRequests]) .await; params.server.inner.cache.access_tokens.clear(); // Destroy test accounts params.destroy_all_mailboxes(account).await; params.assert_is_empty().await; // Check webhook events params .webhook .assert_contains(&["auth.failed", "auth.success", "security.authentication-ban"]); } ================================================ FILE: tests/src/jmap/auth/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod limits; pub mod oauth; pub mod permissions; pub mod quota; ================================================ FILE: tests/src/jmap/auth/oauth.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ imap::{ ImapConnection, Type, pop::{self, Pop3Connection}, }, jmap::{JMAPTest, ManagementApi, mail::delivery::SmtpConnection}, }; use base64::{Engine, engine::general_purpose}; use biscuit::{JWT, SingleOrMultiple, jwk::JWKSet}; use bytes::Bytes; use common::auth::oauth::{ introspect::OAuthIntrospect, oidc::StandardClaims, registration::{ClientRegistrationRequest, ClientRegistrationResponse}, }; use http::auth::oauth::{ DeviceAuthResponse, ErrorType, OAuthCodeRequest, TokenResponse, auth::OAuthMetadata, openid::OpenIdMetadata, }; use imap_proto::ResponseType; use jmap_client::{ client::{Client, Credentials}, mailbox::query::Filter, }; use serde::{Serialize, de::DeserializeOwned}; use std::time::{Duration, Instant}; use store::ahash::AHashMap; #[derive(serde::Deserialize, Debug)] #[allow(dead_code)] struct OAuthCodeResponse { pub code: String, #[serde(rename = "isEnterprise")] pub is_enterprise: bool, } pub async fn test(params: &mut JMAPTest) { println!("Running OAuth tests..."); // Create test account let server = params.server.clone(); let account = params.account("jdoe@example.com"); // Build API let api = ManagementApi::new(8899, "jdoe@example.com", "12345"); // Obtain OAuth metadata let metadata: OAuthMetadata = get("https://127.0.0.1:8899/.well-known/oauth-authorization-server").await; let oidc_metadata: OpenIdMetadata = get("https://127.0.0.1:8899/.well-known/openid-configuration").await; let jwk_set: JWKSet<()> = get(&oidc_metadata.jwks_uri).await; // Register client let registration: ClientRegistrationResponse = post_json( &metadata.registration_endpoint, None, &ClientRegistrationRequest { redirect_uris: vec!["https://localhost".to_string()], ..Default::default() }, ) .await; let client_id = registration.client_id; /*println!("OAuth metadata: {:#?}", metadata); println!("OpenID metadata: {:#?}", oidc_metadata); println!("JWKSet: {:#?}", jwk_set);*/ // ------------------------ // Authorization code flow // ------------------------ // Authenticate with the correct password let response = api .post::( "/api/oauth", &OAuthCodeRequest::Code { client_id: client_id.to_string(), redirect_uri: "https://localhost".to_string().into(), nonce: "abc1234".to_string().into(), }, ) .await .unwrap() .unwrap_data(); // Both client_id and redirect_uri have to match let mut token_params = AHashMap::from_iter([ ("client_id".to_string(), "invalid_client".to_string()), ("redirect_uri".to_string(), "https://localhost".to_string()), ("grant_type".to_string(), "authorization_code".to_string()), ("code".to_string(), response.code), ]); assert_eq!( post::(&metadata.token_endpoint, &token_params).await, TokenResponse::Error { error: ErrorType::InvalidClient } ); token_params.insert("client_id".to_string(), client_id.to_string()); token_params.insert( "redirect_uri".to_string(), "https://some-other.url".to_string(), ); assert_eq!( post::(&metadata.token_endpoint, &token_params).await, TokenResponse::Error { error: ErrorType::InvalidClient } ); // Obtain token token_params.insert("redirect_uri".to_string(), "https://localhost".to_string()); let (token, refresh_token, id_token) = unwrap_oidc_token_response(post(&metadata.token_endpoint, &token_params).await); // Connect to account using token and attempt to search let john_client = Client::new() .credentials(Credentials::bearer(&token)) .accept_invalid_certs(true) .follow_redirects(["127.0.0.1"]) .connect("https://127.0.0.1:8899") .await .unwrap(); assert_eq!(john_client.default_account_id(), account.id_string()); assert!( !john_client .mailbox_query(None::, None::>) .await .unwrap() .ids() .is_empty() ); // Verify ID token using the JWK set let id_token = JWT::::new_encoded(&id_token) .decode_with_jwks(&jwk_set, None) .unwrap(); let claims = id_token.payload().unwrap(); let registered_claims = &claims.registered; let private_claims = &claims.private; assert_eq!(registered_claims.issuer, Some(oidc_metadata.issuer)); assert_eq!( registered_claims.subject, Some(account.id().document_id().to_string()) ); assert_eq!( registered_claims.audience, Some(SingleOrMultiple::Single(client_id.to_string())) ); assert_eq!(private_claims.nonce, Some("abc1234".into())); assert_eq!( private_claims.preferred_username, Some("jdoe@example.com".into()) ); assert_eq!(private_claims.email, Some("jdoe@example.com".into())); // Introspect token let access_introspect: OAuthIntrospect = post_with_auth::( &metadata.introspection_endpoint, token.as_str().into(), &AHashMap::from_iter([("token".to_string(), token.to_string())]), ) .await; assert_eq!(access_introspect.username.unwrap(), "jdoe@example.com"); assert_eq!(access_introspect.token_type.unwrap(), "bearer"); assert_eq!(access_introspect.client_id.unwrap(), client_id); assert!(access_introspect.active); let refresh_introspect = post_with_auth::( &metadata.introspection_endpoint, token.as_str().into(), &AHashMap::from_iter([("token".to_string(), refresh_token.unwrap())]), ) .await; assert_eq!(refresh_introspect.username.unwrap(), "jdoe@example.com"); assert_eq!(refresh_introspect.client_id.unwrap(), client_id); assert!(refresh_introspect.active); assert_eq!( refresh_introspect.iat.unwrap(), access_introspect.iat.unwrap() ); // Try SMTP OAUTHBEARER auth let oauth_bearer_invalid_sasl = general_purpose::STANDARD.encode(format!( "n,a={},\u{1}auth=Bearer {}\u{1}\u{1}", "user@domain", "invalid_token" )); let oauth_bearer_sasl = general_purpose::STANDARD.encode(format!( "n,a={},\u{1}auth=Bearer {}\u{1}\u{1}", "user@domain", token )); let mut smtp = SmtpConnection::connect().await; smtp.send(&format!("AUTH OAUTHBEARER {oauth_bearer_invalid_sasl}",)) .await; smtp.read(1, 4).await; smtp.send(&format!("AUTH OAUTHBEARER {oauth_bearer_sasl}",)) .await; smtp.read(1, 2).await; // Try IMAP OAUTHBEARER auth let mut imap = ImapConnection::connect(b"_x ").await; imap.assert_read(Type::Untagged, ResponseType::Ok).await; imap.send(&format!("AUTHENTICATE OAUTHBEARER {oauth_bearer_sasl}")) .await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; // Try POP3 OAUTHBEARER auth let mut pop3 = Pop3Connection::connect().await; pop3.assert_read(pop::ResponseType::Ok).await; pop3.send(&format!("AUTH OAUTHBEARER {oauth_bearer_sasl}")) .await; pop3.assert_read(pop::ResponseType::Ok).await; // ------------------------ // Device code flow // ------------------------ // Request a device code let device_code_params = AHashMap::from_iter([("client_id".to_string(), client_id.to_string())]); let device_response: DeviceAuthResponse = post(&metadata.device_authorization_endpoint, &device_code_params).await; //println!("Device response: {:#?}", device_response); // Status should be pending let mut token_params = AHashMap::from_iter([ ("client_id".to_string(), client_id.to_string()), ( "grant_type".to_string(), "urn:ietf:params:oauth:grant-type:device_code".to_string(), ), ( "device_code".to_string(), device_response.device_code.to_string(), ), ]); assert_eq!( post::(&metadata.token_endpoint, &token_params).await, TokenResponse::Error { error: ErrorType::AuthorizationPending } ); // Let the code expire and make sure it's invalidated tokio::time::sleep(Duration::from_secs(1)).await; assert!( !api.post::( "/api/oauth", &OAuthCodeRequest::Device { code: device_response.user_code.clone(), }, ) .await .unwrap() .unwrap_data(), "Code should be expired" ); assert_eq!( post::(&metadata.token_endpoint, &token_params).await, TokenResponse::Error { error: ErrorType::ExpiredToken } ); // Authenticate account using a valid code let device_response: DeviceAuthResponse = post(&metadata.device_authorization_endpoint, &device_code_params).await; token_params.insert( "device_code".to_string(), device_response.device_code.to_string(), ); assert!( api.post::( "/api/oauth", &OAuthCodeRequest::Device { code: device_response.user_code.clone(), }, ) .await .unwrap() .unwrap_data(), "Code is invalid" ); // Obtain token let time_first_token = Instant::now(); let (token, refresh_token, _) = unwrap_token_response(post(&metadata.token_endpoint, &token_params).await); let refresh_token = refresh_token.unwrap(); // Authorization codes can only be used once assert_eq!( post::(&metadata.token_endpoint, &token_params).await, TokenResponse::Error { error: ErrorType::ExpiredToken } ); // Connect to account using token and attempt to search let john_client = Client::new() .credentials(Credentials::bearer(&token)) .accept_invalid_certs(true) .follow_redirects(["127.0.0.1"]) .connect("https://127.0.0.1:8899") .await .unwrap(); assert_eq!(john_client.default_account_id(), account.id_string()); assert!( !john_client .mailbox_query(None::, None::>) .await .unwrap() .ids() .is_empty() ); // Connecting using the refresh token should not work assert_unauthorized("https://127.0.0.1:8899", &refresh_token).await; // Refreshing a token using the access token should not work assert_eq!( post::( &metadata.token_endpoint, &AHashMap::from_iter([ ("client_id".to_string(), client_id.to_string()), ("grant_type".to_string(), "refresh_token".to_string()), ("refresh_token".to_string(), token), ]), ) .await, TokenResponse::Error { error: ErrorType::InvalidGrant } ); // Refreshing the access token before expiration should not include a new refresh token let refresh_params = AHashMap::from_iter([ ("client_id".to_string(), client_id.to_string()), ("grant_type".to_string(), "refresh_token".to_string()), ("refresh_token".to_string(), refresh_token), ]); let time_before_post: Instant = Instant::now(); let (token, new_refresh_token, _) = unwrap_token_response(post(&metadata.token_endpoint, &refresh_params).await); assert_eq!( new_refresh_token, None, "Refreshed token in {:?}, since start {:?}", time_before_post.elapsed(), time_first_token.elapsed() ); // Wait 1 second and make sure the access token expired tokio::time::sleep(Duration::from_secs(1)).await; assert_unauthorized("https://127.0.0.1:8899", &token).await; // Wait another second for the refresh token to be about to expire // and expect a new refresh token tokio::time::sleep(Duration::from_secs(1)).await; let (_, new_refresh_token, _) = unwrap_token_response(post(&metadata.token_endpoint, &refresh_params).await); //println!("New refresh token: {:?}", new_refresh_token); assert_ne!(new_refresh_token, None); // Wait another second and make sure the refresh token expired tokio::time::sleep(Duration::from_secs(1)).await; assert_eq!( post::(&metadata.token_endpoint, &refresh_params).await, TokenResponse::Error { error: ErrorType::InvalidGrant } ); // Destroy test accounts server .core .storage .lookup .purge_in_memory_store() .await .unwrap(); params.destroy_all_mailboxes(account).await; params.assert_is_empty().await; } async fn post_bytes( url: &str, auth_token: Option<&str>, params: &AHashMap, ) -> Bytes { let mut client = reqwest::Client::builder() .timeout(Duration::from_millis(500)) .danger_accept_invalid_certs(true) .build() .unwrap_or_default() .post(url); if let Some(auth_token) = auth_token { client = client.bearer_auth(auth_token); } client .form(params) .send() .await .unwrap() .bytes() .await .unwrap() } async fn post_json( url: &str, auth_token: Option<&str>, body: &impl Serialize, ) -> D { let mut client = reqwest::Client::builder() .timeout(Duration::from_millis(500)) .danger_accept_invalid_certs(true) .build() .unwrap_or_default() .post(url); if let Some(auth_token) = auth_token { client = client.bearer_auth(auth_token); } serde_json::from_slice( &client .body(serde_json::to_string(body).unwrap().into_bytes()) .send() .await .unwrap() .bytes() .await .unwrap(), ) .unwrap() } async fn post(url: &str, params: &AHashMap) -> T { post_with_auth(url, None, params).await } async fn post_with_auth( url: &str, auth_token: Option<&str>, params: &AHashMap, ) -> T { serde_json::from_slice(&post_bytes(url, auth_token, params).await).unwrap() } async fn get_bytes(url: &str) -> Bytes { reqwest::Client::builder() .timeout(Duration::from_millis(500)) .danger_accept_invalid_certs(true) .build() .unwrap_or_default() .get(url) .send() .await .unwrap() .bytes() .await .unwrap() } async fn get(url: &str) -> T { serde_json::from_slice(&get_bytes(url).await).unwrap() } async fn assert_unauthorized(base_url: &str, token: &str) { match Client::new() .credentials(Credentials::bearer(token)) .accept_invalid_certs(true) .follow_redirects(["127.0.0.1"]) .connect(base_url) .await { Ok(_) => panic!("Expected unauthorized access."), Err(err) => { let err = err.to_string(); assert!(err.contains("Unauthorized"), "{}", err); } } } fn unwrap_token_response(response: TokenResponse) -> (String, Option, u64) { match response { TokenResponse::Granted(granted) => { assert_eq!(granted.token_type, "bearer"); ( granted.access_token, granted.refresh_token, granted.expires_in, ) } TokenResponse::Error { error } => panic!("Expected granted, got {:?}", error), } } fn unwrap_oidc_token_response(response: TokenResponse) -> (String, Option, String) { match response { TokenResponse::Granted(granted) => { assert_eq!(granted.token_type, "bearer"); ( granted.access_token, granted.refresh_token, granted.id_token.unwrap(), ) } TokenResponse::Error { error } => panic!("Expected granted, got {:?}", error), } } ================================================ FILE: tests/src/jmap/auth/permissions.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ directory::internal::TestInternalDirectory, jmap::{JMAPTest, ManagementApi, server::List}, }; use ahash::AHashSet; use common::auth::{AccessToken, TenantInfo}; use directory::{ Permission, Type, backend::internal::{PrincipalField, PrincipalSet, PrincipalUpdate, PrincipalValue}, }; use email::message::delivery::{IngestMessage, IngestRecipient, LocalDeliveryStatus, MailDelivery}; use std::sync::Arc; pub async fn test(params: &JMAPTest) { println!("Running permissions tests..."); let server = params.server.clone(); // Disable spam filtering to avoid adding extra headers let old_core = params.server.core.clone(); let mut new_core = old_core.as_ref().clone(); new_core.spam.enabled = false; new_core.smtp.session.data.add_delivered_to = false; params.server.inner.shared_core.store(Arc::new(new_core)); // Remove unlimited requests permission for &account in params.accounts.keys() { params .server .store() .remove_permissions(account, [Permission::UnlimitedRequests]) .await; } // Prepare management API let api = ManagementApi::new(8899, "admin", "secret"); // Create a user with the default 'user' role let account_id = api .post::( "/api/principal", &PrincipalSet::new(u32::MAX, Type::Individual) .with_field(PrincipalField::Name, "role_player") .with_field(PrincipalField::Roles, vec!["user".to_string()]) .with_field( PrincipalField::DisabledPermissions, vec![Permission::Pop3Dele.name().to_string()], ), ) .await .unwrap() .unwrap_data(); let revision = server .get_access_token(account_id) .await .unwrap() .validate_permissions( Permission::all().filter(|p| p.is_user_permission() && *p != Permission::Pop3Dele), ) .revision; // Create multiple roles for (role, permissions, parent_role) in &[ ( "pop3_user", vec![Permission::Pop3Authenticate, Permission::Pop3List], vec![], ), ( "imap_user", vec![Permission::ImapAuthenticate, Permission::ImapList], vec![], ), ( "jmap_user", vec![ Permission::JmapEmailQuery, Permission::AuthenticateOauth, Permission::ManageEncryption, ], vec![], ), ( "email_user", vec![Permission::EmailSend, Permission::EmailReceive], vec!["pop3_user", "imap_user", "jmap_user"], ), ] { api.post::( "/api/principal", &PrincipalSet::new(u32::MAX, Type::Role) .with_field(PrincipalField::Name, role.to_string()) .with_field( PrincipalField::EnabledPermissions, permissions .iter() .map(|p| p.name().to_string()) .collect::>(), ) .with_field( PrincipalField::Roles, parent_role .iter() .map(|r| r.to_string()) .collect::>(), ), ) .await .unwrap() .unwrap_data(); } // Update email_user role api.patch::<()>( "/api/principal/email_user", &vec![PrincipalUpdate::add_item( PrincipalField::DisabledPermissions, PrincipalValue::String(Permission::ManageEncryption.name().to_string()), )], ) .await .unwrap() .unwrap_data(); // Update the user role to the nested 'email_user' role api.patch::<()>( "/api/principal/role_player", &vec![PrincipalUpdate::set( PrincipalField::Roles, PrincipalValue::StringList(vec!["email_user".to_string()]), )], ) .await .unwrap() .unwrap_data(); assert_ne!( server .get_access_token(account_id) .await .unwrap() .validate_permissions([ Permission::EmailSend, Permission::EmailReceive, Permission::JmapEmailQuery, Permission::AuthenticateOauth, Permission::ImapAuthenticate, Permission::ImapList, Permission::Pop3Authenticate, Permission::Pop3List, ]) .revision, revision ); // Query all principals api.get::>("/api/principal") .await .unwrap() .unwrap_data() .assert_count(12) .assert_exists( "admin", Type::Individual, [ (PrincipalField::Roles, &["admin"][..]), (PrincipalField::Members, &[][..]), (PrincipalField::EnabledPermissions, &[][..]), (PrincipalField::DisabledPermissions, &[][..]), ], ) .assert_exists( "role_player", Type::Individual, [ (PrincipalField::Roles, &["email_user"][..]), (PrincipalField::Members, &[][..]), (PrincipalField::EnabledPermissions, &[][..]), ( PrincipalField::DisabledPermissions, &[Permission::Pop3Dele.name()][..], ), ], ) .assert_exists( "email_user", Type::Role, [ ( PrincipalField::Roles, &["pop3_user", "imap_user", "jmap_user"][..], ), (PrincipalField::Members, &["role_player"][..]), ( PrincipalField::EnabledPermissions, &[ Permission::EmailReceive.name(), Permission::EmailSend.name(), ][..], ), ( PrincipalField::DisabledPermissions, &[Permission::ManageEncryption.name()][..], ), ], ) .assert_exists( "pop3_user", Type::Role, [ (PrincipalField::Roles, &[][..]), (PrincipalField::Members, &["email_user"][..]), ( PrincipalField::EnabledPermissions, &[ Permission::Pop3Authenticate.name(), Permission::Pop3List.name(), ][..], ), (PrincipalField::DisabledPermissions, &[][..]), ], ) .assert_exists( "imap_user", Type::Role, [ (PrincipalField::Roles, &[][..]), (PrincipalField::Members, &["email_user"][..]), ( PrincipalField::EnabledPermissions, &[ Permission::ImapAuthenticate.name(), Permission::ImapList.name(), ][..], ), (PrincipalField::DisabledPermissions, &[][..]), ], ) .assert_exists( "jmap_user", Type::Role, [ (PrincipalField::Roles, &[][..]), (PrincipalField::Members, &["email_user"][..]), ( PrincipalField::EnabledPermissions, &[ Permission::JmapEmailQuery.name(), Permission::AuthenticateOauth.name(), Permission::ManageEncryption.name(), ][..], ), (PrincipalField::DisabledPermissions, &[][..]), ], ); // Create new tenants let tenant_id = api .post::( "/api/principal", &PrincipalSet::new(u32::MAX, Type::Tenant) .with_field(PrincipalField::Name, "foobar") .with_field( PrincipalField::Roles, vec!["tenant-admin".to_string(), "user".to_string()], ) .with_field( PrincipalField::Quota, PrincipalValue::IntegerList(vec![TENANT_QUOTA, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]), ), ) .await .unwrap() .unwrap_data(); let other_tenant_id = api .post::( "/api/principal", &PrincipalSet::new(u32::MAX, Type::Tenant) .with_field(PrincipalField::Name, "xanadu") .with_field( PrincipalField::Roles, vec!["tenant-admin".to_string(), "user".to_string()], ), ) .await .unwrap() .unwrap_data(); // Creating a tenant without a valid domain should fail api.post::( "/api/principal", &PrincipalSet::new(u32::MAX, Type::Individual) .with_field(PrincipalField::Name, "admin-foobar") .with_field(PrincipalField::Roles, vec!["tenant-admin".to_string()]) .with_field( PrincipalField::Secrets, PrincipalValue::String("mytenantpass".to_string()), ) .with_field( PrincipalField::Tenant, PrincipalValue::String("foobar".to_string()), ), ) .await .unwrap() .expect_error("Principal name must include a valid domain assigned to the tenant"); // Create domain for the tenant and one outside the tenant api.post::( "/api/principal", &PrincipalSet::new(u32::MAX, Type::Domain) .with_field(PrincipalField::Name, "foobar.org") .with_field( PrincipalField::Tenant, PrincipalValue::String("foobar".to_string()), ), ) .await .unwrap() .unwrap_data(); api.post::( "/api/principal", &PrincipalSet::new(u32::MAX, Type::Domain).with_field(PrincipalField::Name, "example.org"), ) .await .unwrap() .unwrap_data(); // Create tenant admin let tenant_admin_id = api .post::( "/api/principal", &PrincipalSet::new(u32::MAX, Type::Individual) .with_field(PrincipalField::Name, "admin@foobar.org") .with_field(PrincipalField::Roles, vec!["tenant-admin".to_string()]) .with_field( PrincipalField::Secrets, PrincipalValue::String("mytenantpass".to_string()), ) .with_field( PrincipalField::Tenant, PrincipalValue::String("foobar".to_string()), ), ) .await .unwrap() .unwrap_data(); // Verify permissions server .get_access_token(tenant_admin_id) .await .unwrap() .validate_permissions(Permission::all().filter(|p| p.is_tenant_admin_permission())) .validate_tenant(tenant_id, TENANT_QUOTA); // Prepare tenant admin API let tenant_api = ManagementApi::new(8899, "admin@foobar.org", "mytenantpass"); // Tenant should not be able to create other tenants or modify its tenant id tenant_api .post::( "/api/principal", &PrincipalSet::new(u32::MAX, Type::Tenant) .with_field(PrincipalField::Name, "subfoobar"), ) .await .unwrap() .expect_request_error("Forbidden"); tenant_api .patch::<()>( "/api/principal/foobar", &vec![PrincipalUpdate::set( PrincipalField::Tenant, PrincipalValue::String("subfoobar".to_string()), )], ) .await .unwrap() .expect_request_error("Forbidden"); tenant_api .get::<()>("/api/principal/foobar") .await .unwrap() .expect_request_error("Forbidden"); tenant_api .get::<()>("/api/principal?type=tenant") .await .unwrap() .expect_request_error("Forbidden"); // Create a second domain for the tenant tenant_api .post::( "/api/principal", &PrincipalSet::new(u32::MAX, Type::Domain) .with_field(PrincipalField::Name, "foobar.com"), ) .await .unwrap() .unwrap_data(); // Creating a third domain should be limited by quota tenant_api .post::( "/api/principal", &PrincipalSet::new(u32::MAX, Type::Domain) .with_field(PrincipalField::Name, "foobar.net"), ) .await .unwrap() .expect_request_error("Tenant quota exceeded"); // Creating a tenant user without a valid domain or with a domain outside the tenant should fail for user in ["mytenantuser", "john@example.org"] { tenant_api .post::( "/api/principal", &PrincipalSet::new(u32::MAX, Type::Individual) .with_field(PrincipalField::Name, user.to_string()) .with_field(PrincipalField::Roles, vec!["tenant-admin".to_string()]), ) .await .unwrap() .expect_error("Principal name must include a valid domain assigned to the tenant"); } // Create an account let tenant_user_id = tenant_api .post::( "/api/principal", &PrincipalSet::new(u32::MAX, Type::Individual) .with_field(PrincipalField::Name, "john@foobar.org") .with_field( PrincipalField::Roles, vec!["tenant-admin".to_string(), "user".to_string()], ) .with_field( PrincipalField::Secrets, PrincipalValue::String("tenantpass".to_string()), ) .with_field( PrincipalField::Tenant, PrincipalValue::String("xanadu".to_string()), ), ) .await .unwrap() .unwrap_data(); // Although super user privileges were used and a different tenant name was provided, this should be ignored server .get_access_token(tenant_user_id) .await .unwrap() .validate_permissions( Permission::all().filter(|p| p.is_tenant_admin_permission() || p.is_user_permission()), ) .validate_tenant(tenant_id, TENANT_QUOTA); // Create a second account should be limited by quota tenant_api .post::( "/api/principal", &PrincipalSet::new(u32::MAX, Type::Individual) .with_field(PrincipalField::Name, "jane@foobar.org") .with_field(PrincipalField::Roles, vec!["tenant-admin".to_string()]), ) .await .unwrap() .expect_request_error("Tenant quota exceeded"); // Create an tenant role tenant_api .post::( "/api/principal", &PrincipalSet::new(u32::MAX, Type::Role) .with_field(PrincipalField::Name, "no-mail-for-you@foobar.com") .with_field( PrincipalField::DisabledPermissions, vec![Permission::EmailReceive.name().to_string()], ), ) .await .unwrap() .unwrap_data(); // Assigning a role that does not belong to the tenant should fail tenant_api .patch::<()>( "/api/principal/john@foobar.org", &vec![PrincipalUpdate::add_item( PrincipalField::Roles, PrincipalValue::String("imap_user".to_string()), )], ) .await .unwrap() .expect_error("notFound"); // Add tenant defined role tenant_api .patch::<()>( "/api/principal/john@foobar.org", &vec![PrincipalUpdate::add_item( PrincipalField::Roles, PrincipalValue::String("no-mail-for-you@foobar.com".to_string()), )], ) .await .unwrap() .unwrap_data(); // Check updated permissions server .get_access_token(tenant_user_id) .await .unwrap() .validate_permissions(Permission::all().filter(|p| { (p.is_tenant_admin_permission() || p.is_user_permission()) && *p != Permission::EmailReceive })); // Changing the tenant of a user should fail tenant_api .patch::<()>( "/api/principal/john@foobar.org", &vec![PrincipalUpdate::set( PrincipalField::Tenant, PrincipalValue::String("xanadu".to_string()), )], ) .await .unwrap() .expect_request_error("Forbidden"); // Renaming a tenant account without a valid domain should fail for user in ["john", "john@example.org"] { tenant_api .patch::<()>( "/api/principal/john@foobar.org", &vec![PrincipalUpdate::set( PrincipalField::Name, PrincipalValue::String(user.to_string()), )], ) .await .unwrap() .expect_error("Principal name must include a valid domain assigned to the tenant"); } // Rename the tenant account and add an email address tenant_api .patch::<()>( "/api/principal/john@foobar.org", &vec![ PrincipalUpdate::set( PrincipalField::Name, PrincipalValue::String("john.doe@foobar.org".to_string()), ), PrincipalUpdate::add_item( PrincipalField::Emails, PrincipalValue::String("john@foobar.org".to_string()), ), ], ) .await .unwrap() .unwrap_data(); // Tenants should only see their own principals tenant_api .get::>("/api/principal?types=individual,group,role,list") .await .unwrap() .unwrap_data() .assert_count(3) .assert_exists( "admin@foobar.org", Type::Individual, [ (PrincipalField::Roles, &["tenant-admin"][..]), (PrincipalField::Members, &[][..]), (PrincipalField::EnabledPermissions, &[][..]), (PrincipalField::DisabledPermissions, &[][..]), ], ) .assert_exists( "john.doe@foobar.org", Type::Individual, [ ( PrincipalField::Roles, &["tenant-admin", "no-mail-for-you@foobar.com", "user"][..], ), (PrincipalField::Members, &[][..]), (PrincipalField::EnabledPermissions, &[][..]), (PrincipalField::DisabledPermissions, &[][..]), ], ) .assert_exists( "no-mail-for-you@foobar.com", Type::Role, [ (PrincipalField::Roles, &[][..]), (PrincipalField::Members, &["john.doe@foobar.org"][..]), (PrincipalField::EnabledPermissions, &[][..]), ( PrincipalField::DisabledPermissions, &[Permission::EmailReceive.name()][..], ), ], ); // John should not be allowed to receive email let (message_blob, _) = server .put_temporary_blob(tenant_user_id, TEST_MESSAGE.as_bytes(), 60) .await .unwrap(); assert_eq!( server .deliver_message(IngestMessage { sender_address: "bill@foobar.org".to_string(), sender_authenticated: true, recipients: vec![IngestRecipient { address: "john@foobar.org".to_string(), is_spam: false }], message_blob: message_blob.clone(), message_size: TEST_MESSAGE.len() as u64, session_id: 0, }) .await .status, vec![LocalDeliveryStatus::PermanentFailure { code: [5, 5, 0], reason: "This account is not authorized to receive email.".into() }] ); // Remove the restriction tenant_api .patch::<()>( "/api/principal/john.doe@foobar.org", &vec![PrincipalUpdate::remove_item( PrincipalField::Roles, PrincipalValue::String("no-mail-for-you@foobar.com".to_string()), )], ) .await .unwrap() .unwrap_data(); server .get_access_token(tenant_user_id) .await .unwrap() .validate_permissions( Permission::all().filter(|p| p.is_tenant_admin_permission() || p.is_user_permission()), ); // Delivery should now succeed assert_eq!( server .deliver_message(IngestMessage { sender_address: "bill@foobar.org".to_string(), sender_authenticated: true, recipients: vec![IngestRecipient { address: "john@foobar.org".to_string(), is_spam: false }], message_blob: message_blob.clone(), message_size: TEST_MESSAGE.len() as u64, session_id: 0, }) .await .status, vec![LocalDeliveryStatus::Success] ); // Quota for the tenant and user should be updated const EXTRA_BYTES: i64 = 19; // Storage overhead assert_eq!( server.get_used_quota(tenant_id).await.unwrap(), TEST_MESSAGE.len() as i64 + EXTRA_BYTES ); assert_eq!( server.get_used_quota(tenant_user_id).await.unwrap(), TEST_MESSAGE.len() as i64 + EXTRA_BYTES ); // Next delivery should fail due to tenant quota assert_eq!( server .deliver_message(IngestMessage { sender_address: "bill@foobar.org".to_string(), sender_authenticated: true, recipients: vec![IngestRecipient { address: "john@foobar.org".to_string(), is_spam: false }], message_blob, message_size: TEST_MESSAGE.len() as u64, session_id: 0, }) .await .status, vec![LocalDeliveryStatus::TemporaryFailure { reason: "Organization over quota.".into() }] ); // Moving a user to another tenant should move its quota too api.patch::<()>( "/api/principal/john.doe@foobar.org", &vec![PrincipalUpdate::set( PrincipalField::Tenant, PrincipalValue::String("xanadu".to_string()), )], ) .await .unwrap() .unwrap_data(); assert_eq!(server.get_used_quota(tenant_id).await.unwrap(), 0); assert_eq!( server.get_used_quota(other_tenant_id).await.unwrap(), TEST_MESSAGE.len() as i64 + EXTRA_BYTES ); // Deleting tenants with data should fail api.delete::<()>("/api/principal/xanadu") .await .unwrap() .expect_error("Tenant has members"); // Delete user api.delete::<()>("/api/principal/john.doe@foobar.org") .await .unwrap() .unwrap_data(); // Quota usage for tenant should be updated assert_eq!(server.get_used_quota(other_tenant_id).await.unwrap(), 0); // Delete tenant api.delete::<()>("/api/principal/xanadu") .await .unwrap() .unwrap_data(); // Delete tenant information for query in [ "/api/principal/no-mail-for-you@foobar.com", "/api/principal/admin@foobar.org", "/api/principal/foobar.org", "/api/principal/foobar.com", ] { api.delete::<()>(query).await.unwrap().unwrap_data(); } // Delete tenant api.delete::<()>("/api/principal/foobar") .await .unwrap() .unwrap_data(); server .core .storage .config .clear("report.domain") .await .unwrap(); params.assert_is_empty().await; } const TENANT_QUOTA: u64 = TEST_MESSAGE.len() as u64; const TEST_MESSAGE: &str = concat!( "From: bill@foobar.org\r\n", "To: jdoe@foobar.com\r\n", "Subject: TPS Report\r\n", "\r\n", "I'm going to need those TPS reports ASAP. ", "So, if you could do that, that'd be great." ); trait ValidatePrincipalList { fn assert_exists<'x>( self, name: &str, typ: Type, items: impl IntoIterator, ) -> Self; fn assert_count(self, count: usize) -> Self; } impl ValidatePrincipalList for List { fn assert_exists<'x>( self, name: &str, typ: Type, items: impl IntoIterator, ) -> Self { for item in &self.items { if item.name() == name { item.validate(typ, items); return self; } } panic!("Principal not found: {}", name); } fn assert_count(self, count: usize) -> Self { assert_eq!(self.items.len(), count, "Principal count failed validation"); assert_eq!(self.total, count, "Principal total failed validation"); self } } trait ValidatePrincipal { fn validate<'x>( &self, typ: Type, items: impl IntoIterator, ); } impl ValidatePrincipal for PrincipalSet { fn validate<'x>( &self, typ: Type, items: impl IntoIterator, ) { assert_eq!(self.typ(), typ, "Type failed validation"); for (field, values) in items { match ( self.get_str_array(field).filter(|v| !v.is_empty()), (!values.is_empty()).then_some(values), ) { (Some(values), Some(expected)) => { assert_eq!( values.iter().map(|s| s.as_str()).collect::>(), expected.iter().copied().collect::>(), "Field {field:?} failed validation: {values:?} != {expected:?}" ); } (None, None) => {} (values, expected) => { panic!("Field {field:?} failed validation: {values:?} != {expected:?}"); } } } } } trait ValidatePermissions { fn validate_permissions( self, expected_permissions: impl IntoIterator, ) -> Self; fn validate_tenant(self, tenant_id: u32, tenant_quota: u64) -> Self; } impl ValidatePermissions for Arc { fn validate_permissions( self, expected_permissions: impl IntoIterator, ) -> Self { let expected_permissions: AHashSet<_> = expected_permissions.into_iter().collect(); let permissions = self.permissions(); for permission in &permissions { assert!( expected_permissions.contains(permission), "Permission {:?} failed validation", permission ); } assert_eq!( permissions.into_iter().collect::>(), expected_permissions ); for permission in Permission::all() { if self.has_permission(permission) { assert!( expected_permissions.contains(&permission), "Permission {:?} failed validation", permission ); } } self } fn validate_tenant(self, tenant_id: u32, tenant_quota: u64) -> Self { assert_eq!( self.tenant, Some(TenantInfo { id: tenant_id, quota: tenant_quota }) ); self } } ================================================ FILE: tests/src/jmap/auth/quota.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ directory::internal::TestInternalDirectory, jmap::{JMAPTest, mail::delivery::SmtpConnection, wait_for_index}, smtp::queue::QueuedEvents, store::cleanup::store_blob_expire_all, }; use common::config::smtp::queue::QueueName; use email::{cache::MessageCacheFetch, mailbox::INBOX_ID}; use http::management::stores::recalculate_quota; use jmap::blob::upload::DISABLE_UPLOAD_QUOTA; use jmap_client::{ core::set::{SetErrorType, SetObject}, email::EmailBodyPart, }; use serde_json::json; use smtp::queue::spool::SmtpSpool; use types::id::Id; pub async fn test(params: &mut JMAPTest) { println!("Running quota tests..."); let server = params.server.clone(); let account = params.account("robert@example.com"); let other_account = params.account("jdoe@example.com"); server .core .storage .data .set_test_quota("robert@example.com", 1024) .await; server .core .storage .data .add_to_group("robert@example.com", "jdoe@example.com") .await; server.inner.cache.access_tokens.clear(); // Delete temporary blobs from previous tests store_blob_expire_all(&server.core.storage.data).await; // Test temporary blob quota (3 files) DISABLE_UPLOAD_QUOTA.store(false, std::sync::atomic::Ordering::Relaxed); let client = account.client(); for i in 0..3 { assert_eq!( client .upload(None, vec![b'A' + i; 1024], None) .await .unwrap() .size(), 1024 ); } match client .upload(None, vec![b'Z'; 1024], None) .await .unwrap_err() { jmap_client::Error::Problem(err) if err.detail().unwrap().contains("quota") => (), other => panic!("Unexpected error: {:?}", other), } store_blob_expire_all(&server.core.storage.data).await; // Test temporary blob quota (50000 bytes) for i in 0..2 { assert_eq!( client .upload(None, vec![b'a' + i; 25000], None) .await .unwrap() .size(), 25000 ); } match client .upload(None, vec![b'z'; 1024], None) .await .unwrap_err() { jmap_client::Error::Problem(err) if err.detail().unwrap().contains("quota") => (), other => panic!("Unexpected error: {:?}", other), } store_blob_expire_all(&server.core.storage.data).await; // Test JMAP Quotas extension let response = account .jmap_method_call( "Quota/get", json!({ "ids": null }), ) .await .to_string(); assert!(response.contains("\"used\":0"), "{}", response); assert!(response.contains("\"hardLimit\":1024"), "{}", response); assert!(response.contains("\"scope\":\"account\""), "{}", response); assert!( response.contains("\"name\":\"robert@example.com\""), "{}", response ); // Test Email/import quota let inbox_id = Id::new(INBOX_ID as u64).to_string(); let mut message_ids = Vec::new(); for i in 0..2 { message_ids.push( client .email_import( create_message_with_size( "jdoe@example.com", "robert@example.com", &format!("Test {i}"), 512, ), vec![&inbox_id], None::>, None, ) .await .unwrap() .take_id(), ); } assert_over_quota( client .email_import( create_message_with_size("test@example.com", "jdoe@example.com", "Test 3", 100), vec![&inbox_id], None::>, None, ) .await, ); // Test JMAP Quotas extension let response = account .jmap_method_call( "Quota/get", json!({ "ids": null }), ) .await .to_string(); assert!(response.contains("\"used\":1024"), "{}", response); assert!(response.contains("\"hardLimit\":1024"), "{}", response); // Delete messages and check available quota for message_id in message_ids { client.email_destroy(&message_id).await.unwrap(); } // Wait for pending index tasks wait_for_index(&server).await; assert_eq!( server .get_used_quota(account.id().document_id()) .await .unwrap(), 0 ); // Test Email/set quota let mut message_ids = Vec::new(); for i in 0..2 { let mut request = client.build(); let create_item = request.set_email().create(); create_item .mailbox_ids([&inbox_id]) .subject(format!("Test {i}")) .from(["jdoe@example.com"]) .to(["robert@example.com"]) .body_value("a".to_string(), String::from_utf8(vec![b'A'; 200]).unwrap()) .text_body(EmailBodyPart::new().part_id("a")); let create_id = create_item.create_id().unwrap(); message_ids.push( request .send_set_email() .await .unwrap() .created(&create_id) .unwrap() .take_id(), ); } let mut request = client.build(); let create_item = request.set_email().create(); create_item .mailbox_ids([&inbox_id]) .subject("Test 3") .from(["jdoe@example.com"]) .to(["robert@example.com"]) .body_value("a".to_string(), String::from_utf8(vec![b'A'; 400]).unwrap()) .text_body(EmailBodyPart::new().part_id("a")); let create_id = create_item.create_id().unwrap(); assert_over_quota(request.send_set_email().await.unwrap().created(&create_id)); // Recalculate quota let prev_quota = server .get_used_quota(account.id().document_id()) .await .unwrap(); recalculate_quota(&server, account.id().document_id()) .await .unwrap(); assert_eq!( server .get_used_quota(account.id().document_id()) .await .unwrap(), prev_quota ); // Delete messages and check available quota for message_id in message_ids { client.email_destroy(&message_id).await.unwrap(); } // Wait for pending index tasks wait_for_index(&server).await; assert_eq!( server .get_used_quota(account.id().document_id()) .await .unwrap(), 0 ); // Test Email/copy quota let other_client = other_account.client(); let mut other_message_ids = Vec::new(); let mut message_ids = Vec::new(); for i in 0..3 { other_message_ids.push( other_client .email_import( create_message_with_size( "jane@example.com", "jdoe@example.com", &format!("Other Test {i}"), 512, ), vec![&inbox_id], None::>, None, ) .await .unwrap() .take_id(), ); } for id in other_message_ids.iter().take(2) { message_ids.push( client .email_copy( other_account.id_string(), id, vec![&inbox_id], None::>, None, ) .await .unwrap() .take_id(), ); } assert_over_quota( client .email_copy( other_account.id_string(), &other_message_ids[2], vec![&inbox_id], None::>, None, ) .await, ); // Delete messages and check available quota for message_id in message_ids { client.email_destroy(&message_id).await.unwrap(); } // Wait for pending index tasks wait_for_index(&server).await; assert_eq!( server .get_used_quota(account.id().document_id()) .await .unwrap(), 0 ); // Test delivery quota let mut lmtp = SmtpConnection::connect().await; for i in 0..2 { lmtp.ingest( "jane@example.com", &["robert@example.com"], &String::from_utf8(create_message_with_size( "jane@example.com", "robert@example.com", &format!("Ingest test {i}"), 513, )) .unwrap(), ) .await; } let quota = server .get_used_quota(account.id().document_id()) .await .unwrap(); assert!(quota > 0 && quota <= 1024, "Quota is {}", quota); assert_eq!( server .get_cached_messages(account.id().document_id()) .await .unwrap() .emails .items .len(), 1, ); DISABLE_UPLOAD_QUOTA.store(true, std::sync::atomic::Ordering::Relaxed); // Remove test data params.destroy_all_mailboxes(account).await; params.destroy_all_mailboxes(other_account).await; for event in server.all_queued_messages().await.messages { server .read_message(event.queue_id, QueueName::default()) .await .unwrap() .remove(&server, event.due.into()) .await; } params.assert_is_empty().await; } fn assert_over_quota(result: Result) { match result { Ok(result) => panic!("Expected error, got {:?}", result), Err(jmap_client::Error::Set(err)) if err.error() == &SetErrorType::OverQuota => (), Err(err) => panic!("Expected OverQuota SetError, got {:?}", err), } } fn create_message_with_size(from: &str, to: &str, subject: &str, size: usize) -> Vec { let mut message = format!( "From: {}\r\nTo: {}\r\nSubject: {}\r\n\r\n", from, to, subject ); for _ in 0..size - message.len() { message.push('A'); } message.into_bytes() } ================================================ FILE: tests/src/jmap/calendar/acl.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::{JMAPTest, JmapUtils}; use calcard::jscalendar::JSCalendarProperty; use jmap_proto::{ object::{calendar::CalendarProperty, share_notification::ShareNotificationProperty}, request::method::MethodObject, }; use serde_json::json; use types::id::Id; pub async fn test(params: &mut JMAPTest) { println!("Running Calendar ACL tests..."); let john = params.account("jdoe@example.com"); let jane = params.account("jane.smith@example.com"); let john_id = john.id_string().to_string(); let jane_id = jane.id_string().to_string(); // Create test calendars let response = john .jmap_create( MethodObject::Calendar, [json!({ "name": "Test #1", })], Vec::<(&str, &str)>::new(), ) .await; let john_calendar_id = response.created(0).id().to_string(); let john_event_id = john .jmap_create( MethodObject::CalendarEvent, [json!({ "@type": "Event", "uid": "a8df6573-0474-496d-8496-033ad45d7fea", "updated": "2020-01-02T18:23:04Z", "title": "John's Simple Event", "start": "2020-01-15T13:00:00", "timeZone": "America/New_York", "duration": "PT1H", "calendarIds": { &john_calendar_id: true }, })], Vec::<(&str, &str)>::new(), ) .await .created(0) .id() .to_string(); let response = jane .jmap_create( MethodObject::Calendar, [json!({ "name": "Test #1", })], Vec::<(&str, &str)>::new(), ) .await; let jane_calendar_id = response.created(0).id().to_string(); let jane_event_id = jane .jmap_create( MethodObject::CalendarEvent, [json!({ "uid": "a8df6575-0474-496d-8496-033ad45d7fea", "updated": "2020-01-02T18:23:04Z", "title": "Jane's Simple Event", "start": "2020-01-15T13:00:00", "timeZone": "America/New_York", "duration": "PT1H", "calendarIds": { &jane_calendar_id: true }, })], Vec::<(&str, &str)>::new(), ) .await .created(0) .id() .to_string(); // Verify myRights john.jmap_get( MethodObject::Calendar, [ CalendarProperty::Id, CalendarProperty::Name, CalendarProperty::MyRights, CalendarProperty::ShareWith, ], [john_calendar_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_calendar_id, "name": "Test #1", "myRights": { "mayReadItems": true, "mayWriteAll": true, "mayDelete": true, "mayShare": true, "mayWriteOwn": true, "mayReadFreeBusy": true, "mayUpdatePrivate": true, "mayRSVP": true }, "shareWith": {} })); // Obtain share notifications let mut jane_share_change_id = jane .jmap_get( MethodObject::ShareNotification, Vec::<&str>::new(), Vec::<&str>::new(), ) .await .state() .to_string(); // Make sure Jane has no access assert_eq!( jane.jmap_get_account( john, MethodObject::Calendar, Vec::<&str>::new(), [john_calendar_id.as_str()], ) .await .method_response() .typ(), "forbidden" ); // Share calendar with Jane john.jmap_update( MethodObject::Calendar, [( &john_calendar_id, json!({ "shareWith": { &jane_id : { "mayReadItems": true, } } }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&john_calendar_id); john.jmap_get( MethodObject::Calendar, [ CalendarProperty::Id, CalendarProperty::Name, CalendarProperty::ShareWith, ], [john_calendar_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_calendar_id, "name": "Test #1", "shareWith": { &jane_id : { "mayReadItems": true, "mayWriteAll": false, "mayDelete": false, "mayShare": false, "mayWriteOwn": false, "mayReadFreeBusy": false, "mayUpdatePrivate": false, "mayRSVP": false } } })); // Verify Jane can access the event jane.jmap_get_account( john, MethodObject::Calendar, [ CalendarProperty::Id, CalendarProperty::Name, CalendarProperty::MyRights, ], [john_calendar_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_calendar_id, "name": "Test #1", "myRights": { "mayReadItems": true, "mayWriteAll": false, "mayDelete": false, "mayShare": false, "mayWriteOwn": false, "mayReadFreeBusy": false, "mayUpdatePrivate": false, "mayRSVP": false } })); jane.jmap_get_account( john, MethodObject::CalendarEvent, [JSCalendarProperty::::Id, JSCalendarProperty::Title], [john_event_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_event_id, "title": "John's Simple Event", })); // Verify Jane received a share notification let response = jane .jmap_changes(MethodObject::ShareNotification, &jane_share_change_id) .await; jane_share_change_id = response.new_state().to_string(); let changes = response.changes().collect::>(); assert_eq!(changes.len(), 1); let share_id = changes[0].as_created(); jane.jmap_get( MethodObject::ShareNotification, [ ShareNotificationProperty::Id, ShareNotificationProperty::ChangedBy, ShareNotificationProperty::ObjectType, ShareNotificationProperty::ObjectAccountId, ShareNotificationProperty::ObjectId, ShareNotificationProperty::OldRights, ShareNotificationProperty::NewRights, ShareNotificationProperty::Name, ], [share_id], ) .await .list()[0] .assert_is_equal(json!({ "id": &share_id, "changedBy": { "principalId": &john_id, "name": "John Doe", "email": "jdoe@example.com" }, "objectType": "Calendar", "objectAccountId": &john_id, "objectId": &john_calendar_id, "oldRights": { "mayReadItems": false, "mayWriteAll": false, "mayDelete": false, "mayShare": false, "mayWriteOwn": false, "mayReadFreeBusy": false, "mayUpdatePrivate": false, "mayRSVP": false }, "newRights": { "mayReadItems": true, "mayWriteAll": false, "mayDelete": false, "mayShare": false, "mayWriteOwn": false, "mayReadFreeBusy": false, "mayUpdatePrivate": false, "mayRSVP": false }, "name": null })); // Updating and deleting should fail assert_eq!( jane.jmap_update_account( john, MethodObject::Calendar, [(&john_calendar_id, json!({}))], Vec::<(&str, &str)>::new(), ) .await .not_updated(&john_calendar_id) .description(), "You are not allowed to modify this calendar." ); assert_eq!( jane.jmap_destroy_account( john, MethodObject::Calendar, [&john_calendar_id], Vec::<(&str, &str)>::new(), ) .await .not_destroyed(&john_calendar_id) .description(), "You are not allowed to delete this calendar." ); assert!( jane.jmap_update_account( john, MethodObject::CalendarEvent, [(&john_event_id, json!({}))], Vec::<(&str, &str)>::new(), ) .await .not_updated(&john_event_id) .description() .contains("You are not allowed to modify calendar"), ); assert!( jane.jmap_destroy_account( john, MethodObject::CalendarEvent, [&john_event_id], Vec::<(&str, &str)>::new(), ) .await .not_destroyed(&john_event_id) .description() .contains("You are not allowed to remove events from calendar"), ); // Grant Jane write access john.jmap_update( MethodObject::Calendar, [( &john_calendar_id, json!({ format!("shareWith/{jane_id}/mayWriteAll"): true, format!("shareWith/{jane_id}/mayDelete"): true, }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&john_calendar_id); jane.jmap_get_account( john, MethodObject::Calendar, [ CalendarProperty::Id, CalendarProperty::Name, CalendarProperty::MyRights, ], [john_calendar_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_calendar_id, "name": "Test #1", "myRights": { "mayReadItems": true, "mayWriteAll": true, "mayDelete": true, "mayShare": false, "mayWriteOwn": false, "mayReadFreeBusy": false, "mayUpdatePrivate": false, "mayRSVP": false } })); // Verify Jane received a share notification with the updated rights let response = jane .jmap_changes(MethodObject::ShareNotification, &jane_share_change_id) .await; jane_share_change_id = response.new_state().to_string(); let changes = response.changes().collect::>(); assert_eq!(changes.len(), 1); let share_id = changes[0].as_created(); jane.jmap_get( MethodObject::ShareNotification, [ ShareNotificationProperty::Id, ShareNotificationProperty::ChangedBy, ShareNotificationProperty::ObjectType, ShareNotificationProperty::ObjectAccountId, ShareNotificationProperty::ObjectId, ShareNotificationProperty::OldRights, ShareNotificationProperty::NewRights, ShareNotificationProperty::Name, ], [share_id], ) .await .list()[0] .assert_is_equal(json!({ "id": &share_id, "changedBy": { "principalId": &john_id, "name": "John Doe", "email": "jdoe@example.com" }, "objectType": "Calendar", "objectAccountId": &john_id, "objectId": &john_calendar_id, "oldRights": { "mayReadItems": true, "mayWriteAll": false, "mayDelete": false, "mayShare": false, "mayWriteOwn": false, "mayReadFreeBusy": false, "mayUpdatePrivate": false, "mayRSVP": false }, "newRights": { "mayReadItems": true, "mayWriteAll": true, "mayDelete": true, "mayShare": false, "mayWriteOwn": false, "mayReadFreeBusy": false, "mayUpdatePrivate": false, "mayRSVP": false }, "name": null })); // Creating a root folder should fail assert_eq!( jane.jmap_create_account( john, MethodObject::Calendar, [json!({ "name": "A new shared calendar", })], Vec::<(&str, &str)>::new() ) .await .not_created(0) .description(), "Cannot create calendars in a shared account." ); // Copy Jane's event into John's calendar let john_copied_event_id = jane .jmap_copy( jane, john, MethodObject::CalendarEvent, [( &jane_event_id, json!({ "calendarIds": { &john_calendar_id: true } }), )], false, ) .await .copied(&jane_event_id) .id() .to_string(); jane.jmap_get_account( john, MethodObject::CalendarEvent, [ JSCalendarProperty::::Id, JSCalendarProperty::CalendarIds, JSCalendarProperty::Title, ], [john_copied_event_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_copied_event_id, "title": "Jane's Simple Event", "calendarIds": { &john_calendar_id: true } })); // Destroy the copied event assert_eq!( jane.jmap_destroy_account( john, MethodObject::CalendarEvent, [john_copied_event_id.as_str()], Vec::<(&str, &str)>::new(), ) .await .destroyed() .collect::>(), [&john_copied_event_id] ); // Update John's event jane.jmap_update_account( john, MethodObject::CalendarEvent, [( &john_event_id, json!({ "title": "John's Updated Event", }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&john_event_id); jane.jmap_get_account( john, MethodObject::CalendarEvent, [JSCalendarProperty::::Id, JSCalendarProperty::Title], [john_event_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_event_id, "title": "John's Updated Event", })); // Update John's calendar name jane.jmap_update_account( john, MethodObject::Calendar, [( &john_calendar_id, json!({ "name": "Jane's version of John's Calendar", "description": "This is John's calendar, but Jane can edit it now" }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&john_calendar_id); jane.jmap_get_account( john, MethodObject::Calendar, [ CalendarProperty::Id, CalendarProperty::Name, CalendarProperty::Description, ], [john_calendar_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_calendar_id, "name": "Jane's version of John's Calendar", "description": "This is John's calendar, but Jane can edit it now" })); // John should still see the old name john.jmap_get( MethodObject::Calendar, [ CalendarProperty::Id, CalendarProperty::Name, CalendarProperty::Description, ], [john_calendar_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_calendar_id, "name": "Test #1", "description": null })); // Revoke Jane's access john.jmap_update( MethodObject::Calendar, [( &john_calendar_id, json!({ format!("shareWith/{jane_id}"): () }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&john_calendar_id); john.jmap_get( MethodObject::Calendar, [ CalendarProperty::Id, CalendarProperty::Name, CalendarProperty::ShareWith, ], [john_calendar_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_calendar_id, "name": "Test #1", "shareWith": {} })); // Verify Jane can no longer access the calendar or its events assert_eq!( jane.jmap_get_account( john, MethodObject::Calendar, Vec::<&str>::new(), [john_calendar_id.as_str()], ) .await .method_response() .typ(), "forbidden" ); // Verify Jane received a share notification with the updated rights let response = jane .jmap_changes(MethodObject::ShareNotification, &jane_share_change_id) .await; let changes = response.changes().collect::>(); assert_eq!(changes.len(), 1); let share_id = changes[0].as_created(); jane.jmap_get( MethodObject::ShareNotification, [ ShareNotificationProperty::Id, ShareNotificationProperty::ChangedBy, ShareNotificationProperty::ObjectType, ShareNotificationProperty::ObjectAccountId, ShareNotificationProperty::ObjectId, ShareNotificationProperty::OldRights, ShareNotificationProperty::NewRights, ShareNotificationProperty::Name, ], [share_id], ) .await .list()[0] .assert_is_equal(json!({ "id": &share_id, "changedBy": { "principalId": &john_id, "name": "John Doe", "email": "jdoe@example.com" }, "objectType": "Calendar", "objectAccountId": &john_id, "objectId": &john_calendar_id, "oldRights": { "mayReadItems": true, "mayWriteAll": true, "mayDelete": true, "mayShare": false, "mayWriteOwn": false, "mayReadFreeBusy": false, "mayUpdatePrivate": false, "mayRSVP": false }, "newRights": { "mayReadItems": false, "mayWriteAll": false, "mayDelete": false, "mayShare": false, "mayWriteOwn": false, "mayReadFreeBusy": false, "mayUpdatePrivate": false, "mayRSVP": false }, "name": null })); // Grant Jane delete access once again john.jmap_update( MethodObject::Calendar, [( &john_calendar_id, json!({ format!("shareWith/{jane_id}/mayReadItems"): true, format!("shareWith/{jane_id}/mayDelete"): true, }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&john_calendar_id); // Verify Jane can delete the calendar assert_eq!( jane.jmap_destroy_account( john, MethodObject::Calendar, [john_calendar_id.as_str()], [("onDestroyRemoveEvents", true)], ) .await .destroyed() .collect::>(), [john_calendar_id.as_str()] ); // Destroy all mailboxes john.destroy_all_calendars().await; jane.destroy_all_calendars().await; params.assert_is_empty().await; } ================================================ FILE: tests/src/jmap/calendar/alarm.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use futures::StreamExt; use jmap_client::{ CalendarAlert, PushObject, client_ws::WebSocketMessage, event_source::PushNotification, }; use jmap_proto::request::method::MethodObject; use mail_parser::DateTime; use serde_json::json; use std::time::Instant; use store::write::now; use tokio::sync::mpsc; use crate::jmap::{IntoJmapSet, JMAPTest, JmapUtils}; pub async fn test(params: &mut JMAPTest) { println!("Running Calendar Alarm tests..."); let account = params.account("jdoe@example.com"); let account_id = account.id_string(); let client = account.client(); let client_ws = account.client_owned().await; // Create test calendar let response = account .jmap_create( MethodObject::Calendar, [json!({ "name": "Alarming Calendar", })], Vec::<(&str, &str)>::new(), ) .await; let calendar_id = response.created(0).id().to_string(); // Connect to EventSource let (event_tx, mut event_rx) = mpsc::channel::(100); let mut notifications = client .event_source(None::>, false, 1.into(), None) .await .unwrap(); tokio::spawn(async move { while let Some(notification) = notifications.next().await { if let Err(_err) = event_tx.send(notification.unwrap()).await { break; } } }); // Connect to WebSocket let mut ws_stream = client_ws.connect_ws().await.unwrap(); let (stream_tx, mut stream_rx) = mpsc::channel::(100); tokio::spawn(async move { while let Some(change) = ws_stream.next().await { if stream_tx.send(change.unwrap()).await.is_err() { break; } } }); client_ws .enable_push_ws(None::>, None::<&str>) .await .unwrap(); // Create test event let response = account .jmap_create( MethodObject::CalendarEvent, [json!({ "@type": "Event", "calendarIds": ([calendar_id.as_str()].into_jmap_set()), "description": "What mirror where?!", "timeZone": "Etc/UTC", "start": DateTime::from_timestamp(now() as i64 + 5) .to_rfc3339().trim_end_matches("Z").to_string(), "title": "See the pretty girl in that mirror there", "alerts": { "k1": { "@type": "Alert", "trigger": { "@type": "OffsetTrigger", "offset": "-PT2S" }, "action": "display" }, "k2": { "trigger": { "@type": "OffsetTrigger", "offset": "-PT4S" }, "action": "display", "@type": "Alert" } }, "locations": { "0b7168ae-ed3e-5eae-9540-89ba3a469b16": { "name": "West Side", "@type": "Location" } }, "uid": "2371c2d9-a136-43b0-bba3-f6ab249ad46e", "duration": "P1D" })], Vec::<(&str, &str)>::new(), ) .await; let event_id = response.created(0).id().to_string(); // Wait for alarm notifications let start = Instant::now(); let mut ws_events = Vec::new(); let mut es_events = Vec::new(); while start.elapsed().as_secs() < 7 && (ws_events.len() < 2 || es_events.len() < 2) { tokio::select! { Some(notification) = event_rx.recv() => { if let PushNotification::CalendarAlert(alert) = notification { es_events.push(alert); } } Some(message) = stream_rx.recv() => { match message { WebSocketMessage::PushNotification(PushObject::CalendarAlert(alert)) => { ws_events.push(alert); } WebSocketMessage::PushNotification(PushObject::Group {entries} ) => { ws_events.extend(entries.into_iter().filter_map(|entry| { if let PushObject::CalendarAlert(alert) = entry { Some(alert) } else { None } })); } _ => {} } } _ = tokio::time::sleep(std::time::Duration::from_secs(6)) => { break; } } } let expected_alerts = vec![ CalendarAlert { account_id: account_id.to_string(), calendar_event_id: event_id.clone(), uid: "2371c2d9-a136-43b0-bba3-f6ab249ad46e".to_string(), recurrence_id: None, alert_id: "k2".to_string(), }, CalendarAlert { account_id: account_id.to_string(), calendar_event_id: event_id.clone(), uid: "2371c2d9-a136-43b0-bba3-f6ab249ad46e".to_string(), recurrence_id: None, alert_id: "k1".to_string(), }, ]; assert_eq!( es_events, expected_alerts, "EventSource alarms do not match" ); assert_eq!(ws_events, expected_alerts, "WebSocket alarms do not match"); // Cleanup account.destroy_all_calendars().await; params.assert_is_empty().await; } ================================================ FILE: tests/src/jmap/calendar/calendars.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::{ChangeType, JMAPTest, JmapUtils}; use jmap_proto::{object::calendar::CalendarProperty, request::method::MethodObject}; use serde_json::json; pub async fn test(params: &mut JMAPTest) { println!("Running Calendar tests..."); let account = params.account("jdoe@example.com"); // Make sure the default calendar exists let response = account .jmap_get( MethodObject::Calendar, [ CalendarProperty::Id, CalendarProperty::Name, CalendarProperty::Description, CalendarProperty::SortOrder, CalendarProperty::Color, CalendarProperty::TimeZone, CalendarProperty::IsSubscribed, CalendarProperty::IsDefault, CalendarProperty::IsVisible, CalendarProperty::IncludeInAvailability, CalendarProperty::DefaultAlertsWithTime, CalendarProperty::DefaultAlertsWithoutTime, ], Vec::<&str>::new(), ) .await; let list = response.list(); assert_eq!(list.len(), 1); let default_calendar_id = list[0].id().to_string(); assert_eq!( list[0], json!({ "id": default_calendar_id, "name": "Stalwart Calendar (jdoe@example.com)", "description": null, "sortOrder": 0, "isSubscribed": false, "isDefault": true, "color": null, "timeZone": null, "isVisible": true, "includeInAvailability": "all", "defaultAlertsWithTime": {}, "defaultAlertsWithoutTime": {} }) ); let change_id = response.state(); // Create Calendar let calendar_id = account .jmap_create( MethodObject::Calendar, [json!({ "name": "Test calendar", "description": "My personal calendar", "sortOrder": 1, "isSubscribed": true, "color": "#ff0000", "timeZone": "Indian/Christmas", "isVisible": false, "includeInAvailability": "attending", "defaultAlertsWithTime": { "0": { "action": "display", "trigger": { "relativeTo": "start", "offset": "PT15M" } }, "1": { "action": "email", "trigger": { "relativeTo": "end", "offset": "PT30M" } } }, "defaultAlertsWithoutTime": { "0": { "action": "display", "trigger": { "relativeTo": "start", "offset": "P1D" } }, "1": { "action": "email", "trigger": { "relativeTo": "end", "offset": "P2D" } } } })], Vec::<(&str, &str)>::new(), ) .await .created(0) .id() .to_string(); // Validate changes assert_eq!( account .jmap_changes(MethodObject::Calendar, change_id) .await .changes() .collect::>(), [ChangeType::Created(&calendar_id)] ); // Get Calendar let response = account .jmap_get( MethodObject::Calendar, [ CalendarProperty::Id, CalendarProperty::Name, CalendarProperty::Description, CalendarProperty::SortOrder, CalendarProperty::Color, CalendarProperty::TimeZone, CalendarProperty::IsSubscribed, CalendarProperty::IsDefault, CalendarProperty::IsVisible, CalendarProperty::IncludeInAvailability, CalendarProperty::DefaultAlertsWithTime, CalendarProperty::DefaultAlertsWithoutTime, ], [&calendar_id], ) .await; response.list()[0].assert_is_equal(json!({ "name": "Test calendar", "description": "My personal calendar", "sortOrder": 1, "isSubscribed": true, "isVisible": false, "isDefault": false, "color": "#ff0000", "timeZone": "Indian/Christmas", "includeInAvailability": "attending", "defaultAlertsWithTime": { "0": { "@type": "Alert", "action": "display", "trigger": { "@type": "OffsetTrigger", "relativeTo": "start", "offset": "PT15M" } }, "1": { "@type": "Alert", "action": "email", "trigger": { "@type": "OffsetTrigger", "relativeTo": "end", "offset": "PT30M" } } }, "defaultAlertsWithoutTime": { "0": { "@type": "Alert", "action": "display", "trigger": { "@type": "OffsetTrigger", "relativeTo": "start", "offset": "P1D" } }, "1": { "@type": "Alert", "action": "email", "trigger": { "@type": "OffsetTrigger", "relativeTo": "end", "offset": "P2D" } } }, "id": calendar_id, })); // Update Calendar and set it as default account .jmap_update( MethodObject::Calendar, [( calendar_id.as_str(), json!({ "name": "Updated calendar", "description": "My updated personal calendar", "sortOrder": 2, "isSubscribed": false, "isVisible": true, "timeZone": null, "color": null, "includeInAvailability": "none", "defaultAlertsWithTime": { "0": { "action": "email", "trigger": { "relativeTo": "start", "offset": "PT10M" } } }, "defaultAlertsWithoutTime/0": { "action": "email", "trigger": { "relativeTo": "start", "offset": "P3D" } }, "defaultAlertsWithoutTime/1": null, "defaultAlertsWithoutTime/2": { "action": "display", "trigger": { "relativeTo": "end", "offset": "P1W" } } }), )], [("onSuccessSetIsDefault", calendar_id.as_str())], ) .await .updated(&calendar_id); // Validate changes let response = account .jmap_get( MethodObject::Calendar, [ CalendarProperty::Id, CalendarProperty::Name, CalendarProperty::Description, CalendarProperty::SortOrder, CalendarProperty::Color, CalendarProperty::TimeZone, CalendarProperty::IsSubscribed, CalendarProperty::IsDefault, CalendarProperty::IsVisible, CalendarProperty::IncludeInAvailability, CalendarProperty::DefaultAlertsWithTime, CalendarProperty::DefaultAlertsWithoutTime, ], [&calendar_id, &default_calendar_id], ) .await; response.list()[0].assert_is_equal(json!({ "id": calendar_id, "name": "Updated calendar", "description": "My updated personal calendar", "sortOrder": 2, "isSubscribed": false, "isDefault": true, "color": null, "timeZone": null, "isVisible": true, "includeInAvailability": "none", "defaultAlertsWithTime": { "0": { "@type": "Alert", "action": "email", "trigger": { "@type": "OffsetTrigger", "relativeTo": "start", "offset": "PT10M" } } }, "defaultAlertsWithoutTime": { "0": { "@type": "Alert", "action": "email", "trigger": { "@type": "OffsetTrigger", "relativeTo": "start", "offset": "P3D" } }, "2": { "@type": "Alert", "action": "display", "trigger": { "@type": "OffsetTrigger", "relativeTo": "end", "offset": "P1W" } } } })); response.list()[1].assert_is_equal(json!({ "id": default_calendar_id, "name": "Stalwart Calendar (jdoe@example.com)", "description": (), "sortOrder": 0, "isSubscribed": false, "isDefault": false, "color": null, "timeZone": null, "isVisible": true, "includeInAvailability": "all", "defaultAlertsWithTime": {}, "defaultAlertsWithoutTime": {} })); // Create an event let _ = account .jmap_create( MethodObject::CalendarEvent, [json!({ "calendarIds": { &calendar_id: true }, "@type": "Event", "uid": "a8df6573-0474-496d-8496-033ad45d7fea", "updated": "2020-01-02T18:23:04Z", "title": "Some event", "start": "2020-01-15T13:00:00", "timeZone": "America/New_York", "duration": "PT1H" })], Vec::<(&str, &str)>::new(), ) .await .created(0) .id(); // Try destroying the calendar (should fail) assert_eq!( account .jmap_destroy( MethodObject::Calendar, [&calendar_id], Vec::<(&str, &str)>::new(), ) .await .not_destroyed(&calendar_id) .typ(), "calendarHasEvent" ); // Destroy using force assert_eq!( account .jmap_destroy( MethodObject::Calendar, [&calendar_id], [("onDestroyRemoveEvents", true)], ) .await .destroyed() .collect::>(), vec![&calendar_id] ); // Destroy all mailboxes account.destroy_all_calendars().await; params.assert_is_empty().await; } ================================================ FILE: tests/src/jmap/calendar/event.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ jmap::{ChangeType, IntoJmapSet, JMAPTest, JmapUtils, wait_for_index}, webdav::DummyWebDavClient, }; use ahash::AHashSet; use calcard::jscalendar::JSCalendarProperty; use groupware::cache::GroupwareCache; use hyper::StatusCode; use jmap_proto::request::method::MethodObject; use serde_json::{Value, json}; use types::{collection::SyncCollection, id::Id}; pub async fn test(params: &mut JMAPTest) { println!("Running Calendar Event tests..."); let account = params.account("jdoe@example.com"); // Create test calendars let response = account .jmap_create( MethodObject::Calendar, [ json!({ "name": "Holy Calendar, Batman!", "timeZone": "Europe/Vatican", }), json!({ "name": "Calendar with Alerts", "defaultAlertsWithTime": { "abc": { "action": "display", "trigger": { "relativeTo": "start", "offset": "PT15M" } } }, }), ], Vec::<(&str, &str)>::new(), ) .await; let calendar1_id = response.created(0).id().to_string(); let calendar2_id = response.created(1).id().to_string(); // Obtain state let change_id = account .jmap_get( MethodObject::CalendarEvent, Vec::<&str>::new(), Vec::<&str>::new(), ) .await .state() .to_string(); // Create test events let event_1 = test_jscalendar_1().with_property( JSCalendarProperty::::CalendarIds, [calendar1_id.as_str()].into_jmap_set(), ); let event_2 = test_jscalendar_2().with_property( JSCalendarProperty::::CalendarIds, [calendar2_id.as_str()].into_jmap_set(), ); let event_3 = test_jscalendar_3().with_property( JSCalendarProperty::::CalendarIds, [calendar1_id.as_str(), calendar2_id.as_str()].into_jmap_set(), ); let event_4 = test_jscalendar_4().with_property( JSCalendarProperty::::CalendarIds, [calendar1_id.as_str()].into_jmap_set(), ); let response = account .jmap_create( MethodObject::CalendarEvent, [ event_1 .clone() .with_property(JSCalendarProperty::::IsDraft, true) .with_property(JSCalendarProperty::::MayInviteSelf, true) .with_property(JSCalendarProperty::::MayInviteOthers, true) .with_property(JSCalendarProperty::::HideAttendees, true), event_2 .clone() .with_property(JSCalendarProperty::::UseDefaultAlerts, true), event_3.clone(), event_4, ], Vec::<(&str, &str)>::new(), ) .await; let event_1_id = response.created(0).id().to_string(); let event_2_id = response.created(1).id().to_string(); let event_3_id = response.created(2).id().to_string(); let event_4_id = response.created(3).id().to_string(); // Destroy tmp event assert_eq!( account .jmap_destroy( MethodObject::CalendarEvent, [event_4_id.as_str()], Vec::<(&str, &str)>::new(), ) .await .destroyed() .next(), Some(event_4_id.as_str()) ); // Validate changes assert_eq!( account .jmap_changes(MethodObject::CalendarEvent, &change_id) .await .changes() .collect::>(), [ ChangeType::Created(&event_1_id), ChangeType::Created(&event_2_id), ChangeType::Created(&event_3_id) ] .into_iter() .collect::>(), ); // Verify event contents let response = account .jmap_get( MethodObject::CalendarEvent, Vec::<&str>::new(), [&event_1_id, &event_2_id, &event_3_id], ) .await; response.list()[0].assert_is_equal( event_1 .with_property(JSCalendarProperty::::Id, event_1_id.as_str()) .with_property(JSCalendarProperty::::IsDraft, true) .with_property(JSCalendarProperty::::IsOrigin, true), ); response.list()[1].assert_is_equal( event_2 .with_property(JSCalendarProperty::::Id, event_2_id.as_str()) .with_property(JSCalendarProperty::::IsDraft, false) .with_property(JSCalendarProperty::::IsOrigin, true) .with_property( JSCalendarProperty::::Alerts, json!({ "k1": { "action": "display", "trigger": { "@type": "OffsetTrigger", "offset": "PT15M" }, "@type": "Alert" } }), ), ); response.list()[2].assert_is_equal( event_3 .with_property(JSCalendarProperty::::Id, event_3_id.as_str()) .with_property(JSCalendarProperty::::IsDraft, false) .with_property(JSCalendarProperty::::IsOrigin, false), ); // Verify JMAP for Calendars properties let response = account .jmap_get( MethodObject::CalendarEvent, [ JSCalendarProperty::::Id, JSCalendarProperty::MayInviteSelf, JSCalendarProperty::MayInviteOthers, JSCalendarProperty::HideAttendees, JSCalendarProperty::UtcStart, JSCalendarProperty::UtcEnd, ], [&event_1_id, &event_2_id, &event_3_id], ) .await; response.list()[0].assert_is_equal(json!({ "id": &event_1_id, "mayInviteSelf": true, "mayInviteOthers": true, "hideAttendees": true, "utcStart": "2006-01-02T15:00:00Z", "utcEnd": "2006-01-02T16:00:00Z" })); response.list()[1].assert_is_equal(json!({ "id": &event_2_id, "mayInviteSelf": false, "mayInviteOthers": false, "hideAttendees": false, "utcStart": "2006-01-02T17:00:00Z", "utcEnd": "2006-01-02T18:00:00Z" })); response.list()[2].assert_is_equal(json!({ "id": &event_3_id, "mayInviteSelf": false, "mayInviteOthers": false, "hideAttendees": false, "utcStart": "2006-01-04T15:00:00Z", "utcEnd": "2006-01-04T16:00:00Z" })); // Test /get parameters let response = account .jmap_method_calls(json!([[ "CalendarEvent/get", { "properties": ["id", "title", "recurrenceOverrides", "participants"], "ids": [&event_2_id, &event_3_id], "recurrenceOverridesBefore": "2006-01-07T00:00:00Z", "recurrenceOverridesAfter": "2006-01-06T00:00:00Z", "reduceParticipants": true, }, "0" ]])) .await; response.list_array().assert_is_equal(json!([ { "title": "Event #2", "recurrenceOverrides": { "2006-01-06T12:00:00": { "updated": "2006-02-06T00:11:21Z", "start": "2006-01-06T14:00:00", "title": "Event #2 bis bis", "duration": "PT1H" } }, "id": "c" }, { "title": "Event #3", "participants": { "3f5bc8c0-c722-5345-b7d9-5a899db08a30": { "calendarAddress": "mailto:cyrus@example.com", "@type": "Participant" } }, "id": "d" } ])); // Creating an event without calendar should fail assert_eq!( account .jmap_create( MethodObject::CalendarEvent, [json!({ "title": "Event #5", "start": "2006-01-22T10:00:00", "duration": "PT1H", "timeZone": "US/Eastern", "calendarIds": {}, }),], Vec::<(&str, &str)>::new() ) .await .not_created(0) .description(), "Event has to belong to at least one calendar." ); // Creating an event with a duplicate UID should fail assert_eq!( account .jmap_create( MethodObject::CalendarEvent, [json!({ "title": "Event #5", "start": "2006-01-22T10:00:00", "duration": "PT1H", "timeZone": "US/Eastern", "uid": "00959BC664CA650E933C892C@example.com", "calendarIds": { &calendar1_id: true }, })], Vec::<(&str, &str)>::new() ) .await .not_created(0) .description(), "An event with UID 00959BC664CA650E933C892C@example.com already exists." ); // Patching tests let response = account .jmap_update( MethodObject::CalendarEvent, [ ( &event_1_id, json!({ "isDraft": false, "mayInviteSelf": false, "mayInviteOthers": false, "hideAttendees": false, "description": null, "title": "Event one", "keywords": {"work": true}, format!("calendarIds/{calendar2_id}"): true }), ), ( &event_2_id, json!({ "calendarIds": { &calendar1_id: true, &calendar2_id: true }, "title": "Event two", "description": "Updated description", "recurrenceOverrides/2006-01-04T12:00:00/title": "Event two overridden", "recurrenceOverrides/2006-01-06T12:00:00/title": "Event two overridden twice", }), ), ( &event_3_id, json!({ format!("calendarIds/{calendar2_id}"): false, "title": "Event three", "utcStart": "2006-01-04T14:00:00Z", "utcEnd": "2006-01-04T16:00:00Z", "participants/3f5bc8c0-c722-5345-b7d9-5a899db08a30/roles/chair": false, "participants/3f5bc8c0-c722-5345-b7d9-5a899db08a30/roles/owner": true, "participants/ec5e7db5-22a3-5ed5-89bf-c8894ab86805" : null, "participants/7f2bd210-6c66-5b64-8562-0176b74462b1": { "calendarAddress": "mailto:rupert@example.com", "@type": "Participant", "participationStatus": "needs-action" } }), ), ], Vec::<(&str, &str)>::new(), ) .await; response.updated(&event_1_id); response.updated(&event_2_id); response.updated(&event_3_id); // Verify patches let response = account .jmap_get( MethodObject::CalendarEvent, [ JSCalendarProperty::::Id, JSCalendarProperty::CalendarIds, JSCalendarProperty::Title, JSCalendarProperty::Start, JSCalendarProperty::Description, JSCalendarProperty::Keywords, JSCalendarProperty::RecurrenceOverrides, JSCalendarProperty::Participants, JSCalendarProperty::MayInviteOthers, JSCalendarProperty::MayInviteSelf, JSCalendarProperty::HideAttendees, JSCalendarProperty::IsDraft, ], [&event_1_id, &event_2_id, &event_3_id], ) .await; response.list()[0].assert_is_equal(json!({ "id": &event_1_id, "calendarIds": { &calendar1_id: true, &calendar2_id: true }, "isDraft": false, "mayInviteSelf": false, "mayInviteOthers": false, "hideAttendees": false, "title": "Event one", "start": "2006-01-02T10:00:00", "keywords": { "work": true } })); response.list()[1].assert_is_equal(json!({ "id": &event_2_id, "calendarIds": { &calendar1_id: true, &calendar2_id: true }, "title": "Event two", "start": "2006-01-02T12:00:00", "description": "Updated description", "recurrenceOverrides": { "2006-01-04T12:00:00": { "title": "Event two overridden", "start": "2006-01-04T14:00:00", "duration": "PT1H", "updated": "2006-02-06T00:11:21Z" }, "2006-01-06T12:00:00": { "title": "Event two overridden twice", "start": "2006-01-06T14:00:00", "duration": "PT1H", "updated": "2006-02-06T00:11:21Z" } }, "title": "Event two", "start": "2006-01-02T12:00:00", "mayInviteOthers": false, "mayInviteSelf": false, "hideAttendees": false, "isDraft": false })); response.list()[2].assert_is_equal(json!({ "id": event_3_id, "calendarIds": { &calendar1_id: true, }, "title": "Event three", "start": "2006-01-04T09:00:00", "participants": { "3f5bc8c0-c722-5345-b7d9-5a899db08a30": { "calendarAddress": "mailto:cyrus@example.com", "@type": "Participant", "roles": { "owner": true }, "participationStatus": "accepted" }, "7f2bd210-6c66-5b64-8562-0176b74462b1": { "calendarAddress": "mailto:rupert@example.com", "@type": "Participant", "participationStatus": "needs-action" } }, "mayInviteOthers": false, "mayInviteSelf": false, "hideAttendees": false, "isDraft": false })); // Query tests wait_for_index(¶ms.server).await; assert_eq!( account .jmap_query( MethodObject::CalendarEvent, [ ("text", "Event one"), ("inCalendar", calendar1_id.as_str()), ("uid", "74855313FA803DA593CD579A@example.com"), ("after", "2006-01-02T10:59:59"), ("before", "2006-01-02T10:00:01"), ], ["start"], [("timeZone", "US/Eastern")], ) .await .ids() .collect::>(), [event_1_id.as_str()].into_iter().collect::>() ); // Recurrence expansion tests let response = account .jmap_query( MethodObject::CalendarEvent, [ ("after", "2006-01-01T00:00:00"), ("before", "2006-01-08T00:00:00"), ], ["start"], [ ("timeZone", Value::String("US/Eastern".into())), ("expandRecurrences", Value::Bool(true)), ], ) .await; let ids = response.ids().collect::>(); assert_eq!(ids.len(), 7); account .jmap_get( MethodObject::CalendarEvent, [ JSCalendarProperty::::Id, JSCalendarProperty::BaseEventId, JSCalendarProperty::Start, JSCalendarProperty::Duration, JSCalendarProperty::TimeZone, JSCalendarProperty::Title, JSCalendarProperty::RecurrenceId, ], ids.clone(), ) .await .list_array() .assert_is_equal(json!([ { "duration": "PT1H", "title": "Event one", "start": "2006-01-02T10:00:00", "timeZone": "US/Eastern", "id": &ids[0], "baseEventId": &event_1_id }, { "recurrenceId": "2006-01-02T12:00:00", "title": "Event two", "duration": "PT1H", "start": "2006-01-02T12:00:00", "timeZone": "US/Eastern", "id": &ids[1], "baseEventId": &event_2_id }, { "duration": "PT1H", "start": "2006-01-03T12:00:00", "timeZone": "US/Eastern", "title": "Event two", "recurrenceId": "2006-01-03T12:00:00", "id": &ids[2], "baseEventId": &event_2_id }, { "start": "2006-01-04T09:00:00", "timeZone": "US/Eastern", "duration": "PT2H", "title": "Event three", "id": &ids[3], "baseEventId": &event_3_id }, { "recurrenceId": "2006-01-04T14:00:00", "title": "Event two overridden", "start": "2006-01-04T14:00:00", "timeZone": "US/Eastern", "duration": "PT1H", "id": &ids[4], "baseEventId": &event_2_id }, { "recurrenceId": "2006-01-05T12:00:00", "duration": "PT1H", "timeZone": "US/Eastern", "start": "2006-01-05T12:00:00", "title": "Event two", "id": &ids[5], "baseEventId": &event_2_id }, { "recurrenceId": "2006-01-06T14:00:00", "duration": "PT1H", "title": "Event two overridden twice", "timeZone": "US/Eastern", "start": "2006-01-06T14:00:00", "id": &ids[6], "baseEventId": &event_2_id } ])); // Parse tests account .jmap_method_calls(json!([ [ "Blob/upload", { "create": { "ical": { "data": [ { "data:asText": r#"BEGIN:VCALENDAR PRODID:-//xyz Corp//NONSGML PDA Calendar Version 1.0//EN VERSION:2.0 BEGIN:VEVENT DTSTAMP:19960704T120000Z UID:uid1@example.com ORGANIZER:mailto:jsmith@example.com DTSTART:19960918T143000Z DTEND:19960920T220000Z STATUS:CONFIRMED CATEGORIES:CONFERENCE SUMMARY:Networld+Interop Conference DESCRIPTION:Networld+Interop Conference and Exhibit\nAtlanta World Congress Center\n Atlanta\, Georgia END:VEVENT END:VCALENDAR "# } ] } } }, "S4" ], [ "CalendarEvent/parse", { "blobIds": [ "#ical" ] }, "G4" ] ])) .await .pointer("/methodResponses/1/1/parsed") .unwrap() .as_object() .unwrap() .iter() .next() .unwrap() .1 .assert_is_equal(json!([ { "updated": "1996-07-04T12:00:00Z", "title": "Networld+Interop Conference", "description": "Networld+Interop Conferenceand Exhibit\nAtlanta World Congress Center\n", "timeZone": "Etc/UTC", "start": "1996-09-18T14:30:00", "status": "confirmed", "iCalendar": { "convertedProperties": { "duration": { "name": "dtend" } }, "name": "vevent" }, "@type": "Event", "uid": "uid1@example.com", "participants": { "25d7647e-52fc-559b-88df-d66f08da079c": { "calendarAddress": "mailto:jsmith@example.com", "@type": "Participant" } }, "keywords": { "CONFERENCE": true }, "organizerCalendarAddress": "mailto:jsmith@example.com", "duration": "P2DT7H30M" } ])); // Deletion tests assert_eq!( account .jmap_destroy( MethodObject::CalendarEvent, [event_2_id.as_str(), event_3_id.as_str()], Vec::<(&str, &str)>::new() ) .await .destroyed() .collect::>(), [event_2_id.as_str(), event_3_id.as_str()] .into_iter() .collect::>() ); // CardDAV compatibility tests let account_id = account.id().document_id(); let dav_client = DummyWebDavClient::new( u32::MAX, account.name(), account.secret(), account.emails()[0], ); let resources = params .server .fetch_dav_resources( ¶ms.server.get_access_token(account_id).await.unwrap(), account_id, SyncCollection::Calendar, ) .await .unwrap(); let path = format!( "{}{}", resources.base_path, resources .paths .iter() .find(|v| v.parent_id.is_some()) .unwrap() .path ); let ical = dav_client .request("GET", &path, "") .await .with_status(StatusCode::OK) .expect_body() .lines() .map(String::from) .collect::>(); let expected_ical = TEST_ICAL_1 .lines() .map(String::from) .collect::>(); assert_eq!(ical, expected_ical); // Clean up account.destroy_all_calendars().await; params.assert_is_empty().await; } pub fn test_jscalendar_1() -> Value { json!({ "duration": "PT1H", "@type": "Event", "description": "Go Steelers!", "updated": "2006-02-06T00:11:02Z", "timeZone": "US/Eastern", "start": "2006-01-02T10:00:00", "title": "Event #1", "uid": "74855313FA803DA593CD579A@example.com" }) } pub fn test_jscalendar_2() -> Value { json!({ "title": "Event #2", "duration": "PT1H", "updated": "2006-02-06T00:11:21Z", "recurrenceRule": { "frequency": "daily", "count": 5 }, "start": "2006-01-02T12:00:00", "uid": "00959BC664CA650E933C892C@example.com", "@type": "Event", "timeZone": "US/Eastern", "recurrenceOverrides": { "2006-01-04T12:00:00": { "title": "Event #2 bis", "start": "2006-01-04T14:00:00", "updated": "2006-02-06T00:11:21Z", "duration": "PT1H" }, "2006-01-06T12:00:00": { "title": "Event #2 bis bis", "start": "2006-01-06T14:00:00", "updated": "2006-02-06T00:11:21Z", "duration": "PT1H" } } }) } pub fn test_jscalendar_3() -> Value { json!({ "duration": "PT1H", "organizerCalendarAddress": "mailto:cyrus@example.com", "@type": "Event", "start": "2006-01-04T10:00:00", "status": "tentative", "uid": "DC6C50A017428C5216A2F1CD@example.com", "sequence": 1, "participants": { "3f5bc8c0-c722-5345-b7d9-5a899db08a30": { "calendarAddress": "mailto:cyrus@example.com", "@type": "Participant", "roles": { "chair": true }, "participationStatus": "accepted" }, "ec5e7db5-22a3-5ed5-89bf-c8894ab86805": { "calendarAddress": "mailto:lisa@example.com", "@type": "Participant", "participationStatus": "needs-action" } }, "title": "Event #3", "updated": "2006-02-06T00:12:20Z", "timeZone": "US/Eastern" }) } pub fn test_jscalendar_4() -> Value { json!({ "duration": "PT1H", "@type": "Event", "description": "Tmp Event", "updated": "2006-02-06T00:11:02Z", "timeZone": "US/Eastern", "start": "2006-01-02T10:00:00", "title": "Tmp Event", "uid": "tmp-event@example.com" }) } const TEST_ICAL_1: &str = r#"BEGIN:VCALENDAR BEGIN:VEVENT DTSTART;TZID=US/Eastern:20060102T100000 UID:74855313FA803DA593CD579A@example.com DURATION:PT1H SUMMARY:Event one DTSTAMP:20060206T001102Z CATEGORIES:work END:VEVENT END:VCALENDAR "#; ================================================ FILE: tests/src/jmap/calendar/identity.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::{JMAPTest, JmapUtils}; use jmap_proto::{ object::participant_identity::ParticipantIdentityProperty, request::method::MethodObject, }; use serde_json::json; use store::write::BatchBuilder; use types::{collection::Collection, field::PrincipalField}; pub async fn test(params: &mut JMAPTest) { println!("Running Participant Identity tests..."); let account = params.account("jdoe@example.com"); // Obtain all identities let response = account .jmap_get( MethodObject::ParticipantIdentity, [ ParticipantIdentityProperty::Id, ParticipantIdentityProperty::Name, ParticipantIdentityProperty::CalendarAddress, ParticipantIdentityProperty::IsDefault, ], Vec::<&str>::new(), ) .await; response.list_array().assert_is_equal(json!([ { "id": "a", "name": "John Doe", "calendarAddress": "mailto:jdoe@example.com", "isDefault": true }, { "id": "b", "name": "John Doe", "calendarAddress": "mailto:john.doe@example.com", "isDefault": false } ])); // Destroy identity b let response = account .jmap_destroy( MethodObject::ParticipantIdentity, ["b"], Vec::<(&str, &str)>::new(), ) .await; assert_eq!(response.destroyed().next(), Some("b")); let response = account .jmap_get( MethodObject::ParticipantIdentity, [ ParticipantIdentityProperty::Id, ParticipantIdentityProperty::Name, ParticipantIdentityProperty::CalendarAddress, ParticipantIdentityProperty::IsDefault, ], Vec::<&str>::new(), ) .await; response.list_array().assert_is_equal(json!([ { "id": "a", "name": "John Doe", "calendarAddress": "mailto:jdoe@example.com", "isDefault": true } ])); // Creating a new identity with an unauthorized calendar address should fail let response = account .jmap_create( MethodObject::ParticipantIdentity, [ json!({ "name": "Work", "calendarAddress": "mailto:work@example.com" }), json!({ "name": "Work", "calendarAddress": "work@example.com" }), ], [("onSuccessSetIsDefault", "#i0")], ) .await; assert_eq!( response.not_created(0).description(), "Calendar address not configured for this account." ); assert_eq!( response.not_created(1).description(), "Calendar address not configured for this account." ); // Create a new identity and set it as default let response = account .jmap_create( MethodObject::ParticipantIdentity, [json!({ "name": "Johnny B Goode", "calendarAddress": "mailto:john.doe@example.com" })], [("onSuccessSetIsDefault", "#i0")], ) .await; response.created(0); let response = account .jmap_get( MethodObject::ParticipantIdentity, [ ParticipantIdentityProperty::Id, ParticipantIdentityProperty::Name, ParticipantIdentityProperty::CalendarAddress, ParticipantIdentityProperty::IsDefault, ], Vec::<&str>::new(), ) .await; response.list_array().assert_is_equal(json!([ { "id": "a", "name": "John Doe", "calendarAddress": "mailto:jdoe@example.com", "isDefault": false }, { "id": "b", "name": "Johnny B Goode", "calendarAddress": "mailto:john.doe@example.com", "isDefault": true } ])); // Cleanup let mut batch = BatchBuilder::new(); batch .with_account_id(account.id().document_id()) .with_collection(Collection::Principal) .with_document(0) .clear(PrincipalField::ParticipantIdentities); params.server.commit_batch(batch).await.unwrap(); params.assert_is_empty().await; } ================================================ FILE: tests/src/jmap/calendar/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod acl; pub mod alarm; pub mod calendars; pub mod event; pub mod identity; pub mod notification; ================================================ FILE: tests/src/jmap/calendar/notification.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::{IntoJmapSet, JMAPTest, JmapUtils, wait_for_index}; use calcard::jscalendar::JSCalendarProperty; use jmap_proto::{ object::calendar_event_notification::CalendarEventNotificationProperty, request::method::MethodObject, }; use mail_parser::DateTime; use serde_json::{Value, json}; use store::write::now; use types::id::Id; pub async fn test(params: &mut JMAPTest) { println!("Running Calendar Event Notification tests..."); let john = params.account("jdoe@example.com"); let jane = params.account("jane.smith@example.com"); let bill = params.account("bill@example.com"); let john_id = john.id_string().to_string(); let jane_id = jane.id_string().to_string(); let bill_id = bill.id_string().to_string(); let mut john_change_id = String::new(); let mut jane_change_id = String::new(); let mut bill_change_id = String::new(); // Obtain share notification change ids for all accounts for (change_id, client) in [ (&mut john_change_id, john), (&mut jane_change_id, jane), (&mut bill_change_id, bill), ] { let response = client .jmap_get( MethodObject::CalendarEventNotification, [CalendarEventNotificationProperty::Id], Vec::<&str>::new(), ) .await; response.list_array().assert_is_equal(json!([])); *change_id = response.state().to_string(); } // Create test calendars let response = john .jmap_create( MethodObject::Calendar, [json!({ "name": "Test Calendar", })], Vec::<(&str, &str)>::new(), ) .await; let john_calendar_id = response.created(0).id().to_string(); // Sent invitation to Jane and Bill let john_event = test_event(); let response = john .jmap_create( MethodObject::CalendarEvent, [john_event.clone().with_property( JSCalendarProperty::::CalendarIds, [john_calendar_id.as_str()].into_jmap_set(), )], [("sendSchedulingMessages", true)], ) .await; let john_event_id = response.created(0).id().to_string(); tokio::time::sleep(std::time::Duration::from_millis(600)).await; wait_for_index(¶ms.server).await; // Verify Jane and Bill received the share notification let mut jane_event_id = String::new(); let mut bill_event_id = String::new(); for (change_id, event_id, client) in [ (&mut jane_change_id, &mut jane_event_id, jane), (&mut bill_change_id, &mut bill_event_id, bill), ] { // Obtain changes let response = client .jmap_changes(MethodObject::CalendarEventNotification, &change_id) .await; let changes = response.changes().collect::>(); assert_eq!(changes.len(), 1); *change_id = response.new_state().to_string(); let notification_id = changes[0].as_created(); // Obtain and verify notification let response = client .jmap_get( MethodObject::CalendarEventNotification, [ CalendarEventNotificationProperty::Id, CalendarEventNotificationProperty::Created, CalendarEventNotificationProperty::ChangedBy, CalendarEventNotificationProperty::Comment, CalendarEventNotificationProperty::Type, CalendarEventNotificationProperty::CalendarEventId, CalendarEventNotificationProperty::IsDraft, CalendarEventNotificationProperty::Event, CalendarEventNotificationProperty::EventPatch, ], [notification_id], ) .await; let notification = &response.list()[0]; *event_id = notification.text_field("calendarEventId").to_string(); notification.assert_is_equal(json!({ "id": ¬ification_id, "created": ¬ification.text_field("created"), "changedBy": { "name": "John Doe", "email": "jdoe@example.com", "principalId": &john_id }, "type": "created", "calendarEventId": event_id, "isDraft": false, "event": john_event .clone() .with_property("sequence", 1) .with_property( "updated", notification .text_field("event/updated") ) })); // Verify the event exists let response = client .jmap_get( MethodObject::CalendarEvent, [JSCalendarProperty::::Id, JSCalendarProperty::Title], [&event_id], ) .await; response.list()[0].assert_is_equal(json!({ "id": &event_id, "title": "Lunch" })); } // Jane and Bill accept the invitation let response = jane .jmap_update( MethodObject::CalendarEvent, [( &jane_event_id, json!({ "participants/a0171748-fe8d-57d8-879e-56036a5251d1/participationStatus": "accepted"}), )], [("sendSchedulingMessages", true)], ) .await; response.updated(&jane_event_id); let response = bill .jmap_update( MethodObject::CalendarEvent, [( &bill_event_id, json!({ "participants/86720268-d67c-58c3-9217-03df7d7ee4d8/participationStatus": "accepted"}), )], [("sendSchedulingMessages", true)], ) .await; response.updated(&bill_event_id); tokio::time::sleep(std::time::Duration::from_millis(200)).await; // Verify John received two share notifications let response = john .jmap_changes(MethodObject::CalendarEventNotification, &john_change_id) .await; let changes = response.changes().collect::>(); assert_eq!(changes.len(), 2); for (i, change) in changes.into_iter().enumerate() { let notification_id = change.as_created(); // Obtain and verify notification let response = john .jmap_get( MethodObject::CalendarEventNotification, [ CalendarEventNotificationProperty::Id, CalendarEventNotificationProperty::ChangedBy, CalendarEventNotificationProperty::Comment, CalendarEventNotificationProperty::Type, CalendarEventNotificationProperty::CalendarEventId, CalendarEventNotificationProperty::IsDraft, ], [notification_id], ) .await; let changed_by = if i == 0 { json!({ "name": "Jane Smith", "email": "jane.smith@example.com", "principalId": &jane_id, }) } else { json!({ "name": "Bill Foobar", "email": "bill@example.com", "principalId": &bill_id, }) }; response.list()[0].assert_is_equal(json!({ "id": ¬ification_id, "changedBy": changed_by, "type": "updated", "calendarEventId": &john_event_id, "isDraft": false })); } // Verify the event was updated let response = john .jmap_get( MethodObject::CalendarEvent, [ JSCalendarProperty::::Id, JSCalendarProperty::Title, JSCalendarProperty::Participants, ], [&john_event_id], ) .await; response.list()[0].assert_is_equal(json!({ "participants": { "8584f8f9-5414-55e3-8a1c-ad6fc2f3ffb6": { "calendarAddress": "mailto:jdoe@example.com", "@type": "Participant", "roles": { "chair": true }, "participationStatus": "accepted" }, "a0171748-fe8d-57d8-879e-56036a5251d1": { "calendarAddress": "mailto:jane.smith@example.com", "@type": "Participant", "participationStatus": "accepted", "kind": "individual" }, "86720268-d67c-58c3-9217-03df7d7ee4d8": { "calendarAddress": "mailto:bill@example.com", "@type": "Participant", "kind": "individual", "participationStatus": "accepted" } }, "title": "Lunch", "id": &john_event_id })); // Jane later declines the invitation let response = jane .jmap_update( MethodObject::CalendarEvent, [( &jane_event_id, json!({ "participants/a0171748-fe8d-57d8-879e-56036a5251d1/participationStatus": "declined"}), )], [("sendSchedulingMessages", true)], ) .await; response.updated(&jane_event_id); tokio::time::sleep(std::time::Duration::from_millis(200)).await; // Make sure John received the update let response = john .jmap_get( MethodObject::CalendarEvent, [ JSCalendarProperty::::Id, JSCalendarProperty::Title, JSCalendarProperty::Participants, ], [&john_event_id], ) .await; response.list()[0].assert_is_equal(json!({ "participants": { "8584f8f9-5414-55e3-8a1c-ad6fc2f3ffb6": { "calendarAddress": "mailto:jdoe@example.com", "@type": "Participant", "roles": { "chair": true }, "participationStatus": "accepted" }, "a0171748-fe8d-57d8-879e-56036a5251d1": { "calendarAddress": "mailto:jane.smith@example.com", "@type": "Participant", "participationStatus": "declined", "kind": "individual" }, "86720268-d67c-58c3-9217-03df7d7ee4d8": { "calendarAddress": "mailto:bill@example.com", "@type": "Participant", "kind": "individual", "participationStatus": "accepted" } }, "title": "Lunch", "id": &john_event_id })); // John deletes the event let response = john .jmap_destroy( MethodObject::CalendarEvent, [&john_event_id], [("sendSchedulingMessages", true)], ) .await; assert_eq!(response.destroyed().collect::>(), [&john_event_id]); tokio::time::sleep(std::time::Duration::from_millis(200)).await; // Verify that only Bill received the cancellation let response = jane .jmap_changes(MethodObject::CalendarEventNotification, &jane_change_id) .await; assert_eq!(response.changes().next(), None); let response = bill .jmap_changes(MethodObject::CalendarEventNotification, &bill_change_id) .await; let changes = response.changes().collect::>(); assert_eq!(changes.len(), 1); let notification_id = changes[0].as_created(); let response = bill .jmap_get( MethodObject::CalendarEventNotification, [ CalendarEventNotificationProperty::Id, CalendarEventNotificationProperty::ChangedBy, CalendarEventNotificationProperty::Comment, CalendarEventNotificationProperty::Type, CalendarEventNotificationProperty::CalendarEventId, CalendarEventNotificationProperty::IsDraft, ], [notification_id], ) .await; response.list()[0].assert_is_equal(json!({ "id": ¬ification_id, "changedBy": { "name": "John Doe", "email": "jdoe@example.com", "principalId": &john_id }, "type": "updated", "calendarEventId": &bill_event_id, "isDraft": false })); // Verify Bill's event was updated let response = bill .jmap_get( MethodObject::CalendarEvent, [ JSCalendarProperty::::Id, JSCalendarProperty::Title, JSCalendarProperty::Status, ], [&bill_event_id], ) .await; response.list()[0].assert_is_equal(json!({ "id": &bill_event_id, "title": "Lunch", "status": "cancelled" })); // Cleanup for client in [john, jane, bill] { client.destroy_all_calendars().await; client.destroy_all_event_notifications().await; params.destroy_all_mailboxes(client).await; } params.assert_is_empty().await; } fn test_event() -> Value { json!({ "uid": "9263504FD3AD", "title": "Lunch", "timeZone": "Europe/London", "start": DateTime::from_timestamp(now() as i64 + 60 * 60) .to_rfc3339().trim_end_matches("Z").to_string(), "duration": "PT1H", "freeBusyStatus": "busy", "updated": "2009-06-02T17:00:00Z", "sequence": 0, "@type": "Event", "participants": { "8584f8f9-5414-55e3-8a1c-ad6fc2f3ffb6": { "calendarAddress": "mailto:jdoe@example.com", "participationStatus": "accepted", "roles": { "chair": true }, "@type": "Participant" }, "a0171748-fe8d-57d8-879e-56036a5251d1": { "calendarAddress": "mailto:jane.smith@example.com", "@type": "Participant", "participationStatus": "needs-action", "kind": "individual" }, "86720268-d67c-58c3-9217-03df7d7ee4d8": { "calendarAddress": "mailto:bill@example.com", "participationStatus": "needs-action", "@type": "Participant", "kind": "individual" } }, "organizerCalendarAddress": "mailto:jdoe@example.com" }) } ================================================ FILE: tests/src/jmap/contacts/acl.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::{JMAPTest, JmapUtils}; use calcard::jscontact::JSContactProperty; use jmap_proto::{ object::{addressbook::AddressBookProperty, share_notification::ShareNotificationProperty}, request::method::MethodObject, }; use serde_json::json; use types::id::Id; pub async fn test(params: &mut JMAPTest) { println!("Running Contacts ACL tests..."); let john = params.account("jdoe@example.com"); let jane = params.account("jane.smith@example.com"); let john_id = john.id_string().to_string(); let jane_id = jane.id_string().to_string(); // Create test address books let response = john .jmap_create( MethodObject::AddressBook, [json!({ "name": "Test #1", })], Vec::<(&str, &str)>::new(), ) .await; let john_book_id = response.created(0).id().to_string(); let john_contact_id = john .jmap_create( MethodObject::ContactCard, [json!({ "uid": "abc123", "name": { "full": "John's Simple Contact", }, "addressBookIds": { &john_book_id: true }, })], Vec::<(&str, &str)>::new(), ) .await .created(0) .id() .to_string(); let response = jane .jmap_create( MethodObject::AddressBook, [json!({ "name": "Test #1", })], Vec::<(&str, &str)>::new(), ) .await; let jane_book_id = response.created(0).id().to_string(); let jane_contact_id = jane .jmap_create( MethodObject::ContactCard, [json!({ "uid": "abc456", "name": { "full": "Jane's Simple Contact", }, "addressBookIds": { &jane_book_id: true }, })], Vec::<(&str, &str)>::new(), ) .await .created(0) .id() .to_string(); // Verify myRights john.jmap_get( MethodObject::AddressBook, [ AddressBookProperty::Id, AddressBookProperty::Name, AddressBookProperty::MyRights, AddressBookProperty::ShareWith, ], [john_book_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_book_id, "name": "Test #1", "myRights": { "mayRead": true, "mayWrite": true, "mayDelete": true, "mayShare": true }, "shareWith": {} })); // Obtain share notifications let mut jane_share_change_id = jane .jmap_get( MethodObject::ShareNotification, Vec::<&str>::new(), Vec::<&str>::new(), ) .await .state() .to_string(); // Make sure Jane has no access assert_eq!( jane.jmap_get_account( john, MethodObject::AddressBook, Vec::<&str>::new(), [john_book_id.as_str()], ) .await .method_response() .typ(), "forbidden" ); // Share address book with Jane john.jmap_update( MethodObject::AddressBook, [( &john_book_id, json!({ "shareWith": { &jane_id : { "mayRead": true, } } }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&john_book_id); john.jmap_get( MethodObject::AddressBook, [ AddressBookProperty::Id, AddressBookProperty::Name, AddressBookProperty::ShareWith, ], [john_book_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_book_id, "name": "Test #1", "shareWith": { &jane_id : { "mayRead": true, "mayWrite": false, "mayDelete": false, "mayShare": false } } })); // Verify Jane can access the contact jane.jmap_get_account( john, MethodObject::AddressBook, [ AddressBookProperty::Id, AddressBookProperty::Name, AddressBookProperty::MyRights, ], [john_book_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_book_id, "name": "Test #1", "myRights": { "mayRead": true, "mayWrite": false, "mayDelete": false, "mayShare": false } })); jane.jmap_get_account( john, MethodObject::ContactCard, [AddressBookProperty::Id, AddressBookProperty::Name], [john_contact_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_contact_id, "name": { "full": "John's Simple Contact" }, })); // Verify Jane received a share notification let response = jane .jmap_changes(MethodObject::ShareNotification, &jane_share_change_id) .await; jane_share_change_id = response.new_state().to_string(); let changes = response.changes().collect::>(); assert_eq!(changes.len(), 1); let share_id = changes[0].as_created(); jane.jmap_get( MethodObject::ShareNotification, [ ShareNotificationProperty::Id, ShareNotificationProperty::ChangedBy, ShareNotificationProperty::ObjectType, ShareNotificationProperty::ObjectAccountId, ShareNotificationProperty::ObjectId, ShareNotificationProperty::OldRights, ShareNotificationProperty::NewRights, ShareNotificationProperty::Name, ], [share_id], ) .await .list()[0] .assert_is_equal(json!({ "id": &share_id, "changedBy": { "principalId": &john_id, "name": "John Doe", "email": "jdoe@example.com" }, "objectType": "AddressBook", "objectAccountId": &john_id, "objectId": &john_book_id, "oldRights": { "mayRead": false, "mayWrite": false, "mayDelete": false, "mayShare": false }, "newRights": { "mayRead": true, "mayWrite": false, "mayDelete": false, "mayShare": false }, "name": null })); // Updating and deleting should fail assert_eq!( jane.jmap_update_account( john, MethodObject::AddressBook, [(&john_book_id, json!({}))], Vec::<(&str, &str)>::new(), ) .await .not_updated(&john_book_id) .description(), "You are not allowed to modify this address book." ); assert_eq!( jane.jmap_destroy_account( john, MethodObject::AddressBook, [&john_book_id], Vec::<(&str, &str)>::new(), ) .await .not_destroyed(&john_book_id) .description(), "You are not allowed to delete this address book." ); assert!( jane.jmap_update_account( john, MethodObject::ContactCard, [(&john_contact_id, json!({}))], Vec::<(&str, &str)>::new(), ) .await .not_updated(&john_contact_id) .description() .contains("You are not allowed to modify address book"), ); assert!( jane.jmap_destroy_account( john, MethodObject::ContactCard, [&john_contact_id], Vec::<(&str, &str)>::new(), ) .await .not_destroyed(&john_contact_id) .description() .contains("You are not allowed to remove contacts from address book"), ); // Grant Jane write access john.jmap_update( MethodObject::AddressBook, [( &john_book_id, json!({ format!("shareWith/{jane_id}/mayWrite"): true, format!("shareWith/{jane_id}/mayDelete"): true, }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&john_book_id); jane.jmap_get_account( john, MethodObject::AddressBook, [ AddressBookProperty::Id, AddressBookProperty::Name, AddressBookProperty::MyRights, ], [john_book_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_book_id, "name": "Test #1", "myRights": { "mayRead": true, "mayWrite": true, "mayDelete": true, "mayShare": false } })); // Verify Jane received a share notification with the updated rights let response = jane .jmap_changes(MethodObject::ShareNotification, &jane_share_change_id) .await; jane_share_change_id = response.new_state().to_string(); let changes = response.changes().collect::>(); assert_eq!(changes.len(), 1); let share_id = changes[0].as_created(); jane.jmap_get( MethodObject::ShareNotification, [ ShareNotificationProperty::Id, ShareNotificationProperty::ChangedBy, ShareNotificationProperty::ObjectType, ShareNotificationProperty::ObjectAccountId, ShareNotificationProperty::ObjectId, ShareNotificationProperty::OldRights, ShareNotificationProperty::NewRights, ShareNotificationProperty::Name, ], [share_id], ) .await .list()[0] .assert_is_equal(json!({ "id": &share_id, "changedBy": { "principalId": &john_id, "name": "John Doe", "email": "jdoe@example.com" }, "objectType": "AddressBook", "objectAccountId": &john_id, "objectId": &john_book_id, "oldRights": { "mayRead": true, "mayWrite": false, "mayDelete": false, "mayShare": false }, "newRights": { "mayRead": true, "mayWrite": true, "mayDelete": true, "mayShare": false }, "name": null })); // Creating a root folder should fail assert_eq!( jane.jmap_create_account( john, MethodObject::AddressBook, [json!({ "name": "A new shared address book", })], Vec::<(&str, &str)>::new() ) .await .not_created(0) .description(), "Cannot create address books in a shared account." ); // Copy Jane's contact into John's address book let john_copied_contact_id = jane .jmap_copy( jane, john, MethodObject::ContactCard, [( &jane_contact_id, json!({ "addressBookIds": { &john_book_id: true } }), )], false, ) .await .copied(&jane_contact_id) .id() .to_string(); jane.jmap_get_account( john, MethodObject::ContactCard, [ JSContactProperty::::Id, JSContactProperty::AddressBookIds, JSContactProperty::Name, ], [john_copied_contact_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_copied_contact_id, "name": { "full": "Jane's Simple Contact" }, "addressBookIds": { &john_book_id: true } })); // Destroy the copied contact assert_eq!( jane.jmap_destroy_account( john, MethodObject::ContactCard, [john_copied_contact_id.as_str()], Vec::<(&str, &str)>::new(), ) .await .destroyed() .collect::>(), [&john_copied_contact_id] ); // Update John's contact jane.jmap_update_account( john, MethodObject::ContactCard, [( &john_contact_id, json!({ "name": { "full": "John's Updated Contact", } }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&john_contact_id); jane.jmap_get_account( john, MethodObject::ContactCard, [JSContactProperty::::Id, JSContactProperty::Name], [john_contact_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_contact_id, "name": { "full": "John's Updated Contact" }, })); // Update John's address book name jane.jmap_update_account( john, MethodObject::AddressBook, [( &john_book_id, json!({ "name": "Jane's version of John's Address Book", "description": "This is John's address book, but Jane can edit it now" }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&john_book_id); jane.jmap_get_account( john, MethodObject::AddressBook, [ AddressBookProperty::Id, AddressBookProperty::Name, AddressBookProperty::Description, ], [john_book_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_book_id, "name": "Jane's version of John's Address Book", "description": "This is John's address book, but Jane can edit it now" })); // John should still see the old name john.jmap_get( MethodObject::AddressBook, [ AddressBookProperty::Id, AddressBookProperty::Name, AddressBookProperty::Description, ], [john_book_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_book_id, "name": "Test #1", "description": null })); // Revoke Jane's access john.jmap_update( MethodObject::AddressBook, [( &john_book_id, json!({ format!("shareWith/{jane_id}"): () }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&john_book_id); john.jmap_get( MethodObject::AddressBook, [ AddressBookProperty::Id, AddressBookProperty::Name, AddressBookProperty::ShareWith, ], [john_book_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_book_id, "name": "Test #1", "shareWith": {} })); // Verify Jane can no longer access the address book or its contacts assert_eq!( jane.jmap_get_account( john, MethodObject::AddressBook, Vec::<&str>::new(), [john_book_id.as_str()], ) .await .method_response() .typ(), "forbidden" ); // Verify Jane received a share notification with the updated rights let response = jane .jmap_changes(MethodObject::ShareNotification, &jane_share_change_id) .await; let changes = response.changes().collect::>(); assert_eq!(changes.len(), 1); let share_id = changes[0].as_created(); jane.jmap_get( MethodObject::ShareNotification, [ ShareNotificationProperty::Id, ShareNotificationProperty::ChangedBy, ShareNotificationProperty::ObjectType, ShareNotificationProperty::ObjectAccountId, ShareNotificationProperty::ObjectId, ShareNotificationProperty::OldRights, ShareNotificationProperty::NewRights, ShareNotificationProperty::Name, ], [share_id], ) .await .list()[0] .assert_is_equal(json!({ "id": &share_id, "changedBy": { "principalId": &john_id, "name": "John Doe", "email": "jdoe@example.com" }, "objectType": "AddressBook", "objectAccountId": &john_id, "objectId": &john_book_id, "oldRights": { "mayRead": true, "mayWrite": true, "mayDelete": true, "mayShare": false }, "newRights": { "mayRead": false, "mayWrite": false, "mayDelete": false, "mayShare": false }, "name": null })); // Grant Jane delete access once again john.jmap_update( MethodObject::AddressBook, [( &john_book_id, json!({ format!("shareWith/{jane_id}/mayRead"): true, format!("shareWith/{jane_id}/mayDelete"): true, }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&john_book_id); // Verify Jane can delete the address book assert_eq!( jane.jmap_destroy_account( john, MethodObject::AddressBook, [john_book_id.as_str()], [("onDestroyRemoveContents", true)], ) .await .destroyed() .collect::>(), [john_book_id.as_str()] ); // Destroy all mailboxes john.destroy_all_addressbooks().await; jane.destroy_all_addressbooks().await; params.assert_is_empty().await; } ================================================ FILE: tests/src/jmap/contacts/addressbook.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use jmap_proto::{object::addressbook::AddressBookProperty, request::method::MethodObject}; use serde_json::json; use crate::jmap::{ChangeType, JMAPTest, JmapUtils}; pub async fn test(params: &mut JMAPTest) { println!("Running AddressBook tests..."); let account = params.account("jdoe@example.com"); // Make sure the default address book exists let response = account .jmap_get( MethodObject::AddressBook, [ AddressBookProperty::Id, AddressBookProperty::Name, AddressBookProperty::Description, AddressBookProperty::SortOrder, AddressBookProperty::IsSubscribed, AddressBookProperty::IsDefault, ], Vec::<&str>::new(), ) .await; let list = response.list(); assert_eq!(list.len(), 1); let default_addressbook_id = list[0].id().to_string(); assert_eq!( list[0], json!({ "name": "Stalwart Address Book (jdoe@example.com)", "description": (), "sortOrder": 0, "isSubscribed": false, "isDefault": true, "id": default_addressbook_id, }) ); let change_id = response.state(); // Create Address Book let addressbook_id = account .jmap_create( MethodObject::AddressBook, [json!({ "name": "Test address book", "description": "My personal address book", "sortOrder": 1, "isSubscribed": true })], Vec::<(&str, &str)>::new(), ) .await .created(0) .id() .to_string(); // Validate changes assert_eq!( account .jmap_changes(MethodObject::AddressBook, change_id) .await .changes() .collect::>(), [ChangeType::Created(&addressbook_id)] ); // Get Address Book let response = account .jmap_get( MethodObject::AddressBook, [ AddressBookProperty::Id, AddressBookProperty::Name, AddressBookProperty::Description, AddressBookProperty::SortOrder, AddressBookProperty::IsSubscribed, AddressBookProperty::IsDefault, ], [&addressbook_id], ) .await; assert_eq!( response.list()[0], json!({ "name": "Test address book", "description": "My personal address book", "sortOrder": 1, "isSubscribed": true, "isDefault": false, "id": addressbook_id, }) ); // Update Address Book and set it as default account .jmap_update( MethodObject::AddressBook, [( addressbook_id.as_str(), json!({ "name": "Updated address book", "description": "My updated personal address book", "sortOrder": 2, "isSubscribed": false }), )], [("onSuccessSetIsDefault", addressbook_id.as_str())], ) .await .updated(&addressbook_id); // Validate changes assert_eq!( account .jmap_get( MethodObject::AddressBook, [ AddressBookProperty::Id, AddressBookProperty::Name, AddressBookProperty::Description, AddressBookProperty::SortOrder, AddressBookProperty::IsSubscribed, AddressBookProperty::IsDefault, ], [&addressbook_id, &default_addressbook_id], ) .await .list(), vec![ json!({ "name": "Updated address book", "description": "My updated personal address book", "sortOrder": 2, "isSubscribed": false, "isDefault": true, "id": addressbook_id, }), json!({ "name": "Stalwart Address Book (jdoe@example.com)", "description": (), "sortOrder": 0, "isSubscribed": false, "isDefault": false, "id": default_addressbook_id, }) ] ); // Create a contact let _ = account .jmap_create( MethodObject::ContactCard, [json!({ "addressBookIds": { &addressbook_id: true }, "name": { "components": [ { "kind": "given", "value": "Joe" }, { "kind": "surname", "value": "Bloggs" } ] }, "emails": { "0": { "address": "joe.bloggs@example.com" } } })], Vec::<(&str, &str)>::new(), ) .await .created(0) .id(); // Try destroying the address book (should fail) assert_eq!( account .jmap_destroy( MethodObject::AddressBook, [&addressbook_id], Vec::<(&str, &str)>::new(), ) .await .not_destroyed(&addressbook_id) .typ(), "addressBookHasContents" ); // Destroy using force assert_eq!( account .jmap_destroy( MethodObject::AddressBook, [&addressbook_id], [("onDestroyRemoveContents", true)], ) .await .destroyed() .collect::>(), vec![&addressbook_id] ); // Destroy all mailboxes account.destroy_all_addressbooks().await; params.assert_is_empty().await; } ================================================ FILE: tests/src/jmap/contacts/contact.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ jmap::{ChangeType, IntoJmapSet, JMAPTest, JmapUtils, wait_for_index}, webdav::DummyWebDavClient, }; use ahash::AHashSet; use calcard::jscontact::JSContactProperty; use groupware::cache::GroupwareCache; use hyper::StatusCode; use jmap_proto::request::method::MethodObject; use serde_json::{Value, json}; use types::{collection::SyncCollection, id::Id}; pub async fn test(params: &mut JMAPTest) { println!("Running Contact Card tests..."); let account = params.account("jdoe@example.com"); // Create test address books let response = account .jmap_create( MethodObject::AddressBook, [ json!({ "name": "Test #1", }), json!({ "name": "Test #2", }), ], Vec::<(&str, &str)>::new(), ) .await; let book1_id = response.created(0).id().to_string(); let book2_id = response.created(1).id().to_string(); // Obtain state let change_id = account .jmap_get( MethodObject::ContactCard, Vec::<&str>::new(), Vec::<&str>::new(), ) .await .state() .to_string(); // Create test contacts let sarah_contact = test_jscontact_1().with_property( JSContactProperty::::AddressBookIds, [book1_id.as_str()].into_jmap_set(), ); let carlos_contact = test_jscontact_2().with_property( JSContactProperty::::AddressBookIds, [book2_id.as_str()].into_jmap_set(), ); let acme_contact = test_jscontact_3().with_property( JSContactProperty::::AddressBookIds, [book1_id.as_str(), book2_id.as_str()].into_jmap_set(), ); let tmp_contact = test_jscontact_4().with_property( JSContactProperty::::AddressBookIds, [book2_id.as_str()].into_jmap_set(), ); let response = account .jmap_create( MethodObject::ContactCard, [ sarah_contact.clone(), carlos_contact.clone(), acme_contact.clone(), tmp_contact, ], Vec::<(&str, &str)>::new(), ) .await; let sarah_contact_id = response.created(0).id().to_string(); let carlos_contact_id = response.created(1).id().to_string(); let acme_contact_id = response.created(2).id().to_string(); let tmp_contact_id = response.created(3).id().to_string(); // Destroy tmp contact assert_eq!( account .jmap_destroy( MethodObject::ContactCard, [tmp_contact_id.as_str()], Vec::<(&str, &str)>::new(), ) .await .destroyed() .next(), Some(tmp_contact_id.as_str()) ); // Validate changes assert_eq!( account .jmap_changes(MethodObject::ContactCard, &change_id) .await .changes() .collect::>(), [ ChangeType::Created(&sarah_contact_id), ChangeType::Created(&carlos_contact_id), ChangeType::Created(&acme_contact_id) ] .into_iter() .collect::>(), ); // Fetch contacts and verify let response = account .jmap_get( MethodObject::ContactCard, Vec::<&str>::new(), [&sarah_contact_id, &carlos_contact_id, &acme_contact_id], ) .await; response.list()[0].assert_is_equal( sarah_contact.with_property(JSContactProperty::::Id, sarah_contact_id.as_str()), ); response.list()[1].assert_is_equal( carlos_contact.with_property(JSContactProperty::::Id, carlos_contact_id.as_str()), ); response.list()[2].assert_is_equal( acme_contact.with_property(JSContactProperty::::Id, acme_contact_id.as_str()), ); // Creating a contact without address book should fail assert_eq!( account .jmap_create( MethodObject::ContactCard, [json!({ "name": { "full": "Simple Contact", }, "addressBookIds": {}, }),], Vec::<(&str, &str)>::new() ) .await .not_created(0) .description(), "Contact has to belong to at least one address book." ); // Creating a contact with a duplicate UID should fail assert!( account .jmap_create( MethodObject::ContactCard, [json!({ "uid": "urn:uuid:f81d4fae-7dec-11d0-a765-00a0c91e6bf6", "name": { "full": "Simple Contact", }, "addressBookIds": { &book1_id: true }, }),], Vec::<(&str, &str)>::new() ) .await .not_created(0) .description() .contains( "Contact with UID urn:uuid:f81d4fae-7dec-11d0-a765-00a0c91e6bf6 already exists" ), ); // Patching tests let response = account .jmap_update( MethodObject::ContactCard, [ ( &sarah_contact_id, json!({ "name/full": "Sarah O'Connor", "name/components/0/value": "O'Connor", format!("addressBookIds/{book2_id}"): true }), ), ( &carlos_contact_id, json!({ "addressBookIds": { &book1_id: true, &book2_id: true }, "nicknames/k1": (), "nicknames/k2": { "name": "Carlitos" }, }), ), ( &acme_contact_id, json!({ format!("addressBookIds/{book2_id}"): false, "keywords/B2B": false, "keywords/B2C": true, }), ), ], Vec::<(&str, &str)>::new(), ) .await; response.updated(&sarah_contact_id); response.updated(&carlos_contact_id); response.updated(&acme_contact_id); // Verify patches let response = account .jmap_get( MethodObject::ContactCard, [ JSContactProperty::::Id, JSContactProperty::AddressBookIds, JSContactProperty::Name, JSContactProperty::Keywords, JSContactProperty::Nicknames, ], [&sarah_contact_id, &carlos_contact_id, &acme_contact_id], ) .await; response.list()[0].assert_is_equal(json!({ "id": &sarah_contact_id, "name": { "full": "Sarah O'Connor", "components": [ { "kind": "surname", "value": "O'Connor" }, { "kind": "given", "value": "Sarah" }, { "kind": "given2", "value": "Marie" }, { "kind": "title", "value": "Dr." }, { "kind": "credential", "value": "Ph.D." } ], "isOrdered": true }, "nicknames": { "k1": { "name": "Sadie" } }, "keywords": { "Work": true, "Research": true, "VIP": true }, "addressBookIds": { &book1_id: true, &book2_id: true }, })); response.list()[1].assert_is_equal(json!({ "id": &carlos_contact_id, "name": { "components": [ { "kind": "surname", "value": "Rodriguez-Martinez" }, { "kind": "given", "value": "Carlos" }, { "kind": "given2", "value": "Alberto" }, { "kind": "title", "value": "Mr." }, { "kind": "credential", "value": "Jr." } ], "isOrdered": true, "full": "Carlos Rodriguez-Martinez" }, "keywords": { "Marketing": true, "Management": true, "International": true }, "nicknames": { "k2": { "name": "Carlitos" } }, "addressBookIds": { &book1_id: true, &book2_id: true }, })); response.list()[2].assert_is_equal(json!({ "id": acme_contact_id, "addressBookIds": { &book1_id: true, }, "name": { "full": "Acme Business Solutions Ltd." }, "keywords": { "Technology": true, "B2C": true, "Solutions": true, "Services": true } })); // Query tests wait_for_index(¶ms.server).await; let email = if !params.server.search_store().is_mysql() { "sarah.johnson@example.com" } else { "sarah.johnson@example" }; assert_eq!( account .jmap_query( MethodObject::ContactCard, [ ("text", "Sarah"), ("inAddressBook", book1_id.as_str()), ("uid", "urn:uuid:f81d4fae-7dec-11d0-a765-00a0c91e6bf6"), ("email", email), ], ["created"], Vec::<(&str, &str)>::new(), ) .await .ids() .collect::>(), [sarah_contact_id.as_str()] .into_iter() .collect::>() ); // Parse tests account .jmap_method_calls(json!([ [ "Blob/upload", { "create": { "vcard": { "data": [ { "data:asText": r#"BEGIN:VCARD VERSION:4.0 KIND:individual FN:Jane Doe ORG:ABC\, Inc.;North American Division;Marketing END:VCARD"# } ] } } }, "S4" ], [ "ContactCard/parse", { "blobIds": [ "#vcard" ] }, "G4" ] ])) .await .pointer("/methodResponses/1/1/parsed") .unwrap() .as_object() .unwrap() .iter() .next() .unwrap() .1 .assert_is_equal(json!({ "name": { "full": "Jane Doe" }, "version": "1.0", "vCard": { "properties": [ [ "version", {}, "unknown", "4.0" ] ] }, "organizations": { "k1": { "name": "ABC, Inc.", "units": [ { "name": "North American Division" }, { "name": "Marketing" } ] } }, "@type": "Card", "kind": "individual" })); // Deletion tests assert_eq!( account .jmap_destroy( MethodObject::ContactCard, [carlos_contact_id.as_str(), acme_contact_id.as_str()], Vec::<(&str, &str)>::new() ) .await .destroyed() .collect::>(), [carlos_contact_id.as_str(), acme_contact_id.as_str()] .into_iter() .collect::>() ); // CardDAV compatibility tests let account_id = account.id().document_id(); let dav_client = DummyWebDavClient::new( u32::MAX, account.name(), account.secret(), account.emails()[0], ); let resources = params .server .fetch_dav_resources( ¶ms.server.get_access_token(account_id).await.unwrap(), account_id, SyncCollection::AddressBook, ) .await .unwrap(); let path = format!( "{}{}", resources.base_path, resources .paths .iter() .find(|v| v.parent_id.is_some()) .unwrap() .path ); let vcard = dav_client .request("GET", &path, "") .await .with_status(StatusCode::OK) .expect_body() .lines() .map(String::from) .collect::>(); let expected_vcard = TEST_VCARD_1 .lines() .map(String::from) .collect::>(); assert_eq!(vcard, expected_vcard); // Clean up account.destroy_all_addressbooks().await; params.assert_is_empty().await; } fn test_jscontact_1() -> Value { json!({ "uid": "urn:uuid:f81d4fae-7dec-11d0-a765-00a0c91e6bf6", "@type": "Card", "preferredLanguages": { "k1": { "language": "en", "contexts": { "work": true }, "pref": 1 }, "k2": { "language": "fr", "contexts": { "work": true }, "pref": 2 } }, "name": { "full": "Sarah Johnson", "components": [ { "kind": "surname", "value": "Johnson" }, { "kind": "given", "value": "Sarah" }, { "kind": "given2", "value": "Marie" }, { "kind": "title", "value": "Dr." }, { "kind": "credential", "value": "Ph.D." } ], "isOrdered": true }, "cryptoKeys": { "k1": { "uri": "https://pgp.example.com/pks/lookup?op=get&search=sarah.johnson@example.com", "contexts": { "pgp": true } } }, "keywords": { "Work": true, "Research": true, "VIP": true }, "anniversaries": { "k1": { "date": { "@type": "PartialDate", "year": 1985, "month": 4, "day": 15 }, "kind": "birth" }, "k2": { "date": { "@type": "PartialDate", "year": 2010, "month": 6, "day": 10 }, "kind": "wedding" } }, "links": { "k1": { "uri": "https://www.example.com/staff/sjohnson", "contexts": { "work": true } }, "k2": { "uri": "https://www.sarahjohnson.example.com", "contexts": { "private": true } } }, "organizations": { "k1": { "name": "Acme Technologies Inc.", "units": [ { "name": "Research Department" } ] } }, "emails": { "k1": { "address": "sarah.johnson@example.com", "contexts": { "work": true } }, "k2": { "address": "sarahjpersonal@example.com", "contexts": { "private": true, "pref": true } } }, "phones": { "k1": { "number": "+1-555-123-4567", "contexts": { "pref": true }, "features": { "mobile": true, "voice": true } }, "k2": { "number": "+1-555-987-6543", "contexts": { "work": true }, "features": { "voice": true } }, "k3": { "number": "+1-555-456-7890", "contexts": { "private": true }, "features": { "voice": true } } }, "version": "1.0", "addresses": { "k1": { "contexts": { "work": true }, "full": "123 Business Ave\nSuite 400\nNew York, NY 10001\nUSA", "components": [ { "kind": "name", "value": "123 Business Ave" }, { "kind": "locality", "value": "New York" }, { "kind": "region", "value": "NY" }, { "kind": "postcode", "value": "10001" }, { "kind": "country", "value": "USA" } ], "timeZone": "Etc/GMT+5", "coordinates": "40.7128;-74.0060", "isOrdered": true }, "k2": { "contexts": { "private": true, "pref": true }, "full": "456 Residential St\nApt 7B\nBrooklyn, NY 11201\nUSA", "components": [ { "kind": "name", "value": "456 Residential St" }, { "kind": "locality", "value": "Brooklyn" }, { "kind": "region", "value": "NY" }, { "kind": "postcode", "value": "11201" }, { "kind": "country", "value": "USA" } ], "isOrdered": true } }, "titles": { "k1": { "name": "Senior Research Scientist", "kind": "title" }, "k2": { "name": "Team Lead", "kind": "role", "organizationId": "k1" } }, "nicknames": { "k1": { "name": "Sadie" } }, "notes": { "k1": { "note": "Sarah prefers video calls over phone calls. Available Mon-Thu 9-5 EST." } }, "updated": "2022-03-15T13:30:00Z" }) } fn test_jscontact_2() -> Value { json!({ "phones": { "k1": { "number": "+34-611-234-567", "contexts": { "pref": true }, "features": { "mobile": true, "voice": true } }, "k2": { "number": "+34-911-876-543", "contexts": { "work": true }, "features": { "voice": true } }, "k3": { "number": "+34-644-321-987", "contexts": { "private": true }, "features": { "voice": true } }, "k4": { "number": "+34-911-876-544", "features": { "fax": true } } }, "keywords": { "Marketing": true, "Management": true, "International": true }, "kind": "individual", "anniversaries": { "k1": { "date": { "@type": "PartialDate", "month": 6, "day": 23 }, "kind": "birth" }, "k2": { "date": { "@type": "PartialDate", "year": 2015, "month": 8, "day": 9 }, "kind": "wedding" } }, "members": { "urn:uuid:03a0e51f-d1aa-4385-8a53-e29025acd8af": true }, "uid": "urn:uuid:e1ee798b-3d4c-41b0-b217-b9c918e4686a", "name": { "components": [ { "kind": "surname", "value": "Rodriguez-Martinez" }, { "kind": "given", "value": "Carlos" }, { "kind": "given2", "value": "Alberto" }, { "kind": "title", "value": "Mr." }, { "kind": "credential", "value": "Jr." } ], "full": "Carlos Rodriguez-Martinez", "isOrdered": true }, "nicknames": { "k1": { "name": "Charlie" } }, "relatedTo": { "urn:uuid:f81d4fae-7dec-11d0-a765-00a0c91e6bf6": { "relation": { "friend": true } } }, "emails": { "k1": { "address": "carlos.rodriguez@example-corp.com", "contexts": { "work": true, "pref": true } }, "k2": { "address": "carlosrm@personalmail.example", "contexts": { "private": true } } }, "directories": { "k1": { "uri": "https://contacts.example.com/carlosrodriguez.vcf", "kind": "entry" } }, "cryptoKeys": { "k1": { "uri": "https://pgp.example.com/pks/lookup?op=get&search=carlos.rodriguez@example-corp.com", "contexts": { "pgp": true } } }, "version": "1.0", "notes": { "k1": { "note": "Carlos speaks English, Spanish, and Portuguese fluently. Prefers communication via email. Do not contact after 7PM CET." } }, "updated": "2023-07-12T09:21:35Z", "links": { "k1": { "uri": "https://www.example-corp.com/team/carlos", "contexts": { "work": true } }, "k2": { "uri": "https://www.carlosrodriguez.example", "contexts": { "private": true } }, "k3": { "uri": "https://linkedin.com/in/carlosrodriguezm", "contexts": { "social": true } } }, "@type": "Card", "titles": { "k1": { "name": "Digital Marketing Director", "kind": "title" }, "k2": { "name": "Department Head", "kind": "role", "organizationId": "k1" } }, "preferredLanguages": { "k1": { "language": "es", "contexts": { "work": true }, "pref": 1 }, "k2": { "language": "en", "contexts": { "work": true }, "pref": 2 }, "k3": { "language": "pt", "contexts": { "work": true }, "pref": 3 } }, "addresses": { "k1": { "contexts": { "work": true }, "full": "Calle Empresarial 42\nPlanta 3\nMadrid, 28001\nSpain", "components": [ { "kind": "name", "value": "Calle Empresarial 42" }, { "kind": "locality", "value": "Madrid" }, { "kind": "postcode", "value": "28001" }, { "kind": "country", "value": "Spain" } ], "timeZone": "Etc/GMT-1", "coordinates": "40.4168;-3.7038", "isOrdered": true }, "k2": { "contexts": { "private": true, "pref": true }, "full": "Avenida Residencial 15\nPiso 7, Puerta C\nMadrid, 28045\nSpain", "components": [ { "kind": "name", "value": "Avenida Residencial 15" }, { "kind": "locality", "value": "Madrid" }, { "kind": "postcode", "value": "28045" }, { "kind": "country", "value": "Spain" } ], "isOrdered": true } }, "organizations": { "k1": { "name": "Global Solutions S.L.", "units": [ { "name": "Marketing Division" } ] } } }) } fn test_jscontact_3() -> Value { json!({ "kind": "org", "organizations": { "k1": { "name": "Acme Business Solutions Ltd.", "units": [ { "name": "Technology Division" } ] } }, "preferredLanguages": { "k1": { "language": "en", "contexts": { "work": true }, "pref": 1 }, "k2": { "language": "de", "contexts": { "work": true }, "pref": 2 }, "k3": { "language": "fr", "contexts": { "work": true }, "pref": 3 } }, "directories": { "k1": { "uri": "https://directory.example.com/acme.vcf", "kind": "entry" } }, "cryptoKeys": { "k1": { "uri": "https://pgp.example.com/pks/lookup?op=get&search=info@acme-solutions.example", "contexts": { "pgp": true } } }, "links": { "k1": { "uri": "https://www.acme-solutions.example", "contexts": { "work": true } }, "k2": { "uri": "https://support.acme-solutions.example", "contexts": { "support": true } } }, "name": { "full": "Acme Business Solutions Ltd." }, "notes": { "k1": { "note": "Business hours: Mon-Fri 9:00-17:30 GMT. Closed on UK bank holidays. VAT Reg: GB123456789" } }, "uid": "urn:uuid:a9e95948-7b1c-46e8-bd85-c729a9e910f2", "@type": "Card", "prodId": "-//Example Corp.//Contact Manager 3.0//EN", "version": "1.0", "emails": { "k1": { "address": "info@acme-solutions.example", "contexts": { "work": true, "pref": true } }, "k2": { "address": "support@acme-solutions.example", "contexts": { "support": true } }, "k3": { "address": "sales@acme-solutions.example", "contexts": { "sales": true } } }, "phones": { "k1": { "number": "+44-20-1234-5678", "contexts": { "work": true, "pref": true }, "features": { "voice": true } }, "k2": { "number": "+44-20-1234-5679", "features": { "fax": true } }, "k3": { "number": "+44-800-987-6543", "contexts": { "support": true } } }, "addresses": { "k1": { "contexts": { "work": true }, "full": "10 Enterprise Way\nTech Park\nLondon, EC1A 1BB\nUnited Kingdom", "components": [ { "kind": "name", "value": "10 Enterprise Way, Tech Park" }, { "kind": "locality", "value": "London" }, { "kind": "postcode", "value": "EC1A 1BB" }, { "kind": "country", "value": "United Kingdom" } ], "timeZone": "Etc/UTC", "coordinates": "51.5074;-0.1278", "isOrdered": true }, "k2": { "contexts": { "branch": true }, "full": "25 Innovation Street\nManchester, M1 5QF\nUnited Kingdom", "components": [ { "kind": "name", "value": "25 Innovation Street" }, { "kind": "locality", "value": "Manchester" }, { "kind": "postcode", "value": "M1 5QF" }, { "kind": "country", "value": "United Kingdom" } ], "isOrdered": true } }, "updated": "2023-04-15T15:30:00Z", "keywords": { "Technology": true, "B2B": true, "Solutions": true, "Services": true }, "relatedTo": { "urn:uuid:b9e93fdb-4d34-45fa-a1e2-47da0428c4a1": { "relation": { "contact": true } }, "urn:uuid:c8e74dfe-6b34-45fa-b1e2-47ea0428c4b2": { "relation": { "contact": true } } } }) } fn test_jscontact_4() -> Value { json!({ "@type": "Card", "version": "1.0", "kind": "individual", "name": { "@type": "Name", "full": "Temporary Contact" }}) } const TEST_VCARD_1: &str = r#"BEGIN:VCARD VERSION:4.0 UID:urn:uuid:f81d4fae-7dec-11d0-a765-00a0c91e6bf6 LANG;TYPE=WORK;PREF=1;PROP-ID=k1:en LANG;TYPE=WORK;PREF=2;PROP-ID=k2:fr FN:Sarah O'Connor N;JSCOMPS=";0;1;2;3;4":O'Connor;Sarah;Marie;Dr.;Ph.D.;; KEY;TYPE=PGP;PROP-ID=k1:https://pgp.example.com/pks/lookup?op=get&search=sar ah.johnson@example.com CATEGORIES:Work,Research,VIP BDAY;PROP-ID=k1:19850415 ANNIVERSARY;PROP-ID=k2:20100610 URL;TYPE=WORK;PROP-ID=k1:https://www.example.com/staff/sjohnson URL;TYPE=HOME;PROP-ID=k2:https://www.sarahjohnson.example.com ORG;PROP-ID=k1:Acme Technologies Inc.;Research Department EMAIL;TYPE=WORK;PROP-ID=k1:sarah.johnson@example.com EMAIL;TYPE=HOME,PREF;PROP-ID=k2:sarahjpersonal@example.com TEL;TYPE=PREF,CELL,VOICE;PROP-ID=k1:+1-555-123-4567 TEL;TYPE=WORK,VOICE;PROP-ID=k2:+1-555-987-6543 TEL;TYPE=HOME,VOICE;PROP-ID=k3:+1-555-456-7890 ADR;TYPE=WORK;LABEL="123 Business Ave\nSuite 400\nNew York, NY 10001\nUSA"; TZ=Etc/GMT+5;GEO="40.7128;-74.0060";PROP-ID=k1;JSCOMPS=";11;3;4;5;6":;;123 B usiness Ave;New York;NY;10001;USA;;;;;123 Business Ave;;;;;; ADR;TYPE=HOME,PREF;LABEL="456 Residential St\nApt 7B\nBrooklyn, NY 11201\nU SA";PROP-ID=k2;JSCOMPS=";11;3;4;5;6":;;456 Residential St;Brooklyn;NY;11201; USA;;;;;456 Residential St;;;;;; TITLE;PROP-ID=k1:Senior Research Scientist JSPROP;JSPTR=titles/k2/organizationId:"k1" ROLE;PROP-ID=k2:Team Lead NICKNAME;PROP-ID=k1:Sadie NOTE;PROP-ID=k1:Sarah prefers video calls over phone calls. Available Mon-Th u 9-5 EST. REV:20220315T133000Z END:VCARD "#; ================================================ FILE: tests/src/jmap/contacts/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod acl; pub mod addressbook; pub mod contact; ================================================ FILE: tests/src/jmap/core/blob.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{jmap::JMAPTest, store::cleanup::store_blob_expire_all}; use email::mailbox::INBOX_ID; use serde_json::{Value, json}; use types::id::Id; pub async fn test(params: &mut JMAPTest) { println!("Running blob tests..."); let server = params.server.clone(); let account = params.account("jdoe@example.com"); store_blob_expire_all(&server.core.storage.data).await; // Blob/set simple test let response = account.jmap_method_call("Blob/upload", json!({ "create": { "abc": { "data" : [ { "data:asBase64": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQMAAAAl21bKAAAAA1BMVEX/AAAZ4gk3AAAAAXRSTlN/gFy0ywAAAApJREFUeJxjYgAAAAYAAzY3fKgAAAAASUVORK5CYII=" } ], "type": "image/png" } } })).await; assert_eq!( response .pointer("/methodResponses/0/1/created/abc/type") .and_then(|v| v.as_str()) .unwrap_or_default(), "image/png", "Response: {:?}", response ); assert_eq!( response .pointer("/methodResponses/0/1/created/abc/size") .and_then(|v| v.as_i64()) .unwrap_or_default(), 95, "Response: {:?}", response ); // Blob/get simple test let blob_id = account .jmap_method_call( "Blob/upload", json!({ "create": { "abc": { "data" : [ { "data:asText": "The quick brown fox jumped over the lazy dog." } ] } } }), ) .await .pointer("/methodResponses/0/1/created/abc/id") .and_then(|v| v.as_str()) .unwrap() .to_string(); let response = account .jmap_method_calls(json!([[ "Blob/get", { "ids" : [ blob_id ], "properties" : [ "data:asText", "digest:sha", "size" ] }, "R1" ], [ "Blob/get", { "ids" : [ blob_id ], "properties" : [ "data:asText", "digest:sha", "digest:sha-256", "size" ], "offset" : 4, "length" : 9 }, "R2" ] ])) .await; for (pointer, expected) in [ ( "/methodResponses/0/1/list/0/data:asText", "The quick brown fox jumped over the lazy dog.", ), ( "/methodResponses/0/1/list/0/digest:sha", "wIVPufsDxBzOOALLDSIFKebu+U4=", ), ("/methodResponses/0/1/list/0/size", "45"), ("/methodResponses/1/1/list/0/data:asText", "quick bro"), ( "/methodResponses/1/1/list/0/digest:sha", "QiRAPtfyX8K6tm1iOAtZ87Xj3Ww=", ), ( "/methodResponses/1/1/list/0/digest:sha-256", "gdg9INW7lwHK6OQ9u0dwDz2ZY/gubi0En0xlFpKt0OA=", ), ] { assert_eq!( response .pointer(pointer) .and_then(|v| match v { Value::String(s) => Some(s.to_string()), Value::Number(n) => Some(n.to_string()), _ => None, }) .unwrap_or_default(), expected, "Pointer {pointer:?} Response: {response:?}", ); } store_blob_expire_all(&server.core.storage.data).await; // Blob/upload Complex Example let response = account .jmap_method_calls(json!([ [ "Blob/upload", { "create": { "b4": { "data": [ { "data:asText": "The quick brown fox jumped over the lazy dog." } ] } } }, "S4" ], [ "Blob/upload", { "create": { "cat": { "data": [ { "data:asText": "How" }, { "blobId": "#b4", "length": 7, "offset": 3 }, { "data:asText": "was t" }, { "blobId": "#b4", "length": 1, "offset": 1 }, { "data:asBase64": "YXQ/" } ] } } }, "CAT" ], [ "Blob/get", { "properties": [ "data:asText", "size" ], "ids": [ "#cat" ] }, "G4" ] ])) .await; for (pointer, expected) in [ ( "/methodResponses/2/1/list/0/data:asText", "How quick was that?", ), ("/methodResponses/2/1/list/0/size", "19"), ] { assert_eq!( response .pointer(pointer) .and_then(|v| match v { Value::String(s) => Some(s.to_string()), Value::Number(n) => Some(n.to_string()), _ => None, }) .unwrap_or_default(), expected, "Pointer {pointer:?} Response: {response:?}", ); } store_blob_expire_all(&server.core.storage.data).await; // Blob/get Example with Range and Encoding Errors let response = account.jmap_method_calls(json!([ [ "Blob/upload", { "create": { "b1": { "data": [ { "data:asBase64": "VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wZWQgb3ZlciB0aGUggYEgZG9nLg==" } ] }, "b2": { "data": [ { "data:asText": "hello world" } ], "type" : "text/plain" } } }, "S1" ], [ "Blob/get", { "ids": [ "#b1", "#b2" ] }, "G1" ], [ "Blob/get", { "ids": [ "#b1", "#b2" ], "properties": [ "data:asText", "size" ] }, "G2" ], [ "Blob/get", { "ids": [ "#b1", "#b2" ], "properties": [ "data:asBase64", "size" ] }, "G3" ], [ "Blob/get", { "offset": 0, "length": 5, "ids": [ "#b1", "#b2" ] }, "G4" ], [ "Blob/get", { "offset": 20, "length": 100, "ids": [ "#b1", "#b2" ] }, "G5" ] ])).await; for (pointer, expected) in [ ( "/methodResponses/1/1/list/0/data:asBase64", "VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wZWQgb3ZlciB0aGUggYEgZG9nLg==", ), ("/methodResponses/1/1/list/1/data:asText", "hello world"), ("/methodResponses/2/1/list/0/isEncodingProblem", "true"), ("/methodResponses/2/1/list/1/data:asText", "hello world"), ( "/methodResponses/3/1/list/0/data:asBase64", "VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wZWQgb3ZlciB0aGUggYEgZG9nLg==", ), ( "/methodResponses/3/1/list/1/data:asBase64", "aGVsbG8gd29ybGQ=", ), ("/methodResponses/4/1/list/0/data:asText", "The q"), ("/methodResponses/4/1/list/1/data:asText", "hello"), ("/methodResponses/5/1/list/0/isEncodingProblem", "true"), ("/methodResponses/5/1/list/0/isTruncated", "true"), ("/methodResponses/5/1/list/1/isTruncated", "true"), ] { assert_eq!( response .pointer(pointer) .and_then(|v| match v { Value::String(s) => Some(s.to_string()), Value::Number(n) => Some(n.to_string()), Value::Bool(b) => Some(b.to_string()), _ => None, }) .unwrap_or_default(), expected, "Pointer {pointer:?} Response: {response:?}", ); } store_blob_expire_all(&server.core.storage.data).await; // Blob/lookup let client = account.client(); let blob_id = client .email_import( concat!( "From: bill@example.com\r\n", "To: jdoe@example.com\r\n", "Subject: TPS Report\r\n", "\r\n", "I'm going to need those TPS reports ASAP. ", "So, if you could do that, that'd be great." ) .as_bytes() .to_vec(), [&Id::from(INBOX_ID).to_string()], None::>, None, ) .await .unwrap() .take_blob_id(); let response = account .jmap_method_call( "Blob/lookup", json!({ "typeNames": [ "Mailbox", "Thread", "Email" ], "ids": [ blob_id, "not-a-blob" ] }), ) .await; for pointer in [ "/methodResponses/0/1/list/0/matchedIds/Email", "/methodResponses/0/1/list/0/matchedIds/Mailbox", "/methodResponses/0/1/list/0/matchedIds/Thread", ] { assert_eq!( response .pointer(pointer) .and_then(|v| v.as_array()) .map(|arr| arr.len()) .unwrap_or_default(), 1, "Pointer {pointer:?} Response: {response:#?}", ); } // Remove test data params.destroy_all_mailboxes(account).await; params.assert_is_empty().await; } ================================================ FILE: tests/src/jmap/core/event_source.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::{JMAPTest, mail::delivery::SmtpConnection}; use email::mailbox::INBOX_ID; use futures::StreamExt; use jmap_client::{ DataType, event_source::{Changes, PushNotification}, mailbox::Role, }; use std::time::Duration; use store::ahash::AHashSet; use tokio::sync::mpsc; use types::id::Id; pub async fn test(params: &mut JMAPTest) { println!("Running EventSource tests..."); // Create test account let account = params.account("jdoe@example.com"); let client = account.client(); let mut changes = client .event_source(None::>, false, 1.into(), None) .await .unwrap(); let (event_tx, mut event_rx) = mpsc::channel::(100); tokio::spawn(async move { while let Some(change) = changes.next().await { if let Err(_err) = event_tx .send(match change.unwrap() { PushNotification::StateChange(changes) => changes, PushNotification::CalendarAlert(_) => unreachable!(), }) .await { //println!("Error sending event: {}", _err); break; } } }); assert_ping(&mut event_rx).await; // Create mailbox and expect state change let mailbox_id = client .mailbox_create("EventSource Test", None::, Role::None) .await .unwrap() .take_id(); assert_state(&mut event_rx, account.id_string(), &[DataType::Mailbox]).await; // Multiple changes should be grouped and delivered in intervals for num in 0..5 { client .mailbox_update_sort_order(&mailbox_id, num) .await .unwrap(); } assert_state(&mut event_rx, account.id_string(), &[DataType::Mailbox]).await; assert_ping(&mut event_rx).await; // Pings are only received in cfg(test) // Ingest email and expect state change let mut lmtp = SmtpConnection::connect().await; lmtp.ingest( "bill@example.com", &["jdoe@example.com"], concat!( "From: bill@example.com\r\n", "To: jdoe@example.com\r\n", "Subject: TPS Report\r\n", "\r\n", "I'm going to need those TPS reports ASAP. ", "So, if you could do that, that'd be great." ), ) .await; lmtp.quit().await; assert_state( &mut event_rx, account.id_string(), &[ DataType::EmailDelivery, DataType::Email, DataType::Thread, DataType::Mailbox, ], ) .await; assert_ping(&mut event_rx).await; // Destroy mailbox client.mailbox_destroy(&mailbox_id, true).await.unwrap(); assert_state(&mut event_rx, account.id_string(), &[DataType::Mailbox]).await; // Destroy Inbox client .mailbox_destroy(&Id::from(INBOX_ID).to_string(), true) .await .unwrap(); assert_state( &mut event_rx, account.id_string(), &[DataType::Email, DataType::Thread, DataType::Mailbox], ) .await; assert_ping(&mut event_rx).await; assert_ping(&mut event_rx).await; params.destroy_all_mailboxes(account).await; params.assert_is_empty().await; } async fn assert_state( event_rx: &mut mpsc::Receiver, account_id: &str, state: &[DataType], ) { match tokio::time::timeout(Duration::from_millis(700), event_rx.recv()).await { Ok(Some(changes)) => { assert_eq!( changes .changes(account_id) .unwrap() .map(|x| x.0) .collect::>(), state.iter().collect::>() ); } result => { panic!("Timeout waiting for event {:?}: {:?}", state, result); } } } async fn assert_ping(event_rx: &mut mpsc::Receiver) { match tokio::time::timeout(Duration::from_millis(1100), event_rx.recv()).await { Ok(Some(changes)) => { assert!(changes.changes("ping").is_some(),); } _ => { panic!("Did not receive ping."); } } } ================================================ FILE: tests/src/jmap/core/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod blob; pub mod event_source; pub mod push_subscription; pub mod websocket; ================================================ FILE: tests/src/jmap/core/push_subscription.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{AssertConfig, add_test_certs, jmap::JMAPTest}; use base64::{Engine, engine::general_purpose}; use common::{Caches, Core, Data, Inner, config::server::Listeners, listener::SessionData}; use ece::EcKeyComponents; use http_proto::{HtmlResponse, ToHttpResponse, request::fetch_body}; use hyper::{StatusCode, body, header::CONTENT_ENCODING, server::conn::http1, service::service_fn}; use hyper_util::rt::TokioIo; use jmap_client::{mailbox::Role, push_subscription::Keys}; use jmap_proto::{response::status::PushObject, types::state::State}; use services::state_manager::ece::ece_encrypt; use std::{ sync::{ Arc, atomic::{AtomicBool, Ordering}, }, time::Duration, }; use store::ahash::AHashSet; use tokio::sync::mpsc; use types::{id::Id, type_state::DataType}; use utils::{config::Config, map::vec_map::VecMap}; const SERVER: &str = r#" [server] hostname = "'jmap-push.example.org'" [http] url = "'https://127.0.0.1:9000'" [server.listener.jmap] bind = ['127.0.0.1:9000'] protocol = 'http' tls.implicit = true [server.socket] reuse-addr = true [certificate.default] cert = '%{file:{CERT}}%' private-key = '%{file:{PK}}%' default = true "#; pub async fn test(params: &mut JMAPTest) { println!("Running Push Subscription tests..."); // ECE roundtrip test ece_roundtrip(); // Create test account let account = params.account("jdoe@example.com"); let client = account.client(); // Create channels let (event_tx, mut event_rx) = mpsc::channel::(100); // Create subscription keys let (keypair, auth_secret) = ece::generate_keypair_and_auth_secret().unwrap(); let pubkey = keypair.pub_as_raw().unwrap(); let keys = Keys::new(&pubkey, &auth_secret); let push_server = Arc::new(PushServer { keypair: keypair.raw_components().unwrap(), auth_secret: auth_secret.to_vec(), tx: event_tx, fail_requests: false.into(), }); // Start mock push server let mut settings = Config::new(add_test_certs(SERVER)).unwrap(); settings.resolve_all_macros().await; let mock_inner = Arc::new(Inner { shared_core: Core::parse(&mut settings, Default::default(), Default::default()) .await .into_shared(), data: Data::parse(&mut settings), cache: Caches::parse(&mut settings), ..Default::default() }); settings.errors.clear(); settings.warnings.clear(); let mut servers = Listeners::parse(&mut settings); servers.parse_tcp_acceptors(&mut settings, mock_inner.clone()); // Start JMAP server servers.bind_and_drop_priv(&mut settings); settings.assert_no_errors(); let _shutdown_tx = servers.spawn(|server, acceptor, shutdown_rx| { server.spawn( SessionManager::from(push_server.clone()), mock_inner.clone(), acceptor, shutdown_rx, ); }); // Register push notification (no encryption) let push_id = client .push_subscription_create("123", "https://127.0.0.1:9000/push", None) .await .unwrap() .take_id(); // Expect push verification let verification = expect_push(&mut event_rx).await.unwrap_verification(); assert_eq!(verification.push_subscription_id, push_id); // Update verification code client .push_subscription_verify(&push_id, verification.verification_code) .await .unwrap(); // Create a mailbox and expect a state change let mailbox_id = client .mailbox_create("PushSubscription Test", None::, Role::None) .await .unwrap() .take_id(); assert_state(&mut event_rx, account.id(), &[DataType::Mailbox]).await; // Receive states just for the requested types client .push_subscription_update_types(&push_id, [jmap_client::DataType::Email].into()) .await .unwrap(); client .mailbox_update_sort_order(&mailbox_id, 123) .await .unwrap(); expect_nothing(&mut event_rx).await; // Destroy subscription client.push_subscription_destroy(&push_id).await.unwrap(); // Only one verification per minute is allowed let push_id = client .push_subscription_create("invalid", "https://127.0.0.1:9000/push", None) .await .unwrap() .take_id(); expect_nothing(&mut event_rx).await; client.push_subscription_destroy(&push_id).await.unwrap(); // Register push notification (with encryption) let push_id = client .push_subscription_create( "123", "https://127.0.0.1:9000/push?skip_checks=true", // skip_checks only works in cfg(test) keys.into(), ) .await .unwrap() .take_id(); // Expect push verification let verification = expect_push(&mut event_rx).await.unwrap_verification(); assert_eq!(verification.push_subscription_id, push_id); // Update verification code client .push_subscription_verify(&push_id, verification.verification_code) .await .unwrap(); // Failed deliveries should be re-attempted push_server.fail_requests.store(true, Ordering::Relaxed); client .mailbox_update_sort_order(&mailbox_id, 101) .await .unwrap(); tokio::time::sleep(Duration::from_millis(200)).await; push_server.fail_requests.store(false, Ordering::Relaxed); assert_state(&mut event_rx, account.id(), &[DataType::Mailbox]).await; // Make a mailbox change and expect state change client .mailbox_rename(&mailbox_id, "My Mailbox") .await .unwrap(); assert_state(&mut event_rx, account.id(), &[DataType::Mailbox]).await; //expect_nothing(&mut event_rx).await; // Multiple change updates should be grouped and pushed in intervals for num in 0..5 { client .mailbox_update_sort_order(&mailbox_id, num) .await .unwrap(); } assert_state(&mut event_rx, account.id(), &[DataType::Mailbox]).await; expect_nothing(&mut event_rx).await; // Destroy mailbox client.push_subscription_destroy(&push_id).await.unwrap(); client.mailbox_destroy(&mailbox_id, true).await.unwrap(); expect_nothing(&mut event_rx).await; params.destroy_all_mailboxes(account).await; params.assert_is_empty().await; } #[derive(Clone)] pub struct SessionManager { pub inner: Arc, } impl From> for SessionManager { fn from(inner: Arc) -> Self { SessionManager { inner } } } pub struct PushServer { keypair: EcKeyComponents, auth_secret: Vec, tx: mpsc::Sender, fail_requests: AtomicBool, } #[derive(serde::Deserialize, Debug)] #[serde(untagged)] enum PushMessage { PushObject(PushObject), Verification(PushVerification), } impl PushMessage { pub fn unwrap_state_change(self) -> VecMap> { match self { PushMessage::PushObject(PushObject::StateChange { changed }) => changed, _ => panic!("Expected PushObject"), } } pub fn unwrap_verification(self) -> PushVerification { match self { PushMessage::Verification(verification) => verification, _ => panic!("Expected Verification"), } } } #[derive(serde::Deserialize, Debug)] enum PushVerificationType { PushVerification, } #[derive(serde::Deserialize, Debug)] struct PushVerification { #[serde(rename = "@type")] _type: PushVerificationType, #[serde(rename = "pushSubscriptionId")] pub push_subscription_id: String, #[serde(rename = "verificationCode")] pub verification_code: String, } impl common::listener::SessionManager for SessionManager { #[allow(clippy::manual_async_fn)] fn handle( self, session: SessionData, ) -> impl std::future::Future + Send { async move { let push = self.inner; let _ = http1::Builder::new() .keep_alive(false) .serve_connection( TokioIo::new(session.stream), service_fn(|mut req: hyper::Request| { let push = push.clone(); async move { if push.fail_requests.load(Ordering::Relaxed) { return Ok(HtmlResponse::with_status( StatusCode::TOO_MANY_REQUESTS, "too many requests".to_string(), ) .into_http_response() .build()); } let is_encrypted = req .headers() .get(CONTENT_ENCODING) .is_some_and(|encoding| encoding.to_str().unwrap() == "aes128gcm"); let body = fetch_body(&mut req, 1024 * 1024, 0).await.unwrap(); let message = serde_json::from_slice::(&if is_encrypted { ece::decrypt( &push.keypair, &push.auth_secret, &general_purpose::URL_SAFE.decode(body).unwrap(), ) .unwrap() } else { body }) .unwrap(); //println!("Push received ({}): {:?}", is_encrypted, message); push.tx.send(message).await.unwrap(); Ok::<_, hyper::Error>( HtmlResponse::new("ok".to_string()) .into_http_response() .build(), ) } }), ) .await; } } #[allow(clippy::manual_async_fn)] fn shutdown(&self) -> impl std::future::Future + Send { async {} } } async fn expect_push(event_rx: &mut mpsc::Receiver) -> PushMessage { match tokio::time::timeout(Duration::from_millis(1500), event_rx.recv()).await { Ok(Some(push)) => { //println!("Push received: {:?}", push); push } result => { panic!("Timeout waiting for push: {:?}", result); } } } async fn expect_nothing(event_rx: &mut mpsc::Receiver) { match tokio::time::timeout(Duration::from_millis(1000), event_rx.recv()).await { Err(_) => {} message => { panic!("Received a message when expecting nothing: {:?}", message); } } } async fn assert_state(event_rx: &mut mpsc::Receiver, id: &Id, state: &[DataType]) { assert_eq!( expect_push(event_rx) .await .unwrap_state_change() .get(id) .unwrap() .iter() .map(|x| x.0) .collect::>(), state.iter().collect::>() ); } fn ece_roundtrip() { for len in [1, 2, 5, 16, 256, 1024, 2048, 4096, 1024 * 1024] { let (keypair, auth_secret) = ece::generate_keypair_and_auth_secret().unwrap(); let bytes: Vec = (0..len).map(|_| store::rand::random::()).collect(); let encrypted_bytes = ece_encrypt(&keypair.pub_as_raw().unwrap(), &auth_secret, &bytes).unwrap(); let decrypted_bytes = ece::decrypt( &keypair.raw_components().unwrap(), &auth_secret, &encrypted_bytes, ) .unwrap(); assert_eq!(bytes, decrypted_bytes, "len: {}", len); } } ================================================ FILE: tests/src/jmap/core/websocket.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::JMAPTest; use ahash::AHashSet; use futures::StreamExt; use jmap_client::{ DataType, PushObject, client_ws::WebSocketMessage, core::{ response::{Response, TaggedMethodResponse}, set::SetObject, }, }; use std::time::Duration; use tokio::sync::mpsc; pub async fn test(params: &mut JMAPTest) { println!("Running WebSockets tests..."); // Authenticate all accounts let account = params.account("jdoe@example.com"); let client = account.client(); let mut ws_stream = client.connect_ws().await.unwrap(); let (stream_tx, mut stream_rx) = mpsc::channel::(100); tokio::spawn(async move { while let Some(change) = ws_stream.next().await { stream_tx.send(change.unwrap()).await.unwrap(); } }); // Create mailbox let mut request = client.build(); let create_id = request .set_mailbox() .create() .name("WebSocket Test") .create_id() .unwrap(); let request_id = request.send_ws().await.unwrap(); let mut response = expect_response(&mut stream_rx).await; assert_eq!(request_id, response.request_id().unwrap()); let mailbox_id = response .pop_method_response() .unwrap() .unwrap_set_mailbox() .unwrap() .created(&create_id) .unwrap() .take_id(); // Enable push notifications client .enable_push_ws(None::>, None::<&str>) .await .unwrap(); // Make changes over standard HTTP and expect a push notification via WebSockets client .mailbox_update_sort_order(&mailbox_id, 1) .await .unwrap(); assert_state(&mut stream_rx, account.id_string(), &[DataType::Mailbox]).await; // Multiple changes should be grouped and delivered in intervals for num in 0..5 { client .mailbox_update_sort_order(&mailbox_id, num) .await .unwrap(); } tokio::time::sleep(Duration::from_millis(500)).await; assert_state(&mut stream_rx, account.id_string(), &[DataType::Mailbox]).await; expect_nothing(&mut stream_rx).await; // Disable push notifications client.disable_push_ws().await.unwrap(); // No more changes should be received let mut request = client.build(); request.set_mailbox().destroy([&mailbox_id]); request.send_ws().await.unwrap(); expect_response(&mut stream_rx) .await .pop_method_response() .unwrap() .unwrap_set_mailbox() .unwrap() .destroyed(&mailbox_id) .unwrap(); expect_nothing(&mut stream_rx).await; params.destroy_all_mailboxes(account).await; params.assert_is_empty().await; } async fn expect_response( stream_rx: &mut mpsc::Receiver, ) -> Response { match tokio::time::timeout(Duration::from_millis(100), stream_rx.recv()).await { Ok(Some(message)) => match message { WebSocketMessage::Response(response) => response, _ => panic!("Expected response, got: {:?}", message), }, result => { panic!("Timeout waiting for websocket: {:?}", result); } } } async fn assert_state( stream_rx: &mut mpsc::Receiver, id: &str, state: &[DataType], ) { match tokio::time::timeout(Duration::from_millis(700), stream_rx.recv()).await { Ok(Some(message)) => match message { WebSocketMessage::PushNotification(PushObject::StateChange { changed }) => { assert_eq!( changed .get(id) .unwrap() .keys() .collect::>(), state.iter().collect::>() ); } _ => panic!("Expected state change, got: {:?}", message), }, result => { panic!("Timeout waiting for websocket: {:?}", result); } } } async fn expect_nothing(stream_rx: &mut mpsc::Receiver) { match tokio::time::timeout(Duration::from_millis(1000), stream_rx.recv()).await { Err(_) => {} message => { panic!("Received a message when expecting nothing: {:?}", message); } } } ================================================ FILE: tests/src/jmap/files/acl.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::{JMAPTest, JmapUtils}; use jmap_proto::{ object::{file_node::FileNodeProperty, share_notification::ShareNotificationProperty}, request::method::MethodObject, }; use serde_json::json; pub async fn test(params: &mut JMAPTest) { println!("Running File Storage ACL tests..."); let john = params.account("jdoe@example.com"); let jane = params.account("jane.smith@example.com"); let john_id = john.id_string().to_string(); let jane_id = jane.id_string().to_string(); // Create test folders let response = john .jmap_create( MethodObject::FileNode, [json!({ "name": "Test #1", })], Vec::<(&str, &str)>::new(), ) .await; let john_folder_id = response.created(0).id().to_string(); // Verify myRights john.jmap_get( MethodObject::FileNode, [ FileNodeProperty::Id, FileNodeProperty::Name, FileNodeProperty::MyRights, FileNodeProperty::ShareWith, ], [john_folder_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_folder_id, "name": "Test #1", "myRights": { "mayRead": true, "mayWrite": true, "mayShare": true }, "shareWith": {} })); // Obtain share notifications let mut jane_share_change_id = jane .jmap_get( MethodObject::ShareNotification, Vec::<&str>::new(), Vec::<&str>::new(), ) .await .state() .to_string(); // Make sure Jane has no access assert_eq!( jane.jmap_get_account( john, MethodObject::FileNode, Vec::<&str>::new(), [john_folder_id.as_str()], ) .await .method_response() .typ(), "forbidden" ); // Share folder with Jane john.jmap_update( MethodObject::FileNode, [( &john_folder_id, json!({ "shareWith": { &jane_id : { "mayRead": true, } } }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&john_folder_id); john.jmap_get( MethodObject::FileNode, [ FileNodeProperty::Id, FileNodeProperty::Name, FileNodeProperty::ShareWith, ], [john_folder_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_folder_id, "name": "Test #1", "shareWith": { &jane_id : { "mayRead": true, "mayWrite": false, "mayShare": false } } })); // Verify Jane can access the contact jane.jmap_get_account( john, MethodObject::FileNode, [ FileNodeProperty::Id, FileNodeProperty::Name, FileNodeProperty::MyRights, ], [john_folder_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_folder_id, "name": "Test #1", "myRights": { "mayRead": true, "mayWrite": false, "mayShare": false } })); // Verify Jane received a share notification let response = jane .jmap_changes(MethodObject::ShareNotification, &jane_share_change_id) .await; jane_share_change_id = response.new_state().to_string(); let changes = response.changes().collect::>(); assert_eq!(changes.len(), 1); let share_id = changes[0].as_created(); jane.jmap_get( MethodObject::ShareNotification, [ ShareNotificationProperty::Id, ShareNotificationProperty::ChangedBy, ShareNotificationProperty::ObjectType, ShareNotificationProperty::ObjectAccountId, ShareNotificationProperty::ObjectId, ShareNotificationProperty::OldRights, ShareNotificationProperty::NewRights, ShareNotificationProperty::Name, ], [share_id], ) .await .list()[0] .assert_is_equal(json!({ "id": &share_id, "changedBy": { "principalId": &john_id, "name": "John Doe", "email": "jdoe@example.com" }, "objectType": "FileNode", "objectAccountId": &john_id, "objectId": &john_folder_id, "oldRights": { "mayRead": false, "mayWrite": false, "mayShare": false }, "newRights": { "mayRead": true, "mayWrite": false, "mayShare": false }, "name": null })); // Updating and deleting should fail assert_eq!( jane.jmap_update_account( john, MethodObject::FileNode, [(&john_folder_id, json!({}))], Vec::<(&str, &str)>::new(), ) .await .not_updated(&john_folder_id) .description(), "You are not allowed to modify this file node." ); assert_eq!( jane.jmap_destroy_account( john, MethodObject::FileNode, [&john_folder_id], Vec::<(&str, &str)>::new(), ) .await .not_destroyed(&john_folder_id) .description(), "You are not allowed to delete this file node." ); // Grant Jane write access john.jmap_update( MethodObject::FileNode, [( &john_folder_id, json!({ format!("shareWith/{jane_id}/mayWrite"): true, }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&john_folder_id); jane.jmap_get_account( john, MethodObject::FileNode, [ FileNodeProperty::Id, FileNodeProperty::Name, FileNodeProperty::MyRights, ], [john_folder_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_folder_id, "name": "Test #1", "myRights": { "mayRead": true, "mayWrite": true, "mayShare": false } })); // Verify Jane received a share notification with the updated rights let response = jane .jmap_changes(MethodObject::ShareNotification, &jane_share_change_id) .await; jane_share_change_id = response.new_state().to_string(); let changes = response.changes().collect::>(); assert_eq!(changes.len(), 1); let share_id = changes[0].as_created(); jane.jmap_get( MethodObject::ShareNotification, [ ShareNotificationProperty::Id, ShareNotificationProperty::ChangedBy, ShareNotificationProperty::ObjectType, ShareNotificationProperty::ObjectAccountId, ShareNotificationProperty::ObjectId, ShareNotificationProperty::OldRights, ShareNotificationProperty::NewRights, ShareNotificationProperty::Name, ], [share_id], ) .await .list()[0] .assert_is_equal(json!({ "id": &share_id, "changedBy": { "principalId": &john_id, "name": "John Doe", "email": "jdoe@example.com" }, "objectType": "FileNode", "objectAccountId": &john_id, "objectId": &john_folder_id, "oldRights": { "mayRead": true, "mayWrite": false, "mayShare": false }, "newRights": { "mayRead": true, "mayWrite": true, "mayShare": false }, "name": null })); // Creating a root folder should fail assert_eq!( jane.jmap_create_account( john, MethodObject::FileNode, [json!({ "name": "A new shared folder", })], Vec::<(&str, &str)>::new() ) .await .not_created(0) .description(), "Cannot create top-level folder in a shared account." ); // Update John's folder name jane.jmap_update_account( john, MethodObject::FileNode, [( &john_folder_id, json!({ "name": "Jane's updated name", }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&john_folder_id); jane.jmap_get_account( john, MethodObject::FileNode, [FileNodeProperty::Id, FileNodeProperty::Name], [john_folder_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_folder_id, "name": "Jane's updated name", })); // Revoke Jane's access john.jmap_update( MethodObject::FileNode, [( &john_folder_id, json!({ format!("shareWith/{jane_id}"): () }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&john_folder_id); john.jmap_get( MethodObject::FileNode, [ FileNodeProperty::Id, FileNodeProperty::Name, FileNodeProperty::ShareWith, ], [john_folder_id.as_str()], ) .await .list()[0] .assert_is_equal(json!({ "id": john_folder_id, "name": "Jane's updated name", "shareWith": {} })); // Verify Jane can no longer access the folder or its contacts assert_eq!( jane.jmap_get_account( john, MethodObject::FileNode, Vec::<&str>::new(), [john_folder_id.as_str()], ) .await .method_response() .typ(), "forbidden" ); // Verify Jane received a share notification with the updated rights let response = jane .jmap_changes(MethodObject::ShareNotification, &jane_share_change_id) .await; let changes = response.changes().collect::>(); assert_eq!(changes.len(), 1); let share_id = changes[0].as_created(); jane.jmap_get( MethodObject::ShareNotification, [ ShareNotificationProperty::Id, ShareNotificationProperty::ChangedBy, ShareNotificationProperty::ObjectType, ShareNotificationProperty::ObjectAccountId, ShareNotificationProperty::ObjectId, ShareNotificationProperty::OldRights, ShareNotificationProperty::NewRights, ShareNotificationProperty::Name, ], [share_id], ) .await .list()[0] .assert_is_equal(json!({ "id": &share_id, "changedBy": { "principalId": &john_id, "name": "John Doe", "email": "jdoe@example.com" }, "objectType": "FileNode", "objectAccountId": &john_id, "objectId": &john_folder_id, "oldRights": { "mayRead": true, "mayWrite": true, "mayShare": false }, "newRights": { "mayRead": false, "mayWrite": false, "mayShare": false }, "name": null })); // Grant Jane delete access once again john.jmap_update( MethodObject::FileNode, [( &john_folder_id, json!({ format!("shareWith/{jane_id}/mayRead"): true, format!("shareWith/{jane_id}/mayWrite"): true, }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&john_folder_id); // Verify Jane can delete the folder assert_eq!( jane.jmap_destroy_account( john, MethodObject::FileNode, [john_folder_id.as_str()], [("onDestroyRemoveChildren", true)], ) .await .destroyed() .collect::>(), [john_folder_id.as_str()] ); // Destroy all mailboxes params.assert_is_empty().await; } ================================================ FILE: tests/src/jmap/files/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod acl; pub mod node; ================================================ FILE: tests/src/jmap/files/node.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::{ChangeType, JMAPTest, JmapUtils}; use ahash::AHashSet; use jmap_proto::{object::file_node::FileNodeProperty, request::method::MethodObject}; use serde_json::json; pub async fn test(params: &mut JMAPTest) { println!("Running File Storage tests..."); let account = params.account("jdoe@example.com"); // Obtain change id let change_id = account .jmap_get( MethodObject::FileNode, [FileNodeProperty::Id], Vec::<&str>::new(), ) .await .state() .to_string(); // Create test folders let response = account .jmap_create( MethodObject::FileNode, [ json!({ "name": "Root Folder", "parentId": null, }), json!({ "name": "Sub Folder", "parentId": "#i0", }), json!({ "name": "Sub-sub Folder", "parentId": "#i1", }), ], Vec::<(&str, &str)>::new(), ) .await; let root_folder_id = response.created(0).id().to_string(); let sub_folder_id = response.created(1).id().to_string(); let sub_sub_folder_id = response.created(2).id().to_string(); // Validate changes assert_eq!( account .jmap_changes(MethodObject::FileNode, change_id) .await .changes() .collect::>(), [ ChangeType::Created(&root_folder_id), ChangeType::Created(&sub_folder_id), ChangeType::Created(&sub_sub_folder_id) ] .into_iter() .collect::>() ); // Verify folder structure let response = account .jmap_get( MethodObject::FileNode, [ FileNodeProperty::Id, FileNodeProperty::Name, FileNodeProperty::ParentId, ], [&root_folder_id, &sub_folder_id, &sub_sub_folder_id], ) .await; let list = response.list(); assert_eq!(list.len(), 3); list[0].assert_is_equal(json!({ "id": &root_folder_id, "name": "Root Folder", "parentId": null, })); list[1].assert_is_equal(json!({ "id": &sub_folder_id, "name": "Sub Folder", "parentId": &root_folder_id, })); list[2].assert_is_equal(json!({ "id": &sub_sub_folder_id, "name": "Sub-sub Folder", "parentId": &sub_folder_id, })); // Create file in root folder let response = account .jmap_method_calls(json!([ [ "Blob/upload", { "create": { "hello": { "data": [ { "data:asText": r#"hello world"# } ] } } }, "S4" ], [ "FileNode/set", { "create": { "i0": { "name": "hello.txt", "parentId": &root_folder_id, "blobId": "#hello", "type": "text/plain", } } }, "G4" ] ])) .await; let file_id = response .pointer("/methodResponses/1/1/created/i0") .unwrap() .id() .to_string(); // Verify file creation let response = account .jmap_get( MethodObject::FileNode, [ FileNodeProperty::Id, FileNodeProperty::BlobId, FileNodeProperty::Name, FileNodeProperty::ParentId, FileNodeProperty::Type, FileNodeProperty::Size, ], [&file_id], ) .await; let blob_id = response.list()[0].blob_id().to_string(); response.list()[0].assert_is_equal(json!({ "id": &file_id, "name": "hello.txt", "parentId": &root_folder_id, "type": "text/plain", "size": 11, "blobId": &blob_id, })); assert_eq!( account .jmap_get(MethodObject::Blob, ["data:asText"], [&blob_id]) .await .list()[0] .text_field("data:asText"), "hello world" ); // Creating folders with invalid names or parent ids should fail let response = account .jmap_create( MethodObject::FileNode, [ json!({ "name": "Sub Folder", "parentId": &root_folder_id, }), json!({ "name": "Folder under file", "parentId": &file_id, }), json!({ "name": "My/Sub/Folder", }), json!({ "name": ".", }), json!({ "name": "..", }), ], Vec::<(&str, &str)>::new(), ) .await; assert_eq!( response.not_created(0).description(), "A node with the same name already exists in this folder." ); assert_eq!( response.not_created(1).description(), "Parent ID does not exist or is not a folder." ); assert_eq!( response.not_created(2).description(), "Field could not be set." ); assert_eq!( response.not_created(3).description(), "Field could not be set." ); assert_eq!( response.not_created(4).description(), "Field could not be set." ); // Circular folder references should fail let response = account .jmap_update( MethodObject::FileNode, [( &root_folder_id, json!({ "parentId": &sub_sub_folder_id, }), )], Vec::<(&str, &str)>::new(), ) .await; assert_eq!( response.not_updated(&root_folder_id).description(), "Circular reference in parent ids." ); // Rename folder and file let response = account .jmap_update( MethodObject::FileNode, [ ( &sub_folder_id, json!({ "name": "Renamed Sub Folder", }), ), ( &file_id, json!({ "name": "renamed-hello.txt", }), ), ], Vec::<(&str, &str)>::new(), ) .await; response.updated(&sub_folder_id); response.updated(&file_id); // Verify rename let response = account .jmap_get( MethodObject::FileNode, [ FileNodeProperty::Id, FileNodeProperty::Name, FileNodeProperty::ParentId, ], [&sub_folder_id, &file_id], ) .await; let list = response.list(); assert_eq!(list.len(), 2); list[0].assert_is_equal(json!({ "id": &sub_folder_id, "name": "Renamed Sub Folder", "parentId": &root_folder_id, })); list[1].assert_is_equal(json!({ "id": &file_id, "name": "renamed-hello.txt", "parentId": &root_folder_id, })); // Destroying a folder with children should fail assert_eq!( account .jmap_destroy( MethodObject::FileNode, [&root_folder_id], Vec::<(&str, &str)>::new(), ) .await .not_destroyed(&root_folder_id) .description(), "Cannot delete non-empty folder." ); // Delete file and sub folders assert_eq!( account .jmap_destroy( MethodObject::FileNode, [&file_id], [("onDestroyRemoveChildren", true)], ) .await .destroyed() .collect::>(), [file_id.as_str(),].into_iter().collect::>() ); assert_eq!( account .jmap_destroy( MethodObject::FileNode, [&root_folder_id], [("onDestroyRemoveChildren", true)], ) .await .destroyed() .collect::>(), [ sub_sub_folder_id.as_str(), sub_folder_id.as_str(), root_folder_id.as_str() ] .into_iter() .collect::>() ); // Make sure everything is gone params.assert_is_empty().await; } ================================================ FILE: tests/src/jmap/mail/acl.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{directory::internal::TestInternalDirectory, jmap::JMAPTest}; use ::email::mailbox::{INBOX_ID, TRASH_ID}; use jmap_client::{ core::{ error::{MethodError, MethodErrorType}, set::{SetError, SetErrorType}, }, email::{self, Property, import::EmailImportResponse, query::Filter}, mailbox::{self, Role}, principal::ACL, }; use std::fmt::Debug; use store::ahash::AHashMap; use types::id::Id; pub async fn test(params: &mut JMAPTest) { println!("Running ACL tests..."); let server = params.server.clone(); // Create a group and three test accounts let inbox_id = Id::new(INBOX_ID as u64).to_string(); let trash_id = Id::new(TRASH_ID as u64).to_string(); let john = params.account("jdoe@example.com"); let jane = params.account("jane.smith@example.com"); let bill = params.account("bill@example.com"); let sales = params.account("sales@example.com"); // Authenticate all accounts let mut john_client = john.client_owned().await; let mut jane_client = jane.client_owned().await; let mut bill_client = bill.client_owned().await; // Insert two emails in each account let mut email_ids = AHashMap::default(); for (client, account_id, name) in [ (&mut john_client, john.id(), "john"), (&mut jane_client, jane.id(), "jane"), (&mut bill_client, bill.id(), "bill"), ( &mut params.account("admin").client_owned().await, sales.id(), "sales", ), ] { let user_name = client.session().username().to_string(); let mut ids = Vec::with_capacity(2); for (mailbox_id, mailbox_name) in [(&inbox_id, "inbox"), (&trash_id, "trash")] { ids.push( client .set_default_account_id(account_id.to_string()) .email_import( format!( concat!( "From: acl_test@example.com\r\n", "To: {}\r\n", "Subject: Owned by {} in {}\r\n", "\r\n", "This message is owned by {}.", ), user_name, name, mailbox_name, name ) .into_bytes(), [mailbox_id], None::>, None, ) .await .unwrap() .take_id(), ); } email_ids.insert(name, ids); } // John should have access to his emails only assert_eq!( john_client .email_get( email_ids.get("john").unwrap().first().unwrap(), [Property::Subject].into(), ) .await .unwrap() .unwrap() .subject() .unwrap(), "Owned by john in inbox" ); assert_forbidden( john_client .set_default_account_id(jane.id_string()) .email_get( email_ids.get("jane").unwrap().first().unwrap(), [Property::Subject].into(), ) .await, ); assert_forbidden( john_client .set_default_account_id(jane.id_string()) .mailbox_get(&inbox_id, None::>) .await, ); assert_forbidden( john_client .set_default_account_id(sales.id_string()) .email_get( email_ids.get("sales").unwrap().first().unwrap(), [Property::Subject].into(), ) .await, ); assert_forbidden( john_client .set_default_account_id(sales.id_string()) .mailbox_get(&inbox_id, None::>) .await, ); assert_forbidden( john_client .set_default_account_id(jane.id_string()) .email_query(None::, None::>) .await, ); // Jane grants Inbox ReadItems access to John jane_client .mailbox_update_acl(&inbox_id, john.id_string(), [ACL::ReadItems]) .await .unwrap(); // John should have ReadItems access to Inbox assert_eq!( john_client .set_default_account_id(jane.id_string()) .email_get( email_ids.get("jane").unwrap().first().unwrap(), [Property::Subject].into(), ) .await .unwrap() .unwrap() .subject() .unwrap(), "Owned by jane in inbox" ); assert_eq!( john_client .set_default_account_id(jane.id_string()) .email_query(None::, None::>) .await .unwrap() .ids(), [email_ids.get("jane").unwrap().first().unwrap().as_str()] ); // John's session resource should contain Jane's account details john_client.refresh_session().await.unwrap(); assert_eq!( john_client .session() .account(jane.id_string()) .unwrap() .name(), "jane.smith@example.com" ); // John should not have access to emails in Jane's Trash folder assert!( john_client .set_default_account_id(jane.id_string()) .email_get( email_ids.get("jane").unwrap().last().unwrap(), [Property::Subject].into(), ) .await .unwrap() .is_none() ); // John should only be able to copy blobs he has access to let blob_id = jane_client .email_get( email_ids.get("jane").unwrap().first().unwrap(), [Property::BlobId].into(), ) .await .unwrap() .unwrap() .take_blob_id(); john_client .set_default_account_id(john.id_string()) .blob_copy(jane.id_string(), &blob_id) .await .unwrap(); let blob_id = jane_client .email_get( email_ids.get("jane").unwrap().last().unwrap(), [Property::BlobId].into(), ) .await .unwrap() .unwrap() .take_blob_id(); assert_forbidden( john_client .set_default_account_id(john.id_string()) .blob_copy(jane.id_string(), &blob_id) .await, ); // John only has ReadItems access to Inbox jane_client .mailbox_update_acl(&inbox_id, john.id_string(), [ACL::ReadItems]) .await .unwrap(); assert_eq!( john_client .set_default_account_id(jane.id_string()) .mailbox_get(&inbox_id, [mailbox::Property::MyRights].into()) .await .unwrap() .unwrap() .my_rights() .unwrap() .acl_list(), vec![ACL::ReadItems] ); // Try to add items using import and copy let blob_id = john_client .set_default_account_id(john.id_string()) .upload( Some(john.id_string()), concat!( "From: acl_test@example.com\r\n", "To: jane.smith@example.com\r\n", "Subject: Created by john in jane's inbox\r\n", "\r\n", "This message is owned by jane.", ) .as_bytes() .to_vec(), None, ) .await .unwrap() .take_blob_id(); let mut request = john_client.set_default_account_id(jane.id_string()).build(); let email_id = request .import_email() .email(&blob_id) .mailbox_ids([&inbox_id]) .create_id(); assert_forbidden( request .send_single::() .await .unwrap() .created(&email_id), ); assert_forbidden( john_client .set_default_account_id(jane.id_string()) .email_copy( john.id_string(), email_ids.get("john").unwrap().last().unwrap(), [&inbox_id], None::>, None, ) .await, ); // Grant access and try again jane_client .mailbox_update_acl(&inbox_id, john.id_string(), [ACL::ReadItems, ACL::AddItems]) .await .unwrap(); let mut request = john_client.set_default_account_id(jane.id_string()).build(); let email_id = request .import_email() .email(&blob_id) .mailbox_ids([&inbox_id]) .create_id(); let email_id = request .send_single::() .await .unwrap() .created(&email_id) .unwrap() .take_id(); let email_id_2 = john_client .set_default_account_id(jane.id_string()) .email_copy( john.id_string(), email_ids.get("john").unwrap().last().unwrap(), [&inbox_id], None::>, None, ) .await .unwrap() .take_id(); assert_eq!( jane_client .email_get(&email_id, [Property::Subject].into(),) .await .unwrap() .unwrap() .subject() .unwrap(), "Created by john in jane's inbox" ); assert_eq!( jane_client .email_get(&email_id_2, [Property::Subject].into(),) .await .unwrap() .unwrap() .subject() .unwrap(), "Owned by john in trash" ); // Try removing items assert_forbidden( john_client .set_default_account_id(jane.id_string()) .email_destroy(&email_id) .await, ); jane_client .mailbox_update_acl( &inbox_id, john.id_string(), [ACL::ReadItems, ACL::AddItems, ACL::RemoveItems], ) .await .unwrap(); john_client .set_default_account_id(jane.id_string()) .email_destroy(&email_id) .await .unwrap(); // Try to set keywords assert_forbidden( john_client .set_default_account_id(jane.id_string()) .email_set_keyword(&email_id_2, "$seen", true) .await, ); jane_client .mailbox_update_acl( &inbox_id, john.id_string(), [ ACL::ReadItems, ACL::AddItems, ACL::RemoveItems, ACL::SetKeywords, ], ) .await .unwrap(); john_client .set_default_account_id(jane.id_string()) .email_set_keyword(&email_id_2, "$seen", true) .await .unwrap(); john_client .set_default_account_id(jane.id_string()) .email_set_keyword(&email_id_2, "my-keyword", true) .await .unwrap(); // Try to create a child assert_forbidden( john_client .set_default_account_id(jane.id_string()) .mailbox_create("John's mailbox", None::<&str>, Role::None) .await, ); jane_client .mailbox_update_acl( &inbox_id, john.id_string(), [ ACL::ReadItems, ACL::AddItems, ACL::RemoveItems, ACL::SetKeywords, ACL::CreateChild, ], ) .await .unwrap(); let mailbox_id = john_client .set_default_account_id(jane.id_string()) .mailbox_create("John's mailbox", Some(&inbox_id), Role::None) .await .unwrap() .take_id(); // Try renaming a mailbox assert_forbidden( john_client .set_default_account_id(jane.id_string()) .mailbox_rename(&mailbox_id, "John's private mailbox") .await, ); jane_client .mailbox_update_acl(&mailbox_id, john.id_string(), [ACL::ReadItems, ACL::Rename]) .await .unwrap(); john_client .set_default_account_id(jane.id_string()) .mailbox_rename(&mailbox_id, "John's private mailbox") .await .unwrap(); // Try moving a message assert_forbidden( john_client .set_default_account_id(jane.id_string()) .email_set_mailbox(&email_id_2, &mailbox_id, true) .await, ); jane_client .mailbox_update_acl( &mailbox_id, john.id_string(), [ACL::ReadItems, ACL::Rename, ACL::AddItems], ) .await .unwrap(); john_client .set_default_account_id(jane.id_string()) .email_set_mailbox(&email_id_2, &mailbox_id, true) .await .unwrap(); // Try deleting a mailbox assert_forbidden( john_client .set_default_account_id(jane.id_string()) .mailbox_destroy(&mailbox_id, true) .await, ); jane_client .mailbox_update_acl( &mailbox_id, john.id_string(), [ACL::ReadItems, ACL::Rename, ACL::AddItems, ACL::Delete], ) .await .unwrap(); assert_forbidden( john_client .set_default_account_id(jane.id_string()) .mailbox_destroy(&mailbox_id, true) .await, ); jane_client .mailbox_update_acl( &mailbox_id, john.id_string(), [ ACL::ReadItems, ACL::Rename, ACL::AddItems, ACL::Delete, ACL::RemoveItems, ], ) .await .unwrap(); john_client .set_default_account_id(jane.id_string()) .mailbox_destroy(&mailbox_id, true) .await .unwrap(); // Try changing ACL assert_forbidden( john_client .set_default_account_id(jane.id_string()) .mailbox_update_acl(&inbox_id, bill.id_string(), [ACL::ReadItems]) .await, ); assert_forbidden( bill_client .set_default_account_id(jane.id_string()) .email_query(None::, None::>) .await, ); jane_client .mailbox_update_acl( &inbox_id, john.id_string(), [ ACL::ReadItems, ACL::AddItems, ACL::RemoveItems, ACL::SetKeywords, ACL::CreateChild, ACL::Rename, ACL::Administer, ], ) .await .unwrap(); assert_eq!( john_client .set_default_account_id(jane.id_string()) .mailbox_get(&inbox_id, [mailbox::Property::MyRights].into()) .await .unwrap() .unwrap() .my_rights() .unwrap() .acl_list(), vec![ ACL::ReadItems, ACL::AddItems, ACL::RemoveItems, ACL::SetSeen, ACL::SetKeywords, ACL::CreateChild, ACL::Rename ] ); john_client .set_default_account_id(jane.id_string()) .mailbox_update_acl(&inbox_id, bill.id_string(), [ACL::ReadItems]) .await .unwrap(); assert_eq!( bill_client .set_default_account_id(jane.id_string()) .email_query( None::, vec![email::query::Comparator::subject()].into() ) .await .unwrap() .ids(), [ email_ids.get("jane").unwrap().first().unwrap().as_str(), &email_id_2 ] ); // Revoke all access to John jane_client .mailbox_update_acl(&inbox_id, john.id_string(), []) .await .unwrap(); assert_forbidden( john_client .set_default_account_id(jane.id_string()) .email_get( email_ids.get("jane").unwrap().first().unwrap(), [Property::Subject].into(), ) .await, ); john_client.refresh_session().await.unwrap(); assert!(john_client.session().account(jane.id_string()).is_none()); assert_eq!( bill_client .set_default_account_id(jane.id_string()) .email_get( email_ids.get("jane").unwrap().first().unwrap(), [Property::Subject].into(), ) .await .unwrap() .unwrap() .subject() .unwrap(), "Owned by jane in inbox" ); // Add John and Jane to the Sales group for name in ["jdoe@example.com", "jane.smith@example.com"] { server .invalidate_principal_caches( server .core .storage .data .add_to_group(name, "sales@example.com") .await, ) .await; } john_client.refresh_session().await.unwrap(); jane_client.refresh_session().await.unwrap(); bill_client.refresh_session().await.unwrap(); assert_eq!( john_client .session() .account(sales.id_string()) .unwrap() .name(), "sales@example.com" ); assert!( !john_client .session() .account(sales.id_string()) .unwrap() .is_personal() ); assert_eq!( jane_client .session() .account(sales.id_string()) .unwrap() .name(), "sales@example.com" ); assert!(bill_client.session().account(sales.id_string()).is_none()); // Insert a message in Sales's inbox let blob_id = john_client .set_default_account_id(sales.id_string()) .upload( Some(sales.id_string()), concat!( "From: acl_test@example.com\r\n", "To: sales@example.com\r\n", "Subject: Created by john in sales\r\n", "\r\n", "This message is owned by sales.", ) .as_bytes() .to_vec(), None, ) .await .unwrap() .take_blob_id(); let mut request = john_client.build(); let email_id = request .import_email() .email(&blob_id) .mailbox_ids([&inbox_id]) .create_id(); let email_id = request .send_single::() .await .unwrap() .created(&email_id) .unwrap() .take_id(); // Both Jane and John should be able to see this message, but not Bill assert_eq!( john_client .set_default_account_id(sales.id_string()) .email_get(&email_id, [Property::Subject].into(),) .await .unwrap() .unwrap() .subject() .unwrap(), "Created by john in sales" ); assert_eq!( jane_client .set_default_account_id(sales.id_string()) .email_get(&email_id, [Property::Subject].into(),) .await .unwrap() .unwrap() .subject() .unwrap(), "Created by john in sales" ); assert_forbidden( bill_client .set_default_account_id(sales.id_string()) .email_get(&email_id, [Property::Subject].into()) .await, ); // Remove John from the sales group server .invalidate_principal_caches( server .core .storage .data .remove_from_group("jdoe@example.com", "sales@example.com") .await, ) .await; assert_forbidden( john_client .set_default_account_id(sales.id_string()) .email_get(&email_id, [Property::Subject].into()) .await, ); // Destroy test account data for id in [john, bill, jane, sales] { params.destroy_all_mailboxes(id).await; } params.assert_is_empty().await; } pub fn assert_forbidden(result: Result) { if !matches!( result, Err(jmap_client::Error::Method(MethodError { p_type: MethodErrorType::Forbidden })) | Err(jmap_client::Error::Set(SetError { type_: SetErrorType::BlobNotFound | SetErrorType::Forbidden, .. })) ) { panic!("Expected forbidden, got {:?}", result); } } ================================================ FILE: tests/src/jmap/mail/antispam.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::Arc; use email::mailbox::{DRAFTS_ID, INBOX_ID, JUNK_ID}; use store::write::now; use types::{id::Id, keyword::Keyword}; use crate::{imap::antispam::*, jmap::JMAPTest}; pub async fn test(params: &mut JMAPTest) { println!("Running Email Spam classifier tests..."); let account = params.account("jdoe@example.com"); let client = account.client(); let account_id = account.id().document_id(); // Make sure there are no training samples spam_delete_samples(¶ms.server).await; assert_eq!(spam_training_samples(¶ms.server).await.total_count, 0); // Import samples let mut spam_ids = vec![]; let mut ham_ids = vec![]; for (idx, samples) in [&SPAM, &HAM].into_iter().enumerate() { let is_spam = idx == 0; for (num, sample) in samples.iter().enumerate() { let mut mailbox_ids = vec![]; let mut keywords = vec![]; if num == 0 { if is_spam { mailbox_ids.push(Id::from(JUNK_ID).to_string()); keywords.push(Keyword::Junk.to_string()); } else { mailbox_ids.push(Id::from(INBOX_ID).to_string()); keywords.push(Keyword::NotJunk.to_string()); } } else { mailbox_ids.push(Id::from(DRAFTS_ID).to_string()); } let mail_id = client .email_import( sample.as_bytes().to_vec(), &mailbox_ids, Some(&keywords), None, ) .await .unwrap() .take_id(); if is_spam { spam_ids.push(mail_id); } else { ham_ids.push(mail_id); } } } let samples = spam_training_samples(¶ms.server).await; assert_eq!(samples.ham_count, 1); assert_eq!(samples.spam_count, 1); // Train the classifier via JMAP for (ids, is_spam) in [(&spam_ids, true), (&ham_ids, false)] { for (idx, id) in ids.iter().skip(1).enumerate() { // Set keywords and mailboxes let mut request = client.build(); let req = request.set_email().update(id); if idx < 5 || !is_spam { // Update via keywords let keyword = if is_spam { Keyword::Junk } else { Keyword::NotJunk } .to_string(); req.keywords([&keyword]); } else { // Update via mailbox let mailbox_id = if is_spam { JUNK_ID } else { INBOX_ID }; req.mailbox_ids([&Id::from(mailbox_id).to_string()]); } request.send_set_email().await.unwrap().updated(id).unwrap(); } } let samples = spam_training_samples(¶ms.server).await; assert_eq!(samples.ham_count, 10); assert_eq!(samples.spam_count, 10); // Reclassifying an email should not add a new sample let mut request = client.build(); request .set_email() .update(&ham_ids[0]) .keywords([Keyword::Junk.to_string()]); request .send_set_email() .await .unwrap() .updated(&ham_ids[0]) .unwrap(); let samples = spam_training_samples(¶ms.server).await; assert_eq!(samples.ham_count, 9); assert_eq!(samples.spam_count, 11); assert_eq!(samples.samples.len(), 20); let hold_for = params .server .core .spam .classifier .as_ref() .unwrap() .hold_samples_for; assert!(hold_for > 2 * 86400); let hold_until = now() + hold_for; let hold_range = (hold_until - 86400)..=hold_until; assert!(samples.samples.iter().all(|s| s.account_id == account_id && s.remove.is_none() && hold_range.contains(&s.until))); // Purging blobs should not remove training samples params .server .store() .purge_blobs(params.server.blob_store().clone()) .await .unwrap(); let samples = spam_training_samples(¶ms.server).await; assert_eq!(samples.ham_count, 9); assert_eq!(samples.spam_count, 11); assert_eq!(samples.samples.len(), 20); // Extend hold period so a new training sample is generated let old_core = params.server.core.clone(); let mut new_core = old_core.as_ref().clone(); new_core.spam.classifier.as_mut().unwrap().hold_samples_for += 2 * 86400; params.server.inner.shared_core.store(Arc::new(new_core)); // Reclassifying an email will now add a new sample let mut request = client.build(); request .set_email() .update(&ham_ids[0]) .keywords([Keyword::NotJunk.to_string()]); request .send_set_email() .await .unwrap() .updated(&ham_ids[0]) .unwrap(); let samples = spam_training_samples(¶ms.server).await; assert_eq!(samples.ham_count, 10); assert_eq!(samples.spam_count, 11); assert_eq!(samples.samples.len(), 21); // Blob purge should remove the duplicated sample params .server .store() .purge_blobs(params.server.blob_store().clone()) .await .unwrap(); let samples = spam_training_samples(¶ms.server).await; assert_eq!(samples.ham_count, 10); assert_eq!(samples.spam_count, 10); assert_eq!(samples.samples.len(), 20); params.destroy_all_mailboxes(account).await; params.assert_is_empty().await; } ================================================ FILE: tests/src/jmap/mail/changes.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::JMAPTest; use jmap_proto::types::state::State; use std::str::FromStr; use store::{ahash::AHashSet, write::BatchBuilder}; use types::{ collection::{Collection, SyncCollection}, id::Id, }; pub async fn test(params: &mut JMAPTest) { println!("Running Email Changes tests..."); let server = params.server.clone(); let account = params.account("jdoe@example.com"); let client = account.client(); let mut states = vec![State::Initial]; for (changes, expected_changelog) in [ ( vec![ LogAction::Insert(0), LogAction::Insert(1), LogAction::Insert(2), ], vec![vec![vec![0, 1, 2], vec![], vec![]]], ), ( vec![ LogAction::Move(0, 3), LogAction::Insert(4), LogAction::Insert(5), LogAction::Update(1), LogAction::Update(2), ], vec![ vec![vec![1, 2, 3, 4, 5], vec![], vec![]], vec![vec![3, 4, 5], vec![1, 2], vec![0]], ], ), ( vec![ LogAction::Delete(1), LogAction::Insert(6), LogAction::Insert(7), LogAction::Update(2), ], vec![ vec![vec![2, 3, 4, 5, 6, 7], vec![], vec![]], vec![vec![3, 4, 5, 6, 7], vec![2], vec![0, 1]], vec![vec![6, 7], vec![2], vec![1]], ], ), ( vec![ LogAction::Update(4), LogAction::Update(5), LogAction::Update(6), LogAction::Update(7), ], vec![ vec![vec![2, 3, 4, 5, 6, 7], vec![], vec![]], vec![vec![3, 4, 5, 6, 7], vec![2], vec![0, 1]], vec![vec![6, 7], vec![2, 4, 5], vec![1]], vec![vec![], vec![4, 5, 6, 7], vec![]], ], ), ( vec![ LogAction::Delete(4), LogAction::Delete(5), LogAction::Delete(6), LogAction::Delete(7), ], vec![ vec![vec![2, 3], vec![], vec![]], vec![vec![3], vec![2], vec![0, 1]], vec![vec![], vec![2], vec![1, 4, 5]], vec![vec![], vec![], vec![4, 5, 6, 7]], vec![vec![], vec![], vec![4, 5, 6, 7]], ], ), ( vec![ LogAction::Insert(8), LogAction::Insert(9), LogAction::Insert(10), LogAction::Update(3), ], vec![ vec![vec![2, 3, 8, 9, 10], vec![], vec![]], vec![vec![3, 8, 9, 10], vec![2], vec![0, 1]], vec![vec![8, 9, 10], vec![2, 3], vec![1, 4, 5]], vec![vec![8, 9, 10], vec![3], vec![4, 5, 6, 7]], vec![vec![8, 9, 10], vec![3], vec![4, 5, 6, 7]], vec![vec![8, 9, 10], vec![3], vec![]], ], ), ( vec![LogAction::Update(2), LogAction::Update(8)], vec![ vec![vec![2, 3, 8, 9, 10], vec![], vec![]], vec![vec![3, 8, 9, 10], vec![2], vec![0, 1]], vec![vec![8, 9, 10], vec![2, 3], vec![1, 4, 5]], vec![vec![8, 9, 10], vec![2, 3], vec![4, 5, 6, 7]], vec![vec![8, 9, 10], vec![2, 3], vec![4, 5, 6, 7]], vec![vec![8, 9, 10], vec![2, 3], vec![]], vec![vec![], vec![2, 8], vec![]], ], ), ( vec![ LogAction::Move(9, 11), LogAction::Move(10, 12), LogAction::Delete(8), ], vec![ vec![vec![2, 3, 11, 12], vec![], vec![]], vec![vec![3, 11, 12], vec![2], vec![0, 1]], vec![vec![11, 12], vec![2, 3], vec![1, 4, 5]], vec![vec![11, 12], vec![2, 3], vec![4, 5, 6, 7]], vec![vec![11, 12], vec![2, 3], vec![4, 5, 6, 7]], vec![vec![11, 12], vec![2, 3], vec![]], vec![vec![11, 12], vec![2], vec![8, 9, 10]], vec![vec![11, 12], vec![], vec![8, 9, 10]], ], ), ] .into_iter() { let mut batch = BatchBuilder::new(); batch .with_account_id(account.id().document_id()) .with_collection(Collection::Email); for change in changes { match change { LogAction::Insert(id) => { batch .with_document(id as u32) .log_item_insert(SyncCollection::Email, None); } LogAction::Update(id) => { batch .with_document(id as u32) .log_item_update(SyncCollection::Email, None); } LogAction::Delete(id) => { batch .with_document(id as u32) .log_item_delete(SyncCollection::Email, None); } LogAction::UpdateChild(id) => { batch.log_container_property_change(SyncCollection::Email, id as u32); } LogAction::Move(old_id, new_id) => { batch .with_document(old_id as u32) .log_item_delete(SyncCollection::Email, None) .with_document(new_id as u32) .log_item_insert(SyncCollection::Email, None); } } } server .core .storage .data .write(batch.build_all()) .await .unwrap(); let mut new_state = State::Initial; for (test_num, state) in (states).iter().enumerate() { let changes = client.email_changes(state.to_string(), None).await.unwrap(); assert_eq!( expected_changelog[test_num], [changes.created(), changes.updated(), changes.destroyed()] .into_iter() .map(|list| { let mut list = list .iter() .map(|i| Id::from_str(i).unwrap().into()) .collect::>(); list.sort_unstable(); list }) .collect::>>(), "test_num: {}, state: {:?}", test_num, state ); if &State::Initial == state { new_state = State::parse_str(changes.new_state()).unwrap(); } for max_changes in 1..=8 { let mut insertions = expected_changelog[test_num][0] .iter() .copied() .collect::>(); let mut updates = expected_changelog[test_num][1] .iter() .copied() .collect::>(); let mut deletions = expected_changelog[test_num][2] .iter() .copied() .collect::>(); let mut int_state = state.clone(); for _ in 0..100 { let changes = client .email_changes(int_state.to_string(), max_changes.into()) .await .unwrap(); assert!( changes.created().len() + changes.updated().len() + changes.destroyed().len() <= max_changes, "{} > {}", changes.created().len() + changes.updated().len() + changes.destroyed().len(), max_changes ); changes.created().iter().for_each(|id| { assert!( insertions.remove(&Id::from_str(id).unwrap()), "{:?} != {}", insertions, Id::from_str(id).unwrap() ); }); changes.updated().iter().for_each(|id| { assert!( updates.remove(&Id::from_str(id).unwrap()), "{:?} != {}", updates, Id::from_str(id).unwrap() ); }); changes.destroyed().iter().for_each(|id| { assert!( deletions.remove(&Id::from_str(id).unwrap()), "{:?} != {}", deletions, Id::from_str(id).unwrap() ); }); int_state = State::parse_str(changes.new_state()).unwrap(); if !changes.has_more_changes() { break; } } assert_eq!( insertions.len(), 0, "test_num: {}, state: {:?}, pending: {:?}", test_num, state, insertions ); assert_eq!( updates.len(), 0, "test_num: {}, state: {:?}, pending: {:?}", test_num, state, updates ); assert_eq!( deletions.len(), 0, "test_num: {}, state: {:?}, pending: {:?}", test_num, state, deletions ); } } states.push(new_state); } let changes = client .email_changes(State::Initial.to_string(), None) .await .unwrap(); let mut created = changes .created() .iter() .map(|i| Id::from_str(i).unwrap().into()) .collect::>(); created.sort_unstable(); assert_eq!(created, vec![2, 3, 11, 12]); assert_eq!(changes.updated(), Vec::::new()); assert_eq!(changes.destroyed(), Vec::::new()); params.destroy_all_mailboxes(account).await; params.assert_is_empty().await; } #[derive(Debug, Clone, Copy)] pub enum LogAction { Insert(u64), Update(u64), Delete(u64), UpdateChild(u64), Move(u64, u64), } pub trait ParseState: Sized { fn parse_str(state: &str) -> Option; } impl ParseState for State { fn parse_str(state: &str) -> Option { State::parse(state) } } ================================================ FILE: tests/src/jmap/mail/copy.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::{JMAPTest, mail::mailbox::destroy_all_mailboxes_for_account}; use jmap_client::mailbox::Role; use types::id::Id; pub async fn test(params: &mut JMAPTest) { println!("Running Email Copy tests..."); let account = params.account("admin"); let mut client = account.client_owned().await; // Create a mailbox on account 1 let ac1_mailbox_id = client .set_default_account_id(Id::new(1).to_string()) .mailbox_create("Copy Test Ac# 1", None::, Role::None) .await .unwrap() .take_id(); // Insert a message on account 1 let ac1_email_id = client .email_import( concat!( "From: bill@example.com\r\n", "To: jdoe@example.com\r\n", "Subject: TPS Report\r\n", "\r\n", "I'm going to need those TPS reports ASAP. ", "So, if you could do that, that'd be great." ) .as_bytes() .to_vec(), [&ac1_mailbox_id], None::>, None, ) .await .unwrap() .take_id(); // Create a mailbox on account 2 let ac2_mailbox_id = client .set_default_account_id(Id::new(2).to_string()) .mailbox_create("Copy Test Ac# 2", None::, Role::None) .await .unwrap() .take_id(); // Copy the email and delete it from the first account let mut request = client.build(); request .copy_email(Id::new(1).to_string()) .on_success_destroy_original(true) .create(&ac1_email_id) .mailbox_id(&ac2_mailbox_id, true) .keyword("$draft", true) .received_at(311923920); let ac2_email_id = request .send() .await .unwrap() .method_response_by_pos(0) .unwrap_copy_email() .unwrap() .created(&ac1_email_id) .unwrap() .take_id(); // Check that the email was copied let email = client .email_get(&ac2_email_id, None::>) .await .unwrap() .unwrap(); assert_eq!( email.preview().unwrap(), "I'm going to need those TPS reports ASAP. So, if you could do that, that'd be great." ); assert_eq!(email.subject().unwrap(), "TPS Report"); assert_eq!(email.mailbox_ids(), &[&ac2_mailbox_id]); assert_eq!(email.keywords(), &["$draft"]); assert_eq!(email.received_at().unwrap(), 311923920); // Check that the email was deleted assert!( client .set_default_account_id(Id::new(1).to_string()) .email_get(&ac1_email_id, None::>) .await .unwrap() .is_none() ); // Empty store destroy_all_mailboxes_for_account(1).await; destroy_all_mailboxes_for_account(2).await; params.assert_is_empty().await; } ================================================ FILE: tests/src/jmap/mail/crypto.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::{JMAPTest, ManagementApi, mail::delivery::SmtpConnection}; use email::message::crypto::{ Algorithm, EncryptMessage, EncryptionMethod, EncryptionParams, EncryptionType, try_parse_certs, }; use mail_parser::{MessageParser, MimeHeaders}; use std::path::PathBuf; use store::{ Deserialize, Serialize, write::{Archive, Archiver}, }; pub async fn test(params: &mut JMAPTest) { println!("Running Encryption-at-rest tests..."); // Check encryption check_is_encrypted(); import_certs_and_encrypt().await; // Create test account let account = params.account("jdoe@example.com"); let client = account.client(); // Build API let api = ManagementApi::new(8899, "jdoe@example.com", "12345"); // Try importing using multiple methods and symmetric algos for (file_name, method, num_certs) in [ ("cert_smime.pem", EncryptionMethod::SMIME, 3), ("cert_pgp.pem", EncryptionMethod::PGP, 1), ] { let certs = std::fs::read_to_string( PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("resources") .join("crypto") .join(file_name), ) .unwrap(); for algo in [Algorithm::Aes128, Algorithm::Aes256] { let request = match method { EncryptionMethod::PGP => EncryptionType::PGP { algo, certs: certs.clone(), allow_spam_training: true, }, EncryptionMethod::SMIME => EncryptionType::SMIME { algo, certs: certs.clone(), allow_spam_training: true, }, }; assert_eq!( api.post::("/api/account/crypto", &request) .await .unwrap() .unwrap_data(), num_certs ); } } // Send a new message, which should be encrypted let mut lmtp = SmtpConnection::connect().await; lmtp.ingest( "bill@example.com", &["jdoe@example.com"], concat!( "From: bill@example.com\r\n", "To: jdoe@example.com\r\n", "Subject: TPS Report (should be encrypted)\r\n", "\r\n", "I'm going to need those TPS reports ASAP. ", "So, if you could do that, that'd be great." ), ) .await; // Send an encrypted message lmtp.ingest( "bill@example.com", &["jdoe@example.com"], concat!( "From: bill@example.com\r\n", "To: jdoe@example.com\r\n", "Subject: TPS Report (already encrypted)\r\n", "Content-Type: application/pkcs7-mime; name=\"smime.p7m\"; smime-type=enveloped-data\r\n", "\r\n", "xjMEZMYfNhYJKwYBBAHaRw8BAQdAYyTN1HzqapLw8xwkCGwa0OjsgT/JqhcB/+Dy", "Ga1fsBrNG0pvaG4gRG9lIDxqb2huQGV4YW1wbGUub3JnPsKJBBMWCAAxFiEEg836", "pwbXpuQ/THMtpJwd4oBfIrUFAmTGHzYCGwMECwkIBwUVCAkKCwUWAgMBAAAKCRCk", "nB3igF8itYhyAQD2jEdeYa3gyQ47X9YWZTK1wEJkN8W9//V1fYl2XQwqlQEA0qBv", "Ai6nUh99oDw+/zQ8DFIKdeb5Ti4tu/X58PdpiQ7OOARkxh82EgorBgEEAZdVAQUB", "AQdAvXz2FbFN0DovQF/ACnZyczTsSIQp0mvmF1PE+aijbC8DAQgHwngEGBYIACAW", "IQSDzfqnBtem5D9Mcy2knB3igF8itQUCZMYfNgIbDAAKCRCknB3igF8itRnoAQC3", "GzPmgx7TnB+SexPuJV/DoKSMJ0/X+hbEFcZkulxaDQEAh+xiJCvf+ZNAKw6kFhsL", "UuZhEDktxnY6Ehz3aB7FawA=", "=KGrr", ), ) .await; // Disable encryption assert_eq!( api.post::>("/api/account/crypto", &EncryptionType::Disabled) .await .unwrap() .unwrap_data(), None ); // Send a new message, which should NOT be encrypted lmtp.ingest( "bill@example.com", &["jdoe@example.com"], concat!( "From: bill@example.com\r\n", "To: jdoe@example.com\r\n", "Subject: TPS Report (plain text)\r\n", "\r\n", "I'm going to need those TPS reports ASAP. ", "So, if you could do that, that'd be great." ), ) .await; // Check messages let mut request = client.build(); request.get_email(); let emails = request.send_get_email().await.unwrap().take_list(); assert_eq!(emails.len(), 3, "3 messages were expected: {:#?}.", emails); for email in emails { let message = String::from_utf8(client.download(email.blob_id().unwrap()).await.unwrap()).unwrap(); if message.contains("should be encrypted") { assert!( message.contains("Content-Type: multipart/encrypted"), "got message {message}, expected encrypted message" ); } else if message.contains("already encrypted") { assert!( message.contains("Content-Type: application/pkcs7-mime") && message.contains("xjMEZMYfNhYJKwYBBAHaRw8BAQdAYy"), "got message {message}, expected message to be left intact" ); } else if message.contains("plain text") { assert!( message.contains("I'm going to need those TPS reports ASAP."), "got message {message}, expected plain text message" ); } else { panic!("Unexpected message: {:#?}", message) } } } pub async fn import_certs_and_encrypt() { for (name, method, expected_certs) in [ ("cert_pgp.pem", EncryptionMethod::PGP, 1), //("cert_pgp.der", EncryptionMethod::PGP, 1), ("cert_smime.pem", EncryptionMethod::SMIME, 3), ("cert_smime.der", EncryptionMethod::SMIME, 1), ] { let mut certs = try_parse_certs( method, std::fs::read( PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("resources") .join("crypto") .join(name), ) .unwrap(), ) .expect(name); assert_eq!(certs.len(), expected_certs); if method == EncryptionMethod::PGP && certs.len() == 2 { // PGP library won't encrypt using EC let mut certs_ = certs.to_vec(); certs_.pop(); certs = certs_.into(); } let mut params = EncryptionParams { certs, flags: method.flags(), }; for algo in [Algorithm::Aes128, Algorithm::Aes256] { let message = MessageParser::new() .parse(b"Subject: test\r\ntest\r\n") .unwrap(); assert!(!message.is_encrypted()); params.flags = algo.flags() | method.flags(); let arch = Archive::deserialize_owned(Archiver::new(params.clone()).serialize().unwrap()) .unwrap(); message .encrypt(arch.unarchive::().unwrap()) .await .unwrap(); } } // S/MIME and PGP should not be allowed mixed assert!( try_parse_certs( EncryptionMethod::PGP, std::fs::read( PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("resources") .join("crypto") .join("cert_mixed.pem"), ) .unwrap(), ) .is_err() ); } pub fn check_is_encrypted() { let messages = std::fs::read_to_string( PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("resources") .join("crypto") .join("is_encrypted.txt"), ) .unwrap(); for raw_message in messages.split("!!!") { let is_encrypted = raw_message.contains("TRUE"); let message = MessageParser::new() .parse(raw_message.trim().as_bytes()) .unwrap(); assert!(message.content_type().is_some()); assert_eq!( message.is_encrypted(), is_encrypted, "failed for {raw_message}" ); } } ================================================ FILE: tests/src/jmap/mail/delivery.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ directory::internal::TestInternalDirectory, imap::antispam::{spam_delete_samples, spam_training_samples}, jmap::JMAPTest, store::cleanup::store_blob_expire_all, webdav::DummyWebDavClient, }; use common::Server; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, mailbox::{INBOX_ID, JUNK_ID, SENT_ID}, message::metadata::MessageMetadata, }; use groupware::DavResourceName; use jmap::blob::download::BlobDownload; use std::{sync::Arc, time::Duration}; use store::{ ValueKey, roaring::RoaringBitmap, write::{AlignedBytes, Archive}, }; use tokio::{ io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf}, net::TcpStream, }; use types::{ blob::{BlobClass, BlobId}, collection::Collection, field::EmailField, id::Id, }; use utils::chained_bytes::ChainedBytes; pub async fn test(params: &mut JMAPTest) { println!("Running message delivery tests..."); // Enable delivered to let old_core = params.server.core.clone(); let mut new_core = old_core.as_ref().clone(); new_core.smtp.session.data.add_delivered_to = true; params.server.inner.shared_core.store(Arc::new(new_core)); // Create a domain name and a test account let server = params.server.clone(); let john = params.account("jdoe@example.com"); let jane = params.account("jane.smith@example.com"); let bill = params.account("bill@example.com"); // Create a mailing list server .store() .create_test_list( "members@example.com", "Mailing List", &[ "jdoe@example.com", "jane.smith@example.com", "bill@example.com", ], ) .await; // Delivering to individuals let mut lmtp = SmtpConnection::connect().await; params.webhook.clear(); lmtp.ingest( "bill@example.com", &["jdoe@example.com"], concat!( "From: bill@example.com\r\n", "To: jdoe@example.com\r\n", "Subject: TPS Report\r\n", "\r\n", "I'm going to need those TPS reports ASAP. ", "So, if you could do that, that'd be great." ), ) .await; let john_cache = server .get_cached_messages(john.id().document_id()) .await .unwrap(); assert_eq!(john_cache.emails.items.len(), 1); assert_eq!(john_cache.in_mailbox(INBOX_ID).count(), 1); assert_eq!(john_cache.in_mailbox(JUNK_ID).count(), 0); // Make sure there are no spam training samples spam_delete_samples(¶ms.server).await; assert_eq!(spam_training_samples(¶ms.server).await.total_count, 0); // Test spam filtering lmtp.ingest( "bill@example.com", &["john.doe@example.com"], concat!( "From: bill@example.com\r\n", "To: john.doe@example.com\r\n", "Subject: XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X\r\n", "\r\n", "--- Forwarded Message ---\r\n\r\n ", "I'm going to need those TPS reports ASAP. ", "So, if you could do that, that'd be great." ), ) .await; let john_cache = server .get_cached_messages(john.id().document_id()) .await .unwrap(); let inbox_ids = john_cache .in_mailbox(INBOX_ID) .map(|e| e.document_id) .collect::(); let junk_ids = john_cache .in_mailbox(JUNK_ID) .map(|e| e.document_id) .collect::(); assert_eq!(john_cache.emails.items.len(), 2); assert_eq!(inbox_ids.len(), 1); assert_eq!(junk_ids.len(), 1); assert_message_headers_contains( &server, john.id().document_id(), junk_ids.min().unwrap(), "X-Spam-Status: Yes", ) .await; assert_eq!(spam_training_samples(¶ms.server).await.total_count, 0); // CardDAV spam override let dav_client = DummyWebDavClient::new(u32::MAX, john.name(), john.secret(), john.emails()[0]); dav_client .request( "PUT", &format!( "{}/jdoe%40example.com/default/bill.vcf", DavResourceName::Card.base_path() ), r#"BEGIN:VCARD VERSION:4.0 FN:Bill Foobar EMAIL;TYPE=WORK:dmarc-bill@example.com UID:urn:uuid:e1ee798b-3d4c-41b0-b217-b9c918e4686f END:VCARD "#, ) .await .with_status(hyper::StatusCode::CREATED); lmtp.ingest( "dmarc-bill@example.com", &["john.doe@example.com"], concat!( "From: dmarc-bill@example.com\r\n", "To: john.doe@example.com\r\n", "Subject: XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X\r\n", "\r\n", "--- Forwarded Message ---\r\n\r\n ", "I'm going to need those TPS reports ASAP. ", "So, if you could do that, that'd be great." ), ) .await; let john_cache = server .get_cached_messages(john.id().document_id()) .await .unwrap(); let inbox_ids = john_cache .in_mailbox(INBOX_ID) .map(|e| e.document_id) .collect::(); let junk_ids = john_cache .in_mailbox(JUNK_ID) .map(|e| e.document_id) .collect::(); assert_eq!(john_cache.emails.items.len(), 3); assert_eq!(inbox_ids.len(), 2); assert_eq!(junk_ids.len(), 1); dav_client.delete_default_containers().await; assert_message_headers_contains( &server, john.id().document_id(), inbox_ids.max().unwrap(), "X-Spam-Status: No, reason=card-exists", ) .await; let samples = spam_training_samples(¶ms.server).await; assert_eq!(samples.ham_count, 1); assert_eq!(samples.spam_count, 0); // Test trusted reply override john.client() .email_import( concat!( "From: john.doe@example.com\r\n", "To: dmarc-bill@example.com\r\n", "Message-ID: \r\n", "Subject: XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X\r\n", "\r\n", "This is a trusted reply." ) .as_bytes() .to_vec(), vec![Id::from(SENT_ID).to_string()], None::>, None, ) .await .unwrap() .take_id(); assert_eq!( server .get_cached_messages(john.id().document_id()) .await .unwrap() .emails .items .len(), 4 ); lmtp.ingest( "dmarc-bill@example.com", &["john.doe@example.com"], concat!( "From: dmarc-bill@example.com\r\n", "To: john.doe@example.com\r\n", "Message-ID: \r\n", "References: \r\n", "Subject: XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X\r\n", "\r\n", "--- Forwarded Message ---\r\n\r\n ", "I'm going to need those TPS reports ASAP. ", "So, if you could do that, that'd be great." ), ) .await; let john_cache = server .get_cached_messages(john.id().document_id()) .await .unwrap(); let inbox_ids = john_cache .in_mailbox(INBOX_ID) .map(|e| e.document_id) .collect::(); let junk_ids = john_cache .in_mailbox(JUNK_ID) .map(|e| e.document_id) .collect::(); assert_eq!(john_cache.emails.items.len(), 5); assert_eq!(inbox_ids.len(), 3); assert_eq!(junk_ids.len(), 1); assert_message_headers_contains( &server, john.id().document_id(), inbox_ids.max().unwrap(), "X-Spam-Status: No, reason=trusted-reply", ) .await; let samples = spam_training_samples(¶ms.server).await; assert_eq!(samples.ham_count, 2); assert_eq!(samples.spam_count, 0); // EXPN and VRFY lmtp.expn("members@example.com", 2) .await .assert_contains("jdoe@example.com") .assert_contains("jane.smith@example.com") .assert_contains("bill@example.com"); lmtp.expn("non_existant@example.com", 5).await; lmtp.expn("jdoe@example.com", 5).await; lmtp.vrfy("jdoe@example.com", 2).await; lmtp.vrfy("members@example.com", 5).await; lmtp.vrfy("non_existant@example.com", 5).await; // Delivering to a mailing list lmtp.ingest( "bill@example.com", &["members@example.com"], concat!( "From: bill@example.com\r\n", "To: members@example.com\r\n", "Subject: WFH policy\r\n", "\r\n", "We need the entire staff back in the office, ", "TPS reports cannot be filed properly from home." ), ) .await; tokio::time::sleep(Duration::from_millis(200)).await; for (account, num_messages) in [(john, 6), (jane, 1), (bill, 1)] { assert_eq!( server .get_cached_messages(account.id().document_id()) .await .unwrap() .emails .items .len(), num_messages, "for {}", account.id_string() ); } // Removing members from the mailing list and chunked ingest params .server .core .storage .data .remove_from_group("jdoe@example.com", "members@example.com") .await; lmtp.ingest_chunked( "bill@example.com", &["members@example.com"], concat!( "From: bill@example.com\r\n", "To: members@example.com\r\n", "Subject: WFH policy (reminder)\r\n", "\r\n", "This is a reminder that we need the entire staff back in the office, ", "TPS reports cannot be filed properly from home." ), 10, ) .await; for (account, num_messages) in [(john, 6), (jane, 2), (bill, 2)] { assert_eq!( server .get_cached_messages(account.id().document_id()) .await .unwrap() .emails .items .len(), num_messages, "for {}", account.id_string() ); } // Deduplication of recipients lmtp.ingest( "bill@example.com", &[ "members@example.com", "jdoe@example.com", "john.doe@example.com", "jane.smith@example.com", "bill@example.com", ], concat!( "From: bill@example.com\r\n", "Bcc: Undisclosed recipients;\r\n", "Subject: Holidays\r\n", "\r\n", "Remember to file your TPS reports before ", "going on holidays." ), ) .await; // Make sure blobs are properly linked store_blob_expire_all(params.server.store()).await; for (account, num_messages) in [(john, 7), (jane, 3), (bill, 3)] { let account_id = account.id().document_id(); let cache = server.get_cached_messages(account_id).await.unwrap(); assert_eq!( cache.emails.items.len(), num_messages, "for {}", account.id_string() ); let access_token = server.get_access_token(account_id).await.unwrap(); for document_id in cache.in_mailbox(INBOX_ID).map(|e| e.document_id) { let metadata = message_metadata(&server, account_id, document_id).await; let partial_message = server .store() .get_blob(metadata.blob_hash.0.as_ref(), 0..usize::MAX) .await .unwrap() .unwrap(); assert_ne!(metadata.blob_body_offset, 0); let expected_full_message = String::from_utf8( ChainedBytes::new(metadata.raw_headers.as_ref()) .with_last( partial_message .get(metadata.blob_body_offset as usize..) .unwrap_or_default(), ) .to_bytes(), ) .unwrap(); assert!( expected_full_message.contains("Delivered-To:") && expected_full_message.contains("Subject:"), "for {account_id}: {expected_full_message}" ); let full_message = String::from_utf8( server .blob_download( &BlobId { hash: metadata.blob_hash, class: BlobClass::Linked { account_id, collection: Collection::Email.into(), document_id, }, section: None, }, &access_token, ) .await .unwrap() .unwrap(), ) .unwrap(); assert_eq!(full_message, expected_full_message, "for {account_id}"); } } // Remove test data for account in [john, jane, bill] { params.destroy_all_mailboxes(account).await; } params.assert_is_empty().await; // Restore core params.server.inner.shared_core.store(old_core); // Check webhook events params.webhook.assert_contains(&[ "message-ingest.", "delivery.dsn", "\"from\": \"bill@example.com\"", "\"john.doe@example.com\"", ]); } async fn assert_message_headers_contains( server: &Server, account_id: u32, document_id: u32, value: &str, ) { let headers = message_headers(server, account_id, document_id).await; assert!( headers.contains(value), "Expected message headers to contain {:?}, got {:?}", value, headers ); } async fn message_headers(server: &Server, account_id: u32, document_id: u32) -> String { std::str::from_utf8( message_metadata(server, account_id, document_id) .await .raw_headers .as_ref(), ) .unwrap() .to_string() } async fn message_metadata(server: &Server, account_id: u32, document_id: u32) -> MessageMetadata { server .store() .get_value::>(ValueKey::property( account_id, Collection::Email, document_id, EmailField::Metadata, )) .await .unwrap() .unwrap() .deserialize::() .unwrap() } pub struct SmtpConnection { reader: Lines>>, writer: WriteHalf, } impl SmtpConnection { pub async fn ingest_with_code( &mut self, from: &str, recipients: &[&str], message: &str, code: u8, ) -> Vec { self.mail_from(from, 2).await; for recipient in recipients { self.rcpt_to(recipient, 2).await; } self.data(3).await; let result = self.data_bytes(message, recipients.len(), code).await; tokio::time::sleep(Duration::from_millis(500)).await; result } pub async fn ingest(&mut self, from: &str, recipients: &[&str], message: &str) { self.ingest_with_code(from, recipients, message, 2).await; } async fn ingest_chunked( &mut self, from: &str, recipients: &[&str], message: &str, chunk_size: usize, ) { self.mail_from(from, 2).await; for recipient in recipients { self.rcpt_to(recipient, 2).await; } for chunk in message.as_bytes().chunks(chunk_size) { self.bdat(std::str::from_utf8(chunk).unwrap(), 2).await; } self.bdat_last("", recipients.len(), 2).await; tokio::time::sleep(Duration::from_millis(500)).await; } pub async fn connect() -> Self { SmtpConnection::connect_port(11200).await } pub async fn connect_port(port: u16) -> Self { let (reader, writer) = tokio::io::split( TcpStream::connect(&format!("127.0.0.1:{port}")) .await .unwrap(), ); let mut conn = SmtpConnection { reader: BufReader::new(reader).lines(), writer, }; conn.read(1, 2).await; conn.lhlo().await; conn } pub async fn lhlo(&mut self) -> Vec { self.send("LHLO localhost").await; self.read(1, 2).await } pub async fn mail_from(&mut self, sender: &str, code: u8) -> Vec { self.send(&format!("MAIL FROM:<{}>", sender)).await; self.read(1, code).await } pub async fn rcpt_to(&mut self, rcpt: &str, code: u8) -> Vec { self.send(&format!("RCPT TO:<{}>", rcpt)).await; self.read(1, code).await } pub async fn vrfy(&mut self, rcpt: &str, code: u8) -> Vec { self.send(&format!("VRFY {}", rcpt)).await; self.read(1, code).await } pub async fn expn(&mut self, rcpt: &str, code: u8) -> Vec { self.send(&format!("EXPN {}", rcpt)).await; self.read(1, code).await } pub async fn data(&mut self, code: u8) -> Vec { self.send("DATA").await; self.read(1, code).await } pub async fn data_bytes( &mut self, message: &str, num_responses: usize, code: u8, ) -> Vec { self.send_raw(message).await; self.send_raw("\r\n.\r\n").await; self.read(num_responses, code).await } pub async fn bdat(&mut self, chunk: &str, code: u8) -> Vec { self.send_raw(&format!("BDAT {}\r\n{}", chunk.len(), chunk)) .await; self.read(1, code).await } pub async fn bdat_last(&mut self, chunk: &str, num_responses: usize, code: u8) -> Vec { self.send_raw(&format!("BDAT {} LAST\r\n{}", chunk.len(), chunk)) .await; self.read(num_responses, code).await } pub async fn rset(&mut self) -> Vec { self.send("RSET").await; self.read(1, 2).await } pub async fn noop(&mut self) -> Vec { self.send("NOOP").await; self.read(1, 2).await } pub async fn quit(&mut self) -> Vec { self.send("QUIT").await; self.read(1, 2).await } pub async fn read(&mut self, mut num_responses: usize, code: u8) -> Vec { let mut lines = Vec::new(); loop { match tokio::time::timeout(Duration::from_millis(1500), self.reader.next_line()).await { Ok(Ok(Some(line))) => { let is_done = line.as_bytes()[3] == b' '; //let c = println!("<- {:?}", line); lines.push(line); if is_done { num_responses -= 1; if num_responses != 0 { continue; } if code != u8::MAX { for line in &lines { if line.as_bytes()[0] - b'0' != code { panic!("Expected completion code {}, got {:?}.", code, lines); } } } return lines; } } Ok(Ok(None)) => { panic!("Invalid response: {:?}.", lines); } Ok(Err(err)) => { panic!("Connection broken: {} ({:?})", err, lines); } Err(_) => panic!("Timeout while waiting for server response: {:?}", lines), } } } pub async fn send(&mut self, text: &str) { //let c = println!("-> {:?}", text); self.writer.write_all(text.as_bytes()).await.unwrap(); self.writer.write_all(b"\r\n").await.unwrap(); self.writer.flush().await.unwrap(); } pub async fn send_raw(&mut self, text: &str) { //let c = println!("-> {:?}", text); self.writer.write_all(text.as_bytes()).await.unwrap(); } } pub trait AssertResult: Sized { fn assert_contains(self, text: &str) -> Self; fn assert_count(self, text: &str, occurrences: usize) -> Self; fn assert_equals(self, text: &str) -> Self; } impl AssertResult for Vec { fn assert_contains(self, text: &str) -> Self { for line in &self { if line.contains(text) { return self; } } panic!("Expected response to contain {:?}, got {:?}", text, self); } fn assert_count(self, text: &str, occurrences: usize) -> Self { assert_eq!( self.iter().filter(|l| l.contains(text)).count(), occurrences, "Expected {} occurrences of {:?}, found {}.", occurrences, text, self.iter().filter(|l| l.contains(text)).count() ); self } fn assert_equals(self, text: &str) -> Self { for line in &self { if line == text { return self; } } panic!("Expected response to be {:?}, got {:?}", text, self); } } ================================================ FILE: tests/src/jmap/mail/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::{JMAPTest, replace_blob_ids}; use ::email::mailbox::INBOX_ID; use jmap_client::email::{self, Header, HeaderForm, import::EmailImportResponse}; use mail_parser::HeaderName; use std::{fs, path::PathBuf}; use types::id::Id; pub async fn test(params: &mut JMAPTest) { println!("Running Email Get tests..."); let mut test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); test_dir.push("resources"); test_dir.push("jmap"); test_dir.push("email_get"); let mailbox_id = Id::from(INBOX_ID).to_string(); let account = params.account("jdoe@example.com"); let client = account.client(); for file_name in fs::read_dir(&test_dir).unwrap() { let mut file_name = file_name.as_ref().unwrap().path(); if file_name.extension().is_none_or(|e| e != "eml") { continue; } let is_headers_test = file_name.file_name().unwrap() == "headers.eml"; let blob = fs::read(&file_name).unwrap(); let blob_len = blob.len(); // Import email let mut request = client.build(); let import_request = request .import_email() .email( client .upload(None, blob, None) .await .unwrap() .take_blob_id(), ) .mailbox_ids([mailbox_id.clone()]) .keywords(["tag".to_string()]) .received_at((blob_len * 1000000) as i64); let id = import_request.create_id(); let mut response = request.send_single::().await.unwrap(); assert_ne!(response.old_state(), Some(response.new_state())); let email = response.created(&id).unwrap(); let mut request = client.build(); request .get_email() .ids([email.id().unwrap()]) .properties([ email::Property::Id, email::Property::BlobId, email::Property::ThreadId, email::Property::MailboxIds, email::Property::Keywords, email::Property::Size, email::Property::ReceivedAt, email::Property::MessageId, email::Property::InReplyTo, email::Property::References, email::Property::Sender, email::Property::From, email::Property::To, email::Property::Cc, email::Property::Bcc, email::Property::ReplyTo, email::Property::Subject, email::Property::SentAt, email::Property::HasAttachment, email::Property::Preview, email::Property::BodyValues, email::Property::TextBody, email::Property::HtmlBody, email::Property::Attachments, email::Property::BodyStructure, ]) .arguments() .body_properties(if !is_headers_test { [ email::BodyProperty::PartId, email::BodyProperty::BlobId, email::BodyProperty::Size, email::BodyProperty::Name, email::BodyProperty::Type, email::BodyProperty::Charset, email::BodyProperty::Headers, email::BodyProperty::Disposition, email::BodyProperty::Cid, email::BodyProperty::Language, email::BodyProperty::Location, ] } else { [ email::BodyProperty::PartId, email::BodyProperty::Size, email::BodyProperty::Name, email::BodyProperty::Type, email::BodyProperty::Charset, email::BodyProperty::Disposition, email::BodyProperty::Cid, email::BodyProperty::Language, email::BodyProperty::Location, email::BodyProperty::Header(Header { name: "X-Custom-Header".into(), form: HeaderForm::Raw, all: false, }), email::BodyProperty::Header(Header { name: "X-Custom-Header-2".into(), form: HeaderForm::Raw, all: false, }), ] }) .fetch_all_body_values(true) .max_body_value_bytes(100); let mut result = request .send_get_email() .await .unwrap() .take_list() .pop() .unwrap() .into_test(); if is_headers_test { for property in all_headers() { let mut request = client.build(); request .get_email() .ids([email.id().unwrap()]) .properties([property]); result.headers.extend( request .send_get_email() .await .unwrap() .take_list() .pop() .unwrap() .into_test() .headers, ); } } let result = replace_blob_ids(serde_json::to_string_pretty(&result).unwrap()); file_name.set_extension("json"); if fs::read(&file_name).unwrap() != result.as_bytes() { file_name.set_extension("failed"); fs::write(&file_name, result.as_bytes()).unwrap(); panic!("Test failed, output saved to {}", file_name.display()); } } params.destroy_all_mailboxes(account).await; params.assert_is_empty().await; } pub fn all_headers() -> Vec { let mut properties = Vec::new(); for header in [ HeaderName::From, HeaderName::To, HeaderName::Cc, HeaderName::Bcc, HeaderName::Other("X-Address-Single".into()), HeaderName::Other("X-Address".into()), HeaderName::Other("X-AddressList-Single".into()), HeaderName::Other("X-AddressList".into()), HeaderName::Other("X-AddressesGroup-Single".into()), HeaderName::Other("X-AddressesGroup".into()), ] { properties.push(email::Property::Header(Header { form: HeaderForm::Raw, name: header.as_str().to_string(), all: true, })); properties.push(email::Property::Header(Header { form: HeaderForm::Raw, name: header.as_str().to_string(), all: false, })); properties.push(email::Property::Header(Header { form: HeaderForm::Addresses, name: header.as_str().to_string(), all: true, })); properties.push(email::Property::Header(Header { form: HeaderForm::Addresses, name: header.as_str().to_string(), all: false, })); properties.push(email::Property::Header(Header { form: HeaderForm::GroupedAddresses, name: header.as_str().to_string(), all: true, })); properties.push(email::Property::Header(Header { form: HeaderForm::GroupedAddresses, name: header.as_str().to_string(), all: false, })); } for header in [ HeaderName::ListPost, HeaderName::ListSubscribe, HeaderName::ListUnsubscribe, HeaderName::ListOwner, HeaderName::Other("X-List-Single".into()), HeaderName::Other("X-List".into()), ] { properties.push(email::Property::Header(Header { form: HeaderForm::Raw, name: header.as_str().to_string(), all: true, })); properties.push(email::Property::Header(Header { form: HeaderForm::Raw, name: header.as_str().to_string(), all: false, })); properties.push(email::Property::Header(Header { form: HeaderForm::URLs, name: header.as_str().to_string(), all: true, })); properties.push(email::Property::Header(Header { form: HeaderForm::URLs, name: header.as_str().to_string(), all: false, })); } for header in [ HeaderName::Date, HeaderName::ResentDate, HeaderName::Other("X-Date-Single".into()), HeaderName::Other("X-Date".into()), ] { properties.push(email::Property::Header(Header { form: HeaderForm::Raw, name: header.as_str().to_string(), all: true, })); properties.push(email::Property::Header(Header { form: HeaderForm::Raw, name: header.as_str().to_string(), all: false, })); properties.push(email::Property::Header(Header { form: HeaderForm::Date, name: header.as_str().to_string(), all: true, })); properties.push(email::Property::Header(Header { form: HeaderForm::Date, name: header.as_str().to_string(), all: false, })); } for header in [ HeaderName::MessageId, HeaderName::References, HeaderName::Other("X-Id-Single".into()), HeaderName::Other("X-Id".into()), ] { properties.push(email::Property::Header(Header { form: HeaderForm::Raw, name: header.as_str().to_string(), all: true, })); properties.push(email::Property::Header(Header { form: HeaderForm::Raw, name: header.as_str().to_string(), all: false, })); properties.push(email::Property::Header(Header { form: HeaderForm::MessageIds, name: header.as_str().to_string(), all: true, })); properties.push(email::Property::Header(Header { form: HeaderForm::MessageIds, name: header.as_str().to_string(), all: false, })); } for header in [ HeaderName::Subject, HeaderName::Keywords, HeaderName::Other("X-Text-Single".into()), HeaderName::Other("X-Text".into()), ] { properties.push(email::Property::Header(Header { form: HeaderForm::Raw, name: header.as_str().to_string(), all: true, })); properties.push(email::Property::Header(Header { form: HeaderForm::Raw, name: header.as_str().to_string(), all: false, })); properties.push(email::Property::Header(Header { form: HeaderForm::Text, name: header.as_str().to_string(), all: true, })); properties.push(email::Property::Header(Header { form: HeaderForm::Text, name: header.as_str().to_string(), all: false, })); } properties } ================================================ FILE: tests/src/jmap/mail/mailbox.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::{Account, JMAPTest, wait_for_index}; use jmap_client::{ Error, Set, client::{Client, Credentials}, core::{ query::Filter, set::{SetError, SetErrorType, SetObject, SetRequest}, }, mailbox::{self, Mailbox, Role}, }; use jmap_proto::types::state::State; use serde::{Deserialize, Serialize}; use std::time::Duration; use store::ahash::AHashMap; use types::id::Id; pub async fn test(params: &mut JMAPTest) { println!("Running Mailbox tests..."); let account = params.account("admin"); let mut client = account.client_owned().await; // Create test mailboxes client.set_default_account_id(Id::from(0u64)); let id_map = create_test_mailboxes(&client).await; // Sort by name assert_eq!( client .mailbox_query( None::, [mailbox::query::Comparator::name()].into() ) .await .unwrap() .ids() .iter() .map(|id| id_map.get(id).unwrap()) .collect::>(), [ "drafts", "spam2", "inbox", "l.1", "l.2", "l.3", "sent", "spam", "1.1", "1.2", "trash", "spam1", "1.1.1.1", "1.1.1.1.1", "1.1.1", "1.2.1" ] ); // Sort by name as tree let mut request = client.build(); request .query_mailbox() .sort([mailbox::query::Comparator::name()]) .arguments() .sort_as_tree(true); assert_eq!( request .send_query_mailbox() .await .unwrap() .ids() .iter() .map(|id| id_map.get(id).unwrap()) .collect::>(), [ "drafts", "inbox", "l.1", "1.1", "1.1.1", "1.1.1.1", "1.1.1.1.1", "1.2", "1.2.1", "l.2", "l.3", "sent", "spam", "spam1", "spam2", "trash" ] ); // Sort as tree with filters let mut request = client.build(); request .query_mailbox() .filter(mailbox::query::Filter::name("level")) .sort([mailbox::query::Comparator::name()]) .arguments() .sort_as_tree(true); assert_eq!( request .send_query_mailbox() .await .unwrap() .ids() .iter() .map(|id| id_map.get(id).unwrap()) .collect::>(), [ "l.1", "1.1", "1.1.1", "1.1.1.1", "1.1.1.1.1", "1.2", "1.2.1", "l.2", "l.3" ] ); // Filter as tree let mut request = client.build(); request .query_mailbox() .filter(mailbox::query::Filter::name("spam")) .sort([mailbox::query::Comparator::name()]) .arguments() .filter_as_tree(true) .sort_as_tree(true); assert_eq!( request .send_query_mailbox() .await .unwrap() .ids() .iter() .map(|id| id_map.get(id).unwrap()) .collect::>(), ["spam", "spam1", "spam2"] ); let mut request = client.build(); request .query_mailbox() .filter(mailbox::query::Filter::name("level")) .sort([mailbox::query::Comparator::name()]) .arguments() .filter_as_tree(true) .sort_as_tree(true); assert_eq!( request.send_query_mailbox().await.unwrap().ids(), Vec::<&str>::new() ); // Filter by role assert_eq!( client .mailbox_query( mailbox::query::Filter::role(Role::Inbox).into(), [mailbox::query::Comparator::name()].into() ) .await .unwrap() .ids() .iter() .map(|id| id_map.get(id).unwrap()) .collect::>(), ["inbox"] ); assert_eq!( client .mailbox_query( mailbox::query::Filter::has_any_role(true).into(), [mailbox::query::Comparator::name()].into() ) .await .unwrap() .ids() .iter() .map(|id| id_map.get(id).unwrap()) .collect::>(), ["drafts", "inbox", "sent", "spam", "trash"] ); // Duplicate role let mut request = client.build(); request .set_mailbox() .update(&id_map["sent"]) .role(Role::Inbox); assert!(matches!( request .send_set_mailbox() .await .unwrap() .updated(&id_map["sent"]), Err(Error::Set(SetError { type_: SetErrorType::InvalidProperties, .. })) )); // Duplicate name let mut request = client.build(); request.set_mailbox().update(&id_map["l.2"]).name("Level 3"); let result = request .send_set_mailbox() .await .unwrap() .updated(&id_map["l.2"]); assert!( matches!( result, Err(Error::Set(SetError { type_: SetErrorType::InvalidProperties, .. })) ), "{result:?}", ); // Circular relationship let mut request = client.build(); request .set_mailbox() .update(&id_map["l.1"]) .parent_id((&id_map["1.1.1.1.1"]).into()); assert!(matches!( request .send_set_mailbox() .await .unwrap() .updated(&id_map["l.1"]), Err(Error::Set(SetError { type_: SetErrorType::InvalidProperties, .. })) )); let mut request = client.build(); request .set_mailbox() .update(&id_map["l.1"]) .parent_id((&id_map["l.1"]).into()); assert!(matches!( request .send_set_mailbox() .await .unwrap() .updated(&id_map["l.1"]), Err(Error::Set(SetError { type_: SetErrorType::InvalidProperties, .. })) )); // Invalid parentId let mut request = client.build(); request .set_mailbox() .update(&id_map["l.1"]) .parent_id(Id::new(u64::MAX).to_string().into()); assert!(matches!( request .send_set_mailbox() .await .unwrap() .updated(&id_map["l.1"]), Err(Error::Set(SetError { type_: SetErrorType::InvalidProperties, .. })) )); // Obtain state let state = client .mailbox_changes(State::Initial.to_string(), 0) .await .unwrap() .new_state() .to_string(); // Rename and move mailbox let mut request = client.build(); request .set_mailbox() .update(&id_map["1.1.1.1.1"]) .name("Renamed and moved") .parent_id((&id_map["l.2"]).into()); assert!( request .send_set_mailbox() .await .unwrap() .updated(&id_map["1.1.1.1.1"]) .is_ok() ); // Verify changes let state = client.mailbox_changes(state, 0).await.unwrap(); assert_eq!(state.created().len(), 0); assert_eq!(state.updated().len(), 1); assert_eq!(state.destroyed().len(), 0); assert_eq!(state.arguments().updated_properties(), None); let state = state.new_state().to_string(); // Insert email into Inbox let mail_id = client .email_import( b"From: test@test.com\nSubject: hey\n\ntest".to_vec(), [&id_map["inbox"]], None::>, None, ) .await .unwrap() .take_id(); // Inbox's total and unread count should have increased let inbox = client .mailbox_get( &id_map["inbox"], [ mailbox::Property::TotalEmails, mailbox::Property::UnreadEmails, mailbox::Property::TotalThreads, mailbox::Property::UnreadThreads, ] .into(), ) .await .unwrap() .unwrap(); assert_eq!(inbox.total_emails(), 1); assert_eq!(inbox.unread_emails(), 1); assert_eq!(inbox.total_threads(), 1); assert_eq!(inbox.unread_threads(), 1); // Set email to read and fetch properties again client .email_set_keyword(&mail_id, "$seen", true) .await .unwrap(); let inbox = client .mailbox_get( &id_map["inbox"], [ mailbox::Property::TotalEmails, mailbox::Property::UnreadEmails, mailbox::Property::TotalThreads, mailbox::Property::UnreadThreads, ] .into(), ) .await .unwrap() .unwrap(); assert_eq!(inbox.total_emails(), 1); assert_eq!(inbox.unread_emails(), 0); assert_eq!(inbox.total_threads(), 1); assert_eq!(inbox.unread_threads(), 0); // Only email properties must have changed let prev_state = state.clone(); let state = client.mailbox_changes(state, 0).await.unwrap(); assert_eq!(state.created().len(), 0); assert_eq!( state .updated() .iter() .map(|s| s.as_str()) .collect::>(), &[&id_map["inbox"]] ); assert_eq!(state.destroyed().len(), 0); assert_eq!( state.arguments().updated_properties(), Some( &[ mailbox::Property::TotalEmails, mailbox::Property::UnreadEmails, mailbox::Property::TotalThreads, mailbox::Property::UnreadThreads, ][..] ) ); let state = state.new_state().to_string(); // Use updatedProperties in a query let mut request = client.build(); let changes_request = request.changes_mailbox(prev_state).max_changes(0); let properties_ref = changes_request.updated_properties_reference(); let updated_ref = changes_request.updated_reference(); request .get_mailbox() .ids_ref(updated_ref) .properties_ref(properties_ref); let mut changed_mailboxes = request .send() .await .unwrap() .unwrap_method_responses() .pop() .unwrap() .unwrap_get_mailbox() .unwrap() .take_list(); assert_eq!(changed_mailboxes.len(), 1); let inbox = changed_mailboxes.pop().unwrap(); assert_eq!(inbox.id().unwrap(), &id_map["inbox"]); assert_eq!(inbox.total_emails(), 1); assert_eq!(inbox.unread_emails(), 0); assert_eq!(inbox.total_threads(), 1); assert_eq!(inbox.unread_threads(), 0); assert_eq!(inbox.name(), None); assert_eq!(inbox.my_rights(), None); // Move email from Inbox to Trash client .email_set_mailboxes(&mail_id, [&id_map["trash"]]) .await .unwrap(); // E-mail properties of both Inbox and Trash must have changed let state = client.mailbox_changes(state, 0).await.unwrap(); assert_eq!(state.created().len(), 0); assert_eq!(state.updated().len(), 2); assert_eq!(state.destroyed().len(), 0); let mut folder_ids = vec![&id_map["trash"], &id_map["inbox"]]; let mut updated_ids = state .updated() .iter() .map(|s| s.as_str()) .collect::>(); updated_ids.sort_unstable(); folder_ids.sort_unstable(); assert_eq!(updated_ids, folder_ids); assert_eq!( state.arguments().updated_properties(), Some( &[ mailbox::Property::TotalEmails, mailbox::Property::UnreadEmails, mailbox::Property::TotalThreads, mailbox::Property::UnreadThreads, ][..] ) ); // Deleting folders with children is not allowed let mut request = client.build(); request.set_mailbox().destroy([&id_map["l.1"]]); assert!(matches!( request .send_set_mailbox() .await .unwrap() .destroyed(&id_map["l.1"]), Err(Error::Set(SetError { type_: SetErrorType::MailboxHasChild, .. })) )); // Deleting folders with contents is not allowed (unless remove_emails is true) let mut request = client.build(); request.set_mailbox().destroy([&id_map["trash"]]); assert!(matches!( request .send_set_mailbox() .await .unwrap() .destroyed(&id_map["trash"]), Err(Error::Set(SetError { type_: SetErrorType::MailboxHasEmail, .. })) )); // Delete Trash folder and its contents let mut request = client.build(); request .set_mailbox() .destroy([&id_map["trash"]]) .arguments() .on_destroy_remove_emails(true); assert!( request .send_set_mailbox() .await .unwrap() .destroyed(&id_map["trash"]) .is_ok() ); // Verify that Trash folder and its contents are gone assert!( client .mailbox_get(&id_map["trash"], None::>) .await .unwrap() .is_none() ); assert!( client .email_get(&mail_id, None::>) .await .unwrap() .is_none() ); // Check search results after changing folder properties let mut request = client.build(); request .set_mailbox() .update(&id_map["drafts"]) .name("Borradores") .sort_order(100) .parent_id((&id_map["l.2"]).into()) .role(Role::None); assert!( request .send_set_mailbox() .await .unwrap() .updated(&id_map["drafts"]) .is_ok() ); assert_eq!( client .mailbox_query( Filter::and([ mailbox::query::Filter::name("Borradores").into(), mailbox::query::Filter::parent_id((&id_map["l.2"]).into()).into(), Filter::not([mailbox::query::Filter::has_any_role(true)]) ]) .into(), [mailbox::query::Comparator::name()].into() ) .await .unwrap() .ids() .iter() .map(|id| id_map.get(id).unwrap()) .collect::>(), ["drafts"] ); assert!( client .mailbox_query( mailbox::query::Filter::name("Drafts").into(), [mailbox::query::Comparator::name()].into() ) .await .unwrap() .ids() .is_empty() ); assert!( client .mailbox_query( mailbox::query::Filter::role(Role::Drafts).into(), [mailbox::query::Comparator::name()].into() ) .await .unwrap() .ids() .is_empty() ); assert_eq!( client .mailbox_query( mailbox::query::Filter::parent_id(None::<&str>).into(), [mailbox::query::Comparator::name()].into() ) .await .unwrap() .ids() .iter() .map(|id| id_map.get(id).unwrap()) .collect::>(), ["inbox", "sent", "spam"] ); assert_eq!( client .mailbox_query( mailbox::query::Filter::has_any_role(true).into(), [mailbox::query::Comparator::name()].into() ) .await .unwrap() .ids() .iter() .map(|id| id_map.get(id).unwrap()) .collect::>(), ["inbox", "sent", "spam"] ); destroy_all_mailboxes_no_wait(&client).await; params.assert_is_empty().await; } async fn create_test_mailboxes(client: &Client) -> AHashMap { let mut mailbox_map = AHashMap::default(); let mut request = client.build(); build_create_query( request.set_mailbox(), &mut mailbox_map, serde_json::from_slice(TEST_MAILBOXES).unwrap(), None, ); let mut result = request.send_set_mailbox().await.unwrap(); let mut id_map = AHashMap::with_capacity(mailbox_map.len()); for (create_id, local_id) in mailbox_map { let server_id = result.created(&create_id).unwrap().take_id(); id_map.insert(local_id.clone(), server_id.clone()); id_map.insert(server_id, local_id); } id_map } fn build_create_query( request: &mut SetRequest>, mailbox_map: &mut AHashMap, mailboxes: Vec, parent_id: Option, ) { for mailbox in mailboxes { let create_mailbox = request .create() .name(mailbox.name) .sort_order(mailbox.order); if let Some(role) = mailbox.role { create_mailbox.role(role); } if let Some(parent_id) = &parent_id { create_mailbox.parent_id_ref(parent_id); } let create_mailbox_id = create_mailbox.create_id().unwrap(); mailbox_map.insert(create_mailbox_id.clone(), mailbox.id); if let Some(children) = mailbox.children { build_create_query(request, mailbox_map, children, create_mailbox_id.into()); } } } impl JMAPTest { pub async fn destroy_all_mailboxes(&self, account: &Account) { wait_for_index(&self.server).await; destroy_all_mailboxes_no_wait(account.client()).await; } } pub async fn destroy_all_mailboxes_for_account(account_id: u32) { let mut client = Client::new() .credentials(Credentials::basic("admin", "secret")) .follow_redirects(["127.0.0.1"]) .timeout(Duration::from_secs(3600)) .accept_invalid_certs(true) .connect("https://127.0.0.1:8899") .await .unwrap(); client.set_default_account_id(Id::from(account_id)); destroy_all_mailboxes_no_wait(&client).await; } pub async fn destroy_all_mailboxes_no_wait(client: &Client) { let mut request = client.build(); request.query_mailbox().arguments().sort_as_tree(true); let mut ids = request.send_query_mailbox().await.unwrap().take_ids(); ids.reverse(); for id in ids { client.mailbox_destroy(&id, true).await.unwrap(); } } #[derive(Serialize, Deserialize)] struct TestMailbox { id: String, name: String, role: Option, order: u32, children: Option>, } const TEST_MAILBOXES: &[u8] = br#" [ { "id": "inbox", "name": "Inbox", "role": "INBOX", "order": 5, "children": [ { "name": "Level 1", "id": "l.1", "order": 4, "children": [ { "name": "Sub-Level 1.1", "id": "1.1", "order": 3, "children": [ { "name": "Z-Sub-Level 1.1.1", "id": "1.1.1", "order": 2, "children": [ { "name": "X-Sub-Level 1.1.1.1", "id": "1.1.1.1", "order": 1, "children": [ { "name": "Y-Sub-Level 1.1.1.1.1", "id": "1.1.1.1.1", "order": 0 } ] } ] } ] }, { "name": "Sub-Level 1.2", "id": "1.2", "order": 7, "children": [ { "name": "Z-Sub-Level 1.2.1", "id": "1.2.1", "order": 6 } ] } ] }, { "name": "Level 2", "id": "l.2", "order": 8 }, { "name": "Level 3", "id": "l.3", "order": 9 } ] }, { "id": "sent", "name": "Sent", "role": "SENT", "order": 15 }, { "id": "drafts", "name": "Drafts", "role": "DRAFTS", "order": 14 }, { "id": "trash", "name": "Trash", "role": "TRASH", "order": 13 }, { "id": "spam", "name": "Spam", "role": "JUNK", "order": 12, "children": [{ "id": "spam1", "name": "Work Spam", "order": 11, "children": [{ "id": "spam2", "name": "Friendly Spam", "order": 10 }] }] } ] "#; ================================================ FILE: tests/src/jmap/mail/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod acl; pub mod antispam; pub mod changes; pub mod copy; pub mod crypto; pub mod delivery; pub mod get; pub mod mailbox; pub mod parse; pub mod query; pub mod query_changes; pub mod search_snippet; pub mod set; pub mod sieve_script; pub mod submission; pub mod thread_get; pub mod thread_merge; pub mod vacation_response; ================================================ FILE: tests/src/jmap/mail/parse.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::{JMAPTest, mail::get::all_headers, replace_blob_ids}; use jmap_client::{ email::{self, Header, HeaderForm}, mailbox::Role, }; use std::{fs, path::PathBuf}; pub async fn test(params: &mut JMAPTest) { println!("Running Email Parse tests..."); let account = params.account("jdoe@example.com"); let client = account.client(); let mut test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); test_dir.push("resources"); test_dir.push("jmap"); test_dir.push("email_parse"); let mailbox_id = client .mailbox_create("JMAP Parse", None::, Role::None) .await .unwrap() .take_id(); // Test parsing an email attachment for test_name in ["attachment.eml", "attachment_b64.eml"] { let mut test_file = test_dir.clone(); test_file.push(test_name); let email = client .email_import( fs::read(&test_file).unwrap(), [mailbox_id.clone()], None::>, None, ) .await .unwrap(); let blob_id = client .email_get(email.id().unwrap(), Some([email::Property::Attachments])) .await .unwrap() .unwrap() .attachments() .unwrap() .first() .unwrap() .blob_id() .unwrap() .to_string(); let email = client .email_parse( &blob_id, [ email::Property::Id, email::Property::BlobId, email::Property::ThreadId, email::Property::MailboxIds, email::Property::Keywords, email::Property::Size, email::Property::ReceivedAt, email::Property::MessageId, email::Property::InReplyTo, email::Property::References, email::Property::Sender, email::Property::From, email::Property::To, email::Property::Cc, email::Property::Bcc, email::Property::ReplyTo, email::Property::Subject, email::Property::SentAt, email::Property::HasAttachment, email::Property::Preview, email::Property::BodyValues, email::Property::TextBody, email::Property::HtmlBody, email::Property::Attachments, email::Property::BodyStructure, ] .into(), [ email::BodyProperty::PartId, email::BodyProperty::BlobId, email::BodyProperty::Size, email::BodyProperty::Name, email::BodyProperty::Type, email::BodyProperty::Charset, email::BodyProperty::Headers, email::BodyProperty::Disposition, email::BodyProperty::Cid, email::BodyProperty::Language, email::BodyProperty::Location, ] .into(), 100.into(), ) .await .unwrap(); if !test_name.contains("_b64") { for parts in [ email.text_body().unwrap(), email.html_body().unwrap(), email.attachments().unwrap(), ] { for part in parts { let blob_id = part.blob_id().unwrap(); let inner_blob = client.download(blob_id).await.unwrap(); test_file.set_extension(format!("part{}", part.part_id().unwrap())); //fs::write(&test_file, inner_blob).unwrap(); let expected_inner_blob = fs::read(&test_file).unwrap(); assert_eq!( inner_blob, expected_inner_blob, "file: {}", test_file.display() ); } } } test_file.set_extension("json"); let result = replace_blob_ids(serde_json::to_string_pretty(&email.into_test()).unwrap()); if fs::read(&test_file).unwrap() != result.as_bytes() { test_file.set_extension("failed"); fs::write(&test_file, result.as_bytes()).unwrap(); panic!("Test failed, output saved to {}", test_file.display()); } } // Test header parsing on a temporary blob let mut test_file = test_dir; test_file.push("headers.eml"); let blob_id = client .upload(None, fs::read(&test_file).unwrap(), None) .await .unwrap() .take_blob_id(); let mut email = client .email_parse( &blob_id, [ email::Property::Id, email::Property::MessageId, email::Property::InReplyTo, email::Property::References, email::Property::Sender, email::Property::From, email::Property::To, email::Property::Cc, email::Property::Bcc, email::Property::ReplyTo, email::Property::Subject, email::Property::SentAt, email::Property::Preview, email::Property::TextBody, email::Property::HtmlBody, email::Property::Attachments, ] .into(), [ email::BodyProperty::Size, email::BodyProperty::Name, email::BodyProperty::Type, email::BodyProperty::Charset, email::BodyProperty::Disposition, email::BodyProperty::Cid, email::BodyProperty::Language, email::BodyProperty::Location, email::BodyProperty::Header(Header { name: "X-Custom-Header".into(), form: HeaderForm::Raw, all: false, }), email::BodyProperty::Header(Header { name: "X-Custom-Header-2".into(), form: HeaderForm::Raw, all: false, }), ] .into(), 100.into(), ) .await .unwrap() .into_test(); for property in all_headers() { email.headers.extend( client .email_parse(&blob_id, [property].into(), [].into(), None) .await .unwrap() .into_test() .headers, ); } test_file.set_extension("json"); let result = replace_blob_ids(serde_json::to_string_pretty(&email).unwrap()); if fs::read(&test_file).unwrap() != result.as_bytes() { test_file.set_extension("failed"); fs::write(&test_file, result.as_bytes()).unwrap(); panic!("Test failed, output saved to {}", test_file.display()); } params.destroy_all_mailboxes(account).await; params.assert_is_empty().await; } ================================================ FILE: tests/src/jmap/mail/query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ jmap::{Account, JMAPTest, wait_for_index}, store::{deflate_test_resource, query::FIELDS}, }; use ::email::{cache::MessageCacheFetch, mailbox::Mailbox}; use ahash::AHashSet; use common::{Server, storage::index::ObjectIndexBuilder}; use jmap_client::{ client::Client, core::query::{Comparator, Filter}, email, }; use mail_builder::{ MessageBuilder, headers::{date::Date, message_id::MessageId, text::Text}, }; use mail_parser::HeaderName; use std::{collections::hash_map::Entry, str::FromStr, time::Instant}; use store::{ ahash::AHashMap, write::{BatchBuilder, now}, }; use types::{collection::Collection, id::Id, special_use::SpecialUse}; const MAX_THREADS: usize = 100; const MAX_MESSAGES: usize = 1000; const MAX_MESSAGES_PER_THREAD: usize = 100; pub async fn test(params: &mut JMAPTest, insert: bool) { println!("Running Email Query tests..."); let server = params.server.clone(); let account = params.account("jdoe@example.com"); let client = account.client(); if insert { // Add some "virtual" mailbox ids so create doesn't fail let mut batch = BatchBuilder::new(); let account_id = Id::from_str(client.default_account_id()) .unwrap() .document_id(); batch .with_account_id(account_id) .with_collection(Collection::Mailbox); for mailbox_id in 1545..3010 { batch .with_document(mailbox_id) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(Mailbox { name: format!("Mailbox {mailbox_id}"), role: SpecialUse::None, parent_id: 0, sort_order: None, uid_validity: 0, subscribers: vec![], acls: vec![], })) .unwrap(); } server .core .storage .data .write(batch.build_all()) .await .unwrap(); // Create test messages println!("Inserting JMAP Mail query test messages..."); create(&server, account).await; assert_eq!( params .server .get_cached_messages(account_id) .await .unwrap() .emails .items .iter() .map(|m| m.thread_id) .collect::>() .len(), MAX_THREADS ); // Wait for indexing to complete wait_for_index(&server).await; } let can_stem = !params.server.search_store().is_mysql(); println!("Running JMAP Mail query tests..."); query(client, can_stem).await; println!("Running JMAP Mail query options tests..."); query_options(client).await; println!("Deleting all messages..."); let mut request = client.build(); let result_ref = request.query_email().result_reference(); request.set_email().destroy_ref(result_ref); let response = request.send().await.unwrap(); response .unwrap_method_responses() .pop() .unwrap() .unwrap_set_email() .unwrap(); params.destroy_all_mailboxes(account).await; params.assert_is_empty().await; } pub async fn query(client: &Client, can_stem: bool) { for (filter, sort, expected_results) in [ ( Filter::and(vec![ (email::query::Filter::after(1850)), (email::query::Filter::from("george")), ]), vec![ email::query::Comparator::subject(), email::query::Comparator::sent_at(), ], vec![ "N01389", "T10115", "N00618", "N03500", "T01587", "T00397", "N01561", "N05250", "N03973", "N04973", "N04057", "N01940", "N01539", "N01612", "N04484", "N01954", "N05998", "T02053", "AR00171", "AR00172", "AR00176", ], ), ( Filter::and(vec![ (email::query::Filter::in_mailbox(Id::new(1768u64).to_string())), (email::query::Filter::cc("canvas")), ]), vec![ email::query::Comparator::from(), email::query::Comparator::sent_at(), ], vec!["T01882", "N04689", "T00925", "N00121"], ), ( Filter::and(vec![ (email::query::Filter::text(if can_stem { "study" } else { "studies" })), (email::query::Filter::in_mailbox_other_than(vec![ Id::new(1991).to_string(), Id::new(1870).to_string(), Id::new(2011).to_string(), Id::new(1951).to_string(), Id::new(1902).to_string(), Id::new(1808).to_string(), Id::new(1963).to_string(), ])), ]), vec![ email::query::Comparator::subject(), email::query::Comparator::sent_at(), ], if can_stem { vec![ "T10330", "N01744", "N01743", "N04885", "N02688", "N02122", "A00059", "A00058", "N02123", "T00651", "T09439", "N05001", "T05848", "T05508", ] } else { vec!["T10330", "N02122", "N02123", "T09439"] }, ), ( Filter::and(vec![ (email::query::Filter::has_keyword("N0")).into(), Filter::not(vec![(email::query::Filter::from("collins"))]), (email::query::Filter::body("bequeathed")).into(), ]), vec![ email::query::Comparator::subject(), email::query::Comparator::sent_at(), ], vec![ "N02640", "A01020", "N01250", "T03430", "N01800", "N00620", "N05250", "N04630", "A01040", ], ), ( email::query::Filter::not_keyword("artist").into(), vec![ email::query::Comparator::subject(), email::query::Comparator::sent_at(), ], vec!["T08626", "T09334", "T09455", "N01737", "T10965"], ), ( Filter::and(vec![ (email::query::Filter::after(1970)), (email::query::Filter::before(1972)), (email::query::Filter::text("colour")), ]), vec![ email::query::Comparator::from(), email::query::Comparator::sent_at(), ], vec!["T01745", "P01436", "P01437"], ), ( Filter::and(vec![(email::query::Filter::text("'cats and dogs'"))]), vec![email::query::Comparator::from()], vec!["P77623"], ), ( Filter::and(vec![ (email::query::Filter::header( HeaderName::Comments.to_string(), Some("attributed"), )), (email::query::Filter::from("john")), (email::query::Filter::cc("oil")), ]), vec![email::query::Comparator::from()], vec!["T10965"], ), ( Filter::and(vec![ (email::query::Filter::all_in_thread_have_keyword("N")), (email::query::Filter::before(1800)), ]), vec![ email::query::Comparator::from(), email::query::Comparator::sent_at(), ], vec![ "N01496", "N05916", "N01046", "N00675", "N01320", "N01321", "N00273", "N01453", "N02984", ], ), ( Filter::and(vec![ (email::query::Filter::none_in_thread_have_keyword("N")), (email::query::Filter::after(1995)), ]), vec![ email::query::Comparator::from(), email::query::Comparator::sent_at(), ], vec![ "AR00163", "AR00164", "AR00472", "P11481", "AR00066", "AR00178", "P77895", "P77896", "P77897", ], ), ( Filter::and(vec![ (email::query::Filter::some_in_thread_have_keyword("Bronze")), (email::query::Filter::before(1878)), ]), vec![ email::query::Comparator::from(), email::query::Comparator::sent_at(), ], vec![ "N04326", "N01610", "N02920", "N01587", "T00167", "T00168", "N01554", "N01535", "N01536", "N01622", "N01754", "N01594", ], ), // Sorting tests ( email::query::Filter::before(1800).into(), vec![ email::query::Comparator::all_in_thread_have_keyword("N"), email::query::Comparator::from(), email::query::Comparator::sent_at(), ], vec![ "N01496", "N05916", "N01046", "N00675", "N01320", "N01321", "N00273", "N01453", "N02984", "T09417", "T01882", "T08820", "N04689", "T08891", "T00986", "N00316", "N03544", "N04296", "N04297", "T08234", "N00112", "T00211", "N01497", "N02639", "N02640", "T00925", "T11683", "T08269", "D00001", "D00002", "D00046", "N00121", "N00126", "T08626", ], ), ( email::query::Filter::before(1800).into(), vec![ email::query::Comparator::all_in_thread_have_keyword("N").descending(), email::query::Comparator::from(), email::query::Comparator::sent_at(), ], vec![ "T09417", "T01882", "T08820", "N04689", "T08891", "T00986", "N00316", "N03544", "N04296", "N04297", "T08234", "N00112", "T00211", "N01497", "N02639", "N02640", "T00925", "T11683", "T08269", "D00001", "D00002", "D00046", "N00121", "N00126", "T08626", "N01496", "N05916", "N01046", "N00675", "N01320", "N01321", "N00273", "N01453", "N02984", ], ), ( Filter::and(vec![ (email::query::Filter::after(1875)), (email::query::Filter::before(1878)), ]), vec![ email::query::Comparator::some_in_thread_have_keyword("Bronze"), email::query::Comparator::from(), email::query::Comparator::sent_at(), ], vec![ "N04326", "N01610", "N02920", "N01587", "T00167", "T00168", "N01554", "N01535", "N01536", "N01622", "N01754", "N01594", "N01559", "N02123", "N01940", "N03594", "N01494", "N04271", ], ), ( Filter::and(vec![ (email::query::Filter::after(1875)), (email::query::Filter::before(1878)), ]), vec![ email::query::Comparator::some_in_thread_have_keyword("Bronze").descending(), email::query::Comparator::from(), email::query::Comparator::sent_at(), ], vec![ "N01559", "N02123", "N01940", "N03594", "N01494", "N04271", "N04326", "N01610", "N02920", "N01587", "T00167", "T00168", "N01554", "N01535", "N01536", "N01622", "N01754", "N01594", ], ), ( Filter::and(vec![ (email::query::Filter::after(1786)), (email::query::Filter::before(1840)), (email::query::Filter::has_keyword("T")), ]), vec![ email::query::Comparator::has_keyword("attributed to"), email::query::Comparator::from(), email::query::Comparator::sent_at(), ], vec![ "T09455", "T09334", "T10965", "T08626", "T09417", "T08951", "T01851", "T01852", "T08761", "T08123", "T08756", "T10561", "T10562", "T10563", "T00986", "T03424", "T03427", "T08234", "T08133", "T06866", "T08897", "T00996", "T00997", "T01095", "T03393", "T09456", "T00188", "T02362", "T09065", "T09547", "T10330", "T09187", "T03433", "T08635", "T02366", "T03436", "T09150", "T01861", "T09759", "T11683", "T02368", "T02369", "T08269", "T01018", "T10066", "T01710", "T01711", "T05764", ], ), ( Filter::and(vec![ (email::query::Filter::after(1786)), (email::query::Filter::before(1840)), (email::query::Filter::has_keyword("T")), ]), vec![ email::query::Comparator::has_keyword("attributed to").descending(), email::query::Comparator::from(), email::query::Comparator::sent_at(), ], vec![ "T09417", "T08951", "T01851", "T01852", "T08761", "T08123", "T08756", "T10561", "T10562", "T10563", "T00986", "T03424", "T03427", "T08234", "T08133", "T06866", "T08897", "T00996", "T00997", "T01095", "T03393", "T09456", "T00188", "T02362", "T09065", "T09547", "T10330", "T09187", "T03433", "T08635", "T02366", "T03436", "T09150", "T01861", "T09759", "T11683", "T02368", "T02369", "T08269", "T01018", "T10066", "T01710", "T01711", "T05764", "T09455", "T09334", "T10965", "T08626", ], ), ] { let mut request = client.build(); let query_request = request .query_email() .filter(filter.clone()) .sort(sort.clone()) .calculate_total(true); query_request.arguments().collapse_threads(false); let query_result_ref = query_request.result_reference(); request .get_email() .ids_ref(query_result_ref) .properties([email::Property::MessageId]); let results = request .send() .await .unwrap_or_else(|_| panic!("invalid response for {filter:?}")) .unwrap_method_responses() .pop() .unwrap_or_else(|| panic!("invalid response for {filter:?}")) .unwrap_get_email() .unwrap_or_else(|_| panic!("invalid response for {filter:?}")) .take_list() .into_iter() .map(|e| e.message_id().unwrap().first().unwrap().to_string()) .collect::>(); let mut missing = Vec::new(); let mut extra = Vec::new(); for &expected in &expected_results { if !results.iter().any(|r| r.as_str() == expected) { missing.push(expected); } } for result in &results { if !expected_results.contains(&result.as_str()) { extra.push(result.as_str()); } } assert_eq!( results, expected_results, "failed test!\nfilter: {filter:?}\nsort: {sort:?}\nmissing: {missing:?}\nextra: {extra:?}" ); } } pub async fn query_options(client: &Client) { for (query, expected_results, expected_results_collapsed) in [ ( EmailQuery { filter: None, sort: vec![ email::query::Comparator::subject(), email::query::Comparator::from(), email::query::Comparator::sent_at(), ], position: 0, anchor: None, anchor_offset: 0, limit: 10, }, vec![ "N01496", "N01320", "N01321", "N05916", "N00273", "N01453", "N02984", "T08820", "N00112", "T00211", ], vec![ "N01496", "N01320", "N05916", "N01453", "T08820", "N01046", "N00675", "T08891", "T01882", "N04296", ], ), ( EmailQuery { filter: None, sort: vec![ email::query::Comparator::subject(), email::query::Comparator::from(), email::query::Comparator::sent_at(), ], position: 10, anchor: None, anchor_offset: 0, limit: 10, }, vec![ "N01046", "N00675", "T08891", "N00126", "T01882", "N04689", "T00925", "N00121", "N04296", "N04297", ], vec![ "T08234", "T09417", "N01110", "T08123", "N01039", "T09456", "T08951", "N01273", "N00373", "T09547", ], ), ( EmailQuery { filter: None, sort: vec![ email::query::Comparator::subject(), email::query::Comparator::from(), email::query::Comparator::sent_at(), ], position: -10, anchor: None, anchor_offset: 0, limit: 0, }, vec![ "T07236", "P11481", "AR00066", "P77895", "P77896", "P77897", "AR00163", "AR00164", "AR00472", "AR00178", ], vec![ "P07639", "P07522", "AR00089", "P02949", "T05820", "P11441", "T06971", "P11481", "AR00163", "AR00164", ], ), ( EmailQuery { filter: None, sort: vec![ email::query::Comparator::subject(), email::query::Comparator::from(), email::query::Comparator::sent_at(), ], position: -20, anchor: None, anchor_offset: 0, limit: 10, }, vec![ "P20079", "AR00024", "AR00182", "P20048", "P20044", "P20045", "P20046", "T06971", "AR00177", "P77935", ], vec![ "T00300", "P06033", "T02310", "T02135", "P04006", "P03166", "P01358", "P07133", "P03138", "T03562", ], ), ( EmailQuery { filter: None, sort: vec![ email::query::Comparator::subject(), email::query::Comparator::from(), email::query::Comparator::sent_at(), ], position: -100000, anchor: None, anchor_offset: 0, limit: 1, }, vec!["N01496"], vec!["N01496"], ), ( EmailQuery { filter: None, sort: vec![ email::query::Comparator::subject(), email::query::Comparator::from(), email::query::Comparator::sent_at(), ], position: -1, anchor: None, anchor_offset: 0, limit: 100000, }, vec!["AR00178"], vec!["AR00164"], ), ( EmailQuery { filter: None, sort: vec![ email::query::Comparator::subject(), email::query::Comparator::from(), email::query::Comparator::sent_at(), ], position: 0, anchor: get_anchor(client, "N01205").await, anchor_offset: 0, limit: 10, }, vec![ "N01205", "N01976", "T01139", "N01525", "T00176", "N01405", "N02396", "N04885", "N01526", "N02134", ], vec![ "N01205", "N01526", "T01455", "N01969", "N05250", "N01781", "N00759", "A00057", "N03527", "N01558", ], ), ( EmailQuery { filter: None, sort: vec![ email::query::Comparator::subject(), email::query::Comparator::from(), email::query::Comparator::sent_at(), ], position: 0, anchor: get_anchor(client, "N01205").await, anchor_offset: 10, limit: 10, }, vec![ "N01933", "N03618", "T03904", "N02398", "N02399", "N02688", "T01455", "N03051", "N01500", "N03411", ], vec![ "N01559", "N04326", "N06017", "N01553", "N01617", "N01528", "N01539", "T09439", "N01593", "N03988", ], ), ( EmailQuery { filter: None, sort: vec![ email::query::Comparator::subject(), email::query::Comparator::from(), email::query::Comparator::sent_at(), ], position: 0, anchor: get_anchor(client, "N01205").await, anchor_offset: -10, limit: 10, }, vec![ "N05779", "N04652", "N01534", "A00845", "N03409", "N03410", "N02061", "N02426", "N00662", "N01205", ], vec![ "N00443", "N02237", "T03025", "N01722", "N01356", "N01800", "T05475", "T01587", "N05779", "N01205", ], ), ( EmailQuery { filter: None, sort: vec![ email::query::Comparator::subject(), email::query::Comparator::from(), email::query::Comparator::sent_at(), ], position: 0, anchor: get_anchor(client, "N01496").await, anchor_offset: -10, limit: 10, }, vec!["N01496"], vec!["N01496"], ), ( EmailQuery { filter: None, sort: vec![ email::query::Comparator::subject(), email::query::Comparator::from(), email::query::Comparator::sent_at(), ], position: 0, anchor: get_anchor(client, "AR00164").await, anchor_offset: 10, limit: 10, }, vec![], vec![], ), ( EmailQuery { filter: None, sort: vec![ email::query::Comparator::subject(), email::query::Comparator::from(), email::query::Comparator::sent_at(), ], position: 0, anchor: get_anchor(client, "AR00164").await, anchor_offset: 0, limit: 0, }, vec!["AR00164", "AR00472", "AR00178"], vec!["AR00164"], ), ] { for (test_num, expected_results) in [expected_results, expected_results_collapsed] .into_iter() .enumerate() { let mut request = client.build(); let query_request = request .query_email() .sort(query.sort.clone()) .position(query.position) .calculate_total(true); if query.limit > 0 { query_request.limit(query.limit); } if let Some(filter) = query.filter.as_ref() { query_request.filter(filter.clone()); } if let Some(anchor) = query.anchor.as_ref() { query_request.anchor(anchor); query_request.anchor_offset(query.anchor_offset); } query_request.arguments().collapse_threads(test_num == 1); if !expected_results.is_empty() { let query_result_ref = query_request.result_reference(); request .get_email() .ids_ref(query_result_ref) .properties([email::Property::MessageId]); assert_eq!( request .send() .await .unwrap() .unwrap_method_responses() .pop() .unwrap() .unwrap_get_email() .unwrap() .take_list() .into_iter() .map(|e| e.message_id().unwrap().first().unwrap().to_string()) .collect::>(), expected_results, "{:#?} ({})", query, test_num == 1 ); } else { assert_eq!( request.send_query_email().await.unwrap().ids(), Vec::<&str>::new() ); } } } } pub async fn create(server: &Server, account: &Account) { let sent_at = now(); let now = Instant::now(); let mut fields = AHashMap::default(); for (field_num, field) in FIELDS.iter().enumerate() { fields.insert(field.to_string(), field_num); } let mut total_messages = 0; let mut total_threads = 0; let mut thread_count = AHashMap::default(); let mut artist_count = AHashMap::default(); let mut messages = Vec::new(); let mut chunks = Vec::new(); 'outer: for (idx, record) in csv::ReaderBuilder::new() .has_headers(true) .from_reader(&deflate_test_resource("artwork_data.csv.gz")[..]) .records() .enumerate() { let record = record.unwrap(); let mut values_str = AHashMap::default(); let mut values_int = AHashMap::default(); for field_name in [ "year", "acquisitionYear", "accession_number", "artist", "artistRole", "medium", "title", "creditLine", "inscription", ] { let field = record.get(fields[field_name]).unwrap(); if field.is_empty() || (field_name == "title" && (field.contains('[') || field.contains(']'))) { continue 'outer; } else if field_name == "year" || field_name == "acquisitionYear" { let field = field.parse::().unwrap_or(0); if field < 1000 { continue 'outer; } values_int.insert(field_name.to_string(), field); } else { values_str.insert(field_name.to_string(), field.to_string()); } } let val = artist_count .entry(values_str["artist"].clone()) .or_insert(0); if *val == 3 { continue; } *val += 1; match thread_count.entry(values_int["year"]) { Entry::Occupied(mut e) => { let messages_per_thread = e.get_mut(); if *messages_per_thread == MAX_MESSAGES_PER_THREAD { continue; } *messages_per_thread += 1; } Entry::Vacant(e) => { if total_threads == MAX_THREADS { continue; } total_threads += 1; e.insert(1); } } total_messages += 1; let mut keywords = Vec::new(); for keyword in [ values_str["medium"].to_string(), values_str["artistRole"].to_string(), values_str["accession_number"][0..1].to_string(), format!( "N{}", &values_str["accession_number"][values_str["accession_number"].len() - 1..] ), ] { if keyword == "attributed to" || keyword == "T" || keyword == "N0" || keyword == "N" || keyword == "artist" || keyword == "Bronze" { keywords.push(keyword); } } let message = MessageBuilder::new() .from((values_str["artist"].as_str(), "artist@domain.com")) .cc((values_str["medium"].as_str(), "cc@domain.com")) .subject(format!("Year {}", values_int["year"])) .date(Date::new(sent_at as i64 + idx as i64)) .message_id(values_str["accession_number"].as_str()) .header("References", MessageId::new(values_int["year"].to_string())) .header("Comments", Text::new(values_str["artistRole"].as_str())) .text_body(format!( "{}\n{}\n", values_str["creditLine"], values_str["inscription"] )) .attachment("text/plain", "details.txt", values_str["title"].as_bytes()) .write_to_vec() .unwrap(); messages.push(( message, [ Id::new(values_int["year"] as u64).to_string(), Id::new((values_int["acquisitionYear"] + 1000) as u64).to_string(), ], keywords, values_int["year"] as i64, )); if messages.len() == 100 { chunks.push(messages); messages = Vec::new(); } if total_messages == MAX_MESSAGES { break; } } if !messages.is_empty() { chunks.push(messages); } let mut tasks = Vec::new(); for chunk in chunks { let client = account.client_owned().await; tasks.push(tokio::spawn(async move { for (raw_message, mailbox_ids, keywords, sent_at) in chunk { client .email_import(raw_message, mailbox_ids, keywords.into(), Some(sent_at)) .await .unwrap(); } })); } for task in tasks { task.await.unwrap(); } wait_for_index(server).await; println!( "Imported {} messages in {} ms (single thread).", total_messages, now.elapsed().as_millis() ); } async fn get_anchor(client: &Client, anchor: &str) -> Option { client .email_query( email::query::Filter::header("Message-Id", anchor.into()).into(), None::>, ) .await .unwrap() .take_ids() .pop() .unwrap() .into() } #[derive(Debug, Clone)] pub struct EmailQuery { pub filter: Option>, pub sort: Vec>, pub position: i32, pub anchor: Option, pub anchor_offset: i32, pub limit: usize, } ================================================ FILE: tests/src/jmap/mail/query_changes.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::{ JMAPTest, mail::changes::{LogAction, ParseState}, }; use ::email::message::metadata::MessageData; use common::storage::index::ObjectIndexBuilder; use jmap_client::{ core::query::{Comparator, Filter}, email, mailbox::Role, }; use jmap_proto::types::state::State; use std::str::FromStr; use store::{ ValueKey, ahash::{AHashMap, AHashSet}, write::{AlignedBytes, Archive, BatchBuilder}, }; use types::{ collection::{Collection, SyncCollection}, id::Id, }; pub async fn test(params: &mut JMAPTest) { println!("Running Email QueryChanges tests..."); let server = params.server.clone(); let account = params.account("jdoe@example.com"); let client = account.client(); let mailbox1_id = client .mailbox_create("JMAP Changes 1", None::, Role::None) .await .unwrap() .take_id(); let mailbox2_id = client .mailbox_create("JMAP Changes 2", None::, Role::None) .await .unwrap() .take_id(); let mut states = vec![State::Initial]; let mut id_map = AHashMap::default(); let mut updated_ids = AHashSet::default(); let mut removed_ids = AHashSet::default(); let mut type1_ids = AHashSet::default(); let mut thread_id_map: AHashMap = AHashMap::default(); let mut thread_id = 100; for (change_num, change) in [ LogAction::Insert(0), LogAction::Insert(1), LogAction::Insert(2), LogAction::Move(0, 3), LogAction::Insert(4), LogAction::Insert(5), LogAction::Update(1), LogAction::Update(2), LogAction::Delete(1), LogAction::Insert(6), LogAction::Insert(7), LogAction::Update(2), LogAction::Update(4), LogAction::Update(5), LogAction::Update(6), LogAction::Update(7), LogAction::Delete(4), LogAction::Delete(5), LogAction::Delete(6), LogAction::Insert(8), LogAction::Insert(9), LogAction::Insert(10), LogAction::Update(3), LogAction::Update(2), LogAction::Update(8), LogAction::Move(9, 11), LogAction::Move(10, 12), LogAction::Delete(8), ] .iter() .enumerate() { match &change { LogAction::Insert(id) => { let jmap_id = Id::from_str( client .email_import( format!( "From: test_{}\nSubject: test_{}\n\ntest", if change_num % 2 == 0 { 1 } else { 2 }, *id ) .into_bytes(), [if change_num % 2 == 0 { &mailbox1_id } else { &mailbox2_id }], [if change_num % 2 == 0 { "1" } else { "2" }].into(), Some(*id as i64), ) .await .unwrap() .id() .unwrap(), ) .unwrap(); id_map.insert(*id, jmap_id); if change_num % 2 == 0 { type1_ids.insert(jmap_id); } thread_id_map.entry(jmap_id.prefix_id()).or_insert(jmap_id); } LogAction::Update(id) => { let id = *id_map.get(id).unwrap(); let mut batch = BatchBuilder::new(); batch .with_document(id.document_id()) .log_item_update(SyncCollection::Email, id.prefix_id().into()); server.store().write(batch.build_all()).await.unwrap(); updated_ids.insert(id); } LogAction::Delete(id) => { let id = *id_map.get(id).unwrap(); client.email_destroy(&id.to_string()).await.unwrap(); removed_ids.insert(id); } LogAction::Move(from, to) => { let id = *id_map.get(from).unwrap(); let new_id = Id::from_parts(thread_id, id.document_id()); //let new_thread_id = store::rand::random::(); let old_message_ = server .store() .get_value::>(ValueKey::archive( account.id().document_id(), Collection::Email, id.document_id(), )) .await .unwrap() .unwrap(); let old_message = old_message_.to_unarchived::().unwrap(); let mut new_message = old_message.deserialize::().unwrap(); new_message.thread_id = thread_id; server .core .storage .data .write( BatchBuilder::new() .with_account_id(account.id().document_id()) .with_collection(Collection::Email) .with_document(id.document_id()) .custom( ObjectIndexBuilder::new() .with_current(old_message) .with_changes(new_message), ) .unwrap() .build_all(), ) .await .unwrap(); id_map.insert(*to, new_id); if type1_ids.contains(&id) { type1_ids.insert(new_id); } removed_ids.insert(id); thread_id_map.insert(new_id.prefix_id(), new_id); thread_id += 1; } LogAction::UpdateChild(_) => unreachable!(), } let mut new_state = State::Initial; for state in &states { for (test_num, query) in vec![ QueryChanges { filter: None, sort: vec![email::query::Comparator::received_at()], since_query_state: state.clone(), max_changes: 0, up_to_id: None, collapse_threads: false, }, QueryChanges { filter: Some(email::query::Filter::from("test_1").into()), sort: vec![email::query::Comparator::received_at()], since_query_state: state.clone(), max_changes: 0, up_to_id: None, collapse_threads: false, }, QueryChanges { filter: Some(email::query::Filter::in_mailbox(&mailbox1_id).into()), sort: vec![email::query::Comparator::received_at()], since_query_state: state.clone(), max_changes: 0, up_to_id: None, collapse_threads: false, }, QueryChanges { filter: None, sort: vec![email::query::Comparator::received_at()], since_query_state: state.clone(), max_changes: 0, up_to_id: id_map .get(&7) .map(|id| id.to_string().into()) .unwrap_or(None), collapse_threads: false, }, QueryChanges { filter: None, sort: vec![email::query::Comparator::received_at()], since_query_state: state.clone(), max_changes: 0, up_to_id: None, collapse_threads: true, }, ] .into_iter() .enumerate() { if (test_num == 3 || test_num == 4) && query.up_to_id.is_none() { continue; } if test_num == 4 && !query.collapse_threads { continue; } let mut request = client.build(); let query_request = request .query_email_changes(query.since_query_state.to_string()) .sort(query.sort); if let Some(filter) = query.filter { query_request.filter(filter); } if let Some(up_to_id) = query.up_to_id { query_request.up_to_id(up_to_id); } if query.collapse_threads { query_request.arguments().collapse_threads(true); } let changes = request.send_query_email_changes().await.unwrap(); if test_num == 0 || test_num == 1 { // Immutable filters should not return modified ids, only deletions. for id in changes.removed() { let id = Id::from_str(id).unwrap(); assert!( removed_ids.contains(&id), "{:?} (id: {:?})", changes, id_map.iter().find(|(_, v)| **v == id).map(|(k, _)| k) ); } } if test_num == 1 || test_num == 2 { // Only type 1 results should be added to the list. for item in changes.added() { let id = Id::from_str(item.id()).unwrap(); assert!( type1_ids.contains(&id), "{:?} (id: {:?})", changes, id_map.iter().find(|(_, v)| **v == id).map(|(k, _)| k) ); } } if test_num == 3 { // Only ids up to 7 should be added to the list. for item in changes.added() { let item_id = Id::from_str(item.id()).unwrap(); let id = id_map.iter().find(|(_, v)| **v == item_id).unwrap().0; assert!(id <= &7, "{:?} (id: {})", changes, id); } } if test_num == 4 { // With collapse_threads, only first email per thread should be added. let mut seen_threads = AHashSet::new(); for item in changes.added() { let item_id = Id::from_str(item.id()).unwrap(); let thread_id = item_id.prefix_id(); assert!( seen_threads.insert(thread_id), "Thread {} appears multiple times with collapse_threads: {:?}", thread_id, changes ); // Verify this is the first email in this thread assert_eq!( thread_id_map.get(&thread_id), Some(&item_id), "Expected first email in thread {}, got {:?}", thread_id, item_id ); } } if let State::Initial = state { new_state = State::parse_str(changes.new_query_state()).unwrap(); } } } states.push(new_state); } params.destroy_all_mailboxes(account).await; params.assert_is_empty().await; } #[derive(Debug, Clone)] pub struct QueryChanges { pub filter: Option>, pub sort: Vec>, pub since_query_state: State, pub max_changes: usize, pub up_to_id: Option, pub collapse_threads: bool, } ================================================ FILE: tests/src/jmap/mail/search_snippet.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::{JMAPTest, wait_for_index}; use email::mailbox::INBOX_ID; use jmap_client::{core::query, email::query::Filter}; use std::{fs, path::PathBuf}; use store::ahash::AHashMap; use types::id::Id; pub async fn test(params: &mut JMAPTest) { println!("Running SearchSnippet tests..."); let server = params.server.clone(); let account = params.account("jdoe@example.com"); let client = account.client(); let mailbox_id = Id::from(INBOX_ID).to_string(); let mut email_ids = AHashMap::default(); let mut test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); test_dir.push("resources"); test_dir.push("jmap"); test_dir.push("email_snippet"); // Import test messages for email_name in [ "html", "subpart", "mixed", "text_plain", "text_plain_chinese", ] { let mut file_name = test_dir.clone(); file_name.push(format!("{}.eml", email_name)); let email_id = client .email_import( fs::read(&file_name).unwrap(), [&mailbox_id], None::>, None, ) .await .unwrap() .take_id(); email_ids.insert(email_name, email_id); } wait_for_index(&server).await; let can_stem = params.server.search_store().internal_fts().is_some(); // Run tests for (filter, email_name, snippet_subject, snippet_preview) in [ ( query::Filter::or(vec![ query::Filter::or(vec![Filter::subject("friend"), Filter::subject("help")]), query::Filter::or(vec![Filter::body("secret"), Filter::body("call")]), ]), "text_plain", Some("Help a friend from Abidjan Côte d'Ivoire"), Some(concat!( "d'Ivoire. He secretly called me on his bedside ", "and told me that he has a sum of $7.5M (Seven Million five Hundred Thousand", " Dollars) left in a suspense account in a local bank here in Abidjan Côte ", "d'Ivoire, that he used my name a" )), ), ( Filter::text("côte").into(), "text_plain", Some("Help a friend from Abidjan Côte d'Ivoire"), Some(concat!( "in Abidjan Côte d'Ivoire. He secretly called me on ", "his bedside and told me that he has a sum of $7.5M (Seven ", "Million five Hundred Thousand Dollars) left in a suspense ", "account in a local bank here in Abidjan Côte d'Ivoire, that " )), ), ( Filter::text("\"your country\"").into(), "text_plain", None, Some(concat!( "over to your country to further my education and ", "to secure a residential permit for me in your country", ". Moreover, I am willing to offer you 30 percent of the total sum as ", "compensation for your effort inp", )), ), ( Filter::text("overseas").into(), "text_plain", None, Some("nominated account overseas. "), ), ( Filter::text("孫子兵法").into(), "text_plain_chinese", Some("孫子兵法"), Some(concat!( "<"孫子兵法："> ", "孫子曰：兵者，國之大事，死生之地，存亡之道，", "不可不察也。 孫子曰：凡用兵之法，馳車千駟" )), ), ( Filter::text("cia").into(), "subpart", None, Some("shouldn't the CIA have something like that? Bill"), ), ( Filter::text("frösche").into(), "html", Some("Die Hasen und die Frösche"), Some(concat!( "und die Frösche Die Hasen klagten einst über ihre mißliche Lage; ", ""wir leben", sprach ein Redner, "in steter Furcht vor Menschen und ", "Tieren, eine Beute der Hunde, der Adler, ja fast aller Raubtiere! ", "Unsere stete Angst ist är" )), ), ( Filter::text(if can_stem { "es:galería vasto biblioteca" } else { "es:galería vastos biblioteca" }) .into(), "mixed", Some("Biblioteca de Babel"), Some(concat!( "llaman la *Biblioteca*) se compone de un número indefinido, y tal ", "vez infinito, de galerías hexagonales, con vastos ", "pozos de ventilación en el medio, cercados por barandas bajísimas. Desde ", "cualquier hexágono se " )), ), ] { let mut request = client.build(); let result_ref = request .query_email() .filter(filter.clone()) .result_reference(); request .get_search_snippet() .filter(filter) .email_ids_ref(result_ref); let response = request .send() .await .unwrap() .unwrap_method_responses() .pop() .unwrap() .unwrap_get_search_snippet() .unwrap(); let snippet = response .snippet(email_ids.get(email_name).unwrap()) .unwrap_or_else(|| panic!("No snippet for {}", email_name)); assert_eq!(snippet_subject, snippet.subject()); assert_eq!(snippet_preview, snippet.preview()); assert!( snippet.preview().map_or(0, |p| p.len()) <= 255, "len: {}", snippet.preview().map_or(0, |p| p.len()) ); } // Destroy test data params.destroy_all_mailboxes(account).await; params.assert_is_empty().await; } ================================================ FILE: tests/src/jmap/mail/set.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::{JMAPTest, find_values, replace_blob_ids, replace_boundaries, replace_values}; use ::email::mailbox::INBOX_ID; use ahash::AHashSet; use jmap_client::{ Error, Set, client::Client, core::set::{SetError, SetErrorType}, email::{self, Email}, mailbox::Role, }; use std::{fs, path::PathBuf}; use types::id::Id; pub async fn test(params: &mut JMAPTest) { println!("Running Email Set tests..."); let account = params.account("jdoe@example.com"); let client = account.client(); let mailbox_id = Id::from(INBOX_ID).to_string(); create(client, &mailbox_id).await; update(client, &mailbox_id).await; params.destroy_all_mailboxes(account).await; params.assert_is_empty().await; } async fn create(client: &Client, mailbox_id: &str) { let mut test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); test_dir.push("resources"); test_dir.push("jmap"); test_dir.push("email_set"); for file_name in fs::read_dir(&test_dir).unwrap() { let mut file_name = file_name.as_ref().unwrap().path(); if file_name.extension().is_none_or(|e| e != "json") { continue; } println!("Creating email from {:?}", file_name); // Upload blobs let mut json_request = String::from_utf8(fs::read(&file_name).unwrap()).unwrap(); let blob_values = find_values(&json_request, "\"blobId\""); if !blob_values.is_empty() { let mut blob_ids = Vec::with_capacity(blob_values.len()); for blob_value in &blob_values { let blob_value = blob_value.replace("\\r", "\r").replace("\\n", "\n"); blob_ids.push( client .upload(None, blob_value.into_bytes(), None) .await .unwrap() .take_blob_id(), ); } json_request = replace_values(json_request, &blob_values, &blob_ids); } // Create message and obtain its blobId let mut request = client.build(); let mut create_item = serde_json::from_slice::>(json_request.as_bytes()).unwrap(); create_item.mailbox_ids([mailbox_id]); let create_id = request.set_email().create_item(create_item); let created_email = request .send_set_email() .await .unwrap() .created(&create_id) .unwrap(); // Download raw message let raw_message = client .download(created_email.blob_id().unwrap()) .await .unwrap(); // Fetch message let mut request = client.build(); request .get_email() .ids([created_email.id().unwrap()]) .properties([ email::Property::Id, email::Property::BlobId, email::Property::ThreadId, email::Property::MailboxIds, email::Property::Keywords, email::Property::ReceivedAt, email::Property::MessageId, email::Property::InReplyTo, email::Property::References, email::Property::Sender, email::Property::From, email::Property::To, email::Property::Cc, email::Property::Bcc, email::Property::ReplyTo, email::Property::Subject, email::Property::SentAt, email::Property::HasAttachment, email::Property::Preview, email::Property::BodyValues, email::Property::TextBody, email::Property::HtmlBody, email::Property::Attachments, email::Property::BodyStructure, ]) .arguments() .body_properties([ email::BodyProperty::PartId, email::BodyProperty::BlobId, email::BodyProperty::Size, email::BodyProperty::Name, email::BodyProperty::Type, email::BodyProperty::Charset, email::BodyProperty::Headers, email::BodyProperty::Disposition, email::BodyProperty::Cid, email::BodyProperty::Language, email::BodyProperty::Location, ]) .fetch_all_body_values(true) .max_body_value_bytes(100); let email = request .send_get_email() .await .unwrap() .pop() .unwrap() .into_test(); // Compare raw message file_name.set_extension("eml"); let result = replace_boundaries(String::from_utf8(raw_message).unwrap()); if fs::read(&file_name).unwrap() != result.as_bytes() { file_name.set_extension("eml_failed"); fs::write(&file_name, result.as_bytes()).unwrap(); panic!("Test failed, output saved to {}", file_name.display()); } // Compare response file_name.set_extension("jmap"); let result = replace_blob_ids(replace_boundaries( serde_json::to_string_pretty(&email).unwrap(), )); if fs::read(&file_name).unwrap() != result.as_bytes() { file_name.set_extension("jmap_failed"); fs::write(&file_name, result.as_bytes()).unwrap(); panic!("Test failed, output saved to {}", file_name.display()); } } } async fn update(client: &Client, root_mailbox_id: &str) { // Obtain all messageIds previously created let mailbox = client .email_query( email::query::Filter::in_mailbox(root_mailbox_id).into(), None::>, ) .await .unwrap(); // Create two test mailboxes let test_mailbox1_id = client .mailbox_create("Test 1", None::, Role::None) .await .unwrap() .take_id(); let test_mailbox2_id = client .mailbox_create("Test 2", None::, Role::None) .await .unwrap() .take_id(); // Set keywords and mailboxes let mut request = client.build(); request .set_email() .update(mailbox.id(0)) .mailbox_ids([&test_mailbox1_id, &test_mailbox2_id]) .keywords(["test1", "test2"]); request .send_set_email() .await .unwrap() .updated(mailbox.id(0)) .unwrap(); assert_email_properties( client, mailbox.id(0), &[&test_mailbox1_id, &test_mailbox2_id], &["test1", "test2"], ) .await; // Patch keywords and mailboxes let mut request = client.build(); request .set_email() .update(mailbox.id(0)) .mailbox_id(&test_mailbox1_id, false) .keyword("test1", true) .keyword("test2", false) .keyword("test3", true); request .send_set_email() .await .unwrap() .updated(mailbox.id(0)) .unwrap(); assert_email_properties( client, mailbox.id(0), &[&test_mailbox2_id], &["test1", "test3"], ) .await; // Orphan messages should not be permitted let mut request = client.build(); request .set_email() .update(mailbox.id(0)) .mailbox_id(&test_mailbox2_id, false); assert!(matches!( request .send_set_email() .await .unwrap() .updated(mailbox.id(0)), Err(Error::Set(SetError { type_: SetErrorType::InvalidProperties, .. })) )); // Updating and destroying the same item should not be allowed let mut request = client.build(); let set_email_request = request.set_email(); set_email_request .update(mailbox.id(0)) .mailbox_id(&test_mailbox2_id, false); set_email_request.destroy([mailbox.id(0)]); assert!(matches!( request .send_set_email() .await .unwrap() .updated(mailbox.id(0)), Err(Error::Set(SetError { type_: SetErrorType::WillDestroy, .. })) )); // Delete some messages let mut request = client.build(); request.set_email().destroy([mailbox.id(1), mailbox.id(2)]); assert_eq!( request .send_set_email() .await .unwrap() .destroyed_ids() .unwrap() .count(), 2 ); let mut request = client.build(); request.get_email().ids([mailbox.id(1), mailbox.id(2)]); assert_eq!(request.send_get_email().await.unwrap().not_found().len(), 2); // Destroy test mailboxes client .mailbox_destroy(&test_mailbox1_id, true) .await .unwrap(); client .mailbox_destroy(&test_mailbox2_id, true) .await .unwrap(); } pub async fn assert_email_properties( client: &Client, message_id: &str, mailbox_ids: &[&str], keywords: &[&str], ) { let result = client .email_get( message_id, [email::Property::MailboxIds, email::Property::Keywords].into(), ) .await .unwrap() .unwrap(); assert_eq!( mailbox_ids.iter().copied().collect::>(), result .mailbox_ids() .iter() .copied() .collect::>() ); assert_eq!( keywords.iter().copied().collect::>(), result.keywords().iter().copied().collect::>() ); } ================================================ FILE: tests/src/jmap/mail/sieve_script.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ jmap::{ JMAPTest, mail::{ delivery::SmtpConnection, submission::{MockMessage, assert_message_delivery, spawn_mock_smtp_server}, }, }, smtp::DnsCache, }; use jmap_client::{ Error, core::set::{SetError, SetErrorType}, email, mailbox, sieve::query::{Comparator, Filter}, }; use std::{ fs, path::PathBuf, time::{Duration, Instant}, }; pub async fn test(params: &mut JMAPTest) { println!("Running Sieve tests..."); let server = params.server.clone(); let account = params.account("jdoe@example.com"); let client = account.client(); // Validate scripts client .sieve_script_validate(get_script("validate_ok")) .await .unwrap(); assert!(matches!( client .sieve_script_validate(get_script("validate_error")) .await, Err(Error::Set(SetError { type_: SetErrorType::InvalidScript, .. })) )); // Create 5 Sieve scripts, all deactivated. let mut script_ids = Vec::new(); for i in 0..5 { script_ids.push( client .sieve_script_create( format!("script_{}", i + 1), format!("require \"fileinto\"; fileinto \"{}\";", i + 1).into_bytes(), false, ) .await .unwrap() .take_id(), ); } let response = client .sieve_script_query(Filter::is_active(false).into(), [Comparator::name()].into()) .await .unwrap(); assert_eq!(response.ids().len(), 5); for (pos, id) in response.ids().iter().enumerate() { let script = client .sieve_script_get(id, None::>) .await .unwrap() .unwrap(); assert_eq!(script.name().unwrap(), format!("script_{}", pos + 1)); assert_eq!( String::from_utf8(client.download(script.blob_id().unwrap()).await.unwrap()).unwrap(), format!("require \"fileinto\"; fileinto \"{}\";", pos + 1) ); } // Activate last script twice and then the first script for _ in 0..2 { client .sieve_script_activate(script_ids.last().unwrap()) .await .unwrap(); assert_eq!( client .sieve_script_query(Filter::is_active(true).into(), [Comparator::name()].into()) .await .unwrap() .ids(), vec![script_ids.last().unwrap().to_string()] ); } client .sieve_script_activate(script_ids.first().unwrap()) .await .unwrap(); assert_eq!( client .sieve_script_query(Filter::is_active(true).into(), [Comparator::name()].into()) .await .unwrap() .ids(), vec![script_ids.first().unwrap().to_string()] ); // Destroying an active script should not work assert!(matches!( client .sieve_script_destroy(script_ids.first().unwrap()) .await, Err(Error::Set(SetError { type_: SetErrorType::ScriptIsActive, .. })) )); // Deactivate all scripts client.sieve_script_deactivate().await.unwrap(); assert_eq!( client .sieve_script_query(Filter::is_active(true).into(), [Comparator::name()].into()) .await .unwrap() .ids(), Vec::::new() ); // Connect to LMTP service let mut lmtp = SmtpConnection::connect().await; // Run mailbox, fileinto, flags tests client .sieve_script_create("test_mailbox", get_script("test_mailbox"), true) .await .unwrap(); lmtp.ingest( "bill@remote.org", &["jdoe@example.com"], concat!( "From: bill@remote.org\r\n", "To: jdoe@example.com\r\n", "Subject: TPS Report\r\n", "\r\n", "I'm going to need those TPS reports ASAP. ", "So, if you could do that, that'd be great." ), ) .await; // Make sure all folders were created let mailbox_names = "My/Nested/Mailbox/with/multiple/levels/Folder" .split('/') .collect::>(); let mut mailbox_ids = Vec::new(); for &mailbox in &mailbox_names { let mut response = client .mailbox_query(mailbox::query::Filter::name(mailbox).into(), None::>) .await .unwrap(); assert!( !response.ids().is_empty(), "Mailbox {} was not created.", mailbox ); mailbox_ids.extend(response.take_ids()); } assert_eq!(mailbox_ids.len(), mailbox_names.len()); // Make sure the message was delivered to the right folders let message_ids = client .email_query(None::, None::>) .await .unwrap() .take_ids(); assert_eq!(message_ids.len(), 1, "too many messages {:?}", message_ids); let email = client .email_get( message_ids.last().unwrap(), [email::Property::MailboxIds, email::Property::Keywords].into(), ) .await .unwrap() .unwrap(); assert_eq!( email.keywords().len(), 2, "Expected 2 keywords, found {:?}.", email.keywords() ); for keyword in ["$important", "$seen"] { if !email.keywords().contains(&keyword) { panic!("Keyword {} not found in {:?}.", keyword, email.keywords()); } } assert_eq!( email.mailbox_ids().len(), 2, "Expected 2 mailbox ids, found {:?}.", email.mailbox_ids() ); for mailbox_pos in [mailbox_ids.len() - 1, mailbox_ids.len() - 2] { if !email .mailbox_ids() .contains(&mailbox_ids[mailbox_pos].as_str()) { panic!( "Mailbox {} ({}) not found in {:?}.", mailbox_names[mailbox_pos], mailbox_ids[mailbox_pos], email.keywords() ); } } // Run discard and duplicate tests client .sieve_script_create( "test_discard_reject", get_script("test_discard_reject"), true, ) .await .unwrap(); lmtp.ingest( "bill@remote.org", &["jdoe@example.com"], concat!( "From: bill@remote.org\r\n", "Bcc: Undisclosed recipients;\r\n", "Message-ID: <1234@example.com>\r\n", "Subject: Holidays\r\n", "\r\n", "Remember to file your TPS reports before ", "going on holidays." ), ) .await; assert_eq!( client .email_query(None::, None::>) .await .unwrap() .ids() .len(), 1, "Discard failed." ); // Let one sec duplicate ids expire tokio::time::sleep(Duration::from_millis(1100)).await; // Start mock SMTP server let (mut smtp_rx, smtp_settings) = spawn_mock_smtp_server(); server.ipv4_add( "localhost", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); // Run reject and duplicate check tests lmtp.ingest( "bill@remote.org", &["jdoe@example.com"], concat!( "From: bill@remote.org\r\n", "Bcc: Undisclosed recipients;\r\n", "Message-ID: <1234@example.com>\r\n", "Subject: Holidays\r\n", "\r\n", "Remember to file your T.P.S. reports before ", "going on holidays." ), ) .await; assert_eq!( client .email_query(None::, None::>) .await .unwrap() .ids() .len(), 1, "Reject failed." ); assert_message_delivery( &mut smtp_rx, MockMessage::new("<>", [""], "@No soup for you"), ) .await; // Run include tests client .sieve_script_create("test_include_this", get_script("test_include_this"), false) .await .unwrap(); client .sieve_script_create("test_include", get_script("test_include"), true) .await .unwrap(); lmtp.ingest( "bill@remote.org", &["jdoe@example.com"], concat!( "From: bill@remote.org\r\n", "Bcc: Undisclosed recipients;\r\n", "Message-ID: <1234@example.com>\r\n", "Subject: Holidays\r\n", "\r\n", "Remember to file your T.P.S. reports before ", "going on holidays." ), ) .await; assert_message_delivery( &mut smtp_rx, MockMessage::new( "<>", [""], "@Rejected from an included script", ), ) .await; // Run include global tests client .sieve_script_create( "test_include_global", get_script("test_include_global"), true, ) .await .unwrap(); lmtp.ingest( "bill@remote.org", &["jdoe@example.com"], concat!( "From: bill@remote.org\r\n", "Bcc: Undisclosed recipients;\r\n", "Message-ID: <1234@example.com>\r\n", "Subject: Holidays\r\n", "\r\n", "Remember to file your T.P.S. reports before ", "going on holidays." ), ) .await; assert_message_delivery( &mut smtp_rx, MockMessage::new( "<>", [""], "@Rejected from a global script", ), ) .await; // Run enclose + redirect tests client .sieve_script_create( "test_redirect_enclose", get_script("test_redirect_enclose"), true, ) .await .unwrap(); lmtp.ingest( "bill@remote.org", &["jdoe@example.com"], concat!( "From: bill@remote.org\r\n", "To: jdoe@example.com\r\n", "Subject: TPS Report\r\n", "\r\n", "I'm going to need those TPS reports ASAP. ", "So, if you could do that, that'd be great." ), ) .await; assert_message_delivery( &mut smtp_rx, MockMessage::new( "", [""], "@Attached you'll find", ), ) .await; assert_eq!( client .email_query(None::, None::>) .await .unwrap() .ids() .len(), 1, "Redirected message was stored." ); // Run notify + editheader + notify + fcc tests client .sieve_script_create("test_notify_fcc", get_script("test_notify_fcc"), true) .await .unwrap(); smtp_settings.lock().do_stop = true; lmtp.ingest( "bill@remote.org", &["jdoe@example.com"], concat!( "From: bill@remote.org\r\n", "To: jdoe@example.com\r\n", "Subject: Urgently I need those TPS Reports\r\n", "\r\n", "I'm going to need those TPS reports ASAP. ", "So, if you could do that, that'd be great." ), ) .await; assert_message_delivery( &mut smtp_rx, MockMessage::new( "", [""], "@It's TPS-o-clock", ), ) .await; let mut request = client.build(); request.get_email().properties([ email::Property::MailboxIds, email::Property::Keywords, email::Property::Subject, ]); let emails = request.send_get_email().await.unwrap().take_list(); assert_eq!( emails.len(), 3, "Two new messages were expected: {:#?}.", emails ); 'outer: for (subject, folder, keywords) in [ ("It's TPS-o-clock", "Notifications", ""), ( "Urgently I need those **censored** Reports", "Inbox", "$seen", ), ] { for email in &emails { if email.subject().unwrap().eq(subject) { if !keywords.is_empty() && !email.keywords().contains(&keywords) { panic!("Keyword {:?} not found in: {:#?}", keywords, email); } let mailbox_id = client .mailbox_query( mailbox::query::Filter::name(folder.to_string()).into(), None::>, ) .await .unwrap() .take_ids() .pop() .unwrap_or_else(|| panic!("Mailbox {:?} not found", folder)); if !email.mailbox_ids().contains(&mailbox_id.as_str()) { panic!( "Mailbox {:?} ({}) not found in: {:#?}", folder, mailbox_id, email ); } continue 'outer; } } panic!("Email {:?} not found in: {:#?}", subject, emails); } // Remove test data client.sieve_script_deactivate().await.unwrap(); let mut request = client.build(); request.query_sieve_script(); for id in request.send_query_sieve_script().await.unwrap().take_ids() { client.sieve_script_destroy(&id).await.unwrap(); } params.destroy_all_mailboxes(account).await; params.assert_is_empty().await; } fn get_script(name: &str) -> Vec { let mut script_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); script_path.push("resources"); script_path.push("jmap"); script_path.push("sieve"); script_path.push(format!("{}.sieve", name)); fs::read(script_path).unwrap() } ================================================ FILE: tests/src/jmap/mail/submission.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ jmap::{JMAPTest, mail::set::assert_email_properties}, smtp::DnsCache, }; use ahash::AHashMap; use jmap_client::{ Error, core::set::{SetError, SetErrorType, SetObject}, email_submission::{Address, Delivered, DeliveryStatus, Displayed, UndoStatus, query::Filter}, mailbox::Role, }; use mail_parser::DateTime; use std::{ sync::Arc, time::{Duration, Instant}, }; use store::parking_lot::Mutex; use tokio::{ io::{AsyncBufReadExt, AsyncWriteExt, BufReader}, net::TcpListener, sync::mpsc, }; use types::id::Id; #[derive(Default, Debug, PartialEq, Eq)] pub struct MockMessage { pub mail_from: String, pub rcpt_to: Vec, pub message: String, } impl MockMessage { pub fn new(mail_from: T, rcpt_to: U, message: T) -> Self where T: Into, U: IntoIterator, { Self { mail_from: mail_from.into(), rcpt_to: rcpt_to.into_iter().map(|s| s.into()).collect(), message: message.into(), } } } #[derive(Default)] pub struct MockSMTPSettings { pub fail_mail_from: bool, pub fail_rcpt_to: bool, pub fail_message: bool, pub do_stop: bool, } #[allow(clippy::disallowed_types)] pub async fn test(params: &mut JMAPTest) { println!("Running E-mail submissions tests..."); // Start mock SMTP server let server = params.server.clone(); let account = params.account("jdoe@example.com"); let client = account.client(); let (mut smtp_rx, smtp_settings) = spawn_mock_smtp_server(); server.ipv4_add( "localhost", vec!["127.0.0.1".parse().unwrap()], Instant::now() + std::time::Duration::from_secs(10), ); // Test automatic identity creation for (identity_id, email) in [(2u64, "jdoe@example.com"), (1u64, "john.doe@example.com")] { let identity = client .identity_get(&Id::from(identity_id).to_string(), None) .await .unwrap() .unwrap(); assert_eq!(identity.email().unwrap(), email); assert_eq!(identity.name().unwrap(), "John Doe"); } // Create an identity without using a valid address should fail match client .identity_create("John Doe", "someaddress@domain.com") .await .unwrap_err() { Error::Set(err) => assert_eq!(err.error(), &SetErrorType::InvalidProperties), err => panic!("Unexpected error: {:?}", err), } // Create an identity let identity_id = client .identity_create("John Doe (manually created)", "jdoe@example.com") .await .unwrap() .take_id(); // Create test mailboxes let mailbox_id = client .mailbox_create("JMAP EmailSubmission", None::, Role::None) .await .unwrap() .take_id(); let mailbox_id_2 = client .mailbox_create("JMAP EmailSubmission 2", None::, Role::None) .await .unwrap() .take_id(); // Import an email without any recipients let email_id = client .email_import( b"From: jdoe@example.com\nSubject: hey\n\ntest".to_vec(), [&mailbox_id], None::>, None, ) .await .unwrap() .take_id(); // Submission without a valid emailId or identityId should fail assert!(matches!( client .email_submission_create(Id::new(123456).to_string(), &identity_id) .await, Err(Error::Set(SetError { type_: SetErrorType::InvalidProperties, .. })) )); assert!(matches!( client .email_submission_create(&email_id, Id::new(123456).to_string()) .await, Err(Error::Set(SetError { type_: SetErrorType::InvalidProperties, .. })) )); // Submissions of e-mails without any recipients should fail assert!(matches!( client .email_submission_create(&email_id, &identity_id) .await, Err(Error::Set(SetError { type_: SetErrorType::NoRecipients, .. })) )); // Submissions with an envelope that does not match // the identity from address should fail assert!(matches!( client .email_submission_create_envelope( &email_id, &identity_id, "other_address@example.com", Vec::
::new(), ) .await, Err(Error::Set(SetError { type_: SetErrorType::ForbiddenFrom, .. })) )); // Submit a valid message submission let email_body = concat!( "From: jdoe@example.com\r\n", "To: jane_smith@remote.org\r\n", "Bcc: bill@remote.org\r\n", "Subject: hey\r\n\r\n", "test" ); let email_id = client .email_import( email_body.as_bytes().to_vec(), [&mailbox_id], None::>, None, ) .await .unwrap() .take_id(); client .email_submission_create(&email_id, &identity_id) .await .unwrap(); // Confirm that the message has been delivered let email_body = email_body.replace("Bcc: bill@remote.org\r\n", ""); assert_message_delivery( &mut smtp_rx, MockMessage::new( "", ["", ""], &email_body, ), ) .await; // Manually add recipients to the envelope and confirm submission let email_submission_id = client .email_submission_create_envelope( &email_id, &identity_id, "jdoe@example.com", [ "tim@foobar.com", // Should be de-duplicated "tim@foobar.com", "tim@foobar.com ", " james@other_domain.com ", // Should be sanitized " secret_rcpt@test.com ", ], ) .await .unwrap() .take_id(); for _ in 0..3 { let mut message = expect_message_delivery(&mut smtp_rx).await; assert_eq!(message.mail_from, ""); let rcpt_to = message.rcpt_to.pop().unwrap(); assert!( [ "", "", "", ] .contains(&rcpt_to.as_str()) ); assert!( message.message.contains(&email_body), "Got [{}], Expected[{}]", message.message, email_body ); } // Confirm that the email submission status was updated tokio::time::sleep(Duration::from_millis(100)).await; let email_submission = client .email_submission_get(&email_submission_id, None) .await .unwrap() .unwrap(); assert_eq!(email_submission.undo_status().unwrap(), &UndoStatus::Final); assert_eq!( email_submission.delivery_status().unwrap(), &AHashMap::from_iter([ ( "tim@foobar.com".to_string(), DeliveryStatus::new("250 2.1.5 Queued", Delivered::Unknown, Displayed::Unknown) ), ( "secret_rcpt@test.com".to_string(), DeliveryStatus::new("250 2.1.5 Queued", Delivered::Unknown, Displayed::Unknown) ), ( "james@other_domain.com".to_string(), DeliveryStatus::new("250 2.1.5 Queued", Delivered::Unknown, Displayed::Unknown) ), ]) ); // SMTP rejects some of the recipients let email_submission_id = client .email_submission_create_envelope( &email_id, &identity_id, "jdoe@example.com", [ "nonexistant@example.com", "delay@other_domain.com", "fail@test.com", "tim@foobar.com", ], ) .await .unwrap() .take_id(); assert_message_delivery( &mut smtp_rx, MockMessage::new("", [""], &email_body), ) .await; expect_nothing(&mut smtp_rx).await; // Verify SMTP replies tokio::time::sleep(Duration::from_millis(100)).await; let email_submission = client .email_submission_get(&email_submission_id, None) .await .unwrap() .unwrap(); assert_eq!( email_submission.undo_status().unwrap(), &UndoStatus::Pending ); assert_eq!( email_submission.delivery_status().unwrap(), &AHashMap::from_iter([ ( "nonexistant@example.com".to_string(), DeliveryStatus::new( "550 5.1.2 Mailbox does not exist.", Delivered::No, Displayed::Unknown ) ), ( "delay@other_domain.com".to_string(), DeliveryStatus::new( "Code: 451, Enhanced code: 4.5.3, Message: Try again later.", Delivered::Queued, Displayed::Unknown ) ), ( "fail@test.com".to_string(), DeliveryStatus::new( "Code: 550, Enhanced code: 0.0.0, Message: I refuse to accept that recipient.", Delivered::No, Displayed::Unknown ) ), ( "tim@foobar.com".to_string(), DeliveryStatus::new( "Code: 250, Enhanced code: 0.0.0, Message: OK", Delivered::Yes, Displayed::Unknown ) ), ]) ); // Cancel submission client .email_submission_change_status(&email_submission_id, UndoStatus::Canceled) .await .unwrap(); let email_submission = client .email_submission_get(&email_submission_id, None) .await .unwrap() .unwrap(); assert_eq!( email_submission.undo_status().unwrap(), &UndoStatus::Canceled ); assert_eq!( email_submission.delivery_status().unwrap(), &AHashMap::from_iter([ ( "nonexistant@example.com".to_string(), DeliveryStatus::new( "550 5.1.2 Mailbox does not exist.", Delivered::No, Displayed::Unknown ) ), ( "delay@other_domain.com".to_string(), DeliveryStatus::new("250 2.1.5 Queued", Delivered::Unknown, Displayed::Unknown) ), ( "fail@test.com".to_string(), DeliveryStatus::new("250 2.1.5 Queued", Delivered::Unknown, Displayed::Unknown) ), ( "tim@foobar.com".to_string(), DeliveryStatus::new("250 2.1.5 Queued", Delivered::Unknown, Displayed::Unknown) ), ]) ); // Confirm that the sendAt property is updated when using FUTURERELEASE let hold_until = DateTime::parse_rfc3339("2079-11-20T05:00:00Z") .unwrap() .to_timestamp(); let email_submission_id = client .email_submission_create_envelope( &email_id, &identity_id, Address::new("jdoe@example.com").parameter("HOLDUNTIL", Some(hold_until.to_string())), ["jane_smith@remote.org"], ) .await .unwrap() .take_id(); tokio::time::sleep(Duration::from_millis(100)).await; let email_submission = client .email_submission_get(&email_submission_id, None) .await .unwrap() .unwrap(); assert_eq!(email_submission.send_at().unwrap(), hold_until); assert_eq!( email_submission.undo_status().unwrap(), &UndoStatus::Pending ); assert_eq!( email_submission.delivery_status().unwrap(), &AHashMap::from_iter([( "jane_smith@remote.org".to_string(), DeliveryStatus::new("250 2.1.5 Queued", Delivered::Queued, Displayed::Unknown) ),]) ); // Verify onSuccessUpdateEmail action let mut request = client.build(); let set_request = request.set_email_submission(); let create_id = set_request .create() .email_id(&email_id) .identity_id(&identity_id) .create_id() .unwrap(); set_request .arguments() .on_success_update_email(&create_id) .keyword("$draft", true) .mailbox_id(&mailbox_id, false) .mailbox_id(&mailbox_id_2, true); request.send().await.unwrap().unwrap_method_responses(); assert_email_properties(client, &email_id, &[&mailbox_id_2], &["$draft"]).await; // Verify onSuccessDestroyEmail action let mut request = client.build(); let set_request = request.set_email_submission(); let create_id = set_request .create() .email_id(&email_id) .identity_id(&identity_id) .create_id() .unwrap(); set_request.arguments().on_success_destroy_email(&create_id); request.send().await.unwrap().unwrap_method_responses(); assert!( client .email_get(&email_id, None::>) .await .unwrap() .is_none() ); smtp_settings.lock().do_stop = true; // Destroy the created mailbox, identity and all submissions for identity_id in [ identity_id, Id::from(1u64).to_string(), Id::from(2u64).to_string(), ] { client.identity_destroy(&identity_id).await.unwrap(); } for id in client .email_submission_query(None::, None::>) .await .unwrap() .take_ids() { let _ = client .email_submission_change_status(&id, UndoStatus::Canceled) .await; client.email_submission_destroy(&id).await.unwrap(); } params.destroy_all_mailboxes(account).await; params.assert_is_empty().await; } pub fn spawn_mock_smtp_server() -> (mpsc::Receiver, Arc>) { // Create channels let (event_tx, event_rx) = mpsc::channel::(100); let _settings = Arc::new(Mutex::new(MockSMTPSettings::default())); let settings = _settings.clone(); // Start mock SMTP server tokio::spawn(async move { let listener = TcpListener::bind("127.0.0.1:9999") .await .unwrap_or_else(|e| { panic!("Failed to bind mock SMTP server to 127.0.0.1:9999: {}", e); }); while let Ok((mut stream, _)) = listener.accept().await { let (rx, mut tx) = stream.split(); let mut rx = BufReader::new(rx); let mut buf = String::with_capacity(128); let mut message = MockMessage::default(); tx.write_all(b"220 [127.0.0.1] Clueless host service ready\r\n") .await .unwrap(); while rx.read_line(&mut buf).await.is_ok() { //print!("-> {}", buf); if buf.starts_with("EHLO") { tx.write_all(b"250 Hi there, but I have no extensions to offer :-(\r\n") .await .unwrap(); } else if buf.starts_with("MAIL FROM") { if settings.lock().fail_mail_from { tx.write_all("552-I do not\r\n552 like that MAIL FROM.\r\n".as_bytes()) .await .unwrap(); } else { message.mail_from = buf.split_once(':').unwrap().1.trim().to_string(); tx.write_all(b"250 OK\r\n").await.unwrap(); } } else if buf.starts_with("RCPT TO") { if buf.contains("fail@") { tx.write_all( "550-I refuse to\r\n550 accept that recipient.\r\n".as_bytes(), ) .await .unwrap(); } else if buf.contains("delay@") { tx.write_all("451 4.5.3 Try again later.\r\n".as_bytes()) .await .unwrap(); } else { message .rcpt_to .push(buf.split(':').nth(1).unwrap().trim().to_string()); tx.write_all(b"250 OK\r\n").await.unwrap(); } } else if buf.starts_with("DATA") { if settings.lock().fail_message { tx.write_all( "503-Thank you but I am\r\n503 saving myself for dessert.\r\n" .as_bytes(), ) .await .unwrap(); } else if !message.mail_from.is_empty() && !message.rcpt_to.is_empty() { tx.write_all(b"354 Start feeding me now some quality content please\r\n") .await .unwrap(); buf.clear(); while rx.read_line(&mut buf).await.is_ok() { if buf.starts_with('.') && buf.len() < 4 { message.message = message.message.trim().to_string(); break; } else { message.message += buf.as_str(); buf.clear(); } } tx.write_all(b"250 Great success!\r\n").await.unwrap(); message.rcpt_to.sort_unstable(); event_tx.send(message).await.unwrap(); message = MockMessage::default(); } else { tx.write_all("554 You forgot to tell me a few things.\r\n".as_bytes()) .await .unwrap(); } } else if buf.starts_with("QUIT") { tx.write_all("250 Arrivederci!\r\n".as_bytes()) .await .unwrap(); break; } else if buf.starts_with("RSET") { tx.write_all("250 Your wish is my command.\r\n".as_bytes()) .await .unwrap(); message = MockMessage::default(); } else { println!("Unknown command: {}", buf.trim()); } buf.clear(); } if settings.lock().do_stop { //println!("Mock SMTP server stopped."); break; } } }); (event_rx, _settings) } pub async fn expect_message_delivery(event_rx: &mut mpsc::Receiver) -> MockMessage { match tokio::time::timeout(Duration::from_millis(3000), event_rx.recv()).await { Ok(Some(message)) => { //println!("Got message [{}]", message.message); message } result => { panic!("Timeout waiting for message, got: {:?}", result); } } } pub async fn assert_message_delivery( event_rx: &mut mpsc::Receiver, expected_message: MockMessage, ) { let message = expect_message_delivery(event_rx).await; assert_eq!(message.mail_from, expected_message.mail_from); assert_eq!(message.rcpt_to, expected_message.rcpt_to); if let Some(needle) = expected_message.message.strip_prefix('@') { assert!( message.message.contains(needle), "[{}] needle = {:?}", message.message, needle ); } else { assert!( message.message.contains(&expected_message.message), "Got [{}], Expected[{}]", message.message, expected_message.message ); } } pub async fn expect_nothing(event_rx: &mut mpsc::Receiver) { match tokio::time::timeout(Duration::from_millis(500), event_rx.recv()).await { Err(_) => {} message => { panic!("Received a message when expecting nothing: {:?}", message); } } } ================================================ FILE: tests/src/jmap/mail/thread_get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::{JMAPTest, wait_for_index}; use jmap_client::mailbox::Role; pub async fn test(params: &mut JMAPTest) { println!("Running Email Thread tests..."); let account = params.account("jdoe@example.com"); let client = account.client(); let mailbox_id = client .mailbox_create("JMAP Get", None::, Role::None) .await .unwrap() .take_id(); let mut expected_result = vec!["".to_string(); 5]; let mut thread_id = "".to_string(); for num in [5, 3, 1, 2, 4] { let mut email = client .email_import( format!("Subject: test\nReferences: <1234>\n\n{}", num).into_bytes(), [&mailbox_id], None::>, Some(10000i64 + num as i64), ) .await .unwrap(); thread_id = email.thread_id().unwrap().to_string(); expected_result[num - 1] = email.take_id(); } wait_for_index(¶ms.server).await; assert_eq!( client .thread_get(&thread_id) .await .unwrap() .unwrap() .email_ids(), expected_result ); params.destroy_all_mailboxes(account).await; params.assert_is_empty().await; } ================================================ FILE: tests/src/jmap/mail/thread_merge.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ jmap::{JMAPTest, mail::mailbox::destroy_all_mailboxes_no_wait, wait_for_index}, store::deflate_test_resource, }; use ::email::{ cache::MessageCacheFetch, mailbox::INBOX_ID, message::ingest::{EmailIngest, IngestEmail, IngestSource}, }; use common::auth::AccessToken; use jmap_client::{email, mailbox::Role}; use mail_parser::{MessageParser, mailbox::mbox::MessageIterator}; use std::{io::Cursor, str::FromStr, time::Duration}; use store::{ ahash::{AHashMap, AHashSet}, rand::{self, Rng}, }; use types::id::Id; pub async fn test(params: &mut JMAPTest) { test_single_thread(params).await; test_multi_thread(params).await; } async fn test_single_thread(params: &mut JMAPTest) { println!("Running Email Merge Threads tests..."); let account = params.account("admin"); let mut client = account.client_owned().await; let mut all_mailboxes = AHashMap::default(); for (base_test_num, test) in [test_1(), test_2(), test_3()].iter().enumerate() { let base_test_num = ((base_test_num * 6) as u32) + 1; let mut messages = Vec::new(); let mut total_messages = 0; let mut messages_per_thread = build_messages(test, &mut messages, &mut total_messages, None, 0); messages_per_thread.sort_unstable(); let mut mailbox_ids = Vec::with_capacity(6); for test_num in 0..=5 { mailbox_ids.push( client .set_default_account_id(Id::new((base_test_num + test_num) as u64).to_string()) .mailbox_create("Thread nightmare", None::, Role::None) .await .unwrap() .take_id(), ); } for message in &messages { client .set_default_account_id(Id::new(base_test_num as u64).to_string()) .email_import( message.to_string().into_bytes(), [mailbox_ids[0].clone()], None::>, None, ) .await .unwrap(); } for message in messages.iter().rev() { client .set_default_account_id(Id::new((base_test_num + 1) as u64).to_string()) .email_import( message.to_string().into_bytes(), [mailbox_ids[1].clone()], None::>, None, ) .await .unwrap(); } for chunk in messages.chunks(5) { client.set_default_account_id(Id::new((base_test_num + 2) as u64).to_string()); for message in chunk { client .email_import( message.to_string().into_bytes(), [mailbox_ids[2].clone()], None::>, None, ) .await .unwrap(); } client.set_default_account_id(Id::new((base_test_num + 3) as u64).to_string()); for message in chunk.iter().rev() { client .email_import( message.to_string().into_bytes(), [mailbox_ids[3].clone()], None::>, None, ) .await .unwrap(); } } for chunk in messages.chunks(5).rev() { client.set_default_account_id(Id::new((base_test_num + 4) as u64).to_string()); for message in chunk { client .email_import( message.to_string().into_bytes(), [mailbox_ids[4].clone()], None::>, None, ) .await .unwrap(); } client.set_default_account_id(Id::new((base_test_num + 5) as u64).to_string()); for message in chunk.iter().rev() { client .email_import( message.to_string().into_bytes(), [mailbox_ids[5].clone()], None::>, None, ) .await .unwrap(); } } wait_for_index(¶ms.server).await; for test_num in 0..=5 { let result = client .set_default_account_id(Id::new((base_test_num + test_num) as u64).to_string()) .email_query( email::query::Filter::in_mailbox(mailbox_ids[test_num as usize].clone()).into(), None::>, ) .await .unwrap(); assert_eq!( result.ids().len(), total_messages, "test# {}/{}", base_test_num, test_num ); let thread_ids: AHashSet = result .ids() .iter() .map(|id| Id::from_str(id).unwrap().prefix_id()) .collect(); assert_eq!( thread_ids.len(), messages_per_thread.len(), "{:?}: test# {}/{}", thread_ids, base_test_num, test_num ); let mut messages_per_thread_db = Vec::new(); for thread_id in thread_ids { messages_per_thread_db.push( client .thread_get(&Id::new(thread_id as u64).to_string()) .await .unwrap() .unwrap() .email_ids() .len(), ); } messages_per_thread_db.sort_unstable(); assert_eq!(messages_per_thread_db, messages_per_thread); println!("passed test# {}/{}", base_test_num, test_num); } all_mailboxes.insert(base_test_num as usize, mailbox_ids); } // Delete all messages and make sure no keys are left in the store. for (base_test_num, mailbox_ids) in all_mailboxes { for (test_num, _) in mailbox_ids.into_iter().enumerate() { client.set_default_account_id(Id::new((base_test_num + test_num) as u64).to_string()); destroy_all_mailboxes_no_wait(&client).await; } } params.assert_is_empty().await; } #[allow(dead_code)] async fn test_multi_thread(params: &mut JMAPTest) { println!("Running Email Merge Threads tests (multi-threaded)..."); let mut handles = vec![]; let account = params.account("jdoe@example.com"); let account_id = account.id().document_id(); let mailbox_id = INBOX_ID; for message in MessageIterator::new(Cursor::new(deflate_test_resource("mailbox.gz"))) .collect::>() .into_iter() { let message = message.unwrap(); let server = params.server.clone(); handles.push(tokio::task::spawn(async move { let mut retry_count = 0; loop { match server .email_ingest(IngestEmail { raw_message: message.contents(), message: MessageParser::new().parse(message.contents()), blob_hash: None, access_token: &AccessToken::from_id(account_id), mailbox_ids: vec![mailbox_id], keywords: vec![], received_at: None, source: IngestSource::Smtp { deliver_to: "test@domain.org", is_sender_authenticated: true, is_spam: false, }, session_id: 0, }) .await { Ok(_) => break, Err(err) => { if err.is_assertion_failure() && retry_count < 10 { //println!("Retrying ingest for {}...", message.from()); let backoff = rand::rng().random_range(50..=300); tokio::time::sleep(Duration::from_millis(backoff)).await; retry_count += 1; continue; } panic!("Failed to ingest message: {:?}", err); } } } })); } // Wait for all tasks to complete let messages = handles.len(); println!("Waiting for {} tasks to complete...", messages); for handle in handles { handle.await.expect("Task panicked"); } assert_eq!( messages, params .server .get_cached_messages(account_id) .await .unwrap() .emails .items .len(), ); println!("Deleting all messages..."); params.destroy_all_mailboxes(account).await; params.assert_is_empty().await; } fn build_message(message: usize, in_reply_to: Option, thread_num: usize) -> String { if let Some(in_reply_to) = in_reply_to { format!( "Message-ID: <{}>\nReferences: <{}>\nSubject: re: T{}\n\nreply\n", message, in_reply_to, thread_num ) } else { format!( "Message-ID: <{}>\nSubject: T{}\n\nmsg\n", message, thread_num ) } } fn build_messages( three: &ThreadTest, messages: &mut Vec, total_messages: &mut usize, in_reply_to: Option, thread_num: usize, ) -> Vec { let mut messages_per_thread = Vec::new(); match three { ThreadTest::Message => { *total_messages += 1; messages.push(build_message(*total_messages, in_reply_to, thread_num)); } ThreadTest::MessageWithReplies(replies) => { *total_messages += 1; messages.push(build_message(*total_messages, in_reply_to, thread_num)); let in_reply_to = Some(*total_messages); for reply in replies { build_messages(reply, messages, total_messages, in_reply_to, thread_num); } } ThreadTest::Root(items) => { for (thread_num, item) in items.iter().enumerate() { let count_start = *total_messages; build_messages(item, messages, total_messages, None, thread_num); messages_per_thread.push(*total_messages - count_start); } } } messages_per_thread } pub fn build_thread_test_messages() -> Vec { let mut messages = Vec::new(); let mut total_messages = 0; build_messages(&test_3(), &mut messages, &mut total_messages, None, 0); messages } pub enum ThreadTest { Message, MessageWithReplies(Vec), Root(Vec), } fn test_1() -> ThreadTest { ThreadTest::Root(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ]), ]), ]), ]), ]), ]), ]), ]) } fn test_2() -> ThreadTest { ThreadTest::Root(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::Message, ThreadTest::Message, ]), ]), ThreadTest::Message, ]), ThreadTest::Message, ]), ThreadTest::Message, ]), ThreadTest::Message, ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies( vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::Message, ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ]), ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ]), ThreadTest::Message, ]), ]), ThreadTest::Message, ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ]), ]), ], )]), ThreadTest::Message, ]), ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), ]), ]), ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ]), ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), ]), ]), ThreadTest::Message, ThreadTest::Message, ])]), ]), ]), ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::Message, ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ]), ]), ThreadTest::Message, ]), ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ]), ]), ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ]), ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ]), ]), ThreadTest::Message, ThreadTest::Message, ]), ]), ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::Message, ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ]), ThreadTest::Message, ]), ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::Message, ThreadTest::Message, ]), ]), ThreadTest::Message, ThreadTest::Message, ]), ]), ThreadTest::Message, ThreadTest::Message, ]), ]), ]), ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ThreadTest::Message, ThreadTest::Message]), ]) } fn test_3() -> ThreadTest { ThreadTest::Root(vec![ ThreadTest::MessageWithReplies(vec![ThreadTest::Message, ThreadTest::Message]), ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), ThreadTest::Message, ]), ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies( vec![ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ])], )]), ThreadTest::Message, ThreadTest::Message, ])]), ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::Message, ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::Message, ]), ]), ]), ]), ]), ThreadTest::Message, ThreadTest::Message, ])]), ThreadTest::Message, ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ])]), ThreadTest::Message, ]), ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ThreadTest::Message, ThreadTest::Message]), ThreadTest::Message, ThreadTest::Message, ])]), ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies( vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ]), ThreadTest::Message, ], )]), ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies( vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), ], )]), ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ]), ])]), ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), ThreadTest::Message, ]), ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), ThreadTest::Message, ]), ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ]), ]), ThreadTest::Message, ]), ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::Message, ThreadTest::Message, ThreadTest::Message, ]), ThreadTest::Message, ]), ThreadTest::Message, ]), ThreadTest::Message, ]), ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ ThreadTest::Message, ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies( vec![ThreadTest::Message, ThreadTest::Message], )]), ThreadTest::Message, ]), ThreadTest::Message, ]), ]), ]), ]), ]) } ================================================ FILE: tests/src/jmap/mail/vacation_response.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ jmap::{ JMAPTest, mail::{ delivery::SmtpConnection, submission::{ MockMessage, assert_message_delivery, expect_nothing, spawn_mock_smtp_server, }, }, }, smtp::DnsCache, }; use chrono::{TimeDelta, Utc}; use std::time::Instant; pub async fn test(params: &mut JMAPTest) { println!("Running Vacation Response tests..."); // Create test account let server = params.server.clone(); let account = params.account("jdoe@example.com"); let client = account.client(); // Start mock SMTP server let (mut smtp_rx, smtp_settings) = spawn_mock_smtp_server(); server.ipv4_add( "localhost", vec!["127.0.0.1".parse().unwrap()], Instant::now() + std::time::Duration::from_secs(10), ); // Let people know that we'll be down in Kokomo client .vacation_response_create( "Off the Florida Keys there's a place called Kokomo", "That's where you wanna go to get away from it all".into(), "That's where you wanna go to get away from it all".into(), ) .await .unwrap(); // Connect to LMTP service let mut lmtp = SmtpConnection::connect().await; // Send a message lmtp.ingest( "bill@remote.org", &["jdoe@example.com"], concat!( "From: bill@remote.org\r\n", "To: jdoe@example.com\r\n", "Subject: TPS Report\r\n", "\r\n", "I'm going to need those TPS reports ASAP. ", "So, if you could do that, that'd be great." ), ) .await; // Await vacation response assert_message_delivery( &mut smtp_rx, MockMessage::new("", [""], "@Kokomo"), ) .await; // Further messages from the same recipient should not // trigger a vacation response lmtp.ingest( "bill@remote.org", &["jdoe@example.com"], concat!( "From: bill@remote.org\r\n", "To: jdoe@example.com\r\n", "Subject: TPS Report -- friendly reminder\r\n", "\r\n", "Listen, are you gonna have those TPS reports for us this afternoon?", ), ) .await; expect_nothing(&mut smtp_rx).await; // Messages from MAILER-DAEMON should not // trigger a vacation response lmtp.ingest( "MAILER-DAEMON@remote.org", &["jdoe@example.com"], concat!( "From: MAILER-DAEMON@example.com\r\n", "To: jdoe@example.com\r\n", "Subject: Delivery Failure\r\n", "\r\n", "I tried so hard and got so far but in the end it wasn't delivered.", ), ) .await; expect_nothing(&mut smtp_rx).await; // Vacation responses should honor the configured date ranges client .vacation_response_set_dates( (Utc::now() + TimeDelta::try_days(1).unwrap_or_default()) .timestamp() .into(), None, ) .await .unwrap(); lmtp.ingest( "jane_smith@remote.org", &["jdoe@example.com"], concat!( "From: jane_smith@remote.org\r\n", "To: jdoe@example.com\r\n", "Subject: When were you going on holidays?\r\n", "\r\n", "I'm asking because Bill really wants those TPS reports.", ), ) .await; expect_nothing(&mut smtp_rx).await; client .vacation_response_set_dates( (Utc::now() - TimeDelta::try_days(1).unwrap_or_default()) .timestamp() .into(), None, ) .await .unwrap(); smtp_settings.lock().do_stop = true; lmtp.ingest( "jane_smith@remote.org", &["jdoe@example.com"], concat!( "From: jane_smith@remote.org\r\n", "To: jdoe@example.com\r\n", "Subject: When were you going on holidays?\r\n", "\r\n", "I'm asking because Bill really wants those TPS reports.", ), ) .await; lmtp.quit().await; assert_message_delivery( &mut smtp_rx, MockMessage::new("", [""], "@Kokomo"), ) .await; // Remove test data client.vacation_response_destroy().await.unwrap(); params.destroy_all_mailboxes(account).await; params.assert_is_empty().await; } ================================================ FILE: tests/src/jmap/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ AssertConfig, add_test_certs, directory::internal::TestInternalDirectory, jmap::server::{ enterprise::{EnterpriseCore, insert_test_metrics}, webhooks::{MockWebhookEndpoint, spawn_mock_webhook_endpoint}, }, store::{ TempDir, build_store_config, cleanup::{search_store_destroy, store_assert_is_empty, store_destroy}, }, }; use ahash::AHashMap; use base64::{ Engine, engine::general_purpose::{self, STANDARD}, }; use common::{ Caches, Core, Data, Inner, Server, config::{ server::{Listeners, ServerProtocol}, telemetry::Telemetry, }, core::BuildServer, manager::{ boot::build_ipc, config::{ConfigManager, Patterns}, }, }; use http::HttpSessionManager; use hyper::{Method, header::AUTHORIZATION}; use imap::core::ImapSessionManager; use jmap_client::client::{Client, Credentials}; use jmap_proto::error::request::RequestError; use managesieve::core::ManageSieveSessionManager; use pop3::Pop3SessionManager; use reqwest::header; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use serde_json::{Value, json}; use services::{ SpawnServices, task_manager::{Task, TaskAction}, }; use smtp::{SpawnQueueManager, core::SmtpSessionManager}; use std::{ fmt::{Debug, Display}, path::PathBuf, sync::Arc, time::Duration, }; use store::{ IterateParams, SUBSPACE_TASK_QUEUE, Stores, U32_LEN, U64_LEN, write::{AnyKey, TaskEpoch, key::DeserializeBigEndian}, }; use tokio::sync::watch; use types::id::Id; use utils::config::Config; pub mod auth; pub mod calendar; pub mod contacts; pub mod core; pub mod files; pub mod mail; pub mod principal; pub mod server; #[tokio::test(flavor = "multi_thread")] async fn jmap_tests() { let delete = std::env::var("NO_DELETE").is_err(); let mut params = init_jmap_tests(delete).await; server::webhooks::test(&mut params).await; mail::get::test(&mut params).await; mail::set::test(&mut params).await; mail::parse::test(&mut params).await; mail::query::test(&mut params, delete).await; mail::search_snippet::test(&mut params).await; mail::changes::test(&mut params).await; mail::query_changes::test(&mut params).await; mail::copy::test(&mut params).await; mail::thread_get::test(&mut params).await; mail::thread_merge::test(&mut params).await; mail::mailbox::test(&mut params).await; mail::delivery::test(&mut params).await; mail::acl::test(&mut params).await; mail::sieve_script::test(&mut params).await; mail::vacation_response::test(&mut params).await; mail::submission::test(&mut params).await; mail::crypto::test(&mut params).await; mail::antispam::test(&mut params).await; core::event_source::test(&mut params).await; core::websocket::test(&mut params).await; core::push_subscription::test(&mut params).await; core::blob::test(&mut params).await; auth::limits::test(&mut params).await; auth::oauth::test(&mut params).await; auth::quota::test(&mut params).await; auth::permissions::test(¶ms).await; contacts::addressbook::test(&mut params).await; contacts::contact::test(&mut params).await; contacts::acl::test(&mut params).await; files::node::test(&mut params).await; files::acl::test(&mut params).await; calendar::calendars::test(&mut params).await; calendar::event::test(&mut params).await; calendar::notification::test(&mut params).await; calendar::alarm::test(&mut params).await; calendar::identity::test(&mut params).await; calendar::acl::test(&mut params).await; principal::get::test(&mut params).await; principal::availability::test(&mut params).await; server::purge::test(&mut params).await; server::enterprise::test(&mut params).await; assert_is_empty(¶ms.server).await; if delete { params.temp_dir.delete(); } } #[ignore] #[tokio::test(flavor = "multi_thread")] pub async fn jmap_metric_tests() { let params = init_jmap_tests(false).await; insert_test_metrics(params.server.core.clone()).await; } #[allow(dead_code)] pub struct JMAPTest { server: Server, accounts: AHashMap<&'static str, Account>, temp_dir: TempDir, webhook: Arc, shutdown_tx: watch::Sender, } pub struct Account { name: &'static str, secret: &'static str, emails: &'static [&'static str], id: Id, id_string: String, client: Client, } impl JMAPTest { pub fn account(&self, name: &str) -> &Account { self.accounts.get(name).unwrap() } pub async fn assert_is_empty(&self) { assert_is_empty(&self.server).await; } } impl Account { pub fn id(&self) -> &Id { &self.id } pub fn id_string(&self) -> &str { &self.id_string } pub fn client(&self) -> &Client { &self.client } pub fn name(&self) -> &'static str { self.name } pub fn secret(&self) -> &'static str { self.secret } pub fn emails(&self) -> &'static [&'static str] { self.emails } pub async fn client_owned(&self) -> Client { Client::new() .credentials(Credentials::basic(self.name(), self.secret())) .timeout(Duration::from_secs(3600)) .accept_invalid_certs(true) .follow_redirects(["127.0.0.1"]) .connect("https://127.0.0.1:8899") .await .unwrap() } } pub async fn wait_for_index(server: &Server) { let mut count = 0; loop { let mut has_index_tasks = None; server .core .storage .data .iterate( IterateParams::new( AnyKey { subspace: SUBSPACE_TASK_QUEUE, key: vec![0u8], }, AnyKey { subspace: SUBSPACE_TASK_QUEUE, key: vec![u8::MAX; 16], }, ) .ascending(), |key, value| { has_index_tasks = Some( Task::::deserialize(key, value).unwrap_or_else(|_| Task { due: TaskEpoch::from_inner( key.deserialize_be_u64(key.len() - U64_LEN).unwrap(), ), account_id: key.deserialize_be_u32(U64_LEN).unwrap(), document_id: key.deserialize_be_u32(U64_LEN + U32_LEN + 1).unwrap(), action: TaskAction::SendImip, }), ); Ok(false) }, ) .await .unwrap(); if let Some(task) = has_index_tasks { count += 1; if count % 10 == 0 { println!("Waiting for pending task {:?}...", task); } tokio::time::sleep(Duration::from_millis(300)).await; } else { break; } } } pub async fn assert_is_empty(server: &Server) { // Wait for pending index tasks wait_for_index(server).await; // Assert is empty store_assert_is_empty(server.store(), server.core.storage.blob.clone(), false).await; search_store_destroy(server.search_store()).await; // Clean caches for cache in [ &server.inner.cache.events, &server.inner.cache.contacts, &server.inner.cache.files, &server.inner.cache.scheduling, ] { cache.clear(); } server.inner.cache.messages.clear(); } async fn init_jmap_tests(delete_if_exists: bool) -> JMAPTest { // Load and parse config let temp_dir = TempDir::new("jmap_tests", delete_if_exists); let mut config = Config::new( add_test_certs(&(build_store_config(&temp_dir.path.to_string_lossy()) + SERVER)) .replace("{TMP}", &temp_dir.path.display().to_string()) .replace( "{LEVEL}", &std::env::var("LOG").unwrap_or_else(|_| "disable".to_string()), ), ) .unwrap(); config.resolve_all_macros().await; // Parse servers let mut servers = Listeners::parse(&mut config); // Bind ports and drop privileges servers.bind_and_drop_priv(&mut config); // Build stores let stores = Stores::parse_all(&mut config, false).await; // Parse core let config_manager = ConfigManager { cfg_local: Default::default(), cfg_local_path: PathBuf::new(), cfg_local_patterns: Patterns::parse(&mut config).into(), cfg_store: config .value("storage.data") .and_then(|id| stores.stores.get(id)) .cloned() .unwrap_or_default(), }; let tracers = Telemetry::parse(&mut config, &stores); let core = Core::parse(&mut config, stores, config_manager) .await .enable_enterprise(); let data = Data::parse(&mut config); let cache = Caches::parse(&mut config); let store = core.storage.data.clone(); let search_store = core.storage.fts.clone(); let (ipc, mut ipc_rxs) = build_ipc(false); let inner = Arc::new(Inner { shared_core: core.into_shared(), data, ipc, cache, }); if delete_if_exists { store_destroy(&store).await; search_store_destroy(&search_store).await; } // Parse acceptors servers.parse_tcp_acceptors(&mut config, inner.clone()); // Enable tracing tracers.enable(true); // Start services config.assert_no_errors(); ipc_rxs.spawn_queue_manager(inner.clone()); ipc_rxs.spawn_services(inner.clone()); // Spawn servers let (shutdown_tx, _) = servers.spawn(|server, acceptor, shutdown_rx| { match &server.protocol { ServerProtocol::Smtp | ServerProtocol::Lmtp => server.spawn( SmtpSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Http => server.spawn( HttpSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Imap => server.spawn( ImapSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Pop3 => server.spawn( Pop3SessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::ManageSieve => server.spawn( ManageSieveSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), }; }); // Create tables let server = inner.build_server(); let mut accounts = AHashMap::new(); for (name, secret, description, emails) in [ ("admin", "secret", "Superuser", &[][..]), ( "jdoe@example.com", "12345", "John Doe", &["jdoe@example.com", "john.doe@example.com"][..], ), ( "jane.smith@example.com", "abcde", "Jane Smith", &["jane.smith@example.com"], ), ( "bill@example.com", "098765", "Bill Foobar", &["bill@example.com"], ), ( "robert@example.com", "aabbcc", "Robert Foobar", &["robert@example.com"][..], ), ] { let id: Id = server .store() .create_test_user(name, secret, description, emails) .await .into(); let id_string = id.to_string(); let mut client = Client::new() .credentials(Credentials::basic(name, secret)) .timeout(Duration::from_secs(3600)) .accept_invalid_certs(true) .follow_redirects(["127.0.0.1"]) .connect("https://127.0.0.1:8899") .await .unwrap(); client.set_default_account_id(id_string.clone()); accounts.insert( name, Account { name, secret, emails, id, id_string, client, }, ); } for (name, description, emails) in [("sales@example.com", "Sales Group", &["sales@example.com"])] { let id: Id = server .store() .create_test_group(name, description, emails) .await .into(); let id_string = id.to_string(); let mut client = Client::new() .credentials(Credentials::basic("admin", "secret")) .timeout(Duration::from_secs(3600)) .accept_invalid_certs(true) .follow_redirects(["127.0.0.1"]) .connect("https://127.0.0.1:8899") .await .unwrap(); client.set_default_account_id(id_string.clone()); accounts.insert( name, Account { name, secret: "", emails, id, id_string, client, }, ); } JMAPTest { server, temp_dir, accounts, shutdown_tx, webhook: spawn_mock_webhook_endpoint(), } } pub struct JmapResponse(pub Value); impl Account { pub async fn jmap_get( &self, object: impl Display, properties: impl IntoIterator, ids: impl IntoIterator, ) -> JmapResponse { self.jmap_get_account(self, object, properties, ids).await } pub async fn jmap_get_account( &self, account: &Account, object: impl Display, properties: impl IntoIterator, ids: impl IntoIterator, ) -> JmapResponse { let ids = ids .into_iter() .map(|id| Value::String(id.to_string())) .collect::>(); self.jmap_method_calls(json!([[ format!("{object}/get"), { "accountId": account.id_string(), "properties": properties .into_iter() .map(|p| Value::String(p.to_string())) .collect::>(), "ids": if !ids.is_empty() { Some(ids) } else { None } }, "0" ]])) .await } pub async fn jmap_query( &self, object: impl Display, filter: impl IntoIterator)>, sort_by: impl IntoIterator, arguments: impl IntoIterator)>, ) -> JmapResponse { let filter = filter .into_iter() .map(|(k, v)| (k.to_string(), v.into())) .collect::>(); let sort_by = sort_by .into_iter() .map(|id| { json! ({ "property": id.to_string() }) }) .collect::>(); let arguments = [ ("filter".to_string(), Value::Object(filter)), ("sort".to_string(), Value::Array(sort_by)), ] .into_iter() .chain( arguments .into_iter() .map(|(k, v)| (k.to_string(), v.into())), ) .collect::>(); self.jmap_method_calls(json!([[format!("{object}/query"), arguments, "0"]])) .await } pub async fn jmap_create( &self, object: impl Display, items: impl IntoIterator, arguments: impl IntoIterator)>, ) -> JmapResponse { self.jmap_create_account(self, object, items, arguments) .await } pub async fn jmap_create_account( &self, account: &Account, object: impl Display, items: impl IntoIterator, arguments: impl IntoIterator)>, ) -> JmapResponse { let create = items .into_iter() .enumerate() .map(|(i, item)| (format!("i{i}"), item)) .collect::>(); let arguments = [ ( "accountId".to_string(), Value::String(account.id_string().to_string()), ), ("create".to_string(), Value::Object(create)), ] .into_iter() .chain( arguments .into_iter() .map(|(k, v)| (k.to_string(), v.into())), ) .collect::>(); self.jmap_method_calls(json!([[format!("{object}/set"), arguments, "0"]])) .await } pub async fn jmap_update( &self, object: impl Display, items: impl IntoIterator, arguments: impl IntoIterator)>, ) -> JmapResponse { self.jmap_update_account(self, object, items, arguments) .await } pub async fn jmap_update_account( &self, account: &Account, object: impl Display, items: impl IntoIterator, arguments: impl IntoIterator)>, ) -> JmapResponse { let update = items .into_iter() .map(|(i, item)| (i.to_string(), item)) .collect::>(); let arguments = [ ( "accountId".to_string(), Value::String(account.id_string().to_string()), ), ("update".to_string(), Value::Object(update)), ] .into_iter() .chain( arguments .into_iter() .map(|(k, v)| (k.to_string(), v.into())), ) .collect::>(); self.jmap_method_calls(json!([[format!("{object}/set"), arguments, "0"]])) .await } pub async fn jmap_destroy( &self, object: impl Display, items: impl IntoIterator, arguments: impl IntoIterator)>, ) -> JmapResponse { self.jmap_destroy_account(self, object, items, arguments) .await } pub async fn jmap_destroy_account( &self, account: &Account, object: impl Display, items: impl IntoIterator, arguments: impl IntoIterator)>, ) -> JmapResponse { let destroy = items .into_iter() .map(|id| Value::String(id.to_string())) .collect::>(); let arguments = [ ( "accountId".to_string(), Value::String(account.id_string().to_string()), ), ("destroy".to_string(), Value::Array(destroy)), ] .into_iter() .chain( arguments .into_iter() .map(|(k, v)| (k.to_string(), v.into())), ) .collect::>(); self.jmap_method_calls(json!([[format!("{object}/set"), arguments, "0"]])) .await } pub async fn jmap_copy( &self, from_account: &Account, to_account: &Account, object: impl Display, items: impl IntoIterator, on_success_destroy: bool, ) -> JmapResponse { self.jmap_method_calls(json!([[ format!("{object}/copy"), { "fromAccountId": from_account.id_string(), "accountId": to_account.id_string(), "onSuccessDestroyOriginal": on_success_destroy, "create": items .into_iter() .map(|(i, item)| (i.to_string(), item)).collect::>() }, "0" ]])) .await } pub async fn jmap_changes(&self, object: impl Display, state: impl Display) -> JmapResponse { self.jmap_method_calls(json!([[ format!("{object}/changes"), { "sinceState": state.to_string() }, "0" ]])) .await } pub async fn jmap_method_call(&self, method_name: &str, body: Value) -> JmapResponse { self.jmap_method_calls(json!([[method_name, body, "0"]])) .await } pub async fn jmap_method_calls(&self, calls: Value) -> JmapResponse { let mut headers = header::HeaderMap::new(); headers.insert( header::AUTHORIZATION, header::HeaderValue::from_str(&format!( "Basic {}", general_purpose::STANDARD.encode(format!("{}:{}", self.name(), self.secret())) )) .unwrap(), ); let body = json!({ "using": [ "urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail", "urn:ietf:params:jmap:quota" ], "methodCalls": calls }); JmapResponse( serde_json::from_slice( &reqwest::Client::builder() .danger_accept_invalid_certs(true) .timeout(Duration::from_millis(1000)) .default_headers(headers) .build() .unwrap() .post("https://127.0.0.1:8899/jmap") .body(body.to_string()) .send() .await .unwrap() .bytes() .await .unwrap(), ) .unwrap(), ) } pub async fn jmap_session_object(&self) -> JmapResponse { let mut headers = header::HeaderMap::new(); headers.insert( header::AUTHORIZATION, header::HeaderValue::from_str(&format!( "Basic {}", general_purpose::STANDARD.encode(format!("{}:{}", self.name(), self.secret())) )) .unwrap(), ); JmapResponse( serde_json::from_slice( &reqwest::Client::builder() .danger_accept_invalid_certs(true) .timeout(Duration::from_millis(1000)) .default_headers(headers) .build() .unwrap() .get("https://127.0.0.1:8899/jmap/session") .send() .await .unwrap() .bytes() .await .unwrap(), ) .unwrap(), ) } pub async fn destroy_all_addressbooks(&self) { self.jmap_method_calls(json!([[ "AddressBook/get", { "ids" : (), "properties" : [ "id" ] }, "R1" ], [ "AddressBook/set", { "#destroy" : { "resultOf": "R1", "name": "AddressBook/get", "path": "/list/*/id" }, "onDestroyRemoveContents" : true }, "R2" ] ])) .await; } pub async fn destroy_all_calendars(&self) { self.jmap_method_calls(json!([[ "Calendar/get", { "ids" : (), "properties" : [ "id" ] }, "R1" ], [ "Calendar/set", { "#destroy" : { "resultOf": "R1", "name": "Calendar/get", "path": "/list/*/id" }, "onDestroyRemoveEvents" : true }, "R2" ] ])) .await; } pub async fn destroy_all_event_notifications(&self) { self.jmap_method_calls(json!([[ "CalendarEventNotification/get", { "ids" : (), "properties" : [ "id" ] }, "R1" ], [ "CalendarEventNotification/set", { "#destroy" : { "resultOf": "R1", "name": "CalendarEventNotification/get", "path": "/list/*/id" } }, "R2" ] ])) .await; } } impl JmapResponse { pub fn created(&self, item_idx: u32) -> &Value { self.0 .pointer(&format!("/methodResponses/0/1/created/i{item_idx}")) .unwrap_or_else(|| panic!("Missing created item {item_idx}: {self:?}")) } pub fn not_created(&self, item_idx: u32) -> &Value { self.0 .pointer(&format!("/methodResponses/0/1/notCreated/i{item_idx}")) .unwrap_or_else(|| panic!("Missing not created item {item_idx}: {self:?}")) } pub fn updated(&self, id: &str) -> &Value { self.0 .pointer(&format!("/methodResponses/0/1/updated/{id}")) .unwrap_or_else(|| panic!("Missing updated item {id}: {self:?}")) } pub fn not_updated(&self, id: &str) -> &Value { self.0 .pointer(&format!("/methodResponses/0/1/notUpdated/{id}")) .unwrap_or_else(|| panic!("Missing not updated item {id}: {self:?}")) } pub fn copied(&self, id: &str) -> &Value { self.0 .pointer(&format!("/methodResponses/0/1/created/{id}")) .unwrap_or_else(|| panic!("Missing updated item {id}: {self:?}")) } pub fn method_response(&self) -> &Value { self.0 .pointer("/methodResponses/0/1") .unwrap_or_else(|| panic!("Missing method response in response: {self:?}")) } pub fn list_array(&self) -> &Value { self.0 .pointer("/methodResponses/0/1/list") .unwrap_or_else(|| panic!("Missing list in response: {self:?}")) } pub fn list(&self) -> &[Value] { self.0 .pointer("/methodResponses/0/1/list") .and_then(|v| v.as_array()) .unwrap_or_else(|| panic!("Missing list in response: {self:?}")) } pub fn not_found(&self) -> impl Iterator { self.0 .pointer("/methodResponses/0/1/notFound") .and_then(|v| v.as_array()) .unwrap_or_else(|| panic!("Missing notFound in response: {self:?}")) .iter() .map(|v| v.as_str().unwrap()) } pub fn ids(&self) -> impl Iterator { self.0 .pointer("/methodResponses/0/1/ids") .and_then(|v| v.as_array()) .unwrap_or_else(|| panic!("Missing ids in response: {self:?}")) .iter() .map(|v| v.as_str().unwrap()) } pub fn destroyed(&self) -> impl Iterator { self.0 .pointer("/methodResponses/0/1/destroyed") .and_then(|v| v.as_array()) .unwrap_or_else(|| panic!("Missing destroyed in response: {self:?}")) .iter() .map(|v| v.as_str().unwrap()) } pub fn not_destroyed(&self, id: &str) -> &Value { self.0 .pointer(&format!("/methodResponses/0/1/notDestroyed/{id}")) .unwrap_or_else(|| panic!("Missing not destroyed item {id}: {self:?}")) } pub fn state(&self) -> &str { self.0 .pointer("/methodResponses/0/1/state") .and_then(|v| v.as_str()) .unwrap_or_else(|| panic!("Missing state in response: {self:?}")) } pub fn new_state(&self) -> &str { self.0 .pointer("/methodResponses/0/1/newState") .and_then(|v| v.as_str()) .unwrap_or_else(|| panic!("Missing new state in response: {self:?}")) } pub fn changes(&self) -> impl Iterator> { self.changes_by_type("created") .map(ChangeType::Created) .chain(self.changes_by_type("updated").map(ChangeType::Updated)) .chain(self.changes_by_type("destroyed").map(ChangeType::Destroyed)) } fn changes_by_type(&self, typ: &str) -> impl Iterator { self.0 .pointer(&format!("/methodResponses/0/1/{typ}")) .and_then(|v| v.as_array()) .unwrap_or_else(|| panic!("Missing {typ} changes in response: {self:?}")) .iter() .map(|v| v.as_str().unwrap()) } pub fn pointer(&self, pointer: &str) -> Option<&Value> { self.0.pointer(pointer) } pub fn into_inner(self) -> Value { self.0 } } pub trait JmapUtils { fn id(&self) -> &str { self.text_field("id") } fn blob_id(&self) -> &str { self.text_field("blobId") } fn typ(&self) -> &str { self.text_field("type") } fn description(&self) -> &str { self.text_field("description") } fn with_property(self, field: impl Display, value: impl Into) -> Self; fn text_field(&self, field: &str) -> &str; fn assert_is_equal(&self, other: Value); } impl JmapUtils for Value { fn text_field(&self, field: &str) -> &str { self.pointer(&format!("/{field}")) .and_then(|v| v.as_str()) .unwrap_or_else(|| panic!("Missing {field} in object: {self:?}")) } fn assert_is_equal(&self, expected: Value) { if self != &expected { panic!( "Values are not equal:\ngot: {}\nexpected: {}", serde_json::to_string_pretty(self).unwrap(), serde_json::to_string_pretty(&expected).unwrap() ); } } fn with_property(mut self, field: impl Display, value: impl Into) -> Self { if let Value::Object(map) = &mut self { map.insert(field.to_string(), value.into()); } else { panic!("Not an object: {self:?}"); } self } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum ChangeType<'x> { Created(&'x str), Updated(&'x str), Destroyed(&'x str), } impl<'x> ChangeType<'x> { pub fn as_created(&self) -> &str { match self { ChangeType::Created(id) => id, _ => panic!("Not a created change: {self:?}"), } } pub fn as_updated(&self) -> &str { match self { ChangeType::Updated(id) => id, _ => panic!("Not an updated change: {self:?}"), } } pub fn as_destroyed(&self) -> &str { match self { ChangeType::Destroyed(id) => id, _ => panic!("Not a destroyed change: {self:?}"), } } } impl Display for JmapResponse { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(&self.0, f) } } impl Debug for JmapResponse { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { serde_json::to_string_pretty(&self.0) .map_err(|_| std::fmt::Error) .and_then(|s| std::fmt::Display::fmt(&s, f)) } } pub trait IntoJmapSet { fn into_jmap_set(self) -> Value; } impl> IntoJmapSet for T { fn into_jmap_set(self) -> Value { Value::Object( self.into_iter() .map(|id| (id.to_string(), Value::Bool(true))) .collect::>(), ) } } pub fn find_values(string: &str, name: &str) -> Vec { let mut last_pos = 0; let mut values = Vec::new(); while let Some(pos) = string[last_pos..].find(name) { let mut value = string[last_pos + pos + name.len()..] .split('"') .nth(1) .unwrap(); if value.ends_with('\\') { value = &value[..value.len() - 1]; } values.push(value.to_string()); last_pos += pos + name.len(); } values } pub fn replace_values(mut string: String, find: &[String], replace: &[String]) -> String { for (find, replace) in find.iter().zip(replace.iter()) { string = string.replace(find, replace); } string } pub fn replace_boundaries(string: String) -> String { let values = find_values(&string, "boundary="); if !values.is_empty() { replace_values( string, &values, &(0..values.len()) .map(|i| format!("boundary_{}", i)) .collect::>(), ) } else { string } } pub fn replace_blob_ids(string: String) -> String { let values = find_values(&string, "blobId\":"); if !values.is_empty() { replace_values( string, &values, &(0..values.len()) .map(|i| format!("blob_{}", i)) .collect::>(), ) } else { string } } #[derive(Deserialize)] #[serde(untagged)] pub enum Response { RequestError(RequestError<'static>), Error { error: String, details: Option, item: Option, reason: Option, }, Data { data: T, }, } pub struct ManagementApi { pub port: u16, pub username: String, pub password: String, } impl Default for ManagementApi { fn default() -> Self { Self { port: 9980, username: "admin".to_string(), password: "secret".to_string(), } } } impl ManagementApi { pub fn new(port: u16, username: &str, password: &str) -> Self { Self { port, username: username.to_string(), password: password.to_string(), } } pub async fn post( &self, query: &str, body: &impl Serialize, ) -> Result, String> { self.request_raw( Method::POST, query, Some(serde_json::to_string(body).unwrap()), ) .await .map(|result| { serde_json::from_str::>(&result) .unwrap_or_else(|err| panic!("{err}: {result}")) }) } pub async fn patch( &self, query: &str, body: &impl Serialize, ) -> Result, String> { self.request_raw( Method::PATCH, query, Some(serde_json::to_string(body).unwrap()), ) .await .map(|result| { serde_json::from_str::>(&result) .unwrap_or_else(|err| panic!("{err}: {result}")) }) } pub async fn delete(&self, query: &str) -> Result, String> { self.request_raw(Method::DELETE, query, None) .await .map(|result| { serde_json::from_str::>(&result) .unwrap_or_else(|err| panic!("{err}: {result}")) }) } pub async fn get(&self, query: &str) -> Result, String> { self.request_raw(Method::GET, query, None) .await .map(|result| { serde_json::from_str::>(&result) .unwrap_or_else(|err| panic!("{err}: {result}")) }) } pub async fn request( &self, method: Method, query: &str, ) -> Result, String> { self.request_raw(method, query, None).await.map(|result| { serde_json::from_str::>(&result) .unwrap_or_else(|err| panic!("{err}: {result}")) }) } async fn request_raw( &self, method: Method, query: &str, body: Option, ) -> Result { let mut request = reqwest::Client::builder() .timeout(Duration::from_millis(500)) .danger_accept_invalid_certs(true) .build() .unwrap() .request(method, format!("https://127.0.0.1:{}{query}", self.port)); if let Some(body) = body { request = request.body(body); } request .header( AUTHORIZATION, format!( "Basic {}", STANDARD.encode(format!("{}:{}", self.username, self.password).as_bytes()) ), ) .send() .await .map_err(|err| err.to_string())? .bytes() .await .map(|bytes| String::from_utf8(bytes.to_vec()).unwrap()) .map_err(|err| err.to_string()) } } impl Response { pub fn unwrap_data(self) -> T { match self { Response::Data { data } => data, Response::Error { error, details, reason, .. } => { panic!("Expected data, found error {error:?}: {details:?} {reason:?}") } Response::RequestError(err) => { panic!("Expected data, found error {err:?}") } } } pub fn try_unwrap_data(self) -> Option { match self { Response::Data { data } => Some(data), Response::RequestError(error) if error.status == 404 => None, Response::Error { error, details, reason, .. } => { panic!("Expected data, found error {error:?}: {details:?} {reason:?}") } Response::RequestError(err) => { panic!("Expected data, found error {err:?}") } } } pub fn unwrap_error(self) -> (String, Option, Option) { match self { Response::Error { error, details, reason, .. } => (error, details, reason), Response::Data { data } => panic!("Expected error, found data: {data:?}"), Response::RequestError(err) => { panic!("Expected error, found request error {err:?}") } } } pub fn unwrap_request_error(self) -> RequestError<'static> { match self { Response::Error { error, details, reason, .. } => { panic!("Expected request error, found error {error:?}: {details:?} {reason:?}") } Response::Data { data } => panic!("Expected request error, found data: {data:?}"), Response::RequestError(err) => err, } } pub fn expect_request_error(self, value: &str) { let err = self.unwrap_request_error(); if !err.detail.contains(value) && !err.title.as_ref().is_some_and(|t| t.contains(value)) { panic!("Expected request error containing {value:?}, found {err:?}") } } pub fn expect_error(self, value: &str) { let (error, details, reason) = self.unwrap_error(); if !error.contains(value) && !details.as_ref().is_some_and(|d| d.contains(value)) && !reason.as_ref().is_some_and(|r| r.contains(value)) { panic!("Expected error containing {value:?}, found {error:?}: {details:?} {reason:?}") } } } const SERVER: &str = r#" [server] hostname = "'jmap.example.org'" [http] url = "'https://127.0.0.1:8899'" [server.listener.jmap] bind = ["127.0.0.1:8899"] protocol = "http" max-connections = 81920 tls.implicit = true [server.listener.imap] bind = ["127.0.0.1:9991"] protocol = "imap" max-connections = 81920 [server.listener.lmtp-debug] bind = ['127.0.0.1:11200'] greeting = 'Test LMTP instance' protocol = 'lmtp' tls.implicit = false [server.listener.pop3] bind = ["127.0.0.1:4110"] protocol = "pop3" max-connections = 81920 tls.implicit = true [server.socket] reuse-addr = true [server.tls] enable = true implicit = false certificate = "default" [server.fail2ban] authentication = "100/5s" [authentication] rate-limit = "100/2s" [session.ehlo] reject-non-fqdn = false [session.rcpt] relay = [ { if = "!is_empty(authenticated_as)", then = true }, { else = false } ] [session.rcpt.errors] total = 5 wait = "1ms" [session.auth] mechanisms = "[plain, login, oauthbearer]" [session.data] spam-filter = "recipients[0] != 'robert@example.com'" [session.data.add-headers] delivered-to = false [queue] path = "{TMP}" hash = 64 [report] path = "{TMP}" hash = 64 [resolver] type = "system" [queue.strategy] route = [ { if = "rcpt_domain == 'example.com'", then = "'local'" }, { if = "contains(['remote.org', 'foobar.com', 'test.com', 'other_domain.com'], rcpt_domain)", then = "'mock-smtp'" }, { else = "'mx'" } ] [queue.route."mock-smtp"] type = "relay" address = "localhost" port = 9999 protocol = "smtp" [queue.route."mock-smtp".tls] implicit = false allow-invalid-certs = true [session.extensions] future-release = [ { if = "!is_empty(authenticated_as)", then = "99999999d"}, { else = false } ] [certificate.default] cert = "%{file:{CERT}}%" private-key = "%{file:{PK}}%" [jmap.protocol.get] max-objects = 100000 [jmap.protocol.set] max-objects = 100000 [jmap.protocol.request] max-concurrent = 8 [jmap.protocol.upload] max-size = 5000000 max-concurrent = 4 ttl = "1m" [jmap.protocol.upload.quota] files = 3 size = 50000 [jmap.rate-limit] account = "1000/1m" anonymous = "100/1m" [jmap.event-source] throttle = "500ms" [jmap.web-sockets] throttle = "500ms" [jmap.push] throttle = "500ms" attempts.interval = "500ms" [email] auto-expunge = "1s" [changes] max-history = "1" [store."auth"] type = "sqlite" path = "{TMP}/auth.db" [store."auth".query] name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = ? AND active = true" members = "SELECT member_of FROM group_members WHERE name = ?" recipients = "SELECT name FROM emails WHERE address = ?" emails = "SELECT address FROM emails WHERE name = ? AND type != 'list' ORDER BY type DESC, address ASC" verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type = 'primary' ORDER BY address LIMIT 5" expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50" domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || ? LIMIT 1" [imap.auth] allow-plain-text = true [oauth] key = "parerga_und_paralipomena" [oauth.auth] max-attempts = 1 [oauth.expiry] user-code = "1s" token = "1s" refresh-token = "3s" refresh-token-renew = "2s" [oauth.client-registration] anonymous = true require = true [oauth.oidc] signature-key = '''-----BEGIN PRIVATE KEY----- MIIEuwIBADANBgkqhkiG9w0BAQEFAASCBKUwggShAgEAAoIBAQDMXJI1bL3z8gaF Ze/6493VjL+jHkFMP2Pc7fLwRF1fhkuIdYTp69LabzrSEJCRCz0UI2NHqPOgtOta +zRHKAMr7c7Z6uKO0K+aXiQYHw4Y70uSG8CnmNl7kb4OM/CAcoO6fePmvBsyESfn TmkJ5bfHEZQFDQEAoDlDjtjxuwYsAQQVQXuAydi8j8pyTWKAJ1RDgnUT+HbOub7j JrQ7sPe6MPCjXv5N76v9RMHKktfYwRNMlkLkxImQU55+vlvghNztgFlIlJDFfNiy UQPV5FTEZJli9BzMoj1JQK3sZyV8WV0W1zN41QQ+glAAC6+K7iTDPRMINBSwbHyn 6Lb9Q6U7AgMBAAECggEAB93qZ5xrhYgEFeoyKO4mUdGsu4qZyJB0zNeWGgdaXCfZ zC4l8zFM+R6osix0EY6lXRtC95+6h9hfFQNa5FWseupDzmIQiEnim1EowjWef87l Eayi0nDRB8TjqZKjR/aLOUhzrPlXHKrKEUk/RDkacCiDklwz9S0LIfLOSXlByBDM /n/eczfX2gUATexMHSeIXs8vN2jpuiVv0r+FPXcRvqdzDZnYSzS8BJ9k6RYXVQ4o NzCbfqgFIpVryB7nHgSTrNX9G7299If8/dXmesXWSFEJvvDSSpcBoINKbfgSlrxd 6ubjiotcEIBUSlbaanRrydwShhLHnXyupNAb7tlvyQKBgQDsIipSK4+H9FGl1rAk Gg9DLJ7P/94sidhoq1KYnj/CxwGLoRq22khZEUYZkSvYXDu1Qkj9Avi3TRhw8uol l2SK1VylL5FQvTLKhWB7b2hjrUd5llMRgS3/NIdLhOgDMB7w3UxJnCA/df/Rj+dM WhkyS1f0x3t7XPLwWGurW0nJcwKBgQDdjhrNfabrK7OQvDpAvNJizuwZK9WUL7CD rR0V0MpDGYW12BTEOY6tUK6XZgiRitAXf4EkEI6R0Q0bFzwDDLrg7TvGdTuzNeg/ 8vm8IlRlOkrdihtHZI4uRB7Ytmz24vzywEBE0p6enA7v4oniscUks/KKmDGr0V90 yT9gIVrjGQKBgQCjnWC5otlHGLDiOgm+WhgtMWOxN9dYAQNkMyF+Alinu4CEoVKD VGhA3sk1ufMpbW8pvw4X0dFIITFIQeift3DBCemxw23rBc2FqjkaDi3EszINO22/ eUTHyjvcxfCFFPi7aHsNnhJyJm7lY9Kegudmg/Ij93zGE7d5darVBuHvpQKBgBBY YovUgFMLR1UfPeD2zUKy52I4BKrJFemxBNtOKw3mPSIcTfPoFymcMTVENs+eARoq svlZK1uAo8ni3e+Pqd3cQrOyhHQFPxwwrdH+amGJemp7vOV4erDZH7l3Q/S27Fhw bI1nSIKFGukBupB58wRxLiyha9C0QqmYC0/pRg5JAn8Rbj5tP26oVCXjZEfWJL8J axxSxsGA4Vol6i6LYnVgZG+1ez2rP8vUORo1lRzmdeP4o1BSJf9TPwXkuppE5J+t UZVKtYGlEn1RqwGNd8I9TiWvU84rcY9nsxlDR86xwKRWFvYqVOiGYtzRyewYRdjU rTs9aqB3v1+OVxGxR6Na -----END PRIVATE KEY----- ''' signature-algorithm = "RS256" [oauth.oidc-ignore] signature-key = '''-----BEGIN PRIVATE KEY----- MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQggybcqc86ulFFiOon WiYrLO4z8/kmkqvA7wGElBok9IqhRANCAAQxZK68FnQtHC0eyh8CA05xRIvxhVHn 0ymka6XBh9aFtW4wfeoKhTkSKjHc/zjh9Rr2dr3kvmYe80fMGhW4ycGA -----END PRIVATE KEY----- ''' signature-algorithm = "ES256" [session.extensions] expn = true vrfy = true [spam-filter] enable = true [spam-filter.list] scores = {"GTUBE_TEST" = "1000.0"} [sharing] allow-directory-query = true [calendar.alarms] minimum-interval = "1s" [tracer.console] type = "console" level = "{LEVEL}" multiline = false ansi = true #disabled-events = ["network.*", "telemetry.webhook-error"] disabled-events = ["network.*", "telemetry.webhook-error", "http.request-body"] [webhook."test"] url = "http://127.0.0.1:8821/hook" events = ["auth.*", "delivery.dsn*", "message-ingest.*", "security.authentication-ban"] signature-key = "ovos-moles" throttle = "100ms" [sieve.untrusted.scripts."common"] contents = ''' require "reject"; reject "Rejected from a global script."; stop; ''' "#; ================================================ FILE: tests/src/jmap/principal/availability.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::{IntoJmapSet, JMAPTest, JmapUtils, calendar::event::*}; use calcard::jscalendar::JSCalendarProperty; use jmap_proto::request::method::MethodObject; use serde_json::json; use types::id::Id; pub async fn test(params: &mut JMAPTest) { println!("Running Principal Availability tests..."); let john = params.account("jdoe@example.com"); let jane = params.account("jane.smith@example.com"); let john_id = john.id_string().to_string(); let jane_id = jane.id_string().to_string(); // Create test calendars let response = john .jmap_create( MethodObject::Calendar, [json!({ "name": "Test Calendar", "includeInAvailability": "all" })], Vec::<(&str, &str)>::new(), ) .await; let calendar1_id = response.created(0).id().to_string(); // Create test events let event_1 = test_jscalendar_1().with_property( JSCalendarProperty::::CalendarIds, [calendar1_id.as_str()].into_jmap_set(), ); let event_2 = test_jscalendar_2().with_property( JSCalendarProperty::::CalendarIds, [calendar1_id.as_str()].into_jmap_set(), ); let event_3 = test_jscalendar_3() .with_property( JSCalendarProperty::::CalendarIds, [calendar1_id.as_str()].into_jmap_set(), ) .with_property( JSCalendarProperty::::Participants, json!({ "3f5bc8c0-c722-5345-b7d9-5a899db08a30": { "calendarAddress": "mailto:jdoe@example.com", "@type": "Participant", "roles": { "attendee": true, "chair": true }, "participationStatus": "accepted" } }), ); let response = john .jmap_create( MethodObject::CalendarEvent, [event_1, event_2, event_3], Vec::<(&str, &str)>::new(), ) .await; let _event_1_id = response.created(0).id().to_string(); let _event_2_id = response.created(1).id().to_string(); let event_3_id = response.created(2).id().to_string(); // Jane should not have access to John's availability let response = jane .jmap_method_calls(json!([[ "Principal/getAvailability", { "id": &john_id, "utcStart": "2006-01-01T00:00:00Z", "utcEnd": "2006-01-08T00:00:00Z", }, "0" ]])) .await; response.list_array().assert_is_equal(json!([])); // Grant Jane free/busy access john.jmap_update( MethodObject::Calendar, [( &calendar1_id, json!({ "shareWith": { &jane_id : { "mayReadFreeBusy": true, } } }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&calendar1_id); // Jane should see John's availability now let response = jane .jmap_method_calls(json!([[ "Principal/getAvailability", { "id": &john_id, "utcStart": "2006-01-01T00:00:00Z", "utcEnd": "2006-01-08T00:00:00Z", }, "0" ]])) .await; response.list_array().assert_is_equal(json!([ { "utcStart": "2006-01-02T15:00:00Z", "utcEnd": "2006-01-02T16:00:00Z", "busyStatus": "confirmed", "event": null }, { "utcStart": "2006-01-02T17:00:00Z", "utcEnd": "2006-01-02T18:00:00Z", "busyStatus": "confirmed", "event": null }, { "utcStart": "2006-01-03T17:00:00Z", "utcEnd": "2006-01-03T18:00:00Z", "busyStatus": "confirmed", "event": null }, { "utcStart": "2006-01-04T15:00:00Z", "utcEnd": "2006-01-04T16:00:00Z", "busyStatus": "confirmed", "event": null }, { "utcStart": "2006-01-04T19:00:00Z", "utcEnd": "2006-01-04T20:00:00Z", "busyStatus": "confirmed", "event": null }, { "utcStart": "2006-01-05T17:00:00Z", "utcEnd": "2006-01-05T18:00:00Z", "busyStatus": "confirmed", "event": null }, { "utcStart": "2006-01-06T19:00:00Z", "utcEnd": "2006-01-06T20:00:00Z", "busyStatus": "confirmed", "event": null } ])); // Update availability to none john.jmap_update( MethodObject::Calendar, [( &calendar1_id, json!({ "includeInAvailability": "none" }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&calendar1_id); // Jane should not see any events now let response = jane .jmap_method_calls(json!([[ "Principal/getAvailability", { "id": &john_id, "utcStart": "2006-01-01T00:00:00Z", "utcEnd": "2006-01-08T00:00:00Z", }, "0" ]])) .await; response.list_array().assert_is_equal(json!([])); // Update availability to attending john.jmap_update( MethodObject::Calendar, [( &calendar1_id, json!({ "includeInAvailability": "attending" }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&calendar1_id); // Jane should only see events where John is attending let response = jane .jmap_method_calls(json!([[ "Principal/getAvailability", { "id": &john_id, "utcStart": "2006-01-01T00:00:00Z", "utcEnd": "2006-01-08T00:00:00Z", }, "0" ]])) .await; response.list_array().assert_is_equal(json!([ { "utcStart": "2006-01-04T15:00:00Z", "utcEnd": "2006-01-04T16:00:00Z", "busyStatus": "confirmed", "event": null } ])); // Update attending event to not attending john.jmap_update( MethodObject::CalendarEvent, [( &event_3_id, json!({ "participants/3f5bc8c0-c722-5345-b7d9-5a899db08a30/participationStatus": "declined" }), )], Vec::<(&str, &str)>::new(), ) .await .updated(&event_3_id); // Jane should not see any events now let response = jane .jmap_method_calls(json!([[ "Principal/getAvailability", { "id": &john_id, "utcStart": "2006-01-01T00:00:00Z", "utcEnd": "2006-01-08T00:00:00Z", }, "0" ]])) .await; response.list_array().assert_is_equal(json!([])); // Cleanup john.destroy_all_calendars().await; params.assert_is_empty().await; } ================================================ FILE: tests/src/jmap/principal/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::jmap::{JMAPTest, JmapUtils}; use jmap_proto::{object::principal::PrincipalProperty, request::method::MethodObject}; use serde_json::json; pub async fn test(params: &mut JMAPTest) { println!("Running Principal get/query tests..."); let john = params.account("jdoe@example.com"); let jane = params.account("jane.smith@example.com"); let bill = params.account("bill@example.com"); let sales = params.account("sales@example.com"); let john_id = john.id_string(); let jane_id = jane.id_string(); let bill_id = bill.id_string(); let sales_id = sales.id_string(); // Validate session object capabilities let response = john.jmap_session_object().await.into_inner(); response.assert_is_equal(json!({ "capabilities": { "urn:ietf:params:jmap:core": { "maxSizeUpload": 5000000, "maxConcurrentUpload": 4, "maxSizeRequest": 10000000, "maxConcurrentRequests": 8, "maxCallsInRequest": 16, "maxObjectsInGet": 100000, "maxObjectsInSet": 100000, "collationAlgorithms": [ "i;ascii-numeric", "i;ascii-casemap", "i;unicode-casemap" ] }, "urn:ietf:params:jmap:mail": {}, "urn:ietf:params:jmap:calendars": {}, "urn:ietf:params:jmap:calendars:parse": {}, "urn:ietf:params:jmap:contacts": {}, "urn:ietf:params:jmap:contacts:parse": {}, "urn:ietf:params:jmap:filenode": {}, "urn:ietf:params:jmap:principals": {}, "urn:ietf:params:jmap:principals:availability": {}, "urn:ietf:params:jmap:submission": {}, "urn:ietf:params:jmap:vacationresponse": {}, "urn:ietf:params:jmap:sieve": { "implementation": "Stalwart v1.0.0" }, "urn:ietf:params:jmap:blob": {}, "urn:ietf:params:jmap:quota": {}, "urn:ietf:params:jmap:websocket": { "url": "wss://127.0.0.1:8899/jmap/ws", "supportsPush": true } }, "accounts": { john_id: { "name": "jdoe@example.com", "isPersonal": true, "isReadOnly": false, "accountCapabilities": { "urn:ietf:params:jmap:mail": { "maxMailboxesPerEmail": null, "maxMailboxDepth": 10, "maxSizeMailboxName": 255, "maxSizeAttachmentsPerEmail": 50000000, "emailQuerySortOptions": [ "receivedAt", "size", "from", "to", "subject", "sentAt", "hasKeyword", "allInThreadHaveKeyword", "someInThreadHaveKeyword" ], "mayCreateTopLevelMailbox": true }, "urn:ietf:params:jmap:submission": { "maxDelayedSend": 2592000, "submissionExtensions": { "FUTURERELEASE": [], "SIZE": [], "DSN": [], "DELIVERYBY": [], "MT-PRIORITY": [ "MIXER" ], "REQUIRETLS": [] } }, "urn:ietf:params:jmap:vacationresponse": {}, "urn:ietf:params:jmap:contacts": { "maxAddressBooksPerCard": null, "mayCreateAddressBook": true }, "urn:ietf:params:jmap:contacts:parse": {}, "urn:ietf:params:jmap:calendars": { "maxCalendarsPerEvent": null, "minDateTime": "0001-01-01T00:00:00Z", "maxDateTime": "65534-12-31T23:59:59Z", "maxExpandedQueryDuration": "P52W1D", "maxParticipantsPerEvent": 20, "mayCreateCalendar": true }, "urn:ietf:params:jmap:calendars:parse": {}, "urn:ietf:params:jmap:websocket": {}, "urn:ietf:params:jmap:sieve": { "maxSizeScriptName": 512, "maxSizeScript": 1048576, "maxNumberScripts": 100, "maxNumberRedirects": 1, "sieveExtensions": [ "body", "comparator-elbonia", "comparator-i;ascii-casemap", "comparator-i;ascii-numeric", "comparator-i;octet", "convert", "copy", "date", "duplicate", "editheader", "enclose", "encoded-character", "enotify", "envelope", "envelope-deliverby", "envelope-dsn", "environment", "ereject", "extlists", "extracttext", "fcc", "fileinto", "foreverypart", "ihave", "imap4flags", "imapsieve", "include", "index", "mailbox", "mailboxid", "mboxmetadata", "mime", "redirect-deliverby", "redirect-dsn", "regex", "reject", "relational", "replace", "servermetadata", "spamtest", "spamtestplus", "special-use", "subaddress", "vacation", "vacation-seconds", "variables", "virustest" ], "notificationMethods": [ "mailto" ], "externalLists": null }, "urn:ietf:params:jmap:blob": { "maxSizeBlobSet": 7499488, "maxDataSources": 16, "supportedTypeNames": [ "Email", "Thread", "SieveScript" ], "supportedDigestAlgorithms": [ "sha", "sha-256", "sha-512" ] }, "urn:ietf:params:jmap:quota": {}, "urn:ietf:params:jmap:principals": { "currentUserPrincipalId": john_id }, "urn:ietf:params:jmap:principals:availability": { "maxAvailabilityDuration": "P52W1D", }, "urn:ietf:params:jmap:filenode": { "maxFileNodeDepth": null, "maxSizeFileNodeName": 255, "fileNodeQuerySortOptions": [], "mayCreateTopLevelFileNode": true } } } }, "primaryAccounts": { "urn:ietf:params:jmap:mail": john_id, "urn:ietf:params:jmap:submission": john_id, "urn:ietf:params:jmap:vacationresponse": john_id, "urn:ietf:params:jmap:contacts": john_id, "urn:ietf:params:jmap:contacts:parse": john_id, "urn:ietf:params:jmap:calendars": john_id, "urn:ietf:params:jmap:calendars:parse": john_id, "urn:ietf:params:jmap:websocket": john_id, "urn:ietf:params:jmap:sieve": john_id, "urn:ietf:params:jmap:blob": john_id, "urn:ietf:params:jmap:quota": john_id, "urn:ietf:params:jmap:principals": john_id, "urn:ietf:params:jmap:principals:availability": john_id, "urn:ietf:params:jmap:filenode": john_id }, "username": "jdoe@example.com", "apiUrl": "https://127.0.0.1:8899/jmap/", "downloadUrl": "https://127.0.0.1:8899/jmap/download/{accountId}/{blobId}/{name}?accept={type}", "uploadUrl": "https://127.0.0.1:8899/jmap/upload/{accountId}/", "eventSourceUrl": "https://127.0.0.1:8899/jmap/eventsource/?types={types}&closeafter={closeafter}&ping={ping}", "state": response.text_field("state") })); // Obtain principal ids for Jane, Bill and the sales group let response = john .jmap_query( MethodObject::Principal, [("email", "john.doe@example.com")], ["name"], Vec::<(&str, &str)>::new(), ) .await; assert_eq!(response.ids().collect::>(), [john_id]); let response = john .jmap_query( MethodObject::Principal, [("name", "bill@example.com")], ["name"], Vec::<(&str, &str)>::new(), ) .await; assert_eq!(response.ids().collect::>(), [bill_id]); let response = john .jmap_query( MethodObject::Principal, [("accountIds", [jane_id])], ["name"], Vec::<(&str, &str)>::new(), ) .await; assert_eq!(response.ids().collect::>(), [jane_id]); let response = john .jmap_query( MethodObject::Principal, [("text", "sales group")], ["name"], Vec::<(&str, &str)>::new(), ) .await; assert_eq!(response.ids().collect::>(), [sales_id]); // Validate principal contents let response = john .jmap_get( MethodObject::Principal, [ PrincipalProperty::Id, PrincipalProperty::Type, PrincipalProperty::Email, PrincipalProperty::Description, PrincipalProperty::Name, PrincipalProperty::Timezone, PrincipalProperty::Capabilities, PrincipalProperty::Accounts, ], [john_id, jane_id, bill_id, sales_id], ) .await; let list = response.list(); assert_eq!(list.len(), 4); list[0].assert_is_equal(json!({ "id": john_id, "type": "individual", "email": "jdoe@example.com", "description": "John Doe", "name": "jdoe@example.com", "timezone": null, "capabilities": { "urn:ietf:params:jmap:mail": {}, "urn:ietf:params:jmap:contacts": {}, "urn:ietf:params:jmap:calendars": {}, "urn:ietf:params:jmap:filenode": {}, "urn:ietf:params:jmap:principals": {} }, "accounts": { john_id: { "urn:ietf:params:jmap:mail": {}, "urn:ietf:params:jmap:contacts": {}, "urn:ietf:params:jmap:calendars": { "accountId": john_id, "mayGetAvailability": true, "mayShareWith": true, "calendarAddress": "mailto:jdoe@example.com" }, "urn:ietf:params:jmap:filenode": {}, "urn:ietf:params:jmap:principals": {}, "urn:ietf:params:jmap:principals:owner": { "accountIdForPrincipal": john_id, "principalId": john_id } } } })); list[1].assert_is_equal(json!({ "id": jane_id, "type": "individual", "email": "jane.smith@example.com", "description": "Jane Smith", "name": "jane.smith@example.com", "timezone": null, "capabilities": { "urn:ietf:params:jmap:mail": {}, "urn:ietf:params:jmap:contacts": {}, "urn:ietf:params:jmap:calendars": {}, "urn:ietf:params:jmap:filenode": {}, "urn:ietf:params:jmap:principals": {} }, "accounts": { jane_id: { "urn:ietf:params:jmap:mail": {}, "urn:ietf:params:jmap:contacts": {}, "urn:ietf:params:jmap:calendars": { "accountId": jane_id, "mayGetAvailability": true, "mayShareWith": true, "calendarAddress": "mailto:jane.smith@example.com" }, "urn:ietf:params:jmap:filenode": {}, "urn:ietf:params:jmap:principals": {}, "urn:ietf:params:jmap:principals:owner": { "accountIdForPrincipal": jane_id, "principalId": jane_id } } } })); list[2].assert_is_equal(json!({ "id": bill_id, "type": "individual", "email": "bill@example.com", "description": "Bill Foobar", "name": "bill@example.com", "timezone": null, "capabilities": { "urn:ietf:params:jmap:mail": {}, "urn:ietf:params:jmap:contacts": {}, "urn:ietf:params:jmap:calendars": {}, "urn:ietf:params:jmap:filenode": {}, "urn:ietf:params:jmap:principals": {} }, "accounts": { bill_id: { "urn:ietf:params:jmap:mail": {}, "urn:ietf:params:jmap:contacts": {}, "urn:ietf:params:jmap:calendars": { "accountId": bill_id, "mayGetAvailability": true, "mayShareWith": true, "calendarAddress": "mailto:bill@example.com" }, "urn:ietf:params:jmap:filenode": {}, "urn:ietf:params:jmap:principals": {}, "urn:ietf:params:jmap:principals:owner": { "accountIdForPrincipal": bill_id, "principalId": bill_id } } } })); list[3].assert_is_equal(json!({ "id": sales_id, "type": "group", "email": "sales@example.com", "description": "Sales Group", "name": "sales@example.com", "timezone": null, "capabilities": { "urn:ietf:params:jmap:mail": {}, "urn:ietf:params:jmap:contacts": {}, "urn:ietf:params:jmap:calendars": {}, "urn:ietf:params:jmap:filenode": {}, "urn:ietf:params:jmap:principals": {} }, "accounts": { sales_id: { "urn:ietf:params:jmap:mail": {}, "urn:ietf:params:jmap:contacts": {}, "urn:ietf:params:jmap:calendars": { "accountId": sales_id, "mayGetAvailability": true, "mayShareWith": true, "calendarAddress": "mailto:sales@example.com" }, "urn:ietf:params:jmap:filenode": {}, "urn:ietf:params:jmap:principals": {}, "urn:ietf:params:jmap:principals:owner": { "accountIdForPrincipal": sales_id, "principalId": sales_id } } } })); } ================================================ FILE: tests/src/jmap/principal/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod availability; pub mod get; ================================================ FILE: tests/src/jmap/server/enterprise.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: LicenseRef-SEL * * This file is subject to the Stalwart Enterprise License Agreement (SEL) and * is NOT open source software. * */ use crate::{ AssertConfig, directory::internal::TestInternalDirectory, imap::{ImapConnection, Type}, jmap::{ JMAPTest, ManagementApi, mail::delivery::{AssertResult, SmtpConnection}, server::List, wait_for_index, }, }; use common::{ Core, Server, config::telemetry::{StoreTracer, TelemetrySubscriberType}, core::BuildServer, enterprise::{ Enterprise, MetricStore, TraceStore, Undelete, config::parse_metric_alerts, license::LicenseKey, }, telemetry::{ metrics::store::{Metric, MetricsStore, SharedMetricHistory}, tracers::store::TracingStore, }, }; use directory::{QueryBy, backend::internal::manage::ManageDirectory}; use http::management::{ enterprise::undelete::{ DeletedBlobResponse, DeletedItemResponse, UndeleteRequest, UndeleteResponse, }, stores::destroy_account_data, }; use imap_proto::ResponseType; use nlp::language::Language; use std::{sync::Arc, time::Duration}; use store::{ rand::{self, Rng}, search::{ SearchField, SearchFilter, SearchOperator, SearchQuery, SearchValue, TracingSearchField, }, write::{SearchIndex, now}, }; use trc::{ ipc::{bitset::Bitset, subscriber::SubscriberBuilder}, *, }; use utils::config::{Config, cron::SimpleCron}; const METRICS_CONFIG: &str = r#" [metrics.alerts.expected] enable = true condition = "domain_count > 1 && cluster_publisher_error > 3" [metrics.alerts.expected.notify.event] enable = true message = "Yikes! Found %{cluster.publisher-error}% cluster errors!" [metrics.alerts.expected.notify.email] enable = true from-name = "Alert Subsystem" from-addr = "alert@example.com" to = ["jdoe@example.com"] subject = "Found %{cluster.publisher-error}% cluster errors" body = "Sorry for the bad news, but we found %{domain.count}% domains and %{cluster.publisher-error}% cluster errors." [metrics.alerts.unexpected] enable = true condition = "domain_count < 1 || cluster_publisher_error < 3" [metrics.alerts.unexpected.notify.event] enable = true message = "this should not have happened" "#; const RAW_MESSAGE: &str = "From: john@example.com To: john@example.com Subject: undelete test test "; pub async fn test(params: &mut JMAPTest) { // Enable Enterprise println!("Running Enterprise tests..."); let mut core = params.server.inner.shared_core.load_full().as_ref().clone(); let mut config = Config::new(METRICS_CONFIG).unwrap(); core.enterprise = Enterprise { license: LicenseKey { valid_to: now() + 3600, valid_from: now() - 3600, domain: String::new(), accounts: 100, }, undelete: Undelete { retention: Duration::from_secs(2), } .into(), trace_store: TraceStore { retention: Some(Duration::from_secs(1)), store: core.storage.data.clone(), } .into(), metrics_store: MetricStore { retention: Some(Duration::from_secs(1)), store: core.storage.data.clone(), interval: SimpleCron::Day { hour: 0, minute: 0 }, } .into(), metrics_alerts: parse_metric_alerts(&mut config), logo_url: None, ai_apis: Default::default(), spam_filter_llm: None, template_calendar_alarm: None, template_scheduling_email: None, template_scheduling_web: None, } .into(); config.assert_no_errors(); assert_ne!(core.enterprise.as_ref().unwrap().metrics_alerts.len(), 0); params.server.inner.shared_core.store(core.into()); assert!( params .server .inner .shared_core .load() .is_enterprise_edition() ); // Create test account let server = params.server.inner.build_server(); let account_id = server .store() .create_test_user( "jdoe@example.com", "12345", "John Doe", &["jdoe@example.com"], ) .await; alerts(&server).await; undelete(params).await; tracing(params).await; metrics(params).await; // Delete test account server .store() .delete_principal(QueryBy::Id(account_id)) .await .unwrap(); destroy_account_data(&server, account_id, true) .await .unwrap(); params.assert_is_empty().await; params.server.inner.shared_core.store( params .server .inner .shared_core .load_full() .as_ref() .clone() .enable_enterprise() .into(), ); } pub trait EnterpriseCore { fn enable_enterprise(self) -> Self; } impl EnterpriseCore for Core { fn enable_enterprise(mut self) -> Self { self.enterprise = Enterprise { license: LicenseKey { valid_to: now() + 3600, valid_from: now() - 3600, domain: String::new(), accounts: 100, }, undelete: None, trace_store: None, metrics_store: None, metrics_alerts: vec![], logo_url: None, ai_apis: Default::default(), spam_filter_llm: None, template_calendar_alarm: None, template_scheduling_email: None, template_scheduling_web: None, } .into(); self } } async fn alerts(server: &Server) { // Make sure the required metrics are set to 0 assert_eq!( Collector::read_event_metric(EventType::Cluster(ClusterEvent::PublisherError).id()), 0 ); assert_eq!(Collector::read_metric(MetricType::DomainCount), 0.0); assert_eq!( Collector::read_event_metric(EventType::Telemetry(TelemetryEvent::Alert).id()), 0 ); // Increment metrics to trigger alerts Collector::update_event_counter(EventType::Cluster(ClusterEvent::PublisherError), 5); Collector::update_gauge(MetricType::DomainCount, 3); // Make sure the values were set assert_eq!( Collector::read_event_metric(EventType::Cluster(ClusterEvent::PublisherError).id()), 5 ); assert_eq!(Collector::read_metric(MetricType::DomainCount), 3.0); // Process alerts let message = server.process_alerts().await.unwrap().pop().unwrap(); assert_eq!(message.from, "alert@example.com"); assert_eq!(message.to, vec!["jdoe@example.com".to_string()]); let body = String::from_utf8(message.body).unwrap(); assert!( body.contains("Sorry for the bad news, but we found 3 domains and 5 cluster errors."), "{body:?}" ); assert!(body.contains("Subject: Found 5 cluster errors"), "{body:?}"); assert!( body.contains("From: \"Alert Subsystem\" "), "{body:?}" ); assert!(body.contains("To: "), "{body:?}"); // Make sure the event was triggered assert_eq!( Collector::read_event_metric(EventType::Telemetry(TelemetryEvent::Alert).id()), 1 ); } async fn tracing(params: &mut JMAPTest) { // Enable tracing let store = params.server.core.storage.data.clone(); let query = params.server.core.storage.fts.clone(); TelemetrySubscriberType::StoreTracer(StoreTracer { store: store.clone(), }) .spawn( SubscriberBuilder::new("store-tracer".to_string()).with_interests(Box::new(Bitset::all())), true, ); // Make sure there are no span entries in the db store .purge_spans(Duration::from_secs(0), Some(&query)) .await .unwrap(); assert_eq!( query .query_global(SearchQuery::new(SearchIndex::Tracing).with_filters(vec![ SearchFilter::Operator { field: SearchField::Tracing(TracingSearchField::EventType), op: SearchOperator::Equal, value: SearchValue::Uint(EventType::Smtp(SmtpEvent::ConnectionStart).code()) } ])) .await .unwrap(), Vec::::new() ); // Send an email let mut lmtp = SmtpConnection::connect().await; lmtp.ingest( "bill@example.com", &["jdoe@example.com"], concat!( "From: bill@example.com\r\n", "To: jdoe@example.com\r\n", "Subject: TPS Report\r\n", "X-Spam-Status: No\r\n", "\r\n", "I'm going to need those TPS reports ASAP. ", "So, if you could do that, that'd be great." ), ) .await; lmtp.quit().await; tokio::time::sleep(Duration::from_millis(300)).await; params.server.notify_task_queue(); wait_for_index(¶ms.server).await; // Purge should not delete anything at this point store .purge_spans(Duration::from_secs(2), Some(&query)) .await .unwrap(); // There should be a span entry in the db for span_type in [ EventType::Delivery(DeliveryEvent::AttemptStart), EventType::Smtp(SmtpEvent::ConnectionStart), ] { let spans = query .query_global(SearchQuery::new(SearchIndex::Tracing).with_filters(vec![ SearchFilter::Operator { field: SearchField::Tracing(TracingSearchField::EventType), op: SearchOperator::Equal, value: SearchValue::Uint(span_type.code()), }, ])) .await .unwrap(); assert_eq!(spans.len(), 1, "{span_type:?}"); assert_eq!( store.get_span(spans[0]).await.unwrap()[0].inner.typ, span_type ); } // Try searching for keyword in ["bill@example.com", "jdoe@example.com", "example.com"] { let spans = query .query_global(SearchQuery::new(SearchIndex::Tracing).with_filters(vec![ SearchFilter::Operator { field: SearchField::Tracing(TracingSearchField::Keywords), op: SearchOperator::Equal, value: SearchValue::Text { value: keyword.to_string(), language: Language::None, }, }, ])) .await .unwrap(); assert_eq!(spans.len(), 2, "keyword: {keyword}"); assert!(spans[0] != spans[1], "keyword: {keyword}"); } // Purge should delete the span entries tokio::time::sleep(Duration::from_millis(800)).await; store .purge_spans(Duration::from_secs(1), Some(&query)) .await .unwrap(); assert_eq!( query .query_global(SearchQuery::new(SearchIndex::Tracing).with_filters(vec![ SearchFilter::Operator { field: SearchField::Id, op: SearchOperator::GreaterThan, value: SearchValue::Uint(0), }, ])) .await .unwrap(), Vec::::new() ); } async fn metrics(params: &mut JMAPTest) { // Make sure there are no span entries in the db let store = params.server.core.storage.data.clone(); assert_eq!( store.query_metrics(0, u64::MAX).await.unwrap(), Vec::>::new() ); insert_test_metrics(params.server.core.clone()).await; let total = store.query_metrics(0, u64::MAX).await.unwrap(); assert!(!total.is_empty(), "{total:?}"); store.purge_metrics(Duration::from_secs(0)).await.unwrap(); assert_eq!( store.query_metrics(0, u64::MAX).await.unwrap(), Vec::>::new() ); } async fn undelete(params: &mut JMAPTest) { // Authenticate let mut imap = ImapConnection::connect(b"_x ").await; imap.authenticate("jdoe@example.com", "12345").await; // Insert test message imap.send("STATUS INBOX (MESSAGES)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("MESSAGES 0"); imap.send(&format!("APPEND INBOX {{{}}}", RAW_MESSAGE.len())) .await; imap.assert_read(Type::Continuation, ResponseType::Ok).await; imap.send_untagged(RAW_MESSAGE).await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; // Make sure the message is there imap.send("STATUS INBOX (MESSAGES)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("MESSAGES 1"); imap.send("SELECT INBOX").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; // Fetch message body imap.send("FETCH 1 BODY[]").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("Subject: undelete test"); // Delete and expunge message imap.send("STORE 1 +FLAGS (\\Deleted)").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("EXPUNGE").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; // Logout and reconnect imap.send("LOGOUT").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; let mut imap = ImapConnection::connect(b"_x ").await; imap.authenticate("jdoe@example.com", "12345").await; // Make sure the message is gone imap.send("STATUS INBOX (MESSAGES)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("MESSAGES 0"); // Query undelete API let api = ManagementApi::new(8899, "admin", "secret"); api.get::("/api/store/purge/account/jdoe@example.com") .await .unwrap(); wait_for_index(¶ms.server).await; tokio::time::sleep(Duration::from_millis(200)).await; let deleted = api .get::>("/api/store/undelete/jdoe@example.com") .await .unwrap() .unwrap_data() .items; assert_eq!(deleted.len(), 1); let deleted = deleted.into_iter().next().unwrap(); match deleted.item { DeletedItemResponse::Email { from, subject, .. } => { assert_eq!(subject.as_ref(), "undelete test"); assert_eq!(from.as_ref(), "john@example.com"); } other => { panic!("Unexpected deleted item response: {:?}", other); } } // Undelete let result = api .post::>( "/api/store/undelete/jdoe@example.com", &vec![UndeleteRequest { hash: deleted.hash, collection: "email".to_string(), time: deleted.deleted_at, cancel_deletion: deleted.expires_at.into(), }], ) .await .unwrap() .unwrap_data(); assert_eq!(result, vec![UndeleteResponse::Success]); // Make sure the message is back imap.send("STATUS INBOX (MESSAGES)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("MESSAGES 1"); imap.send("SELECT INBOX").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; // Fetch message body imap.send("FETCH 1 BODY[]").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("Subject: undelete test"); } pub async fn insert_test_metrics(core: Arc) { let store = core.storage.data.clone(); store.purge_metrics(Duration::from_secs(0)).await.unwrap(); let mut start_time = now() - (90 * 24 * 60 * 60); let timestamp = now(); let history = SharedMetricHistory::default(); while start_time < timestamp { for event_type in [ EventType::Smtp(SmtpEvent::ConnectionStart), EventType::Imap(ImapEvent::ConnectionStart), EventType::Pop3(Pop3Event::ConnectionStart), EventType::ManageSieve(ManageSieveEvent::ConnectionStart), EventType::Http(HttpEvent::ConnectionStart), EventType::Delivery(DeliveryEvent::AttemptStart), EventType::Queue(QueueEvent::QueueMessage), EventType::Queue(QueueEvent::QueueMessageAuthenticated), EventType::Queue(QueueEvent::QueueDsn), EventType::Queue(QueueEvent::QueueReport), EventType::MessageIngest(MessageIngestEvent::Ham), EventType::MessageIngest(MessageIngestEvent::Spam), EventType::Auth(AuthEvent::Failed), EventType::Security(SecurityEvent::AuthenticationBan), EventType::Security(SecurityEvent::ScanBan), EventType::Security(SecurityEvent::AbuseBan), EventType::Security(SecurityEvent::LoiterBan), EventType::Security(SecurityEvent::IpBlocked), EventType::IncomingReport(IncomingReportEvent::DmarcReport), EventType::IncomingReport(IncomingReportEvent::DmarcReportWithWarnings), EventType::IncomingReport(IncomingReportEvent::TlsReport), EventType::IncomingReport(IncomingReportEvent::TlsReportWithWarnings), ] { // Generate a random value between 0 and 100 Collector::update_event_counter(event_type, rand::rng().random_range(0..=100)) } Collector::update_gauge(MetricType::QueueCount, rand::rng().random_range(0..=1000)); Collector::update_gauge( MetricType::ServerMemory, rand::rng().random_range(100 * 1024 * 1024..=300 * 1024 * 1024), ); for metric_type in [ MetricType::MessageIngestionTime, MetricType::MessageFtsIndexTime, MetricType::DeliveryTime, MetricType::DnsLookupTime, ] { Collector::update_histogram(metric_type, rand::rng().random_range(2..=1000)) } Collector::update_histogram( MetricType::DeliveryTotalTime, rand::rng().random_range(1000..=5000), ); store .write_metrics(core.clone(), start_time, history.clone()) .await .unwrap(); start_time += 60 * 60; } } ================================================ FILE: tests/src/jmap/server/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod enterprise; pub mod purge; pub mod webhooks; #[derive(serde::Deserialize, Debug)] #[allow(dead_code)] pub(crate) struct List { pub items: Vec, pub total: usize, } ================================================ FILE: tests/src/jmap/server/purge.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ imap::{AssertResult, ImapConnection, Type}, jmap::{JMAPTest, wait_for_index}, }; use ahash::AHashSet; use common::Server; use directory::{QueryBy, backend::internal::manage::ManageDirectory}; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, mailbox::{INBOX_ID, JUNK_ID, TRASH_ID}, message::delete::EmailDeletion, }; use http::management::stores::destroy_account_data; use imap_proto::ResponseType; use store::{IterateParams, LogKey, U32_LEN, U64_LEN, write::key::DeserializeBigEndian}; use types::id::Id; pub async fn test(params: &mut JMAPTest) { println!("Running purge tests..."); let server = params.server.clone(); let inbox_id = Id::from(INBOX_ID).to_string(); let trash_id = Id::from(TRASH_ID).to_string(); let junk_id = Id::from(JUNK_ID).to_string(); let account = params.account("jdoe@example.com"); let client = account.client(); let mut imap = ImapConnection::connect(b"_x ").await; imap.assert_read(Type::Untagged, ResponseType::Ok).await; imap.send("LOGIN \"jdoe@example.com\" \"12345\"").await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; imap.send("STATUS INBOX (UIDNEXT MESSAGES UNSEEN)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("MESSAGES 0"); // Create test messages let mut message_ids = Vec::new(); let mut pass = 0; let mut changes = AHashSet::new(); loop { pass += 1; for folder_id in [&inbox_id, &trash_id, &junk_id] { message_ids.push( client .email_import( format!( concat!( "From: bill@example.com\r\n", "To: jdoe@example.com\r\n", "Subject: TPS Report #{} {}\r\n", "\r\n", "I'm going to need those TPS reports ASAP. ", "So, if you could do that, that'd be great." ), pass, folder_id ) .into_bytes(), [folder_id], None::>, None, ) .await .unwrap() .take_id(), ); } if pass == 1 { let (changes_, is_truncated) = get_changes(&server).await; assert!(!is_truncated); changes = changes_; tokio::time::sleep(std::time::Duration::from_secs(1)).await; } else { break; } } // Check IMAP status imap.send("LIST \"\" \"*\" RETURN (STATUS (MESSAGES))") .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("\"INBOX\" (MESSAGES 2)") .assert_contains("\"Deleted Items\" (MESSAGES 2)") .assert_contains("\"Junk Mail\" (MESSAGES 2)"); // Make sure both messages and changes are present assert_eq!( server .get_cached_messages(account.id().document_id()) .await .unwrap() .emails .items .len(), 6 ); // Purge junk/trash messages and old changes server.purge_account(account.id().document_id()).await; let cache = server .get_cached_messages(account.id().document_id()) .await .unwrap(); // Only 4 messages should remain assert_eq!( server .get_cached_messages(account.id().document_id()) .await .unwrap() .emails .items .len(), 4 ); assert_eq!(cache.in_mailbox(INBOX_ID).count(), 2); assert_eq!(cache.in_mailbox(TRASH_ID).count(), 1); assert_eq!(cache.in_mailbox(JUNK_ID).count(), 1); // Check IMAP status imap.send("LIST \"\" \"*\" RETURN (STATUS (MESSAGES))") .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("\"INBOX\" (MESSAGES 2)") .assert_contains("\"Deleted Items\" (MESSAGES 1)") .assert_contains("\"Junk Mail\" (MESSAGES 1)"); // Compare changes let (new_changes, is_truncated) = get_changes(&server).await; assert!(!changes.is_empty()); assert!(!new_changes.is_empty()); assert!(is_truncated); for change in &changes { assert!( !new_changes.contains(change), "Change {change:?} was not purged, expected {} changes, got {}", changes.len(), new_changes.len() ); } // Delete account wait_for_index(&server).await; server .store() .delete_principal(QueryBy::Id(account.id().document_id())) .await .unwrap(); destroy_account_data(&server, account.id().document_id(), true) .await .unwrap(); params.assert_is_empty().await; } async fn get_changes(server: &Server) -> (AHashSet<(u64, u8)>, bool) { let mut changes = AHashSet::new(); let mut is_truncated = false; server .core .storage .data .iterate( IterateParams::new( LogKey { account_id: 0, collection: 0, change_id: 0, }, LogKey { account_id: u32::MAX, collection: u8::MAX, change_id: u64::MAX, }, ) .ascending(), |key, value| { if !value.is_empty() { changes.insert(( key.deserialize_be_u64(key.len() - U64_LEN).unwrap(), key[U32_LEN], )); } else { is_truncated = true; } Ok(true) }, ) .await .unwrap(); (changes, is_truncated) } ================================================ FILE: tests/src/jmap/server/webhooks.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ sync::{ Arc, atomic::{AtomicBool, Ordering}, }, time::Duration, }; use crate::jmap::JMAPTest; use base64::{Engine, engine::general_purpose::STANDARD}; use common::manager::webadmin::Resource; use http_proto::{ToHttpResponse, request::fetch_body}; use hyper::{body, server::conn::http1, service::service_fn}; use hyper_util::rt::TokioIo; use jmap::api::ToJmapHttpResponse; use jmap_proto::error::request::RequestError; use ring::hmac; use store::parking_lot::Mutex; use tokio::{net::TcpListener, sync::watch}; pub struct MockWebhookEndpoint { pub tx: watch::Sender, pub events: Mutex>, pub reject: AtomicBool, } pub async fn test(params: &mut JMAPTest) { println!("Running Webhook tests..."); // Webhooks endpoint starts disabled by default, make sure there are no events. tokio::time::sleep(Duration::from_millis(200)).await; params.webhook.assert_is_empty(); // Enable the endpoint params.webhook.accept(); tokio::time::sleep(Duration::from_millis(1000)).await; // Check for events params.webhook.assert_contains(&["auth.success"]); } impl MockWebhookEndpoint { pub fn assert_contains(&self, expected: &[&str]) { let events = serde_json::to_string_pretty(&self.events.lock().drain(..).collect::>()) .unwrap(); for string in expected { if !events.contains(string) { panic!( "Expected events to contain '{}', but it did not. Events: {}", string, events ); } } } pub fn accept(&self) { self.reject.store(false, Ordering::Relaxed); } pub fn reject(&self) { self.reject.store(true, Ordering::Relaxed); } pub fn clear(&self) { self.events.lock().clear(); } pub fn assert_is_empty(&self) { assert!(self.events.lock().is_empty()); } } pub fn spawn_mock_webhook_endpoint() -> Arc { let (tx, rx) = watch::channel(true); let endpoint_ = Arc::new(MockWebhookEndpoint { tx, events: Mutex::new(vec![]), reject: true.into(), }); let endpoint = endpoint_.clone(); tokio::spawn(async move { let listener = TcpListener::bind("127.0.0.1:8821") .await .unwrap_or_else(|e| { panic!("Failed to bind mock Webhooks server to 127.0.0.1:8821: {e}"); }); let mut rx_ = rx.clone(); loop { tokio::select! { stream = listener.accept() => { match stream { Ok((stream, _)) => { let _ = http1::Builder::new() .keep_alive(false) .serve_connection( TokioIo::new(stream), service_fn(|mut req: hyper::Request| { let endpoint = endpoint.clone(); async move { // Verify HMAC signature let key = hmac::Key::new(hmac::HMAC_SHA256, "ovos-moles".as_bytes()); let body = fetch_body(&mut req, usize::MAX, 0).await.unwrap(); let tag = STANDARD.decode(req.headers().get("X-Signature").unwrap().to_str().unwrap()).unwrap(); hmac::verify(&key, &body, &tag).expect("Invalid signature"); // Deserialize JSON #[derive(serde::Deserialize)] struct WebhookRequest { events: Vec, } let request = serde_json::from_slice::(&body) .expect("Failed to parse JSON"); if !endpoint.reject.load(Ordering::Relaxed) { //let c = print!("received webhook: {}", serde_json::to_string_pretty(&request).unwrap()); // Add events endpoint.events.lock().extend(request.events); Ok::<_, hyper::Error>( Resource::new("application/json", "[]".to_string().into_bytes()) .into_http_response().build(), ) } else { //let c = print!("rejected webhook: {}", serde_json::to_string_pretty(&request).unwrap()); Ok::<_, hyper::Error>( RequestError::not_found().into_http_response().build() ) } } }), ) .await; } Err(err) => { panic!("Something went wrong: {err}" ); } } }, _ = rx_.changed() => { //println!("Mock jMilter server stopping"); break; } }; } }); endpoint_ } ================================================ FILE: tests/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::path::PathBuf; #[cfg(not(target_env = "msvc"))] use jemallocator::Jemalloc; #[cfg(test)] use trc::Collector; #[cfg(not(target_env = "msvc"))] #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; #[cfg(test)] pub mod cluster; #[cfg(test)] pub mod directory; #[cfg(test)] pub mod http_server; #[cfg(test)] pub mod imap; #[cfg(test)] pub mod jmap; #[cfg(test)] pub mod smtp; #[cfg(test)] pub mod store; #[cfg(test)] pub mod webdav; pub fn add_test_certs(config: &str) -> String { let mut cert_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); cert_path.push("resources"); let mut cert = cert_path.clone(); cert.push("tls_cert.pem"); let mut pk = cert_path.clone(); pk.push("tls_privatekey.pem"); config .replace("{CERT}", cert.as_path().to_str().unwrap()) .replace("{PK}", pk.as_path().to_str().unwrap()) } #[cfg(test)] pub trait AssertConfig { fn assert_no_errors(self) -> Self; fn assert_no_warnings(self) -> Self; } #[cfg(test)] impl AssertConfig for utils::config::Config { fn assert_no_errors(self) -> Self { if !self.errors.is_empty() { panic!("Errors: {:#?}", self.errors); } self } fn assert_no_warnings(self) -> Self { if !self.warnings.is_empty() { panic!("Warnings: {:#?}", self.warnings); } self } } #[cfg(test)] pub fn enable_logging() { use common::config::telemetry::Telemetry; if let Ok(level) = std::env::var("LOG") && !Collector::is_enabled() { Telemetry::test_tracer(level.parse().expect("Invalid log level")); } } pub const TEST_USERS: &[(&str, &str, &str, &str)] = &[ ("admin", "secret", "Superuser", "admin@example.com"), ("john", "secret2", "John Doe", "jdoe@example.com"), ( "jane", "secret3", "Jane Doe-Smith", "jane.smith@example.com", ), ("bill", "secret4", "Bill Foobar", "bill@example.com"), ("mike", "secret5", "Mike Noquota", "mike@example.com"), ]; ================================================ FILE: tests/src/smtp/config.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{fs, net::IpAddr, path::PathBuf, sync::Arc, time::Duration}; use common::{ Server, config::{ server::{Listener, Listeners, ServerProtocol, TcpListener}, smtp::*, }, expr::{functions::ResolveVariable, if_block::*, tokenizer::TokenMap, *}, }; use compact_str::ToCompactString; use throttle::parse_queue_rate_limiter; use tokio::net::TcpSocket; use utils::config::{Config, Rate}; use super::add_test_certs; struct TestEnvelope { pub local_ip: IpAddr, pub remote_ip: IpAddr, pub sender_domain: String, pub sender: String, pub rcpt_domain: String, pub rcpt: String, pub helo_domain: String, pub authenticated_as: String, pub mx: String, pub listener_id: String, pub priority: i16, } #[test] fn parse_if_blocks() { let mut file = PathBuf::from(env!("CARGO_MANIFEST_DIR")); file.push("resources"); file.push("smtp"); file.push("config"); file.push("if-blocks.toml"); let mut config = Config::new(fs::read_to_string(file).unwrap()).unwrap(); // Create context and add some conditions let token_map = TokenMap::default().with_variables(&[ V_RECIPIENT, V_RECIPIENT_DOMAIN, V_SENDER, V_SENDER_DOMAIN, V_AUTHENTICATED_AS, V_LISTENER, V_REMOTE_IP, V_LOCAL_IP, V_PRIORITY, ]); assert_eq!( IfBlock::try_parse(&mut config, "durations", &token_map).unwrap(), IfBlock { key: "durations".into(), if_then: vec![ IfThen { expr: Expression { items: vec![ ExpressionItem::Variable(V_SENDER), ExpressionItem::Constant(Constant::String("jdoe".into())), ExpressionItem::BinaryOperator(BinaryOperator::Eq) ] }, then: Expression { items: vec![ExpressionItem::Constant(Constant::Integer(432000000))] } }, IfThen { expr: Expression { items: vec![ ExpressionItem::Variable(V_PRIORITY), ExpressionItem::Constant(Constant::Integer(1)), ExpressionItem::UnaryOperator(UnaryOperator::Minus), ExpressionItem::BinaryOperator(BinaryOperator::Eq), ExpressionItem::JmpIf { val: true, pos: 4 }, ExpressionItem::Variable(V_RECIPIENT), ExpressionItem::Constant(Constant::String("jane".into())), ExpressionItem::Function { id: 29, num_args: 2 }, ExpressionItem::BinaryOperator(BinaryOperator::Or) ] }, then: Expression { items: vec![ExpressionItem::Constant(Constant::Integer(3600000))] } } ], default: Expression { items: vec![ExpressionItem::Constant(Constant::Integer(0))] } } ); assert_eq!( IfBlock::try_parse(&mut config, "string-list", &token_map).unwrap(), IfBlock { key: "string-list".into(), if_then: vec![ IfThen { expr: Expression { items: vec![ ExpressionItem::Variable(V_SENDER), ExpressionItem::Constant(Constant::String("jdoe".into())), ExpressionItem::BinaryOperator(BinaryOperator::Eq) ] }, then: Expression { items: vec![ ExpressionItem::Constant(Constant::String("From".into())), ExpressionItem::Constant(Constant::String("To".into())), ExpressionItem::Constant(Constant::String("Date".into())), ExpressionItem::ArrayBuild(3) ] } }, IfThen { expr: Expression { items: vec![ ExpressionItem::Variable(V_PRIORITY), ExpressionItem::Constant(Constant::Integer(1)), ExpressionItem::UnaryOperator(UnaryOperator::Minus), ExpressionItem::BinaryOperator(BinaryOperator::Eq), ExpressionItem::JmpIf { val: true, pos: 4 }, ExpressionItem::Variable(V_RECIPIENT), ExpressionItem::Constant(Constant::String("jane".into())), ExpressionItem::Function { id: 29, num_args: 2 }, ExpressionItem::BinaryOperator(BinaryOperator::Or) ] }, then: Expression { items: vec![ExpressionItem::Constant(Constant::String( "Other-ID".into() ))] } } ], default: Expression { items: vec![ExpressionItem::ArrayBuild(0)] } } ); assert_eq!( IfBlock::try_parse(&mut config, "string-list-bis", &token_map).unwrap(), IfBlock { key: "string-list-bis".into(), if_then: vec![ IfThen { expr: Expression { items: vec![ ExpressionItem::Variable(V_SENDER), ExpressionItem::Constant(Constant::String("jdoe".into())), ExpressionItem::BinaryOperator(BinaryOperator::Eq) ] }, then: Expression { items: vec![ ExpressionItem::Constant(Constant::String("From".into())), ExpressionItem::Constant(Constant::String("To".into())), ExpressionItem::Constant(Constant::String("Date".into())), ExpressionItem::ArrayBuild(3) ] } }, IfThen { expr: Expression { items: vec![ ExpressionItem::Variable(V_PRIORITY), ExpressionItem::Constant(Constant::Integer(1)), ExpressionItem::UnaryOperator(UnaryOperator::Minus), ExpressionItem::BinaryOperator(BinaryOperator::Eq), ExpressionItem::JmpIf { val: true, pos: 4 }, ExpressionItem::Variable(V_RECIPIENT), ExpressionItem::Constant(Constant::String("jane".into())), ExpressionItem::Function { id: 29, num_args: 2 }, ExpressionItem::BinaryOperator(BinaryOperator::Or) ] }, then: Expression { items: vec![ExpressionItem::ArrayBuild(0)] } } ], default: Expression { items: vec![ ExpressionItem::Constant(Constant::String("ID-Bis".into())), ExpressionItem::ArrayBuild(1) ] } } ); assert_eq!( IfBlock::try_parse(&mut config, "single-value", &token_map).unwrap(), IfBlock { key: "single-value".into(), if_then: vec![], default: Expression { items: vec![ExpressionItem::Constant(Constant::String( "hello world".into() ))] } } ); for bad_rule in [ "bad-if-without-then", "bad-if-without-else", "bad-multiple-else", ] { if let Some(value) = IfBlock::try_parse(&mut config, bad_rule, &token_map) { panic!("Condition {bad_rule:?} had unexpected result {value:?}"); } } } #[test] fn parse_throttles() { let mut file = PathBuf::from(env!("CARGO_MANIFEST_DIR")); file.push("resources"); file.push("smtp"); file.push("config"); file.push("throttle.toml"); let mut config = Config::new(fs::read_to_string(file).unwrap()).unwrap(); let throttle = parse_queue_rate_limiter( &mut config, "throttle", &TokenMap::default().with_variables(&[ V_RECIPIENT, V_RECIPIENT_DOMAIN, V_SENDER, V_SENDER_DOMAIN, V_AUTHENTICATED_AS, V_LISTENER, V_REMOTE_IP, V_LOCAL_IP, V_PRIORITY, ]), u16::MAX, ); assert_eq!( throttle, vec![ QueueRateLimiter { id: "0000".into(), expr: Expression { items: vec![ ExpressionItem::Variable(8), ExpressionItem::Constant(Constant::String("127.0.0.1".into())), ExpressionItem::BinaryOperator(BinaryOperator::Eq) ] }, keys: THROTTLE_REMOTE_IP | THROTTLE_AUTH_AS, rate: Rate { requests: 50, period: Duration::from_secs(30) } }, QueueRateLimiter { id: "0001".into(), expr: Expression::default(), keys: THROTTLE_SENDER_DOMAIN, rate: Rate { requests: 50, period: Duration::from_secs(30) } } ] ); } #[test] fn parse_servers() { let mut file = PathBuf::from(env!("CARGO_MANIFEST_DIR")); file.push("resources"); file.push("smtp"); file.push("config"); file.push("servers.toml"); let toml = add_test_certs(&fs::read_to_string(file).unwrap()); // Parse servers let mut config = Config::new(toml).unwrap(); let servers = Listeners::parse(&mut config).servers; let id_generator = Arc::new(utils::snowflake::SnowflakeIdGenerator::new()); let expected_servers = vec![ Listener { id: "smtp".into(), protocol: ServerProtocol::Smtp, listeners: vec![TcpListener { socket: TcpSocket::new_v4().unwrap(), addr: "127.0.0.1:9925".parse().unwrap(), ttl: 3600.into(), backlog: 1024.into(), linger: None, nodelay: true, }], max_connections: 8192, proxy_networks: vec![], span_id_gen: id_generator.clone(), }, Listener { id: "smtps".into(), protocol: ServerProtocol::Smtp, listeners: vec![ TcpListener { socket: TcpSocket::new_v4().unwrap(), addr: "127.0.0.1:9465".parse().unwrap(), ttl: 4096.into(), backlog: 1024.into(), linger: None, nodelay: true, }, TcpListener { socket: TcpSocket::new_v4().unwrap(), addr: "127.0.0.1:9466".parse().unwrap(), ttl: 4096.into(), backlog: 1024.into(), linger: None, nodelay: true, }, ], max_connections: 1024, proxy_networks: vec![], span_id_gen: id_generator.clone(), }, Listener { id: "submission".into(), protocol: ServerProtocol::Smtp, listeners: vec![TcpListener { socket: TcpSocket::new_v4().unwrap(), addr: "127.0.0.1:9991".parse().unwrap(), ttl: 3600.into(), backlog: 2048.into(), linger: None, nodelay: true, }], max_connections: 8192, proxy_networks: vec![], span_id_gen: id_generator.clone(), }, ]; for (server, expected_server) in servers.into_iter().zip(expected_servers) { assert_eq!( server.id, expected_server.id, "failed for {}", expected_server.id ); assert_eq!( server.protocol, expected_server.protocol, "failed for {}", expected_server.id ); for (listener, expected_listener) in server.listeners.into_iter().zip(expected_server.listeners) { assert_eq!( listener.addr, expected_listener.addr, "failed for {}", expected_server.id ); assert_eq!( listener.ttl, expected_listener.ttl, "failed for {}", expected_server.id ); assert_eq!( listener.backlog, expected_listener.backlog, "failed for {}", expected_server.id ); } } } #[tokio::test] async fn eval_if() { let mut file = PathBuf::from(env!("CARGO_MANIFEST_DIR")); file.push("resources"); file.push("smtp"); file.push("config"); file.push("rules-eval.toml"); let mut config = Config::new(fs::read_to_string(file).unwrap()).unwrap(); let envelope = TestEnvelope::from_config(&mut config); let token_map = TokenMap::default().with_variables(&[ V_RECIPIENT, V_RECIPIENT_DOMAIN, V_SENDER, V_SENDER_DOMAIN, V_AUTHENTICATED_AS, V_LISTENER, V_REMOTE_IP, V_LOCAL_IP, V_PRIORITY, V_MX, ]); let core = Server::default(); for (key, _) in config.keys.clone() { if !key.starts_with("rule.") { continue; } //println!("============= Testing {:?} ==================", key); let (_, expected_result) = key.rsplit_once('-').unwrap(); assert_eq!( core.eval_if::( &IfBlock { key: key.to_string(), if_then: vec![IfThen { expr: Expression::try_parse(&mut config, key.as_str(), &token_map).unwrap(), then: Expression::from(true), }], default: Expression::from(false), }, &envelope, 0 ) .await .unwrap() .to_bool(), expected_result.parse::().unwrap(), "failed for {key:?}" ); } } #[tokio::test] async fn eval_dynvalue() { let mut file = PathBuf::from(env!("CARGO_MANIFEST_DIR")); file.push("resources"); file.push("smtp"); file.push("config"); file.push("rules-dynvalue.toml"); let mut config = Config::new(fs::read_to_string(file).unwrap()).unwrap(); let envelope = TestEnvelope::from_config(&mut config); let token_map = TokenMap::default().with_variables(&[ V_RECIPIENT, V_RECIPIENT_DOMAIN, V_SENDER, V_SENDER_DOMAIN, V_AUTHENTICATED_AS, V_LISTENER, V_REMOTE_IP, V_LOCAL_IP, V_PRIORITY, V_MX, ]); let core = Server::default(); for test_name in config.sub_keys("eval", "") { //println!("============= Testing {:?} ==================", key); let if_block = IfBlock::try_parse( &mut config, ("eval", test_name.as_str(), "test"), &token_map, ) .unwrap(); let expected = config .property_require::>(("eval", test_name.as_str(), "expect")) .unwrap_or_else(|| panic!("Missing expect for test {test_name:?}")); assert_eq!( core.eval_if::(&if_block, &envelope, 0).await, expected, "failed for test {test_name:?}" ); } } impl ResolveVariable for TestEnvelope { fn resolve_variable(&self, variable: u32) -> Variable<'_> { match variable { V_RECIPIENT => self.rcpt.as_str().into(), V_RECIPIENT_DOMAIN => self.rcpt_domain.as_str().into(), V_SENDER => self.sender.as_str().into(), V_SENDER_DOMAIN => self.sender_domain.as_str().into(), V_AUTHENTICATED_AS => self.authenticated_as.as_str().into(), V_LISTENER => self.listener_id.to_compact_string().into(), V_REMOTE_IP => self.remote_ip.to_compact_string().into(), V_LOCAL_IP => self.local_ip.to_compact_string().into(), V_PRIORITY => self.priority.to_compact_string().into(), V_MX => self.mx.as_str().into(), V_HELO_DOMAIN => self.helo_domain.as_str().into(), _ => Default::default(), } } fn resolve_global(&self, _: &str) -> Variable<'_> { Variable::Integer(0) } } impl TestEnvelope { pub fn from_config(config: &mut Config) -> Self { Self { local_ip: config.property_require("envelope.local-ip").unwrap(), remote_ip: config.property_require("envelope.remote-ip").unwrap(), sender_domain: config.property_require("envelope.sender-domain").unwrap(), sender: config.property_require("envelope.sender").unwrap(), rcpt_domain: config.property_require("envelope.rcpt-domain").unwrap(), rcpt: config.property_require("envelope.rcpt").unwrap(), authenticated_as: config .property_require("envelope.authenticated-as") .unwrap(), mx: config.property_require("envelope.mx").unwrap(), listener_id: config.property_require("envelope.listener").unwrap(), priority: config.property_require("envelope.priority").unwrap(), helo_domain: config.property_require("envelope.helo-domain").unwrap(), } } } ================================================ FILE: tests/src/smtp/inbound/antispam.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ http_server::{HttpMessage, spawn_mock_http_server}, jmap::server::enterprise::EnterpriseCore, smtp::{DnsCache, TempDir, TestSMTP, session::TestSession}, }; use ahash::{AHashMap, AHashSet}; use common::{ Core, Server, auth::AccessToken, config::spamfilter::SpamFilterAction, enterprise::{ SpamFilterLlmConfig, llm::{ AiApiConfig, ChatCompletionChoice, ChatCompletionRequest, ChatCompletionResponse, Message, }, }, }; use email::message::ingest::EmailIngest; use http_proto::{JsonResponse, ToHttpResponse}; use hyper::Method; use mail_auth::{ ArcOutput, DkimOutput, DkimResult, DmarcResult, IprevOutput, IprevResult, MX, SpfOutput, SpfResult, dkim::Signature, dmarc::Policy, }; use mail_parser::MessageParser; use smtp::core::{Session, SessionAddress}; use smtp_proto::{MAIL_BODY_8BITMIME, MAIL_SMTPUTF8}; use spam_filter::{ SpamFilterInput, analysis::{ classifier::SpamFilterAnalyzeClassify, date::SpamFilterAnalyzeDate, dmarc::SpamFilterAnalyzeDmarc, domain::SpamFilterAnalyzeDomain, ehlo::SpamFilterAnalyzeEhlo, from::SpamFilterAnalyzeFrom, headers::SpamFilterAnalyzeHeaders, html::SpamFilterAnalyzeHtml, init::SpamFilterInit, ip::SpamFilterAnalyzeIp, llm::SpamFilterAnalyzeLlm, messageid::SpamFilterAnalyzeMid, mime::SpamFilterAnalyzeMime, pyzor::SpamFilterAnalyzePyzor, received::SpamFilterAnalyzeReceived, recipient::SpamFilterAnalyzeRecipient, replyto::SpamFilterAnalyzeReplyTo, rules::SpamFilterAnalyzeRules, score::SpamFilterAnalyzeScore, subject::SpamFilterAnalyzeSubject, url::SpamFilterAnalyzeUrl, }, modules::{ classifier::{SpamClassifier, Token}, html::{HtmlToken, html_to_tokens}, }, }; use std::{ fs, path::PathBuf, sync::Arc, time::{Duration, Instant}, }; use store::{Stores, write::BatchBuilder}; use utils::config::Config; const CONFIG: &str = r#" [spam-filter.score] spam = "5.0" [spam-filter.llm] enable = true model = "dummy" prompt = "You are an AI assistant specialized in analyzing email content to detect unsolicited, commercial, or harmful messages. Format your response as follows, separated by commas: Category,Confidence,Explanation Here's the email to analyze, please provide your analysis based on the above instructions, ensuring your response is in the specified comma-separated format." separator = "," categories = ["Unsolicited", "Commercial", "Harmful", "Legitimate"] confidence = ["High", "Medium", "Low"] [spam-filter.llm.index] category = 0 confidence = 1 explanation = 2 [spam-filter.classifier.samples] min-ham = 10 min-spam = 10 [session.rcpt] relay = true [storage] data = "spamdb" lookup = "spamdb" blob = "spamdb" fts = "spamdb" directory = "spamdb" [directory."spamdb"] type = "internal" store = "spamdb" [store."spamdb"] type = "rocksdb" path = "{PATH}/test_antispam.db" #[store."redis"] #type = "redis" #url = "redis://127.0.0.1" [http-lookup.STWT_OPENPHISH] enable = true url = "https://openphish.com/feed.txt" format = "list" retry = "1h" refresh = "12h" timeout = "30s" limits.size = 104857600 limits.entries = 900000 limits.entry-size = 512 [http-lookup.STWT_PHISHTANK] enable = true url = "http://data.phishtank.com/data/online-valid.csv.gz" format = "csv" separator = "," index.key = 1 skip-first = true gzipped = true retry = "1h" refresh = "6h" timeout = "30s" limits.size = 104857600 limits.entries = 900000 limits.entry-size = 512 [http-lookup.STWT_DISPOSABLE_DOMAINS] enable = true url = "https://disposable.github.io/disposable-email-domains/domains_mx.txt" format = "list" retry = "1h" refresh = "24h" timeout = "30s" limits.size = 104857600 limits.entries = 900000 limits.entry-size = 512 [http-lookup.STWT_FREE_DOMAINS] enable = true url = "https://gist.githubusercontent.com/okutbay/5b4974b70673dfdcc21c517632c1f984/raw/993a35930a8d24a1faab1b988d19d38d92afbba4/free_email_provider_domains.txt" format = "list" retry = "1h" refresh = "720h" timeout = "30s" limits.size = 104857600 limits.entries = 900000 limits.entry-size = 512 [enterprise.ai.dummy] url = "https://127.0.0.1:9090/v1/chat/completions" type = "chat" model = "gpt-dummy" allow-invalid-certs = true [spam-filter.list] "file-extensions" = { "html" = "text/html|BAD", "pdf" = "application/pdf|NZ", "txt" = "text/plain|message/disposition-notification|text/rfc822-headers", "zip" = "AR", "js" = "BAD|NZ", "hta" = "BAD|NZ" } [lookup] "url-redirectors" = {"bit.ly", "redirect.io", "redirect.me", "redirect.org", "redirect.com", "redirect.net", "t.ly", "tinyurl.com"} "spam-traps" = {"spamtrap@*"} "trusted-domains" = {"stalw.art"} "surbl-hashbl" = {"bit.ly", "drive.google.com", "lnkiy.in"} "#; #[tokio::test(flavor = "multi_thread")] async fn antispam() { // Enable logging crate::enable_logging(); // Prepare config let tmp_dir = TempDir::new("smtp_antispam_test", true); let mut config = CONFIG.replace("{PATH}", tmp_dir.temp_dir.as_path().to_str().unwrap()); let base_path = PathBuf::from( std::env::var("SPAM_RULES_DIR") .unwrap_or_else(|_| "/Users/me/code/spam-filter".to_string()), ); for section in ["rules", "lists"] { for entry in fs::read_dir(base_path.join(section)).unwrap() { let entry = entry.unwrap(); let path = entry.path(); if path.is_file() { let file_name = path.file_name().unwrap().to_str().unwrap(); if file_name.ends_with(".toml") && ((section == "rules" && file_name != "llm.toml") || (section == "lists" && file_name == "scores.toml")) { let contents = fs::read_to_string(&path).unwrap(); config.push_str("\n\n"); config.push_str(&contents); } } } } // Parse config let mut config = Config::new(&config).unwrap(); config.resolve_all_macros().await; let stores = Stores::parse_all(&mut config, false).await; let mut core = Core::parse(&mut config, stores, Default::default()) .await .enable_enterprise(); let ai_apis = AHashMap::from_iter([( "dummy".to_string(), AiApiConfig::parse(&mut config, "dummy").unwrap().into(), )]); core.enterprise.as_mut().unwrap().spam_filter_llm = SpamFilterLlmConfig::parse(&mut config, &ai_apis); crate::AssertConfig::assert_no_errors(config); let server = TestSMTP::from_core(core).server; // Add mock DNS entries for (domain, ip) in [ ("bank.com", "127.0.0.1"), ("apple.com", "127.0.0.1"), ("youtube.com", "127.0.0.1"), ("twitter.com", "127.0.0.3"), ("dkimtrusted.org.dwl.dnswl.org", "127.0.0.3"), ("sh-malware.com.dbl.spamhaus.org", "127.0.1.5"), ("surbl-abuse.com.multi.surbl.org", "127.0.0.64"), ("uribl-grey.com.multi.uribl.com", "127.0.0.4"), ("sem-uribl.com.uribl.spameatingmonkey.net", "127.0.0.2"), ("sem-fresh15.com.fresh15.spameatingmonkey.net", "127.0.0.2"), ( "b4a64d60f67529b0b18df66ea2f292e09e43c975.ebl.msbl.org", "127.0.0.2", ), ( "a95bd658068a8315dc1864d6bb79632f47692621.ebl.msbl.org", "127.0.1.3", ), ( "ba76e47680ba70a0cbff8d6c92139683.hashbl.surbl.org", "127.0.0.16", ), ( "0ac5b387a1c6d8461a78bbf7b172a2a1.hashbl.surbl.org", "127.0.0.64", ), ( "637d6717761b5de0c84108c894bb68f2.hashbl.surbl.org", "127.0.0.8", ), ] { server.ipv4_add( domain, vec![ip.parse().unwrap()], Instant::now() + Duration::from_secs(100), ); server.dnsbl_add( domain, vec![ip.parse().unwrap()], Instant::now() + Duration::from_secs(100), ); } for mx in [ "domain.org", "domain.co.uk", "gmail.com", "custom.disposable.org", ] { server.mx_add( mx, vec![MX { exchanges: vec!["127.0.0.1".parse().unwrap()], preference: 10, }], Instant::now() + Duration::from_secs(100), ); } // Spawn mock OpenAI server let _tx = spawn_mock_http_server(Arc::new(|req: HttpMessage| { assert_eq!(req.uri.path(), "/v1/chat/completions"); assert_eq!(req.method, Method::POST); let req = serde_json::from_slice::(req.body.as_ref().unwrap()).unwrap(); assert_eq!(req.model, "gpt-dummy"); let message = &req.messages[0].content; assert!(message.contains("You are an AI assistant specialized in analyzing email")); JsonResponse::new(&ChatCompletionResponse { created: 0, object: String::new(), id: String::new(), model: req.model, choices: vec![ChatCompletionChoice { index: 0, finish_reason: "stop".to_string(), message: Message { role: "assistant".to_string(), content: message.split_once("Subject: ").unwrap().1.to_string(), }, }], }) .into_http_response() })) .await; // Run tests let base_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("resources") .join("smtp") .join("antispam"); let filter_test = std::env::var("TEST_NAME").ok(); for test_name in [ "combined", "ip", "helo", "received", "messageid", "date", "from", "subject", "replyto", "recipient", "headers", "url", "html", "mime", "bounce", "dmarc", "rbl", "spamtrap", "classifier_html", "classifier_features", "classifier", "pyzor", "llm", ] { if filter_test .as_ref() .is_some_and(|s| !s.eq_ignore_ascii_case(test_name)) { continue; } println!("===== {test_name} ====="); let contents = fs::read_to_string(base_path.join(format!("{test_name}.test"))).unwrap(); match test_name { "classifier_html" => { html_tokens(contents); continue; } "classifier_features" => { classifier_features(&server, contents).await; continue; } "classifier" => { let mut batch = BatchBuilder::new(); batch.with_account_id(u32::MAX); for class in ["spam", "ham"] { let contents = fs::read_to_string(base_path.join(format!("classifier.{class}"))).unwrap(); for sample in contents.split("") { let sample = sample.trim_start(); if sample.is_empty() { continue; } let (hash, blob_hold) = server .put_temporary_blob(u32::MAX, sample.as_bytes(), 60) .await .unwrap(); server.add_spam_sample(&mut batch, hash, class == "spam", true, 0); batch.clear(blob_hold); } } assert!(!batch.is_empty()); server.store().write(batch.build_all()).await.unwrap(); server.spam_train(false).await.unwrap(); } _ => {} } let mut lines = contents.lines(); let mut has_more = true; while has_more { let mut message = String::new(); let mut in_params = true; // Build session let mut session = Session::test(server.clone()); let mut arc_result = None; let mut dkim_result = None; let mut dkim_signatures = vec![]; let mut dmarc_result = None; let mut dmarc_policy = None; let mut expected_tags: AHashSet = AHashSet::new(); let mut expect_headers = String::new(); let mut body_params = 0; let mut is_tls = false; for line in lines.by_ref() { if in_params { if line.is_empty() { in_params = false; continue; } let (param, value) = line.split_once(' ').unwrap(); let value = value.trim(); match param { "remote_ip" => { session.data.remote_ip_str = value.to_string(); session.data.remote_ip = value.parse().unwrap(); } "helo_domain" => { session.data.helo_domain = value.to_string(); } "authenticated_as" => { session.data.authenticated_as = Some(Arc::new(AccessToken { name: value.to_string(), ..Default::default() })); } "spf.result" | "spf_ehlo.result" => { session.data.spf_mail_from = Some(SpfOutput::default().with_result(SpfResult::from_str(value))); } "iprev.result" => { session .data .iprev .get_or_insert(IprevOutput { result: IprevResult::None, ptr: None, }) .result = IprevResult::from_str(value); } "dkim.result" => { dkim_result = match DkimResult::from_str(value) { DkimResult::Pass => DkimOutput::pass(), DkimResult::Neutral(error) => DkimOutput::neutral(error), DkimResult::Fail(error) => DkimOutput::fail(error), DkimResult::PermError(error) => DkimOutput::perm_err(error), DkimResult::TempError(error) => DkimOutput::temp_err(error), DkimResult::None => unreachable!(), } .into(); } "arc.result" => { arc_result = ArcOutput::default() .with_result(DkimResult::from_str(value)) .into(); } "dkim.domains" => { dkim_signatures = value .split_ascii_whitespace() .map(|s| Signature { d: s.to_lowercase(), ..Default::default() }) .collect(); } "envelope_from" => { session.data.mail_from = Some(SessionAddress::new(value.to_string())); } "envelope_to" => { session .data .rcpt_to .push(SessionAddress::new(value.to_string())); } "iprev.ptr" => { session .data .iprev .get_or_insert(IprevOutput { result: IprevResult::None, ptr: None, }) .ptr = Some(Arc::new(vec![value.to_string()])); } "dmarc.result" => { dmarc_result = DmarcResult::from_str(value).into(); } "dmarc.policy" => { dmarc_policy = Policy::from_str(value).into(); } "expect" => { expected_tags .extend(value.split_ascii_whitespace().map(|v| v.to_uppercase())); } "expect_header" => { let value = value.trim(); if !value.is_empty() { if !expect_headers.is_empty() { expect_headers.push(' '); } expect_headers.push_str(value); } } "param.smtputf8" => { body_params |= MAIL_SMTPUTF8; } "param.8bitmime" => { body_params |= MAIL_BODY_8BITMIME; } "tls.version" => { is_tls = true; } _ => panic!("Invalid parameter {param:?}"), } } else { has_more = line.trim().eq_ignore_ascii_case(""); if !has_more { message.push_str(line); message.push_str("\r\n"); } else { break; } } } if message.is_empty() { panic!("No message found"); } if body_params != 0 { session .data .mail_from .get_or_insert_with(|| SessionAddress::new("".to_string())) .flags = body_params; } // Build input let mut dkim_domains = vec![]; if let Some(dkim_result) = dkim_result { if dkim_signatures.is_empty() { dkim_signatures.push(Signature { d: "unknown.org".to_string(), ..Default::default() }); } for signature in &dkim_signatures { dkim_domains.push(dkim_result.clone().with_signature(signature)); } } let parsed_message = MessageParser::new().parse(&message).unwrap(); // Combined tests if test_name == "combined" { match session .spam_classify( &parsed_message, &dkim_domains, arc_result.as_ref(), dmarc_result.as_ref(), dmarc_policy.as_ref(), ) .await { SpamFilterAction::Allow(score) => { let mut last_ch = 'x'; let mut result = String::with_capacity(score.headers.len()); for ch in score.headers.chars() { if !ch.is_whitespace() { if last_ch.is_whitespace() { result.push(' '); } result.push(ch); } last_ch = ch; } assert_eq!(result, expect_headers); } other => panic!("Unexpected action {other:?}"), } continue; } // Initialize filter let mut spam_input = session.build_spam_input( &parsed_message, &dkim_domains, arc_result.as_ref(), dmarc_result.as_ref(), dmarc_policy.as_ref(), ); spam_input.is_tls = is_tls; let mut spam_ctx = server.spam_filter_init(spam_input); match test_name { "html" => { server.spam_filter_analyze_html(&mut spam_ctx).await; server.spam_filter_analyze_rules(&mut spam_ctx).await; } "subject" => { server.spam_filter_analyze_headers(&mut spam_ctx).await; spam_ctx.result.tags.retain(|t| t.starts_with("X_HDR_")); server.spam_filter_analyze_subject(&mut spam_ctx).await; server.spam_filter_analyze_rules(&mut spam_ctx).await; spam_ctx.result.tags.retain(|t| !t.starts_with("X_HDR_")); } "received" => { server.spam_filter_analyze_headers(&mut spam_ctx).await; spam_ctx.result.tags.retain(|t| t.starts_with("X_HDR_")); server.spam_filter_analyze_received(&mut spam_ctx).await; server.spam_filter_analyze_rules(&mut spam_ctx).await; spam_ctx.result.tags.retain(|t| !t.starts_with("X_HDR_")); } "messageid" => { server.spam_filter_analyze_message_id(&mut spam_ctx).await; } "date" => { server.spam_filter_analyze_date(&mut spam_ctx).await; } "from" => { server.spam_filter_analyze_from(&mut spam_ctx).await; server.spam_filter_analyze_domain(&mut spam_ctx).await; server.spam_filter_analyze_rules(&mut spam_ctx).await; } "replyto" => { server.spam_filter_analyze_reply_to(&mut spam_ctx).await; server.spam_filter_analyze_domain(&mut spam_ctx).await; server.spam_filter_analyze_rules(&mut spam_ctx).await; } "recipient" => { server.spam_filter_analyze_headers(&mut spam_ctx).await; spam_ctx.result.tags.retain(|t| t.starts_with("X_HDR_")); server.spam_filter_analyze_recipient(&mut spam_ctx).await; server.spam_filter_analyze_domain(&mut spam_ctx).await; server.spam_filter_analyze_subject(&mut spam_ctx).await; server.spam_filter_analyze_url(&mut spam_ctx).await; server.spam_filter_analyze_rules(&mut spam_ctx).await; spam_ctx.result.tags.retain(|t| !t.starts_with("X_HDR_")); } "mime" => { server.spam_filter_analyze_mime(&mut spam_ctx).await; } "headers" => { server.spam_filter_analyze_headers(&mut spam_ctx).await; server.spam_filter_analyze_rules(&mut spam_ctx).await; spam_ctx.result.tags.retain(|t| !t.starts_with("X_HDR_")); } "url" => { server.spam_filter_analyze_url(&mut spam_ctx).await; server.spam_filter_analyze_rules(&mut spam_ctx).await; } "dmarc" => { server.spam_filter_analyze_dmarc(&mut spam_ctx).await; server.spam_filter_analyze_headers(&mut spam_ctx).await; server.spam_filter_analyze_rules(&mut spam_ctx).await; spam_ctx.result.tags.retain(|t| !t.starts_with("X_HDR_")); } "ip" => { server.spam_filter_analyze_ip(&mut spam_ctx).await; } "helo" => { server.spam_filter_analyze_ehlo(&mut spam_ctx).await; } "bounce" => { server.spam_filter_analyze_mime(&mut spam_ctx).await; server.spam_filter_analyze_headers(&mut spam_ctx).await; server.spam_filter_analyze_rules(&mut spam_ctx).await; spam_ctx.result.tags.retain(|t| !t.starts_with("X_HDR_")); } "rbl" => { server.spam_filter_analyze_url(&mut spam_ctx).await; server.spam_filter_analyze_ip(&mut spam_ctx).await; server.spam_filter_analyze_domain(&mut spam_ctx).await; } "spamtrap" => { server.spam_filter_analyze_spam_trap(&mut spam_ctx).await; server.spam_filter_finalize(&mut spam_ctx).await; } "classifier" => { server.spam_filter_analyze_classify(&mut spam_ctx).await; match server.spam_filter_finalize(&mut spam_ctx).await { SpamFilterAction::Allow(r) => spam_ctx.result.tags.extend( r.headers .split_ascii_whitespace() .filter(|t| t.starts_with("PROB_")) .map(|t| t.to_string()), ), _ => unreachable!(), } } "pyzor" => { server.spam_filter_analyze_pyzor(&mut spam_ctx).await; } "llm" => { server.spam_filter_analyze_llm(&mut spam_ctx).await; } _ => panic!("Invalid test {test_name:?}"), } // Compare tags if spam_ctx.result.tags != expected_tags { for tag in &spam_ctx.result.tags { if !expected_tags.contains(tag) { println!("Unexpected tag: {tag:?}"); } } for tag in &expected_tags { if !spam_ctx.result.tags.contains(tag) { println!("Missing tag: {tag:?}"); } } panic!("Tags mismatch, expected {expected_tags:?}"); } else { println!("Tags matched: {expected_tags:?}"); } } } } async fn classifier_features(server: &Server, contents: String) { let mut num_tests = 0; for test in contents.split("") { let test = test.trim(); if test.is_empty() { continue; } let (input, expected) = test.split_once("").unwrap(); let input = input.trim(); let expected = expected.trim(); // Build features let message = MessageParser::new().parse(input).unwrap_or_default(); let mut ctx = server.spam_filter_init(SpamFilterInput::from_message(&message, 0).train_mode()); server.spam_filter_analyze_domain(&mut ctx).await; server.spam_filter_analyze_url(&mut ctx).await; let mut tokens = server .spam_build_tokens(&ctx) .await .0 .into_keys() .collect::>(); tokens.sort(); assert!(!tokens.is_empty(), "No tokens parsed for input: {}", input); let expected_tokens: Vec> = serde_json::from_str(expected).unwrap(); if tokens != expected_tokens { eprintln!("Input: {}", input); eprintln!("Expected Tokens: {}", expected); eprintln!( "Parsed Tokens: {}", serde_json::to_string_pretty(&tokens).unwrap() ); panic!("Tokens do not match"); } num_tests += 1; } assert_eq!(num_tests, 11, "Expected number of tests to run"); } fn html_tokens(contents: String) { let mut num_tests = 0; for test in contents.split("") { let test = test.trim(); if test.is_empty() { continue; } let (input, expected) = test.split_once("").unwrap(); let input = input.trim(); let expected = expected.trim(); let tokens = html_to_tokens(input); assert!(!tokens.is_empty(), "No tokens parsed for input: {}", input); let expected_tokens: Vec = serde_json::from_str(expected).unwrap(); assert_eq!(tokens, expected_tokens, "Input: {}", input); num_tests += 1; } assert_eq!(num_tests, 12, "Expected number of tests to run"); } trait ParseConfigValue: Sized { fn from_str(value: &str) -> Self; } impl ParseConfigValue for SpfResult { fn from_str(value: &str) -> Self { match value { "pass" => SpfResult::Pass, "fail" => SpfResult::Fail, "softfail" => SpfResult::SoftFail, "neutral" => SpfResult::Neutral, "none" => SpfResult::None, "temperror" => SpfResult::TempError, "permerror" => SpfResult::PermError, _ => panic!("Invalid SPF result"), } } } impl ParseConfigValue for IprevResult { fn from_str(value: &str) -> Self { match value { "pass" => IprevResult::Pass, "fail" => IprevResult::Fail(mail_auth::Error::NotAligned), "temperror" => IprevResult::TempError(mail_auth::Error::NotAligned), "permerror" => IprevResult::PermError(mail_auth::Error::NotAligned), "none" => IprevResult::None, _ => panic!("Invalid IPREV result"), } } } impl ParseConfigValue for DkimResult { fn from_str(value: &str) -> Self { match value { "pass" => DkimResult::Pass, "none" => DkimResult::None, "neutral" => DkimResult::Neutral(mail_auth::Error::NotAligned), "fail" => DkimResult::Fail(mail_auth::Error::NotAligned), "permerror" => DkimResult::PermError(mail_auth::Error::NotAligned), "temperror" => DkimResult::TempError(mail_auth::Error::NotAligned), _ => panic!("Invalid DKIM result"), } } } impl ParseConfigValue for DmarcResult { fn from_str(value: &str) -> Self { match value { "pass" => DmarcResult::Pass, "fail" => DmarcResult::Fail(mail_auth::Error::NotAligned), "temperror" => DmarcResult::TempError(mail_auth::Error::NotAligned), "permerror" => DmarcResult::PermError(mail_auth::Error::NotAligned), "none" => DmarcResult::None, _ => panic!("Invalid DMARC result"), } } } impl ParseConfigValue for Policy { fn from_str(value: &str) -> Self { match value { "reject" => Policy::Reject, "quarantine" => Policy::Quarantine, "none" => Policy::None, _ => panic!("Invalid DMARC policy"), } } } ================================================ FILE: tests/src/smtp/inbound/asn.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ #[cfg(test)] mod tests { use std::time::{Duration, Instant}; use common::{Core, Server, config::network::AsnGeoLookupConfig}; #[tokio::test] #[ignore] async fn lookup_asn_country_dns() { let mut core = Core::default(); core.network.asn_geo_lookup = AsnGeoLookupConfig::Dns { zone_ipv4: "origin.asn.cymru.com".to_string(), zone_ipv6: "origin6.asn.cymru.com".to_string(), separator: '|'.to_string(), index_asn: 0, index_asn_name: 3.into(), index_country: 2.into(), }; let server = Server { core: core.into(), inner: Default::default(), }; for (ip, asn, asn_name, country) in [ ("8.8.8.8", 15169, "arin", "US"), ("1.1.1.1", 13335, "apnic", "AU"), ("2a01:4f9:c011:b43c::1", 24940, "ripencc", "DE"), ("1.33.1.1", 2514, "apnic", "JP"), ] { let result = server.lookup_asn_country(ip.parse().unwrap()).await; println!("{ip}: {result:?}"); assert_eq!(result.asn.as_ref().map(|r| r.id), Some(asn)); assert_eq!( result.asn.as_ref().and_then(|r| r.name.as_deref()), Some(asn_name) ); assert_eq!(result.country.as_ref().map(|s| s.as_str()), Some(country)); } } #[tokio::test] #[ignore] async fn lookup_asn_country_http() { let mut core = Core::default(); core.network.asn_geo_lookup = AsnGeoLookupConfig::Resource { expires: Duration::from_secs(86400), timeout: Duration::from_secs(100), max_size: 100 * 1024 * 1024, headers: Default::default(), asn_resources: vec![ //url: "file:///Users/me/code/playground/asn-ipv4.csv".to_string(), //url: "file:///Users/me/code/playground/asn-ipv6.csv".to_string(), "https://cdn.jsdelivr.net/npm/@ip-location-db/asn/asn-ipv4.csv".to_string(), "https://cdn.jsdelivr.net/npm/@ip-location-db/asn/asn-ipv6.csv".to_string(), ], geo_resources: vec![ //url: "file:///Users/me/code/playground/geolite2-geo-whois-asn-country-ipv4.csv" // .to_string(), //url: "file:///Users/me/code/playground/geolite2-geo-whois-asn-country-ipv6.csv" // .to_string(), concat!( "https://cdn.jsdelivr.net/npm/@ip-location-db/geolite2-geo-whois-", "asn-country/geolite2-geo-whois-asn-country-ipv4.csv" ) .to_string(), concat!( "https://cdn.jsdelivr.net/npm/@ip-location-db/geolite2-geo-whois-", "asn-country/geolite2-geo-whois-asn-country-ipv6.csv" ) .to_string(), ], }; let server = Server { core: core.into(), inner: Default::default(), }; server.lookup_asn_country("8.8.8.8".parse().unwrap()).await; let time = Instant::now(); loop { tokio::time::sleep(Duration::from_millis(500)).await; if server.inner.data.asn_geo_data.lock.available_permits() > 0 { break; } } println!("Fetch took {:?}", time.elapsed()); for (ip, asn, asn_name, country) in [ ("8.8.8.8", 15169, "Google LLC", "US"), ("1.1.1.1", 13335, "Cloudflare, Inc.", "AU"), ("2a01:4f9:c011:b43c::1", 24940, "Hetzner Online GmbH", "FI"), ("1.33.1.1", 2514, "NTT PC Communications, Inc.", "JP"), ] { let result = server.lookup_asn_country(ip.parse().unwrap()).await; println!("{ip}: {result:?}"); assert_eq!(result.asn.as_ref().map(|r| r.id), Some(asn)); assert_eq!( result.asn.as_ref().and_then(|r| r.name.as_deref()), Some(asn_name) ); assert_eq!(result.country.as_ref().map(|s| s.as_str()), Some(country)); } } } ================================================ FILE: tests/src/smtp/inbound/auth.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::Core; use store::Stores; use utils::config::Config; use crate::{ AssertConfig, smtp::{ TempDir, TestSMTP, session::{TestSession, VerifyResponse}, }, }; use smtp::core::{Session, State}; const CONFIG: &str = r#" [storage] data = "rocksdb" lookup = "rocksdb" blob = "rocksdb" fts = "rocksdb" directory = "local" [store."rocksdb"] type = "rocksdb" path = "{TMP}/queue.db" [directory."local"] type = "memory" [[directory."local".principals]] name = "john" description = "John Doe" secret = "secret" email = ["john@example.org", "jdoe@example.org", "john.doe@example.org"] email-list = ["info@example.org"] member-of = ["sales"] [[directory."local".principals]] name = "jane" description = "Jane Doe" secret = "p4ssw0rd" email = "jane@example.org" email-list = ["info@example.org"] member-of = ["sales", "support"] [session.auth] require = [{if = "remote_ip = '10.0.0.1'", then = true}, {else = false}] mechanisms = [{if = "remote_ip = '10.0.0.1' && is_tls", then = "[plain, login]"}, {else = 0}] directory = [{if = "remote_ip = '10.0.0.1'", then = "'local'"}, {else = false}] must-match-sender = true [session.auth.errors] total = [{if = "remote_ip = '10.0.0.1'", then = 2}, {else = 3}] wait = "100ms" [session.extensions] future-release = [{if = '!is_empty(authenticated_as)', then = '1d'}, {else = false}] "#; #[tokio::test] async fn auth() { // Enable logging crate::enable_logging(); let tmp_dir = TempDir::new("smtp_auth_test", true); let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; config.assert_no_errors(); // EHLO should not advertise plain text auth without TLS let mut session = Session::test(TestSMTP::from_core(core).server); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.stream.tls = false; session .ehlo("mx.foobar.org") .await .assert_not_contains(" PLAIN") .assert_not_contains(" LOGIN"); // EHLO should advertise AUTH for 10.0.0.1 session.stream.tls = true; session .ehlo("mx.foobar.org") .await .assert_contains("AUTH ") .assert_contains(" PLAIN") .assert_contains(" LOGIN") .assert_not_contains("FUTURERELEASE"); // Invalid password should be rejected session .cmd("AUTH PLAIN AGpvaG4AY2hpbWljaGFuZ2Fz", "535 5.7.8") .await; // Session should be disconnected after second invalid auth attempt session .ingest(b"AUTH PLAIN AGpvaG4AY2hpbWljaGFuZ2Fz\r\n") .await .unwrap_err(); session.response().assert_code("455 4.3.0"); // Should not be able to send without authenticating session.state = State::default(); session.mail_from("bill@foobar.org", "503 5.5.1").await; // Successful PLAIN authentication session.data.auth_errors = 0; session .cmd("AUTH PLAIN AGpvaG4Ac2VjcmV0", "235 2.7.0") .await; // Users should be able to send emails only from their own email addresses session.mail_from("bill@foobar.org", "501 5.5.4").await; session.mail_from("john@example.org", "250").await; session.data.mail_from.take(); // Should not be able to authenticate twice session .cmd("AUTH PLAIN AGpvaG4Ac2VjcmV0", "503 5.5.1") .await; // FUTURERELEASE extension should be available after authenticating session .ehlo("mx.foobar.org") .await .assert_not_contains("AUTH ") .assert_not_contains(" PLAIN") .assert_not_contains(" LOGIN") .assert_contains("FUTURERELEASE 86400"); // Successful LOGIN authentication session.data.authenticated_as.take(); session.cmd("AUTH LOGIN", "334").await; session.cmd("amFuZQ==", "334").await; session.cmd("cDRzc3cwcmQ=", "235 2.7.0").await; // Login should not be advertised to 10.0.0.2 session.data.remote_ip_str = "10.0.0.2".into(); session.eval_session_params().await; session.stream.tls = true; session .ehlo("mx.foobar.org") .await .assert_not_contains("AUTH ") .assert_not_contains(" PLAIN") .assert_not_contains(" LOGIN"); session .cmd("AUTH PLAIN AGpvaG4Ac2VjcmV0", "503 5.5.1") .await; } ================================================ FILE: tests/src/smtp/inbound/basic.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::Core; use smtp::core::Session; use crate::smtp::{ TestSMTP, session::{TestSession, VerifyResponse}, }; #[tokio::test] async fn basic_commands() { // Enable logging crate::enable_logging(); let mut session = Session::test(TestSMTP::from_core(Core::default()).server); // STARTTLS should be available on clear text connections session.stream.tls = false; session .ehlo("mx.foobar.org") .await .assert_contains("STARTTLS"); assert!(!session.ingest(b"STARTTLS\r\n").await.unwrap()); session.response().assert_contains("220 2.0.0"); // STARTTLS should not be offered on TLS connections session.stream.tls = true; session .ehlo("mx.foobar.org") .await .assert_not_contains("STARTTLS"); session.cmd("STARTTLS", "504 5.7.4").await; // Test NOOP session.cmd("NOOP", "250").await; // Test RSET session.cmd("RSET", "250").await; // Test HELP session.cmd("HELP QUIT", "250").await; // Test LHLO on SMTP channel session.cmd("LHLO domain.org", "502").await; // Test QUIT session.ingest(b"QUIT\r\n").await.unwrap_err(); session.response().assert_code("221"); } ================================================ FILE: tests/src/smtp/inbound/data.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::Core; use store::Stores; use utils::config::Config; use crate::{ AssertConfig, smtp::{ TempDir, TestSMTP, inbound::TestMessage, session::{TestSession, VerifyResponse, load_test_message}, }, store::cleanup::store_assert_is_empty, }; use smtp::core::Session; const CONFIG: &str = r#" [storage] data = "rocksdb" lookup = "rocksdb" blob = "rocksdb" fts = "rocksdb" directory = "local" [store."rocksdb"] type = "rocksdb" path = "{TMP}/queue.db" [spam-filter] enable = false [directory."local"] type = "memory" [[directory."local".principals]] name = "john" description = "John Doe" secret = "secret" email = ["john@foobar.org", "jdoe@example.org", "john.doe@example.org"] [[directory."local".principals]] name = "jane" description = "Jane Doe" secret = "p4ssw0rd" email = "jane@domain.net" [[directory."local".principals]] name = "bill" description = "Bill Foobar" secret = "p4ssw0rd" email = "bill@foobar.org" [[directory."local".principals]] name = "mike" description = "Mike Foobar" secret = "p4ssw0rd" email = "mike@test.com" [session.rcpt] directory = "'local'" [session.data.limits] messages = [{if = "remote_ip = '10.0.0.1'", then = 1}, {else = 100}] received-headers = 3 [session.data.add-headers] received = [{if = "remote_ip = '10.0.0.3'", then = true}, {else = false}] received-spf = [{if = "remote_ip = '10.0.0.3'", then = true}, {else = false}] auth-results = [{if = "remote_ip = '10.0.0.3'", then = true}, {else = false}] message-id = [{if = "remote_ip = '10.0.0.3'", then = true}, {else = false}] date = [{if = "remote_ip = '10.0.0.3'", then = true}, {else = false}] return-path = [{if = "remote_ip = '10.0.0.3'", then = true}, {else = false}] [[queue.quota]] match = "sender = 'john@doe.org'" key = ['sender'] messages = 1 [[queue.quota]] match = "rcpt_domain = 'foobar.org'" key = ['rcpt_domain'] size = 450 enable = true [[queue.quota]] match = "rcpt = 'jane@domain.net'" key = ['rcpt'] size = 450 enable = true "#; #[tokio::test] async fn data() { // Enable logging crate::enable_logging(); // Create temp dir for queue let tmp_dir = TempDir::new("smtp_data_test", true); let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; config.assert_no_errors(); // Test queue message builder let test = TestSMTP::from_core(core); let mut qr = test.queue_receiver; let mut session = Session::test(test.server.clone()); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.test_builder().await; // Send DATA without RCPT session.ehlo("mx.doe.org").await; session.ingest(b"DATA\r\n").await.unwrap(); session.response().assert_code("503 5.5.1"); // Send broken message session .send_message("john@doe.org", &["bill@foobar.org"], "invalid", "550 5.7.7") .await; // Naive Loop detection session .send_message( "john@doe.org", &["bill@foobar.org"], "test:loop", "450 4.4.6", ) .await; // No headers should be added to messages from 10.0.0.1 session .send_message("john@test.org", &["mike@test.com"], "test:no_msgid", "250") .await; assert_eq!( qr.expect_message().await.read_message(&qr).await, load_test_message("no_msgid", "messages") ); // Maximum one message per session is allowed for 10.0.0.1 session.mail_from("john@doe.org", "250").await; session.rcpt_to("bill@foobar.org", "250").await; session.ingest(b"DATA\r\n").await.unwrap(); session.response().assert_code("452 4.4.5"); session.rset().await; // Headers should be added to messages from 10.0.0.3 session.data.remote_ip_str = "10.0.0.3".into(); session.eval_session_params().await; session .send_message("bill@doe.org", &["mike@test.com"], "test:no_msgid", "250") .await; qr.expect_message() .await .read_lines(&qr) .await .assert_contains("From: ") .assert_contains("To: ") .assert_contains("Subject: ") .assert_contains("Date: ") .assert_contains("Message-ID: ") .assert_contains("Return-Path: ") .assert_contains("Received: ") .assert_contains("Authentication-Results: ") .assert_contains("Received-SPF: "); // Only one message is allowed in the queue from john@doe.org session.data.remote_ip_str = "10.0.0.2".into(); session.eval_session_params().await; session .send_message("john@doe.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; session .send_message( "john@doe.org", &["bill@foobar.org"], "test:no_dkim", "452 4.3.1", ) .await; // Release quota qr.clear_queue(&test.server).await; // Only 1500 bytes are allowed in the queue to domain foobar.org session .send_message( "jane@foobar.org", &["bill@foobar.org"], "test:no_dkim", "250", ) .await; session .send_message( "jane@foobar.org", &["bill@foobar.org"], "test:no_dkim", "452 4.3.1", ) .await; // Only 1500 bytes are allowed in the queue to recipient jane@domain.net session .send_message( "jane@foobar.org", &["jane@domain.net"], "test:no_dkim", "250", ) .await; session .send_message( "jane@foobar.org", &["jane@domain.net"], "test:no_dkim", "452 4.3.1", ) .await; // Make sure store is empty qr.clear_queue(&test.server).await; store_assert_is_empty(test.server.store(), test.server.blob_store().clone(), false).await; } ================================================ FILE: tests/src/smtp/inbound/dmarc.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::{Duration, Instant}; use common::{Core, config::smtp::report::AggregateFrequency}; use mail_auth::{ common::{parse::TxtRecordParser, verify::DomainKey}, dkim::DomainKeyReport, dmarc::Dmarc, report::DmarcResult, spf::Spf, }; use store::Stores; use utils::config::Config; use crate::smtp::{ DnsCache, TempDir, TestSMTP, inbound::{TestMessage, TestReportingEvent, sign::SIGNATURES}, session::{TestSession, VerifyResponse}, }; use smtp::core::Session; const CONFIG: &str = r#" [storage] data = "rocksdb" lookup = "rocksdb" blob = "rocksdb" fts = "rocksdb" [store."rocksdb"] type = "rocksdb" path = "{TMP}/queue.db" [directory."local"] type = "memory" [[directory."local".principals]] name = "john" description = "John Doe" secret = "secret" email = ["jdoe@example.com"] [session.rcpt] directory = "'local'" [session.data.add-headers] received = true received-spf = true auth-results = true message-id = true date = true return-path = false [report.dkim] send = "[1, 1s]" sign = "['rsa']" [report.spf] send = "[1, 1s]" sign = "['rsa']" [report.dmarc] send = "[1, 1s]" sign = "['rsa']" [report.dmarc.aggregate] send = "daily" [auth.spf.verify] ehlo = [{if = "remote_ip = '10.0.0.2'", then = 'strict'}, { else = 'relaxed' }] mail-from = [{if = "remote_ip = '10.0.0.2'", then = 'strict'}, { else = 'relaxed' }] [auth.dmarc] verify = "strict" [auth.arc] verify = "strict" [auth.dkim] verify = [{if = "sender_domain = 'test.net'", then = 'relaxed'}, { else = 'strict' }] "#; #[tokio::test] async fn dmarc() { // Enable logging crate::enable_logging(); let tmp_dir = TempDir::new("smtp_dmarc_test", true); let mut config = Config::new(tmp_dir.update_config(CONFIG.to_string() + SIGNATURES)).unwrap(); let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; let test = TestSMTP::from_core(core); // Add SPF, DKIM and DMARC records test.server.txt_add( "mx.example.com", Spf::parse(b"v=spf1 ip4:10.0.0.1 ip4:10.0.0.2 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); test.server.txt_add( "example.com", Spf::parse(b"v=spf1 ip4:10.0.0.1 -all ra=spf-failures rr=e:f:s:n").unwrap(), Instant::now() + Duration::from_secs(5), ); test.server.txt_add( "foobar.com", Spf::parse(b"v=spf1 ip4:10.0.0.1 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); test.server.txt_add( "ed._domainkey.example.com", DomainKey::parse( concat!( "v=DKIM1; k=ed25519; ", "p=11qYAYKxCrfVS/7TyWQHOg7hcvPapiMlrwIaaPcHURo=" ) .as_bytes(), ) .unwrap(), Instant::now() + Duration::from_secs(5), ); test.server.txt_add( "default._domainkey.example.com", DomainKey::parse( concat!( "v=DKIM1; t=s; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQ", "KBgQDwIRP/UC3SBsEmGqZ9ZJW3/DkMoGeLnQg1fWn7/zYt", "IxN2SnFCjxOCKG9v3b4jYfcTNh5ijSsq631uBItLa7od+v", "/RtdC2UzJ1lWT947qR+Rcac2gbto/NMqJ0fzfVjH4OuKhi", "tdY9tf6mcwGjaNBcWToIMmPSPDdQPNUYckcQ2QIDAQAB", ) .as_bytes(), ) .unwrap(), Instant::now() + Duration::from_secs(5), ); test.server.txt_add( "_report._domainkey.example.com", DomainKeyReport::parse(b"ra=dkim-failures; rp=100; rr=d:o:p:s:u:v:x;").unwrap(), Instant::now() + Duration::from_secs(5), ); test.server.txt_add( "_dmarc.example.com", Dmarc::parse( concat!( "v=DMARC1; p=reject; sp=quarantine; np=None; aspf=s; adkim=s; fo=1;", "rua=mailto:dmarc-feedback@example.com;", "ruf=mailto:dmarc-failures@example.com" ) .as_bytes(), ) .unwrap(), Instant::now() + Duration::from_secs(5), ); // SPF must pass let mut rr = test.report_receiver; let mut qr = test.queue_receiver; let mut session = Session::test(test.server.clone()); session.data.remote_ip_str = "10.0.0.2".into(); session.data.remote_ip = session.data.remote_ip_str.parse().unwrap(); session.eval_session_params().await; session.ehlo("mx.example.com").await; session.mail_from("bill@example.com", "550 5.7.23").await; // Expect SPF auth failure report let message = qr.expect_message().await; assert_eq!( message.message.recipients.last().unwrap().address(), "spf-failures@example.com" ); message .read_lines(&qr) .await .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") .assert_contains("To: spf-failures@example.com") .assert_contains("Feedback-Type: auth-failure") .assert_contains("Auth-Failure: spf"); // Second DKIM failure report should be rate limited session.mail_from("bill@example.com", "550 5.7.23").await; qr.assert_no_events(); // Invalid DKIM signatures should be rejected session.data.remote_ip_str = "10.0.0.1".into(); session.data.remote_ip = session.data.remote_ip_str.parse().unwrap(); session.eval_session_params().await; session .send_message( "bill@example.com", &["jdoe@example.com"], "test:invalid_dkim", "550 5.7.20", ) .await; // Expect DKIM auth failure report let message = qr.expect_message().await; assert_eq!( message.message.recipients.last().unwrap().address(), "dkim-failures@example.com" ); message .read_lines(&qr) .await .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") .assert_contains("To: dkim-failures@example.com") .assert_contains("Feedback-Type: auth-failure") .assert_contains("Auth-Failure: bodyhash"); // Second DKIM failure report should be rate limited session .send_message( "bill@example.com", &["jdoe@example.com"], "test:invalid_dkim", "550 5.7.20", ) .await; qr.assert_no_events(); // Invalid ARC should be rejected session .send_message( "bill@example.com", &["jdoe@example.com"], "test:invalid_arc", "550 5.7.29", ) .await; qr.assert_no_events(); // Unaligned DMARC should be rejected test.server.txt_add( "test.net", Spf::parse(b"v=spf1 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); session .send_message( "joe@test.net", &["jdoe@example.com"], "test:invalid_dkim", "550 5.7.1", ) .await; // Expect DMARC auth failure report let message = qr.expect_message().await; assert_eq!( message.message.recipients.last().unwrap().address(), "dmarc-failures@example.com" ); message .read_lines(&qr) .await .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") .assert_contains("To: dmarc-failures@example.com") .assert_contains("Feedback-Type: auth-failure") .assert_contains("Auth-Failure: dmarc") .assert_contains("dmarc=3Dnone"); // Expect DMARC aggregate report let report = rr.read_report().await.unwrap_dmarc(); assert_eq!(report.domain, "example.com"); assert_eq!(report.interval, AggregateFrequency::Daily); assert_eq!(report.dmarc_record.rua().len(), 1); assert_eq!(report.report_record.dmarc_spf_result(), DmarcResult::Fail); // Second DMARC failure report should be rate limited session .send_message( "joe@test.net", &["jdoe@example.com"], "test:invalid_dkim", "550 5.7.1", ) .await; qr.assert_no_events(); // Messages passing DMARC should be accepted session .send_message( "bill@example.com", &["jdoe@example.com"], "test:dkim", "250", ) .await; qr.expect_message() .await .read_lines(&qr) .await .assert_contains("dkim=pass") .assert_contains("spf=pass") .assert_contains("dmarc=pass") .assert_contains("Received-SPF: pass"); } ================================================ FILE: tests/src/smtp/inbound/ehlo.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::{Duration, Instant}; use common::Core; use mail_auth::{SpfResult, common::parse::TxtRecordParser, spf::Spf}; use smtp::core::Session; use utils::config::Config; use crate::smtp::{ DnsCache, TestSMTP, session::{TestSession, VerifyResponse}, }; const CONFIG: &str = r#" [session.data.limits] size = [{if = "remote_ip = '10.0.0.1'", then = 1024}, {else = 2048}] [session.extensions] future-release = [{if = "remote_ip = '10.0.0.1'", then = '1h'}, {else = false}] mt-priority = [{if = "remote_ip = '10.0.0.1'", then = 'nsep'}, {else = false}] [session.ehlo] reject-non-fqdn = "starts_with(remote_ip, '10.0.0.')" [auth.spf.verify] ehlo = [{if = "remote_ip = '10.0.0.2'", then = 'strict'}, {else = 'relaxed'}] "#; #[tokio::test] async fn ehlo() { // Enable logging crate::enable_logging(); let mut config = Config::new(CONFIG).unwrap(); let core = Core::parse(&mut config, Default::default(), Default::default()).await; let server = TestSMTP::from_core(core).server; server.txt_add( "mx1.foobar.org", Spf::parse(b"v=spf1 ip4:10.0.0.1 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); server.txt_add( "mx2.foobar.org", Spf::parse(b"v=spf1 ip4:10.0.0.2 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); // Reject non-FQDN domains let mut session = Session::test(server); session.data.remote_ip_str = "10.0.0.1".into(); session.data.remote_ip = session.data.remote_ip_str.parse().unwrap(); session.stream.tls = false; session.eval_session_params().await; session.cmd("EHLO domain", "550 5.5.0").await; // EHLO capabilities evaluation session .cmd("EHLO mx1.foobar.org", "250") .await .assert_contains("SIZE 1024") .assert_contains("MT-PRIORITY NSEP") .assert_contains("FUTURERELEASE 3600") .assert_contains("STARTTLS"); // SPF should be a Pass for 10.0.0.1 assert_eq!( session.data.spf_ehlo.as_ref().unwrap().result(), SpfResult::Pass ); // Test SPF strict mode session.data.helo_domain = "".into(); session.data.remote_ip_str = "10.0.0.2".into(); session.data.remote_ip = session.data.remote_ip_str.parse().unwrap(); session.stream.tls = true; session.eval_session_params().await; session.ingest(b"EHLO mx1.foobar.org\r\n").await.unwrap(); session.response().assert_code("550 5.7.23"); // EHLO capabilities evaluation session.ingest(b"EHLO mx2.foobar.org\r\n").await.unwrap(); assert_eq!( session.data.spf_ehlo.as_ref().unwrap().result(), SpfResult::Pass ); session .response() .assert_code("250") .assert_contains("SIZE 2048") .assert_not_contains("MT-PRIORITY") .assert_not_contains("FUTURERELEASE") .assert_not_contains("STARTTLS"); } ================================================ FILE: tests/src/smtp/inbound/limits.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::{Duration, Instant}; use common::Core; use tokio::sync::watch; use smtp::core::Session; use utils::config::Config; use crate::smtp::{ TestSMTP, session::{TestSession, VerifyResponse}, }; const CONFIG: &str = r#" [session] transfer-limit = [{if = "remote_ip = '10.0.0.1'", then = 10}, {else = 1024}] timeout = [{if = "remote_ip = '10.0.0.2'", then = '500ms'}, {else = '30m'}] duration = [{if = "remote_ip = '10.0.0.3'", then = '500ms'}, {else = '60m'}] "#; #[tokio::test] async fn limits() { // Enable logging crate::enable_logging(); let mut config = Config::new(CONFIG).unwrap(); let core = Core::parse(&mut config, Default::default(), Default::default()).await; let (_tx, rx) = watch::channel(true); // Exceed max line length let mut session = Session::test_with_shutdown(TestSMTP::from_core(core).server, rx); session.data.remote_ip_str = "10.0.0.1".into(); let mut buf = vec![b'A'; 4097]; session.ingest(&buf).await.unwrap(); session.ingest(b"\r\n").await.unwrap(); session.response().assert_code("554 5.3.4"); // Invalid command buf.extend_from_slice(b"\r\n"); session.ingest(&buf).await.unwrap(); session.response().assert_code("500 5.5.1"); // Exceed transfer quota session.eval_session_params().await; session.write_rx("MAIL FROM:\r\n"); session.handle_conn().await; session.response().assert_code("452 4.7.28"); // Loitering session.data.remote_ip_str = "10.0.0.3".into(); session.data.valid_until = Instant::now(); session.eval_session_params().await; tokio::time::sleep(Duration::from_millis(600)).await; session.write_rx("MAIL FROM:\r\n"); session.handle_conn().await; session.response().assert_code("421 4.3.2"); // Timeout session.data.remote_ip_str = "10.0.0.2".into(); session.data.valid_until = Instant::now(); session.eval_session_params().await; session.write_rx("MAIL FROM:\r\n"); session.handle_conn().await; session.response().assert_code("221 2.0.0"); } ================================================ FILE: tests/src/smtp/inbound/mail.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::{Duration, Instant, SystemTime}; use common::Core; use mail_auth::{IprevResult, SpfResult, common::parse::TxtRecordParser, spf::Spf}; use smtp_proto::{MAIL_BY_NOTIFY, MAIL_BY_RETURN, MAIL_REQUIRETLS}; use smtp::core::Session; use store::Stores; use utils::config::Config; use crate::smtp::{ DnsCache, TempDir, TestSMTP, session::{TestSession, VerifyResponse}, }; const CONFIG: &str = r#" [storage] data = "rocksdb" lookup = "rocksdb" blob = "rocksdb" fts = "rocksdb" [store."rocksdb"] type = "rocksdb" path = "{TMP}/data.db" [session.ehlo] require = true [auth.spf.verify] ehlo = 'relaxed' mail-from = [{if = "remote_ip = '10.0.0.2'", then = 'strict'}, {else = 'relaxed'}] [auth.iprev] verify = [{if = "remote_ip = '10.0.0.2'", then = 'strict'}, {else = 'relaxed'}] [session.extensions] future-release = [{if = "remote_ip = '10.0.0.2'", then = '1d'}, {else = false}] deliver-by = [{if = "remote_ip = '10.0.0.2'", then = '1d'}, {else = false}] requiretls = [{if = "remote_ip = '10.0.0.2'", then = true}, {else = false}] mt-priority = [{if = "remote_ip = '10.0.0.2'", then = 'nsep'}, {else = false}] [session.mail] is-allowed = "sender_domain != 'blocked.com'" [session.data.limits] size = [{if = "remote_ip = '10.0.0.2'", then = 2048}, {else = 1024}] [[queue.limiter.inbound]] match = "remote_ip = '10.0.0.1'" key = 'sender' rate = '2/1s' enable = true "#; #[tokio::test] async fn mail() { // Enable logging crate::enable_logging(); let tmp_dir = TempDir::new("smtp_mail_test", true); let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; let server = TestSMTP::from_core(core).server; server.txt_add( "foobar.org", Spf::parse(b"v=spf1 ip4:10.0.0.1 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); server.txt_add( "mx1.foobar.org", Spf::parse(b"v=spf1 ip4:10.0.0.1 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); server.ptr_add( "10.0.0.1".parse().unwrap(), vec!["mx1.foobar.org.".to_string()], Instant::now() + Duration::from_secs(5), ); server.ipv4_add( "mx1.foobar.org.", vec!["10.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(5), ); server.ptr_add( "10.0.0.2".parse().unwrap(), vec!["mx2.foobar.org.".to_string()], Instant::now() + Duration::from_secs(5), ); // Be rude and do not say EHLO let mut session = Session::test(server.clone()); session.data.remote_ip_str = "10.0.0.1".into(); session.data.remote_ip = session.data.remote_ip_str.parse().unwrap(); session.eval_session_params().await; session .ingest(b"MAIL FROM:\r\n") .await .unwrap(); session.response().assert_code("503 5.5.1"); // Test sender not allowed session.ingest(b"EHLO mx1.foobar.org\r\n").await.unwrap(); session.response().assert_code("250"); session .ingest(b"MAIL FROM:\r\n") .await .unwrap(); session.response().assert_code("550 5.7.1"); // Both IPREV and SPF should pass session .ingest(b"MAIL FROM:\r\n") .await .unwrap(); session.response().assert_code("250"); assert_eq!( session.data.spf_ehlo.as_ref().unwrap().result(), SpfResult::Pass ); assert_eq!( session.data.spf_mail_from.as_ref().unwrap().result(), SpfResult::Pass ); assert_eq!( session.data.iprev.as_ref().unwrap().result(), &IprevResult::Pass ); // Multiple MAIL FROMs should not be allowed session .ingest(b"MAIL FROM:\r\n") .await .unwrap(); session.response().assert_code("503 5.5.1"); // Test rate limit for n in 0..2 { session.rset().await; session .ingest(b"MAIL FROM:\r\n") .await .unwrap(); session .response() .assert_code(if n == 0 { "250" } else { "452 4.4.5" }); } // Test disabled extensions for param in [ "HOLDFOR=123", "HOLDUNTIL=49374347", "MT-PRIORITY=3", "BY=120;R", "REQUIRETLS", ] { session .ingest(format!("MAIL FROM: {param}\r\n").as_bytes()) .await .unwrap(); session.response().assert_code("501 5.5.4"); } // Test size with a large value session .ingest(b"MAIL FROM: SIZE=1512\r\n") .await .unwrap(); session.response().assert_code("552 5.3.4"); // Test strict IPREV session.data.remote_ip_str = "10.0.0.2".into(); session.data.remote_ip = session.data.remote_ip_str.parse().unwrap(); session.data.iprev = None; session.eval_session_params().await; session .ingest(b"MAIL FROM:\r\n") .await .unwrap(); session.response().assert_code("550 5.7.25"); session.data.iprev = None; server.ipv4_add( "mx2.foobar.org.", vec!["10.0.0.2".parse().unwrap()], Instant::now() + Duration::from_secs(5), ); // Test strict SPF session .ingest(b"MAIL FROM:\r\n") .await .unwrap(); session.response().assert_code("550 5.7.23"); server.txt_add( "foobar.org", Spf::parse(b"v=spf1 ip4:10.0.0.1 ip4:10.0.0.2 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); session .ingest(b"MAIL FROM:\r\n") .await .unwrap(); session.response().assert_code("250"); let mail_from = session.data.mail_from.as_ref().unwrap(); assert_eq!(mail_from.domain, "foobar.org"); assert_eq!(mail_from.address, "Jane@FooBar.org"); assert_eq!(mail_from.address_lcase, "jane@foobar.org"); session.rset().await; // Test SIZE extension session .ingest(b"MAIL FROM: SIZE=1023\r\n") .await .unwrap(); session.response().assert_code("250"); session.rset().await; // Test MT-PRIORITY extension session .ingest(b"MAIL FROM: MT-PRIORITY=-3\r\n") .await .unwrap(); session.response().assert_code("250"); assert_eq!(session.data.priority, -3); session.rset().await; // Test REQUIRETLS extension session .ingest(b"MAIL FROM: REQUIRETLS\r\n") .await .unwrap(); session.response().assert_code("250"); assert!((session.data.mail_from.as_ref().unwrap().flags & MAIL_REQUIRETLS) != 0); session.rset().await; // Test DELIVERBY extension with by-mode=R session .ingest(b"MAIL FROM: BY=120;R\r\n") .await .unwrap(); session.response().assert_code("250"); assert!((session.data.mail_from.as_ref().unwrap().flags & MAIL_BY_RETURN) != 0); assert_eq!(session.data.delivery_by, 120); session.rset().await; // Test DELIVERBY extension with by-mode=N session .ingest(b"MAIL FROM: BY=-456;N\r\n") .await .unwrap(); session.response().assert_code("250"); assert!((session.data.mail_from.as_ref().unwrap().flags & MAIL_BY_NOTIFY) != 0); assert_eq!(session.data.delivery_by, -456); session.rset().await; // Test DELIVERBY extension with invalid by-mode=R session .ingest(b"MAIL FROM: BY=-1;R\r\n") .await .unwrap(); session.response().assert_code("501 5.5.4"); session.rset().await; session .ingest(b"MAIL FROM: BY=99999;R\r\n") .await .unwrap(); session.response().assert_code("501 5.5.4"); session.rset().await; // Test FUTURERELEASE extension with HOLDFOR session .ingest(b"MAIL FROM: HOLDFOR=1234\r\n") .await .unwrap(); session.response().assert_code("250"); assert_eq!(session.data.future_release, 1234); session.rset().await; // Test FUTURERELEASE extension with invalid HOLDFOR falue session .ingest(b"MAIL FROM: HOLDFOR=99999\r\n") .await .unwrap(); session.response().assert_code("501 5.5.4"); session.rset().await; // Test FUTURERELEASE extension with HOLDUNTIL let now = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()); session .ingest(format!("MAIL FROM: HOLDUNTIL={}\r\n", now + 10).as_bytes()) .await .unwrap(); session.response().assert_code("250"); assert_eq!(session.data.future_release, 10); session.rset().await; // Test FUTURERELEASE extension with invalid HOLDUNTIL value session .ingest(format!("MAIL FROM: HOLDUNTIL={}\r\n", now + 99999).as_bytes()) .await .unwrap(); session.response().assert_code("501 5.5.4"); session.rset().await; } ================================================ FILE: tests/src/smtp/inbound/milter.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{fs, net::SocketAddr, path::PathBuf, sync::Arc, time::Duration}; use ahash::AHashSet; use common::{ Core, config::smtp::session::{Milter, MilterVersion, Stage}, expr::if_block::IfBlock, manager::webadmin::Resource, }; use http_proto::{ToHttpResponse, request::fetch_body}; use hyper::{body, server::conn::http1, service::service_fn}; use hyper_util::rt::TokioIo; use mail_auth::AuthenticatedMessage; use mail_parser::MessageParser; use serde::Deserialize; use smtp::{ core::{Session, SessionData}, inbound::{ hooks::{self, Request, SmtpResponse}, milter::{ Action, Command, Macros, MilterClient, Modification, Options, Response, receiver::{FrameResult, Receiver}, }, }, }; use store::Stores; use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, net::{TcpListener, TcpStream}, sync::watch, }; use utils::config::Config; use crate::smtp::{ TempDir, TestSMTP, inbound::TestMessage, session::{TestSession, VerifyResponse, load_test_message}, }; #[derive(Debug, Deserialize)] struct HeaderTest { modifications: Vec, result: String, } const CONFIG_MILTER: &str = r#" [storage] data = "rocksdb" lookup = "rocksdb" blob = "rocksdb" fts = "rocksdb" [store."rocksdb"] type = "rocksdb" path = "{TMP}/queue.db" [session.rcpt] relay = true [[session.milter]] hostname = "127.0.0.1" port = 9332 #port = 11332 #port = 7357 enable = true options.version = 6 tls = false stages = ["data"] "#; const CONFIG_JMILTER: &str = r#" [storage] data = "rocksdb" lookup = "rocksdb" blob = "rocksdb" fts = "rocksdb" [store."rocksdb"] type = "rocksdb" path = "{TMP}/queue.db" [session.rcpt] relay = true [[session.hook]] url = "http://127.0.0.1:9333" enable = true stages = ["data"] "#; #[tokio::test] async fn milter_session() { // Enable logging crate::enable_logging(); // Configure tests let tmp_dir = TempDir::new("smtp_milter_test", true); let mut config = Config::new(tmp_dir.update_config(CONFIG_MILTER)).unwrap(); let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; let _rx = spawn_mock_milter_server(); tokio::time::sleep(Duration::from_millis(100)).await; // Build session let test = TestSMTP::from_core(core); let mut qr = test.queue_receiver; let mut session = Session::test(test.server); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.doe.org").await; // Test reject session .send_message( "reject@doe.org", &["bill@foobar.org"], "test:no_dkim", "503 5.5.3", ) .await; qr.assert_no_events(); // Test discard session .send_message( "discard@doe.org", &["bill@foobar.org"], "test:no_dkim", "250 2.0.0", ) .await; qr.assert_no_events(); // Test temp fail session .send_message( "temp_fail@doe.org", &["bill@foobar.org"], "test:no_dkim", "451 4.3.5", ) .await; qr.assert_no_events(); // Test shutdown session .send_message( "shutdown@doe.org", &["bill@foobar.org"], "test:no_dkim", "421 4.3.0", ) .await; qr.assert_no_events(); // Test reply code session .send_message( "reply_code@doe.org", &["bill@foobar.org"], "test:no_dkim", "321", ) .await; qr.assert_no_events(); // Test accept with header addition session .send_message( "0@doe.org", &["bill@foobar.org"], "test:no_dkim", "250 2.0.0", ) .await; qr.expect_message() .await .read_lines(&qr) .await .assert_contains("X-Hello: World") .assert_contains("Subject: Is dinner ready?") .assert_contains("Are you hungry yet?"); // Test accept with header replacement session .send_message( "3@doe.org", &["bill@foobar.org"], "test:no_dkim", "250 2.0.0", ) .await; qr.expect_message() .await .read_lines(&qr) .await .assert_contains("Subject: [SPAM] Saying Hello") .assert_count("References: ", 1) .assert_contains("Are you hungry yet?"); // Test accept with body replacement session .send_message( "2@doe.org", &["bill@foobar.org"], "test:no_dkim", "250 2.0.0", ) .await; qr.expect_message() .await .read_lines(&qr) .await .assert_contains("X-Spam: Yes") .assert_contains("123456"); } #[tokio::test] async fn mta_hook_session() { // Enable logging /*let disable = "true"; tracing::subscriber::set_global_default( tracing_subscriber::FmtSubscriber::builder() .with_max_level(tracing::Level::TRACE) .finish(), ) .unwrap();*/ // Configure tests let tmp_dir = TempDir::new("smtp_mta_hook_test", true); let mut config = Config::new(tmp_dir.update_config(CONFIG_JMILTER)).unwrap(); let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; let _rx = spawn_mock_mta_hook_server(); tokio::time::sleep(Duration::from_millis(100)).await; // Build session let test = TestSMTP::from_core(core); let mut qr = test.queue_receiver; let mut session = Session::test(test.server); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.doe.org").await; // Test reject session .send_message( "reject@doe.org", &["bill@foobar.org"], "test:no_dkim", "503 5.5.3", ) .await; qr.assert_no_events(); // Test discard session .send_message( "discard@doe.org", &["bill@foobar.org"], "test:no_dkim", "250 2.0.0", ) .await; qr.assert_no_events(); // Test temp fail session .send_message( "temp_fail@doe.org", &["bill@foobar.org"], "test:no_dkim", "451 4.3.5", ) .await; qr.assert_no_events(); // Test shutdown session .send_message( "shutdown@doe.org", &["bill@foobar.org"], "test:no_dkim", "421 4.3.0", ) .await; qr.assert_no_events(); // Test reply code session .send_message( "reply_code@doe.org", &["bill@foobar.org"], "test:no_dkim", "321", ) .await; qr.assert_no_events(); // Test accept with header addition session .send_message( "0@doe.org", &["bill@foobar.org"], "test:no_dkim", "250 2.0.0", ) .await; qr.expect_message() .await .read_lines(&qr) .await .assert_contains("X-Hello: World") .assert_contains("Subject: Is dinner ready?") .assert_contains("Are you hungry yet?"); // Test accept with header replacement session .send_message( "3@doe.org", &["bill@foobar.org"], "test:no_dkim", "250 2.0.0", ) .await; qr.expect_message() .await .read_lines(&qr) .await .assert_contains("Subject: [SPAM] Saying Hello") .assert_count("References: ", 1) .assert_contains("Are you hungry yet?"); // Test accept with body replacement session .send_message( "2@doe.org", &["bill@foobar.org"], "test:no_dkim", "250 2.0.0", ) .await; qr.expect_message() .await .read_lines(&qr) .await .assert_contains("X-Spam: Yes") .assert_contains("123456"); } #[test] fn milter_address_modifications() { let test_message = fs::read_to_string( PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("resources") .join("smtp") .join("milter") .join("message.eml"), ) .unwrap(); let parsed_test_message = AuthenticatedMessage::parse(test_message.as_bytes()).unwrap(); let mut data = SessionData::new( "127.0.0.1".parse().unwrap(), 0, "127.0.0.1".parse().unwrap(), 0, Default::default(), 0, ); // ChangeFrom assert!( data.apply_milter_modifications( vec![Modification::ChangeFrom { sender: "<>".into(), args: "".into(), }], &parsed_test_message ) .is_none() ); let addr = data.mail_from.as_ref().unwrap(); assert_eq!(addr.address_lcase, ""); assert_eq!(addr.dsn_info, None); assert_eq!(addr.flags, 0); // ChangeFrom with parameters assert!( data.apply_milter_modifications( vec![Modification::ChangeFrom { sender: "john@example.org".into(), args: "REQUIRETLS ENVID=abc123".into(), //"NOTIFY=SUCCESS,FAILURE ENVID=abc123\n".into() }], &parsed_test_message ) .is_none() ); let addr = data.mail_from.as_ref().unwrap(); assert_eq!(addr.address_lcase, "john@example.org"); assert_ne!(addr.flags, 0); assert_eq!(addr.dsn_info, Some("abc123".into())); // Add recipients assert!( data.apply_milter_modifications( vec![ Modification::AddRcpt { recipient: "bill@example.org".into(), args: "".into(), }, Modification::AddRcpt { recipient: "jane@foobar.org".into(), args: "NOTIFY=SUCCESS,FAILURE ORCPT=rfc822;Jane.Doe@Foobar.org".into(), }, Modification::AddRcpt { recipient: "".into(), args: "".into(), }, Modification::AddRcpt { recipient: "<>".into(), args: "".into(), }, ], &parsed_test_message ) .is_none() ); assert_eq!(data.rcpt_to.len(), 2); let addr = data.rcpt_to.first().unwrap(); assert_eq!(addr.address_lcase, "bill@example.org"); assert_eq!(addr.dsn_info, None); assert_eq!(addr.flags, 0); let addr = data.rcpt_to.last().unwrap(); assert_eq!(addr.address_lcase, "jane@foobar.org"); assert_ne!(addr.flags, 0); assert_eq!(addr.dsn_info, Some("Jane.Doe@Foobar.org".into())); // Remove recipients assert!( data.apply_milter_modifications( vec![ Modification::DeleteRcpt { recipient: "bill@example.org".into(), }, Modification::DeleteRcpt { recipient: "<>".into(), }, ], &parsed_test_message ) .is_none() ); assert_eq!(data.rcpt_to.len(), 1); let addr = data.rcpt_to.last().unwrap(); assert_eq!(addr.address_lcase, "jane@foobar.org"); assert_ne!(addr.flags, 0); assert_eq!(addr.dsn_info, Some("Jane.Doe@Foobar.org".into())); } #[test] fn milter_message_modifications() { // Read test message let milter_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("resources") .join("smtp") .join("milter"); let test_message = fs::read_to_string(milter_path.join("message.eml")).unwrap(); let tests = serde_json::from_str::>( &fs::read_to_string(milter_path.join("message.json")).unwrap(), ) .unwrap(); let parsed_test_message = AuthenticatedMessage::parse(test_message.as_bytes()).unwrap(); let mut session_data = SessionData::new( "127.0.0.1".parse().unwrap(), 0, "127.0.0.1".parse().unwrap(), 0, Default::default(), 0, ); for test in tests { assert_eq!( test.result, String::from_utf8( session_data .apply_milter_modifications(test.modifications, &parsed_test_message) .unwrap() ) .unwrap() ) } } #[test] fn milter_frame_receiver() { let mut stream = Vec::new(); for i in 0u32..100u32 { stream.extend_from_slice((i + 1).to_be_bytes().as_ref()); stream.push(i as u8); for v in 0..i { stream.push(v as u8); } } for chunk_size in [stream.len(), 1, 2, 3, 4, 10, 20, 30, 40, 100, 200, 300, 400] { let mut receiver = Receiver::with_max_frame_len(100); let mut frame_num = 0; 'outer: for chunk in stream.chunks(chunk_size) { loop { match receiver.read_frame(chunk) { FrameResult::Frame(bytes) => { /*println!( "frame {frame_num}, chunk: {chunk_size}, {}", if matches!(bytes, std::borrow::Cow::Borrowed(_)) { "borrowed" } else { "owned" } );*/ assert_eq!(*bytes.first().unwrap(), frame_num); assert_eq!(bytes.len(), frame_num as usize + 1); frame_num += 1; } FrameResult::Incomplete => continue 'outer, FrameResult::TooLarge(size) => { panic!("Frame too large: {size}") } } } } assert_eq!(frame_num, 100, "chunk_size: {}", chunk_size); } } #[tokio::test] #[ignore] async fn milter_client_test() { //const PORT : u16 = 11332; const PORT: u16 = 7357; let mut client = MilterClient::connect( &Milter { enable: IfBlock::empty(""), id: Arc::new("test".into()), addrs: vec![SocketAddr::from(([127, 0, 0, 1], PORT))], hostname: "localhost".into(), port: PORT, timeout_connect: Duration::from_secs(10), timeout_command: Duration::from_secs(30), timeout_data: Duration::from_secs(30), tls: false, tls_allow_invalid_certs: false, tempfail_on_error: false, max_frame_len: 5000000, protocol_version: MilterVersion::V6, flags_actions: None, flags_protocol: None, run_on_stage: AHashSet::from([Stage::Data]), }, 0, ) .await .unwrap(); client.init().await.unwrap(); let raw_message = load_test_message("arc", "messages"); let message = MessageParser::new().parse(raw_message.as_bytes()).unwrap(); let r = client .connection( "gmail.com", "127.0.0.1".parse().unwrap(), 1235, Macros::new(), ) .await .unwrap(); println!("CONNECT: {:?}", r); let r = client .mail_from("john@gmail.com", None::<&[&str]>, Macros::new()) .await .unwrap(); println!("MAIL FROM: {:?}", r); let r = client .rcpt_to("user@gmail.com", None::<&[&str]>, Macros::new()) .await .unwrap(); println!("RCPT TO: {:?}", r); let r = client.data().await.unwrap(); println!("DATA: {:?}", r); let r = client.headers(message.headers_raw()).await.unwrap(); println!("HEADERS: {:?}", r); let r = client .body(&message.raw_message()[message.root_part().raw_body_offset() as usize..]) .await .unwrap(); println!("BODY: {:?}", r); client.quit().await.unwrap(); } pub fn spawn_mock_milter_server() -> watch::Sender { let (tx, rx) = watch::channel(true); let tests = Arc::new( serde_json::from_str::>( &fs::read_to_string( PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("resources") .join("smtp") .join("milter") .join("message.json"), ) .unwrap(), ) .unwrap(), ); tokio::spawn(async move { let listener = TcpListener::bind("127.0.0.1:9332") .await .unwrap_or_else(|e| { panic!("Failed to bind mock Milter server to 127.0.0.1:9332: {e}"); }); let mut rx_ = rx.clone(); //println!("Mock Milter server listening on port 9332"); loop { tokio::select! { stream = listener.accept() => { match stream { Ok((stream, _)) => { tokio::spawn(accept_milter(stream, rx.clone(), tests.clone())); } Err(err) => { panic!("Something went wrong: {err}" ); } } }, _ = rx_.changed() => { //println!("Mock Milter server stopping"); break; } }; } }); tx } async fn accept_milter( mut stream: TcpStream, mut rx: watch::Receiver, tests: Arc>, ) { let mut buf = vec![0u8; 1024]; let mut receiver = Receiver::with_max_frame_len(5000000); let mut action = None; let mut modifications = None; 'outer: loop { let br = tokio::select! { br = stream.read(&mut buf) => { match br { Ok(br) => { br } Err(_) => { break; } } }, _ = rx.changed() => { break; } }; if br == 0 { break; } loop { match receiver.read_frame(&buf[..br]) { FrameResult::Frame(bytes) => { let cmd = Command::deserialize(bytes.as_ref()); println!("CMD: {cmd}"); let response = match cmd { Command::Abort | Command::Macro { .. } => continue, Command::Body { .. } | Command::Data | Command::Connect { .. } | Command::Header { .. } | Command::Helo { .. } | Command::Rcpt { .. } | Command::QuitNewConnection | Command::EndOfHeader => Response::Action(Action::Accept), Command::OptionNegotiation(_) => Response::OptionNegotiation(Options { version: 6, actions: 0, protocol: 0, }), Command::MailFrom { sender, .. } => { let sender = std::str::from_utf8(sender).unwrap(); action = match sender .strip_prefix('<') .unwrap() .split_once('@') .unwrap() .0 { "accept" => Action::Accept, "reject" => Action::Reject, "discard" => Action::Discard, "temp_fail" => Action::TempFail, "shutdown" => Action::Shutdown, "conn_fail" => Action::ConnectionFailure, "reply_code" => Action::ReplyCode { code: [b'3', b'2', b'1'], text: "test".into(), }, test_num => { modifications = tests[test_num.parse::().unwrap()] .modifications .clone() .into(); Action::Accept } } .into(); Response::Action(Action::Accept) } Command::Quit => break 'outer, Command::EndOfBody => { if let Some(modifications) = modifications.take() { for modification in modifications { // Write modifications stream .write_all( &Response::Modification(modification).serialize(), ) .await .unwrap(); } } Response::Action(action.take().unwrap()) } }; // Write response stream.write_all(&response.serialize()).await.unwrap(); } FrameResult::Incomplete => continue 'outer, FrameResult::TooLarge(size) => { panic!("Frame too large: {size}") } } } } } pub fn spawn_mock_mta_hook_server() -> watch::Sender { let (tx, rx) = watch::channel(true); let tests = Arc::new( serde_json::from_str::>( &fs::read_to_string( PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("resources") .join("smtp") .join("milter") .join("message.json"), ) .unwrap(), ) .unwrap(), ); tokio::spawn(async move { let listener = TcpListener::bind("127.0.0.1:9333") .await .unwrap_or_else(|e| { panic!("Failed to bind mock Milter server to 127.0.0.1:9333: {e}"); }); let mut rx_ = rx.clone(); //println!("Mock jMilter server listening on port 9333"); loop { tokio::select! { stream = listener.accept() => { match stream { Ok((stream, _)) => { let _ = http1::Builder::new() .keep_alive(false) .serve_connection( TokioIo::new(stream), service_fn(|mut req: hyper::Request| { let tests = tests.clone(); async move { let request = serde_json::from_slice::(&fetch_body(&mut req, 1024 * 1024,0).await.unwrap()) .unwrap(); let response = handle_mta_hook(request, tests); Ok::<_, hyper::Error>( Resource::new("application/json", serde_json::to_string(&response).unwrap().into_bytes()) .into_http_response().build(), ) } }), ) .await; } Err(err) => { panic!("Something went wrong: {err}" ); } } }, _ = rx_.changed() => { //println!("Mock jMilter server stopping"); break; } }; } }); tx } fn handle_mta_hook(request: Request, tests: Arc>) -> hooks::Response { match request .envelope .unwrap() .from .address .split_once('@') .unwrap() .0 { "accept" => hooks::Response { action: hooks::Action::Accept, response: None, modifications: vec![], }, "reject" => hooks::Response { action: hooks::Action::Reject, response: None, modifications: vec![], }, "discard" => hooks::Response { action: hooks::Action::Discard, response: None, modifications: vec![], }, "temp_fail" => hooks::Response { action: hooks::Action::Reject, response: SmtpResponse { status: 451.into(), enhanced_status: Some("4.3.5".into()), message: Some("Unable to accept message at this time.".into()), disconnect: false, } .into(), modifications: vec![], }, "shutdown" => hooks::Response { action: hooks::Action::Reject, response: SmtpResponse { status: 421.into(), enhanced_status: Some("4.3.0".into()), message: Some("Server shutting down".into()), disconnect: false, } .into(), modifications: vec![], }, "conn_fail" => hooks::Response { action: hooks::Action::Accept, response: SmtpResponse { disconnect: true, ..Default::default() } .into(), modifications: vec![], }, "reply_code" => hooks::Response { action: hooks::Action::Reject, response: SmtpResponse { status: 321.into(), enhanced_status: Some("3.1.1".into()), message: Some("Test".into()), disconnect: false, } .into(), modifications: vec![], }, test_num => hooks::Response { action: hooks::Action::Accept, response: None, modifications: tests[test_num.parse::().unwrap()] .modifications .iter() .map(|m| match m { Modification::ChangeFrom { sender, args } => hooks::Modification::ChangeFrom { value: sender.clone(), parameters: args .split_whitespace() .map(|arg| { let (key, value) = arg.split_once('=').unwrap(); (key.into(), Some(value.into())) }) .collect(), }, Modification::AddRcpt { recipient, args } => { hooks::Modification::AddRecipient { value: recipient.clone(), parameters: args .split_whitespace() .map(|arg| { let (key, value) = arg.split_once('=').unwrap(); (key.into(), Some(value.into())) }) .collect(), } } Modification::DeleteRcpt { recipient } => { hooks::Modification::DeleteRecipient { value: recipient.clone(), } } Modification::ReplaceBody { value } => hooks::Modification::ReplaceContents { value: String::from_utf8(value.clone()).unwrap(), }, Modification::AddHeader { name, value } => hooks::Modification::AddHeader { name: name.clone(), value: value.clone(), }, Modification::InsertHeader { index, name, value } => { hooks::Modification::InsertHeader { index: *index, name: name.clone(), value: value.clone(), } } Modification::ChangeHeader { index, name, value } => { hooks::Modification::ChangeHeader { index: *index, name: name.clone(), value: value.clone(), } } Modification::Quarantine { reason } => hooks::Modification::AddHeader { name: "X-Quarantine".into(), value: reason.clone(), }, }) .collect(), }, } } ================================================ FILE: tests/src/smtp/inbound/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use common::{ Server, ipc::{DmarcEvent, QueueEvent, QueueEventStatus, ReportingEvent, TlsEvent}, }; use store::{ Deserialize, IterateParams, U64_LEN, ValueKey, write::{ AlignedBytes, Archive, QueueClass, ReportEvent, ValueClass, key::DeserializeBigEndian, }, }; use tokio::sync::mpsc::error::TryRecvError; use smtp::queue::{Message, MessageWrapper, QueueId, QueuedMessage}; use super::{QueueReceiver, ReportReceiver}; pub mod antispam; pub mod asn; pub mod auth; pub mod basic; pub mod data; pub mod dmarc; pub mod ehlo; pub mod limits; pub mod mail; pub mod milter; pub mod rcpt; pub mod rewrite; pub mod scripts; pub mod sign; pub mod throttle; pub mod vrfy; impl QueueReceiver { pub async fn read_event(&mut self) -> QueueEvent { match tokio::time::timeout(Duration::from_millis(100), self.queue_rx.recv()).await { Ok(Some(event)) => event, Ok(None) => panic!("Channel closed."), Err(_) => panic!("No queue event received."), } } pub async fn try_read_event(&mut self) -> Option { match tokio::time::timeout(Duration::from_millis(100), self.queue_rx.recv()).await { Ok(Some(event)) => Some(event), Ok(None) => panic!("Channel closed."), Err(_) => None, } } pub fn assert_no_events(&mut self) { match self.queue_rx.try_recv() { Err(TryRecvError::Empty) => (), Ok(event) => panic!("Expected empty queue but got {event:?}"), Err(err) => panic!("Queue error: {err:?}"), } } pub async fn assert_queue_is_empty(&self) { assert_eq!(self.read_queued_messages().await, vec![]); assert_eq!(self.read_queued_events().await, vec![]); } pub async fn assert_report_is_empty(&self) { assert_eq!(self.read_report_events().await, vec![]); for (from_key, to_key) in [ ( ValueKey::from(ValueClass::Queue(QueueClass::TlsReportEvent(ReportEvent { due: 0, policy_hash: 0, seq_id: 0, domain: String::new(), }))), ValueKey::from(ValueClass::Queue(QueueClass::TlsReportEvent(ReportEvent { due: u64::MAX, policy_hash: 0, seq_id: 0, domain: String::new(), }))), ), ( ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportEvent( ReportEvent { due: 0, policy_hash: 0, seq_id: 0, domain: String::new(), }, ))), ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportEvent( ReportEvent { due: u64::MAX, policy_hash: 0, seq_id: 0, domain: String::new(), }, ))), ), ] { self.store .iterate( IterateParams::new(from_key, to_key).ascending().no_values(), |key, _| { panic!("Unexpected report event: {key:?}"); }, ) .await .unwrap(); } } pub async fn expect_message(&mut self) -> MessageWrapper { self.read_event().await.assert_refresh(); self.last_queued_message().await } pub async fn consume_message(&mut self, server: &Server) -> MessageWrapper { self.read_event().await.assert_refresh(); let message = self.last_queued_message().await; message .clone() .remove(server, self.last_queued_due().await.into()) .await; message } pub async fn expect_message_then_deliver(&mut self) -> QueuedMessage { let message = self.expect_message().await; self.delivery_attempt(message.queue_id).await } pub async fn delivery_attempt(&mut self, queue_id: u64) -> QueuedMessage { QueuedMessage { due: self.message_due(queue_id).await, queue_id, queue_name: Default::default(), } } pub async fn read_queued_events(&self) -> Vec { let mut events = Vec::new(); let from_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( store::write::QueueEvent { due: 0, queue_id: 0, queue_name: [0; 8], }, ))); let to_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( store::write::QueueEvent { due: u64::MAX, queue_id: u64::MAX, queue_name: [u8::MAX; 8], }, ))); self.store .iterate( IterateParams::new(from_key, to_key).ascending().no_values(), |key, _| { events.push(store::write::QueueEvent { due: key.deserialize_be_u64(0)?, queue_id: key.deserialize_be_u64(U64_LEN)?, queue_name: key[U64_LEN + 1..U64_LEN + 9] .try_into() .expect("Queue name must be 8 bytes"), }); Ok(true) }, ) .await .unwrap(); events } pub async fn read_queued_messages(&self) -> Vec { let from_key = ValueKey::from(ValueClass::Queue(QueueClass::Message(0))); let to_key = ValueKey::from(ValueClass::Queue(QueueClass::Message(u64::MAX))); let mut messages = Vec::new(); self.store .iterate( IterateParams::new(from_key, to_key).descending(), |key, value| { messages.push(MessageWrapper { queue_id: key.deserialize_be_u64(0)?, queue_name: Default::default(), is_multi_queue: false, span_id: 0, message: as Deserialize>::deserialize(value)? .deserialize::()?, }); Ok(true) }, ) .await .unwrap(); messages } pub async fn read_report_events(&self) -> Vec { let from_key = ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportHeader( ReportEvent { due: 0, policy_hash: 0, seq_id: 0, domain: String::new(), }, ))); let to_key = ValueKey::from(ValueClass::Queue(QueueClass::TlsReportHeader( ReportEvent { due: u64::MAX, policy_hash: 0, seq_id: 0, domain: String::new(), }, ))); let mut events = Vec::new(); self.store .iterate( IterateParams::new(from_key, to_key).ascending().no_values(), |key, _| { let event = ReportEvent::deserialize(key)?; // Skip lock if event.seq_id != 0 { events.push(if *key.last().unwrap() == 0 { QueueClass::DmarcReportHeader(event) } else { QueueClass::TlsReportHeader(event) }); } Ok(true) }, ) .await .unwrap(); events } pub async fn last_queued_message(&self) -> MessageWrapper { self.read_queued_messages() .await .into_iter() .next() .expect("No messages found in queue") } pub async fn last_queued_due(&self) -> u64 { self.message_due(self.last_queued_message().await.queue_id) .await } pub async fn message_due(&self, queue_id: QueueId) -> u64 { self.read_queued_events() .await .iter() .find_map(|event| { if event.queue_id == queue_id { Some(event.due) } else { None } }) .expect("No event found in queue for message") } pub async fn clear_queue(&self, server: &Server) { for message in self.read_queued_messages().await { let due = self.message_due(message.queue_id).await; message.remove(server, due.into()).await; } } } impl ReportReceiver { pub async fn read_report(&mut self) -> ReportingEvent { match tokio::time::timeout(Duration::from_millis(100), self.report_rx.recv()).await { Ok(Some(event)) => event, Ok(None) => panic!("Channel closed."), Err(_) => panic!("No report event received."), } } pub async fn try_read_report(&mut self) -> Option { match tokio::time::timeout(Duration::from_millis(100), self.report_rx.recv()).await { Ok(Some(event)) => Some(event), Ok(None) => panic!("Channel closed."), Err(_) => None, } } pub fn assert_no_reports(&mut self) { match self.report_rx.try_recv() { Err(TryRecvError::Empty) => (), Ok(event) => panic!("Expected no reports but got {event:?}"), Err(err) => panic!("Report error: {err:?}"), } } } pub trait TestQueueEvent { fn assert_refresh(self); fn assert_done(self); fn assert_refresh_or_done(self); } impl TestQueueEvent for QueueEvent { fn assert_refresh(self) { match self { QueueEvent::Refresh | QueueEvent::WorkerDone { status: QueueEventStatus::Deferred, .. } => (), e => panic!("Unexpected event: {e:?}"), } } fn assert_done(self) { match self { QueueEvent::WorkerDone { status: QueueEventStatus::Completed, .. } => (), e => panic!("Unexpected event: {e:?}"), } } fn assert_refresh_or_done(self) { match self { QueueEvent::WorkerDone { status: QueueEventStatus::Completed | QueueEventStatus::Deferred, .. } => (), e => panic!("Unexpected event: {e:?}"), } } } pub trait TestReportingEvent { fn unwrap_dmarc(self) -> Box; fn unwrap_tls(self) -> Box; } impl TestReportingEvent for ReportingEvent { fn unwrap_dmarc(self) -> Box { match self { ReportingEvent::Dmarc(event) => event, e => panic!("Unexpected event: {e:?}"), } } fn unwrap_tls(self) -> Box { match self { ReportingEvent::Tls(event) => event, e => panic!("Unexpected event: {e:?}"), } } } #[allow(async_fn_in_trait)] pub trait TestMessage { async fn read_message(&self, core: &QueueReceiver) -> String; async fn read_lines(&self, core: &QueueReceiver) -> Vec; } impl TestMessage for MessageWrapper { async fn read_message(&self, core: &QueueReceiver) -> String { String::from_utf8( core.blob_store .get_blob(self.message.blob_hash.as_slice(), 0..usize::MAX) .await .unwrap() .expect("Message blob not found"), ) .unwrap() } async fn read_lines(&self, core: &QueueReceiver) -> Vec { self.read_message(core) .await .split('\n') .map(|l| l.to_string()) .collect() } } ================================================ FILE: tests/src/smtp/inbound/rcpt.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use common::Core; use smtp_proto::{RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_SUCCESS}; use store::Stores; use utils::config::Config; use smtp::core::{Session, State}; use crate::smtp::{ TempDir, TestSMTP, session::{TestSession, VerifyResponse}, }; const CONFIG: &str = r#" [storage] data = "rocksdb" lookup = "rocksdb" blob = "rocksdb" fts = "rocksdb" [store."rocksdb"] type = "rocksdb" path = "{TMP}/queue.db" [directory."local"] type = "memory" [[directory."local".principals]] name = "john" description = "John Doe" secret = "secret" email = "john@foobar.org" [[directory."local".principals]] name = "jane" description = "Jane Doe" secret = "p4ssw0rd" email = "jane@foobar.org" [[directory."local".principals]] name = "bill" description = "Bill Foobar" secret = "p4ssw0rd" email = "bill@foobar.org" [[directory."local".principals]] name = "mike" description = "Mike Foobar" secret = "p4ssw0rd" email = "mike@foobar.org" [session.rcpt] directory = "'local'" max-recipients = [{if = "remote_ip = '10.0.0.1'", then = 3}, {else = 5}] relay = [{if = "remote_ip = '10.0.0.1'", then = false}, {else = true}] [session.rcpt.errors] total = [{if = "remote_ip = '10.0.0.1'", then = 3}, {else = 100}] wait = [{if = "remote_ip = '10.0.0.1'", then = '5ms'}, {else = '1s'}] [session.extensions] dsn = [{if = "remote_ip = '10.0.0.1'", then = false}, {else = true}] [[queue.limiter.inbound]] match = "remote_ip = '10.0.0.1' && !is_empty(rcpt)" key = 'sender' rate = '2/1s' enable = true "#; #[tokio::test] async fn rcpt() { // Enable logging crate::enable_logging(); let tmp_dir = TempDir::new("smtp_rcpt_test", true); let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; // RCPT without MAIL FROM let mut session = Session::test(TestSMTP::from_core(core).server); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx1.foobar.org").await; session.rcpt_to("jane@foobar.org", "503 5.5.1").await; // Relaying is disabled for 10.0.0.1 session.mail_from("john@example.net", "250").await; session.rcpt_to("external@domain.com", "550 5.1.2").await; // DSN is disabled for 10.0.0.1 session .ingest(b"RCPT TO: NOTIFY=SUCCESS,FAILURE,DELAY\r\n") .await .unwrap(); session.response().assert_code("501 5.5.4"); // Send to non-existing user session.rcpt_to("tom@foobar.org", "550 5.1.2").await; // Exceeding max number of errors session .ingest(b"RCPT TO:\r\n") .await .unwrap_err(); session.response().assert_code("451 4.3.0"); // Rate limit session.data.rcpt_errors = 0; session.state = State::default(); session.rcpt_to("Jane@FooBar.org", "250").await; session.rcpt_to("Bill@FooBar.org", "250").await; session.rcpt_to("Mike@FooBar.org", "452 4.4.5").await; // Restore rate limit tokio::time::sleep(Duration::from_millis(1100)).await; session.rcpt_to("Mike@FooBar.org", "250").await; session.rcpt_to("john@foobar.org", "455 4.5.3").await; // Check recipients assert_eq!(session.data.rcpt_to.len(), 3); for (rcpt, expected) in session .data .rcpt_to .iter() .zip(["Jane@FooBar.org", "Bill@FooBar.org", "Mike@FooBar.org"]) { assert_eq!(rcpt.address, expected); assert_eq!(rcpt.domain, "foobar.org"); assert_eq!(rcpt.address_lcase, expected.to_lowercase()); } // Relaying should be allowed for 10.0.0.2 session.data.remote_ip_str = "10.0.0.2".into(); session.eval_session_params().await; session.rset().await; session.mail_from("john@example.net", "250").await; session.rcpt_to("external@domain.com", "250").await; // DSN is enabled for 10.0.0.2 session .ingest(b"RCPT TO: NOTIFY=SUCCESS,FAILURE,DELAY ORCPT=rfc822;Jane.Doe@Foobar.org\r\n") .await .unwrap(); session.response().assert_code("250"); let rcpt = session.data.rcpt_to.last().unwrap(); assert!((rcpt.flags & (RCPT_NOTIFY_DELAY | RCPT_NOTIFY_SUCCESS | RCPT_NOTIFY_FAILURE)) != 0); assert_eq!(rcpt.dsn_info.as_ref().unwrap(), "Jane.Doe@Foobar.org"); } ================================================ FILE: tests/src/smtp/inbound/rewrite.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::Core; use smtp::core::Session; use utils::config::Config; use crate::smtp::{TestSMTP, session::TestSession}; const CONFIG: &str = r#" [session.mail] rewrite = [ { if = "ends_with(sender_domain, '.foobar.net') & matches('^([^.]+)@([^.]+)\.(.+)$', sender)", then = "$1 + '+' + $2 + '@' + $3"}, { else = false } ] script = [ { if = "sender_domain = 'foobar.org'", then = "'mail'" }, { else = false } ] [session.rcpt] rewrite = [ { if = "rcpt_domain = 'foobar.net' & matches('^([^.]+)\\.([^.]+)@(.+)$', rcpt)", then = "$1 + '+' + $2 + '@' + $3"}, { else = false } ] script = [ { if = "rcpt_domain = 'foobar.org'", then = "'rcpt'" }, { else = false } ] relay = true [sieve.trusted] from-name = "Sieve Daemon" from-addr = "sieve@foobar.org" return-path = "" hostname = "mx.foobar.org" [sieve.trusted.limits] redirects = 3 out-messages = 5 received-headers = 50 cpu = 10000 nested-includes = 5 duplicate-expiry = "7d" [sieve.trusted.scripts."mail"] contents = ''' require ["variables", "envelope"]; if allof( envelope :domain :is "from" "foobar.org", envelope :localpart :contains "from" "admin" ) { set "envelope.from" "MAILER-DAEMON@foobar.org"; } ''' [sieve.trusted.scripts."rcpt"] contents = ''' require ["variables", "envelope", "regex"]; if allof( envelope :localpart :contains "to" ".", envelope :regex "to" "(.+)@(.+)$") { set :replace "." "" "to" "${1}"; set "envelope.to" "${to}@${2}"; } ''' "#; #[tokio::test] async fn address_rewrite() { // Enable logging crate::enable_logging(); // Prepare config let mut config = Config::new(CONFIG).unwrap(); let core = Core::parse(&mut config, Default::default(), Default::default()).await; // Init session let mut session = Session::test(TestSMTP::from_core(core).server); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.doe.org").await; // Sender rewrite using regex session.mail_from("bill@doe.foobar.net", "250").await; assert_eq!( session.data.mail_from.as_ref().unwrap().address, "bill+doe@foobar.net" ); session.reset(); // Sender rewrite using sieve session.mail_from("this_is_admin@foobar.org", "250").await; assert_eq!( session.data.mail_from.as_ref().unwrap().address_lcase, "mailer-daemon@foobar.org" ); // Recipient rewrite using regex session.rcpt_to("mary.smith@foobar.net", "250").await; assert_eq!( session.data.rcpt_to.last().unwrap().address, "mary+smith@foobar.net" ); // Remove duplicates session.rcpt_to("mary.smith@foobar.net", "250").await; assert_eq!(session.data.rcpt_to.len(), 1); // Recipient rewrite using sieve session.rcpt_to("m.a.r.y.s.m.i.t.h@foobar.org", "250").await; assert_eq!( session.data.rcpt_to.last().unwrap().address, "marysmith@foobar.org" ); } ================================================ FILE: tests/src/smtp/inbound/scripts.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use core::panic; use std::{fmt::Write, fs, path::PathBuf}; use crate::{ AssertConfig, enable_logging, smtp::{ TempDir, TestSMTP, inbound::{TestMessage, TestQueueEvent, sign::SIGNATURES}, session::{TestSession, VerifyResponse}, }, }; use common::Core; use smtp::{ core::Session, scripts::{ScriptResult, event_loop::RunScript}, }; use store::Stores; use utils::config::Config; const CONFIG: &str = r#" [storage] data = "sql" lookup = "sql" blob = "sql" fts = "sql" directory = "local" [store."sql"] type = "sqlite" path = "{TMP}/smtp_sieve.db" [store."sql".pool] max-connections = 10 min-connections = 0 idle-timeout = "5m" [spam-filter] enable = false [sieve.trusted] from-name = "'Sieve Daemon'" from-addr = "'sieve@foobar.org'" return-path = "''" hostname = "mx.foobar.org" sign = "['rsa']" [sieve.trusted.limits] redirects = 3 out-messages = 5 received-headers = 50 cpu = 10000 nested-includes = 5 duplicate-expiry = "7d" [session.connect] script = "'stage_connect'" greeting = "'mx.example.org at your service'" [session.ehlo] script = "'stage_ehlo'" [session.mail] script = "'stage_mail'" [session.rcpt] script = "'stage_rcpt'" relay = true [session.data] script = "'stage_data'" [session.data.add-headers] received = true received-spf = true auth-results = true message-id = true date = true return-path = false [directory."local"] type = "memory" [[directory."local".principals]] name = "john" description = "John Doe" secret = "secret" email = ["john@localdomain.org", "jdoe@localdomain.org", "john.doe@localdomain.org"] email-list = ["info@localdomain.org"] member-of = ["sales"] "#; #[tokio::test] async fn sieve_scripts() { // Enable logging enable_logging(); // Add test scripts let mut config = CONFIG.to_string() + SIGNATURES; for entry in fs::read_dir( PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("resources") .join("smtp") .join("sieve"), ) .unwrap() { let entry = entry.unwrap(); writeln!( &mut config, "[sieve.trusted.scripts.{}]\ncontents = \"%{{file:{}}}%\"", entry .file_name() .to_str() .unwrap() .split_once('.') .unwrap() .0, entry.path().to_str().unwrap() ) .unwrap(); } // Prepare config let tmp_dir = TempDir::new("smtp_sieve_test", true); let mut config = Config::new(tmp_dir.update_config(config)).unwrap(); config.resolve_all_macros().await; let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; config.assert_no_errors(); // Build session let test = TestSMTP::from_core(core); let mut qr = test.queue_receiver; let mut session = Session::test(test.server.clone()); session.data.remote_ip_str = "10.0.0.88".parse().unwrap(); session.data.remote_ip = session.data.remote_ip_str.parse().unwrap(); assert!(!session.init_conn().await); // Run tests for (name, script) in &test.server.core.sieve.trusted_scripts { if name.starts_with("stage_") || name.ends_with("_include") { continue; } let script = script.clone(); let params = session .build_script_parameters("data") .set_variable("from", "john.doe@example.org") .with_envelope(&test.server, &session, 0) .await; match test.server.run_script(name.into(), script, params).await { ScriptResult::Accept { .. } => (), ScriptResult::Reject(message) => panic!("{}", message), err => { panic!("Unexpected script result {err:?}"); } } } // Test connect script session .response() .assert_contains("503 5.5.3 Your IP '10.0.0.88' is not welcomed here"); session.data.remote_ip_str = "10.0.0.5".parse().unwrap(); session.data.remote_ip = session.data.remote_ip_str.parse().unwrap(); assert!(session.init_conn().await); session .response() .assert_contains("220 mx.example.org at your service"); // Test EHLO script session .cmd( "EHLO spammer.org", "551 5.1.1 Your domain 'spammer.org' has been blocklisted", ) .await; session.cmd("EHLO foobar.net", "250").await; // Test MAIL-FROM script session .mail_from("spammer@domain.com", "450 4.1.1 Invalid address") .await; session .mail_from( "marketing@spam-domain.com", "503 5.5.3 Your address has been blocked", ) .await; session.mail_from("bill@foobar.org", "250").await; // Test RCPT-TO script session .rcpt_to( "jane@foobar.org", "422 4.2.2 You have been greylisted '10.0.0.5.bill@foobar.org.jane@foobar.org'.", ) .await; session.rcpt_to("jane@foobar.org", "250").await; // Expect a modified message session.data("test:multipart", "250").await; qr.expect_message() .await .read_lines(&qr) .await .assert_contains("X-Part-Number: 5") .assert_contains("THIS IS A PIECE OF HTML TEXT"); qr.assert_no_events(); // Expect rejection for bill@foobar.net session .send_message( "test@example.net", &["bill@foobar.net"], "test:multipart", "503 5.5.3 Bill cannot receive messages", ) .await; qr.assert_no_events(); qr.clear_queue(&test.server).await; // Expect message delivery plus a notification session .send_message( "test@example.net", &["john@foobar.net"], "test:multipart", "250", ) .await; qr.read_event().await.assert_refresh(); qr.read_event().await.assert_refresh(); let messages = qr.read_queued_messages().await; assert_eq!(messages.len(), 2); let mut messages = messages.into_iter(); let notification = messages.next().unwrap(); assert_eq!(notification.message.return_path.as_ref(), ""); assert_eq!(notification.message.recipients.len(), 2); assert_eq!( notification.message.recipients.first().unwrap().address(), "john@example.net" ); assert_eq!( notification.message.recipients.last().unwrap().address(), "jane@example.org" ); notification .read_lines(&qr) .await .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") .assert_contains("From: \"Sieve Daemon\" ") .assert_contains("To: ") .assert_contains("Cc: ") .assert_contains("Subject: You have got mail") .assert_contains("One Two Three Four"); messages .next() .unwrap() .read_lines(&qr) .await .assert_contains("One Two Three Four") .assert_contains("multi-part message in MIME format") .assert_not_contains("X-Part-Number: 5") .assert_not_contains("THIS IS A PIECE OF HTML TEXT"); qr.assert_no_events(); qr.clear_queue(&test.server).await; // Expect a modified message delivery plus a notification session .send_message( "test@example.net", &["jane@foobar.net"], "test:multipart", "250", ) .await; qr.read_event().await.assert_refresh(); qr.read_event().await.assert_refresh(); let messages = qr.read_queued_messages().await; assert_eq!(messages.len(), 2); let mut messages = messages.into_iter(); messages .next() .unwrap() .read_lines(&qr) .await .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") .assert_contains("From: \"Sieve Daemon\" ") .assert_contains("To: ") .assert_contains("Cc: ") .assert_contains("Subject: You have got mail") .assert_contains("One Two Three Four"); messages .next() .unwrap() .read_lines(&qr) .await .assert_contains("X-Part-Number: 5") .assert_contains("THIS IS A PIECE OF HTML TEXT") .assert_not_contains("X-My-Header: true"); qr.clear_queue(&test.server).await; // Expect a modified redirected message session .send_message( "test@example.net", &["thomas@foobar.gov"], "test:no_dkim", "250", ) .await; let redirect = qr.expect_message().await; assert_eq!(redirect.message.return_path.as_ref(), ""); assert_eq!(redirect.message.recipients.len(), 1); assert_eq!( redirect.message.recipients.first().unwrap().address(), "redirect@here.email" ); redirect .read_lines(&qr) .await .assert_contains("From: no-reply@my.domain") .assert_contains("To: Suzie Q ") .assert_contains("Subject: Is dinner ready?") .assert_contains("Message-ID: <20030712040037.46341.5F8J@football.example.com>") .assert_contains("Received: ") .assert_not_contains("From: Joe SixPack "); qr.assert_no_events(); // Expect an intact redirected message session .send_message( "test@example.net", &["bob@foobar.gov"], "test:no_dkim", "250", ) .await; let redirect = qr.expect_message().await; assert_eq!(redirect.message.return_path.as_ref(), ""); assert_eq!(redirect.message.recipients.len(), 1); assert_eq!( redirect.message.recipients.first().unwrap().address(), "redirect@somewhere.email" ); redirect .read_lines(&qr) .await .assert_not_contains("From: no-reply@my.domain") .assert_contains("To: Suzie Q ") .assert_contains("Subject: Is dinner ready?") .assert_contains("Message-ID: <20030712040037.46341.5F8J@football.example.com>") .assert_contains("From: Joe SixPack ") .assert_contains("Received: ") .assert_contains("Authentication-Results: "); qr.assert_no_events(); } ================================================ FILE: tests/src/smtp/inbound/sign.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::{Duration, Instant}; use common::Core; use mail_auth::{ common::{parse::TxtRecordParser, verify::DomainKey}, spf::Spf, }; use store::Stores; use utils::config::Config; use crate::smtp::{ DnsCache, TempDir, TestSMTP, inbound::TestMessage, session::{TestSession, VerifyResponse}, }; use smtp::core::Session; pub const SIGNATURES: &str = " [signature.rsa] private-key = ''' -----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEAv9XYXG3uK95115mB4nJ37nGeNe2CrARm1agrbcnSk5oIaEfM ZLUR/X8gPzoiNHZcfMZEVR6bAytxUhc5EvZIZrjSuEEeny+fFd/cTvcm3cOUUbIa UmSACj0dL2/KwW0LyUaza9z9zor7I5XdIl1M53qVd5GI62XBB76FH+Q0bWPZNkT4 NclzTLspD/MTpNCCPhySM4Kdg5CuDczTH4aNzyS0TqgXdtw6A4Sdsp97VXT9fkPW 9rso3lrkpsl/9EQ1mR/DWK6PBmRfIuSFuqnLKY6v/z2hXHxF7IoojfZLa2kZr9Ae d4l9WheQOTA19k5r2BmlRw/W9CrgCBo0Sdj+KQIDAQABAoIBAFPChEi/OvnulReB ECQWhOUYuNKlFKQU++2YEvZJ4+bMn5UgnE7wfJ1pj2Pr9xlfALz+OMHNrjMxGbaV KzdrT2uCkYcf78XjnhuH9gKIiXDUv4L4N+P3u6w8yOx4bFgOS9IjS53yDOPM7SC5 g6dIg5aigHaHlffqIuFFv4yQMI/+Ai+zBKxS7wRhxK/7nnAuo28fe5MEdp57ho9/ AGlDNsdg9zCgjwhokwFE3+AaD+bkUFm4gQ1XjkUFrlmnQn8vDQ0i9toEWhCj+UPY iOKL63MJnr90MXTXWLHoFj99wBp//mYygbF9Lj8fa28/oa8LWp3Jhb7QeMgH46iv 3aLHbTECgYEA5M2dAw+nyMw9vYlkMejhwObKYP8Mr/6zcGMLCalYvRJM5iUAM0JI H6sM6pV9/nv167cbKocj3xYPdtE7FPOn4132MLM8Ne1f8nPE64Qrcbj5WBXvLnU8 hpWbwe2Z8h7UUMKx6q4F1/TXYkc3ScxYwfjM4mP/pLsAOgVzRSEEgrUCgYEA1qNQ xaQHNWZ1O8WuTnqWd5JSsic6iURAmUcLeFDZY2PWhVoaQ8L/xMQhDYs1FIbLWArW 4Qq3Ibu8AbSejAKuaJz7Uf26PX+PYVUwAOO0qamCJ8d/qd6So7qWMDyAY2yXI39Y 1nMqRjr7bkEsggAZao7BKqA7ZtmogjOusBT38iUCgYEA06agJ8TDoKvOMRZ26PRU YO0dKLzGL8eclcoI29cbj0rud7aiiMg3j5PbTuUat95TjsjDCIQaWrM9etvxm2AJ Xfn9Uu96MyhyKQWOk46f4YMKpMElkARDCPw8KRhx39dE77AqhLyWCz8iPndCXbH6 KPTOEl4OjYOuof2Is9nnIkECgYBh948RdsnXhNlzm8nwhiGRmBbou+EK8D0v+O5y Tyy6IcKzgSnFzgZh8EdJ4EUtBk1f9SqY8wQdgIvSl3daXorusuA/TzkngsaV3YUY ktZOLlF7CKLrjOyPkMWmZKcROmpNyH1q/IvKHHfQnizLdXIkYd4nL5WNX0F7lE1i j1+QhQKBgB2lviBK7rJFwlFYdQUP1NAN2dKxMZk8uJS8JglHrM0+8nRI83HbTdEQ vB0ManEKBkbS4T5n+gRtdEqKSDmWDTXDlrBfcdCHNQLwYtBpOotCqQn/AmfjcPBl byAbwh4+HiZ5JISoRZpiZqy67aJNVoXmdtb/E9mi7ozzytpxMNql -----END RSA PRIVATE KEY-----''' domain = 'example.com' selector = 'rsa' headers = ['From', 'To', 'Date', 'Subject', 'Message-ID'] algorithm = 'rsa-sha256' canonicalization = 'simple/relaxed' expire = '10d' set-body-length = true report = true [signature.ed] private-key = '-----BEGIN PRIVATE KEY----- MC4CAQAwBQYDK2VwBCIEIAO3hAf144lTAVjTkht3ZwBTK0CMCCd1bI0alggneN3B -----END PRIVATE KEY-----' domain = 'example.com' selector = 'ed' headers = ['From', 'To', 'Date', 'Subject', 'Message-ID'] algorithm = 'ed25519-sha256' canonicalization = 'relaxed/simple' set-body-length = false "; const CONFIG: &str = r#" [storage] data = "rocksdb" lookup = "rocksdb" blob = "rocksdb" fts = "rocksdb" [store."rocksdb"] type = "rocksdb" path = "{TMP}/queue.db" [directory."local"] type = "memory" [[directory."local".principals]] name = "john" description = "John Doe" secret = "secret" email = ["jdoe@example.com"] [session.rcpt] directory = "'local'" [session.data.add-headers] received = true received-spf = true auth-results = true message-id = true date = true return-path = false [auth.spf.verify] ehlo = "relaxed" mail-from = "relaxed" [auth.dkim] verify = "relaxed" sign = "['rsa']" [auth.arc] verify = "relaxed" seal = "'ed'" [auth.dmarc] verify = "relaxed" "#; #[tokio::test] async fn sign_and_seal() { // Enable logging crate::enable_logging(); let tmp_dir = TempDir::new("smtp_sign_test", true); let mut config = Config::new(tmp_dir.update_config(CONFIG.to_string() + SIGNATURES)).unwrap(); let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; let test = TestSMTP::from_core(core); // Add SPF, DKIM and DMARC records test.server.txt_add( "mx.example.com", Spf::parse(b"v=spf1 ip4:10.0.0.1 ip4:10.0.0.2 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); test.server.txt_add( "example.com", Spf::parse(b"v=spf1 ip4:10.0.0.1 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); test.server.txt_add( "ed._domainkey.scamorza.org", DomainKey::parse( concat!( "v=DKIM1; k=ed25519; ", "p=11qYAYKxCrfVS/7TyWQHOg7hcvPapiMlrwIaaPcHURo=" ) .as_bytes(), ) .unwrap(), Instant::now() + Duration::from_secs(5), ); test.server.txt_add( "rsa._domainkey.manchego.org", DomainKey::parse( concat!( "v=DKIM1; t=s; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQ", "KBgQDwIRP/UC3SBsEmGqZ9ZJW3/DkMoGeLnQg1fWn7/zYt", "IxN2SnFCjxOCKG9v3b4jYfcTNh5ijSsq631uBItLa7od+v", "/RtdC2UzJ1lWT947qR+Rcac2gbto/NMqJ0fzfVjH4OuKhi", "tdY9tf6mcwGjaNBcWToIMmPSPDdQPNUYckcQ2QIDAQAB", ) .as_bytes(), ) .unwrap(), Instant::now() + Duration::from_secs(5), ); // Test DKIM signing let mut qr = test.queue_receiver; let mut session = Session::test(test.server); session.data.remote_ip_str = "10.0.0.2".into(); session.eval_session_params().await; session.ehlo("mx.example.com").await; session .send_message( "bill@foobar.org", &["jdoe@example.com"], "test:no_dkim", "250", ) .await; qr.expect_message() .await .read_lines(&qr) .await .assert_contains( "DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com; c=simple/relaxed;", ); // Test ARC verify and seal session .send_message("bill@foobar.org", &["jdoe@example.com"], "test:arc", "250") .await; qr.expect_message() .await .read_lines(&qr) .await .assert_contains("ARC-Seal: i=3; a=ed25519-sha256; s=ed; d=example.com; cv=pass;") .assert_contains( "ARC-Message-Signature: i=3; a=ed25519-sha256; s=ed; d=example.com; c=relaxed/simple;", ); // Test ARC sealing of a DKIM signed message session .send_message("bill@foobar.org", &["jdoe@example.com"], "test:dkim", "250") .await; qr.expect_message() .await .read_lines(&qr) .await .assert_contains("ARC-Seal: i=1; a=ed25519-sha256; s=ed; d=example.com; cv=none;") .assert_contains( "ARC-Message-Signature: i=1; a=ed25519-sha256; s=ed; d=example.com; c=relaxed/simple;", ); } ================================================ FILE: tests/src/smtp/inbound/throttle.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use crate::smtp::{TempDir, TestSMTP, session::TestSession}; use common::Core; use smtp::core::{Session, SessionAddress}; use store::Stores; use utils::config::Config; const CONFIG: &str = r#" [storage] data = "rocksdb" lookup = "rocksdb" blob = "rocksdb" fts = "rocksdb" [store."rocksdb"] type = "rocksdb" path = "{TMP}/data.db" [[queue.limiter.inbound]] match = "remote_ip = '10.0.0.1'" key = 'remote_ip' rate = '2/1s' enable = true [[queue.limiter.inbound]] key = 'sender' rate = '2/1s' enable = true [[queue.limiter.inbound]] key = ['remote_ip', 'rcpt'] rate = '2/1s' enable = true "#; #[tokio::test] async fn throttle_inbound() { // Enable logging crate::enable_logging(); let tmp_dir = TempDir::new("smtp_inbound_throttle", true); let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; // Test connection rate limit let mut session = Session::test(TestSMTP::from_core(core).server); session.data.remote_ip_str = "10.0.0.1".into(); assert!(session.is_allowed().await, "Rate limiter too strict."); assert!(session.is_allowed().await, "Rate limiter too strict."); assert!(!session.is_allowed().await, "Rate limiter failed."); tokio::time::sleep(Duration::from_millis(1100)).await; assert!( session.is_allowed().await, "Rate limiter did not restore quota." ); // Test mail from rate limit session.data.mail_from = SessionAddress { address: "sender@test.org".into(), address_lcase: "sender@test.org".into(), domain: "test.org".into(), flags: 0, dsn_info: None, } .into(); assert!(session.is_allowed().await, "Rate limiter too strict."); assert!(session.is_allowed().await, "Rate limiter too strict."); assert!(!session.is_allowed().await, "Rate limiter failed."); session.data.mail_from = SessionAddress { address: "other-sender@test.org".into(), address_lcase: "other-sender@test.org".into(), domain: "test.org".into(), flags: 0, dsn_info: None, } .into(); assert!(session.is_allowed().await, "Rate limiter failed."); // Test recipient rate limit session.data.rcpt_to.push(SessionAddress { address: "recipient@example.org".into(), address_lcase: "recipient@example.org".into(), domain: "example.org".into(), flags: 0, dsn_info: None, }); assert!(session.is_allowed().await, "Rate limiter too strict."); assert!(session.is_allowed().await, "Rate limiter too strict."); assert!(!session.is_allowed().await, "Rate limiter failed."); session.data.remote_ip_str = "10.0.0.2".into(); assert!(session.is_allowed().await, "Rate limiter too strict."); } ================================================ FILE: tests/src/smtp/inbound/vrfy.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::Core; use store::Stores; use utils::config::Config; use smtp::core::Session; use crate::{ AssertConfig, smtp::{ TempDir, TestSMTP, session::{TestSession, VerifyResponse}, }, }; const CONFIG: &str = r#" [storage] data = "rocksdb" lookup = "rocksdb" blob = "rocksdb" fts = "rocksdb" directory = "local" [store."rocksdb"] type = "rocksdb" path = "{TMP}/data.db" [directory."local"] type = "memory" [[directory."local".principals]] name = "john" description = "John Doe" secret = "secret" email = ["john@foobar.org"] email-list = ["sales@foobar.org"] [[directory."local".principals]] name = "jane" description = "Jane Doe" secret = "p4ssw0rd" email = "jane@foobar.org" email-list = ["sales@foobar.org"] [[directory."local".principals]] name = "bill" description = "Bill Foobar" secret = "p4ssw0rd" email = "bill@foobar.org" email-list = ["sales@foobar.org"] [session.rcpt] directory = "'local'" [session.extensions] vrfy = [{if = "remote_ip = '10.0.0.1'", then = true}, {else = false}] expn = [{if = "remote_ip = '10.0.0.1'", then = true}, {else = false}] "#; #[tokio::test] async fn vrfy_expn() { // Enable logging crate::enable_logging(); let tmp_dir = TempDir::new("smtp_vrfy_test", true); let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; config.assert_no_errors(); // EHLO should not advertise VRFY/EXPN to 10.0.0.2 let mut session = Session::test(TestSMTP::from_core(core).server); session.data.remote_ip_str = "10.0.0.2".into(); session.eval_session_params().await; session .ehlo("mx.foobar.org") .await .assert_not_contains("EXPN") .assert_not_contains("VRFY"); session.cmd("VRFY john", "252 2.5.1").await; session.cmd("EXPN sales@foobar.org", "252 2.5.1").await; // EHLO should advertise VRFY/EXPN for 10.0.0.1 session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session .ehlo("mx.foobar.org") .await .assert_contains("EXPN") .assert_contains("VRFY"); // Successful VRFY session.cmd("VRFY john", "250 john@foobar.org").await; // Successful EXPN session .cmd("EXPN sales@foobar.org", "250") .await .assert_contains("250-john@foobar.org") .assert_contains("250-jane@foobar.org") .assert_contains("250 bill@foobar.org"); // Non-existent VRFY session.cmd("VRFY robert", "550 5.1.2").await; // Non-existent EXPN session.cmd("EXPN procurement", "550 5.1.2").await; } ================================================ FILE: tests/src/smtp/lookup/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod sql; pub mod utils; ================================================ FILE: tests/src/smtp/lookup/sql.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::{Duration, Instant}; use common::{ Core, expr::{tokenizer::TokenMap, *}, }; use directory::{ QueryParams, Type, backend::internal::{PrincipalField, PrincipalSet, PrincipalValue, manage::ManageDirectory}, }; use mail_auth::MX; use store::Stores; use utils::config::Config; use crate::{ directory::DirectoryStore, smtp::{ DnsCache, TempDir, TestSMTP, session::{TestSession, VerifyResponse}, }, }; use smtp::{core::Session, queue::RecipientDomain}; const CONFIG: &str = r#" [storage] data = "sql" blob = "sql" fts = "sql" lookup = "sql" directory = "sql" [store."sql"] type = "sqlite" path = "{TMP}/smtp_sql.db" [store."sql".query] name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = ? AND active = true" members = "SELECT member_of FROM group_members WHERE name = ?" recipients = "SELECT name FROM emails WHERE address = ?" emails = "SELECT address FROM emails WHERE name = ? AND type != 'list' ORDER BY type DESC, address ASC" verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type = 'primary' ORDER BY address LIMIT 5" expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50" domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || ? LIMIT 1" [directory."sql"] type = "sql" store = "sql" [directory."sql".columns] name = "name" description = "description" secret = "secret" email = "address" quota = "quota" class = "type" [session.auth] directory = "'sql'" mechanisms = "[plain, login]" errors.wait = "5ms" [session.rcpt] directory = "'sql'" relay = false errors.wait = "5ms" [session.extensions] requiretls = [{if = "sql_query('sql', 'SELECT addr FROM allowed_ips WHERE addr = ? LIMIT 1', remote_ip)", then = true}, {else = false}] expn = true vrfy = true [test."sql"] expr = "sql_query('sql', 'SELECT description FROM domains WHERE name = ?', 'foobar.org')" expect = "Main domain" [test."dns"] expr = "dns_query(rcpt_domain, 'mx')[0]" expect = "mx.foobar.org" [test."key_get"] expr = "key_get('sql', 'hello') + '-' + key_exists('sql', 'hello') + '-' + key_set('sql', 'hello', 'world') + '-' + key_get('sql', 'hello') + '-' + key_exists('sql', 'hello')" expect = "-0-1-world-1" [test."counter_get"] expr = "counter_get('sql', 'county') + '-' + counter_incr('sql', 'county', 1) + '-' + counter_incr('sql', 'county', 1) + '-' + counter_get('sql', 'county')" expect = "0-1-2-2" "#; #[tokio::test] async fn lookup_sql() { // Enable logging crate::enable_logging(); // Parse settings let temp_dir = TempDir::new("smtp_lookup_tests", true); let mut config = Config::new(temp_dir.update_config(CONFIG)).unwrap(); let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; // Obtain directory handle let handle = DirectoryStore { store: core.storage.stores.get("sql").unwrap().clone(), }; let test = TestSMTP::from_core(core); test.server.mx_add( "test.org", vec![MX { exchanges: vec!["mx.foobar.org".to_string()], preference: 10, }], Instant::now() + Duration::from_secs(10), ); // Create tables handle.create_test_directory().await; // Create test records handle .create_test_user_with_email("jane@foobar.org", "s3cr3tp4ss", "Jane") .await; handle .create_test_user_with_email("john@foobar.org", "mypassword", "John") .await; handle .create_test_user_with_email("bill@foobar.org", "123456", "Bill") .await; handle .create_test_user_with_email("mike@foobar.net", "098765", "Mike") .await; for query in [ "CREATE TABLE domains (name TEXT PRIMARY KEY, description TEXT);", "INSERT INTO domains (name, description) VALUES ('foobar.org', 'Main domain');", "INSERT INTO domains (name, description) VALUES ('foobar.net', 'Secondary domain');", "CREATE TABLE allowed_ips (addr TEXT PRIMARY KEY);", "INSERT INTO allowed_ips (addr) VALUES ('10.0.0.50');", ] { handle .store .sql_query::(query, Vec::new()) .await .unwrap(); } // Create local domains let internal_store = &test.server.core.storage.data; for name in ["foobar.org", "foobar.net"] { internal_store .create_principal( PrincipalSet::new(0, Type::Domain).with_field(PrincipalField::Name, name), None, None, ) .await .unwrap(); } // Create lists internal_store .create_principal( PrincipalSet::new(0, Type::List) .with_field(PrincipalField::Name, "support@foobar.org") .with_field(PrincipalField::Emails, "support@foobar.org") .with_field( PrincipalField::ExternalMembers, PrincipalValue::StringList(vec!["mike@foobar.net".to_string()]), ), None, None, ) .await .unwrap(); internal_store .create_principal( PrincipalSet::new(0, Type::List) .with_field(PrincipalField::Name, "sales@foobar.org") .with_field(PrincipalField::Emails, "sales@foobar.org") .with_field( PrincipalField::ExternalMembers, PrincipalValue::StringList(vec![ "jane@foobar.org".to_string(), "john@foobar.org".to_string(), "bill@foobar.org".to_string(), ]), ), None, None, ) .await .unwrap(); // Test expression functions let token_map = TokenMap::default().with_variables(&[ V_RECIPIENT, V_RECIPIENT_DOMAIN, V_SENDER, V_SENDER_DOMAIN, V_MX, V_HELO_DOMAIN, V_AUTHENTICATED_AS, V_LISTENER, V_REMOTE_IP, V_LOCAL_IP, V_PRIORITY, ]); for test_name in ["sql", "dns", "key_get", "counter_get"] { let e = Expression::try_parse(&mut config, ("test", test_name, "expr"), &token_map).unwrap(); assert_eq!( test.server .eval_expr::(&e, &RecipientDomain::new("test.org"), "text", 0) .await .unwrap(), config.value(("test", test_name, "expect")).unwrap(), "failed for '{}'", test_name ); } let mut session = Session::test(test.server); session.data.remote_ip_str = "10.0.0.50".parse().unwrap(); session.eval_session_params().await; session.stream.tls = true; session .ehlo("mx.foobar.org") .await .assert_contains("REQUIRETLS"); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session .ehlo("mx1.foobar.org") .await .assert_not_contains("REQUIRETLS"); // Test RCPT session.mail_from("john@example.net", "250").await; // External domain session.rcpt_to("user@otherdomain.org", "550 5.1.2").await; // Non-existent user session.rcpt_to("jack@foobar.org", "550 5.1.2").await; // Valid users session.rcpt_to("jane@foobar.org", "250").await; session.rcpt_to("john@foobar.org", "250").await; session.rcpt_to("bill@foobar.org", "250").await; // Lists session.rcpt_to("sales@foobar.org", "250").await; // Test EXPN session .cmd("EXPN sales@foobar.org", "250") .await .assert_contains("jane@foobar.org") .assert_contains("john@foobar.org") .assert_contains("bill@foobar.org"); session .cmd("EXPN support@foobar.org", "250") .await .assert_contains("mike@foobar.net"); session.cmd("EXPN marketing@foobar.org", "550 5.1.2").await; // Test VRFY session .server .core .storage .directory .query(QueryParams::name("john@foobar.org").with_return_member_of(true)) .await .unwrap() .unwrap(); session .server .core .storage .directory .query(QueryParams::name("jane@foobar.org").with_return_member_of(true)) .await .unwrap() .unwrap(); session .cmd("VRFY john", "250") .await .assert_contains("john@foobar.org"); session .cmd("VRFY jane", "250") .await .assert_contains("jane@foobar.org"); session.cmd("VRFY tim", "550 5.1.2").await; // Test AUTH session .cmd( "AUTH PLAIN AGphbmVAZm9vYmFyLm9yZwB3cm9uZ3Bhc3M=", "535 5.7.8", ) .await; session .cmd( "AUTH PLAIN AGphbmVAZm9vYmFyLm9yZwBzM2NyM3RwNHNz", "235 2.7.0", ) .await; } ================================================ FILE: tests/src/smtp/lookup/utils.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::net::IpAddr; use crate::smtp::TestSMTP; use ::smtp::outbound::NextHop; use common::{ Core, config::smtp::{ queue::{MxConfig, QueueExpiry, QueueName}, report::AggregateFrequency, resolver::{Mode, MxPattern, Policy}, }, }; use mail_auth::{IpLookupStrategy, MX}; use mail_parser::DateTime; use smtp::{ outbound::{ lookup::{SourceIp, ToNextHop}, mta_sts::parse::ParsePolicy, }, queue::{ Error, ErrorDetails, FROM_AUTHENTICATED, Message, QueueEnvelope, Recipient, Schedule, Status, }, reporting::AggregateTimestamp, }; use store::write::now; use utils::config::Config; const CONFIG: &str = r#" [queue.connection.test.timeout] connect = "10s" [queue.connection.test] ehlo-hostname = "test.example.com" source-ips = ["10.0.0.1", "10.0.0.2", "10.0.0.3", "10.0.0.4", "a:b::1", "a:b::2", "a:b::3", "a:b::4"] [queue.source-ip."10.0.0.1"] ehlo-hostname = "test1.example.com" [queue.source-ip."10.0.0.2"] ehlo-hostname = "test2.example.com" [queue.source-ip."10.0.0.3"] ehlo-hostname = "test3.example.com" [queue.source-ip."10.0.0.4"] ehlo-hostname = "test4.example.com" [queue.source-ip."a:b::1"] ehlo-hostname = "test5.example.com" [queue.source-ip."a:b::2"] ehlo-hostname = "test6.example.com" [queue.source-ip."a:b::3"] ehlo-hostname = "test7.example.com" [queue.source-ip."a:b::4"] ehlo-hostname = "test8.example.com" [queue.test-v4.type] type = "mx" ip-lookup-strategy = "ipv4_then_ipv6" [queue.test-v6.type] type = "mx" ip-lookup-strategy = "ipv6_then_ipv4" [queue.strategy] schedule = "source + ' ' + received_from_ip + ' ' + received_via_port + ' ' + queue_name + ' ' + last_error + ' ' + rcpt_domain + ' ' + size + ' ' + queue_age" "#; #[tokio::test] async fn strategies() { // Enable logging crate::enable_logging(); let ipv6: [IpAddr; 4] = [ "a:b::1".parse().unwrap(), "a:b::2".parse().unwrap(), "a:b::3".parse().unwrap(), "a:b::4".parse().unwrap(), ]; let ipv4: [IpAddr; 4] = [ "10.0.0.1".parse().unwrap(), "10.0.0.2".parse().unwrap(), "10.0.0.3".parse().unwrap(), "10.0.0.4".parse().unwrap(), ]; let ipv4_hosts = [ "test1.example.com".to_string(), "test2.example.com".to_string(), "test3.example.com".to_string(), "test4.example.com".to_string(), ]; let ipv6_hosts = [ "test5.example.com".to_string(), "test6.example.com".to_string(), "test7.example.com".to_string(), "test8.example.com".to_string(), ]; let mut config = Config::new(CONFIG).unwrap(); let test = TestSMTP::from_core(Core::parse(&mut config, Default::default(), Default::default()).await); let conn = test .server .core .smtp .queue .connection_strategy .get("test") .unwrap(); assert_eq!(conn.ehlo_hostname.as_ref().unwrap(), "test.example.com"); for is_ipv4 in [true, false] { for _ in 0..10 { let ip_host = conn.source_ip(is_ipv4).unwrap(); if is_ipv4 { assert_eq!( &ipv4_hosts[ipv4.iter().position(|&ip| ip == ip_host.ip).unwrap()], ip_host.host.as_ref().unwrap() ); } else { assert_eq!( &ipv6_hosts[ipv6.iter().position(|&ip| ip == ip_host.ip).unwrap()], ip_host.host.as_ref().unwrap() ); } } } // Test strategy resolution let message = Message { created: now() - 123, blob_hash: Default::default(), received_from_ip: "1.2.3.4".parse().unwrap(), received_via_port: 7911, return_path: "test@example.com".into(), recipients: vec![Recipient { address: "recipient@foobar.com".into(), retry: Schedule::now(), notify: Schedule::now(), expires: QueueExpiry::Ttl(3600), queue: QueueName::new("test").unwrap(), status: Status::TemporaryFailure(ErrorDetails { entity: "test.example.com".into(), details: Error::TlsError("TLS handshake failed".into()), }), flags: 0, orcpt: None, }], flags: FROM_AUTHENTICATED, env_id: None, priority: 0, size: 978, quota_keys: Default::default(), }; assert_eq!( test.server .eval_if::( &test.server.core.smtp.queue.queue, &QueueEnvelope::new(&message, &message.recipients[0]), 0, ) .await .unwrap_or_else(|| "default".to_string()), "authenticated 1.2.3.4 7911 test tls foobar.com 978 123" ); } #[test] fn to_remote_hosts() { let mx = vec![ MX { exchanges: vec!["mx1".to_string(), "mx2".to_string()], preference: 10, }, MX { exchanges: vec![ "mx3".to_string(), "mx4".to_string(), "mx5".to_string(), "mx6".to_string(), ], preference: 20, }, MX { exchanges: vec!["mx7".to_string(), "mx8".to_string()], preference: 10, }, MX { exchanges: vec!["mx9".to_string(), "mxA".to_string()], preference: 10, }, ]; let mx_config = MxConfig { max_mx: 7, max_multi_homed: 2, ip_lookup_strategy: IpLookupStrategy::Ipv4thenIpv6, }; let hosts = mx.to_remote_hosts("domain", &mx_config).unwrap(); assert_eq!(hosts.len(), 7); for host in hosts { if let NextHop::MX { host, .. } = host { assert!((*host.as_bytes().last().unwrap() - b'0') <= 8); } } let mx = vec![MX { exchanges: vec![".".to_string()], preference: 0, }]; assert!(mx.to_remote_hosts("domain", &mx_config).is_none()); } #[test] fn parse_policy() { for (policy, expected_policy) in [ ( r"version: STSv1 mode: enforce mx: mail.example.com mx: *.example.net mx: backupmx.example.com max_age: 604800", Policy { id: "abc".to_string(), mode: Mode::Enforce, mx: vec![ MxPattern::Equals("mail.example.com".to_string()), MxPattern::StartsWith("example.net".to_string()), MxPattern::Equals("backupmx.example.com".to_string()), ], max_age: 604800, }, ), ( r"version: STSv1 mode: testing mx: gmail-smtp-in.l.google.com mx: *.gmail-smtp-in.l.google.com max_age: 86400 ", Policy { id: "abc".to_string(), mode: Mode::Testing, mx: vec![ MxPattern::Equals("gmail-smtp-in.l.google.com".to_string()), MxPattern::StartsWith("gmail-smtp-in.l.google.com".to_string()), ], max_age: 86400, }, ), ] { assert_eq!( Policy::parse(policy, expected_policy.id.to_string()).unwrap(), expected_policy ); } } #[test] fn aggregate_to_timestamp() { for (freq, date, expected) in [ ( AggregateFrequency::Hourly, "2023-01-24T09:10:40Z", "2023-01-24T09:00:00Z", ), ( AggregateFrequency::Daily, "2023-01-24T09:10:40Z", "2023-01-24T00:00:00Z", ), ( AggregateFrequency::Weekly, "2023-01-24T09:10:40Z", "2023-01-22T00:00:00Z", ), ( AggregateFrequency::Weekly, "2023-01-28T23:59:59Z", "2023-01-22T00:00:00Z", ), ( AggregateFrequency::Weekly, "2023-01-22T23:59:59Z", "2023-01-22T00:00:00Z", ), ] { assert_eq!( DateTime::from_timestamp( freq.to_timestamp_(DateTime::parse_rfc3339(date).unwrap()) as i64 ) .to_rfc3339(), expected, "failed for {freq:?} {date} {expected}" ); } } ================================================ FILE: tests/src/smtp/management/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod queue; pub mod report; ================================================ FILE: tests/src/smtp/management/queue.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::{Duration, Instant}; use ahash::{AHashMap, HashMap, HashSet}; use common::config::server::ServerProtocol; use http::management::queue::Message; use mail_auth::MX; use mail_parser::DateTime; use reqwest::{Method, StatusCode, header::AUTHORIZATION}; use crate::{ jmap::ManagementApi, smtp::{DnsCache, TestSMTP, session::TestSession}, }; use smtp::queue::{QueueId, Status, manager::SpawnQueue}; const LOCAL: &str = r#" [storage] directory = "local" [directory."local"] type = "memory" [[directory."local".principals]] name = "admin" type = "admin" description = "Superuser" secret = "secret" class = "admin" [queue.schedule.default] retry = "1000s" notify = "2000s" expire = "3000s" queue-name = "default" [session.rcpt] relay = true max-recipients = 100 [session.extensions] dsn = true future-release = "1h" "#; const REMOTE: &str = r#" [session.ehlo] reject-non-fqdn = false [session.rcpt] relay = true "#; #[derive(serde::Deserialize, Debug)] #[allow(dead_code)] pub(super) struct List { pub items: Vec, pub total: usize, } #[tokio::test] #[serial_test::serial] async fn manage_queue() { // Enable logging crate::enable_logging(); // Start remote test server let mut remote = TestSMTP::new("smtp_manage_queue_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Smtp]).await; let remote_core = remote.build_smtp(); // Start local management interface let local = TestSMTP::new("smtp_manage_queue_local", LOCAL).await; // Add mock DNS entries let core = local.build_smtp(); core.mx_add( "foobar.org", vec![MX { exchanges: vec!["mx1.foobar.org".to_string()], preference: 10, }], Instant::now() + Duration::from_secs(10), ); core.ipv4_add( "mx1.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); let _rx_manage = local.start(&[ServerProtocol::Http]).await; // Send test messages let envelopes = HashMap::from_iter([ ( "a", ( "bill1@foobar.net", vec![ "rcpt1@example1.org", "rcpt1@example2.org", "rcpt1@example2.org", ], ), ), ( "b", ( "bill2@foobar.net", vec!["rcpt3@example1.net", "rcpt4@example1.net"], ), ), ( "c", ( "bill3@foobar.net", vec![ "rcpt5@example1.com", "rcpt6@example2.com", "rcpt7@example2.com", "rcpt8@example3.com", "rcpt9@example4.com", ], ), ), ("d", ("bill4@foobar.net", vec!["delay@foobar.org"])), ("e", ("bill5@foobar.net", vec!["john@foobar.org"])), ("f", ("", vec!["success@foobar.org", "delay@foobar.org"])), ]); let mut session = local.new_session(); local .queue_receiver .queue_rx .spawn(local.server.inner.clone()); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("foobar.net").await; for test_num in 0..6 { let env_id = char::from(b'a' + test_num).to_string(); let hold_for = ((test_num + 1) as u32) * 100; let (sender, recipients) = envelopes.get(env_id.as_str()).unwrap(); session .send_message( &if env_id != "f" { format!("<{sender}> ENVID={env_id} HOLDFOR={hold_for}") } else { format!("<{sender}> ENVID={env_id}") }, recipients, "test:no_dkim", "250", ) .await; } // Expect delivery to success@foobar.org tokio::time::sleep(Duration::from_millis(100)).await; assert_eq!( remote .queue_receiver .consume_message(&remote_core) .await .message .recipients .into_iter() .map(|r| r.address().to_string()) .collect::>(), vec!["success@foobar.org"] ); // Fetch and validate messages let api = ManagementApi::default(); let ids = api .request::>(Method::GET, "/api/queue/messages") .await .unwrap() .unwrap_data() .items; assert_eq!(ids.len(), 6); let mut id_map = AHashMap::new(); let mut id_map_rev = AHashMap::new(); let mut test_search = String::new(); for (message, id) in api.get_messages(&ids).await.into_iter().zip(ids) { let message = message.unwrap(); let env_id = message.env_id.as_ref().unwrap().clone(); // Validate return path and recipients let (sender, recipients) = envelopes.get(env_id.as_str()).unwrap(); assert_eq!(&message.return_path, sender); 'outer: for recipient in recipients { for rcpt in &message.recipients { if &rcpt.address == recipient { continue 'outer; } } panic!("Recipient {recipient} not found in message."); } // Validate status and datetimes let created = message.created.to_timestamp(); let hold_for = (env_id.as_bytes().first().unwrap() - b'a' + 1) as i64 * 100; let next_retry = created + hold_for; let next_notify = created + 2000 + hold_for; let expires = created + 3000 + hold_for; for rcpt in &message.recipients { if env_id == "c" { let mut dt = *rcpt.next_retry.as_ref().unwrap(); dt.second -= 1; test_search = dt.to_rfc3339(); } if env_id != "f" { // HOLDFOR messages assert_eq!(rcpt.retry_num, 0); assert_timestamp( rcpt.next_retry.as_ref().unwrap(), next_retry, "retry", &message, ); assert_timestamp( rcpt.next_notify.as_ref().unwrap(), next_notify, "notify", &message, ); assert_timestamp(&rcpt.expires.unwrap(), expires, "expires", &message); assert_eq!(&rcpt.status, &Status::Scheduled, "{message:#?}"); } else if rcpt.address == "success@foobar.org" { assert_eq!(rcpt.retry_num, 0); assert!( matches!(&rcpt.status, Status::Completed(_)), "{:?}", rcpt.status ); } else { assert_eq!(rcpt.retry_num, 1); assert!( matches!(&rcpt.status, Status::TemporaryFailure(_)), "{:?}", rcpt.status ); } } id_map.insert(env_id.clone(), id); id_map_rev.insert(id, env_id); } assert_eq!(id_map.len(), 6); // Test list search for (query, expected_ids) in [ ( "/api/queue/messages?from=bill1@foobar.net".to_string(), vec!["a"], ), ( "/api/queue/messages?to=foobar.org".to_string(), vec!["d", "e", "f"], ), ( "/api/queue/messages?from=bill3@foobar.net&to=rcpt5@example1.com".to_string(), vec!["c"], ), ( format!("/api/queue/messages?before={test_search}"), vec!["a", "b"], ), ( format!("/api/queue/messages?after={test_search}"), vec!["d", "e", "f", "c"], ), ] { let expected_ids = HashSet::from_iter(expected_ids.into_iter().map(|s| s.to_string())); let ids = api .request::>(Method::GET, &query) .await .unwrap() .unwrap_data() .items .into_iter() .map(|id| id_map_rev.get(&id).unwrap().clone()) .collect::>(); assert_eq!(ids, expected_ids, "failed for {query}"); } // Retry delivery for id in [id_map.get("e").unwrap(), id_map.get("f").unwrap()] { assert!( api.request::(Method::PATCH, &format!("/api/queue/messages/{id}",)) .await .unwrap() .unwrap_data(), ); } assert!( api.request::( Method::PATCH, &format!( "/api/queue/messages/{}?filter=example1.org&at=2200-01-01T00:00:00Z", id_map.get("a").unwrap(), ) ) .await .unwrap() .unwrap_data() ); // Expect delivery to john@foobar.org tokio::time::sleep(Duration::from_millis(100)).await; assert_eq!( remote .queue_receiver .consume_message(&remote_core) .await .message .recipients .into_iter() .map(|r| r.address().to_string()) .collect::>(), vec!["john@foobar.org".to_string()] ); // Message 'e' should be gone, 'f' should have retry_num == 2 // while 'a' should have a retry time of 2200-01-01T00:00:00Z for example1.org let mut messages = api .get_messages(&[ *id_map.get("e").unwrap(), *id_map.get("f").unwrap(), *id_map.get("a").unwrap(), ]) .await .into_iter(); assert_eq!(messages.next().unwrap(), None); assert_eq!( messages .next() .unwrap() .unwrap() .recipients .first() .unwrap() .retry_num, 2 ); for rcpt in messages.next().unwrap().unwrap().recipients { let next_retry = rcpt.next_retry.as_ref().unwrap().to_rfc3339(); let matched = ["2200-01-01T00:00:00Z", "2199-12-31T23:59:59Z"].contains(&next_retry.as_str()); if rcpt.address.ends_with("example1.org") { assert!(matched, "{next_retry}"); } else { assert!(!matched, "{next_retry}"); } } // Cancel deliveries for (id, filter) in [ ("a", "example2.org"), ("b", "example1.net"), ("c", "rcpt6@example2.com"), ("d", ""), ] { assert!( api.request::( Method::DELETE, &format!( "/api/queue/messages/{}{}{}", id_map.get(id).unwrap(), if !filter.is_empty() { "?filter=" } else { "" }, filter ) ) .await .unwrap() .unwrap_data(), "failed for {id}: {filter}" ); } assert_eq!( api.request::>(Method::GET, "/api/queue/messages") .await .unwrap() .unwrap_data() .items .len(), 3 ); for (message, id) in api .get_messages(&[ *id_map.get("a").unwrap(), *id_map.get("b").unwrap(), *id_map.get("c").unwrap(), *id_map.get("d").unwrap(), ]) .await .into_iter() .zip(["a", "b", "c", "d"]) { if ["b", "d"].contains(&id) { assert_eq!(message, None); } else { let message = message.unwrap(); assert!(!message.recipients.is_empty()); for rcpt in message.recipients { match id { "a" => { if rcpt.address.ends_with("example2.org") { assert!(matches!(&rcpt.status, Status::PermanentFailure(_))); } else { assert!(matches!(&rcpt.status, Status::Scheduled)); } } "c" => { if rcpt.address.ends_with("example2.com") { if rcpt.address == "rcpt6@example2.com" { assert!(matches!(&rcpt.status, Status::PermanentFailure(_))); } else { assert!(matches!(&rcpt.status, Status::Scheduled)); } } else { assert!(matches!(&rcpt.status, Status::Scheduled)); } } _ => unreachable!(), } } } } // Bulk cancel assert_eq!( api.request::>(Method::GET, "/api/queue/messages?values=1") .await .unwrap() .unwrap_data() .items .len(), 3 ); assert!( api.request::(Method::DELETE, "/api/queue/messages?text=example2.com") .await .unwrap() .unwrap_data() ); tokio::time::sleep(Duration::from_millis(100)).await; assert_eq!( api.request::>(Method::GET, "/api/queue/messages") .await .unwrap() .unwrap_data() .items .len(), 2 ); assert!( api.request::(Method::DELETE, "/api/queue/messages") .await .unwrap() .unwrap_data() ); tokio::time::sleep(Duration::from_millis(100)).await; assert_eq!( api.request::>(Method::GET, "/api/queue/messages") .await .unwrap() .unwrap_data() .items .len(), 0 ); // Test authentication error assert_eq!( reqwest::Client::builder() .timeout(Duration::from_millis(500)) .danger_accept_invalid_certs(true) .build() .unwrap() .get("https://127.0.0.1:9980/api/queue/messages") .header(AUTHORIZATION, "Basic YWRtaW46aGVsbG93b3JsZA==") .send() .await .unwrap() .status(), StatusCode::UNAUTHORIZED ); } fn assert_timestamp(timestamp: &DateTime, expected: i64, ctx: &str, message: &Message) { let timestamp = timestamp.to_timestamp(); let diff = timestamp - expected; if ![-2, -1, 0, 1, 2].contains(&diff) { panic!( "Got timestamp {timestamp}, expected {expected} (diff {diff} for {ctx}) for {message:?}" ); } } impl ManagementApi { async fn get_messages(&self, ids: &[QueueId]) -> Vec> { let mut results = Vec::with_capacity(ids.len()); for id in ids { let message = self .request::(Method::GET, &format!("/api/queue/messages/{id}",)) .await .unwrap() .try_unwrap_data(); results.push(message); } results } } ================================================ FILE: tests/src/smtp/management/report.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::Arc; use ahash::{AHashMap, HashSet}; use common::{ config::{server::ServerProtocol, smtp::report::AggregateFrequency}, ipc::{DmarcEvent, PolicyType, TlsEvent}, }; use http::management::queue::Report; use mail_auth::{ common::parse::TxtRecordParser, dmarc::Dmarc, mta_sts::TlsRpt, report::{ ActionDisposition, DmarcResult, Record, tlsrpt::{FailureDetails, ResultType}, }, }; use reqwest::Method; use crate::{ jmap::ManagementApi, smtp::{TestSMTP, management::queue::List}, }; use smtp::reporting::{SmtpReporting, scheduler::SpawnReport}; const CONFIG: &str = r#" [storage] directory = "local" [directory."local"] type = "memory" [[directory."local".principals]] name = "admin" type = "admin" description = "Superuser" secret = "secret" class = "admin" [session.rcpt] relay = true [report.dmarc.aggregate] max-size = 1024 [report.tls.aggregate] max-size = 1024 "#; #[tokio::test] #[serial_test::serial] async fn manage_reports() { // Enable logging crate::enable_logging(); // Start reporting service let local = TestSMTP::new("smtp_manage_reports", CONFIG).await; let _rx = local.start(&[ServerProtocol::Http]).await; let core = local.build_smtp(); local .report_receiver .report_rx .spawn(local.server.inner.clone()); // Send test reporting events core.schedule_report(DmarcEvent { domain: "foobar.org".to_string(), report_record: Record::new() .with_source_ip("192.168.1.2".parse().unwrap()) .with_action_disposition(ActionDisposition::Pass) .with_dmarc_dkim_result(DmarcResult::Pass) .with_dmarc_spf_result(DmarcResult::Fail) .with_envelope_from("hello@example.org") .with_envelope_to("other@example.org") .with_header_from("bye@example.org"), dmarc_record: Arc::new( Dmarc::parse(b"v=DMARC1; p=reject; rua=mailto:reports@foobar.org").unwrap(), ), interval: AggregateFrequency::Daily, }) .await; core.schedule_report(DmarcEvent { domain: "foobar.net".to_string(), report_record: Record::new() .with_source_ip("a:b:c::e:f".parse().unwrap()) .with_action_disposition(ActionDisposition::Reject) .with_dmarc_dkim_result(DmarcResult::Fail) .with_dmarc_spf_result(DmarcResult::Pass), dmarc_record: Arc::new( Dmarc::parse( b"v=DMARC1; p=quarantine; rua=mailto:reports@foobar.net,mailto:reports@example.net", ) .unwrap(), ), interval: AggregateFrequency::Weekly, }) .await; core.schedule_report(TlsEvent { domain: "foobar.org".to_string(), policy: PolicyType::None, failure: None, tls_record: Arc::new(TlsRpt::parse(b"v=TLSRPTv1;rua=mailto:reports@foobar.org").unwrap()), interval: AggregateFrequency::Daily, }) .await; core.schedule_report(TlsEvent { domain: "foobar.net".to_string(), policy: PolicyType::Sts(None), failure: FailureDetails::new(ResultType::StsPolicyInvalid).into(), tls_record: Arc::new(TlsRpt::parse(b"v=TLSRPTv1;rua=mailto:reports@foobar.net").unwrap()), interval: AggregateFrequency::Weekly, }) .await; // List reports let api = ManagementApi::default(); let ids = api .request::>(Method::GET, "/api/queue/reports") .await .unwrap() .unwrap_data() .items; assert_eq!(ids.len(), 4); let mut id_map = AHashMap::new(); let mut id_map_rev = AHashMap::new(); for (report, id) in api.get_reports(&ids).await.into_iter().zip(ids) { let mut parts = id.split('!'); let report = report.unwrap(); let mut id_num = if parts.next().unwrap() == "t" { assert!(matches!(report, Report::Tls { .. })); 2 } else { assert!(matches!(report, Report::Dmarc { .. })); 0 }; let (domain, range_to, range_from) = match report { Report::Dmarc { domain, range_to, range_from, .. } => (domain, range_to, range_from), Report::Tls { domain, range_to, range_from, .. } => (domain, range_to, range_from), }; assert_eq!(parts.next().unwrap(), domain); let diff = range_to.to_timestamp() - range_from.to_timestamp(); if domain == "foobar.org" { assert_eq!(diff, 86400); } else { assert_eq!(diff, 7 * 86400); id_num += 1; } id_map.insert(char::from(b'a' + id_num).to_string(), id.clone()); id_map_rev.insert(id, char::from(b'a' + id_num).to_string()); } // Test list search for (query, expected_ids) in [ ("/api/queue/reports?type=dmarc", vec!["a", "b"]), ("/api/queue/reports?type=tls", vec!["c", "d"]), ("/api/queue/reports?domain=foobar.org", vec!["a", "c"]), ("/api/queue/reports?domain=foobar.net", vec!["b", "d"]), ("/api/queue/reports?domain=foobar.org&type=dmarc", vec!["a"]), ("/api/queue/reports?domain=foobar.net&type=tls", vec!["d"]), ] { let expected_ids = HashSet::from_iter(expected_ids.into_iter().map(|s| s.to_string())); let ids = api .request::>(Method::GET, query) .await .unwrap() .unwrap_data() .items .into_iter() .map(|id| id_map_rev.get(&id).unwrap().clone()) .collect::>(); assert_eq!(ids, expected_ids, "failed for {query}"); } // Cancel reports for id in ["a", "b"] { assert!( api.request::( Method::DELETE, &format!("/api/queue/reports/{}", id_map.get(id).unwrap(),) ) .await .unwrap() .unwrap_data(), "failed for {id}" ); } assert_eq!( api.request::>(Method::GET, "/api/queue/reports") .await .unwrap() .unwrap_data() .items .len(), 2 ); let mut ids = api .get_reports(&[ id_map.get("a").unwrap().clone(), id_map.get("b").unwrap().clone(), id_map.get("c").unwrap().clone(), id_map.get("d").unwrap().clone(), ]) .await .into_iter(); assert!(ids.next().unwrap().is_none()); assert!(ids.next().unwrap().is_none()); assert!(ids.next().unwrap().is_some()); assert!(ids.next().unwrap().is_some()); // Cancel all reports assert!( api.request::(Method::DELETE, "/api/queue/reports") .await .unwrap() .unwrap_data() ); assert_eq!( api.request::>(Method::GET, "/api/queue/reports") .await .unwrap() .unwrap_data() .items .len(), 0 ); } impl ManagementApi { async fn get_reports(&self, ids: &[String]) -> Vec> { let mut results = Vec::with_capacity(ids.len()); for id in ids { let report = self .request::(Method::GET, &format!("/api/queue/reports/{id}",)) .await .unwrap() .try_unwrap_data(); results.push(report); } results } } ================================================ FILE: tests/src/smtp/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ net::{IpAddr, Ipv4Addr, Ipv6Addr}, path::PathBuf, sync::Arc, }; use common::{ Core, Data, Inner, Server, config::{ server::{Listeners, ServerProtocol}, smtp::resolver::Tlsa, spamfilter::IpResolver, }, ipc::{QueueEvent, ReportingEvent}, manager::boot::{IpcReceivers, build_ipc}, }; use http::HttpSessionManager; use mail_auth::{MX, Txt, common::resolver::IntoFqdn}; use session::{DummyIo, TestSession}; use smtp::core::{Session, SmtpSessionManager}; use store::{BlobStore, Store, Stores}; use tokio::sync::{mpsc, watch}; use utils::config::Config; use crate::{AssertConfig, store::cleanup::store_destroy}; pub mod config; pub mod inbound; pub mod lookup; pub mod management; pub mod outbound; pub mod queue; pub mod reporting; pub mod session; pub struct TempDir { pub temp_dir: PathBuf, pub delete: bool, } impl TempDir { pub fn new(name: &str, delete: bool) -> TempDir { let mut temp_dir = std::env::temp_dir(); temp_dir.push(name); if !temp_dir.exists() { let _ = std::fs::create_dir(&temp_dir); } else if delete { let _ = std::fs::remove_dir_all(&temp_dir); let _ = std::fs::create_dir(&temp_dir); } TempDir { temp_dir, delete } } pub fn update_config(&self, config: impl AsRef) -> String { config .as_ref() .replace("{TMP}", self.temp_dir.to_str().unwrap()) } } impl Drop for TempDir { fn drop(&mut self) { if self.delete { let _ = std::fs::remove_dir_all(&self.temp_dir); } } } pub fn add_test_certs(config: &str) -> String { let mut cert_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); cert_path.push("resources"); cert_path.push("smtp"); cert_path.push("certs"); let mut cert = cert_path.clone(); cert.push("tls_cert.pem"); let mut pk = cert_path.clone(); pk.push("tls_privatekey.pem"); config .replace("{CERT}", cert.as_path().to_str().unwrap()) .replace("{PK}", pk.as_path().to_str().unwrap()) } pub struct QueueReceiver { store: Store, blob_store: BlobStore, pub queue_rx: mpsc::Receiver, } pub struct ReportReceiver { pub report_rx: mpsc::Receiver, } pub struct TestSMTP { pub server: Server, pub temp_dir: Option, pub queue_receiver: QueueReceiver, pub report_receiver: ReportReceiver, } const CONFIG: &str = r#" [session.connect] hostname = "'mx.example.org'" greeting = "'Test SMTP instance'" [server.listener.smtp-debug] bind = ['127.0.0.1:9925'] protocol = 'smtp' [server.listener.lmtp-debug] bind = ['127.0.0.1:9924'] protocol = 'lmtp' tls.implicit = true [server.listener.management-debug] bind = ['127.0.0.1:9980'] protocol = 'http' tls.implicit = true [server.socket] reuse-addr = true [server.tls] enable = true implicit = false certificate = 'default' [certificate.default] cert = '%{file:{CERT}}%' private-key = '%{file:{PK}}%' [storage] data = "{STORE}" fts = "{STORE}" blob = "{STORE}" lookup = "{STORE}" [store."rocksdb"] type = "rocksdb" path = "{TMP}/queue.db" #[store."foundationdb"] #type = "foundationdb" [store."postgresql"] type = "postgresql" host = "localhost" port = 5432 database = "stalwart" user = "postgres" password = "mysecretpassword" [store."mysql"] type = "mysql" host = "localhost" port = 3307 database = "stalwart" user = "root" password = "password" "#; impl TestSMTP { pub fn from_core(core: Core) -> Self { Self::from_core_and_tempdir(core, Default::default(), None) } pub fn inner_with_rxs(&self) -> (Arc, IpcReceivers) { let (ipc, ipc_rxs) = build_ipc(false); ( Inner { shared_core: self.server.core.as_ref().clone().into_shared(), data: Default::default(), ipc, cache: Default::default(), } .into(), ipc_rxs, ) } fn from_core_and_tempdir(core: Core, data: Data, temp_dir: Option) -> Self { let store = core.storage.data.clone(); let blob_store = core.storage.blob.clone(); let shared_core = core.into_shared(); let (ipc, mut ipc_rxs) = build_ipc(false); TestSMTP { queue_receiver: QueueReceiver { store, blob_store, queue_rx: ipc_rxs.queue_rx.take().unwrap(), }, report_receiver: ReportReceiver { report_rx: ipc_rxs.report_rx.take().unwrap(), }, server: Server { core: shared_core.load_full(), inner: Inner { shared_core, data, ipc, cache: Default::default(), } .into(), }, temp_dir, } } pub async fn new(name: &str, config: impl AsRef) -> TestSMTP { Self::with_database(name, config, "rocksdb").await } pub async fn with_database( name: &str, config: impl AsRef, store_id: impl AsRef, ) -> TestSMTP { let temp_dir = TempDir::new(name, true); let mut config = Config::new( temp_dir .update_config(add_test_certs(CONFIG) + config.as_ref()) .replace("{STORE}", store_id.as_ref()), ) .unwrap(); config.resolve_all_macros().await; let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; let data = Data::parse(&mut config); store_destroy(&core.storage.data).await; Self::from_core_and_tempdir(core, data, Some(temp_dir)) } pub async fn start(&self, protocols: &[ServerProtocol]) -> watch::Sender { // Spawn listeners let mut config = Config::new(CONFIG).unwrap(); let mut servers = Listeners::parse(&mut config); servers.parse_tcp_acceptors(&mut config, self.server.inner.clone()); // Filter out protocols servers .servers .retain(|server| protocols.contains(&server.protocol)); // Start servers servers.bind_and_drop_priv(&mut config); config.assert_no_errors(); servers .spawn(|server, acceptor, shutdown_rx| { match &server.protocol { ServerProtocol::Smtp | ServerProtocol::Lmtp => server.spawn( SmtpSessionManager::new(self.server.inner.clone()), self.server.inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Http => server.spawn( HttpSessionManager::new(self.server.inner.clone()), self.server.inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Imap | ServerProtocol::Pop3 | ServerProtocol::ManageSieve => { unreachable!() } }; }) .0 } pub fn new_session(&self) -> Session { Session::test(self.server.clone()) } pub fn build_smtp(&self) -> Server { self.server.clone() } } pub trait DnsCache { fn txt_add<'x>( &self, name: impl IntoFqdn<'x>, value: impl Into, valid_until: std::time::Instant, ); fn ipv4_add<'x>( &self, name: impl IntoFqdn<'x>, value: Vec, valid_until: std::time::Instant, ); fn ipv6_add<'x>( &self, name: impl IntoFqdn<'x>, value: Vec, valid_until: std::time::Instant, ); fn dnsbl_add(&self, name: &str, value: Vec, valid_until: std::time::Instant); fn ptr_add(&self, name: IpAddr, value: Vec, valid_until: std::time::Instant); fn mx_add<'x>(&self, name: impl IntoFqdn<'x>, value: Vec, valid_until: std::time::Instant); fn tlsa_add<'x>( &self, name: impl IntoFqdn<'x>, value: Arc, valid_until: std::time::Instant, ); } impl DnsCache for Server { fn txt_add<'x>( &self, name: impl IntoFqdn<'x>, value: impl Into, valid_until: std::time::Instant, ) { self.inner.cache.dns_txt.insert_with_expiry( name.into_fqdn().into_owned(), value.into(), valid_until, ); } fn ipv4_add<'x>( &self, name: impl IntoFqdn<'x>, value: Vec, valid_until: std::time::Instant, ) { self.inner.cache.dns_ipv4.insert_with_expiry( name.into_fqdn().into_owned(), Arc::new(value), valid_until, ); } fn dnsbl_add(&self, name: &str, value: Vec, valid_until: std::time::Instant) { self.inner.cache.dns_rbl.insert_with_expiry( name.to_string(), Some(Arc::new(IpResolver::new( value .iter() .copied() .next() .unwrap_or(Ipv4Addr::BROADCAST) .into(), ))), valid_until, ); } fn ipv6_add<'x>( &self, name: impl IntoFqdn<'x>, value: Vec, valid_until: std::time::Instant, ) { self.inner.cache.dns_ipv6.insert_with_expiry( name.into_fqdn().into_owned(), Arc::new(value), valid_until, ); } fn ptr_add(&self, name: IpAddr, value: Vec, valid_until: std::time::Instant) { self.inner .cache .dns_ptr .insert_with_expiry(name, Arc::new(value), valid_until); } fn mx_add<'x>(&self, name: impl IntoFqdn<'x>, value: Vec, valid_until: std::time::Instant) { self.inner.cache.dns_mx.insert_with_expiry( name.into_fqdn().into_owned(), Arc::new(value), valid_until, ); } fn tlsa_add<'x>( &self, name: impl IntoFqdn<'x>, value: Arc, valid_until: std::time::Instant, ) { self.inner.cache.dns_tlsa.insert_with_expiry( name.into_fqdn().into_owned(), value, valid_until, ); } } ================================================ FILE: tests/src/smtp/outbound/dane.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::smtp::{ DnsCache, TestSMTP, inbound::{TestMessage, TestQueueEvent, TestReportingEvent}, session::{TestSession, VerifyResponse}, }; use common::{ Core, config::{ server::ServerProtocol, smtp::resolver::{DnssecResolver, Resolvers, Tlsa, TlsaEntry}, }, ipc::PolicyType, }; use mail_auth::{ MX, MessageAuthenticator, common::parse::TxtRecordParser, hickory_resolver::{ TokioResolver, config::{ResolverConfig, ResolverOpts}, name_server::TokioConnectionProvider, }, mta_sts::{ReportUri, TlsRpt}, report::tlsrpt::ResultType, }; use rustls_pki_types::CertificateDer; use smtp::outbound::dane::{dnssec::TlsaLookup, verify::TlsaVerify}; use smtp::queue::{Error, ErrorDetails, Status}; use std::{ collections::BTreeSet, fs::{self, File}, io::{BufRead, BufReader}, num::ParseIntError, path::PathBuf, sync::Arc, time::{Duration, Instant}, }; const LOCAL: &str = r#" [session.rcpt] relay = true [report.tls.aggregate] send = "weekly" [queue.tls.default] dane = "require" starttls = "require" "#; const REMOTE: &str = " [session.ehlo] reject-non-fqdn = false [session.rcpt] relay = true [session.data.add-headers] received = true received-spf = true auth-results = true message-id = true date = true return-path = false "; #[tokio::test] #[serial_test::serial] async fn dane_verify() { // Enable logging crate::enable_logging(); // Start test server let mut remote = TestSMTP::new("smtp_dane_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Smtp]).await; // Fail on missing TLSA record let mut local = TestSMTP::new("smtp_dane_local", LOCAL).await; // Add mock DNS entries let core = local.build_smtp(); core.mx_add( "foobar.org", vec![MX { exchanges: vec!["mx.foobar.org".to_string()], preference: 10, }], Instant::now() + Duration::from_secs(10), ); core.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); core.txt_add( "_smtp._tls.foobar.org", TlsRpt::parse(b"v=TLSRPTv1; rua=mailto:reports@foobar.org").unwrap(), Instant::now() + Duration::from_secs(10), ); let mut session = local.new_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); local .queue_receiver .expect_message() .await .read_lines(&local.queue_receiver) .await .assert_contains(" (DANE failed to authenticate") .assert_contains("No TLSA reco=") .assert_contains("rds found"); local.queue_receiver.read_event().await.assert_done(); local.queue_receiver.assert_no_events(); // Expect TLS failure report let report = local.report_receiver.read_report().await.unwrap_tls(); assert_eq!(report.domain, "foobar.org"); assert_eq!(report.policy, PolicyType::Tlsa(None)); assert_eq!( report.failure.as_ref().unwrap().result_type, ResultType::DaneRequired ); assert_eq!( report.failure.as_ref().unwrap().receiving_mx_hostname, Some("mx.foobar.org".to_string()) ); assert_eq!( report.tls_record.rua, vec![ReportUri::Mail("reports@foobar.org".to_string())] ); // DANE failure with no matching certificates let tlsa = Arc::new(Tlsa { entries: vec![TlsaEntry { is_end_entity: true, is_sha256: true, is_spki: true, data: vec![1, 2, 3], }], has_end_entities: true, has_intermediates: false, }); core.tlsa_add( "_25._tcp.mx.foobar.org", tlsa.clone(), Instant::now() + Duration::from_secs(10), ); session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); local .queue_receiver .expect_message() .await .read_lines(&local.queue_receiver) .await .assert_contains(" (DANE failed to authenticate") .assert_contains("No matching ") .assert_contains("certificates found"); local.queue_receiver.read_event().await.assert_done(); local.queue_receiver.assert_no_events(); // Expect TLS failure report let report = local.report_receiver.read_report().await.unwrap_tls(); assert_eq!(report.policy, PolicyType::Tlsa(tlsa.into())); assert_eq!( report.failure.as_ref().unwrap().result_type, ResultType::ValidationFailure ); remote.queue_receiver.assert_no_events(); // DANE successful delivery let tlsa = Arc::new(Tlsa { entries: vec![TlsaEntry { is_end_entity: true, is_sha256: true, is_spki: true, data: vec![ 73, 186, 44, 106, 13, 198, 100, 180, 0, 44, 158, 188, 15, 195, 39, 198, 61, 254, 215, 237, 100, 26, 15, 155, 219, 235, 120, 64, 128, 172, 17, 0, ], }], has_end_entities: true, has_intermediates: false, }); core.tlsa_add( "_25._tcp.mx.foobar.org", tlsa.clone(), Instant::now() + Duration::from_secs(10), ); session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); local.queue_receiver.read_event().await.assert_done(); local.queue_receiver.assert_no_events(); remote .queue_receiver .expect_message() .await .read_lines(&remote.queue_receiver) .await .assert_contains("using TLSv1.3 with cipher"); // Expect TLS success report let report = local.report_receiver.read_report().await.unwrap_tls(); assert_eq!(report.policy, PolicyType::Tlsa(tlsa.into())); assert!(report.failure.is_none()); } #[tokio::test] async fn dane_test() { let conf = ResolverConfig::cloudflare_tls(); let mut opts = ResolverOpts::default(); opts.validate = true; opts.try_tcp_on_error = true; let mut core = Core::default(); core.smtp.resolvers = Resolvers { dns: MessageAuthenticator::new_cloudflare().unwrap(), dnssec: DnssecResolver { resolver: TokioResolver::builder_with_config(conf, TokioConnectionProvider::default()) .with_options(opts) .build(), }, }; let r = TestSMTP::from_core(core).build_smtp(); // Add dns entries let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); path.push("resources"); path.push("smtp"); path.push("dane"); let mut file = path.clone(); file.push("dns.txt"); let mut hosts = BTreeSet::new(); let mut tlsa = Tlsa { entries: Vec::new(), has_end_entities: false, has_intermediates: false, }; let mut hostname = String::new(); for line in BufReader::new(File::open(file).unwrap()).lines() { let line = line.unwrap(); let mut is_end_entity = false; for (pos, item) in line.split_whitespace().enumerate() { match pos { 0 => { if hostname != item && !hostname.is_empty() { r.tlsa_add( hostname, tlsa.into(), Instant::now() + Duration::from_secs(30), ); tlsa = Tlsa { entries: Vec::new(), has_end_entities: false, has_intermediates: false, }; } hosts.insert(item.strip_prefix("_25._tcp.").unwrap().to_string()); hostname = item.to_string(); } 1 => { is_end_entity = item == "3"; } 4 => { if is_end_entity { tlsa.has_end_entities = true; } else { tlsa.has_intermediates = true; } tlsa.entries.push(TlsaEntry { is_end_entity, is_sha256: true, is_spki: true, data: decode_hex(item).unwrap(), }); } _ => (), } } } r.tlsa_add( hostname, tlsa.into(), Instant::now() + Duration::from_secs(30), ); // Add certificates assert!(!hosts.is_empty()); for host in hosts { // Add certificates let mut certs = Vec::new(); for num in 0..6 { let mut file = path.clone(); file.push(format!("{host}.{num}.cert")); if file.exists() { certs.push(CertificateDer::from(fs::read(file).unwrap())); } else { break; } } // Successful DANE verification let tlsa = r .tlsa_lookup(format!("_25._tcp.{host}.")) .await .unwrap() .unwrap(); assert_eq!(tlsa.verify(0, &host, Some(&certs)), Ok(())); // Failed DANE verification certs.remove(0); assert_eq!( tlsa.verify(0, &host, Some(&certs)), Err(Status::PermanentFailure(ErrorDetails { entity: host.into(), details: Error::DaneError("No matching certificates found in TLSA records".into()) })) ); } } pub fn decode_hex(s: &str) -> Result, ParseIntError> { (0..s.len()) .step_by(2) .map(|i| u8::from_str_radix(&s[i..i + 2], 16)) .collect() } ================================================ FILE: tests/src/smtp/outbound/extensions.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::{Duration, Instant}; use common::config::server::ServerProtocol; use mail_auth::MX; use smtp_proto::{MAIL_REQUIRETLS, MAIL_RET_HDRS, MAIL_SMTPUTF8, RCPT_NOTIFY_NEVER}; use crate::smtp::{ DnsCache, TestSMTP, inbound::{TestMessage, TestQueueEvent}, session::{TestSession, VerifyResponse}, }; const LOCAL: &str = r#" [session.rcpt] relay = true [session.extensions] dsn = true "#; const REMOTE: &str = r#" [session.ehlo] reject-non-fqdn = false [session.rcpt] relay = true [session.data.limits] size = 1500 [session.extensions] dsn = true requiretls = true [session.data.add-headers] received = true received-spf = true auth-results = true message-id = true date = true return-path = false "#; #[tokio::test] #[serial_test::serial] async fn extensions() { // Enable logging crate::enable_logging(); // Start test server let mut remote = TestSMTP::new("smtp_ext_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Smtp]).await; // Successful delivery with DSN let mut local = TestSMTP::new("smtp_ext_local", LOCAL).await; // Add mock DNS entries let core = local.build_smtp(); core.mx_add( "foobar.org", vec![MX { exchanges: vec!["mx.foobar.org".to_string()], preference: 10, }], Instant::now() + Duration::from_secs(10), ); core.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); let mut session = local.new_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; session .send_message( "john@test.org", &[" NOTIFY=SUCCESS,FAILURE"], "test:no_dkim", "250", ) .await; local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); local .queue_receiver .expect_message() .await .read_lines(&local.queue_receiver) .await .assert_contains(" (delivered to") .assert_contains("Final-Recipient: rfc822;bill@foobar.org") .assert_contains("Action: delivered"); local.queue_receiver.read_event().await.assert_done(); remote .queue_receiver .expect_message() .await .read_lines(&remote.queue_receiver) .await .assert_contains("using TLSv1.3 with cipher"); // Test SIZE extension session .send_message("john@test.org", &["bill@foobar.org"], "test:arc", "250") .await; local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); local .queue_receiver .expect_message() .await .read_lines(&local.queue_receiver) .await .assert_contains(" (host 'mx.foobar.org' rejected command 'MAIL FROM:") .assert_contains("Action: failed") .assert_contains("Diagnostic-Code: smtp;552") .assert_contains("Status: 5.3.4"); local.queue_receiver.read_event().await.assert_done(); remote.queue_receiver.assert_no_events(); // Test DSN, SMTPUTF8 and REQUIRETLS extensions session .send_message( " ENVID=abc123 RET=HDRS REQUIRETLS SMTPUTF8", &[" NOTIFY=NEVER"], "test:no_dkim", "250", ) .await; local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); local.queue_receiver.read_event().await.assert_done(); let message = remote.queue_receiver.expect_message().await; assert_eq!(message.message.env_id, Some("abc123".into())); assert!((message.message.flags & MAIL_RET_HDRS) != 0); assert!((message.message.flags & MAIL_REQUIRETLS) != 0); assert!((message.message.flags & MAIL_SMTPUTF8) != 0); assert!((message.message.recipients.last().unwrap().flags & RCPT_NOTIFY_NEVER) != 0); } ================================================ FILE: tests/src/smtp/outbound/fallback_relay.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::{Duration, Instant}; use common::config::server::ServerProtocol; use mail_auth::MX; use store::write::now; use crate::smtp::{DnsCache, TestSMTP, session::TestSession}; const LOCAL: &str = r#" [queue.strategy] route = [{if = "retry_num > 0", then = "'fallback'"}, {else = "'mx'"}] [session.rcpt] relay = true max-recipients = 100 [session.extensions] dsn = true [queue.route.fallback] type = "relay" address = fallback.foobar.org port = 9925 protocol = 'smtp' concurrency = 5 [queue.route.fallback.tls] implicit = false allow-invalid-certs = true "#; const REMOTE: &str = r#" [session.rcpt] relay = true [session.ehlo] reject-non-fqdn = false [session.extensions] dsn = true chunking = false "#; #[tokio::test] #[serial_test::serial] async fn fallback_relay() { // Enable logging crate::enable_logging(); // Start test server let mut remote = TestSMTP::new("smtp_fallback_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Smtp]).await; let mut local = TestSMTP::new("smtp_fallback_local", LOCAL).await; // Add mock DNS entries let core = local.build_smtp(); core.mx_add( "foobar.org", vec![MX { exchanges: vec!["_dns_error.foobar.org".to_string()], preference: 10, }], Instant::now() + Duration::from_secs(10), ); /*core.ipv4_add( "unreachable.foobar.org", vec!["127.0.0.2".parse().unwrap()], Instant::now() + Duration::from_secs(10), );*/ core.ipv4_add( "fallback.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); let mut session = local.new_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); let mut retry = local.queue_receiver.expect_message().await; let prev_due = retry.message.recipients[0].retry.due; let next_due = now(); let queue_id = retry.queue_id; retry.message.recipients[0].retry.due = next_due; retry.save_changes(&core, prev_due.into()).await; local .queue_receiver .delivery_attempt(queue_id) .await .try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; remote.queue_receiver.expect_message().await; } ================================================ FILE: tests/src/smtp/outbound/ip_lookup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::{Duration, Instant}; use common::config::server::ServerProtocol; use mail_auth::{IpLookupStrategy, MX}; use crate::smtp::{DnsCache, TestSMTP, session::TestSession}; const LOCAL: &str = r#" [session.rcpt] relay = true [queue.route.mx] ip-lookup = "ipv6_then_ipv4" "#; const REMOTE: &str = r#" [session.ehlo] reject-non-fqdn = false [session.rcpt] relay = true "#; #[tokio::test] #[serial_test::serial] async fn ip_lookup_strategy() { // Enable logging crate::enable_logging(); // Start test server let mut remote = TestSMTP::new("smtp_iplookup_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Smtp]).await; for strategy in [IpLookupStrategy::Ipv6Only, IpLookupStrategy::Ipv6thenIpv4] { //println!("-> Strategy: {:?}", strategy); // Add mock DNS entries let mut local = TestSMTP::new("smtp_iplookup_local", LOCAL).await; let core = local.build_smtp(); core.mx_add( "foobar.org", vec![MX { exchanges: vec!["mx.foobar.org".to_string()], preference: 10, }], Instant::now() + Duration::from_secs(10), ); if matches!(strategy, IpLookupStrategy::Ipv6thenIpv4) { core.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); } core.ipv6_add( "mx.foobar.org", vec!["::1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); // Retry on failed STARTTLS let mut session = local.new_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; if matches!(strategy, IpLookupStrategy::Ipv6thenIpv4) { remote.queue_receiver.expect_message().await; } else { let message = local.queue_receiver.last_queued_message().await; let status = message.message.recipients[0].status.to_string(); assert!( status.contains("Connection refused"), "Message: {:?}", message ); } } } ================================================ FILE: tests/src/smtp/outbound/lmtp.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::{Duration, Instant}; use crate::smtp::{ DnsCache, TestSMTP, inbound::TestMessage, queue::QueuedEvents, session::{TestSession, VerifyResponse}, }; use common::{ config::{server::ServerProtocol, smtp::queue::QueueName}, ipc::QueueEvent, }; use smtp::queue::spool::{QUEUE_REFRESH, SmtpSpool}; use store::write::now; const REMOTE: &str = " [session.ehlo] reject-non-fqdn = false [session.rcpt] relay = true [session.extensions] dsn = true "; const LOCAL: &str = r#" [queue.strategy] route = [{if = "rcpt_domain = 'foobar.org'", then = "'lmtp'"}, {else = "'mx'"}] schedule = [{if = "rcpt_domain = 'foobar.org'", then = "'foobar'"}, {else = "'default'"}] [session.rcpt] relay = true max-recipients = 100 [session.extensions] dsn = true [queue.schedule.default] retry = "1s" notify = "1s" expire = "5s" queue-name = "default" [queue.schedule.foobar] retry = "1s" notify = ["1s", "2s"] expire = "4s" queue-name = "default" [queue.connection.default.timeout] connect = "1s" data = "50ms" [queue.route.lmtp] type = "relay" address = lmtp.foobar.org port = 9924 protocol = 'lmtp' concurrency = 5 [queue.route.lmtp.tls] implicit = true allow-invalid-certs = true "#; #[tokio::test] #[serial_test::serial] async fn lmtp_delivery() { // Enable logging crate::enable_logging(); // Start test server let mut remote = TestSMTP::new("lmtp_delivery_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Lmtp]).await; // Multiple delivery attempts let mut local = TestSMTP::new("lmtp_delivery_local", LOCAL).await; // Add mock DNS entries let core = local.build_smtp(); core.ipv4_add( "lmtp.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); let mut session = local.new_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; session .send_message( "john@test.org", &[ " NOTIFY=SUCCESS,DELAY,FAILURE", " NOTIFY=SUCCESS,DELAY,FAILURE", " NOTIFY=SUCCESS,DELAY,FAILURE", " NOTIFY=SUCCESS,DELAY,FAILURE", " NOTIFY=SUCCESS,DELAY,FAILURE", " NOTIFY=SUCCESS,DELAY,FAILURE", ], "test:no_dkim", "250", ) .await; local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); let mut dsn = Vec::new(); loop { match local.queue_receiver.try_read_event().await { Some(QueueEvent::Refresh | QueueEvent::WorkerDone { .. }) => {} Some(QueueEvent::Paused(_)) | Some(QueueEvent::ReloadSettings) => unreachable!(), None | Some(QueueEvent::Stop) => break, } let mut events = core.all_queued_messages().await; if events.messages.is_empty() { let now = now(); if events.next_refresh < now + QUEUE_REFRESH { tokio::time::sleep(Duration::from_secs(events.next_refresh - now)).await; events = core.all_queued_messages().await; } else { break; } } for event in events.messages { let message = core .read_message(event.queue_id, QueueName::default()) .await .unwrap(); if message.message.return_path.is_empty() { message.clone().remove(&core, event.due.into()).await; dsn.push(message); } else { event.try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; } } } local.queue_receiver.assert_queue_is_empty().await; assert_eq!(dsn.len(), 4); let mut dsn = dsn.into_iter(); dsn.next() .unwrap() .read_lines(&local.queue_receiver) .await .assert_contains(" (delivered to") .assert_contains(" (delivered to") .assert_contains(" (delivered to") .assert_contains(" (failed to lookup") .assert_contains(" (host 'lmtp.foobar.org' rejected command"); dsn.next() .unwrap() .read_lines(&local.queue_receiver) .await .assert_contains(" (host 'lmtp.foobar.org' rejected") .assert_contains("Action: delayed"); dsn.next() .unwrap() .read_lines(&local.queue_receiver) .await .assert_contains(" (host 'lmtp.foobar.org' rejected") .assert_contains("Action: delayed"); dsn.next() .unwrap() .read_lines(&local.queue_receiver) .await .assert_contains(" (host 'lmtp.foobar.org' rejected") .assert_contains("Action: failed"); assert_eq!( remote .queue_receiver .expect_message() .await .message .recipients .into_iter() .map(|r| r.address().to_string()) .collect::>(), vec![ "bill@foobar.org".to_string(), "jane@foobar.org".to_string(), "john@foobar.org".to_string() ] ); remote.queue_receiver.assert_no_events(); } ================================================ FILE: tests/src/smtp/outbound/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod dane; pub mod extensions; pub mod fallback_relay; pub mod ip_lookup; pub mod lmtp; pub mod mta_sts; pub mod smtp; pub mod throttle; pub mod tls; ================================================ FILE: tests/src/smtp/outbound/mta_sts.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ sync::Arc, time::{Duration, Instant}, }; use common::{ config::{server::ServerProtocol, smtp::resolver::Policy}, ipc::PolicyType, }; use mail_auth::{ MX, common::parse::TxtRecordParser, mta_sts::{MtaSts, ReportUri, TlsRpt}, report::tlsrpt::ResultType, }; use crate::smtp::{ DnsCache, TestSMTP, inbound::{TestMessage, TestQueueEvent, TestReportingEvent}, session::{TestSession, VerifyResponse}, }; use smtp::outbound::mta_sts::{lookup::STS_TEST_POLICY, parse::ParsePolicy}; const LOCAL: &str = r#" [session.rcpt] relay = true [queue.tls.default] mta-sts = "require" allow-invalid-certs = false [report.tls.aggregate] send = "weekly" "#; const REMOTE: &str = r#" [session.ehlo] reject-non-fqdn = false [session.rcpt] relay = true [session.data.add-headers] received = true received-spf = true auth-results = true message-id = true date = true return-path = false "#; #[tokio::test] #[serial_test::serial] async fn mta_sts_verify() { // Enable logging crate::enable_logging(); // Start test server let mut remote = TestSMTP::new("smtp_mta_sts_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Smtp]).await; // Fail on missing MTA-STS record let mut local = TestSMTP::new("smtp_mta_sts_local", LOCAL).await; // Add mock DNS entries let core = local.build_smtp(); core.mx_add( "foobar.org", vec![MX { exchanges: vec!["mx.foobar.org".to_string()], preference: 10, }], Instant::now() + Duration::from_secs(10), ); core.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); core.txt_add( "_smtp._tls.foobar.org", TlsRpt::parse(b"v=TLSRPTv1; rua=mailto:reports@foobar.org").unwrap(), Instant::now() + Duration::from_secs(10), ); let mut session = local.new_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); local .queue_receiver .expect_message() .await .read_lines(&local.queue_receiver) .await .assert_contains(" (MTA-STS failed to authenticate") .assert_contains("Record not f=") .assert_contains("ound"); local.queue_receiver.read_event().await.assert_done(); // Expect TLS failure report let report = local.report_receiver.read_report().await.unwrap_tls(); assert_eq!(report.domain, "foobar.org"); assert_eq!(report.policy, PolicyType::Sts(None)); assert_eq!( report.failure.as_ref().unwrap().result_type, ResultType::Other ); assert_eq!( report.tls_record.rua, vec![ReportUri::Mail("reports@foobar.org".to_string())] ); // MTA-STS policy fetch failure core.txt_add( "_mta-sts.foobar.org", MtaSts::parse(b"v=STSv1; id=policy_will_fail;").unwrap(), Instant::now() + Duration::from_secs(10), ); session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); local .queue_receiver .expect_message() .await .read_lines(&local.queue_receiver) .await .assert_contains(" (MTA-STS failed to authenticate") .assert_contains("No 'mx' entries found"); local.queue_receiver.read_event().await.assert_done(); // Expect TLS failure report let report = local.report_receiver.read_report().await.unwrap_tls(); assert_eq!(report.policy, PolicyType::Sts(None)); assert_eq!( report.failure.as_ref().unwrap().result_type, ResultType::StsPolicyInvalid ); // MTA-STS policy does not authorize mx.foobar.org let policy = concat!( "version: STSv1\n", "mode: enforce\n", "mx: mail.foobar.net\n", "max_age: 604800\n" ); STS_TEST_POLICY.lock().extend_from_slice(policy.as_bytes()); session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); local .queue_receiver .expect_message() .await .read_lines(&local.queue_receiver) .await .assert_contains(" (MTA-STS failed to authenticate") .assert_contains("not authorized by policy"); local.queue_receiver.read_event().await.assert_done(); // Expect TLS failure report let report = local.report_receiver.read_report().await.unwrap_tls(); assert_eq!( report.policy, PolicyType::Sts( Arc::new(Policy::parse(policy, "policy_will_fail".to_string()).unwrap()).into() ) ); assert_eq!( report.failure.as_ref().unwrap().receiving_mx_hostname, Some("mx.foobar.org".to_string()) ); assert_eq!( report.failure.as_ref().unwrap().result_type, ResultType::ValidationFailure ); remote.queue_receiver.assert_no_events(); // MTA-STS successful validation core.txt_add( "_mta-sts.foobar.org", MtaSts::parse(b"v=STSv1; id=policy_will_work;").unwrap(), Instant::now() + Duration::from_secs(10), ); let policy = concat!( "version: STSv1\n", "mode: enforce\n", "mx: *.foobar.org\n", "max_age: 604800\n" ); STS_TEST_POLICY.lock().clear(); STS_TEST_POLICY.lock().extend_from_slice(policy.as_bytes()); session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); local.queue_receiver.read_event().await.assert_done(); remote .queue_receiver .expect_message() .await .read_lines(&remote.queue_receiver) .await .assert_contains("using TLSv1.3 with cipher"); // Expect TLS success report let report = local.report_receiver.read_report().await.unwrap_tls(); assert_eq!( report.policy, PolicyType::Sts( Arc::new(Policy::parse(policy, "policy_will_work".to_string()).unwrap()).into() ) ); assert!(report.failure.is_none()); } ================================================ FILE: tests/src/smtp/outbound/smtp.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::{Duration, Instant}; use common::{ config::{server::ServerProtocol, smtp::queue::QueueName}, ipc::QueueEvent, }; use mail_auth::MX; use store::write::now; use crate::smtp::{ DnsCache, TestSMTP, inbound::{TestMessage, TestQueueEvent}, queue::QueuedEvents, session::{TestSession, VerifyResponse}, }; use smtp::queue::spool::{QUEUE_REFRESH, SmtpSpool}; const LOCAL: &str = r#" [session.rcpt] relay = true max-recipients = 100 [session.extensions] dsn = true [queue.schedule.default] retry = "1s" notify = "1s" expire = "7s" queue-name = "default" [queue.schedule.foobar-org] retry = "1s" notify = ["1s", "2s"] expire = "6s" queue-name = "default" [queue.schedule.foobar-com] retry = "1s" notify = ["5s", "6s"] expire = "7s" queue-name = "default" [queue.strategy] schedule = [{if = "rcpt_domain == 'foobar.org'", then = "'foobar-org'"}, {if = "rcpt_domain == 'foobar.com'", then = "'foobar-com'"}, {else = "'default'"}] [spam-filter] enable = false "#; const REMOTE: &str = r#" [session.ehlo] reject-non-fqdn = false [session.rcpt] relay = true [session.extensions] dsn = true chunking = false [spam-filter] enable = false "#; const SMUGGLER: &str = r#"From: Joe SixPack To: Suzie Q Subject: Is dinner ready? Hi. We lost the game. Are you hungry yet? .hey Joe. . MAIL FROM: RCPT TO: DATA From: Joe SixPack To: Suzie Q Subject: smuggled message This is a smuggled message "#; #[tokio::test] #[serial_test::serial] async fn smtp_delivery() { // Enable logging crate::enable_logging(); // Start test server let mut remote = TestSMTP::new("smtp_delivery_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Smtp]).await; let remote_core = remote.build_smtp(); // Multiple delivery attempts let mut local = TestSMTP::new("smtp_delivery_local", LOCAL).await; // Add mock DNS entries let core = local.build_smtp(); for domain in ["foobar.org", "foobar.net", "foobar.com"] { core.mx_add( domain, vec![MX { exchanges: vec![format!("mx1.{domain}"), format!("mx2.{domain}")], preference: 10, }], Instant::now() + Duration::from_secs(10), ); core.ipv4_add( format!("mx1.{domain}"), vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(30), ); core.ipv4_add( format!("mx2.{domain}"), vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(30), ); } let mut session = local.new_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; session .send_message( "john@test.org", &[ " NOTIFY=SUCCESS,DELAY,FAILURE", " NOTIFY=SUCCESS,DELAY,FAILURE", " NOTIFY=SUCCESS,DELAY,FAILURE", " NOTIFY=SUCCESS,DELAY,FAILURE", " NOTIFY=SUCCESS,DELAY,FAILURE", " NOTIFY=SUCCESS,DELAY,FAILURE", " NOTIFY=SUCCESS,DELAY,FAILURE", ], "test:no_dkim", "250", ) .await; let message = local.queue_receiver.expect_message().await; let num_recipients = message.message.recipients.len(); assert_eq!(num_recipients, 7); local .queue_receiver .delivery_attempt(message.queue_id) .await .try_deliver(core.clone()); let mut dsn = Vec::new(); let mut rcpt_retries = vec![0; num_recipients]; loop { match local.queue_receiver.try_read_event().await { Some(QueueEvent::Refresh | QueueEvent::WorkerDone { .. }) => {} Some(QueueEvent::Paused(_)) | Some(QueueEvent::ReloadSettings) => unreachable!(), None | Some(QueueEvent::Stop) => { break; } } let mut events = core.all_queued_messages().await; if events.messages.is_empty() { let now = now(); if events.next_refresh < now + QUEUE_REFRESH { tokio::time::sleep(Duration::from_secs(events.next_refresh - now)).await; events = core.all_queued_messages().await; } else { break; } } for event in events.messages { let message = core .read_message(event.queue_id, QueueName::default()) .await .unwrap(); if message.message.return_path.is_empty() { message.clone().remove(&core, event.due.into()).await; dsn.push(message); } else { for (idx, rcpt) in message.message.recipients.iter().enumerate() { rcpt_retries[idx] = rcpt.retry.inner; } event.try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; } } } assert_eq!(rcpt_retries[0], 0, "retries {rcpt_retries:?}"); assert!(rcpt_retries[1] >= 5, "retries {rcpt_retries:?}"); assert_eq!(rcpt_retries[2], 0, "retries {rcpt_retries:?}"); assert_eq!(rcpt_retries[3], 0, "retries {rcpt_retries:?}"); assert!(rcpt_retries[4] >= 5, "retries {rcpt_retries:?}"); assert_eq!(rcpt_retries[5], 0, "retries {rcpt_retries:?}"); assert_eq!(rcpt_retries[6], 0, "retries {rcpt_retries:?}"); assert!( rcpt_retries[1] >= rcpt_retries[4], "retries {rcpt_retries:?}" ); local.queue_receiver.assert_queue_is_empty().await; assert_eq!(dsn.len(), 5); let mut dsn = dsn.into_iter(); dsn.next() .unwrap() .read_lines(&local.queue_receiver) .await .assert_contains(" (delivered to") .assert_contains(" (delivered to") .assert_contains(" (failed to lookup") .assert_contains(" (host ") .assert_contains(" (host "); dsn.next() .unwrap() .read_lines(&local.queue_receiver) .await .assert_contains(" (host ") .assert_contains(" (host ") .assert_contains("Action: delayed"); dsn.next() .unwrap() .read_lines(&local.queue_receiver) .await .assert_contains(" (host ") .assert_contains("Action: delayed"); dsn.next() .unwrap() .read_lines(&local.queue_receiver) .await .assert_contains(" (host "); dsn.next() .unwrap() .read_lines(&local.queue_receiver) .await .assert_contains(" (host ") .assert_contains("Action: failed"); let mut recipients = remote .queue_receiver .consume_message(&remote_core) .await .message .recipients .into_iter() .map(|r| r.address().to_string()) .collect::>(); recipients.extend( remote .queue_receiver .consume_message(&remote_core) .await .message .recipients .into_iter() .map(|r| r.address().to_string()), ); recipients.sort(); assert_eq!( recipients, vec!["ok@foobar.net".to_string(), "ok@foobar.org".to_string()] ); remote.queue_receiver.assert_no_events(); // SMTP smuggling for separator in ["\n", "\r"].iter() { session.data.remote_ip_str = "10.0.0.2".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; let message = SMUGGLER .replace('\r', "") .replace('\n', "\r\n") .replace("", separator); session .send_message("john@doe.org", &["bill@foobar.com"], &message, "250") .await; local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); local .queue_receiver .read_event() .await .assert_refresh_or_done(); let message = remote .queue_receiver .consume_message(&remote_core) .await .read_message(&remote.queue_receiver) .await; assert!( message.contains("This is a smuggled message"), "message: {:?}", message ); assert!( message.contains("We lost the game."), "message: {:?}", message ); assert!( message.contains(&format!("{separator}..\r\nMAIL FROM:<",)), "message: {:?}", message ); } } ================================================ FILE: tests/src/smtp/outbound/throttle.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::smtp::{ DnsCache, TestSMTP, inbound::TestQueueEvent, queue::{build_rcpt, manager::new_message}, session::TestSession, }; use mail_auth::MX; use smtp::queue::{Message, QueueEnvelope, Recipient, throttle::IsAllowed}; use std::{ net::{IpAddr, Ipv4Addr}, time::{Duration, Instant}, }; use store::write::now; const CONFIG: &str = r#" [session.rcpt] relay = true [queue.schedule.default] retry = "1h" notify = "1h" expire = "1h" [[queue.limiter.outbound]] match = "sender_domain = 'foobar.org'" key = 'sender_domain' enable = true [[queue.limiter.outbound]] match = "sender_domain = 'foobar.net'" key = 'sender_domain' rate = '1/30m' enable = true [[queue.limiter.outbound]] match = "rcpt_domain = 'example.org'" key = 'rcpt_domain' enable = true [[queue.limiter.outbound]] match = "rcpt_domain = 'example.net'" key = 'rcpt_domain' rate = '1/40m' enable = true [[queue.limiter.outbound]] match = "mx = 'mx.test.org'" key = 'mx' enable = true [[queue.limiter.outbound]] match = "mx = 'mx.test.net'" key = 'mx' rate = '1/50m' enable = true "#; #[tokio::test] async fn throttle_outbound() { // Enable logging crate::enable_logging(); // Build test message let mut test_message = new_message(0).message; test_message.return_path = "test@foobar.org".into(); test_message .recipients .push(build_rcpt("bill@test.org", 0, 0, 0)); let mut local = TestSMTP::new("smtp_throttle_outbound", CONFIG).await; let core = local.build_smtp(); let mut session = local.new_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; session .send_message("john@foobar.org", &["bill@test.org"], "test:no_dkim", "250") .await; assert_eq!( local.queue_receiver.last_queued_due().await as i64 - now() as i64, 0 ); // Throttle sender let throttle = &core.core.smtp.queue.outbound_limiters; for t in &throttle.sender { core.is_allowed( t, &QueueEnvelope::test(&test_message, &test_message.recipients[0], ""), 0, ) .await .unwrap(); } // Expect concurrency throttle for sender domain 'foobar.org' /*local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; local.queue_receiver.read_event().await.assert_on_hold();*/ // Expect rate limit throttle for sender domain 'foobar.net' test_message.return_path = "test@foobar.net".into(); for t in &throttle.sender { core.is_allowed( t, &QueueEnvelope::test(&test_message, &test_message.recipients[0], ""), 0, ) .await .unwrap(); } test_message.recipients.clear(); session .send_message("john@foobar.net", &["bill@test.org"], "test:no_dkim", "250") .await; local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; local.queue_receiver.read_event().await.assert_refresh(); let due = local.queue_receiver.last_queued_due().await - now(); assert!(due > 0, "Due: {}", due); // Expect concurrency throttle for recipient domain 'example.org' test_message.return_path = "test@test.net".into(); test_message .recipients .push(build_rcpt("test@example.org", 0, 0, 0)); for t in &throttle.rcpt { core.is_allowed( t, &QueueEnvelope::test(&test_message, &test_message.recipients[0], ""), 0, ) .await .unwrap(); } /*session .send_message( "john@test.net", &["jane@example.org"], "test:no_dkim", "250", ) .await; local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; local.queue_receiver.read_event().await.assert_on_hold();*/ // Expect rate limit throttle for recipient domain 'example.net' test_message .recipients .push(build_rcpt("test@example.net", 0, 0, 0)); for t in &throttle.rcpt { core.is_allowed( t, &QueueEnvelope::test(&test_message, &test_message.recipients[1], ""), 0, ) .await .unwrap(); } session .send_message( "john@test.net", &["jane@example.net"], "test:no_dkim", "250", ) .await; local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; local.queue_receiver.read_event().await.assert_refresh(); let due = local.queue_receiver.last_queued_due().await - now(); assert!(due > 0, "Due: {}", due); // Expect concurrency throttle for mx 'mx.test.org' core.mx_add( "test.org", vec![MX { exchanges: vec!["mx.test.org".into()], preference: 10, }], Instant::now() + Duration::from_secs(10), ); core.ipv4_add( "mx.test.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); test_message .recipients .push(build_rcpt("test@test.org", 0, 0, 0)); for t in &throttle.remote { core.is_allowed( t, &QueueEnvelope::test(&test_message, &test_message.recipients[2], "mx.test.org"), 0, ) .await .unwrap(); } /*session .send_message("john@test.net", &["jane@test.org"], "test:no_dkim", "250") .await; local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); local.queue_receiver.read_event().await.assert_on_hold();*/ // Expect rate limit throttle for mx 'mx.test.net' core.mx_add( "test.net", vec![MX { exchanges: vec!["mx.test.net".into()], preference: 10, }], Instant::now() + Duration::from_secs(10), ); core.ipv4_add( "mx.test.net", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); for t in &throttle.remote { core.is_allowed( t, &QueueEnvelope::test(&test_message, &test_message.recipients[1], "mx.test.net"), 0, ) .await .unwrap(); } session .send_message("john@test.net", &["jane@test.net"], "test:no_dkim", "250") .await; local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; local.queue_receiver.read_event().await.assert_refresh(); let due = local.queue_receiver.last_queued_due().await - now(); assert!(due > 0, "Due: {}", due); } pub trait TestQueueEnvelope<'x> { fn test(message: &'x Message, rcpt: &'x Recipient, mx: &'x str) -> Self; } impl<'x> TestQueueEnvelope<'x> for QueueEnvelope<'x> { fn test(message: &'x Message, rcpt: &'x Recipient, mx: &'x str) -> Self { QueueEnvelope { message, mx, remote_ip: IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), local_ip: IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), domain: rcpt.domain_part(), rcpt, } } } ================================================ FILE: tests/src/smtp/outbound/tls.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::{Duration, Instant}; use common::config::server::ServerProtocol; use mail_auth::MX; use store::write::now; use crate::smtp::{ DnsCache, TestSMTP, inbound::TestMessage, session::{TestSession, VerifyResponse}, }; const LOCAL: &str = r#" [session.rcpt] relay = true [queue.connection.default] ehlo-hostname = "badtls.foobar.org" [queue.strategy] tls = [ { if = "retry_num > 0 && last_error == 'tls'", then = "'no-tls'"}, { else = "'default'" }] [queue.tls.no-tls] starttls = false allow-invalid-certs = true "#; const REMOTE: &str = r#" [session.rcpt] relay = true [session.ehlo] reject-non-fqdn = false [session.extensions] dsn = true chunking = false "#; #[tokio::test] #[serial_test::serial] async fn starttls_optional() { // Enable logging crate::enable_logging(); // Start test server let mut remote = TestSMTP::new("smtp_starttls_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Smtp]).await; // Retry on failed STARTTLS let mut local = TestSMTP::new("smtp_starttls_local", LOCAL).await; // Add mock DNS entries let core = local.build_smtp(); core.mx_add( "foobar.org", vec![MX { exchanges: vec!["mx.foobar.org".to_string()], preference: 10, }], Instant::now() + Duration::from_secs(10), ); core.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); let mut session = local.new_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local .queue_receiver .expect_message_then_deliver() .await .try_deliver(core.clone()); let mut retry = local.queue_receiver.expect_message().await; let prev_due = retry.message.recipients[0].retry.due; let next_due = now(); let queue_id = retry.queue_id; retry.message.recipients[0].retry.due = next_due; retry.save_changes(&core, prev_due.into()).await; local .queue_receiver .delivery_attempt(queue_id) .await .try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; remote .queue_receiver .expect_message() .await .read_lines(&remote.queue_receiver) .await .assert_not_contains("using TLSv1.3 with cipher"); } ================================================ FILE: tests/src/smtp/queue/concurrent.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::{Duration, Instant}; use common::{config::server::ServerProtocol, core::BuildServer, ipc::QueueEvent}; use mail_auth::MX; use crate::{ smtp::{DnsCache, TestSMTP, session::TestSession}, store::cleanup::store_assert_is_empty, }; use smtp::queue::manager::Queue; const LOCAL: &str = r#" [spam-filter] enable = false [session.rcpt] relay = true [session.data.limits] messages = 2000 [queue.virtual.default] threads-per-node = 4 [queue.schedule.default] retry = "1s" notify = "1d" expire = "1d" queue-name = "default" "#; const REMOTE: &str = r#" [session.ehlo] reject-non-fqdn = false [session.rcpt] relay = true [spam-filter] enable = false "#; const NUM_MESSAGES: usize = 100; const NUM_QUEUES: usize = 10; #[tokio::test(flavor = "multi_thread", worker_threads = 18)] #[serial_test::serial] async fn concurrent_queue() { // Enable logging crate::enable_logging(); // Start test server let remote = TestSMTP::new("smtp_concurrent_queue_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Smtp]).await; let local = TestSMTP::with_database("smtp_concurrent_queue_local", LOCAL, "mysql").await; // Add mock DNS entries let core = local.build_smtp(); core.mx_add( "foobar.org", vec![MX { exchanges: vec!["mx.foobar.org".to_string()], preference: 10, }], Instant::now() + Duration::from_secs(100), ); core.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(100), ); let mut session = local.new_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; // Spawn concurrent queues let mut inners = vec![]; for _ in 0..NUM_QUEUES { let (inner, rxs) = local.inner_with_rxs(); let server = inner.build_server(); server.mx_add( "foobar.org", vec![MX { exchanges: vec!["mx.foobar.org".to_string()], preference: 10, }], Instant::now() + Duration::from_secs(100), ); server.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(100), ); inners.push(inner.clone()); tokio::spawn(async move { Queue::new(inner, rxs.queue_rx.unwrap()).start().await; }); } tokio::time::sleep(Duration::from_millis(200)).await; // Send 1000 test messages for _ in 0..(NUM_MESSAGES / 2) { session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; } // Wake up all queues for inner in &inners { inner.ipc.queue_tx.send(QueueEvent::Refresh).await.unwrap(); } for _ in 0..(NUM_MESSAGES / 2) { session .send_message( "john@test.org", &["delay-random@foobar.org"], "test:no_dkim", "250", ) .await; } loop { tokio::time::sleep(Duration::from_millis(1500)).await; let m = local.queue_receiver.read_queued_messages().await.len(); let e = local.queue_receiver.read_queued_events().await.len(); if m + e != 0 { println!("Queue still has {} messages and {} events", m, e); /*for inner in &inners { inner.ipc.queue_tx.send(QueueEvent::Refresh).await.unwrap(); }*/ } else { break; } } local.queue_receiver.assert_queue_is_empty().await; let remote_messages = remote.queue_receiver.read_queued_messages().await; assert_eq!(remote_messages.len(), NUM_MESSAGES); // Make sure local store is queue store_assert_is_empty( &core.core.storage.data, core.core.storage.blob.clone(), false, ) .await; } ================================================ FILE: tests/src/smtp/queue/dsn.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::smtp::{QueueReceiver, TestSMTP, inbound::sign::SIGNATURES}; use common::config::smtp::queue::{QueueExpiry, QueueName}; use smtp::queue::{ Error, ErrorDetails, HostResponse, Message, MessageWrapper, Recipient, Schedule, Status, UnexpectedResponse, dsn::SendDsn, }; use smtp_proto::{RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_SUCCESS, Response}; use std::{ fs, net::{IpAddr, Ipv4Addr}, path::PathBuf, time::SystemTime, }; use store::write::now; use types::blob_hash::BlobHash; const CONFIG: &str = r#" [report] submitter = "'mx.example.org'" [session.ehlo] reject-non-fqdn = false [session.rcpt] relay = true [report.dsn] from-name = "'Mail Delivery Subsystem'" from-address = "'MAILER-DAEMON@example.org'" sign = "['rsa']" "#; #[tokio::test] async fn generate_dsn() { // Enable logging crate::enable_logging(); let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); path.push("resources"); path.push("smtp"); path.push("dsn"); path.push("original.txt"); let size = fs::metadata(&path).unwrap().len() as u64; let dsn_original = fs::read_to_string(&path).unwrap(); let flags = RCPT_NOTIFY_FAILURE | RCPT_NOTIFY_DELAY | RCPT_NOTIFY_SUCCESS; let mut message = MessageWrapper { queue_id: 0, span_id: 0, is_multi_queue: false, queue_name: QueueName::default(), message: Message { size, created: SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()), return_path: "sender@foobar.org".into(), recipients: vec![Recipient { address: "foobar@example.org".into(), status: Status::PermanentFailure(ErrorDetails { entity: "mx.example.org".into(), details: Error::UnexpectedResponse(UnexpectedResponse { command: "RCPT TO:".into(), response: Response { code: 550, esc: [5, 1, 2], message: "User does not exist".into(), }, }), }), flags: 0, orcpt: None, retry: Schedule::now(), notify: Schedule::now(), expires: QueueExpiry::Ttl(10), queue: QueueName::default(), }], flags: 0, env_id: None, priority: 0, blob_hash: BlobHash::generate(dsn_original.as_bytes()), quota_keys: Default::default(), received_from_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), received_via_port: 0, }, }; // Load config let mut local = TestSMTP::new("smtp_dsn_test", CONFIG.to_string() + SIGNATURES).await; let core = local.build_smtp(); let qr = &mut local.queue_receiver; // Create temp dir for queue qr.blob_store .put_blob( message.message.blob_hash.as_slice(), dsn_original.as_bytes(), ) .await .unwrap(); // Disabled DSN core.send_dsn(&mut message).await; qr.assert_no_events(); qr.assert_queue_is_empty().await; // Failure DSN message.message.recipients[0].flags = flags; core.send_dsn(&mut message).await; let dsn_message = qr.expect_message().await; qr.compare_dsn(dsn_message.message, "failure.eml").await; // Success DSN message.message.recipients.push(Recipient { address: "jane@example.org".into(), status: Status::Completed(HostResponse { hostname: "mx2.example.org".into(), response: Response { code: 250, esc: [2, 1, 5], message: "Message accepted for delivery".into(), }, }), flags, orcpt: None, retry: Schedule::now(), notify: Schedule::now(), expires: QueueExpiry::Ttl(10), queue: QueueName::default(), }); core.send_dsn(&mut message).await; let dsn_message = qr.expect_message().await; qr.compare_dsn(dsn_message.message, "success.eml").await; // Delay DSN message.message.recipients.push(Recipient { address: "john.doe@example.org".into(), status: Status::TemporaryFailure(ErrorDetails { entity: "mx.domain.org".into(), details: Error::ConnectionError("Connection timeout".into()), }), flags, orcpt: Some("jdoe@example.org".into()), retry: Schedule::now(), notify: Schedule::now(), expires: QueueExpiry::Ttl(10), queue: QueueName::default(), }); core.send_dsn(&mut message).await; let dsn_message = qr.expect_message().await; qr.compare_dsn(dsn_message.message, "delay.eml").await; // Mixed DSN for rcpt in &mut message.message.recipients { rcpt.flags = flags; } message.message.recipients.last_mut().unwrap().notify.due = now(); core.send_dsn(&mut message).await; let dsn_message = qr.expect_message().await; qr.compare_dsn(dsn_message.message, "mixed.eml").await; // Load queue let queue = qr.read_queued_messages().await; assert_eq!(queue.len(), 4); } impl QueueReceiver { async fn compare_dsn(&self, message: Message, test: &str) { let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); path.push("resources"); path.push("smtp"); path.push("dsn"); path.push(test); let bytes = self .blob_store .get_blob(message.blob_hash.as_slice(), 0..usize::MAX) .await .unwrap() .unwrap(); let dsn = remove_ids(bytes); let dsn_expected = fs::read_to_string(&path).unwrap(); if dsn != dsn_expected { let mut failed = PathBuf::from(&path); failed.set_extension("failed"); fs::write(&failed, dsn.as_bytes()).unwrap(); panic!( "Failed for {}, output saved to {}", path.display(), failed.display() ); } } } fn remove_ids(message: Vec) -> String { let old_message = String::from_utf8(message).unwrap(); let mut message = String::with_capacity(old_message.len()); let mut found_dkim = false; let mut skip = false; let mut boundary = ""; for line in old_message.split("\r\n") { if skip { if line.chars().next().unwrap().is_ascii_whitespace() { continue; } else { skip = false; } } if line.starts_with("Date:") || line.starts_with("Message-ID:") { continue; } else if !found_dkim && line.starts_with("DKIM-Signature:") { found_dkim = true; skip = true; continue; } else if line.starts_with("--") { message.push_str(&line.replace(boundary, "mime_boundary")); } else if let Some((_, boundary_)) = line.split_once("boundary=\"") { boundary = boundary_.split_once('"').unwrap().0; message.push_str(&line.replace(boundary, "mime_boundary")); } else if line.starts_with("Arrival-Date:") { message.push_str("Arrival-Date: "); } else if line.starts_with("Will-Retry-Until:") { message.push_str("Will-Retry-Until: "); } else { message.push_str(line); } message.push_str("\r\n"); } if !found_dkim { panic!("No DKIM signature found in: {old_message}"); } message } ================================================ FILE: tests/src/smtp/queue/manager.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::smtp::{ TestSMTP, queue::{QueuedEvents, build_rcpt}, }; use common::config::smtp::queue::QueueName; use smtp::queue::{ Error, ErrorDetails, Message, MessageWrapper, Recipient, Status, spool::SmtpSpool, }; use std::{ net::{IpAddr, Ipv4Addr}, time::Duration, }; use store::write::now; const CONFIG: &str = r#" [session.ehlo] reject-non-fqdn = false [session.rcpt] relay = true "#; #[tokio::test] async fn queue_due() { // Enable logging crate::enable_logging(); let local = TestSMTP::new("smtp_queue_due_test", CONFIG).await; let core = local.build_smtp(); let qr = &local.queue_receiver; let mut message = new_message(0); message.message.recipients.push(build_rcpt("c", 3, 8, 9)); message.save_changes(&core, 0.into()).await; let mut message = new_message(1); message.message.recipients.push(build_rcpt("b", 2, 6, 7)); message.save_changes(&core, 0.into()).await; let mut message = new_message(2); message.message.recipients.push(build_rcpt("a", 1, 4, 5)); message.save_changes(&core, 0.into()).await; for domain in vec!["a", "b", "c"].into_iter() { let now = now(); let queued = core.all_queued_messages().await; if queued.messages.is_empty() { let wake_up = queued.next_refresh - now; assert_eq!(wake_up, 1); std::thread::sleep(Duration::from_secs(wake_up)); } for queue_event in core.all_queued_messages().await.messages { if let Some(message) = core .read_message(queue_event.queue_id, QueueName::default()) .await { message.message.rcpt(domain); message.remove(&core, queue_event.due.into()).await; } else { panic!("Message not found"); } } } qr.assert_queue_is_empty().await; } #[test] fn delivery_events() { let mut message = new_message(0).message; message.created = now(); message.recipients.push(build_rcpt("a", 1, 2, 3)); message.recipients.push(build_rcpt("b", 4, 5, 6)); message.recipients.push(build_rcpt("c", 7, 8, 9)); for t in 0..2 { assert_eq!( message.next_event(None).unwrap(), message.rcpt("a").retry.due ); assert_eq!( message.next_delivery_event(None).unwrap(), message.rcpt("a").retry.due ); assert_eq!( next_event_after( &message, None, message.rcpt("a").expiration_time(message.created).unwrap() ) .unwrap(), message.rcpt("b").retry.due ); assert_eq!( next_event_after( &message, None, message.rcpt("b").expiration_time(message.created).unwrap() ) .unwrap(), message.rcpt("c").retry.due ); assert_eq!( next_event_after(&message, None, message.rcpt("c").notify.due).unwrap(), message.rcpt("c").expiration_time(message.created).unwrap() ); assert!( next_event_after( &message, None, message.rcpt("c").expiration_time(message.created).unwrap() ) .is_none() ); if t == 0 { message.recipients.reverse(); } else { message.recipients.swap(0, 1); } } message.rcpt_mut("a").status = Status::PermanentFailure(ErrorDetails { entity: "localhost".into(), details: Error::ConcurrencyLimited, }); assert_eq!( message.next_event(None).unwrap(), message.rcpt("b").retry.due ); assert_eq!( message.next_delivery_event(None).unwrap(), message.rcpt("b").retry.due ); message.rcpt_mut("b").status = Status::PermanentFailure(ErrorDetails { entity: "localhost".into(), details: Error::ConcurrencyLimited, }); assert_eq!( message.next_event(None).unwrap(), message.rcpt("c").retry.due ); assert_eq!( message.next_delivery_event(None).unwrap(), message.rcpt("c").retry.due ); message.rcpt_mut("c").status = Status::PermanentFailure(ErrorDetails { entity: "localhost".into(), details: Error::ConcurrencyLimited, }); assert!(message.next_event(None).is_none()); } pub fn new_message(queue_id: u64) -> MessageWrapper { MessageWrapper { queue_id, span_id: 0, queue_name: QueueName::default(), is_multi_queue: false, message: Message { size: 0, created: now(), return_path: "sender@foobar.org".into(), recipients: vec![], flags: 0, env_id: None, priority: 0, quota_keys: Default::default(), blob_hash: Default::default(), received_from_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), received_via_port: 0, }, } } fn next_event_after(message: &Message, queue: Option, instant: u64) -> Option { let mut next_event = None; for rcpt in &message.recipients { if matches!(rcpt.status, Status::Scheduled | Status::TemporaryFailure(_)) && queue.is_none_or(|q| rcpt.queue == q) { if rcpt.retry.due > instant && next_event.as_ref().is_none_or(|ne| rcpt.retry.due.lt(ne)) { next_event = rcpt.retry.due.into(); } if rcpt.notify.due > instant && next_event.as_ref().is_none_or(|ne| rcpt.notify.due.lt(ne)) { next_event = rcpt.notify.due.into(); } if let Some(expires) = rcpt.expiration_time(message.created) && expires > instant && next_event.as_ref().is_none_or(|ne| expires.lt(ne)) { next_event = expires.into(); } } } next_event } pub trait TestMessage { fn rcpt(&self, name: &str) -> &Recipient; fn rcpt_mut(&mut self, name: &str) -> &mut Recipient; } impl TestMessage for Message { fn rcpt(&self, name: &str) -> &Recipient { self.recipients .iter() .find(|d| d.address() == name) .unwrap_or_else(|| panic!("Expected rcpt {name} not found in {:?}", self.recipients)) } fn rcpt_mut(&mut self, name: &str) -> &mut Recipient { self.recipients .iter_mut() .find(|d| d.address() == name) .unwrap() } } ================================================ FILE: tests/src/smtp/queue/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use common::{ Server, config::smtp::queue::{QueueExpiry, QueueName}, }; use smtp::queue::{ Recipient, Schedule, Status, manager::Queue, spool::{QueuedMessages, SmtpSpool}, }; use tokio::sync::mpsc; pub mod concurrent; pub mod dsn; pub mod manager; pub mod retry; pub mod virtualq; pub fn build_rcpt(address: &str, retry: u64, notify: u64, expires: u64) -> Recipient { Recipient { address: address.into(), retry: Schedule::later(retry), notify: Schedule::later(notify), expires: QueueExpiry::Ttl(expires), status: Status::Scheduled, flags: 0, orcpt: None, queue: QueueName::default(), } } pub trait QueuedEvents: Sync + Send { fn all_queued_messages(&self) -> impl Future + Send; } impl QueuedEvents for Server { async fn all_queued_messages(&self) -> QueuedMessages { self.next_event(&mut Queue::new(self.inner.clone(), mpsc::channel(100).1)) .await } } ================================================ FILE: tests/src/smtp/queue/retry.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use crate::smtp::{ TestSMTP, inbound::{TestMessage, TestQueueEvent}, queue::QueuedEvents, session::{TestSession, VerifyResponse}, }; use ahash::AHashSet; use common::{ config::smtp::queue::QueueName, ipc::{QueueEvent, QueueEventStatus}, }; use smtp::queue::spool::{QUEUE_REFRESH, SmtpSpool}; use store::write::now; const CONFIG: &str = r#" [session.ehlo] reject-non-fqdn = false [session.rcpt] relay = true [session.extensions] deliver-by = "1h" future-release = "1h" [queue.schedule.sender-default] retry = ["1s", "2s", "3s"] notify = ["15h", "22h"] expire = "1d" queue-name = "default" [queue.schedule.sender-test] retry = ["1s", "2s", "3s"] notify = ["1s", "2s"] expire = "6s" #max-attempts = 3 queue-name = "default" [queue.strategy] schedule = [{if = "sender_domain == 'test.org'", then = "'sender-test'"}, {else = "'sender-default'"}] "#; #[tokio::test] async fn queue_retry() { // Enable logging crate::enable_logging(); // Create temp dir for queue let mut local = TestSMTP::new("smtp_queue_retry_test", CONFIG).await; // Create test message let core = local.build_smtp(); let mut session = local.new_session(); let qr = &mut local.queue_receiver; session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; let attempt = qr.expect_message_then_deliver().await; // Expect a failed DSN attempt.try_deliver(core.clone()); let message = qr.expect_message().await; assert_eq!(message.message.return_path.as_ref(), ""); assert_eq!( message.message.recipients.first().unwrap().address(), "john@test.org" ); message .read_lines(qr) .await .assert_contains("Content-Type: multipart/report") .assert_contains("Final-Recipient: rfc822;bill@foobar.org") .assert_contains("Action: failed"); qr.read_event().await.assert_done(); qr.clear_queue(&core).await; // Expect a failed DSN for foobar.org, followed by two delayed DSN and // a final failed DSN for _dns_error.org. session .send_message( "john@test.org", &["bill@foobar.org", "jane@_dns_error.org"], "test:no_dkim", "250", ) .await; let mut in_fight = AHashSet::new(); let attempt = qr.expect_message_then_deliver().await; let mut dsn = Vec::new(); let mut retries = Vec::new(); in_fight.insert(attempt.queue_id); attempt.try_deliver(core.clone()); loop { match qr.try_read_event().await { Some(QueueEvent::WorkerDone { queue_id, status, .. }) => { in_fight.remove(&queue_id); match &status { QueueEventStatus::Completed | QueueEventStatus::Deferred => (), _ => panic!("unexpected status {queue_id}: {status:?}"), } } Some(QueueEvent::Refresh) | Some(QueueEvent::ReloadSettings) => (), None | Some(QueueEvent::Stop) | Some(QueueEvent::Paused(_)) => break, } let now = now(); let mut events = core.all_queued_messages().await; if events.messages.is_empty() { if events.next_refresh < now + QUEUE_REFRESH { tokio::time::sleep(Duration::from_secs(events.next_refresh - now)).await; events = core.all_queued_messages().await; } else if in_fight.is_empty() { break; } } for event in events.messages { if in_fight.contains(&event.queue_id) { continue; } let message = core .read_message(event.queue_id, QueueName::default()) .await .unwrap(); if message.message.return_path.is_empty() { message.clone().remove(&core, event.due.into()).await; dsn.push(message); } else { retries.push(event.due.saturating_sub(now)); in_fight.insert(event.queue_id); event.try_deliver(core.clone()); tokio::time::sleep(Duration::from_millis(100)).await; } } } qr.assert_queue_is_empty().await; assert_eq!(retries, vec![1, 2, 3]); assert_eq!(dsn.len(), 4); let mut dsn = dsn.into_iter(); dsn.next() .unwrap() .read_lines(qr) .await .assert_contains(" (failed to lookup 'foobar.org'") .assert_contains("Final-Recipient: rfc822;bill@foobar.org") .assert_contains("Action: failed"); dsn.next() .unwrap() .read_lines(qr) .await .assert_contains(" (failed to lookup '_dns_error.org'") .assert_contains("Final-Recipient: rfc822;jane@_dns_error.org") .assert_contains("Action: delayed"); dsn.next() .unwrap() .read_lines(qr) .await .assert_contains(" (failed to lookup '_dns_error.org'") .assert_contains("Final-Recipient: rfc822;jane@_dns_error.org") .assert_contains("Action: delayed"); dsn.next() .unwrap() .read_lines(qr) .await .assert_contains(" (failed to lookup '_dns_error.org'") .assert_contains("Final-Recipient: rfc822;jane@_dns_error.org") .assert_contains("Action: failed"); // Test FUTURERELEASE + DELIVERBY (RETURN) session.data.remote_ip_str = "10.0.0.2".into(); session.eval_session_params().await; session .send_message( " HOLDFOR=60 BY=3600;R", &["john@test.net"], "test:no_dkim", "250", ) .await; let now_ = now(); let message = qr.expect_message().await; assert!([59, 60].contains(&(qr.message_due(message.queue_id).await - now_))); assert!([59, 60].contains(&(message.message.next_delivery_event(None).unwrap() - now_))); assert!( [3599, 3600].contains( &(message .message .recipients .first() .unwrap() .expiration_time(message.message.created) .unwrap() - now_) ) ); assert!( [54059, 54060].contains(&(message.message.recipients.first().unwrap().notify.due - now_)) ); // Test DELIVERBY (NOTIFY) session .send_message( " BY=3600;N", &["john@test.net"], "test:no_dkim", "250", ) .await; let schedule = qr.expect_message().await; assert!( [3599, 3600].contains(&(schedule.message.recipients.first().unwrap().notify.due - now())) ); } ================================================ FILE: tests/src/smtp/queue/virtualq.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::{Duration, Instant}; use common::{ config::{server::ServerProtocol, smtp::queue::QueueName}, core::BuildServer, ipc::QueueEvent, }; use mail_auth::MX; use crate::{ smtp::{DnsCache, TestSMTP, session::TestSession}, store::cleanup::store_assert_is_empty, }; use smtp::queue::manager::Queue; const LOCAL: &str = r#" [spam-filter] enable = false [session.rcpt] relay = true [session.data.limits] messages = 2000 [queue.strategy] schedule = [ { if = "rcpt == 'delay-random@foobar.org'", then = "'q2'" }, { else = "'q1'"} ] [queue.virtual.q1] threads-per-node = 5 [queue.virtual.q2] threads-per-node = 4 [queue.schedule.q1] retry = "1s" notify = "1d" expire = "1d" queue-name = "q1" [queue.schedule.q2] retry = "1s" notify = "1d" expire = "1d" queue-name = "q2" "#; const REMOTE: &str = r#" [session.ehlo] reject-non-fqdn = false [session.rcpt] relay = true [spam-filter] enable = false "#; const NUM_MESSAGES: usize = 100; const NUM_QUEUES: usize = 10; #[tokio::test(flavor = "multi_thread", worker_threads = 18)] #[serial_test::serial] async fn virtual_queue() { // Enable logging crate::enable_logging(); // Start test server let remote = TestSMTP::new("smtp_virtual_queue_remote", REMOTE).await; let _rx = remote.start(&[ServerProtocol::Smtp]).await; let local = TestSMTP::with_database("smtp_virtual_queue_local", LOCAL, "mysql").await; // Validate parsing for value in ["a", "ab", "abcdefgh"] { let queue_name = QueueName::new(value).unwrap(); assert_eq!(queue_name.to_string(), value); } assert_eq!( local .server .core .smtp .queue .virtual_queues .get(&QueueName::new("q1").unwrap()) .unwrap() .threads, 5 ); assert_eq!( local .server .core .smtp .queue .virtual_queues .get(&QueueName::new("q2").unwrap()) .unwrap() .threads, 4 ); // Add mock DNS entries let core = local.build_smtp(); core.mx_add( "foobar.org", vec![MX { exchanges: vec!["mx.foobar.org".to_string()], preference: 10, }], Instant::now() + Duration::from_secs(100), ); core.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(100), ); let mut session = local.new_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; // Spawn concurrent queues let mut inners = vec![]; for _ in 0..NUM_QUEUES { let (inner, rxs) = local.inner_with_rxs(); let server = inner.build_server(); server.mx_add( "foobar.org", vec![MX { exchanges: vec!["mx.foobar.org".to_string()], preference: 10, }], Instant::now() + Duration::from_secs(100), ); server.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(100), ); inners.push(inner.clone()); tokio::spawn(async move { Queue::new(inner, rxs.queue_rx.unwrap()).start().await; }); } tokio::time::sleep(Duration::from_millis(200)).await; // Send 1000 test messages for _ in 0..(NUM_MESSAGES / 2) { session .send_message( "john@test.org", &["bill@foobar.org", "delay-random@foobar.org"], "test:no_dkim", "250", ) .await; } // Wake up all queues for inner in &inners { inner.ipc.queue_tx.send(QueueEvent::Refresh).await.unwrap(); } for _ in 0..(NUM_MESSAGES / 2) { session .send_message( "john@test.org", &["bill@foobar.org", "delay-random@foobar.org"], "test:no_dkim", "250", ) .await; } loop { tokio::time::sleep(Duration::from_millis(1500)).await; let m = local.queue_receiver.read_queued_messages().await; let e = local.queue_receiver.read_queued_events().await; if m.len() + e.len() != 0 { println!( "Queue still has {} messages and {} events", m.len(), e.len() ); /*for inner in &inners { inner.ipc.queue_tx.send(QueueEvent::Refresh).await.unwrap(); }*/ } else { break; } } local.queue_receiver.assert_queue_is_empty().await; let remote_messages = remote.queue_receiver.read_queued_messages().await; assert_eq!(remote_messages.len(), NUM_MESSAGES * 2); // Make sure local store is queue store_assert_is_empty( &core.core.storage.data, core.core.storage.blob.clone(), false, ) .await; } ================================================ FILE: tests/src/smtp/reporting/analyze.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use crate::smtp::{TestSMTP, inbound::TestQueueEvent, session::TestSession}; use store::{ IterateParams, ValueKey, write::{ReportClass, ValueClass}, }; const CONFIG: &str = r#" [session.rcpt] relay = true [session.data.limits] messages = 100 [report.analysis] addresses = ["reports@*", "*@dmarc.foobar.org", "feedback@foobar.org"] forward = false store = "1s" "#; #[tokio::test(flavor = "multi_thread")] async fn report_analyze() { // Enable logging crate::enable_logging(); // Create temp dir for queue let mut local = TestSMTP::new("smtp_analyze_report_test", CONFIG).await; // Create test message let mut session = local.new_session(); let qr = &mut local.queue_receiver; session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; let addresses = [ "reports@foobar.org", "rep@dmarc.foobar.org", "feedback@foobar.org", ]; let mut ac = 0; let mut total_reports_received = 0; for (test, num_tests) in [("arf", 5), ("dmarc", 5), ("tls", 2)] { for num_test in 1..=num_tests { total_reports_received += 1; session .send_message( "john@test.org", &[addresses[ac % addresses.len()]], &format!("report:{test}{num_test}"), "250", ) .await; qr.assert_no_events(); ac += 1; } } tokio::time::sleep(Duration::from_millis(200)).await; //let c = tokio::time::sleep(Duration::from_secs(86400)).await; // Purging the database shouldn't remove the reports qr.store.purge_store().await.unwrap(); // Make sure the reports are in the store let mut total_reports = 0; qr.store .iterate( IterateParams::new( ValueKey::from(ValueClass::Report(ReportClass::Tls { id: 0, expires: 0 })), ValueKey::from(ValueClass::Report(ReportClass::Arf { id: u64::MAX, expires: u64::MAX, })), ), |_, _| { total_reports += 1; Ok(true) }, ) .await .unwrap(); assert_eq!(total_reports, total_reports_received); // Wait one second, purge, and make sure they are gone tokio::time::sleep(Duration::from_secs(1)).await; qr.store.purge_store().await.unwrap(); let mut total_reports = 0; qr.store .iterate( IterateParams::new( ValueKey::from(ValueClass::Report(ReportClass::Tls { id: 0, expires: 0 })), ValueKey::from(ValueClass::Report(ReportClass::Arf { id: u64::MAX, expires: u64::MAX, })), ), |_, _| { total_reports += 1; Ok(true) }, ) .await .unwrap(); assert_eq!(total_reports, 0); // Test delivery to non-report addresses session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; qr.read_event().await.assert_refresh(); qr.last_queued_message().await; } ================================================ FILE: tests/src/smtp/reporting/dmarc.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{ net::IpAddr, sync::Arc, time::{Duration, Instant}, }; use common::{config::smtp::report::AggregateFrequency, ipc::DmarcEvent}; use mail_auth::{ common::parse::TxtRecordParser, dmarc::Dmarc, report::{ActionDisposition, Disposition, DmarcResult, Record, Report}, }; use smtp::reporting::dmarc::DmarcReporting; use store::write::QueueClass; use crate::smtp::{ DnsCache, TestSMTP, inbound::{TestMessage, sign::SIGNATURES}, session::VerifyResponse, }; const CONFIG: &str = r#" [session.rcpt] relay = true [server] hostname = "mx.example.org" [report] submitter = "'mx.example.org'" [report.dmarc.aggregate] from-name = "'DMARC Report'" from-address = "'reports@' + config_get('report.domain')" org-name = "'Foobar, Inc.'" contact-info = "'https://foobar.org/contact'" send = "daily" max-size = 4096 sign = "['rsa']" "#; #[tokio::test] async fn report_dmarc() { // Enable logging crate::enable_logging(); // Create scheduler let mut local = TestSMTP::new("smtp_report_dmarc_test", CONFIG.to_string() + SIGNATURES).await; // Authorize external report for foobar.org let core = local.build_smtp(); core.txt_add( "foobar.org._report._dmarc.foobar.net", Dmarc::parse(b"v=DMARC1;").unwrap(), Instant::now() + Duration::from_secs(10), ); let qr = &mut local.queue_receiver; // Schedule two events with a same policy and another one with a different policy let dmarc_record = Arc::new( Dmarc::parse( b"v=DMARC1; p=quarantine; rua=mailto:reports@foobar.net,mailto:reports@example.net", ) .unwrap(), ); assert_eq!(dmarc_record.rua().len(), 2); for _ in 0..2 { core.schedule_dmarc(Box::new(DmarcEvent { domain: "foobar.org".to_string(), report_record: Record::new() .with_source_ip("192.168.1.2".parse().unwrap()) .with_action_disposition(ActionDisposition::Pass) .with_dmarc_dkim_result(DmarcResult::Pass) .with_dmarc_spf_result(DmarcResult::Fail) .with_envelope_from("hello@example.org") .with_envelope_to("other@example.org") .with_header_from("bye@example.org"), dmarc_record: dmarc_record.clone(), interval: AggregateFrequency::Weekly, })) .await; } core.schedule_dmarc(Box::new(DmarcEvent { domain: "foobar.org".to_string(), report_record: Record::new() .with_source_ip("a:b:c::e:f".parse().unwrap()) .with_action_disposition(ActionDisposition::Reject) .with_dmarc_dkim_result(DmarcResult::Fail) .with_dmarc_spf_result(DmarcResult::Pass), dmarc_record: dmarc_record.clone(), interval: AggregateFrequency::Weekly, })) .await; tokio::time::sleep(Duration::from_millis(200)).await; let reports = qr.read_report_events().await; assert_eq!(reports.len(), 1); match reports.into_iter().next().unwrap() { QueueClass::DmarcReportHeader(event) => { core.send_dmarc_aggregate_report(event).await; } _ => unreachable!(), } // Expect report let message = qr.expect_message().await; qr.assert_no_events(); assert_eq!(message.message.recipients.len(), 1); assert_eq!( message.message.recipients.last().unwrap().address(), "reports@foobar.net" ); assert_eq!(message.message.return_path.as_ref(), "reports@example.org"); message .read_lines(qr) .await .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") .assert_contains("To: ") .assert_contains("Report Domain: foobar.org") .assert_contains("Submitter: mx.example.org"); // Verify generated report let report = Report::parse_rfc5322(message.read_message(qr).await.as_bytes()).unwrap(); assert_eq!(report.domain(), "foobar.org"); assert_eq!(report.email(), "reports@example.org"); assert_eq!(report.org_name(), "Foobar, Inc."); assert_eq!( report.extra_contact_info().unwrap(), "https://foobar.org/contact" ); assert_eq!(report.p(), Disposition::Quarantine); assert_eq!(report.records().len(), 2); for record in report.records() { let source_ip = record.source_ip().unwrap(); if source_ip == "192.168.1.2".parse::().unwrap() { assert_eq!(record.count(), 2); assert_eq!(record.action_disposition(), ActionDisposition::Pass); assert_eq!(record.envelope_from(), "hello@example.org"); assert_eq!(record.header_from(), "bye@example.org"); assert_eq!(record.envelope_to().unwrap(), "other@example.org"); } else if source_ip == "a:b:c::e:f".parse::().unwrap() { assert_eq!(record.count(), 1); assert_eq!(record.action_disposition(), ActionDisposition::Reject); } else { panic!("unexpected ip {source_ip}"); } } qr.assert_report_is_empty().await; } ================================================ FILE: tests/src/smtp/reporting/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod analyze; pub mod dmarc; pub mod scheduler; pub mod tls; ================================================ FILE: tests/src/smtp/reporting/scheduler.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::sync::Arc; use common::{ config::smtp::report::AggregateFrequency, ipc::{DmarcEvent, PolicyType, TlsEvent}, }; use mail_auth::{ common::parse::TxtRecordParser, dmarc::{Dmarc, URI}, mta_sts::TlsRpt, report::{ActionDisposition, Alignment, Disposition, DmarcResult, PolicyPublished, Record}, }; use store::write::QueueClass; use smtp::reporting::{ dmarc::{DmarcFormat, DmarcReporting}, tls::TlsReporting, }; use crate::smtp::TestSMTP; const CONFIG: &str = r#" [session.rcpt] relay = true [report.dmarc.aggregate] max-size = 500 send = "daily" [report.tls.aggregate] max-size = 550 send = "daily" "#; #[tokio::test] async fn report_scheduler() { // Enable logging crate::enable_logging(); // Create scheduler let local = TestSMTP::new("smtp_report_queue_test", CONFIG).await; let core = local.build_smtp(); let qr = &local.queue_receiver; // Schedule two events with a same policy and another one with a different policy let dmarc_record = Arc::new(Dmarc::parse(b"v=DMARC1; p=quarantine; rua=mailto:dmarc@foobar.org").unwrap()); core.schedule_dmarc(Box::new(DmarcEvent { domain: "foobar.org".to_string(), report_record: Record::new() .with_source_ip("192.168.1.2".parse().unwrap()) .with_action_disposition(ActionDisposition::Pass) .with_dmarc_dkim_result(DmarcResult::Pass) .with_dmarc_spf_result(DmarcResult::Fail) .with_envelope_from("hello@example.org") .with_envelope_to("other@example.org") .with_header_from("bye@example.org"), dmarc_record: dmarc_record.clone(), interval: AggregateFrequency::Weekly, })) .await; // No records should be added once the 550 bytes max size is reached for _ in 0..10 { core.schedule_dmarc(Box::new(DmarcEvent { domain: "foobar.org".to_string(), report_record: Record::new() .with_source_ip("192.168.1.2".parse().unwrap()) .with_action_disposition(ActionDisposition::Pass) .with_dmarc_dkim_result(DmarcResult::Pass) .with_dmarc_spf_result(DmarcResult::Fail) .with_envelope_from("hello@example.org") .with_envelope_to("other@example.org") .with_header_from("bye@example.org"), dmarc_record: dmarc_record.clone(), interval: AggregateFrequency::Weekly, })) .await; } let dmarc_record = Arc::new(Dmarc::parse(b"v=DMARC1; p=reject; rua=mailto:dmarc@foobar.org").unwrap()); core.schedule_dmarc(Box::new(DmarcEvent { domain: "foobar.org".to_string(), report_record: Record::new() .with_source_ip("a:b:c::e:f".parse().unwrap()) .with_action_disposition(ActionDisposition::Reject) .with_dmarc_dkim_result(DmarcResult::Fail) .with_dmarc_spf_result(DmarcResult::Pass), dmarc_record: dmarc_record.clone(), interval: AggregateFrequency::Weekly, })) .await; // Schedule TLS event let tls_record = Arc::new(TlsRpt::parse(b"v=TLSRPTv1;rua=mailto:reports@foobar.org").unwrap()); core.schedule_tls(Box::new(TlsEvent { domain: "foobar.org".to_string(), policy: PolicyType::Tlsa(None), failure: None, tls_record: tls_record.clone(), interval: AggregateFrequency::Daily, })) .await; core.schedule_tls(Box::new(TlsEvent { domain: "foobar.org".to_string(), policy: PolicyType::Tlsa(None), failure: None, tls_record: tls_record.clone(), interval: AggregateFrequency::Daily, })) .await; core.schedule_tls(Box::new(TlsEvent { domain: "foobar.org".to_string(), policy: PolicyType::Sts(None), failure: None, tls_record: tls_record.clone(), interval: AggregateFrequency::Daily, })) .await; core.schedule_tls(Box::new(TlsEvent { domain: "foobar.org".to_string(), policy: PolicyType::None, failure: None, tls_record: tls_record.clone(), interval: AggregateFrequency::Daily, })) .await; // Verify sizes and counts let mut total_tls = 0; let mut total_tls_policies = 0; let mut total_dmarc_policies = 0; let mut last_domain = String::new(); for report in qr.read_report_events().await { match report { QueueClass::DmarcReportHeader(event) => { total_dmarc_policies += 1; assert_eq!(event.due - event.seq_id, 7 * 86400); } QueueClass::TlsReportHeader(event) => { if event.domain != last_domain { last_domain.clone_from(&event.domain); total_tls += 1; } total_tls_policies += 1; assert_eq!(event.due - event.seq_id, 86400); } _ => unreachable!(), } } assert_eq!(total_tls, 1); assert_eq!(total_tls_policies, 3); assert_eq!(total_dmarc_policies, 2); } #[test] fn report_strip_json() { let mut d = DmarcFormat { rua: vec![ URI { uri: "hello".to_string(), max_size: 0, }, URI { uri: "world".to_string(), max_size: 0, }, ], policy: PolicyPublished { domain: "example.org".to_string(), version_published: None, adkim: Alignment::Relaxed, aspf: Alignment::Strict, p: Disposition::Quarantine, sp: Disposition::Reject, testing: false, fo: None, }, records: vec![ Record::default() .with_count(1) .with_envelope_from("domain.net") .with_envelope_to("other.org"), ], }; let mut s = serde_json::to_string(&d).unwrap(); s.truncate(s.len() - 2); let r = Record::default() .with_count(2) .with_envelope_from("otherdomain.net") .with_envelope_to("otherother.org"); let rs = serde_json::to_string(&r).unwrap(); d.records.push(r); assert_eq!( serde_json::from_str::(&format!("{s},{rs}]}}")).unwrap(), d ); } ================================================ FILE: tests/src/smtp/reporting/tls.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{io::Read, sync::Arc, time::Duration}; use common::{config::smtp::report::AggregateFrequency, ipc::TlsEvent}; use mail_auth::{ common::parse::TxtRecordParser, flate2::read::GzDecoder, mta_sts::TlsRpt, report::tlsrpt::{FailureDetails, PolicyType, ResultType, TlsReport}, }; use store::write::QueueClass; use smtp::reporting::tls::{TLS_HTTP_REPORT, TlsReporting}; use crate::smtp::{ TestSMTP, inbound::{TestMessage, sign::SIGNATURES}, session::VerifyResponse, }; const CONFIG: &str = r#" [session.rcpt] relay = true [report] submitter = "'mx.example.org'" [report.tls.aggregate] from-name = "'Report Subsystem'" from-address = "'reports@example.org'" org-name = "'Foobar, Inc.'" contact-info = "'https://foobar.org/contact'" send = "daily" max-size = 1532 sign = "['rsa']" "#; #[tokio::test] async fn report_tls() { // Enable logging crate::enable_logging(); // Create scheduler let mut local = TestSMTP::new("smtp_report_tls_test", CONFIG.to_string() + SIGNATURES).await; let core = local.build_smtp(); let qr = &mut local.queue_receiver; // Schedule TLS reports to be delivered via email let tls_record = Arc::new(TlsRpt::parse(b"v=TLSRPTv1;rua=mailto:reports@foobar.org").unwrap()); for _ in 0..2 { // Add two successful records core.schedule_tls(Box::new(TlsEvent { domain: "foobar.org".to_string(), policy: common::ipc::PolicyType::None, failure: None, tls_record: tls_record.clone(), interval: AggregateFrequency::Daily, })) .await; } for (policy, rt) in [ ( common::ipc::PolicyType::None, // Quota limited at 1532 bytes, this should not be included in the report. ResultType::CertificateExpired, ), (common::ipc::PolicyType::Tlsa(None), ResultType::TlsaInvalid), ( common::ipc::PolicyType::Sts(None), ResultType::StsPolicyFetchError, ), ( common::ipc::PolicyType::Sts(None), ResultType::StsPolicyInvalid, ), ( common::ipc::PolicyType::Sts(None), ResultType::StsWebpkiInvalid, ), ] { core.schedule_tls(Box::new(TlsEvent { domain: "foobar.org".to_string(), policy, failure: FailureDetails::new(rt).into(), tls_record: tls_record.clone(), interval: AggregateFrequency::Daily, })) .await; } // Wait for flush tokio::time::sleep(Duration::from_millis(200)).await; let reports = qr.read_report_events().await; assert_eq!(reports.len(), 3); let mut tls_reports = Vec::with_capacity(3); for report in reports { match report { QueueClass::TlsReportHeader(event) => { tls_reports.push(event); } _ => unreachable!(), } } core.send_tls_aggregate_report(tls_reports).await; // Expect report let message = qr.expect_message().await; assert_eq!( message.message.recipients.last().unwrap().address(), "reports@foobar.org" ); assert_eq!(message.message.return_path.as_ref(), "reports@example.org"); message .read_lines(qr) .await .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") .assert_contains("To: ") .assert_contains("Report Domain: foobar.org") .assert_contains("Submitter: mx.example.org"); // Verify generated report let report = TlsReport::parse_rfc5322(message.read_message(qr).await.as_bytes()).unwrap(); assert_eq!(report.organization_name.unwrap(), "Foobar, Inc."); assert_eq!(report.contact_info.unwrap(), "https://foobar.org/contact"); assert_eq!(report.policies.len(), 3); let mut seen = [false; 3]; for policy in report.policies { match policy.policy.policy_type { PolicyType::Tlsa => { seen[0] = true; assert_eq!(policy.summary.total_failure, 1); assert_eq!(policy.summary.total_success, 0); assert_eq!(policy.policy.policy_domain, "foobar.org"); assert_eq!(policy.failure_details.len(), 1); assert_eq!( policy.failure_details.first().unwrap().result_type, ResultType::TlsaInvalid ); } PolicyType::Sts => { seen[1] = true; assert_eq!(policy.summary.total_failure, 2); assert_eq!(policy.summary.total_success, 0); assert_eq!(policy.policy.policy_domain, "foobar.org"); assert_eq!(policy.failure_details.len(), 2); assert!( policy .failure_details .iter() .any(|d| d.result_type == ResultType::StsPolicyFetchError) ); assert!( policy .failure_details .iter() .any(|d| d.result_type == ResultType::StsPolicyInvalid) ); } PolicyType::NoPolicyFound => { seen[2] = true; assert_eq!(policy.summary.total_failure, 1); assert_eq!(policy.summary.total_success, 2); assert_eq!(policy.policy.policy_domain, "foobar.org"); assert_eq!(policy.failure_details.len(), 1); /*assert_eq!( policy.failure_details.first().unwrap().result_type, ResultType::CertificateExpired );*/ } PolicyType::Other => unreachable!(), } } assert!(seen[0]); assert!(seen[1]); assert!(seen[2]); // Schedule TLS reports to be delivered via https let tls_record = Arc::new(TlsRpt::parse(b"v=TLSRPTv1;rua=https://127.0.0.1/tls").unwrap()); for _ in 0..2 { // Add two successful records core.schedule_tls(Box::new(TlsEvent { domain: "foobar.org".to_string(), policy: common::ipc::PolicyType::None, failure: None, tls_record: tls_record.clone(), interval: AggregateFrequency::Daily, })) .await; } let reports = qr.read_report_events().await; assert_eq!(reports.len(), 1); match reports.into_iter().next().unwrap() { QueueClass::TlsReportHeader(event) => { core.send_tls_aggregate_report(vec![event]).await; } _ => unreachable!(), } tokio::time::sleep(Duration::from_millis(200)).await; // Uncompress report { let gz_report = TLS_HTTP_REPORT.lock(); let mut file = GzDecoder::new(&gz_report[..]); let mut buf = Vec::new(); file.read_to_end(&mut buf).unwrap(); let report = TlsReport::parse_json(&buf).unwrap(); assert_eq!(report.organization_name.unwrap(), "Foobar, Inc."); assert_eq!(report.contact_info.unwrap(), "https://foobar.org/contact"); assert_eq!(report.policies.len(), 1); } qr.assert_report_is_empty().await; } ================================================ FILE: tests/src/smtp/session.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::{borrow::Cow, path::PathBuf, sync::Arc}; use common::{ Server, config::server::ServerProtocol, listener::{ServerInstance, SessionStream, TcpAcceptor, limiter::ConcurrencyLimiter}, }; use rustls::{ServerConfig, server::ResolvesServerCert}; use tokio::{ io::{AsyncRead, AsyncWrite}, sync::watch, }; use smtp::core::{Session, SessionAddress, SessionData, SessionParameters, State}; use tokio_rustls::TlsAcceptor; use utils::snowflake::SnowflakeIdGenerator; pub struct DummyIo { pub tx_buf: Vec, pub rx_buf: Vec, pub tls: bool, } impl AsyncRead for DummyIo { fn poll_read( mut self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>, buf: &mut tokio::io::ReadBuf<'_>, ) -> std::task::Poll> { if !self.rx_buf.is_empty() { buf.put_slice(&self.rx_buf); self.rx_buf.clear(); std::task::Poll::Ready(Ok(())) } else { std::task::Poll::Pending } } } impl AsyncWrite for DummyIo { fn poll_write( mut self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>, buf: &[u8], ) -> std::task::Poll> { self.tx_buf.extend_from_slice(buf); std::task::Poll::Ready(Ok(buf.len())) } fn poll_flush( self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>, ) -> std::task::Poll> { std::task::Poll::Ready(Ok(())) } fn poll_shutdown( self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>, ) -> std::task::Poll> { std::task::Poll::Ready(Ok(())) } } impl SessionStream for DummyIo { fn is_tls(&self) -> bool { self.tls } fn tls_version_and_cipher(&self) -> (Cow<'static, str>, Cow<'static, str>) { ("".into(), "".into()) } } impl Unpin for DummyIo {} #[allow(async_fn_in_trait)] pub trait TestSession { fn test(server: Server) -> Self; fn test_with_shutdown(server: Server, shutdown_rx: watch::Receiver) -> Self; fn response(&mut self) -> Vec; fn write_rx(&mut self, data: &str); async fn rset(&mut self); async fn cmd(&mut self, cmd: &str, expected_code: &str) -> Vec; async fn ehlo(&mut self, host: &str) -> Vec; async fn mail_from(&mut self, from: &str, expected_code: &str); async fn rcpt_to(&mut self, to: &str, expected_code: &str); async fn data(&mut self, data: &str, expected_code: &str); async fn send_message(&mut self, from: &str, to: &[&str], data: &str, expected_code: &str); async fn test_builder(&self); } impl TestSession for Session { fn test_with_shutdown(server: Server, shutdown_rx: watch::Receiver) -> Self { Self { state: State::default(), instance: Arc::new(ServerInstance::test_with_shutdown(shutdown_rx)), server, stream: DummyIo { rx_buf: vec![], tx_buf: vec![], tls: false, }, data: SessionData::new( "127.0.0.1".parse().unwrap(), 0, "127.0.0.1".parse().unwrap(), 0, Default::default(), 0, ), params: SessionParameters::default(), hostname: "localhost".into(), } } fn test(server: Server) -> Self { Self::test_with_shutdown(server, watch::channel(false).1) } fn response(&mut self) -> Vec { if !self.stream.tx_buf.is_empty() { let response = std::str::from_utf8(&self.stream.tx_buf) .unwrap() .split("\r\n") .filter_map(|r| { if !r.is_empty() { r.to_string().into() } else { None } }) .collect::>(); self.stream.tx_buf.clear(); response } else { panic!("There was no response."); } } fn write_rx(&mut self, data: &str) { self.stream.rx_buf.extend_from_slice(data.as_bytes()); } async fn rset(&mut self) { self.ingest(b"RSET\r\n").await.unwrap(); self.response().assert_code("250"); } async fn cmd(&mut self, cmd: &str, expected_code: &str) -> Vec { self.ingest(format!("{cmd}\r\n").as_bytes()).await.unwrap(); self.response().assert_code(expected_code) } async fn ehlo(&mut self, host: &str) -> Vec { self.ingest(format!("EHLO {host}\r\n").as_bytes()) .await .unwrap(); self.response().assert_code("250") } async fn mail_from(&mut self, from: &str, expected_code: &str) { self.ingest( if !from.starts_with('<') { format!("MAIL FROM:<{from}>\r\n") } else { format!("MAIL FROM:{from}\r\n") } .as_bytes(), ) .await .unwrap(); self.response().assert_code(expected_code); } async fn rcpt_to(&mut self, to: &str, expected_code: &str) { self.ingest( if !to.starts_with('<') { format!("RCPT TO:<{to}>\r\n") } else { format!("RCPT TO:{to}\r\n") } .as_bytes(), ) .await .unwrap(); self.response().assert_code(expected_code); } async fn data(&mut self, data: &str, expected_code: &str) { self.ingest(b"DATA\r\n").await.unwrap(); self.response().assert_code("354"); if let Some(file) = data.strip_prefix("test:") { self.ingest(load_test_message(file, "messages").as_bytes()) .await .unwrap(); } else if let Some(file) = data.strip_prefix("report:") { self.ingest(load_test_message(file, "reports").as_bytes()) .await .unwrap(); } else { self.ingest(data.as_bytes()).await.unwrap(); } self.ingest(b"\r\n.\r\n").await.unwrap(); self.response().assert_code(expected_code); } async fn send_message(&mut self, from: &str, to: &[&str], data: &str, expected_code: &str) { self.mail_from(from, "250").await; for to in to { self.rcpt_to(to, "250").await; } self.data(data, expected_code).await; } async fn test_builder(&self) { let message = self .build_message( SessionAddress { address: "bill@foobar.org".into(), address_lcase: "bill@foobar.org".into(), domain: "foobar.org".into(), flags: 123, dsn_info: Some("envelope1".into()), }, vec![ SessionAddress { address: "a@foobar.org".into(), address_lcase: "a@foobar.org".into(), domain: "foobar.org".into(), flags: 1, dsn_info: None, }, SessionAddress { address: "b@test.net".into(), address_lcase: "b@test.net".into(), domain: "test.net".into(), flags: 2, dsn_info: None, }, SessionAddress { address: "c@foobar.org".into(), address_lcase: "c@foobar.org".into(), domain: "foobar.org".into(), flags: 3, dsn_info: None, }, SessionAddress { address: "d@test.net".into(), address_lcase: "d@test.net".into(), domain: "test.net".into(), flags: 4, dsn_info: None, }, ], self.server.inner.data.queue_id_gen.generate(), 0, ) .await; let rcpts = ["a@foobar.org", "b@test.net", "c@foobar.org", "d@test.net"]; for rcpt in &message.message.recipients { let idx = (rcpt.flags - 1) as usize; assert_eq!(rcpts[idx], rcpt.address()); } } } pub fn load_test_message(file: &str, test: &str) -> String { let mut test_file = PathBuf::from(env!("CARGO_MANIFEST_DIR")); test_file.push("resources"); test_file.push("smtp"); test_file.push(test); test_file.push(format!("{file}.eml")); std::fs::read_to_string(test_file).unwrap() } pub trait VerifyResponse { fn assert_code(self, expected_code: &str) -> Self; fn assert_contains(self, expected_text: &str) -> Self; fn assert_not_contains(self, expected_text: &str) -> Self; fn assert_count(self, text: &str, occurrences: usize) -> Self; } impl VerifyResponse for Vec { fn assert_code(self, expected_code: &str) -> Self { if self.last().expect("response").starts_with(expected_code) { self } else { panic!("Expected {:?} but got {}.", expected_code, self.join("\n")); } } fn assert_contains(self, expected_text: &str) -> Self { if self.iter().any(|line| line.contains(expected_text)) { self } else { panic!("Expected {:?} but got {}.", expected_text, self.join("\n")); } } fn assert_not_contains(self, expected_text: &str) -> Self { if !self.iter().any(|line| line.contains(expected_text)) { self } else { panic!( "Not expecting {:?} but got it {}.", expected_text, self.join("\n") ); } } fn assert_count(self, text: &str, occurrences: usize) -> Self { assert_eq!( self.iter().filter(|l| l.contains(text)).count(), occurrences, "Expected {} occurrences of {:?}, found {}.", occurrences, text, self.iter().filter(|l| l.contains(text)).count() ); self } } pub trait TestServerInstance { fn test_with_shutdown(shutdown_rx: watch::Receiver) -> Self; } impl TestServerInstance for ServerInstance { fn test_with_shutdown(shutdown_rx: watch::Receiver) -> Self { let tls_config = Arc::new( ServerConfig::builder() .with_no_client_auth() .with_cert_resolver(Arc::new(DummyCertResolver)), ); Self { id: "smtp".to_string(), protocol: ServerProtocol::Smtp, acceptor: TcpAcceptor::Tls { config: tls_config.clone(), acceptor: TlsAcceptor::from(tls_config), implicit: false, }, limiter: ConcurrencyLimiter::new(100), shutdown_rx, proxy_networks: vec![], span_id_gen: Arc::new(SnowflakeIdGenerator::new()), } } } #[derive(Debug)] pub struct DummyCertResolver; impl ResolvesServerCert for DummyCertResolver { fn resolve(&self, _: rustls::server::ClientHello) -> Option> { None } } pub fn test_server_instance() -> ServerInstance { ServerInstance::test_with_shutdown(watch::channel(false).1) } ================================================ FILE: tests/src/store/blob.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::store::{CONFIG, TempDir, cleanup::store_destroy}; use ahash::AHashMap; use common::{Core, Inner, Server, config::storage::Storage}; use email::message::metadata::MessageMetadata; use http::management::stores::destroy_account_blobs; use std::sync::Arc; use store::{ BlobStore, Serialize, SerializeInfallible, Stores, write::{Archiver, BatchBuilder, BlobLink, BlobOp, ValueClass, blob::BlobQuota, now}, }; use types::{blob::BlobClass, blob_hash::BlobHash, collection::Collection, field::EmailField}; use utils::config::Config; #[tokio::test] pub async fn blob_tests() { let temp_dir = TempDir::new("blob_tests", true); let mut config = Config::new(CONFIG.replace("{TMP}", temp_dir.path.as_path().to_str().unwrap())).unwrap(); let stores = Stores::parse_all(&mut config, false).await; for (store_id, blob_store) in &stores.blob_stores { println!("Testing blob store {}...", store_id); test_store(blob_store.clone()).await; } for (store_id, store) in stores.stores { println!("Testing blob management on store {}...", store_id); // Init store store_destroy(&store).await; // Test internal blob store let blob_store: BlobStore = store.clone().into(); let server = Server { inner: Arc::new(Inner::default()), core: Arc::new(Core { storage: Storage { data: store.clone(), blob: blob_store.clone(), ..Default::default() }, ..Default::default() }), }; // Blob hash exists let hash = BlobHash::generate(b"abc".as_slice()); assert!(!store.blob_exists(&hash).await.unwrap()); // Reserve blob let until = now() + 1; store .write( BatchBuilder::new() .with_account_id(0) .set( BlobOp::Link { to: BlobLink::Temporary { until }, hash: hash.clone(), }, 1024u32.serialize(), ) .build_all(), ) .await .unwrap(); // Uncommitted blob, should not exist assert!(!store.blob_exists(&hash).await.unwrap()); // Write blob to store blob_store.put_blob(hash.as_ref(), b"abc").await.unwrap(); // Commit blob store .write( BatchBuilder::new() .set(BlobOp::Commit { hash: hash.clone() }, Vec::new()) .build_all(), ) .await .unwrap(); // Blob hash should now exist assert!(store.blob_exists(&hash).await.unwrap()); assert!( blob_store .get_blob(hash.as_ref(), 0..usize::MAX) .await .unwrap() .is_some() ); // AccountId 0 should be able to read blob assert!( store .blob_has_access( &hash, BlobClass::Reserved { account_id: 0, expires: until } ) .await .unwrap() ); // AccountId 1 should not be able to read blob assert!( !store .blob_has_access( &hash, BlobClass::Reserved { account_id: 1, expires: until } ) .await .unwrap() ); // Blob already expired, quota should be 0 tokio::time::sleep(std::time::Duration::from_secs(1)).await; assert_eq!( store.blob_quota(0).await.unwrap(), BlobQuota { bytes: 0, count: 0 } ); // Purge expired blobs store.purge_blobs(blob_store.clone()).await.unwrap(); // Blob hash should no longer exist assert!(!store.blob_exists(&hash).await.unwrap()); // AccountId 0 should not be able to read blob assert!( !store .blob_has_access( &hash, BlobClass::Reserved { account_id: 0, expires: until } ) .await .unwrap() ); // Blob should no longer be in store assert!( blob_store .get_blob(hash.as_ref(), 0..usize::MAX) .await .unwrap() .is_none() ); // Upload one linked blob to accountId 1, two linked blobs to accountId 0, and three unlinked (reserved) blobs to accountId 2 let expiry_times = AHashMap::from_iter([ (b"abc", now() - 10), (b"efg", now() + 10), (b"hij", now() + 10), ]); for (document_id, (blob, blob_value)) in [ (b"123", vec![]), (b"456", vec![]), (b"789", vec![]), (b"abc", 5000u32.serialize()), (b"efg", 1000u32.serialize()), (b"hij", 2000u32.serialize()), ] .into_iter() .enumerate() { let hash = BlobHash::generate(blob.as_slice()); let mut batch = BatchBuilder::new(); batch .with_account_id(if document_id > 0 { 0 } else { 1 }) .with_collection(Collection::Email) .with_document(document_id as u32); if let Some(until) = expiry_times.get(blob) { if !blob_value.is_empty() { batch.set( BlobOp::Quota { hash: hash.clone(), until: *until, }, blob_value, ); } batch.set( BlobOp::Link { hash: hash.clone(), to: BlobLink::Temporary { until: *until }, }, vec![], ); } else { batch .set( BlobOp::Link { hash: hash.clone(), to: BlobLink::Document, }, vec![], ) .set( ValueClass::Property(EmailField::Metadata.into()), Archiver::new(MessageMetadata { contents: Default::default(), rcvd_attach: Default::default(), blob_hash: hash.clone(), blob_body_offset: Default::default(), preview: Default::default(), raw_headers: Default::default(), }) .serialize() .unwrap(), ); }; batch.set(BlobOp::Commit { hash: hash.clone() }, vec![]); store.write(batch.build_all()).await.unwrap(); blob_store .put_blob(hash.as_ref(), blob.as_slice()) .await .unwrap(); } // One of the reserved blobs expired and should not count towards quota assert_eq!( store.blob_quota(0).await.unwrap(), BlobQuota { bytes: 3000, count: 2 } ); assert_eq!( store.blob_quota(1).await.unwrap(), BlobQuota { bytes: 0, count: 0 } ); // Purge expired blobs and make sure nothing else is deleted store.purge_blobs(blob_store.clone()).await.unwrap(); for (pos, (blob, blob_class)) in [ ( b"abc", BlobClass::Reserved { account_id: 0, expires: expiry_times[&b"abc"], }, ), ( b"123", BlobClass::Linked { account_id: 1, collection: 0, document_id: 0, }, ), ( b"456", BlobClass::Linked { account_id: 0, collection: 0, document_id: 1, }, ), ( b"789", BlobClass::Linked { account_id: 0, collection: 0, document_id: 2, }, ), ( b"efg", BlobClass::Reserved { account_id: 0, expires: expiry_times[&b"efg"], }, ), ( b"hij", BlobClass::Reserved { account_id: 0, expires: expiry_times[&b"hij"], }, ), ] .into_iter() .enumerate() { let ct = pos == 0; let hash = BlobHash::generate(blob.as_slice()); assert!(store.blob_has_access(&hash, blob_class).await.unwrap() ^ ct); assert!(store.blob_exists(&hash).await.unwrap() ^ ct); assert!( blob_store .get_blob(hash.as_ref(), 0..usize::MAX) .await .unwrap() .is_some() ^ ct ); } // AccountId 0 should not have access to accountId 1's blobs assert!( !store .blob_has_access( BlobHash::generate(b"123".as_slice()), BlobClass::Linked { account_id: 0, collection: 0, document_id: 0, } ) .await .unwrap() ); // Unlink blob store .write( BatchBuilder::new() .with_account_id(0) .with_collection(Collection::Email) .with_document(2) .clear(BlobOp::Link { hash: BlobHash::generate(b"789".as_slice()), to: BlobLink::Document, }) .build_all(), ) .await .unwrap(); // Purge and make sure blob is deleted store.purge_blobs(blob_store.clone()).await.unwrap(); for (pos, (blob, blob_class)) in [ ( b"789", BlobClass::Linked { account_id: 0, collection: 0, document_id: 2, }, ), ( b"123", BlobClass::Linked { account_id: 1, collection: 0, document_id: 0, }, ), ( b"456", BlobClass::Linked { account_id: 0, collection: 0, document_id: 1, }, ), ( b"efg", BlobClass::Reserved { account_id: 0, expires: expiry_times[&b"efg"], }, ), ( b"hij", BlobClass::Reserved { account_id: 0, expires: expiry_times[&b"hij"], }, ), ] .into_iter() .enumerate() { let ct = pos == 0; let hash = BlobHash::generate(blob.as_slice()); assert!(store.blob_has_access(&hash, blob_class).await.unwrap() ^ ct); assert!(store.blob_exists(&hash).await.unwrap() ^ ct); assert!( blob_store .get_blob(hash.as_ref(), 0..usize::MAX) .await .unwrap() .is_some() ^ ct ); } // Unlink all blobs from accountId 1 and purge destroy_account_blobs(&server, 1).await.unwrap(); store.purge_blobs(blob_store.clone()).await.unwrap(); // Make sure only accountId 0's blobs are left for (pos, (blob, blob_class)) in [ ( b"123", BlobClass::Linked { account_id: 1, collection: 0, document_id: 0, }, ), ( b"456", BlobClass::Linked { account_id: 0, collection: 0, document_id: 1, }, ), ( b"efg", BlobClass::Reserved { account_id: 0, expires: expiry_times[&b"efg"], }, ), ( b"hij", BlobClass::Reserved { account_id: 0, expires: expiry_times[&b"hij"], }, ), ] .into_iter() .enumerate() { let ct = pos == 0; let hash = BlobHash::generate(blob.as_slice()); assert!(store.blob_has_access(&hash, blob_class).await.unwrap() ^ ct); assert!(store.blob_exists(&hash).await.unwrap() ^ ct); assert!( blob_store .get_blob(hash.as_ref(), 0..usize::MAX) .await .unwrap() .is_some() ^ ct ); } } temp_dir.delete(); } async fn test_store(store: BlobStore) { // Test small blob const DATA: &[u8] = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce erat nisl, dignissim a porttitor id, varius nec arcu. Sed mauris."; let hash = BlobHash::generate(DATA); store.put_blob(hash.as_slice(), DATA).await.unwrap(); assert_eq!( String::from_utf8( store .get_blob(hash.as_slice(), 0..usize::MAX) .await .unwrap() .unwrap() ) .unwrap(), std::str::from_utf8(DATA).unwrap() ); assert_eq!( String::from_utf8( store .get_blob(hash.as_slice(), 11..57) .await .unwrap() .unwrap() ) .unwrap(), std::str::from_utf8(&DATA[11..57]).unwrap() ); assert!(store.delete_blob(hash.as_slice()).await.unwrap()); assert!( store .get_blob(hash.as_slice(), 0..usize::MAX) .await .unwrap() .is_none() ); // Test large blob let mut data = Vec::with_capacity(50 * 1024 * 1024); while data.len() < 50 * 1024 * 1024 { data.extend_from_slice(DATA); let marker = format!(" [{}] ", data.len()); data.extend_from_slice(marker.as_bytes()); } let hash = BlobHash::generate(&data); store.put_blob(hash.as_slice(), &data).await.unwrap(); assert_eq!( String::from_utf8( store .get_blob(hash.as_slice(), 0..usize::MAX) .await .unwrap() .unwrap() ) .unwrap(), std::str::from_utf8(&data).unwrap() ); assert_eq!( String::from_utf8( store .get_blob(hash.as_slice(), 3000111..4000999) .await .unwrap() .unwrap() ) .unwrap(), std::str::from_utf8(&data[3000111..4000999]).unwrap() ); assert!(store.delete_blob(hash.as_slice()).await.unwrap()); assert!( store .get_blob(hash.as_slice(), 0..usize::MAX) .await .unwrap() .is_none() ); } ================================================ FILE: tests/src/store/cleanup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use store::{ ValueKey, write::{key::DeserializeBigEndian, *}, *, }; use trc::AddContext; use types::blob_hash::{BLOB_HASH_LEN, BlobHash}; pub async fn store_destroy(store: &Store) { store_destroy_sql_indexes(store).await; for subspace in [ SUBSPACE_ACL, SUBSPACE_DIRECTORY, SUBSPACE_TASK_QUEUE, SUBSPACE_INDEXES, SUBSPACE_BLOB_EXTRA, SUBSPACE_BLOB_LINK, SUBSPACE_LOGS, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_COUNTER, SUBSPACE_PROPERTY, SUBSPACE_SETTINGS, SUBSPACE_BLOBS, SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUEUE_EVENT, SUBSPACE_QUOTA, SUBSPACE_REPORT_OUT, SUBSPACE_REPORT_IN, SUBSPACE_TELEMETRY_SPAN, SUBSPACE_TELEMETRY_METRIC, SUBSPACE_SEARCH_INDEX, ] { if subspace == SUBSPACE_SEARCH_INDEX && store.is_pg_or_mysql() { continue; } store .delete_range( AnyKey { subspace, key: vec![0u8], }, AnyKey { subspace, key: vec![u8::MAX; 16], }, ) .await .unwrap(); } } pub async fn search_store_destroy(store: &SearchStore) { match &store { SearchStore::Store(store) => { store_destroy_sql_indexes(store).await; } SearchStore::ElasticSearch(store) => { if let Err(err) = store.drop_indexes().await { eprintln!("Failed to drop elasticsearch indexes: {}", err); } store.create_indexes(3, 0, false).await.unwrap(); } SearchStore::MeiliSearch(store) => { if let Err(err) = store.drop_indexes().await { eprintln!("Failed to drop meilisearch indexes: {}", err); } store.create_indexes().await.unwrap(); } } } #[allow(unused_variables)] async fn store_destroy_sql_indexes(store: &Store) { #[cfg(any(feature = "postgres", feature = "mysql"))] { if store.is_pg_or_mysql() { for index in [ SearchIndex::Email, SearchIndex::Calendar, SearchIndex::Contacts, SearchIndex::Tracing, ] { #[cfg(feature = "postgres")] let table = index.psql_table(); #[cfg(feature = "mysql")] let table = index.mysql_table(); store .sql_query::(&format!("TRUNCATE TABLE {table}"), vec![]) .await .unwrap(); } } } } pub async fn store_blob_expire_all(store: &Store) { // Delete all temporary hashes let from_key = ValueKey { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Blob(BlobOp::Commit { hash: BlobHash::default(), }), }; let to_key = ValueKey { account_id: u32::MAX, collection: u8::MAX, document_id: u32::MAX, class: ValueClass::Blob(BlobOp::Link { hash: BlobHash::new_max(), to: BlobLink::Document, }), }; let mut batch = BatchBuilder::new(); let mut last_account_id = u32::MAX; store .iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { if key.len() == BLOB_HASH_LEN + U32_LEN + U64_LEN { let account_id = key .deserialize_be_u32(BLOB_HASH_LEN) .caused_by(trc::location!())?; if account_id != last_account_id { last_account_id = account_id; batch.with_account_id(account_id); } let hash = BlobHash::try_from_hash_slice(key.get(..BLOB_HASH_LEN).unwrap()).unwrap(); let until = key .deserialize_be_u64(BLOB_HASH_LEN + U32_LEN) .caused_by(trc::location!())?; match value.first().copied() { Some(BlobLink::QUOTA_LINK) => { batch.clear(ValueClass::Blob(BlobOp::Quota { hash: hash.clone(), until, })); } Some(BlobLink::UNDELETE_LINK) => { batch.clear(ValueClass::Blob(BlobOp::Undelete { hash: hash.clone(), until, })); } Some(BlobLink::SPAM_SAMPLE_LINK) => { batch.clear(ValueClass::Blob(BlobOp::SpamSample { hash: hash.clone(), until, })); } _ => {} } batch.clear(ValueClass::Blob(BlobOp::Link { hash, to: BlobLink::Temporary { until }, })); } Ok(true) }, ) .await .unwrap(); store.write(batch.build_all()).await.unwrap(); } pub async fn store_lookup_expire_all(store: &Store) { // Delete all temporary counters let from_key = ValueKey::from(ValueClass::InMemory(InMemoryClass::Key(vec![0u8]))); let to_key = ValueKey::from(ValueClass::InMemory(InMemoryClass::Key(vec![u8::MAX; 10]))); let mut expired_keys = Vec::new(); let mut expired_counters = Vec::new(); store .iterate(IterateParams::new(from_key, to_key), |key, value| { let expiry = value.deserialize_be_u64(0).caused_by(trc::location!())?; if expiry == 0 { expired_counters.push(key.to_vec()); } else if expiry != u64::MAX { expired_keys.push(key.to_vec()); } Ok(true) }) .await .unwrap(); if !expired_keys.is_empty() { let mut batch = BatchBuilder::new(); for key in expired_keys { batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Key(key)), op: ValueOp::Clear, }); if batch.is_large_batch() { store.write(batch.build_all()).await.unwrap(); batch = BatchBuilder::new(); } } if !batch.is_empty() { store.write(batch.build_all()).await.unwrap(); } } if !expired_counters.is_empty() { let mut batch = BatchBuilder::new(); for key in expired_counters { batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Counter(key.clone())), op: ValueOp::Clear, }); batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Key(key)), op: ValueOp::Clear, }); if batch.is_large_batch() { store.write(batch.build_all()).await.unwrap(); batch = BatchBuilder::new(); } } if !batch.is_empty() { store.write(batch.build_all()).await.unwrap(); } } } #[allow(unused_variables)] pub async fn store_assert_is_empty(store: &Store, blob_store: BlobStore, include_directory: bool) { store_blob_expire_all(store).await; store_lookup_expire_all(store).await; store.purge_blobs(blob_store).await.unwrap(); store.purge_store().await.unwrap(); let store = store.clone(); let mut failed = false; for (subspace, with_values) in [ (SUBSPACE_ACL, true), (SUBSPACE_DIRECTORY, true), (SUBSPACE_TASK_QUEUE, true), (SUBSPACE_IN_MEMORY_VALUE, true), (SUBSPACE_IN_MEMORY_COUNTER, false), (SUBSPACE_PROPERTY, true), (SUBSPACE_SETTINGS, true), (SUBSPACE_QUEUE_MESSAGE, true), (SUBSPACE_QUEUE_EVENT, true), (SUBSPACE_REPORT_OUT, true), (SUBSPACE_REPORT_IN, true), (SUBSPACE_BLOB_EXTRA, true), (SUBSPACE_BLOB_LINK, true), (SUBSPACE_BLOBS, true), (SUBSPACE_COUNTER, false), (SUBSPACE_QUOTA, false), (SUBSPACE_INDEXES, false), (SUBSPACE_TELEMETRY_SPAN, true), (SUBSPACE_TELEMETRY_METRIC, true), (SUBSPACE_SEARCH_INDEX, true), ] { if (subspace == SUBSPACE_SEARCH_INDEX && store.is_pg_or_mysql()) || (subspace == SUBSPACE_DIRECTORY && !include_directory) { continue; } let from_key = AnyKey { subspace, key: vec![0u8], }; let to_key = AnyKey { subspace, key: vec![u8::MAX; 10], }; store .iterate( IterateParams::new(from_key, to_key).set_values(with_values), |key, value| { match subspace { SUBSPACE_COUNTER if key.len() == U32_LEN + 1 || key.len() == U32_LEN => { // Message ID and change ID counters return Ok(true); } SUBSPACE_INDEXES => { println!( concat!( "Found index key, account {}, collection {}, ", "document {}, property {}, value {:?}: {:?}" ), u32::from_be_bytes(key[0..4].try_into().unwrap()), key[4], u32::from_be_bytes(key[key.len() - 4..].try_into().unwrap()), key[5], String::from_utf8_lossy(&key[6..key.len() - 4]), key ); } _ => { println!( "Found key in {:?}: {:?} ({:?}) = {:?} ({:?})", char::from(subspace), key, String::from_utf8_lossy(key), value, String::from_utf8_lossy(value) ); } } failed = true; Ok(true) }, ) .await .unwrap(); } // Delete logs and counters store .delete_range( AnyKey { subspace: SUBSPACE_LOGS, key: &[0u8], }, AnyKey { subspace: SUBSPACE_LOGS, key: &[ u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX, ], }, ) .await .unwrap(); store .delete_range( AnyKey { subspace: SUBSPACE_COUNTER, key: &[0u8], }, AnyKey { subspace: SUBSPACE_COUNTER, key: (u32::MAX / 2).to_be_bytes().as_slice(), }, ) .await .unwrap(); if failed { panic!("Store is not empty."); } } ================================================ FILE: tests/src/store/import_export.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::store::{ TempDir, cleanup::{store_assert_is_empty, store_destroy}, }; use ahash::AHashSet; use common::{Core, DATABASE_SCHEMA_VERSION, manager::backup::BackupParams}; use store::{ rand, write::{ AnyClass, AnyKey, BatchBuilder, BlobLink, BlobOp, DirectoryClass, Operation, QueueClass, QueueEvent, ValueClass, }, *, }; use types::{ blob_hash::BlobHash, collection::{Collection, SyncCollection}, field::{Field, MailboxField}, }; pub async fn test(db: Store) { let mut core = Core::default(); core.storage.data = db.clone(); core.storage.blob = db.clone().into(); core.storage.fts = db.clone().into(); core.storage.lookup = db.clone().into(); // Make sure the store is empty store_assert_is_empty(&db, db.clone().into(), true).await; // Create blobs println!("Creating blobs..."); let mut batch = BatchBuilder::new(); batch.set( ValueClass::Any(AnyClass { subspace: SUBSPACE_PROPERTY, key: vec![0u8], }), DATABASE_SCHEMA_VERSION.serialize(), ); let mut blob_hashes = Vec::new(); for blob_size in [16, 128, 1024, 2056, 102400] { let data = random_bytes(blob_size); let hash = BlobHash::generate(data.as_slice()); blob_hashes.push(hash.clone()); core.storage .blob .put_blob(hash.as_ref(), &data) .await .unwrap(); batch.set(ValueClass::Blob(BlobOp::Commit { hash }), vec![]); } db.write(batch.build_all()).await.unwrap(); // Create account data println!("Creating account data..."); for account_id in 0u32..10u32 { let mut batch = BatchBuilder::new(); batch.with_account_id(account_id); // Create properties of different sizes for collection in [ Collection::Email, Collection::Mailbox, Collection::Thread, Collection::Identity, ] { batch.with_collection(collection); for document_id in [0, 10, 20, 30, 40] { batch.with_document(document_id); if collection == Collection::Mailbox { batch .set( ValueClass::Property(Field::ARCHIVE.into()), random_bytes(10), ) .add( ValueClass::Property(MailboxField::UidCounter.into()), rand::random(), ); } for (idx, value_size) in [16, 128, 1024, 2056, 102400].into_iter().enumerate() { batch.set(ValueClass::Property(idx as u8), random_bytes(value_size)); } for grant_account_id in 0u32..10u32 { if account_id != grant_account_id { batch.set( ValueClass::Acl(grant_account_id), vec![account_id as u8, grant_account_id as u8, document_id as u8], ); } } for hash in &blob_hashes { batch.set( ValueClass::Blob(BlobOp::Link { hash: hash.clone(), to: BlobLink::Document, }), vec![], ); } batch.log_item_insert(SyncCollection::from(collection), None); for field in 0..5 { batch.any_op(Operation::Index { field, key: random_bytes(field as usize + 2), set: true, }); } } } db.write(batch.build_all()).await.unwrap(); } // Create queue, config and lookup data println!("Creating queue, config and lookup data..."); let mut batch = BatchBuilder::new(); for idx in [1, 2, 3, 4, 5] { batch.set( ValueClass::Queue(QueueClass::Message(rand::random())), random_bytes(idx), ); batch.set( ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { due: rand::random(), queue_id: rand::random(), queue_name: rand::random(), })), random_bytes(idx), ); /*batch.set( ValueClass::InMemory(InMemoryClass::Key(random_bytes(idx))), random_bytes(idx), ); batch.add( ValueClass::InMemory(InMemoryClass::Counter(random_bytes(idx))), rand::random(), );*/ batch.set( ValueClass::Config(random_bytes(idx + 10)), random_bytes(idx + 10), ); } db.write(batch.build_all()).await.unwrap(); // Create directory data println!("Creating directory data..."); let mut batch = BatchBuilder::new(); batch .with_account_id(u32::MAX) .with_collection(Collection::Principal); for account_id in [1, 2, 3, 4, 5] { batch .with_document(account_id) .add( ValueClass::Directory(DirectoryClass::UsedQuota(account_id)), rand::random(), ) .set( ValueClass::Directory(DirectoryClass::NameToId(random_bytes( 2 + account_id as usize, ))), random_bytes(4), ) .set( ValueClass::Directory(DirectoryClass::EmailToId(random_bytes( 4 + account_id as usize, ))), random_bytes(4), ) .set( ValueClass::Directory(DirectoryClass::Principal(account_id)), random_bytes(30), ) .set( ValueClass::Directory(DirectoryClass::MemberOf { principal_id: account_id, member_of: rand::random(), }), random_bytes(15), ) .set( ValueClass::Directory(DirectoryClass::Members { principal_id: account_id, has_member: rand::random(), }), random_bytes(15), ); } db.write(batch.build_all()).await.unwrap(); // Obtain store hash println!("Calculating store hash..."); let snapshot = Snapshot::new(&db).await; assert!(!snapshot.keys.is_empty(), "Store hash counts are empty",); // Export store println!("Exporting store..."); let temp_dir = TempDir::new("art_vandelay_tests", true); core.backup(BackupParams::new(temp_dir.path.clone())).await; // Destroy store println!("Destroying store..."); store_destroy(&db).await; store_assert_is_empty(&db, db.clone().into(), true).await; // Import store println!("Importing store..."); core.restore(temp_dir.path.clone()).await; // Verify hash print!("Verifying store hash..."); snapshot.assert_is_eq(&Snapshot::new(&db).await); println!(" GREAT SUCCESS!"); // Destroy store store_destroy(&db).await; store_assert_is_empty(&db, db.clone().into(), true).await; temp_dir.delete(); } #[derive(Debug, PartialEq, Eq)] struct Snapshot { keys: AHashSet, } #[derive(Debug, PartialEq, Eq, Hash)] struct KeyValue { subspace: u8, key: Vec, value: Vec, } impl Snapshot { async fn new(db: &Store) -> Self { let is_sql = db.is_sql(); let mut keys = AHashSet::new(); for (subspace, with_values) in [ (SUBSPACE_ACL, true), (SUBSPACE_DIRECTORY, true), (SUBSPACE_TASK_QUEUE, true), (SUBSPACE_INDEXES, false), (SUBSPACE_BLOB_EXTRA, true), (SUBSPACE_BLOB_LINK, true), (SUBSPACE_BLOBS, true), (SUBSPACE_LOGS, true), (SUBSPACE_COUNTER, !is_sql), (SUBSPACE_IN_MEMORY_COUNTER, !is_sql), (SUBSPACE_IN_MEMORY_VALUE, true), (SUBSPACE_PROPERTY, true), (SUBSPACE_SETTINGS, true), (SUBSPACE_QUEUE_MESSAGE, true), (SUBSPACE_QUEUE_EVENT, true), (SUBSPACE_QUOTA, !is_sql), (SUBSPACE_REPORT_OUT, true), (SUBSPACE_REPORT_IN, true), ] { let from_key = AnyKey { subspace, key: vec![0u8], }; let to_key = AnyKey { subspace, key: vec![u8::MAX; 10], }; db.iterate( IterateParams::new(from_key, to_key).set_values(with_values), |key, value| { keys.insert(KeyValue { subspace, key: key.to_vec(), value: value.to_vec(), }); Ok(true) }, ) .await .unwrap(); } Snapshot { keys } } fn assert_is_eq(&self, other: &Self) { let mut is_err = false; for key in &self.keys { if !other.keys.contains(key) { println!( "Subspace {}, Key {:?} not found in restored snapshot", char::from(key.subspace), key.key, ); is_err = true; } } for key in &other.keys { if !self.keys.contains(key) { println!( "Subspace {}, Key {:?} not found in original snapshot", char::from(key.subspace), key.key, ); is_err = true; } } if is_err { panic!("Snapshot mismatch"); } } } fn random_bytes(len: usize) -> Vec { (0..len).map(|_| rand::random::()).collect() } ================================================ FILE: tests/src/store/lookup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::time::Duration; use store::{InMemoryStore, Stores, dispatch::lookup::KeyValue}; use utils::config::{Config, Rate}; use crate::{ AssertConfig, store::{ CONFIG, TempDir, cleanup::{store_assert_is_empty, store_destroy}, }, }; #[tokio::test] pub async fn lookup_tests() { let temp_dir = TempDir::new("lookup_tests", true); let mut config = Config::new(CONFIG.replace("{TMP}", temp_dir.path.as_path().to_str().unwrap())) .unwrap() .assert_no_errors(); let stores = Stores::parse_all(&mut config, false).await; let rate = Rate { requests: 1, period: Duration::from_secs(1), }; for (store_id, store) in stores.in_memory_stores { println!("Testing in-memory store {}...", store_id); if let InMemoryStore::Store(store) = &store { store_destroy(store).await; } else { // Reset redis counter store .key_set(KeyValue::new("abc", "0".as_bytes().to_vec())) .await .unwrap(); } // Test key let key = "xyz".as_bytes().to_vec(); store .key_set(KeyValue::new(key.clone(), "world".to_string().into_bytes())) .await .unwrap(); store.purge_in_memory_store().await.unwrap(); assert_eq!( store.key_get::(key.clone()).await.unwrap(), Some("world".to_string()) ); // Test value expiry store .key_set(KeyValue::new(key.clone(), "hello".to_string().into_bytes()).expires(1)) .await .unwrap(); assert_eq!( store.key_get::(key.clone()).await.unwrap(), Some("hello".to_string()) ); tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; assert_eq!(None, store.key_get::(key.clone()).await.unwrap()); store.purge_in_memory_store().await.unwrap(); if let InMemoryStore::Store(store) = &store { store_assert_is_empty(store, store.clone().into(), false).await; } // Test counter let key = "abc".as_bytes().to_vec(); store .counter_incr(KeyValue::new(key.clone(), 1), true) .await .unwrap(); assert_eq!(1, store.counter_get(key.clone()).await.unwrap()); store .counter_incr(KeyValue::new(key.clone(), 2), true) .await .unwrap(); assert_eq!(3, store.counter_get(key.clone()).await.unwrap()); store .counter_incr(KeyValue::new(key.clone(), -3), false) .await .unwrap(); assert_eq!(0, store.counter_get(key.clone()).await.unwrap()); // Test counter expiry let key = "fgh".as_bytes().to_vec(); store .counter_incr(KeyValue::new(key.clone(), 1).expires(1), false) .await .unwrap(); assert_eq!(1, store.counter_get(key.clone()).await.unwrap()); tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; store.purge_in_memory_store().await.unwrap(); assert_eq!(0, store.counter_get(key.clone()).await.unwrap()); // Test rate limiter assert!( store .is_rate_allowed(0, "rate".as_bytes(), &rate, false) .await .unwrap() .is_none() ); assert!( store .is_rate_allowed(0, "rate".as_bytes(), &rate, false) .await .unwrap() .is_some() ); tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; assert!( store .is_rate_allowed(0, "rate".as_bytes(), &rate, false) .await .unwrap() .is_none() ); tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; store.purge_in_memory_store().await.unwrap(); if let InMemoryStore::Store(store) = &store { store_assert_is_empty(store, store.clone().into(), false).await; } // Test locking for iteration in [1, 2] { let mut tasks = Vec::new(); for _ in 0..100 { let store = store.clone(); tasks.push(tokio::spawn(async move { store.try_lock(0, "lock".as_bytes(), 1).await.unwrap() })); } // Only one should return true let mut count = 0; for task in tasks { if task.await.unwrap() { count += 1; } } assert_eq!(1, count, "Iteration {}", iteration); // Wait 2 seconds for the lock to expire tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; } store.purge_in_memory_store().await.unwrap(); if let InMemoryStore::Store(store) = &store { store_assert_is_empty(store, store.clone().into(), false).await; } // Test prefix delete store .key_set(KeyValue::with_prefix( 1, [0], "hello".to_string().into_bytes(), )) .await .unwrap(); for v in 0u32..2020u32 { store .key_set(KeyValue::with_prefix( 0, pack_u32(0, v), "world".to_string().into_bytes(), )) .await .unwrap(); store .counter_incr( KeyValue::with_prefix(0, pack_u32(1, v), 123).expires(3600), false, ) .await .unwrap(); } // Make sure the keys are there assert_eq!( Some("hello"), store .key_get::(KeyValue::<()>::build_key(1, [0])) .await .unwrap() .as_deref() ); for v in [0, 1000, 1001, 2000, 2001] { assert_eq!( Some("world"), store .key_get::(KeyValue::<()>::build_key(0, pack_u32(0, v))) .await .unwrap() .as_deref() ); } for v in [0, 1000, 1001, 2000, 2001] { assert_ne!( 0, store .counter_get(KeyValue::<()>::build_key(0, pack_u32(1, v))) .await .unwrap() ); } // Delete [0, 0, 0, 0, 1] prefix and make sure only the keys with that prefix are gone store .key_delete_prefix(&KeyValue::<()>::build_key(0, 1u32.to_be_bytes())) .await .unwrap(); assert_eq!( Some("hello"), store .key_get::(KeyValue::<()>::build_key(1, [0])) .await .unwrap() .as_deref() ); for v in [0, 1000, 1001, 2000, 2001] { assert_eq!( Some("world"), store .key_get::(KeyValue::<()>::build_key(0, pack_u32(0, v))) .await .unwrap() .as_deref() ); } for v in [0, 1000, 1001, 2000, 2001] { assert_eq!( 0, store .counter_get(KeyValue::<()>::build_key(0, pack_u32(1, v))) .await .unwrap() ); } // Delete [0, 0, 0, 0, 0] prefix and make sure only the keys with that prefix are gone store .key_delete_prefix(&KeyValue::<()>::build_key(0, 0u32.to_be_bytes())) .await .unwrap(); assert_eq!( Some("hello"), store .key_get::(KeyValue::<()>::build_key(1, [0])) .await .unwrap() .as_deref() ); for v in [0, 1000, 1001, 2000, 2001] { assert_eq!( None, store .key_get::(KeyValue::<()>::build_key(0, pack_u32(0, v))) .await .unwrap() .as_deref() ); } // Delete [1, ...] prefix and make sure it's all gone store.key_delete_prefix(&[1u8]).await.unwrap(); assert_eq!( None, store .key_get::(KeyValue::<()>::build_key(1, [0])) .await .unwrap() .as_deref() ); if let InMemoryStore::Store(store) = &store { store_assert_is_empty(store, store.clone().into(), false).await; } } } fn pack_u32(a: u32, b: u32) -> Vec { (((a as u64) << 32) | b as u64).to_be_bytes().to_vec() } ================================================ FILE: tests/src/store/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod blob; pub mod cleanup; pub mod import_export; pub mod lookup; pub mod ops; pub mod query; use crate::{ AssertConfig, store::cleanup::{search_store_destroy, store_destroy}, }; use std::io::Read; use store::Stores; use utils::config::Config; pub struct TempDir { pub path: std::path::PathBuf, } #[tokio::test(flavor = "multi_thread")] pub async fn store_tests() { let insert = true; let temp_dir = TempDir::new("store_tests", insert); let mut config = Config::new(build_store_config(&temp_dir.path.to_string_lossy())) .unwrap() .assert_no_errors(); let stores = Stores::parse_all(&mut config, false).await; let store_id = std::env::var("STORE") .expect("Missing store type. Try running `STORE= cargo test`"); let store = stores .stores .get(&store_id) .expect("Store not found") .clone(); println!("Testing store {}...", store_id); if insert { store_destroy(&store).await; } import_export::test(store.clone()).await; ops::test(store.clone()).await; if insert { temp_dir.delete(); } } #[tokio::test(flavor = "multi_thread")] pub async fn search_tests() { let insert = std::env::var("NO_INSERT").is_err(); let temp_dir = TempDir::new("search_store_tests", insert); let mut config = Config::new(build_store_config(&temp_dir.path.to_string_lossy())) .unwrap() .assert_no_errors(); let stores = Stores::parse_all(&mut config, false).await; let store_id = std::env::var("SEARCH_STORE") .expect("Missing store type. Try running `SEARCH_STORE= cargo test`"); let store = stores .search_stores .get(&store_id) .expect("Store not found") .clone(); println!("Testing store {}...", store_id); if insert { search_store_destroy(&store).await; } query::test(store, insert).await; if insert { temp_dir.delete(); } } pub fn deflate_test_resource(name: &str) -> Vec { let mut csv_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); csv_path.push("resources"); csv_path.push(name); let mut decoder = flate2::bufread::GzDecoder::new(std::io::BufReader::new( std::fs::File::open(csv_path).unwrap(), )); let mut result = Vec::new(); decoder.read_to_end(&mut result).unwrap(); result } impl TempDir { pub fn new(name: &str, delete_if_exists: bool) -> Self { let mut path = std::env::temp_dir(); path.push(name); if delete_if_exists && path.exists() { std::fs::remove_dir_all(&path).unwrap(); } std::fs::create_dir_all(&path).unwrap(); Self { path } } pub fn delete(&self) { std::fs::remove_dir_all(&self.path).unwrap(); } } pub fn build_store_config(temp_dir: &str) -> String { let store = std::env::var("STORE") .expect("Missing store type. Try running `STORE= cargo test`"); let fts_store = std::env::var("SEARCH_STORE").unwrap_or_else(|_| store.clone()); let blob_store = std::env::var("BLOB_STORE").unwrap_or_else(|_| store.clone()); let lookup_store = std::env::var("LOOKUP_STORE").unwrap_or_else(|_| store.clone()); CONFIG .replace("{STORE}", &store) .replace("{SEARCH_STORE}", &fts_store) .replace("{BLOB_STORE}", &blob_store) .replace("{LOOKUP_STORE}", &lookup_store) .replace("{TMP}", temp_dir) .replace( "{ELASTIC_ENABLED}", if fts_store != "elastic" { "true" } else { "false" }, ) .replace( "{MEILI_ENABLED}", if fts_store != "meili" { "true" } else { "false" }, ) } const CONFIG: &str = r#" [store."sqlite"] type = "sqlite" path = "{TMP}/sqlite.db" [store."rocksdb"] type = "rocksdb" path = "{TMP}/rocks.db" [store."foundationdb"] type = "foundationdb" [store."postgresql"] type = "postgresql" host = "localhost" port = 5432 database = "stalwart" user = "postgres" password = "mysecretpassword" [store."mysql"] type = "mysql" host = "localhost" port = 3307 database = "stalwart" user = "root" password = "password" [store."elastic"] type = "elasticsearch" url = "https://localhost:9200" tls.allow-invalid-certs = true disable = {ELASTIC_ENABLED} [store."elastic".auth] username = "elastic" secret = "changeme" [store."meili"] type = "meilisearch" url = "http://localhost:7700" tls.allow-invalid-certs = true disable = {MEILI_ENABLED} [store."meili".task] poll-interval = "100ms" #[store."meili".auth] #username = "meili" #secret = "changeme" #[store."s3"] #type = "s3" #access-key = "minioadmin" #secret-key = "minioadmin" #region = "eu-central-1" #endpoint = "http://localhost:9000" #bucket = "tmp" [store."fs"] type = "fs" path = "{TMP}" [store."redis"] type = "redis" urls = "redis://127.0.0.1" redis-type = "single" #[store."psql-replica"] #type = "sql-read-replica" #primary = "postgresql" #replicas = "postgresql" [storage] data = "{STORE}" fts = "{SEARCH_STORE}" blob = "{BLOB_STORE}" lookup = "{LOOKUP_STORE}" directory = "{STORE}" [directory."{STORE}"] type = "internal" store = "{STORE}" [session.rcpt] directory = "'{STORE}'" [session.auth] directory = "'{STORE}'" "#; ================================================ FILE: tests/src/store/ops.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use ahash::AHashSet; use std::collections::HashSet; use store::{ Store, ValueKey, rand::{self, Rng}, write::{ AlignedBytes, Archive, Archiver, BatchBuilder, DirectoryClass, MergeResult, Params, ValueClass, }, }; use types::collection::{Collection, SyncCollection}; use crate::store::cleanup::store_assert_is_empty; // FDB max value const MAX_VALUE_SIZE: usize = 100000; #[cfg(feature = "foundationdb")] fn value_gen(chunks: impl IntoIterator) -> Vec { let mut value = Vec::new(); for (byte, size) in chunks { value.extend(std::iter::repeat_n(byte, size)); } value } pub async fn test(db: Store) { #[cfg(feature = "foundationdb")] if matches!(db, Store::FoundationDb(_)) { use types::collection::Collection; println!("Running FoundationDB chunked iterator test..."); let kvs = [ ("a", value_gen([(b'a', 1)])), ("b", value_gen([(b'b', MAX_VALUE_SIZE), (b'0', 1)])), ( "c", value_gen([ (b'c', MAX_VALUE_SIZE), (b'1', MAX_VALUE_SIZE), (b'2', MAX_VALUE_SIZE), ]), ), ( "d", value_gen([(b'd', MAX_VALUE_SIZE), (b'3', MAX_VALUE_SIZE)]), ), ("e", value_gen([(b'e', 1)])), ]; let mut batch = BatchBuilder::new(); batch .with_account_id(0) .with_collection(Collection::Email) .with_document(0); for (key, value) in &kvs { batch.set(ValueClass::Config(key.as_bytes().to_vec()), value.clone()); } db.write(batch.build_all()).await.unwrap(); // Iterate over all keys let mut results = Vec::new(); db.iterate( store::IterateParams::new( ValueKey { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Config(b"".to_vec()), }, ValueKey { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Config(b"\xFF".to_vec()), }, ), |key, value| { results.push((String::from_utf8(key.to_vec()).unwrap(), value.to_vec())); Ok(true) }, ) .await .unwrap(); assert_eq!(results.len(), kvs.len()); db.delete_range( ValueKey { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Config(b"".to_vec()), }, ValueKey { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Config(b"\xFF".to_vec()), }, ) .await .unwrap(); if std::env::var("SLOW_FDB_TRX").is_ok() { println!("Running FoundationDB slow transaction tests..."); // Create 900000 keys let mut batch = BatchBuilder::new(); batch .with_account_id(0) .with_collection(Collection::Email) .with_document(0); for n in 0..900000 { batch.set( ValueClass::Config(format!("key{n:10}").into_bytes()), format!("value{n:10}").into_bytes(), ); if n % 10000 == 0 { db.write(batch.build_all()).await.unwrap(); batch = BatchBuilder::new(); batch .with_account_id(0) .with_collection(Collection::Email) .with_document(0); } } db.write(batch.build_all()).await.unwrap(); println!("Created 900.000 keys..."); // Iterate over all keys let mut n = 0; db.iterate( store::IterateParams::new( ValueKey { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Config(b"".to_vec()), }, ValueKey { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Config(b"\xFF".to_vec()), }, ), |key, value| { assert_eq!(std::str::from_utf8(key).unwrap(), format!("key{n:10}")); assert_eq!(std::str::from_utf8(value).unwrap(), format!("value{n:10}")); n += 1; if n % 10000 == 0 { println!("Iterated over {n} keys"); std::thread::sleep(std::time::Duration::from_millis(1000)); } Ok(true) }, ) .await .unwrap(); // Delete 100 keys let mut batch = BatchBuilder::new(); batch .with_account_id(0) .with_collection(Collection::Email) .with_document(0); for n in 0..900000 { batch.clear(ValueClass::Config(format!("key{n:10}").into_bytes())); if n % 10000 == 0 { db.write(batch.build_all()).await.unwrap(); batch = BatchBuilder::new(); batch .with_account_id(0) .with_collection(Collection::Email) .with_document(0); } } db.write(batch.build_all()).await.unwrap(); } } // Merge values 1000 times concurrently let mut handles = Vec::new(); println!("Merge values 1000 times concurrently..."); for _ in 0..1000 { handles.push({ let db = db.clone(); tokio::spawn(async move { for _ in 0..5 { let mut builder = BatchBuilder::new(); builder .with_account_id(0) .with_collection(Collection::Email) .with_document(0) .merge_fnc( ValueClass::Property(3), Params::with_capacity(0), |_, _, bytes| { if let Some(bytes) = bytes { Ok(MergeResult::Update( (u64::from_be_bytes(bytes.try_into().unwrap()) + 1) .to_be_bytes() .to_vec(), )) } else { Ok(MergeResult::Update(0u64.to_be_bytes().to_vec())) } }, ); match db.write(builder.build_all()).await { Ok(_) => { break; } Err(e) if e.is_assertion_failure() => { // Retry on assertion failures continue; } Err(e) => { panic!("Merge failed: {:?}", e); } } } }) }); } for handle in handles { handle.await.unwrap(); } assert_eq!( 999, db.get_value::(ValueKey { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Property(3), }) .await .unwrap() .unwrap() ); // Increment a counter 1000 times concurrently let mut handles = Vec::new(); let mut assigned_ids = HashSet::new(); println!("Incrementing counter 1000 times concurrently..."); for _ in 0..1000 { handles.push({ let db = db.clone(); tokio::spawn(async move { let mut builder = BatchBuilder::new(); builder .with_account_id(0) .with_collection(Collection::Email) .with_document(0) .add_and_get(ValueClass::Directory(DirectoryClass::UsedQuota(0)), 1); db.write(builder.build_all()) .await .unwrap() .last_counter_id() .unwrap() }) }); } for handle in handles { let assigned_id = handle.await.unwrap(); assert!( assigned_ids.insert(assigned_id), "counter assigned {assigned_id} twice or more times." ); } assert_eq!(assigned_ids.len(), 1000); assert_eq!( db.get_counter(ValueKey { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Directory(DirectoryClass::UsedQuota(0)), }) .await .unwrap(), 1000 ); // Concurrent changelog let mut handles = Vec::new(); let mut assigned_ids = AHashSet::new(); print!("Incrementing changeId 1000 times concurrently..."); let time = std::time::Instant::now(); for document_id in 0..1000 { handles.push({ let db = db.clone(); tokio::spawn(async move { let mut builder = BatchBuilder::new(); let value = if document_id != 0 { (0..rand::rng().random_range(1..=100)) .map(|_| rand::rng().random_range(0..=255)) .collect::>() } else { vec![0u8; 100000] }; let (offset, archived_value) = Archiver::new(value).serialize_versioned().unwrap(); builder .with_account_id(0) .with_collection(Collection::Email) .with_document(document_id) .set_fnc( ValueClass::Property(5), Params::with_capacity(2) .with_bytes(archived_value) .with_u64(offset), |params, ids| { let change_id = ids.current_change_id()?; let archive = params.bytes(0); let offset = params.u64(1); let mut bytes = Vec::with_capacity(archive.len()); bytes.extend_from_slice(&archive[..offset as usize]); bytes.extend_from_slice(&change_id.to_be_bytes()[..]); bytes.push(archive.last().copied().unwrap()); // Marker Ok(bytes) }, ) .log_container_insert(SyncCollection::Email); db.write(builder.build_all()) .await .unwrap() .last_change_id(0) .unwrap() }) }); } for handle in handles { let assigned_id = handle.await.unwrap(); assert!( assigned_ids.insert(assigned_id), "counter assigned {assigned_id} twice or more times: {:?}.", assigned_ids ); } assert_eq!(assigned_ids.len(), 1000); println!(" done in {:?}ms", time.elapsed().as_millis()); let mut change_ids = AHashSet::new(); for document_id in 0..1000 { let archive = db .get_value::>(ValueKey { account_id: 0, collection: 0, document_id, class: ValueClass::Property(5), }) .await .unwrap() .unwrap(); change_ids.insert(archive.version.change_id().unwrap()); archive.unarchive_untrusted::>().unwrap(); } assert_eq!(change_ids, assigned_ids); println!("Running chunking tests..."); for (test_num, value) in [ vec![b'A'; 0], vec![b'A'; 1], vec![b'A'; 100], vec![b'A'; MAX_VALUE_SIZE], vec![b'B'; MAX_VALUE_SIZE + 1], vec![b'C'; MAX_VALUE_SIZE] .into_iter() .chain(vec![b'D'; MAX_VALUE_SIZE]) .chain(vec![b'E'; MAX_VALUE_SIZE]) .collect::>(), vec![b'F'; MAX_VALUE_SIZE] .into_iter() .chain(vec![b'G'; MAX_VALUE_SIZE]) .chain(vec![b'H'; MAX_VALUE_SIZE + 1]) .collect::>(), ] .into_iter() .enumerate() { // Write value let test_len = value.len(); db.write( BatchBuilder::new() .with_account_id(0) .with_collection(Collection::Email) .with_document(0) .set(ValueClass::Property(1), value.as_slice()) .set(ValueClass::Property(0), "check1".as_bytes()) .set(ValueClass::Property(2), "check2".as_bytes()) .build_all(), ) .await .unwrap(); // Fetch value assert_eq!( String::from_utf8(value).unwrap(), db.get_value::(ValueKey { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Property(1), }) .await .unwrap() .unwrap_or_else(|| panic!("no value for test {test_num} with value length {test_len}")), "failed for test {test_num} with value length {test_len}" ); // Delete value db.write( BatchBuilder::new() .with_account_id(0) .with_collection(Collection::Email) .with_document(0) .clear(ValueClass::Property(1)) .build_all(), ) .await .unwrap(); // Make sure value is deleted assert_eq!( None, db.get_value::(ValueKey { account_id: 0, collection: 0, document_id: 0, class: ValueClass::Property(1), }) .await .unwrap() ); // Make sure other values are still there for (class, value) in [ (ValueClass::Property(0), "check1"), (ValueClass::Property(2), "check2"), ] { assert_eq!( Some(value.to_string()), db.get_value::(ValueKey { account_id: 0, collection: 0, document_id: 0, class, }) .await .unwrap() ); } // Delete everything let mut batch = BatchBuilder::new(); batch .with_account_id(0) .with_collection(Collection::Email) .with_account_id(0) .with_document(0) .clear(ValueClass::Property(0)) .clear(ValueClass::Property(2)) .clear(ValueClass::Property(3)) .clear(ValueClass::Directory(DirectoryClass::UsedQuota(0))) .clear(ValueClass::ChangeId); for document_id in 0..1000 { batch .with_document(document_id) .clear(ValueClass::Property(5)); } db.write(batch.build_all()).await.unwrap(); // Make sure everything is deleted store_assert_is_empty(&db, db.clone().into(), false).await; } } ================================================ FILE: tests/src/store/query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::store::deflate_test_resource; use ahash::AHashSet; use nlp::language::Language; use std::{ io::Write, sync::{Arc, Mutex}, time::Instant, }; use store::{ SearchStore, ahash::AHashMap, rand::{self, Rng, distr::Alphanumeric}, roaring::RoaringBitmap, search::{ EmailSearchField, IndexDocument, SearchComparator, SearchField, SearchFilter, SearchOperator, SearchQuery, SearchValue, TracingSearchField, }, write::SearchIndex, }; use utils::map::vec_map::VecMap; pub const FIELDS: [&str; 20] = [ "id", "accession_number", "artist", "artistRole", "artistId", "title", "dateText", "medium", "creditLine", "year", "acquisitionYear", "dimensions", "width", "height", "depth", "units", "inscription", "thumbnailCopyright", "thumbnailUrl", "url", ]; /* "title", // Subject "year". // ReceivedAt "width", // Size "height", // SentAt "artist" // Headers "artistRole" // Cc "medium", // From "creditLine" // Body "acquisitionYear" // Bcc "accession_number" // To */ const FIELD_MAPPINGS: [EmailSearchField; 20] = [ EmailSearchField::HasAttachment, // "id", EmailSearchField::To, // "accession_number", EmailSearchField::Headers, // "artist", EmailSearchField::Cc, // "artistRole", EmailSearchField::HasAttachment, // "artistId", EmailSearchField::Subject, // "title", EmailSearchField::HasAttachment, // "dateText", EmailSearchField::From, // "medium", EmailSearchField::Body, // "creditLine", EmailSearchField::ReceivedAt, // "year", EmailSearchField::Bcc, // "acquisitionYear", EmailSearchField::HasAttachment, // "dimensions", EmailSearchField::Size, // "width", EmailSearchField::SentAt, // "height", EmailSearchField::HasAttachment, // "depth", EmailSearchField::HasAttachment, // "units", EmailSearchField::HasAttachment, // "inscription", EmailSearchField::HasAttachment, // "thumbnailCopyright", EmailSearchField::HasAttachment, // "thumbnailUrl", EmailSearchField::HasAttachment, // "url", ]; const ALL_IDS: &[&str] = &[ "p11293", "p79426", "p79427", "p79428", "p79429", "p79430", "d05503", "d00399", "d05352", "p01764", "t05843", "n02478", "n02479", "n03568", "n03658", "n04327", "n04328", "n04721", "n04739", "n05095", "n05096", "n05145", "n05157", "n05158", "n05159", "n05298", "n05303", "n06070", "t01181", "t03571", "t05805", "t05806", "t12147", "t12154", "t12155", "ar00039", "t12600", "p80203", "t13209", "t13560", "t13561", "t13655", "t13811", "p13352", "p13351", "p13350", "p13349", "p13348", "p13347", "p13346", "p13345", "p13344", "p13342", "p13341", "p13340", "p13339", "p13338", "p13337", "p13336", "p13335", "p13334", "p13333", "p13332", "p13331", "p13330", "p13329", "p13328", "p13327", "p13326", "p13325", "p13324", "p13323", "t13786", "p13322", "p13321", "p13320", "p13319", "p13318", "p13317", "p13316", "p13315", "p13314", "t13588", "t13587", "t13586", "t13585", "t13584", "t13540", "t13444", "ar01154", "ar01153", "t03681", "t12601", "ar00166", "t12625", "t12915", "p04182", "t06483", "ar00703", "t07671", "ar00021", "t05557", "t07918", "p06298", "p05465", "p06640", "t12855", "t01355", "t12800", "t12557", "t02078", "ar00052", "ar00627", "t00352", "t07275", "t12318", "t04931", "t13683", "t13686", "t13687", "t13688", "t13689", "t13690", "t13691", "t13769", "t13773", "t07151", "t13684", "t07523", "t12369", "t12567", "ar00627", "ar00052", "t00352", "t07275", "t12318", "t04931", "t13683", "t13686", "t13687", "t13688", "t13689", "t13690", "t13691", "t07766", "t07918", "t12993", "ar00044", "t13326", "t07614", "t12414", ]; #[allow(clippy::mutex_atomic)] pub async fn test(store: SearchStore, do_insert: bool) { println!("Running Store query tests..."); let pool = rayon::ThreadPoolBuilder::new() .num_threads(8) .build() .unwrap(); let now = Instant::now(); let documents = Arc::new(Mutex::new(Vec::new())); let mut mask = RoaringBitmap::new(); let mut fields = AHashMap::new(); // Global ids test println!("Running global id filtering tests..."); test_global(store.clone()).await; // Large document insert test println!("Running large document insert tests..."); let mut large_text = String::with_capacity(20 * 1024 * 1024); while large_text.len() < 20 * 1024 * 1024 { let word = rand::rng() .sample_iter(&Alphanumeric) .take(rand::rng().random_range(3..10)) .map(char::from) .collect::(); large_text.push_str(&word); large_text.push(' '); } let mut document = IndexDocument::new(SearchIndex::Email) .with_account_id(1) .with_document_id(1); for field in [ EmailSearchField::From, EmailSearchField::To, EmailSearchField::Cc, EmailSearchField::Bcc, EmailSearchField::Subject, ] { document.index_text(field, &large_text[..10 * 1024], Language::English); } for field in [EmailSearchField::Body, EmailSearchField::Attachment] { document.index_text(field, &large_text, Language::English); } for field in [ EmailSearchField::ReceivedAt, EmailSearchField::SentAt, EmailSearchField::Size, ] { document.index_unsigned(field, rand::rng().random_range(100u64..1_000_000u64)); } store.index(vec![document]).await.unwrap(); // Refresh if let SearchStore::ElasticSearch(store) = &store { store.refresh_index(SearchIndex::Email).await.unwrap(); } println!("Running account filtering tests..."); let filter_ids = std::env::var("QUICK_TEST").is_ok().then(|| { let mut ids = AHashSet::new(); for &id in ALL_IDS { ids.insert(id.to_string()); let id = id.as_bytes(); if id.last().unwrap() > &b'0' { let mut alt_id = id.to_vec(); *alt_id.last_mut().unwrap() -= 1; ids.insert(String::from_utf8(alt_id).unwrap()); } if id.last().unwrap() < &b'9' { let mut alt_id = id.to_vec(); *alt_id.last_mut().unwrap() += 1; ids.insert(String::from_utf8(alt_id).unwrap()); } } ids }); pool.scope_fifo(|s| { for (document_id, record) in csv::ReaderBuilder::new() .has_headers(true) .from_reader(&deflate_test_resource("artwork_data.csv.gz")[..]) .records() .enumerate() { let record = record.unwrap(); let documents = documents.clone(); if let Some(filter_ids) = &filter_ids { let id = record.get(1).unwrap().to_lowercase(); if !filter_ids.contains(&id) { continue; } } s.spawn_fifo(move |_| { let mut document = IndexDocument::new(SearchIndex::Email) .with_account_id(0) .with_document_id(document_id as u32); for (pos, field) in record.iter().enumerate() { match FIELD_MAPPINGS[pos] { EmailSearchField::From | EmailSearchField::To | EmailSearchField::Cc | EmailSearchField::Bcc => { document.index_text( FIELD_MAPPINGS[pos].clone(), &field.to_lowercase(), Language::None, ); } EmailSearchField::Subject | EmailSearchField::Body | EmailSearchField::Attachment => { document.index_text( FIELD_MAPPINGS[pos].clone(), &field .replace(|ch: char| !ch.is_alphanumeric(), " ") .to_lowercase(), Language::English, ); } EmailSearchField::Headers => { document.insert_key_value( EmailSearchField::Headers, "artist", field.to_lowercase(), ); } EmailSearchField::ReceivedAt | EmailSearchField::SentAt | EmailSearchField::Size => { document.index_unsigned( FIELD_MAPPINGS[pos].clone(), field.parse::().unwrap_or(0), ); } _ => { continue; } }; } documents.lock().unwrap().push(document); }); } }); println!( "Parsed {} entries in {} ms.", documents.lock().unwrap().len(), now.elapsed().as_millis() ); let now = Instant::now(); let batches = documents.lock().unwrap().drain(..).collect::>(); print!("Inserting... ",); let mut chunks = Vec::new(); let mut chunk = Vec::new(); for document in batches { let mut document_id = None; let mut to_field = None; for (key, value) in document.fields() { if key == &SearchField::DocumentId { if let SearchValue::Uint(id) = value { document_id = Some(*id as u32); } } else if key == &SearchField::Email(EmailSearchField::To) && let SearchValue::Text { value, .. } = value { to_field = Some(value.to_string()); } } let document_id = document_id.unwrap(); let to_field = to_field.unwrap(); mask.insert(document_id); fields.insert(document_id, to_field); chunk.push(document); if chunk.len() == 10 { chunks.push(chunk); chunk = Vec::new(); } } if !chunk.is_empty() { chunks.push(chunk); } if do_insert { let mut tasks = Vec::new(); for chunk in chunks { let chunk_instance = Instant::now(); tasks.push({ let db = store.clone(); tokio::spawn(async move { db.index(chunk).await }) }); if tasks.len() == 100 { for handle in tasks { handle.await.unwrap().unwrap(); } print!(" [{} ms]", chunk_instance.elapsed().as_millis()); std::io::stdout().flush().unwrap(); tasks = Vec::new(); } } if !tasks.is_empty() { for handle in tasks { handle.await.unwrap().unwrap(); } } // Refresh if let SearchStore::ElasticSearch(store) = &store { store.refresh_index(SearchIndex::Email).await.unwrap(); } println!("\nInsert took {} ms.", now.elapsed().as_millis()); } if store.internal_fts().is_none() { let ids = store .query_account( SearchQuery::new(SearchIndex::Email) .with_filters(vec![SearchFilter::eq(SearchField::AccountId, 0u32)]) .with_comparator(SearchComparator::ascending(EmailSearchField::ReceivedAt)) .with_mask(mask.clone()), ) .await .unwrap() .into_iter() .collect::(); assert_eq!(ids, mask); let ids = store .query_account( SearchQuery::new(SearchIndex::Email) .with_filters(vec![ SearchFilter::eq(SearchField::AccountId, 0u32), SearchFilter::ge(SearchField::DocumentId, 0u32), ]) .with_mask(mask.clone()), ) .await .unwrap() .into_iter() .collect::(); assert_eq!(ids, mask); } println!("Running account filter tests..."); let now = Instant::now(); test_filter(store.clone(), &fields, &mask).await; println!("Filtering took {} ms.", now.elapsed().as_millis()); println!("Running account sort tests..."); let now = Instant::now(); test_sort(store.clone(), &fields, &mask).await; println!("Sorting took {} ms.", now.elapsed().as_millis()); println!("Running unindex tests..."); let now = Instant::now(); test_unindex(store.clone(), &fields).await; println!("Unindexing took {} ms.", now.elapsed().as_millis()); } async fn test_filter(store: SearchStore, fields: &AHashMap, mask: &RoaringBitmap) { let can_stem = !store.is_mysql(); let tests = [ ( vec![ SearchFilter::eq(SearchField::AccountId, 0u32), SearchFilter::has_english_text(EmailSearchField::Subject, "water"), SearchFilter::eq(EmailSearchField::ReceivedAt, 1979u32), ], vec!["p11293"], ), ( vec![ SearchFilter::eq(SearchField::AccountId, 0u32), SearchFilter::has_keyword(EmailSearchField::From, "gelatin"), SearchFilter::gt(EmailSearchField::ReceivedAt, 2000u32), SearchFilter::lt(EmailSearchField::Size, 180u32), SearchFilter::gt(EmailSearchField::Size, 0u32), ], vec!["p79426", "p79427", "p79428", "p79429", "p79430"], ), ( vec![ SearchFilter::eq(SearchField::AccountId, 0u32), SearchFilter::has_english_text(EmailSearchField::Subject, "'rustic bridge'"), ], vec!["d05503"], ), ( vec![ SearchFilter::eq(SearchField::AccountId, 0u32), SearchFilter::has_english_text(EmailSearchField::Subject, "'rustic'"), SearchFilter::has_english_text( EmailSearchField::Subject, if can_stem { "study" } else { "studies" }, ), ], vec!["d00399", "d05352"], ), ( vec![ SearchFilter::eq(SearchField::AccountId, 0u32), SearchFilter::cond( EmailSearchField::Headers, SearchOperator::Contains, SearchValue::KeyValues(VecMap::from_iter([( "artist".to_string(), "kunst, mauro".to_string(), )])), ), SearchFilter::has_keyword(EmailSearchField::Cc, "artist"), SearchFilter::Or, SearchFilter::eq(EmailSearchField::ReceivedAt, 1969u32), SearchFilter::eq(EmailSearchField::ReceivedAt, 1971u32), SearchFilter::End, ], vec!["p01764", "t05843"], ), ( vec![ SearchFilter::eq(SearchField::AccountId, 0u32), SearchFilter::Not, SearchFilter::has_keyword(EmailSearchField::From, "oil"), SearchFilter::End, SearchFilter::has_english_text( EmailSearchField::Body, if can_stem { "bequeath" } else { "bequeathed" }, ), SearchFilter::Or, SearchFilter::And, SearchFilter::ge(EmailSearchField::ReceivedAt, 1900u32), SearchFilter::lt(EmailSearchField::ReceivedAt, 1910u32), SearchFilter::End, SearchFilter::And, SearchFilter::ge(EmailSearchField::ReceivedAt, 2000u32), SearchFilter::lt(EmailSearchField::ReceivedAt, 2010u32), SearchFilter::End, SearchFilter::End, ], vec![ "n02478", "n02479", "n03568", "n03658", "n04327", "n04328", "n04721", "n04739", "n05095", "n05096", "n05145", "n05157", "n05158", "n05159", "n05298", "n05303", "n06070", "t01181", "t03571", "t05805", "t05806", "t12147", "t12154", "t12155", ], ), ( vec![ SearchFilter::And, SearchFilter::eq(SearchField::AccountId, 0u32), SearchFilter::cond( EmailSearchField::Headers, SearchOperator::Contains, SearchValue::KeyValues(VecMap::from_iter([( "artist".to_string(), "warhol".to_string(), )])), ), SearchFilter::Not, SearchFilter::has_english_text(EmailSearchField::Subject, "'campbell'"), SearchFilter::End, SearchFilter::Not, SearchFilter::Or, SearchFilter::gt(EmailSearchField::ReceivedAt, 1980u32), SearchFilter::And, SearchFilter::gt(EmailSearchField::Size, 500u32), SearchFilter::gt(EmailSearchField::SentAt, 500u32), SearchFilter::End, SearchFilter::End, SearchFilter::End, SearchFilter::eq(EmailSearchField::Bcc, "2008".to_string()), SearchFilter::End, ], vec!["ar00039", "t12600"], ), ( if can_stem { vec![ SearchFilter::eq(SearchField::AccountId, 0u32), SearchFilter::has_english_text(EmailSearchField::Subject, "study"), SearchFilter::has_keyword(EmailSearchField::From, "paper"), SearchFilter::has_english_text(EmailSearchField::Body, "'purchased'"), SearchFilter::Not, SearchFilter::Or, SearchFilter::has_english_text(EmailSearchField::Subject, "'anatomical'"), SearchFilter::has_english_text(EmailSearchField::Subject, "'discarded'"), SearchFilter::has_english_text(EmailSearchField::Subject, "'untitled'"), SearchFilter::has_english_text(EmailSearchField::Subject, "'girl'"), SearchFilter::End, SearchFilter::End, SearchFilter::gt(EmailSearchField::ReceivedAt, 1900u32), SearchFilter::gt(EmailSearchField::Bcc, "2008".to_string()), ] } else { vec![ SearchFilter::eq(SearchField::AccountId, 0u32), SearchFilter::Or, SearchFilter::has_english_text(EmailSearchField::Subject, "study"), SearchFilter::has_english_text(EmailSearchField::Subject, "studies"), SearchFilter::End, SearchFilter::has_keyword(EmailSearchField::From, "paper"), SearchFilter::has_english_text(EmailSearchField::Body, "'purchased'"), SearchFilter::Not, SearchFilter::Or, SearchFilter::has_english_text(EmailSearchField::Subject, "'anatomical'"), SearchFilter::has_english_text(EmailSearchField::Subject, "'discarded'"), SearchFilter::has_english_text(EmailSearchField::Subject, "'untitled'"), SearchFilter::has_english_text(EmailSearchField::Subject, "'girl'"), SearchFilter::End, SearchFilter::End, SearchFilter::gt(EmailSearchField::ReceivedAt, 1900u32), SearchFilter::gt(EmailSearchField::Bcc, "2008".to_string()), ] }, vec!["p80203", "t13209", "t13560", "t13561"], ), ]; for (filters, expected_results) in tests { //println!("Running test: {:?}", filter); let ids = store .query_account( SearchQuery::new(SearchIndex::Email) .with_filters(filters) .with_comparator(SearchComparator::ascending(EmailSearchField::To)) .with_mask(mask.clone()), ) .await .unwrap(); let mut results = Vec::new(); for document_id in ids { results.push(fields.get(&document_id).unwrap()); } assert_eq!(results, expected_results); } } async fn test_sort(store: SearchStore, fields: &AHashMap, mask: &RoaringBitmap) { let is_reversed = store.is_postgres(); let tests = [ ( vec![ SearchFilter::eq(SearchField::AccountId, 0u32), SearchFilter::gt(EmailSearchField::ReceivedAt, 0u32), SearchFilter::gt(EmailSearchField::Bcc, "0000".to_string()), SearchFilter::gt(EmailSearchField::Size, 0u32), ], vec![ SearchComparator::descending(EmailSearchField::ReceivedAt), SearchComparator::ascending(EmailSearchField::Bcc), SearchComparator::ascending(EmailSearchField::Size), SearchComparator::descending(EmailSearchField::To), ], vec![ "t13655", "t13811", "p13352", "p13351", "p13350", "p13349", "p13348", "p13347", "p13346", "p13345", "p13344", "p13342", "p13341", "p13340", "p13339", "p13338", "p13337", "p13336", "p13335", "p13334", "p13333", "p13332", "p13331", "p13330", "p13329", "p13328", "p13327", "p13326", "p13325", "p13324", "p13323", "t13786", "p13322", "p13321", "p13320", "p13319", "p13318", "p13317", "p13316", "p13315", "p13314", "t13588", "t13587", "t13586", "t13585", "t13584", "t13540", "t13444", "ar01154", "ar01153", ], ), ( vec![ SearchFilter::eq(SearchField::AccountId, 0u32), SearchFilter::gt(EmailSearchField::Size, 0u32), SearchFilter::gt(EmailSearchField::SentAt, 0u32), ], vec![ SearchComparator::descending(EmailSearchField::Size), SearchComparator::ascending(EmailSearchField::SentAt), ], vec![ "t03681", "t12601", "ar00166", "t12625", "t12915", "p04182", "t06483", "ar00703", "t07671", "ar00021", "t05557", "t07918", "p06298", "p05465", "p06640", "t12855", "t01355", "t12800", "t12557", "t02078", ], ), ( vec![SearchFilter::eq(SearchField::AccountId, 0u32)], vec![ SearchComparator::descending(EmailSearchField::From), SearchComparator::descending(EmailSearchField::Cc), SearchComparator::ascending(EmailSearchField::To), ], if is_reversed { vec![ "ar00052", "ar00627", "t00352", "t07275", "t12318", "t04931", "t13683", "t13686", "t13687", "t13688", "t13689", "t13690", "t13691", "t13769", "t13773", "t07151", "t13684", "t07523", "t12369", "t12567", ] } else { vec![ "ar00627", "ar00052", "t00352", "t07275", "t12318", "t04931", "t13683", "t13686", "t13687", "t13688", "t13689", "t13690", "t13691", "t07766", "t07918", "t12993", "ar00044", "t13326", "t07614", "t12414", ] }, ), ]; for (filters, comparators, expected_results) in tests { //println!("Running test: {:?}", sort); let ids = store .query_account( SearchQuery::new(SearchIndex::Email) .with_filters(filters) .with_comparators(comparators) .with_mask(mask.clone()), ) .await .unwrap(); let mut results = Vec::new(); for document_id in ids.into_iter().take(expected_results.len()) { results.push(fields.get(&document_id).unwrap()); } assert_eq!(results, expected_results); } } async fn test_unindex(store: SearchStore, fields: &AHashMap) { let ids = store .query_account( SearchQuery::new(SearchIndex::Email) .with_mask(RoaringBitmap::from_iter(fields.keys().copied())) .with_filters(vec![ SearchFilter::has_keyword(EmailSearchField::From, "gelatin"), SearchFilter::gt(EmailSearchField::ReceivedAt, 2000u32), SearchFilter::lt(EmailSearchField::Size, 180u32), SearchFilter::gt(EmailSearchField::Size, 0u32), ]) .with_account_id(0), ) .await .unwrap(); assert!(!ids.is_empty()); let expected_count = ids.len().saturating_sub(10); let mut query = SearchQuery::new(SearchIndex::Email) .with_account_id(0) .with_filter(SearchFilter::Or); for id in ids.into_iter().take(10) { query = query.with_filter(SearchFilter::eq(SearchField::DocumentId, id)); } query = query.with_filter(SearchFilter::End); store.unindex(query).await.unwrap(); // Refresh if let SearchStore::ElasticSearch(store) = &store { store.refresh_index(SearchIndex::Email).await.unwrap(); } assert_eq!( store .query_account( SearchQuery::new(SearchIndex::Email) .with_filters(vec![ SearchFilter::has_keyword(EmailSearchField::From, "gelatin"), SearchFilter::gt(EmailSearchField::ReceivedAt, 2000u32), SearchFilter::lt(EmailSearchField::Size, 180u32), SearchFilter::gt(EmailSearchField::Size, 0u32), ]) .with_account_id(0) .with_mask(RoaringBitmap::from_iter(fields.keys().copied())), ) .await .unwrap() .len(), expected_count ); } async fn test_global(store: SearchStore) { // Insert global ids for (id, queue_id, etyp, keywords) in [ (0, 1000u64, 1u64, "init start"), (1, 1000u64, 2u64, "init complete"), (2, 1001u64, 1u64, "process start"), (3, 1001u64, 2u64, "process complete"), (4, 1002u64, 1u64, "cleanup start"), (5, 1002u64, 2u64, "cleanup complete"), ] { let mut document = IndexDocument::new(SearchIndex::Tracing).with_id(id); document.index_unsigned(TracingSearchField::QueueId, queue_id); document.index_unsigned(TracingSearchField::EventType, etyp); document.index_text(TracingSearchField::Keywords, keywords, Language::None); store.index(vec![document]).await.unwrap(); } // Refresh if let SearchStore::ElasticSearch(store) = &store { store.refresh_index(SearchIndex::Tracing).await.unwrap(); } // Query all assert_eq!( store .query_global( SearchQuery::new(SearchIndex::Tracing) .with_filter(SearchFilter::ge(SearchField::Id, 0u64)) ) .await .unwrap() .into_iter() .collect::>(), AHashSet::from_iter([0, 1, 2, 3, 4, 5]) ); // Query with filter assert_eq!( store .query_global( SearchQuery::new(SearchIndex::Tracing) .with_filter(SearchFilter::gt(SearchField::Id, 1u64)) .with_filter(SearchFilter::lt(SearchField::Id, 5u64)) .with_filter(SearchFilter::has_keyword( TracingSearchField::Keywords, "start", )), ) .await .unwrap() .into_iter() .collect::>(), AHashSet::from_iter([2, 4]) ); // Delete by filter store .unindex( SearchQuery::new(SearchIndex::Tracing) .with_filter(SearchFilter::lt(SearchField::Id, 3u64)), ) .await .unwrap(); // Refresh if let SearchStore::ElasticSearch(store) = &store { store.refresh_index(SearchIndex::Tracing).await.unwrap(); } assert_eq!( store .query_global( SearchQuery::new(SearchIndex::Tracing) .with_filter(SearchFilter::ge(SearchField::Id, 0u64)) ) .await .unwrap() .into_iter() .collect::>(), AHashSet::from_iter([3, 4, 5]) ); } ================================================ FILE: tests/src/webdav/acl.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use dav_proto::schema::property::{DavProperty, WebDavProperty}; use groupware::DavResourceName; use hyper::StatusCode; use crate::webdav::GenerateTestDavResource; use super::{DavResponse, DummyWebDavClient, WebDavTest}; pub async fn test(test: &WebDavTest) { let owner_client = test.client("bill"); let sharee_client = test.client("john"); for resource_type in [ DavResourceName::File, DavResourceName::Cal, DavResourceName::Card, ] { println!("Running ACL tests ({})...", resource_type.base_path()); let is_file = resource_type == DavResourceName::File; let sharee_principal = format!("{}/john/", DavResourceName::Principal.base_path()); let sharee_base_path = format!("{}/john/", resource_type.base_path()); let owner_principal = format!("{}/bill/", DavResourceName::Principal.base_path()); let owner_base_path = format!("{}/bill/", resource_type.base_path()); // Create a resource for the owner let owner_folder = format!("{owner_base_path}test-shared/"); let owner_folder_private = format!("{owner_base_path}test-private/"); let owner_file = format!("{owner_folder}test-file"); let owner_file_content = resource_type.generate(); let owner_file_private = format!("{owner_folder_private}test-file-private"); let owner_file_content_private = resource_type.generate(); let sharee_created_file = format!("{owner_folder}test-file-sharee"); for (folder, file, content) in [ (&owner_folder, &owner_file, &owner_file_content), ( &owner_folder_private, &owner_file_private, &owner_file_content_private, ), ] { owner_client .request("MKCOL", folder, "") .await .with_status(StatusCode::CREATED); owner_client .request("PUT", file, content) .await .with_status(StatusCode::CREATED); } // Create a resource for the sharee let sharee_folder = format!("{sharee_base_path}test-folder/"); let sharee_file = format!("{sharee_folder}test-file"); let sharee_file_content = resource_type.generate(); sharee_client .request("MKCOL", &sharee_folder, "") .await .with_status(StatusCode::CREATED); sharee_client .request("PUT", &sharee_file, &sharee_file_content) .await .with_status(StatusCode::CREATED); // Test 1: Sharee should only see their own resources sharee_client .propfind_with_headers( resource_type.collection_path(), [DavProperty::WebDav(WebDavProperty::GetETag)], [("prefer", "depth-noroot")], ) .await .with_hrefs([sharee_base_path.as_str()]); // Test 2: Share a resource and make sure the root folder is visible owner_client .acl(&owner_folder, sharee_principal.as_str(), ["read"]) .await .with_status(StatusCode::OK); if is_file { owner_client .acl(&owner_file, sharee_principal.as_str(), ["read"]) .await .with_status(StatusCode::OK); } sharee_client .propfind_with_headers( resource_type.collection_path(), [DavProperty::WebDav(WebDavProperty::GetETag)], [("prefer", "depth-noroot")], ) .await .with_hrefs([sharee_base_path.as_str(), owner_base_path.as_str()]); // Test 3: Verify that only the shared resource is visible sharee_client .propfind_with_headers( &owner_base_path, [DavProperty::WebDav(WebDavProperty::GetETag)], [("prefer", "depth-noroot")], ) .await .with_hrefs([owner_folder.as_str()]); // Test 4: Verify that the sharee can access the shared resource sharee_client .propfind( &owner_folder, [DavProperty::WebDav(WebDavProperty::GetETag)], ) .await .with_hrefs([owner_folder.as_str(), owner_file.as_str()]); sharee_client .request("GET", &owner_file, "") .await .with_status(StatusCode::OK) .with_body(&owner_file_content); match resource_type { DavResourceName::Cal => { sharee_client .multiget_calendar(&owner_folder, &[&owner_file]) .await .properties(&owner_file) .with_status(StatusCode::OK) .is_defined(DavProperty::WebDav(WebDavProperty::GetETag)); } DavResourceName::Card => { sharee_client .multiget_addressbook(&owner_folder, &[&owner_file]) .await .properties(&owner_file) .with_status(StatusCode::OK) .is_defined(DavProperty::WebDav(WebDavProperty::GetETag)); } _ => {} } // Test 5: Read ACL as owner let response = owner_client .propfind(&owner_folder, [DavProperty::WebDav(WebDavProperty::Acl)]) .await; response .properties(&owner_folder) .get(DavProperty::WebDav(WebDavProperty::Acl)) .with_values([ format!("D:ace.D:principal.D:href:{sharee_principal}").as_str(), "D:ace.D:grant.D:privilege.D:read", "D:ace.D:grant.D:privilege.D:read-current-user-privilege-set", ]); // Test 6: acl-principal-prop-set REPORT let response = owner_client .request("REPORT", &owner_folder, ACL_PRINCIPAL_QUERY) .await .with_status(StatusCode::MULTI_STATUS) .into_propfind_response(None); response .properties(&sharee_principal) .get(DavProperty::WebDav(WebDavProperty::DisplayName)) .with_values(["John Doe"]); // Test 7: Verify current-user-privilege-set and owner let response = sharee_client .propfind( &owner_folder, [ DavProperty::WebDav(WebDavProperty::CurrentUserPrivilegeSet), DavProperty::WebDav(WebDavProperty::Owner), ], ) .await; for href in [owner_folder.as_str(), owner_file.as_str()] { let props = response.properties(href); props .get(DavProperty::WebDav(WebDavProperty::CurrentUserPrivilegeSet)) .with_values([ "D:privilege.D:read", "D:privilege.D:read-current-user-privilege-set", ]); props .get(DavProperty::WebDav(WebDavProperty::Owner)) .with_values([format!("D:href:{owner_principal}").as_str()]); } // Test 8: Write operations should fail for (path, dest, dest_copy) in [ ( &owner_folder, &sharee_folder, Some(format!("{sharee_base_path}copied/")), ), (&owner_file, &sharee_file, None), ] { sharee_client .proppatch( path, [(DavProperty::WebDav(WebDavProperty::DisplayName), "test")], [], [], ) .await .with_status(StatusCode::FORBIDDEN); sharee_client .request("DELETE", path, "") .await .with_status(StatusCode::FORBIDDEN); sharee_client .request_with_headers("MOVE", path, [("destination", dest.as_str())], "") .await .with_status(StatusCode::FORBIDDEN); if let Some(dest_copy) = dest_copy { sharee_client .request_with_headers("COPY", path, [("destination", dest_copy.as_str())], "") .await .with_status(StatusCode::CREATED); } } sharee_client .request("PUT", &owner_file, resource_type.generate()) .await .with_status(StatusCode::FORBIDDEN); sharee_client .request("PUT", &sharee_created_file, resource_type.generate()) .await .with_status(StatusCode::FORBIDDEN); // Test 9: Grant write access to the sharee owner_client .acl( &owner_folder, sharee_principal.as_str(), ["read", "write-content", "write-properties"], ) .await .with_status(StatusCode::OK); if is_file { owner_client .acl( &owner_file, sharee_principal.as_str(), ["read", "write-content", "write-properties"], ) .await .with_status(StatusCode::OK); } let response = owner_client .propfind(&owner_folder, [DavProperty::WebDav(WebDavProperty::Acl)]) .await; response .properties(&owner_folder) .get(DavProperty::WebDav(WebDavProperty::Acl)) .with_values([ format!("D:ace.D:principal.D:href:{sharee_principal}").as_str(), "D:ace.D:grant.D:privilege.D:read", "D:ace.D:grant.D:privilege.D:read-current-user-privilege-set", "D:ace.D:grant.D:privilege.D:write-content", "D:ace.D:grant.D:privilege.D:write-properties", ]); let response = sharee_client .propfind( &owner_folder, [DavProperty::WebDav(WebDavProperty::CurrentUserPrivilegeSet)], ) .await; for href in [owner_folder.as_str(), owner_file.as_str()] { response .properties(href) .get(DavProperty::WebDav(WebDavProperty::CurrentUserPrivilegeSet)) .with_values([ "D:privilege.D:read", "D:privilege.D:read-current-user-privilege-set", "D:privilege.D:write-content", "D:privilege.D:write-properties", ]); } // Test 10: Delete operations should fail for (path, dest) in [(&owner_folder, &sharee_folder), (&owner_file, &sharee_file)] { sharee_client .proppatch( path, [(DavProperty::WebDav(WebDavProperty::DisplayName), "test")], [], [], ) .await .with_status(StatusCode::MULTI_STATUS); sharee_client .request("DELETE", path, "") .await .with_status(StatusCode::FORBIDDEN); sharee_client .request_with_headers("MOVE", path, [("destination", dest.as_str())], "") .await .with_status(StatusCode::FORBIDDEN); } sharee_client .request("PUT", &owner_file, &owner_file_content) .await .with_status(StatusCode::NO_CONTENT); sharee_client .request("PUT", &sharee_created_file, resource_type.generate()) .await .with_status(StatusCode::CREATED); // Test 11: Grant delete access to the sharee and verify owner_client .acl(&owner_folder, sharee_principal.as_str(), ["read", "write"]) .await .with_status(StatusCode::OK); if is_file { owner_client .acl(&owner_file, sharee_principal.as_str(), ["read", "write"]) .await .with_status(StatusCode::OK); owner_client .acl( &sharee_created_file, sharee_principal.as_str(), ["read", "write"], ) .await .with_status(StatusCode::OK); } sharee_client .request_with_headers( "MOVE", &owner_file, [("destination", sharee_file.as_str())], "", ) .await .with_status(StatusCode::NO_CONTENT); sharee_client .request("DELETE", &sharee_created_file, "") .await .with_status(StatusCode::NO_CONTENT); sharee_client .request("DELETE", &owner_folder, "") .await .with_status(StatusCode::NO_CONTENT); // Test 12: Share and unshare a resource owner_client .acl(&owner_folder_private, sharee_principal.as_str(), ["read"]) .await .with_status(StatusCode::OK); sharee_client .propfind_with_headers( resource_type.collection_path(), [DavProperty::WebDav(WebDavProperty::GetETag)], [("prefer", "depth-noroot")], ) .await .with_hrefs([sharee_base_path.as_str(), owner_base_path.as_str()]); sharee_client .propfind_with_headers( &owner_base_path, [DavProperty::WebDav(WebDavProperty::GetETag)], [("prefer", "depth-noroot")], ) .await .with_hrefs([owner_folder_private.as_str()]); owner_client .acl(&owner_folder_private, sharee_principal.as_str(), []) .await .with_status(StatusCode::OK); sharee_client .propfind_with_headers( resource_type.collection_path(), [DavProperty::WebDav(WebDavProperty::GetETag)], [("prefer", "depth-noroot")], ) .await .with_hrefs([sharee_base_path.as_str()]); // Delete resources owner_client .request("DELETE", &owner_folder_private, "") .await .with_status(StatusCode::NO_CONTENT); sharee_client .request("DELETE", &sharee_folder, "") .await .with_status(StatusCode::NO_CONTENT); sharee_client .request("DELETE", &format!("{sharee_base_path}copied/"), "") .await .with_status(StatusCode::NO_CONTENT); } sharee_client.delete_default_containers().await; owner_client.delete_default_containers().await; test.assert_is_empty().await; } impl DummyWebDavClient { pub async fn acl<'x>( &self, query: &str, principal_href: &str, grant: impl IntoIterator, ) -> DavResponse { let body = ACL_QUERY.replace("$HREF", principal_href).replace( "$GRANT", &grant.into_iter().fold(String::new(), |mut output, g| { use std::fmt::Write; let _ = write!(output, ""); output }), ); self.request("ACL", query, &body).await } } const ACL_QUERY: &str = r#" $HREF $GRANT "#; const ACL_PRINCIPAL_QUERY: &str = r#" "#; ================================================ FILE: tests/src/webdav/basic.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use dav_proto::Depth; use hyper::StatusCode; use super::WebDavTest; pub async fn test(test: &WebDavTest) { println!("Running basic tests..."); let john = test.client("john"); let jane = test.client("jane"); // Test OPTIONS request john.request("OPTIONS", "/dav/file", "") .await .with_header( "dav", concat!( "1, 2, 3, access-control, extended-mkcol, calendar-access, ", "calendar-auto-schedule, calendar-no-timezone, addressbook" ), ) .with_header( "allow", concat!( "OPTIONS, GET, HEAD, POST, PUT, DELETE, COPY, MOVE, ", "MKCALENDAR, MKCOL, PROPFIND, PROPPATCH, LOCK, UNLOCK, REPORT, ACL" ), ); // Test Discovery john.request("PROPFIND", "/.well-known/carddav", "") .await .with_values( "D:multistatus.D:response.D:href", ["/dav/card/", "/dav/card/john/"], ); jane.request("PROPFIND", "/.well-known/caldav", "") .await .with_values( "D:multistatus.D:response.D:href", ["/dav/cal/", "/dav/cal/jane/", "/dav/cal/support/"], ); // Test 404 responses jane.sync_collection( "/dav/cal/jane/default/", "", Depth::Infinity, None, ["D:getetag"], ) .await; jane.sync_collection( "/dav/cal/jane/test-404/", "", Depth::Infinity, None, ["D:getetag"], ) .await; jane.request("PROPFIND", "/dav/cal/jane/default/", "") .await .with_status(StatusCode::MULTI_STATUS); jane.request( "REPORT", "/dav/cal/jane/default/", concat!( r#""#, r#""#, r#""#, r#""# ), ) .await .with_status(StatusCode::MULTI_STATUS); jane.request( "REPORT", "/dav/cal/jane/test-404/", concat!( r#""#, r#""#, r#""#, r#""# ), ) .await .with_status(StatusCode::MULTI_STATUS); jane.request("PROPFIND", "/dav/cal/jane/test-404/", "") .await .with_status(StatusCode::NOT_FOUND); john.delete_default_containers().await; jane.delete_default_containers().await; jane.delete_default_containers_by_account("support").await; test.assert_is_empty().await; } ================================================ FILE: tests/src/webdav/cal_alarm.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::WebDavTest; use crate::jmap::mail::mailbox::destroy_all_mailboxes_for_account; use email::cache::MessageCacheFetch; use hyper::StatusCode; use mail_parser::{DateTime, MessageParser}; use store::write::now; pub async fn test(test: &WebDavTest) { println!("Running calendar e-mail alarms tests..."); let client = test.client("john"); client .request_with_headers( "PUT", "/dav/cal/john/default/its-alarming-how-charming-i-feel.ics", [("content-type", "text/calendar; charset=utf-8")], TEST_ALARM_1.replace( "$START", &DateTime::from_timestamp(now() as i64 + 5) .to_rfc3339() .replace(['-', ':'], ""), ), ) .await .with_status(StatusCode::CREATED); tokio::time::sleep(std::time::Duration::from_secs(6)).await; // Check that the alarm was sent let messages = test .server .get_cached_messages(client.account_id) .await .unwrap(); assert_eq!(messages.emails.items.len(), 2); for (idx, message) in messages.emails.items.iter().enumerate() { let contents = test .fetch_email(client.account_id, message.document_id) .await; let message = MessageParser::new().parse(&contents).unwrap(); let contents = message .html_bodies() .next() .unwrap() .text_contents() .unwrap(); if idx == 0 { // First alarm does not have a summary or description assert!( contents.contains("See the pretty girl in that mirror there"), "failed for {contents}" ); assert!( contents.contains("What mirror where?!"), "failed for {contents}" ); } else { assert!( contents.contains("I feel pretty and witty and gay"), "failed for {contents}" ); assert!( contents.contains("It's alarming how charming I feel."), "failed for {contents}" ); } assert!( contents.contains(concat!( "/dav/cal/john/default/", "its-alarming-how-charming-i-feel.ics" )), "failed for {contents}" ); } client.delete_default_containers().await; destroy_all_mailboxes_for_account(client.account_id).await; test.assert_is_empty().await } const TEST_ALARM_1: &str = r#"BEGIN:VCALENDAR VERSION:2.0 BEGIN:VEVENT UID: 2371c2d9-a136-43b0-bba3-f6ab249ad46e SUMMARY:See the pretty girl in that mirror there DESCRIPTION:What mirror where?! DTSTART:$START DTEND;TZID=America/New_York:21250221T180000 LOCATION:West Side BEGIN:VALARM TRIGGER:-P2S ACTION:EMAIL ATTENDEE:mailto:john_doe@unknown.com SUMMARY:I feel pretty and witty and gay DESCRIPTION:I feel charming, Oh, so charming, It's alarming how charming I feel. END:VALARM BEGIN:VALARM TRIGGER:-P4S ACTION:EMAIL END:VALARM END:VEVENT END:VCALENDAR "#; ================================================ FILE: tests/src/webdav/cal_itip.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use ahash::AHashMap; use calcard::{ common::{IanaString, PartialDateTime}, icalendar::{ICalendar, ICalendarProperty, ICalendarValue}, }; use groupware::scheduling::{ ItipMessage, ItipSummary, event_cancel::itip_cancel, event_create::itip_create, event_update::itip_update, inbound::{MergeResult, itip_import_message, itip_merge_changes, itip_process_message}, snapshot::itip_snapshot, }; use std::{collections::hash_map::Entry, path::PathBuf}; struct Test { test_name: String, command: Command, line_num: usize, parameters: Vec, payload: String, } #[derive(Debug, PartialEq, Eq)] enum Command { Put, Get, Delete(bool), Expect, Send, Reset, Itip, } pub fn test() { for entry in std::fs::read_dir( PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("resources") .join("itip"), ) .unwrap() { let entry = entry.unwrap(); let path = entry.path(); if path.extension().is_none_or(|ext| ext != "txt") { continue; } let file_name = path.file_name().unwrap().to_str().unwrap(); let rules = std::fs::read_to_string(&path).unwrap(); let mut last_comment = ""; let mut last_command = ""; let mut last_line_num = 0; let mut payload = String::new(); let mut commands = Vec::new(); for (line_num, line) in rules.lines().enumerate() { if line.starts_with('#') { last_comment = line.trim_start_matches('#').trim(); } else if let Some(command) = line.strip_prefix("> ") { last_command = command.trim(); last_line_num = line_num; } else if !line.is_empty() { payload.push_str(line); payload.push('\n'); } else { if last_command.is_empty() && payload.is_empty() { continue; } let mut command_and_args = last_command.split_whitespace(); let command = match command_and_args .next() .expect("Command should not be empty") { "put" => Command::Put, "get" => Command::Get, "expect" => Command::Expect, "send" => Command::Send, "delete" => Command::Delete(false), "delete-force-send" => Command::Delete(true), "reset" => Command::Reset, "itip" => Command::Itip, _ => panic!("Unknown command: {}", last_command), }; commands.push(Test { command, test_name: last_comment.to_string(), line_num: last_line_num, parameters: command_and_args.map(String::from).collect(), payload: payload.trim().to_string(), }); last_command = ""; last_line_num = 0; payload.clear(); } } if commands.is_empty() { panic!("No commands found in file: {}", file_name); } else if !last_command.is_empty() { panic!( "File ended with command '{}' at line {} without payload", last_command, last_line_num ); } println!("====== Running test: {} ======", file_name); let mut store: AHashMap> = AHashMap::new(); let mut dtstamp_map: AHashMap = AHashMap::new(); let mut last_itip = None; for command in &commands { if command.command != Command::Put { println!("{} (line {})", command.test_name, command.line_num); } match command.command { Command::Put => { let account = command .parameters .first() .expect("Account parameter is required"); let name = command .parameters .get(1) .expect("Name parameter is required"); let mut ical = ICalendar::parse(&command.payload) .expect("Failed to parse iCalendar payload"); match store .entry(account.to_string()) .or_default() .entry(name.to_string()) { Entry::Occupied(mut entry) => { last_itip = Some(itip_update( &mut ical, entry.get_mut(), &[account.to_string()], )); entry.insert(ical); } Entry::Vacant(entry) => { last_itip = Some(itip_create(&mut ical, &[account.to_string()])); entry.insert(ical); } } } Command::Get => { let account = command .parameters .first() .expect("Account parameter is required") .as_str(); let name = command .parameters .get(1) .expect("Name parameter is required") .as_str(); let ical = ICalendar::parse(&command.payload) .expect("Failed to parse iCalendar payload") .to_string() .replace("\r\n", "\n"); store .get(account) .and_then(|account_store| account_store.get(name)) .map(|stored_ical| { let stored_ical = normalize_ical(stored_ical.clone(), &mut dtstamp_map); if stored_ical != ical { panic!( "ICalendar mismatch for {}: expected {}, got {}", command.test_name, ical, stored_ical ); } }) .unwrap_or_else(|| { panic!( "ICalendar not found for account: {}, name: {}", account, name ); }); } Command::Delete(force_send) => { let account = command .parameters .first() .expect("Account parameter is required") .as_str(); let name = command .parameters .get(1) .expect("Name parameter is required") .as_str(); let store = store.get_mut(account).expect("Account not found in store"); if let Some(ical) = store.remove(name) { last_itip = Some( itip_cancel(&ical, &[account.to_string()], force_send) .map(|message| vec![message]), ); } else { panic!( "ICalendar not found for account: {}, name: {}", account, name ); } } Command::Expect => { let last_itip_str = match last_itip .as_ref() .expect("No last iTIP message to compare against") { Ok(m) => { let mut result = String::new(); for (i, m) in m.iter().enumerate() { if i > 0 { result.push_str("================================\n"); } result.push_str(&m.to_string(&mut dtstamp_map)); } result } Err(e) => format!("{e:?}"), }; assert_eq!( command.payload.trim(), last_itip_str.trim(), "iTIP message mismatch for {} at line {}\nEXPECTED {}\n\nRECEIVED {}", command.test_name, command.line_num, command.payload, last_itip_str ); } Command::Send => { let mut results = String::new(); match last_itip { Some(Ok(messages)) => { for message in messages { for rcpt in &message.to { let result = match itip_snapshot( &message.message, &[rcpt.to_string()], false, ) { Ok(itip_snapshots) => { match store .entry(rcpt.to_string()) .or_default() .entry(itip_snapshots.uid.to_string()) { Entry::Occupied(mut entry) => { let ical = entry.get_mut(); let snapshots = itip_snapshot( ical, &[rcpt.to_string()], false, ) .expect("Failed to create iTIP snapshot"); match itip_process_message( ical, snapshots, &message.message, itip_snapshots, message.from.clone(), ) { Ok(result) => match result { MergeResult::Actions(changes) => { itip_merge_changes(ical, changes); Ok(None) } MergeResult::Message(message) => { Ok(Some(message)) } MergeResult::None => Ok(None), }, Err(err) => Err(err), } } Entry::Vacant(entry) => { let mut message = message.message.clone(); itip_import_message(&mut message) .expect("Failed to import iTIP message"); entry.insert(message); Ok(None) } } } Err(err) => Err(err), }; match result { Ok(Some(itip_message)) => { results.push_str( &itip_message.to_string(&mut dtstamp_map), ); } Ok(None) => {} Err(e) => { results.push_str(&format!("{e:?}")); } } } } assert_eq!( results.trim(), command.payload.trim(), "iTIP send result mismatch for {} at line {}: expected {}, got {}", command.test_name, command.line_num, command.payload, results ); } Some(Err(e)) => { panic!( "Failed to create iTIP message for {} at line {}: {:?}", command.test_name, command.line_num, e ); } None => { panic!( "No iTIP message to send for {} at line {}", command.test_name, command.line_num ); } } last_itip = None; } Command::Itip => { let mut commands = command.parameters.iter(); last_itip = Some(Ok(vec![ItipMessage { from_organizer: false, from: commands .next() .expect("From parameter is required") .to_string(), to: commands.map(|s| s.to_string()).collect::>(), summary: ItipSummary::Invite(vec![]), message: ICalendar::parse(&command.payload) .expect("Failed to parse iCalendar payload"), }])) } Command::Reset => { store.clear(); dtstamp_map.clear(); last_itip = None; } } } } } trait ItipMessageExt { fn to_string(&self, map: &mut AHashMap) -> String; } impl ItipMessageExt for ItipMessage { fn to_string(&self, map: &mut AHashMap) -> String { use std::fmt::Write; let mut f = String::new(); let mut to = self.to.iter().map(|t| t.as_str()).collect::>(); to.sort_unstable(); writeln!(&mut f, "from: {}", self.from).unwrap(); writeln!(&mut f, "to: {}", to.join(", ")).unwrap(); write!(&mut f, "summary: ").unwrap(); let mut fields = Vec::new(); match &self.summary { ItipSummary::Invite(itip_fields) => { writeln!(&mut f, "invite").unwrap(); fields.push(itip_fields); } ItipSummary::Update { method, current, previous, } => { writeln!(&mut f, "update {}", method.as_str()).unwrap(); fields.push(current); fields.push(previous); } ItipSummary::Cancel(itip_fields) => { writeln!(&mut f, "cancel").unwrap(); fields.push(itip_fields); } ItipSummary::Rsvp { part_stat, current } => { writeln!(&mut f, "rsvp {}", part_stat.as_str()).unwrap(); fields.push(current); } } for (pos, fields) in fields.into_iter().enumerate() { let prefix = if pos > 0 { "~summary." } else { "summary." }; let mut fields = fields .iter() .map(|f| format!("{}: {:?}", f.name.as_str().to_lowercase(), f.value)) .collect::>(); fields.sort_unstable(); for field in fields { writeln!(&mut f, "{prefix}{}", field).unwrap(); } } write!(&mut f, "{}", normalize_ical(self.message.clone(), map)).unwrap(); f } } fn normalize_ical(mut ical: ICalendar, map: &mut AHashMap) -> String { let mut comps = ical .components .iter() .enumerate() .filter(|(comp_id, _)| { ical.components[0] .component_ids .contains(&(*comp_id as u32)) }) .collect::>(); comps.sort_unstable_by_key(|(_, comp)| *comp); ical.components[0].component_ids = comps.iter().map(|(comp_id, _)| *comp_id as u32).collect(); for comp in &mut ical.components { for entry in &mut comp.entries { if let (ICalendarProperty::Dtstamp, Some(ICalendarValue::PartialDateTime(dt))) = (&entry.name, entry.values.first()) { if let Some(index) = map.get(dt) { entry.values = vec![ICalendarValue::Integer(*index as i64)]; } else { let index = map.len(); map.insert(dt.as_ref().clone(), index); entry.values = vec![ICalendarValue::Integer(index as i64)]; } } } comp.entries.sort_unstable(); } ical.to_string().replace("\r\n", "\n") } ================================================ FILE: tests/src/webdav/cal_query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::WebDavTest; use ahash::AHashSet; use calcard::{common::timezone::Tz, icalendar::ICalendar}; use groupware::{ DavResourceName, calendar::{CalendarEventData, alarm::ExpandAlarm, expand::CalendarEventExpansion}, }; use hyper::StatusCode; use store::write::serialize::rkyv_unarchive; use types::TimeRange; pub async fn test(test: &WebDavTest) { println!("Running REPORT calendar-query & free-busy-query tests..."); let client = test.client("john"); let cal_path = format!("{}/john/default/", DavResourceName::Cal.base_path()); #[allow(clippy::never_loop)] for (num, ics) in [ (1, ICAL_RFC_ABCD1_ICS), (2, ICAL_RFC_ABCD2_ICS), (3, ICAL_RFC_ABCD3_ICS), (4, ICAL_RFC_ABCD4_ICS), (5, ICAL_RFC_ABCD5_ICS), (6, ICAL_RFC_ABCD6_ICS), (7, ICAL_RFC_ABCD7_ICS), (8, ICAL_RFC_ABCD8_ICS), ] { roundtrip_expansion(ics, false); client .request("PUT", &rfc_file_name(num), ics) .await .with_status(StatusCode::CREATED); } // Test 1: Partial Retrieval of Events by Time Range let response = client .request("REPORT", &cal_path, REPORT_1) .await .with_status(StatusCode::MULTI_STATUS) .with_hrefs([rfc_file_name(2).as_str(), rfc_file_name(3).as_str()]) .into_propfind_response(None); response .properties(&rfc_file_name(2)) .calendar_data() .with_values([REPORT_1_EXPECTED_ABCD2.replace('\n', "\r\n").as_str()]); response .properties(&rfc_file_name(3)) .calendar_data() .with_values([REPORT_1_EXPECTED_ABCD3.replace('\n', "\r\n").as_str()]); // Test 2: Partial Retrieval of Recurring Events let response = client .request("REPORT", &cal_path, REPORT_2) .await .with_status(StatusCode::MULTI_STATUS) .with_hrefs([rfc_file_name(2).as_str(), rfc_file_name(3).as_str()]) .into_propfind_response(None); response .properties(&rfc_file_name(2)) .calendar_data() .with_values([REPORT_2_EXPECTED_ABCD2.replace('\n', "\r\n").as_str()]); response .properties(&rfc_file_name(3)) .calendar_data() .with_values([REPORT_2_EXPECTED_ABCD3.replace('\n', "\r\n").as_str()]); // Test 3: Expanded Retrieval of Recurring Events let response = client .request("REPORT", &cal_path, REPORT_3) .await .with_status(StatusCode::MULTI_STATUS) .with_hrefs([rfc_file_name(2).as_str(), rfc_file_name(3).as_str()]) .into_propfind_response(None); response .properties(&rfc_file_name(2)) .calendar_data() .with_values([REPORT_3_EXPECTED_ABCD2.replace('\n', "\r\n").as_str()]); response .properties(&rfc_file_name(3)) .calendar_data() .with_values([REPORT_3_EXPECTED_ABCD3.replace('\n', "\r\n").as_str()]); // Test 4: Partial Retrieval of Stored Free Busy Components let response = client .request("REPORT", &cal_path, REPORT_4) .await .with_status(StatusCode::MULTI_STATUS) .with_hrefs([rfc_file_name(8).as_str()]) .into_propfind_response(None); response .properties(&rfc_file_name(8)) .calendar_data() .with_values([REPORT_4_EXPECTED_ABCD8.replace('\n', "\r\n").as_str()]); // Test 5: Retrieval of To-Dos by Alarm Time Range let response = client .request("REPORT", &cal_path, REPORT_5) .await .with_status(StatusCode::MULTI_STATUS) .with_hrefs([rfc_file_name(5).as_str()]) .into_propfind_response(None); response .properties(&rfc_file_name(5)) .calendar_data() .with_values([ICAL_RFC_ABCD5_ICS.replace('\n', "\r\n").as_str()]); // Test 6: Retrieval of Event by UID client .request("REPORT", &cal_path, REPORT_6) .await .with_status(StatusCode::MULTI_STATUS) .with_hrefs([rfc_file_name(3).as_str()]) .into_propfind_response(None); // Test 7: Retrieval of Events by PARTSTAT client .request("REPORT", &cal_path, REPORT_7) .await .with_status(StatusCode::MULTI_STATUS) .with_hrefs([rfc_file_name(3).as_str()]) .into_propfind_response(None); // Test 8: Retrieval of Events Only client .request("REPORT", &cal_path, REPORT_8) .await .with_status(StatusCode::MULTI_STATUS) .with_hrefs([ rfc_file_name(1).as_str(), rfc_file_name(2).as_str(), rfc_file_name(3).as_str(), ]) .into_propfind_response(None); // Test 9: Retrieval of All Pending To-Dos client .request("REPORT", &cal_path, REPORT_9) .await .with_status(StatusCode::MULTI_STATUS) .with_hrefs([rfc_file_name(4).as_str(), rfc_file_name(5).as_str()]) .into_propfind_response(None); // Test 10: Successful CALDAV:free-busy-query REPORT assert_eq!( remove_dtstamp( client .request("REPORT", &cal_path, REPORT_10) .await .with_status(StatusCode::OK) .body .as_ref() .unwrap() ), remove_dtstamp(REPORT_10_RESPONSE) ); assert_eq!( remove_dtstamp( client .request("REPORT", &cal_path, REPORT_11) .await .with_status(StatusCode::OK) .body .as_ref() .unwrap() ), remove_dtstamp(REPORT_11_RESPONSE) ); client.delete_default_containers().await; test.assert_is_empty().await; } #[test] #[ignore] fn ical_roundtrip_expansion() { for entry in std::fs::read_dir("/Users/me/code/calcard/resources/ical").unwrap() { let entry = entry.unwrap(); let path = entry.path(); if path.extension().is_some_and(|ext| ext == "ics") { println!("Testing: {:?}", path); let input = match String::from_utf8(std::fs::read(&path).unwrap()) { Ok(input) => input, Err(err) => { // ISO-8859-1 err.as_bytes() .iter() .map(|&b| b as char) .collect::() } }; roundtrip_expansion(&input, true); } } } fn roundtrip_expansion(ics: &str, ignore_errors: bool) { let ical = if let Ok(ical) = ICalendar::parse(ics) { ical } else if ignore_errors { return; } else { panic!("Failed to parse ICalendar {}", ics); }; let expanded = ical.expand_dates(Tz::UTC, 100); if !ignore_errors { assert!(expanded.errors.is_empty()); } let mut min_utc = i64::MAX; let mut max_utc = i64::MIN; let mut events = expanded .events .into_iter() .enumerate() .map(|(i, e)| { let e = e.try_into_date_time().unwrap(); let start = e.start.timestamp(); let end = e.end.timestamp(); let mut min = std::cmp::min(start, end); let mut max = std::cmp::max(start, end); for alarm in ical.alarms_for_id(e.comp_id) { if let Some(alarm_time) = alarm .expand_alarm(0, 0) .and_then(|alarm| alarm.delta.to_timestamp(start, end, Tz::UTC)) { if alarm_time < min { min = alarm_time; } if alarm_time > max { max = alarm_time; } } } if min < min_utc { min_utc = min; } if max > max_utc { max_utc = max; } CalendarEventExpansion { comp_id: e.comp_id, expansion_id: i as u32, start, end, } }) .collect::>(); // Verify min/max UTC timestamps let event_data = CalendarEventData::new(ical, Tz::UTC, 100, &mut None); let from_time = event_data.base_time_utc as i64 + event_data.base_offset; let to_time = from_time + event_data.duration as i64; if min_utc != i64::MAX { assert_eq!( from_time, min_utc, "diff: {}, failed for {}", from_time - min_utc, ics ); assert_eq!( to_time, max_utc, "diff: {}, failed for {}", to_time - max_utc, ics ); } // Validate archive expansion let expanded_bytes = rkyv::to_bytes::(&event_data).unwrap(); let expanded_archive = rkyv_unarchive::(&expanded_bytes).unwrap(); let mut events_archive = expanded_archive .expand( Tz::UTC, TimeRange { start: i64::MIN, end: i64::MAX, }, ) .unwrap(); assert_eq!( events_archive, event_data .expand_from_ids( &mut events .iter() .map(|e| e.expansion_id) .collect::>(), Tz::UTC ) .unwrap() ); events.sort_by(|a, b| { if a.comp_id == b.comp_id { a.start.cmp(&b.start) } else { a.comp_id.cmp(&b.comp_id) } }); events_archive.sort_by(|a, b| { if a.comp_id == b.comp_id { a.start.cmp(&b.start) } else { a.comp_id.cmp(&b.comp_id) } }); for event in events.iter_mut().chain(events_archive.iter_mut()) { event.expansion_id = 0; } assert_eq!(events, events_archive); } fn rfc_file_name(num: usize) -> String { format!( "{}/john/default/abcd{num}.ics", DavResourceName::Cal.base_path() ) } const REPORT_1: &str = r#" "#; const REPORT_1_EXPECTED_ABCD2: &str = r#"BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VTIMEZONE LAST-MODIFIED:20040110T032845Z TZID:US/Eastern BEGIN:DAYLIGHT DTSTART:20000404T020000 RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4 TZNAME:EDT TZOFFSETFROM:-0500 TZOFFSETTO:-0400 END:DAYLIGHT BEGIN:STANDARD DTSTART:20001026T020000 RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 TZNAME:EST TZOFFSETFROM:-0400 TZOFFSETTO:-0500 END:STANDARD END:VTIMEZONE BEGIN:VEVENT DTSTART;TZID=US/Eastern:20060102T120000 DURATION:PT1H RRULE:FREQ=DAILY;COUNT=5 SUMMARY:Event #2 UID:00959BC664CA650E933C892C@example.com END:VEVENT BEGIN:VEVENT DTSTART;TZID=US/Eastern:20060104T140000 DURATION:PT1H RECURRENCE-ID;TZID=US/Eastern:20060104T120000 SUMMARY:Event #2 bis UID:00959BC664CA650E933C892C@example.com END:VEVENT BEGIN:VEVENT DTSTART;TZID=US/Eastern:20060106T140000 DURATION:PT1H RECURRENCE-ID;TZID=US/Eastern:20060106T120000 SUMMARY:Event #2 bis bis UID:00959BC664CA650E933C892C@example.com END:VEVENT END:VCALENDAR "#; const REPORT_1_EXPECTED_ABCD3: &str = r#"BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VTIMEZONE LAST-MODIFIED:20040110T032845Z TZID:US/Eastern BEGIN:DAYLIGHT DTSTART:20000404T020000 RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4 TZNAME:EDT TZOFFSETFROM:-0500 TZOFFSETTO:-0400 END:DAYLIGHT BEGIN:STANDARD DTSTART:20001026T020000 RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 TZNAME:EST TZOFFSETFROM:-0400 TZOFFSETTO:-0500 END:STANDARD END:VTIMEZONE BEGIN:VEVENT DTSTART;TZID=US/Eastern:20060104T100000 DURATION:PT1H SUMMARY:Event #3 UID:DC6C50A017428C5216A2F1CD@example.com END:VEVENT END:VCALENDAR "#; const REPORT_2: &str = r#" "#; const REPORT_2_EXPECTED_ABCD2: &str = r#"BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VTIMEZONE LAST-MODIFIED:20040110T032845Z TZID:US/Eastern BEGIN:DAYLIGHT DTSTART:20000404T020000 RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4 TZNAME:EDT TZOFFSETFROM:-0500 TZOFFSETTO:-0400 END:DAYLIGHT BEGIN:STANDARD DTSTART:20001026T020000 RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 TZNAME:EST TZOFFSETFROM:-0400 TZOFFSETTO:-0500 END:STANDARD END:VTIMEZONE BEGIN:VEVENT DTSTAMP:20060206T001121Z DTSTART;TZID=US/Eastern:20060102T120000 DURATION:PT1H RRULE:FREQ=DAILY;COUNT=5 SUMMARY:Event #2 UID:00959BC664CA650E933C892C@example.com END:VEVENT BEGIN:VEVENT DTSTAMP:20060206T001121Z DTSTART;TZID=US/Eastern:20060104T140000 DURATION:PT1H RECURRENCE-ID;TZID=US/Eastern:20060104T120000 SUMMARY:Event #2 bis UID:00959BC664CA650E933C892C@example.com END:VEVENT END:VCALENDAR "#; const REPORT_2_EXPECTED_ABCD3: &str = r#"BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VTIMEZONE LAST-MODIFIED:20040110T032845Z TZID:US/Eastern BEGIN:DAYLIGHT DTSTART:20000404T020000 RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4 TZNAME:EDT TZOFFSETFROM:-0500 TZOFFSETTO:-0400 END:DAYLIGHT BEGIN:STANDARD DTSTART:20001026T020000 RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 TZNAME:EST TZOFFSETFROM:-0400 TZOFFSETTO:-0500 END:STANDARD END:VTIMEZONE BEGIN:VEVENT ATTENDEE;PARTSTAT=ACCEPTED;ROLE=CHAIR:mailto:cyrus@example.com ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:lisa@example.com DTSTAMP:20060206T001220Z DTSTART;TZID=US/Eastern:20060104T100000 DURATION:PT1H LAST-MODIFIED:20060206T001330Z ORGANIZER:mailto:cyrus@example.com SEQUENCE:1 STATUS:TENTATIVE SUMMARY:Event #3 UID:DC6C50A017428C5216A2F1CD@example.com X-ABC-GUID:E1CX5Dr-0007ym-Hz@example.com END:VEVENT END:VCALENDAR "#; const REPORT_3: &str = r#" "#; const REPORT_3_EXPECTED_ABCD2: &str = r#"BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VEVENT DTSTART:20060103T170000Z RECURRENCE-ID:20060103T170000Z DTSTAMP:20060206T001121Z DURATION:PT1H SUMMARY:Event #2 UID:00959BC664CA650E933C892C@example.com END:VEVENT BEGIN:VEVENT DTSTART:20060104T190000Z RECURRENCE-ID:20060104T190000Z DTSTAMP:20060206T001121Z DURATION:PT1H SUMMARY:Event #2 bis UID:00959BC664CA650E933C892C@example.com END:VEVENT END:VCALENDAR "#; const REPORT_3_EXPECTED_ABCD3: &str = r#"BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VEVENT DTSTART:20060104T150000Z ATTENDEE;PARTSTAT=ACCEPTED;ROLE=CHAIR:mailto:cyrus@example.com ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:lisa@example.com DTSTAMP:20060206T001220Z DURATION:PT1H LAST-MODIFIED:20060206T001330Z ORGANIZER:mailto:cyrus@example.com SEQUENCE:1 STATUS:TENTATIVE SUMMARY:Event #3 UID:DC6C50A017428C5216A2F1CD@example.com X-ABC-GUID:E1CX5Dr-0007ym-Hz@example.com END:VEVENT END:VCALENDAR "#; const REPORT_4: &str = r#" "#; const REPORT_4_EXPECTED_ABCD8: &str = r#"BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VFREEBUSY ORGANIZER;CN="Bernard Desruisseaux":mailto:bernard@example.com UID:76ef34-54a3d2@example.com DTSTAMP:20050530T123421Z DTSTART:20060101T000000Z DTEND:20060108T000000Z FREEBUSY;FBTYPE=BUSY-TENTATIVE:20060102T100000Z/20060102T120000Z END:VFREEBUSY END:VCALENDAR "#; const REPORT_5: &str = r#" "#; const REPORT_6: &str = r#" DC6C50A017428C5216A2F1CD@example.com "#; const REPORT_7: &str = r#" mailto:lisa@example.com NEEDS-ACTION "#; const REPORT_8: &str = r#" "#; const REPORT_9: &str = r#" CANCELLED "#; const REPORT_10: &str = r#" "#; const REPORT_10_RESPONSE: &str = r#"BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Stalwart Labs LLC//Stalwart Server//EN BEGIN:VFREEBUSY DTSTART:20060104T140000Z DTEND:20060105T220000Z FREEBUSY;FBTYPE=BUSY-TENTATIVE:20060104T150000Z/20060104T160000Z FREEBUSY;FBTYPE=BUSY:20060104T190000Z/20060104T200000Z,20060105T170000Z/20060105T180000Z FREEBUSY;FBTYPE=BUSY-UNAVAILABLE:20060105T100000Z/20060105T120000Z END:VFREEBUSY END:VCALENDAR "#; const REPORT_11: &str = r#" "#; const REPORT_11_RESPONSE: &str = r#"BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Stalwart Labs LLC//Stalwart Server//EN BEGIN:VFREEBUSY DTSTART:20060101T000000Z DTEND:20060104T140000Z DTSTAMP:20250505T105255Z FREEBUSY;FBTYPE=BUSY-TENTATIVE:20060102T100000Z/20060102T120000Z FREEBUSY;FBTYPE=BUSY:20060102T150000Z/20060102T160000Z,20060102T170000Z/20060102T180000Z, 20060103T100000Z/20060103T120000Z,20060103T170000Z/20060103T180000Z,20060104T100000Z/20060104T120000Z END:VFREEBUSY END:VCALENDAR "#; const ICAL_RFC_ABCD1_ICS: &str = r#"BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VTIMEZONE LAST-MODIFIED:20040110T032845Z TZID:US/Eastern BEGIN:DAYLIGHT DTSTART:20000404T020000 RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4 TZNAME:EDT TZOFFSETFROM:-0500 TZOFFSETTO:-0400 END:DAYLIGHT BEGIN:STANDARD DTSTART:20001026T020000 RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 TZNAME:EST TZOFFSETFROM:-0400 TZOFFSETTO:-0500 END:STANDARD END:VTIMEZONE BEGIN:VEVENT DTSTAMP:20060206T001102Z DTSTART;TZID=US/Eastern:20060102T100000 DURATION:PT1H SUMMARY:Event #1 Description:Go Steelers! UID:74855313FA803DA593CD579A@example.com END:VEVENT END:VCALENDAR "#; const ICAL_RFC_ABCD2_ICS: &str = r#"BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VTIMEZONE LAST-MODIFIED:20040110T032845Z TZID:US/Eastern BEGIN:DAYLIGHT DTSTART:20000404T020000 RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4 TZNAME:EDT TZOFFSETFROM:-0500 TZOFFSETTO:-0400 END:DAYLIGHT BEGIN:STANDARD DTSTART:20001026T020000 RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 TZNAME:EST TZOFFSETFROM:-0400 TZOFFSETTO:-0500 END:STANDARD END:VTIMEZONE BEGIN:VEVENT DTSTAMP:20060206T001121Z DTSTART;TZID=US/Eastern:20060102T120000 DURATION:PT1H RRULE:FREQ=DAILY;COUNT=5 SUMMARY:Event #2 UID:00959BC664CA650E933C892C@example.com END:VEVENT BEGIN:VEVENT DTSTAMP:20060206T001121Z DTSTART;TZID=US/Eastern:20060104T140000 DURATION:PT1H RECURRENCE-ID;TZID=US/Eastern:20060104T120000 SUMMARY:Event #2 bis UID:00959BC664CA650E933C892C@example.com END:VEVENT BEGIN:VEVENT DTSTAMP:20060206T001121Z DTSTART;TZID=US/Eastern:20060106T140000 DURATION:PT1H RECURRENCE-ID;TZID=US/Eastern:20060106T120000 SUMMARY:Event #2 bis bis UID:00959BC664CA650E933C892C@example.com END:VEVENT END:VCALENDAR "#; const ICAL_RFC_ABCD3_ICS: &str = r#"BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VTIMEZONE LAST-MODIFIED:20040110T032845Z TZID:US/Eastern BEGIN:DAYLIGHT DTSTART:20000404T020000 RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4 TZNAME:EDT TZOFFSETFROM:-0500 TZOFFSETTO:-0400 END:DAYLIGHT BEGIN:STANDARD DTSTART:20001026T020000 RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 TZNAME:EST TZOFFSETFROM:-0400 TZOFFSETTO:-0500 END:STANDARD END:VTIMEZONE BEGIN:VEVENT ATTENDEE;PARTSTAT=ACCEPTED;ROLE=CHAIR:mailto:cyrus@example.com ATTENDEE;PARTSTAT=NEEDS-ACTION:mailto:lisa@example.com DTSTAMP:20060206T001220Z DTSTART;TZID=US/Eastern:20060104T100000 DURATION:PT1H LAST-MODIFIED:20060206T001330Z ORGANIZER:mailto:cyrus@example.com SEQUENCE:1 STATUS:TENTATIVE SUMMARY:Event #3 UID:DC6C50A017428C5216A2F1CD@example.com X-ABC-GUID:E1CX5Dr-0007ym-Hz@example.com END:VEVENT END:VCALENDAR "#; const ICAL_RFC_ABCD4_ICS: &str = r#"BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VTODO DTSTAMP:20060205T235335Z DUE;VALUE=DATE:20060104 STATUS:NEEDS-ACTION SUMMARY:Task #1 UID:DDDEEB7915FA61233B861457@example.com BEGIN:VALARM ACTION:AUDIO TRIGGER;RELATED=START:-PT10M END:VALARM END:VTODO END:VCALENDAR "#; const ICAL_RFC_ABCD5_ICS: &str = r#"BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VTODO DTSTAMP:20060205T235300Z DUE;TZID=US/Eastern:20060106T120000 LAST-MODIFIED:20060205T235308Z SEQUENCE:1 STATUS:NEEDS-ACTION SUMMARY:Task #2 UID:E10BA47467C5C69BB74E8720@example.com BEGIN:VALARM ACTION:AUDIO TRIGGER;RELATED=START:-PT10M END:VALARM END:VTODO END:VCALENDAR "#; const ICAL_RFC_ABCD6_ICS: &str = r#"BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VTODO COMPLETED:20051223T122322Z DTSTAMP:20060205T235400Z DUE;VALUE=DATE:20051225 LAST-MODIFIED:20060205T235308Z SEQUENCE:1 STATUS:COMPLETED SUMMARY:Task #3 UID:E10BA47467C5C69BB74E8722@example.com END:VTODO END:VCALENDAR "#; const ICAL_RFC_ABCD7_ICS: &str = r#"BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VTODO DTSTAMP:20060205T235600Z DUE;VALUE=DATE:20060101 LAST-MODIFIED:20060205T235308Z SEQUENCE:1 STATUS:CANCELLED SUMMARY:Task #4 UID:E10BA47467C5C69BB74E8725@example.com END:VTODO END:VCALENDAR "#; const ICAL_RFC_ABCD8_ICS: &str = r#"BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VFREEBUSY ORGANIZER;CN="Bernard Desruisseaux":mailto:bernard@example.com UID:76ef34-54a3d2@example.com DTSTAMP:20050530T123421Z DTSTART:20060101T000000Z DTEND:20060108T000000Z FREEBUSY:20050531T230000Z/20050601T010000Z FREEBUSY;FBTYPE=BUSY-TENTATIVE:20060102T100000Z/20060102T120000Z FREEBUSY:20060103T100000Z/20060103T120000Z FREEBUSY:20060104T100000Z/20060104T120000Z FREEBUSY;FBTYPE=BUSY-UNAVAILABLE:20060105T100000Z/20060105T120000Z FREEBUSY:20060106T100000Z/20060106T120000Z END:VFREEBUSY END:VCALENDAR "#; fn remove_dtstamp(ics: &str) -> AHashSet { let mut result = AHashSet::new(); for line in ics.lines() { if !line.starts_with("DTSTAMP:") { result.insert(line.to_string()); } } result } ================================================ FILE: tests/src/webdav/cal_scheduling.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::WebDavTest; use crate::{ jmap::mail::mailbox::destroy_all_mailboxes_for_account, webdav::{DummyWebDavClient, prop::ALL_DAV_PROPERTIES}, }; use calcard::{ common::timezone::Tz, icalendar::{ ICalendarDay, ICalendarFrequency, ICalendarMethod, ICalendarParticipationStatus, ICalendarProperty, ICalendarRecurrenceRule, ICalendarWeekday, }, }; use common::{Server, auth::AccessToken}; use dav_proto::schema::property::{CalDavProperty, DavProperty, WebDavProperty}; use email::cache::MessageCacheFetch; use groupware::{ cache::GroupwareCache, scheduling::{ ArchivedItipSummary, ItipField, ItipParticipant, ItipSummary, ItipTime, ItipValue, }, }; use hyper::StatusCode; use mail_parser::{DateTime, MessageParser}; use services::task_manager::imip::build_itip_template; use std::str::FromStr; use store::write::now; use types::collection::SyncCollection; pub async fn test(test: &WebDavTest) { println!("Running calendar scheduling tests..."); let bill_client = test.client("bill"); let jane_client = test.client("jane"); let john_client = test.client("john"); // Validate hierarchy of scheduling resources let response = jane_client .propfind_with_headers("/dav/itip/jane/", ALL_DAV_PROPERTIES, [("depth", "1")]) .await; let properties = response .with_hrefs([ "/dav/itip/jane/", "/dav/itip/jane/inbox/", "/dav/itip/jane/outbox/", ]) .properties("/dav/itip/jane/inbox/"); // Validate schedule inbox properties properties .get(DavProperty::WebDav(WebDavProperty::ResourceType)) .with_values(["D:collection", "A:schedule-inbox"]); properties .get(DavProperty::CalDav( CalDavProperty::ScheduleDefaultCalendarURL, )) .with_values(["D:href:/dav/cal/jane/default/"]) .with_status(StatusCode::OK); properties .get(DavProperty::WebDav(WebDavProperty::SupportedPrivilegeSet)) .with_some_values([ "D:supported-privilege.D:privilege.D:all", concat!( "D:supported-privilege.D:supported-privilege.", "D:privilege.D:read" ), concat!( "D:supported-privilege.D:supported-privilege.", "D:privilege.A:schedule-deliver" ), concat!( "D:supported-privilege.D:supported-privilege.", "D:supported-privilege.D:privilege.A:schedule-deliver-invite" ), concat!( "D:supported-privilege.D:supported-privilege.", "D:supported-privilege.D:privilege.A:schedule-deliver-reply" ), concat!( "D:supported-privilege.D:supported-privilege.", "D:supported-privilege.D:privilege.A:schedule-query-freebusy" ), ]); properties .get(DavProperty::WebDav(WebDavProperty::CurrentUserPrivilegeSet)) .with_values([ "D:privilege.D:write-properties", "D:privilege.A:schedule-deliver-invite", "D:privilege.D:write-content", "D:privilege.A:schedule-deliver", "D:privilege.D:read", "D:privilege.D:all", "D:privilege.A:schedule-query-freebusy", "D:privilege.D:read-acl", "D:privilege.D:write-acl", "D:privilege.A:schedule-deliver-reply", "D:privilege.D:write", "D:privilege.D:read-current-user-privilege-set", ]); // Validate schedule outbox properties let properties = response.properties("/dav/itip/jane/outbox/"); properties .get(DavProperty::WebDav(WebDavProperty::ResourceType)) .with_values(["D:collection", "A:schedule-outbox"]); properties .get(DavProperty::WebDav(WebDavProperty::SupportedPrivilegeSet)) .with_some_values([ "D:supported-privilege.D:privilege.D:all", concat!( "D:supported-privilege.D:supported-privilege.", "D:privilege.D:read" ), concat!( "D:supported-privilege.D:supported-privilege.", "D:privilege.A:schedule-send" ), concat!( "D:supported-privilege.D:supported-privilege.", "D:supported-privilege.D:privilege.A:schedule-send-invite" ), concat!( "D:supported-privilege.D:supported-privilege.", "D:supported-privilege.D:privilege.A:schedule-send-reply" ), concat!( "D:supported-privilege.D:supported-privilege.", "D:supported-privilege.D:privilege.A:schedule-send-freebusy" ), ]); properties .get(DavProperty::WebDav(WebDavProperty::CurrentUserPrivilegeSet)) .with_values([ "D:privilege.D:write-properties", "D:privilege.A:schedule-send-invite", "D:privilege.D:write-content", "D:privilege.A:schedule-send", "D:privilege.D:read", "D:privilege.D:all", "D:privilege.A:schedule-send-freebusy", "D:privilege.D:read-acl", "D:privilege.D:write-acl", "D:privilege.A:schedule-send-reply", "D:privilege.D:write", "D:privilege.D:read-current-user-privilege-set", ]); // Send invitation to Bill and Mike let test_itip = TEST_ITIP .replace( "$START", &DateTime::from_timestamp(now() as i64 + 60 * 60) .to_rfc3339() .replace(['-', ':'], ""), ) .replace( "$END", &DateTime::from_timestamp(now() as i64 + 5 * 60 * 60) .to_rfc3339() .replace(['-', ':'], ""), ); john_client .request_with_headers( "PUT", "/dav/cal/john/default/itip.ics", [("content-type", "text/calendar; charset=utf-8")], &test_itip, ) .await .with_status(StatusCode::CREATED); tokio::time::sleep(std::time::Duration::from_millis(200)).await; // Check that the invitation was received by Bill and Mike for client in [bill_client, jane_client] { let messages = test .server .get_cached_messages(client.account_id) .await .unwrap(); assert_eq!(messages.emails.items.len(), 1); let access_token = test .server .get_access_token(client.account_id) .await .unwrap(); let events = test .server .fetch_dav_resources(&access_token, client.account_id, SyncCollection::Calendar) .await .unwrap(); assert_eq!(events.resources.len(), 2); let events = test .server .fetch_dav_resources( &access_token, client.account_id, SyncCollection::CalendarEventNotification, ) .await .unwrap(); assert_eq!(events.resources.len(), 3); } // Validate iTIP let itips = fetch_and_remove_itips(jane_client).await; assert_eq!(itips.len(), 1); let itip = itips.first().unwrap(); assert!( itip.contains("SUMMARY:Lunch") && itip.contains("METHOD:REQUEST"), "failed for itip: {itip}" ); // Fetch added calendar entry let cals = fetch_icals(jane_client).await; assert_eq!(cals.len(), 1); let cal = cals.into_iter().next().unwrap(); // Using an invalid schedule tag should fail let rsvp_ical = cal.ical.replace( "PARTSTAT=NEEDS-ACTION:mailto:jane.smith", "PARTSTAT=ACCEPTED:mailto:jane.smith", ); jane_client .request_with_headers( "PUT", &cal.href, [ ("content-type", "text/calendar; charset=utf-8"), ("if-schedule-tag-match", "\"9999999\""), ], &rsvp_ical, ) .await .with_status(StatusCode::PRECONDITION_FAILED); // RSVP the invitation jane_client .request_with_headers( "PUT", &cal.href, [ ("content-type", "text/calendar; charset=utf-8"), ("if-schedule-tag-match", cal.schedule_tag.as_str()), ], &rsvp_ical, ) .await .with_status(StatusCode::NO_CONTENT); // Make sure that the schedule has not changed assert_eq!( fetch_icals(jane_client).await[0].schedule_tag, cal.schedule_tag ); // Check that John received the RSVP tokio::time::sleep(std::time::Duration::from_millis(200)).await; test.wait_for_index().await; let itips = fetch_and_remove_itips(john_client).await; assert_eq!(itips.len(), 1); assert!( itips[0].contains("METHOD:REPLY") && itips[0].contains("PARTSTAT=ACCEPTED:mailto:jane.smith"), "failed for itip: {}", itips[0] ); let cals = fetch_icals(john_client).await; assert_eq!(cals.len(), 1); assert!( cals[0] .ical .contains("PARTSTAT=ACCEPTED;SCHEDULE-STATUS=2.0:mailto:jane"), "failed for cal: {}", cals[0].ical ); // Changing the event name should not trigger a new iTIP let updated_ical = rsvp_ical.replace("Lunch", "Dinner"); jane_client .request_with_headers( "PUT", &cal.href, [("content-type", "text/calendar; charset=utf-8")], &updated_ical, ) .await .with_status(StatusCode::NO_CONTENT); tokio::time::sleep(std::time::Duration::from_millis(200)).await; assert_eq!( fetch_and_remove_itips(john_client).await, Vec::::new() ); // Deleting the event should send a cancellation jane_client .request("DELETE", &cal.href, "") .await .with_status(StatusCode::NO_CONTENT); tokio::time::sleep(std::time::Duration::from_millis(200)).await; let itips = fetch_and_remove_itips(john_client).await; assert_eq!(itips.len(), 1); assert!( itips[0].contains("METHOD:REPLY") && itips[0].contains("PARTSTAT=DECLINED:mailto:jane.smith"), "failed for itip: {}", itips[0] ); let cals = fetch_icals(john_client).await; assert_eq!(cals.len(), 1); let cal = cals.into_iter().next().unwrap(); assert!( cal.ical.contains("PARTSTAT=DECLINED:mailto:jane"), "failed for cal: {}", cal.ical ); // Fetch Bill's email invitation and RSVP via HTTP let document_id = test .server .get_cached_messages(bill_client.account_id) .await .unwrap() .emails .items[0] .document_id; let contents = test.fetch_email(bill_client.account_id, document_id).await; let message = MessageParser::new().parse(&contents).unwrap(); let contents = message .html_bodies() .next() .unwrap() .text_contents() .unwrap(); let url = contents .split("href=\"") .filter_map(|s| { let url = s.split_once('\"').map(|(url, _)| url)?; if url.contains("m=ACCEPTED") { Some(url.strip_prefix("https://webdav.example.org").unwrap()) } else { None } }) .next() .unwrap_or_else(|| { panic!("Failed to find RSVP link in email contents: {contents}"); }); let response = jane_client .request("GET", url, "") .await .with_status(StatusCode::OK) .body .unwrap(); assert!( response.contains("Lunch") && response.contains("RSVP has been recorded"), "failed for response: {response}" ); let cals = fetch_icals(john_client).await; assert_eq!(cals.len(), 1); let cal = cals.into_iter().next().unwrap(); assert!( cal.ical.contains("PARTSTAT=ACCEPTED:mailto:bill"), "failed for cal: {}", cal.ical ); // Test the schedule outbox let test_outbox = TEST_FREEBUSY .replace( "$START", &DateTime::from_timestamp(now() as i64) .to_rfc3339() .replace(['-', ':'], ""), ) .replace( "$END", &DateTime::from_timestamp(now() as i64 + 100 * 60 * 60) .to_rfc3339() .replace(['-', ':'], ""), ); let response = john_client .request_with_headers( "POST", "/dav/itip/john/outbox/", [("content-type", "text/calendar; charset=utf-8")], &test_outbox, ) .await .with_status(StatusCode::OK); let mut account = ""; let mut found_data = false; for (key, value) in &response.xml { match key.as_str() { "A:schedule-response.A:response.A:recipient.D:href" => { account = value.strip_prefix("mailto:").unwrap(); } "A:schedule-response.A:response.A:request-status" => { if account == "unknown@example.com" { assert_eq!( value, "3.7;Invalid calendar user or insufficient permissions" ); } else { assert_eq!(value, "2.0;Success"); } } "A:schedule-response.A:response.A:calendar-data" => { assert!( value.contains("BEGIN:VFREEBUSY"), "missing freebusy data in response: {response:?}" ); if account == "jdoe@example.com" { assert!( value.contains("FREEBUSY;FBTYPE=BUSY:"), "missing freebusy data in response: {response:?}" ); found_data = true; } } _ => {} } } assert!( found_data, "Missing calendar data in response: {response:?}" ); // Modifying john's event should only send updates to bill let updated_ical = cal.ical.replace("Lunch", "Breakfast at Tiffany's"); john_client .request_with_headers( "PUT", &cal.href, [("content-type", "text/calendar; charset=utf-8")], &updated_ical, ) .await .with_status(StatusCode::NO_CONTENT); // Make sure that the schedule has changed assert_ne!( fetch_icals(john_client).await[0].schedule_tag, cal.schedule_tag ); let main_event_href = cal.href; // Check that Bill received the update tokio::time::sleep(std::time::Duration::from_millis(200)).await; test.wait_for_index().await; let mut itips = fetch_and_remove_itips(bill_client).await; itips.sort_unstable_by(|a, _| { if a.contains("Lunch") { std::cmp::Ordering::Less } else { std::cmp::Ordering::Greater } }); assert_eq!(itips.len(), 2); assert!( itips[0].contains("METHOD:REQUEST") && itips[0].contains("Lunch"), "failed for itip: {}", itips[0] ); assert!( itips[1].contains("METHOD:REQUEST") && itips[1].contains("Breakfast at Tiffany's"), "failed for itip: {}", itips[1] ); let cals = fetch_icals(bill_client).await; assert_eq!(cals.len(), 1); let cal = cals.into_iter().next().unwrap(); assert!( cal.ical.contains("SUMMARY:Breakfast at Tiffany's") && cal.ical.contains("PARTSTAT=ACCEPTED:mailto:bill"), "failed for cal: {}", cal.ical ); let attendee_href = cal.href; assert_eq!( fetch_and_remove_itips(jane_client).await, Vec::::new() ); // Removing the event should from John's calendar send a cancellation to Bill john_client .request("DELETE", &main_event_href, "") .await .with_status(StatusCode::NO_CONTENT); tokio::time::sleep(std::time::Duration::from_millis(200)).await; let itips = fetch_and_remove_itips(bill_client).await; assert_eq!(itips.len(), 1); assert!( itips[0].contains("METHOD:CANCEL") && itips[0].contains("STATUS:CANCELLED"), "failed for itip: {}", itips[0] ); let cals = fetch_icals(bill_client).await; assert_eq!(cals.len(), 1); let cal = cals.into_iter().next().unwrap(); assert!( cal.ical.contains("STATUS:CANCELLED"), "failed for cal: {}", cal.ical ); assert_eq!( fetch_and_remove_itips(jane_client).await, Vec::::new() ); // Delete the event from Bill's calendar disabling schedule replies bill_client .request_with_headers("DELETE", &attendee_href, [("Schedule-Reply", "F")], "") .await .with_status(StatusCode::NO_CONTENT); tokio::time::sleep(std::time::Duration::from_millis(200)).await; assert_eq!( fetch_and_remove_itips(john_client).await, Vec::::new() ); for client in [bill_client, jane_client, john_client] { client.delete_default_containers().await; destroy_all_mailboxes_for_account(client.account_id).await; } test.assert_is_empty().await; } async fn fetch_and_remove_itips(client: &DummyWebDavClient) -> Vec { let inbox_href = format!("/dav/itip/{}/inbox/", client.name); let response = client .propfind_with_headers(&inbox_href, ALL_DAV_PROPERTIES, [("depth", "1")]) .await; let mut itips = vec![]; for href in response.hrefs.keys().filter(|&href| href != &inbox_href) { let itip = client .request("GET", href, "") .await .with_status(StatusCode::OK) .body .expect("Missing body"); client .request("DELETE", href, "") .await .with_status(StatusCode::NO_CONTENT); itips.push(itip); } itips } #[derive(Debug)] struct CalEntry { href: String, ical: String, schedule_tag: String, } async fn fetch_icals(client: &DummyWebDavClient) -> Vec { let cal_inbox = format!("/dav/cal/{}/default/", client.name); let response = client .propfind_with_headers(&cal_inbox, ALL_DAV_PROPERTIES, [("depth", "1")]) .await; let mut cals = vec![]; for href in response.hrefs.keys().filter(|&href| href != &cal_inbox) { let ical = client .request("GET", href, "") .await .with_status(StatusCode::OK) .body .expect("Missing body"); let properties = response.properties(href); assert!( !ical.contains("METHOD:"), "iTIP method found in calendar entry: {ical}" ); cals.push(CalEntry { href: href.to_string(), ical, schedule_tag: properties .get(DavProperty::CalDav(CalDavProperty::ScheduleTag)) .value() .to_string(), }); } cals } pub async fn test_build_itip_templates(server: &Server) { let dummy_access_token = AccessToken::from_id(0); for (idx, summary) in [ ItipSummary::Invite(vec![ ItipField { name: ICalendarProperty::Summary, value: ItipValue::Text("Lunch".to_string()), }, ItipField { name: ICalendarProperty::Description, value: ItipValue::Text("Lunch at the cafe".to_string()), }, ItipField { name: ICalendarProperty::Location, value: ItipValue::Text("Cafe Corner".to_string()), }, ItipField { name: ICalendarProperty::Dtstart, value: ItipValue::Time(ItipTime { start: 1750616068, tz_id: Tz::from_str("New Zealand").unwrap().as_id(), }), }, ItipField { name: ICalendarProperty::Attendee, value: ItipValue::Participants(vec![ ItipParticipant { email: "jdoe@domain.com".to_string(), name: Some("John Doe".to_string()), is_organizer: true, }, ItipParticipant { email: "jane@domain.com".to_string(), name: Some("Jane Smith".to_string()), is_organizer: false, }, ]), }, ]), ItipSummary::Cancel(vec![ ItipField { name: ICalendarProperty::Summary, value: ItipValue::Text("Lunch".to_string()), }, ItipField { name: ICalendarProperty::Description, value: ItipValue::Text("Lunch at the cafe".to_string()), }, ItipField { name: ICalendarProperty::Location, value: ItipValue::Text("Cafe Corner".to_string()), }, ItipField { name: ICalendarProperty::Dtstart, value: ItipValue::Time(ItipTime { start: 1750616068, tz_id: Tz::from_str("New Zealand").unwrap().as_id(), }), }, ]), ItipSummary::Rsvp { part_stat: ICalendarParticipationStatus::Accepted, current: vec![ ItipField { name: ICalendarProperty::Summary, value: ItipValue::Text("Lunch".to_string()), }, ItipField { name: ICalendarProperty::Description, value: ItipValue::Text("Lunch at the cafe".to_string()), }, ItipField { name: ICalendarProperty::Location, value: ItipValue::Text("Cafe Corner".to_string()), }, ItipField { name: ICalendarProperty::Dtstart, value: ItipValue::Time(ItipTime { start: 1750616068, tz_id: Tz::from_str("New Zealand").unwrap().as_id(), }), }, ItipField { name: ICalendarProperty::Rrule, value: ItipValue::Rrule(Box::new(ICalendarRecurrenceRule { freq: ICalendarFrequency::Weekly, until: None, count: Some(2), interval: Some(3), bysecond: Default::default(), byday: vec![ ICalendarDay { ordwk: None, weekday: ICalendarWeekday::Monday, }, ICalendarDay { ordwk: None, weekday: ICalendarWeekday::Wednesday, }, ], ..Default::default() })), }, ], }, ItipSummary::Rsvp { part_stat: ICalendarParticipationStatus::Declined, current: vec![ ItipField { name: ICalendarProperty::Summary, value: ItipValue::Text("Lunch".to_string()), }, ItipField { name: ICalendarProperty::Description, value: ItipValue::Text("Lunch at the cafe".to_string()), }, ItipField { name: ICalendarProperty::Location, value: ItipValue::Text("Cafe Corner".to_string()), }, ItipField { name: ICalendarProperty::Dtstart, value: ItipValue::Time(ItipTime { start: 1750616068, tz_id: Tz::from_str("New Zealand").unwrap().as_id(), }), }, ], }, ItipSummary::Update { method: ICalendarMethod::Request, current: vec![ ItipField { name: ICalendarProperty::Summary, value: ItipValue::Text("Lunch".to_string()), }, ItipField { name: ICalendarProperty::Description, value: ItipValue::Text("Lunch at the cafe".to_string()), }, ItipField { name: ICalendarProperty::Location, value: ItipValue::Text("Cafe Corner".to_string()), }, ItipField { name: ICalendarProperty::Dtstart, value: ItipValue::Time(ItipTime { start: 1750616068, tz_id: Tz::from_str("New Zealand").unwrap().as_id(), }), }, ItipField { name: ICalendarProperty::Attendee, value: ItipValue::Participants(vec![ ItipParticipant { email: "jdoe@domain.com".to_string(), name: Some("John Doe".to_string()), is_organizer: true, }, ItipParticipant { email: "jane@domain.com".to_string(), name: Some("Jane Smith".to_string()), is_organizer: false, }, ]), }, ], previous: vec![ ItipField { name: ICalendarProperty::Summary, value: ItipValue::Text("Dinner".to_string()), }, ItipField { name: ICalendarProperty::Description, value: ItipValue::Text("Dinner at the cafe".to_string()), }, ItipField { name: ICalendarProperty::Dtstart, value: ItipValue::Time(ItipTime { start: 1750916068, tz_id: Tz::from_str("New Zealand").unwrap().as_id(), }), }, ], }, ] .into_iter() .enumerate() { let bytes = rkyv::to_bytes::(&summary) .unwrap() .to_vec(); let summary = rkyv::access::(&bytes).unwrap(); let html = build_itip_template( server, &dummy_access_token, 0, 1, "john.doe@example.org", "jane.smith@example.net", summary, "124", ) .await; println!("iTIP template {idx}: {}", html.subject); std::fs::write(format!("itip_template_{idx}.html"), html.body) .expect("Failed to write iTIP template to file"); } } const TEST_ITIP: &str = r#"BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN BEGIN:VEVENT UID:9263504FD3AD SEQUENCE:0 DTSTART:$START DTEND:$END DTSTAMP:20090602T170000Z TRANSP:OPAQUE SUMMARY:Lunch ORGANIZER:mailto:jdoe@example.com ATTENDEE;CUTYPE=INDIVIDUAL:mailto:jane.smith@example.com ATTENDEE;CUTYPE=INDIVIDUAL:mailto:bill@example.com END:VEVENT END:VCALENDAR "#; const TEST_FREEBUSY: &str = r#"BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN METHOD:REQUEST BEGIN:VFREEBUSY UID:4FD3AD926350 DTSTAMP:20090602T190420Z DTSTART:$START DTEND:$END ORGANIZER:mailto:jdoe@example.com ATTENDEE:mailto:jdoe@example.com ATTENDEE:mailto:jane.smith@example.com ATTENDEE:mailto:bill@example.com ATTENDEE:mailto:unknown@example.com END:VFREEBUSY END:VCALENDAR "#; ================================================ FILE: tests/src/webdav/card_query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::WebDavTest; use dav_proto::schema::property::{CardDavProperty, DavProperty, WebDavProperty}; use groupware::DavResourceName; use hyper::StatusCode; pub async fn test(test: &WebDavTest) { println!("Running REPORT addressbook-query tests..."); let client = test.client("john"); // Create test data let default_path = format!("{}/john/default/", DavResourceName::Card.base_path()); let mut hrefs = Vec::with_capacity(3); for (i, vcard) in [VCARD1, VCARD2, VCARD3].iter().enumerate() { let href = format!("{default_path}contact-{i}.vcf",); client .request("PUT", &href, *vcard) .await .with_status(hyper::StatusCode::CREATED); hrefs.push(href); } let uri_sarah = hrefs[0].as_str(); let uri_carlos = hrefs[1].as_str(); let uri_acme = hrefs[2].as_str(); // Test 1: RFC6352 8.6.3 example 1 let response = client .request("REPORT", &default_path, QUERY1) .await .with_status(StatusCode::MULTI_STATUS) .with_hrefs([uri_carlos]) .into_propfind_response(None); let props = response.properties(uri_carlos); props .get(DavProperty::WebDav(WebDavProperty::GetETag)) .is_not_empty(); props .get(DavProperty::CardDav(CardDavProperty::AddressData( Default::default(), ))) .with_values([r#"BEGIN:VCARD VERSION:4.0 FN:Carlos Rodriguez-Martinez NICKNAME:Charlie EMAIL;TYPE=WORK,pref:carlos.rodriguez@example-corp.com EMAIL;TYPE=HOME:carlosrm@personalmail.example UID:urn:uuid:e1ee798b-3d4c-41b0-b217-b9c918e4686a END:VCARD "# .replace('\n', "\r\n") .as_str()]); // Test 2: RFC6352 8.6.3 example 2 let response = client .request("REPORT", &default_path, QUERY2) .await .with_status(StatusCode::MULTI_STATUS) .with_hrefs([uri_carlos, uri_sarah]) .into_propfind_response(None); let props = response.properties(uri_carlos); props .get(DavProperty::WebDav(WebDavProperty::GetETag)) .is_not_empty(); props .get(DavProperty::CardDav(CardDavProperty::AddressData( Default::default(), ))) .with_values([r#"BEGIN:VCARD FN:Carlos Rodriguez-Martinez BDAY:--0623 CATEGORIES:Marketing,Management,International LANG;TYPE=WORK;PREF=1:es LANG;TYPE=WORK;PREF=2:en LANG;TYPE=WORK;PREF=3:pt END:VCARD "# .replace('\n', "\r\n") .as_str()]); let props = response.properties(uri_sarah); props .get(DavProperty::WebDav(WebDavProperty::GetETag)) .is_not_empty(); props .get(DavProperty::CardDav(CardDavProperty::AddressData( Default::default(), ))) .with_values([r#"BEGIN:VCARD FN:Sarah Johnson BDAY:19850415 CATEGORIES:Work,Research,VIP LANG;TYPE=WORK;PREF=1:en LANG;TYPE=WORK;PREF=2:fr END:VCARD "# .replace('\n', "\r\n") .as_str()]); // Test 3: Search within parameters let response = client .request("REPORT", &default_path, QUERY3) .await .with_status(StatusCode::MULTI_STATUS) .with_hrefs([uri_acme]) .into_propfind_response(None); let props = response.properties(uri_acme); props .get(DavProperty::CardDav(CardDavProperty::AddressData( Default::default(), ))) .with_values([VCARD3.replace('\n', "\r\n").as_str()]); // Test 4: Search using limit client .request("REPORT", &default_path, QUERY4) .await .with_status(StatusCode::MULTI_STATUS) .with_value( "D:multistatus.D:response.D:status", "HTTP/1.1 507 Insufficient Storage", ) .with_value( "D:multistatus.D:response.D:error.D:number-of-matches-within-limits", "", ) .with_value( "D:multistatus.D:response.D:responsedescription", "The number of matches exceeds the limit of 2", ) .with_href_count(3); client.delete_default_containers().await; test.assert_is_empty().await; } const QUERY1: &str = r#" charlie "#; const QUERY2: &str = r#" john rodriguez "#; const QUERY3: &str = r#" enterprise "#; const QUERY4: &str = r#" acme global 2 "#; const VCARD1: &str = r#"BEGIN:VCARD VERSION:4.0 FN:Sarah Johnson N:Johnson;Sarah;Marie;Dr.;Ph.D. NICKNAME:Sadie GENDER:F BDAY:19850415 ANNIVERSARY:20100610 EMAIL;TYPE=work:sarah.johnson@example.com EMAIL;TYPE=home,pref:sarahjpersonal@example.com TEL;TYPE=cell,voice,pref:+1-555-123-4567 TEL;TYPE=work,voice:+1-555-987-6543 TEL;TYPE=home,voice:+1-555-456-7890 ADR;TYPE=work;LABEL="123 Business Ave\nSuite 400\nNew York, NY 10001\nUSA":;;123 Business Ave;New York;NY;10001;USA ADR;TYPE=home,pref;LABEL="456 Residential St\nApt 7B\nBrooklyn, NY 11201\nUSA":;;456 Residential St;Brooklyn;NY;11201;USA ORG:Acme Technologies Inc.;Research Department TITLE:Senior Research Scientist ROLE:Team Lead CATEGORIES:Work,Research,VIP URL;TYPE=work:https://www.example.com/staff/sjohnson URL;TYPE=home:https://www.sarahjohnson.example.com KEY;TYPE=PGP:https://pgp.example.com/pks/lookup?op=get&search=sarah.johnson@example.com NOTE:Sarah prefers video calls over phone calls. Available Mon-Thu 9-5 EST. LANG;TYPE=work;PREF=1:en LANG;TYPE=work;PREF=2:fr TZ:-0500 GEO:40.7128;-74.0060 UID:urn:uuid:f81d4fae-7dec-11d0-a765-00a0c91e6bf6 REV:20220315T133000Z END:VCARD "#; const VCARD2: &str = r#"BEGIN:VCARD VERSION:4.0 FN:Carlos Rodriguez-Martinez N:Rodriguez-Martinez;Carlos;Alberto;Mr.;Jr. NICKNAME:Charlie GENDER:M BDAY:--0623 ANNIVERSARY:20150809 EMAIL;TYPE=work,pref:carlos.rodriguez@example-corp.com EMAIL;TYPE=home:carlosrm@personalmail.example TEL;TYPE=cell,voice,pref:+34-611-234-567 TEL;TYPE=work,voice:+34-911-876-543 TEL;TYPE=home,voice:+34-644-321-987 TEL;TYPE=fax:+34-911-876-544 ADR;TYPE=work;LABEL="Calle Empresarial 42\nPlanta 3\nMadrid, 28001\nSpain":;;Calle Empresarial 42;Madrid;;28001;Spain ADR;TYPE=home,pref;LABEL="Avenida Residencial 15\nPiso 7, Puerta C\nMadrid, 28045\nSpain":;;Avenida Residencial 15;Madrid;;28045;Spain ORG:Global Solutions S.L.;Marketing Division TITLE:Digital Marketing Director ROLE:Department Head CATEGORIES:Marketing,Management,International URL;TYPE=work:https://www.example-corp.com/team/carlos URL;TYPE=home:https://www.carlosrodriguez.example URL;TYPE=social:https://linkedin.com/in/carlosrodriguezm KEY;TYPE=PGP:https://pgp.example.com/pks/lookup?op=get&search=carlos.rodriguez@example-corp.com NOTE:Carlos speaks English, Spanish, and Portuguese fluently. Prefers communication via email. Do not contact after 7PM CET. LANG;TYPE=work;PREF=1:es LANG;TYPE=work;PREF=2:en LANG;TYPE=work;PREF=3:pt TZ:+0100 GEO:40.4168;-3.7038 UID:urn:uuid:e1ee798b-3d4c-41b0-b217-b9c918e4686a REV:20230712T092135Z SOURCE:https://contacts.example.com/carlosrodriguez.vcf KIND:individual MEMBER:urn:uuid:03a0e51f-d1aa-4385-8a53-e29025acd8af RELATED;TYPE=friend:urn:uuid:f81d4fae-7dec-11d0-a765-00a0c91e6bf6 END:VCARD "#; const VCARD3: &str = r#"BEGIN:VCARD VERSION:4.0 FN:Acme Business Solutions Ltd. N:;;;; KIND:ORG ORG:Acme Business Solutions Ltd.;Technology Division EMAIL;TYPE=WORK,pref:info@acme-solutions.example EMAIL;TYPE=support:support@acme-solutions.example EMAIL;TYPE=sales:sales@acme-solutions.example TEL;TYPE=WORK,VOICE,pref:+44-20-1234-5678 TEL;TYPE=FAX:+44-20-1234-5679 TEL;TYPE=support:+44-800-987-6543 ADR;TYPE=WORK;LABEL="10 Enterprise Way\nTech Park\nLondon, EC1A 1BB\nUnited Kingdom":;;10 Enterprise Way\, Tech Park;London;;EC1A 1BB;United Kingdom ADR;TYPE=branch;LABEL="25 Innovation Street\nManchester, M1 5QF\nUnited Kin gdom":;;25 Innovation Street;Manchester;;M1 5QF;United Kingdom URL;TYPE=WORK:https://www.acme-solutions.example URL;TYPE=support:https://support.acme-solutions.example CATEGORIES:Technology,B2B,Solutions,Services NOTE:Business hours: Mon-Fri 9:00-17:30 GMT. Closed on UK bank holidays. VAT Reg: GB123456789 TZ:Z GEO:51.5074;-0.1278 KEY;TYPE=PGP:https://pgp.example.com/pks/lookup?op=get&search=info@acme-solu tions.example UID:urn:uuid:a9e95948-7b1c-46e8-bd85-c729a9e910f2 REV:20230415T153000Z LANG;TYPE=WORK;PREF=1:en LANG;TYPE=WORK;PREF=2:de LANG;TYPE=WORK;PREF=3:fr SOURCE:https://directory.example.com/acme.vcf RELATED;TYPE=CONTACT:urn:uuid:b9e93fdb-4d34-45fa-a1e2-47da0428c4a1 RELATED;TYPE=CONTACT:urn:uuid:c8e74dfe-6b34-45fa-b1e2-47ea0428c4b2 X-ABLabel:Company PRODID:-//Example Corp.//Contact Manager 3.0//EN END:VCARD "#; ================================================ FILE: tests/src/webdav/copy_move.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{DavResponse, WebDavTest}; use crate::webdav::GenerateTestDavResource; use ahash::AHashSet; use dav_proto::Depth; use groupware::DavResourceName; use hyper::StatusCode; pub async fn test(test: &WebDavTest, assisted_discovery: bool) { let client = test.client("jane"); let mike_noquota = test.client("mike"); for resource_type in [ DavResourceName::File, DavResourceName::Cal, DavResourceName::Card, ] { println!("Running COPY/MOVE tests ({})...", resource_type.base_path()); let user_base_path = format!("{}/jane", resource_type.base_path()); let group_base_path = format!("{}/support", resource_type.base_path()); let default_test_depth = if resource_type == DavResourceName::File { 2 } else { 0 }; // Obtain sync token let response = client .sync_collection(&user_base_path, "", Depth::Infinity, None, ["D:getetag"]) .await; // TODO: Fix tests for assisted discovery assert_eq!( response.hrefs().len(), if resource_type == DavResourceName::File { 1 } else { 2 + usize::from(assisted_discovery) }, "{:?}", response.hrefs() ); // Create nested files and folders let (hierarchy_root, mut hierarchy) = client .create_hierarchy(&user_base_path, default_test_depth, 2, 3) .await; let prev_sync_token = response.sync_token(); let response = client .sync_collection( &user_base_path, prev_sync_token, Depth::Infinity, None, ["D:getetag"], ) .await; let sync_token = response.sync_token(); let changed_hrefs = response.hrefs(); assert_ne!(sync_token, prev_sync_token); assert_eq!( changed_hrefs, hierarchy.iter().map(|x| x.0.as_str()).collect::>(), "lengths {} & {}", changed_hrefs.len(), hierarchy.len() ); client.validate_values(&hierarchy).await; // Delete cache an resync test.clear_cache(); let response = client .sync_collection( &user_base_path, prev_sync_token, Depth::Infinity, None, ["D:getetag"], ) .await; let sync_token = response.sync_token(); let changed_hrefs = response.hrefs(); assert_ne!(sync_token, prev_sync_token); assert_eq!( changed_hrefs, hierarchy.iter().map(|x| x.0.as_str()).collect::>(), "lengths {} & {}", changed_hrefs.len(), hierarchy.len() ); // Copying and moving to the same or root containers is invalid for method in ["COPY", "MOVE"] { for destination in [ "/dav", "/dav/cal", "/dav/card", "/dav/file", "/dav/pal", hierarchy_root.as_str(), ] { client .request_with_headers( method, &hierarchy_root, [("destination", destination)], "", ) .await .with_status(StatusCode::BAD_GATEWAY); } } // Test 1: Rename container let new_hierarchy_root = format!("{user_base_path}/Test_Folder/"); client .request_with_headers( "MOVE", &hierarchy_root, [("destination", new_hierarchy_root.as_str())], "", ) .await .with_status(StatusCode::CREATED); let response = client .sync_collection(&user_base_path, "", Depth::Infinity, None, ["D:getetag"]) .await; replace_prefix(&mut hierarchy, &hierarchy_root, &new_hierarchy_root); assert_result(&response, &hierarchy); client.validate_values(&hierarchy).await; // Validate changes let changes = client .sync_collection( &user_base_path, sync_token, Depth::Infinity, None, ["D:getetag"], ) .await .with_href_count(2) .into_propfind_response(None); changes .properties(&hierarchy_root) .with_status(StatusCode::NOT_FOUND); changes .properties(&new_hierarchy_root) .with_status(StatusCode::OK); let hierarchy_root = new_hierarchy_root; // Test 2: Copy container let new_hierarchy_root = format!("{user_base_path}/Test_Folder_Copy/"); client .request_with_headers( "COPY", &hierarchy_root, [("destination", new_hierarchy_root.as_str())], "", ) .await .with_status(StatusCode::CREATED); let response = client .sync_collection(&user_base_path, "", Depth::Infinity, None, ["D:getetag"]) .await; let mut copied_hierarchy = hierarchy.clone(); replace_prefix(&mut copied_hierarchy, &hierarchy_root, &new_hierarchy_root); copied_hierarchy.extend_from_slice(&hierarchy); assert_result(&response, &copied_hierarchy); client.validate_values(&copied_hierarchy).await; // Test 3: Delete original container client .request("DELETE", &new_hierarchy_root, "") .await .with_status(StatusCode::NO_CONTENT); let response = client .sync_collection(&user_base_path, "", Depth::Infinity, None, ["D:getetag"]) .await; assert_result(&response, &hierarchy); client.validate_values(&hierarchy).await; // Test 4: Create a shallow container and overwrite the previous one using MOVE let (new_hierarchy_root, mut hierarchy) = client.create_hierarchy(&user_base_path, 0, 0, 3).await; let sync_token = client .sync_collection( &user_base_path, sync_token, Depth::Infinity, None, ["D:getetag"], ) .await .sync_token() .to_string(); client .request_with_headers( "MOVE", &new_hierarchy_root, [("destination", hierarchy_root.as_str())], "", ) .await .with_status(StatusCode::NO_CONTENT); let response = client .sync_collection(&user_base_path, "", Depth::Infinity, None, ["D:getetag"]) .await; replace_prefix(&mut hierarchy, &new_hierarchy_root, &hierarchy_root); assert_result(&response, &hierarchy); client.validate_values(&hierarchy).await; // Validate changes let changes = client .sync_collection( &user_base_path, &sync_token, Depth::Infinity, None, ["D:getetag"], ) .await .into_propfind_response(None); changes .properties(&new_hierarchy_root) .with_status(StatusCode::NOT_FOUND); changes .properties(&hierarchy_root) .with_status(StatusCode::OK); // Test 5: Create a deep container and overwrite the previous one using COPY let (new_hierarchy_root, new_hierarchy) = client .create_hierarchy(&user_base_path, default_test_depth, 1, 2) .await; client .request_with_headers( "COPY", &new_hierarchy_root, [("destination", hierarchy_root.as_str())], "", ) .await .with_status(StatusCode::NO_CONTENT); let response = client .sync_collection(&user_base_path, "", Depth::Infinity, None, ["D:getetag"]) .await; let mut orig_hierarchy = new_hierarchy.clone(); replace_prefix(&mut orig_hierarchy, &new_hierarchy_root, &hierarchy_root); let mut full_hierarchy = new_hierarchy.clone(); full_hierarchy.extend_from_slice(&orig_hierarchy); assert_result(&response, &full_hierarchy); client.validate_values(&full_hierarchy).await; // Test 6: Copy and move containers to a shared account let shared_hierarchy_root_1 = format!("{group_base_path}/Test_Shared_Folder_1/"); let shared_hierarchy_root_2 = format!("{group_base_path}/Test_Shared_Folder_2/"); client .request_with_headers( "MOVE", &new_hierarchy_root, [("destination", shared_hierarchy_root_1.as_str())], "", ) .await .with_status(StatusCode::CREATED); client .request_with_headers( "COPY", &hierarchy_root, [("destination", shared_hierarchy_root_2.as_str())], "", ) .await .with_status(StatusCode::CREATED); let response = client .sync_collection(&user_base_path, "", Depth::Infinity, None, ["D:getetag"]) .await; assert_result(&response, &orig_hierarchy); client.validate_values(&orig_hierarchy).await; let response = client .sync_collection(&group_base_path, "", Depth::Infinity, None, ["D:getetag"]) .await; replace_prefix( &mut full_hierarchy, &new_hierarchy_root, &shared_hierarchy_root_1, ); replace_prefix( &mut full_hierarchy, &hierarchy_root, &shared_hierarchy_root_2, ); assert_result(&response, &full_hierarchy); client.validate_values(&full_hierarchy).await; // Delete all containers for shared_container in [ shared_hierarchy_root_1, shared_hierarchy_root_2, hierarchy_root, ] { client .request("DELETE", &shared_container, "") .await .with_status(StatusCode::NO_CONTENT); } // Create test containers let mut hierarchy = vec![]; for folder_name in ["folder1", "folder2", "folder3"] { let folder_path = format!("{user_base_path}/{folder_name}/"); client .mkcol("MKCOL", &folder_path, [], []) .await .with_status(StatusCode::CREATED); for file_name in ["file1", "file2", "file3"] { let file_path = format!("{folder_path}{file_name}"); let file_contents = resource_type.generate(); client .request("PUT", &file_path, &file_contents) .await .with_status(StatusCode::CREATED); hierarchy.push((file_path, file_contents)); } hierarchy.push((folder_path, "".to_string())); } let response = client .sync_collection(&user_base_path, "", Depth::Infinity, None, ["D:getetag"]) .await; assert_result(&response, &hierarchy); client.validate_values(&hierarchy).await; // Test 7: Copying or moving files to the root container is not allowed let folder1_file1 = format!("{user_base_path}/folder1/file1"); if resource_type != DavResourceName::File { for method in ["COPY", "MOVE"] { client .request_with_headers( method, &folder1_file1, [("destination", user_base_path.as_str())], "", ) .await .with_status(StatusCode::BAD_GATEWAY); client .request_with_headers( method, &folder1_file1, [("destination", format!("{user_base_path}/folder2").as_str())], "", ) .await .with_status(StatusCode::BAD_GATEWAY); } } // Test 8: Copying or moving to the same location is not allowed for method in ["COPY", "MOVE"] { client .request_with_headers( method, &folder1_file1, [("destination", folder1_file1.as_str())], "", ) .await .with_status(StatusCode::BAD_GATEWAY); } // Test 9: Rename file let folder1_file1_new = format!("{user_base_path}/folder1/file1_new"); client .request_with_headers( "MOVE", &folder1_file1, [("destination", folder1_file1_new.as_str())], "", ) .await .with_status(StatusCode::CREATED); rename(&mut hierarchy, &folder1_file1, &folder1_file1_new); let response = client .sync_collection(&user_base_path, "", Depth::Infinity, None, ["D:getetag"]) .await; assert_result(&response, &hierarchy); client.validate_values(&hierarchy).await; // Test 10: Move a file under a different container let folder2_file1_from_folder1 = format!("{user_base_path}/folder2/file1_from_folder1"); client .request_with_headers( "MOVE", &folder1_file1_new, [("destination", folder2_file1_from_folder1.as_str())], "", ) .await .with_status(StatusCode::CREATED); rename( &mut hierarchy, &folder1_file1_new, &folder2_file1_from_folder1, ); let response = client .sync_collection(&user_base_path, "", Depth::Infinity, None, ["D:getetag"]) .await; assert_result(&response, &hierarchy); client.validate_values(&hierarchy).await; // Test 11: Move and overwrite a file under a different container let folder1_file2 = format!("{user_base_path}/folder1/file2"); client .request_with_headers( "MOVE", &folder2_file1_from_folder1, [("destination", folder1_file2.as_str())], "", ) .await .with_status(StatusCode::NO_CONTENT); delete(&mut hierarchy, &folder1_file2); rename(&mut hierarchy, &folder2_file1_from_folder1, &folder1_file2); let response = client .sync_collection(&user_base_path, "", Depth::Infinity, None, ["D:getetag"]) .await; assert_result(&response, &hierarchy); client.validate_values(&hierarchy).await; // Test 12: Copy a file under a different container let file3_path = format!("{user_base_path}/folder1/file3"); let folder3_file3_from_folder1 = format!("{user_base_path}/folder3/file3_from_folder1"); client .request_with_headers( "COPY", &file3_path, [("destination", folder3_file3_from_folder1.as_str())], "", ) .await .with_status(StatusCode::CREATED); copy(&mut hierarchy, &file3_path, &folder3_file3_from_folder1); let response = client .sync_collection(&user_base_path, "", Depth::Infinity, None, ["D:getetag"]) .await; assert_result(&response, &hierarchy); client.validate_values(&hierarchy).await; // Test 12: Copy and overwrite a file under a different container let folder2_file2 = format!("{user_base_path}/folder2/file2"); client .request_with_headers( "COPY", &folder3_file3_from_folder1, [("destination", folder2_file2.as_str())], "", ) .await .with_status(StatusCode::NO_CONTENT); delete(&mut hierarchy, &folder2_file2); copy(&mut hierarchy, &folder3_file3_from_folder1, &folder2_file2); let response = client .sync_collection(&user_base_path, "", Depth::Infinity, None, ["D:getetag"]) .await; assert_result(&response, &hierarchy); client.validate_values(&hierarchy).await; // Test 13: Copy and move files to a shared container let shared_hierarchy_root = format!("{group_base_path}/Test_Child_Folder/"); let folder3_file1 = format!("{user_base_path}/folder3/file1"); let shared_file_1 = format!("{shared_hierarchy_root}shared_file_1"); let shared_file_2 = format!("{shared_hierarchy_root}shared_file_2"); client .mkcol("MKCOL", &shared_hierarchy_root, [], []) .await .with_status(StatusCode::CREATED); client .request_with_headers( "MOVE", &folder3_file1, [("destination", shared_file_1.as_str())], "", ) .await .with_status(StatusCode::CREATED); client .request_with_headers( "COPY", &folder1_file2, [("destination", shared_file_2.as_str())], "", ) .await .with_status(StatusCode::CREATED); let shared_hierarchy = vec![ (shared_hierarchy_root.clone(), "".to_string()), ( shared_file_1, get_contents(&hierarchy, &folder3_file1).unwrap(), ), ( shared_file_2, get_contents(&hierarchy, &folder1_file2).unwrap(), ), ]; delete(&mut hierarchy, &folder3_file1); let response = client .sync_collection(&user_base_path, "", Depth::Infinity, None, ["D:getetag"]) .await; assert_result(&response, &hierarchy); client.validate_values(&hierarchy).await; let response = client .sync_collection(&group_base_path, "", Depth::Infinity, None, ["D:getetag"]) .await; assert_result(&response, &shared_hierarchy); client.validate_values(&shared_hierarchy).await; client .request("DELETE", &shared_hierarchy_root, "") .await .with_status(StatusCode::NO_CONTENT); if resource_type == DavResourceName::File { // Test 14: Move a container under a different container let folder2 = format!("{user_base_path}/folder2/"); let folder3 = format!("{user_base_path}/folder3/"); let folder2_folder3 = format!("{user_base_path}/folder2/folder3/"); client .request_with_headers( "MOVE", &folder3, [("destination", folder2_folder3.as_str())], "", ) .await .with_status(StatusCode::CREATED); replace_prefix(&mut hierarchy, &folder3, &folder2_folder3); let response = client .sync_collection(&user_base_path, "", Depth::Infinity, None, ["D:getetag"]) .await; assert_result(&response, &hierarchy); client.validate_values(&hierarchy).await; // Test 15: Moving or copying a parent under a child is not allowed for method in ["MOVE", "COPY"] { client .request_with_headers( method, &folder2_folder3, [("destination", folder2.as_str())], "", ) .await .with_status(StatusCode::BAD_GATEWAY); } // Test 16: Copy a container under a different container let folder1 = format!("{user_base_path}/folder1/"); let folder2_folder1 = format!("{user_base_path}/folder2/folder1/"); client .request_with_headers( "COPY", &folder1, [("destination", folder2_folder1.as_str())], "", ) .await .with_status(StatusCode::CREATED); let response = client .sync_collection(&user_base_path, "", Depth::Infinity, None, ["D:getetag"]) .await; copy_prefix(&mut hierarchy, &folder1, &folder2_folder1); assert_result(&response, &hierarchy); client.validate_values(&hierarchy).await; } else { // Test 17: UID collision let folder1 = format!("{user_base_path}/folder1/"); let folder2 = format!("{user_base_path}/folder2/"); let file_contents = resource_type.generate(); for folder_path in [&folder1, &folder2] { let file_path = format!("{folder_path}uid_test"); client .request("PUT", &file_path, file_contents.as_str()) .await .with_status(StatusCode::CREATED); } let uid_file_src = format!("{folder1}uid_test"); let uid_file_dest = format!("{folder2}uid_test_dup"); for method in ["COPY", "MOVE"] { client .request_with_headers( method, &uid_file_src, [("destination", uid_file_dest.as_str())], "", ) .await .with_status(StatusCode::PRECONDITION_FAILED) .with_failed_precondition( if resource_type == DavResourceName::Cal { "A:no-uid-conflict.D:href" } else { "B:no-uid-conflict.D:href" }, &format!("{folder2}uid_test"), ); } } // Delete all containers and create a new one client .request("DELETE", &format!("{user_base_path}/folder3/"), "") .await .with_status(if resource_type == DavResourceName::File { StatusCode::NOT_FOUND } else { StatusCode::NO_CONTENT }); for folder in ["folder1", "folder2"] { let folder_path = format!("{user_base_path}/{folder}/"); client .request("DELETE", &folder_path, "") .await .with_status(StatusCode::NO_CONTENT); } // Create a new test container and file let test_base_path = format!("{user_base_path}/My_Test_Folder/"); client .mkcol("MKCOL", &test_base_path, [], []) .await .with_status(StatusCode::CREATED); let test_contents_1 = resource_type.generate(); let test_contents_2 = resource_type.generate(); let test_file1_path = format!("{test_base_path}test_file_1"); let test_file2_path = format!("{test_base_path}test_file_2"); let test_etag_1 = client .request("PUT", &test_file1_path, test_contents_1.as_str()) .await .with_status(StatusCode::CREATED) .etag() .to_string(); let test_etag_2 = client .request("PUT", &test_file2_path, test_contents_2.as_str()) .await .with_status(StatusCode::CREATED) .etag() .to_string(); // Test 18: Failed DAV preconditions for method in ["COPY", "MOVE"] { client .request_with_headers( method, &test_file1_path, [ ("destination", test_file2_path.as_str()), ("overwrite", "F"), ], "", ) .await .with_status(StatusCode::PRECONDITION_FAILED) .with_empty_body(); client .request_with_headers( method, &test_file1_path, [ ("destination", test_file2_path.as_str()), ("if-none-match", "*"), ], "", ) .await .with_status(StatusCode::PRECONDITION_FAILED) .with_empty_body(); let iff = format!( "<{test_file1_path}> (Not [{test_etag_1}]) <{test_file2_path}> (Not [{test_etag_2}])", ); client .request_with_headers( method, &test_file1_path, [ ("destination", test_file2_path.as_str()), ("if", iff.as_str()), ], "", ) .await .with_status(StatusCode::PRECONDITION_FAILED) .with_empty_body(); } // Test 18: Successful DAV preconditions let iff = format!("<{test_file1_path}> ([{test_etag_1}]) <{test_file2_path}> ([{test_etag_2}])",); client .request_with_headers( "MOVE", &test_file1_path, [ ("destination", test_file2_path.as_str()), ("if", iff.as_str()), ], "", ) .await .with_status(StatusCode::NO_CONTENT); // Delete the test container client .request("DELETE", &test_base_path, "") .await .with_status(StatusCode::NO_CONTENT); // Test 19: Quota enforcement (on CalDAV/CardDAV items are linked, not copied therefore there is no quota increase) if resource_type == DavResourceName::File { let path = format!("{}/mike/quota-test/", resource_type.base_path()); let content = resource_type.generate(); mike_noquota .mkcol("MKCOL", &path, [], []) .await .with_status(StatusCode::CREATED); mike_noquota .request_with_headers("PUT", &format!("{path}file"), [], &content) .await .with_status(StatusCode::CREATED); let mut num_success = 0; let mut did_fail = false; for i in 0..100 { let response = mike_noquota .request_with_headers( "COPY", &path, [( "destination", format!("{}/mike/quota-test{i}", resource_type.base_path()).as_str(), )], &content, ) .await; match response.status { StatusCode::CREATED => { num_success += 1; } StatusCode::PRECONDITION_FAILED => { did_fail = true; break; } _ => panic!("Unexpected status code: {:?}", response.status), } } if !did_fail { panic!("Quota test failed: {} files created", num_success); } if num_success == 0 { panic!("Quota test failed: no files created"); } mike_noquota .request("DELETE", &path, "") .await .with_status(StatusCode::NO_CONTENT); for i in 0..num_success { mike_noquota .request( "DELETE", &format!("{}/mike/quota-test{i}", resource_type.base_path()), "", ) .await .with_status(StatusCode::NO_CONTENT); } } } client.delete_default_containers().await; client.delete_default_containers_by_account("support").await; mike_noquota.delete_default_containers().await; test.assert_is_empty().await; } fn assert_result(response: &DavResponse, hierarchy: &[(String, String)]) { assert!(!hierarchy.is_empty()); let response = response .hrefs() .into_iter() .filter(|h| { !h.ends_with("/jane/") && !h.ends_with("/support/") && !h.ends_with("/default/") }) .collect::>(); let hierarchy = hierarchy .iter() .map(|x| x.0.as_str()) .collect::>(); if hierarchy != response { println!("\nMissing: {:?}", hierarchy.difference(&response)); println!("\nExtra: {:?}", response.difference(&hierarchy)); panic!( "Hierarchy mismatch: expected {} items, received {} items", hierarchy.len(), response.len() ); } } fn replace_prefix(items: &mut [(String, String)], old_prefix: &str, new_prefix: &str) { let mut did_replace = false; for (href, _) in items.iter_mut() { if let Some(value) = href.strip_prefix(old_prefix) { *href = format!("{new_prefix}{value}"); did_replace = true; } } if !did_replace { panic!("Prefix not found: {}", old_prefix); } } fn rename(items: &mut [(String, String)], old_name: &str, new_name: &str) { for (href, _) in items.iter_mut() { if href == old_name { *href = new_name.to_string(); return; } } panic!("Item not found: {}", old_name); } fn delete(items: &mut Vec<(String, String)>, name: &str) { let mut did_delete = false; items.retain(|(href, _)| { did_delete = did_delete || href == name; href != name }); if !did_delete { panic!("Item not found: {}", name); } } fn copy(items: &mut Vec<(String, String)>, old_name: &str, new_name: &str) { for (href, contents) in items.iter_mut() { if href == old_name { let value = (new_name.to_string(), contents.to_string()); items.push(value); return; } } panic!("Item not found: {}", old_name); } fn copy_prefix(items: &mut Vec<(String, String)>, old_prefix: &str, new_prefix: &str) { let mut new_items = vec![]; for (href, contents) in items.iter() { if let Some(value) = href.strip_prefix(old_prefix) { new_items.push((format!("{new_prefix}{value}"), contents.to_string())); } } if !new_items.is_empty() { items.extend(new_items); } else { panic!("Prefix not found: {}", old_prefix); } } fn get_contents(items: &[(String, String)], name: &str) -> Option { for (href, contents) in items.iter() { if href == name { return Some(contents.to_string()); } } None } ================================================ FILE: tests/src/webdav/lock.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{DavResponse, DummyWebDavClient, WebDavTest}; use crate::webdav::GenerateTestDavResource; use dav_proto::schema::property::{DavProperty, WebDavProperty}; use groupware::DavResourceName; use hyper::StatusCode; pub async fn test(test: &WebDavTest) { let client = test.client("john"); for resource_type in [ DavResourceName::File, DavResourceName::Cal, DavResourceName::Card, ] { println!( "Running LOCK/UNLOCK tests ({})...", resource_type.base_path() ); let base_path = format!("{}/john", resource_type.base_path()); // Test 1: Creating a collection under an unmapped resource without providing a lock token should fail let path = format!("{base_path}/do-not-write"); let response = client .lock_create(&path, "super-owner", true, "infinity", "Second-123") .await .with_status(StatusCode::CREATED); let lock_token = response .with_value( "D:prop.D:lockdiscovery.D:activelock.D:owner.href", "super-owner", ) .with_value("D:prop.D:lockdiscovery.D:activelock.D:depth", "infinity") .with_value( "D:prop.D:lockdiscovery.D:activelock.D:timeout", "Second-123", ) .lock_token() .to_string(); // Test 2: Refreshing a lock token with an invalid a lock token should fail client .lock_refresh(&path, "urn:stalwart:davlock:1234", "infinity", "Second-456") .await .with_status(StatusCode::PRECONDITION_FAILED); // Test 3: Refreshing a lock token with valid a lock token should succeed client .lock_refresh(&path, &lock_token, "infinity", "Second-456") .await .with_status(StatusCode::OK) .with_value( "D:prop.D:lockdiscovery.D:activelock.D:owner.href", "super-owner", ) .with_any_value( "D:prop.D:lockdiscovery.D:activelock.D:timeout", ["Second-456", "Second-455"], ); // Test 3: Creating a collection under an unmapped resource with a lock token should fail client .request_with_headers("MKCOL", &path, [], "") .await .with_status(StatusCode::LOCKED) .with_value("D:error.D:lock-token-submitted.D:href", &path); // Test 4: Creating a collection under a mapped resource with a lock token should succeed client .request_with_headers( "MKCOL", &path, [("if", format!("(<{lock_token}>)").as_str())], "", ) .await .with_status(StatusCode::CREATED); // Test 5: Creating a lock under an infinity locked resource should fail let file_path = format!("{path}/file.txt"); client .lock_create(&file_path, "super-owner", true, "0", "Second-123") .await .with_status(StatusCode::LOCKED) .with_value("D:error.D:lock-token-submitted.D:href", &path); // Test 6: Creating a file under a locked resource without a lock token should fail let contents = resource_type.generate(); client .request("PUT", &file_path, &contents) .await .with_status(StatusCode::LOCKED) .with_value("D:error.D:lock-token-submitted.D:href", &path); // Test 7: Creating a file under a locked resource with a lock token should succeed client .request_with_headers( "PUT", &file_path, [("if", format!("(<{lock_token}>)").as_str())], &contents, ) .await .with_status(StatusCode::CREATED); // Test 8: Locks should be included in propfind responses let response = client .propfind(&path, [DavProperty::WebDav(WebDavProperty::LockDiscovery)]) .await; for href in [path.clone() + "/", file_path] { let props = response.properties(&href); props .get(DavProperty::WebDav(WebDavProperty::LockDiscovery)) .with_some_values([ "D:activelock.D:owner.href:super-owner", "D:activelock.D:depth:infinity", format!("D:activelock.D:locktoken.D:href:{lock_token}").as_str(), format!("D:activelock.D:lockroot.D:href:{path}").as_str(), "D:activelock.D:locktype.D:write", "D:activelock.D:lockscope.D:exclusive", ]) .with_any_values([ "D:activelock.D:timeout:Second-456", "D:activelock.D:timeout:Second-455", ]); } // Test 9: Delete with and without a lock token client .request("DELETE", &path, "") .await .with_status(StatusCode::LOCKED) .with_value("D:error.D:lock-token-submitted.D:href", &path); client .request_with_headers( "DELETE", &path, [("if", format!("(<{lock_token}>)").as_str())], "", ) .await .with_status(StatusCode::NO_CONTENT); // Test 10: Unlock with and without a lock token client .unlock(&path, "urn:stalwart:davlock:1234") .await .with_status(StatusCode::CONFLICT) .with_value("D:error.D:lock-token-matches-request-uri", ""); client .unlock(&path, &lock_token) .await .with_status(StatusCode::NO_CONTENT); // Test 11: Locking with a large dead property should fail let path = format!("{base_path}/invalid-lock"); client .lock_create( &path, (0..=test.server.core.groupware.dead_property_size.unwrap() + 1) .map(|_| "a") .collect::() .as_str(), true, "infinity", "Second-123", ) .await .with_status(StatusCode::PAYLOAD_TOO_LARGE); // Test 12: Too many locks should fail for i in 0..test.server.core.groupware.max_locks_per_user { client .lock_create( &format!("{base_path}/invalid-lock-{i}"), "super-owner", true, "infinity", "Second-123", ) .await .with_status(StatusCode::CREATED); } client .lock_create( &format!("{base_path}/invalid-lock-greedy"), "super-owner", true, "infinity", "Second-123", ) .await .with_status(StatusCode::TOO_MANY_REQUESTS); } client.delete_default_containers().await; test.assert_is_empty().await; } const LOCK_REQUEST: &str = r#" $OWNER "#; impl DummyWebDavClient { pub async fn lock_create( &self, path: &str, owner: &str, is_exclusive: bool, depth: &str, timeout: &str, ) -> DavResponse { let lock_request = LOCK_REQUEST .replace("$TYPE", if is_exclusive { "exclusive" } else { "shared" }) .replace("$OWNER", owner); self.request_with_headers( "LOCK", path, [("depth", depth), ("timeout", timeout)], &lock_request, ) .await } pub async fn lock_refresh( &self, path: &str, lock_token: &str, depth: &str, timeout: &str, ) -> DavResponse { let condition = format!("(<{lock_token}>)"); self.request_with_headers( "LOCK", path, [ ("if", condition.as_str()), ("depth", depth), ("timeout", timeout), ], "", ) .await } pub async fn unlock(&self, path: &str, lock_token: &str) -> DavResponse { let condition = format!("<{lock_token}>"); self.request_with_headers("UNLOCK", path, [("lock-token", condition.as_str())], "") .await } } impl DavResponse { pub fn lock_token(&self) -> &str { self.value("D:prop.D:lockdiscovery.D:activelock.D:locktoken.D:href") } } ================================================ FILE: tests/src/webdav/mkcol.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use hyper::StatusCode; use crate::webdav::{TEST_FILE_1, TEST_ICAL_1, TEST_VCARD_1, TEST_VTIMEZONE_1}; use super::{DavResponse, DummyWebDavClient, WebDavTest}; pub async fn test(test: &WebDavTest) { println!("Running MKCOL tests..."); let client = test.client("john"); // Creating collections in root elements is not allowed for path in [ "/dav/file/test", "/dav/card/test", "/dav/cal/test", "/dav/test", ] { client .request("MKCOL", path, "") .await .with_status(StatusCode::NOT_FOUND); } // Create collections using MKCOL (empty body) for path in [ "/dav/file/john/my-files", "/dav/card/john/my-cards", "/dav/cal/john/my-events", ] { client .request("MKCOL", path, "") .await .with_status(StatusCode::CREATED); } // Create resources under the newly created collections for (path, content) in [ ("/dav/file/john/my-files/file1.txt", TEST_FILE_1), ("/dav/card/john/my-cards/card1.vcf", TEST_VCARD_1), ("/dav/cal/john/my-events/event1.ics", TEST_ICAL_1), ] { client .request("PUT", path, content) .await .with_status(StatusCode::CREATED); } // Creating a collection on a mapped resource should fail for path in [ "/dav/file/john/my-files", "/dav/card/john/my-cards", "/dav/cal/john/my-events", "/dav/file/john/my-files/file1.txt", "/dav/card/john/my-cards/card1.vcf", "/dav/cal/john/my-events/event1.ics", ] { client .request("MKCOL", path, "") .await .with_status(StatusCode::METHOD_NOT_ALLOWED); } // Creating a sub-collections is allowed in FileDAV but in CalDAV and CardDAV for (path, expected_status) in [ ("/dav/file/john/my-files/my-sub-files", StatusCode::CREATED), ( "/dav/card/john/my-cards/my-sub-cards", StatusCode::METHOD_NOT_ALLOWED, ), ( "/dav/cal/john/my-events/my-sub-events", StatusCode::METHOD_NOT_ALLOWED, ), ] { client .request("MKCOL", path, "") .await .with_status(expected_status); } // Extended MKCOL with an unsupported resource types should fail for (path, resource_type) in [ ("/dav/file/john/my-named-files", "B:addressbook"), ("/dav/card/john/my-named-cards", "A:calendar"), ("/dav/cal/john/my-named-events", "B:addressbook"), ] { client .mkcol("MKCOL", path, ["D:collection", resource_type], []) .await .with_status(StatusCode::FORBIDDEN) .with_value( "D:mkcol-response.D:propstat.D:error.D:valid-resourcetype", "", ) .with_value("D:mkcol-response.D:propstat.D:prop.D:resourcetype", ""); } // Create using extended MKCOL for (path, expected_properties, resource_types) in [ ( "/dav/file/john/my-named-files/", [("D:displayname", "Named Files")].as_slice(), ["D:collection"].as_slice(), ), ( "/dav/card/john/my-named-cards/", [ ("D:displayname", "Named Cards"), ("B:addressbook-description", "Some amazing contacts"), ] .as_slice(), ["D:collection", "B:addressbook"].as_slice(), ), ( "/dav/cal/john/my-named-events/", [ ("D:displayname", "Named Events"), ("A:calendar-description", "Some amazing events"), ( "A:calendar-timezone", &TEST_VTIMEZONE_1.replace("\n", "\r\n"), ), ] .as_slice(), ["D:collection", "A:calendar"].as_slice(), ), ] { let response = client .mkcol( "MKCOL", path, resource_types.iter().copied(), expected_properties.iter().copied(), ) .await .with_status(StatusCode::CREATED) .into_propfind_response("D:mkcol-response".into()); let properties = response.properties(""); for (property, _) in expected_properties { properties .get(property) .with_status(StatusCode::OK) .with_values([""]); } // Check the properties of the created collection let response = client .propfind(path, expected_properties.iter().map(|x| x.0)) .await; let properties = response.properties(path); for (property, value) in expected_properties { properties .get(property) .with_status(StatusCode::OK) .with_values([*value]); } } // Test MKCALENDAR client .mkcol( "MKCALENDAR", "/dav/cal/john/my-named-events2", [], [ ("D:displayname", "Named Events 2"), ("A:calendar-description", ""), ], ) .await .with_status(StatusCode::CREATED) .with_value("A:mkcalendar-response.D:propstat.D:prop.D:displayname", "") .with_values( "A:mkcalendar-response.D:propstat.D:status", ["HTTP/1.1 200 OK"], ); client .mkcol( "MKCALENDAR", "/dav/cal/john/my-named-events3", [], [ ("D:displayname", "Named Events 3"), ( "A:supported-calendar-component-set", "", ), ], ) .await .with_status(StatusCode::CREATED) .with_value("A:mkcalendar-response.D:propstat.D:prop.D:displayname", "") .with_values( "A:mkcalendar-response.D:propstat.D:status", ["HTTP/1.1 200 OK"], ); // Check the properties of the created calendars client .propfind( "/dav/cal/john/my-named-events2/", ["A:supported-calendar-component-set"], ) .await .properties("/dav/cal/john/my-named-events2/") .get("A:supported-calendar-component-set") .with_status(StatusCode::OK) .with_values([ "A:comp.[name]:VJOURNAL", "A:comp.[name]:VTIMEZONE", "A:comp.[name]:VAVAILABILITY", "A:comp.[name]:VALARM", "A:comp.[name]:VRESOURCE", "A:comp.[name]:AVAILABLE", "A:comp.[name]:VTODO", "A:comp.[name]:VFREEBUSY", "A:comp.[name]:VEVENT", "A:comp.[name]:STANDARD", "A:comp.[name]:DAYLIGHT", "A:comp.[name]:VLOCATION", "A:comp.[name]:PARTICIPANT", ]); client .propfind( "/dav/cal/john/my-named-events3/", ["A:supported-calendar-component-set"], ) .await .properties("/dav/cal/john/my-named-events3/") .get("A:supported-calendar-component-set") .with_status(StatusCode::OK) .with_values(["A:comp.[name]:VEVENT", "A:comp.[name]:VTODO"]); // Delete everything for path in [ "/dav/file/john/my-files", "/dav/card/john/my-cards", "/dav/cal/john/my-events", "/dav/file/john/my-named-files", "/dav/card/john/my-named-cards", "/dav/cal/john/my-named-events", "/dav/cal/john/my-named-events2", "/dav/cal/john/my-named-events3", ] { client .request("DELETE", path, "") .await .with_status(StatusCode::NO_CONTENT); } client.delete_default_containers().await; test.assert_is_empty().await; } impl DummyWebDavClient { pub async fn mkcol( &self, method: &str, path: &str, resource_types: impl IntoIterator, properties: impl IntoIterator, ) -> DavResponse { let mut request = concat!( "", "", "" ) .to_string(); let mut has_resource_type = false; for (idx, resource_type) in resource_types.into_iter().enumerate() { if idx == 0 { request.push_str(""); } request.push_str(&format!("<{resource_type}/>")); has_resource_type = true; } if has_resource_type { request.push_str(""); } for (key, value) in properties { request.push_str(&format!("<{key}>{value}")); } request.push_str(""); if method == "MKCALENDAR" { request = request.replace("D:mkcol", "A:mkcalendar"); } self.request(method, path, &request).await } } ================================================ FILE: tests/src/webdav/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ AssertConfig, TEST_USERS, add_test_certs, directory::internal::TestInternalDirectory, jmap::{assert_is_empty, wait_for_index}, store::{ TempDir, build_store_config, cleanup::{search_store_destroy, store_destroy}, }, }; use ::managesieve::core::ManageSieveSessionManager; use ::store::Stores; use ahash::{AHashMap, AHashSet}; use base64::{Engine, engine::general_purpose::STANDARD}; use common::{ Caches, Core, Data, DavResource, DavResources, Inner, Server, config::{ server::{Listeners, ServerProtocol}, telemetry::Telemetry, }, core::BuildServer, manager::boot::build_ipc, }; use dav_proto::{ schema::property::{DavProperty, WebDavProperty}, xml_pretty_print, }; use directory::Permission; use email::message::metadata::MessageMetadata; use groupware::{DavResourceName, cache::GroupwareCache}; use http::HttpSessionManager; use hyper::{HeaderMap, Method, StatusCode, header::AUTHORIZATION}; use imap::core::ImapSessionManager; use pop3::Pop3SessionManager; use quick_xml::Reader; use quick_xml::events::Event; use services::SpawnServices; use smtp::{SpawnQueueManager, core::SmtpSessionManager}; use std::{borrow::Cow, str}; use std::{ sync::Arc, time::{Duration, Instant}, }; use store::{ ValueKey, rand::{Rng, distr::Alphanumeric, rng}, write::{AlignedBytes, Archive}, }; use tokio::sync::watch; use types::{collection::Collection, field::EmailField}; use utils::config::Config; pub mod acl; pub mod basic; pub mod cal_alarm; pub mod cal_itip; pub mod cal_query; pub mod cal_scheduling; pub mod card_query; pub mod copy_move; pub mod lock; pub mod mkcol; pub mod multiget; pub mod principals; pub mod prop; pub mod put_get; pub mod sync; #[test] fn webdav_tests() { //test_build_itip_templates(&handle.server).await; tokio::runtime::Builder::new_multi_thread() .thread_stack_size(8 * 1024 * 1024) // 8MB stack .enable_all() .build() .unwrap() .block_on(async { // Prepare settings let assisted_discovery = std::env::var("ASSISTED_DISCOVERY").unwrap_or_default() == "1"; let start_time = Instant::now(); let delete = true; let handle = init_webdav_tests(assisted_discovery, delete).await; basic::test(&handle).await; put_get::test(&handle).await; mkcol::test(&handle).await; copy_move::test(&handle, assisted_discovery).await; prop::test(&handle, assisted_discovery).await; multiget::test(&handle).await; sync::test(&handle).await; lock::test(&handle).await; principals::test(&handle, assisted_discovery).await; acl::test(&handle).await; card_query::test(&handle).await; cal_query::test(&handle).await; cal_alarm::test(&handle).await; cal_itip::test(); cal_scheduling::test(&handle).await; // Print elapsed time let elapsed = start_time.elapsed(); println!( "Elapsed: {}.{:03}s", elapsed.as_secs(), elapsed.subsec_millis() ); // Remove test data if delete { handle.temp_dir.delete(); } }); } #[allow(dead_code)] pub struct WebDavTest { server: Server, clients: AHashMap<&'static str, DummyWebDavClient>, temp_dir: TempDir, shutdown_tx: watch::Sender, } async fn init_webdav_tests(assisted_discovery: bool, delete_if_exists: bool) -> WebDavTest { // Load and parse config let temp_dir = TempDir::new("webdav_tests", delete_if_exists); let mut config = Config::new( add_test_certs(&(build_store_config(&temp_dir.path.to_string_lossy()) + SERVER)) .replace("{TMP}", &temp_dir.path.display().to_string()) .replace("{ASSISTED_DISCOVERY}", &assisted_discovery.to_string()) .replace( "{LEVEL}", &std::env::var("LOG").unwrap_or_else(|_| "disable".to_string()), ), ) .unwrap(); config.resolve_all_macros().await; // Parse servers let mut servers = Listeners::parse(&mut config); // Bind ports and drop privileges servers.bind_and_drop_priv(&mut config); // Build stores let stores = Stores::parse_all(&mut config, false).await; // Parse core let tracers = Telemetry::parse(&mut config, &stores); let core = Core::parse(&mut config, stores, Default::default()).await; let data = Data::parse(&mut config); let cache = Caches::parse(&mut config); let store = core.storage.data.clone(); let search_store = core.storage.fts.clone(); let (ipc, mut ipc_rxs) = build_ipc(false); let inner = Arc::new(Inner { shared_core: core.into_shared(), data, ipc, cache, }); // Parse acceptors servers.parse_tcp_acceptors(&mut config, inner.clone()); // Enable tracing tracers.enable(true); // Start services config.assert_no_errors(); ipc_rxs.spawn_queue_manager(inner.clone()); ipc_rxs.spawn_services(inner.clone()); // Spawn servers let (shutdown_tx, _) = servers.spawn(|server, acceptor, shutdown_rx| { match &server.protocol { ServerProtocol::Smtp | ServerProtocol::Lmtp => server.spawn( SmtpSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Http => server.spawn( HttpSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Imap => server.spawn( ImapSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::Pop3 => server.spawn( Pop3SessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), ServerProtocol::ManageSieve => server.spawn( ManageSieveSessionManager::new(inner.clone()), inner.clone(), acceptor, shutdown_rx, ), }; }); if delete_if_exists { store_destroy(&store).await; search_store_destroy(&search_store).await; } // Create test accounts let mut clients = AHashMap::new(); for (account, secret, name, email) in TEST_USERS { let account_id = store .create_test_user(account, secret, name, &[email]) .await; clients.insert( *account, DummyWebDavClient::new(account_id, account, secret, email), ); store .add_permissions( account, [Permission::DavPrincipalList, Permission::DavPrincipalSearch], ) .await; if *account == "mike" { store.set_test_quota(account, 1024).await; } } store .create_test_group("support", "Support Group", &["support@example.com"]) .await; store.add_to_group("jane", "support").await; WebDavTest { server: inner.build_server(), clients, temp_dir, shutdown_tx, } } impl WebDavTest { pub fn client(&self, name: &'static str) -> &DummyWebDavClient { self.clients.get(name).unwrap() } pub async fn resources(&self, name: &'static str, collection: Collection) -> Arc { let account_id = self.client(name).account_id; let access_token = self.server.get_access_token(account_id).await.unwrap(); self.server .fetch_dav_resources(&access_token, account_id, collection.into()) .await .unwrap() } pub fn clear_cache(&self) { for cache in [ &self.server.inner.cache.events, &self.server.inner.cache.contacts, &self.server.inner.cache.files, ] { cache.clear(); } } pub async fn assert_is_empty(&self) { assert_is_empty(&self.server).await; self.clear_cache(); } pub async fn wait_for_index(&self) { wait_for_index(&self.server).await; } } #[allow(dead_code)] #[derive(Debug)] pub struct DummyWebDavClient { account_id: u32, name: &'static str, email: &'static str, credentials: String, } #[derive(Debug)] pub struct DavResponse { headers: AHashMap, status: StatusCode, body: Result, xml: Vec<(String, String)>, } impl DummyWebDavClient { pub fn new( account_id: u32, name: &'static str, secret: &'static str, email: &'static str, ) -> Self { Self { account_id, name, email, credentials: format!( "Basic {}", STANDARD.encode(format!("{name}:{secret}").as_bytes()) ), } } pub async fn request(&self, method: &str, query: &str, body: impl Into) -> DavResponse { self.request_with_headers(method, query, [].into_iter(), body) .await } pub async fn request_with_headers( &self, method: &str, query: &str, headers: impl IntoIterator, body: impl Into, ) -> DavResponse { let mut request = reqwest::Client::builder() .timeout(Duration::from_millis(500)) .danger_accept_invalid_certs(true) .build() .unwrap() .request( Method::from_bytes(method.as_bytes()).unwrap(), format!("https://127.0.0.1:8899{query}"), ); let body = body.into(); if !body.is_empty() { request = request.body(body); } let mut request_headers = HeaderMap::new(); for (key, value) in headers { request_headers.insert(key, value.parse().unwrap()); } request_headers.insert(AUTHORIZATION, self.credentials.parse().unwrap()); let response = request.headers(request_headers).send().await.unwrap(); let status = response.status(); let headers = response .headers() .iter() .map(|(k, v)| { ( k.to_string().to_lowercase(), v.to_str().unwrap().to_string(), ) }) .collect(); let body = response .bytes() .await .map(|bytes| String::from_utf8(bytes.to_vec()).unwrap()) .map_err(|err| err.to_string()); let xml = match &body { Ok(body) if body.starts_with(" flatten_xml(body), _ => vec![], }; DavResponse { headers, status, body, xml, } } pub async fn available_quota(&self, path: &str) -> u64 { self.propfind( path, [DavProperty::WebDav(WebDavProperty::QuotaAvailableBytes)], ) .await .properties(path) .get(DavProperty::WebDav(WebDavProperty::QuotaAvailableBytes)) .value() .parse() .unwrap() } pub async fn create_hierarchy( &self, base_path: &str, max_depth: usize, containers_per_level: usize, files_per_container: usize, ) -> (String, Vec<(String, String)>) { let resource_type = if base_path.starts_with("/dav/card/") { DavResourceName::Card } else if base_path.starts_with("/dav/cal/") { DavResourceName::Cal } else { DavResourceName::File }; let mut created_resources = Vec::new(); self.create_hierarchy_recursive( resource_type, base_path, max_depth, containers_per_level, files_per_container, 0, &mut created_resources, ) .await; let root_folder = created_resources.first().unwrap().0.clone(); created_resources.sort_unstable_by(|a, b| a.0.cmp(&b.0)); (root_folder, created_resources) } #[allow(clippy::too_many_arguments)] async fn create_hierarchy_recursive( &self, resource_type: DavResourceName, base_path: &str, max_depth: usize, containers_per_level: usize, files_per_container: usize, current_depth: usize, created_resources: &mut Vec<(String, String)>, ) { let folder_name = generate_random_name(4); let folder_path = format!("{base_path}/Folder_{folder_name}"); self.mkcol("MKCOL", &folder_path, [], []) .await .with_status(StatusCode::CREATED); created_resources.push((format!("{folder_path}/"), "".to_string())); for _ in 0..files_per_container { let file_name = generate_random_name(8); let file_path = format!( "{folder_path}/{file_name}.{}", match resource_type { DavResourceName::Card => "vcf", DavResourceName::Cal => "ics", DavResourceName::File => "txt", _ => unreachable!(), } ); let content = match resource_type { DavResourceName::Card => generate_random_vcard(), DavResourceName::Cal => generate_random_ical(), DavResourceName::File => generate_random_content(100, 500), _ => unreachable!(), }; self.request("PUT", &file_path, &content) .await .with_status(StatusCode::CREATED); created_resources.push((file_path, content)); } if current_depth < max_depth { for _ in 0..containers_per_level { Box::pin(self.create_hierarchy_recursive( resource_type, &folder_path, max_depth, containers_per_level, files_per_container, current_depth + 1, created_resources, )) .await; } } } pub async fn validate_values(&self, items: &[(String, String)]) { for (path, value) in items { if !path.ends_with('/') { self.request("GET", path, "") .await .with_status(StatusCode::OK) .with_body(value); } } } pub async fn delete_default_containers(&self) { self.delete_default_containers_by_account(self.name).await; } pub async fn delete_default_containers_by_account(&self, account: &str) { for col in ["card", "cal"] { self.request("DELETE", &format!("/dav/{col}/{account}/default"), "") .await .with_status(StatusCode::NO_CONTENT); } } } impl DavResponse { pub fn with_status(self, status: StatusCode) -> Self { if self.status != status { self.dump_response(); panic!("Expected {status} but got {}", self.status) } self } pub fn with_redirect_to(self, url: &str) -> Self { self.with_status(StatusCode::TEMPORARY_REDIRECT) .with_header("location", url) } pub fn with_header(self, header: &str, value: &str) -> Self { if self.headers.get(header).is_some_and(|v| v == value) { self } else { self.dump_response(); panic!("Header {header}:{value} not found.") } } pub fn with_body(self, expect_body: impl AsRef) -> Self { let expect_body = expect_body.as_ref(); if self.body.is_ok() { let body = self.body.as_ref().unwrap(); if body != expect_body { self.dump_response(); assert_eq!(body, &expect_body); } self } else { self.dump_response(); panic!("Expected body {expect_body:?} but no body was returned.") } } pub fn with_empty_body(self) -> Self { if self.body.is_ok() { let body = self.body.as_ref().unwrap(); if !body.is_empty() { self.dump_response(); panic!("Expected empty body but got {body:?}"); } self } else { self.dump_response(); panic!("Expected empty body but no body was returned.") } } pub fn expect_body(&self) -> &str { if self.body.is_ok() { self.body.as_ref().unwrap() } else { self.dump_response(); panic!("Expected body but no body was returned.") } } pub fn header(&self, header: &str) -> &str { if let Some(value) = self.headers.get(header) { value } else { self.dump_response(); panic!("Header {header} not found.") } } pub fn etag(&self) -> &str { self.header("etag") } pub fn sync_token(&self) -> &str { self.find_keys("D:multistatus.D:sync-token") .next() .filter(|v| !v.is_empty()) .unwrap_or_else(|| { self.dump_response(); panic!("Sync token not found.") }) } pub fn hrefs(&self) -> Vec<&str> { let mut hrefs = self .find_keys("D:multistatus.D:response.D:href") .collect::>(); hrefs.sort_unstable(); hrefs } pub fn with_href_count(self, count: usize) -> Self { let href_count = self.find_keys("D:multistatus.D:response.D:href").count(); if href_count != count { self.dump_response(); panic!("Expected {} hrefs but got {}", count, href_count); } self } pub fn with_hrefs<'x>(self, hrefs: impl IntoIterator) -> Self { let expected_hrefs = hrefs.into_iter().collect::>(); let hrefs = self .find_keys("D:multistatus.D:response.D:href") .collect::>(); if expected_hrefs != hrefs { self.dump_response(); println!("\nMissing: {:?}", expected_hrefs.difference(&hrefs)); println!("\nExtra: {:?}", hrefs.difference(&expected_hrefs)); panic!( "Hierarchy mismatch: expected {} items, received {} items", expected_hrefs.len(), hrefs.len() ); } self } fn dump_response(&self) { eprintln!("-------------------------------------"); eprintln!("Status: {}", self.status); eprintln!("Headers:"); for (key, value) in self.headers.iter() { eprintln!(" {}: {:?}", key, value); } if !self.xml.is_empty() { eprintln!("XML: {}", xml_pretty_print(self.body.as_ref().unwrap())); for (key, value) in self.xml.iter() { eprintln!("{} -> {:?}", key, value); } } else { eprintln!("Body: {:?}", self.body); } } fn find_keys(&self, name: &str) -> impl Iterator { self.xml .iter() .filter(move |(key, _)| name == key) .map(|(_, value)| value.as_str()) } pub fn value(&self, name: &str) -> &str { self.find_keys(name).next().unwrap_or_else(|| { self.dump_response(); panic!("Key {name} not found.") }) } // Poor man's XPath pub fn with_value(self, query: &str, expect: impl AsRef) -> Self { let expect = expect.as_ref(); if let Some(value) = self.find_keys(query).next() { if value != expect { self.dump_response(); panic!("Expected {query} = {expect:?} but got {value:?}"); } } else { self.dump_response(); panic!("Key {query} not found."); } self } pub fn with_any_value<'x>( self, query: &str, expect: impl IntoIterator, ) -> Self { let expect = expect.into_iter().collect::>(); if let Some(value) = self.find_keys(query).next() { if !expect.contains(value) { self.dump_response(); panic!("Expected {query} = {expect:?} but got {value:?}"); } } else { self.dump_response(); panic!("Key {query} not found."); } self } pub fn with_values(self, query: &str, expect: I) -> Self where I: IntoIterator, T: AsRef, { let expect_owned: Vec = expect.into_iter().collect(); let expect = expect_owned.iter().map(|s| s.as_ref()).collect::>(); let found = self.find_keys(query).collect::>(); if expect != found { self.dump_response(); panic!("Expected {query} = {expect:?} but got {found:?}"); } self } pub fn with_failed_precondition(self, precondition: &str, value: &str) -> Self { let error = format!("D:error.{precondition}"); if self.find_keys(&error).next().is_none_or(|v| v != value) { self.dump_response(); panic!("Precondition {precondition} did not match."); } self } } pub trait DavResourcesTest { fn items(&self) -> Vec; } impl DavResourcesTest for DavResources { fn items(&self) -> Vec { self.resources.clone() } } fn flatten_xml(xml: &str) -> Vec<(String, String)> { let mut reader = Reader::from_str(xml); let mut path: Vec = Vec::new(); let mut result: Vec<(String, String)> = Vec::new(); let mut buf = Vec::new(); let mut text_content: Option = None; loop { match reader.read_event_into(&mut buf).unwrap() { Event::Start(ref e) => { let name = str::from_utf8(e.name().as_ref()).unwrap().to_string(); path.push(name); let base_path = path.join("."); for attr in e.attributes() { let attr = attr.unwrap(); let key = str::from_utf8(attr.key.as_ref()).unwrap().to_string(); let value = attr.unescape_value().unwrap(); let value_str = value.trim().to_string(); result.push((format!("{}.[{}]", base_path, key), value_str)); } text_content = None; } Event::Empty(ref e) => { let name = str::from_utf8(e.name().as_ref()).unwrap().to_string(); let base_path = format!("{}.{}", path.join("."), name); let mut has_attrs = false; for attr in e.attributes() { let attr = attr.unwrap(); let key = str::from_utf8(attr.key.as_ref()).unwrap().to_string(); let value = attr.unescape_value().unwrap(); let value_str = value.trim().to_string(); has_attrs = true; result.push((format!("{}.[{}]", base_path, key), value_str)); } if !has_attrs { result.push((base_path, "".to_string())); } } Event::Text(e) => { let text = e.xml_content().unwrap(); let trimmed = text.trim(); if !trimmed.is_empty() { if let Some(text_content) = text_content.as_mut() { text_content.push_str(trimmed); } else { text_content = Some(trimmed.to_string()); } } } Event::GeneralRef(entity) => { let value: Cow = match entity.as_ref() { b"lt" => "<".into(), b"gt" => ">".into(), b"amp" => "&".into(), b"apos" => "'".into(), b"quot" => "\"".into(), _ => { if let Ok(Some(gr)) = entity.resolve_char_ref() { gr.to_string().into() } else { std::str::from_utf8(entity.as_ref()) .unwrap_or_default() .into() } } }; if let Some(text_content) = text_content.as_mut() { text_content.push_str(value.as_ref()); } else { text_content = Some(value.into_owned()); } } Event::CData(e) => { text_content = Some(std::str::from_utf8(e.as_ref()).unwrap().to_string()); } Event::End(_) => { if let Some(text) = text_content.take() { result.push((path.join("."), text)); } if !path.is_empty() { path.pop(); } } Event::Eof => break, _ => {} } buf.clear(); } result } pub const TEST_VCARD_1: &str = r#"BEGIN:VCARD VERSION:4.0 UID:18F098B5-7383-4FD6-B482-48F2181D73AA X-TEST:SEQ1 N:Coyote;Wile;E.;; FN:Wile E. Coyote ORG:ACME Inc.; END:VCARD "#; pub const TEST_VCARD_2: &str = r#"BEGIN:VCARD VERSION:4.0 UID:6exhjr32bt783wwlr9u0sr8lfqse5x7zqc8y X-TEST:SEQ1 FN:Joe Citizen N:Citizen;Joe;;; NICKNAME:human_being EMAIL;TYPE=pref:jcitizen@foo.com REV:20200411T072429Z END:VCARD "#; pub const TEST_ICAL_1: &str = r#"BEGIN:VCALENDAR SOURCE;VALUE=URI:http://calendar.example.com/event_with_html.ics X-TEST:SEQ1 BEGIN:VEVENT UID: 2371c2d9-a136-43b0-bba3-f6ab249ad46e SUMMARY:What a nice present: 🎁 DTSTART;TZID=America/New_York:20190221T170000 DTEND;TZID=America/New_York:20190221T180000 LOCATION:Germany DESCRIPTION:

Title

  • first Row
  • < i>second Row

END:VEVENT END:VCALENDAR "#; pub const TEST_ICAL_2: &str = r#"BEGIN:VCALENDAR X-TEST:SEQ1 BEGIN:VEVENT UID:0000001 SUMMARY:Treasure Hunting DTSTART;TZID=America/Los_Angeles:20150706T120000 DTEND;TZID=America/Los_Angeles:20150706T130000 RRULE:FREQ=DAILY;COUNT=10 EXDATE;TZID=America/Los_Angeles:20150708T120000 EXDATE;TZID=America/Los_Angeles:20150710T120000 END:VEVENT BEGIN:VEVENT UID:0000001 SUMMARY:More Treasure Hunting LOCATION:The other island DTSTART;TZID=America/Los_Angeles:20150709T150000 DTEND;TZID=America/Los_Angeles:20150707T160000 RECURRENCE-ID;TZID=America/Los_Angeles:20150707T120000 END:VEVENT END:VCALENDAR "#; pub const TEST_FILE_1: &str = r#"this is a test file with some text and some more text X-TEST:SEQ1 "#; pub const TEST_FILE_2: &str = r#"another test file with amazing content and some more text X-TEST:SEQ1 "#; pub const TEST_VTIMEZONE_1: &str = r#"BEGIN:VCALENDAR PRODID:-//Example Corp.//CalDAV Client//EN VERSION:2.0 BEGIN:VTIMEZONE TZID:US-Eastern LAST-MODIFIED:19870101T000000Z BEGIN:STANDARD DTSTART:19671029T020000 RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 TZOFFSETFROM:-0400 TZOFFSETTO:-0500 TZNAME:Eastern Standard Time (US Canada) END:STANDARD BEGIN:DAYLIGHT DTSTART:19870405T020000 RRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=4 TZOFFSETFROM:-0500 TZOFFSETTO:-0400 TZNAME:Eastern Daylight Time (US Canada) END:DAYLIGHT END:VTIMEZONE END:VCALENDAR "#; pub trait GenerateTestDavResource { fn generate(&self) -> String; } impl GenerateTestDavResource for DavResourceName { fn generate(&self) -> String { match self { DavResourceName::Card => generate_random_vcard(), DavResourceName::Cal => generate_random_ical(), DavResourceName::File => generate_random_content(100, 200), _ => unreachable!(), } } } fn generate_random_vcard() -> String { r#"BEGIN:VCARD VERSION:4.0 UID:$UID FN:$NAME END:VCARD "# .replace("$UID", &generate_random_name(8)) .replace("$NAME", &generate_random_name(10)) .replace('\n', "\r\n") } fn generate_random_ical() -> String { r#"BEGIN:VCALENDAR VERSION:2.0 BEGIN:VEVENT UID:$UID SUMMARY:$SUMMARY DESCRIPTION:$DESCRIPTION END:VEVENT END:VCALENDAR "# .replace("$UID", &generate_random_name(8)) .replace("$SUMMARY", &generate_random_name(10)) .replace("$DESCRIPTION", &generate_random_name(20)) .replace('\n', "\r\n") } fn generate_random_content(min_chars: usize, max_chars: usize) -> String { let mut rng = rng(); let length = rng.random_range(min_chars..=max_chars); let words = [ "lorem", "ipsum", "dolor", "sit", "amet", "consectetur", "adipiscing", "elit", "sed", "do", "eiusmod", "tempor", "incididunt", "ut", "labore", "et", "dolore", "magna", "aliqua", "ut", "enim", "ad", "minim", "veniam", "quis", "nostrud", "exercitation", "ullamco", "laboris", "nisi", "ut", "aliquip", "ex", "ea", "commodo", "consequat", ]; let mut content = String::with_capacity(length); while content.len() < length { let word_idx = rng.random_range(0..words.len()); if !content.is_empty() { content.push(' '); } if rng.random_ratio(1, 10) { content.push('.'); let word = words[word_idx]; let mut chars = word.chars(); if let Some(first_char) = chars.next() { content.push_str(&first_char.to_uppercase().to_string()); content.push_str(chars.as_str()); } } else { content.push_str(words[word_idx]); } } if !content.ends_with('.') { content.push('.'); } content } fn generate_random_name(length: usize) -> String { let mut rng = rng(); (0..length) .map(|_| rng.sample(Alphanumeric) as char) .collect() } impl WebDavTest { pub async fn fetch_email(&self, account_id: u32, document_id: u32) -> Vec { let metadata_ = self .server .store() .get_value::>(ValueKey::property( account_id, Collection::Email, document_id, EmailField::Metadata, )) .await .unwrap() .unwrap(); self.server .blob_store() .get_blob( metadata_ .unarchive::() .unwrap() .blob_hash .0 .as_slice(), 0..usize::MAX, ) .await .unwrap() .unwrap() } } const SERVER: &str = r#" [server] hostname = "webdav.example.org" [spam-filter] enable = false [http] url = "'https://127.0.0.1:8899'" [server.listener.webdav] bind = ["127.0.0.1:8899"] protocol = "http" max-connections = 81920 tls.implicit = true [server.socket] reuse-addr = true [server.tls] enable = true implicit = false certificate = "default" [session.ehlo] reject-non-fqdn = false [session.rcpt] relay = [ { if = "!is_empty(authenticated_as)", then = true }, { else = false } ] [session.rcpt.errors] total = 5 wait = "1ms" [resolver] type = "system" [queue.strategy] route = [ { if = "rcpt_domain == 'example.com'", then = "'local'" }, { else = "'mx'" } ] [session.data.add-headers] delivered-to = false [session.extensions] future-release = [ { if = "!is_empty(authenticated_as)", then = "99999999d"}, { else = false } ] [certificate.default] cert = "%{file:{CERT}}%" private-key = "%{file:{PK}}%" [jmap.protocol] set.max-objects = 100000 [jmap.protocol.request] max-concurrent = 8 [jmap.protocol.upload] max-size = 5000000 max-concurrent = 4 ttl = "1m" [jmap.protocol.upload.quota] files = 3 size = 50000 [jmap.rate-limit] account = "1000/1m" authentication = "100/2s" anonymous = "100/1m" [calendar.alarms] minimum-interval = "1s" [calendar.scheduling.inbound] auto-add = true [dav.collection] assisted-discovery = {ASSISTED_DISCOVERY} [sharing] allow-directory-query = true [store."auth"] type = "sqlite" path = "{TMP}/auth.db" [store."auth".query] name = "SELECT name, type, secret, description, quota FROM accounts WHERE name = ? AND active = true" members = "SELECT member_of FROM group_members WHERE name = ?" recipients = "SELECT name FROM emails WHERE address = ?" emails = "SELECT address FROM emails WHERE name = ? AND type != 'list' ORDER BY type DESC, address ASC" verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type = 'primary' ORDER BY address LIMIT 5" expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50" domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || ? LIMIT 1" [oauth] key = "parerga_und_paralipomena" [oauth.auth] max-attempts = 1 [oauth.expiry] user-code = "1s" token = "1s" refresh-token = "3s" refresh-token-renew = "2s" [tracer.console] type = "console" level = "{LEVEL}" multiline = false ansi = true disabled-events = ["network.*"] "#; ================================================ FILE: tests/src/webdav/multiget.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::WebDavTest; use crate::webdav::{DummyWebDavClient, GenerateTestDavResource, prop::DavMultiStatus}; use dav_proto::schema::property::{CalDavProperty, CardDavProperty, DavProperty, WebDavProperty}; use groupware::DavResourceName; use hyper::StatusCode; const MULTIGET_CALENDAR: &str = r#" $PATH "#; const MULTIGET_ADDRESSBOOK: &str = r#" $PATH "#; pub async fn test(test: &WebDavTest) { let client = test.client("john"); for resource_type in [DavResourceName::Cal, DavResourceName::Card] { println!( "Running REPORT multiget tests ({})...", resource_type.base_path() ); let mut paths = Vec::new(); for name in ["file1", "file2"] { let contents = resource_type.generate(); let path = format!("{}/john/default/{}", resource_type.base_path(), name); let etag = client .request("PUT", &path, contents.as_str()) .await .with_status(StatusCode::CREATED) .etag() .to_string(); paths.push((path, etag, contents)); } if resource_type == DavResourceName::Cal { let path = format!("{}/john", resource_type.base_path()); let response = client .multiget_calendar(&path, &[&paths[0].0, &paths[1].0]) .await; for (path, etag, contents) in paths { let props = response.properties(&path); props .get(DavProperty::WebDav(WebDavProperty::GetETag)) .with_values([etag.as_str()]); props .get(DavProperty::CalDav(CalDavProperty::CalendarData( Default::default(), ))) .with_values([contents.as_str()]); } } else { let path = format!("{}/john", resource_type.base_path()); let response = client .multiget_addressbook(&path, &[&paths[0].0, &paths[1].0]) .await; for (path, etag, contents) in paths { let props = response.properties(&path); props .get(DavProperty::WebDav(WebDavProperty::GetETag)) .with_values([etag.as_str()]); props .get(DavProperty::CardDav(CardDavProperty::AddressData( Default::default(), ))) .with_values([contents.as_str()]); } } } client.delete_default_containers().await; test.assert_is_empty().await; } impl DummyWebDavClient { pub async fn multiget_calendar(&self, path: &str, uris: &[&str]) -> DavMultiStatus { let mut paths = String::new(); for uri in uris { paths.push_str(&format!("{}", uri)); } self.request("REPORT", path, &MULTIGET_CALENDAR.replace("$PATH", &paths)) .await .with_status(StatusCode::MULTI_STATUS) .into_propfind_response(None) } pub async fn multiget_addressbook(&self, path: &str, uris: &[&str]) -> DavMultiStatus { let mut paths = String::new(); for uri in uris { paths.push_str(&format!("{}", uri)); } self.request( "REPORT", path, &MULTIGET_ADDRESSBOOK.replace("$PATH", &paths), ) .await .with_status(StatusCode::MULTI_STATUS) .into_propfind_response(None) } } ================================================ FILE: tests/src/webdav/principals.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::WebDavTest; use crate::{TEST_USERS, webdav::prop::ALL_DAV_PROPERTIES}; use dav_proto::schema::property::{DavProperty, PrincipalProperty, WebDavProperty}; use groupware::DavResourceName; use hyper::StatusCode; pub async fn test(test: &WebDavTest, assisted_discovery: bool) { println!("Running principals tests..."); let client = test.client("jane"); let principal_path = format!("D:href:{}/", DavResourceName::Principal.base_path()); let jane_principal_path = format!("D:href:{}/jane/", DavResourceName::Principal.base_path()); let path_support_card = format!("D:href:{}/support/", DavResourceName::Card.base_path()); let path_support_cal = format!("D:href:{}/support/", DavResourceName::Cal.base_path()); // Test 1: PROPFIND on /dav/pal should return all principals let response = client .propfind( DavResourceName::Principal.collection_path(), ALL_DAV_PROPERTIES, ) .await; for (account, _, name, email) in TEST_USERS { let props = response.properties(&format!( "{}/{}/", DavResourceName::Principal.base_path(), account )); let path_pal = format!( "D:href:{}/{}/", DavResourceName::Principal.base_path(), account ); let path_card = format!("D:href:{}/{}/", DavResourceName::Card.base_path(), account); let path_cal = format!("D:href:{}/{}/", DavResourceName::Cal.base_path(), account); props .get(DavProperty::WebDav(WebDavProperty::DisplayName)) .with_values([*name]) .with_status(StatusCode::OK); props .get(DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal)) .with_values([jane_principal_path.as_str()]) .with_status(StatusCode::OK); props .get(DavProperty::Principal(PrincipalProperty::PrincipalURL)) .with_values([path_pal.as_str()]) .with_status(StatusCode::OK); props .get(DavProperty::WebDav(WebDavProperty::Owner)) .with_values([path_pal.as_str()]) .with_status(StatusCode::OK); if *account == "jane" && !assisted_discovery { props .get(DavProperty::Principal(PrincipalProperty::CalendarHomeSet)) .with_values([path_cal.as_str(), path_support_cal.as_str()]) .with_status(StatusCode::OK); props .get(DavProperty::Principal( PrincipalProperty::AddressbookHomeSet, )) .with_values([path_card.as_str(), path_support_card.as_str()]) .with_status(StatusCode::OK); } else { props .get(DavProperty::Principal(PrincipalProperty::CalendarHomeSet)) .with_values([path_cal.as_str()]) .with_status(StatusCode::OK); props .get(DavProperty::Principal( PrincipalProperty::AddressbookHomeSet, )) .with_values([path_card.as_str()]) .with_status(StatusCode::OK); } props .get(DavProperty::WebDav(WebDavProperty::PrincipalCollectionSet)) .with_values([principal_path.as_str()]) .with_status(StatusCode::OK); props .get(DavProperty::WebDav(WebDavProperty::SupportedReportSet)) .with_values([ "D:supported-report.D:report.D:principal-property-search", "D:supported-report.D:report.D:principal-search-property-set", "D:supported-report.D:report.D:principal-match", ]) .with_status(StatusCode::OK); props .get(DavProperty::WebDav(WebDavProperty::ResourceType)) .with_values(["D:principal", "D:collection"]) .with_status(StatusCode::OK); // Scheduling properties props .get(DavProperty::Principal( PrincipalProperty::CalendarUserAddressSet, )) .with_values([format!("D:href:mailto:{email}",).as_str()]) .with_status(StatusCode::OK); props .get(DavProperty::Principal(PrincipalProperty::CalendarUserType)) .with_values(["INDIVIDUAL"]) .with_status(StatusCode::OK); props .get(DavProperty::Principal(PrincipalProperty::ScheduleInboxURL)) .with_values([format!( "D:href:{}/{account}/inbox/", DavResourceName::Scheduling.base_path() ) .as_str()]) .with_status(StatusCode::OK); props .get(DavProperty::Principal(PrincipalProperty::ScheduleOutboxURL)) .with_values([format!( "D:href:{}/{account}/outbox/", DavResourceName::Scheduling.base_path() ) .as_str()]) .with_status(StatusCode::OK); } // Test 2: PROPFIND on /dav/[resource] should return user and shared resources for resource_type in [ DavResourceName::File, DavResourceName::Cal, DavResourceName::Card, ] { let supported_reports = match resource_type { DavResourceName::File => [ "D:supported-report.D:report.D:sync-collection", "D:supported-report.D:report.D:acl-principal-prop-set", "D:supported-report.D:report.D:principal-match", ] .as_slice(), DavResourceName::Cal => [ "D:supported-report.D:report.A:free-busy-query", "D:supported-report.D:report.A:calendar-query", "D:supported-report.D:report.D:expand-property", "D:supported-report.D:report.D:sync-collection", "D:supported-report.D:report.D:acl-principal-prop-set", "D:supported-report.D:report.D:principal-match", "D:supported-report.D:report.A:calendar-multiget", ] .as_slice(), DavResourceName::Card => [ "D:supported-report.D:report.B:addressbook-query", "D:supported-report.D:report.D:acl-principal-prop-set", "D:supported-report.D:report.D:expand-property", "D:supported-report.D:report.B:addressbook-multiget", "D:supported-report.D:report.D:principal-match", "D:supported-report.D:report.D:sync-collection", ] .as_slice(), _ => unreachable!(), }; let privilege_set = if resource_type == DavResourceName::Cal { [ "D:privilege.D:read-current-user-privilege-set", "D:privilege.D:write-acl", "D:privilege.A:read-free-busy", "D:privilege.D:read-acl", "D:privilege.D:write-properties", "D:privilege.D:write", "D:privilege.D:write-content", "D:privilege.D:unlock", "D:privilege.D:all", "D:privilege.D:read", "D:privilege.D:bind", "D:privilege.D:unbind", ] .as_slice() } else { [ "D:privilege.D:all", "D:privilege.D:read", "D:privilege.D:write", "D:privilege.D:write-properties", "D:privilege.D:write-content", "D:privilege.D:unlock", "D:privilege.D:read-acl", "D:privilege.D:read-current-user-privilege-set", "D:privilege.D:write-acl", "D:privilege.D:bind", "D:privilege.D:unbind", ] .as_slice() }; let response = client .propfind(resource_type.collection_path(), ALL_DAV_PROPERTIES) .await; let props = response.properties(resource_type.collection_path()); props .get(DavProperty::WebDav(WebDavProperty::SupportedReportSet)) .with_values(supported_reports.iter().copied()) .with_status(StatusCode::OK); props .get(DavProperty::WebDav(WebDavProperty::ResourceType)) .with_values(["D:collection"]) .with_status(StatusCode::OK); props .get(DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal)) .with_values([jane_principal_path.as_str()]) .with_status(StatusCode::OK); if assisted_discovery { props .get(DavProperty::Principal(PrincipalProperty::CalendarHomeSet)) .with_values( [format!("D:href:{}/jane/", DavResourceName::Cal.base_path()).as_str()], ) .with_status(StatusCode::OK); props .get(DavProperty::Principal( PrincipalProperty::AddressbookHomeSet, )) .with_values([ format!("D:href:{}/jane/", DavResourceName::Card.base_path()).as_str(), ]) .with_status(StatusCode::OK); } else { props .get(DavProperty::Principal(PrincipalProperty::CalendarHomeSet)) .with_values([ format!("D:href:{}/jane/", DavResourceName::Cal.base_path()).as_str(), format!("D:href:{}/support/", DavResourceName::Cal.base_path()).as_str(), ]) .with_status(StatusCode::OK); props .get(DavProperty::Principal( PrincipalProperty::AddressbookHomeSet, )) .with_values([ format!("D:href:{}/jane/", DavResourceName::Card.base_path()).as_str(), format!("D:href:{}/support/", DavResourceName::Card.base_path()).as_str(), ]) .with_status(StatusCode::OK); } for (account, _, name, _) in TEST_USERS .iter() .filter(|(account, _, _, _)| ["jane", "support"].contains(account)) { let path_card = format!("D:href:{}/{}/", DavResourceName::Card.base_path(), account); let path_cal = format!("D:href:{}/{}/", DavResourceName::Cal.base_path(), account); let path_pal = format!( "D:href:{}/{}/", DavResourceName::Principal.base_path(), account ); let props = response.properties(&format!("{}/{account}/", resource_type.base_path())); props .get(DavProperty::WebDav(WebDavProperty::DisplayName)) .with_values([*name]) .with_status(StatusCode::OK); props .get(DavProperty::WebDav(WebDavProperty::ResourceType)) .with_values(["D:collection"]) .with_status(StatusCode::OK); props .get(DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal)) .with_values([jane_principal_path.as_str()]) .with_status(StatusCode::OK); props .get(DavProperty::WebDav(WebDavProperty::CurrentUserPrivilegeSet)) .with_values(privilege_set.iter().copied()) .with_status(StatusCode::OK); props .get(DavProperty::WebDav(WebDavProperty::SupportedReportSet)) .with_values(supported_reports.iter().copied()) .with_status(StatusCode::OK); props .get(DavProperty::Principal(PrincipalProperty::PrincipalURL)) .with_values([path_pal.as_str()]) .with_status(StatusCode::OK); props .get(DavProperty::WebDav(WebDavProperty::PrincipalCollectionSet)) .with_values([principal_path.as_str()]) .with_status(StatusCode::OK); props .get(DavProperty::WebDav(WebDavProperty::Owner)) .with_values([path_pal.as_str()]) .with_status(StatusCode::OK); if *account == "jane" && !assisted_discovery { props .get(DavProperty::Principal(PrincipalProperty::CalendarHomeSet)) .with_values([path_cal.as_str(), path_support_cal.as_str()]) .with_status(StatusCode::OK); props .get(DavProperty::Principal( PrincipalProperty::AddressbookHomeSet, )) .with_values([path_card.as_str(), path_support_card.as_str()]) .with_status(StatusCode::OK); } else { props .get(DavProperty::Principal(PrincipalProperty::CalendarHomeSet)) .with_values([path_cal.as_str()]) .with_status(StatusCode::OK); props .get(DavProperty::Principal( PrincipalProperty::AddressbookHomeSet, )) .with_values([path_card.as_str()]) .with_status(StatusCode::OK); } props .get(DavProperty::WebDav(WebDavProperty::SyncToken)) .with_status(StatusCode::OK) .is_not_empty(); props .get(DavProperty::WebDav(WebDavProperty::QuotaAvailableBytes)) .with_status(StatusCode::OK) .is_not_empty(); props .get(DavProperty::WebDav(WebDavProperty::QuotaUsedBytes)) .with_status(StatusCode::OK) .is_not_empty(); } // Test 3: principal-match-query on resources let response = client .request( "REPORT", resource_type.collection_path(), PRINCIPAL_MATCH_QUERY, ) .await .with_status(StatusCode::MULTI_STATUS) .into_propfind_response(None); response.with_hrefs([ format!("{}/jane/", resource_type.base_path()).as_str(), format!("{}/support/", resource_type.base_path()).as_str(), ]); } // Test 4: principal-match-query on principals let response = client .request( "REPORT", DavResourceName::Principal.collection_path(), PRINCIPAL_MATCH_QUERY, ) .await .with_status(StatusCode::MULTI_STATUS) .into_propfind_response(None); response.with_hrefs([ format!("{}/jane/", DavResourceName::Principal.base_path()).as_str(), format!("{}/support/", DavResourceName::Principal.base_path()).as_str(), ]); // Test 5: principal-search-property-set REPORT let response = client .request( "REPORT", DavResourceName::Principal.collection_path(), PRINCIPAL_SEARCH_PROPERTY_SET_QUERY, ) .await .with_status(StatusCode::OK); response .with_value( "D:principal-search-property-set.D:principal-search-property.D:prop.D:displayname", "", ) .with_value( "D:principal-search-property-set.D:principal-search-property.D:description", "Account or Group name", ); // Test 6: principal-property-search REPORT let response = client .request( "REPORT", DavResourceName::Principal.collection_path(), PRINCIPAL_PROPERTY_SEARCH_QUERY.replace("$NAME", "doe"), ) .await .with_status(StatusCode::MULTI_STATUS) .into_propfind_response(None); response.with_hrefs([ format!("{}/jane/", DavResourceName::Principal.base_path()).as_str(), format!("{}/john/", DavResourceName::Principal.base_path()).as_str(), ]); response .properties(&format!("{}/jane/", DavResourceName::Principal.base_path())) .get(DavProperty::WebDav(WebDavProperty::DisplayName)) .with_values([TEST_USERS .iter() .find(|(account, _, _, _)| *account == "jane") .unwrap() .2]) .with_status(StatusCode::OK); client .request( "REPORT", DavResourceName::Principal.collection_path(), PRINCIPAL_PROPERTY_SEARCH_QUERY.replace("$NAME", "support"), ) .await .with_status(StatusCode::MULTI_STATUS) .into_propfind_response(None) .with_hrefs([format!("{}/support/", DavResourceName::Principal.base_path()).as_str()]); client.delete_default_containers().await; client.delete_default_containers_by_account("support").await; test.assert_is_empty().await; } const PRINCIPAL_MATCH_QUERY: &str = r#" "#; const PRINCIPAL_SEARCH_PROPERTY_SET_QUERY: &str = r#""#; const PRINCIPAL_PROPERTY_SEARCH_QUERY: &str = r#" $NAME "#; ================================================ FILE: tests/src/webdav/prop.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{DavResponse, DummyWebDavClient, WebDavTest}; use crate::webdav::{GenerateTestDavResource, TEST_ICAL_2, TEST_VTIMEZONE_1}; use ahash::{AHashMap, AHashSet}; use dav_proto::schema::property::{ CalDavProperty, CardDavProperty, DavProperty, PrincipalProperty, WebDavProperty, }; use groupware::DavResourceName; use hyper::StatusCode; use types::dead_property::DeadElementTag; pub async fn test(test: &WebDavTest, assisted_discovery: bool) { let client = test.client("jane"); for resource_type in [ DavResourceName::File, DavResourceName::Cal, DavResourceName::Card, ] { println!( "Running PROPFIND/PROPPATCH tests ({})...", resource_type.base_path() ); let user_base_path = format!("{}/jane", resource_type.base_path()); let group_base_path = format!("{}/support", resource_type.base_path()); // Create a new test container and file let test_base_path = format!("{user_base_path}/PropFind_Folder/"); let etag_folder = client .mkcol("MKCOL", &test_base_path, [], []) .await .with_status(StatusCode::CREATED) .etag() .to_string(); let test_contents = resource_type.generate(); let test_path = format!("{test_base_path}test_file"); let etag_file = client .request_with_headers( "PUT", &test_path, [("content-type", "text/x-other")], test_contents.as_str(), ) .await .with_status(StatusCode::CREATED) .etag() .to_string(); // Test 1: PROPFIND Depth 0 on root client .request_with_headers("PROPFIND", resource_type.base_path(), [("depth", "0")], "") .await .with_status(StatusCode::MULTI_STATUS) .with_hrefs([resource_type.collection_path()]); // Test 2: PROPFIND Depth 0 on user base path client .request_with_headers("PROPFIND", &user_base_path, [("depth", "0")], "") .await .with_status(StatusCode::MULTI_STATUS) .with_hrefs([format!("{user_base_path}/").as_str()]); // Test 3: PROPFIND Depth 1 on root client .request_with_headers("PROPFIND", resource_type.base_path(), [("depth", "1")], "") .await .with_status(StatusCode::MULTI_STATUS) .with_hrefs([ resource_type.collection_path(), format!("{user_base_path}/").as_str(), format!("{group_base_path}/").as_str(), ]); // Test 4: Infinity depth is not allowed for path in [resource_type.base_path(), user_base_path.as_str()] { client .request_with_headers("PROPFIND", path, [("depth", "infinity")], "") .await .with_status(StatusCode::FORBIDDEN); } // Test 5: PROPFIND Depth 1 on user base path client .request_with_headers("PROPFIND", &user_base_path, [("depth", "1")], "") .await .with_status(StatusCode::MULTI_STATUS) .with_hrefs( [ format!("{group_base_path}/default/").as_str(), format!("{user_base_path}/default/").as_str(), format!("{user_base_path}/").as_str(), &test_base_path, ] .into_iter() .skip(if resource_type == DavResourceName::File { 2 } else if !assisted_discovery { 1 } else { 0 }), ); // Test 6: PROPFIND Depth 1 on created collection client .request_with_headers("PROPFIND", &test_base_path, [("depth", "1")], "") .await .with_status(StatusCode::MULTI_STATUS) .with_hrefs([test_base_path.as_str(), test_path.as_str()]); // Test 7: Infinity depth is not allowed on file containers client .request_with_headers("PROPFIND", &test_base_path, [("depth", "infinity")], "") .await .with_status(if resource_type == DavResourceName::File { StatusCode::FORBIDDEN } else { StatusCode::MULTI_STATUS }); // Test 8 PROPFIND with depth-no-root client .request_with_headers( "PROPFIND", &user_base_path, [("depth", "1"), ("prefer", "depth-noroot")], "", ) .await .with_status(StatusCode::MULTI_STATUS) .with_hrefs( [ format!("{group_base_path}/default/").as_str(), format!("{user_base_path}/default/").as_str(), &test_base_path, ] .into_iter() .skip(if resource_type == DavResourceName::File { 2 } else if !assisted_discovery { 1 } else { 0 }), ); client .request_with_headers( "PROPFIND", &test_base_path, [("depth", "1"), ("prefer", "depth-noroot")], "", ) .await .with_status(StatusCode::MULTI_STATUS) .with_hrefs([test_path.as_str()]); // Test 8 PROPFIND with prefer return=minimal let response = client .propfind_with_headers(&test_base_path, ALL_DAV_PROPERTIES, []) .await; response .properties(&test_base_path) .is_defined(DavProperty::WebDav(WebDavProperty::GetETag)) .is_defined(DavProperty::Principal(PrincipalProperty::GroupMembership)); let response = client .propfind_with_headers( &test_base_path, ALL_DAV_PROPERTIES, [("prefer", "return=minimal")], ) .await; response .properties(&test_base_path) .is_defined(DavProperty::WebDav(WebDavProperty::GetETag)) .is_undefined(DavProperty::Principal(PrincipalProperty::GroupMembership)); // Test 9: Retrieve all static properties for (path, etag, is_file) in [ (&test_base_path, &etag_folder, false), (&test_path, &etag_file, true), ] { let response = client.propfind(path, ALL_DAV_PROPERTIES).await; let properties = response.properties(path); properties .get(DavProperty::WebDav(WebDavProperty::CreationDate)) .is_not_empty(); properties .get(DavProperty::WebDav(WebDavProperty::GetLastModified)) .is_not_empty(); properties .get(DavProperty::WebDav(WebDavProperty::SyncToken)) .is_not_empty(); properties .get(DavProperty::WebDav(WebDavProperty::GetETag)) .with_values([etag.as_str()]); properties .get(DavProperty::WebDav(WebDavProperty::SupportedLock)) .with_values([ "D:lockentry.D:lockscope.D:exclusive", "D:lockentry.D:locktype.D:write", "D:lockentry.D:lockscope.D:shared", "D:lockentry.D:locktype.D:write", ]); properties .get(DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal)) .with_values([ format!("D:href:{}/jane/", DavResourceName::Principal.base_path()).as_str(), ]); properties .get(DavProperty::WebDav(WebDavProperty::Owner)) .with_values([ format!("D:href:{}/jane/", DavResourceName::Principal.base_path()).as_str(), ]); properties .get(DavProperty::WebDav(WebDavProperty::SupportedPrivilegeSet)) .is_not_empty(); properties .get(DavProperty::WebDav(WebDavProperty::AclRestrictions)) .with_values(["D:grant-only", "D:no-invert"]); properties .get(DavProperty::WebDav(WebDavProperty::PrincipalCollectionSet)) .with_values([ format!("D:href:{}", DavResourceName::Principal.collection_path()).as_str(), ]); if is_file { // File specific properties properties .get(DavProperty::WebDav(WebDavProperty::GetContentType)) .with_values([match resource_type { DavResourceName::File => "text/x-other", DavResourceName::Cal => "text/calendar", DavResourceName::Card => "text/vcard", _ => unreachable!(), }]); properties .get(DavProperty::WebDav(WebDavProperty::GetContentLength)) .with_values([test_contents.len().to_string().as_str()]); } else { // Collection specific properties properties .get(DavProperty::WebDav(WebDavProperty::GetCTag)) .is_not_empty(); properties .get(DavProperty::WebDav(WebDavProperty::ResourceType)) .with_values(match resource_type { DavResourceName::File => ["D:collection"].as_slice().iter().copied(), DavResourceName::Cal => { ["D:collection", "A:calendar"].as_slice().iter().copied() } DavResourceName::Card => { ["D:collection", "B:addressbook"].as_slice().iter().copied() } _ => unreachable!(), }); let used_bytes: u64 = properties .get(DavProperty::WebDav(WebDavProperty::QuotaUsedBytes)) .value() .parse() .unwrap(); let available_bytes: u64 = properties .get(DavProperty::WebDav(WebDavProperty::QuotaAvailableBytes)) .value() .parse() .unwrap(); assert!(used_bytes > 0); assert!(available_bytes > 0); properties .get(DavProperty::WebDav(WebDavProperty::SupportedReportSet)) .with_values(match resource_type { DavResourceName::File => [ "D:supported-report.D:report.D:sync-collection", "D:supported-report.D:report.D:acl-principal-prop-set", "D:supported-report.D:report.D:principal-match", ] .as_slice() .iter() .copied(), DavResourceName::Cal => [ "D:supported-report.D:report.A:calendar-query", "D:supported-report.D:report.D:sync-collection", "D:supported-report.D:report.D:acl-principal-prop-set", "D:supported-report.D:report.D:expand-property", "D:supported-report.D:report.A:free-busy-query", "D:supported-report.D:report.A:calendar-multiget", "D:supported-report.D:report.D:principal-match", ] .as_slice() .iter() .copied(), DavResourceName::Card => [ "D:supported-report.D:report.B:addressbook-multiget", "D:supported-report.D:report.D:sync-collection", "D:supported-report.D:report.D:acl-principal-prop-set", "D:supported-report.D:report.D:principal-match", "D:supported-report.D:report.B:addressbook-query", "D:supported-report.D:report.D:expand-property", ] .as_slice() .iter() .copied(), _ => unreachable!(), }); if resource_type == DavResourceName::Cal { properties .get(DavProperty::WebDav(WebDavProperty::CurrentUserPrivilegeSet)) .with_values([ "D:privilege.D:all", "D:privilege.D:read", "D:privilege.D:write", "D:privilege.D:write-properties", "D:privilege.D:write-content", "D:privilege.D:unlock", "D:privilege.D:read-acl", "D:privilege.D:read-current-user-privilege-set", "D:privilege.D:write-acl", "D:privilege.D:bind", "D:privilege.D:unbind", "D:privilege.A:read-free-busy", ]); properties .get(DavProperty::CalDav( CalDavProperty::SupportedCalendarComponentSet, )) .with_values([ "A:comp.[name]:VAVAILABILITY", "A:comp.[name]:AVAILABLE", "A:comp.[name]:VRESOURCE", "A:comp.[name]:VTODO", "A:comp.[name]:DAYLIGHT", "A:comp.[name]:STANDARD", "A:comp.[name]:VLOCATION", "A:comp.[name]:VTIMEZONE", "A:comp.[name]:VFREEBUSY", "A:comp.[name]:VEVENT", "A:comp.[name]:VJOURNAL", "A:comp.[name]:PARTICIPANT", "A:comp.[name]:VALARM", ]); properties .get(DavProperty::CalDav(CalDavProperty::SupportedCalendarData)) .with_values([ concat!("A:calendar-data-type.", "[content-type]:text/calendar"), "A:calendar-data-type.[version]:2.0", "A:calendar-data-type.[version]:1.0", ]); properties .get(DavProperty::CalDav(CalDavProperty::SupportedCollationSet)) .with_values([ "A:supported-collation:i;unicode-casemap", "A:supported-collation:i;ascii-casemap", ]); properties .get(DavProperty::CalDav(CalDavProperty::MinDateTime)) .with_values(["0001-01-01T00:00:00Z"]); properties .get(DavProperty::CalDav(CalDavProperty::MaxDateTime)) .with_values(["9999-12-31T23:59:59Z"]); for (key, value) in [ ( DavProperty::CalDav(CalDavProperty::MaxResourceSize), test.server.core.groupware.max_ical_size, ), ( DavProperty::CalDav(CalDavProperty::MaxInstances), test.server.core.groupware.max_ical_instances, ), ( DavProperty::CalDav(CalDavProperty::MaxAttendeesPerInstance), test.server.core.groupware.max_ical_attendees_per_instance, ), ] { properties .get(key) .with_values([value.to_string().as_str()]); } } else { if resource_type == DavResourceName::Card { properties .get(DavProperty::CardDav(CardDavProperty::SupportedAddressData)) .with_values([ concat!("B:address-data-type.", "[content-type]:text/vcard"), "B:address-data-type.[version]:3.0", "B:address-data-type.[version]:4.0", "B:address-data-type.[version]:2.1", ]); properties .get(DavProperty::CardDav(CardDavProperty::SupportedCollationSet)) .with_values([ "B:supported-collation:i;unicode-casemap", "B:supported-collation:i;ascii-casemap", ]); properties .get(DavProperty::CardDav(CardDavProperty::MaxResourceSize)) .with_values([test .server .core .groupware .max_vcard_size .to_string() .as_str()]); } properties .get(DavProperty::WebDav(WebDavProperty::CurrentUserPrivilegeSet)) .with_values([ "D:privilege.D:all", "D:privilege.D:read", "D:privilege.D:write", "D:privilege.D:write-properties", "D:privilege.D:write-content", "D:privilege.D:unlock", "D:privilege.D:read-acl", "D:privilege.D:read-current-user-privilege-set", "D:privilege.D:write-acl", "D:privilege.D:bind", "D:privilege.D:unbind", ]); } } } // Test 10: expand-property report for path in [&test_base_path, &test_path] { let response = client .request("REPORT", path, EXPAND_REPORT_QUERY) .await .with_status(StatusCode::MULTI_STATUS) .into_propfind_response(None); let properties = response.properties(path); for prop in [ DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal), DavProperty::WebDav(WebDavProperty::Owner), ] { properties.get(prop).with_some_values([ format!( "D:response.D:href:{}/jane/", DavResourceName::Principal.base_path(), ) .as_str(), "D:response.D:propstat.D:prop.D:displayname:Jane Doe-Smith", ]); } } for (path, etag, is_file) in [ (&test_base_path, &etag_folder, false), (&test_path, &etag_file, true), ] { // Test 11: PROPPATCH should fail when a precondition fails client .proppatch( path, [( DavProperty::WebDav(WebDavProperty::DisplayName), "Magnific name", )], [], [("if", format!("(Not [{etag}])").as_str())], ) .await .with_status(StatusCode::PRECONDITION_FAILED); client .proppatch( path, [( DavProperty::WebDav(WebDavProperty::DisplayName), "Magnific name - second try", )], [], [("if", format!("([{etag}])").as_str())], ) .await .with_status(StatusCode::MULTI_STATUS); client .propfind(path, [DavProperty::WebDav(WebDavProperty::GetETag)]) .await .properties(path) .get(DavProperty::WebDav(WebDavProperty::GetETag)) .with_status(StatusCode::OK) .without_values([etag.as_str()]); // Test 12: PROPPATCH set on DAV properties client .patch_and_check( path, [ ( DavProperty::WebDav(WebDavProperty::DisplayName), "New display name", ), ( DavProperty::WebDav(WebDavProperty::CreationDate), "2000-01-01T00:00:00Z", ), ( DavProperty::DeadProperty(DeadElementTag::new( "my-dead-element".to_string(), Some("xmlns=\"http://example.com/ns/\" prop=\"abc\"".to_string()), )), "this is a dead but exciting element", ), ], ) .await; client .patch_and_check( path, [( DavProperty::DeadProperty(DeadElementTag::new( "my-dead-element".to_string(), Some("xmlns=\"http://example.com/ns/\" prop=\"xyz\"".to_string()), )), "this is a modified dead but exciting element", )], ) .await; // Test 13: PROPPATCH remove on DAV properties let mut props = vec![ ( DavProperty::DeadProperty(DeadElementTag::new( "my-dead-element".to_string(), Some("xmlns=\"http://example.com/ns/\"".to_string()), )), "", ), (DavProperty::WebDav(WebDavProperty::DisplayName), ""), ]; if !is_file { // DisplayName can't be removed from calendar/contact collections props.pop(); } client.patch_and_check(path, props).await; match resource_type { DavResourceName::File if is_file => { // Test 14: Change a file's content-type client .patch_and_check( path, [( DavProperty::WebDav(WebDavProperty::GetContentType), "text/x-yadda-yadda", )], ) .await; } DavResourceName::Cal if !is_file => { // Test 15: Change a calendar's properties client .patch_and_check( path, [ ( DavProperty::CalDav(CalDavProperty::CalendarDescription), "New calendar description", ), ( DavProperty::CalDav(CalDavProperty::TimezoneId), "Europe/Ljubljana", ), ], ) .await; client .patch_and_check( path, [ (DavProperty::CalDav(CalDavProperty::CalendarDescription), ""), (DavProperty::CalDav(CalDavProperty::TimezoneId), ""), ], ) .await; client .patch_and_check( path, [( DavProperty::CalDav(CalDavProperty::CalendarTimezone), TEST_VTIMEZONE_1.replace('\n', "\r\n").as_str(), )], ) .await; } DavResourceName::Card if !is_file => { // Test 16: Change an addressbook's properties client .patch_and_check( path, [( DavProperty::CardDav(CardDavProperty::AddressbookDescription), "New calendar description", )], ) .await; client .patch_and_check( path, [( DavProperty::CardDav(CardDavProperty::AddressbookDescription), "", )], ) .await; } _ => (), } // Test 17: PROPPATCH should fail on large properties let mut chunky_props = vec![ DavProperty::WebDav(WebDavProperty::DisplayName), DavProperty::DeadProperty(DeadElementTag::new( "my-chunky-dead-element".to_string(), Some("xmlns=\"http://example.com/ns/\"".to_string()), )), ]; if !is_file { if resource_type == DavResourceName::Cal { chunky_props.push(DavProperty::CalDav(CalDavProperty::CalendarDescription)); } else if resource_type == DavResourceName::Card { chunky_props.push(DavProperty::CardDav( CardDavProperty::AddressbookDescription, )); } } let chunky_live_contents = (0..=(test.server.core.groupware.live_property_size + 1)) .map(|_| "a") .collect::(); let chunky_dead_contents = (0..=(test.server.core.groupware.dead_property_size.unwrap() + 1)) .map(|_| "a") .collect::(); let response = client .proppatch( path, chunky_props.iter().map(|prop| { ( prop.clone(), if matches!(prop, DavProperty::DeadProperty(_)) { &chunky_dead_contents } else { &chunky_live_contents } .as_str(), ) }), [], [], ) .await .into_propfind_response(None); let props = response.properties(path); for prop in chunky_props { props .get(prop) .with_status(StatusCode::INSUFFICIENT_STORAGE) .with_description("Property value is too long"); } // Test 18: PROPPATCH should fail on invalid calendar property values if !is_file && resource_type == DavResourceName::Cal { let response = client .proppatch( path, [ ( DavProperty::CalDav(CalDavProperty::TimezoneId), "unknown/zone", ), ( DavProperty::CalDav(CalDavProperty::CalendarTimezone), TEST_ICAL_2, ), ], [], [], ) .await .into_propfind_response(None); let props = response.properties(path); props .get(DavProperty::CalDav(CalDavProperty::TimezoneId)) .with_status(StatusCode::PRECONDITION_FAILED) .with_description("Invalid timezone ID"); props .get(DavProperty::CalDav(CalDavProperty::CalendarTimezone)) .with_status(StatusCode::PRECONDITION_FAILED) .with_description("Invalid calendar timezone"); } } client .request("DELETE", &test_base_path, "") .await .with_status(StatusCode::NO_CONTENT); } client.delete_default_containers().await; client.delete_default_containers_by_account("support").await; test.assert_is_empty().await; } #[derive(Debug)] pub struct DavMultiStatus { pub response: DavResponse, pub hrefs: AHashMap, } #[derive(Debug, serde::Serialize)] pub struct DavItem { #[serde(serialize_with = "serialize_status_code")] pub status: StatusCode, pub values: AHashMap>, pub error: Vec, pub description: Option, } #[derive(Debug, serde::Serialize)] pub struct DavProperties { #[serde(skip)] status: StatusCode, props: Vec, } impl DavMultiStatus { pub fn properties(&self, href: &str) -> DavPropertyResult<'_> { DavPropertyResult { response: &self.response, properties: self.hrefs.get(href).unwrap_or_else(|| { self.response.dump_response(); panic!( "No properties found for href: {href} in {}", serde_json::to_string_pretty(&self.hrefs).unwrap() ) }), } } pub fn with_hrefs<'x>(&self, expect_hrefs: impl IntoIterator) -> &Self { let expect_hrefs: AHashSet<_> = expect_hrefs.into_iter().collect(); let hrefs: AHashSet<_> = self.hrefs.keys().map(|s| s.as_str()).collect(); if hrefs != expect_hrefs { self.response.dump_response(); panic!("Expected hrefs {expect_hrefs:?}, but got {hrefs:?}",); } self } } pub struct DavPropertyResult<'x> { pub response: &'x DavResponse, pub properties: &'x DavProperties, } pub struct DavQueryResult<'x> { pub response: &'x DavResponse, pub prop: &'x DavItem, pub values: &'x [String], } impl DavPropertyResult<'_> { pub fn get(&self, name: impl AsRef) -> DavQueryResult<'_> { let name = name.as_ref(); self.properties .props .iter() .find_map(|prop| { prop.values.get(name).map(|values| DavQueryResult { response: self.response, prop, values, }) }) .unwrap_or_else(|| { self.response.dump_response(); panic!( "No property found for name: {name} in {}", serde_json::to_string_pretty(&self.properties.props).unwrap() ) }) } pub fn with_status(&self, status: StatusCode) -> &Self { if self.properties.status != status { self.response.dump_response(); panic!( "Expected status {status}, but got {}", self.properties.status ); } self } pub fn is_defined(&self, name: impl AsRef) -> &Self { if self .properties .props .iter() .any(|prop| prop.values.contains_key(name.as_ref())) { self } else { self.response.dump_response(); panic!("Expected property {} to be defined", name.as_ref()); } } pub fn is_undefined(&self, name: impl AsRef) -> &Self { if self .properties .props .iter() .any(|prop| prop.values.contains_key(name.as_ref())) { self.response.dump_response(); panic!("Expected property {} to be undefined", name.as_ref()); } self } pub fn calendar_data(&self) -> DavQueryResult<'_> { self.get(DavProperty::CalDav(CalDavProperty::CalendarData( Default::default(), ))) } } impl<'x> DavQueryResult<'x> { pub fn with_values(&self, expected_values: impl IntoIterator) -> &Self { let expected_values = AHashSet::from_iter(expected_values); let values = self .values .iter() .map(|s| s.as_str()) .collect::>(); if values != expected_values { self.response.dump_response(); assert_eq!(values, expected_values,); } self } pub fn with_some_values(&self, expected_values: impl IntoIterator) -> &Self { let values = self .values .iter() .map(|s| s.as_str()) .collect::>(); for expected_value in expected_values { if !values.contains(expected_value) { self.response.dump_response(); panic!("Expected at least one of {expected_value:?} values, but got {values:?}",); } } self } pub fn with_any_values(&self, expected_values: impl IntoIterator) -> &Self { let values = self .values .iter() .map(|s| s.as_str()) .collect::>(); let expected_values = AHashSet::from_iter(expected_values); if values.is_disjoint(&expected_values) { self.response.dump_response(); panic!("Expected at least one of {expected_values:?} values, but got {values:?}",); } self } pub fn without_values(&self, expected_values: impl IntoIterator) -> &Self { let expected_values = AHashSet::from_iter(expected_values); let values = self .values .iter() .map(|s| s.as_str()) .collect::>(); if !expected_values.is_disjoint(&values) { self.response.dump_response(); panic!("Expected no {expected_values:?} values, but got {values:?}",); } self } pub fn is_not_empty(&self) -> &Self { if self.values.is_empty() || self.values.iter().all(|s| s.is_empty()) { self.response.dump_response(); panic!("Expected non-empty values, but got {:?}", self.values); } self } pub fn value(&self) -> &str { if let Some(value) = self.values.iter().find(|s| !s.is_empty()) { value } else { self.response.dump_response(); panic!("Expected a value, but got {:?}", self.values); } } pub fn with_status(&self, status: StatusCode) -> &Self { if self.prop.status != status { self.response.dump_response(); panic!("Expected status {status}, but got {}", self.prop.status); } self } pub fn with_description(&self, description: &str) -> &Self { if self.prop.description.as_deref() != Some(description) { self.response.dump_response(); panic!( "Expected description {description}, but got {:?}", self.prop.description ); } self } pub fn with_error(&self, error: &str) -> &Self { if !self.prop.error.contains(&error.to_string()) { self.response.dump_response(); panic!("Expected error {error}, but got {:?}", self.prop.error); } self } } impl DavResponse { pub fn into_propfind_response(mut self, prop_prefix: Option<&str>) -> DavMultiStatus { if let Some(prop_prefix) = prop_prefix { for (key, _) in self.xml.iter_mut() { if let Some(suffix) = key.strip_prefix(prop_prefix) { *key = format!("D:multistatus.D:response{suffix}"); } } self.xml.push(( "D:multistatus.D:response.D:href".to_string(), "".to_string(), )); } let mut result = DavMultiStatus { response: self, hrefs: AHashMap::new(), }; let mut href = None; let mut href_status = StatusCode::OK; let mut props = Vec::new(); let mut prop = DavItem::default(); for (key, value) in &result.response.xml { match key.as_str() { "D:multistatus.D:response.D:href" => { if let Some(href) = href.take() { if !prop.is_empty() { props.push(std::mem::take(&mut prop)); } result.hrefs.insert( href, DavProperties { status: href_status, props: std::mem::take(&mut props), }, ); href_status = StatusCode::OK; } href = Some(value.to_string()); } "D:multistatus.D:response.D:status" => { href_status = value .split_ascii_whitespace() .nth(1) .unwrap_or_default() .parse() .unwrap(); } "D:multistatus.D:response.D:propstat.D:status" => { prop.status = value .split_ascii_whitespace() .nth(1) .unwrap_or_default() .parse() .unwrap(); } "D:multistatus.D:response.D:propstat.D:responsedescription" => { prop.description = Some(value.to_string()); } _ => { if let Some(prop_name) = key.strip_prefix("D:multistatus.D:response.D:propstat.D:prop.") { if prop.status != StatusCode::PROXY_AUTHENTICATION_REQUIRED { props.push(std::mem::take(&mut prop)); } let (prop_name, prop_value) = if let Some((prop_name, prop_sub_name)) = prop_name.split_once('.') { if value.is_empty() { (prop_name, prop_sub_name.to_string()) } else { (prop_name, format!("{}:{}", prop_sub_name, value)) } } else { (prop_name, value.to_string()) }; prop.values .entry(prop_name.to_string()) .or_default() .push(prop_value); } } } } if let Some(href) = href.take() { if !prop.is_empty() { props.push(prop); } result.hrefs.insert( href, DavProperties { status: href_status, props, }, ); } result } } impl DummyWebDavClient { pub async fn patch_and_check( &self, path: &str, properties: impl IntoIterator, ) where T: AsRef + Clone, { let mut expect_set = Vec::new(); let mut expect_remove = Vec::new(); for (key, value) in properties { if !value.is_empty() { expect_set.push((key, value)); } else { expect_remove.push(key); } } let response = self .proppatch( path, expect_set.iter().cloned(), expect_remove.iter().cloned(), [], ) .await .with_status(StatusCode::MULTI_STATUS) .into_propfind_response(None); let patch_prop = response.properties(path); for (key, _) in &expect_set { patch_prop.get(key.as_ref()).with_status(StatusCode::OK); } for key in &expect_remove { patch_prop .get(key.as_ref()) .with_status(StatusCode::NO_CONTENT); } let response = self .propfind( path, expect_set .iter() .map(|(k, _)| k) .chain(expect_remove.iter()), ) .await; let prop = response.properties(path); for (key, value) in expect_set { prop.get(key.as_ref()) .with_values([value]) .with_status(StatusCode::OK); } for key in expect_remove { prop.get(key.as_ref()).with_status(StatusCode::NOT_FOUND); } } pub async fn propfind(&self, path: &str, properties: I) -> DavMultiStatus where I: IntoIterator, T: AsRef, { self.propfind_with_headers(path, properties, []).await } pub async fn propfind_with_headers( &self, path: &str, properties: I, headers: impl IntoIterator, ) -> DavMultiStatus where I: IntoIterator, T: AsRef, { let mut request = concat!( "", "", "" ) .to_string(); for property in properties { request.push_str(&format!("<{}/>", property.as_ref())); } request.push_str(""); self.request_with_headers("PROPFIND", path, headers, &request) .await .with_status(StatusCode::MULTI_STATUS) .into_propfind_response(None) } pub async fn proppatch( &self, path: &str, set: impl IntoIterator, clear: impl IntoIterator, headers: impl IntoIterator, ) -> DavResponse where T: AsRef, { let mut request = concat!( "", "", "" ) .to_string(); for property in clear { request.push_str(&format!("<{}/>", property.as_ref())); } request.push_str(""); for (key, value) in set { let key = key.as_ref(); request.push_str(&format!("<{key}>{value}")); } request.push_str(""); self.request_with_headers("PROPPATCH", path, headers, &request) .await } } impl DavItem { pub fn is_empty(&self) -> bool { self.values.is_empty() && self.status == StatusCode::PROXY_AUTHENTICATION_REQUIRED && self.error.is_empty() && self.description.is_none() } } impl Default for DavItem { fn default() -> Self { DavItem { status: StatusCode::PROXY_AUTHENTICATION_REQUIRED, values: AHashMap::new(), error: Vec::new(), description: None, } } } const EXPAND_REPORT_QUERY: &str = r#" "#; pub const ALL_DAV_PROPERTIES: &[DavProperty] = &[ DavProperty::WebDav(WebDavProperty::CreationDate), DavProperty::WebDav(WebDavProperty::DisplayName), DavProperty::WebDav(WebDavProperty::GetContentLanguage), DavProperty::WebDav(WebDavProperty::GetContentLength), DavProperty::WebDav(WebDavProperty::GetContentType), DavProperty::WebDav(WebDavProperty::GetETag), DavProperty::WebDav(WebDavProperty::GetLastModified), DavProperty::WebDav(WebDavProperty::ResourceType), DavProperty::WebDav(WebDavProperty::LockDiscovery), DavProperty::WebDav(WebDavProperty::SupportedLock), DavProperty::WebDav(WebDavProperty::CurrentUserPrincipal), DavProperty::WebDav(WebDavProperty::QuotaAvailableBytes), DavProperty::WebDav(WebDavProperty::QuotaUsedBytes), DavProperty::WebDav(WebDavProperty::SupportedReportSet), DavProperty::WebDav(WebDavProperty::SyncToken), DavProperty::WebDav(WebDavProperty::Owner), DavProperty::WebDav(WebDavProperty::Group), DavProperty::WebDav(WebDavProperty::SupportedPrivilegeSet), DavProperty::WebDav(WebDavProperty::CurrentUserPrivilegeSet), DavProperty::WebDav(WebDavProperty::Acl), DavProperty::WebDav(WebDavProperty::AclRestrictions), DavProperty::WebDav(WebDavProperty::InheritedAclSet), DavProperty::WebDav(WebDavProperty::PrincipalCollectionSet), DavProperty::WebDav(WebDavProperty::GetCTag), DavProperty::CardDav(CardDavProperty::AddressbookDescription), DavProperty::CardDav(CardDavProperty::SupportedAddressData), DavProperty::CardDav(CardDavProperty::SupportedCollationSet), DavProperty::CardDav(CardDavProperty::MaxResourceSize), DavProperty::CalDav(CalDavProperty::CalendarDescription), DavProperty::CalDav(CalDavProperty::CalendarTimezone), DavProperty::CalDav(CalDavProperty::SupportedCalendarComponentSet), DavProperty::CalDav(CalDavProperty::SupportedCalendarData), DavProperty::CalDav(CalDavProperty::SupportedCollationSet), DavProperty::CalDav(CalDavProperty::MaxResourceSize), DavProperty::CalDav(CalDavProperty::MinDateTime), DavProperty::CalDav(CalDavProperty::MaxDateTime), DavProperty::CalDav(CalDavProperty::MaxInstances), DavProperty::CalDav(CalDavProperty::MaxAttendeesPerInstance), DavProperty::CalDav(CalDavProperty::TimezoneServiceSet), DavProperty::CalDav(CalDavProperty::TimezoneId), DavProperty::CalDav(CalDavProperty::ScheduleDefaultCalendarURL), DavProperty::CalDav(CalDavProperty::ScheduleTag), DavProperty::CalDav(CalDavProperty::ScheduleCalendarTransp), DavProperty::Principal(PrincipalProperty::AlternateURISet), DavProperty::Principal(PrincipalProperty::PrincipalURL), DavProperty::Principal(PrincipalProperty::GroupMemberSet), DavProperty::Principal(PrincipalProperty::GroupMembership), DavProperty::Principal(PrincipalProperty::CalendarHomeSet), DavProperty::Principal(PrincipalProperty::AddressbookHomeSet), DavProperty::Principal(PrincipalProperty::PrincipalAddress), DavProperty::Principal(PrincipalProperty::CalendarUserAddressSet), DavProperty::Principal(PrincipalProperty::CalendarUserType), DavProperty::Principal(PrincipalProperty::ScheduleInboxURL), DavProperty::Principal(PrincipalProperty::ScheduleOutboxURL), ]; fn serialize_status_code(status_code: &StatusCode, serializer: S) -> Result where S: serde::Serializer, { serializer.serialize_str(&status_code.to_string()) } ================================================ FILE: tests/src/webdav/put_get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use types::collection::Collection; use super::WebDavTest; use crate::webdav::*; pub async fn test(test: &WebDavTest) { println!("Running PUT/GET tests..."); let client = test.client("john"); // Simple PUT let mut files = AHashMap::new(); for (path, ct, content) in [ ("/dav/file/john/file1.txt", "text/plain", TEST_FILE_1), ("/dav/file/john/file2.txt", "text/x-other", TEST_FILE_2), ( "/dav/card/john/default/card1.vcf", "text/vcard; charset=utf-8", TEST_VCARD_1, ), ( "/dav/card/john/default/card2.vcf", "text/vcard; charset=utf-8", TEST_VCARD_2, ), ( "/dav/cal/john/default/event1.ics", "text/calendar; charset=utf-8", TEST_ICAL_1, ), ( "/dav/cal/john/default/event2.ics", "text/calendar; charset=utf-8", TEST_ICAL_2, ), ] { let content = content.replace("\n", "\r\n"); let etag = client .request_with_headers("PUT", path, [("content-type", ct)], &content) .await .with_status(StatusCode::CREATED) .etag() .to_string(); files.insert(path, (content, ct, etag)); } // Test GET for (path, (content, ct, etag)) in &files { client .request("GET", path, "") .await .with_status(StatusCode::OK) .with_header("etag", etag) .with_header("content-type", ct) .with_body(content); } // PUT under a non-existing parent should fail for (path, contents) in [ ("/dav/file/john/foo/file1.txt", TEST_FILE_1), ("/dav/card/john/foo/card1.vcf", TEST_VCARD_1), ("/dav/cal/john/foo/event1.ics", TEST_ICAL_1), ] { client .request("PUT", path, contents) .await .with_status(StatusCode::CONFLICT); } // PUT under resources should fail for (path, contents) in [ ("/dav/file/john/file1.txt/other-file.txt", TEST_FILE_1), ( "/dav/card/john/default/card1.vcf/other-file.vcf", TEST_VCARD_1, ), ( "/dav/cal/john/default/event1.ics/other-file.ical", TEST_ICAL_1, ), ] { client .request("PUT", path, contents) .await .with_status(StatusCode::METHOD_NOT_ALLOWED); } // PUT a non-vCard/iCalendar file should fail for (path, ct, content, precondition) in [ ( "/dav/card/john/card3.vcf", "text/vcard; charset=utf-8", TEST_FILE_1, "B:supported-address-data", ), ( "/dav/cal/john/event3.ics", "text/calendar; charset=utf-8", TEST_FILE_2, "A:supported-calendar-data", ), ] { client .request_with_headers("PUT", path, [("content-type", ct)], content) .await .with_status(StatusCode::PRECONDITION_FAILED) .with_failed_precondition(precondition, ""); } // Exceeding the configured file limits should fail let conf = &test.server.core.groupware; for (path, contents, max_size, expect) in [ ( "/dav/file/john/chunky-file1.txt", TEST_FILE_1, conf.max_file_size, None, ), ( "/dav/card/john/chunky-card1.vcf", TEST_VCARD_1, conf.max_vcard_size, Some("B:max-resource-size"), ), ( "/dav/cal/john/chunky-event1.ics", TEST_ICAL_1, conf.max_ical_size, Some("A:max-resource-size"), ), ] { let mut chunky_contents = String::with_capacity(max_size + contents.len()); while chunky_contents.len() < max_size { chunky_contents.push_str(contents); } let response = client .request("PUT", path, chunky_contents) .await .with_status( expect .map(|_| StatusCode::PRECONDITION_FAILED) .unwrap_or(StatusCode::PAYLOAD_TOO_LARGE), ); if let Some(expect) = expect { response.with_failed_precondition(expect, &max_size.to_string()); } } // PUT requests cannot exceed quota let mike_noquota = test.client("mike"); for resource_type in [ DavResourceName::File, DavResourceName::Card, DavResourceName::Cal, ] { let path = format!("{}/mike/quota-test/", resource_type.base_path()); mike_noquota .mkcol("MKCOL", &path, [], []) .await .with_status(StatusCode::CREATED); let mut num_success = 0; let mut did_fail = false; for i in 0..100 { let content = resource_type.generate(); let available = mike_noquota.available_quota(&path).await; let response = mike_noquota .request_with_headers("PUT", &format!("{path}file{i}"), [], &content) .await; if available > content.len() as u64 { num_success += 1; response.with_status(StatusCode::CREATED); } else { response .with_status(StatusCode::PRECONDITION_FAILED) .with_failed_precondition("D:quota-not-exceeded", ""); did_fail = true; break; } } if !did_fail { panic!("Quota test failed: {} files created", num_success); } if num_success == 0 { panic!("Quota test failed: no files created"); } mike_noquota .request("DELETE", &path, "") .await .with_status(StatusCode::NO_CONTENT); } // PUT precondition enforcement let modseq = [ test.resources("john", Collection::FileNode) .await .highest_change_id, test.resources("john", Collection::Calendar) .await .highest_change_id, test.resources("john", Collection::AddressBook) .await .highest_change_id, ]; for (path, ct, content) in [ ("/dav/file/john/file1.txt", "text/plain", TEST_FILE_1), ( "/dav/card/john/default/card1.vcf", "text/vcard; charset=utf-8", TEST_VCARD_1, ), ( "/dav/cal/john/default/event1.ics", "text/calendar; charset=utf-8", TEST_ICAL_1, ), ] { let content = content.replace("\n", "\r\n"); client .request_with_headers( "PUT", path, [("content-type", ct), ("if-none-match", "*")], &content, ) .await .with_status(StatusCode::PRECONDITION_FAILED); client .request_with_headers( "PUT", path, [("content-type", ct), ("overwrite", "F")], &content, ) .await .with_status(StatusCode::PRECONDITION_FAILED); client .request_with_headers( "PUT", path, [("content-type", ct), ("if", "([\"3827\"])")], &content, ) .await .with_status(StatusCode::PRECONDITION_FAILED); client .request_with_headers( "PUT", path, [ ("content-type", ct), ("if", "([\"3827\"])"), ("prefer", "return=representation"), ], &content, ) .await .with_status(StatusCode::PRECONDITION_FAILED) .with_header("preference-applied", "return=representation") .with_body(&content); } assert_eq!( [ test.resources("john", Collection::FileNode) .await .highest_change_id, test.resources("john", Collection::Calendar) .await .highest_change_id, test.resources("john", Collection::AddressBook) .await .highest_change_id, ], modseq ); // Update files using etags for (path, (content, ct, etag)) in &mut files { let condition = format!("([{}])", etag); *content = content.replace("X-TEST:SEQ1", "X-TEST:SEQ2"); *etag = client .request_with_headers( "PUT", path, [("content-type", &**ct), ("if", condition.as_str())], content.as_str(), ) .await .with_status(StatusCode::NO_CONTENT) .etag() .to_string(); } // Test GET for (path, (content, ct, etag)) in &files { client .request("GET", path, "") .await .with_status(StatusCode::OK) .with_header("etag", etag) .with_header("content-type", ct) .with_body(content); } // PUT requests require unique UIDs for (path, ct, content, precond_key, precond_value) in [ ( "/dav/card/john/default/card5.vcf", "text/vcard; charset=utf-8", TEST_VCARD_1, "B:no-uid-conflict.D:href", "/dav/card/john/default/card1.vcf", ), ( "/dav/cal/john/default/event5.ics", "text/calendar; charset=utf-8", TEST_ICAL_1, "A:no-uid-conflict.D:href", "/dav/cal/john/default/event1.ics", ), ] { client .request_with_headers( "PUT", path, [("content-type", ct), ("if-none-match", "*")], content, ) .await .with_status(StatusCode::PRECONDITION_FAILED) .with_failed_precondition(precond_key, precond_value); } // iCal containing different component types should fail client .request_with_headers( "PUT", "/dav/cal/john/default/invalid.ics", [ ("content-type", "text/calendar; charset=utf-8"), ("if-none-match", "*"), ], r#"BEGIN:VCALENDAR VERSION:2.0 BEGIN:VEVENT UID:1234567890 SUMMARY:Test Event DTSTART;TZID=Europe/London:20231001T120000 DTEND;TZID=Europe/London:20231001T130000 END:VEVENT BEGIN:VTODO UID:1234567890 SUMMARY:Test Task DTSTART;TZID=Europe/London:20231001T120000 DTEND;TZID=Europe/London:20231001T130000 END:VTODO END:VCALENDAR "#, ) .await .with_status(StatusCode::PRECONDITION_FAILED) .with_failed_precondition("A:valid-calendar-object-resource", ""); // iCal referencing more than one UID should fail client .request_with_headers( "PUT", "/dav/cal/john/default/invalid.ics", [ ("content-type", "text/calendar; charset=utf-8"), ("if-none-match", "*"), ], r#"BEGIN:VCALENDAR VERSION:2.0 BEGIN:VEVENT UID:1234567890 SUMMARY:Test Event 1 DTSTART;TZID=Europe/London:20231001T120000 DTEND;TZID=Europe/London:20231001T130000 END:VEVENT BEGIN:VEVENT UID:1234567891 SUMMARY:Test Event 2 DTSTART;TZID=Europe/London:20231001T120000 DTEND;TZID=Europe/London:20231001T130000 END:VEVENT END:VCALENDAR "#, ) .await .with_status(StatusCode::PRECONDITION_FAILED) .with_failed_precondition("A:valid-calendar-object-resource", ""); // Deleting unknown/invalid destinations should fail for (path, expect) in [ ("/dav/file/john/unknown.txt", StatusCode::NOT_FOUND), ("/dav/card/john/default/unknown.txt", StatusCode::NOT_FOUND), ("/dav/cal/john/default/unknown.txt", StatusCode::NOT_FOUND), ("/dav/file/john", StatusCode::FORBIDDEN), ("/dav/cal/john", StatusCode::FORBIDDEN), ("/dav/card/john", StatusCode::FORBIDDEN), ("/dav/pal/john", StatusCode::METHOD_NOT_ALLOWED), ("/dav/file", StatusCode::FORBIDDEN), ("/dav/cal", StatusCode::FORBIDDEN), ("/dav/card", StatusCode::FORBIDDEN), ("/dav/pal", StatusCode::METHOD_NOT_ALLOWED), ] { client.request("DELETE", path, "").await.with_status(expect); } // Delete files for (path, (_, _, etag)) in &files { client .request_with_headers("DELETE", path, [("if", "([\"3827\"])")], "") .await .with_status(StatusCode::PRECONDITION_FAILED); let condition = format!("([{}])", etag); client .request_with_headers("DELETE", path, [("if", condition.as_str())], "") .await .with_status(StatusCode::NO_CONTENT); client .request("DELETE", path, "") .await .with_status(StatusCode::NOT_FOUND); } client.delete_default_containers().await; mike_noquota.delete_default_containers().await; test.assert_is_empty().await; } ================================================ FILE: tests/src/webdav/sync.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::{DavResponse, DummyWebDavClient, WebDavTest}; use crate::webdav::GenerateTestDavResource; use ahash::AHashSet; use dav_proto::Depth; use groupware::DavResourceName; use hyper::StatusCode; pub async fn test(test: &WebDavTest) { let client = test.client("john"); for resource_type in [ DavResourceName::File, DavResourceName::Cal, DavResourceName::Card, ] { println!( "Running REPORT sync-collection tests ({})...", resource_type.base_path() ); let user_base_path = format!("{}/john/", resource_type.base_path()); // Test 1: Initial sync let response = client .sync_collection(&user_base_path, "", Depth::Infinity, None, ["D:getetag"]) .await; assert_eq!( response.hrefs().len(), if resource_type == DavResourceName::File { 1 } else { 2 }, "{:?}", response.hrefs() ); let sync_token_1 = response.sync_token().to_string(); // Test 2: No changes since last sync let response = client .sync_collection( &user_base_path, &sync_token_1, Depth::Infinity, None, ["D:getetag"], ) .await; assert_eq!(response.hrefs(), Vec::::new()); // Test 3: Create a collection and make sure it is synced let new_collection = format!("{}new-collection/", user_base_path); client .mkcol("MKCOL", &new_collection, [], []) .await .with_status(StatusCode::CREATED); let response = client .sync_collection( &user_base_path, &sync_token_1, Depth::Infinity, None, ["D:getetag"], ) .await; assert_eq!(response.hrefs(), vec![new_collection.clone()]); let sync_token_2 = response.sync_token().to_string(); // Test 4: Create a file and make sure it is synced let new_file = format!("{new_collection}new-file"); let contents = resource_type.generate(); client .request("PUT", &new_file, &contents) .await .with_status(StatusCode::CREATED); let response = client .sync_collection( &user_base_path, &sync_token_1, Depth::Infinity, None, ["D:getetag"], ) .await; assert_eq!( response.hrefs(), vec![new_collection.clone(), new_file.clone()] ); let sync_token_3 = response.sync_token().to_string(); let response = client .sync_collection( &user_base_path, &sync_token_2, Depth::Infinity, None, ["D:getetag"], ) .await; assert_eq!(response.hrefs(), vec![new_file.clone()]); // Test 5: sync-token with Depth 1 let response = client .sync_collection( &user_base_path, &sync_token_1, Depth::One, None, ["D:getetag"], ) .await; assert_eq!(response.hrefs(), vec![new_collection.clone()]); // Test 6: sync-token with Depth 0 let response = client .sync_collection( &new_collection, &sync_token_1, Depth::Zero, None, ["D:getetag"], ) .await; assert_eq!(response.hrefs(), vec![new_collection.clone()]); // Test 7: Outdated sync-token in If header should fail let new_file2 = format!("{new_collection}new-file2"); let contents = resource_type.generate(); let condition = format!("(<{sync_token_2}>)"); client .request_with_headers( "PUT", &new_file2, [("if", condition.as_str())], contents.as_str(), ) .await .with_status(StatusCode::PRECONDITION_FAILED) .with_empty_body(); // Test 8: Correct sync-token in If header should work let condition = format!("(<{sync_token_3}>)"); client .request_with_headers( "PUT", &new_file2, [("if", condition.as_str())], contents.as_str(), ) .await .with_status(StatusCode::CREATED) .with_empty_body(); // Test 9: Limit let mut sync_token = client .sync_collection( &new_collection, &sync_token_3, Depth::Zero, None, ["D:getetag"], ) .await .sync_token() .to_string(); let (folder_name, files) = client .create_hierarchy(user_base_path.trim_end_matches('/'), 1, 0, 10) .await; let mut expected_changes = files .iter() .map(|x| x.0.as_str()) .chain([folder_name.as_str()]) .collect::>(); for _ in 0..10 { let response = client .sync_collection( &user_base_path, &sync_token, Depth::Infinity, 2.into(), ["D:getetag"], ) .await; sync_token = response.sync_token().to_string(); let hrefs = response.hrefs(); if hrefs.is_empty() { break; } let mut has_user_base_path = false; let mut item_count = 0; for href in hrefs { if href == user_base_path { has_user_base_path = true; } else if expected_changes.remove(href) { item_count += 1; } else { panic!("Unexpected href: {href}"); } } if has_user_base_path { assert_eq!(item_count, 2); response .with_value( "D:multistatus.D:response.D:status", "HTTP/1.1 507 Insufficient Storage", ) .with_value( "D:multistatus.D:response.D:error.D:number-of-matches-within-limits", "", ) .with_value( "D:multistatus.D:response.D:responsedescription", "The number of matches exceeds the limit of 2", ); } else { assert!(item_count <= 2); break; } } assert!(expected_changes.is_empty(), "{:?}", expected_changes); // Test 10: Expect changes after deletion client .request("DELETE", &new_file, "") .await .with_status(StatusCode::NO_CONTENT); let response = client .sync_collection( &user_base_path, &sync_token, Depth::Infinity, None, ["D:getetag"], ) .await; sync_token = response.sync_token().to_string(); response .with_href_count(1) .with_value("D:multistatus.D:response.D:href", &new_file) .with_value( "D:multistatus.D:response.D:status", "HTTP/1.1 404 Not Found", ); client .request("DELETE", &new_collection, "") .await .with_status(StatusCode::NO_CONTENT); let response = client .sync_collection( &user_base_path, &sync_token, Depth::Infinity, None, ["D:getetag"], ) .await; sync_token = response.sync_token().to_string(); response .with_href_count(1) .with_value("D:multistatus.D:response.D:href", &new_collection) .with_value( "D:multistatus.D:response.D:status", "HTTP/1.1 404 Not Found", ); client .request("DELETE", &folder_name, "") .await .with_status(StatusCode::NO_CONTENT); client .sync_collection( &user_base_path, &sync_token, Depth::Infinity, None, ["D:getetag"], ) .await .with_href_count(1) .with_value("D:multistatus.D:response.D:href", &folder_name) .with_value( "D:multistatus.D:response.D:status", "HTTP/1.1 404 Not Found", ); } client.delete_default_containers().await; test.assert_is_empty().await; } impl DummyWebDavClient { pub async fn sync_collection( &self, path: &str, sync_token: &str, depth: Depth, limit: Option, properties: impl IntoIterator, ) -> DavResponse { let mut request = concat!( "", "", "" ) .to_string(); for property in properties { request.push_str(&format!("<{property}/>")); } request.push_str(""); request.push_str(sync_token); request.push_str(""); request.push_str(match depth { Depth::One => "1", Depth::Infinity => "infinite", _ => "0", }); request.push_str(""); if let Some(limit) = limit { request.push_str(""); request.push_str(&limit.to_string()); request.push_str(""); } request.push_str(""); self.request("REPORT", path, &request) .await .with_status(StatusCode::MULTI_STATUS) } }

: serde::Serialize"))] pub struct SetError { #[serde(rename = "type")] pub type_: SetErrorType, #[serde(skip_serializing_if = "Option::is_none")] pub description: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub properties: Option>>, #[serde(rename = "existingId")] #[serde(skip_serializing_if = "Option::is_none")] existing_id: Option, } #[derive(Debug, Clone)] pub enum InvalidProperty { Property(Key<'static, T>), Path(Vec>), } #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] pub enum SetErrorType { #[serde(rename = "forbidden")] Forbidden, #[serde(rename = "overQuota")] OverQuota, #[serde(rename = "tooLarge")] TooLarge, #[serde(rename = "rateLimit")] RateLimit, #[serde(rename = "notFound")] NotFound, #[serde(rename = "invalidPatch")] InvalidPatch, #[serde(rename = "willDestroy")] WillDestroy, #[serde(rename = "invalidProperties")] InvalidProperties, #[serde(rename = "singleton")] Singleton, #[serde(rename = "mailboxHasChild")] MailboxHasChild, #[serde(rename = "mailboxHasEmail")] MailboxHasEmail, #[serde(rename = "blobNotFound")] BlobNotFound, #[serde(rename = "tooManyKeywords")] TooManyKeywords, #[serde(rename = "tooManyMailboxes")] TooManyMailboxes, #[serde(rename = "forbiddenFrom")] ForbiddenFrom, #[serde(rename = "invalidEmail")] InvalidEmail, #[serde(rename = "tooManyRecipients")] TooManyRecipients, #[serde(rename = "noRecipients")] NoRecipients, #[serde(rename = "invalidRecipients")] InvalidRecipients, #[serde(rename = "forbiddenMailFrom")] ForbiddenMailFrom, #[serde(rename = "forbiddenToSend")] ForbiddenToSend, #[serde(rename = "cannotUnsend")] CannotUnsend, #[serde(rename = "alreadyExists")] AlreadyExists, #[serde(rename = "invalidScript")] InvalidScript, #[serde(rename = "scriptIsActive")] ScriptIsActive, #[serde(rename = "addressBookHasContents")] AddressBookHasContents, #[serde(rename = "nodeHasChildren")] NodeHasChildren, #[serde(rename = "calendarHasEvent")] CalendarHasEvent, } impl SetErrorType { pub fn as_str(&self) -> &'static str { match self { SetErrorType::Forbidden => "forbidden", SetErrorType::OverQuota => "overQuota", SetErrorType::TooLarge => "tooLarge", SetErrorType::RateLimit => "rateLimit", SetErrorType::NotFound => "notFound", SetErrorType::InvalidPatch => "invalidPatch", SetErrorType::WillDestroy => "willDestroy", SetErrorType::InvalidProperties => "invalidProperties", SetErrorType::Singleton => "singleton", SetErrorType::BlobNotFound => "blobNotFound", SetErrorType::MailboxHasChild => "mailboxHasChild", SetErrorType::MailboxHasEmail => "mailboxHasEmail", SetErrorType::TooManyKeywords => "tooManyKeywords", SetErrorType::TooManyMailboxes => "tooManyMailboxes", SetErrorType::ForbiddenFrom => "forbiddenFrom", SetErrorType::InvalidEmail => "invalidEmail", SetErrorType::TooManyRecipients => "tooManyRecipients", SetErrorType::NoRecipients => "noRecipients", SetErrorType::InvalidRecipients => "invalidRecipients", SetErrorType::ForbiddenMailFrom => "forbiddenMailFrom", SetErrorType::ForbiddenToSend => "forbiddenToSend", SetErrorType::CannotUnsend => "cannotUnsend", SetErrorType::AlreadyExists => "alreadyExists", SetErrorType::InvalidScript => "invalidScript", SetErrorType::ScriptIsActive => "scriptIsActive", SetErrorType::AddressBookHasContents => "addressBookHasContents", SetErrorType::NodeHasChildren => "nodeHasChildren", SetErrorType::CalendarHasEvent => "calendarHasEvent", } } } impl SetError { pub fn new(type_: SetErrorType) -> Self { SetError { type_, description: None, properties: None, existing_id: None, } } pub fn with_description(mut self, description: impl Into>) -> Self { self.description = description.into().into(); self } pub fn with_property(mut self, property: impl Into>) -> Self { self.properties = vec![property.into()].into(); self } pub fn with_properties( mut self, properties: impl IntoIterator>>, ) -> Self { self.properties = properties .into_iter() .map(Into::into) .collect::>() .into(); self } pub fn with_existing_id(mut self, id: Id) -> Self { self.existing_id = id.into(); self } pub fn invalid_properties() -> Self { Self::new(SetErrorType::InvalidProperties) } pub fn forbidden() -> Self { Self::new(SetErrorType::Forbidden) } pub fn not_found() -> Self { Self::new(SetErrorType::NotFound) } pub fn blob_not_found() -> Self { Self::new(SetErrorType::BlobNotFound) } pub fn over_quota() -> Self { Self::new(SetErrorType::OverQuota).with_description("Account quota exceeded.") } pub fn already_exists() -> Self { Self::new(SetErrorType::AlreadyExists) } pub fn too_large() -> Self { Self::new(SetErrorType::TooLarge) } pub fn will_destroy() -> Self { Self::new(SetErrorType::WillDestroy).with_description("ID will be destroyed.") } pub fn address_book_has_contents() -> Self { Self::new(SetErrorType::AddressBookHasContents) .with_description("Address book is not empty.") } pub fn node_has_children() -> Self { Self::new(SetErrorType::NodeHasChildren).with_description("Cannot delete non-empty folder.") } pub fn calendar_has_event() -> Self { Self::new(SetErrorType::CalendarHasEvent).with_description("Calendar is not empty.") } } impl From for InvalidProperty { fn from(property: T) -> Self { InvalidProperty::Property(Key::Property(property)) } } impl From<(T, T)> for InvalidProperty { fn from((a, b): (T, T)) -> Self { InvalidProperty::Path(vec![Key::Property(a), Key::Property(b)]) } } impl From> for InvalidProperty { fn from(property: Key<'static, T>) -> Self { InvalidProperty::Property(property) } } impl serde::Serialize for InvalidProperty { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { match self { InvalidProperty::Property(p) => p.serialize(serializer), InvalidProperty::Path(p) => { use std::fmt::Write; let mut path = String::with_capacity(64); for (i, p) in p.iter().enumerate() { if i > 0 { path.push('/'); } let _ = write!(path, "{}", p.to_string()); } path.serialize(serializer) } } } } ================================================ FILE: crates/jmap-proto/src/lib.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ pub mod error; pub mod method; pub mod object; pub mod references; pub mod request; pub mod response; pub mod types; ================================================ FILE: crates/jmap-proto/src/method/availability.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ request::{ MaybeInvalid, deserialize::{DeserializeArguments, deserialize_request}, }, types::date::UTCDate, }; use calcard::jscalendar::{JSCalendar, JSCalendarProperty}; use serde::{Deserialize, Deserializer, Serialize}; use types::{blob::BlobId, id::Id}; #[derive(Debug, Clone, Default)] pub struct GetAvailabilityRequest { pub account_id: Id, pub id: Id, pub utc_start: UTCDate, pub utc_end: UTCDate, pub show_details: bool, pub event_properties: Option>>>, } #[derive(Debug, Serialize, Clone)] #[serde(rename_all = "camelCase")] pub struct GetAvailabilityResponse { pub list: Vec, } #[derive(Debug, Serialize, Clone)] #[serde(rename_all = "camelCase")] pub struct BusyPeriod { pub utc_start: UTCDate, pub utc_end: UTCDate, pub busy_status: Option, pub event: Option>, } #[derive(Debug, Serialize, Clone, Copy, PartialOrd, Ord, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum BusyStatus { Confirmed, Tentative, Unavailable, } impl<'de> DeserializeArguments<'de> for GetAvailabilityRequest { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"accountId" => { self.account_id = map.next_value()?; }, b"utcStart" => { self.utc_start = map.next_value()?; }, b"utcEnd" => { self.utc_end = map.next_value()?; }, b"id" => { self.id = map.next_value()?; }, b"showDetails" => { self.show_details = map.next_value()?; }, b"eventProperties" => { self.event_properties = map.next_value()?; }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> Deserialize<'de> for GetAvailabilityRequest { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserialize_request(deserializer) } } ================================================ FILE: crates/jmap-proto/src/method/changes.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ method::PropertyWrapper, object::JmapObject, request::deserialize::{DeserializeArguments, deserialize_request}, types::state::State, }; use serde::{Deserialize, Deserializer}; use types::id::Id; #[derive(Debug, Clone, Default)] pub struct ChangesRequest { pub account_id: Id, pub since_state: State, pub max_changes: Option, } #[derive(Debug, Clone, serde::Serialize)] pub struct ChangesResponse { #[serde(rename = "accountId")] pub account_id: Id, #[serde(rename = "oldState")] pub old_state: State, #[serde(rename = "newState")] pub new_state: State, #[serde(rename = "hasMoreChanges")] pub has_more_changes: bool, pub created: Vec, pub updated: Vec, pub destroyed: Vec, #[serde(rename = "updatedProperties")] #[serde(skip_serializing_if = "Option::is_none")] pub updated_properties: Option>>, } impl<'de> DeserializeArguments<'de> for ChangesRequest { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"accountId" => { self.account_id = map.next_value()?; }, b"sinceState" => { self.since_state = map.next_value()?; }, b"maxChanges" => { self.max_changes = map.next_value()?; }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> Deserialize<'de> for ChangesRequest { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserialize_request(deserializer) } } impl ChangesResponse { pub fn has_changes(&self) -> bool { !self.created.is_empty() || !self.updated.is_empty() || !self.destroyed.is_empty() } } ================================================ FILE: crates/jmap-proto/src/method/copy.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ error::set::SetError, object::{JmapObject, blob::BlobProperty}, request::{ MaybeInvalid, deserialize::{DeserializeArguments, deserialize_request}, reference::MaybeIdReference, }, types::state::State, }; use jmap_tools::{Key, Map, Value}; use serde::{Deserialize, Deserializer, Serialize}; use types::{blob::BlobId, id::Id}; use utils::map::vec_map::VecMap; #[derive(Debug, Clone)] pub struct CopyRequest<'x, T: JmapObject> { pub from_account_id: Id, pub if_from_in_state: Option, pub account_id: Id, pub if_in_state: Option, pub create: VecMap, Value<'x, T::Property, T::Element>>, pub on_success_destroy_original: Option, pub destroy_from_if_in_state: Option, } #[derive(Debug, Clone, serde::Serialize)] pub struct CopyResponse { #[serde(rename = "fromAccountId")] pub from_account_id: Id, #[serde(rename = "accountId")] pub account_id: Id, #[serde(rename = "oldState")] pub old_state: State, #[serde(rename = "newState")] pub new_state: State, #[serde(rename = "created")] #[serde(skip_serializing_if = "VecMap::is_empty")] pub created: VecMap>, #[serde(rename = "notCreated")] #[serde(skip_serializing_if = "VecMap::is_empty")] pub not_created: VecMap>, } #[derive(Debug, Clone, Default)] pub struct CopyBlobRequest { pub from_account_id: Id, pub account_id: Id, pub blob_ids: Vec>, } #[derive(Debug, Clone, Serialize)] pub struct CopyBlobResponse { #[serde(rename = "fromAccountId")] pub from_account_id: Id, #[serde(rename = "accountId")] pub account_id: Id, #[serde(rename = "copied")] #[serde(skip_serializing_if = "VecMap::is_empty")] pub copied: VecMap, #[serde(rename = "notCopied")] #[serde(skip_serializing_if = "VecMap::is_empty")] pub not_copied: VecMap>, } impl<'de, T: JmapObject> DeserializeArguments<'de> for CopyRequest<'de, T> { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"accountId" => { self.account_id = map.next_value()?; }, b"ifInState" => { self.if_in_state = map.next_value()?; }, b"fromAccountId" => { self.from_account_id = map.next_value()?; }, b"ifFromInState" => { self.if_from_in_state = map.next_value()?; }, b"create" => { self.create = map.next_value()?; }, b"onSuccessDestroyOriginal" => { self.on_success_destroy_original = map.next_value()?; }, b"destroyFromIfInState" => { self.destroy_from_if_in_state = map.next_value()?; }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> DeserializeArguments<'de> for CopyBlobRequest { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"accountId" => { self.account_id = map.next_value()?; }, b"fromAccountId" => { self.from_account_id = map.next_value()?; }, b"blobIds" => { self.blob_ids = map.next_value()?; }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl<'de, T: JmapObject> Deserialize<'de> for CopyRequest<'de, T> { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserialize_request(deserializer) } } impl<'de> Deserialize<'de> for CopyBlobRequest { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserialize_request(deserializer) } } impl<'de, T: JmapObject> Default for CopyRequest<'de, T> { fn default() -> Self { CopyRequest { from_account_id: Id::default(), if_from_in_state: None, account_id: Id::default(), if_in_state: None, create: VecMap::new(), on_success_destroy_original: None, destroy_from_if_in_state: None, } } } impl CopyResponse { pub fn created(&mut self, id: Id, document_id: impl Into) { let document_id = document_id.into(); self.created.append( id, Value::Object(Map::from(vec![( Key::Property(T::ID_PROPERTY), Value::Element(document_id.into()), )])), ); } } ================================================ FILE: crates/jmap-proto/src/method/get.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ object::JmapObject, request::{ IntoValid, MaybeInvalid, deserialize::{DeserializeArguments, deserialize_request}, reference::{MaybeIdReference, MaybeResultReference, ResultReference}, }, types::state::State, }; use jmap_tools::Value; use serde::{Deserialize, Deserializer}; use types::id::Id; #[derive(Debug, Clone)] pub struct GetRequest { pub account_id: Id, pub ids: Option>>>, pub properties: Option>>>, pub arguments: T::GetArguments, } #[derive(Debug, Clone, serde::Serialize)] pub struct GetResponse { #[serde(rename = "accountId")] #[serde(skip_serializing_if = "Option::is_none")] pub account_id: Option, #[serde(skip_serializing_if = "Option::is_none")] pub state: Option, pub list: Vec>, #[serde(rename = "notFound")] pub not_found: Vec, } impl<'de, T: JmapObject> DeserializeArguments<'de> for GetRequest { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"accountId" => { self.account_id = map.next_value()?; }, b"ids" => { self.ids = map.next_value::>>>()?.map(MaybeResultReference::Value); }, b"properties" => { self.properties = map.next_value::>>>()?.map(MaybeResultReference::Value); }, b"#ids" => { self.ids = Some(MaybeResultReference::Reference(map.next_value::()?)); }, b"#properties" => { self.properties = Some(MaybeResultReference::Reference(map.next_value::()?)); }, _ => { self.arguments.deserialize_argument(key, map)?; } ); Ok(()) } } impl<'de, T: JmapObject> Deserialize<'de> for GetRequest { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserialize_request(deserializer) } } impl Default for GetRequest { fn default() -> Self { Self { account_id: Id::default(), ids: None, properties: None, arguments: T::GetArguments::default(), } } } impl GetRequest { pub fn unwrap_properties(&mut self, default: &[T::Property]) -> Vec { if let Some(properties_) = self.properties.take().map(|p| p.unwrap()) { let mut properties = Vec::with_capacity(properties_.len()); let id_prop = T::ID_PROPERTY; let mut has_id = false; for prop in properties_ { if let MaybeInvalid::Value(p) = prop { if p == id_prop { has_id = true; } properties.push(p); } } if !has_id { properties.push(id_prop); } properties } else { default.to_vec() } } pub fn unwrap_ids(&mut self, max_objects_in_get: usize) -> trc::Result>> { if let Some(ids) = self.ids.take() { let ids = ids.unwrap(); if ids.len() <= max_objects_in_get { Ok(Some(ids.into_valid().collect::>())) } else { Err(trc::JmapEvent::RequestTooLarge.into_err()) } } else { Ok(None) } } } ================================================ FILE: crates/jmap-proto/src/method/import.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ error::set::SetError, method::JmapDict, object::{ AnyId, email::{EmailProperty, EmailValue}, }, request::{ MaybeInvalid, deserialize::{DeserializeArguments, deserialize_request}, reference::{MaybeIdReference, MaybeResultReference, ResultReference}, }, response::Response, types::{date::UTCDate, state::State}, }; use jmap_tools::{Key, Value}; use serde::{Deserialize, Deserializer}; use types::{blob::BlobId, id::Id, keyword::Keyword}; use utils::map::vec_map::VecMap; #[derive(Debug, Clone, Default)] pub struct ImportEmailRequest { pub account_id: Id, pub if_in_state: Option, pub emails: VecMap, } #[derive(Debug, Clone, Default)] pub struct ImportEmail { pub blob_id: MaybeInvalid, pub mailbox_ids: MaybeResultReference>>, pub keywords: Vec, pub received_at: Option, } #[derive(Debug, Clone, serde::Serialize)] pub struct ImportEmailResponse { #[serde(rename = "accountId")] pub account_id: Id, #[serde(rename = "oldState")] #[serde(skip_serializing_if = "Option::is_none")] pub old_state: Option, #[serde(rename = "newState")] pub new_state: State, #[serde(rename = "created")] #[serde(skip_serializing_if = "VecMap::is_empty")] pub created: VecMap>, #[serde(rename = "notCreated")] #[serde(skip_serializing_if = "VecMap::is_empty")] pub not_created: VecMap>, } impl<'de> DeserializeArguments<'de> for ImportEmailRequest { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"accountId" => { self.account_id = map.next_value()?; }, b"ifInState" => { self.if_in_state = map.next_value()?; }, b"emails" => { self.emails = map.next_value()?; } _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> DeserializeArguments<'de> for ImportEmail { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"blobId" => { self.blob_id = map.next_value()?; }, b"keywords" => { self.keywords = map.next_value::>()?.0; }, b"receivedAt" => { self.received_at = map.next_value()?; }, b"mailboxIds" => { self.mailbox_ids = MaybeResultReference::Value(map.next_value::>>()?.0); }, b"#mailboxIds" => { self.mailbox_ids = MaybeResultReference::Reference(map.next_value::()?); }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> Deserialize<'de> for ImportEmail { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserialize_request(deserializer) } } impl<'de> Deserialize<'de> for ImportEmailRequest { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserialize_request(deserializer) } } impl ImportEmailResponse { pub fn update_created_ids(&self, response: &mut Response) { for (user_id, obj) in &self.created { if let Value::Object(obj) = obj && let Some(Value::Element(EmailValue::Id(id))) = obj.get(&Key::Property(EmailProperty::Id)) { response.created_ids.insert(user_id.clone(), AnyId::Id(*id)); } } } } ================================================ FILE: crates/jmap-proto/src/method/lookup.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::request::{ MaybeInvalid, deserialize::{DeserializeArguments, deserialize_request}, }; use serde::{Deserialize, Deserializer}; use types::{blob::BlobId, id::Id, type_state::DataType}; use utils::map::vec_map::VecMap; #[derive(Debug, Clone, Default)] pub struct BlobLookupRequest { pub account_id: Id, pub type_names: Vec>, pub ids: Vec>, } #[derive(Debug, Clone, Default, serde::Serialize)] pub struct BlobLookupResponse { #[serde(rename = "accountId")] pub account_id: Id, #[serde(rename = "list")] pub list: Vec, #[serde(rename = "notFound")] pub not_found: Vec, } #[derive(Debug, Clone, Default, serde::Serialize)] pub struct BlobInfo { pub id: BlobId, #[serde(rename = "matchedIds")] pub matched_ids: VecMap>, } impl<'de> DeserializeArguments<'de> for BlobLookupRequest { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"accountId" => { self.account_id = map.next_value()?; }, b"typeNames" => { self.type_names = map.next_value()?; }, b"ids" => { self.ids = map.next_value()?; }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> Deserialize<'de> for BlobLookupRequest { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserialize_request(deserializer) } } ================================================ FILE: crates/jmap-proto/src/method/mod.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use ahash::AHashMap; use jmap_tools::Property; use serde::{ Deserialize, Deserializer, Serialize, Serializer, de::{self, MapAccess, Visitor}, }; use std::{borrow::Cow, fmt, str::FromStr}; pub mod availability; pub mod changes; pub mod copy; pub mod get; pub mod import; pub mod lookup; pub mod parse; pub mod query; pub mod query_changes; pub mod search_snippet; pub mod set; pub mod upload; pub mod validate; #[inline(always)] fn ahash_is_empty(map: &AHashMap) -> bool { map.is_empty() } #[derive(Debug, Clone)] #[repr(transparent)] pub struct PropertyWrapper(pub T); impl From for PropertyWrapper { fn from(value: T) -> Self { Self(value) } } impl Serialize for PropertyWrapper { fn serialize(&self, serializer: S) -> Result where S: Serializer, { serializer.serialize_str(self.0.to_cow().as_ref()) } } pub(crate) struct JmapDict(pub Vec); struct JmapDictVisitor<'de, T: FromStr> { marker: std::marker::PhantomData<&'de T>, } impl<'de, T: FromStr> Visitor<'de> for JmapDictVisitor<'de, T> { type Value = JmapDict; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("a map") } fn visit_map(self, mut access: M) -> Result where M: MapAccess<'de>, { let mut vec = Vec::with_capacity(3); while let Some(key) = access.next_key::>()? { let key = T::from_str(&key).map_err(|_| de::Error::custom("invalid dictionary key"))?; if access.next_value::>()?.unwrap_or(false) { vec.push(key); } } Ok(JmapDict(vec)) } } impl<'de, T: FromStr + 'static> Deserialize<'de> for JmapDict { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserializer.deserialize_map(JmapDictVisitor { marker: std::marker::PhantomData, }) } } ================================================ FILE: crates/jmap-proto/src/method/parse.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ object::JmapObject, request::{ MaybeInvalid, deserialize::{DeserializeArguments, deserialize_request}, reference::MaybeIdReference, }, }; use jmap_tools::Value; use serde::{Deserialize, Deserializer}; use types::{blob::BlobId, id::Id}; use utils::map::vec_map::VecMap; #[derive(Debug, Clone)] pub struct ParseRequest { pub account_id: Id, pub blob_ids: Vec>, pub properties: Option>>, pub arguments: T::ParseArguments, } #[derive(Debug, Clone, serde::Serialize)] pub struct ParseResponse { #[serde(rename = "accountId")] pub account_id: Id, #[serde(rename = "parsed")] #[serde(skip_serializing_if = "VecMap::is_empty")] pub parsed: VecMap>, #[serde(rename = "notParsable")] #[serde(skip_serializing_if = "Vec::is_empty")] pub not_parsable: Vec, #[serde(rename = "notFound")] #[serde(skip_serializing_if = "Vec::is_empty")] pub not_found: Vec, } impl<'de, T: JmapObject> DeserializeArguments<'de> for ParseRequest { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"accountId" => { self.account_id = map.next_value()?; }, b"blobIds" => { self.blob_ids = map.next_value()?; }, b"properties" => { self.properties = map.next_value()?; }, _ => { self.arguments.deserialize_argument(key, map)?; } ); Ok(()) } } impl<'de, T: JmapObject> Deserialize<'de> for ParseRequest { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserialize_request(deserializer) } } impl Default for ParseRequest { fn default() -> Self { Self { account_id: Id::default(), blob_ids: Vec::default(), properties: None, arguments: T::ParseArguments::default(), } } } ================================================ FILE: crates/jmap-proto/src/method/query.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ object::JmapObject, request::deserialize::{DeserializeArguments, deserialize_request}, types::state::State, }; use serde::{ Deserialize, Deserializer, de::{self, DeserializeSeed, MapAccess, SeqAccess, Visitor}, }; use std::{ borrow::Cow, fmt::{self}, }; use types::id::Id; #[derive(Debug, Clone)] pub struct QueryRequest { pub account_id: Id, pub filter: Vec>, pub sort: Option>>, pub position: Option, pub anchor: Option, pub anchor_offset: Option, pub limit: Option, pub calculate_total: Option, pub arguments: T::QueryArguments, } #[derive(Debug, Clone, serde::Serialize)] pub struct QueryResponse { #[serde(rename = "accountId")] pub account_id: Id, #[serde(rename = "queryState")] pub query_state: State, #[serde(rename = "canCalculateChanges")] pub can_calculate_changes: bool, #[serde(rename = "position")] pub position: i32, #[serde(rename = "ids")] pub ids: Vec, #[serde(rename = "total")] #[serde(skip_serializing_if = "Option::is_none")] pub total: Option, #[serde(rename = "limit")] #[serde(skip_serializing_if = "Option::is_none")] pub limit: Option, } #[derive(Clone, Debug)] pub enum Filter where T: for<'de> DeserializeArguments<'de> + Default, { Property(T), And, Or, Not, Close, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct Comparator where T: for<'de> DeserializeArguments<'de> + Default, { pub is_ascending: bool, pub collation: Option, pub property: T, } impl<'de, T: JmapObject> DeserializeArguments<'de> for QueryRequest { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"accountId" => { self.account_id = map.next_value()?; }, b"filter" => { self.filter = map.next_value::>()?.0; }, b"sort" => { self.sort = map.next_value()?; }, b"calculateTotal" => { self.calculate_total = map.next_value()?; }, b"position" => { self.position = map.next_value()?; }, b"anchor" => { self.anchor = map.next_value()?; }, b"anchorOffset" => { self.anchor_offset = map.next_value()?; }, b"limit" => { self.limit = map.next_value()?; }, _ => { self.arguments.deserialize_argument(key, map)?; } ); Ok(()) } } impl<'de, T: JmapObject> Deserialize<'de> for QueryRequest { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserialize_request(deserializer) } } impl Default for QueryRequest { fn default() -> Self { Self { account_id: Id::default(), filter: vec![], sort: None, position: None, anchor: None, anchor_offset: None, limit: None, calculate_total: None, arguments: T::QueryArguments::default(), } } } struct FilterMapCollector<'x, T: 'x>(&'x mut Vec>) where T: for<'de> DeserializeArguments<'de> + Default; struct FilterListCollector<'x, T: 'x>(&'x mut Vec>) where T: for<'de> DeserializeArguments<'de> + Default; pub(super) struct FilterWrapper(pub Vec>) where T: for<'de> DeserializeArguments<'de> + Default; impl<'de, T> Deserialize<'de> for FilterWrapper where T: for<'de2> DeserializeArguments<'de2> + Default, { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { let mut items = Vec::new(); FilterMapCollector(&mut items) .deserialize(deserializer) .map(|_| FilterWrapper(items)) } } impl<'de, 'x, T> DeserializeSeed<'de> for FilterMapCollector<'x, T> where T: for<'de2> DeserializeArguments<'de2> + Default, { type Value = (); fn deserialize(self, deserializer: D) -> Result where D: Deserializer<'de>, { struct FilterVisitor<'x, T: 'x>(&'x mut Vec>) where T: for<'de2> DeserializeArguments<'de2> + Default; impl<'de, 'x, T> Visitor<'de> for FilterVisitor<'x, T> where T: for<'de2> DeserializeArguments<'de2> + Default, { type Value = (); fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { write!(formatter, "a filter object") } fn visit_map(self, mut map: V) -> Result<(), V::Error> where V: MapAccess<'de>, { let mut filter = None; let mut has_multiple_filters = false; let mut has_conditions = None; let mut op = None; while let Some(key) = map.next_key::>()? { match key.len() { 8 if key == "operator" => { let op_ = hashify::tiny_map!( map.next_value::<&str>()?.as_bytes(), "AND" => Filter::And, "OR" => Filter::Or, "NOT" => Filter::Not, ) .ok_or_else(|| { de::Error::custom(format!("Unknown filter operator: {}", key)) })?; if let Some(pos) = has_conditions { self.0[pos] = op_; } else { op = Some(op_); } } 10 if key == "conditions" => { has_conditions = Some(self.0.len()); self.0.push(op.take().unwrap_or(Filter::And)); map.next_value_seed(FilterListCollector(self.0))?; self.0.push(Filter::Close); } _ => { if let Some(filter) = filter { if !has_multiple_filters { self.0.push(Filter::And); has_multiple_filters = true; } self.0.push(Filter::Property(filter)); } let mut new_filter = T::default(); new_filter.deserialize_argument(&key, &mut map)?; filter = Some(new_filter); } } } if let Some(filter) = filter { if has_conditions.is_some() { return Err(de::Error::custom( "Cannot mix conditions with property filters", )); } self.0.push(Filter::Property(filter)); if has_multiple_filters { self.0.push(Filter::Close); } } Ok(()) } } deserializer.deserialize_map(FilterVisitor(self.0)) } } impl<'de, 'x, T> DeserializeSeed<'de> for FilterListCollector<'x, T> where T: for<'de2> DeserializeArguments<'de2> + Default, { type Value = (); fn deserialize(self, deserializer: D) -> Result where D: Deserializer<'de>, { struct FilterVisitor<'x, T: 'x>(&'x mut Vec>) where T: for<'de2> DeserializeArguments<'de2> + Default; impl<'de, 'x, T> Visitor<'de> for FilterVisitor<'x, T> where T: for<'de2> DeserializeArguments<'de2> + Default, { type Value = (); fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { write!(formatter, "a filter list") } fn visit_seq(self, mut seq: A) -> Result<(), A::Error> where A: SeqAccess<'de>, { while let Some(()) = seq.next_element_seed(FilterMapCollector(self.0))? {} Ok(()) } } deserializer.deserialize_seq(FilterVisitor(self.0)) } } impl<'de, T> DeserializeArguments<'de> for Comparator where T: for<'de2> DeserializeArguments<'de2> + Default, { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"isAscending" => { self.is_ascending = map.next_value()?; }, b"collation" => { self.collation = map.next_value()?; }, _ => { self.property.deserialize_argument(key, map)?; } ); Ok(()) } } impl<'de, T> Deserialize<'de> for Comparator where T: for<'de2> DeserializeArguments<'de2> + Default, { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserialize_request(deserializer) } } impl Comparator where T: for<'de> DeserializeArguments<'de> + Default, { pub fn descending(property: T) -> Self { Self { property, is_ascending: false, collation: None, } } pub fn ascending(property: T) -> Self { Self { property, is_ascending: true, collation: None, } } } impl Default for Comparator where T: for<'de> DeserializeArguments<'de> + Default, { fn default() -> Self { Self { is_ascending: true, collation: None, property: T::default(), } } } ================================================ FILE: crates/jmap-proto/src/method/query_changes.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ method::query::{Comparator, Filter, FilterWrapper, QueryRequest}, object::JmapObject, request::deserialize::{DeserializeArguments, deserialize_request}, types::state::State, }; use serde::{Deserialize, Deserializer}; use types::id::Id; #[derive(Debug, Clone)] pub struct QueryChangesRequest { pub account_id: Id, pub filter: Vec>, pub sort: Option>>, pub since_query_state: State, pub max_changes: Option, pub up_to_id: Option, pub calculate_total: Option, pub arguments: T::QueryArguments, } #[derive(Debug, Clone, serde::Serialize)] pub struct QueryChangesResponse { #[serde(rename = "accountId")] pub account_id: Id, #[serde(rename = "oldQueryState")] pub old_query_state: State, #[serde(rename = "newQueryState")] pub new_query_state: State, #[serde(rename = "total")] #[serde(skip_serializing_if = "Option::is_none")] pub total: Option, #[serde(rename = "removed")] pub removed: Vec, #[serde(rename = "added")] pub added: Vec, } #[derive(Debug, Clone, serde::Serialize)] pub struct AddedItem { pub id: Id, pub index: usize, } impl AddedItem { pub fn new(id: Id, index: usize) -> Self { Self { id, index } } } impl<'de, T: JmapObject> DeserializeArguments<'de> for QueryChangesRequest { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"accountId" => { self.account_id = map.next_value()?; }, b"filter" => { self.filter = map.next_value::>()?.0; }, b"sort" => { self.sort = map.next_value()?; }, b"sinceQueryState" => { self.since_query_state = map.next_value()?; }, b"maxChanges" => { self.max_changes = map.next_value()?; }, b"upToId" => { self.up_to_id = map.next_value()?; }, b"calculateTotal" => { self.calculate_total = map.next_value()?; }, _ => { self.arguments.deserialize_argument(key, map)?; } ); Ok(()) } } impl<'de, T: JmapObject> Deserialize<'de> for QueryChangesRequest { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserialize_request(deserializer) } } impl Default for QueryChangesRequest { fn default() -> Self { Self { account_id: Id::default(), filter: Vec::new(), sort: None, since_query_state: State::default(), max_changes: None, up_to_id: None, calculate_total: None, arguments: T::QueryArguments::default(), } } } impl From> for QueryRequest { fn from(request: QueryChangesRequest) -> Self { QueryRequest { account_id: request.account_id, filter: request.filter, sort: request.sort, position: None, anchor: None, anchor_offset: None, limit: None, calculate_total: request.calculate_total, arguments: request.arguments, } } } ================================================ FILE: crates/jmap-proto/src/method/search_snippet.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::query::Filter; use crate::{ method::query::FilterWrapper, object::email::EmailFilter, request::{ MaybeInvalid, deserialize::{DeserializeArguments, deserialize_request}, reference::{MaybeResultReference, ResultReference}, }, }; use serde::{Deserialize, Deserializer}; use types::id::Id; #[derive(Debug, Clone)] pub struct GetSearchSnippetRequest { pub account_id: Id, pub filter: Vec>, pub email_ids: MaybeResultReference>>, } #[derive(Debug, Clone, serde::Serialize)] pub struct GetSearchSnippetResponse { #[serde(rename = "accountId")] pub account_id: Id, #[serde(rename = "list")] pub list: Vec, #[serde(rename = "notFound")] #[serde(skip_serializing_if = "Vec::is_empty")] pub not_found: Vec, } #[derive(serde::Serialize, Clone, Debug)] pub struct SearchSnippet { #[serde(rename = "emailId")] pub email_id: Id, #[serde(skip_serializing_if = "Option::is_none")] pub subject: Option, #[serde(skip_serializing_if = "Option::is_none")] pub preview: Option, } impl<'de> DeserializeArguments<'de> for GetSearchSnippetRequest { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"accountId" => { self.account_id = map.next_value()?; }, b"filter" => { self.filter = map.next_value::>()?.0; }, b"emailIds" => { self.email_ids = MaybeResultReference::Value(map.next_value::>>()?); }, b"#emailIds" => { self.email_ids = MaybeResultReference::Reference(map.next_value::()?); }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> Deserialize<'de> for GetSearchSnippetRequest { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserialize_request(deserializer) } } impl Default for GetSearchSnippetRequest { fn default() -> Self { Self { account_id: Id::default(), filter: Vec::new(), email_ids: MaybeResultReference::Value(Vec::new()), } } } ================================================ FILE: crates/jmap-proto/src/method/set.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use super::ahash_is_empty; use crate::{ error::set::{InvalidProperty, SetError}, object::{JmapObject, JmapObjectId}, request::{ MaybeInvalid, deserialize::{DeserializeArguments, deserialize_request}, reference::{MaybeResultReference, ResultReference}, }, response::Response, types::state::State, }; use ahash::AHashMap; use jmap_tools::{Key, Map, Value}; use serde::{Deserialize, Deserializer}; use types::id::Id; use utils::map::vec_map::VecMap; #[derive(Debug, Clone)] #[allow(clippy::type_complexity)] pub struct SetRequest<'x, T: JmapObject> { pub account_id: Id, pub if_in_state: Option, pub create: Option>>, pub update: Option, Value<'x, T::Property, T::Element>>>, pub destroy: Option>>>, pub arguments: T::SetArguments<'x>, } #[derive(Debug, Clone, Default, serde::Serialize)] #[allow(clippy::type_complexity)] pub struct SetResponse { #[serde(rename = "accountId")] #[serde(skip_serializing_if = "Option::is_none")] pub account_id: Option, #[serde(rename = "oldState")] #[serde(skip_serializing_if = "Option::is_none")] pub old_state: Option, #[serde(rename = "newState")] #[serde(skip_serializing_if = "Option::is_none")] pub new_state: Option, #[serde(rename = "created")] #[serde(skip_serializing_if = "ahash_is_empty")] pub created: AHashMap>, #[serde(rename = "updated")] #[serde(skip_serializing_if = "VecMap::is_empty")] pub updated: VecMap>>, #[serde(rename = "destroyed")] #[serde(skip_serializing_if = "Vec::is_empty")] pub destroyed: Vec, #[serde(rename = "notCreated")] #[serde(skip_serializing_if = "VecMap::is_empty")] pub not_created: VecMap>, #[serde(rename = "notUpdated")] #[serde(skip_serializing_if = "VecMap::is_empty")] pub not_updated: VecMap>, #[serde(rename = "notDestroyed")] #[serde(skip_serializing_if = "VecMap::is_empty")] pub not_destroyed: VecMap>, } impl<'de, T: JmapObject> DeserializeArguments<'de> for SetRequest<'de, T> { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"accountId" => { self.account_id = map.next_value()?; }, b"ifInState" => { self.if_in_state = map.next_value()?; }, b"create" => { self.create = map.next_value()?; }, b"update" => { self.update = map.next_value()?; }, b"destroy" => { self.destroy = map.next_value::>>>()?.map(MaybeResultReference::Value); }, b"#destroy" => { self.destroy = Some(MaybeResultReference::Reference(map.next_value::()?)); } _ => { self.arguments.deserialize_argument(key, map)?; } ); Ok(()) } } impl<'de, T: JmapObject> Deserialize<'de> for SetRequest<'de, T> { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserialize_request(deserializer) } } impl<'x, T: JmapObject> Default for SetRequest<'x, T> { fn default() -> Self { Self { account_id: Id::default(), if_in_state: None, create: None, update: None, destroy: None, arguments: T::SetArguments::default(), } } } impl<'x, T: JmapObject> SetRequest<'x, T> { pub fn validate(&self, max_objects_in_set: usize) -> trc::Result<()> { if self.create.as_ref().map_or(0, |objs| objs.len()) + self.update.as_ref().map_or(0, |objs| objs.len()) + self.destroy.as_ref().map_or(0, |objs| { if let MaybeResultReference::Value(ids) = objs { ids.len() } else { 0 } }) > max_objects_in_set { Err(trc::JmapEvent::RequestTooLarge.into_err()) } else { Ok(()) } } pub fn has_updates(&self) -> bool { self.update.as_ref().is_some_and(|objs| !objs.is_empty()) } pub fn has_creates(&self) -> bool { self.create.as_ref().is_some_and(|objs| !objs.is_empty()) } pub fn unwrap_create(&mut self) -> VecMap> { self.create.take().unwrap_or_default() } pub fn unwrap_update( &mut self, ) -> VecMap, Value<'x, T::Property, T::Element>> { self.update.take().unwrap_or_default() } pub fn unwrap_destroy(&mut self) -> Vec> { self.destroy .take() .map(|ids| ids.unwrap()) .unwrap_or_default() } } impl SetResponse { pub fn from_request(request: &SetRequest, max_objects: usize) -> trc::Result { let n_create = request.create.as_ref().map_or(0, |objs| objs.len()); let n_update = request.update.as_ref().map_or(0, |objs| objs.len()); let n_destroy = request.destroy.as_ref().map_or(0, |objs| { if let MaybeResultReference::Value(ids) = objs { ids.len() } else { 0 } }); if n_create + n_update + n_destroy <= max_objects { Ok(SetResponse { account_id: if request.account_id.is_valid() { request.account_id.into() } else { None }, new_state: None, old_state: None, created: AHashMap::with_capacity(n_create), updated: VecMap::with_capacity(n_update), destroyed: Vec::with_capacity(n_destroy), not_created: VecMap::new(), not_updated: VecMap::new(), not_destroyed: VecMap::new(), }) } else { Err(trc::JmapEvent::RequestTooLarge.into_err()) } } pub fn with_state(mut self, state: State) -> Self { self.old_state = Some(state.clone()); self.new_state = Some(state); self } pub fn created(&mut self, id: String, document_id: impl Into) { self.created.insert( id, Value::Object(Map::from(vec![( Key::Property(T::ID_PROPERTY), Value::Element(document_id.into().into()), )])), ); } pub fn invalid_property_create( &mut self, id: String, property: impl Into>, ) { self.not_created.append( id, SetError::invalid_properties() .with_property(property) .with_description("Invalid property or value.".to_string()), ); } pub fn invalid_property_update( &mut self, id: Id, property: impl Into>, ) { self.not_updated.append( id, SetError::invalid_properties() .with_property(property) .with_description("Invalid property or value.".to_string()), ); } pub fn update_created_ids(&self, response: &mut Response) { for (user_id, obj) in &self.created { if let Value::Object(obj) = obj && let Some(Value::Element(id)) = obj.get(&Key::Property(T::ID_PROPERTY)) && let Some(id) = id.as_any_id() { response.created_ids.insert(user_id.clone(), id); } } } pub fn get_object_by_id( &mut self, id: Id, ) -> Option<&mut Value<'static, T::Property, T::Element>> { if let Some(obj) = self.updated.get_mut(&id) { if let Some(obj) = obj { return Some(obj); } else { *obj = Some(Value::Object(Map::with_capacity(1))); return obj.as_mut().unwrap().into(); } } (&mut self.created) .into_iter() .map(|(_, obj)| obj) .find(|obj| { obj.as_object_and_get(&Key::Property(T::ID_PROPERTY)) .and_then(|v| v.as_element()) .and_then(|v| v.as_id()) .is_some_and(|oid| oid == id) }) } pub fn has_changes(&self) -> bool { !self.created.is_empty() || !self.updated.is_empty() || !self.destroyed.is_empty() } } ================================================ FILE: crates/jmap-proto/src/method/upload.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use std::borrow::Cow; use super::ahash_is_empty; use crate::{ error::set::SetError, object::{AnyId, blob::BlobProperty}, request::{ deserialize::{DeserializeArguments, deserialize_request}, reference::MaybeIdReference, }, response::Response, }; use ahash::AHashMap; use mail_parser::decoders::base64::base64_decode; use serde::{Deserialize, Deserializer}; use types::{blob::BlobId, id::Id}; use utils::map::vec_map::VecMap; #[derive(Debug, Clone, Default)] pub struct BlobUploadRequest { pub account_id: Id, pub create: VecMap, } #[derive(Debug, Clone, Default)] pub struct UploadObject { pub type_: Option, pub data: Vec, } #[derive(Debug, Clone, PartialEq, Eq, Default)] pub enum DataSourceObject { Id { id: MaybeIdReference, length: Option, offset: Option, }, Value(Vec), #[default] Null, } #[derive(Debug, Clone, Default, serde::Serialize)] pub struct BlobUploadResponse { #[serde(rename = "accountId")] pub account_id: Id, #[serde(rename = "created")] #[serde(skip_serializing_if = "ahash_is_empty")] pub created: AHashMap, #[serde(rename = "notCreated")] #[serde(skip_serializing_if = "VecMap::is_empty")] pub not_created: VecMap>, } #[derive(Debug, Clone, Default, serde::Serialize)] pub struct BlobUploadResponseObject { pub id: BlobId, #[serde(rename = "type")] #[serde(skip_serializing_if = "Option::is_none")] pub type_: Option, pub size: usize, } impl<'de> DeserializeArguments<'de> for BlobUploadRequest { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"accountId" => { self.account_id = map.next_value()?; }, b"create" => { self.create = map.next_value()?; } _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> DeserializeArguments<'de> for UploadObject { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"type" => { self.type_ = map.next_value()?; }, b"data" => { self.data = map.next_value()?; }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> DeserializeArguments<'de> for DataSourceObject { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"data:asText" => { *self = DataSourceObject::Value(map.next_value::().map(|v| v.into_bytes())?); }, b"data:asBase64" => { *self = DataSourceObject::Value(base64_decode(map.next_value::>()?.as_bytes()).ok_or_else(|| serde::de::Error::custom("Failed to decode base64 data"))?); }, b"blobId" => { match self { DataSourceObject::Id { id, .. } => { *id = map.next_value()?; }, _ => { *self = DataSourceObject::Id { id: map.next_value()?, length: None, offset: None, }; } } }, b"offset" => { match self { DataSourceObject::Id { offset, .. } => { *offset = map.next_value()?; }, _ => { *self = DataSourceObject::Id { id: MaybeIdReference::Invalid("".into()), length: None, offset: map.next_value()?, }; } } }, b"length" => { match self { DataSourceObject::Id { length, .. } => { *length = map.next_value()?; }, _ => { *self = DataSourceObject::Id { id: MaybeIdReference::Invalid("".into()), length: map.next_value()?, offset: None, }; } } }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl BlobUploadResponse { pub fn update_created_ids(&self, response: &mut Response) { for (user_id, obj) in &self.created { response .created_ids .insert(user_id.clone(), AnyId::BlobId(obj.id.clone())); } } } impl<'de> Deserialize<'de> for DataSourceObject { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserialize_request(deserializer) } } impl<'de> Deserialize<'de> for UploadObject { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserialize_request(deserializer) } } impl<'de> Deserialize<'de> for BlobUploadRequest { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserialize_request(deserializer) } } ================================================ FILE: crates/jmap-proto/src/method/validate.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ error::set::SetError, object::sieve::SieveProperty, request::{ MaybeInvalid, deserialize::{DeserializeArguments, deserialize_request}, }, }; use serde::{Deserialize, Deserializer, Serialize}; use types::{blob::BlobId, id::Id}; #[derive(Debug, Clone, Default)] pub struct ValidateSieveScriptRequest { pub account_id: Id, pub blob_id: MaybeInvalid, } #[derive(Debug, Serialize)] pub struct ValidateSieveScriptResponse { #[serde(rename = "accountId")] pub account_id: Id, pub error: Option>, } impl<'de> DeserializeArguments<'de> for ValidateSieveScriptRequest { fn deserialize_argument(&mut self, key: &str, map: &mut A) -> Result<(), A::Error> where A: serde::de::MapAccess<'de>, { hashify::fnc_map!(key.as_bytes(), b"accountId" => { self.account_id = map.next_value()?; }, b"blobId" => { self.blob_id = map.next_value()?; }, _ => { let _ = map.next_value::()?; } ); Ok(()) } } impl<'de> Deserialize<'de> for ValidateSieveScriptRequest { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserialize_request(deserializer) } } ================================================ FILE: crates/jmap-proto/src/object/addressbook.rs ================================================ /* * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use crate::{ object::{ AnyId, JmapObject, JmapObjectId, JmapRight, JmapSharedObject, MaybeReference, parse_ref, }, request::{deserialize::DeserializeArguments, reference::MaybeIdReference}, }; use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, Property}; use std::{borrow::Cow, str::FromStr}; use types::{acl::Acl, id::Id, special_use::SpecialUse}; #[derive(Debug, Clone, Default)] pub struct AddressBook; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum AddressBookProperty { Id, Name, Description, SortOrder, IsDefault, IsSubscribed, ShareWith, MyRights, // Other IdValue(Id), Rights(AddressBookRight), Pointer(JsonPointer), } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum AddressBookRight { MayRead, MayWrite, MayShare, MayDelete, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum AddressBookValue { Id(Id), IdReference(String), Role(SpecialUse), } impl Property for AddressBookProperty { fn try_parse(key: Option<&Key<'_, Self>>, value: &str) -> Option { let allow_patch = key.is_none(); if let Some(Key::Property(key)) = key { match key.patch_or_prop() { AddressBookProperty::ShareWith => { Id::from_str(value).ok().map(AddressBookProperty::IdValue) } _ => AddressBookProperty::parse(value, allow_patch), } } else { AddressBookProperty::parse(value, allow_patch) } } fn to_cow(&self) -> Cow<'static, str> { match self { AddressBookProperty::Id => "id", AddressBookProperty::Name => "name", AddressBookProperty::Description => "description", AddressBookProperty::SortOrder => "sortOrder", AddressBookProperty::IsDefault => "isDefault", AddressBookProperty::IsSubscribed => "isSubscribed", AddressBookProperty::ShareWith => "shareWith", AddressBookProperty::MyRights => "myRights", AddressBookProperty::Rights(addressbook_right) => addressbook_right.as_str(), AddressBookProperty::Pointer(json_pointer) => return json_pointer.to_string().into(), AddressBookProperty::IdValue(id) => return id.to_string().into(), } .into() } } impl AddressBookRight { pub fn as_str(&self) -> &'static str { match self { AddressBookRight::MayRead => "mayRead", AddressBookRight::MayWrite => "mayWrite", AddressBookRight::MayShare => "mayShare", AddressBookRight::MayDelete => "mayDelete", } } } impl Element for AddressBookValue { type Property = AddressBookProperty; fn try_parse