Showing preview only (264K chars total). Download the full file or copy to clipboard to get everything.
Repository: rustyio/super-imap
Branch: master
Commit: 98b407b86b84
Files: 192
Total size: 222.4 KB
Directory structure:
gitextract_xuq40opm/
├── .gitignore
├── Gemfile
├── Procfile
├── Procfile.development
├── Procfile.imap-client-px
├── Procfile.stress-test
├── README.md
├── Rakefile
├── app/
│ ├── admin/
│ │ ├── admin_user.rb
│ │ ├── delayed_job.rb
│ │ ├── imap_provider.rb
│ │ ├── mail_log.rb
│ │ ├── partner.rb
│ │ ├── partner_connection.rb
│ │ ├── tracer_log.rb
│ │ ├── transmit_log.rb
│ │ └── user.rb
│ ├── assets/
│ │ ├── images/
│ │ │ └── .keep
│ │ ├── javascripts/
│ │ │ ├── active_admin.js.coffee
│ │ │ └── application.js
│ │ └── stylesheets/
│ │ ├── active_admin.css.scss
│ │ └── application.css
│ ├── controllers/
│ │ ├── api/
│ │ │ └── v1/
│ │ │ ├── connections_controller.rb
│ │ │ └── users_controller.rb
│ │ ├── application_controller.rb
│ │ ├── concerns/
│ │ │ ├── .keep
│ │ │ └── link_rel.rb
│ │ ├── users/
│ │ │ ├── base_callback_controller.rb
│ │ │ ├── connects_controller.rb
│ │ │ └── disconnects_controller.rb
│ │ └── webhook_test_controller.rb
│ ├── helpers/
│ │ ├── application_helper.rb
│ │ ├── oauth2/
│ │ │ ├── connects_helper.rb
│ │ │ └── disconnects_helper.rb
│ │ └── plain/
│ │ ├── connects_helper.rb
│ │ └── disconnects_helper.rb
│ ├── interactors/
│ │ ├── base_webhook.rb
│ │ ├── call_new_mail_webhook.rb
│ │ ├── call_user_connected_webhook.rb
│ │ ├── call_user_disconnected_webhook.rb
│ │ └── schedule_tracer_emails.rb
│ ├── mailers/
│ │ ├── .keep
│ │ └── tracer_mailer.rb
│ ├── models/
│ │ ├── .keep
│ │ ├── admin_user.rb
│ │ ├── concerns/
│ │ │ ├── .keep
│ │ │ ├── auth_method_helper.rb
│ │ │ └── connection_fields.rb
│ │ ├── delayed_job.rb
│ │ ├── imap_daemon_heartbeat.rb
│ │ ├── imap_provider.rb
│ │ ├── mail_log.rb
│ │ ├── oauth2/
│ │ │ ├── imap_provider.rb
│ │ │ ├── partner_connection.rb
│ │ │ └── user.rb
│ │ ├── partner.rb
│ │ ├── partner_connection.rb
│ │ ├── plain/
│ │ │ ├── imap_provider.rb
│ │ │ ├── partner_connection.rb
│ │ │ └── user.rb
│ │ ├── tracer_log.rb
│ │ ├── transmit_log.rb
│ │ └── user.rb
│ ├── processes/
│ │ ├── common/
│ │ │ ├── csv_log.rb
│ │ │ ├── db_connection.rb
│ │ │ ├── light_sleep.rb
│ │ │ ├── stoppable.rb
│ │ │ ├── worker_pool.rb
│ │ │ └── wrapped_thread.rb
│ │ ├── imap_client/
│ │ │ ├── daemon.rb
│ │ │ ├── process_uid.rb
│ │ │ ├── rendezvous_hash.rb
│ │ │ └── user_thread.rb
│ │ ├── imap_client.rb
│ │ ├── imap_test_server/
│ │ │ ├── daemon.rb
│ │ │ ├── mailboxes.rb
│ │ │ └── socket_state.rb
│ │ └── imap_test_server.rb
│ └── views/
│ ├── api/
│ │ └── v1/
│ │ ├── connections/
│ │ │ ├── index.json.rb
│ │ │ └── show.json.rb
│ │ └── users/
│ │ ├── index.json.rb
│ │ └── show.json.rb
│ ├── layouts/
│ │ ├── application.html.erb
│ │ └── blank.html.erb
│ └── tracer_mailer/
│ └── tracer_email.html.erb
├── app.json
├── bin/
│ ├── bundle
│ ├── delayed_job
│ ├── rails
│ ├── rake
│ └── spring
├── config/
│ ├── application.rb
│ ├── boot.rb
│ ├── database.yml.example
│ ├── environment.rb
│ ├── environments/
│ │ ├── development.rb
│ │ ├── performance.rb
│ │ ├── production.rb
│ │ ├── stress.rb
│ │ └── test.rb
│ ├── initializers/
│ │ ├── active_admin.rb
│ │ ├── airbrake.rb
│ │ ├── assets.rb
│ │ ├── backtrace_silencers.rb
│ │ ├── cookies_serializer.rb
│ │ ├── delayed_job.rb
│ │ ├── devise.rb
│ │ ├── filter_parameter_logging.rb
│ │ ├── inflections.rb
│ │ ├── log.rb
│ │ ├── mime_types.rb
│ │ ├── ruby_template_handler.rb
│ │ ├── session_store.rb
│ │ └── wrap_parameters.rb
│ ├── locales/
│ │ ├── devise.en.yml
│ │ └── en.yml
│ ├── newrelic.yml
│ ├── routes.rb
│ ├── secrets.yml
│ └── unicorn.rb
├── config.ru
├── db/
│ ├── migrate/
│ │ ├── 20141029214610_devise_create_admin_users.rb
│ │ ├── 20141029214612_create_active_admin_comments.rb
│ │ ├── 20141029215033_create_partners.rb
│ │ ├── 20141029215101_create_mail_logs.rb
│ │ ├── 20141029215105_create_transmit_logs.rb
│ │ ├── 20141031010321_create_imap_providers.rb
│ │ ├── 20141031010353_create_partner_connections.rb
│ │ ├── 20141031010433_create_users.rb
│ │ ├── 20141104202256_create_imap_daemon_heartbeats.rb
│ │ ├── 20141111204248_add_archived_to_users.rb
│ │ ├── 20141113163147_add_type_to_users_and_imap_providers.rb
│ │ ├── 20141114233206_add_locked_at_to_admin_user.rb
│ │ ├── 20141118170010_add_type_to_partner_connection.rb
│ │ ├── 20141121152941_add_oauth2_fields_to_imap_provider.rb
│ │ ├── 20141121182537_add_redirect_urls_to_partner.rb
│ │ ├── 20141121184010_rename_fields.rb
│ │ ├── 20141205024759_rename_partner_webhook_columns.rb
│ │ ├── 20141207191800_add_webhooks_to_partner.rb
│ │ ├── 20141207200312_create_delayed_jobs.rb
│ │ ├── 20141215194630_add_tracer_to_users.rb
│ │ ├── 20141215194652_create_tracer_logs.rb
│ │ ├── 20141215194754_add_smtp_settings_to_imap_provider.rb
│ │ ├── 20141215212628_remove_oauth1_fields.rb
│ │ ├── 20150119185401_encrypt_existing_data.rb
│ │ └── 20150611134232_add_more_indexes_to_mail_logs.rb
│ ├── schema.rb
│ ├── seeds-development.rb
│ ├── seeds-production.rb
│ ├── seeds-stress.rb
│ ├── seeds-test.rb
│ └── seeds.rb
├── lib/
│ ├── assets/
│ │ └── .keep
│ ├── tasks/
│ │ ├── .keep
│ │ └── imap.rake
│ └── xoauth2_authenticator.rb
├── log/
│ └── .keep
├── public/
│ ├── 404.html
│ ├── 422.html
│ ├── 500.html
│ ├── failure.html
│ ├── index.html
│ ├── robots.txt
│ └── success.html
├── script/
│ ├── analyze-stress-test.R
│ └── stress-test
├── test/
│ ├── controllers/
│ │ ├── .keep
│ │ └── api/
│ │ └── v1/
│ │ ├── connections_controller_test.rb
│ │ └── users_controller_test.rb
│ ├── fixtures/
│ │ ├── .keep
│ │ ├── imap_daemon_heartbeats.yml
│ │ └── tracer_logs.yml
│ ├── helpers/
│ │ └── .keep
│ ├── imap/
│ │ └── rendezvous_hash_test.rb
│ ├── integration/
│ │ └── .keep
│ ├── mailers/
│ │ ├── .keep
│ │ ├── previews/
│ │ │ └── tracer_mailer_preview.rb
│ │ └── tracer_mailer_test.rb
│ ├── models/
│ │ ├── .keep
│ │ ├── admin_user_test.rb
│ │ ├── imap_daemon_heartbeat_test.rb
│ │ ├── imap_provider_test.rb
│ │ ├── mail_log_test.rb
│ │ ├── partner_connection_test.rb
│ │ ├── partner_credential_test.rb
│ │ ├── partner_test.rb
│ │ ├── tracer_log_test.rb
│ │ ├── transmit_log_test.rb
│ │ └── user_test.rb
│ └── test_helper.rb
└── vendor/
└── assets/
├── javascripts/
│ └── .keep
└── stylesheets/
└── .keep
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
# See https://help.github.com/articles/ignoring-files for more about ignoring files.
#
# If you find yourself ignoring temporary files generated by your text editor
# or operating system, you probably want to add a global ignore instead:
# git config --global core.excludesfile '~/.gitignore_global'
# Ignore bundler config.
/.bundle
# Ignore the default SQLite database.
/db/*.sqlite3
/db/*.sqlite3-journal
# Ignore all logfiles and tempfiles.
/log/stress
/stress-test-results.pdf
/log/*.log
/tmp
config/database.yml
.ruby-version
================================================
FILE: Gemfile
================================================
source 'https://rubygems.org'
ruby "2.2.0"
gem 'rails' , '4.1.6'
gem 'sass-rails' , '~> 4.0.3'
gem 'uglifier' , '>= 1.3.0'
gem 'coffee-rails' , '~> 4.0.0'
gem 'jquery-rails'
gem 'turbolinks'
gem 'jbuilder' , '~> 2.0'
gem 'pg' , '~> 0.17.1'
gem 'delayed_job_active_record' , '~> 4.0.2'
gem 'unicorn' , '~> 4.8.3'
gem 'devise' , '~> 3.4.1'
gem 'rest-client' , '~> 1.7.2'
gem 'oauth' , '~> 0.4.7'
gem 'oauth2' , '~> 0.9.3'
gem 'airbrake' , '~> 4.1.0'
gem 'activeadmin' , '1.0.0.pre', :github => 'activeadmin', :ref => '0becbef0'
gem 'gibberish' , '~> 1.4.0'
gem 'foreman' , '~> 0.75.0'
gem 'rails_stdout_logging' , :group => [:staging, :production]
gem 'rails_12factor' , :group => :production
# gem 'newrelic_rpm' , :group => :production
gem 'spring' , :group => :development
gem 'sqlite3' , :group => :development
gem 'pry' , :group => :development
gem 'pry-byebug' , :group => :development
gem 'ruby-prof' , :group => :stress
gem 'sdoc' , '~> 0.4.0', :group => :doc
================================================
FILE: Procfile
================================================
web: bundle exec unicorn -p $PORT -c ./config/unicorn.rb
worker: bundle exec rake jobs:work
# Heroku dynos have thread limits. (1x = 256, 2x = 512, px =
# 32767). For 1x and 2x dynos, we set MAX_USER_THREADS way below the
# limit because IMAP also consumes some threads. For px dynos, we use
# foreman to create multiple imap_client instances on a single box.
#
# See https://devcenter.heroku.com/articles/limits#processes-threads
imap_client_1x: NUM_WORKER_THREADS=1 MAX_USER_THREADS=200 bundle exec rake imap:client
imap_client_2x: NUM_WORKER_THREADS=2 MAX_USER_THREADS=500 bundle exec rake imap:client
imap_client_px: bundle exec foreman s -f Procfile.imap-client-px
================================================
FILE: Procfile.development
================================================
web: bundle exec unicorn -p $PORT -c ./config/unicorn.rb
worker: bundle exec rake jobs:work
imap_client: bundle exec rake imap:client
================================================
FILE: Procfile.imap-client-px
================================================
north: DYNO=$DYNO-1 NUM_WORKER_THREADS=3 MAX_USER_THREADS=3750 bundle exec rake imap:client
south: DYNO=$DYNO-2 NUM_WORKER_THREADS=3 MAX_USER_THREADS=3750 bundle exec rake imap:client
east: DYNO=$DYNO-3 NUM_WORKER_THREADS=3 MAX_USER_THREADS=3750 bundle exec rake imap:client
west: DYNO=$DYNO-4 NUM_WORKER_THREADS=3 MAX_USER_THREADS=3750 bundle exec rake imap:client
================================================
FILE: Procfile.stress-test
================================================
imap_test_server: rake imap:test_server
imap_client_1: rake imap:client STRESS_TEST_MODE=true ENABLE_PROFILER=$ENABLE_PROFILER
imap_client_2: rake imap:client STRESS_TEST_MODE=true ENABLE_PROFILER=false
imap_client_3: rake imap:client STRESS_TEST_MODE=true ENABLE_PROFILER=false
================================================
FILE: README.md
================================================
# SuperIMAP - Version 0.1.2
SuperIMAP helps you build email-driven applications. It takes care of
connecting to a customer's IMAP inbox, watching for new email, and
triggering a webhook to your application when a new email arrives,
typically within seconds.
SuperIMAP is built for
scale. [FiveStreet.com](https://www.fivestreet.com) built SuperIMAP as
an alternative to Context.io. It contains a subset of Context.io Lite
API functionality. As of July 2015, the FiveStreet team runs a
SuperIMAP cluster processing ~400k emails per day for thousands of
users.
SuperIMAP is written in Ruby and open sourced under the MIT license. [Why Ruby?](#why-ruby)
[](https://heroku.com/deploy?template=https://github.com/rustyio/super-imap)
## Contents
* [Installation](#installation)
* [Installing on Heroku](#installing-on-heroku)
* [Installing Elsewhere](#installing-elsewhere)
* [Usage](#usage)
* [Set up a Partner](#set-up-a-partner)
* [Set up a Partner Connection](#set-up-a-partner-connection)
* [Add and Connect a User](#add-and-connect-a-user)
* [Add and Connect a User Programatically](#add-and-connect-a-user-programatically)
* [Disconnect a User Programatically](#disconnect-a-user-programatically)
* [Security](#security)
* [Webhooks](#webhooks)
* [Webhook Security](#webhook-security)
* [New Mail Webhook](#new-mail-webhook)
* [User Connected Webhook](#user-connected-webhook)
* [User Disconnected Webhook](#user-disconnected-webhook)
* [API](#api)
* [Manage Partner Connections](#apiv1connections)
* [Manage a Partner Connection](#apiv1connectionsimap_provider_code)
* [Manage Users](#apiv1connectionsimap_provider_codeusers)
* [Manage a User](#apiv1connectionsimap_provider_codeuserstag)
* [Operations](#operations)
* [Process Types](#process-types)
* [Why Ruby?](#why-ruby)
* [Environment Variables](#environment-variables)
* [Scaling](#scaling)
* [Monitoring](#monitoring)
* [Performance](#performance)
* [Tracer Emails](#tracer-emails)
* [Appendix](#appendix)
* [Understanding OAuth 2.0](#understanding-oauth-20)
* [Understanding IMAP](#understanding-imap)
* [Development Tasks](#development-tasks)
* [Running Unit Tests](#running-unit-tests)
* [Running Stress Tests](#running-stress-tests)
* [Future Work](#future-work)
* [Contributions](#contributions)
* [Changes](#changes)
* [License](#license)
## Screenshot
## Installation
SuperIMAP was built to run on Heroku, but can run in any environment that supports Rails.
#### Installing on Heroku
1. Provision a new Heroku project.
2. Add an encrypted database.
3. Set the `SECRET_KEY_BASE` and `ENCRYPTION_KEY` environment
variables to something really long and complicated.
4. Add a Heroku remote endpoint, and push the code.
5. Ramp up some workers.
5. Seed the database with `heroku run rake db:setup db:seed`
Then, log in as the default user: `admin@example.com` / `password`.
**Remember: Change the username and password immediately!**
The production Procfile assumes that you are installing on Heroku. As
a result it has multiple definitions for the `imap_client` process
corresponding to different sized Heroku dynos. In order for load
balancing to work correctly, you should have all `imap_client`
processes use the same dyno size. Do not mix and match boxes.
#### Installing Elsewhere
1. Get the code: `git clone https://github.com/rustyio/super-imap.git`
2. Update `config/database.yml`. (Use `config/database.yml.example`)
3. Run `bundle` to install dependencies.
4. Set the `SECRET_KEY_BASE` and `ENCRYPTION_KEY` environment
variables to something really long and complicated.
5. Seed the database with `RAILS_ENV=production rake db:setup db:seed`
6. Start the processes: `foreman start -c "web=1, worker=1, imap_client_1x=1`
Then, log in as the default user: `admin@example.com` / `password`.
**Remember: Change the username and password immediately!**
The production Procfile assumes that you are installing on Heroku. As
a result it has multiple definitions for the `imap_client` process
corresponding to different sized Heroku dynos. In order for load
balancing to work correctly, you should ensure that all `imap_client`
processes are the same size and point to the same database.
## Usage
#### Set up a Partner
A single SuperIMAP instance can support multiple applications (and/or multiple environments for a single application.) This is done by creating a new "Partner" for every different application (and/or environment).
1. Open the SuperIMAP dashboard.
2. Click on the "Partners" tab. Click "New Partner".
3. Set webhooks to notify your application of the following events:
* A new email has arrived.
* A user has connected their email account.
* A user has disconnected their email account.
#### Set up a Partner Connection
1. Still within the dashboard, click the "Partners" tab.
2. Click on the "Connections" link next to your Partner.
3. Click on "New Partner Connection".
4. Choose an authentication type, and fill out any necessary credentials.
For Gmail, you will need to get OAuth 2.0 Client credentials here from the [Developer Console](https://console.developers.google.com/project).
#### Add and Connect a User
1. Still within the dashboard, click on "Partners".
2. Click on the "Connections" link next to your partner.
3. Click on the "Users" link next to your connection.
4. Click on the "New User" button.
5. Click the 'Connect' link to connect the user to an IMAP provider.
6. Send yourself email, and watch the logs!
#### Add and Connect a User Programatically
The next step is to update your application to handle the process of creating and connecting to SuperIMAP users. Here is some example code:
```ruby
require 'rest-client'
url = "https://my-app.com/api/v1/connections/GMAIL_OAUTH2/users"
users = RestClient::Resource.new(url, :headers => {
:'x-api-key' => "$API_KEY$",
:content_type => :json,
:accept => :json
})
# Create the user.
users.post(:tag => "MY_USER")
# Get the connect url.
response = users["MY_USER"].get
connect_url = JSON.parse(response)['connect_url']
# Set up the success and failure callbacks.
callbacks = {
:success => "http://my-app.com/connect_callback?success=1",
:failure => "http://my-app.com/connect_callback?failure=1"
}
# Redirect the user to the connect url.
redirect_to connect_url + "?" + callbacks.to_query
```
#### Disconnect a User Programatically
Below is sample code to disconnect a user:
```ruby
url = "https://my-host.com/api/v1/connections/GMAIL_OAUTH2/users"
users = RestClient::Resource.new(url, :headers => {
:'x-api-key' => "$API_KEY$",
:content_type => :json,
:accept => :json
})
# Later, if you want to disconnect the user.
response = users["MY_USER"].get
disconnect_url = JSON.parse(response)['disconnect_url']
# Set up the success and failure callbacks.
callbacks = {
:success => "http://my-app.com/disconnect_callback?success=1"
}
# Redirect the user to the disconnect url.
redirect_to disconnect_url + "?" + callbacks.to_query
```
#### Security
This is a good time to mention security. It is a big responsibility to
hold the keys to someone's email. Treat it with the appropriate amount
of caution.
If you use this code:
* *PLEASE* ensure that you use very strong, safeguarded passwords.
* *PLEASE* use enable 2-factor-authentication for your Heroku account.
* *PLEASE* make sure your entire database is encrypted at rest.
Other security measures within SuperIMAP:
+ SSL is *required* in production.
+ Secure fields (e.g. passwords and other credentials) are never exposed via the web interface, and are encrypted in the database.
+ Passwords are not recoverable by email.
+ Accounts are locked for an hour after three invalid password attempts.
## Webhooks
SuperIMAP sends new email events (and other events) to your
applications through webhooks:
+ All webhooks are dispatched through delayed jobs.
+ Webhooks will be retried up to 6 times, with exponential backoff.
+ Webhooks will fail if the receiving server takes more than 30 seconds to respond.
+ Webhooks expect a success response (HTTP code 200 - 206).
+ A "Forbidden" response code of 403 will automatically archive the user.
+ Any other response codes count as an error, and will trigger a retry.
#### Webhook Security
All webhooks are signed. You can validate the signature as follows:
```ruby
# Parse the incoming JSON body.
json_params = JSON.parse(request.raw_post)
# Calculate expected signature.
digest = OpenSSL::Digest.new('sha256')
api_key = Rails.application.config.super_imap_api_key
sha1 = json_params['sha1']
timestamp = json_params['timestamp']
expected_signature = OpenSSL::HMAC.hexdigest(digest, api_key, "#{timestamp}#{sha1}")
# Get actual signature.
actual_signature = json_params['signature']
# Compare signatures.
valid = expected_signature == actual_signature
```
#### New Mail Webhook
Called when a new mail arrives in a user's inbox.
+ `timestamp` - Timestamp the webhook was sent. Seconds since Jan 1, 1970.
+ `sha1` - The SHA1 hash of the rfc822 parameter.
+ `imap_provider_code` - The IMAP provider code (e.g. "GMAIL_OAUTH2")
+ `user_tag` - The user's tag.
+ `envelope - The email envelope, including date, subject, from, sender, reply_to, to, cc, bcc, in_reply_to, and message_id.
+ `rfc822` - The raw body of the email. http://www.w3.org/Protocols/rfc822/
#### User Connected Webhook
Called when a user has successfully authenticated with an IMAP
provider. Only applies to OAuth connections at the moment.
+ `timestamp` - Timestamp the webhook was sent. Seconds since Jan 1, 1970.
+ `sha1` - The SHA1 hash of the user's tag.
+ `imap_provider_code` - The IMAP provider code (e.g. "GMAIL_OAUTH2")
+ `user_tag` - The user's tag.
+ `email` - The email address with which the user authenticated.
#### User Disconnected Webhook
Called when a user has disconnected from an IMAP provider. Only
applies to OAuth connections at the moment.
+ `timestamp` - Timestamp the webhook was sent. Seconds since Jan 1, 1970.
+ `sha1` - The SHA1 hash of the user's tag.
+ `imap_provider_code` - The IMAP provider code (e.g. "GMAIL_OAUTH2")
+ `user_tag` - The user's tag.
## API
All API calls are scoped by partner. To authenticate, send the
Partner's API key using a header or a parameter. (A header is
preferred because it won't normally appear in HTTP logs.)
```sh
# Access the API curl:
curl -H "Accept: json" \
-H "x-api-key:APIKEY" \
https://my-host.com/api/v1/connections
```
```ruby
# Access the API using the rest-client gem:
url = "https://my-host.com/api/v1"
resource = RestClient::Resource.new(url, :headers => {
:'x-api-key' => "$API_KEY$",
:content_type => :json,
:accept => :json
})
resource['connections'].get
```
___
#### /api/v1/connections
**GET**
Get a list of connections for the specified partner.
**POST**
Create a new connection.
* `imap_provider_code` is required.
* Other required parameters depend on the IMAP Provider used.
___
#### /api/v1/connections/:IMAP_PROVIDER_CODE
**GET**
Get information about a given connection.
**PUT**
Update settings for a given connection. The required parameters depend on the IMAP provider used.
**DELETE**
Delete a connection and all underlying user data.
___
#### /api/v1/connections/:IMAP_PROVIDER_CODE/users
**GET**
Get a list of users for the specified IMAP Provider.
**POST**
Create a new user.
* `tag` - Required, a unique tag within the scope of a partner
connection, selected by the partner application.
* Other required parameters depend on the IMAP Provider used.
___
#### /api/v1/connections/:IMAP_PROVIDER_CODE/users/:TAG
**GET**
Get information about the given user, including:
* `email` - The IMAP email address to which the user's account is connected.
* `connected_at` - The date when the user's account was connected. Present only if connected.
* `connect_url` - Redirect to this url to connect a user to a provider. For OAuth based IMAP providers, this begins the OAuth dance.
* `disconnect_url` - Redirect to this url to disconnect a user from a provider.
**PUT**
Update a user. The required parameters depend on the IMAP provider used.
**DELETE**
Archive a user. The user can be restored in the web interface, or by
updating the user (ie: a PUT request.
## Operations
#### Process Types
SuperIMAP consists of 3 different processes, all written in Ruby / Rails:
+ `web` - Serves the admin interface and the API.
+ `imap_client` - Handles the task of connecting to IMAP providers and listening for email.
+ `worker` - Processes background jobs generated by the 'imap_client' process.
#### Environment Variables
+ `ENCRYPTION_KEY` - If provided, used to encrypt passwords and secret keys. Required in production.
+ `MAX_USER_THREADS` - Change the maximum number of user threads. Default is 500.
+ `NUM_WORKER_THREADS` - Change the number of worker threads. Default is 5.
+ `MAX_EMAIL_SIZE` - Change the maximum email size. Default is 1 MiB (1,048,576 bytes).
+ `TRACER_INTERVAL` - Interval, in seconds, between outgoing tracer emails. Default is 600 seconds (10 minutes).
+ `NUM_TRACERS` - Number of tracers to send at the end of each tracer interval. Default is 3.
#### Scaling
To scale SuperIMAP, you will mainly want to increase the number of IMAP
Client processes. The IMAP Client processes automatically publish a
heartbeat every 10 seconds. Other instances look for this heartbeat
and re-calculate which neighboring processes are alive based on any
processes that have published a heartbeat within the last 30 seconds.
The IMAP Client processes re-balance users every 10 seconds. If no new
instances have entered or left the pool, then re-balancing will have
no effect.
If a new IMAP Client instance is started, then a small number of users
will be taken from each running instance and handed to the new
instance. If one of the IMAP Client instances is stopped it is removed
from the pool, then it's users will be evenly distributed to the
remaining instances (assuming they are still below the
`MAX_USER_THREADS` threshold.)
There is no "master" process that decides which IMAP Client process
should handle a given user. SuperIMAP uses a
[Rendezvous Hash](http://en.wikipedia.org/wiki/Rendezvous_hashing) to
allow IMAP Client instances to agree on how to evenly assign users
without any central coordination. The algorithm assumes that all
SuperIMAP instances have roughly the same number of resources.
#### Monitoring
SuperIMAP publishes some useful monitoring information in the logs.
This includes:
+ `imap_client.user_thread.count` - The size of the imap client work queue. Backups may indicate that your servers are overloaded.
+ `imap_client.total_emails_processed` - The total number of emails processed since the instance was started.
+ `imap_client.work_queue.length` - The number of user threads. This indicates how many users are connected on a given IMAP Client instance.
+ `work_queue.latency` - The latency, in seconds, between when an item is added to the work queue and when it is processed.
These metrics are published in a format that can be consumed by the
Librato Add-On in Heroku. See
https://devcenter.heroku.com/articles/librato#custom-log-based-metrics
for more information.
Apart from keeping an eye on these metrics, SuperIMAP should need no other regular metrics.
You may also want to keep an eye out for any failing Delayed Job tasks. You can view these from the Admin site.
#### Tracer Emails
SuperIMAP has the ability to give you useful monitoring information
through "tracer emails". The system will send a specially formatted
email to an account, wait for the incoming email, and log the
results. The logs can be accessed through the "Tracer Logs" tab.
To enable Tracer Emails, navigate to a user and check the "Enable
Tracer" checkbox. It is recommended that you create a few dummy email
addresses to use for tracer emails.
By default, a cluster of three tracers are sent every ten minutes from
each `imap_client` instance to a random tracer-enabled user managed by
that instance.
Keep in mind that this could generate a lot of email. Three emails
every ten minutes works out to ~430 emails per day.
#### Performance
SuperIMAP's architecture makes judicious use of system resources:
All connections to the IMAP server are managed by separate "user
threads", but these threads sit dormant most of the time. When
anything interesting happens that requires real work, the operation is
queued and handled by a worker in a worker pool. The size of the
worker pool is controlled by the `NUM_WORKER_THREADS` environment
variable. Only worker pools threads, and a small number of other
system threads, require a database connection.
In terms of tradeoffs, this architecture chooses to slightly degrade
an individual user's response time in favor of making sure that the
system will not get overloaded when things get rough. When things get
busy, the work simply builds up in the queue. The size of the worker
queue, and the queue latency, becomes a rough measure of system
health.
Typically, a SuperIMAP box is resource-limited by the number of user
processes that can be started. SuperIMAP requires 2 user processes for
each user's IMAP connection. On Heroku, the number of user processes
are limited at 256 for a 1X box, 512 for a 2X box, and 32,767 for a PX
box. You can set this at home using `ulimit -u`. Divide this in half
to get the maximum number of users that the SuperIMAP process can
manage.
[FiveStreet.com](https://www.fivestreet.com) uses SuperIMAP to manages
thousands of users and process over 1M incoming emails per week (as of
January 2015). We currently run this load on a single Heroku PX dyno,
with plenty of headroom. Our SuperIMAP instance serving thousands of
users requires just 10 database connections, uses about 3GB of RAM,
and has a 0.50 load average. The work queue usually sits near 0, with
a latency of < 0.5 seconds.
#### Why Ruby?
At first glance, and from a purely technical point-of-view, Ruby is a
poor choice for an application like SuperIMAP. SuperIMAP is highly
concurrent, and Ruby is bad at concurrency.
Specifically, the `imap_client` process spawns what could technically
be described as a "boatload" of threads (2 threads per connected user,
plus a handful of other threads). Ruby threads are heavyweight, so the
interpreter has to burn significant resources just to create and
schedule the threads before it can do any real work.
Using Erlang, Go, or Rust (all of which support lightweight threads
and actor-style programming) would have made the concurrent bits of
SuperIMAP less tricky to write, and would have required fewer
computing resources, possibly allowing a single box to handle tens of
thousands of active users.
So, why Ruby? A few reasons:
+ **FiveStreet uses Ruby** - The primary goal of SuperIMAP is to power
a critical part of FiveStreet's application. FiveStreet.com is
written in Ruby. It is built and maintained by a small team of Ruby
engineers. Introducing a new language would force the team to spend
dozens of hours learning a new stack and maintaining a new
development environment.
+ **Low barrier to entry** - A secondary goal of SuperIMAP is to become
a healthy open-source project. Based only on language popularity, it
is more likely that another team can use, troubleshoot, and contribute to
a Ruby-based SuperIMAP than an Erlang/Go/Rust-based SuperIMAP.
+ **For us, the cost savings are small** - FiveStreet's SuperIMAP
cluster currently runs on three commodity servers and easily handles
thousands of users. It's possible that if SuperIMAP were written in
a different language, we could handle the load on a single machine,
saving us a few hundred dollars a month. Not worth changing our
stack for it.
+ **The concurrency is not complicated** - The concurrency in SuperIMAP is
fairly straightforward -- one parent process, many child
processes. It's a little painful to solve the problem in Ruby, but
not impossible.
+ **Ruby has mature IMAP and OAuth 2.0 libraries** - Ruby has a
[built-in IMAP library](http://ruby-doc.org/stdlib-2.0.0/libdoc/net/imap/rdoc/Net/IMAP.html),
and a
[widely-used OAuth 2.0 library](https://github.com/intridea/oauth2). As
of 2015, the IMAP and OAuth 2.0 libraries for other languages are
[far](https://github.com/alexcrichton/oauth2-rs)
[less](https://github.com/boorad/erlimap)
[mature](https://github.com/mxk/go-imap).
Side Note: This was a deeply considered choice. I (Rusty Klophaus, the
author of SuperIMAP) spent about 4 years writing Erlang
professionally. It's a fascinating language.
## Appendix
#### Understanding OAuth 2.0
SuperIMAP can authenticate to email providers using OAuth 2.0. OAuth
2.0 can be difficult to understand. Here is how it works, in the
context of a user authenticating to Gmail through SuperIMAP:
On your application:
* John visits a "Connect Your Email" page on your website.
* On the server side, your application uses the SuperIMAP API to fetch a special url called a `connect_url`. ([sample code](#add-and-connect-a-user-programatically))
* John clicks a link and is redirected to the `connect_url`.
On your SuperIMAP instance:
* The `connect_url` links to a SuperIMAP page.
* The SuperIMAP page construct a special URL and redirects to Google.
On Google:
* Google displays a page asking John to confirm certain permissions for your application.
* John clicks the "Approve" button.
* Google redirects John to SuperIMAP and includes a "refresh token" parameter.
On your SuperIMAP instance:
* SuperIMAP grabs the "refresh token", and issues a server side request for an "access token".
* SuperIMAP saves the "access token". This is what allows SuperIMAP to connect to Google on behalf of the user in the future.
* SuperIMAP redirects the user back to the "success" page provided in step #2 above.
On your application:
* Your application tells John that the connection was successful.
To get one step more complicated, OAuth is secured in a few different ways:
+ Google requires your application to pre-register callback URLS for your app. Sending the user back to a non-registered URL will fail.
+ The access token is tied to a client id and a client secret on our site. Someone needs all three to authenticate as the user.
+ There is more too it, but that's the extent of my knowledge.
If you want more detail, here's a video tutorial: https://www.youtube.com/watch?v=tFYrq3d54Dc
The OAuth settings are configured through the Google Developer console: https://console.developers.google.com/
#### Understanding IMAP
Once you are authenticated to an IMAP server, IMAP itself is a fairly
straightforward protocol. It consists of simple plain text commands
and responses. The commands and responses are tagged, allowing
multiple commands to run in parallel.
Below is a sample IMAP session, taken directly from the [Internet Message Access Protocol RFC (3501)](https://tools.ietf.org/html/rfc3501#section-8):
```
S: * OK IMAP4rev1 Service Ready
C: a001 login mrc secret
S: a001 OK LOGIN completed
C: a002 select inbox
S: * 18 EXISTS
S: * FLAGS (\Answered \Flagged \Deleted \Seen \Draft)
S: * 2 RECENT
S: * OK [UNSEEN 17] Message 17 is the first unseen message
S: * OK [UIDVALIDITY 3857529045] UIDs valid
S: a002 OK [READ-WRITE] SELECT completed
C: a003 fetch 12 full
S: * 12 FETCH (FLAGS (\Seen) INTERNALDATE "17-Jul-1996 02:44:25 -0700"
RFC822.SIZE 4286 ENVELOPE ("Wed, 17 Jul 1996 02:23:25 -0700 (PDT)"
"IMAP4rev1 WG mtg summary and minutes"
(("Terry Gray" NIL "gray" "cac.washington.edu"))
(("Terry Gray" NIL "gray" "cac.washington.edu"))
(("Terry Gray" NIL "gray" "cac.washington.edu"))
((NIL NIL "imap" "cac.washington.edu"))
((NIL NIL "minutes" "CNRI.Reston.VA.US")
("John Klensin" NIL "KLENSIN" "MIT.EDU")) NIL NIL
"<B27397-0100000@cac.washington.edu>")
BODY ("TEXT" "PLAIN" ("CHARSET" "US-ASCII") NIL NIL "7BIT" 3028 92))
S: a003 OK FETCH completed
C: a004 fetch 12 body[header]
S: * 12 FETCH (BODY[HEADER] {342}
S: Date: Wed, 17 Jul 1996 02:23:25 -0700 (PDT)
S: From: Terry Gray <gray@cac.washington.edu>
S: Subject: IMAP4rev1 WG mtg summary and minutes
S: To: imap@cac.washington.edu
S: cc: minutes@CNRI.Reston.VA.US, John Klensin <KLENSIN@MIT.EDU>
S: Message-Id: <B27397-0100000@cac.washington.edu>
S: MIME-Version: 1.0
S: Content-Type: TEXT/PLAIN; CHARSET=US-ASCII
S:
S: )
S: a004 OK FETCH completed
C: a005 store 12 +flags \deleted
S: * 12 FETCH (FLAGS (\Seen \Deleted))
S: a005 OK +FLAGS completed
C: a006 logout
S: * BYE IMAP4rev1 server terminating connection
S: a006 OK LOGOUT completed
```
SuperIMAP uses the IDLE command to wait for incoming email, defined in
the
[IMAP4 Idle Command RFC (2177)](https://tools.ietf.org/html/rfc2177). The
IMAP client sends an IDLE command to the server and awaits a
response. When an IMAP connection is in IDLE mode, no other commands
are allowed.
## Development Tasks
The information below is mainly intended at developers who want to modify the SuperIMAP codebase.
#### Running Unit Tests
Run this once:
RAILS_ENV=test rake db:setup db:seed
Then run all tests:
rake test:all
#### Running Stress Tests
The stress test exercises the multi-threaded aspects of SuperIMAP, as
well as the error recovery code. To do this, we point the SuperIMAP IMAP
client code against a local IMAP server and generate a bunch of fake emails
for many users.
Additionally, the IMAP server generates 'chaotic' events; it will
intentionally generate incorrect or gibberish responses. The SuperIMAP
IMAP client code is expected to recover gracefully while using a
minimal amount of system resources.
Run this once:
RAILS_ENV=stress rake db:setup db:seed
Then run the stress test:
script/stress-test
#### Future Work
* Configure stress test to report code coverage.
* Make a way to "sweep" a user's inbox, generating webhook events for all emails.
#### Contributions
To contribute to this project, please fork and file a pull
request. Small patches will be accepted more quickly than large
patches.
## Changes
#### Version 0.1.2
* Re-organize and improve documentation in README.md.
* Detect and handle race condition around changing UIDVALIDITY.
* Properly escape OAuth 2.0 disconnect URL.
* Improve usage of Rails connection pool.
* Clean up heartbeat records during exit.
* Synchronize access to shared objects.
## License
The MIT License (MIT)
Copyright (c) 2015 Rusty Klophaus / FiveStreet.com
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
================================================
FILE: Rakefile
================================================
# Add your own tasks in files placed in lib/tasks ending in .rake,
# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
require File.expand_path('../config/application', __FILE__)
Rails.application.load_tasks
================================================
FILE: app/admin/admin_user.rb
================================================
ActiveAdmin.register AdminUser do
permit_params :email, :password, :password_confirmation
menu priority: 100
config.sort_order = "email_asc"
config.filters = false
index do
column :tag do |obj|
link_to obj.email, admin_admin_user_path(obj)
end
column :last_sign_in_at
end
show do |obj|
panel "Details" do
attributes_table_for obj do
row :id
row :email
row :sign_in_count
row :current_sign_in_at
row :current_sign_in_ip
row :last_sign_in_at
row :last_sign_in_ip
row :created_at
end
end
end
form do |f|
f.inputs "Admin Details" do
f.input :email
f.input :password
f.input :password_confirmation
end
f.actions
end
end
================================================
FILE: app/admin/delayed_job.rb
================================================
ActiveAdmin.register DelayedJob do
menu priority: 90
config.sort_order = "created_at_desc"
config.batch_actions = true
# Only allow viewing and deleting.
actions :all, :except => [:new, :edit]
filter :handler
filter :last_error
filter :queue
index do
column "Handler" do |obj|
link_to obj.handler.slice(0, 250), admin_delayed_job_path(obj)
end
column :created_at
column :failed_at
column :attempts
column :queue
end
show do |obj|
panel "Details" do
attributes_table_for obj do
row :id
row :queue
row :created_at
row :failed_at if obj.attempts > 0
row :attempts if obj.attempts > 0
row :message do
link_to "Download Message", message_admin_delayed_job_path(obj, "eml")
end if /CallNewMailWebhook/.match(obj.handler)
end
end
panel "Handler" do
pre obj.handler
end
panel "Last Error" do
pre obj.last_error
end if obj.attempts > 0
end
member_action :message, :method => :get do
# HACK - Make sure the class is loaded.
CallNewMailWebhook
job = YAML.load(resource.handler)
if job.object.class == CallNewMailWebhook
render :text => job.object.raw_eml, :content_type => 'message/rfc822'
else
render :text => "There was a problem."
end
end
end
================================================
FILE: app/admin/imap_provider.rb
================================================
ActiveAdmin.register ImapProvider do
config.sort_order = "code_asc"
permit_params :code, :title,
:imap_host, :imap_port, :imap_use_ssl,
:smtp_host, :smtp_port, :smtp_domain, :smtp_enable_starttls_auto,
*Plain::ImapProvider.connection_fields,
*Oauth2::ImapProvider.connection_fields
config.filters = false
config.clear_action_items!
actions :all, :except => [:edit, :destroy]
index do
column "Connection Type" do |obj|
link_to "#{obj.title} (#{obj.code})", admin_imap_provider_path(obj)
end
column "IMAP Server" do |obj|
"#{obj.imap_host}:#{obj.imap_port}"
end
column "SMTP Server" do |obj|
"#{obj.smtp_host}:#{obj.smtp_port}"
end
end
show do |obj|
panel "Details" do
attributes_table_for obj do
row :code
row :title
row :type
end
end
panel "IMAP Settings" do
attributes_table_for obj do
row :imap_host
row :imap_port
row :imap_use_ssl
end
end
panel "SMTP Settings" do
attributes_table_for obj do
row :smtp_host
row :smtp_port
row :smtp_domain
row :smtp_enable_starttls_auto
end
end
panel "Connection Settings" do
attributes_table_for obj do
obj.connection_fields.map do |field|
row field
end
end
end if obj.connection_fields.present?
end
end
================================================
FILE: app/admin/mail_log.rb
================================================
ActiveAdmin.register MailLog do
belongs_to :user
config.sort_order = "created_at_desc"
# Only allow viewing.
config.clear_action_items!
actions :all, :except => [:new, :edit, :destroy]
filter :message_id
breadcrumb do
user = User.find(params[:user_id])
connection = user.partner_connection
partner = connection.partner
[
link_to("Partners", admin_partners_path),
link_to(partner.name, admin_partner_path(partner)),
link_to("Connections", admin_partner_partner_connections_path(partner)),
link_to(connection.imap_provider.code, admin_partner_partner_connection_path(partner, connection)),
link_to("Users", admin_partner_connection_users_path(connection)),
link_to(user.email, admin_partner_connection_user_path(connection, user)),
link_to("Mail Logs", admin_user_mail_logs_path(user))
]
end
index do
column :created_at
column "Message ID" do |obj|
link_to obj.message_id, admin_user_mail_log_path(obj.user, obj)
end
column "Transmit Logs", :sortable => :transmit_logs_count do |obj|
link_to("Transmit Logs (#{obj.transmit_logs_count})", admin_mail_log_transmit_logs_path(obj))
end
actions
end
show do
attributes_table do
row :created_at
row :id
row :message_id
end
end
end
================================================
FILE: app/admin/partner.rb
================================================
ActiveAdmin.register Partner do
menu :priority => 0
permit_params :name, :api_key,
:success_url, :failure_url,
:new_mail_webhook,
:user_connected_webhook,
:user_disconnected_webhook
breadcrumb do
[
link_to("Partners", admin_partners_path)
]
end
config.filters = false
index do
column :name do |partner|
link_to partner.name, admin_partner_path(partner)
end
column :links do |partner|
link_to("Connections (#{partner.partner_connections_count})",
admin_partner_partner_connections_path(partner))
end
actions
end
show do |obj|
panel "Connection Settings" do
attributes_table_for obj do
row :name
row :api_key
end
end
panel "Client Side Redirects" do
attributes_table_for obj do
row :success_url
row :failure_url
end
end
panel "Webhooks" do
attributes_table_for obj do
row :user_connected_webhook
row :user_disconnected_webhook
row :new_mail_webhook
end
end
end
form do |f|
f.inputs "Details" do
f.input :name
f.input :api_key unless f.object.new_record?
end
f.inputs "Client Side Redirects" do
f.input :success_url
f.input :failure_url
end
f.inputs "Webhooks" do
f.input :user_connected_webhook
f.input :user_disconnected_webhook
f.input :new_mail_webhook
end
f.actions
end
end
================================================
FILE: app/admin/partner_connection.rb
================================================
ActiveAdmin.register PartnerConnection do
belongs_to :partner
permit_params :imap_provider_id,
*Plain::PartnerConnection.connection_fields,
*Oauth2::PartnerConnection.connection_fields
controller do
def create
imap_provider = ImapProvider.find(params[:partner_connection][:imap_provider_id])
new_type = imap_provider.type.gsub("::ImapProvider", "::PartnerConnection")
params[:partner_connection].merge!(:type => new_type)
super
end
end
breadcrumb do
partner = Partner.find(params[:partner_id])
[
link_to("Partners", admin_partners_path),
link_to(partner.name, admin_partner_path(partner)),
link_to("Connections", admin_partner_partner_connections_path(partner))
]
end
config.filters = false
index do
column "Imap Provider" do |obj|
link_to obj.imap_provider.title, admin_partner_partner_connection_path(obj.partner, obj)
end
column "Links" do |obj|
raw [
link_to("Connection Type",
admin_imap_provider_path(obj)),
link_to("Users (#{obj.users_count})",
admin_partner_connection_users_path(obj))
].join(", ")
end
actions
end
show do |obj|
panel "Details" do
attributes_table_for obj do
row :imap_provider_code
end
end
panel "Connection Settings" do
attributes_table_for obj do
obj.connection_fields.map do |field|
row field
end
end
end if obj.connection_fields.present?
end
form do |f|
f.inputs "Details" do
f.input :imap_provider, :label => "Auth Mechanism",
:as => :select, :collection => ImapProvider.pluck(:title, :id)
end if f.object.new_record?
if !f.object.new_record? && f.object.connection_fields.present?
f.inputs "Connection Settings" do
f.object.connection_fields.each do |field|
f.input field, :input_html => { :value => f.object.send(field) }
end
end
end
f.actions
end
end
================================================
FILE: app/admin/tracer_log.rb
================================================
ActiveAdmin.register TracerLog do
config.sort_order = "created_at_desc"
# Only allow viewing.
config.clear_action_items!
actions :all, :except => [:new, :edit, :destroy]
config.filters = false
index do
column :user
column :uid do |obj|
link_to obj.uid, admin_tracer_log_path(obj)
end
column :created_at
column :detected_at
column "Elapsed" do |obj|
if obj.detected_at && obj.created_at
seconds = obj.detected_at - obj.created_at
"#{seconds.round(2)} s"
end
end
end
end
================================================
FILE: app/admin/transmit_log.rb
================================================
ActiveAdmin.register TransmitLog do
belongs_to :mail_log
config.sort_order = "created_at_desc"
# Only allow viewing.
config.clear_action_items!
actions :all, :except => [:new, :edit, :destroy]
filter :response_code
filter :response_body
breadcrumb do
mail_log = MailLog.find(params[:mail_log_id])
user = mail_log.user
connection = user.partner_connection
partner = connection.partner
[
link_to("Partners", admin_partners_path),
link_to(partner.name, admin_partner_path(partner)),
link_to("Connections", admin_partner_partner_connections_path(partner)),
link_to(connection.imap_provider.code, admin_partner_partner_connection_path(partner, connection)),
link_to("Users", admin_partner_connection_users_path(connection)),
link_to(user.email, admin_partner_connection_user_path(connection, user)),
link_to("Mail Logs", admin_user_mail_logs_path(user)),
link_to(mail_log.id, admin_user_mail_log_path(user, mail_log)),
link_to("Transmit Logs", admin_mail_log_transmit_logs_path(mail_log))
]
end
index do
column :response_code
column :response_body
column :created_at
actions
end
show do
attributes_table do
row :created_at
row :response_code
row :response_body
end
end
end
================================================
FILE: app/admin/user.rb
================================================
ActiveAdmin.register User do
belongs_to :partner_connection
config.sort_order = "email_asc"
permit_params :tag, :enable_tracer,
*Plain::User.connection_fields,
*Oauth2::User.connection_fields
actions :all, :except => [:destroy]
action_item :only => :show do
if user.archived
link_to('Restore User', restore_admin_partner_connection_user_path(params[:partner_connection_id], user.id))
else
link_to('Archive User', archive_admin_partner_connection_user_path(params[:partner_connection_id], user.id))
end
end
member_action :archive, :method => :get do
user = User.find(params[:id])
user.update_attributes!(:archived => true)
redirect_to({:action => :show}, {:notice => "User archived!"})
end
member_action :restore, :method => :get do
user = User.find(params[:id])
user.update_attributes!(:archived => false)
redirect_to({:action => :show}, {:notice => "User restored!"})
end
breadcrumb do
connection = PartnerConnection.find(params[:partner_connection_id])
partner = connection.partner
[
link_to("Partners", admin_partners_path),
link_to(partner.name, admin_partner_path(partner)),
link_to("Connections", admin_partner_partner_connections_path(partner)),
link_to(connection.imap_provider_code,
admin_partner_partner_connection_path(partner, connection)),
link_to("Users",
admin_partner_connection_users_path(connection))
]
end
filter :tag
filter :email
scope :active
scope :archived
scope :tracer
index do
column :tag do |obj|
link_to obj.tag, admin_partner_connection_user_path(obj.connection, obj)
end
column :email do |obj|
if obj.email
link_to obj.email, admin_partner_connection_user_path(obj.connection, obj)
end
end
column "Mail Logs", :sortable => :mail_logs_count do |obj|
link_to("Mail Logs (#{obj.mail_logs_count})",
admin_user_mail_logs_path(obj))
end
column :connected_at
column :last_login_at
column :last_email_at
column :tracer do |obj|
"YES" if obj.enable_tracer
end
column :archived do |obj|
"YES" if obj.archived
end
end
show do |obj|
panel "Details" do
attributes_table_for obj do
row :id
row :tag
row :connected_at
row :last_login_at
row :last_email_at
row :last_uid
row :last_uid_validity
row :type
row "Links" do
link_to("Connect", new_users_connect_url(obj.signed_request_params)) +
", " +
link_to("Disconnect", new_users_disconnect_url(obj.signed_request_params))
# [
# ].join(", ")
end
row :enable_tracer
row :archived
end
end
panel "Connection Settings" do
attributes_table_for obj do
obj.connection_fields.map do |field|
row field
end
end
end if obj.connection_fields.present?
end
form do |f|
f.inputs "Details" do
f.input :tag
f.input :enable_tracer
end
if !f.object.new_record? && f.object.connection_fields.present?
f.inputs "Connection Settings" do
f.object.connection_fields.each do |field|
f.input field, :input_html => { :value => f.object.send(field) }
end
end
end
f.actions
end
end
================================================
FILE: app/assets/images/.keep
================================================
================================================
FILE: app/assets/javascripts/active_admin.js.coffee
================================================
#= require active_admin/base
================================================
FILE: app/assets/javascripts/application.js
================================================
// This is a manifest file that'll be compiled into application.js, which will include all the files
// listed below.
//
// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
// or vendor/assets/javascripts of plugins, if any, can be referenced here using a relative path.
//
// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
// compiled file.
//
// Read Sprockets README (https://github.com/sstephenson/sprockets#sprockets-directives) for details
// about supported directives.
//
//= require jquery
//= require jquery_ujs
//= require turbolinks
//= require_tree .
================================================
FILE: app/assets/stylesheets/active_admin.css.scss
================================================
// SASS variable overrides must be declared before loading up Active Admin's styles.
//
// To view the variables that Active Admin provides, take a look at
// `app/assets/stylesheets/active_admin/mixins/_variables.css.scss` in the
// Active Admin source.
//
// For example, to change the sidebar width:
// $sidebar-width: 242px;
$body-background-color: #FFF !default;
$primary-color: #8B1B89 !default;
$secondary-color: #ffffff !default;
$text-color: #333333 !default;
$table-stripe-color: lighten($primary-color, 66%) !default;
// Active Admin's got SASS!
@import "active_admin/mixins";
@import "active_admin/base";
body {
font-size: 14px;
}
th {
white-space: nowrap;
}
================================================
FILE: app/assets/stylesheets/application.css
================================================
/*
* This is a manifest file that'll be compiled into application.css, which will include all the files
* listed below.
*
* Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
* or vendor/assets/stylesheets of plugins, if any, can be referenced here using a relative path.
*
* You're free to add application-wide styles to this file and they'll appear at the bottom of the
* compiled file so the styles you add here take precedence over styles defined in any styles
* defined in the other CSS/SCSS files in this directory. It is generally better to create a new
* file per style scope.
*
*= require_tree .
*= require_self
*/
================================================
FILE: app/controllers/api/v1/connections_controller.rb
================================================
class Api::V1::ConnectionsController < ApplicationController
layout "blank"
respond_to :json
skip_before_action :verify_authenticity_token
before_action :default_format_json
before_action :load_partner
before_action :load_imap_provider, :only => [:create, :update, :show, :destroy]
before_action :load_connection, :only => [:update, :show, :destroy]
attr_accessor :partner, :imap_provider, :connection
def index
@connections = self.partner.connections
end
def create
self.connection = self.partner.connections.where(:imap_provider_id => imap_provider.id).build
self.connection.update_attributes!(connection_params)
render :show
rescue ActiveRecord::RecordInvalid => e
render :status => :bad_request, :text => e.to_s
end
def update
self.connection.update_attributes!(connection_params)
render :show
rescue ActiveRecord::RecordInvalid => e
render :status => :bad_request, :text => e.to_s
end
def show
# Pass.
end
def destroy
self.connection.destroy
render :status => :no_content, :text => "Deleted connection."
end
private
def default_format_json
request.format = "json" unless params[:format]
end
def load_partner
api_key = request.headers['x-api-key'] || params[:api_key]
self.partner = Partner.find_by_api_key(api_key)
if self.partner.nil?
render :status => :not_found, :text => "Partner not found. Check your api_key."
end
end
def load_imap_provider
code = params[:imap_provider_code]
self.imap_provider = ImapProvider.find_by_code(code)
if self.imap_provider.nil?
render :status => :not_found, :text => "Imap Provider not found for '#{code}'."
end
end
def load_connection
self.connection = self.partner.connections.where(:imap_provider_id => imap_provider.id).first
if self.connection.nil?
render :status => :not_found, :text => "Connection not found."
end
end
def connection_params
if self.connection
params.permit(self.connection.connection_fields)
else
params.permit()
end
end
end
================================================
FILE: app/controllers/api/v1/users_controller.rb
================================================
class Api::V1::UsersController < ApplicationController
layout "blank"
respond_to :json
skip_before_action :verify_authenticity_token
before_action :default_format_json
before_action :load_partner
before_action :load_imap_provider
before_action :load_connection
before_action :load_user, :only => [:update, :show, :destroy]
attr_accessor :partner, :imap_provider, :connection, :user
def index
@users = self.connection.users.order(:email)
end
def create
self.user = self.connection.new_typed_user
self.user.update_attributes!(user_params)
render :show
rescue ActiveRecord::RecordInvalid => e
render :status => :bad_request, :text => e.to_s
end
def update
self.user.update_attributes!(user_params)
render :show
rescue ActiveRecord::RecordInvalid => e
render :status => :bad_request, :text => e.to_s
end
def show
# pass
end
def destroy
self.user.update_attributes(:archived => true)
render :status => :no_content, :text => "Archived user."
end
private
def default_format_json
request.format = "json" unless params[:format]
end
def load_partner
api_key = request.headers['x-api-key'] || params[:api_key]
self.partner = Partner.find_by_api_key(api_key)
if partner.nil?
render :status => :not_found, :text => "Partner not found. Check your api_key."
end
end
def load_imap_provider
code = params[:connection_imap_provider_code]
self.imap_provider = ImapProvider.find_by_code(code)
if self.imap_provider.nil?
render :status => :not_found, :text => "Imap Provider not found for '#{code}'."
end
end
def load_connection
self.connection = self.partner.connections.find_by_imap_provider_id(self.imap_provider.id)
if self.connection.nil?
render :status => :not_found, :text => "Connection not found."
end
end
def load_user
tag = params[:tag]
self.user = self.connection.users.find_by_tag(tag)
if self.user.nil?
render :status => :not_found, :text => "User not found for '#{tag}'."
end
end
def user_params
if self.user
params.permit([:tag, :email, :archived] + self.user.connection_fields)
else
params.permit([:tag, :email, :archived])
end
end
end
================================================
FILE: app/controllers/application_controller.rb
================================================
class ApplicationController < ActionController::Base
# Prevent CSRF attacks by raising an exception.
# For APIs, you may want to use :null_session instead.
protect_from_forgery with: :exception
before_action :ensure_secure
def ensure_secure
if !request.ssl? && Rails.env.production?
redirect_to request.original_url.gsub(/^http:/, "https:")
end
end
end
================================================
FILE: app/controllers/concerns/.keep
================================================
================================================
FILE: app/controllers/concerns/link_rel.rb
================================================
module LinkRel
extend ActiveSupport::Concern
included do
def self.link_rel(tag, url)
@links ||= []
@links << %(<#{url}; rel="#{tag}")
headers['Link'] = @links.join(', ') if links.present?
end
end
end
================================================
FILE: app/controllers/users/base_callback_controller.rb
================================================
class Users::BaseCallbackController < ApplicationController
before_action :load_user
before_action :validate_signature
attr_accessor :user
def new
# Store user in session.
self.user.signed_request_params.each do |key, value|
session[key] = value
end
# Set up the callback URLs.
session[:success_url] = params[:success] || partner.success_url
session[:failure_url] = params[:failure] || partner.failure_url
apply_helper
end
def callback
apply_helper
end
private
def apply_helper
helper = self.user.imap_provider.helper_for(params[:action])
self.send(helper)
end
def load_user(user_id = nil)
# Load from params or for a specific auth method.
self.user = User.find_by_id(user_id || params[:user_id] || session[:user_id])
if self.user.nil?
render :status => :not_found, :text => "User not found."
end
end
def validate_signature(options = {})
# Validate from params or for a specific auth method.
is_valid =
self.user.valid_signature?(options) ||
self.user.valid_signature?(params) ||
self.user.valid_signature?(session)
if !is_valid
render :status => :not_found, :text => "User not found."
end
end
def connection
self.user.connection
end
def partner
self.user.connection.partner
end
def imap_provider
self.user.connection.imap_provider
end
def redirect_to_success_url
redirect_to session[:success_url]
end
def redirect_to_failure_url
redirect_to session[:failure_url]
end
end
================================================
FILE: app/controllers/users/connects_controller.rb
================================================
class Users::ConnectsController < Users::BaseCallbackController
include Plain::ConnectsHelper
include Oauth2::ConnectsHelper
# The new and callback actions are contained in
# Users::BaseCallbackController, which itself calls helpers
# according to the authentication type.
end
================================================
FILE: app/controllers/users/disconnects_controller.rb
================================================
class Users::DisconnectsController < Users::BaseCallbackController
include Plain::DisconnectsHelper
include Oauth2::DisconnectsHelper
# The new and callback actions are contained in
# Users::BaseCallbackController, which itself calls helpers
# according to the authentication type.
end
================================================
FILE: app/controllers/webhook_test_controller.rb
================================================
class WebhookTestController < ApplicationController
layout "blank"
skip_before_action :verify_authenticity_token
attr_accessor :json_params, :user
def new_mail
Log.info request.body
render :status => :ok, :text => "OK"
end
def user_connected
Log.info request.body
render :status => :ok, :text => "OK"
end
def user_disconnected
Log.info request.body
render :status => :ok, :text => "OK"
end
end
================================================
FILE: app/helpers/application_helper.rb
================================================
module ApplicationHelper
def array_to_hash(values)
hash = {}
values.each do |k,v|
hash[k] = v
end
return hash
end
end
================================================
FILE: app/helpers/oauth2/connects_helper.rb
================================================
module Oauth2::ConnectsHelper
BadRequestError = Class.new(StandardError)
attr_accessor :oauth2_token
def oauth2_new_helper
# Construct the client.
client = OAuth2::Client.new(
connection.oauth2_client_id,
connection.oauth2_client_secret_secure,
:site => imap_provider.oauth2_site,
:authorize_url => imap_provider.oauth2_authorize_url)
# Construct the auth url.
auth_url = client.auth_code.authorize_url(
:redirect_uri => callback_users_connect_url(),
:response_type => imap_provider.oauth2_response_type,
:state => "",
:scope => imap_provider.oauth2_scope,
:access_type => imap_provider.oauth2_access_type,
:approval_prompt => imap_provider.oauth2_approval_prompt)
# Redirect.
redirect_to auth_url
end
def oauth2_callback_helper
# Exchange the code for a refresh token.
# https://developers.google.com/accounts/docs/OAuth2WebServer
client = OAuth2::Client.new(
connection.oauth2_client_id,
connection.oauth2_client_secret_secure,
:site => imap_provider.oauth2_site,
:token_url => imap_provider.oauth2_token_url)
self.oauth2_token = client.auth_code.get_token(
params[:code],
:redirect_uri => callback_users_connect_url())
user.update_attributes!(
:email => oauth2_email,
:oauth2_refresh_token => oauth2_token.refresh_token,
:connected_at => Time.now)
begin
CallUserConnectedWebhook.new(user).run
rescue => e
CallUserConnectedWebhook.new(user).delay.run
end
redirect_to_success_url
rescue => e
Log.exception(e)
redirect_to_failure_url
end
def oauth2_email
method = "#{imap_provider.code.downcase}_email".to_sym
send(method)
end
def gmail_oauth2_email
# Get the google email address.
data = JSON.parse(oauth2_token.get("https://www.googleapis.com/userinfo/email?alt=json").body)
return data["data"]["email"]
end
end
================================================
FILE: app/helpers/oauth2/disconnects_helper.rb
================================================
require 'uri'
module Oauth2::DisconnectsHelper
def oauth2_new_helper
# Disconnect the user. Assume that this succeeds.
token = self.user.oauth2_refresh_token_secure || ""
url = "https://accounts.google.com/o/oauth2/revoke?token=#{URI.escape(token)}"
Net::HTTP.get_response(URI(url))
# Throw away our credentials.
self.user.update_attributes!(
:email => nil,
:oauth2_refresh_token => nil,
:connected_at => nil)
begin
CallUserDisconnectedWebhook.new(user).run
rescue => e
CallUserDisconnectedWebhook.new(user).delay.run
end
# Redirect.
redirect_to_success_url
end
end
================================================
FILE: app/helpers/plain/connects_helper.rb
================================================
module Plain::ConnectsHelper
def plain_new_helper
raise :todo
end
def plain_callback_helper
raise :todo
end
end
================================================
FILE: app/helpers/plain/disconnects_helper.rb
================================================
module Plain::DisconnectsHelper
def plain_new_helper
raise :todo
end
end
================================================
FILE: app/interactors/base_webhook.rb
================================================
require 'timeout'
require 'net/imap'
class BaseWebhook
private unless Rails.env.test?
def calculate_signature(api_key, uid, timestamp)
digest = OpenSSL::Digest.new('sha256')
return OpenSSL::HMAC.hexdigest(digest, api_key, "#{timestamp}#{uid}")
end
end
================================================
FILE: app/interactors/call_new_mail_webhook.rb
================================================
# encoding: utf-8
class CallNewMailWebhook < BaseWebhook
attr_accessor :mail_log, :envelope, :raw_eml
def initialize(mail_log, envelope, raw_eml)
self.mail_log = mail_log
self.envelope = envelope
self.raw_eml = raw_eml
end
def run
user = mail_log.user
partner = user.partner_connection.partner
if partner.new_mail_webhook.blank?
return false
end
# Assemble the payload.
data = {
:timestamp => Time.now.to_i,
:sha1 => mail_log.sha1,
:user_tag => user.tag,
:imap_provider_code => user.connection.imap_provider_code,
:envelope => envelope,
:rfc822 => raw_eml
}
data[:signature] = calculate_signature(partner.api_key, data[:sha1], data[:timestamp])
# START DEBUGGING!
begin
envelope.to_json
rescue => e
Log.info("Problem converting to JSON:\n#{envelope}.")
end
begin
raw_eml.to_json
rescue => e
Log.info("Problem converting to JSON:\n#{raw_eml}.")
end
# END DEBUGGING!
# Post the data
begin
transmit_log = mail_log.transmit_logs.create()
# Post the data.
webhook = RestClient::Resource.new(partner.new_mail_webhook)
response = Timeout::timeout(30) do
webhook.post(data.to_json, :content_type => :json, :accept => :json)
end
# Update the transmit log record.
transmit_log.update_attributes!(:response_code => response.code.to_i,
:response_body => response.to_s.slice(0, 1024))
Log.librato(:count, 'app.call_new_mail_webhook.count', 1)
return true
rescue RestClient::Forbidden => e
response = e.response
transmit_log.update_attributes!(:response_code => response.code.to_i,
:response_body => response.to_s.slice(0, 1024))
# The server understood the request but refused it. Mark the
# user as archived, but only if it's not a tracer user.
if !user.enable_tracer
user.update_attributes!(:archived => true)
end
rescue RestClient::Exception => e
response = e.response
transmit_log.update_attributes!(:response_code => response.code.to_i,
:response_body => response.to_s.slice(0, 1024))
raise e
rescue => e
transmit_log.update_attributes!(:response_code => "ERROR",
:response_body => e.to_s.slice(0, 1024))
raise e
end
end
end
================================================
FILE: app/interactors/call_user_connected_webhook.rb
================================================
class CallUserConnectedWebhook < BaseWebhook
attr_accessor :user
def initialize(user)
self.user = user
end
def run
partner = user.partner_connection.partner
if partner.user_connected_webhook.blank?
return false
end
# Assemble the payload.
data = {
:timestamp => Time.now.to_i,
:sha1 => Digest::SHA1.hexdigest(user.tag),
:user_tag => user.tag,
:imap_provider_code => user.connection.imap_provider_code,
:email => user.email
}
data[:signature] = calculate_signature(partner.api_key, data[:sha1], data[:timestamp])
# Post the data.
begin
webhook = RestClient::Resource.new(partner.user_connected_webhook)
response = Timeout::timeout(30) do
webhook.post(data.to_json, :content_type => :json, :accept => :json)
end
Log.librato(:count, 'app.call_user_connected_webhook.count', 1)
rescue RestClient::Forbidden => e
# The server understood the request but refused it. Mark the
# user as archived, but only if it's not a tracer user.
if !user.enable_tracer
user.update_attributes!(:archived => true)
end
end
end
end
================================================
FILE: app/interactors/call_user_disconnected_webhook.rb
================================================
class CallUserDisconnectedWebhook < BaseWebhook
attr_accessor :user
def initialize(user)
self.user = user
end
def run
partner = user.partner_connection.partner
if partner.user_disconnected_webhook.blank?
return false
end
# Assemble the payload.
data = {
:timestamp => Time.now.to_i,
:sha1 => Digest::SHA1.hexdigest(user.tag),
:user_tag => user.tag,
:imap_provider_code => user.connection.imap_provider_code
}
data[:signature] = calculate_signature(partner.api_key, data[:sha1], data[:timestamp])
begin
# Post the data.
webhook = RestClient::Resource.new(partner.user_disconnected_webhook)
response = Timeout::timeout(30) do
webhook.post(data.to_json, :content_type => :json, :accept => :json)
end
Log.librato(:count, 'app.call_user_disconnected_webhook.count', 1)
rescue RestClient::Forbidden => e
# The server understood the request but refused it. Mark the
# user as archived, but only if it's not a tracer user.
if !user.enable_tracer
user.update_attributes!(:archived => true)
end
end
end
end
================================================
FILE: app/interactors/schedule_tracer_emails.rb
================================================
class ScheduleTracerEmails
attr_accessor :user, :num_tracers
def initialize(user, num_tracers)
self.user = user
self.num_tracers = num_tracers
end
def run
num_tracers.times.each do |n|
send_tracer_to_user(user)
end
end
def send_tracer_to_user(user)
# Deliver the mail.
uid = SecureRandom.hex(10)
mail = TracerMailer.tracer_email(user, uid)
user.connection.imap_provider.authenticate_smtp(mail, user)
mail.deliver
Log.librato(:count, 'app.schedule_tracer_email.count', 1)
# Log the tracer.
user.tracer_logs.create!(:uid => uid)
end
end
================================================
FILE: app/mailers/.keep
================================================
================================================
FILE: app/mailers/tracer_mailer.rb
================================================
class TracerMailer < ActionMailer::Base
def tracer_email(user, uid)
@uid = uid
mail(:from => user.email,
:to => user.email,
:subject => "TRACER: #{uid}")
end
end
================================================
FILE: app/models/.keep
================================================
================================================
FILE: app/models/admin_user.rb
================================================
class AdminUser < ActiveRecord::Base
# Include default devise modules. Others available are:
# :confirmable, :lockable, :timeoutable and :omniauthable
devise :database_authenticatable, :trackable, :validatable, :lockable
end
================================================
FILE: app/models/concerns/.keep
================================================
================================================
FILE: app/models/concerns/auth_method_helper.rb
================================================
module AuthMethodHelper
def auth_method_plain?
/^plain$/i.match(self.auth_method)
end
def auth_method_oauth2?
/^oauth2$/i.match(self.oauth_method)
end
end
================================================
FILE: app/models/concerns/connection_fields.rb
================================================
# coding: utf-8
module ConnectionFields
extend ActiveSupport::Concern
included do
@connection_fields = []
def self.encrypt(value)
if Rails.application.config.encryption_cipher && value.present?
Rails.application.config.encryption_cipher.encrypt(value)
else
value
end
end
def self.decrypt(value)
begin
if Rails.application.config.encryption_cipher && value.present?
Rails.application.config.encryption_cipher.decrypt(value)
else
value
end
rescue
value
end
end
def self.connection_field(field, options = {})
@connection_fields ||= []
@connection_fields << field
# Maybe validate presence.
if options[:required]
validates_presence_of(field)
end
# Maybe obscure the actual value.
if options[:secure]
define_method(field) do |secure = false|
if !secure && self[field].present?
"- encrypted -"
else
self.class.decrypt(super())
end
end
define_method("#{field}_secure".to_sym) do
self.send(field, true)
end
define_method("#{field}=".to_sym) do |value|
if value != self.send(field)
super(self.class.encrypt(value))
end
end
end
end
def self.connection_fields
@connection_fields || []
end
def connection_fields
self.class.connection_fields
end
end
end
================================================
FILE: app/models/delayed_job.rb
================================================
class DelayedJob < ActiveRecord::Base
# Run all existing delayed_job records. Log errors, return true if
# everything was run.
def self.flush
count = 0
while job = Delayed::Job.where("locked_at IS NULL").first do
count += 1
raise "Delayed Job loop?" if count > 10
begin
job.invoke_job
job.destroy
rescue => e
print "Problem processing delayed job:\n#{job.to_yaml}"
raise e
end
end
return true
end
end
================================================
FILE: app/models/imap_daemon_heartbeat.rb
================================================
class ImapDaemonHeartbeat < ActiveRecord::Base
end
================================================
FILE: app/models/imap_provider.rb
================================================
class ImapProvider < ActiveRecord::Base
include ConnectionFields
has_many :partner_connections
def display_name
self.code
end
# Public: Single Table Inheritance helper. Returns the correct
# inherited class depending on the ImapProvider class.
#
# Usage:
#
# imap_provider = Oauth2::ImapProvider.new
# imap_provider.class_for(User) => Oauth2::User
#
# Returns a class.
def class_for(c)
(self.class.parent_name + "::" + c.base_class.name).constantize
end
# Public: Single Table Inheritance helper. Returns the name of a
# helper method depending on the ImapProvider class.
#
# Usage:
#
# imap_provider = Oauth2::ImapProvider.new
# imap_provider.helper_for(:connects, :new) => :oauth2_new_connects_helper
#
# Returns a symbol.
def helper_for(action)
"#{self.class.parent_name.underscore}_#{action}_helper".to_sym
end
end
================================================
FILE: app/models/mail_log.rb
================================================
class MailLog < ActiveRecord::Base
belongs_to :user, :counter_cache => true
has_many :transmit_logs, :dependent => :destroy
end
================================================
FILE: app/models/oauth2/imap_provider.rb
================================================
require 'xoauth2_authenticator'
class Oauth2::ImapProvider < ImapProvider
include ConnectionFields
connection_field :oauth2_grant_type, :required => true
connection_field :oauth2_scope, :required => true
connection_field :oauth2_site, :required => true
connection_field :oauth2_token_method, :required => true
connection_field :oauth2_token_url, :required => true
connection_field :oauth2_authorize_url, :required => true
connection_field :oauth2_response_type, :required => true
connection_field :oauth2_access_type, :required => true
connection_field :oauth2_approval_prompt, :required => true
def authenticate_imap(client, user)
client.authenticate('XOAUTH2', user.email, _access_token(user))
end
def authenticate_smtp(mail, user)
mail.delivery_method.settings.merge!(
:address => smtp_host,
:port => smtp_port,
:domain => smtp_domain,
:user_name => user.email,
:password => _access_token(user),
:authentication => :xoauth2,
:enable_starttls_auto => smtp_enable_starttls_auto
)
end
private
def _access_token(user)
partner_connection = user.connection
oauth_client = OAuth2::Client.new(
partner_connection.oauth2_client_id,
partner_connection.oauth2_client_secret_secure,
{
:site => oauth2_site,
:token_url => oauth2_token_url,
:token_method => oauth2_token_method.to_sym,
:grant_type => oauth2_grant_type,
:scope => oauth2_scope
})
oauth2_access_token = oauth_client.get_token(
{
:client_id => partner_connection.oauth2_client_id,
:client_secret => partner_connection.oauth2_client_secret_secure,
:refresh_token => user.oauth2_refresh_token_secure,
:grant_type => oauth2_grant_type
})
oauth2_access_token.token
end
end
================================================
FILE: app/models/oauth2/partner_connection.rb
================================================
class Oauth2::PartnerConnection < PartnerConnection
include ConnectionFields
connection_field :oauth2_client_id, :required => true
connection_field :oauth2_client_secret, :required => true, :secure => true
end
================================================
FILE: app/models/oauth2/user.rb
================================================
class Oauth2::User < User
include ConnectionFields
before_save :update_connected_at
connection_field :email
connection_field :oauth2_refresh_token, :secure => true
def update_connected_at
if email.present? && oauth2_refresh_token.present?
self.connected_at ||= Time.now
else
self.connected_at = nil
end
end
end
================================================
FILE: app/models/partner.rb
================================================
class Partner < ActiveRecord::Base
# Magic.
before_save :ensure_api_key
# Relations
has_many :partner_connections, :dependent => :destroy
alias_method :connections, :partner_connections
# Validations
validates :name, :presence => true
def ensure_api_key
self.api_key ||= SecureRandom.hex(10)
end
# Public: Create a new connection that bases it's type on the
# provided imap_provider. In other words, if this is an
# Oauth2::ImapProvider, then return an Oauth2::ImapProvider.
def new_typed_connection(imap_provider)
connection = imap_provider.class_for(PartnerConnection).new
self.connections << connection
connection
end
end
================================================
FILE: app/models/partner_connection.rb
================================================
class PartnerConnection < ActiveRecord::Base
include ConnectionFields
# Magic.
before_validation :fix_type
# Relations.
belongs_to :partner, :counter_cache => true
belongs_to :imap_provider, :counter_cache => true
has_many :users, :dependent => :destroy
# Validation.
validates_presence_of :imap_provider_id
validates_uniqueness_of :imap_provider_id, :scope => :partner_id
# Public: Used by ActiveAdmin.
def display_name
self.imap_provider_code
end
def imap_provider_code
self.imap_provider.code
end
# Public: Create a new user that bases it's type on the
# PartnerConnection type. In other words, if this is an
# Oauth2::PartnerConnection, then return an Oauth2::User.
def new_typed_user
user = self.imap_provider.class_for(User).new
self.users << user
user
end
private
# Private: Automatically set the STI type based on the imap_provider.
def fix_type
self.type ||= self.imap_provider.class_for(PartnerConnection).to_s
end
end
================================================
FILE: app/models/plain/imap_provider.rb
================================================
class Plain::ImapProvider < ImapProvider
include ConnectionFields
def authenticate_smtp(mail, user)
mail.delivery_method.settings.merge!(
:address => smtp_host,
:port => smtp_port,
:domain => smtp_domain,
:user_name => user.login_username,
:password => user.login_password,
:authentication => :plain,
:enable_starttls_auto => enable_starttls_auto
)
client.login(user.login_username, user.login_password_secure)
end
def authenticate_imap(client, user)
client.login(user.login_username, user.login_password_secure)
end
end
================================================
FILE: app/models/plain/partner_connection.rb
================================================
class Plain::PartnerConnection < PartnerConnection
include ConnectionFields
end
================================================
FILE: app/models/plain/user.rb
================================================
class Plain::User < User
include ConnectionFields
before_save :update_connected_at
connection_field :login_username
connection_field :login_password, :secure => true
def update_connected_at
if login_username.present? && login_password.present?
self.connected_at ||= Time.now
else
self.connected_at = nil
end
end
end
================================================
FILE: app/models/tracer_log.rb
================================================
class TracerLog < ActiveRecord::Base
belongs_to :user
end
================================================
FILE: app/models/transmit_log.rb
================================================
class TransmitLog < ActiveRecord::Base
belongs_to :mail_log, :counter_cache => true
end
================================================
FILE: app/models/user.rb
================================================
class User < ActiveRecord::Base
include ConnectionFields
# Magic.
before_validation :fix_type
# Scopes.
scope :active, proc { where(:archived => false).where.not(:last_email_at => nil) }
scope :archived, proc { where(:archived => true) }
scope :tracer, proc { where(:enable_tracer => true) }
# Relations.
has_many :mail_logs, :dependent => :destroy
has_many :tracer_logs, :dependent => :destroy
belongs_to :partner_connection, :counter_cache => true
alias_method :connection, :partner_connection
# Validations.
validates_presence_of :tag
validates_uniqueness_of :tag, :case_sensitive => false,
:scope => :partner_connection_id,
:conditions => -> { where.not(:archived => true) },
:if => Proc.new { |object| object.tag_changed? }
validates_uniqueness_of :email, :case_sensitive => false,
:scope => :partner_connection_id,
:allow_nil => true,
:conditions => -> { where.not(:archived => true) },
:if => Proc.new { |object| object.email_changed? }
def imap_provider
self.connection.imap_provider
end
# Public: Calculate a timestamped signature. Used to sign redirect
# URLs. Returns a hash.
def signed_request_params(timestamp = nil)
timestamp ||= Time.now.to_i
data = "#{self.id} - #{timestamp} - #{self.connection.partner.api_key}"
{
'user_id' => id,
'ts' => timestamp,
'sig' => Digest::SHA1.hexdigest(data).slice(0, 10)
}
end
# Public: Verify a timestamp signature.
def valid_signature?(params)
(Time.at(params['ts'].to_i) > 30.minutes.ago) &&
params['sig'] == signed_request_params(params['ts'])['sig']
end
private
# Private: Automatically set the STI type based on the imap_provider.
def fix_type
self.type ||= self.partner_connection.imap_provider.class_for(User).to_s
end
end
================================================
FILE: app/processes/common/csv_log.rb
================================================
class Common::CsvLog
include Common::Stoppable
include Common::WrappedThread
attr_accessor :log_path
attr_accessor :log_filehandle
attr_accessor :log_queue
attr_accessor :log_thread
def initialize(log_path)
init_stoppable
self.log_path = log_path
self.log_queue = Queue.new
self.log_thread = wrapped_thread do
_thread_runner
end
end
def log(*values)
self.log_queue << values
end
private
def _thread_runner
self.log_filehandle = File.open(log_path, "w")
while running?
_drain_queue
sleep 0.1
end
_drain_queue
_close_file
end
def _drain_queue
while true
# Don't block, otherwise we can't exit.
values = log_queue.pop(true)
log_filehandle.write(values.join(",") + "\n")
end
rescue ThreadError => e
# Thrown when queue is empty.
log_filehandle.flush()
end
def _close_file
log_filehandle.close
rescue IOError
# May fire if we've already closed the stream elsewhere.
end
end
================================================
FILE: app/processes/common/db_connection.rb
================================================
module Common::DbConnection
def db_config
ActiveRecord::Base.configurations[Rails.env] ||
Rails.application.config.database_configuration[Rails.env]
end
def set_db_connection_pool_size(size)
ActiveRecord::Base.connection_pool.disconnect!
ActiveSupport.on_load(:active_record) do
config = ActiveRecord::Base.configurations[Rails.env] ||
Rails.application.config.database_configuration[Rails.env]
config['pool'] = size
ActiveRecord::Base.establish_connection(config)
end
end
end
================================================
FILE: app/processes/common/light_sleep.rb
================================================
module Common::LightSleep
include Common::Stoppable
def light_sleep(seconds = nil)
now = Time.now
while running?
break if seconds.present? && ((Time.now - now) >= seconds)
sleep 1
end
end
end
================================================
FILE: app/processes/common/stoppable.rb
================================================
module Common::Stoppable
def init_stoppable
@stop = false
@stop_lock = Mutex.new
end
def trap_signals
Signal.trap("INT") do self.stop! end
Signal.trap("TERM") do self.stop! end
end
def stop!
@stop_lock.synchronize do
@stop = true
end
end
def running?
@stop_lock.synchronize do
@stop != true
end
end
def stopping?
@stop_lock.synchronize do
@stop == true
end
end
end
================================================
FILE: app/processes/common/worker_pool.rb
================================================
module Common::WorkerPool
include Common::Stoppable
include Common::WrappedThread
include Common::DbConnection
attr_accessor :worker_rhash
attr_accessor :work_queues, :work_queues_lock
attr_accessor :worker_threads, :worker_threads_lock
attr_accessor :work_queue_latency
def init_worker_pool
self.work_queues = []
self.work_queues_lock = Mutex.new
self.worker_threads = []
self.worker_threads_lock = Mutex.new
self.worker_rhash = ImapClient::RendezvousHash.new
end
# Public: Start a number of worker threads and begin processing
# scheduled work.
def start_worker_pool(num_worker_threads)
# Create work queues.
work_queues_lock.synchronize do
worker_threads_lock.synchronize do
num_worker_threads.times do |n|
work_queue = Queue.new
worker_thread = _start_worker_thread(work_queue)
self.work_queues << work_queue
self.worker_threads << worker_thread
end
end
end
tags = num_worker_threads.times.map(&:to_i)
self.worker_rhash.site_tags = tags
end
# Public: Schedule a task to be executed on one of the work
# queues. If a :hash option is provided, then use this to
# consistently send the work to the same worker. This allows us to
# effectively "single thread" some lines of work.
#
# s - The command to schedule.
# options - Options for the command.
#
# Returns nothing.
def schedule_work(s, options)
raise "No hash specified!" if options[:hash].nil?
index = worker_rhash.hash(options[:hash])
options.merge!(:'$action' => s, :'$time' => Time.now)
work_queues_lock.synchronize do
work_queues[index] << options
end
end
# Public: Return the total number of scheduled items in the work queue.
def work_queue_length
work_queues_lock.synchronize do
work_queues.map(&:size).inject(&:+)
end
end
# Public: Wait for the worker pool to finish processing all items.
def terminate_worker_pool
Log.info("Waiting for worker threads...")
worker_threads_lock.synchronize do
worker_threads.present? && worker_threads.map(&:terminate)
end
end
private
def _start_worker_thread(work_queue)
wrapped_thread do
ActiveRecord::Base.connection_pool.with_connection do |conn|
_worker_thread_runner(work_queue)
end
end
end
def _worker_thread_runner(work_queue)
# Create a work queue.
while running?
_worker_thread_next_action(work_queue)
end
end
def _worker_thread_next_action(queue)
# Don't block, otherwise we can't exit.
options = queue.pop(true)
method = "action_#{options[:'$action']}".to_sym
# Run the action.
begin
self.send(method.to_sym, options)
rescue => e
Log.exception(e)
end
# Track the most recent latency.
self.work_queue_latency = Time.now - options[:'$time']
rescue ThreadError => e
# Queue is empty.
sleep 0.1
end
end
================================================
FILE: app/processes/common/wrapped_thread.rb
================================================
module Common::WrappedThread
def wrapped_thread(&block)
Thread.new do
begin
yield
rescue => e
Log.exception(e)
end
end
end
end
================================================
FILE: app/processes/imap_client/daemon.rb
================================================
# ImapClient::Daemon - Main entry point for the IMAP client. Reads
# credentials from a database, connects to IMAP servers, listens for
# email, generates webhook events.
#
# Contains load-balancing code so that when multiple ImapClient::Daemon
# processes are running they claim users evenly. The ImapClient::Daemon
# coordinate through the database.
#
# Starts a few different types of threads:
#
# + Heartbeat thread - Publishes our heartbeat to the database. Runs every 10 seconds.
# + Discovery thread - Listen for the heartbeats of other ImapClient::Daemon processes. Runs every 30 seconds.
# + Claim thread - Claims an even share of users. Runs every 30 seconds.
# + Worker threads - A small pool of threads to run CPU intensive operations. (5 by default.)
# + User threads - A list of threads created on demand to managed communication with IMAP server. (Limited to 500 by default.)
require 'net/imap'
class ImapClient::Daemon
include Common::Stoppable
include Common::WorkerPool
include Common::LightSleep
include Common::WrappedThread
include Common::DbConnection
# Config variables.
attr_accessor :stress_test_mode, :chaos_mode, :enable_profiler
attr_accessor :num_worker_threads, :max_user_threads, :max_email_size, :tracer_interval, :num_tracers
# Threads.
attr_accessor :heartbeat_thread, :discovery_thread, :claim_thread, :tracer_thread
# Instance variables.
attr_accessor :server_tag, :server_rhash
# Shared variables.
attr_accessor :user_threads, :user_threads_lock
attr_accessor :error_counts, :error_counts_lock
attr_accessor :tracer_emails_processed
attr_accessor :total_emails_processed
# Stress testing variables.
attr_accessor :processed_log
def initialize(options = {})
# Settings.
self.stress_test_mode = options.fetch(:stress_test_mode)
self.chaos_mode = options.fetch(:enable_chaos)
self.num_worker_threads = options.fetch(:num_worker_threads)
self.max_user_threads = options.fetch(:max_user_threads)
self.max_email_size = options.fetch(:max_email_size)
self.tracer_interval = options.fetch(:tracer_interval)
self.num_tracers = options.fetch(:num_tracers)
self.enable_profiler = options.fetch(:enable_profiler)
# Initialize mixins.
init_stoppable
init_worker_pool
# Load balancing stuff.
self.server_tag = SecureRandom.hex(10)
self.server_rhash = ImapClient::RendezvousHash.new
# User stuff.
self.user_threads = {}
self.user_threads_lock = Mutex.new
# Error count stuff.
self.error_counts = {}
self.error_counts_lock = Mutex.new
# Stats.
self.total_emails_processed = 0
end
#
# Thread safe methods for `user_threads` object.
#
# Public: Get number of user threads.
def user_threads_count
user_threads_lock.synchronize do
user_threads.count
end
end
# Public: Get user thread for a user.
def get_user_thread(user_id)
user_threads_lock.synchronize do
user_threads[user_id]
end
end
# Public: Set user thread for a user.
def set_user_thread(user_id, user_thread)
user_threads_lock.synchronize do
user_threads[user_id] = user_thread
end
end
# Public: Delete user thread.
def delete_user_thread(user_id)
user_threads_lock.synchronize do
user_threads.delete(user_id)
end
end
# Public: Execute a block on all user threads. Block takes user_id,
# user_thread parameters.
def walk_user_threads
user_threads_lock.synchronize do
user_threads.each do |user_id, user_thread|
yield(user_id, user_thread)
end
end
end
#
# Thread safe methods for `error_counts`.
#
# Public: Returns the number of errors for a user.
def get_error_count(user_id)
error_counts_lock.synchronize do
self.error_counts[user_id] ||= 0
end
end
# Public: Increment the error count for a user
def increment_error_count(user_id)
error_counts_lock.synchronize do
self.error_counts[user_id] ||= 0
self.error_counts[user_id] += 1
end
end
# Public:
def clear_error_count(user_id)
error_counts_lock.synchronize do
self.error_counts[user_id] = 0
end
end
def run
trap_signals
force_class_loading
maybe_start_profiling
# If stress testing, start a log.
if self.stress_test_mode
self.processed_log = Common::CsvLog.new("./log/stress/processed_emails_#{server_tag}.csv")
end
# Start our threads. We need one thread for each worker, plus four
# additional threads -- one each for the heartbeat, discovery,
# claim, and tracer threads.
set_db_connection_pool_size(self.num_worker_threads + 5)
start_worker_pool(num_worker_threads)
start_heartbeat_thread
start_discovery_thread
start_claim_thread
start_tracer_thread
# Sleep until we are stopped.
light_sleep
rescue Exception => e
Log.error("ImapClient::Daemon is stopping because of an exception.")
Log.exception(e)
stop!
ensure
stop!
Log.info("ImapClient::Daemon is stopping.")
heartbeat_thread && heartbeat_thread.terminate
discovery_thread && discovery_thread.terminate
claim_thread && claim_thread.terminate
terminate_worker_pool
terminate_user_threads
self.processed_log.stop! if self.processed_log
stop_profiling
end
private
def force_class_loading
# Force ImapDaemonHeartbeat to load before we create any
# threads. This fixes a "Circular dependency detected while
# autoloading constant ImapDaemonHeartbeat" error.
ImapDaemonHeartbeat
CallNewMailWebhook
end
def start_heartbeat_thread
self.heartbeat_thread = Thread.new do
ActiveRecord::Base.connection_pool.with_connection do |conn|
heartbeat_thread_runner
end
end
end
def start_discovery_thread
self.discovery_thread = Thread.new do
ActiveRecord::Base.connection_pool.with_connection do |conn|
discovery_thread_runner
end
end
# Wait for servers.
while running? && server_rhash.size == 0
Log.info("Discovering other daemons...")
light_sleep 1
end
end
def start_claim_thread
self.claim_thread = Thread.new do
ActiveRecord::Base.connection_pool.with_connection do |conn|
claim_thread_runner
end
end
end
def start_tracer_thread
self.tracer_thread = Thread.new do
ActiveRecord::Base.connection_pool.with_connection do |conn|
tracer_thread_runner
end
end
end
# Private: Creates/updates an ImapDaemonHeartbeat record in the
# database every 10 seconds.
def heartbeat_thread_runner
heartbeat = ImapDaemonHeartbeat.create(:tag => server_tag)
while running?
# Update the heartbeat.
heartbeat.touch
# Log Heroku / Librato stats.
Log.librato(:measure, 'imap_client.thread.count', Thread.list.count)
Log.librato(:measure, 'imap_client.work_queue.length', work_queue_length)
Log.librato(:measure, 'imap_client.user_thread.count', user_threads_count)
Log.librato(:measure, 'work_queue.latency', work_queue_latency)
Log.librato(:sample, 'imap_client.total_emails_processed', total_emails_processed)
light_sleep 10
end
rescue Exception => e
Log.exception(e)
raise e
ensure
Log.info("Stopping heartbeat thread.")
heartbeat.delete if heartbeat
end
# Private: Fetches all recently updated ImapDaemonHeartbeat records
# in the database very 30 seconds. Create a new RendezvousHash from
# the associated tags.
def discovery_thread_runner
while running?
tags = ImapDaemonHeartbeat.where("updated_at >= ?", 30.seconds.ago).map(&:tag)
Log.info("There are #{tags.count} daemons running.")
self.server_rhash.site_tags = tags
if server_rhash.size == 0
light_sleep 1
else
light_sleep 10
end
end
rescue Exception => e
Log.exception(e)
raise e
ensure
Log.info("Stopping discovery thread.")
end
# Private: Iterate through users and schedule a connect or
# disconnect task depending on whether the user is hashed to this
# server.
def claim_thread_runner
while running?
User.select(:id, :email, :connected_at, :archived).find_each do |user|
if server_rhash.hash(user.id) == server_tag && !user.archived && user.connected_at
schedule_work(:connect_user, :hash => user.id, :user_id => user.id)
else
schedule_work(:disconnect_user, :hash => user.id, :user_id => user.id)
end
# Try not to peg the processor.
sleep 0.01
end
light_sleep 10
end
rescue Exception => e
Log.exception(e)
raise e
ensure
Log.info("Stopping claim thread.")
end
# Private: Schedule a tracer emails to random tracer users.
def tracer_thread_runner
while running?
# Get a random user assigned to this server.
user = User.where(:enable_tracer => true, :archived => false).select(:id, :connected_at).all.select do |user|
server_rhash.hash(user.id) == server_tag && user.connected_at
end.shuffle.first
# If we found a user, schedule a tracer email.
if user
user.reload
ScheduleTracerEmails.new(user, self.num_tracers).delay.run
end
light_sleep self.tracer_interval
end
rescue Exception => e
Log.exception(e)
raise e
ensure
Log.info("Stopping tracer thread.")
end
# Private: Construct and return user thread options.
def user_options
@user_options ||= {
:max_email_size => self.max_email_size
}
end
# Private: Create a new user thread for the specified user.
#
# options[:user_id] - The user id.
def action_connect_user(options)
user_id = options[:user_id]
# Nothing to do if stopped.
return if stopping?
# Are we allowed to create a new user thread?
if user_threads_count > max_user_threads
Log.error("Error: Reached maximum number of users (#{max_user_threads}).")
return
end
# Nothing to do if already a running (or sleeping) thread.
user_thread = get_user_thread(user_id)
return if user_thread.present? && user_thread.alive?
# Load the user; preload connection information.
user = User.find(user_id)
user.connection.imap_provider
# Start the thread.
user_thread = wrapped_thread do
Log.info("Connecting #{user.email}...")
ImapClient::UserThread.new(self, user, user_options).run
end
set_user_thread(user_id, user_thread)
rescue => e
Log.exception(e)
end
# Private: Disconnect a user and destroy the user thread.
#
# options[:user_id] - The user id.
def action_disconnect_user(options)
# Nothing to do if no thread.
user_id = options[:user_id]
return if get_user_thread(user_id).nil?
# Tell the thread to stop.
thread = delete_user_thread(user_id)
thread.terminate
rescue => e
Log.exception(e)
end
# Private: Stop and disconnect all user threads.
def terminate_user_threads
# Terminate all threads, ignore exceptions.
walk_user_threads do |user_id, user_thread|
user_thread.terminate rescue nil
end
# Wait for all threads to finish.
walk_user_threads do |user_id, user_thread|
user_thread.join if user_thread.alive?
end
end
# Private: Run a function, then restart a user thread.
#
# See UserThread#schedule for more details.
#
# options[:block] - The block to run.
# options[:thread] - The thread to restart.
def action_callback(options)
# Wait until the calling thread goes to sleep.
while options[:thread].status == "run"
sleep 0.1
end
# Run the block.
if options[:thread].status == "sleep"
# Call the callback.
options[:block].call
end
rescue => e
Log.exception(e)
ensure
# Wake up the thread.
if options[:thread].status == "sleep"
options[:thread].run
end
end
# Private.
def maybe_start_profiling
return unless enable_profiler
Log.info("Starting profiler...")
require 'ruby-prof'
RubyProf.start
end
# Private.
def stop_profiling
return unless enable_profiler
Log.info("Stopping profiler...")
result = RubyProf.stop
printer = RubyProf::CallStackPrinter.new(result)
File.open("./tmp/profile.html", 'w') do |file|
printer.print(file)
end
end
end
================================================
FILE: app/processes/imap_client/process_uid.rb
================================================
# Private: Read and act on a single email. This is one place where
# Ruby support for monads would be useful. The challenge is that we
# have to verify a lot of data in a very specific sequence, and at any
# time we could either abort (ie: skip the remaining operations) or
# raise an exception.
#
# We build something similar by creating a 'Maybe' control structure
# where we submit blocks of code to an object. A block is considered
# successful if it returns true and doesn't throw any exceptions. If a
# block is not successful, we skip the remaining blocks.
#
# Note that every database touch is wrapped in a call to
# `user_thread.schedule(&block)`. This allows us to avoid creating a
# separate database connection for each user thread.
class ProcessUid
attr_accessor :user_thread, :uid
attr_accessor :internal_date, :message_size
attr_accessor :raw_eml, :envelope
attr_accessor :message_id, :sha1
attr_accessor :mail_log
def initialize(user_thread, uid)
self.user_thread = user_thread
self.uid = uid
end
# Public: Process the email.
def run
# Run all the steps below. Stop as soon as one of them returns
# false or throws an error.
true &&
fetch_internal_date_and_size &&
check_for_really_old_internal_date &&
check_for_pre_creation_internal_date &&
check_for_relapsed_internal_date &&
check_for_big_messages &&
fetch_uid_envelope_rfc822 &&
update_user_mark_email_processed &&
handle_tracer_email &&
check_for_duplicate_message_id &&
check_for_duplicate_sha1 &&
create_mail_log &&
deploy_webhook &&
update_daemon_stats
ensure
clean_up
end
# Private: The User model.
def user
user_thread.user
end
# Private: The IMAP client instance.
def client
user_thread.client
end
# Private: The imap_client daemon.
def daemon
user_thread.daemon
end
private
def confirm_tracer(tracer_uid)
user_thread.schedule do
tracer = TracerLog.find_by_uid(tracer_uid) || TracerLog.new(:uid => tracer_uid)
tracer.update_attributes!(:detected_at => Time.now)
end
end
def fetch_internal_date_and_size
responses = Timeout::timeout(30) do
client.uid_fetch([uid], ["INTERNALDATE", "RFC822.SIZE"])
end
response = responses && responses.first
# If there was no response, then skip this message.
if response.nil?
user_thread.update_user(:last_uid => uid)
return false
end
# Save the internal_date and message_size for later.
self.internal_date = Time.parse(response.attr["INTERNALDATE"])
self.message_size = (response.attr["RFC822.SIZE"] || 0).to_i
return true
rescue Timeout::Error => e
# If this email triggered a timeout, then skip it.
user_thread.update_user(:last_uid => uid)
raise e
end
# Private: Check for a really old date. If it's old, then we should
# stop counting on our UID knowledge and go back to loading UIDs by
# date.
def check_for_really_old_internal_date
if internal_date < 4.days.ago
Log.librato(:count, "system.process_uid.really_old_internal_date", 1)
user_thread.update_user(:last_uid => nil, :last_uid_validity => nil)
user_thread.stop!
return false
else
return true
end
end
# Private: Don't process emails that arrived before this user was
# created.
def check_for_pre_creation_internal_date
if internal_date < user.created_at
Log.librato(:count, "system.process_uid.pre_creation_internal_date", 1)
user_thread.update_user(:last_uid => uid)
return false
else
return true
end
end
# Private: Don't process emails that are significantly older than
# the last internal date that we've processed.
def check_for_relapsed_internal_date
if user.last_internal_date && internal_date < (user.last_internal_date - 1.hour)
Log.librato(:count, "system.process_uid.relapsed_internal_date", 1)
user_thread.update_user(:last_uid => uid)
return false
else
return true
end
end
# Private: Skip emails that are too big.
def check_for_big_messages
if message_size > user_thread.options[:max_email_size]
Log.librato(:count, "system.process_uid.big_message", 1)
user_thread.update_user(:last_uid => uid)
return false
else
return true
end
end
def fetch_uid_envelope_rfc822
# Load the email body.
responses = Timeout::timeout(30) do
self.client.uid_fetch([uid], ["UID", "ENVELOPE", "RFC822"])
end
response = responses && responses.first
# If there was no response, then skip this message.
if response.nil?
Log.librato(:count, "system.process_uid.uid_fetch_no_response", 1)
user_thread.update_user(:last_uid => uid)
return false
end
# Save the internal_date and message_size for later.
self.uid = response.attr["UID"]
self.raw_eml = to_utf8(response.attr["RFC822"])
self.envelope = response.attr["ENVELOPE"]
self.message_id = (envelope.message_id || "#{user.email} - #{uid} - #{internal_date}").slice(0, 255)
return true
rescue Timeout::Error => e
# If this email triggered a timeout, then skip it.
user_thread.update_user(:last_uid => uid)
raise e
end
# Private: Update the high-water mark for which emails we've
# processed.
def update_user_mark_email_processed
# Ignore any suspicious looking internal dates. Sometimes
# misconfigured email servers means that email arrives from the
# future.
if internal_date > Time.now
Log.librato(:count, "system.process_uid.fix_suspicious_internal_date", 1)
self.internal_date = user.last_internal_date
end
# Update the user.
user_thread.update_user(:last_uid => uid,
:last_email_at => Time.now,
:last_internal_date => internal_date)
return true
end
# Private: Is this a tracer? If so, update the TracerLog and stop
# processing.
def handle_tracer_email
if m = /^TRACER: (.+)$/.match(envelope.subject)
tracer_uid = m[1]
confirm_tracer(tracer_uid)
user_thread.update_user(:last_uid => uid)
daemon.total_emails_processed += 1
return false
else
return true
end
end
# Private: Have we already processed this message_id?
def check_for_duplicate_message_id
old_mail_log = nil
user_thread.schedule do
old_mail_log = user.mail_logs.find_by_message_id(message_id)
end
if old_mail_log
Log.librato(:count, "system.process_uid.duplicate_message_id", 1)
return false
else
return true
end
end
# Private: Have we already processed this sha1 hash? This helps us
# catch rare cases where an email doesn't have a message_id so we
# make one up, so the message_id is unique, but the email is a
# duplicate. This may be unnecessary.
def check_for_duplicate_sha1
# Generate the SHA1.
self.sha1 = Digest::SHA1.hexdigest(raw_eml)
old_mail_log = nil
user_thread.schedule do
old_mail_log = user.mail_logs.find_by_sha1(sha1)
end
if old_mail_log
Log.librato(:count, "system.process_uid.duplicate_sha1", 1)
return false
else
return true
end
end
# Private: Log the mail.
def create_mail_log
user_thread.schedule do
self.mail_log = user.mail_logs.create(:message_id => message_id, :sha1 => sha1)
end
return true
end
# Private: Deploy the web hook.
def deploy_webhook
unless daemon.stress_test_mode
user_thread.schedule do
CallNewMailWebhook.new(mail_log, envelope, raw_eml).delay.run
end
end
return true
end
# Private: Update stats
def update_daemon_stats
daemon.clear_error_count(user.id)
daemon.processed_log &&
daemon.processed_log.log(Time.now, user.email, message_id)
daemon.total_emails_processed += 1
return true
end
# Private: Help the garbage collector know what it can collect.
def clean_up
self.user_thread = nil
self.uid = nil
self.internal_date = nil
self.raw_eml = nil
self.envelope = nil
self.message_id = nil
self.sha1 = nil
self.mail_log = nil
end
# Private: Convert a string UTF-8 format.
def to_utf8(s)
return nil if s.nil?
# Attempt to politely transcode the string.
s.encode("UTF-8").scrub
rescue
# If that doesn't work, then overwrite the existing encoding and
# clobber any strange characters.
s.force_encoding("UTF-8").scrub
end
end
================================================
FILE: app/processes/imap_client/rendezvous_hash.rb
================================================
# http://en.wikipedia.org/wiki/Rendezvous_hashing
class ImapClient::RendezvousHash
attr_accessor :lock
def initialize
@site_tags = []
@lock = Mutex.new
end
def site_tags=(site_tags)
lock.synchronize do
@site_tags = site_tags
end
end
# Return the number of sites.
def size
lock.synchronize do
return @site_tags.length
end
end
# Return the highest priority item.
def hash(object_tag)
hashes = self.lock.synchronize do
@site_tags.map do |site_tag|
h = Digest::SHA1.hexdigest("#{site_tag} - #{object_tag}")
[h, site_tag]
end
end
priority = hashes.sort
if priority.length > 0
priority[0][1]
end
end
end
================================================
FILE: app/processes/imap_client/user_thread.rb
================================================
# ImapClient::UserThread - Manages the interactions of a single user
# against an IMAP server. Written in a "crash-only" style. If
# something goes wrong, we tear down the whole thread and start from
# scratch.
#
# The basic lifecycle is:
#
# + Connects to the server.
# + Catch up on email we may have missed.
# + Go into IMAP IDLE mode.
# + Pop out of IDLE mode, catch up on emails.
# + Repeat until we get a "terminate" signal.
# + When terminated, or on error, disconnect ourselves.
require 'net/imap'
require 'timeout'
class ImapClient::UserThread
include Common::LightSleep
include Common::Stoppable
attr_accessor :daemon, :options
attr_accessor :user, :client, :folder_name
attr_accessor :uid_validity
def initialize(daemon, user, options)
init_stoppable
self.daemon = daemon
self.user = user
self.options = options
end
# Private: Update a user record in a scheduled way.
def update_user(hash)
schedule do
if !user.update_attributes(hash)
Log.error("Could not update #{user.email} - #{user.errors.to_h}")
stop!
end
end
end
def run
delay_start
connect if running?
authenticate if running?
choose_folder if running?
update_uid_validity if running?
main_loop if running?
rescue => e
log_exception(e)
stat_exception(e)
self.daemon.increment_error_count(user.id)
stop!
ensure
stop!
daemon.schedule_work(:disconnect_user, :hash => user.id, :user_id => user.id)
disconnect
Log.info("Disconnected #{user.email}.")
clean_up
end
# Private: Schedule a block of code to run in a worker thread
# (rather than a user thread).
#
# We do this in order to prevent ourselves from using too many
# database connections, exhausting memory, and redlining the CPU at
# startup or at times of lots of email activity. This helps smooth
# out the load.
#
# Without this, we would have all the user threads maintaining
# separate database connections and trying to crunch through emails
# as quickly as possible at the same time.
def schedule(&block)
# Schedule the block to run on a worker thread, and put ourselves to sleep.
daemon.schedule_work(:callback,
:hash => user.id,
:user_id => user.id,
:block => block,
:thread => Thread.current)
# Put ourselves to sleep. The worker will call Thread.run to wake us back up.
sleep
end
# Private: Log exceptions. Be less verbose if we're stress testing.
def log_exception(e)
imap_exceptions = [
Net::IMAP::Error,
Net::IMAP::ResponseParseError,
IOError,
EOFError,
Errno::EPIPE
]
# Minimally log imap exceptions when stress testing.
if imap_exceptions.include?(e.class) && self.daemon.stress_test_mode
Log.info("#{e.class} - #{e.to_s}")
else
Log.info("Encountered error for #{user.email}: #{e.class} - #{e.to_s}")
Log.exception(e)
end
end
# Private: Log operations metrics. Be less verbose if we're stress testing.
def stat_exception(e)
return if self.daemon.stress_test_mode
category = "error." + e.class.to_s.gsub("::", "_")
Log.librato(:count, category, 1)
end
private unless Rails.env.test?
# Private: Exponentially backoff based on the number of errors we
# are seeing for a given user. At most, wait 5 minutes before trying
# to connect.
def delay_start
errors = self.daemon.get_error_count(user.id)
seconds = (errors ** 3) - 1
seconds = [seconds, 300].min
Log.librato(:measure, 'user_thread.delayed_start', seconds) if seconds > 0
light_sleep seconds
end
# Private: Connect to the server, set the client.
def connect
conn_type = user.imap_provider
self.client = Net::IMAP.new(conn_type.imap_host,
:port => conn_type.imap_port,
:ssl => conn_type.imap_use_ssl)
end
# Private: Authenticate a user to the server.
def authenticate
user.connection.imap_provider.authenticate_imap(client, user)
update_user(:last_login_at => Time.now)
rescue OAuth::Error,
Net::IMAP::NoResponseError,
Net::IMAP::ByeResponseError,
OAuth2::Error => e
# If we encounter an OAuth error during authentication, then the
# credentials are probably invalid. We don't want to log every
# occurrance of this, and we don't want to archive the user, so
# we'll just back off from reconnecting again.
stat_exception(e)
Log.info("Encountered error for #{user.email}: #{e.class} - #{e.to_s}")
self.daemon.increment_error_count(user.id)
stop!
end
# Private: Fetch a list of folders, choose the first one that looks
# promising.
def choose_folder
# TODO: This should probably live in the imap_provider model.
best_folders = [
"[Gmail]/All Mail",
"[Google Mail]/All Mail",
"INBOX"
]
# Discover the folder.
client.list("", "*").each do |folder|
if best_folders.include?(folder.name)
self.folder_name = folder.name
break
end
end
# Examine the folder.
client.examine(folder_name)
end
# Private: Return true if our knowledge of the server's uid is still
# valid. See "http://tools.ietf.org/html/rfc4549#section-4.1"
def update_uid_validity
# Get the latest validity value.
response = client.status(folder_name.to_s, attrs=['UIDVALIDITY'])
self.uid_validity = response['UIDVALIDITY']
if user.last_uid_validity.to_s != self.uid_validity.to_s
# Update the user with the new validity value, invalidate the
# old last_uid value.
update_user(:last_uid_validity => self.uid_validity, :last_uid => nil)
end
end
# Private: Start a loop that alternates between idling and reading
# email.
def main_loop
while running?
verify_uid_validity if running?
jumpstart_stalled_account if running?
# Read emails until we have read everything there is to
# read. Then go into idle mode.
last_read_count = 9999
while running? && last_read_count > 0
if user.last_uid.present?
last_read_count = read_email_by_uid
else
last_read_count = read_email_by_date
end
end
wait_for_email if running?
end
end
# Private: Stop if the uid_validity was changed by someone else.
#
# Because SuperIMAP tries to avoid locks, there is a chance that two
# machines in a cluster may briefly own the same user. If this
# happens and the server manages to change the UIDVALIDITY settings,
# then there is a chance that one server could change the
# `uid_validity` attribute, and the other server could set the
# `last_uid` attribute. This would put the user out of sync, we
# would begin querying for UIDS that no longer exist, and we'd miss
# emails.
#
# To work around this, we refresh the user object and make sure that
# the `uid_validity` field hasn't changed from what we know it to
# be. If it has, then we'll stop this instance of the thread.
def verify_uid_validity
schedule do
user.reload
end
stop! if self.uid_validity.present? && (self.uid_validity.to_s != user.last_uid_validity.to_s)
end
# Private: Check for accounts that haven't had an email in 24 hours.
# When this happens, start reading emails by date, not UID. This
# hopefully guards against accounts getting into a weird state and
# getting locked.
def jumpstart_stalled_account
if user.last_email_at && user.last_email_at < 24.hours.ago
update_user(:last_uid => nil)
end
end
# Private: Search for new email by uid. See
# "https://tools.ietf.org/html/rfc3501#section-2.3.1.1"e
#
# Returns the number of emails read.
def read_email_by_uid
# Read in batches of 100 emails.
batch_size = 100
uids = client.uid_search(["UID", "#{user.last_uid + 1}:#{user.last_uid + batch_size}"])
uids.each do |uid|
break if stopping?
process_uid(uid) unless stopping?
end
return uids.count
end
# Private: Search for new email by date. See
# "https://tools.ietf.org/html/rfc3501#section-6.4.4"
#
# Returns the number of uids read.
def read_email_by_date
# Search by date. Unfortunately, IMAP date searching isn't very
# granular. To ensure we get all emails we go back two full
# days. We filter out duplicates later.
date_string = 2.days.ago.strftime("%d-%b-%Y")
uids = client.uid_search(["SINCE", date_string])
uids.each do |uid|
break if stopping?
process_uid(uid) unless stopping?
end
return uids.count
end
# Private: Put the connection into idle mode, exit when we receive a
# new EXISTS message.
# See "https://tools.ietf.org/html/rfc3501#section-7.3.1"
def wait_for_email
client.idle do |response|
if response &&
response.respond_to?(:name) &&
response.name == "EXISTS"
client.idle_done()
elsif stopping?
client.idle_done()
end
end
rescue Net::IMAP::Error,
EOFError => e
# Recover gracefully.
stat_exception(e)
Log.info("Encountered error for #{user.email}: #{e.class} - #{e.to_s}")
self.daemon.increment_error_count(user.id)
stop!
end
# Private: Read and act on a single email. Calls the ProcessUid
# interactor.
#
# + uid - The UID of the email.
def process_uid(uid)
ProcessUid.new(self, uid).run
rescue Timeout::Error => e
# Recover gracefully.
stat_exception(e)
Log.info("Encountered error for #{user.email}: #{e.class} - #{e.to_s}")
self.daemon.increment_error_count(user.id)
stop!
end
# Private: Logout the user, disconnect the client.
def disconnect
begin
client && client.logout
rescue => e
# Ignore errors.
end
begin
client && client.disconnect
rescue => e
# Ignore errors.
end
# The client is no longer connected.
self.client = nil
end
# Private: Help the garbage collector know what it can collect.
def clean_up
self.daemon = nil
self.user = nil
self.client = nil
end
end
================================================
FILE: app/processes/imap_client.rb
================================================
class ImapClient
VERSION="1.0.0"
end
require 'imap_client/rendezvous_hash'
require 'imap_client/process_uid'
require 'imap_client/user_thread'
require 'imap_client/daemon'
================================================
FILE: app/processes/imap_test_server/daemon.rb
================================================
# ImapTestServer::Daemon - A test IMAP server that can respond to all
# calls made by the ImapClient process. Generates test data and
# (muhahaha) deliberately responds to some calls with gibberish in
# order to test how well ImapClient recovers.
#
# The daemon has three threads:
# + The Connection thread listens for incoming connections.
# + The New Mail thread generates new emails to users.
# + The Process Sockets (main) thread sits in a tight loop, sending and receiving IMAP commands.
#
# The code is organized as follows:
#
# + The Daemon class contains high level connection logic.
# + SocketState holds the state of a socket and contains our IMAP logic.
# + Mailboxes holds mail for all users.
require 'socket'
class ImapTestServer::Daemon
include Common::Stoppable
include Common::LightSleep
include Common::WrappedThread
include Common::DbConnection
attr_accessor :port, :enable_chaos, :emails_per_minute, :length_of_test
attr_accessor :stats_thread
attr_accessor :connection_thread
attr_accessor :new_sockets, :sockets, :socket_states
attr_accessor :mailboxes
attr_accessor :total_emails_generated, :total_emails_fetched
attr_accessor :generated_log, :fetched_log, :events_log
def initialize(options = {})
# Initialize mixins.
init_stoppable
# Config stuff.
self.port = options.fetch(:port)
self.enable_chaos = options.fetch(:enable_chaos)
self.emails_per_minute = options.fetch(:emails_per_minute)
self.length_of_test = options.fetch(:length_of_test)
# Socket stuff.
self.new_sockets = Queue.new
self.sockets = []
self.socket_states = {}
# Mailboxes.
self.mailboxes = Mailboxes.new()
# Stats.
self.total_emails_generated = 0
self.total_emails_fetched = 0
end
# Public: Start threads and begin servicing connections.
def run
trap_signals
self.generated_log = Common::CsvLog.new("./log/stress/generated_emails.csv")
self.fetched_log = Common::CsvLog.new("./log/stress/fetched_emails.csv")
self.events_log = Common::CsvLog.new("./log/stress/events.csv")
start_stats_thread
start_connection_thread
start_new_mail_thread
start_process_sockets_thread
stop_time = self.length_of_test.minutes.from_now
while running?
if Time.now > stop_time
Log.info("Test finished! Shutting down.")
stop!
end
light_sleep 1.0
end
rescue => e
stop!
Log.exception(e)
raise e
ensure
stop!
connection_thread && connection_thread.terminate
sockets.map(&:close)
self.generated_log.stop!
self.fetched_log.stop!
self.events_log.stop!
Log.info("Generated #{total_emails_generated} emails.")
Log.info("Served #{total_emails_fetched} emails.")
end
private
def start_stats_thread
self.stats_thread = wrapped_thread do
stats_thread_runner
end
end
def start_connection_thread
self.connection_thread = wrapped_thread do
connection_thread_runner
end
end
def start_new_mail_thread
self.connection_thread = wrapped_thread do
new_mail_thread_runner
end
end
def stats_thread_runner
while running?
Log.info("Stats (connections = #{sockets.count}, emails_generated = #{total_emails_generated}, emails_fetched = #{total_emails_fetched})")
light_sleep 10
end
end
# Private: Accepts incoming connections.
def connection_thread_runner
Log.info("Waiting for connections on port 0.0.0.0:#{port}.")
server = TCPServer.new("0.0.0.0", port)
while running?
begin
socket = server.accept
new_sockets << socket
rescue IO::EAGAINWaitReadable
sleep 0.2
rescue => e
Log.exception(e)
end
end
rescue => e
Log.exception(e)
end
# Private: Sends and receives IMAP commands.
def start_process_sockets_thread
wrapped_thread do
process_sockets_runner
end
end
def process_sockets_runner
while running?
process_new_sockets
process_incoming_messages
send_exists_messages
sleep 0.1
end
end
def process_new_sockets
while running? && !new_sockets.empty?
socket = new_sockets.pop(true)
process_new_socket(socket)
end
end
def process_new_socket(socket)
options = {
:enable_chaos => self.enable_chaos
}
socket_state = ImapTestServer::SocketState.new(self, socket, options)
socket_state.handle_connect
# Add to our list of existing sockets.
self.sockets << socket
self.socket_states[socket.hash] = socket_state
rescue => e
Log.exception(e)
close_socket(socket)
end
def process_incoming_messages
# Which sockets need attention?
response = IO.select(sockets, [], [], 0)
return if response.nil?
# Attend to the sockets.
read_sockets, _, _ = response
read_sockets.each do |socket|
process_incoming_message(socket)
end
end
def process_incoming_message(socket)
command = socket.gets
if command.present?
socket_state = socket_states[socket.hash]
socket_state.handle_command(command)
else
close_socket(socket)
end
rescue ImapTestServer::SocketState::NormalDisconnect => e
close_socket(socket)
rescue ImapTestServer::SocketState::ChaosDisconnect => e
close_socket(socket)
rescue => e
Log.exception(e)
close_socket(socket)
end
def send_exists_messages
socket_states.values.each do |socket_state|
send_exists_message(socket_state)
end
end
def send_exists_message(socket_state)
socket_state.send_exists_messages
rescue => e
Log.exception(e)
close_socket(socket_state.socket)
end
def close_socket(socket)
sockets.delete(socket)
socket_states.delete(socket.hash)
socket.close()
rescue => e
Log.exception(e)
end
def new_mail_thread_runner
sleep_seconds = 1
while running?
if self.mailboxes.count > 0
n = (1.0 * sleep_seconds / 60) * emails_per_minute
generate_new_mail(n)
end
light_sleep sleep_seconds
end
end
def generate_new_mail(n)
# What's our chance of generating an email for an individual user?
prob_of_email = n / self.mailboxes.count
self.mailboxes.each do |mailbox|
if rand() < prob_of_email
self.total_emails_generated += 1
mailbox.add_fake_message do |message_id|
self.generated_log.log(Time.now, mailbox.username, message_id)
end
end
end
end
end
================================================
FILE: app/processes/imap_test_server/mailboxes.rb
================================================
class ImapTestServer::Mailboxes
attr_accessor :mailboxes, :mailboxes_mutex
def initialize(options = {})
self.mailboxes = {}
self.mailboxes_mutex = Mutex.new
end
def count
return self.mailboxes.count
end
def find(username)
self.mailboxes_mutex.synchronize do
self.mailboxes[username] ||= Mailbox.new(username)
end
end
def each(&block)
usernames = self.mailboxes_mutex.synchronize do
self.mailboxes.keys.dup
end
usernames.each do |username|
yield mailboxes[username]
end
end
private
class Mailbox
attr_accessor :username
attr_accessor :last_uid, :mails
MailStruct = Struct.new(:uid, :date, :message_id)
def initialize(username)
self.username = username
self.last_uid = rand(999999)
self.mails = []
end
def count
return mails.length
end
def add_fake_message
self.last_uid += 1
message_id = "message-#{username}-#{last_uid}-#{rand(999999)}@localhost"
self.mails << MailStruct.new(last_uid, Time.now, message_id)
yield(message_id)
end
def uid_search(from_uid, to_uid)
mails.select do |mail|
mail.uid >= from_uid && mail.uid <= to_uid
end.map(&:uid)
end
def date_search(since_date)
mails.select do |mail|
mail.date > since_date
end.map(&:uid)
end
def fetch(uid)
email = self.username
mail = mails.find do |mail|
mail.uid == uid
end
Mail.new do
from email
to email
date mail.date
message_id mail.message_id
subject "MySubject"
body "MyBody"
end
end
end
end
================================================
FILE: app/processes/imap_test_server/socket_state.rb
================================================
require 'date'
class ImapTestServer::SocketState
NormalDisconnect = Class.new(StandardError)
ChaosDisconnect = Class.new(StandardError)
attr_accessor :daemon, :mailbox, :socket
attr_accessor :enable_chaos
attr_accessor :username
attr_accessor :uid_validity
attr_accessor :last_count
attr_accessor :idling, :idle_tag
attr_accessor :new_email, :inbox
def initialize(daemon, socket, options)
self.daemon = daemon
self.socket = socket
self.enable_chaos = options[:enable_chaos]
self.uid_validity = rand(999)
self.idling = false
self.last_count = 0
end
# Public: Return true if we are idling.
def idling?
self.idling
end
# Public: Greet the new connection.
def handle_connect()
handle_command("TAG HELLO")
end
# Public: Handle the specified IMAP command, respond to the socket.
def handle_command(s)
tag, verb, args = parse_command(s)
method = verb_to_method(verb)
self.daemon.events_log.log(Time.now, username, method)
send(method, tag, args)
end
def send_exists_messages
if self.mailbox && self.last_count < self.mailbox.count
respond("*", "#{self.mailbox.count} EXISTS")
self.last_count = mailbox.count
end
end
private
def parse_command(s)
# Get the tag.
tag, s = /(.+?)\s+(.*)/.match(s).captures
# Special case for an IDLE done command.
return [nil, "DONE", []] if tag == "DONE"
# Parse the rest of the command.
verb, s = /(UID SEARCH|UID FETCH|\w+)\s*(.*)/.match(s).captures
args = s.split(/\s+/)
[tag, verb, args]
end
# Private: Given an IMAP verb, return a method. This is our chance
# to inject some chaos into the system. (Muhahaha.)
def verb_to_method(verb)
verb = verb.downcase.gsub(/\s/, "_")
choices = [[400, "imap_#{verb}".to_sym]]
if self.enable_chaos
choices += [
[3, "imap_#{verb}_chaos".to_sym],
[1, :imap_chaos_respond_no],
[1, :imap_chaos_respond_bad],
[1, :imap_chaos_gibberish_tagged],
[1, :imap_chaos_gibberish_untagged],
[1, :imap_chaos_soft_disconnect],
[1, :imap_chaos_hard_disconnect],
]
end
choose(choices)
end
# Private: Choose from a list of weighted choices, of the form
# [[weight1, choice1], [weight2, choice2], ...]. Weights are
# normalized, they do not need to add to 1.0.
#
# Returns the selected choice.
def choose(choices)
r = rand()
w = 0
total_weight = choices.map(&:first).inject(&:+)
choices.each do |weight, choice|
w += (1.0 * weight / total_weight)
return choice if r <= w
end
end
# Private: Write a response to the socket.
def respond(tag, s)
socket.write("#{tag} #{s}\r\n")
socket.flush
end
# CONNECT
def imap_hello(tag, args)
respond("*", "OK ImapTestServer ready.")
end
def imap_hello_chaos(tag, args)
respond("*", "ERROR Not ready.")
end
# LOGIN Command
# https://tools.ietf.org/html/rfc3501#section-6.2.3
def imap_login(tag, args)
self.username = args[0]
self.mailbox = self.daemon.mailboxes.find(username)
respond(tag, "OK Logged in.")
end
def imap_login_chaos(tag, args)
respond(tag, "NO")
end
# LIST Command
# https://tools.ietf.org/html/rfc3501#section-6.3.8
def imap_list(tag, args)
respond("*", %(LIST (\HasNoChildren) "." "INBOX"))
respond("*", %(LIST (\HasNoChildren) "." "FOLDER1"))
respond("*", %(LIST (\HasNoChildren) "." "FOLDER2"))
respond(tag, "OK LIST Completed")
end
def imap_list_chaos(tag, args)
respond("*", %(LIST (\HasNoChildren) "." "FOLDER1"))
respond("*", %(LIST (\HasNoChildren) "." "FOLDER2"))
respond(tag, "OK LIST Completed")
end
# EXAMINE Command
# https://tools.ietf.org/html/rfc3501#section-6.3.2
def imap_examine(tag, args)
respond("*", %(FLAGS (\Answered \Flagged \Deleted \Seen \Draft)))
respond("*", %(OK [PERMANENTFLAGS ()] Read-only mailbox.))
respond("*", %(#{mailbox.count} EXISTS))
respond("*", %(OK [UIDVALIDITY #{uid_validity}] UIDs valid))
respond(tag, %(OK [READ-ONLY] Select completed.))
end
def imap_examine_chaos(tag, args)
self.uid_validity = rand(999)
imap_examine(tag, args)
end
# STATUS Command
# https://tools.ietf.org/html/rfc3501#section-6.3.10
def imap_status(tag, args)
mailbox_name = args[0]
respond("*", %(STATUS #{mailbox_name} (UIDVALIDITY #{uid_validity})))
respond(tag, %(OK STATUS completed))
end
def imap_status_chaos(tag, args)
self.uid_validity = rand(999)
imap_status(tag, args)
end
# UID SEARCH Command
# https://tools.ietf.org/html/rfc3501#section-6.4.4
def imap_uid_search(tag, args)
if args.index("UID")
imap_uid_search_by_uid(tag, args)
elsif args.index("SINCE")
imap_uid_search_by_date(tag, args)
else
raise "Unhandled search: #{args}"
end
end
def imap_uid_search_by_uid(tag, args)
# Parse the search request.
from_uid, to_uid = args[args.index("UID") + 1].split(":")
from_uid = from_uid.to_i - self.uid_validity
to_uid = to_uid.to_i - self.uid_validity
# Get a list of uids, offset by uid validity.
uids = self.mailbox.uid_search(from_uid, to_uid).map do |uid|
uid + self.uid_validity
end
respond("*", %(SEARCH #{uids.join(' ')}))
respond(tag, %(OK SEARCH completed))
end
def imap_uid_search_by_date(tag, args)
# Parse the search request.
since_date = Time.parse(args[args.index("SINCE") + 1])
# Make sure we have a valid date.
if since_date.nil?
raise "Unhandled date: #{args}"
end
# Get a list of uids, offset by uid validity.
uids = mailbox.date_search(since_date).map do |uid|
uid + self.uid_validity
end
respond("*", %(SEARCH #{uids.join(' ')}))
respond(tag, %(OK SEARCH completed))
end
def imap_uid_search_chaos(tag, args)
imap_uid_search(tag, args)
end
# UID FETCH Command
# https://tools.ietf.org/html/rfc3501#section-6.4.5
# https://tools.ietf.org/html/rfc3501#section-7.4.2
def imap_uid_fetch(tag, args)
# Looks like this: 1103963 (INTERNALDATE RFC822.SIZE UID)
m = /(\d+)\s\((.*)\)/.match(args.join(' '))
uid = m[1].to_i
fields = m[2].split
mail = mailbox.fetch(uid - self.uid_validity)
values = fields.map do |field|
case field
when "UID"
[field, as_integer(uid)]
when "INTERNALDATE"
[field, as_date(mail.date)]
when "ENVELOPE"
[field, as_list(
as_date(mail.date),
as_string(mail.subject),
as_address_structure(mail.from),
as_address_structure(mail.from),
as_address_structure(mail.reply_to),
as_address_structure(mail.to),
as_string(nil),
as_string(nil),
as_string(nil),
as_string(mail.message_id)
)]
when "RFC822.SIZE"
[field, as_integer(mail.encoded.size)]
when "RFC822"
self.daemon.total_emails_fetched += 1
self.daemon.fetched_log.log(Time.now, username, mail.message_id)
[field, as_multiline_string(mail.encoded)]
else
raise "Unknown field: #{field}"
end
end
values += ["UID", as_integer(uid)] unless fields.include?("RFC822")
respond("*", "#{uid} FETCH #{as_list(values)}")
respond(tag, "OK FETCH complete")
end
def as_list(*values)
s = values.map do |value|
if value.instance_of?(Array)
value.join(" ")
else
value
end
end.join(' ')
return "(#{s})"
end
def as_address_structure(addresses)
# https://tools.ietf.org/html/rfc3501#section-7.4.2
return as_string(nil) if addresses.blank?
values = addresses.map do |address|
if !address.instance_of?(Mail::Address)
address = Mail::Address.new(address)
end
as_list(as_string(address.display_name),
as_string(nil),
as_string(address.local),
as_string(address.domain))
end
as_list(values)
end
def as_date(date)
as_string(date.strftime("%a, %b %e %Y %H:%M:%S %z (%Z)"))
end
def as_integer(n)
n.to_s
end
def as_string(s)
s.nil? ? "NIL" : "\"#{s}\""
end
def as_multiline_string(s)
"{#{s.length}}\r\n#{s}"
end
def imap_uid_fetch_chaos(tag, args)
imap_uid_fetch(tag, args)
end
# IDLE Command
# http://tools.ietf.org/html/rfc2177
def imap_idle(tag, args)
self.idling = true
self.idle_tag = tag
respond("+", "idling")
end
def imap_idle_chaos(tag, args)
imap_idle(tag, args)
end
def imap_done(tag, args)
respond(idle_tag, "OK IDLE terminated")
end
def imap_done_chaos(tag, args)
imap_done(tag, args)
end
# LOGOUT Command
# https://tools.ietf.org/html/rfc3501#section-6.1.3
def imap_logout(tag, args)
respond("*", "BYE ImapTestServer logging out")
respond(tag, "OK LOGOUT completed")
raise NormalDisconnect.new()
end
def imap_logout_chaos(tag, args)
imap_logout(tag, args)
end
# GENERAL CHAOS
def imap_chaos_respond_no(tag, args)
respond(tag, "BAD")
end
def imap_chaos_respond_bad(tag, args)
respond(tag, "BAD")
end
def imap_chaos_gibberish_tagged(tag, args)
respond(tag, "ZZZ")
end
def imap_chaos_gibberish_untagged(tag, args)
respond("*", "ZZZ")
end
def imap_chaos_soft_disconnect(tag, args)
respond("*", "BYE")
end
def imap_chaos_hard_disconnect(tag, args)
raise ChaosDisconnect.new("Disconnect!")
end
end
================================================
FILE: app/processes/imap_test_server.rb
================================================
class ImapTestServer
VERSION="1.0.0"
end
require 'imap_test_server/daemon'
================================================
FILE: app/views/api/v1/connections/index.json.rb
================================================
fields = [
:imap_provider_code,
:users_count
]
@connections.map do |user|
values = fields.map do |field|
[field, user.send(field)]
end
array_to_hash(values)
end.to_json
================================================
FILE: app/views/api/v1/connections/show.json.rb
================================================
fields = [
:imap_provider_code,
:users_count
]
values = fields.map do |field|
[field, @connection.send(field)]
end
array_to_hash(values).to_json
================================================
FILE: app/views/api/v1/users/index.json.rb
================================================
fields = [:tag, :email]
@users.map do |user|
values = fields.map do |field|
[field, user.send(field)]
end
array_to_hash(values)
end.to_json
================================================
FILE: app/views/api/v1/users/show.json.rb
================================================
{
:tag => @user.tag,
:email => @user.email,
:connect_url => new_users_connect_url(@user.signed_request_params),
:disconnect_url => new_users_disconnect_url(@user.signed_request_params),
:connected_at => @user.connected_at
}.to_json
================================================
FILE: app/views/layouts/application.html.erb
================================================
<!DOCTYPE html>
<html>
<head>
<title>SuperIMAP</title>
<%= stylesheet_link_tag 'application', media: 'all', 'data-turbolinks-track' => true %>
<%= javascript_include_tag 'application', 'data-turbolinks-track' => true %>
<%= csrf_meta_tags %>
</head>
<body>
<p class="notice"><%= notice %></p>
<p class="alert"><%= alert %></p>
<%= yield %>
</body>
</html>
================================================
FILE: app/views/layouts/blank.html.erb
================================================
<%= yield %>
================================================
FILE: app/views/tracer_mailer/tracer_email.html.erb
================================================
TRACER: <%= @uid %>
================================================
FILE: app.json
================================================
{
"name": "SuperIMAP",
"description": "Monitor inboxes for incoming email, at scale.",
"repository": "https://github.com/rustyio/super-imap",
"keywords": ["IMAP", "mail", "webhook", "client", "ruby"],
"success_url": "/admin",
"scripts": {
"postdeploy": "bundle exec rake db:seed"
},
"addons": [
"heroku-postgresql"
],
"env": {
"ENCRYPTION_KEY": {
"description": "Used to encrypt information in the database.",
"generator": "secret"
},
"SECRET_KEY_BASE": {
"description": "A secret key for verifying the integrity of signed cookies.",
"generator": "secret"
}
}
}
================================================
FILE: bin/bundle
================================================
#!/usr/bin/env ruby
ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
load Gem.bin_path('bundler', 'bundle')
================================================
FILE: bin/delayed_job
================================================
#!/usr/bin/env ruby
require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment'))
require 'delayed/command'
Delayed::Command.new(ARGV).daemonize
================================================
FILE: bin/rails
================================================
#!/usr/bin/env ruby
begin
load File.expand_path("../spring", __FILE__)
rescue LoadError
end
APP_PATH = File.expand_path('../../config/application', __FILE__)
require_relative '../config/boot'
require 'rails/commands'
================================================
FILE: bin/rake
================================================
#!/usr/bin/env ruby
begin
load File.expand_path("../spring", __FILE__)
rescue LoadError
end
require_relative '../config/boot'
require 'rake'
Rake.application.run
================================================
FILE: bin/spring
================================================
#!/usr/bin/env ruby
# This file loads spring without using Bundler, in order to be fast
# It gets overwritten when you run the `spring binstub` command
unless defined?(Spring)
require "rubygems"
require "bundler"
if match = Bundler.default_lockfile.read.match(/^GEM$.*?^ spring \((.*?)\)$.*?^$/m)
ENV["GEM_PATH"] = ([Bundler.bundle_path.to_s] + Gem.path).join(File::PATH_SEPARATOR)
ENV["GEM_HOME"] = ""
Gem.paths = ENV
gem "spring", match[1]
require "spring/binstub"
end
end
================================================
FILE: config/application.rb
================================================
require File.expand_path('../boot', __FILE__)
require 'rails/all'
# Require the gems listed in Gemfile, including any gems
# you've limited to :test, :development, or :production.
Bundler.require(*Rails.groups)
module SuperIMAP
class Application < Rails::Application
# Settings in config/environments/* take precedence over those specified here.
# Application configuration should go into files in config/initializers
# -- all .rb files in that directory are automatically loaded.
# Set Time.zone default to the specified zone and make Active Record auto-convert to this zone.
# Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC.
# config.time_zone = 'Central Time (US & Canada)'
# The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded.
# config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
# config.i18n.default_locale = :de
config.autoload_paths += Dir["#{config.root}/app/interactors"]
config.autoload_paths += Dir["#{config.root}/app/processes"]
encryption_key = ENV['ENCRYPTION_KEY']
if encryption_key.present?
config.encryption_cipher = Gibberish::AES.new(encryption_key)
else
config.encryption_cipher = nil
end
config.log_level = String(ENV['LOG_LEVEL'] || "info").upcase
end
end
================================================
FILE: config/boot.rb
================================================
# Set up gems listed in the Gemfile.
ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
================================================
FILE: config/database.yml.example
================================================
default: &default
adapter: postgresql
host: localhost
pool: 5
timeout: 5000
user: username
password: password
development:
<<: *default
database: super_imap_development
stress:
<<: *default
database: super_imap_stress
test:
<<: *default
database: super_imap_test
================================================
FILE: config/environment.rb
================================================
# Load the Rails application.
require File.expand_path('../application', __FILE__)
# Initialize the Rails application.
Rails.application.initialize!
================================================
FILE: config/environments/development.rb
================================================
Rails.application.configure do
# Settings specified here will take precedence over those in config/application.rb.
# In the development environment your application's code is reloaded on
# every request. This slows down response time but is perfect for development
# since you don't have to restart the web server when you make code changes.
config.cache_classes = false
# Do not eager load code on boot.
config.eager_load = false
# Show full error reports and disable caching.
config.consider_all_requests_local = true
config.action_controller.perform_caching = false
# Don't care if the mailer can't send.
config.action_mailer.raise_delivery_errors = false
# Print deprecation notices to the Rails logger.
config.active_support.deprecation = :log
# Raise an error on page load if there are pending migrations.
config.active_record.migration_error = :page_load
# Debug mode disables concatenation and preprocessing of assets.
# This option may cause significant delays in view rendering with a large
# number of complex assets.
config.assets.debug = true
# Adds additional error checking when serving assets at runtime.
# Checks for improperly declared sprockets dependencies.
# Raises helpful error messages.
config.assets.raise_runtime_errors = true
# Raises error for missing translations
# config.action_view.raise_on_missing_translations = true
end
================================================
FILE: config/environments/performance.rb
================================================
Rails.application.configure do
# Settings specified here will take precedence over those in config/application.rb.
# Code is not reloaded between requests.
config.cache_classes = true
# Eager load code on boot. This eager loads most of Rails and
# your application in memory, allowing both threaded web servers
# and those relying on copy on write to perform better.
# Rake tasks automatically ignore this option for performance.
config.eager_load = true
# Full error reports are disabled and caching is turned on.
config.consider_all_requests_local = false
config.action_controller.perform_caching = true
# Enable Rack::Cache to put a simple HTTP cache in front of your application
# Add `rack-cache` to your Gemfile before enabling this.
# For large-scale production use, consider using a caching reverse proxy like nginx, varnish or squid.
# config.action_dispatch.rack_cache = true
# Disable Rails's static asset server (Apache or nginx will already do this).
config.serve_static_assets = false
# Compress JavaScripts and CSS.
config.assets.js_compressor = :uglifier
# config.assets.css_compressor = :sass
# Do not fallback to assets pipeline if a precompiled asset is missed.
config.assets.compile = false
# Generate digests for assets URLs.
config.assets.digest = true
# `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb
# Specifies the header that your server uses for sending files.
# config.action_dispatch.x_sendfile_header = "X-Sendfile" # for apache
# config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for nginx
# Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies.
# config.force_ssl = true
# Set to :debug to see everything in the log.
config.log_level = :info
# Prepend all log lines with the following tags.
# config.log_tags = [ :subdomain, :uuid ]
# Use a different logger for distributed setups.
# config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new)
# Use a different cache store in production.
# config.cache_store = :mem_cache_store
# Enable serving of images, stylesheets, and JavaScripts from an asset server.
# config.action_controller.asset_host = "http://assets.example.com"
# Ignore bad email addresses and do not raise email delivery errors.
# Set this to true and configure the email server for immediate delivery to raise delivery errors.
# config.action_mailer.raise_delivery_errors = false
# Enable locale fallbacks for I18n (makes lookups for any locale fall back to
# the I18n.default_locale when a translation cannot be found).
config.i18n.fallbacks = true
# Send deprecation notices to registered listeners.
config.active_support.deprecation = :notify
# Disable automatic flushing of the log to improve performance.
# config.autoflush_log = false
# Use default logging formatter so that PID and timestamp are not suppressed.
config.log_formatter = ::Logger::Formatter.new
# Do not dump schema after migrations.
config.active_record.dump_schema_after_migration = false
end
================================================
FILE: config/environments/production.rb
================================================
Rails.application.configure do
# Settings specified here will take precedence over those in config/application.rb.
# Code is not reloaded between requests.
config.cache_classes = true
# Eager load code on boot. This eager loads most of Rails and
# your application in memory, allowing both threaded web servers
# and those relying on copy on write to perform better.
# Rake tasks automatically ignore this option for performance.
config.eager_load = true
# Full error reports are disabled and caching is turned on.
config.consider_all_requests_local = false
config.action_controller.perform_caching = true
# Enable Rack::Cache to put a simple HTTP cache in front of your application
# Add `rack-cache` to your Gemfile before enabling this.
# For large-scale production use, consider using a caching reverse proxy like nginx, varnish or squid.
# config.action_dispatch.rack_cache = true
# Disable Rails's static asset server (Apache or nginx will already do this).
config.serve_static_assets = false
# Compress JavaScripts and CSS.
config.assets.js_compressor = :uglifier
# config.assets.css_compressor = :sass
# Do not fallback to assets pipeline if a precompiled asset is missed.
config.assets.compile = false
# Generate digests for assets URLs.
config.assets.digest = true
# `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb
# Specifies the header that your server uses for sending files.
# config.action_dispatch.x_sendfile_header = "X-Sendfile" # for apache
# config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for nginx
# Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies.
# config.force_ssl = true
# Set to :debug to see everything in the log.
config.log_level = :info
# Prepend all log lines with the following tags.
# config.log_tags = [ :subdomain, :uuid ]
# Use a different logger for distributed setups.
# config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new)
# Use a different cache store in production.
# config.cache_store = :mem_cache_store
# Enable serving of images, stylesheets, and JavaScripts from an asset server.
# config.action_controller.asset_host = "http://assets.example.com"
# Ignore bad email addresses and do not raise email delivery errors.
# Set this to true and configure the email server for immediate delivery to raise delivery errors.
# config.action_mailer.raise_delivery_errors = false
# Enable locale fallbacks for I18n (makes lookups for any locale fall back to
# the I18n.default_locale when a translation cannot be found).
config.i18n.fallbacks = true
# Send deprecation notices to registered listeners.
config.active_support.deprecation = :notify
# Disable automatic flushing of the log to improve performance.
# config.autoflush_log = false
# Use default logging formatter so that PID and timestamp are not suppressed.
config.log_formatter = ::Logger::Formatter.new
# Do not dump schema after migrations.
config.active_record.dump_schema_after_migration = false
if config.encryption_cipher.nil?
raise "Must set ENCRYPTION_KEY environment variable."
end
end
================================================
FILE: config/environments/stress.rb
================================================
Rails.application.configure do
# Settings specified here will take precedence over those in config/application.rb.
# In the development environment your application's code is reloaded on
# every request. This slows down response time but is perfect for development
# since you don't have to restart the web server when you make code changes.
config.cache_classes = false
# Do not eager load code on boot.
config.eager_load = false
# Show full error reports and disable caching.
config.consider_all_requests_local = true
config.action_controller.perform_caching = false
# Don't care if the mailer can't send.
config.action_mailer.raise_delivery_errors = false
# Print deprecation notices to the Rails logger.
config.active_support.deprecation = :log
# Raise an error on page load if there are pending migrations.
config.active_record.migration_error = :page_load
# Debug mode disables concatenation and preprocessing of assets.
# This option may cause significant delays in view rendering with a large
# number of complex assets.
config.assets.debug = true
# Adds additional error checking when serving assets at runtime.
# Checks for improperly declared sprockets dependencies.
# Raises helpful error messages.
config.assets.raise_runtime_errors = true
# Raises error for missing translations
# config.action_view.raise_on_missing_translations = true
end
================================================
FILE: config/environments/test.rb
================================================
Rails.application.configure do
# Settings specified here will take precedence over those in config/application.rb.
# The test environment is used exclusively to run your application's
# test suite. You never need to work with it otherwise. Remember that
# your test database is "scratch space" for the test suite and is wiped
# and recreated between test runs. Don't rely on the data there!
config.cache_classes = true
# Do not eager load code on boot. This avoids loading your whole application
# just for the purpose of running a single test. If you are using a tool that
# preloads Rails for running tests, you may have to set it to true.
config.eager_load = false
# Configure static asset server for tests with Cache-Control for performance.
config.serve_static_assets = true
config.static_cache_control = 'public, max-age=3600'
# Show full error reports and disable caching.
config.consider_all_requests_local = true
config.action_controller.perform_caching = false
# Raise exceptions instead of rendering exception templates.
config.action_dispatch.show_exceptions = false
# Disable request forgery protection in test environment.
config.action_controller.allow_forgery_protection = false
# Tell Action Mailer not to deliver emails to the real world.
# The :test delivery method accumulates sent emails in the
# ActionMailer::Base.deliveries array.
config.action_mailer.delivery_method = :test
# Print deprecation notices to the stderr.
config.active_support.deprecation = :stderr
# Raises error for missing translations
# config.action_view.raise_on_missing_translations = true
end
================================================
FILE: config/initializers/active_admin.rb
================================================
# coding: utf-8
ActiveAdmin.setup do |config|
# == Site Title
#
# Set the title that is displayed on the main layout
# for each of the active admin pages.
#
config.site_title = "SuperIMAP"
# Set the link url for the title. For example, to take
# users to your main site. Defaults to no link.
#
# config.site_title_link = "/"
# Set an optional image to be displayed for the header
# instead of a string (overrides :site_title)
#
# Note: Aim for an image that's 21px high so it fits in the header.
#
# config.site_title_image = "logo.png"
# == Default Namespace
#
# Set the default namespace each administration resource
# will be added to.
#
# eg:
# config.default_namespace = :hello_world
#
# This will create resources in the HelloWorld module and
# will namespace routes to /hello_world/*
#
# To set no namespace by default, use:
# config.default_namespace = false
#
# Default:
# config.default_namespace = :admin
#
# You can customize the settings for each namespace by using
# a namespace block. For example, to change the site title
# within a namespace:
#
# config.namespace :admin do |admin|
# admin.site_title = "Custom Admin Title"
# end
#
# This will ONLY change the title for the admin section. Other
# namespaces will continue to use the main "site_title" configuration.
# == User Authentication
#
# Active Admin will automatically call an authentication
# method in a before filter of all controller actions to
# ensure that there is a currently logged in admin user.
#
# This setting changes the method which Active Admin calls
# within the application controller.
config.authentication_method = :authenticate_admin_user!
# == User Authorization
#
# Active Admin will automatically call an authorization
# method in a before filter of all controller actions to
# ensure that there is a user with proper rights. You can use
# CanCanAdapter or make your own. Please refer to documentation.
# config.authorization_adapter = ActiveAdmin::CanCanAdapter
# You can customize your CanCan Ability class name here.
# config.cancan_ability_class = "Ability"
# You can specify a method to be called on unauthorized access.
# This is necessary in order to prevent a redirect loop which happens
# because, by default, user gets redirected to Dashboard. If user
# doesn't have access to Dashboard, he'll end up in a redirect loop.
# Method provided here should be defined in application_controller.rb.
# config.on_unauthorized_access = :access_denied
# == Current User
#
# Active Admin will associate actions with the current
# user performing them.
#
# This setting changes the method which Active Admin calls
# (within the application controller) to return the currently logged in user.
config.current_user_method = :current_admin_user
# == Logging Out
#
# Active Admin displays a logout link on each screen. These
# settings configure the location and method used for the link.
#
# This setting changes the path where the link points to. If it's
# a string, the strings is used as the path. If it's a Symbol, we
# will call the method to return the path.
#
# Default:
config.logout_link_path = :destroy_admin_user_session_path
# This setting changes the http method used when rendering the
# link. For example :get, :delete, :put, etc..
#
# Default:
# config.logout_link_method = :get
# == Root
#
# Set the action to call for the root path. You can set different
# roots for each namespace.
#
# Default:
config.root_to = 'partners#index'
# == Admin Comments
#
# This allows your users to comment on any resource registered with Active Admin.
#
# You can completely disable comments:
config.allow_comments = false
#
# You can disable the menu item for the comments index page:
# config.show_comments_in_menu = false
#
# You can change the name under which comments are registered:
# config.comments_registration_name = 'AdminComment'
# == Batch Actions
#
# Enable and disable Batch Actions
#
config.batch_actions = false
# == Controller Filters
#
# You can add before, after and around filters to all of your
# Active Admin resources and pages from here.
#
# config.before_filter :do_something_awesome
# == Setting a Favicon
#
# config.favicon = '/assets/favicon.ico'
# == Removing Breadcrumbs
#
# Breadcrumbs are enabled by default. You can customize them for individual
# resources or you can disable them globally from here.
#
# config.breadcrumb = false
# == Register Stylesheets & Javascripts
#
# We recommend using the built in Active Admin layout and loading
# up your own stylesheets / javascripts to customize the look
# and feel.
#
# To load a stylesheet:
# config.register_stylesheet 'my_stylesheet.css'
#
# You can provide an options hash for more control, which is passed along to stylesheet_link_tag():
# config.register_stylesheet 'my_print_stylesheet.css', :media => :print
#
# To load a javascript file:
# config.register_javascript 'my_javascript.js'
# == CSV options
#
# Set the CSV builder separator
# config.csv_options = { :col_sep => ';' }
#
# Force the use of quotes
# config.csv_options = { :force_quotes => true }
# == Menu System
#
# You can add a navigation menu to be used in your application, or configure a provided menu
#
# To change the default utility navigation to show a link to your website & a logout btn
#
# config.namespace :admin do |admin|
# admin.build_menu :utility_navigation do |menu|
# menu.add label: "My Great Website", url: "http://www.mygreatwebsite.com", html_options: { target: :blank }
# admin.add_logout_button_to_menu menu
# end
# end
#
# If you wanted to add a static menu item to the default menu provided:
#
# config.namespace :admin do |admin|
# admin.build_menu :default do |menu|
# menu.add label: "My Great Website", url: "http://www.mygreatwebsite.com", html_options: { target: :blank }
# end
# end
# == Download Links
#
# You can disable download links on resource listing pages,
# or customize the formats shown per namespace/globally
#
# To disable/customize for the :admin namespace:
#
config.namespace :admin do |admin|
# Disable the links entirely
admin.download_links = false
# # Only show XML & PDF options
# admin.download_links = [:xml, :pdf]
# # Enable/disable the links based on block
# # (for example, with cancan)
# admin.download_links = proc { can?(:view_download_links) }
end
# == Pagination
#
# Pagination is enabled by default for all resources.
# You can control the default per page count for all resources here.
#
# config.default_per_page = 30
# == Filters
#
# By default the index screen includes a “Filters” sidebar on the right
# hand side with a filter for each attribute of the registered model.
# You can enable or disable them for all resources here.
#
# config.filters = true
end
================================================
FILE: config/initializers/airbrake.rb
================================================
if defined?(Airbrake) && ENV['AIRBRAKE_KEY']
Airbrake.configure do |config|
config.api_key = ENV['AIRBRAKE_KEY']
end
end
================================================
FILE: config/initializers/assets.rb
================================================
# Be sure to restart your server when you modify this file.
# Version of your assets, change this if you want to expire all your assets.
Rails.application.config.assets.version = '1.0'
# Precompile additional assets.
# application.js, application.css, and all non-JS/CSS in app/assets folder are already added.
# Rails.application.config.assets.precompile += %w( search.js )
================================================
FILE: config/initializers/backtrace_silencers.rb
================================================
# Be sure to restart your server when you modify this file.
# You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces.
# Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ }
# You can also remove all the silencers if you're trying to debug a problem that might stem from framework code.
# Rails.backtrace_cleaner.remove_silencers!
================================================
FILE: config/initializers/cookies_serializer.rb
================================================
# Be sure to restart your server when you modify this file.
Rails.application.config.action_dispatch.cookies_serializer = :json
================================================
FILE: config/initializers/delayed_job.rb
================================================
# config/initializers/delayed_job.rb
# Update settings.
Delayed::Worker.destroy_failed_jobs = false
Delayed::Worker.read_ahead = 10
Delayed::Worker.default_priority = 10
Delayed::Worker.max_run_time = 20.minutes
Delayed::Worker.default_queue_name = "worker"
Delayed::Worker.max_attempts = 6
if !Delayed::Worker.instance_methods.include?(:handle_failed_job)
raise "Could not update Delayed::Worker!"
end
# Patch to log errors.
class Delayed::Worker
alias_method :original_handle_failed_job, :handle_failed_job
def handle_failed_job(job, error)
begin
# Send an alert.
Log.exception(error)
rescue => e
# Shouldn't get here, but if it does, at least log it.
Log.exception(e)
end
# Handle as usual.
original_handle_failed_job(job, error)
end
end
================================================
FILE: config/initializers/devise.rb
================================================
# Use this hook to configure devise mailer, warden hooks and so forth.
# Many of these configuration options can be set straight in your model.
Devise.setup do |config|
# The secret key used by Devise. Devise uses this key to generate
# random tokens. Changing this key will render invalid all existing
# confirmation, reset password and unlock tokens in the database.
config.secret_key = ENV['SECRET_KEY_BASE'] || Rails.env
# ==> Mailer Configuration
# Configure the e-mail address which will be shown in Devise::Mailer,
# note that it will be overwritten if you use your own mailer class
# with default "from" parameter.
# config.mailer_sender = 'please-change-me-at-config-initializers-devise@example.com'
# Configure the class responsible to send e-mails.
# config.mailer = 'Devise::Mailer'
# ==> ORM configuration
# Load and configure the ORM. Supports :active_record (default) and
# :mongoid (bson_ext recommended) by default. Other ORMs may be
# available as additional gems.
require 'devise/orm/active_record'
# ==> Configuration for any authentication mechanism
# Configure which keys are used when authenticating a user. The default is
# just :email. You can configure it to use [:username, :subdomain], so for
# authenticating a user, both parameters are required. Remember that those
# parameters are used only when authenticating and not when retrieving from
# session. If you need permissions, you should implement that in a before filter.
# You can also supply a hash where the value is a boolean determining whether
# or not authentication should be aborted when the value is not present.
# config.authentication_keys = [ :email ]
# Configure parameters from the request object used for authentication. Each entry
# given should be a request method and it will automatically be passed to the
# find_for_authentication method and considered in your model lookup. For instance,
# if you set :request_keys to [:subdomain], :subdomain will be used on authentication.
# The same considerations mentioned for authentication_keys also apply to request_keys.
# config.request_keys = []
# Configure which authentication keys should be case-insensitive.
# These keys will be downcased upon creating or modifying a user and when used
# to authenticate or find a user. Default is :email.
config.case_insensitive_keys = [ :email ]
# Configure which authentication keys should have whitespace stripped.
# These keys will have whitespace before and after removed upon creating or
# modifying a user and when used to authenticate or find a user. Default is :email.
config.strip_whitespace_keys = [ :email ]
# Tell if authentication through request.params is enabled. True by default.
# It can be set to an array that will enable params authentication only for the
# given strategies, for example, `config.params_authenticatable = [:database]` will
# enable it only for database (email + password) authentication.
# config.params_authenticatable = true
# Tell if authentication through HTTP Auth is enabled. False by default.
# It can be set to an array that will enable http authentication only for the
# given strategies, for example, `config.http_authenticatable = [:database]` will
# enable it only for database authentication. The supported strategies are:
# :database = Support basic authentication with authentication key + password
# config.http_authenticatable = false
# If 401 status code should be returned for AJAX requests. True by default.
# config.http_authenticatable_on_xhr = true
# The realm used in Http Basic Authentication. 'Application' by default.
# config.http_authentication_realm = 'Application'
# It will change confirmation, password recovery and other workflows
# to behave the same regardless if the e-mail provided was right or wrong.
# Does not affect registerable.
# config.paranoid = true
# By default Devise will store the user in session. You can skip storage for
# particular strategies by setting this option.
# Notice that if you are skipping storage for all authentication paths, you
# may want to disable generating routes to Devise's sessions controller by
# passing skip: :sessions to `devise_for` in your config/routes.rb
config.skip_session_storage = [:http_auth]
# By default, Devise cleans up the CSRF token on authentication to
# avoid CSRF token fixation attacks. This means that, when using AJAX
# requests for sign in and sign up, you need to get a new CSRF token
# from the server. You can disable this option at your own risk.
# config.clean_up_csrf_token_on_authentication = true
# ==> Configuration for :database_authenticatable
# For bcrypt, this is the cost for hashing the password and defaults to 10. If
# using other encryptors, it sets how many times you want the password re-encrypted.
#
# Limiting the stretches to just one in testing will increase the performance of
# your test suite dramatically. However, it is STRONGLY RECOMMENDED to not use
# a value less than 10 in other environments. Note that, for bcrypt (the default
# encryptor), the cost increases exponentially with the number of stretches (e.g.
# a value of 20 is already extremely slow: approx. 60 seconds for 1 calculation).
config.stretches = Rails.env.test? ? 1 : 10
# Setup a pepper to generate the encrypted password.
# config.pepper = 'd8e8f6d62a78b830c1d1521b0f06d0dc4b4b6795fb27ca47daf1c7e7e03d47ca02abcb2587c823fa7f2fcf272e01cd702cd0fcb5ae33a44fa20c00b5912e9c15'
# ==> Configuration for :confirmable
# A period that the user is allowed to access the website even without
# confirming their account. For instance, if set to 2.days, the user will be
# able to access the website for two days without confirming their account,
# access will be blocked just in the third day. Default is 0.days, meaning
# the user cannot access the website without confirming their account.
# config.allow_unconfirmed_access_for = 2.days
# A period that the user is allowed to confirm their account before their
# token becomes invalid. For example, if set to 3.days, the user can confirm
# their account within 3 days after the mail was sent, but on the fourth day
# their account can't be confirmed with the token any more.
# Default is nil, meaning there is no restriction on how long a user can take
# before confirming their account.
# config.confirm_within = 3.days
# If true, requires any email changes to be confirmed (exactly the same way as
# initial account confirmation) to be applied. Requires additional unconfirmed_email
# db field (see migrations). Until confirmed, new email is stored in
# unconfirmed_email column, and copied to email column on successful confirmation.
config.reconfirmable = true
# Defines which key will be used when confirming an account
# config.confirmation_keys = [ :email ]
# ==> Configuration for :rememberable
# The time the user will be remembered without asking for credentials again.
# config.remember_for = 2.weeks
# Invalidates all the remember me tokens when the user signs out.
config.expire_all_remember_me_on_sign_out = true
# If true, extends the user's remember period when remembered via cookie.
# config.extend_remember_period = false
# Options to be passed to the created cookie. For instance, you can set
# secure: true in order to force SSL only cookies.
# config.rememberable_options = {}
# ==> Configuration for :validatable
# Range for password length.
config.password_length = 8..128
# Email regex used to validate email formats. It simply asserts that
# one (and only one) @ exists in the given string. This is mainly
# to give user feedback and not to assert the e-mail validity.
# config.email_regexp = /\A[^@]+@[^@]+\z/
# ==> Configuration for :timeoutable
# The time you want to timeout the user session without activity. After this
# time the user will be asked for credentials again. Default is 30 minutes.
config.timeout_in = 30.minutes
# If true, expires auth token on session timeout.
config.expire_auth_token_on_timeout = true
# ==> Configuration for :lockable
# Defines which strategy will be used to lock an account.
# :failed_attempts = Locks an account after a number of failed attempts to sign in.
# :none = No lock strategy. You should handle locking by yourself.
# config.lock_strategy = :failed_attempts
# Defines which key will be used when locking and unlocking an account
# config.unlock_keys = [ :email ]
# Defines which strategy will be used to unlock an account.
# :email = Sends an unlock link to the user email
# :time = Re-enables login after a certain amount of time (see :unlock_in below)
# :both = Enables both strategies
# :none = No unlock strategy. You should handle unlocking by yourself.
config.unlock_strategy = :time
# Number of authentication tries before locking an account if lock_strategy
# is failed attempts.
config.maximum_attempts = 3
# Time interval to unlock the account if :time is enabled as unlock_strategy.
config.unlock_in = 1.hour
# Warn on the last attempt before the account is locked.
config.last_attempt_warning = true
# ==> Configuration for :recoverable
#
# Defines which key will be used when recovering the password for an account
# config.reset_password_keys = [ :email ]
# Time interval you can reset your password with a reset password key.
# Don't put a too small interval or your users won't have the time to
# change their passwords.
config.reset_password_within = 6.hours
# ==> Configuration for :encryptable
# Allow you to use another encryption algorithm besides bcrypt (default). You can use
# :sha1, :sha512 or encryptors from others authentication tools as :clearance_sha1,
# :authlogic_sha512 (then you should set stretches above to 20 for default behavior)
# and :restful_authentication_sha1 (then you should set stretches to 10, and copy
# REST_AUTH_SITE_KEY to pepper).
#
# Require the `devise-encryptable` gem when using anything other than bcrypt
# config.encryptor = :sha512
# ==> Scopes configuration
# Turn scoped views on. Before rendering "sessions/new", it will first check for
# "users/sessions/new". It's turned off by default because it's slower if you
# are using only default views.
# config.scoped_views = false
# Configure the default scope given to Warden. By default it's the first
# devise role declared in your routes (usually :user).
# config.default_scope = :user
# Set this configuration to false if you want /users/sign_out to sign out
# only the current scope. By default, Devise signs out all scopes.
# config.sign_out_all_scopes = true
# ==> Navigation configuration
# Lists the formats that should be treated as navigational. Formats like
# :html, should redirect to the sign in page when the user does not have
# access, but formats like :xml or :json, should return 401.
#
# If you have any extra navigational formats, like :iphone or :mobile, you
# should add them to the navigational formats lists.
#
# The "*/*" below is required to match Internet Explorer requests.
# config.navigational_formats = ['*/*', :html]
# The default HTTP method used to sign out a resource. Default is :delete.
config.sign_out_via = :delete
# ==> OmniAuth
# Add a new OmniAuth provider. Check the wiki for more information on setting
# up on your models and hooks.
# config.omniauth :github, 'APP_ID', 'APP_SECRET', scope: 'user,public_repo'
# ==> Warden configuration
# If you want to use other strategies, that are not supported by Devise, or
# change the failure app, you can configure them inside the config.warden block.
#
# config.warden do |manager|
# manager.intercept_401 = false
# manager.default_strategies(scope: :user).unshift :some_external_strategy
# end
# ==> Mountable engine configurations
# When using Devise inside an engine, let's call it `MyEngine`, and this engine
# is mountable, there are some extra configurations to be taken into account.
# The following options are available, assuming the engine is mounted as:
#
# mount MyEngine, at: '/my_engine'
#
# The router that invoked `devise_for`, in the example above, would be:
# config.router_name = :my_engine
#
# When using omniauth, Devise cannot automatically set Omniauth path,
# so you need to do it manually. For the users scope, it would be:
# config.omniauth_path_prefix = '/my_engine/users/auth'
end
================================================
FILE: config/initializers/filter_parameter_logging.rb
================================================
# Be sure to restart your server when you modify this file.
# Configure sensitive parameters which will be filtered from the log file.
Rails.application.config.filter_parameters += [:password]
================================================
FILE: config/initializers/inflections.rb
================================================
# Be sure to restart your server when you modify this file.
# Add new inflection rules using the following format. Inflections
# are locale specific, and you may define rules for as many different
# locales as you wish. All of these examples are active by default:
# ActiveSupport::Inflector.inflections(:en) do |inflect|
# inflect.plural /^(ox)$/i, '\1en'
# inflect.singular /^(ox)en/i, '\1'
# inflect.irregular 'person', 'people'
# inflect.uncountable %w( fish sheep )
# end
# These inflection rules are supported but not enabled by default:
# ActiveSupport::Inflector.inflections(:en) do |inflect|
# inflect.acronym 'RESTful'
# end
================================================
FILE: config/initializers/log.rb
================================================
class MyLogger
def initialize
end
def exception(exception)
msg = "#{exception.class} (#{exception.message}):\n " +
clean_backtrace(exception).join("\n ")
self.error(msg)
# Trigger an exception email...
if defined?(Airbrake)
Airbrake.notify(exception)
end
rescue => e
print e.to_s
end
def clean_backtrace(exception)
if backtrace = exception.backtrace
if defined?(RAILS_ROOT)
return backtrace.map { |line| line.sub RAILS_ROOT, '' }
else
return backtrace
end
else
return []
end
end
def librato(mode, key, value)
dyno_name = ENV['DYNO']
if dyno_name
info("source=#{dyno_name} #{mode}\##{key}=#{value}")
else
info("#{mode}\##{key}=#{value}")
end
end
private
def method_missing(method, *args, &block)
Rails.logger.send(method, *args, &block)
end
end
Log = MyLogger.new()
================================================
FILE: config/initializers/mime_types.rb
================================================
# Be sure to restart your server when you modify this file.
# Add new mime types for use in respond_to blocks:
# Mime::Type.register "text/richtext", :rtf
================================================
FILE: config/initializers/ruby_template_handler.rb
================================================
# handler = ->(template) { template.source }
ActionView::Template.register_template_handler(:rb, :source.to_proc)
================================================
FILE: config/initializers/session_store.rb
================================================
# Be sure to restart your server when you modify this file.
Rails.application.config.session_store :cookie_store, key: "_SuperIMAP_session_#{Rails.env}"
================================================
FILE: config/initializers/wrap_parameters.rb
================================================
# Be sure to restart your server when you modify this file.
# This file contains settings for ActionController::ParamsWrapper which
# is enabled by default.
# Enable parameter wrapping for JSON. You can disable this by setting :format to an empty array.
ActiveSupport.on_load(:action_controller) do
wrap_parameters format: [:json] if respond_to?(:wrap_parameters)
end
# To enable root element in JSON for ActiveRecord objects.
# ActiveSupport.on_load(:active_record) do
# self.include_root_in_json = true
# end
================================================
FILE: config/locales/devise.en.yml
================================================
# Additional translations at https://github.com/plataformatec/devise/wiki/I18n
en:
devise:
confirmations:
confirmed: "Your email address has been successfully confirmed."
send_instructions: "You will receive an email with instructions for how to confirm your email address in a few minutes."
send_paranoid_instructions: "If your email address exists in our database, you will receive an email with instructions for how to confirm your email address in a few minutes."
failure:
already_authenticated: "You are already signed in."
inactive: "Your account is not activated yet."
invalid: "Invalid %{authentication_keys} or password."
locked: "Your account is locked."
last_attempt: "You have one more attempt before your account is locked."
not_found_in_database: "Invalid %{authentication_keys} or password."
timeout: "Your session expired. Please sign in again to continue."
unauthenticated: "You need to sign in before continuing."
unconfirmed: "You have to confirm your email address before continuing."
mailer:
confirmation_instructions:
subject: "Confirmation instructions"
reset_password_instructions:
subject: "Reset password instructions"
unlock_instructions:
subject: "Unlock instructions"
omniauth_callbacks:
failure: "Could not authenticate you from %{kind} because \"%{reason}\"."
success: "Successfully authenticated from %{kind} account."
passwords:
no_token: "You can't access this page without coming from a password reset email. If you do come from a password reset email, please make sure you used the full URL provided."
send_instructions: "You will receive an email with instructions on how to reset your password in a few minutes."
send_paranoid_instructions: "If your email address exists in our database, you will receive a password recovery link at your email address in a few minutes."
updated: "Your password has been changed successfully. You are now signed in."
updated_not_active: "Your password has been changed successfully."
registrations:
destroyed: "Bye! Your account has been successfully cancelled. We hope to see you again soon."
signed_up: "Welcome! You have signed up successfully."
signed_up_but_inactive: "You have signed up successfully. However, we could not sign you in because your account is not yet activated."
signed_up_but_locked: "You have signed up successfully. However, we could not sign you in because your account is locked."
signed_up_but_unconfirmed: "A message with a confirmation link has been sent to your email address. Please follow the link to activate your account."
update_needs_confirmation: "You updated your account successfully, but we need to verify your new email address. Please check your email and follow the confirm link to confirm your new email address."
updated: "Your account has been updated successfully."
sessions:
signed_in: "Signed in successfully."
signed_out: "Signed out successfully."
already_signed_out: "Signed out successfully."
unlocks:
send_instructions: "You will receive an email with instructions for how to unlock your account in a few minutes."
send_paranoid_instructions: "If your account exists, you will receive an email with instructions for how to unlock it in a few minutes."
unlocked: "Your account has been unlocked successfully. Please sign in to continue."
errors:
messages:
already_confirmed: "was already confirmed, please try signing in"
confirmation_period_expired: "needs to be confirmed within %{period}, please request a new one"
expired: "has expired, please request a new one"
not_found: "not found"
not_locked: "was not locked"
not_saved:
one: "1 error prohibited this %{resource} from being saved:"
other: "%{count} errors prohibited this %{resource} from being saved:"
================================================
FILE: config/locales/en.yml
================================================
# Files in the config/locales directory are used for internationalization
# and are automatically loaded by Rails. If you want to use locales other
# than English, add the necessary files in this directory.
#
# To use the locales, use `I18n.t`:
#
# I18n.t 'hello'
#
# In views, this is aliased to just `t`:
#
# <%= t('hello') %>
#
# To use a different locale, set it with `I18n.locale`:
#
# I18n.locale = :es
#
# This would use the information in config/locales/es.yml.
#
# To learn more, please read the Rails Internationalization guide
# available at http://guides.rubyonrails.org/i18n.html.
en:
hello: "Hello world"
================================================
FILE: config/newrelic.yml
================================================
#
# This file configures the New Relic Agent. New Relic monitors
# Ruby, Java, .NET, PHP, and Python applications with deep visibility and low overhead.
# For more information, visit www.newrelic.com.
#
# Generated September 02, 2013
#
# This configuration file is custom generated for app17845580@heroku.com
# Here are the settings that are common to all environments
common: &default_settings
# ============================== LICENSE KEY ===============================
license_key: '<%= ENV["NEW_RELIC_LICENSE_KEY"] %>'
# Agent Enabled (Ruby/Rails Only)
# Use this setting to force the agent to run or not run.
# Default is 'auto' which means the agent will install and run only
# if a valid dispatcher such as Mongrel is running. This prevents
# it from running with Rake or the console. Set to false to
# completely turn the agent off regardless of the other settings.
# Valid values are true, false and auto.
#
# agent_enabled: auto
# Application Name Set this to be the name of your application as
# you'd like it show up in New Relic. The service will then auto-map
# instances of your application into an "application" on your
# dashboard page. If you want to map this instance into multiple
# apps, like "AJAX Requests" and "All UI" then specify a semicolon
# separated list of up to three distinct names, or a yaml list.
# Defaults to the capitalized RAILS_ENV or RACK_ENV (i.e.,
# Production, Staging, etc)
#
# Example:
#
# app_name:
# - Ajax Service
# - All Services
#
app_name: My Application
# When "true", the agent collects performance data about your
# application and reports this data to the New Relic service at
# newrelic.com. This global switch is normally overridden for each
# environment below. (formerly called 'enabled')
monitor_mode: true
# Developer mode should be off in every environment but
# development as it has very high overhead in memory.
developer_mode: false
# The newrelic agent generates its own log file to keep its logging
# information separate from that of your application. Specify its
# log level here.
log_level: info
# Optionally set the path to the log file This is expanded from the
# root directory (may be relative or absolute, e.g. 'log/' or
# '/var/log/') The agent will attempt to create this directory if it
# does not exist.
# log_file_path: 'log'
# Optionally set the name of the log file, defaults to 'newrelic_agent.log'
# log_file_name: 'newrelic_agent.log'
# The newrelic agent communicates with the service via https by default. This
# prevents eavesdropping on the performance metrics transmitted by the agent.
# The encryption required by SSL introduces a nominal amount of CPU overhead,
# which is performed asynchronously in a background thread. If you'd prefer
# to send your metrics over http uncomment the following line.
# ssl: false
#============================== Browser Monitoring ===============================
# New Relic Real User Monitoring gives you insight into the performance real users are
# experiencing with your website. This is accomplished by measuring the time it takes for
# your users' browsers to download and render your web pages by injecting a small amount
# of JavaScript code into the header and footer of each page.
browser_monitoring:
# By default the agent automatically injects the monitoring JavaScript
# into web pages. Set this attribute to false to turn off this behavior.
auto_instrument: true
# Proxy settings for connecting to the New Relic server.
#
# If a proxy is used, the host setting is required. Other settings
# are optional. Default port is 8080.
#
# proxy_host: hostname
# proxy_port: 8080
# proxy_user:
# proxy_pass:
# The agent can optionally log all data it sends to New Relic servers to a
# separate log file for human inspection and auditing purposes. To enable this
# feature, change 'enabled' below to true.
# See: https://newrelic.com/docs/ruby/audit-log
audit_log:
enabled: false
# Tells transaction tracer and error collector (when enabled)
# whether or not to capture HTTP params. When true, frameworks can
# exclude HTTP parameters from being captured.
# Rails: the RoR filter_parameter_logging excludes parameters
# Java: create a config setting called "ignored_params" and set it to
# a comma separated list of HTTP parameter names.
# ex: ignored_params: credit_card, ssn, password
capture_params: false
# Transaction tracer captures deep information about slow
# transactions and sends this to the New Relic service once a
# minute. Included in the transaction is the exact call sequence of
# the transactions including any SQL statements issued.
transaction_tracer:
# Transaction tracer is enabled by default. Set this to false to
# turn it off. This feature is only available at the Professional
# and above product levels.
enabled: true
# Threshold in seconds for when to collect a transaction
# trace. When the response time of a controller action exceeds
# this threshold, a transaction trace will be recorded and sent to
# New Relic. Valid values are any float value, or (default) "apdex_f",
# which will use the threshold for an dissatisfying Apdex
# controller action - four times the Apdex T value.
transaction_threshold: apdex_f
# When transaction tracer is on, SQL statements can optionally be
# recorded. The recorder has three modes, "off" which sends no
# SQL, "raw" which sends the SQL statement in its original form,
# and "obfuscated", which strips out numeric and string literals.
record_sql: obfuscated
# Threshold in seconds for when to collect stack trace for a SQL
# call. In other words, when SQL statements exceed this threshold,
# then capture and send to New Relic the current stack trace. This is
# helpful for pinpointing where long SQL calls originate from.
stack_trace_threshold: 0.500
# Determines whether the agent will capture query plans for slow
# SQL queries. Only supported in mysql and postgres. Should be
# set to false when using other adapters.
# explain_enabled: true
# Threshold for query execution time below which query plans will
# not be captured. Relevant only when `explain_enabled` is true.
# explain_threshold: 0.5
# Error collector captures information about uncaught exceptions and
# sends them to New Relic for viewing
error_collector:
# Error collector is enabled by default. Set this to false to turn
# it off. This feature is only available at the Professional and above
# product levels.
enabled: true
# Rails Only - tells error collector whether or not to capture a
# source snippet around the place of the error when errors are View
# related.
capture_source: true
# To stop specific errors from reporting to New Relic, set this property
# to comma-separated values. Default is to ignore routing errors,
# which are how 404's get triggered.
ignore_errors: "ActionController::RoutingError,Sinatra::NotFound"
# If you're interested in capturing memcache keys as though they
# were SQL uncomment this flag. Note that this does increase
# overhead slightly on every memcached call, and can have security
# implications if your memcached keys are sensitive
# capture_memcache_keys: true
# Application Environments
# ------------------------------------------
# Environment-specific settings are in this section.
# For Rails applications, RAILS_ENV is used to determine the environment.
# For Java applications, pass -Dnewrelic.environment <environment> to set
# the environment.
# NOTE if your application has other named environments, you should
# provide newrelic configuration settings for these environments here.
development:
<<: *default_settings
# Turn off communication to New Relic service in development mode (also
# 'enabled').
# NOTE: for initial evaluation purposes, you may want to temporarily
# turn the agent on in development mode.
monitor_mode: false
# Rails Only - when running in Develo
gitextract_xuq40opm/
├── .gitignore
├── Gemfile
├── Procfile
├── Procfile.development
├── Procfile.imap-client-px
├── Procfile.stress-test
├── README.md
├── Rakefile
├── app/
│ ├── admin/
│ │ ├── admin_user.rb
│ │ ├── delayed_job.rb
│ │ ├── imap_provider.rb
│ │ ├── mail_log.rb
│ │ ├── partner.rb
│ │ ├── partner_connection.rb
│ │ ├── tracer_log.rb
│ │ ├── transmit_log.rb
│ │ └── user.rb
│ ├── assets/
│ │ ├── images/
│ │ │ └── .keep
│ │ ├── javascripts/
│ │ │ ├── active_admin.js.coffee
│ │ │ └── application.js
│ │ └── stylesheets/
│ │ ├── active_admin.css.scss
│ │ └── application.css
│ ├── controllers/
│ │ ├── api/
│ │ │ └── v1/
│ │ │ ├── connections_controller.rb
│ │ │ └── users_controller.rb
│ │ ├── application_controller.rb
│ │ ├── concerns/
│ │ │ ├── .keep
│ │ │ └── link_rel.rb
│ │ ├── users/
│ │ │ ├── base_callback_controller.rb
│ │ │ ├── connects_controller.rb
│ │ │ └── disconnects_controller.rb
│ │ └── webhook_test_controller.rb
│ ├── helpers/
│ │ ├── application_helper.rb
│ │ ├── oauth2/
│ │ │ ├── connects_helper.rb
│ │ │ └── disconnects_helper.rb
│ │ └── plain/
│ │ ├── connects_helper.rb
│ │ └── disconnects_helper.rb
│ ├── interactors/
│ │ ├── base_webhook.rb
│ │ ├── call_new_mail_webhook.rb
│ │ ├── call_user_connected_webhook.rb
│ │ ├── call_user_disconnected_webhook.rb
│ │ └── schedule_tracer_emails.rb
│ ├── mailers/
│ │ ├── .keep
│ │ └── tracer_mailer.rb
│ ├── models/
│ │ ├── .keep
│ │ ├── admin_user.rb
│ │ ├── concerns/
│ │ │ ├── .keep
│ │ │ ├── auth_method_helper.rb
│ │ │ └── connection_fields.rb
│ │ ├── delayed_job.rb
│ │ ├── imap_daemon_heartbeat.rb
│ │ ├── imap_provider.rb
│ │ ├── mail_log.rb
│ │ ├── oauth2/
│ │ │ ├── imap_provider.rb
│ │ │ ├── partner_connection.rb
│ │ │ └── user.rb
│ │ ├── partner.rb
│ │ ├── partner_connection.rb
│ │ ├── plain/
│ │ │ ├── imap_provider.rb
│ │ │ ├── partner_connection.rb
│ │ │ └── user.rb
│ │ ├── tracer_log.rb
│ │ ├── transmit_log.rb
│ │ └── user.rb
│ ├── processes/
│ │ ├── common/
│ │ │ ├── csv_log.rb
│ │ │ ├── db_connection.rb
│ │ │ ├── light_sleep.rb
│ │ │ ├── stoppable.rb
│ │ │ ├── worker_pool.rb
│ │ │ └── wrapped_thread.rb
│ │ ├── imap_client/
│ │ │ ├── daemon.rb
│ │ │ ├── process_uid.rb
│ │ │ ├── rendezvous_hash.rb
│ │ │ └── user_thread.rb
│ │ ├── imap_client.rb
│ │ ├── imap_test_server/
│ │ │ ├── daemon.rb
│ │ │ ├── mailboxes.rb
│ │ │ └── socket_state.rb
│ │ └── imap_test_server.rb
│ └── views/
│ ├── api/
│ │ └── v1/
│ │ ├── connections/
│ │ │ ├── index.json.rb
│ │ │ └── show.json.rb
│ │ └── users/
│ │ ├── index.json.rb
│ │ └── show.json.rb
│ ├── layouts/
│ │ ├── application.html.erb
│ │ └── blank.html.erb
│ └── tracer_mailer/
│ └── tracer_email.html.erb
├── app.json
├── bin/
│ ├── bundle
│ ├── delayed_job
│ ├── rails
│ ├── rake
│ └── spring
├── config/
│ ├── application.rb
│ ├── boot.rb
│ ├── database.yml.example
│ ├── environment.rb
│ ├── environments/
│ │ ├── development.rb
│ │ ├── performance.rb
│ │ ├── production.rb
│ │ ├── stress.rb
│ │ └── test.rb
│ ├── initializers/
│ │ ├── active_admin.rb
│ │ ├── airbrake.rb
│ │ ├── assets.rb
│ │ ├── backtrace_silencers.rb
│ │ ├── cookies_serializer.rb
│ │ ├── delayed_job.rb
│ │ ├── devise.rb
│ │ ├── filter_parameter_logging.rb
│ │ ├── inflections.rb
│ │ ├── log.rb
│ │ ├── mime_types.rb
│ │ ├── ruby_template_handler.rb
│ │ ├── session_store.rb
│ │ └── wrap_parameters.rb
│ ├── locales/
│ │ ├── devise.en.yml
│ │ └── en.yml
│ ├── newrelic.yml
│ ├── routes.rb
│ ├── secrets.yml
│ └── unicorn.rb
├── config.ru
├── db/
│ ├── migrate/
│ │ ├── 20141029214610_devise_create_admin_users.rb
│ │ ├── 20141029214612_create_active_admin_comments.rb
│ │ ├── 20141029215033_create_partners.rb
│ │ ├── 20141029215101_create_mail_logs.rb
│ │ ├── 20141029215105_create_transmit_logs.rb
│ │ ├── 20141031010321_create_imap_providers.rb
│ │ ├── 20141031010353_create_partner_connections.rb
│ │ ├── 20141031010433_create_users.rb
│ │ ├── 20141104202256_create_imap_daemon_heartbeats.rb
│ │ ├── 20141111204248_add_archived_to_users.rb
│ │ ├── 20141113163147_add_type_to_users_and_imap_providers.rb
│ │ ├── 20141114233206_add_locked_at_to_admin_user.rb
│ │ ├── 20141118170010_add_type_to_partner_connection.rb
│ │ ├── 20141121152941_add_oauth2_fields_to_imap_provider.rb
│ │ ├── 20141121182537_add_redirect_urls_to_partner.rb
│ │ ├── 20141121184010_rename_fields.rb
│ │ ├── 20141205024759_rename_partner_webhook_columns.rb
│ │ ├── 20141207191800_add_webhooks_to_partner.rb
│ │ ├── 20141207200312_create_delayed_jobs.rb
│ │ ├── 20141215194630_add_tracer_to_users.rb
│ │ ├── 20141215194652_create_tracer_logs.rb
│ │ ├── 20141215194754_add_smtp_settings_to_imap_provider.rb
│ │ ├── 20141215212628_remove_oauth1_fields.rb
│ │ ├── 20150119185401_encrypt_existing_data.rb
│ │ └── 20150611134232_add_more_indexes_to_mail_logs.rb
│ ├── schema.rb
│ ├── seeds-development.rb
│ ├── seeds-production.rb
│ ├── seeds-stress.rb
│ ├── seeds-test.rb
│ └── seeds.rb
├── lib/
│ ├── assets/
│ │ └── .keep
│ ├── tasks/
│ │ ├── .keep
│ │ └── imap.rake
│ └── xoauth2_authenticator.rb
├── log/
│ └── .keep
├── public/
│ ├── 404.html
│ ├── 422.html
│ ├── 500.html
│ ├── failure.html
│ ├── index.html
│ ├── robots.txt
│ └── success.html
├── script/
│ ├── analyze-stress-test.R
│ └── stress-test
├── test/
│ ├── controllers/
│ │ ├── .keep
│ │ └── api/
│ │ └── v1/
│ │ ├── connections_controller_test.rb
│ │ └── users_controller_test.rb
│ ├── fixtures/
│ │ ├── .keep
│ │ ├── imap_daemon_heartbeats.yml
│ │ └── tracer_logs.yml
│ ├── helpers/
│ │ └── .keep
│ ├── imap/
│ │ └── rendezvous_hash_test.rb
│ ├── integration/
│ │ └── .keep
│ ├── mailers/
│ │ ├── .keep
│ │ ├── previews/
│ │ │ └── tracer_mailer_preview.rb
│ │ └── tracer_mailer_test.rb
│ ├── models/
│ │ ├── .keep
│ │ ├── admin_user_test.rb
│ │ ├── imap_daemon_heartbeat_test.rb
│ │ ├── imap_provider_test.rb
│ │ ├── mail_log_test.rb
│ │ ├── partner_connection_test.rb
│ │ ├── partner_credential_test.rb
│ │ ├── partner_test.rb
│ │ ├── tracer_log_test.rb
│ │ ├── transmit_log_test.rb
│ │ └── user_test.rb
│ └── test_helper.rb
└── vendor/
└── assets/
├── javascripts/
│ └── .keep
└── stylesheets/
└── .keep
SYMBOL INDEX (399 symbols across 101 files)
FILE: app/admin/partner_connection.rb
function create (line 9) | def create
FILE: app/controllers/api/v1/connections_controller.rb
class Api::V1::ConnectionsController (line 1) | class Api::V1::ConnectionsController < ApplicationController
method index (line 12) | def index
method create (line 16) | def create
method update (line 24) | def update
method show (line 31) | def show
method destroy (line 35) | def destroy
method default_format_json (line 42) | def default_format_json
method load_partner (line 46) | def load_partner
method load_imap_provider (line 54) | def load_imap_provider
method load_connection (line 62) | def load_connection
method connection_params (line 69) | def connection_params
FILE: app/controllers/api/v1/users_controller.rb
class Api::V1::UsersController (line 1) | class Api::V1::UsersController < ApplicationController
method index (line 13) | def index
method create (line 17) | def create
method update (line 25) | def update
method show (line 32) | def show
method destroy (line 36) | def destroy
method default_format_json (line 43) | def default_format_json
method load_partner (line 47) | def load_partner
method load_imap_provider (line 55) | def load_imap_provider
method load_connection (line 63) | def load_connection
method load_user (line 70) | def load_user
method user_params (line 78) | def user_params
FILE: app/controllers/application_controller.rb
class ApplicationController (line 1) | class ApplicationController < ActionController::Base
method ensure_secure (line 8) | def ensure_secure
FILE: app/controllers/concerns/link_rel.rb
type LinkRel (line 1) | module LinkRel
function link_rel (line 5) | def self.link_rel(tag, url)
FILE: app/controllers/users/base_callback_controller.rb
class Users::BaseCallbackController (line 1) | class Users::BaseCallbackController < ApplicationController
method new (line 7) | def new
method callback (line 20) | def callback
method apply_helper (line 26) | def apply_helper
method load_user (line 31) | def load_user(user_id = nil)
method validate_signature (line 39) | def validate_signature(options = {})
method connection (line 51) | def connection
method partner (line 55) | def partner
method imap_provider (line 59) | def imap_provider
method redirect_to_success_url (line 63) | def redirect_to_success_url
method redirect_to_failure_url (line 67) | def redirect_to_failure_url
FILE: app/controllers/users/connects_controller.rb
class Users::ConnectsController (line 1) | class Users::ConnectsController < Users::BaseCallbackController
FILE: app/controllers/users/disconnects_controller.rb
class Users::DisconnectsController (line 1) | class Users::DisconnectsController < Users::BaseCallbackController
FILE: app/controllers/webhook_test_controller.rb
class WebhookTestController (line 1) | class WebhookTestController < ApplicationController
method new_mail (line 7) | def new_mail
method user_connected (line 12) | def user_connected
method user_disconnected (line 17) | def user_disconnected
FILE: app/helpers/application_helper.rb
type ApplicationHelper (line 1) | module ApplicationHelper
function array_to_hash (line 2) | def array_to_hash(values)
FILE: app/helpers/oauth2/connects_helper.rb
type Oauth2::ConnectsHelper (line 1) | module Oauth2::ConnectsHelper
function oauth2_new_helper (line 6) | def oauth2_new_helper
function oauth2_callback_helper (line 27) | def oauth2_callback_helper
function oauth2_email (line 57) | def oauth2_email
function gmail_oauth2_email (line 62) | def gmail_oauth2_email
FILE: app/helpers/oauth2/disconnects_helper.rb
type Oauth2::DisconnectsHelper (line 3) | module Oauth2::DisconnectsHelper
function oauth2_new_helper (line 4) | def oauth2_new_helper
FILE: app/helpers/plain/connects_helper.rb
type Plain::ConnectsHelper (line 1) | module Plain::ConnectsHelper
function plain_new_helper (line 2) | def plain_new_helper
function plain_callback_helper (line 6) | def plain_callback_helper
FILE: app/helpers/plain/disconnects_helper.rb
type Plain::DisconnectsHelper (line 1) | module Plain::DisconnectsHelper
function plain_new_helper (line 2) | def plain_new_helper
FILE: app/interactors/base_webhook.rb
class BaseWebhook (line 4) | class BaseWebhook
method calculate_signature (line 7) | def calculate_signature(api_key, uid, timestamp)
FILE: app/interactors/call_new_mail_webhook.rb
class CallNewMailWebhook (line 3) | class CallNewMailWebhook < BaseWebhook
method initialize (line 6) | def initialize(mail_log, envelope, raw_eml)
method run (line 12) | def run
FILE: app/interactors/call_user_connected_webhook.rb
class CallUserConnectedWebhook (line 1) | class CallUserConnectedWebhook < BaseWebhook
method initialize (line 4) | def initialize(user)
method run (line 8) | def run
FILE: app/interactors/call_user_disconnected_webhook.rb
class CallUserDisconnectedWebhook (line 1) | class CallUserDisconnectedWebhook < BaseWebhook
method initialize (line 4) | def initialize(user)
method run (line 8) | def run
FILE: app/interactors/schedule_tracer_emails.rb
class ScheduleTracerEmails (line 1) | class ScheduleTracerEmails
method initialize (line 4) | def initialize(user, num_tracers)
method run (line 9) | def run
method send_tracer_to_user (line 15) | def send_tracer_to_user(user)
FILE: app/mailers/tracer_mailer.rb
class TracerMailer (line 1) | class TracerMailer < ActionMailer::Base
method tracer_email (line 2) | def tracer_email(user, uid)
FILE: app/models/admin_user.rb
class AdminUser (line 1) | class AdminUser < ActiveRecord::Base
FILE: app/models/concerns/auth_method_helper.rb
type AuthMethodHelper (line 1) | module AuthMethodHelper
function auth_method_plain? (line 2) | def auth_method_plain?
function auth_method_oauth2? (line 6) | def auth_method_oauth2?
FILE: app/models/concerns/connection_fields.rb
type ConnectionFields (line 2) | module ConnectionFields
function encrypt (line 8) | def self.encrypt(value)
function decrypt (line 16) | def self.decrypt(value)
function connection_field (line 28) | def self.connection_field(field, options = {})
function connection_fields (line 59) | def self.connection_fields
function connection_fields (line 63) | def connection_fields
FILE: app/models/delayed_job.rb
class DelayedJob (line 1) | class DelayedJob < ActiveRecord::Base
method flush (line 4) | def self.flush
FILE: app/models/imap_daemon_heartbeat.rb
class ImapDaemonHeartbeat (line 1) | class ImapDaemonHeartbeat < ActiveRecord::Base
FILE: app/models/imap_provider.rb
class ImapProvider (line 1) | class ImapProvider < ActiveRecord::Base
method display_name (line 5) | def display_name
method class_for (line 18) | def class_for(c)
method helper_for (line 31) | def helper_for(action)
FILE: app/models/mail_log.rb
class MailLog (line 1) | class MailLog < ActiveRecord::Base
FILE: app/models/oauth2/imap_provider.rb
class Oauth2::ImapProvider (line 3) | class Oauth2::ImapProvider < ImapProvider
method authenticate_imap (line 16) | def authenticate_imap(client, user)
method authenticate_smtp (line 20) | def authenticate_smtp(mail, user)
method _access_token (line 34) | def _access_token(user)
FILE: app/models/oauth2/partner_connection.rb
class Oauth2::PartnerConnection (line 1) | class Oauth2::PartnerConnection < PartnerConnection
FILE: app/models/oauth2/user.rb
class Oauth2::User (line 1) | class Oauth2::User < User
method update_connected_at (line 8) | def update_connected_at
FILE: app/models/partner.rb
class Partner (line 1) | class Partner < ActiveRecord::Base
method ensure_api_key (line 12) | def ensure_api_key
method new_typed_connection (line 19) | def new_typed_connection(imap_provider)
FILE: app/models/partner_connection.rb
class PartnerConnection (line 1) | class PartnerConnection < ActiveRecord::Base
method display_name (line 17) | def display_name
method imap_provider_code (line 21) | def imap_provider_code
method new_typed_user (line 28) | def new_typed_user
method fix_type (line 37) | def fix_type
FILE: app/models/plain/imap_provider.rb
class Plain::ImapProvider (line 1) | class Plain::ImapProvider < ImapProvider
method authenticate_smtp (line 4) | def authenticate_smtp(mail, user)
method authenticate_imap (line 18) | def authenticate_imap(client, user)
FILE: app/models/plain/partner_connection.rb
class Plain::PartnerConnection (line 1) | class Plain::PartnerConnection < PartnerConnection
FILE: app/models/plain/user.rb
class Plain::User (line 1) | class Plain::User < User
method update_connected_at (line 8) | def update_connected_at
FILE: app/models/tracer_log.rb
class TracerLog (line 1) | class TracerLog < ActiveRecord::Base
FILE: app/models/transmit_log.rb
class TransmitLog (line 1) | class TransmitLog < ActiveRecord::Base
FILE: app/models/user.rb
class User (line 1) | class User < ActiveRecord::Base
method imap_provider (line 31) | def imap_provider
method signed_request_params (line 37) | def signed_request_params(timestamp = nil)
method valid_signature? (line 48) | def valid_signature?(params)
method fix_type (line 56) | def fix_type
FILE: app/processes/common/csv_log.rb
class Common::CsvLog (line 1) | class Common::CsvLog
method initialize (line 10) | def initialize(log_path)
method log (line 19) | def log(*values)
method _thread_runner (line 25) | def _thread_runner
method _drain_queue (line 35) | def _drain_queue
method _close_file (line 46) | def _close_file
FILE: app/processes/common/db_connection.rb
type Common::DbConnection (line 1) | module Common::DbConnection
function db_config (line 2) | def db_config
function set_db_connection_pool_size (line 7) | def set_db_connection_pool_size(size)
FILE: app/processes/common/light_sleep.rb
type Common::LightSleep (line 1) | module Common::LightSleep
function light_sleep (line 4) | def light_sleep(seconds = nil)
FILE: app/processes/common/stoppable.rb
type Common::Stoppable (line 1) | module Common::Stoppable
function init_stoppable (line 2) | def init_stoppable
function trap_signals (line 7) | def trap_signals
function stop! (line 12) | def stop!
function running? (line 18) | def running?
function stopping? (line 24) | def stopping?
FILE: app/processes/common/worker_pool.rb
type Common::WorkerPool (line 1) | module Common::WorkerPool
function init_worker_pool (line 11) | def init_worker_pool
function start_worker_pool (line 21) | def start_worker_pool(num_worker_threads)
function schedule_work (line 47) | def schedule_work(s, options)
function work_queue_length (line 57) | def work_queue_length
function terminate_worker_pool (line 64) | def terminate_worker_pool
function _start_worker_thread (line 73) | def _start_worker_thread(work_queue)
function _worker_thread_runner (line 81) | def _worker_thread_runner(work_queue)
function _worker_thread_next_action (line 88) | def _worker_thread_next_action(queue)
FILE: app/processes/common/wrapped_thread.rb
type Common::WrappedThread (line 1) | module Common::WrappedThread
function wrapped_thread (line 2) | def wrapped_thread(&block)
FILE: app/processes/imap_client.rb
class ImapClient (line 1) | class ImapClient
FILE: app/processes/imap_client/daemon.rb
class ImapClient::Daemon (line 19) | class ImapClient::Daemon
method initialize (line 46) | def initialize(options = {})
method user_threads_count (line 82) | def user_threads_count
method get_user_thread (line 89) | def get_user_thread(user_id)
method set_user_thread (line 96) | def set_user_thread(user_id, user_thread)
method delete_user_thread (line 103) | def delete_user_thread(user_id)
method walk_user_threads (line 111) | def walk_user_threads
method get_error_count (line 125) | def get_error_count(user_id)
method increment_error_count (line 132) | def increment_error_count(user_id)
method clear_error_count (line 140) | def clear_error_count(user_id)
method run (line 147) | def run
method force_class_loading (line 189) | def force_class_loading
method start_heartbeat_thread (line 197) | def start_heartbeat_thread
method start_discovery_thread (line 205) | def start_discovery_thread
method start_claim_thread (line 219) | def start_claim_thread
method start_tracer_thread (line 227) | def start_tracer_thread
method heartbeat_thread_runner (line 237) | def heartbeat_thread_runner
method discovery_thread_runner (line 263) | def discovery_thread_runner
method claim_thread_runner (line 286) | def claim_thread_runner
method tracer_thread_runner (line 308) | def tracer_thread_runner
method user_options (line 331) | def user_options
method action_connect_user (line 340) | def action_connect_user(options)
method action_disconnect_user (line 373) | def action_disconnect_user(options)
method terminate_user_threads (line 386) | def terminate_user_threads
method action_callback (line 404) | def action_callback(options)
method maybe_start_profiling (line 425) | def maybe_start_profiling
method stop_profiling (line 433) | def stop_profiling
FILE: app/processes/imap_client/process_uid.rb
class ProcessUid (line 15) | class ProcessUid
method initialize (line 22) | def initialize(user_thread, uid)
method run (line 28) | def run
method user (line 50) | def user
method client (line 55) | def client
method daemon (line 60) | def daemon
method confirm_tracer (line 66) | def confirm_tracer(tracer_uid)
method fetch_internal_date_and_size (line 73) | def fetch_internal_date_and_size
method check_for_really_old_internal_date (line 99) | def check_for_really_old_internal_date
method check_for_pre_creation_internal_date (line 112) | def check_for_pre_creation_internal_date
method check_for_relapsed_internal_date (line 124) | def check_for_relapsed_internal_date
method check_for_big_messages (line 135) | def check_for_big_messages
method fetch_uid_envelope_rfc822 (line 145) | def fetch_uid_envelope_rfc822
method update_user_mark_email_processed (line 174) | def update_user_mark_email_processed
method handle_tracer_email (line 192) | def handle_tracer_email
method check_for_duplicate_message_id (line 205) | def check_for_duplicate_message_id
method check_for_duplicate_sha1 (line 223) | def check_for_duplicate_sha1
method create_mail_log (line 241) | def create_mail_log
method deploy_webhook (line 249) | def deploy_webhook
method update_daemon_stats (line 259) | def update_daemon_stats
method clean_up (line 268) | def clean_up
method to_utf8 (line 280) | def to_utf8(s)
FILE: app/processes/imap_client/rendezvous_hash.rb
class ImapClient::RendezvousHash (line 2) | class ImapClient::RendezvousHash
method initialize (line 5) | def initialize
method site_tags= (line 10) | def site_tags=(site_tags)
method size (line 17) | def size
method hash (line 24) | def hash(object_tag)
FILE: app/processes/imap_client/user_thread.rb
class ImapClient::UserThread (line 18) | class ImapClient::UserThread
method initialize (line 26) | def initialize(daemon, user, options)
method update_user (line 34) | def update_user(hash)
method run (line 43) | def run
method schedule (line 74) | def schedule(&block)
method log_exception (line 87) | def log_exception(e)
method stat_exception (line 106) | def stat_exception(e)
method delay_start (line 117) | def delay_start
method connect (line 126) | def connect
method authenticate (line 134) | def authenticate
method choose_folder (line 153) | def choose_folder
method update_uid_validity (line 175) | def update_uid_validity
method main_loop (line 189) | def main_loop
method verify_uid_validity (line 223) | def verify_uid_validity
method jumpstart_stalled_account (line 235) | def jumpstart_stalled_account
method read_email_by_uid (line 245) | def read_email_by_uid
method read_email_by_date (line 262) | def read_email_by_date
method wait_for_email (line 279) | def wait_for_email
method process_uid (line 302) | def process_uid(uid)
method disconnect (line 313) | def disconnect
method clean_up (line 331) | def clean_up
FILE: app/processes/imap_test_server.rb
class ImapTestServer (line 1) | class ImapTestServer
FILE: app/processes/imap_test_server/daemon.rb
class ImapTestServer::Daemon (line 19) | class ImapTestServer::Daemon
method initialize (line 33) | def initialize(options = {})
method run (line 57) | def run
method start_stats_thread (line 96) | def start_stats_thread
method start_connection_thread (line 102) | def start_connection_thread
method start_new_mail_thread (line 108) | def start_new_mail_thread
method stats_thread_runner (line 114) | def stats_thread_runner
method connection_thread_runner (line 122) | def connection_thread_runner
method start_process_sockets_thread (line 140) | def start_process_sockets_thread
method process_sockets_runner (line 146) | def process_sockets_runner
method process_new_sockets (line 155) | def process_new_sockets
method process_new_socket (line 162) | def process_new_socket(socket)
method process_incoming_messages (line 177) | def process_incoming_messages
method process_incoming_message (line 189) | def process_incoming_message(socket)
method send_exists_messages (line 206) | def send_exists_messages
method send_exists_message (line 212) | def send_exists_message(socket_state)
method close_socket (line 219) | def close_socket(socket)
method new_mail_thread_runner (line 227) | def new_mail_thread_runner
method generate_new_mail (line 239) | def generate_new_mail(n)
FILE: app/processes/imap_test_server/mailboxes.rb
class ImapTestServer::Mailboxes (line 1) | class ImapTestServer::Mailboxes
method initialize (line 4) | def initialize(options = {})
method count (line 9) | def count
method find (line 13) | def find(username)
method each (line 19) | def each(&block)
class Mailbox (line 31) | class Mailbox
method initialize (line 36) | def initialize(username)
method count (line 42) | def count
method add_fake_message (line 46) | def add_fake_message
method uid_search (line 53) | def uid_search(from_uid, to_uid)
method date_search (line 59) | def date_search(since_date)
method fetch (line 65) | def fetch(uid)
FILE: app/processes/imap_test_server/socket_state.rb
class ImapTestServer::SocketState (line 3) | class ImapTestServer::SocketState
method initialize (line 15) | def initialize(daemon, socket, options)
method idling? (line 25) | def idling?
method handle_connect (line 30) | def handle_connect()
method handle_command (line 35) | def handle_command(s)
method send_exists_messages (line 42) | def send_exists_messages
method parse_command (line 51) | def parse_command(s)
method verb_to_method (line 66) | def verb_to_method(verb)
method choose (line 90) | def choose(choices)
method respond (line 101) | def respond(tag, s)
method imap_hello (line 108) | def imap_hello(tag, args)
method imap_hello_chaos (line 112) | def imap_hello_chaos(tag, args)
method imap_login (line 119) | def imap_login(tag, args)
method imap_login_chaos (line 125) | def imap_login_chaos(tag, args)
method imap_list (line 132) | def imap_list(tag, args)
method imap_list_chaos (line 139) | def imap_list_chaos(tag, args)
method imap_examine (line 148) | def imap_examine(tag, args)
method imap_examine_chaos (line 156) | def imap_examine_chaos(tag, args)
method imap_status (line 164) | def imap_status(tag, args)
method imap_status_chaos (line 170) | def imap_status_chaos(tag, args)
method imap_uid_search (line 178) | def imap_uid_search(tag, args)
method imap_uid_search_by_uid (line 188) | def imap_uid_search_by_uid(tag, args)
method imap_uid_search_by_date (line 204) | def imap_uid_search_by_date(tag, args)
method imap_uid_search_chaos (line 222) | def imap_uid_search_chaos(tag, args)
method imap_uid_fetch (line 230) | def imap_uid_fetch(tag, args)
method as_list (line 270) | def as_list(*values)
method as_address_structure (line 282) | def as_address_structure(addresses)
method as_date (line 298) | def as_date(date)
method as_integer (line 302) | def as_integer(n)
method as_string (line 306) | def as_string(s)
method as_multiline_string (line 310) | def as_multiline_string(s)
method imap_uid_fetch_chaos (line 314) | def imap_uid_fetch_chaos(tag, args)
method imap_idle (line 321) | def imap_idle(tag, args)
method imap_idle_chaos (line 327) | def imap_idle_chaos(tag, args)
method imap_done (line 331) | def imap_done(tag, args)
method imap_done_chaos (line 335) | def imap_done_chaos(tag, args)
method imap_logout (line 342) | def imap_logout(tag, args)
method imap_logout_chaos (line 348) | def imap_logout_chaos(tag, args)
method imap_chaos_respond_no (line 354) | def imap_chaos_respond_no(tag, args)
method imap_chaos_respond_bad (line 358) | def imap_chaos_respond_bad(tag, args)
method imap_chaos_gibberish_tagged (line 362) | def imap_chaos_gibberish_tagged(tag, args)
method imap_chaos_gibberish_untagged (line 366) | def imap_chaos_gibberish_untagged(tag, args)
method imap_chaos_soft_disconnect (line 370) | def imap_chaos_soft_disconnect(tag, args)
method imap_chaos_hard_disconnect (line 374) | def imap_chaos_hard_disconnect(tag, args)
FILE: config/application.rb
type SuperIMAP (line 9) | module SuperIMAP
class Application (line 10) | class Application < Rails::Application
FILE: config/initializers/delayed_job.rb
class Delayed::Worker (line 16) | class Delayed::Worker
method handle_failed_job (line 19) | def handle_failed_job(job, error)
FILE: config/initializers/log.rb
class MyLogger (line 1) | class MyLogger
method initialize (line 2) | def initialize
method exception (line 5) | def exception(exception)
method clean_backtrace (line 18) | def clean_backtrace(exception)
method librato (line 30) | def librato(mode, key, value)
method method_missing (line 42) | def method_missing(method, *args, &block)
FILE: db/migrate/20141029214610_devise_create_admin_users.rb
class DeviseCreateAdminUsers (line 1) | class DeviseCreateAdminUsers < ActiveRecord::Migration
method migrate (line 2) | def migrate(direction)
method change (line 8) | def change
FILE: db/migrate/20141029214612_create_active_admin_comments.rb
class CreateActiveAdminComments (line 1) | class CreateActiveAdminComments < ActiveRecord::Migration
method up (line 2) | def self.up
method down (line 16) | def self.down
FILE: db/migrate/20141029215033_create_partners.rb
class CreatePartners (line 1) | class CreatePartners < ActiveRecord::Migration
method change (line 2) | def change
FILE: db/migrate/20141029215101_create_mail_logs.rb
class CreateMailLogs (line 1) | class CreateMailLogs < ActiveRecord::Migration
method change (line 2) | def change
FILE: db/migrate/20141029215105_create_transmit_logs.rb
class CreateTransmitLogs (line 1) | class CreateTransmitLogs < ActiveRecord::Migration
method change (line 2) | def change
FILE: db/migrate/20141031010321_create_imap_providers.rb
class CreateImapProviders (line 1) | class CreateImapProviders < ActiveRecord::Migration
method change (line 2) | def change
FILE: db/migrate/20141031010353_create_partner_connections.rb
class CreatePartnerConnections (line 1) | class CreatePartnerConnections < ActiveRecord::Migration
method change (line 2) | def change
FILE: db/migrate/20141031010433_create_users.rb
class CreateUsers (line 1) | class CreateUsers < ActiveRecord::Migration
method change (line 2) | def change
FILE: db/migrate/20141104202256_create_imap_daemon_heartbeats.rb
class CreateImapDaemonHeartbeats (line 1) | class CreateImapDaemonHeartbeats < ActiveRecord::Migration
method change (line 2) | def change
FILE: db/migrate/20141111204248_add_archived_to_users.rb
class AddArchivedToUsers (line 1) | class AddArchivedToUsers < ActiveRecord::Migration
method change (line 2) | def change
FILE: db/migrate/20141113163147_add_type_to_users_and_imap_providers.rb
class AddTypeToUsersAndImapProviders (line 1) | class AddTypeToUsersAndImapProviders < ActiveRecord::Migration
method change (line 2) | def change
FILE: db/migrate/20141114233206_add_locked_at_to_admin_user.rb
class AddLockedAtToAdminUser (line 1) | class AddLockedAtToAdminUser < ActiveRecord::Migration
method change (line 2) | def change
FILE: db/migrate/20141118170010_add_type_to_partner_connection.rb
class AddTypeToPartnerConnection (line 1) | class AddTypeToPartnerConnection < ActiveRecord::Migration
method change (line 2) | def change
FILE: db/migrate/20141121152941_add_oauth2_fields_to_imap_provider.rb
class AddOauth2FieldsToImapProvider (line 1) | class AddOauth2FieldsToImapProvider < ActiveRecord::Migration
method change (line 2) | def change
FILE: db/migrate/20141121182537_add_redirect_urls_to_partner.rb
class AddRedirectUrlsToPartner (line 1) | class AddRedirectUrlsToPartner < ActiveRecord::Migration
method change (line 2) | def change
FILE: db/migrate/20141121184010_rename_fields.rb
class RenameFields (line 1) | class RenameFields < ActiveRecord::Migration
method change (line 2) | def change
FILE: db/migrate/20141205024759_rename_partner_webhook_columns.rb
class RenamePartnerWebhookColumns (line 1) | class RenamePartnerWebhookColumns < ActiveRecord::Migration
method change (line 2) | def change
FILE: db/migrate/20141207191800_add_webhooks_to_partner.rb
class AddWebhooksToPartner (line 1) | class AddWebhooksToPartner < ActiveRecord::Migration
method change (line 2) | def change
FILE: db/migrate/20141207200312_create_delayed_jobs.rb
class CreateDelayedJobs (line 1) | class CreateDelayedJobs < ActiveRecord::Migration
method up (line 2) | def self.up
method down (line 19) | def self.down
FILE: db/migrate/20141215194630_add_tracer_to_users.rb
class AddTracerToUsers (line 1) | class AddTracerToUsers < ActiveRecord::Migration
method change (line 2) | def change
FILE: db/migrate/20141215194652_create_tracer_logs.rb
class CreateTracerLogs (line 1) | class CreateTracerLogs < ActiveRecord::Migration
method change (line 2) | def change
FILE: db/migrate/20141215194754_add_smtp_settings_to_imap_provider.rb
class AddSmtpSettingsToImapProvider (line 1) | class AddSmtpSettingsToImapProvider < ActiveRecord::Migration
method change (line 2) | def change
FILE: db/migrate/20141215212628_remove_oauth1_fields.rb
class RemoveOauth1Fields (line 1) | class RemoveOauth1Fields < ActiveRecord::Migration
method change (line 2) | def change
FILE: db/migrate/20150119185401_encrypt_existing_data.rb
class EncryptExistingData (line 1) | class EncryptExistingData < ActiveRecord::Migration
method up (line 2) | def up
method down (line 29) | def down
FILE: db/migrate/20150611134232_add_more_indexes_to_mail_logs.rb
class AddMoreIndexesToMailLogs (line 1) | class AddMoreIndexesToMailLogs < ActiveRecord::Migration
method up (line 2) | def up
method down (line 7) | def down
FILE: db/seeds-development.rb
function create_transmit_log (line 30) | def create_transmit_log(mail_log, n)
function create_mail_log (line 34) | def create_mail_log(user, n)
function create_user (line 42) | def create_user(connection, n)
function create_partner_connection (line 54) | def create_partner_connection(partner, imap_provider)
FILE: db/seeds-stress.rb
function create_user (line 10) | def create_user(connection, n)
function create_partner_connection (line 19) | def create_partner_connection(partner, imap_provider)
FILE: db/seeds-test.rb
function create_transmit_log (line 10) | def create_transmit_log(mail_log, n)
function create_mail_log (line 14) | def create_mail_log(user, n)
function create_user (line 22) | def create_user(connection, n)
function create_partner_connection (line 34) | def create_partner_connection(partner, imap_provider)
FILE: lib/xoauth2_authenticator.rb
class Net::IMAP (line 3) | class Net::IMAP
class XOAuth2Authenticator (line 4) | class XOAuth2Authenticator
method initialize (line 5) | def initialize(email_address, access_token)
method process (line 10) | def process(s)
class Net::SMTP (line 18) | class Net::SMTP
method auth_xoauth2 (line 19) | def auth_xoauth2(email_address, access_token)
FILE: test/controllers/api/v1/connections_controller_test.rb
class Api::V1::ConnectionsControllerTest (line 3) | class Api::V1::ConnectionsControllerTest < ActionController::TestCase
FILE: test/controllers/api/v1/users_controller_test.rb
class Api::V1::UsersControllerTest (line 3) | class Api::V1::UsersControllerTest < ActionController::TestCase
FILE: test/imap/rendezvous_hash_test.rb
class RendezvousHashTest (line 4) | class RendezvousHashTest < ActiveSupport::TestCase
FILE: test/mailers/previews/tracer_mailer_preview.rb
class TracerMailerPreview (line 2) | class TracerMailerPreview < ActionMailer::Preview
FILE: test/mailers/tracer_mailer_test.rb
class TracerMailerTest (line 3) | class TracerMailerTest < ActionMailer::TestCase
FILE: test/models/admin_user_test.rb
class AdminUserTest (line 3) | class AdminUserTest < ActiveSupport::TestCase
FILE: test/models/imap_daemon_heartbeat_test.rb
class ImapDaemonHeartbeatTest (line 3) | class ImapDaemonHeartbeatTest < ActiveSupport::TestCase
FILE: test/models/imap_provider_test.rb
class ImapProviderTest (line 3) | class ImapProviderTest < ActiveSupport::TestCase
FILE: test/models/mail_log_test.rb
class MailLogTest (line 3) | class MailLogTest < ActiveSupport::TestCase
FILE: test/models/partner_connection_test.rb
class PartnerConnectionTest (line 3) | class PartnerConnectionTest < ActiveSupport::TestCase
FILE: test/models/partner_credential_test.rb
class PartnerCredentialTest (line 3) | class PartnerCredentialTest < ActiveSupport::TestCase
FILE: test/models/partner_test.rb
class PartnerTest (line 3) | class PartnerTest < ActiveSupport::TestCase
FILE: test/models/tracer_log_test.rb
class TracerLogTest (line 3) | class TracerLogTest < ActiveSupport::TestCase
FILE: test/models/transmit_log_test.rb
class TransmitLogTest (line 3) | class TransmitLogTest < ActiveSupport::TestCase
FILE: test/models/user_test.rb
class UserTest (line 3) | class UserTest < ActiveSupport::TestCase
FILE: test/test_helper.rb
class ActiveSupport::TestCase (line 5) | class ActiveSupport::TestCase
Condensed preview — 192 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (249K chars).
[
{
"path": ".gitignore",
"chars": 537,
"preview": "# See https://help.github.com/articles/ignoring-files for more about ignoring files.\n#\n# If you find yourself ignoring t"
},
{
"path": "Gemfile",
"chars": 1371,
"preview": "source 'https://rubygems.org'\n\nruby \"2.2.0\"\n\ngem 'rails' , '4.1.6'\ngem 'sass-rails' ,"
},
{
"path": "Procfile",
"chars": 691,
"preview": "web: bundle exec unicorn -p $PORT -c ./config/unicorn.rb\nworker: bundle exec rake jobs:work\n\n# Heroku"
},
{
"path": "Procfile.development",
"chars": 147,
"preview": "web: bundle exec unicorn -p $PORT -c ./config/unicorn.rb\nworker: bundle exec rake jobs:work\nimap_client: bu"
},
{
"path": "Procfile.imap-client-px",
"chars": 366,
"preview": "north: DYNO=$DYNO-1 NUM_WORKER_THREADS=3 MAX_USER_THREADS=3750 bundle exec rake imap:client\nsouth: DYNO=$DYNO-2 NUM_WORK"
},
{
"path": "Procfile.stress-test",
"chars": 279,
"preview": "imap_test_server: rake imap:test_server\nimap_client_1: rake imap:client STRESS_TEST_MODE=true ENABLE_PROFILER=$ENABLE_PR"
},
{
"path": "README.md",
"chars": 28129,
"preview": "# SuperIMAP - Version 0.1.2\n\nSuperIMAP helps you build email-driven applications. It takes care of\nconnecting to a custo"
},
{
"path": "Rakefile",
"chars": 249,
"preview": "# Add your own tasks in files placed in lib/tasks ending in .rake,\n# for example lib/tasks/capistrano.rake, and they wil"
},
{
"path": "app/admin/admin_user.rb",
"chars": 770,
"preview": "ActiveAdmin.register AdminUser do\n permit_params :email, :password, :password_confirmation\n menu priority: 100\n confi"
},
{
"path": "app/admin/delayed_job.rb",
"chars": 1351,
"preview": "ActiveAdmin.register DelayedJob do\n menu priority: 90\n config.sort_order = \"created_at_desc\"\n config.batch_actions = "
},
{
"path": "app/admin/imap_provider.rb",
"chars": 1463,
"preview": "ActiveAdmin.register ImapProvider do\n config.sort_order = \"code_asc\"\n permit_params :code, :title,\n :im"
},
{
"path": "app/admin/mail_log.rb",
"chars": 1326,
"preview": "ActiveAdmin.register MailLog do\n belongs_to :user\n\n config.sort_order = \"created_at_desc\"\n\n # Only allow viewing.\n c"
},
{
"path": "app/admin/partner.rb",
"chars": 1518,
"preview": "ActiveAdmin.register Partner do\n menu :priority => 0\n permit_params :name, :api_key,\n :success_url, :fa"
},
{
"path": "app/admin/partner_connection.rb",
"chars": 2053,
"preview": "ActiveAdmin.register PartnerConnection do\n belongs_to :partner\n\n permit_params :imap_provider_id,\n *Pla"
},
{
"path": "app/admin/tracer_log.rb",
"chars": 549,
"preview": "ActiveAdmin.register TracerLog do\n config.sort_order = \"created_at_desc\"\n\n # Only allow viewing.\n config.clear_action"
},
{
"path": "app/admin/transmit_log.rb",
"chars": 1331,
"preview": "ActiveAdmin.register TransmitLog do\n belongs_to :mail_log\n\n config.sort_order = \"created_at_desc\"\n\n # Only allow view"
},
{
"path": "app/admin/user.rb",
"chars": 3425,
"preview": "ActiveAdmin.register User do\n belongs_to :partner_connection\n config.sort_order = \"email_asc\"\n permit_params :tag, :e"
},
{
"path": "app/assets/images/.keep",
"chars": 0,
"preview": ""
},
{
"path": "app/assets/javascripts/active_admin.js.coffee",
"chars": 29,
"preview": "#= require active_admin/base\n"
},
{
"path": "app/assets/javascripts/application.js",
"chars": 664,
"preview": "// This is a manifest file that'll be compiled into application.js, which will include all the files\n// listed below.\n//"
},
{
"path": "app/assets/stylesheets/active_admin.css.scss",
"chars": 683,
"preview": "// SASS variable overrides must be declared before loading up Active Admin's styles.\n//\n// To view the variables that Ac"
},
{
"path": "app/assets/stylesheets/application.css",
"chars": 683,
"preview": "/*\n * This is a manifest file that'll be compiled into application.css, which will include all the files\n * listed below"
},
{
"path": "app/controllers/api/v1/connections_controller.rb",
"chars": 2095,
"preview": "class Api::V1::ConnectionsController < ApplicationController\n layout \"blank\"\n respond_to :json\n skip_before_action :v"
},
{
"path": "app/controllers/api/v1/users_controller.rb",
"chars": 2269,
"preview": "class Api::V1::UsersController < ApplicationController\n layout \"blank\"\n respond_to :json\n skip_before_action :verify_"
},
{
"path": "app/controllers/application_controller.rb",
"chars": 381,
"preview": "class ApplicationController < ActionController::Base\n # Prevent CSRF attacks by raising an exception.\n # For APIs, you"
},
{
"path": "app/controllers/concerns/.keep",
"chars": 0,
"preview": ""
},
{
"path": "app/controllers/concerns/link_rel.rb",
"chars": 233,
"preview": "module LinkRel\n extend ActiveSupport::Concern\n\n included do\n def self.link_rel(tag, url)\n @links ||= []\n "
},
{
"path": "app/controllers/users/base_callback_controller.rb",
"chars": 1563,
"preview": "class Users::BaseCallbackController < ApplicationController\n before_action :load_user\n before_action :validate_signatu"
},
{
"path": "app/controllers/users/connects_controller.rb",
"chars": 288,
"preview": "class Users::ConnectsController < Users::BaseCallbackController\n include Plain::ConnectsHelper\n include Oauth2::Connec"
},
{
"path": "app/controllers/users/disconnects_controller.rb",
"chars": 297,
"preview": "class Users::DisconnectsController < Users::BaseCallbackController\n include Plain::DisconnectsHelper\n include Oauth2::"
},
{
"path": "app/controllers/webhook_test_controller.rb",
"chars": 440,
"preview": "class WebhookTestController < ApplicationController\n layout \"blank\"\n skip_before_action :verify_authenticity_token\n\n "
},
{
"path": "app/helpers/application_helper.rb",
"chars": 144,
"preview": "module ApplicationHelper\n def array_to_hash(values)\n hash = {}\n values.each do |k,v|\n hash[k] = v\n end\n "
},
{
"path": "app/helpers/oauth2/connects_helper.rb",
"chars": 2031,
"preview": "module Oauth2::ConnectsHelper\n BadRequestError = Class.new(StandardError)\n\n attr_accessor :oauth2_token\n\n def oauth2_"
},
{
"path": "app/helpers/oauth2/disconnects_helper.rb",
"chars": 671,
"preview": "require 'uri'\n\nmodule Oauth2::DisconnectsHelper\n def oauth2_new_helper\n # Disconnect the user. Assume that this succ"
},
{
"path": "app/helpers/plain/connects_helper.rb",
"chars": 129,
"preview": "module Plain::ConnectsHelper\n def plain_new_helper\n raise :todo\n end\n\n def plain_callback_helper\n raise :todo\n "
},
{
"path": "app/helpers/plain/disconnects_helper.rb",
"chars": 81,
"preview": "module Plain::DisconnectsHelper\n def plain_new_helper\n raise :todo\n end\nend\n"
},
{
"path": "app/interactors/base_webhook.rb",
"chars": 268,
"preview": "require 'timeout'\nrequire 'net/imap'\n\nclass BaseWebhook\n private unless Rails.env.test?\n\n def calculate_signature(api_"
},
{
"path": "app/interactors/call_new_mail_webhook.rb",
"chars": 2548,
"preview": "# encoding: utf-8\n\nclass CallNewMailWebhook < BaseWebhook\n attr_accessor :mail_log, :envelope, :raw_eml\n\n def initiali"
},
{
"path": "app/interactors/call_user_connected_webhook.rb",
"chars": 1216,
"preview": "class CallUserConnectedWebhook < BaseWebhook\n attr_accessor :user\n\n def initialize(user)\n self.user = user\n end\n\n "
},
{
"path": "app/interactors/call_user_disconnected_webhook.rb",
"chars": 1189,
"preview": "class CallUserDisconnectedWebhook < BaseWebhook\n attr_accessor :user\n\n def initialize(user)\n self.user = user\n end"
},
{
"path": "app/interactors/schedule_tracer_emails.rb",
"chars": 607,
"preview": "class ScheduleTracerEmails\n attr_accessor :user, :num_tracers\n\n def initialize(user, num_tracers)\n self.user = user"
},
{
"path": "app/mailers/.keep",
"chars": 0,
"preview": ""
},
{
"path": "app/mailers/tracer_mailer.rb",
"chars": 192,
"preview": "class TracerMailer < ActionMailer::Base\n def tracer_email(user, uid)\n @uid = uid\n mail(:from => user.email,\n "
},
{
"path": "app/models/.keep",
"chars": 0,
"preview": ""
},
{
"path": "app/models/admin_user.rb",
"chars": 231,
"preview": "class AdminUser < ActiveRecord::Base\n # Include default devise modules. Others available are:\n # :confirmable, :lockab"
},
{
"path": "app/models/concerns/.keep",
"chars": 0,
"preview": ""
},
{
"path": "app/models/concerns/auth_method_helper.rb",
"chars": 172,
"preview": "module AuthMethodHelper\n def auth_method_plain?\n /^plain$/i.match(self.auth_method)\n end\n\n def auth_method_oauth2?"
},
{
"path": "app/models/concerns/connection_fields.rb",
"chars": 1517,
"preview": "# coding: utf-8\nmodule ConnectionFields\n extend ActiveSupport::Concern\n\n included do\n @connection_fields = []\n\n "
},
{
"path": "app/models/delayed_job.rb",
"chars": 487,
"preview": "class DelayedJob < ActiveRecord::Base\n # Run all existing delayed_job records. Log errors, return true if\n # everythin"
},
{
"path": "app/models/imap_daemon_heartbeat.rb",
"chars": 51,
"preview": "class ImapDaemonHeartbeat < ActiveRecord::Base\nend\n"
},
{
"path": "app/models/imap_provider.rb",
"chars": 902,
"preview": "class ImapProvider < ActiveRecord::Base\n include ConnectionFields\n has_many :partner_connections\n\n def display_name\n "
},
{
"path": "app/models/mail_log.rb",
"chars": 132,
"preview": "class MailLog < ActiveRecord::Base\n belongs_to :user, :counter_cache => true\n has_many :transmit_logs, :dependent => :"
},
{
"path": "app/models/oauth2/imap_provider.rb",
"chars": 1942,
"preview": "require 'xoauth2_authenticator'\n\nclass Oauth2::ImapProvider < ImapProvider\n include ConnectionFields\n\n connection_fiel"
},
{
"path": "app/models/oauth2/partner_connection.rb",
"chars": 216,
"preview": "class Oauth2::PartnerConnection < PartnerConnection\n include ConnectionFields\n connection_field :oauth2_client_id, :re"
},
{
"path": "app/models/oauth2/user.rb",
"chars": 349,
"preview": "class Oauth2::User < User\n include ConnectionFields\n before_save :update_connected_at\n\n connection_field :email\n con"
},
{
"path": "app/models/partner.rb",
"chars": 673,
"preview": "class Partner < ActiveRecord::Base\n # Magic.\n before_save :ensure_api_key\n\n # Relations\n has_many :partner_connectio"
},
{
"path": "app/models/partner_connection.rb",
"chars": 1009,
"preview": "class PartnerConnection < ActiveRecord::Base\n include ConnectionFields\n\n # Magic.\n before_validation :fix_type\n\n # R"
},
{
"path": "app/models/plain/imap_provider.rb",
"chars": 664,
"preview": "class Plain::ImapProvider < ImapProvider\n include ConnectionFields\n\n def authenticate_smtp(mail, user)\n mail.delive"
},
{
"path": "app/models/plain/partner_connection.rb",
"chars": 82,
"preview": "class Plain::PartnerConnection < PartnerConnection\n include ConnectionFields\nend\n"
},
{
"path": "app/models/plain/user.rb",
"chars": 354,
"preview": "class Plain::User < User\n include ConnectionFields\n before_save :update_connected_at\n\n connection_field :login_userna"
},
{
"path": "app/models/tracer_log.rb",
"chars": 60,
"preview": "class TracerLog < ActiveRecord::Base\n belongs_to :user\nend\n"
},
{
"path": "app/models/transmit_log.rb",
"chars": 90,
"preview": "class TransmitLog < ActiveRecord::Base\n belongs_to :mail_log, :counter_cache => true\nend\n"
},
{
"path": "app/models/user.rb",
"chars": 2001,
"preview": "class User < ActiveRecord::Base\n include ConnectionFields\n\n # Magic.\n before_validation :fix_type\n\n # Scopes.\n scop"
},
{
"path": "app/processes/common/csv_log.rb",
"chars": 1021,
"preview": "class Common::CsvLog\n include Common::Stoppable\n include Common::WrappedThread\n\n attr_accessor :log_path\n attr_acces"
},
{
"path": "app/processes/common/db_connection.rb",
"chars": 541,
"preview": "module Common::DbConnection\n def db_config\n ActiveRecord::Base.configurations[Rails.env] ||\n Rails.application."
},
{
"path": "app/processes/common/light_sleep.rb",
"chars": 223,
"preview": "module Common::LightSleep\n include Common::Stoppable\n\n def light_sleep(seconds = nil)\n now = Time.now\n while run"
},
{
"path": "app/processes/common/stoppable.rb",
"chars": 447,
"preview": "module Common::Stoppable\n def init_stoppable\n @stop = false\n @stop_lock = Mutex.new\n end\n\n def trap_signals\n "
},
{
"path": "app/processes/common/worker_pool.rb",
"chars": 2965,
"preview": "module Common::WorkerPool\n include Common::Stoppable\n include Common::WrappedThread\n include Common::DbConnection\n\n "
},
{
"path": "app/processes/common/wrapped_thread.rb",
"chars": 173,
"preview": "module Common::WrappedThread\n def wrapped_thread(&block)\n Thread.new do\n begin\n yield\n rescue => e\n"
},
{
"path": "app/processes/imap_client/daemon.rb",
"chars": 12436,
"preview": "# ImapClient::Daemon - Main entry point for the IMAP client. Reads\n# credentials from a database, connects to IMAP ser"
},
{
"path": "app/processes/imap_client/process_uid.rb",
"chars": 8592,
"preview": "# Private: Read and act on a single email. This is one place where\n# Ruby support for monads would be useful. The challe"
},
{
"path": "app/processes/imap_client/rendezvous_hash.rb",
"chars": 716,
"preview": "# http://en.wikipedia.org/wiki/Rendezvous_hashing\nclass ImapClient::RendezvousHash\n attr_accessor :lock\n\n def initiali"
},
{
"path": "app/processes/imap_client/user_thread.rb",
"chars": 10271,
"preview": "# ImapClient::UserThread - Manages the interactions of a single user\n# against an IMAP server. Written in a \"crash-onl"
},
{
"path": "app/processes/imap_client.rb",
"chars": 175,
"preview": "class ImapClient\n VERSION=\"1.0.0\"\nend\n\nrequire 'imap_client/rendezvous_hash'\nrequire 'imap_client/process_uid'\nrequire "
},
{
"path": "app/processes/imap_test_server/daemon.rb",
"chars": 6541,
"preview": "# ImapTestServer::Daemon - A test IMAP server that can respond to all\n# calls made by the ImapClient process. Generate"
},
{
"path": "app/processes/imap_test_server/mailboxes.rb",
"chars": 1676,
"preview": "class ImapTestServer::Mailboxes\n attr_accessor :mailboxes, :mailboxes_mutex\n\n def initialize(options = {})\n self.ma"
},
{
"path": "app/processes/imap_test_server/socket_state.rb",
"chars": 9558,
"preview": "require 'date'\n\nclass ImapTestServer::SocketState\n NormalDisconnect = Class.new(StandardError)\n ChaosDisconnect = Clas"
},
{
"path": "app/processes/imap_test_server.rb",
"chars": 78,
"preview": "class ImapTestServer\n VERSION=\"1.0.0\"\nend\n\nrequire 'imap_test_server/daemon'\n"
},
{
"path": "app/views/api/v1/connections/index.json.rb",
"chars": 184,
"preview": "fields = [\n :imap_provider_code,\n :users_count\n]\n\n@connections.map do |user|\n values = fields.map do |field|\n [fie"
},
{
"path": "app/views/api/v1/connections/show.json.rb",
"chars": 151,
"preview": "fields = [\n :imap_provider_code,\n :users_count\n]\nvalues = fields.map do |field|\n [field, @connection.send(field)]\nend"
},
{
"path": "app/views/api/v1/users/index.json.rb",
"chars": 151,
"preview": "fields = [:tag, :email]\n\n@users.map do |user|\n values = fields.map do |field|\n [field, user.send(field)]\n end\n arr"
},
{
"path": "app/views/api/v1/users/show.json.rb",
"chars": 267,
"preview": "{\n :tag => @user.tag,\n :email => @user.email,\n :connect_url => new_users_connect_url(@user.sig"
},
{
"path": "app/views/layouts/application.html.erb",
"chars": 398,
"preview": "<!DOCTYPE html>\n<html>\n <head>\n <title>SuperIMAP</title>\n <%= stylesheet_link_tag 'application', media: 'all',"
},
{
"path": "app/views/layouts/blank.html.erb",
"chars": 13,
"preview": "<%= yield %>\n"
},
{
"path": "app/views/tracer_mailer/tracer_email.html.erb",
"chars": 20,
"preview": "TRACER: <%= @uid %>\n"
},
{
"path": "app.json",
"chars": 702,
"preview": "{\n \"name\": \"SuperIMAP\",\n \"description\": \"Monitor inboxes for incoming email, at scale.\",\n \"repository\": \"https:"
},
{
"path": "bin/bundle",
"chars": 129,
"preview": "#!/usr/bin/env ruby\nENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)\nload Gem.bin_path('bundler', '"
},
{
"path": "bin/delayed_job",
"chars": 175,
"preview": "#!/usr/bin/env ruby\n\nrequire File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment'))\nrequire "
},
{
"path": "bin/rails",
"chars": 220,
"preview": "#!/usr/bin/env ruby\nbegin\n load File.expand_path(\"../spring\", __FILE__)\nrescue LoadError\nend\nAPP_PATH = File.expand_pat"
},
{
"path": "bin/rake",
"chars": 164,
"preview": "#!/usr/bin/env ruby\nbegin\n load File.expand_path(\"../spring\", __FILE__)\nrescue LoadError\nend\nrequire_relative '../confi"
},
{
"path": "bin/spring",
"chars": 510,
"preview": "#!/usr/bin/env ruby\n\n# This file loads spring without using Bundler, in order to be fast\n# It gets overwritten when you "
},
{
"path": "config/application.rb",
"chars": 1376,
"preview": "require File.expand_path('../boot', __FILE__)\n\nrequire 'rails/all'\n\n# Require the gems listed in Gemfile, including any "
},
{
"path": "config/boot.rb",
"chars": 170,
"preview": "# Set up gems listed in the Gemfile.\nENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)\n\nrequire 'bun"
},
{
"path": "config/database.yml.example",
"chars": 290,
"preview": "default: &default\n adapter: postgresql\n host: localhost\n pool: 5\n timeout: 5000\n user: username\n password: passwor"
},
{
"path": "config/environment.rb",
"chars": 150,
"preview": "# Load the Rails application.\nrequire File.expand_path('../application', __FILE__)\n\n# Initialize the Rails application.\n"
},
{
"path": "config/environments/development.rb",
"chars": 1422,
"preview": "Rails.application.configure do\n # Settings specified here will take precedence over those in config/application.rb.\n\n "
},
{
"path": "config/environments/performance.rb",
"chars": 3163,
"preview": "Rails.application.configure do\n # Settings specified here will take precedence over those in config/application.rb.\n\n "
},
{
"path": "config/environments/production.rb",
"chars": 3263,
"preview": "Rails.application.configure do\n # Settings specified here will take precedence over those in config/application.rb.\n\n "
},
{
"path": "config/environments/stress.rb",
"chars": 1422,
"preview": "Rails.application.configure do\n # Settings specified here will take precedence over those in config/application.rb.\n\n "
},
{
"path": "config/environments/test.rb",
"chars": 1661,
"preview": "Rails.application.configure do\n # Settings specified here will take precedence over those in config/application.rb.\n\n "
},
{
"path": "config/initializers/active_admin.rb",
"chars": 7192,
"preview": "# coding: utf-8\nActiveAdmin.setup do |config|\n\n # == Site Title\n #\n # Set the title that is displayed on the main lay"
},
{
"path": "config/initializers/airbrake.rb",
"chars": 129,
"preview": "if defined?(Airbrake) && ENV['AIRBRAKE_KEY']\n Airbrake.configure do |config|\n config.api_key = ENV['AIRBRAKE_KEY']\n "
},
{
"path": "config/initializers/assets.rb",
"chars": 377,
"preview": "# Be sure to restart your server when you modify this file.\n\n# Version of your assets, change this if you want to expire"
},
{
"path": "config/initializers/backtrace_silencers.rb",
"chars": 404,
"preview": "# Be sure to restart your server when you modify this file.\n\n# You can add backtrace silencers for libraries that you're"
},
{
"path": "config/initializers/cookies_serializer.rb",
"chars": 128,
"preview": "# Be sure to restart your server when you modify this file.\n\nRails.application.config.action_dispatch.cookies_serializer"
},
{
"path": "config/initializers/delayed_job.rb",
"chars": 825,
"preview": "# config/initializers/delayed_job.rb\n\n# Update settings.\nDelayed::Worker.destroy_failed_jobs = false\nDelayed::Worker.rea"
},
{
"path": "config/initializers/devise.rb",
"chars": 12644,
"preview": "# Use this hook to configure devise mailer, warden hooks and so forth.\n# Many of these configuration options can be set "
},
{
"path": "config/initializers/filter_parameter_logging.rb",
"chars": 194,
"preview": "# Be sure to restart your server when you modify this file.\n\n# Configure sensitive parameters which will be filtered fro"
},
{
"path": "config/initializers/inflections.rb",
"chars": 647,
"preview": "# Be sure to restart your server when you modify this file.\n\n# Add new inflection rules using the following format. Infl"
},
{
"path": "config/initializers/log.rb",
"chars": 927,
"preview": "class MyLogger\n def initialize\n end\n\n def exception(exception)\n msg = \"#{exception.class} (#{exception.message}):\\"
},
{
"path": "config/initializers/mime_types.rb",
"chars": 156,
"preview": "# Be sure to restart your server when you modify this file.\n\n# Add new mime types for use in respond_to blocks:\n# Mime::"
},
{
"path": "config/initializers/ruby_template_handler.rb",
"chars": 114,
"preview": "# handler = ->(template) { template.source }\nActionView::Template.register_template_handler(:rb, :source.to_proc)\n"
},
{
"path": "config/initializers/session_store.rb",
"chars": 154,
"preview": "# Be sure to restart your server when you modify this file.\n\nRails.application.config.session_store :cookie_store, key: "
},
{
"path": "config/initializers/wrap_parameters.rb",
"chars": 517,
"preview": "# Be sure to restart your server when you modify this file.\n\n# This file contains settings for ActionController::ParamsW"
},
{
"path": "config/locales/devise.en.yml",
"chars": 4014,
"preview": "# Additional translations at https://github.com/plataformatec/devise/wiki/I18n\n\nen:\n devise:\n confirmations:\n c"
},
{
"path": "config/locales/en.yml",
"chars": 634,
"preview": "# Files in the config/locales directory are used for internationalization\n# and are automatically loaded by Rails. If yo"
},
{
"path": "config/newrelic.yml",
"chars": 9378,
"preview": "#\n# This file configures the New Relic Agent. New Relic monitors\n# Ruby, Java, .NET, PHP, and Python applications with "
},
{
"path": "config/routes.rb",
"chars": 578,
"preview": "Rails.application.routes.draw do\n devise_for :admin_users, ActiveAdmin::Devise.config\n ActiveAdmin.routes(self)\n\n nam"
},
{
"path": "config/secrets.yml",
"chars": 964,
"preview": "# Be sure to restart your server when you modify this file.\n\n# Your secret key is used for verifying the integrity of si"
},
{
"path": "config/unicorn.rb",
"chars": 644,
"preview": "# config/unicorn.rb\n\nworker_processes Integer(ENV['WEB_CONCURRENCY'] || 2)\ntimeout Integer(ENV['WEB_TIMEOUT'] || 30)\npre"
},
{
"path": "config.ru",
"chars": 154,
"preview": "# This file is used by Rack-based servers to start the application.\n\nrequire ::File.expand_path('../config/environment',"
},
{
"path": "db/migrate/20141029214610_devise_create_admin_users.rb",
"chars": 1576,
"preview": "class DeviseCreateAdminUsers < ActiveRecord::Migration\n def migrate(direction)\n super\n # Create a default user\n "
},
{
"path": "db/migrate/20141029214612_create_active_admin_comments.rb",
"chars": 581,
"preview": "class CreateActiveAdminComments < ActiveRecord::Migration\n def self.up\n create_table :active_admin_comments do |t|\n "
},
{
"path": "db/migrate/20141029215033_create_partners.rb",
"chars": 315,
"preview": "class CreatePartners < ActiveRecord::Migration\n def change\n create_table :partners do |t|\n t.string :api_key, :"
},
{
"path": "db/migrate/20141029215101_create_mail_logs.rb",
"chars": 297,
"preview": "class CreateMailLogs < ActiveRecord::Migration\n def change\n create_table :mail_logs do |t|\n t.references :user,"
},
{
"path": "db/migrate/20141029215105_create_transmit_logs.rb",
"chars": 260,
"preview": "class CreateTransmitLogs < ActiveRecord::Migration\n def change\n create_table :transmit_logs do |t|\n t.reference"
},
{
"path": "db/migrate/20141031010321_create_imap_providers.rb",
"chars": 635,
"preview": "class CreateImapProviders < ActiveRecord::Migration\n def change\n create_table :imap_providers do |t|\n t.string "
},
{
"path": "db/migrate/20141031010353_create_partner_connections.rb",
"chars": 430,
"preview": "class CreatePartnerConnections < ActiveRecord::Migration\n def change\n create_table :partner_connections do |t|\n "
},
{
"path": "db/migrate/20141031010433_create_users.rb",
"chars": 596,
"preview": "class CreateUsers < ActiveRecord::Migration\n def change\n create_table :users do |t|\n t.references :partner_conn"
},
{
"path": "db/migrate/20141104202256_create_imap_daemon_heartbeats.rb",
"chars": 178,
"preview": "class CreateImapDaemonHeartbeats < ActiveRecord::Migration\n def change\n create_table :imap_daemon_heartbeats do |t|\n"
},
{
"path": "db/migrate/20141111204248_add_archived_to_users.rb",
"chars": 136,
"preview": "class AddArchivedToUsers < ActiveRecord::Migration\n def change\n add_column :users, :archived, :boolean, :default => "
},
{
"path": "db/migrate/20141113163147_add_type_to_users_and_imap_providers.rb",
"chars": 171,
"preview": "class AddTypeToUsersAndImapProviders < ActiveRecord::Migration\n def change\n add_column :imap_providers, :type, :stri"
},
{
"path": "db/migrate/20141114233206_add_locked_at_to_admin_user.rb",
"chars": 129,
"preview": "class AddLockedAtToAdminUser < ActiveRecord::Migration\n def change\n add_column :admin_users, :locked_at, :datetime\n "
},
{
"path": "db/migrate/20141118170010_add_type_to_partner_connection.rb",
"chars": 134,
"preview": "class AddTypeToPartnerConnection < ActiveRecord::Migration\n def change\n add_column :partner_connections, :type, :str"
},
{
"path": "db/migrate/20141121152941_add_oauth2_fields_to_imap_provider.rb",
"chars": 337,
"preview": "class AddOauth2FieldsToImapProvider < ActiveRecord::Migration\n def change\n add_column :imap_providers, :oauth2_autho"
},
{
"path": "db/migrate/20141121182537_add_redirect_urls_to_partner.rb",
"chars": 176,
"preview": "class AddRedirectUrlsToPartner < ActiveRecord::Migration\n def change\n add_column :partners, :success_url, :string\n "
},
{
"path": "db/migrate/20141121184010_rename_fields.rb",
"chars": 337,
"preview": "class RenameFields < ActiveRecord::Migration\n def change\n remove_column :users, :last_connected_at, :datetime\n ad"
},
{
"path": "db/migrate/20141205024759_rename_partner_webhook_columns.rb",
"chars": 194,
"preview": "class RenamePartnerWebhookColumns < ActiveRecord::Migration\n def change\n rename_column :partners, :success_webhook, "
},
{
"path": "db/migrate/20141207191800_add_webhooks_to_partner.rb",
"chars": 197,
"preview": "class AddWebhooksToPartner < ActiveRecord::Migration\n def change\n add_column :partners, :user_connected_webhook, :st"
},
{
"path": "db/migrate/20141207200312_create_delayed_jobs.rb",
"chars": 1350,
"preview": "class CreateDelayedJobs < ActiveRecord::Migration\n def self.up\n create_table :delayed_jobs, :force => true do |table"
},
{
"path": "db/migrate/20141215194630_add_tracer_to_users.rb",
"chars": 178,
"preview": "class AddTracerToUsers < ActiveRecord::Migration\n def change\n add_column :users, :enable_tracer, :boolean, :default "
},
{
"path": "db/migrate/20141215194652_create_tracer_logs.rb",
"chars": 273,
"preview": "class CreateTracerLogs < ActiveRecord::Migration\n def change\n create_table :tracer_logs do |t|\n t.references :u"
},
{
"path": "db/migrate/20141215194754_add_smtp_settings_to_imap_provider.rb",
"chars": 478,
"preview": "class AddSmtpSettingsToImapProvider < ActiveRecord::Migration\n def change\n rename_column :imap_providers, :host, :im"
},
{
"path": "db/migrate/20141215212628_remove_oauth1_fields.rb",
"chars": 566,
"preview": "class RemoveOauth1Fields < ActiveRecord::Migration\n def change\n remove_column :imap_providers, :oauth1_access_token_"
},
{
"path": "db/migrate/20150119185401_encrypt_existing_data.rb",
"chars": 940,
"preview": "class EncryptExistingData < ActiveRecord::Migration\n def up\n # Update Oauth2::PartnerConnection entries.\n connect"
},
{
"path": "db/migrate/20150611134232_add_more_indexes_to_mail_logs.rb",
"chars": 188,
"preview": "class AddMoreIndexesToMailLogs < ActiveRecord::Migration\n def up\n add_index :mail_logs, [:user_id, :message_id]\n "
},
{
"path": "db/schema.rb",
"chars": 6946,
"preview": "# encoding: UTF-8\n# This file is auto-generated from the current state of the database. Instead\n# of editing this file, "
},
{
"path": "db/seeds-development.rb",
"chars": 2570,
"preview": "AdminUser.new(:email => \"admin@example.com\", :password => \"password\").save\n\nplain_provider = Plain::ImapProvider.create("
},
{
"path": "db/seeds-production.rb",
"chars": 1043,
"preview": "Oauth2::ImapProvider.create!(\n :code => 'GMAIL_OAUTH2',\n :title => \"Google Ma"
},
{
"path": "db/seeds-stress.rb",
"chars": 969,
"preview": "AdminUser.new(:email => \"admin@example.com\", :password => \"password\").save\n\nimap_provider = Plain::ImapProvider.create!("
},
{
"path": "db/seeds-test.rb",
"chars": 1408,
"preview": "AdminUser.new(:email => \"admin@example.com\", :password => \"password\").save\n\nplain_provider = Plain::ImapProvider.create!"
},
{
"path": "db/seeds.rb",
"chars": 43,
"preview": "require_relative \"./seeds-#{Rails.env}.rb\"\n"
},
{
"path": "lib/assets/.keep",
"chars": 0,
"preview": ""
},
{
"path": "lib/tasks/.keep",
"chars": 0,
"preview": ""
},
{
"path": "lib/tasks/imap.rake",
"chars": 1394,
"preview": "namespace :imap do\n task :client => :environment do\n Log.info(\"Starting an IMAP Client process...\")\n\n # Read envi"
},
{
"path": "lib/xoauth2_authenticator.rb",
"chars": 641,
"preview": "require 'net/imap'\n\nclass Net::IMAP\n class XOAuth2Authenticator\n def initialize(email_address, access_token)\n @"
},
{
"path": "log/.keep",
"chars": 0,
"preview": ""
},
{
"path": "public/404.html",
"chars": 1564,
"preview": "<!DOCTYPE html>\n<html>\n<head>\n <title>The page you were looking for doesn't exist (404)</title>\n <meta name=\"viewport\""
},
{
"path": "public/422.html",
"chars": 1547,
"preview": "<!DOCTYPE html>\n<html>\n<head>\n <title>The change you wanted was rejected (422)</title>\n <meta name=\"viewport\" content="
},
{
"path": "public/500.html",
"chars": 1477,
"preview": "<!DOCTYPE html>\n<html>\n<head>\n <title>We're sorry, but something went wrong (500)</title>\n <meta name=\"viewport\" conte"
},
{
"path": "public/failure.html",
"chars": 8,
"preview": "FAILURE\n"
},
{
"path": "public/index.html",
"chars": 63,
"preview": "<script>\n document.location.href = \"/\" + \"admin\";\n</script>\n"
},
{
"path": "public/robots.txt",
"chars": 202,
"preview": "# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file\n#\n# To ban all spiders"
},
{
"path": "public/success.html",
"chars": 8,
"preview": "SUCCESS\n"
},
{
"path": "script/analyze-stress-test.R",
"chars": 2689,
"preview": "#!/usr/bin/env Rscript\n\nlibrary(ggplot2)\n\n###\n### READ DATA\n###\n\n## Read CSV files.\ncat(\"Reading CSV files.\")\ngenerated "
},
{
"path": "script/stress-test",
"chars": 2150,
"preview": "#!/usr/bin/env bash\n\nulimit -n 2048\n\n[ -z \"$LENGTH_OF_TEST\" ] && LENGTH_OF_TEST=\"1\"\n[ -z \"$EMAILS_PER_MINUTE\" ] "
},
{
"path": "test/controllers/.keep",
"chars": 0,
"preview": ""
},
{
"path": "test/controllers/api/v1/connections_controller_test.rb",
"chars": 1076,
"preview": "require 'test_helper'\n\nclass Api::V1::ConnectionsControllerTest < ActionController::TestCase\n setup do\n @partner = P"
},
{
"path": "test/controllers/api/v1/users_controller_test.rb",
"chars": 1390,
"preview": "require 'test_helper'\n\nclass Api::V1::UsersControllerTest < ActionController::TestCase\n setup do\n @partner = Partner"
},
{
"path": "test/fixtures/.keep",
"chars": 0,
"preview": ""
},
{
"path": "test/fixtures/imap_daemon_heartbeats.yml",
"chars": 133,
"preview": "# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html\n\none:\n tag: MyString\n\ntwo:\n t"
},
{
"path": "test/fixtures/tracer_logs.yml",
"chars": 227,
"preview": "# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html\n\none:\n user_id: \n uid: MyStri"
},
{
"path": "test/helpers/.keep",
"chars": 0,
"preview": ""
},
{
"path": "test/imap/rendezvous_hash_test.rb",
"chars": 371,
"preview": "require 'imap_client'\nrequire 'test_helper'\n\nclass RendezvousHashTest < ActiveSupport::TestCase\n test \"Hashing\" do\n "
},
{
"path": "test/integration/.keep",
"chars": 0,
"preview": ""
},
{
"path": "test/mailers/.keep",
"chars": 0,
"preview": ""
},
{
"path": "test/mailers/previews/tracer_mailer_preview.rb",
"chars": 129,
"preview": "# Preview all emails at http://localhost:3000/rails/mailers/tracer_mailer\nclass TracerMailerPreview < ActionMailer::Prev"
},
{
"path": "test/mailers/tracer_mailer_test.rb",
"chars": 125,
"preview": "require 'test_helper'\n\nclass TracerMailerTest < ActionMailer::TestCase\n # test \"the truth\" do\n # assert true\n # end"
},
{
"path": "test/models/.keep",
"chars": 0,
"preview": ""
},
{
"path": "test/models/admin_user_test.rb",
"chars": 123,
"preview": "require 'test_helper'\n\nclass AdminUserTest < ActiveSupport::TestCase\n # test \"the truth\" do\n # assert true\n # end\ne"
},
{
"path": "test/models/imap_daemon_heartbeat_test.rb",
"chars": 133,
"preview": "require 'test_helper'\n\nclass ImapDaemonHeartbeatTest < ActiveSupport::TestCase\n # test \"the truth\" do\n # assert true"
},
{
"path": "test/models/imap_provider_test.rb",
"chars": 126,
"preview": "require 'test_helper'\n\nclass ImapProviderTest < ActiveSupport::TestCase\n # test \"the truth\" do\n # assert true\n # en"
},
{
"path": "test/models/mail_log_test.rb",
"chars": 121,
"preview": "require 'test_helper'\n\nclass MailLogTest < ActiveSupport::TestCase\n # test \"the truth\" do\n # assert true\n # end\nend"
},
{
"path": "test/models/partner_connection_test.rb",
"chars": 131,
"preview": "require 'test_helper'\n\nclass PartnerConnectionTest < ActiveSupport::TestCase\n # test \"the truth\" do\n # assert true\n "
},
{
"path": "test/models/partner_credential_test.rb",
"chars": 131,
"preview": "require 'test_helper'\n\nclass PartnerCredentialTest < ActiveSupport::TestCase\n # test \"the truth\" do\n # assert true\n "
},
{
"path": "test/models/partner_test.rb",
"chars": 121,
"preview": "require 'test_helper'\n\nclass PartnerTest < ActiveSupport::TestCase\n # test \"the truth\" do\n # assert true\n # end\nend"
},
{
"path": "test/models/tracer_log_test.rb",
"chars": 123,
"preview": "require 'test_helper'\n\nclass TracerLogTest < ActiveSupport::TestCase\n # test \"the truth\" do\n # assert true\n # end\ne"
},
{
"path": "test/models/transmit_log_test.rb",
"chars": 125,
"preview": "require 'test_helper'\n\nclass TransmitLogTest < ActiveSupport::TestCase\n # test \"the truth\" do\n # assert true\n # end"
},
{
"path": "test/models/user_test.rb",
"chars": 118,
"preview": "require 'test_helper'\n\nclass UserTest < ActiveSupport::TestCase\n # test \"the truth\" do\n # assert true\n # end\nend\n"
},
{
"path": "test/test_helper.rb",
"chars": 312,
"preview": "ENV['RAILS_ENV'] ||= 'test'\nrequire File.expand_path('../../config/environment', __FILE__)\nrequire 'rails/test_help'\n\ncl"
},
{
"path": "vendor/assets/javascripts/.keep",
"chars": 0,
"preview": ""
},
{
"path": "vendor/assets/stylesheets/.keep",
"chars": 0,
"preview": ""
}
]
About this extraction
This page contains the full source code of the rustyio/super-imap GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 192 files (222.4 KB), approximately 63.1k tokens, and a symbol index with 399 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.