Repository: andyatkinson/rideshare Branch: main Commit: f6c2fb913f72 Files: 269 Total size: 269.0 KB Directory structure: gitextract_qyu9k5q7/ ├── .circleci/ │ └── config.yml ├── .erdconfig ├── .git-blame-ignore-revs ├── .gitignore ├── .rubocop.yml ├── .ruby-version ├── GUIDES.md ├── Gemfile ├── README.md ├── Rakefile ├── TESTING.md ├── app/ │ ├── assets/ │ │ ├── config/ │ │ │ └── manifest.js │ │ ├── images/ │ │ │ └── .keep │ │ └── stylesheets/ │ │ └── application.css │ ├── channels/ │ │ └── application_cable/ │ │ ├── channel.rb │ │ └── connection.rb │ ├── controllers/ │ │ ├── api/ │ │ │ ├── trip_requests_controller.rb │ │ │ └── trips_controller.rb │ │ ├── api_controller.rb │ │ ├── application_controller.rb │ │ ├── authentication_controller.rb │ │ └── concerns/ │ │ └── .keep │ ├── helpers/ │ │ └── application_helper.rb │ ├── javascript/ │ │ └── application.js │ ├── jobs/ │ │ └── application_job.rb │ ├── lib/ │ │ └── pgslice_helper.rb │ ├── mailers/ │ │ └── application_mailer.rb │ ├── models/ │ │ ├── application_record.rb │ │ ├── concerns/ │ │ │ └── .keep │ │ ├── driver.rb │ │ ├── fast_search_result.rb │ │ ├── location.rb │ │ ├── rider.rb │ │ ├── search_result.rb │ │ ├── trip.rb │ │ ├── trip_position.rb │ │ ├── trip_request.rb │ │ ├── user.rb │ │ ├── vehicle.rb │ │ ├── vehicle_reservation.rb │ │ └── vehicle_status.rb │ ├── queries/ │ │ └── top_drivers.sql │ ├── serializers/ │ │ ├── driver_serializer.rb │ │ └── trip_serializer.rb │ ├── services/ │ │ ├── book_reservation.rb │ │ ├── trip_creator.rb │ │ └── trip_search.rb │ └── validators/ │ ├── drivers_license_validator.rb │ └── email_validator.rb ├── bin/ │ ├── bundle │ ├── importmap │ ├── partition_conversion.sh │ ├── pgslice │ ├── rails │ ├── rails_best_practices │ ├── rake │ └── setup ├── config/ │ ├── application.rb │ ├── boot.rb │ ├── cable.yml │ ├── credentials.yml.enc │ ├── database-multiple.sample.yml │ ├── database-slow-clients.sample.yml │ ├── database.yml │ ├── environment.rb │ ├── environments/ │ │ ├── development.rb │ │ ├── production.rb │ │ └── test.rb │ ├── importmap.rb │ ├── initializers/ │ │ ├── application_controller_renderer.rb │ │ ├── assets.rb │ │ ├── backtrace_silencers.rb │ │ ├── cookies_serializer.rb │ │ ├── filter_parameter_logging.rb │ │ ├── geocoder.rb │ │ ├── inflections.rb │ │ ├── mime_types.rb │ │ ├── slow_query_subscriber.rb │ │ ├── strong_migrations.rb │ │ └── wrap_parameters.rb │ ├── locales/ │ │ └── en.yml │ ├── puma.rb │ ├── routes.rb │ └── schedule.rb ├── config.ru ├── db/ │ ├── README.md │ ├── alter_default_privileges_public.sql │ ├── alter_default_privileges_readonly.sql │ ├── alter_default_privileges_readwrite.sql │ ├── create_database.sql │ ├── create_grants_database.sql │ ├── create_grants_schema.sql │ ├── create_role_app_readonly.sql │ ├── create_role_app_user.sql │ ├── create_role_owner.sql │ ├── create_role_readonly_users.sql │ ├── create_role_readwrite_users.sql │ ├── create_schema.sql │ ├── env_vars_sample.sh │ ├── functions/ │ │ ├── scrub_email_v01.sql │ │ ├── scrub_email_v02.sql │ │ ├── scrub_text_v01.sql │ │ └── scrub_text_v02.sql │ ├── migrate/ │ │ ├── 20191107212726_create_users.rb │ │ ├── 20191108221519_create_locations.rb │ │ ├── 20191111151637_create_trip_requests.rb │ │ ├── 20191112165848_create_trips.rb │ │ ├── 20191121175429_install_blazer.rb │ │ ├── 20191203212055_add_foreign_key_constraints.rb │ │ ├── 20191203213103_validate_foreign_key_constraints.rb │ │ ├── 20200603150442_add_column_users_password_digest.rb │ │ ├── 20220711010541_add_db_comments_to_users.rb │ │ ├── 20220711015454_create_function_scrub_email.rb │ │ ├── 20220711015524_create_function_scrub_text.rb │ │ ├── 20220716020213_add_index_users_last_name.rb │ │ ├── 20220729014635_create_vehicle_reservations.rb │ │ ├── 20220729020430_create_vehicles.rb │ │ ├── 20220801140121_add_exclusion_constraint_vehicle_registrations.rb │ │ ├── 20220814175213_add_trips_count_to_users.rb │ │ ├── 20220916171314_create_search_results.rb │ │ ├── 20221007184855_create_fast_search_results.rb │ │ ├── 20221108172933_add_status_column_to_vehicles.rb │ │ ├── 20221108175321_remove_status_column_from_vehicles.rb │ │ ├── 20221108175619_add_status_column_db_enum_type_to_vehicles.rb │ │ ├── 20221110020532_add_drivers_license_number_to_users.rb │ │ ├── 20221111212740_add_trip_rating_check_constraint.rb │ │ ├── 20221111213918_validate_add_trip_rating_check_constraint.rb │ │ ├── 20221219164626_add_unique_address_to_locations.rb │ │ ├── 20221220201836_enable_extension_pg_stat_statements.rb │ │ ├── 20221221052616_change_column_trips_trip_request_id.rb │ │ ├── 20221223161403_create_trip_positions.rb │ │ ├── 20221230200725_add_unique_constraint_users_email.rb │ │ ├── 20221230203627_fix_canceled_column_default.rb │ │ ├── 20230125003531_add_searchable_full_name_to_users.rb │ │ ├── 20230125003946_add_index_searchable_full_name_to_users.rb │ │ ├── 20230126025656_remove_blazer_from_rideshare.rb │ │ ├── 20230314204931_create_trip_positions_partitioned_intermediate_table.rb │ │ ├── 20230314210022_add_trip_positions_intermediate_default_partition.rb │ │ ├── 20230619213546_add_locations_city_state.rb │ │ ├── 20230620030038_remove_unused_indexes.rb │ │ ├── 20230625151410_add_foreign_keys.rb │ │ ├── 20230711015123_add_fast_count_gem.rb │ │ ├── 20230713150550_update_function_scrub_email_to_version_2.rb │ │ ├── 20230713150710_update_function_scrub_text_to_version_2.rb │ │ ├── 20230714013609_trips_check_constraints.rb │ │ ├── 20230716174139_add_foreign_key_column_vehicle_reservations.rb │ │ ├── 20230726020548_add_not_null_trip_positions_position.rb │ │ ├── 20230925150207_add_position_to_locations.rb │ │ ├── 20230925150831_drop_locations_latitude_longitude.rb │ │ ├── 20231018153441_update_fast_search_results_to_version_2.rb │ │ ├── 20231018153712_add_unique_index_fast_search_results.rb │ │ ├── 20231208050516_drop_column_searchable_full_name.rb │ │ ├── 20231213045957_add_constraints_locations_state.rb │ │ ├── 20231218215836_remove_trip_positions_intermediate.rb │ │ └── 20231220043547_install_fast_count.rb │ ├── pgbouncer_prepared_statements_check.sh │ ├── reset.sh │ ├── revoke_drop_public_schema.sql │ ├── scripts/ │ │ ├── README.md │ │ ├── benchmark.sh │ │ ├── bulk_load.sh │ │ ├── bulk_load_extended.sh │ │ ├── list_table_comments.sh │ │ ├── queries.sql │ │ └── simulate_bloat.sh │ ├── scrubbing/ │ │ ├── .gitignore │ │ ├── README.md │ │ ├── assign_sequence.sql │ │ ├── create_tables.sql │ │ ├── drop_and_swap_users.sql │ │ ├── dump_foreign_keys_ddl_target_table.sql │ │ ├── dump_sequence_creation_ddl.sql │ │ ├── dump_views_ddl.sql │ │ ├── generate_add_constraint_statements.sql │ │ ├── scrub_batched_direct_updates.sql │ │ ├── scrub_users.sql │ │ └── scrubber.sh │ ├── setup.sh │ ├── setup_test_database.sh │ ├── structure.sql │ ├── teardown.sh │ ├── teardown_remove_default_privileges.sql │ └── views/ │ ├── fast_search_results_v01.sql │ ├── fast_search_results_v02.sql │ └── search_results_v01.sql ├── docker/ │ ├── README.md │ ├── db01_create_publication.sh │ ├── db01_create_replication_slot.sh │ ├── db01_create_replication_user.sh │ ├── db03_create_subscription.sh │ ├── db03_create_subscription_prepare.sh │ ├── dump_rideshare_local_to_db01.sh │ ├── pg_hba_reset.sh │ ├── reset_docker_instances.sh │ ├── run_db_db01_primary.sh │ ├── run_db_db02_replica.sh │ ├── run_db_db03_replica.sh │ ├── run_pg_basebackup.sh │ └── teardown_docker.sh ├── docs/ │ ├── design_document.md │ ├── dev_tips.md │ ├── development.md │ ├── development_iterations.md │ ├── project_documentation.md │ ├── search.md │ └── workshop/ │ ├── 0_introduction.md │ ├── 1_psql_basics.md │ ├── 2_shell_scripts.md │ ├── 3_query_planning.md │ ├── 4_query_optimization.md │ ├── 5_query_optimization_part_2.md │ ├── 6_macro_overview_part_1.md │ ├── 7_macro_overview_part_2.md │ ├── 8_active_record_multi-db_prep_part_1.md │ ├── 9_active_record_multi-db_roles.md │ └── README.md ├── lib/ │ ├── assets/ │ │ └── .keep │ ├── json_web_token.rb │ └── tasks/ │ ├── .keep │ ├── auto_generate_diagram.rake │ ├── benchmarks.rake │ ├── custom.rake │ ├── data_generators.rake │ ├── fake_data_generator.rake │ ├── migration_hooks.rake │ └── simulate_app_activity.rake ├── log/ │ └── .keep ├── postgresql/ │ ├── .pg_service.sample.conf │ ├── .pgpass.sample │ ├── .psqlrc.sample │ ├── README.md │ ├── pg_hba.sample.conf │ ├── pgbouncer.sample.ini │ ├── postgresql.sample.conf │ └── userlist.sample.txt ├── public/ │ ├── 404.html │ ├── 422.html │ ├── 500.html │ └── robots.txt └── test/ ├── application_system_test_case.rb ├── controllers/ │ ├── .keep │ ├── api/ │ │ ├── trip_requests_controller_test.rb │ │ └── trips_controller_test.rb │ └── authentication_controller_test.rb ├── fixtures/ │ ├── .keep │ ├── drivers.yml │ ├── files/ │ │ └── .keep │ ├── locations.yml │ ├── riders.yml │ ├── trip_requests.yml │ ├── trips.yml │ ├── vehicle_reservations.yml │ └── vehicles.yml ├── helpers/ │ └── .keep ├── mailers/ │ └── .keep ├── models/ │ ├── .keep │ ├── driver_test.rb │ ├── location_test.rb │ ├── rider_test.rb │ ├── trip_request_test.rb │ ├── trip_test.rb │ ├── user_test.rb │ ├── vehicle_reservation_test.rb │ └── vehicle_test.rb ├── services/ │ ├── book_reservation_test.rb │ ├── trip_creator_test.rb │ └── trip_search_test.rb ├── system/ │ └── .keep └── test_helper.rb ================================================ FILE CONTENTS ================================================ ================================================ FILE: .circleci/config.yml ================================================ # https://circleci.com/developer/orbs/orb/circleci/ruby version: 2.1 # https://circleci.com/developer/orbs/orb/circleci/ruby orbs: ruby: circleci/ruby@2.1.0 jobs: build: docker: - image: cimg/ruby:3.2.2 steps: - checkout - ruby/install-deps test: parallelism: 3 docker: - image: cimg/ruby:3.2.2 - image: cimg/postgres:16.0 environment: POSTGRES_USER: postgres POSTGRES_DB: rideshare_test POSTGRES_PASSWORD: postgres environment: BUNDLE_JOBS: "3" BUNDLE_RETRY: "3" PGHOST: 127.0.0.1 RAILS_ENV: test PGSLICE_URL: "postgres://postgres:postgres@localhost:5432/rideshare_test" steps: - checkout - ruby/install-deps - run: name: Wait for DB command: dockerize -wait tcp://localhost:5432 -timeout 1m - run: name: Test Database setup command: sh db/setup_test_database.sh - run: name: Database schema load command: bundle exec rails db:schema:load --trace - run: name: Partition conversion command: sh bin/partition_conversion.sh - run: name: run tests command: bin/rails test workflows: version: 2 build_and_test: jobs: - build - test: requires: - build ================================================ FILE: .erdconfig ================================================ attributes: - content - primary_keys - foreign_keys - inheritance - timestamps disconnected: false filename: erd filetype: pdf indirect: true inheritance: false markup: true notation: simple orientation: horizontal polymorphism: true sort: true warn: true title: Rideshare exclude: null only: null only_recursion_depth: null prepend_primary: false cluster: false splines: spline fonts: normal: "Arial" bold: "Arial Bold" italic: "Arial Italic" ================================================ FILE: .git-blame-ignore-revs ================================================ ================================================ FILE: .gitignore ================================================ # See https://help.github.com/articles/ignoring-files for more about ignoring files. # # If you find yourself ignoring temporary files generated by your text editor # or operating system, you probably want to add a global ignore instead: # git config --global core.excludesfile '~/.gitignore_global' # Ignore bundler config. /.bundle # Ignore the default SQLite database. /db/*.sqlite3 /db/*.sqlite3-journal # Ignore all logfiles and tempfiles. /log/* /tmp/* !/log/.keep !/tmp/.keep # Ignore uploaded files in development. /storage/* !/storage/.keep /public/assets .byebug_history # Ignore master key for decrypting credentials and more. /config/master.key /public/packs /public/packs-test /node_modules /yarn-error.log yarn-debug.log* .yarn-integrity .sql # Docker volume directory docker/postgres-docker/* postgres-docker/ # Ignore backup files docker/pg_hba.backup.conf docker/postgresql.backup.conf docker/pg_hba.conf docker/postgresql.conf docker/replication_user.sql docker/.pgpass output.log .pgpass ================================================ FILE: .rubocop.yml ================================================ AllCops: NewCops: enable Exclude: - "db/schema.rb" - "db/structure.sql" - "Gemfile" - "lib/tasks/*.rake" - "bin/*" - "config/puma.rb" - "config/spring.rb" - "config/environments/development.rb" - "config/environments/production.rb" - "spec/spec_helper.rb" Style/Documentation: Enabled: false # Disable suggestions for rubocop-rails for now AllCops: SuggestExtensions: false ================================================ FILE: .ruby-version ================================================ 3.2.2 ================================================ FILE: GUIDES.md ================================================ # Guides ## Set Up Databases ```sh sh db/setup.sh sh db/setup_test_database.sh ``` ## Set Database Connection Use the readwrite `owner` role for schema modifications. This is not a superuser role, although it does have write capabilities. The password is supplied from `~/.pgpass`. ```sh export DATABASE_URL="postgres://app:@localhost:5432/rideshare_development" ``` The `app` user cannot `TRUNCATE` tables. ## Teardown Databases This is mostly useful for testing the "setup" automation. You wouldn't want to do this normally because you'd lose your data. ```sh sh db/teardown.sh ``` ## Generate Data ```sh bin/rails db:reset bin/rails data_generators:generate_all ``` ## Simulate App Activity Start up the server in one terminal: ```sh bin/rails server ``` In another terminal, run the script: ```sh bin/rails simulate:app_activity ``` Or run it with a iteration count, for example 2 or more: ```sh bin/rails simulate:app_activity[2] ``` ## Local Circle CI Inspired by [Issue #99](https://github.com/andyatkinson/rideshare/issues/99) from @momer. Using this configuration, Circle CI can use its configuration and run locally. Currently there is this error: "invalid UTS mode" ```sh brew install circleci circleci local execute -c process.yml build # works circleci local execute -c process.yml test # error ``` ## Scrub Database ```sh cd db sh scrubbing/scrubber.sh ``` ## PgBouncer Prepared Statements * Configure `pool_mode` to be `statement` in the PgBouncer config file * Disable Query Logs (unfortunately) (`config/application.rb`) * Make sure Prepared Statements aren't disabled in `config/database.yml` * Connect through port 6432 and confirm prepared statements work correctly ================================================ FILE: Gemfile ================================================ source 'https://rubygems.org' gem 'activerecord-import', '~> 1.5' gem 'bcrypt', '~> 3.1' # Use ActiveModel has_secure_password gem 'fast_jsonapi', '~> 1.5' gem 'geocoder', '~> 1.8' gem 'jwt', '~> 2.7' gem 'pg', '~> 1.5' gem 'pg_query', '~> 6.1' gem 'pg_search', '~> 2.3' gem 'prosopite', '~> 1.4' # identify N+1 queries gem 'puma', '~> 6.4' gem 'rails', '>= 7.2', '~> 7.2' # , git: 'https://github.com/rails/rails.git' gem 'whenever', '~> 1.0', require: false # manage scheduled jobs gem 'fast_count', '~> 0.3' # assets gems default Rails 7 app gem 'importmap-rails', '~> 1.2' gem 'sprockets-rails', '~> 3.4' # Forks gem 'pghero', git: 'https://github.com/andyatkinson/pghero.git' gem 'pgslice', git: 'https://github.com/andyatkinson/pgslice.git' # Keep these updated gem 'fx', '~> 0.9' # manage DB functions, triggers gem 'scenic', '~> 1.9' # manage DB views, materialized views gem 'strong_migrations', '~> 2.4' # Use safe Migration patterns gem 'rubocop', '~> 1.77' group :development, :test do gem 'active_record_doctor', '~> 1.15' gem 'benchmark-ips', '~> 2.14' gem 'benchmark-memory', '~> 0.2' gem 'database_consistency', '~> 2.0' gem 'dotenv-rails', '~> 3.1' # Manage .env gem 'faker', '~> 3.5', require: false gem 'faraday', '~> 2.13' gem 'json', '~> 2.1' gem 'pry', '~> 0.15' gem 'rails_best_practices', '~> 1.23' gem 'rails-erd', '~> 1.7' gem 'rails-pg-extras', '~> 5.6' end ================================================ FILE: README.md ================================================ [![CircleCI](https://circleci.com/gh/andyatkinson/rideshare.svg?style=svg)](https://circleci.com/gh/andyatkinson/rideshare) # 📚 High Performance PostgreSQL for Rails Rideshare is the Rails application supporting the book "High Performance PostgreSQL for Rails" , published by Pragmatic Programmers in 2024. # Installation Prepare your development machine.
🎥 Installation - Rideshare on a Mac, Ruby, PostgreSQL, Gems
## Homebrew Packages First, install [Homebrew](https://brew.sh). ### Graphviz ```sh brew install graphviz ``` ## Ruby Version Manager Before installing Ruby, install a *Ruby version manager*. The recommended one is [Rbenv](https://github.com/rbenv/rbenv). Run: ```sh brew install rbenv ``` ## PostgreSQL PostgreSQL 16 or greater is required. Installation may be via Homebrew, although the recommended method is [Postgres.app](https://postgresapp.com) ### PostgresApp - Once installed, from the Menu Bar app, choose "Open Postgres" then click the "+" icon to create a new PostgreSQL 16 server ## Ruby Run `cat .ruby-version` from the Rideshare directory to find the needed version of Ruby. For example, if `3.2.2` is listed, run: ```sh rbenv install 3.2.2 ``` Run `rbenv versions` to confirm the correct version is active. The current version has an asterisk. ```sh system * 3.2.2 (set by /Users/andy/Projects/rideshare/.ruby-version) ``` Running into rbenv trouble? Review *Learn how to load rbenv in your shell* using [`rbenv init`](https://github.com/rbenv/rbenv). ## Bundler and Gems Bundler is included when you install Ruby using Rbenv. You're ready to install the Ruby gems for Rideshare. Run the following command from the Rideshare directory: ```sh bundle install ``` ## Rideshare Development Database ⚠️ This scripts expects PostgreSQL version 16. If you see syntax errors with underscore numbers like `10_000`, it's probably from using an older version that doesn't support that number style. ⚠️ Normally in Ruby on Rails applications, you'd run `bin/rails db:create` to create the development and test databases. Don't do that here. Rideshare uses a custom script. The script is called [`db/setup.sh`](db/setup.sh). Don't run it yet. The video below shows common issues for this section.
🎥 Rideshare DB setup. Common issues running db/setup.sh
Before you run it, let's set some environment variables. Open the file `db/setup.sh` and read the comments at the top for more info about these env vars: - `RIDESHARE_DB_PASSWORD` - `DB_URL` ⚠️ The script generates a password value using `openssl`, assuming it's installed and available. Once you've set values, before running the script, run `echo $RIDESHARE_DB_PASSWORD` (and `echo $DB_URL`) to make sure they're set. Once both are set, you're ready to run the script. Let's capture the output of the script. Use the command below to do that. The script output goes into `output.log` file so we can more easily review it for errors. ```sh sh db/setup.sh 2>&1 | tee -a output.log ``` Since you set `RIDESHARE_DB_PASSWORD` earlier, create or update the special `~/.pgpass` file with the password you generated. This allows us to put the PostgreSQL user in the connection string, without needing to also supply the password. Refer to `postgresql/.pgpass.sample` for an example, and copy the example into your own `~/.pgpass` file, replacing the password with your generated one. When you've updated `~/.pgpass`, it should look like the line below. The last segment (`2C6uw3LprgUMwSLQ` below) is the password you generated. ```sh localhost:5432:rideshare_development:owner:2C6uw3LprgUMwSLQ ``` Run `chmod 0600 ~/.pgpass` to change the file mode (permissions). Finally, run `export DATABASE_URL=`, getting the value from the `.env` file in this project, set as the value of the `DATABASE_URL` environment variable. Confirm that's a non-empty value by running `echo $DATABASE_URL`. Once `DATABASE_URL` is set, we'll use it as an argument to `psql` to connect to the database. Run `psql $DATABASE_URL` to do that. Once connected, you're good to go. If you'd like to do more checks, expand the checks and run through them below.
Installation Checks From within psql, run this: ```sql SELECT current_user; ``` Confirm user `owner` is displayed. ```sql owner@localhost:5432 rideshare_development# select current_user; current_user -------------- owner ``` From psql, run the *describe namespace* meta-command: ```sql \dn ``` Verify the `rideshare` schema is displayed. ```sql owner@localhost:5432 rideshare_development# \dn List of schemas Name | Owner -----------+------- rideshare | owner ``` Now that you've confirmed the `owner` user and the `rideshare` schema have been set up correctly, you can run the migrations to create Rideshare's tables.
## Run Migrations Run migrations the standard way: ```sh bin/rails db:migrate ``` Run the *describe table* meta command next: `\dt`. Rideshare tables like `users`, `trips` are listed. Note that migrations are preceded by the command `SET role = owner`, so they're run with `owner` as the owner of database objects. See `lib/tasks/migration_hooks.rake` for more details. If migrations ran successfully, you're good to go! ## Data Loads To load some sample data, check out: [db/README.md](db/README.md) # Development Guides and Documentation ## Troubleshooting The Rideshare repository has many `README.md` files within subdirectories. Run `find . -name 'README.md'` to see them all. - For expanded installation and troubleshooting, visit: [Development Guides](https://github.com/andyatkinson/development_guides) - For DB things: [db/README.md](db/README.md) - For database scripts: [db/scripts/README.md](db/scripts/README.md) - For PostgreSQL things: [postgresql/README.md](postgresql/README.md) - For Docker things: [docker/README.md](docker/README.md) - For DB scrubbing: [db/scrubbing/README.md](db/scrubbing/README.md) - For test environment details in Rideshare, check out: [TESTING.md](TESTING.md) - For Guides and Tasks in this repo, check out: [Guides](GUIDES.md) # User Interfaces Although Rideshare is an *API-only* app, there are some UI elements. Rideshare runs [PgHero](https://github.com/ankane/pghero) which has a UI. Connect to it: ```sh bin/rails server ``` Once that's running, visit in your browser to see it. ![Screenshot of PgHero for Rideshare](https://i.imgur.com/VduvxSK.png) ================================================ FILE: Rakefile ================================================ # Add your own tasks in files placed in lib/tasks ending in .rake, # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake. require_relative 'config/application' Rails.application.load_tasks Rake::Task['db:reset'].clear namespace :db do desc 'Custom database tasks' task :reset do Rake::Task['custom:db_reset'].invoke end end ================================================ FILE: TESTING.md ================================================ # Test Environment Installation In the development database, you'll use good practices like a custom schema and user, with reduced privileges. For the test database, we'll keep things simpler. The `postgres` superuser is used along with the `public` schema. This configuration is also used for Circle CI. From the Rideshare directory, run: 1. `sh db/setup_test_database.sh`, which sets up `rideshare_test` 1. `RAILS_ENV=test bin/rails db:migrate` 1. `bin/rails test` Refer to `.circleci/config.yml` for the Circle CI config. You should now have a test database, and tests should have passed. ================================================ FILE: app/assets/config/manifest.js ================================================ // app/assets/config/manifest.js //= link_tree ../images //= link_directory ../stylesheets .css //= link_tree ../../javascript .js ================================================ FILE: app/assets/images/.keep ================================================ ================================================ FILE: app/assets/stylesheets/application.css ================================================ /* * This is a manifest file that'll be compiled into application.css, which will include all the files * listed below. * * Any CSS and SCSS file within this directory, lib/assets/stylesheets, or any plugin's * vendor/assets/stylesheets directory can be referenced here using a relative path. * * You're free to add application-wide styles to this file and they'll appear at the bottom of the * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS * files in this directory. Styles in this file should be added after the last require_* statement. * It is generally better to create a new file per style scope. * *= require_tree . *= require_self */ ================================================ FILE: app/channels/application_cable/channel.rb ================================================ module ApplicationCable class Channel < ActionCable::Channel::Base end end ================================================ FILE: app/channels/application_cable/connection.rb ================================================ module ApplicationCable class Connection < ActionCable::Connection::Base end end ================================================ FILE: app/controllers/api/trip_requests_controller.rb ================================================ class Api::TripRequestsController < ApiController def create if start_location && end_location && current_rider trip_request = current_rider.trip_requests.create!( start_location: start_location, end_location: end_location ) TripCreator.new( trip_request_id: trip_request.id ).create_trip! render json: { trip_request_id: trip_request.id }, status: :created else render nothing: true, status: :unprocessable_entity end end def show if current_trip_request render json: { trip_request_id: current_trip_request.id, trip_id: created_trip&.id } else render nothing: true, status: :unprocessable_entity end end private def trip_request_params params .require(:trip_request) .permit(:rider_id, :start_address, :end_address) end def current_trip_request @trip_request ||= TripRequest.find(params[:id]) end def created_trip return unless Trip.exists?(trip_request_id: params[:id]) Trip.find_by(trip_request_id: params[:id]) end def current_rider @rider ||= Rider.find(trip_request_params[:rider_id]) end def start_location @start_location ||= Location.find_or_create_by( address: trip_request_params[:start_address] ) end def end_location @end_location ||= Location.find_or_create_by( address: trip_request_params[:end_address] ) end end ================================================ FILE: app/controllers/api/trips_controller.rb ================================================ class Api::TripsController < ApiController before_action :authorize_request, only: :my # Search params: `start_location` # => `New%20York%2C%20NY` def index search = TripSearch.new(search_params) trips = Trip.apply_scopes( search.start_location, search.driver_name, search.rider_name ) render json: trips end def show expires_in 1.minute, public: true @trip = Trip.find(params[:id]) return unless stale?(@trip) render json: @trip end # Get more details about a single trip # TODO add JSON API mime type def details options = {} # include=driver # fields[driver]=average_rating if params[:fields] driver_fields = params[:fields].permit(:driver).to_h .each_with_object({}) do |(k, v), h| h[k.to_sym] = v.split(',').map(&:to_sym) end options.merge!(fields: driver_fields) end # multiple associated resources are comma-separated options[:include] = params[:include].split(',').map(&:to_sym) if params[:include] @trip = Trip.includes(:driver).find_by(id: params[:id]) render json: TripSerializer.new(@trip, options).serializable_hash end # TODO: add JSON API mime type def my @trips = Trip.completed .includes(:driver, { trip_request: :rider }) .joins(trip_request: :rider) .where(users: { id: params[:rider_id] }) options = {} # JSON API: https://jsonapi.org/format/#fetching-sparse-fieldsets # fast_jsonapi: https://github.com/Netflix/fast_jsonapi#sparse-fieldsets # # convert input params to options arguments if params[:fields] trip_params = params[:fields].permit(:trips).to_h .each_with_object({}) do |(k, v), h| h[k.singularize.to_sym] = v.split(',').map(&:to_sym) end options.merge!(fields: trip_params) end render json: TripSerializer.new(@trips, options).serializable_hash end private def search_params params.permit( :start_location, :driver_name, :rider_name ) end end ================================================ FILE: app/controllers/api_controller.rb ================================================ class ApiController < ActionController::API def authorize_request header = request.headers['Authorization'] header = header.split(' ').last if header begin @decoded = JsonWebToken.decode(header) @current_user = User.find(@decoded[:user_id]) rescue ActiveRecord::RecordNotFound => e render json: { errors: e.message }, status: :unauthorized rescue JWT::DecodeError => e render json: { errors: e.message }, status: :unauthorized end end end ================================================ FILE: app/controllers/application_controller.rb ================================================ class ApplicationController < ActionController::Base end ================================================ FILE: app/controllers/authentication_controller.rb ================================================ class AuthenticationController < ApiController before_action :authorize_request, except: :login # POST /auth/login def login @user = User.find_by(email: login_params[:email]) if @user&.authenticate(login_params[:password]) token = JsonWebToken.encode(user_id: @user.id) time = Time.now + 24.hours.to_i render json: { token: token, exp: time.strftime('%m-%d-%Y %H:%M'), username: @user.display_name }, status: :ok else render json: { error: 'unauthorized' }, status: :unauthorized end end private def login_params params.permit(:email, :password) end end ================================================ FILE: app/controllers/concerns/.keep ================================================ ================================================ FILE: app/helpers/application_helper.rb ================================================ module ApplicationHelper end ================================================ FILE: app/javascript/application.js ================================================ // Configure your import map in config/importmap.rb. Read more: https://github.com/rails/importmap-rails ================================================ FILE: app/jobs/application_job.rb ================================================ class ApplicationJob < ActiveJob::Base # Automatically retry jobs that encountered a deadlock # retry_on ActiveRecord::Deadlocked # Most jobs are safe to ignore if the underlying records are no longer available # discard_on ActiveJob::DeserializationError end ================================================ FILE: app/lib/pgslice_helper.rb ================================================ # Safe by default, add dry_run=false when ready # Prep: # export PGSLICE_URL # - Retire default # - bin/rails runner "PgsliceHelper.new.retire_default_partition(table_name: 'trip_positions')" # - bin/rails runner "PgsliceHelper.new.add_partitions(table_name: 'trip_positions', past: 0, future: 3, dry_run: false)" # - bin/rails runner "PgsliceHelper.new.fill(table_name: 'trip_positions', from_date: '2021-01-01')" # - bin/rails runner "PgsliceHelper.new.analyze(table_name: 'trip_positions')" # # Data export (Safe by default, add dry_run=false when ready) # - bin/rails runner "PgsliceHelper.new.dump_retired_table(table_name: 'trip_positions')" # - bin/rails runner "PgsliceHelper.new.drop_retired_table(table_name: 'trip_positions')" # # To test app compatibility: # - Make sure latest changes from dev DB are applied: `bin/rails db:test:prepare` # - change PGSLICE_URL in .env, specify test DB # - run `bin/rails test` class PgsliceHelper DEFAULT_COLUMN = 'created_at' def add_partitions(table_name:, past:, future:, intermediate: true, dry_run: true) cmd = %(./bin/pgslice add_partitions #{table_name} \ #{'--intermediate ' if intermediate} \ #{"--past #{past}" if past} \ #{"--future #{future}" if future} \ #{'--dry-run' if dry_run} \ ).squish log("dry_run=#{dry_run} invoking: #{cmd}") system(cmd) end def fill(table_name:, from_date:, partition_column: DEFAULT_COLUMN, swapped: false) cmd = %(./bin/pgslice fill #{table_name} #{"--where \"date(#{partition_column}) >= date('#{from_date}')\"" if from_date} #{'--swapped' if swapped} ).squish log("fill cmd: #{cmd}") system(cmd) end def analyze(table_name:) cmd = %(./bin/pgslice analyze #{table_name}).squish log("cmd: #{cmd}") system(cmd) end def swap(table_name:) cmd = %(./bin/pgslice swap #{table_name}).squish log("cmd: #{cmd}") system(cmd) end def unswap(table_name:) cmd = %(./bin/pgslice unswap #{table_name}).squish log("cmd: #{cmd}") system(cmd) end # default partitions cannot be detached concurrently # "ERROR: cannot detach partitions concurrently when a default partition exists" def retire_default_partition(table_name:, dry_run: true) tbl_name = "#{table_name}_intermediate" # assumes intermediate table partition_name = "#{tbl_name}_default" retired_name = "#{partition_name}_retired" sql = %( BEGIN; ALTER TABLE #{tbl_name} \ DETACH PARTITION #{partition_name}; ALTER TABLE #{partition_name} RENAME TO #{retired_name}; COMMIT; ).squish cmd = %(psql $PGSLICE_URL -c '#{sql}') log("detaching and retiring dry_run=#{dry_run} cmd=#{cmd}") log("cmd=#{cmd}") system(cmd) unless dry_run end def unretire_default_partition(table_name:, dry_run: false) table_name = "#{table_name}_intermediate" # assumes intermediate table partition_name = "#{table_name}_default" retired_name = "#{partition_name}_retired" sql = %( BEGIN; ALTER TABLE #{retired_name} RENAME TO #{partition_name}; ALTER TABLE #{table_name} ATTACH PARTITION #{partition_name} DEFAULT; COMMIT; ).squish cmd = %(psql $PGSLICE_URL -c '#{sql}') log("unretiring and attaching. dry_run=#{dry_run}") log("cmd=#{cmd}") system(cmd) unless dry_run end def dump_retired_table(table_name:, dry_run: true) retired_name = "#{table_name}_retired" dump_name = "#{retired_name}.dump" cmd = %(pg_dump -c -Fc -t #{retired_name} $PGSLICE_URL > #{dump_name}) log("cmd=#{cmd}") system(cmd) unless dry_run end def drop_retired_table(table_name:, dry_run: true) retired_name = "#{table_name}_retired" cmd = %(psql -c 'DROP TABLE #{retired_name}' $PGSLICE_URL) log("cmd: #{cmd}") system(cmd) unless dry_run end private def log(line) Rails.logger.info "[pgslice] #{line}" end end ================================================ FILE: app/mailers/application_mailer.rb ================================================ class ApplicationMailer < ActionMailer::Base default from: 'from@example.com' layout 'mailer' end ================================================ FILE: app/models/application_record.rb ================================================ class ApplicationRecord < ActiveRecord::Base self.abstract_class = true # connects_to database: { # writing: :rideshare, # reading: :rideshare_replica # } end ================================================ FILE: app/models/concerns/.keep ================================================ ================================================ FILE: app/models/driver.rb ================================================ class Driver < User has_many :trips validates :drivers_license_number, presence: true, uniqueness: true, drivers_license: true # X out of 5, with 1 to 5 options selected by Riders def average_rating trips.average(:rating) end end ================================================ FILE: app/models/fast_search_result.rb ================================================ class FastSearchResult < ApplicationRecord # this isn't strictly necessary, but it will prevent # rails from calling save, which would fail anyway. def readonly? true end def self.refresh(concurrently: false) Scenic.database.refresh_materialized_view( table_name, concurrently: concurrently, cascade: false ) end end ================================================ FILE: app/models/location.rb ================================================ class Location < ApplicationRecord validates :address, presence: true, uniqueness: true # simple approach, assumes fully address, all parts validates :position, presence: true validates :state, presence: true, length: { is: 2 } geocoded_by :address after_validation :geocode, if: ->(obj) { obj.address_changed? && obj.position.nil? } end ================================================ FILE: app/models/rider.rb ================================================ class Rider < User has_many :trip_requests has_many :trips, through: :trip_requests end ================================================ FILE: app/models/search_result.rb ================================================ class SearchResult < ApplicationRecord # this isn't strictly necessary, but it will prevent # rails from calling save, which would fail anyway. def readonly? true end end ================================================ FILE: app/models/trip.rb ================================================ class Trip < ApplicationRecord belongs_to :trip_request belongs_to :driver, class_name: 'User', counter_cache: true has_many :trip_positions delegate :rider, to: :trip_request, allow_nil: false validates :trip_request, :rider, :driver, presence: true validates :rating, numericality: { only_integer: true, greater_than_or_equal_to: 1, less_than_or_equal_to: 5 }, allow_nil: true validate :rating_requires_completed_trip scope :with_start_location, lambda { |text| joins(trip_request: :start_location) .where('locations.address ILIKE ?', "%#{text}%") } scope :with_driver_name, lambda { |text| joins(:driver) .where('users.first_name ILIKE ?', "%#{text}%") } scope :with_rider_name, lambda { |text| joins(trip_request: :rider) .where('users.first_name ILIKE ?', "%#{text}%") } scope :completed, -> { where.not(completed_at: nil) } def rating_requires_completed_trip return unless rating_changed? && completed_at.nil? errors.add(:rating, 'must be completed before a rating can be added') end def self.apply_scopes(*filters) filters.inject(all) do |scope_chain, filter| scope_chain.merge(filter) end end end ================================================ FILE: app/models/trip_position.rb ================================================ class TripPosition < ApplicationRecord belongs_to :trip validates :trip_id, presence: true validates :position, presence: true end ================================================ FILE: app/models/trip_request.rb ================================================ class TripRequest < ApplicationRecord belongs_to :rider, class_name: 'User' belongs_to :start_location, class_name: 'Location' belongs_to :end_location, class_name: 'Location' has_one :trip has_many :vehicle_reservations # A unique trip request could be per driver, start and end location, that is # in progress. In other words, in order to avoid duplicated data, require that # only trip could be in progress for a rider between the same locations. validates :rider, :start_location, :end_location, presence: true end ================================================ FILE: app/models/user.rb ================================================ class User < ApplicationRecord has_secure_password validates :first_name, :last_name, presence: true validates :drivers_license_number, length: { maximum: 100 } include PgSearch::Model # searchable_full_name column combines # first_name and last_name # Each receives a weight, in the stored generated column # definition pg_search_scope :search_by_full_name, against: { first_name: 'A', # highest weight last_name: 'B' } # Swap the config above for the one on the next line, # after adding the column `searchable_full_name` # against: :searchable_full_name, # stored generated column tsvector # using: { # tsearch: { # dictionary: 'english', # tsvector_column: 'searchable_full_name' # } # } pg_search_scope :unaccent_search, against: %i[first_name last_name], ignoring: :accents validates :email, presence: true, uniqueness: true, email: true # custom validator validates :password, length: { minimum: 6 }, confirmation: true, # automatically added by has_secure_password, prob. redundant if: -> { new_record? || !password.nil? } validates :type, presence: true # NOTE: on password confirmation: # Validation only called when password_confirmation attribute is present def display_name "#{first_name.capitalize} #{last_name[0].capitalize}." end end ================================================ FILE: app/models/vehicle.rb ================================================ class Vehicle < ApplicationRecord validates :name, presence: true, uniqueness: true attr_accessor :status has_many :vehicle_reservations, dependent: :destroy enum :status, { draft: VehicleStatus::DRAFT, published: VehicleStatus::PUBLISHED }, prefix: true validates :status, inclusion: { in: VehicleStatus::VALID_STATUSES }, presence: true end ================================================ FILE: app/models/vehicle_reservation.rb ================================================ class VehicleReservation < ApplicationRecord belongs_to :vehicle belongs_to :trip_request validates :vehicle_id, :starts_at, :ends_at, presence: true end ================================================ FILE: app/models/vehicle_status.rb ================================================ class VehicleStatus DRAFT = 'draft'.freeze PUBLISHED = 'published'.freeze VALID_STATUSES = [ DRAFT, PUBLISHED ] end ================================================ FILE: app/queries/top_drivers.sql ================================================ -- With new drivers WITH new_drivers AS ( SELECT * FROM users WHERE created_at >= (NOW() - INTERVAL '30 days') -- And top rated trips ), top_rated_trips AS ( SELECT id, driver_id FROM trips WHERE rating IS NOT NULL ) -- display their name and average rating SELECT trips.driver_id, CONCAT(users.first_name, ' ', users.last_name) AS driver_name, ROUND(AVG(trips.rating), 2) as avg_rating FROM trips JOIN users ON trips.driver_id = users.id WHERE users.type = 'Driver' AND users.id IN (select id from new_drivers) AND trips.id IN (select id from top_rated_trips) GROUP by 1, 2 ORDER BY 3 DESC LIMIT 10; ================================================ FILE: app/serializers/driver_serializer.rb ================================================ class DriverSerializer include FastJsonapi::ObjectSerializer attribute :display_name attribute :average_rating do |driver| driver.average_rating.round(2) end end ================================================ FILE: app/serializers/trip_serializer.rb ================================================ class TripSerializer include FastJsonapi::ObjectSerializer attribute :rider_name do |trip| trip.rider.display_name end attribute :driver_name do |trip| trip.driver.display_name end belongs_to :driver end ================================================ FILE: app/services/book_reservation.rb ================================================ class BookReservation def initialize(vehicle_id:, rider_id:, start_location_id:, end_location_id:, starts_at:, ends_at:) @vehicle = Vehicle.find(vehicle_id) @rider = Rider.find(rider_id) @start_location = Location.find(start_location_id) @end_location = Location.find(end_location_id) @starts_at = starts_at @ends_at = ends_at end def reserve! ActiveRecord::Base.transaction do trip_request = TripRequest.create!( rider: @rider, start_location: @start_location, end_location: @end_location ) trip_request.vehicle_reservations.create!( vehicle: @vehicle, starts_at: @starts_at, ends_at: @ends_at ) end end end ================================================ FILE: app/services/trip_creator.rb ================================================ class TripCreator class TripCreationFailure < StandardError; end attr_reader :trip_request_id def initialize(trip_request_id:) @trip_request_id = trip_request_id end def create_trip! trip = Trip.new( trip_request_id: trip_request.id, driver: best_available_driver ) raise TripCreationFailure unless trip.valid? trip.save! end private # NOTE: this would be a place to add intelligence # to the selection process: # available? completing a trip nearby? other business # criteria like tenure, driver score etc. def best_available_driver Driver.all.sample end def trip_request @trip_request ||= TripRequest.find(trip_request_id) end end ================================================ FILE: app/services/trip_search.rb ================================================ class TripSearch attr_reader :params def initialize(params) @params = params end def start_location if text = params[:start_location] Trip.with_start_location(sanitize(text)) else Trip.all end end def driver_name if text = params[:driver_name] Trip.with_driver_name(sanitize(text)) else Trip.all end end def rider_name if text = params[:rider_name] Trip.with_rider_name(sanitize(text)) else Trip.all end end private def sanitize(text) CGI.unescape(text.to_s) end end ================================================ FILE: app/validators/drivers_license_validator.rb ================================================ class DriversLicenseValidator < ActiveModel::EachValidator # https://success.myshn.net/Data_Protection/Data_Identifiers/U.S._Driver%27s_License_Numbers # valid example: P800000224322 DL_MN_REGEXP_FORMAT = /[a-zA-Z]\d{12}/i DEFAULT_MESSAGE = "is not a valid driver's license number" def validate_each(record, attribute, value) return if value =~ DL_MN_REGEXP_FORMAT record.errors.add( attribute, options[:message] || DEFAULT_MESSAGE ) end end ================================================ FILE: app/validators/email_validator.rb ================================================ # https://guides.rubyonrails.org/active_record_validations.html#custom-validators class EmailValidator < ActiveModel::EachValidator EMAIL_REGEXP_FORMAT = /\A([^@\s]+)@((?:[-a-z0-9]+\.)+[a-z]{2,})\z/i def validate_each(record, attribute, value) return if value =~ EMAIL_REGEXP_FORMAT record.errors.add(attribute, options[:message] || 'is not an email') end end ================================================ FILE: bin/bundle ================================================ #!/usr/bin/env ruby # frozen_string_literal: true # # This file was generated by Bundler. # # The application 'bundle' is installed as part of a gem, and # this file is here to facilitate running it. # require 'rubygems' m = Module.new do module_function def invoked_as_script? File.expand_path($0) == File.expand_path(__FILE__) end def env_var_version ENV['BUNDLER_VERSION'] end def cli_arg_version return unless invoked_as_script? # don't want to hijack other binstubs return unless 'update'.start_with?(ARGV.first || ' ') # must be running `bundle update` bundler_version = nil update_index = nil ARGV.each_with_index do |a, i| bundler_version = a if update_index && update_index.succ == i && a =~ Gem::Version::ANCHORED_VERSION_PATTERN next unless a =~ /\A--bundler(?:[= ](#{Gem::Version::VERSION_PATTERN}))?\z/ bundler_version = Regexp.last_match(1) || '>= 0.a' update_index = i end bundler_version end def gemfile gemfile = ENV['BUNDLE_GEMFILE'] return gemfile if gemfile && !gemfile.empty? File.expand_path('../Gemfile', __dir__) end def lockfile lockfile = case File.basename(gemfile) when 'gems.rb' then gemfile.sub(/\.rb$/, gemfile) else "#{gemfile}.lock" end File.expand_path(lockfile) end def lockfile_version return unless File.file?(lockfile) lockfile_contents = File.read(lockfile) return unless lockfile_contents =~ /\n\nBUNDLED WITH\n\s{2,}(#{Gem::Version::VERSION_PATTERN})\n/ Regexp.last_match(1) end def bundler_version @bundler_version ||= env_var_version || cli_arg_version || lockfile_version || "#{Gem::Requirement.default}.a" end def load_bundler! ENV['BUNDLE_GEMFILE'] ||= gemfile # must dup string for RG < 1.8 compatibility activate_bundler(bundler_version.dup) end def activate_bundler(bundler_version) if Gem::Version.correct?(bundler_version) && Gem::Version.new(bundler_version).release < Gem::Version.new('2.0') bundler_version = '< 2' end gem_error = activation_error_handling do gem 'bundler', bundler_version end return if gem_error.nil? require_error = activation_error_handling do require 'bundler/version' end if require_error.nil? && Gem::Requirement.new(bundler_version).satisfied_by?(Gem::Version.new(Bundler::VERSION)) return end warn "Activating bundler (#{bundler_version}) failed:\n#{gem_error.message}\n\nTo install the version of bundler this project requires, run `gem install bundler -v '#{bundler_version}'`" exit 42 end def activation_error_handling yield nil rescue StandardError, LoadError => e e end end m.load_bundler! load Gem.bin_path('bundler', 'bundle') if m.invoked_as_script? ================================================ FILE: bin/importmap ================================================ #!/usr/bin/env ruby require_relative '../config/application' require 'importmap/commands' ================================================ FILE: bin/partition_conversion.sh ================================================ #!/bin/bash echo "A script for the test DB" bin/rails db:test:prepare echo "Reminder: Set PGSLICE_URL to test DB in .env" echo "Value is:" echo $PGSLICE_URL bin/rails runner "PgsliceHelper.new.retire_default_partition(table_name: 'trip_positions', dry_run: false)" bin/rails runner "PgsliceHelper.new.add_partitions(table_name: 'trip_positions', past: 0, future: 3, dry_run: false)" bin/rails runner "PgsliceHelper.new.fill(table_name: 'trip_positions', from_date: '2023-03-01')" bin/rails runner "PgsliceHelper.new.analyze(table_name: 'trip_positions')" bin/rails runner "PgsliceHelper.new.swap(table_name: 'trip_positions')" ================================================ FILE: bin/pgslice ================================================ #!/usr/bin/env ruby # frozen_string_literal: true # # This file was generated by Bundler. # # The application 'pgslice' is installed as part of a gem, and # this file is here to facilitate running it. # require 'pathname' ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', Pathname.new(__FILE__).realpath) bundle_binstub = File.expand_path('bundle', __dir__) if File.file?(bundle_binstub) if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/ load(bundle_binstub) else abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") end end require 'rubygems' require 'bundler/setup' load Gem.bin_path('pgslice', 'pgslice') ================================================ FILE: bin/rails ================================================ #!/usr/bin/env ruby APP_PATH = File.expand_path('../config/application', __dir__) require_relative '../config/boot' require 'rails/commands' ================================================ FILE: bin/rails_best_practices ================================================ #!/usr/bin/env ruby # frozen_string_literal: true # # This file was generated by Bundler. # # The application 'rails_best_practices' is installed as part of a gem, and # this file is here to facilitate running it. # require 'pathname' ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', Pathname.new(__FILE__).realpath) bundle_binstub = File.expand_path('bundle', __dir__) if File.file?(bundle_binstub) if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/ load(bundle_binstub) else abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") end end require 'rubygems' require 'bundler/setup' load Gem.bin_path('rails_best_practices', 'rails_best_practices') ================================================ FILE: bin/rake ================================================ #!/usr/bin/env ruby require_relative '../config/boot' require 'rake' Rake.application.run ================================================ FILE: bin/setup ================================================ #!/usr/bin/env ruby require 'fileutils' # path to your application root. APP_ROOT = File.expand_path('..', __dir__) def system!(*args) system(*args) || abort("\n== Command #{args} failed ==") end FileUtils.chdir APP_ROOT do # This script is a way to setup or update your development environment automatically. # This script is idempotent, so that you can run it at anytime and get an expectable outcome. # Add necessary setup steps to this file. puts '== Installing dependencies ==' system! 'gem install bundler --conservative' system('bundle check') || system!('bundle install') # Install JavaScript dependencies # system('bin/yarn') # puts "\n== Copying sample files ==" # unless File.exist?('config/database.yml') # FileUtils.cp 'config/database.yml.sample', 'config/database.yml' # end puts "\n== Preparing database ==" system! 'bin/rails db:prepare' puts "\n== Removing old logs and tempfiles ==" system! 'bin/rails log:clear tmp:clear' puts "\n== Restarting application server ==" system! 'bin/rails restart' end ================================================ FILE: config/application.rb ================================================ require_relative 'boot' # https://andycroll.com/ruby/turn-off-the-bits-of-rails-you-dont-use/ # require 'rails/all' require 'rails' # Pick the frameworks you want: require 'active_model/railtie' require 'active_job/railtie' require 'active_record/railtie' # # require "active_storage/engine" require 'action_controller/railtie' require 'action_mailer/railtie' # # require "action_mailbox/engine" # # require "action_text/engine" require 'action_view/railtie' require 'action_cable/engine' require 'sprockets/railtie' require 'rails/test_unit/railtie' # Require the gems listed in Gemfile, including any gems # you've limited to :test, :development, or :production. Bundler.require(*Rails.groups) module Rideshare class Application < Rails::Application # Initialize configuration defaults for originally generated Rails version. config.load_defaults 7.1 # Settings in config/environments/* take precedence over those specified here. # Application configuration can go into files in config/initializers # -- all .rb files in that directory are automatically loaded after loading # the framework and any gems in your application. # https://blog.bigbinary.com/2016/08/29/rails-5-disables-autoloading-after-booting-the-app-in-production.html config.eager_load_paths << Rails.root.join('app/services') config.eager_load_paths << Rails.root.join('lib') # Use structure.sql # https://edgeguides.rubyonrails.org/configuring.html#config-active-record-schema-format config.active_record.schema_format = :sql # set a timezone. Times are generally stored as # timestamps without a time zone. This application # would need to treat times based on the user's timezone. config.time_zone = 'Central Time (US & Canada)' # Enable Query Logging # NOTE: Disable in order to use Prepared Statements # config.active_record.query_log_tags_enabled = true # https://www.bigbinary.com/blog/rails-7-adds-setting-for-enumerating-columns-in-select-statements# # config.active_record.enumerate_columns_in_select_statements = true # Add '--if-exists' flag to pg_dump # https://github.com/rails/rails/issues/38695#issuecomment-763588402 ActiveRecord::Tasks::DatabaseTasks.structure_dump_flags = ['--clean', '--if-exists'] # Consider limiting the conversion of timestamp without time zone columns to UTC # https://engineering.ezcater.com/youre-not-in-the-zone # ActiveRecord::Base.time_zone_aware_types = [:datetime] # Consider timestamps in the local time zone # This is because the app used "timestamp without time zone" columns and times are # stored in the local timezone (CST). config.active_record.default_timezone = :local end end ================================================ FILE: config/boot.rb ================================================ ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__) require 'bundler/setup' # Set up gems listed in the Gemfile. ================================================ FILE: config/cable.yml ================================================ development: adapter: async test: adapter: test production: adapter: redis url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %> channel_prefix: rideshare_production ================================================ FILE: config/credentials.yml.enc ================================================ itjGwmz6U75xCi1uE8pPIYsLQH/TmelhEw1qDOxAfjyT+F7kKtHQ9kFBFmfmqVu9kcVPLg1ajw7ejk79XodWo+193YdLwpRvj4On5KgPCOfXrGxJleasmqP2lU+Hfwv93CSSGipFFWlwB6kjtvCsqYycnxERqh1PKyoXQUcb9Niely5We+et32LQo7Rb6rkEYKPWcTZp9YNIMLKtNMSObXsJoUVmpafIhtqJ2UC6zpz6RW+7VVpTGoQz++Dc+itByNX0KRSi1/BwKKPfwSqlc2uYtWUQ6VDZWS1lS1lSeSip0YyZKmYgXlUDmZoYBOWaM/PRor7gN9oMKr/J2C+YeNM93kAwUh21RtSJ36q0V7qtjaFsMN7GqQfcf9pwzq2VreM3gSrHVYwhsbwcT/O6vhzhex/KUalNyzWG--eImVIKRRjaAVTrI/--MwJavKULW+yt/C67osImsg== ================================================ FILE: config/database-multiple.sample.yml ================================================ default: &default adapter: postgresql pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> variables: statement_timeout: 5000 development: rideshare: <<: *default database: rideshare_development url: <%= ENV['DATABASE_URL_PRIMARY'] %> schema_search_path: rideshare rideshare_replica: <<: *default database: rideshare_development url: <%= ENV['DATABASE_URL_REPLICA'] %> schema_search_path: rideshare replica: true database_tasks: true #default:true, false=physical https://guides.rubyonrails.org/active_record_multiple_databases.html#connecting-to-databases-without-managing-schema-and-migrations test: <<: *default url: postgresql://postgres:@localhost/rideshare_test ================================================ FILE: config/database-slow-clients.sample.yml ================================================ # # Configuring Active Record: # # # Database Connection Control Functions # # default: &default adapter: postgresql schema_search_path: rideshare prepared_statements: true # enabled by default advisory_locks: true # enabled by default # Optional (PostgreSQL): # checkout_timeout, read_timeout test: <<: *default pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> url: postgresql://postgres:@localhost/rideshare_test development: <<: *default pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> url: <%= ENV['DATABASE_URL'] %> database: rideshare_development variables: # https://www.postgresql.org/docs/current/runtime-config-client.html statement_timeout: 5000 # seconds, set at client level idle_in_transaction_session_timeout: 300000 # milliseconds # PostgreSQL params: # idle_timeout # lock_timeout # idle_session_timeout # class SlowClientModel < ApplicationRecord # self.establish_connection :slow_clients # end # # Put "allowed" slow code in SlowClientModel # or a class that inherits from it. Slow clients: # # - Use fewer, limited (up to) database connections # - Queries are permitted a higher statement_timeout # slow_clients: <<: *default pool: <%= 2 %> url: <%= ENV['DATABASE_URL'] %> database: rideshare_development variables: # https://www.postgresql.org/docs/current/runtime-config-client.html statement_timeout: 60000 # seconds, set at client level idle_in_transaction_session_timeout: 300000 # milliseconds ================================================ FILE: config/database.yml ================================================ # # Configuring Active Record: # # # Database Connection Control Functions # # default: &default adapter: postgresql pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> schema_search_path: rideshare prepared_statements: true # enabled by default advisory_locks: true # enabled by default # Optional (PostgreSQL): # checkout_timeout, read_timeout test: <<: *default url: postgresql://postgres:@localhost/rideshare_test development: <<: *default url: <%= ENV['DATABASE_URL'] %> database: rideshare_development variables: # https://www.postgresql.org/docs/current/runtime-config-client.html statement_timeout: 20000 # (in seconds, consider lowering to 5s for OLTP) idle_in_transaction_session_timeout: 300000 # milliseconds # Consider setting all these params: # idle_timeout # lock_timeout # idle_session_timeout ================================================ FILE: config/environment.rb ================================================ # Load the Rails application. require_relative 'application' # Initialize the Rails application. Rails.application.initialize! ================================================ FILE: config/environments/development.rb ================================================ Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. # In the development environment your application's code is reloaded on # every request. This slows down response time but is perfect for development # since you don't have to restart the web server when you make code changes. config.cache_classes = false # Do not eager load code on boot. config.eager_load = false # Show full error reports. config.consider_all_requests_local = true # Enable/disable caching. By default caching is disabled. # Run rails dev:cache to toggle caching. if Rails.root.join('tmp', 'caching-dev.txt').exist? config.action_controller.perform_caching = true config.action_controller.enable_fragment_cache_logging = true config.cache_store = :memory_store config.public_file_server.headers = { 'Cache-Control' => "public, max-age=#{2.days.to_i}" } else config.action_controller.perform_caching = false config.cache_store = :null_store end # Store uploaded files on the local file system (see config/storage.yml for options). # config.active_storage.service = :local # Don't care if the mailer can't send. config.action_mailer.raise_delivery_errors = false config.action_mailer.perform_caching = false # Print deprecation notices to the Rails logger. config.active_support.deprecation = :log # Raise an error on page load if there are pending migrations. config.active_record.migration_error = :page_load # Highlight code that triggered database queries in logs. config.active_record.verbose_query_logs = true # Debug mode disables concatenation and preprocessing of assets. # This option may cause significant delays in view rendering with a large # number of complex assets. # config.assets.debug = true # Suppress logger output for asset requests. # config.assets.quiet = true # Raises error for missing translations. # config.action_view.raise_on_missing_translations = true # https://github.com/rails/sprockets-rails/issues/376#issuecomment-287560399 logger = ActiveSupport::Logger.new(STDOUT) logger.formatter = config.logger config.logger = ActiveSupport::TaggedLogging.new(logger) # config.active_record.database_selector = { delay: 2.seconds } # config.active_record.database_resolver = ActiveRecord::Middleware::DatabaseSelector::Resolver # config.active_record.database_resolver_context = ActiveRecord::Middleware::DatabaseSelector::Resolver::Session end ================================================ FILE: config/environments/production.rb ================================================ Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. # Code is not reloaded between requests. config.cache_classes = true # Eager load code on boot. This eager loads most of Rails and # your application in memory, allowing both threaded web servers # and those relying on copy on write to perform better. # Rake tasks automatically ignore this option for performance. config.eager_load = true # Full error reports are disabled and caching is turned on. config.consider_all_requests_local = false config.action_controller.perform_caching = true # Ensures that a master key has been made available in either ENV["RAILS_MASTER_KEY"] # or in config/master.key. This key is used to decrypt credentials (and other encrypted files). # config.require_master_key = true # Disable serving static files from the `/public` folder by default since # Apache or NGINX already handles this. config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present? # Compress CSS using a preprocessor. # config.assets.css_compressor = :sass # Do not fallback to assets pipeline if a precompiled asset is missed. config.assets.compile = false # Enable serving of images, stylesheets, and JavaScripts from an asset server. # config.action_controller.asset_host = 'http://assets.example.com' # Specifies the header that your server uses for sending files. # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX # Store uploaded files on the local file system (see config/storage.yml for options). # config.active_storage.service = :local # Mount Action Cable outside main process or domain. # config.action_cable.mount_path = nil # config.action_cable.url = 'wss://example.com/cable' # config.action_cable.allowed_request_origins = [ 'http://example.com', /http:\/\/example.*/ ] # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. # config.force_ssl = true # Use the lowest log level to ensure availability of diagnostic information # when problems arise. config.log_level = :debug # Prepend all log lines with the following tags. config.log_tags = [:request_id] # Use a different cache store in production. # config.cache_store = :mem_cache_store # Use a real queuing backend for Active Job (and separate queues per environment). # config.active_job.queue_adapter = :resque # config.active_job.queue_name_prefix = "rideshare_production" config.action_mailer.perform_caching = false # Ignore bad email addresses and do not raise email delivery errors. # Set this to true and configure the email server for immediate delivery to raise delivery errors. # config.action_mailer.raise_delivery_errors = false # Enable locale fallbacks for I18n (makes lookups for any locale fall back to # the I18n.default_locale when a translation cannot be found). config.i18n.fallbacks = true # Send deprecation notices to registered listeners. config.active_support.deprecation = :notify # Use default logging formatter so that PID and timestamp are not suppressed. config.log_formatter = ::Logger::Formatter.new # Use a different logger for distributed setups. # require 'syslog/logger' # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name') if ENV['RAILS_LOG_TO_STDOUT'].present? logger = ActiveSupport::Logger.new(STDOUT) logger.formatter = config.log_formatter config.logger = ActiveSupport::TaggedLogging.new(logger) end # Do not dump schema after migrations. config.active_record.dump_schema_after_migration = false # Inserts middleware to perform automatic connection switching. # The `database_selector` hash is used to pass options to the DatabaseSelector # middleware. The `delay` is used to determine how long to wait after a write # to send a subsequent read to the primary. # # The `database_resolver` class is used by the middleware to determine which # database is appropriate to use based on the time delay. # # The `database_resolver_context` class is used by the middleware to set # timestamps for the last write to the primary. The resolver uses the context # class timestamps to determine how long to wait before reading from the # replica. # # By default Rails will store a last write timestamp in the session. The # DatabaseSelector middleware is designed as such you can define your own # strategy for connection switching and pass that into the middleware through # these configuration options. # config.active_record.database_selector = { delay: 2.seconds } # config.active_record.database_resolver = ActiveRecord::Middleware::DatabaseSelector::Resolver # config.active_record.database_resolver_context = ActiveRecord::Middleware::DatabaseSelector::Resolver::Session end ================================================ FILE: config/environments/test.rb ================================================ # The test environment is used exclusively to run your application's # test suite. You never need to work with it otherwise. Remember that # your test database is "scratch space" for the test suite and is wiped # and recreated between test runs. Don't rely on the data there! Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. config.cache_classes = false # Do not eager load code on boot. This avoids loading your whole application # just for the purpose of running a single test. If you are using a tool that # preloads Rails for running tests, you may have to set it to true. config.eager_load = false # Configure public file server for tests with Cache-Control for performance. config.public_file_server.enabled = true config.public_file_server.headers = { 'Cache-Control' => "public, max-age=#{1.hour.to_i}" } # Show full error reports and disable caching. config.consider_all_requests_local = true config.action_controller.perform_caching = false config.cache_store = :null_store # Raise exceptions instead of rendering exception templates. config.action_dispatch.show_exceptions = false # Disable request forgery protection in test environment. config.action_controller.allow_forgery_protection = false # Store uploaded files on the local file system in a temporary directory. # config.active_storage.service = :test config.action_mailer.perform_caching = false # Tell Action Mailer not to deliver emails to the real world. # The :test delivery method accumulates sent emails in the # ActionMailer::Base.deliveries array. config.action_mailer.delivery_method = :test # Print deprecation notices to the stderr. config.active_support.deprecation = :stderr # Raises error for missing translations. # config.action_view.raise_on_missing_translations = true # # # NOTE: For the test database, we don't want to dump after migrating, # especially since the test database is using UNLOGGED tables # which will modify the content of db/structure.sql, adding that # keyword to the dump output config.active_record.dump_schema_after_migration = false end # Rails Guides: # https://guides.rubyonrails.org/configuring.html\ # activerecord-connectionadapters-postgresqladapter-create-unlogged-tables ActiveSupport.on_load(:active_record_postgresqladapter) do self.create_unlogged_tables = true end ================================================ FILE: config/importmap.rb ================================================ # Pin npm packages by running ./bin/importmap pin 'application', preload: true ================================================ FILE: config/initializers/application_controller_renderer.rb ================================================ # Be sure to restart your server when you modify this file. # ActiveSupport::Reloader.to_prepare do # ApplicationController.renderer.defaults.merge!( # http_host: 'example.org', # https: false # ) # end ================================================ FILE: config/initializers/assets.rb ================================================ # Be sure to restart your server when you modify this file. # Version of your assets, change this if you want to expire all your assets. Rails.application.config.assets.version = '1.0' # Add additional assets to the asset load path. # Rails.application.config.assets.paths << Emoji.images_path # Add Yarn node_modules folder to the asset load path. # Rails.application.config.assets.paths << Rails.root.join('node_modules') # Precompile additional assets. # application.js, application.css, and all non-JS/CSS in the app/assets # folder are already added. # Rails.application.config.assets.precompile += %w( admin.js admin.css ) ================================================ FILE: config/initializers/backtrace_silencers.rb ================================================ # Be sure to restart your server when you modify this file. # You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces. # Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ } # You can also remove all the silencers if you're trying to debug a problem that might stem from framework code. # Rails.backtrace_cleaner.remove_silencers! ================================================ FILE: config/initializers/cookies_serializer.rb ================================================ # Be sure to restart your server when you modify this file. # Specify a serializer for the signed and encrypted cookie jars. # Valid options are :json, :marshal, and :hybrid. Rails.application.config.action_dispatch.cookies_serializer = :json ================================================ FILE: config/initializers/filter_parameter_logging.rb ================================================ # Be sure to restart your server when you modify this file. # Configure sensitive parameters which will be filtered from the log file. Rails.application.config.filter_parameters += [:password] ================================================ FILE: config/initializers/geocoder.rb ================================================ Geocoder.configure # Geocoding options # timeout: 3, # geocoding service timeout (secs) # lookup: :nominatim, # name of geocoding service (symbol) # ip_lookup: :ipinfo_io, # name of IP address geocoding service (symbol) # language: :en, # ISO-639 language code # use_https: false, # use HTTPS for lookup requests? (if supported) # http_proxy: nil, # HTTP proxy server (user:pass@host:port) # https_proxy: nil, # HTTPS proxy server (user:pass@host:port) # api_key: nil, # API key for geocoding service # cache: nil, # cache object (must respond to #[], #[]=, and #del) # cache_prefix: 'geocoder:', # prefix (string) to use for all cache keys # Exceptions that should not be rescued by default # (if you want to implement custom error handling); # supports SocketError and Timeout::Error # always_raise: [], # Calculation options # units: :mi, # :km for kilometers or :mi for miles # distances: :linear # :spherical or :linear ================================================ FILE: config/initializers/inflections.rb ================================================ # Be sure to restart your server when you modify this file. # Add new inflection rules using the following format. Inflections # are locale specific, and you may define rules for as many different # locales as you wish. All of these examples are active by default: # ActiveSupport::Inflector.inflections(:en) do |inflect| # inflect.plural /^(ox)$/i, '\1en' # inflect.singular /^(ox)en/i, '\1' # inflect.irregular 'person', 'people' # inflect.uncountable %w( fish sheep ) # end # These inflection rules are supported but not enabled by default: # ActiveSupport::Inflector.inflections(:en) do |inflect| # inflect.acronym 'RESTful' # end ================================================ FILE: config/initializers/mime_types.rb ================================================ # Be sure to restart your server when you modify this file. # Add new mime types for use in respond_to blocks: # Mime::Type.register "text/richtext", :rtf ================================================ FILE: config/initializers/slow_query_subscriber.rb ================================================ # Inspiration: https://twitter.com/kukicola/status/1578842934849724416 class SlowQuerySubscriber < ActiveSupport::Subscriber SECONDS_THRESHOLD = 1.0 ActiveSupport::Notifications.subscribe('sql.active_record') do |name, start, finish, _unique_id, data| duration = finish - start if duration > SECONDS_THRESHOLD sql = data[:sql] Rails.logger.info "[#{name}] #{duration} #{sql}" end end end ================================================ FILE: config/initializers/strong_migrations.rb ================================================ # Strong Migrations initializer StrongMigrations.lock_timeout = 10.seconds StrongMigrations.statement_timeout = 1.hour ================================================ FILE: config/initializers/wrap_parameters.rb ================================================ # Be sure to restart your server when you modify this file. # This file contains settings for ActionController::ParamsWrapper which # is enabled by default. # Enable parameter wrapping for JSON. You can disable this by setting :format to an empty array. ActiveSupport.on_load(:action_controller) do wrap_parameters format: [:json] end # To enable root element in JSON for ActiveRecord objects. # ActiveSupport.on_load(:active_record) do # self.include_root_in_json = true # end ================================================ FILE: config/locales/en.yml ================================================ # Files in the config/locales directory are used for internationalization # and are automatically loaded by Rails. If you want to use locales other # than English, add the necessary files in this directory. # # To use the locales, use `I18n.t`: # # I18n.t 'hello' # # In views, this is aliased to just `t`: # # <%= t('hello') %> # # To use a different locale, set it with `I18n.locale`: # # I18n.locale = :es # # This would use the information in config/locales/es.yml. # # The following keys must be escaped otherwise they will not be retrieved by # the default I18n backend: # # true, false, on, off, yes, no # # Instead, surround them with single quotes. # # en: # 'true': 'foo' # # To learn more, please read the Rails Internationalization guide # available at https://guides.rubyonrails.org/i18n.html. en: hello: "Hello world" ================================================ FILE: config/puma.rb ================================================ # Puma can serve each request in a thread from an internal thread pool. # The `threads` method setting takes two numbers: a minimum and maximum. # Any libraries that use thread pools should be configured to match # the maximum value specified for Puma. Default is set to 5 threads for minimum # and maximum; this matches the default thread size of Active Record. # max_threads_count = ENV.fetch('RAILS_MAX_THREADS') { 5 } min_threads_count = ENV.fetch('RAILS_MIN_THREADS') { max_threads_count } threads min_threads_count, max_threads_count # Specifies the `port` that Puma will listen on to receive requests; default is 3000. # port ENV.fetch('PORT') { 3000 } # Specifies the `environment` that Puma will run in. # environment ENV.fetch('RAILS_ENV') { 'development' } # Specifies the `pidfile` that Puma will use. pidfile ENV.fetch('PIDFILE') { 'tmp/pids/server.pid' } # Specifies the number of `workers` to boot in clustered mode. # Workers are forked web server processes. If using threads and workers together # the concurrency of the application would be max `threads` * `workers`. # Workers do not work on JRuby or Windows (both of which do not support # processes). # # workers ENV.fetch("WEB_CONCURRENCY") { 2 } # Use the `preload_app!` method when specifying a `workers` number. # This directive tells Puma to first boot the application and load code # before forking the application. This takes advantage of Copy On Write # process behavior so workers use less memory. # # preload_app! # Allow puma to be restarted by `rails restart` command. plugin :tmp_restart ================================================ FILE: config/routes.rb ================================================ Rails.application.routes.draw do mount PgHero::Engine, at: 'pghero' namespace :api do resources :trips, only: %i[index show] do collection do get :my end member do get :details end end resources :trip_requests, only: %i[create show] end post '/auth/login', to: 'authentication#login' end ================================================ FILE: config/schedule.rb ================================================ # Use this file to easily define all of your cron jobs. # # It's helpful, but not entirely necessary to understand cron before proceeding. # http://en.wikipedia.org/wiki/Cron # Example: # # set :output, "/path/to/my/cron_log.log" # # every 2.hours do # command "/usr/bin/some_great_command" # runner "MyModel.some_method" # rake "some:great:rake:task" # end # # every 4.days do # runner "AnotherModel.prune_old_records" # end # Learn more: http://github.com/javan/whenever # every 15.minutes do runner 'FastSearchResult.refresh' end every 1.month do command "pgslice add_partitions trip_positions --future 6 --url postgres://owner:@localhost/rideshare_development" end ================================================ FILE: config.ru ================================================ # This file is used by Rack-based servers to start the application. require_relative 'config/environment' run Rails.application ================================================ FILE: db/README.md ================================================ # Database Setup ## PostgreSQL Version Make sure you're running PostgreSQL 16 or newer. We recommend Postgres.app, however Homebrew is popular. Make sure you've used this formula: ## Fake data Fake data generated from Ruby, using the Faker gem, may be generated using the following commands. This will generate around 20K user records which is useful for most tests. More data will be needed for performance testing. ```sh bin/rails data_generators:generate_all bin/rails data_generators:drivers bin/rails data_generators:trips_and_requests ``` For more data, see SQL scripts in: [db/scripts/README.md](db/scripts/README.md) ```sh sh db/scripts/bulk_load.sh sh db/scripts/bulk_load_extended.sh ``` ## Data Loads Video Demo To see a demonstration of both methods:
🎥 Rideshare - Loading data using a Rake task and Shell Script
## Security Goals The *Principle of least privilege*[^prin] is followed by creating explicit `GRANT` commands for the `owner`, `app`, and `app_readonly` users. The configuration is based on *My GOTO Postgres Configuration for Web Services*.[^gotocon] One of the other goals besides minimizing access, is to prevent accidental table drops. Since the schema `rideshare` is created, the `public` schema is not needed and is removed. For `psql` commands, use a `DATABASE_URL` environment variable that's set in your terminal. The connection string connects to the Rideshare database, using the `owner` user. The value of `DATABASE_URL` is a connection string, with the format `protocol://role:password@host:port/databasename`. An example is checked in to `.env`. [^prin]: [^gotocon]: ## Configuring Host Based Authentication (HBA) You may want to configure *Host Based Authentication* (`HBA`)[^pghba]. Do that by editing your `pg_hba.conf` file. Changes in `pg_hba.conf` can be applied by *reloading* PostgreSQL. ## Reloading your PostgreSQL configuration Finding config file: `psql -U postgres -c 'SHOW config_file'` To reload your configuration, run: `pg_ctl reload` in your terminal. If you run into the following message, read on for more information. ```sh pg_ctl: no database directory specified and environment variable PGDATA unset Try "pg_ctl --help" for more information. ``` This command assumes the `PGDATA` environment variable is set, and points to the data directory for your PostgreSQL installation. Run `echo $PGDATA` to confirm it's set and see the value. How do you set the value if it's empty? Run the following commands in your terminal: ```sh # Look up the value psql -U postgres -c 'SHOW data_directory' # Assign the value to PGDATA export PGDATA="$(psql -U postgres \ -c 'SHOW data_directory' \ --tuples-only | sed 's/^[ \t]*//')" echo "Set PGDATA: $PGDATA" ``` When you've confirmed `PGDATA` is set, run `pg_ctl reload` again. The command should reload the PostgreSQL config, referencing your data directory via `PGDATA`. [^pghba]: ## Docker Reset everything: ```sh sh reset_docker_instances.sh ``` Tear down docker: ```sh sh teardown_docker.sh ``` ## Slow Clients Replace `config/database.yml` (or just the "slow clients" section) ``` cp config/database-slow-clients.sample.yml config/database.yml ``` With that in place, create a model: ```ruby class SlowClientModel < ApplicationRecord self.establish_connection :slow_clients end ``` Run query code that takes 5 seconds, and verify that it's canceled in the normal configuration. The "slow client" configuration allows it since it has a higher statement timeout configured. ```rb Trip.connection.execute("SELECT PG_SLEEP(5)") SlowClientModel.connection.execute("SELECT PG_SLEEP(5)").first ``` ## pg_cron [Scheduling maintenance with pg_cron](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/PostgreSQL_pg_cron.html) - The extension is created using the postgres superuser - The superuser grants usage privileges to the owner role, for the cron schema - Now the owner user can schedule their own jobs, for objects they own ```sql psql -U postgres -d rideshare_development; CREATE EXTENSION pg_cron; GRANT USAGE ON SCHEMA cron TO owner; ``` Run a job: ```sql SELECT cron.schedule( 'rideshare trips manual vacuum', '10 * * * *', 'VACUUM (ANALYZE) rideshare.trips' ); ``` View the jobs: ```sql SELECT * FROM cron.job; ``` View job runs: ```sql SELECT * FROM cron.job_run_details; ``` ![Screenshot of PgHero Scheduled Jobs](https://i.imgur.com/rxRf7Qn.png) ## active-record-doctor Run the tool from your terminal: ```sh bundle exec rake active_record_doctor: ``` ## database_consistency Run the tool from your terminal: ```sh database_consistency ``` ## rails-pg-extras Specify a custom schema for table_cache_hit ```sh bin/rails runner \ 'RailsPgExtras.table_cache_hit(args: { schema: "rideshare" })' ``` Or for version >= 5.3.1, set a schema using an environment variable: ```sh export PG_EXTRAS_SCHEMA=rideshare ``` For example, we can search for unused indexes, and indexes within the expected schema (`rideshare`) are examined ```sh bin/rails pg_extras:unused_indexes ``` ```sh bin/rails pg_extras:diagnose ``` ## rails_best_practices ```sh bin/rails_best_practices . ``` ## PgBouncer Prepared Statements - Run `brew services` and confirm PgBouncer is running on port 6432 - Set `DATABASE_URL` to be port 6432 - Disable Query Logs in `config/application.rb` (currently incompatible) - Restart PgBouncer to clear out the prepared statements Run the following script to observe how prepared statements are populated: ```sh sh pgbouncer_prepared_statements_check.sh ``` ## pgbench We can use pgbench and some pre-made SQL queries forming a transaction, to measure the transactions per second (TPS) that the server is capable of. ```sh sh db/scripts/benchmark.sh ``` ================================================ FILE: db/alter_default_privileges_public.sql ================================================ -- -- tables, sequences, functions, types, schemas -- \c rideshare_development ALTER DEFAULT PRIVILEGES FOR ROLE owner REVOKE ALL PRIVILEGES ON TABLES FROM PUBLIC; ALTER DEFAULT PRIVILEGES FOR ROLE owner REVOKE ALL PRIVILEGES ON SEQUENCES FROM PUBLIC; ALTER DEFAULT PRIVILEGES FOR ROLE owner REVOKE ALL PRIVILEGES ON FUNCTIONS FROM PUBLIC; ALTER DEFAULT PRIVILEGES FOR ROLE owner REVOKE ALL PRIVILEGES ON TYPES FROM PUBLIC; ALTER DEFAULT PRIVILEGES FOR ROLE owner REVOKE ALL PRIVILEGES ON SCHEMAS FROM PUBLIC; ================================================ FILE: db/alter_default_privileges_readonly.sql ================================================ -- Schema -- readonly role -- \c rideshare_development ALTER DEFAULT PRIVILEGES FOR ROLE owner IN SCHEMA rideshare GRANT SELECT ON TABLES TO readonly_users; ALTER DEFAULT PRIVILEGES FOR ROLE owner IN SCHEMA rideshare GRANT USAGE, SELECT ON SEQUENCES TO readonly_users; ALTER DEFAULT PRIVILEGES FOR ROLE owner IN SCHEMA rideshare GRANT EXECUTE ON FUNCTIONS TO readonly_users; ALTER DEFAULT PRIVILEGES FOR ROLE owner IN SCHEMA rideshare GRANT USAGE ON TYPES TO readonly_users; ================================================ FILE: db/alter_default_privileges_readwrite.sql ================================================ -- https://tightlycoupled.io/my-goto-postgres-configuration-for-web-services/ -- Schema default privileges -- readwrite role -- \c rideshare_development ALTER DEFAULT PRIVILEGES FOR ROLE owner IN SCHEMA rideshare GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO readwrite_users; ALTER DEFAULT PRIVILEGES FOR ROLE owner IN SCHEMA rideshare GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO readwrite_users; ALTER DEFAULT PRIVILEGES FOR ROLE owner IN SCHEMA rideshare GRANT EXECUTE ON FUNCTIONS TO readwrite_users; ALTER DEFAULT PRIVILEGES FOR ROLE owner IN SCHEMA rideshare GRANT USAGE ON TYPES TO readwrite_users; ================================================ FILE: db/create_database.sql ================================================ CREATE DATABASE rideshare_development WITH OWNER owner ENCODING UTF8; -- LC_COLLATE 'en_US.UTF-8' -- LC_CTYPE 'en_US.UTF-8'; ================================================ FILE: db/create_grants_database.sql ================================================ \c rideshare_development GRANT CONNECT ON DATABASE rideshare_development TO readwrite_users; GRANT TEMPORARY ON DATABASE rideshare_development TO readwrite_users; GRANT CONNECT ON DATABASE rideshare_development TO readonly_users; GRANT CONNECT ON DATABASE rideshare_development TO app_readonly; ================================================ FILE: db/create_grants_schema.sql ================================================ \c rideshare_development GRANT USAGE ON SCHEMA rideshare TO readwrite_users; GRANT USAGE ON SCHEMA rideshare TO readonly_users; -- Not needed, but being explicit helps with \dn+ GRANT CREATE, USAGE ON SCHEMA rideshare TO owner; -- Grants for app_readonly GRANT USAGE ON SCHEMA rideshare TO app_readonly; GRANT SELECT ON ALL TABLES IN SCHEMA rideshare TO app_readonly; GRANT USAGE ON ALL SEQUENCES IN SCHEMA rideshare TO app_readonly; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA rideshare TO app_readonly; -- Use pg_read_all_data instead of using Default Privileges GRANT pg_read_all_data TO app_readonly; -- Needed for: SHOW data_directory; -- export PGDATA="$(psql $DATABASE_URL -c 'SHOW data_directory' --tuples-only)" GRANT pg_read_all_settings TO owner; GRANT pg_read_all_data TO owner; GRANT pg_read_all_stats TO owner; ================================================ FILE: db/create_role_app_readonly.sql ================================================ -- A login role -- https://www.crunchydata.com/blog/creating-a-read-only-postgres-user CREATE ROLE app_readonly LOGIN ENCRYPTED PASSWORD :'password_to_save' CONNECTION LIMIT 3; ================================================ FILE: db/create_role_app_user.sql ================================================ -- https://tightlycoupled.io/my-goto-postgres-configuration-for-web-services/ -- CREATE ROLE app WITH LOGIN ENCRYPTED PASSWORD :'password_to_save' -- https://stackoverflow.com/a/72985243/126688 CONNECTION LIMIT 90 -- because of postgres default of 100 IN ROLE readwrite_users; ALTER ROLE app SET statement_timeout = 1000; ALTER ROLE app SET lock_timeout = 750; -- v9.6+ ALTER ROLE app SET idle_in_transaction_session_timeout = 1000; ALTER ROLE app SET search_path = rideshare; ================================================ FILE: db/create_role_owner.sql ================================================ -- https://tightlycoupled.io/my-goto-postgres-configuration-for-web-services/ CREATE ROLE owner LOGIN ENCRYPTED PASSWORD :'password_to_save' -- https://stackoverflow.com/a/72985243/126688 CONNECTION LIMIT 10; ALTER ROLE owner SET statement_timeout = 20000; ALTER ROLE owner SET lock_timeout = 3000; ================================================ FILE: db/create_role_readonly_users.sql ================================================ CREATE ROLE readonly_users NOLOGIN; ================================================ FILE: db/create_role_readwrite_users.sql ================================================ CREATE ROLE readwrite_users NOLOGIN; ================================================ FILE: db/create_schema.sql ================================================ \c rideshare_development SET ROLE owner; CREATE SCHEMA rideshare; RESET ROLE; -- set up owner earlier: -- https://tightlycoupled.io/my-goto-postgres-configuration-for-web-services/ ALTER ROLE owner SET search_path TO rideshare; SET search_path TO rideshare; ================================================ FILE: db/env_vars_sample.sh ================================================ # Replace postgres/postgres with "owner" or "app" credentials # Use the password created at provision time export DATABASE_URL_PRIMARY="postgres://postgres:postgres@localhost:54321/rideshare_development" export DATABASE_URL_REPLICA="postgres://postgres:postgres@localhost:54322/rideshare_development" ================================================ FILE: db/functions/scrub_email_v01.sql ================================================ CREATE OR REPLACE FUNCTION scrub_email(email_address varchar(255)) RETURNS varchar(255) AS $$ BEGIN RETURN -- take random MD5 text that is the same -- length as the first part of the email address -- EXCEPT when it's less than 5 chars, since we might -- have a collision. In that case use 5: greatest(length,6) CONCAT( substr( md5(random()::text), 0, greatest(length(split_part(email_address, '@', 1)) + 1, 6) ), '@', split_part(email_address, '@', 2) ); END; $$ LANGUAGE plpgsql; ================================================ FILE: db/functions/scrub_email_v02.sql ================================================ -- replace email_address with random text that is the same -- length as the unique portion of an email address -- before the "@" symbol. -- Make the minimum length 5 characters to avoid -- MD5 text generation collisions CREATE OR REPLACE FUNCTION scrub_email(email_address varchar(255)) RETURNS varchar(255) AS $$ SELECT CONCAT( SUBSTR( MD5(RANDOM()::text), 0, GREATEST(LENGTH(SPLIT_PART(email_address, '@', 1)) + 1, 6) ), '@', SPLIT_PART(email_address, '@', 2) ); $$ LANGUAGE SQL; ================================================ FILE: db/functions/scrub_text_v01.sql ================================================ CREATE OR REPLACE FUNCTION scrub_text(text varchar(255)) RETURNS varchar(255) AS $$ BEGIN RETURN -- replace from position 0, to max(length or 6) substr( md5(random()::text), 0, greatest(length(text) + 1, 6) ); END; $$ LANGUAGE plpgsql; ================================================ FILE: db/functions/scrub_text_v02.sql ================================================ CREATE OR REPLACE FUNCTION scrub_text(input varchar(255)) RETURNS varchar(255) AS $$ SELECT -- replace from position 0, to max(length or 6) SUBSTR( MD5(RANDOM()::text), 0, GREATEST(LENGTH(input) + 1, 6) ); $$ LANGUAGE SQL; ================================================ FILE: db/migrate/20191107212726_create_users.rb ================================================ class CreateUsers < ActiveRecord::Migration[6.0] def change # index: Rails adds PK index create_table :users do |t| t.string :first_name, null: false # index: no t.string :last_name, null: false # index: no t.string :email, null: false, index: true, unique: true # index: true t.string :type, null: false # index: maybe in future, partitioning t.timestamps # nullability: Rails adds null: false end end end ================================================ FILE: db/migrate/20191108221519_create_locations.rb ================================================ class CreateLocations < ActiveRecord::Migration[6.0] def change # index: Rails adds PK index create_table :locations do |t| t.string :address, null: false, index: true # store the string form of the address [See below], index: yes, search t.decimal :latitude, precision: 15, scale: 10, null: false, index: true # index: yes, search t.decimal :longitude, precision: 15, scale: 10, null: false, index: true # index: yes, search t.timestamps # Nullability: Rails adds null: false end end end # NOTE: We could also make separate fields for house number, street address, city, state etc. # This is a simplified version # # NOTE: We expect to search on address text, or on latitude and longitude ================================================ FILE: db/migrate/20191111151637_create_trip_requests.rb ================================================ class CreateTripRequests < ActiveRecord::Migration[6.0] def change # Indexes: Rails adds PK index # Nullability: no nulls create_table :trip_requests do |t| t.integer :rider_id, index: true, null: false # index: FK t.integer :start_location_id, index: true, null: false # index: FK t.integer :end_location_id, index: true, null: false # index: FK t.timestamps # Rails adds null: false end end end ================================================ FILE: db/migrate/20191112165848_create_trips.rb ================================================ class CreateTrips < ActiveRecord::Migration[6.0] def change # Indexes: Rails adds PK index create_table :trips do |t| t.integer :trip_request_id, index: true, null: false # index: FK t.integer :driver_id, index: true, null: false # index: FK t.timestamp :completed_at # nullable t.integer :rating, index: true # index: aggregate queries t.timestamps # Rails adds null: false end end end ================================================ FILE: db/migrate/20191121175429_install_blazer.rb ================================================ class InstallBlazer < ActiveRecord::Migration[6.0] def change create_table :blazer_queries do |t| t.references :creator t.string :name t.text :description t.text :statement t.string :data_source t.timestamps null: false end create_table :blazer_audits do |t| t.references :user t.references :query t.text :statement t.string :data_source t.timestamp :created_at end create_table :blazer_dashboards do |t| t.references :creator t.text :name t.timestamps null: false end create_table :blazer_dashboard_queries do |t| t.references :dashboard t.references :query t.integer :position t.timestamps null: false end create_table :blazer_checks do |t| t.references :creator t.references :query t.string :state t.string :schedule t.text :emails t.text :slack_channels t.string :check_type t.text :message t.timestamp :last_run_at t.timestamps null: false end end end ================================================ FILE: db/migrate/20191203212055_add_foreign_key_constraints.rb ================================================ class AddForeignKeyConstraints < ActiveRecord::Migration[6.0] def change # https://guides.rubyonrails.org/active_record_migrations.html#foreign-keys # # Strong migrations provides a warning: # # === Dangerous operation detected #strong_migrations === # New foreign keys are validated by default. This acquires an AccessExclusiveLock, # which is expensive on large tables. Instead, validate it in a separate migration # with a more agreeable RowShareLock. # # We could de-couple the introduction of the FK from the validation of it. add_foreign_key :trip_requests, :locations, column: :start_location_id, validate: false add_foreign_key :trip_requests, :locations, column: :end_location_id, validate: false # Because of STI, we want author_id to be a FK to users.id add_foreign_key :trip_requests, :users, column: :rider_id, primary_key: :id, validate: false add_foreign_key :trips, :trip_requests, validate: false add_foreign_key :trips, :users, column: :driver_id, primary_key: :id, validate: false end end ================================================ FILE: db/migrate/20191203213103_validate_foreign_key_constraints.rb ================================================ class ValidateForeignKeyConstraints < ActiveRecord::Migration[6.0] def change # https://github.com/ankane/strong_migrations#good-5 validate_foreign_key :trip_requests, :locations, column: :start_location_id validate_foreign_key :trip_requests, :locations, column: :end_location_id validate_foreign_key :trip_requests, :users, column: :rider_id, primary_key: :id validate_foreign_key :trips, :trip_requests validate_foreign_key :trips, :users, column: :driver_id, primary_key: :id end end ================================================ FILE: db/migrate/20200603150442_add_column_users_password_digest.rb ================================================ class AddColumnUsersPasswordDigest < ActiveRecord::Migration[6.0] def change add_column :users, :password_digest, :string end end ================================================ FILE: db/migrate/20220711010541_add_db_comments_to_users.rb ================================================ class AddDbCommentsToUsers < ActiveRecord::Migration[7.0] def change comment = 'sensitive_fields|first_name:scrub_text,last_name:scrub_text,email:scrub_email' change_table_comment :users, from: nil, to: comment end end ================================================ FILE: db/migrate/20220711015454_create_function_scrub_email.rb ================================================ class CreateFunctionScrubEmail < ActiveRecord::Migration[7.0] def change create_function :scrub_email end end ================================================ FILE: db/migrate/20220711015524_create_function_scrub_text.rb ================================================ class CreateFunctionScrubText < ActiveRecord::Migration[7.0] def change create_function :scrub_text end end ================================================ FILE: db/migrate/20220716020213_add_index_users_last_name.rb ================================================ class AddIndexUsersLastName < ActiveRecord::Migration[7.0] disable_ddl_transaction! def change add_index :users, :last_name, algorithm: :concurrently end end ================================================ FILE: db/migrate/20220729014635_create_vehicle_reservations.rb ================================================ class CreateVehicleReservations < ActiveRecord::Migration[7.0] # https://wiki.postgresql.org/wiki/Don%27t_Do_This#Don.27t_use_timestamp_.28without_time_zone.29 # https://discuss.rubyonrails.org/t/postgres-timestampz-by-default-in-rails-6-2/76537 # # db/schema.rb does not seem to capture the timestamptz column type # https://blog.appsignal.com/2020/01/15/the-pros-and-cons-of-using-structure-sql-in-your-ruby-on-rails-application.html # def change create_table :vehicle_reservations do |t| t.integer :vehicle_id, null: false, index: true t.integer :trip_request_id, null: false t.boolean :canceled, null: false, default: false t.timestamptz :starts_at, null: false t.timestamptz :ends_at, null: false t.timestamps end end end ================================================ FILE: db/migrate/20220729020430_create_vehicles.rb ================================================ class CreateVehicles < ActiveRecord::Migration[7.0] def change create_table :vehicles do |t| t.string :name, null: false, index: { unique: true } t.timestamps end end end ================================================ FILE: db/migrate/20220801140121_add_exclusion_constraint_vehicle_registrations.rb ================================================ class AddExclusionConstraintVehicleRegistrations < ActiveRecord::Migration[7.0] def change # NOTE: Depends on btree_gist extension being created in scripts/db_setup.sh by superuser # # Prevent overlapping reservations for # the same vehicle # # - vehicle_id is the vehicle being reserved # - starts_at is the start time of the reservation # - ends_at is the end time of the reservation # - a reservation is associated with a trip_request_id # - a reservation may be canceled safety_assured do execute <<-SQL ALTER TABLE vehicle_reservations ADD CONSTRAINT non_overlapping_vehicle_registration EXCLUDE USING gist ( int4range(vehicle_id, vehicle_id, '[]') WITH =, tstzrange(starts_at, ends_at) WITH && ) WHERE (not canceled) SQL end # Error: data type integer has no default operator class for access method "gist" # #=> Needed to enable the extension # # Error: PG::InvalidObjectDefinition: ERROR: functions in index expression must be marked IMMUTABLE # #=> Changed from tstzrange operator to tsrange operator, starts_at, ends_at are timestamp columns # # https://www.cybertec-postgresql.com/en/postgresql-exclude-beyond-unique/ end end ================================================ FILE: db/migrate/20220814175213_add_trips_count_to_users.rb ================================================ class AddTripsCountToUsers < ActiveRecord::Migration[7.0] def change add_column :users, :trips_count, :integer end end ================================================ FILE: db/migrate/20220916171314_create_search_results.rb ================================================ class CreateSearchResults < ActiveRecord::Migration[7.0] def change create_view :search_results end end ================================================ FILE: db/migrate/20221007184855_create_fast_search_results.rb ================================================ class CreateFastSearchResults < ActiveRecord::Migration[7.0] def change create_view :fast_search_results, materialized: true end end ================================================ FILE: db/migrate/20221108172933_add_status_column_to_vehicles.rb ================================================ class AddStatusColumnToVehicles < ActiveRecord::Migration[7.0] def change add_column :vehicles, :status, :string, null: false, default: VehicleStatus::DRAFT end end ================================================ FILE: db/migrate/20221108175321_remove_status_column_from_vehicles.rb ================================================ class RemoveStatusColumnFromVehicles < ActiveRecord::Migration[7.0] def change # removing this to replace it with a DB enum # NOTE: if this was in production, do not immediately # drop this column, but create a new one to begin using # migrate to, and then retire the old column safety_assured do remove_column :vehicles, :status end end end ================================================ FILE: db/migrate/20221108175619_add_status_column_db_enum_type_to_vehicles.rb ================================================ class AddStatusColumnDbEnumTypeToVehicles < ActiveRecord::Migration[7.0] def change create_enum :vehicle_status, [ VehicleStatus::DRAFT, VehicleStatus::PUBLISHED ] add_column :vehicles, :status, :enum, enum_type: :vehicle_status, default: VehicleStatus::DRAFT, null: false end end ================================================ FILE: db/migrate/20221110020532_add_drivers_license_number_to_users.rb ================================================ class AddDriversLicenseNumberToUsers < ActiveRecord::Migration[7.0] def change add_column :users, :drivers_license_number, :string, limit: 100 end end ================================================ FILE: db/migrate/20221111212740_add_trip_rating_check_constraint.rb ================================================ class AddTripRatingCheckConstraint < ActiveRecord::Migration[7.0] def change add_check_constraint :trips, 'rating IS NULL OR (rating >= 1 AND rating <= 5)', name: 'rating_check', validate: false end end ================================================ FILE: db/migrate/20221111213918_validate_add_trip_rating_check_constraint.rb ================================================ class ValidateAddTripRatingCheckConstraint < ActiveRecord::Migration[7.0] def change validate_check_constraint :trips, name: 'rating_check' end end ================================================ FILE: db/migrate/20221219164626_add_unique_address_to_locations.rb ================================================ class AddUniqueAddressToLocations < ActiveRecord::Migration[7.1] disable_ddl_transaction! def change remove_index :locations, :address add_index :locations, :address, unique: true, algorithm: :concurrently end end ================================================ FILE: db/migrate/20221220201836_enable_extension_pg_stat_statements.rb ================================================ class EnableExtensionPgStatStatements < ActiveRecord::Migration[7.1] # PGSS = 'pg_stat_statements' # # def change # # prereq: added to shared_preload_libraries='pg_stat_statements' # enable_extension(PGSS) unless extension_enabled?(PGSS) # end # Replaced by: # sh scripts/setup_db.sh # # Extension should be enabled by superuser end ================================================ FILE: db/migrate/20221221052616_change_column_trips_trip_request_id.rb ================================================ class ChangeColumnTripsTripRequestId < ActiveRecord::Migration[7.1] # Purpose: changing int->bigint # for FK column trip_requests.trip_id # bundle exec rake active_record_doctor # def change # don't do this in prod # https://github.com/ankane/strong_migrations#changing-the-type-of-a-column safety_assured do # not in prod, so just performing it change_column :trips, :trip_request_id, :bigint end end end ================================================ FILE: db/migrate/20221223161403_create_trip_positions.rb ================================================ class CreateTripPositions < ActiveRecord::Migration[7.1] def change create_table :trip_positions do |t| t.point :position t.bigint :trip_id, null: false t.timestamps end # Skipping FK for now since a lot of data will be inserted, # preferring faster inserts. `trip_id` would also likely # be indexed. # # new table, so skipping safety checks # safety_assured do # add_foreign_key :trip_positions, :trips # end end end ================================================ FILE: db/migrate/20221230200725_add_unique_constraint_users_email.rb ================================================ class AddUniqueConstraintUsersEmail < ActiveRecord::Migration[7.1] def change # Potentially unsafe in production, but ok # to add here (only used locally) # remove former index that does not support # unique constraint remove_index(:users, :email) if index_exists?(:users, :email) safety_assured do add_index :users, [:email], unique: true end end end ================================================ FILE: db/migrate/20221230203627_fix_canceled_column_default.rb ================================================ class FixCanceledColumnDefault < ActiveRecord::Migration[7.1] def change # by default, reservations should be canceled=false change_column_default :vehicle_reservations, :canceled, false end end ================================================ FILE: db/migrate/20230125003531_add_searchable_full_name_to_users.rb ================================================ class AddSearchableFullNameToUsers < ActiveRecord::Migration[7.1] def change safety_assured do # executing in non-prod execute <<-SQL ALTER TABLE users ADD COLUMN searchable_full_name TSVECTOR GENERATED ALWAYS AS ( SETWEIGHT(TO_TSVECTOR('english', COALESCE(first_name, '')), 'A') || SETWEIGHT(TO_TSVECTOR('english', COALESCE(last_name,'')), 'B') ) STORED; SQL end end end ================================================ FILE: db/migrate/20230125003946_add_index_searchable_full_name_to_users.rb ================================================ class AddIndexSearchableFullNameToUsers < ActiveRecord::Migration[7.1] disable_ddl_transaction! def change add_index :users, :searchable_full_name, using: :gin, # GIN index algorithm: :concurrently end end ================================================ FILE: db/migrate/20230126025656_remove_blazer_from_rideshare.rb ================================================ class RemoveBlazerFromRideshare < ActiveRecord::Migration[7.1] def change # No longer using Blazer drop_table(:blazer_queries) drop_table(:blazer_audits) drop_table(:blazer_dashboards) drop_table(:blazer_dashboard_queries) drop_table(:blazer_checks) end end ================================================ FILE: db/migrate/20230314204931_create_trip_positions_partitioned_intermediate_table.rb ================================================ class CreateTripPositionsPartitionedIntermediateTable < ActiveRecord::Migration[7.1] def change safety_assured do # skipping Strong Migrations safeguard execute <<-SQL.squish BEGIN; CREATE TABLE trip_positions_intermediate ( LIKE trip_positions INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING STORAGE INCLUDING COMMENTS ) PARTITION BY RANGE ("created_at"); COMMENT ON TABLE trip_positions_intermediate IS 'column:created_at,period:month,cast:date,version:3'; COMMIT; SQL end end end ================================================ FILE: db/migrate/20230314210022_add_trip_positions_intermediate_default_partition.rb ================================================ class AddTripPositionsIntermediateDefaultPartition < ActiveRecord::Migration[7.1] def change safety_assured do execute <<-SQL.squish CREATE TABLE "trip_positions_intermediate_default" PARTITION OF "trip_positions_intermediate" DEFAULT; ALTER TABLE "trip_positions_intermediate_default" ADD PRIMARY KEY ("id"); SQL end end end ================================================ FILE: db/migrate/20230619213546_add_locations_city_state.rb ================================================ class AddLocationsCityState < ActiveRecord::Migration[7.1] def change add_column :locations, :city, :string add_column :locations, :state, 'character(2)' end end ================================================ FILE: db/migrate/20230620030038_remove_unused_indexes.rb ================================================ class RemoveUnusedIndexes < ActiveRecord::Migration[7.1] disable_ddl_transaction! def change remove_index :locations, :latitude, name: 'index_locations_on_latitude', algorithm: :concurrently remove_index :locations, :longitude, name: 'index_locations_on_longitude', algorithm: :concurrently end end ================================================ FILE: db/migrate/20230625151410_add_foreign_keys.rb ================================================ class AddForeignKeys < ActiveRecord::Migration[7.1] def change safety_assured do add_foreign_key :trip_positions, :trips add_foreign_key :vehicle_reservations, :vehicles end end end ================================================ FILE: db/migrate/20230711015123_add_fast_count_gem.rb ================================================ class AddFastCountGem < ActiveRecord::Migration[7.1] def change FastCount.install end end ================================================ FILE: db/migrate/20230713150550_update_function_scrub_email_to_version_2.rb ================================================ class UpdateFunctionScrubEmailToVersion2 < ActiveRecord::Migration[7.1] def change update_function :scrub_email, version: 2, revert_to_version: 1 end end ================================================ FILE: db/migrate/20230713150710_update_function_scrub_text_to_version_2.rb ================================================ class UpdateFunctionScrubTextToVersion2 < ActiveRecord::Migration[7.1] def change update_function :scrub_text, version: 2, revert_to_version: 1 end end ================================================ FILE: db/migrate/20230714013609_trips_check_constraints.rb ================================================ class TripsCheckConstraints < ActiveRecord::Migration[7.1] def change safety_assured do # Add it back with the NULL check, which is unnecessary remove_check_constraint :trips, name: 'rating_check' add_check_constraint :trips, 'rating >= 1 AND rating <= 5', name: 'rating_check' add_check_constraint :trips, 'completed_at > created_at', validate: false # Some existing data in pre-made dump violates this end end end ================================================ FILE: db/migrate/20230716174139_add_foreign_key_column_vehicle_reservations.rb ================================================ class AddForeignKeyColumnVehicleReservations < ActiveRecord::Migration[7.1] def change safety_assured do add_foreign_key :vehicle_reservations, :trip_requests end end end ================================================ FILE: db/migrate/20230726020548_add_not_null_trip_positions_position.rb ================================================ class AddNotNullTripPositionsPosition < ActiveRecord::Migration[7.1] def change # Not on a live system safety_assured do change_column_null :trip_positions, :position, false end end end ================================================ FILE: db/migrate/20230925150207_add_position_to_locations.rb ================================================ class AddPositionToLocations < ActiveRecord::Migration[7.1] def change add_column :locations, :position, :point, null: false end end ================================================ FILE: db/migrate/20230925150831_drop_locations_latitude_longitude.rb ================================================ class DropLocationsLatitudeLongitude < ActiveRecord::Migration[7.1] def change # migrated these to a single point type column=>"position" safety_assured do remove_column :locations, :latitude remove_column :locations, :longitude end end end ================================================ FILE: db/migrate/20231018153441_update_fast_search_results_to_version_2.rb ================================================ class UpdateFastSearchResultsToVersion2 < ActiveRecord::Migration[7.1] def change update_view :fast_search_results, version: 2, revert_to_version: 1, materialized: true end end ================================================ FILE: db/migrate/20231018153712_add_unique_index_fast_search_results.rb ================================================ class AddUniqueIndexFastSearchResults < ActiveRecord::Migration[7.1] disable_ddl_transaction! def change add_index :fast_search_results, :driver_id, unique: true, algorithm: :concurrently end end ================================================ FILE: db/migrate/20231208050516_drop_column_searchable_full_name.rb ================================================ class DropColumnSearchableFullName < ActiveRecord::Migration[7.1] def change # Add this migration back in order to use: # `searchable_full_name` in the User model: # - concatenates first_name and last_name # - Configures it with pg_search # - Index added for this column # db/migrate/20230125003531_add_searchable_full_name_to_users.rb safety_assured do remove_column :users, :searchable_full_name end end end ================================================ FILE: db/migrate/20231213045957_add_constraints_locations_state.rb ================================================ class AddConstraintsLocationsState < ActiveRecord::Migration[7.1] def change # I've verified all the locations have a 2-char state # This opts out of Strong Migrations checks safety_assured do change_column_null(:locations, :state, false) end # Opt-out of Strong Migrations checks safety_assured do add_check_constraint :locations, 'LENGTH(state) = 2', name: 'state_length_check', validate: true end end end ================================================ FILE: db/migrate/20231218215836_remove_trip_positions_intermediate.rb ================================================ class RemoveTripPositionsIntermediate < ActiveRecord::Migration[7.1] def change safety_assured do drop_table :trip_positions_intermediate end end end ================================================ FILE: db/migrate/20231220043547_install_fast_count.rb ================================================ class InstallFastCount < ActiveRecord::Migration[7.1] def change # We are upgrading the gem, so we want to replace the current fast_count function safety_assured do execute('DROP FUNCTION IF EXISTS fast_count') end FastCount.install end end ================================================ FILE: db/pgbouncer_prepared_statements_check.sh ================================================ #!/bin/bash # # Disable Query Logs if they're enabled # # Configure DATABASE_URL with password # (can't read from ~/.pgpass), set port 6432 # # Overwrite DATABASE_URL to use PgBouncer port conn="postgres://owner:" conn+="@localhost:6432/rideshare_development" export DATABASE_URL="${conn}" # Confirm prepared statements are initially empty echo "List Prepared Statements results (empty to start):" bin/rails runner "puts ActiveRecord::Base.connection. execute('SELECT * FROM pg_prepared_statements').values" echo "Run a query to populate prepared statements:" bin/rails runner "Trip.first" # Check again echo "List Prepared Statements results again:" bin/rails runner "puts ActiveRecord::Base.connection. execute('SELECT * FROM pg_prepared_statements').values" ================================================ FILE: db/reset.sh ================================================ sh db/teardown.sh sh db/setup.sh ================================================ FILE: db/revoke_drop_public_schema.sql ================================================ \c rideshare_development REVOKE ALL ON DATABASE rideshare_development FROM PUBLIC; DROP SCHEMA public; ================================================ FILE: db/scripts/README.md ================================================ # DB Scripts Run all scripts from the `db` directory. From the Rideshare root, `cd` into `db`. ## Bulk Load Create `10_000_000` records, mix of Drivers and Riders, in `rideshare.users` using SQL Inspiration: ```sh sh scripts/bulk_load.sh ``` ## pgbench ```sh sh scripts/benchmark.sh ``` ## List table comments ```sh sh scripts/list_table_comments.sh ``` ## Simulate bloat ```sh sh scripts/simulate_bloat.sh ``` ================================================ FILE: db/scripts/benchmark.sh ================================================ #!/bin/bash # # https://access.crunchydata.com/documentation/postgresql11/11.5/pgbench.html # # Tested on 16.0 echo "Running pgbench" pgbench \ --username owner \ --protocol prepared \ --time 10 \ --jobs 2 \ --client 2 \ --no-vacuum \ --file scripts/queries.sql \ --report-per-command \ rideshare_development ================================================ FILE: db/scripts/bulk_load.sh ================================================ #!/bin/bash # USAGE: # sh bulk_load.sh # # PURPOSE: Create 10_000_000 users table records for performance testing # - Mix of Drivers and Riders # Technique credit: # query=" INSERT INTO rideshare.users( first_name, last_name, email, type, created_at, updated_at ) SELECT 'fname' || seq, 'lname' || seq, 'user_' || seq || '@' || ( CASE (RANDOM() * 2)::INT WHEN 0 THEN 'gmail' WHEN 1 THEN 'hotmail' WHEN 2 THEN 'yahoo' END ) || '.com' AS email, CASE (seq % 2) WHEN 0 THEN 'Driver' ELSE 'Rider' END, NOW(), NOW() FROM GENERATE_SERIES(1, 10_000_000) seq; -- To add additional batches of 10 million rows that -- with unique values, uncomment the following lines --FROM GENERATE_SERIES(10_000_001, 20_000_000) seq; --FROM GENERATE_SERIES(20_000_001, 30_000_000) seq; --FROM GENERATE_SERIES(30_000_001, 40_000_000) seq; --FROM GENERATE_SERIES(40_000_001, 50_000_000) seq; " if [ -z "$DATABASE_URL" ]; then echo "Error: DATABASE_URL is not set, which provides connection information for this script." echo "To set it, run the following in your terminal:" echo echo "export DATABASE_URL='postgres://owner:@localhost:5432/rideshare_development'" exit 1 fi echo "Creating batch of rideshare.users rows, raising statement_timeout to 30min" psql $DATABASE_URL -c "SET statement_timeout = '30min'; $query"; echo "ANALYZE rideshare.users" psql $DATABASE_URL -c "ANALYZE rideshare.users"; echo "Estimated count:" psql $DATABASE_URL -c "SELECT reltuples::numeric FROM pg_class WHERE relname IN ('users');" ================================================ FILE: db/scripts/bulk_load_extended.sh ================================================ #!/bin/bash # PURPOSE: # - Adds millions of trips and trip_requests records # for performance testing # # USAGE: # sh bulk_load_extended.sh # echo "Loading millions of records for trip_requests, trips..." ######################## # # TRIP REQUESTS # - Fake data, optimizing more for load speed vs. realistic data # ######################## query=" INSERT INTO rideshare.trip_requests( rider_id, start_location_id, end_location_id, created_at, updated_at ) SELECT (SELECT id FROM users WHERE type = 'Rider' ORDER BY RANDOM() LIMIT 1), (SELECT id FROM locations WHERE address = 'New York, NY'), (SELECT id FROM locations WHERE address = 'Boston, MA'), NOW(), NOW() FROM GENERATE_SERIES(1, 1_000_000) seq; " if [ -z "$DATABASE_URL" ]; then echo "Error: DATABASE_URL is not set." echo "Run: export DATABASE_URL='postgres://owner:@localhost:5432/rideshare_development'" exit 1 fi echo "Raising statement_timeout to 30 minutes, running $query..." psql $DATABASE_URL -c "SET statement_timeout = '30min'; $query"; psql $DATABASE_URL -c "ANALYZE (VERBOSE) rideshare.trip_requests"; ######################## # # TRIPS # - Fake data, optimizing more for load speed vs. realistic data # - Trip records are created before they're completed, CHECK constraint enforces that # ######################## query=" WITH last_90_days AS ( SELECT NOW() - ((RANDOM()*90)::INTEGER || 'day')::INTERVAL AS timestamp ) INSERT INTO rideshare.trips( trip_request_id, driver_id, completed_at, rating, created_at, updated_at ) SELECT (SELECT id FROM trip_requests ORDER BY RANDOM() LIMIT 1), (SELECT id FROM users WHERE type = 'Driver' ORDER BY RANDOM() LIMIT 1), (SELECT timestamp FROM last_90_days), (SELECT (RANDOM()*5)::INTEGER), (SELECT (timestamp - INTERVAL '1 day') from last_90_days), NOW() FROM GENERATE_SERIES(1, 10_000_000) seq; " if [ -z "$DATABASE_URL" ]; then echo "Error: DATABASE_URL is not set." echo "Run: export DATABASE_URL='postgres://owner:@localhost:5432/rideshare_development'" exit 1 fi echo "Raising statement_timeout to 30 minutes, running $query..." psql $DATABASE_URL -c "SET statement_timeout = '30min'; $query"; psql $DATABASE_URL -c "ANALYZE (VERBOSE) rideshare.trips"; echo "Estimated counts:" query="SELECT relname AS tablename, reltuples::numeric AS estimated_count FROM pg_class WHERE relname IN ('trips', 'trip_requests'); " psql $DATABASE_URL -c "$query" ================================================ FILE: db/scripts/list_table_comments.sh ================================================ #!/bin/bash # # Or run: \dt+ # # choose tables with a table level comment query="SELECT relname, obj_description(oid) FROM pg_class WHERE relkind = 'r' AND obj_description(oid) is not null" # this should find the "users" table which has table comments # the value for the comment can be inspected and parsed echo "Listing comments from: $DATABASE_URL" echo psql $DATABASE_URL -c "$query" --csv | head -3 | tail -1 ================================================ FILE: db/scripts/queries.sql ================================================ -- Don't remove -- Used by ./benchmark.sh -- one "Transaction" counts as one run of this file -- but file can contain multiple SQL statements, terminated -- by a semicolon -- Drivers with average rating, trip count, presented as -- First name and Last name -- Consider adding: expression index SELECT CONCAT(d.first_name, ' ', d.last_name) AS driver_name, AVG(t.rating) AS avg_rating, COUNT(t.rating) AS trip_count FROM trips t JOIN users d ON t.driver_id = d.id GROUP BY t.driver_id, d.first_name, d.last_name ORDER BY COUNT(t.rating) DESC; -- Groups the users, consider adding an index on 'type' SELECT COUNT(*), type FROM users GROUP BY type; -- Adds average trip length to earlier query SELECT CONCAT(d.first_name, ' ', d.last_name) AS driver_name, COUNT(t.id) AS trip_count, AVG(t.rating) AS avg_rating, AVG(t.completed_at - t.created_at) AS avg_trip_length FROM trips t JOIN users d ON t.driver_id = d.id AND d.type = 'Driver' GROUP BY t.driver_id, d.first_name, d.last_name ORDER BY COUNT(t.rating) DESC; ================================================ FILE: db/scripts/simulate_bloat.sh ================================================ #!/bin/bash # # first run scripts/bulk_load.sh # which will load at least 100,000 user records # consider working with 1 million or 10 million records # measure the estimated bloat percentage # for the indexes on the users table # update a portion of the rows # for all the "even" primary key id numbers # update their first name to Bill # query=" UPDATE users SET first_name = CASE (seq % 2) WHEN 0 THEN 'Bill' || FLOOR(RANDOM() * 10) || FLOOR(RANDOM() * 10) ELSE 'Jane' END FROM GENERATE_SERIES(1,100_000) seq WHERE id = seq; " psql $DATABASE_URL -c "$query"; ================================================ FILE: db/scrubbing/.gitignore ================================================ temp_*.sql ================================================ FILE: db/scrubbing/README.md ================================================ # Scrubbing In this section, we're looking at how to scrub sensitive columns within table rows. The example assumes you've started from a physical or logical copy of rows, for all tables. You'll apply scrubbing only to columns that contain sensitive data, tracking which ones they are using a simple and maintainable system. For an example to work with, you'll use the `rideshare.users` table. You'll consider a couple of the fields within `rideshare.users` to be sensitive. Since the scrubbing is all done with standard PostgreSQL procedural language, shell scripts, and without extensions or Ruby gems, this solution is portable to anywhere PostgreSQL is running. The following scripts clone the table structure, without row data. The scripts fill in rows from the original table and perform scrubbing on the fly. You'll also learn a basic mechanism to track which columns are sensitive, allowing you to maintain that information over time using your normal Rails Migrations process. Compare rows before and after running the script. ## Run Scrubbing ```sh cd db sh scrubbing/scrubber.sh ``` ## View Comments Database comments are used to record which fields are sensitive. ```sh sh db/list_table_comments.sh ``` ## Batching Review the batched `UPDATE` example: [scrub_batched_direct_updates.sql](scrub_batched_direct_updates.sql) For more information, please check out [High Performance PostgreSQL for Rails](https://pragprog.com/titles/aapsql/high-performance-postgresql-for-rails/), where this section is covered extensively in a full "Performance Database" chapter. ================================================ FILE: db/scrubbing/assign_sequence.sql ================================================ -- assumes the sequence was already created ALTER SEQUENCE rideshare.users_id_seq OWNED BY rideshare.users.id; ALTER TABLE rideshare.users ALTER COLUMN id SET DEFAULT nextval('users_id_seq'::regclass); ================================================ FILE: db/scrubbing/create_tables.sql ================================================ -- Among tables: -- users, locations, trip_requests, trips, vehicles, vehicle_reservations -- Only sensitive fields in tables: users DROP TABLE IF EXISTS users_copy CASCADE; CREATE TABLE users_copy (LIKE users INCLUDING ALL); ================================================ FILE: db/scrubbing/drop_and_swap_users.sql ================================================ BEGIN; DROP TABLE IF EXISTS users CASCADE; ALTER TABLE users_copy RENAME TO users; COMMIT; ================================================ FILE: db/scrubbing/dump_foreign_keys_ddl_target_table.sql ================================================ SELECT 'ALTER TABLE ' || nsp.nspname || '.' || cls.relname || ' ADD CONSTRAINT ' || conname || ' FOREIGN KEY (' || STRING_AGG(att.attname, ', ') OVER(PARTITION BY conname) || ')' || ' REFERENCES ' || refnsp.nspname || '.' || refcls.relname || ' (' || STRING_AGG(refatt.attname, ', ') OVER(PARTITION BY conname) || ')' || CASE WHEN confupdtype = 'c' THEN ' ON UPDATE CASCADE' WHEN confupdtype = 'n' THEN ' ON UPDATE SET NULL' WHEN confupdtype = 'd' THEN ' ON UPDATE SET DEFAULT' ELSE '' END || CASE WHEN confdeltype = 'c' THEN ' ON DELETE CASCADE' WHEN confdeltype = 'n' THEN ' ON DELETE SET NULL' WHEN confdeltype = 'd' THEN ' ON DELETE SET DEFAULT' ELSE '' END || ';' FROM pg_constraint con JOIN pg_class cls ON con.conrelid = cls.oid JOIN pg_namespace nsp ON cls.relnamespace = nsp.oid JOIN pg_class refcls ON con.confrelid = refcls.oid JOIN pg_namespace refnsp ON refcls.relnamespace = refnsp.oid JOIN pg_attribute att ON att.attnum = ANY(con.conkey) AND att.attrelid = con.conrelid JOIN pg_attribute refatt ON refatt.attnum = ANY(con.confkey) AND refatt.attrelid = con.confrelid WHERE refcls.relname = 'users' -- replace with your table name AND refnsp.nspname = 'rideshare' -- replace with your schema if different GROUP BY conname, nsp.nspname, cls.relname, refnsp.nspname, refcls.relname, confupdtype, confdeltype, att.attname, refatt.attname; ================================================ FILE: db/scrubbing/dump_sequence_creation_ddl.sql ================================================ SELECT 'CREATE SEQUENCE ' || schemaname || '.' || sequencename || ' INCREMENT ' || increment_by || ' MINVALUE ' || min_value || ' MAXVALUE ' || max_value || ' START ' || start_value || ';' FROM pg_sequences WHERE schemaname = 'rideshare' -- adjust this for your schema if necessary AND sequencename = 'users_id_seq'; -- replace with your sequence name ================================================ FILE: db/scrubbing/dump_views_ddl.sql ================================================ SELECT 'CREATE VIEW ' || viewname || ' AS ' || definition FROM pg_views WHERE schemaname = 'rideshare' -- adjust the schema if your view is in another schema AND viewname = 'search_results';-- replace with your view name SELECT 'CREATE MATERIALIZED VIEW ' || matviewname || ' AS ' || definition || ';' || COALESCE(E'\n\nREFRESH MATERIALIZED VIEW ' || matviewname || ' WITH ' || CASE WHEN matviewname IN (SELECT conname FROM pg_constraint WHERE contype = 'p') THEN 'NO DATA;' ELSE 'DATA;' END, '') FROM pg_matviews WHERE schemaname = 'rideshare' -- adjust the schema if your view is in another schema AND matviewname = 'fast_search_results'; -- replace with your materialized view name ================================================ FILE: db/scrubbing/generate_add_constraint_statements.sql ================================================ CREATE OR REPLACE FUNCTION generate_add_constraint_statements() RETURNS TABLE(stmt text) AS $$ DECLARE v_table_name text; v_statement text; BEGIN FOR v_table_name IN (SELECT tablename FROM pg_tables WHERE schemaname = 'rideshare' AND tablename IN ('users')) -- could add more tables in future LOOP SELECT string_agg('ALTER TABLE '||nspname||'.'||relname||' ADD CONSTRAINT '||conname||' '|| pg_get_constraintdef(pg_constraint.oid)||';', '') INTO v_statement FROM pg_constraint INNER JOIN pg_class ON conrelid=pg_class.oid INNER JOIN pg_namespace ON pg_namespace.oid=pg_class.relnamespace WHERE nspname = 'rideshare' AND relname = v_table_name; stmt := v_statement; RETURN NEXT; END LOOP; -- end loop END; -- end BEGIN $$ LANGUAGE plpgsql; ================================================ FILE: db/scrubbing/scrub_batched_direct_updates.sql ================================================ CREATE OR REPLACE PROCEDURE SCRUB_BATCHES() LANGUAGE PLPGSQL AS $$ DECLARE current_id INT := (SELECT MIN(id) FROM users); max_id INT := (SELECT MAX(id) FROM users); batch_size INT := 1000; rows_updated INT; BEGIN WHILE current_id <= max_id LOOP -- the UPDATE by `id` range UPDATE users SET email = SCRUB_EMAIL(email) WHERE id >= current_id AND id < current_id + batch_size; GET DIAGNOSTICS rows_updated = ROW_COUNT; COMMIT; RAISE NOTICE 'current_id: % - Number of rows updated: %', current_id, rows_updated; current_id := current_id + batch_size + 1; END LOOP; END; $$; -- Call the Procedure CALL SCRUB_BATCHES(); ================================================ FILE: db/scrubbing/scrub_users.sql ================================================ INSERT INTO users_copy(id, first_name, last_name, email, type, created_at, updated_at, password_digest, trips_count, drivers_license_number) ( SELECT id, scrub_text(first_name), scrub_text(last_name), scrub_email(email), type, created_at, updated_at, password_digest, trips_count, scrub_text(drivers_license_number) FROM users ) ON CONFLICT DO NOTHING; ================================================ FILE: db/scrubbing/scrubber.sh ================================================ #!/bin/bash export SOURCE_DB="postgres://owner:@localhost:5432/rideshare_development" echo "STARTING scrub process..." echo "5 rows BEFORE scrubbing:" psql $SOURCE_DB -c "SELECT * FROM users ORDER BY id ASC LIMIT 5" # Set a seed value psql $SOURCE_DB -c "SELECT SETSEED(0.5);" echo "Dump views DDL" psql $SOURCE_DB -f scrubbing/dump_views_ddl.sql \ --tuples-only \ --no-align \ -o scrubbing/temp_views_ddl.sql echo "------------------" echo "Dump target table foreign keys creation DDL" psql $SOURCE_DB -f scrubbing/dump_foreign_keys_ddl_target_table.sql \ --tuples-only \ --no-align \ -o scrubbing/temp_foreign_keys_ddl.sql echo "------------------" echo "Dump primary key sequence creation DDL" psql $SOURCE_DB -f scrubbing/dump_sequence_creation_ddl.sql \ --tuples-only \ --no-align \ -o scrubbing/temp_sequences.sql echo "------------------" echo "Create the users_copy table" sleep 1 psql $SOURCE_DB -f scrubbing/create_tables.sql echo "------------------" echo "Fill users_copy with scrubbed values" sleep 1 psql $SOURCE_DB -f scrubbing/scrub_users.sql echo "------------------" # There are no constraints besides the PK constraint which was already copied # echo "Add the generate add constraint statements function" # psql $SOURCE_DB -c "\i ./generate_add_constraint_statements.sql" # echo "Add function to generate constraints" # psql $SOURCE_DB -c "\i scrubbing/generate_add_constraint_statements.sql" # echo "Remove existing temp_constraints.sql" # rm scrubbing/temp_constraints.sql # echo "Dump table constraints for tables to file" # psql $SOURCE_DB -c "SELECT generate_add_constraint_statements()" \ # --tuples-only \ # -o scrubbing/temp_constraints.sql echo "Drop and rename users table" psql $SOURCE_DB -f scrubbing/drop_and_swap_users.sql echo "------------------" # echo "Add constraints" # psql $SOURCE_DB -f scrubbing/temp_constraints.sql echo "Add views and materialized views for target table" psql $SOURCE_DB -f scrubbing/temp_views_ddl.sql echo "------------------" echo "Add constraints that refer to target table, dropped from CASCADE" psql $SOURCE_DB -f scrubbing/temp_foreign_keys_ddl.sql echo "------------------" echo "Add sequence for target table, dropped from CASCADE" psql $SOURCE_DB -f scrubbing/temp_sequences.sql echo "------------------" echo "Assign sequence for target table" psql $SOURCE_DB -f scrubbing/assign_sequence.sql echo "------------------" echo "Success!" echo "View 10 rows from user:" psql $SOURCE_DB -c "SELECT * FROM users ORDER BY id ASC LIMIT 5" ================================================ FILE: db/setup.sh ================================================ #!/bin/bash # NOTE: This script expects you've generated a password. # You can do that using "openssl" as follows, or you could use any password # generation mechanism you like. # # Generate a password value using "openssl": # openssl rand -hex 12 # # Generate and assign the value to RIDESHARE_DB_PASSWORD: # export RIDESHARE_DB_PASSWORD=$(openssl rand -hex 12) # # Later, you'll create the special password file ~/.pgpass, and # place your generated password in it. # # COMPATIBILITY: Requires PostgreSQL 16 # ENV VARS: [DB_URL, RIDESHARE_DB_PASSWORD] # Make sure password is set if [ -z "$RIDESHARE_DB_PASSWORD" ]; then echo "Error: 'RIDESHARE_DB_PASSWORD' not set, can't continue." echo echo "Check for an existing value in file: ~/.pgpass" echo "If there's a value, set it like this:" echo 'export RIDESHARE_DB_PASSWORD="HSnDDgFtyW9fyFI"' echo "OR generate a new value (See comments in: db/setup.sh)" exit 1 fi # Check if the environment variable DB_URL is set if [ -z "$DB_URL" ]; then echo "Error: 'DB_URL' not set, can't continue." echo "This is the connection to your instance, using a superuser like 'postgres'." echo "The password for 'postgres' is also 'postgres'" echo "Connect to the 'postgres' database to issue these commands" echo echo "See: db/setup.sh" echo "Run: export DB_URL='postgres://postgres:@localhost:5432/postgres'" exit 1 fi # Set up Roles and Users on your PostgreSQL instance psql $DB_URL -v password_to_save=$RIDESHARE_DB_PASSWORD -a -f db/create_role_owner.sql psql $DB_URL -a -f db/create_role_readwrite_users.sql psql $DB_URL -a -f db/create_role_readonly_users.sql psql $DB_URL -v password_to_save=$RIDESHARE_DB_PASSWORD -a -f db/create_role_app_user.sql psql $DB_URL -v password_to_save=$RIDESHARE_DB_PASSWORD -a -f db/create_role_app_readonly.sql # Set up Rideshare development database psql $DB_URL -a -f db/create_database.sql # Revoke database privileges on public, drop public schema psql $DB_URL -a -f db/revoke_drop_public_schema.sql # Create rideshare schema psql $DB_URL -a -f db/create_schema.sql # Perform GRANT operations psql $DB_URL -a -f db/create_grants_database.sql psql $DB_URL -a -f db/create_grants_schema.sql # Alter the default privileges psql $DB_URL -a -f db/alter_default_privileges_readwrite.sql psql $DB_URL -a -f db/alter_default_privileges_readonly.sql psql $DB_URL -a -f db/alter_default_privileges_public.sql # Add generated password to ~/.pgpass file echo "Add to ~/.pgpass" echo "localhost:5432:rideshare_development:owner:$RIDESHARE_DB_PASSWORD localhost:6432:rideshare_development:owner:$RIDESHARE_DB_PASSWORD localhost:5432:rideshare_development:app:$RIDESHARE_DB_PASSWORD localhost:54321:rideshare_development:owner:$RIDESHARE_DB_PASSWORD localhost:54322:rideshare_development:owner:$RIDESHARE_DB_PASSWORD *:*:*:replication_user:$RIDESHARE_DB_PASSWORD *:*:*:app_readonly:$RIDESHARE_DB_PASSWORD" >> ~/.pgpass # Set file ownership and permissions echo "chmod ~/.pgpass" chmod 0600 ~/.pgpass echo echo "DONE! 🎉" echo "Notes:" echo "Make sure 'graphviz' is installed: 'brew install graphviz'" echo echo "Next: run 'bin/rails db:migrate' to apply pending migrations" echo echo "If you ran as: 'sh db/setup.sh 2>&1 | tee -a output.log'" echo "Open the 'output.log' file and check for errors" echo echo "The ~/.pgpass file was generated or new values were added to it." echo echo "Set the 'DATABASE_URL' env var, which you can find in the .env file:" echo "To set it in your terminal, run:" echo echo "export $(cat .env|grep DATABASE_URL|head -n1)" ================================================ FILE: db/setup_test_database.sh ================================================ #!/bin/bash export DB_URL=postgres://postgres:@localhost:5432/postgres # run as OS user/superuser/admin export APP_TEST_DB_NAME=rideshare_test export APP_TEST_USER=rideshare_test export TEST_DB_URL=postgres://postgres:@localhost:5432/rideshare_test # run as OS user/superuser/admin echo "%%%%%%%%%%%" echo "Test DB" echo "%%%%%%%%%%%" # ROLES echo "SELECT 'CREATE USER $APP_TEST_USER WITH LOGIN' WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '$APP_TEST_USER')\gexec" | psql $DB_URL # DATABASE echo "Creating database $APP_TEST_DB_NAME" echo "SELECT 'CREATE DATABASE $APP_TEST_DB_NAME' WHERE NOT EXISTS (SELECT datname FROM pg_database WHERE datname = '$APP_TEST_DB_NAME')\gexec" | psql $DB_URL; psql $DB_URL -c "ALTER DATABASE $APP_TEST_DB_NAME OWNER TO $APP_TEST_USER" # SUPERUSER ONLY(!) for rideshare_test database test user # SUPERUSER required to drop all Foreign Key Constraints, which is done when truncating tables # https://stackoverflow.com/a/32213455/126688 psql $DB_URL -c "ALTER USER $APP_TEST_USER WITH SUPERUSER" # CONNECT psql $DB_URL -c "GRANT CONNECT ON DATABASE $APP_TEST_DB_NAME TO $APP_TEST_USER;" psql -U $APP_TEST_USER -d $APP_TEST_DB_NAME -c "CREATE SCHEMA rideshare;" psql -U $APP_TEST_USER -d $APP_TEST_DB_NAME -c "ALTER ROLE $APP_TEST_USER SET search_path TO rideshare;" psql -U $APP_TEST_USER -d $APP_TEST_DB_NAME -c "SET search_path TO rideshare;" ================================================ FILE: db/structure.sql ================================================ SET statement_timeout = 0; SET lock_timeout = 0; SET idle_in_transaction_session_timeout = 0; SET client_encoding = 'UTF8'; SET standard_conforming_strings = on; SELECT pg_catalog.set_config('search_path', '', false); SET check_function_bodies = false; SET xmloption = content; SET client_min_messages = warning; SET row_security = off; ALTER TABLE IF EXISTS ONLY rideshare.trip_requests DROP CONSTRAINT IF EXISTS fk_rails_fa2679b626; ALTER TABLE IF EXISTS ONLY rideshare.trips DROP CONSTRAINT IF EXISTS fk_rails_e7560abc33; ALTER TABLE IF EXISTS ONLY rideshare.trip_requests DROP CONSTRAINT IF EXISTS fk_rails_c17a139554; ALTER TABLE IF EXISTS ONLY rideshare.trip_positions DROP CONSTRAINT IF EXISTS fk_rails_9688ac8706; ALTER TABLE IF EXISTS ONLY rideshare.vehicle_reservations DROP CONSTRAINT IF EXISTS fk_rails_7edc8e666a; ALTER TABLE IF EXISTS ONLY rideshare.trips DROP CONSTRAINT IF EXISTS fk_rails_6d92acb430; ALTER TABLE IF EXISTS ONLY rideshare.vehicle_reservations DROP CONSTRAINT IF EXISTS fk_rails_59996232fc; ALTER TABLE IF EXISTS ONLY rideshare.trip_requests DROP CONSTRAINT IF EXISTS fk_rails_3fdebbfaca; DROP INDEX IF EXISTS rideshare.index_vehicles_on_name; DROP INDEX IF EXISTS rideshare.index_vehicle_reservations_on_vehicle_id; DROP INDEX IF EXISTS rideshare.index_users_on_last_name; DROP INDEX IF EXISTS rideshare.index_users_on_email; DROP INDEX IF EXISTS rideshare.index_trips_on_trip_request_id; DROP INDEX IF EXISTS rideshare.index_trips_on_rating; DROP INDEX IF EXISTS rideshare.index_trips_on_driver_id; DROP INDEX IF EXISTS rideshare.index_trip_requests_on_start_location_id; DROP INDEX IF EXISTS rideshare.index_trip_requests_on_rider_id; DROP INDEX IF EXISTS rideshare.index_trip_requests_on_end_location_id; DROP INDEX IF EXISTS rideshare.index_locations_on_address; DROP INDEX IF EXISTS rideshare.index_fast_search_results_on_driver_id; ALTER TABLE IF EXISTS ONLY rideshare.vehicles DROP CONSTRAINT IF EXISTS vehicles_pkey; ALTER TABLE IF EXISTS ONLY rideshare.vehicle_reservations DROP CONSTRAINT IF EXISTS vehicle_reservations_pkey; ALTER TABLE IF EXISTS ONLY rideshare.users DROP CONSTRAINT IF EXISTS users_pkey; ALTER TABLE IF EXISTS ONLY rideshare.trips DROP CONSTRAINT IF EXISTS trips_pkey; ALTER TABLE IF EXISTS ONLY rideshare.trip_requests DROP CONSTRAINT IF EXISTS trip_requests_pkey; ALTER TABLE IF EXISTS ONLY rideshare.trip_positions DROP CONSTRAINT IF EXISTS trip_positions_pkey; ALTER TABLE IF EXISTS ONLY rideshare.schema_migrations DROP CONSTRAINT IF EXISTS schema_migrations_pkey; ALTER TABLE IF EXISTS ONLY rideshare.vehicle_reservations DROP CONSTRAINT IF EXISTS non_overlapping_vehicle_registration; ALTER TABLE IF EXISTS ONLY rideshare.locations DROP CONSTRAINT IF EXISTS locations_pkey; ALTER TABLE IF EXISTS rideshare.trips DROP CONSTRAINT IF EXISTS chk_rails_4743ddc2d2; ALTER TABLE IF EXISTS ONLY rideshare.ar_internal_metadata DROP CONSTRAINT IF EXISTS ar_internal_metadata_pkey; ALTER TABLE IF EXISTS rideshare.vehicles ALTER COLUMN id DROP DEFAULT; ALTER TABLE IF EXISTS rideshare.vehicle_reservations ALTER COLUMN id DROP DEFAULT; ALTER TABLE IF EXISTS rideshare.users ALTER COLUMN id DROP DEFAULT; ALTER TABLE IF EXISTS rideshare.trips ALTER COLUMN id DROP DEFAULT; ALTER TABLE IF EXISTS rideshare.trip_requests ALTER COLUMN id DROP DEFAULT; ALTER TABLE IF EXISTS rideshare.trip_positions ALTER COLUMN id DROP DEFAULT; ALTER TABLE IF EXISTS rideshare.locations ALTER COLUMN id DROP DEFAULT; DROP SEQUENCE IF EXISTS rideshare.vehicles_id_seq; DROP TABLE IF EXISTS rideshare.vehicles; DROP SEQUENCE IF EXISTS rideshare.vehicle_reservations_id_seq; DROP TABLE IF EXISTS rideshare.vehicle_reservations; DROP SEQUENCE IF EXISTS rideshare.users_id_seq; DROP SEQUENCE IF EXISTS rideshare.trips_id_seq; DROP SEQUENCE IF EXISTS rideshare.trip_requests_id_seq; DROP TABLE IF EXISTS rideshare.trip_requests; DROP SEQUENCE IF EXISTS rideshare.trip_positions_id_seq; DROP TABLE IF EXISTS rideshare.trip_positions; DROP VIEW IF EXISTS rideshare.search_results; DROP TABLE IF EXISTS rideshare.schema_migrations; DROP SEQUENCE IF EXISTS rideshare.locations_id_seq; DROP TABLE IF EXISTS rideshare.locations; DROP MATERIALIZED VIEW IF EXISTS rideshare.fast_search_results; DROP TABLE IF EXISTS rideshare.users; DROP TABLE IF EXISTS rideshare.trips; DROP TABLE IF EXISTS rideshare.ar_internal_metadata; DROP FUNCTION IF EXISTS rideshare.scrub_text(input character varying); DROP FUNCTION IF EXISTS rideshare.scrub_email(email_address character varying); DROP FUNCTION IF EXISTS rideshare.fast_count(identifier text, threshold bigint); DROP TYPE IF EXISTS rideshare.vehicle_status; DROP SCHEMA IF EXISTS rideshare; -- -- Name: rideshare; Type: SCHEMA; Schema: -; Owner: - -- CREATE SCHEMA rideshare; -- -- Name: vehicle_status; Type: TYPE; Schema: rideshare; Owner: - -- CREATE TYPE rideshare.vehicle_status AS ENUM ( 'draft', 'published' ); -- -- Name: fast_count(text, bigint); Type: FUNCTION; Schema: rideshare; Owner: - -- CREATE FUNCTION rideshare.fast_count(identifier text, threshold bigint) RETURNS bigint LANGUAGE plpgsql AS $$ DECLARE count bigint; table_parts text[]; schema_name text; table_name text; BEGIN SELECT PARSE_IDENT(identifier) INTO table_parts; IF ARRAY_LENGTH(table_parts, 1) = 2 THEN schema_name := ''''|| table_parts[1] ||''''; table_name := ''''|| table_parts[2] ||''''; ELSE schema_name := 'ANY (current_schemas(false))'; table_name := ''''|| table_parts[1] ||''''; END IF; EXECUTE ' WITH tables_counts AS ( -- inherited and partitioned tables counts SELECT ((SUM(child.reltuples::float) / greatest(SUM(child.relpages), 1))) * (SUM(pg_relation_size(child.oid))::float / (current_setting(''block_size'')::float))::integer AS estimate FROM pg_inherits INNER JOIN pg_class parent ON pg_inherits.inhparent = parent.oid LEFT JOIN pg_namespace n ON n.oid = parent.relnamespace INNER JOIN pg_class child ON pg_inherits.inhrelid = child.oid WHERE n.nspname = '|| schema_name ||' AND parent.relname = '|| table_name ||' UNION ALL -- table count SELECT (reltuples::float / greatest(relpages, 1)) * (pg_relation_size(c.oid)::float / (current_setting(''block_size'')::float))::integer AS estimate FROM pg_class c LEFT JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = '|| schema_name ||' AND c.relname = '|| table_name ||' ) SELECT CASE WHEN SUM(estimate) < '|| threshold ||' THEN (SELECT COUNT(*) FROM '|| identifier ||') ELSE SUM(estimate) END AS count FROM tables_counts' INTO count; RETURN count; END $$; -- -- Name: scrub_email(character varying); Type: FUNCTION; Schema: rideshare; Owner: - -- CREATE FUNCTION rideshare.scrub_email(email_address character varying) RETURNS character varying LANGUAGE sql AS $$ SELECT CONCAT( SUBSTR( MD5(RANDOM()::text), 0, GREATEST(LENGTH(SPLIT_PART(email_address, '@', 1)) + 1, 6) ), '@', SPLIT_PART(email_address, '@', 2) ); $$; -- -- Name: scrub_text(character varying); Type: FUNCTION; Schema: rideshare; Owner: - -- CREATE FUNCTION rideshare.scrub_text(input character varying) RETURNS character varying LANGUAGE sql AS $$ SELECT -- replace from position 0, to max(length or 6) SUBSTR( MD5(RANDOM()::text), 0, GREATEST(LENGTH(input) + 1, 6) ); $$; SET default_tablespace = ''; SET default_table_access_method = heap; -- -- Name: ar_internal_metadata; Type: TABLE; Schema: rideshare; Owner: - -- CREATE TABLE rideshare.ar_internal_metadata ( key character varying NOT NULL, value character varying, created_at timestamp(6) without time zone NOT NULL, updated_at timestamp(6) without time zone NOT NULL ); -- -- Name: trips; Type: TABLE; Schema: rideshare; Owner: - -- CREATE TABLE rideshare.trips ( id bigint NOT NULL, trip_request_id bigint NOT NULL, driver_id integer NOT NULL, completed_at timestamp without time zone, rating integer, created_at timestamp(6) without time zone NOT NULL, updated_at timestamp(6) without time zone NOT NULL, CONSTRAINT rating_check CHECK (((rating >= 1) AND (rating <= 5))) ); -- -- Name: users; Type: TABLE; Schema: rideshare; Owner: - -- CREATE TABLE rideshare.users ( id bigint NOT NULL, first_name character varying NOT NULL, last_name character varying NOT NULL, email character varying NOT NULL, type character varying NOT NULL, created_at timestamp(6) without time zone NOT NULL, updated_at timestamp(6) without time zone NOT NULL, password_digest character varying, trips_count integer, drivers_license_number character varying(100) ); -- -- Name: TABLE users; Type: COMMENT; Schema: rideshare; Owner: - -- COMMENT ON TABLE rideshare.users IS 'sensitive_fields|first_name:scrub_text,last_name:scrub_text,email:scrub_email'; -- -- Name: fast_search_results; Type: MATERIALIZED VIEW; Schema: rideshare; Owner: - -- CREATE MATERIALIZED VIEW rideshare.fast_search_results AS SELECT t.driver_id, concat(d.first_name, ' ', d.last_name) AS driver_name, avg(t.rating) AS avg_rating, count(t.rating) AS trip_count FROM (rideshare.trips t JOIN rideshare.users d ON ((t.driver_id = d.id))) GROUP BY t.driver_id, d.first_name, d.last_name ORDER BY (count(t.rating)) DESC WITH NO DATA; -- -- Name: locations; Type: TABLE; Schema: rideshare; Owner: - -- CREATE TABLE rideshare.locations ( id bigint NOT NULL, address character varying NOT NULL, created_at timestamp(6) without time zone NOT NULL, updated_at timestamp(6) without time zone NOT NULL, city character varying, state character(2) NOT NULL, "position" point NOT NULL, CONSTRAINT state_length_check CHECK ((length(state) = 2)) ); -- -- Name: locations_id_seq; Type: SEQUENCE; Schema: rideshare; Owner: - -- CREATE SEQUENCE rideshare.locations_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; -- -- Name: locations_id_seq; Type: SEQUENCE OWNED BY; Schema: rideshare; Owner: - -- ALTER SEQUENCE rideshare.locations_id_seq OWNED BY rideshare.locations.id; -- -- Name: schema_migrations; Type: TABLE; Schema: rideshare; Owner: - -- CREATE TABLE rideshare.schema_migrations ( version character varying NOT NULL ); -- -- Name: search_results; Type: VIEW; Schema: rideshare; Owner: - -- CREATE VIEW rideshare.search_results AS SELECT concat(d.first_name, ' ', d.last_name) AS driver_name, avg(t.rating) AS avg_rating, count(t.rating) AS trip_count FROM (rideshare.trips t JOIN rideshare.users d ON ((t.driver_id = d.id))) GROUP BY t.driver_id, d.first_name, d.last_name ORDER BY (count(t.rating)) DESC; -- -- Name: trip_positions; Type: TABLE; Schema: rideshare; Owner: - -- CREATE TABLE rideshare.trip_positions ( id bigint NOT NULL, "position" point NOT NULL, trip_id bigint NOT NULL, created_at timestamp(6) without time zone NOT NULL, updated_at timestamp(6) without time zone NOT NULL ); -- -- Name: trip_positions_id_seq; Type: SEQUENCE; Schema: rideshare; Owner: - -- CREATE SEQUENCE rideshare.trip_positions_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; -- -- Name: trip_positions_id_seq; Type: SEQUENCE OWNED BY; Schema: rideshare; Owner: - -- ALTER SEQUENCE rideshare.trip_positions_id_seq OWNED BY rideshare.trip_positions.id; -- -- Name: trip_requests; Type: TABLE; Schema: rideshare; Owner: - -- CREATE TABLE rideshare.trip_requests ( id bigint NOT NULL, rider_id integer NOT NULL, start_location_id integer NOT NULL, end_location_id integer NOT NULL, created_at timestamp(6) without time zone NOT NULL, updated_at timestamp(6) without time zone NOT NULL ); -- -- Name: trip_requests_id_seq; Type: SEQUENCE; Schema: rideshare; Owner: - -- CREATE SEQUENCE rideshare.trip_requests_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; -- -- Name: trip_requests_id_seq; Type: SEQUENCE OWNED BY; Schema: rideshare; Owner: - -- ALTER SEQUENCE rideshare.trip_requests_id_seq OWNED BY rideshare.trip_requests.id; -- -- Name: trips_id_seq; Type: SEQUENCE; Schema: rideshare; Owner: - -- CREATE SEQUENCE rideshare.trips_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; -- -- Name: trips_id_seq; Type: SEQUENCE OWNED BY; Schema: rideshare; Owner: - -- ALTER SEQUENCE rideshare.trips_id_seq OWNED BY rideshare.trips.id; -- -- Name: users_id_seq; Type: SEQUENCE; Schema: rideshare; Owner: - -- CREATE SEQUENCE rideshare.users_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; -- -- Name: users_id_seq; Type: SEQUENCE OWNED BY; Schema: rideshare; Owner: - -- ALTER SEQUENCE rideshare.users_id_seq OWNED BY rideshare.users.id; -- -- Name: vehicle_reservations; Type: TABLE; Schema: rideshare; Owner: - -- CREATE TABLE rideshare.vehicle_reservations ( id bigint NOT NULL, vehicle_id integer NOT NULL, trip_request_id integer NOT NULL, canceled boolean DEFAULT false NOT NULL, starts_at timestamp with time zone NOT NULL, ends_at timestamp with time zone NOT NULL, created_at timestamp(6) without time zone NOT NULL, updated_at timestamp(6) without time zone NOT NULL ); -- -- Name: vehicle_reservations_id_seq; Type: SEQUENCE; Schema: rideshare; Owner: - -- CREATE SEQUENCE rideshare.vehicle_reservations_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; -- -- Name: vehicle_reservations_id_seq; Type: SEQUENCE OWNED BY; Schema: rideshare; Owner: - -- ALTER SEQUENCE rideshare.vehicle_reservations_id_seq OWNED BY rideshare.vehicle_reservations.id; -- -- Name: vehicles; Type: TABLE; Schema: rideshare; Owner: - -- CREATE TABLE rideshare.vehicles ( id bigint NOT NULL, name character varying NOT NULL, created_at timestamp(6) without time zone NOT NULL, updated_at timestamp(6) without time zone NOT NULL, status rideshare.vehicle_status DEFAULT 'draft'::rideshare.vehicle_status NOT NULL ); -- -- Name: vehicles_id_seq; Type: SEQUENCE; Schema: rideshare; Owner: - -- CREATE SEQUENCE rideshare.vehicles_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; -- -- Name: vehicles_id_seq; Type: SEQUENCE OWNED BY; Schema: rideshare; Owner: - -- ALTER SEQUENCE rideshare.vehicles_id_seq OWNED BY rideshare.vehicles.id; -- -- Name: locations id; Type: DEFAULT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.locations ALTER COLUMN id SET DEFAULT nextval('rideshare.locations_id_seq'::regclass); -- -- Name: trip_positions id; Type: DEFAULT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.trip_positions ALTER COLUMN id SET DEFAULT nextval('rideshare.trip_positions_id_seq'::regclass); -- -- Name: trip_requests id; Type: DEFAULT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.trip_requests ALTER COLUMN id SET DEFAULT nextval('rideshare.trip_requests_id_seq'::regclass); -- -- Name: trips id; Type: DEFAULT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.trips ALTER COLUMN id SET DEFAULT nextval('rideshare.trips_id_seq'::regclass); -- -- Name: users id; Type: DEFAULT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.users ALTER COLUMN id SET DEFAULT nextval('rideshare.users_id_seq'::regclass); -- -- Name: vehicle_reservations id; Type: DEFAULT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.vehicle_reservations ALTER COLUMN id SET DEFAULT nextval('rideshare.vehicle_reservations_id_seq'::regclass); -- -- Name: vehicles id; Type: DEFAULT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.vehicles ALTER COLUMN id SET DEFAULT nextval('rideshare.vehicles_id_seq'::regclass); -- -- Name: ar_internal_metadata ar_internal_metadata_pkey; Type: CONSTRAINT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.ar_internal_metadata ADD CONSTRAINT ar_internal_metadata_pkey PRIMARY KEY (key); -- -- Name: trips chk_rails_4743ddc2d2; Type: CHECK CONSTRAINT; Schema: rideshare; Owner: - -- ALTER TABLE rideshare.trips ADD CONSTRAINT chk_rails_4743ddc2d2 CHECK ((completed_at > created_at)) NOT VALID; -- -- Name: locations locations_pkey; Type: CONSTRAINT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.locations ADD CONSTRAINT locations_pkey PRIMARY KEY (id); -- -- Name: vehicle_reservations non_overlapping_vehicle_registration; Type: CONSTRAINT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.vehicle_reservations ADD CONSTRAINT non_overlapping_vehicle_registration EXCLUDE USING gist (int4range(vehicle_id, vehicle_id, '[]'::text) WITH =, tstzrange(starts_at, ends_at) WITH &&) WHERE ((NOT canceled)); -- -- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.schema_migrations ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); -- -- Name: trip_positions trip_positions_pkey; Type: CONSTRAINT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.trip_positions ADD CONSTRAINT trip_positions_pkey PRIMARY KEY (id); -- -- Name: trip_requests trip_requests_pkey; Type: CONSTRAINT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.trip_requests ADD CONSTRAINT trip_requests_pkey PRIMARY KEY (id); -- -- Name: trips trips_pkey; Type: CONSTRAINT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.trips ADD CONSTRAINT trips_pkey PRIMARY KEY (id); -- -- Name: users users_pkey; Type: CONSTRAINT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.users ADD CONSTRAINT users_pkey PRIMARY KEY (id); -- -- Name: vehicle_reservations vehicle_reservations_pkey; Type: CONSTRAINT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.vehicle_reservations ADD CONSTRAINT vehicle_reservations_pkey PRIMARY KEY (id); -- -- Name: vehicles vehicles_pkey; Type: CONSTRAINT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.vehicles ADD CONSTRAINT vehicles_pkey PRIMARY KEY (id); -- -- Name: index_fast_search_results_on_driver_id; Type: INDEX; Schema: rideshare; Owner: - -- CREATE UNIQUE INDEX index_fast_search_results_on_driver_id ON rideshare.fast_search_results USING btree (driver_id); -- -- Name: index_locations_on_address; Type: INDEX; Schema: rideshare; Owner: - -- CREATE UNIQUE INDEX index_locations_on_address ON rideshare.locations USING btree (address); -- -- Name: index_trip_requests_on_end_location_id; Type: INDEX; Schema: rideshare; Owner: - -- CREATE INDEX index_trip_requests_on_end_location_id ON rideshare.trip_requests USING btree (end_location_id); -- -- Name: index_trip_requests_on_rider_id; Type: INDEX; Schema: rideshare; Owner: - -- CREATE INDEX index_trip_requests_on_rider_id ON rideshare.trip_requests USING btree (rider_id); -- -- Name: index_trip_requests_on_start_location_id; Type: INDEX; Schema: rideshare; Owner: - -- CREATE INDEX index_trip_requests_on_start_location_id ON rideshare.trip_requests USING btree (start_location_id); -- -- Name: index_trips_on_driver_id; Type: INDEX; Schema: rideshare; Owner: - -- CREATE INDEX index_trips_on_driver_id ON rideshare.trips USING btree (driver_id); -- -- Name: index_trips_on_rating; Type: INDEX; Schema: rideshare; Owner: - -- CREATE INDEX index_trips_on_rating ON rideshare.trips USING btree (rating); -- -- Name: index_trips_on_trip_request_id; Type: INDEX; Schema: rideshare; Owner: - -- CREATE INDEX index_trips_on_trip_request_id ON rideshare.trips USING btree (trip_request_id); -- -- Name: index_users_on_email; Type: INDEX; Schema: rideshare; Owner: - -- CREATE UNIQUE INDEX index_users_on_email ON rideshare.users USING btree (email); -- -- Name: index_users_on_last_name; Type: INDEX; Schema: rideshare; Owner: - -- CREATE INDEX index_users_on_last_name ON rideshare.users USING btree (last_name); -- -- Name: index_vehicle_reservations_on_vehicle_id; Type: INDEX; Schema: rideshare; Owner: - -- CREATE INDEX index_vehicle_reservations_on_vehicle_id ON rideshare.vehicle_reservations USING btree (vehicle_id); -- -- Name: index_vehicles_on_name; Type: INDEX; Schema: rideshare; Owner: - -- CREATE UNIQUE INDEX index_vehicles_on_name ON rideshare.vehicles USING btree (name); -- -- Name: trip_requests fk_rails_3fdebbfaca; Type: FK CONSTRAINT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.trip_requests ADD CONSTRAINT fk_rails_3fdebbfaca FOREIGN KEY (end_location_id) REFERENCES rideshare.locations(id); -- -- Name: vehicle_reservations fk_rails_59996232fc; Type: FK CONSTRAINT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.vehicle_reservations ADD CONSTRAINT fk_rails_59996232fc FOREIGN KEY (trip_request_id) REFERENCES rideshare.trip_requests(id); -- -- Name: trips fk_rails_6d92acb430; Type: FK CONSTRAINT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.trips ADD CONSTRAINT fk_rails_6d92acb430 FOREIGN KEY (trip_request_id) REFERENCES rideshare.trip_requests(id); -- -- Name: vehicle_reservations fk_rails_7edc8e666a; Type: FK CONSTRAINT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.vehicle_reservations ADD CONSTRAINT fk_rails_7edc8e666a FOREIGN KEY (vehicle_id) REFERENCES rideshare.vehicles(id); -- -- Name: trip_positions fk_rails_9688ac8706; Type: FK CONSTRAINT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.trip_positions ADD CONSTRAINT fk_rails_9688ac8706 FOREIGN KEY (trip_id) REFERENCES rideshare.trips(id); -- -- Name: trip_requests fk_rails_c17a139554; Type: FK CONSTRAINT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.trip_requests ADD CONSTRAINT fk_rails_c17a139554 FOREIGN KEY (rider_id) REFERENCES rideshare.users(id); -- -- Name: trips fk_rails_e7560abc33; Type: FK CONSTRAINT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.trips ADD CONSTRAINT fk_rails_e7560abc33 FOREIGN KEY (driver_id) REFERENCES rideshare.users(id); -- -- Name: trip_requests fk_rails_fa2679b626; Type: FK CONSTRAINT; Schema: rideshare; Owner: - -- ALTER TABLE ONLY rideshare.trip_requests ADD CONSTRAINT fk_rails_fa2679b626 FOREIGN KEY (start_location_id) REFERENCES rideshare.locations(id); -- -- PostgreSQL database dump complete -- SET search_path TO rideshare; INSERT INTO "schema_migrations" (version) VALUES ('20231220043547'), ('20231218215836'), ('20231213045957'), ('20231208050516'), ('20231018153712'), ('20231018153441'), ('20230925150831'), ('20230925150207'), ('20230726020548'), ('20230716174139'), ('20230714013609'), ('20230713150710'), ('20230713150550'), ('20230711015123'), ('20230625151410'), ('20230620030038'), ('20230619213546'), ('20230314210022'), ('20230314204931'), ('20230126025656'), ('20230125003946'), ('20230125003531'), ('20221230203627'), ('20221230200725'), ('20221223161403'), ('20221221052616'), ('20221220201836'), ('20221219164626'), ('20221111213918'), ('20221111212740'), ('20221110020532'), ('20221108175619'), ('20221108175321'), ('20221108172933'), ('20221007184855'), ('20220916171314'), ('20220814175213'), ('20220801140121'), ('20220729020430'), ('20220729014635'), ('20220716020213'), ('20220711015524'), ('20220711015454'), ('20220711010541'), ('20200603150442'), ('20191203213103'), ('20191203212055'), ('20191121175429'), ('20191112165848'), ('20191111151637'), ('20191108221519'), ('20191107212726'); ================================================ FILE: db/teardown.sh ================================================ export DB_URL="postgres://postgres:@localhost:5432/postgres" psql $DB_URL -c "DROP DATABASE IF EXISTS rideshare_development" psql $DB_URL -c "DROP DATABASE IF EXISTS rideshare_test" # https://stackoverflow.com/a/54078230/126688 psql $DB_URL -a -f db/teardown_remove_default_privileges.sql psql $DB_URL -c "DROP ROLE IF EXISTS owner" psql $DB_URL -c "DROP ROLE IF EXISTS readwrite_users" psql $DB_URL -c "DROP ROLE IF EXISTS readonly_users" psql $DB_URL -c "DROP ROLE IF EXISTS app" psql $DB_URL -c "DROP ROLE IF EXISTS app_readonly" ================================================ FILE: db/teardown_remove_default_privileges.sql ================================================ -- Reverse all the DEFAULT PRIVILEGES ....or -- https://stackoverflow.com/a/54078230/126688 -- Simpler solution: -- https://dba.stackexchange.com/a/155356/272968 REASSIGN OWNED BY owner TO postgres; DROP OWNED BY owner; ================================================ FILE: db/views/fast_search_results_v01.sql ================================================ -- list all drivers, to search within SELECT CONCAT(d.first_name, ' ', d.last_name) AS driver_name, AVG(t.rating) AS avg_rating, COUNT(t.rating) AS trip_count FROM trips t JOIN users d ON t.driver_id = d.id GROUP BY t.driver_id, d.first_name, d.last_name ORDER BY COUNT(t.rating) DESC; ================================================ FILE: db/views/fast_search_results_v02.sql ================================================ -- list all drivers, to search within SELECT t.driver_id, CONCAT(d.first_name, ' ', d.last_name) AS driver_name, AVG(t.rating) AS avg_rating, COUNT(t.rating) AS trip_count FROM trips t JOIN users d ON t.driver_id = d.id GROUP BY t.driver_id, d.first_name, d.last_name ORDER BY COUNT(t.rating) DESC; ================================================ FILE: db/views/search_results_v01.sql ================================================ -- list all drivers, to search within SELECT CONCAT(d.first_name, ' ', d.last_name) AS driver_name, AVG(t.rating) AS avg_rating, COUNT(t.rating) AS trip_count FROM trips t JOIN users d ON t.driver_id = d.id GROUP BY t.driver_id, d.first_name, d.last_name ORDER BY COUNT(t.rating) DESC; ================================================ FILE: docker/README.md ================================================ # Docker Docker is used to run PostgreSQL instances within a container, using a Docker network, and with different host names. For example "db01" is the primary host, and "db02" is a secondary host. These commands are intended in general to run as shell scripts, from this directory. ```sh sh docker/run_db_db01_primary.sh sh docker/run_db_db02_replica.sh docker ps ``` ## Disable Docker Messages ```sh export DOCKER_CLI_HINTS=false ``` ## Restarting container ```sh pg_ctl: cannot be run as root ``` docker restart ## Replacing `pg_hba.conf` content ```sh docker cp db01:/var/lib/postgresql/data/pg_hba.conf . cp pg_hba.conf pg_hba.backup.conf vim pg_hba.conf host replication replication_user 172.19.0.3/32 md5 docker cp pg_hba.conf db01:/var/lib/postgresql/data/. docker restart db01 ``` ## Standby process 1. Create replication slot 1. Create `pg_hba.conf` entries for replication_user. Use the IP address from db02 and db03 /32 version (IPv4) 1. Make sure there is a `standby.signal` file 1. Restart it (should restart in recovery mode) ## Docker permissions - Run `chown` and `chmod` on the `.pgpass` file - Use the `postgres` user ```sh docker exec --user root -it db02 chown postgres:root /var/lib/postgresql/.pgpass docker exec --user root -it db02 chmod 0600 /var/lib/postgresql/.pgpass ``` ================================================ FILE: docker/db01_create_publication.sh ================================================ #!/bin/bash # # Purpose: Create replication slot on primary db01 PGPASSWORD=postgres docker exec -it db01 \ psql -U postgres -c \ "CREATE PUBLICATION my_pub_inserts_only FOR ALL TABLES WITH (PUBLISH = 'INSERT');" ================================================ FILE: docker/db01_create_replication_slot.sh ================================================ #!/bin/bash # # Purpose: Create replication slot on primary db01 # PGPASSWORD=postgres docker exec -it db01 \ psql -U postgres -c \ "SELECT PG_CREATE_PHYSICAL_REPLICATION_SLOT('rideshare_slot');" # To remove the slot: # PGPASSWORD=postgres docker exec -it db01 \ # psql -U postgres -c \ # "SELECT PG_DROP_REPLICATION_SLOT('rideshare_slot');" ================================================ FILE: docker/db01_create_replication_user.sh ================================================ #!/bin/bash # # Purpose: # - Generate password, and place in .pgpass # - Create replication_user using generated password, on db01 # - Copy .pgpass to db02 # # The .pgpass password is used to authenticate replication_user, # when they run pg_basebackup # # Precondition: Make sure db01 and db02 are running # running_containers=$(docker ps --format "{{.Names}}") if echo "$running_containers" | grep -q "db01"; then echo "db01 is running...continuing" else echo "db01 is not running" echo "Exiting." exit 1 fi if echo "$running_containers" | grep -q "db02"; then echo "db02 is running...continuing" else echo "db02 is not running" echo "Exiting." exit 1 fi # Password for replication_user export REP_USER_PASSWORD=$(openssl rand -hex 12) echo "Create REP_USER_PASSWORD for replication_user" echo $REP_USER_PASSWORD # "rm replication_user.sql" for a clean starting point # CREATE USER statement as SQL file # Set password to DB_PASSWORD value rm -f replication_user.sql echo "CREATE USER replication_user WITH ENCRYPTED PASSWORD '$REP_USER_PASSWORD' REPLICATION LOGIN; GRANT SELECT ON ALL TABLES IN SCHEMA public TO replication_user; ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO replication_user;" >> replication_user.sql rm -f .pgpass echo "*:*:*:replication_user:$REP_USER_PASSWORD" >> .pgpass # Copy replication_user.sql to db01 docker cp replication_user.sql db01:. echo "Copy .pgpass, chown, chmod it for db02" # Copy .pgpass to db02 postgres home dir docker cp .pgpass db02:/var/lib/postgresql/. docker exec --user root -it db02 chown postgres:root /var/lib/postgresql/.pgpass docker exec --user root -it db02 chmod 0600 /var/lib/postgresql/.pgpass # Create replication_user on db01 docker exec -it db01 \ psql -U postgres \ -f /replication_user.sql ================================================ FILE: docker/db03_create_subscription.sh ================================================ # Preconditions: # - db01: wal_level = logical # - docker exec --user postgres -it db01 psql -c "SHOW wal_level" # - db03 is running # - db01 permits access from IP address of db03: # - See: ./db03_create_subscription_prepare.sh # - db01 has publication "my_pub_inserts_only" # Connect to db03 as "postgres" docker exec --user postgres -it db03 /bin/bash # To remove the subscription from /bin/bash db03 if needed: # This also removes "my_sub" replication slot on db01 # psql -U postgres -c "DROP SUBSCRIPTION my_sub" # Generate snippet and send to psql echo "CREATE SUBSCRIPTION my_sub CONNECTION 'dbname=postgres host=db01 user=replication_user' PUBLICATION my_pub_inserts_only;" | psql # View subscriptions psql -c "SELECT * FROM pg_subscription;" ================================================ FILE: docker/db03_create_subscription_prepare.sh ================================================ #!/bin/bash # # Purpose: start db03 # Copy .pgpass to it # # Precondition: .pgpass file exists/made earlier # sh run_db_db03_replica.sh echo "Copy .pgpass, chown, chmod it for db03" # Copy .pgpass to db03 postgres home dir docker cp .pgpass db03:/var/lib/postgresql/. docker exec --user root -it db03 chown postgres:root /var/lib/postgresql/.pgpass docker exec --user root -it db03 chmod 0600 /var/lib/postgresql/.pgpass echo "Getting IP address for db03..." ip2=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' db03) echo "$ip2" echo "Add this entry to pg_hba.conf" echo "host replication replication_user $ip2/32 md5" echo echo "When done, reload:" echo 'docker exec --user postgres -it db01 \ psql -c "SELECT pg_reload_conf();"' ================================================ FILE: docker/dump_rideshare_local_to_db01.sh ================================================ # Copy Rideshare db/setup.sh to db01 # including all the supporting SQL files docker exec -it db01 mkdir db docker cp db db01:. # Run "db/setup.sh" on db01, which should provision an empty # rideshare_development database on the db01 instance # On db01, the file is at "/setup.sh" in the root dir # Preconditions: # - env var DB_URL is set # - env var RIDESHARE_DB_PASSWORD is set # # These should be set locally *first* # so that they can be supplied to the container # docker exec --env DB_URL="$DB_URL" \ --env RIDESHARE_DB_PASSWORD="$RIDESHARE_DB_PASSWORD" \ db01 sh -c "/setup.sh" # Once created, we won't migrate there, since we'll be copying # tables using pg_dump # Connect to db01 and confirm: # - schema "rideshare" exists (\dn) # - database "rideshare_development" exists # - database is empty (has no tables) docker exec --user postgres -it db01 \ psql -d rideshare_development # Dump the local rideshare_development database into a file pg_dump -U postgres \ -h localhost rideshare_development > rideshare_dump.sql # Check the size du -h rideshare_dump.sql # Restore rideshare_development from the file # to db01 # Warning: this might take a few moments! PGPASSWORD=postgres psql -U postgres \ -h localhost \ -p 54321 \ -d rideshare_development < rideshare_dump.sql # Connect again and confirm the tables and row data # have been loaded # NOTE: connect as "owner" # docker exec --user postgres -it db01 \ psql -U owner -d rideshare_development # SELECT COUNT(*) FROM users; -- 20210 ================================================ FILE: docker/pg_hba_reset.sh ================================================ # Run from the "docker" directory in Rideshare # # Remove any existing file if exists rm -f pg_hba.conf echo "Getting IP address for db02..." ip_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' db02) echo "$ip_address" entry="host replication replication_user $ip_address/32 md5" echo "Generating pg_hba.conf file" cat <> pg_hba.conf # TYPE DATABASE USER ADDRESS METHOD # Replication $(echo "$entry") local all all trust # IPv4 local connections: host all all 127.0.0.1/32 trust # IPv6 local connections: host all all ::1/128 trust host all all all scram-sha-256 EOF cat pg_hba.conf echo echo "Copy pg_hba.conf to db01" docker cp pg_hba.conf db01:/var/lib/postgresql/data/. ================================================ FILE: docker/reset_docker_instances.sh ================================================ #!/bin/bash # We've assumed this was copied locally from the db01 container conf_file="postgresql.conf" if [ -e "$conf_file" ]; then echo "File '$conf_file' exists...continuing" else echo "File '$conf_file' does not exist. Run:" echo echo "docker cp db01:/var/lib/postgresql/data/$conf_file ." echo echo "Then try again." exit 1 fi trap 'echo "An error occurred with command: $BASH_COMMAND";' ERR docker stop db01 && docker rm db01 docker stop db02 && docker rm db02 docker stop db03 && docker rm db03 echo "Stopped containers, waiting a moment" sleep 1 sh run_db_db01_primary.sh sh run_db_db02_replica.sh echo "Started containers" docker ps sleep 1 sh pg_hba_reset.sh echo "Restart db01 received new file" docker restart db01 sleep 2 echo "Create replication slot on db01" sh db01_create_replication_slot.sh echo "Configure replication_user" sh db01_create_replication_user.sh echo "Copy existing postgresql.conf to db01" docker cp postgresql.conf db01:/var/lib/postgresql/data/. echo "restart db01" docker restart db01 ================================================ FILE: docker/run_db_db01_primary.sh ================================================ #!/bin/bash # # Run from Rideshare dir # Use bind dir: ./postgres-docker/db01 # network: "rideshare-net" docker run \ --name db01 \ --volume ${PWD}/postgres-docker/db01:/var/lib/postgresql \ --publish 54321:5432 \ --env POSTGRES_USER=postgres \ --env POSTGRES_PASSWORD=postgres \ --net=rideshare-net \ --detach postgres:16.1 ================================================ FILE: docker/run_db_db02_replica.sh ================================================ #!/bin/bash # # Run from Rideshare dir # Use bind dir: ./postgres-docker/db02 # network: "rideshare-net" docker run \ --name db02 \ --volume ${PWD}/postgres-docker/db02:/var/lib/postgresql/data \ --publish 54322:5432 \ --env POSTGRES_USER=postgres \ --env POSTGRES_PASSWORD=postgres \ --net=rideshare-net \ --detach postgres:16.1 ================================================ FILE: docker/run_db_db03_replica.sh ================================================ #!/bin/bash # # db03 uses Logical Replication # docker run \ --name db03 \ --volume ${PWD}/postgres-docker/db03:/var/lib/postgresql/data \ --publish 54323:5432 \ --env POSTGRES_USER=postgres \ --env POSTGRES_PASSWORD=postgres \ --net=rideshare-net \ --detach postgres:16.1 ================================================ FILE: docker/run_pg_basebackup.sh ================================================ # Connect to db02 as "postgres" # replication_user - authenticates from db02 host docker exec --user postgres -it db02 /bin/bash # ############# WARNING ############ # # Copy the "rm" and "pg_basebackup" commands # to clipboard at once, so they can be pasted together # # Dependencies: # - "rideshare_slot" exists # - replication_user exists, with password supplied from ~/.pgpass # - db01 and db02 are running # # ################################## rm -rf /var/lib/postgresql/data/* && \ pg_basebackup --host db01 \ --username replication_user \ --pgdata /var/lib/postgresql/data \ --verbose \ --progress \ --wal-method stream \ --write-recovery-conf \ --slot=rideshare_slot # Container "stops" from removing the data directory # NOTE: Start it again, and it should use the same # replaced data directory docker start db02 # Review live logs docker logs -f db02 ================================================ FILE: docker/teardown_docker.sh ================================================ #!/bin/bash # # Drop slots # - my_subscription # - rideshare_slot PGPASSWORD=postgres docker exec -it db01 \ psql -U postgres -c \ "SELECT pg_drop_replication_slot('my_sub');" PGPASSWORD=postgres docker exec -it db01 \ psql -U postgres -c \ "SELECT pg_drop_replication_slot('rideshare_slot');" PGPASSWORD=postgres docker exec -it db01 \ psql -U postgres -c \ "REASSIGN OWNED BY replication_user TO postgres;" PGPASSWORD=postgres docker exec -it db01 \ psql -U postgres -c \ "DROP OWNED BY replication_user;" docker exec -it db01 \ psql -U postgres \ -c "DROP USER IF EXISTS replication_user" echo "Stop everything if needed" docker stop db01 && docker rm db01 docker stop db02 && docker rm db02 docker stop db03 && docker rm db03 echo "Removing local postgres-docker directory" rm -rf postgres-docker ================================================ FILE: docs/design_document.md ================================================ ## Welcome This document has entries that were chunks of work on this app. The entries are ordered reverse chronologically, so the first entry is on the bottom. Start from the bottom and work up to navigate the design decisions made along the way. ## 2019-11-21 Add `strong_migrations` gem, which helps prevent migrations that introduce downtime. I think this is a great project and wanted to rep it here. Add `blazer` gem for a demonstration of data reporting. ## 2019-11-15 Adding Trip Search on at least 2 dimensions Uniqueness on Trip Requests, they should not have the same start and end location. ## 2019-11-12 Trip model. Add rating. Ensure Trip is complete before it can be rated. Use a `completed_at` timestamp as the initial way to record the trip status, either complete or not. We may wish to add a state machine later, and states like `pending`->`in_progress`->`completed` etc. Idea: User communication (also becomes a uniqueness dimension): add email address? Indexing Dos and Dont's Use `db/schema.rb` and not `db/structure.sql` Trip has a `trip_request_id` FK, we could ensure it exists before create :bulb: Best practice: using `delegate`. Since `TripRequest` already has a Rider, and Trip references TripRequest, we can use `delegate` to access the Rider for a Trip Idea: DB check constraint on rating, `completed_at` IS NOT NULL, pros and cons Idea: Consider using an [Architectural Design Record (ADR)](https://adr.github.io/) style for the Iterations log? ## 2019-11-11 :bulb: Patterns: Introduce Geocoder gem. In order to automatically `geocode` on create, we can use the `after_validation` hook. :bulb: Patterns: has_one/belongs_to it's about where the FK is. For trip_requests, the foreign key to a rider is on the table, so a TripRequest `belongs_to` a Rider. :bulb: Trade-off: with this Location data model, we couldn't take advantage currently of common locations being shared among trip requests. Patterns: ActionController::API, lightweight version of `ActionController::Base` . We're creating an `ApiController` that extends `ActionController::API` as we're intending this to be an API app. :bulb: Patterns: Strong params for Trip Request creation, model attribute params are forbidden to be used for mass assignment until they have been permitted Patterns: Fixtures. Use fixtures for test objects that will be re-used, and not change often (like riders) Patterns: use namespace for `/api` routes :bulb: Trade-off: move current_rider to API controller, requires unnesting the rider_id inside the trip request :bulb: Best practice: render 201 when trip request was created, or unprocessable entity (422) when it failed Best practice: use the geocoder initializer `rails generate geocoder:config`, and customize the testing behavior so lookups are not happening in test mode. NOTE: uniqueness among trip request records. The same rider may travel the same trip, so we might want another dimension for uniqueness. NOTE: We could nest requests and ratings under trips, e.g. /api/trips, /api/trips/requests, /api/trips/ratings ## 2019-11-08 Keeping the `Location` simple for now, a trip would have a start and end location, a location is a lat/lng pair. A `TripRequest` would be a geocoded rider position based on their current location, and a geocoded address of their destination (will need a geocoder) Skip TripRating for now and put a `rating:integer` on the Trip for now. :bulb: Trade-off: this is simpler than a dedicate model. We can still do these aggregate calculations with a simple integer field: * Average trip rating rider has provided * Average trip rating for a driver * Avoiding the mutual rating feature (`rider->driver, driver->rider`) for now :bulb: Pattern: Rails STI: use a `type` column and by creating the object using the subclass type, the type information will be stored as a string in the table. The same works when querying, by asking for a particular record, the type information is surfaced as the type of the class. ## 2019-11-07 Initial thoughts The purpose of this app is to model car-based ride sharing, like Uber or Lyft. This is my take on some objects and their interactions that model this domain. The main model is a Trip and then there are Drivers that provide the trip, and Riders that take the trip. Some Active Record model ideas and notes below. Single-table inheritance can be used for both the Driver and Rider in a `users` table. :bulb: Trade-off: this saves a bit of initial work having separate models and potentially, duplication between two similar models. ``` Driver(name:string) (use STI?) Rider(name:string) (use STI?) Location(driver_id:integer,rider_id:integer,latitude:decimal,longitude:decimal) TripRequest(rider_id:integer,start:location_id,end:location_id) Trip(trip_request_id:integer,driver_id:integer,rider_id:integer,rating:integer) ~~TripRating(trip_id:integer,rating:integer)~~ ``` Integer IDs (PK and FKs). :bulb: Trade-off: integer primary keys can be exhausted at large scale, and auto increment IDs can be guessable which has security concerns. UUIDs or GUIDs are an alternative, but reduce usability. Use cases: * A rider makes a trip request, including a start location (geolocate current) and end location (enter destination) * A driver accepts the trip request * A trip involving a driver and rider begins, the location is tracked (includes driver and rider) * A trip involving a driver and rider completes (driver and rider) * A rider can rate a trip ================================================ FILE: docs/dev_tips.md ================================================ ## Postgres versions I had Homebrew Postgres set up in my path for `pg_dump` but wanted to use the version with Postgres.app. Undesired version: ``` /opt/homebrew/opt/libpq/bin/pg_dump ``` Desired version: ``` /Applications/Postgres.app/Contents/Versions/18/bin/pg_dump ``` Since I use fish shell I fixed it by running: ```sh set -U fish_user_paths /Applications/Postgres.app/Contents/Versions/18/bin $fish_user_paths ``` Verify: ```sh pg_dump --version ``` ================================================ FILE: docs/development.md ================================================ ## Ctags I use it, and running: `ctags -R --exclude=.git --exclude=test` ================================================ FILE: docs/development_iterations.md ================================================ # Development Iterations Development was done in small iterations over time, and the work was tracked here. Consider this a development journal that's a "build in public" that may be interesting to others, although it was mostly written for my own needs as journal. ## Iteration 27 Partition the `trip_positions` table using `pgslice`. * Add `pgslice` to Gemfile and install the binstub ```sh # add 'pgslice` to Gemfile bundle install bundle binstubs pgslice ``` Invoke it with `rails runner`, e.g. `bin/rails runner "PgsliceHelper.new.add_partitions"` TODO, but deferred * `insert_all` compatibility ## Iteration 26 (2023) - Remove webpacker, and most front-end JS (this is an API app) - Retire Blazer. It's a great tool, but no longer part of the goal of this app. ```sh gem update --system brew upgrade ruby-build rbenv install 3.2.0 gem install bundler bundle install bundle update bin/rails test ``` ## Iteration 25 Add Full Text Search (FTS). Add `pg_search` to evaluate the features. - tsearch - Full text search, which is built-in to PostgreSQL - trigram - Trigram search, which requires the trigram extension - dmetaphone - Double Metaphone search, which requires the fuzzystrmatch extension ## Iteration 24 Add slow query logging using Active Support Instrumentation without 3rd party gems or PostgreSQL extensions When configured to log at >= 1 second duration, test it with: ```rb ActiveRecord::Base.connection.execute("select pg_sleep(1)") ``` ## Iteration 23 - Add Trip Position model, and populate it with sample rows - Remove some experimental PG extensions from the application DB - Perform a conversion from unpartitioned to partitioned trip_positions table using pgslice `drop extension sslinfo`, `drop extension pg_buffercache` for now, these may return later. This cleans up the `db/structure.sql` so that it reflects the extensions in use by the application. ## Iteration 22 - Maintain the data generators - Disable Prepared Statements for now - Start using Active Record Doctor gem: `bundle exec rake active_record_doctor` for more insights ## Iteration 21 Trip rating database CHECK constraint. ## Iteration 20 Counter cache example for trips that belong to a driver. ## Iteration 19 Vehicle Reservation concept (e.g. special car, limo). Has a reservation duration. When vehicle is reserved, cannot be overlapping reservation. Create an exclusion constraint. Run a specific test like this: `rails test test/services/book_reservation_test.rb -n BookReservationTest#test_can_NOT_book_overlapping_reservation` ## Iteration 18 Rails Entity Relationship Diagram (ERD) [Customization](https://voormedia.github.io/rails-erd/customise.html) ``` bundle exec rake erd \ inheritance=true \ only="Driver,Rider,User,Location,TripRequest,Trip,Vehicle,VehicleReservation" \ attributes=foreign_keys,primary_keys ``` ## Iteration 17 Start a pgbench benchmark basics. Add fx gem to manage DB functions (pl/pgsql). Add data scrub functions. Add paranoia gem and create some deleted users for the purposes of different query types. ## Iteration 16 (2022) * Upgrade to Rails 7. Remove some gems. ## Iteration 15 * Add [PgHero](https://github.com/ankane/pghero) * Use new CircleCI docker configuration ## Iteration 14 Upgrade Rails 6.0->6.1 ## Iteration 13 Add JSON Web Token support for authenticated API actions. More details TBD. ## Iteration 12 Plan out a public API. A rider's "my trips" API. Includes driver details, maybe additional information like my rating, average rating. Includes start and end location. Use fast_jsonapi and some of the JSON API features, like sparse fieldsets and compound documents. ## Iteration 11 Introduce ETag HTTP caching to the trips API. `ETag` is content-based HTTP caching built in to Rails. ETags can be strong or weak, weak ETags are used by default in Rails, and are identified with a `W/` on the front, e.g. `W/"02d4d6729566d6bb56f0aa9e644c8c93"`. Collections (an `ActiveRecord::Relation`) are supported, although they will be covered here in the future, for now this uses a `trips#show` API as a demonstration. Sending a curl request and asking for headers only, we can see an ETag as a response header, and a 200 status code. Using that ETag value as a request header, for example below, if the content for this trip has not changed, we'll see a `304 Not Modified` response. ``` curl -I --header 'If-None-Match: W/"02d4d6729566d6bb56f0aa9e644c8c93"' localhost:3000/api/trips/1 ``` We can open a console and updated this trip, e.g. `Trip.find(1).touch`, and then sending the same ETag, we'll see the trip is rendered again, and we get a 200 response as expected, since the content of the trip has changed (the `updated_at` timestamp was updated). Another response header that `stale?` introduces (this header doesn't seem to appear with a regular `render`) is `Last-Modified`, e.g. as a header and value an example is `Last-Modified: Thu, 14 May 2020 01:42:08 GMT`. Now we can create a curl request with the request header `If-Modified-Since` and this timestamp, e.g. ``` curl -i --header 'If-Modified-Since: Thu, 14 May 2020 01:42:08 GMT' localhost:3000/api/trips/1 ``` And confirm that we receive a `304 Not Modified`. Updating the trip and sending an equivalent request responds with a `200`, which makes sense since the trip has been updated. And similarly, if we replace the timestamp value with the new value from the `Last-Modified`, we are back to getting a `304 Not Modified` response. ## Iteration 10 Use Circle CI as a CI system. Set it up so that pushes on master kick off a test test. The repo has a status badge indicating whether the tests are passing or not. ## Iteration 9 Add two great tools, [Strong Migrations](https://github.com/ankane/strong_migrations) and [Blazer](https://github.com/ankane/blazer). Strong Migrations ensures that migrations will be safe to run in production, avoiding known risky operations. Blazer is a simple platform for doing data analysis and data pulls. We used this extensively at a previous job and allowed any team member with SQL experience to learn about the data, satisfying their own reporting needs, and served as a repository of knowledge about common operations-related data and queries. I created a Driver and Rider dashboard here with some queries to look at Top Rated Drivers, and the most Active Riders. Driver and Rider Blazer dashboard ## Iteration 8 Improve test code coverage and maintain a `1:0.6` code to test ratio. `rake stats` ``` Code LOC: 198 Test LOC: 115 Code to Test Ratio: 1:0.6 ``` Put together a [Trip Search Sequence Diagram](https://www.planttext.com/). ``` @startuml title "Trip Search Sequence Diagram" actor User boundary "TripSearch" User -> TripSearch : Search by start location, driver name, rider name TripSearch -> User : Respond with matching Trips @enduml ``` Trip search ## Iteration 7 Dockerize the application. * Change the `database.yml` and set the `host: db` * Install `yarn` ### Docker Commands * `docker-compose build` * `docker-compose up` * `docker-compose run web bundle exec rake db:create` * `docker-compose run web bundle exec rake db:migrate` * `docker-compose run web bundle exec rake data_generators:trips` Now query for some data: `curl http://localhost:3000/api/trips?start_location=New%20York&driver_name=Kasie` ## Iteration 6 Add integration and model tests for trip search. Add trip search by multiple dimensions (Driver name, Rider name, Location). ## Iteration 5 Generate sample Driver, Trip, Rider, and Rating data (`rake data_generators:trips`). Add basic Driver dashboard. Show driver stats. Driver dashboard ## Iteration 4 UML Sequence Diagram of Rider, Driver, Trip Request, Trip, and Rating messages ``` @startuml title "Rider, Driver, Trip Sequence Diagram" actor Rider boundary "TripRequest" actor Driver entity Trip Rider -> Rider : Enters Start and End Location Rider -> TripRequest : Requests Trip Driver -> TripRequest : Accepts Trip Request TripRequest -> Trip : Trip Starts Trip -> Trip : Trip Ends Rider -> Trip : Rider Rates Trip @enduml ``` Trips Sequence diagram ## Iteration 3 * Trip model, created when a Driver accepts a Trip Request * Ratings: Completed Trips can be rated ## Iteration 2 * Location (Geo coordinates) and Trip Request models * API base controller * Trip Requests `index` and `create` API endpoints ## Iteration 1 * Started with a [Design Document](/docs/design_document.md) * Wrote out use cases of Riders, Drivers etc. * Planning models, database tables, constraints, validations * Using single-table inheritance for Driver and Rider instances in a Users table ================================================ FILE: docs/project_documentation.md ================================================ ## My Rails Best Practices and Patterns Demonstrations of each of these items can be found in the app * Data Integrity (in the DB and application) * Enforce Null Constraints * Foreign key constraints for referential integrity * Unique constraints * Exclusion * Code Quality * Rails best practices gem (`bin/rails_best_practices .`) * Strong Migrations * Use `delegate` in models * Strong Params * Performance * DB indexes * Primary, uniqueness, indexed foreign key columns * Named Scopes * Search functionality * Automatic Geocoding * Use callbacks * Disable geocoding in the test environment * Testing * Fixtures and factories * Minimum Code to Test Ratio: 1:0.6 (use `bin/rails stats`) * Fake data generators for local development (`faker` gem, rake task), SQL data loads * API Application * We only need an API, use `ActionController::API` for lighter weight API code * Use `/api` namespace * JSON:API for API standardization * Sparse Fieldsets * Compound Documents * Status codes * `201` on created * `422` on error * HTTP Caching (ETag, Last Modified, static content) * Use [Single table inheritance](https://api.rubyonrails.org/v7.1.4/classes/ActiveRecord/Base.html#class-ActiveRecord::Base-label-Single+table+inheritance) when appropriate * Link: [DB migration commit](https://github.com/andyatkinson/rideshare/commit/39232da339c2c04966e49e3e4ff03d88c2e66842#diff-7d736cc988a61ff29b4b9b2466b7a6ab) ================================================ FILE: docs/search.md ================================================ ## Search - Using `pg_search` gem, adds AR scopes - `tsearch` is built in, PostgreSQL Full Text Search - Creates a `tsvector` from document text - Search it using a `tsquery` - Rank fields using `ts_rank()` - Store tsvectors using a Generated column - Index tsvectors using a GIN index - Add `users.searchable_full_name` `GENERATED ALWAYS` - For unaccent, consider an expression based index ================================================ FILE: docs/workshop/0_introduction.md ================================================ # Introduction ## Prerequisites Checklist You have Rideshare running: - `rideshare_development` database is reachable - `bin/rails console` works - DB creation scripts ran - Migrations ran (`bin/rails db:migrate`) If any of these aren't completed, go back to the main [Workshop README](/docs/workshop/README.md) ## Setup - Run shell scripts from Rideshare root directory - Learn to add psql to your `bin/rails console` command-line tools - Create indexes without Active Record ## Performance - Individual query optimization (micro) - Macro query optimization, reduce system load # Micro Optimization - Benefit: Lessen load on server - Query planning basics - Index design basics - Index design more advanced # Macro Optimization - Benefit: Lessen load, distribute load - Find worst performing queries - Move read only queries to a read replica ## What's Next? Visit [1 - Psql Basics](/docs/workshop/1_psql_basics.md) to continue. ================================================ FILE: docs/workshop/1_psql_basics.md ================================================ # psql basics psql is the command-line client that comes with PostgreSQL. We will use it. Running `bin/rails dbconsole` (or `db` for short), it launches psql. The connection string is supplied from the .env file We want the one called `DATABASE_URL`. ```sql cd rideshare cat .env | grep DATABASE_URL bin/rails db ``` We can also connect without `bin/rails dbconsole` and use psql directly. ```sh export DATABASE_URL=postgres://owner@localhost:5432/rideshare_development psql $DATABASE_URL ``` ## What's Next? Visit [2 - Shell Scripts](/docs/workshop/2_shell_scripts.md) to continue. ================================================ FILE: docs/workshop/2_shell_scripts.md ================================================ # Shell Script Basics Let's load more data. You may remove all data if needed. ⚠️ (Optional) WARNING: Run this to remove all data and start over. ```sh bin/rails db:reset ``` If you've migrated the database and it's empty, let's first load some sample data from Rake scripts you're familiar with. ```sh cd rideshare bin/rails data_generators:generate_all ``` Bulk load via SQL commands running in psql ```sh sh db/scripts/bulk_load.sh sh db/scripts/bulk_load_extended.sh ``` ## What's Next? Visit [3 - Query Planning](/docs/workshop/3_query_planning.md) to continue. ================================================ FILE: docs/workshop/3_query_planning.md ================================================ # Query Planning We'll use psql (or run `bin/rails db`). ```sql psql $DATABASE_URL ``` Tip to clear: `\! clear`. ## Section 1: We need a query We need a query. Let's get all users that have a certain **first** name. Let's find one from the existing rows. ```sql SELECT first_name FROM users ORDER BY id ASC LIMIT 1; first_name ------------ Alphonso ``` ## Section 2: Enabling timing Toggle timing to `on`. ```sql \timing Timing is on. ``` ```sql SELECT * FROM users WHERE first_name = 'Alphonso'; -- Type "q" to exit results -- Time: 2012.136 ms (00:02.012) ``` On my machine, this returns 8 rows, taking around 2 seconds. Two seconds is quite slow! Let's look at the query plan. To do that we'll use the `EXPLAIN` keyword. ```sql EXPLAIN SELECT * FROM users WHERE first_name = 'Alphonso'; ``` ## Section 3: Intro to [`EXPLAIN`](https://www.postgresql.org/docs/current/using-explain.html) Let's understand the parts of what we're seeing. - Plan step is contained within the one above it - Filter operation, condition to match, rows removed by filter (when using `ANALYZE`) - Sequential scan on `users` table - Parallel sequential scan using 2 workers - Estimated to match one row (`rows=1`) but we know there are more - Width is "estimated average width of rows" - The cost is based on how many disk pages are accessed Let's get into the cost details more. ## Section 4: Pages Intro and Cost calculation Data in PostgreSQL is stored in "pages" which are fixed size 8kb (by default) chunks. Row data and index data are stored in the pages. For this workshop, we won't go into greater depth. Just know that more pages = slower query. Less pages = faster query. Let's look at a simplified version of the query: - Let's disable parallel sequential scans (max worker of 1) ```sql SET max_parallel_workers_per_gather = 1; ``` - Let's scan the whole table with no `WHERE` clause - Let's get the number of pages for the table - Let's manually reproduce the cost formula calculation Let's run the simplified query from psql: ```sql EXPLAIN SELECT * FROM users; QUERY PLAN ------------------------------------------------------------------- Seq Scan on users (cost=0.00..247873.94 rows=10020294 width=129) ``` The rounded estimated cost is 247874. Let's recalculate 247874. To start, get the number of pages from psql used to store all the rows: ```sql SELECT relpages AS pages, reltuples::numeric AS estimated_rows FROM pg_class WHERE relname = 'users'; pages | estimated_rows --------+---------------- 147671 | 10020300 ``` Cost formula from docs: `(pages * seq_page_cost) + (estimated_rows * cpu_tuple_cost)` Cost calculation components: - Pages: `147671` - Estimated rows: `10020300` - `SHOW seq_page_cost;` (`1`) - `SHOW cpu_tuple_cost;` (`0.1`) ```sql SELECT FLOOR((147671 * 1) + (10020300 * 0.01)) AS estimated_cost; estimated_cost ---------------- 247874 ``` Now we understand some planner information, let's continue on with query optimization. ## What's Next? Visit [4 - Query Optimization](/docs/workshop/4_query_optimization.md) to continue. ================================================ FILE: docs/workshop/4_query_optimization.md ================================================ # Query Optimization Part 1 We're still in psql. We've enabled timing. We have a slow query of user rows filtered by first name: ```sql SELECT * FROM users WHERE first_name = 'Alphonso'; ``` We know the plan type is a sequential scan. ## Section 1: Index Design Basics The most significant way we can improve performance for this query is to add an index that supports the query. Why is that? The index *duplicates* the `first_name` column value from every row, into an ordered data structure. Benefits: - The index is faster to scan and filter on, being ordered (in ascending or descending order) - The index entries are maintained for us as new writes happen Downsides: - Indexed fields add latency to write operations, since the fields are maintained as index entries Optimization Game Plan: - Identify the column we are filtering on. - We are filtering on `users.first_name` - Create a B-Tree index that includes the first name column Do this in psql. We can replay it in Active Record later. ```sql -- Enable timing to see build time \timing CREATE INDEX idx_first_name ON users (first_name); ``` This took around 10 seconds to build. Before analyzing the improvement, let's discuss the details. ## Section 2: Index Definition Analysis and Query Results - This is a "single column" index - This is using the default index type which is B-Tree, since it's unspecified - We are picking all rows from the table - We are using the default sort order - We are using the default `NULL` handling (although `first_name` doesn't allow nulls) Let's view our index in psql: ```sql \d users ``` With the index in place, let's re-run the query. Make sure `\timing` is enabled. Remember the query time before was around 0.5-1.5 seconds. ```sql SELECT * FROM users WHERE first_name = 'Alphonso'; ``` The query now takes milliseconds or less to run, which is tremendously faster. Why is that? ## Section 3: Index Design Concepts Let's look at the query plan. Let's introduce `ANALYZE` now to run the query. ```sql EXPLAIN (ANALYZE) SELECT * FROM users WHERE first_name = 'Alphonso'; ``` ```sql \dt+ users -- 1154MB size \di+ idx_first_name -- 301MB ``` Table size vs. index size: - Now we're scanning the index which is smaller, it contains one column, and it's in order - This is an Index Scan using the index we created `idx_first_name` - We still "filter" on the index, but with much less data access - Startup and actual costs are much lower compared with before - Actual rows shows 8 rows, 1 loop PostgreSQL still needs to access more fields (`SELECT *`) from the heap/table storage, but for a small filtered set of rows. Can we do better? ## What's Next? Visit [5 - Query Optimization Part 2](/docs/workshop/5_query_optimization_part_2.md) to continue. ================================================ FILE: docs/workshop/5_query_optimization_part_2.md ================================================ # Query Optimization: Part 2 ## Section 1: Efficiency Design Concepts - Add more restrictions to the query - Add indexes to support more restricted query ```sql EXPLAIN (ANALYZE) SELECT * FROM users WHERE first_name = 'Alphonso'; ``` ## Section 2: Filtering On Rows We can reduce the rows in our index. When we do that, we're making a [Partial Index](https://www.postgresql.org/docs/current/indexes-partial.html). Let's explore our data and loop for opportunities. We store different `type` values in this table, so let's `COUNT()` by type for first name "Alphonso". ```sql SELECT type, COUNT(*) FROM users WHERE first_name = 'Alphonso' GROUP BY type; type | count --------+------- Driver | 4 Rider | 4 ``` Imagine we only wanted to index the Driver type, since this query finds drivers. We can limit our index to just the Drivers. Let's drop our current index, and add it back with the same name. ```sql -- Drop existing index DROP INDEX IF EXISTS idx_first_name; CREATE INDEX idx_first_name ON users (first_name) WHERE (type = 'Driver'); ``` Let's run: `\di+ idx_first_name;` again and this time we see the index is half the size at 151MB vs. 301MB. Let's run our query again: ```sql EXPLAIN (ANALYZE) SELECT * FROM users WHERE first_name = 'Alphonso'; ``` 😲 It's slow! We need to add this same condition we added to the index, to the query. Let's try that: ```sql EXPLAIN (ANALYZE) SELECT * FROM users WHERE first_name = 'Alphonso' AND type = 'Driver'; -- This is the new condition ``` Now it's super fast again. It's using our index. There are only 4 result rows which makes sense. Can we do better? ## Section 2: Filtering On Columns Besides filtering rows in our index, we can filter columns picked in both our query and index definition. By including the exact set of columns our query needs instead of `SELECT *`, PostgreSQL can get all needed data from the index alone, which is very fast. Let's imagine we needed the `id` of the `Driver` types of `users` named "Alphonso". Let's change our query first and see if it's better: ```sql EXPLAIN (ANALYZE) SELECT id, first_name FROM users WHERE first_name = 'Alphonso' AND type = 'Driver'; -- This is the new condition ``` It's not really any better despite reducing our columns. This is because our current index does not include the `id` column. PostgreSQL can't get all field data from the index alone. Let's replace it with a new definition that includes those two columns. ```sql -- Drop existing index DROP INDEX IF EXISTS idx_first_name; CREATE INDEX idx_first_name ON users (first_name, id) WHERE (type = 'Driver'); ``` We now have: - A partial index - A multicolumn index, where the leading column is our filtered column Let's check the query plan. ```sql EXPLAIN (ANALYZE) SELECT id, first_name FROM users WHERE first_name = 'Alphonso' AND type = 'Driver'; ``` We're now getting the most efficient plan type possible, which is the "Index Only Scan." This is because our index contains the full set of needed columns for the query, meaning PostgreSQL only needs to access the index. ```sql QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------- Index Only Scan using idx_first_name on users (cost=0.43..8.45 rows=1 width=20) (actual time=0.032..0.034 rows=4 loops=1) Index Cond: (first_name = 'Alphonso'::text) Heap Fetches: 0 Planning Time: 0.126 ms Execution Time: 0.055 ms ``` ## What's Next? Visit [6 - Macro Query Optimization Part 1](/docs/workshop/6_macro_overview_part_1.md) to continue. ================================================ FILE: docs/workshop/6_macro_overview_part_1.md ================================================ # Macro Query Optimization Part 1 In the last few sections, we learned about "micro" or individual query optimization. To make broad improvements, we can apply the same concepts across all our queries. - Tactic #1: Find all the slow queries, and focus on high impact ones - Tactic #2: For read-only queries, i.e. the `SELECT` queries but not `INSERT`, `UPDATE`, and `DELETE`, distribute them to a second read-only PostgreSQL instance (a.k.a. replica, follower, secondary) To do that, we will explore: - The `pg_stat_statements` extension - Read and Write Splitting with Active Record Let's improve our DBA skills!
🎥 Configuring and using pg_stat_statements data, creating generic query exec plans
## Section 1: Configure `pg_stat_statements` While being an extension, it's officially supported by PostgreSQL and distributed with it, but is not enabled by default. We need to enable it using a superuser, for the `rideshare_development` database, in the `rideshare` schema. ⚠️ This part won't be included in the workshop due to time, or can be a self-study opportunity. Presenter will demo. ```sh vim "/Users/andy/Library/Application Support/Postgres/var-16/postgresql.conf" # edit shared_preload_libraries shared_preload_libraries = 'pg_stat_statements' # Restart PostgreSQL pg_ctl restart --pgdata "/Users/andy/Library/Application Support/Postgres/var-16/" # Connect as superuser, e.g. "postgres" psql -U postgres -d rideshare_development # Enable the extension (run `CREATE EXTENSION`) postgres@[local]:5432 rideshare_development# \dx List of installed extensions Name | Version | Schema | Description ---------+---------+------------+------------------------------ plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language # Loads into current database CREATE EXTENSION IF NOT EXISTS pg_stat_statements SCHEMA rideshare; # Reset (Requires superuser) WARNING: Removes stats data SET search_path = 'rideshare'; SELECT pg_stat_statements_reset(); \q -- quit psql ``` We can go back to our less-privileged app user `owner`. Now we're ready to view the PGSS data. Let's connect in psql and then look for the `rideshare_development` DB: ```sql SELECT pg_database.oid FROM pg_database WHERE pg_database.datname = 'rideshare_development'; oid --------- 1462704 ``` Filter in `pg_stat_statements` on `dbid` and the `owner` `userid`: ```sql \x -- vertical presentation WITH mydb AS ( SELECT pg_database.oid AS mydbid FROM pg_database WHERE pg_database.datname = 'rideshare_development' ), me AS ( SELECT oid AS myuserid FROM pg_roles WHERE rolname = 'owner' ) SELECT * FROM pg_stat_statements JOIN mydb ON dbid = mydb.mydbid JOIN me ON userid = me.myuserid; ``` Let's populate some query statistics rows. Run our earlier slow query, to act as slow query data: ```sql SELECT * FROM users WHERE first_name = 'Alphonso'; ``` We can get a query from [`andyatkinson/pg_scripts`](https://github.com/andyatkinson/pg_scripts) for PGSS, adapting the 10 worst performers, to get the single worst one. Run this: ```sql SELECT queryid, query as normalized_query, mean_exec_time AS avg_ms, calls, (rows / calls) AS avg_rows FROM pg_stat_statements ORDER BY 3 DESC LIMIT 1; ``` Notes: - Get a generic plan on 16+ with `EXPLAIN (GENERIC_PLAN) SELECT * FROM users WHERE first_name = $1;` - Re-run the query a few times and observe the growth of "calls" We can now identify our slowest queries and apply our micro optimization tactics to them. ## What's Next? Visit [7 - Macro Query Optimization Part 2](/docs/workshop/7_macro_overview_part_2.md) to continue. ================================================ FILE: docs/workshop/7_macro_overview_part_2.md ================================================ # Macro Query Optimization Part 2 In this section, we'll begin to work with multiple PostgreSQL instances. Remember this hierarchy: ``` __________________________________________________________________ | |--Instance (the server) (localhost, db01, db02, etc.) | |----Cluster (*all databases*, e.g. postgres, rideshare_development) | |------Database (postgres, rideshare_development) | |--------Schema (public, rideshare) | -------------------------------------------------------- ``` We'll run these using Docker. Start up Docker. ## Part 1: Docker PostgreSQL Containers - Boot up Docker. There may be zero containers running. (`docker ps`) - Docker containers are in `docker` directory. Read README: - Create a docker network (`rideshare-net`) the containers can use - Run the script to start the `db01` container - Run the script to start the `db02` container - Verify they're running with `docker ps` ```sh # Clean-up from past runs: cd docker rm postgresql.conf rm -rf postgres-docker/ # Starting point: docker network create rideshare-net sh run_db_db01_primary.sh sh run_db_db02_replica.sh ``` Let's configure them. ## Part 2: Enabling Physical Replication Prep: Remove annoying Docker messages: ```sh export DOCKER_CLI_HINTS=false ``` - `db01` and `db02` are now running. Review the network, host names, basics of connection to each instance. ```sh docker ps ``` - We're running two instances of Postgres in containers, simulating two different hosts Go to the `docker` directory and run `sh reset_docker_instances.sh`. If you're missing `postgresql.conf`, you'll be prompted to create it. ```sh cd docker sh reset_docker_instances.sh ``` Follow the commands to copy down `postgresql.conf`. Edit the settings `wal_level = logical` and save the changes. The script copies postgresql.conf to db01. Run the command again to do that: ```sh sh reset_docker_instances.sh ``` Let's walk through the highlights: - Replaced postgresql.conf config file on db01 - Created replication slot on primary db01 - Created `replication_user` user on db01 with a unique password and permissions - Created `pg_hba.conf` on db01 to allow access - Placed password in `.pgpass` and copied to db01 and db02 for `replication_user` - Restarted db01 Check logs on db02: ```sh docker logs -f db02 ``` Initially system identifier won't be the same: ```sh docker exec --user postgres -it db01 \ psql -c "SELECT system_identifier FROM pg_control_system();" docker exec --user postgres -it db02 \ psql -c "SELECT system_identifier FROM pg_control_system();" ``` We'll need to turn the db02 instance into a physical copy of db01. To do that, we'll replace the data directory on db02 with a copy of db01, where it will then be kept in sync. ## Part 3: Run `pg_basebackup` Now we have the two instances configured, and db02 can reach db01. We need to turn db02 into a read replica, by running `pg_basebackup` on it. To do that, open the file `run_pg_basebackup.sh` in the docker directory, but don't run it as a script. Instead, this file is a reference of individual commands. Copy and paste each one into db02. 💻 Do that now! After running the main `pg_basebackup` command as demonstrated, a success message looks like this: ```sh pg_basebackup: base backup completed ``` The container will exit. You'll want to start it again using `docker start db02`.
🎥 Rideshare - PostgreSQL physical replication with Docker containers
If everything works, you'll have replication enabled between both instances with a `replication_user` user, and a replication slow. ```sh pg_basebackup: initiating base backup, waiting for checkpoint to complete pg_basebackup: checkpoint completed pg_basebackup: write-ahead log start point: 0/5000028 on timeline 1 pg_basebackup: starting background WAL receiver 31481/31481 kB (100%), 1/1 tablespace pg_basebackup: write-ahead log end point: 0/5000100 pg_basebackup: waiting for background process to finish streaming ... pg_basebackup: syncing data to disk ... pg_basebackup: renaming backup_manifest.tmp to backup_manifest pg_basebackup: base backup completed ``` And on db02, something like this: ``` 2024-05-01 02:07:18.636 UTC [30] LOG: entering standby mode 2024-05-01 02:07:18.641 UTC [30] LOG: redo starts at 0/6000028 2024-05-01 02:07:18.641 UTC [30] LOG: consistent recovery state reached at 0/6000138 2024-05-01 02:07:18.641 UTC [1] LOG: database system is ready to accept read-only connections 2024-05-01 02:07:18.648 UTC [31] LOG: started streaming WAL from primary at 0/7000000 on timeline 1 ``` ## Conclusion That concludes the basics of setting up a replica instance. In the next section we'll continue with adding content, then layer on application-level configuration. ## Appendix: Debugging and Troubleshooting Check for connectivity from db02 to db01: ```sh docker exec --user postgres -it db02 bin/bash psql -U replication_user -h db01 -d postgres ``` Check for replication slot: ```sh docker exec -it db01 psql -U postgres \x select * from pg_replication_slots; ``` If needed, remove the slot: ```sql SELECT pg_drop_replication_slot('rideshare_slot'); ``` To start over fully, completely remove the locally mapped data directory: ```sh # Local volume for container data, remove this directory if starting over rm -rf docker-postgres ``` Start over from the beginning. ## What's Next? Visit [8 - Active Record Multi-DB Part 1](/docs/workshop/8_active_record_multi-db_prep_part_1.md) to continue. ================================================ FILE: docs/workshop/8_active_record_multi-db_prep_part_1.md ================================================ # Active Record Multiple Databases Part 1 Now we have `db01` and `db02` running. Let's create the Rideshare DB, and configure it. We'll work with `db01`, which is mapped to local port 54321. We'll set `DB_URL` and `RIDESHARE_DB_PASSWORD`. ## Section 1: Primary and Secondary DB config We use `postgres/postgres`, and connect to `postgres` on port 54321 (db01). ```sh export DB_URL="postgres://postgres:postgres@localhost:54321/postgres" cd rideshare # Run setup, will complain if RIDESHARE_DB_PASSWORD is not set sh db/setup.sh 2>&1 | tee -a db01_output.log # Check for any errors: vim db01_output.log ``` Now let's connect as the owner role using a single-DB config: ```sh export DATABASE_URL="postgres://owner:@localhost:54321/rideshare_development" ``` Verify port 54321 is listed. There should be no tables here: `\dt`. We should see the `rideshare` schema: `\dn`. Now we're ready to run migrations on db01: ```sh bin/rails db:migrate ``` Let's see if the tables were replicated! We're gradually moving to a multi-DB setup, but still using a single-DB setup. To prepare, let's configure new env vars. These are in the `.env` for Rideshare. ```sh export DATABASE_URL_PRIMARY="postgres://owner:@localhost:54321/rideshare_development" export DATABASE_URL_REPLICA="postgres://owner:@localhost:54322/rideshare_development" ``` Let's connect to the replica and check for tables. If you see tables on the replica, it's because they were created via replication not from running migrations there. Migrations only run on the primary instance. ```sh psql $DATABASE_URL_REPLICA ``` Note: - We automatically got the `owner` role - We're connected to port 54322 (one greater), which is the locally mapped port to db02 - We see the tables in the `rideshare` schema Cool! If we check row counts on both, all the tables are empty. Let's populate data so that we work on queries. Note that this still uses `DATABASE_URL`, but that now points at db01. ```sh bin/rails data_generators:generate_all ``` Connect again to db02 and verify the row counts are the same. There should be data on db02! This data came from db01 via replication. With data on both instances, we're ready to move to Active Record configuration. ## Section 2: Database config multiple databases In this section, we're going to move to a multi-DB configuration. Copy and paste the contents from the file below, replacing the current contents of `db/config.yml`: ```sh config/database-multiple.sample.yml ``` Replace the contents of `config/database.yml` with the file contents above. Take note of: - These reference the env vars you set earlier: `DATABASE_URL_PRIMARY` and `DATABASE_URL_REPLICA` - "Named" configurations for both: `rideshare` (db01) and `rideshare_replica` (db02) - Database names are `rideshare_development` for both instances - db02 has `replica: true` config - `schema_search_path` is set to `rideshare` for both - `database_tasks: false` for db02, we don't want to run migrations there Now we can try these out! Now when running migrations, they should only run on db01 primary instance: ```sh bin/rails db:migrate ``` Test the new configurations, first using `db`: ```sh bin/rails db --database rideshare bin/rails db --database rideshare_replica ``` This concludes the configuration portion of Active Record Multiple Databases. In the last section, we'll wrap things up with application level configuration and usage: See you there! ## Appendix: Troubleshooting Tip: Log all statements if desired. Run this on the db01 or db02 instance. ```sql ALTER DATABASE rideshare_development SET log_statement = 'all'; ``` ## What's Next? Visit [9 - Active Record Multi-DB Part 2](/docs/workshop/9_active_record_multi-db_roles.md) to continue. ================================================ FILE: docs/workshop/9_active_record_multi-db_roles.md ================================================ # Active Record Multiple Databases - Part 2 With multiple databases configured, we're ready to leverage Active Record Multiple Databases. We can move things up to a higher layer of abstraction, by configuring model code, then making different calls to the primary or replica instance by "role". What are roles? ## Section 1 - Roles Let's change the main application model that classes inherit from. We'll specify "writing" and "reading" roles we can connect to. - Writing role: db01 - Reading role: db02 ## Section 2 - Configuration Edit `app/models/application_record.rb` to uncomment the `connects_to` code. ```rb connects_to database: { writing: :rideshare, reading: :rideshare_replica } ``` Let's try out that new configuration. Use the rails console now instead of db: ```sh bin/rails console ``` From there, we can establish connections to one role or the other. Try out queries to each: ```rb ActiveRecord::Base.connected_to(role: :writing) { Driver.first } ActiveRecord::Base.connected_to(role: :reading) { Driver.first } ``` ⚠️ Let's try an update to the reader. This won't work because it's running in read-only mode. ```rb ActiveRecord::Base.connected_to(role: :reading) do Driver.first.update_attribute(:first_name, "Andrew") end ``` We get an error like: ``` Write query attempted while in readonly mode ``` Let's send that to the writer: ```rb ActiveRecord::Base.connected_to(role: :writing) do Driver.first.update_attribute(:first_name, "Andrew") end ``` Great! If that committed, in a few moments it will be replicated. Let's make sure it's replicated: ```rb ActiveRecord::Base.connected_to(role: :reading) { Driver.first.first_name } ``` That should have returned "Andrew". ## Section 3 - Role Switching What you saw earlier was "manual" role switching. Active Record also supports [Automatic Role Switching](https://guides.rubyonrails.org/active_record_multiple_databases.html#activating-automatic-role-switching) based on the HTTP request and other factors. Let's try that out. ```sh bin/rails g active_record:multi_db ``` Add to `config/application.rb`: ```rb config.active_record.database_selector = { delay: 2.seconds } config.active_record.database_resolver = ActiveRecord::Middleware::DatabaseSelector::Resolver config.active_record.database_resolver_context = ActiveRecord::Middleware::DatabaseSelector::Resolver::Session ``` Let's log all queries. We'd like to verify that sending a GET request runs on db02, although we make this change on db01: ```sh docker exec --user postgres -it db01 psql ALTER DATABASE rideshare_development SET log_statement = 'all'; ``` Let's tail db01 and db02 logs in different terminals: ```sh docker logs -f db01 docker logs -f db02 ``` Start up the rails server: ```sh bin/rails server ``` Send a GET request: ```sh curl localhost:3000/api/trips ``` 💥 Boom. We don't see any queries logged on db01, and we see `SELECT * FROM trips;` logged on db02. The query is automatically sent to the replica. It's working! ## Wrap Up We've now seen how to use multiple PostgreSQL databases to distribute the database work, splitting up writes and read queries. Scaling read traffic separately is part of building High Performance Active Record apps. Beyond write/read role switching, for even more advanced scalability options, Active Record supports Horizontal Sharding, which has a similar pattern to what you've done here for "shard switching." ================================================ FILE: docs/workshop/README.md ================================================ # Workshop Hello! This is meant to be a two hour long workshop, facilitated by Andrew Atkinson. ## Prerequisites For prerequisites, you'll use the Rideshare app. Follow the instructions in the main [Rideshare README.md](/README.md) to fully set it up. When you've installed the app, verify that: - Running `bundle install` in the Rideshare directory installs all Ruby gems - `sh db/setup.sh` ran and created the `rideshare_development` database, users, etc. - `bin/rails db:migrate` run and created empty tables, indexes, etc. - `bin/rails data_generators:generate_all` ran and created a base set of fake data The workshop uses content from my book ["High Performance PostgreSQL for Rails"](https://andyatkinson.com/pgrailsbook). For book references, check Chapters "7 - Query Performance & 8 - Optimized Indexes for Fast Retrieval" for the first half of the workshop. Check Chapter "13 - Scaling with Replication and Sharding" for the second half of the workshop. ## Workshop Structure - Two 1 hr. halves, with a short break - Numbered files from 0 through 9, with "Sections" in the files - Each section has runnable code in backticks blocks, that's expected to be run by participants, unless flagged as "instructor only" ## Support As an independent consultant, your support is very meaningful! If you'd like to support me financially, please consider [buying my book](https://andyatkinson.com/pgrailsbook) and telling your colleagues about it! To get a discount, ask me about codes. Usually there are active discounts during events like conferences. If your team needs help, please visit my [Consulting page](http://andyatkinson.com/consulting), where you can find information about what I offer and how to hire me. ## Rideshare and Workshop Loom Videos
🎥 Installation - Rideshare on a Mac, Ruby, PostgreSQL, Gems
🎥 Rideshare DB setup. Common issues running db/setup.sh
🎥 Rideshare - Loading data using a Rake task and Shell Script
🎥 Configuring and using pg_stat_statements data, creating generic query exec plans
🎥 Rideshare - PostgreSQL physical replication with Docker containers
## Let's Get Started In each section, you'll find links at the bottom to the next topic. Click the [0 - Introduction](/docs/workshop/0_introduction.md) to get started. ================================================ FILE: lib/assets/.keep ================================================ ================================================ FILE: lib/json_web_token.rb ================================================ class JsonWebToken SECRET_KEY = Rails.application.credentials.secret_key_base.to_s def self.encode(payload, exp = 24.hours.from_now) payload[:exp] = exp.to_i JWT.encode(payload, SECRET_KEY) end def self.decode(token) decoded = JWT.decode(token, SECRET_KEY)[0] HashWithIndifferentAccess.new(decoded) end end ================================================ FILE: lib/tasks/.keep ================================================ ================================================ FILE: lib/tasks/auto_generate_diagram.rake ================================================ # NOTE: only doing this in development as some production environments (Heroku) # NOTE: are sensitive to local FS writes, and besides -- it's just not proper # NOTE: to have a dev-mode tool do its thing in production. RailsERD.load_tasks if Rails.env.development? ================================================ FILE: lib/tasks/benchmarks.rake ================================================ namespace :benchmarks do desc 'Code benchmarks' task active_record: :environment do Benchmark.memory do |x| x.report('.select_all() single User') do ActiveRecord::Base.connection.select_all('SELECT * FROM users ORDER BY id LIMIT 1') end x.report('User.first') { User.first } x.compare! end end end ================================================ FILE: lib/tasks/custom.rake ================================================ # # This overrides the built-in db:reset task # namespace :custom do desc 'Custom database tasks' task :db_reset do sh 'db/teardown.sh' sh 'db/setup.sh' sh 'db/setup_test_database.sh' Rake::Task['db:migrate'].invoke end end ================================================ FILE: lib/tasks/data_generators.rake ================================================ require 'faker' namespace :data_generators do desc 'Generate Drivers and Riders' task drivers_and_riders: :environment do |_t, _args| Benchmark.measure do 10_000.times.to_a.in_groups_of(10_000).each do |group| [Driver, Rider].each do |klass| batch = group.map do |i| first_name = Faker::Name.first_name last_name = Faker::Name.last_name drivers_license_number = (random_mn_drivers_license_number(first_name, i) if klass.equal?(Driver)) klass.new( first_name: first_name, last_name: last_name, email: "#{first_name}-#{last_name}-#{klass.name.downcase}-#{i}@email.com", password_digest: SecureRandom.hex, drivers_license_number: drivers_license_number ) end.map do |d| d.attributes.symbolize_keys.slice( :first_name, :last_name, :email, :password, :type, :drivers_license_number ) end puts "bulk insert batch size: #{batch.size}" klass.insert_all(batch) end end end end desc 'Generate Trips and Trip Requests' task trips_and_requests: :environment do |_t, _args| drivers = [] 100.times do |i| fname = Faker::Name.first_name lname = Faker::Name.last_name drivers_license_number = random_mn_drivers_license_number(fname, i) drivers << Driver.create!( first_name: fname, last_name: lname, email: "#{fname}-#{lname}-#{i}@email.com", password: SecureRandom.hex, drivers_license_number: drivers_license_number ) end riders = [] 100.times do |i| fname = Faker::Name.first_name lname = Faker::Name.last_name riders << Rider.create!( first_name: fname, last_name: lname, email: "#{fname}-#{lname}-#{i}@email.com", password: SecureRandom.hex ) end nyc = Location.where( address: 'New York, NY' ).first_or_create do |loc| loc.position = '(40.7143528,-74.0059731)' loc.state = 'NY' end bos = Location.where( address: 'Boston, MA' ).first_or_create do |loc| loc.position = '(42.361145,-71.057083)' loc.state = 'MA' end puts 'creating Trip Requests and Trips' 1000.times do |i| request = TripRequest.create!( rider: riders.sample, start_location: nyc, end_location: bos ) # for about 1/4 of the trips, give them a random rating rating = i % 4 == 0 ? rand(1..5) : nil request.create_trip!( driver: drivers.sample, completed_at: 1.minute.from_now, rating: rating ) end end desc 'Generate Vehicles and Reservations' task vehicles_and_reservations: :environment do |_t, _args| riders = [] 10.times do |i| fname = Faker::Name.first_name lname = Faker::Name.last_name riders << Rider.create!( first_name: fname, last_name: lname, email: "#{fname}-#{lname}-#{i}@email.com", password: SecureRandom.hex ) end nyc = Location.where( address: 'New York, NY' ).first_or_create do |loc| loc.position = '(40.7143528,-74.0059731)' end bos = Location.where( address: 'Boston, MA' ).first_or_create do |loc| loc.position = '(42.361145,-71.057083)' end puts 'creating trip requests and trips' 10.times do |_i| TripRequest.create!( rider: riders.sample, start_location: nyc, end_location: bos ) end Vehicle.destroy_all ['Party Bus', 'Limo', 'Ice Cream Truck', 'Food Truck'].each do |name| Vehicle.create!( name: name, status: VehicleStatus::PUBLISHED ) end # create reservation vehicle = Vehicle.order(Arel.sql('RANDOM()')).first trip_request = TripRequest.order(Arel.sql('RANDOM()')).first starts_at = (rand * 1000).floor.hours.from_now ends_at = starts_at + 1.hour puts "v=#{vehicle.id} tr=#{trip_request.id} from=#{starts_at} to=#{ends_at}" VehicleReservation.create!( vehicle: vehicle, trip_request: trip_request, starts_at: starts_at, ends_at: ends_at, canceled: false ) VehicleReservation.create!( vehicle: vehicle, trip_request: trip_request, starts_at: starts_at + 1.hour, ends_at: ends_at + 1.hour, canceled: true ) VehicleReservation.create!( vehicle: vehicle, trip_request: trip_request, starts_at: starts_at + 1.hour, ends_at: ends_at + 1.hour, canceled: false ) end # bin/rails data_generators:generate_trip_positions desc 'Generate simulated historical trip positions data' task generate_trip_positions: :environment do |_t, _args| # Generate data from 1 year ago, 3 months ago, 2 months ago, 1 month ago # and current month puts 'From 1 year ago' 5.times do |_i| @trip = Trip.all.sample TripPosition.create!( position: '(651096.993815166,667028.1146045981)', trip: @trip, created_at: 1.year.ago ) end puts 'From 3 months ago' 5.times do |_i| @trip = Trip.all.sample TripPosition.create!( position: '(651096.993815166,667028.1146045981)', trip: @trip, created_at: 3.months.ago ) end puts 'From 2 months ago' 5.times do |_i| @trip = Trip.all.sample TripPosition.create!( position: '(651096.993815166,667028.1146045981)', trip: @trip, created_at: 2.months.ago ) end puts 'From 1 month ago' 5.times do |_i| @trip = Trip.all.sample TripPosition.create!( position: '(651096.993815166,667028.1146045981)', trip: @trip, created_at: 1.month.ago ) end puts 'This month' 5.times do |_i| @trip = Trip.all.sample TripPosition.create!( position: '(651096.993815166,667028.1146045981)', trip: @trip ) end puts "Created #{TripPosition.count} records." end desc 'Run ANALYZE on all involved tables' task analyze_tables: :environment do |_t, _args| %w[ users trips trip_requests trip_positions locations vehicles vehicle_reservations ].each do |table_name| ActiveRecord::Base.connection.execute("ANALYZE #{table_name}") end end desc 'Generate All Data' task generate_all: :environment do |_t, _args| Rake::Task['data_generators:drivers_and_riders'].invoke Rake::Task['data_generators:trips_and_requests'].invoke Rake::Task['data_generators:vehicles_and_reservations'].invoke Rake::Task['data_generators:generate_trip_positions'].invoke Rake::Task['data_generators:analyze_tables'].invoke end end def random_mn_drivers_license_number(fname, i) [ "#{fname.first.upcase}", '800000', (rand * 10).to_i.to_s, (rand * 10).to_i.to_s, (rand * 10).to_i.to_s, (rand * 10).to_i.to_s, (rand * 10).to_i.to_s, "#{i}" ].join end ================================================ FILE: lib/tasks/fake_data_generator.rake ================================================ require 'faker' namespace :data_generators do desc 'Generator Drivers' task drivers: :environment do |_t, _args| TOTAL = 20_000 BATCH_SIZE = 10_000 results = Benchmark.measure do TOTAL.times.to_a.in_groups_of(BATCH_SIZE).each do |group| batch = group.map do |i| first_name = Faker::Name.first_name last_name = Faker::Name.last_name Driver.new( first_name: first_name, last_name: last_name, email: "#{first_name}-#{last_name}-#{i}@email.com", password_digest: SecureRandom.hex ) end.map do |d| d.attributes.symbolize_keys.slice( :first_name, :last_name, :email, :password, :type ) end Driver.insert_all(batch) puts "Created #{batch.size} drivers." end end puts 'VACUUM (ANALYZE) users' Driver.connection.execute('VACUUM (ANALYZE) users') puts results end end ================================================ FILE: lib/tasks/migration_hooks.rake ================================================ # lib/tasks/migration_hooks.rb # # https://www.dan-manges.com/blog/modifying-rake-tasks namespace :migration_hooks do task set_role: :environment do if Rails.env.development? puts 'Setting role for development' ActiveRecord::Base.connection.execute('SET ROLE owner') end end end # https://dev.to/molly/rake-task-enhance-method-explained-3bo0 Rake::Task['db:migrate'].enhance(['migration_hooks:set_role']) ================================================ FILE: lib/tasks/simulate_app_activity.rake ================================================ # # bin/rails simulate:app_activity # # Set an optional iterations (default: 1) # bin/rails simulate:app_activity[10] # namespace :simulate do desc 'Simulate App Activity' task :app_activity, [:iteration_count] => :environment do |_t, args| args.with_defaults(iteration_count: 1) # Steps in end-to-end cycle # 1. (API) Rider creates trip_request # 1. (API) Rider polls for trip_request status # 1. Best available driver picks up trip request, updates status, trip created # 1. (API) Rider polls for trip status # 1. Driver completes trip iterations = args[:iteration_count].to_i puts "Running script #{iterations} times..." iterations.times do # 1. create trip request url = 'http://localhost:3000/api/trip_requests' request_body = { trip_request: { rider_id: Rider.first.id, start_address: 'Boston, MA', end_address: 'New York, NY' } } request_headers = { 'Accept' => 'application/json' } puts '[trip_request] creating trip request...' response = Faraday.post(url, request_body, request_headers) resp = JSON.parse(response.body, symbolize_names: true) trip_request_id = resp[:trip_request_id] puts "[trip_request] got trip_request_id: #{trip_request_id}" next unless trip_request_id # 1. poll for trip request status, until trip exists # Polling is not implemented, would need some async processing # in the app like Sidekiq or another background processor begin puts '[trip_request] checking for trip_id...' attempts ||= 1 url = "http://localhost:3000/api/trip_requests/#{trip_request_id}" show_response = Faraday.get(url) show_resp = JSON.parse(show_response.body, symbolize_names: true) puts "[trip_request] show_resp: #{show_resp.inspect}" trip_id = show_resp[:trip_id] if trip_id puts "[trip] Got a trip_id: #{trip_id}" else puts '[trip] no trip_id...' raise end rescue StandardError if (attempts += 1) < 5 # go back to begin block if condition ok puts 'retrying...' retry end end end end end ================================================ FILE: log/.keep ================================================ ================================================ FILE: postgresql/.pg_service.sample.conf ================================================ [rideshare_dev] host=localhost user=owner dbname=rideshare_development port=5432 ================================================ FILE: postgresql/.pgpass.sample ================================================ localhost:5432:rideshare_development:owner:HSnDDgFtyW9fyFI localhost:54321:rideshare_development:owner:HSnDDgFtyW9fyFI localhost:54322:rideshare_development:owner:HSnDDgFtyW9fyFI localhost:5432:rideshare_development:app:HSTnDDgFtyW9fyFI localhost:6432:rideshare_development:owner:HSnDDgFtyW9fyFI *:*:*:replication_user:cd58b7e22c0af34a34c1572a *:*:*:app_readonly:ee0e8cc80c5c244e6582b0de ================================================ FILE: postgresql/.psqlrc.sample ================================================ \encoding unicode \set PROMPT1 '%n@%M:%>%x %/# ' \set PROMPT2 '' \setenv PAGER 'less -S' ================================================ FILE: postgresql/README.md ================================================ # PostgreSQL ## `postgresql.conf` Review the sample file in this directory. Remove `sample` from the file name ## `pg_hba.conf` * Remove `sample` from filename ## `.pg_service.conf` * Remove `sample` * Copy to `~/.pg_service.conf` * Edit the service info with your config ## `~/.pgpass` * Remove `sample` from the sample file * Copy file content to `~/.pgpass` (scripts will populate it as well) * Edit the file with your specific credentials * Perform the following changes: ```sh chown : /home/dir/.pgpass chmod 0600 /home/dir/.pgpass ``` Replace `/home/dir` with the path to the home directory of the user. On PostgreSQL docker containers, that's `/var/lib/postgresql/` For user and group on PostgreSQL docker containers, the user is `postgres` and the group is `root` ## `.psqlrc` * Remove `sample` * Copy to `~/.psqlrc` ## PgBouncer > The mode that results in a more sane balance of improved concurrency and retained critical database features is transaction mode. From: [PgBouncer is useful, important, and fraught with peril](https://jpcamara.com/2023/04/12/pgbouncer-is-useful.html) * For 1.21.0, recommend `transaction` pool mode (compatible with multi-statement transactions) * For macOS, install with Homebrew * Copy changes from `pgbouncer.sample.ini` file * Restart with `brew services restart pgbouncer` ================================================ FILE: postgresql/pg_hba.sample.conf ================================================ # PostgreSQL Client Authentication Configuration File # =================================================== # # Refer to the "Client Authentication" section in the PostgreSQL # documentation for a complete description of this file. A short # synopsis follows. # # This file controls: which hosts are allowed to connect, how clients # are authenticated, which PostgreSQL user names they can use, which # databases they can access. Records take one of these forms: # # local DATABASE USER METHOD [OPTIONS] # host DATABASE USER ADDRESS METHOD [OPTIONS] # hostssl DATABASE USER ADDRESS METHOD [OPTIONS] # hostnossl DATABASE USER ADDRESS METHOD [OPTIONS] # hostgssenc DATABASE USER ADDRESS METHOD [OPTIONS] # hostnogssenc DATABASE USER ADDRESS METHOD [OPTIONS] # # (The uppercase items must be replaced by actual values.) # # The first field is the connection type: # - "local" is a Unix-domain socket # - "host" is a TCP/IP socket (encrypted or not) # - "hostssl" is a TCP/IP socket that is SSL-encrypted # - "hostnossl" is a TCP/IP socket that is not SSL-encrypted # - "hostgssenc" is a TCP/IP socket that is GSSAPI-encrypted # - "hostnogssenc" is a TCP/IP socket that is not GSSAPI-encrypted # # DATABASE can be "all", "sameuser", "samerole", "replication", a # database name, or a comma-separated list thereof. The "all" # keyword does not match "replication". Access to replication # must be enabled in a separate record (see example below). # # USER can be "all", a user name, a group name prefixed with "+", or a # comma-separated list thereof. In both the DATABASE and USER fields # you can also write a file name prefixed with "@" to include names # from a separate file. # # ADDRESS specifies the set of hosts the record matches. It can be a # host name, or it is made up of an IP address and a CIDR mask that is # an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that # specifies the number of significant bits in the mask. A host name # that starts with a dot (.) matches a suffix of the actual host name. # Alternatively, you can write an IP address and netmask in separate # columns to specify the set of hosts. Instead of a CIDR-address, you # can write "samehost" to match any of the server's own IP addresses, # or "samenet" to match any address in any subnet that the server is # directly connected to. # # METHOD can be "trust", "reject", "md5", "password", "scram-sha-256", # "gss", "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert". # Note that "password" sends passwords in clear text; "md5" or # "scram-sha-256" are preferred since they send encrypted passwords. # # OPTIONS are a set of options for the authentication in the format # NAME=VALUE. The available options depend on the different # authentication methods -- refer to the "Client Authentication" # section in the documentation for a list of which options are # available for which authentication methods. # # Database and user names containing spaces, commas, quotes and other # special characters must be quoted. Quoting one of the keywords # "all", "sameuser", "samerole" or "replication" makes the name lose # its special character, and just match a database or username with # that name. # # This file is read on server startup and when the server receives a # SIGHUP signal. If you edit the file on a running system, you have to # SIGHUP the server for the changes to take effect, run "pg_ctl reload", # or execute "SELECT pg_reload_conf()". # # Put your actual configuration here # ---------------------------------- # # If you want to allow non-local connections, you need to add more # "host" records. In that case you will also need to make PostgreSQL # listen on a non-local interface via the listen_addresses # configuration parameter, or via the -i or -h command line switches. # CAUTION: Configuring the system for local "trust" authentication # allows any local user to connect as any PostgreSQL user, including # the database superuser. If you do not trust all your local users, # use another authentication method. # TYPE DATABASE USER ADDRESS METHOD # "local" is for Unix domain socket connections only # local all all trust # # IPv4 local connections: # host all all 127.0.0.1/32 trust # # IPv6 local connections: # host all all ::1/128 trust # # Allow replication connections from localhost, by a user with the # # replication privilege. # local replication all trust # host replication all 127.0.0.1/32 trust # host replication all ::1/128 trust local rideshare_development owner md5 host rideshare_development owner 127.0.0.1/32 md5 host rideshare_development owner ::1/128 md5 local rideshare_development app md5 host rideshare_development app 127.0.0.1/32 md5 host rideshare_development app ::1/128 md5 local all andy trust local all andy trust host all postgres localhost trust ================================================ FILE: postgresql/pgbouncer.sample.ini ================================================ [databases] rideshare_development = host=127.0.0.1 port=5432 dbname=rideshare_development #rideshare_development = host=127.0.0.1 port=5432 dbname=rideshare_development pool_mode=transaction # For userlist.txt, refer to ./userlist.sample.txt # Copy the file, remove the "sample" portion, and place the file where it's # reachable, i.e. /usr/local/etc/userlist.txt [pgbouncer] listen_port = 6432 listen_addr = 127.0.0.1 auth_type = md5 auth_file = userlist.txt logfile = pgbouncer.log pidfile = pgbouncer.pid admin_users = owner ================================================ FILE: postgresql/postgresql.sample.conf ================================================ # ----------------------------- # PostgreSQL configuration file # ----------------------------- # # This file consists of lines of the form: # # name = value # # (The "=" is optional.) Whitespace may be used. Comments are introduced with # "#" anywhere on a line. The complete list of parameter names and allowed # values can be found in the PostgreSQL documentation. # # The commented-out settings shown in this file represent the default values. # Re-commenting a setting is NOT sufficient to revert it to the default value; # you need to reload the server. # # This file is read on server startup and when the server receives a SIGHUP # signal. If you edit the file on a running system, you have to SIGHUP the # server for the changes to take effect, run "pg_ctl reload", or execute # "SELECT pg_reload_conf()". Some parameters, which are marked below, # require a server shutdown and restart to take effect. # # Any parameter can also be given as a command-line option to the server, e.g., # "postgres -c log_connections=on". Some parameters can be changed at run time # with the "SET" SQL command. # # Memory units: B = bytes Time units: us = microseconds # kB = kilobytes ms = milliseconds # MB = megabytes s = seconds # GB = gigabytes min = minutes # TB = terabytes h = hours # d = days #------------------------------------------------------------------------------ # FILE LOCATIONS #------------------------------------------------------------------------------ # The default values of these variables are driven from the -D command-line # option or PGDATA environment variable, represented here as ConfigDir. #data_directory = 'ConfigDir' # use data in another directory # (change requires restart) #hba_file = 'ConfigDir/pg_hba.conf' # host-based authentication file # (change requires restart) #ident_file = 'ConfigDir/pg_ident.conf' # ident configuration file # (change requires restart) # If external_pid_file is not explicitly set, no extra PID file is written. #external_pid_file = '' # write an extra PID file # (change requires restart) #------------------------------------------------------------------------------ # CONNECTIONS AND AUTHENTICATION #------------------------------------------------------------------------------ # - Connection Settings - #listen_addresses = 'localhost' # what IP address(es) to listen on; # comma-separated list of addresses; # defaults to 'localhost'; use '*' for all # (change requires restart) #port = 5432 # (change requires restart) max_connections = 100 # (change requires restart) #superuser_reserved_connections = 3 # (change requires restart) #unix_socket_directories = '/tmp' # comma-separated list of directories # (change requires restart) #unix_socket_group = '' # (change requires restart) #unix_socket_permissions = 0777 # begin with 0 to use octal notation # (change requires restart) #bonjour = off # advertise server via Bonjour # (change requires restart) #bonjour_name = '' # defaults to the computer name # (change requires restart) # - TCP settings - # see "man tcp" for details #tcp_keepalives_idle = 0 # TCP_KEEPIDLE, in seconds; # 0 selects the system default #tcp_keepalives_interval = 0 # TCP_KEEPINTVL, in seconds; # 0 selects the system default #tcp_keepalives_count = 0 # TCP_KEEPCNT; # 0 selects the system default #tcp_user_timeout = 0 # TCP_USER_TIMEOUT, in milliseconds; # 0 selects the system default #client_connection_check_interval = 0 # time between checks for client # disconnection while running queries; # 0 for never # - Authentication - #authentication_timeout = 1min # 1s-600s #password_encryption = scram-sha-256 # scram-sha-256 or md5 #db_user_namespace = off # GSSAPI using Kerberos #krb_server_keyfile = 'FILE:${sysconfdir}/krb5.keytab' #krb_caseins_users = off # - SSL - #ssl = off #ssl_ca_file = '' #ssl_cert_file = 'server.crt' #ssl_crl_file = '' #ssl_crl_dir = '' #ssl_key_file = 'server.key' #ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' # allowed SSL ciphers #ssl_prefer_server_ciphers = on #ssl_ecdh_curve = 'prime256v1' #ssl_min_protocol_version = 'TLSv1.2' #ssl_max_protocol_version = '' #ssl_dh_params_file = '' #ssl_passphrase_command = '' #ssl_passphrase_command_supports_reload = off #------------------------------------------------------------------------------ # RESOURCE USAGE (except WAL) #------------------------------------------------------------------------------ # - Memory - shared_buffers = 128MB # min 128kB # (change requires restart) #huge_pages = try # on, off, or try # (change requires restart) #huge_page_size = 0 # zero for system default # (change requires restart) #temp_buffers = 8MB # min 800kB #max_prepared_transactions = 0 # zero disables the feature # (change requires restart) # Caution: it is not advisable to set max_prepared_transactions nonzero unless # you actively intend to use prepared transactions. #work_mem = 4MB # min 64kB #hash_mem_multiplier = 2.0 # 1-1000.0 multiplier on hash table work_mem #maintenance_work_mem = 64MB # min 1MB #autovacuum_work_mem = -1 # min 1MB, or -1 to use maintenance_work_mem #logical_decoding_work_mem = 64MB # min 64kB #max_stack_depth = 2MB # min 100kB #shared_memory_type = mmap # the default is the first option # supported by the operating system: # mmap # sysv # windows # (change requires restart) dynamic_shared_memory_type = posix # the default is usually the first option # supported by the operating system: # posix # sysv # windows # mmap # (change requires restart) #min_dynamic_shared_memory = 0MB # (change requires restart) # - Disk - #temp_file_limit = -1 # limits per-process temp file space # in kilobytes, or -1 for no limit # - Kernel Resources - #max_files_per_process = 1000 # min 64 # (change requires restart) # - Cost-Based Vacuum Delay - #vacuum_cost_delay = 0 # 0-100 milliseconds (0 disables) #vacuum_cost_page_hit = 1 # 0-10000 credits #vacuum_cost_page_miss = 2 # 0-10000 credits #vacuum_cost_page_dirty = 20 # 0-10000 credits #vacuum_cost_limit = 200 # 1-10000 credits # - Background Writer - #bgwriter_delay = 200ms # 10-10000ms between rounds #bgwriter_lru_maxpages = 100 # max buffers written/round, 0 disables #bgwriter_lru_multiplier = 2.0 # 0-10.0 multiplier on buffers scanned/round #bgwriter_flush_after = 0 # measured in pages, 0 disables # - Asynchronous Behavior - #backend_flush_after = 0 # measured in pages, 0 disables #effective_io_concurrency = 0 # 1-1000; 0 disables prefetching #maintenance_io_concurrency = 10 # 1-1000; 0 disables prefetching #max_worker_processes = 8 # (change requires restart) #max_parallel_workers_per_gather = 2 # taken from max_parallel_workers #max_parallel_maintenance_workers = 2 # taken from max_parallel_workers #max_parallel_workers = 8 # maximum number of max_worker_processes that # can be used in parallel operations #parallel_leader_participation = on #old_snapshot_threshold = -1 # 1min-60d; -1 disables; 0 is immediate # (change requires restart) #------------------------------------------------------------------------------ # WRITE-AHEAD LOG #------------------------------------------------------------------------------ # - Settings - #wal_level = replica # minimal, replica, or logical # (change requires restart) #fsync = on # flush data to disk for crash safety # (turning this off can cause # unrecoverable data corruption) #synchronous_commit = on # synchronization level; # off, local, remote_write, remote_apply, or on #wal_sync_method = fsync # the default is the first option # supported by the operating system: # open_datasync # fdatasync (default on Linux and FreeBSD) # fsync # fsync_writethrough # open_sync #full_page_writes = on # recover from partial page writes #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_compression = off # enables compression of full-page writes; # off, pglz, lz4, zstd, or on #wal_init_zero = on # zero-fill new WAL files #wal_recycle = on # recycle WAL files #wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers # (change requires restart) #wal_writer_delay = 200ms # 1-10000 milliseconds #wal_writer_flush_after = 1MB # measured in pages, 0 disables #wal_skip_threshold = 2MB #commit_delay = 0 # range 0-100000, in microseconds #commit_siblings = 5 # range 1-1000 # - Checkpoints - #checkpoint_timeout = 5min # range 30s-1d #checkpoint_completion_target = 0.9 # checkpoint target duration, 0.0 - 1.0 #checkpoint_flush_after = 0 # measured in pages, 0 disables #checkpoint_warning = 30s # 0 disables max_wal_size = 1GB min_wal_size = 80MB # - Prefetching during recovery - #recovery_prefetch = try # prefetch pages referenced in the WAL? #wal_decode_buffer_size = 512kB # lookahead window used for prefetching # (change requires restart) # - Archiving - #archive_mode = off # enables archiving; off, on, or always # (change requires restart) #archive_library = '' # library to use to archive a logfile segment # (empty string indicates archive_command should # be used) #archive_command = '' # command to use to archive a logfile segment # placeholders: %p = path of file to archive # %f = file name only # e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f' #archive_timeout = 0 # force a logfile segment switch after this # number of seconds; 0 disables # - Archive Recovery - # These are only used in recovery mode. #restore_command = '' # command to use to restore an archived logfile segment # placeholders: %p = path of file to restore # %f = file name only # e.g. 'cp /mnt/server/archivedir/%f %p' #archive_cleanup_command = '' # command to execute at every restartpoint #recovery_end_command = '' # command to execute at completion of recovery # - Recovery Target - # Set these only when performing a targeted recovery. #recovery_target = '' # 'immediate' to end recovery as soon as a # consistent state is reached # (change requires restart) #recovery_target_name = '' # the named restore point to which recovery will proceed # (change requires restart) #recovery_target_time = '' # the time stamp up to which recovery will proceed # (change requires restart) #recovery_target_xid = '' # the transaction ID up to which recovery will proceed # (change requires restart) #recovery_target_lsn = '' # the WAL LSN up to which recovery will proceed # (change requires restart) #recovery_target_inclusive = on # Specifies whether to stop: # just after the specified recovery target (on) # just before the recovery target (off) # (change requires restart) #recovery_target_timeline = 'latest' # 'current', 'latest', or timeline ID # (change requires restart) #recovery_target_action = 'pause' # 'pause', 'promote', 'shutdown' # (change requires restart) #------------------------------------------------------------------------------ # REPLICATION #------------------------------------------------------------------------------ # - Sending Servers - # Set these on the primary and on any standby that will send replication data. #max_wal_senders = 10 # max number of walsender processes # (change requires restart) #max_replication_slots = 10 # max number of replication slots # (change requires restart) #wal_keep_size = 0 # in megabytes; 0 disables #max_slot_wal_keep_size = -1 # in megabytes; -1 disables #wal_sender_timeout = 60s # in milliseconds; 0 disables #track_commit_timestamp = off # collect timestamp of transaction commit # (change requires restart) # - Primary Server - # These settings are ignored on a standby server. #synchronous_standby_names = '' # standby servers that provide sync rep # method to choose sync standbys, number of sync standbys, # and comma-separated list of application_name # from standby(s); '*' = all #vacuum_defer_cleanup_age = 0 # number of xacts by which cleanup is delayed # - Standby Servers - # These settings are ignored on a primary server. #primary_conninfo = '' # connection string to sending server #primary_slot_name = '' # replication slot on sending server #promote_trigger_file = '' # file name whose presence ends recovery #hot_standby = on # "off" disallows queries during recovery # (change requires restart) #max_standby_archive_delay = 30s # max delay before canceling queries # when reading WAL from archive; # -1 allows indefinite delay #max_standby_streaming_delay = 30s # max delay before canceling queries # when reading streaming WAL; # -1 allows indefinite delay #wal_receiver_create_temp_slot = off # create temp slot if primary_slot_name # is not set #wal_receiver_status_interval = 10s # send replies at least this often # 0 disables #hot_standby_feedback = off # send info from standby to prevent # query conflicts #wal_receiver_timeout = 60s # time that receiver waits for # communication from primary # in milliseconds; 0 disables #wal_retrieve_retry_interval = 5s # time to wait before retrying to # retrieve WAL after a failed attempt #recovery_min_apply_delay = 0 # minimum delay for applying changes during recovery # - Subscribers - # These settings are ignored on a publisher. #max_logical_replication_workers = 4 # taken from max_worker_processes # (change requires restart) #max_sync_workers_per_subscription = 2 # taken from max_logical_replication_workers #------------------------------------------------------------------------------ # QUERY TUNING #------------------------------------------------------------------------------ # - Planner Method Configuration - #enable_async_append = on #enable_bitmapscan = on #enable_gathermerge = on #enable_hashagg = on #enable_hashjoin = on #enable_incremental_sort = on #enable_indexscan = on #enable_indexonlyscan = on #enable_material = on #enable_memoize = on #enable_mergejoin = on #enable_nestloop = on #enable_parallel_append = on #enable_parallel_hash = on #enable_partition_pruning = on #enable_partitionwise_join = off #enable_partitionwise_aggregate = off #enable_seqscan = on #enable_sort = on #enable_tidscan = on # - Planner Cost Constants - #seq_page_cost = 1.0 # measured on an arbitrary scale #random_page_cost = 4.0 # same scale as above #cpu_tuple_cost = 0.01 # same scale as above #cpu_index_tuple_cost = 0.005 # same scale as above #cpu_operator_cost = 0.0025 # same scale as above #parallel_setup_cost = 1000.0 # same scale as above #parallel_tuple_cost = 0.1 # same scale as above #min_parallel_table_scan_size = 8MB #min_parallel_index_scan_size = 512kB #effective_cache_size = 4GB #jit_above_cost = 100000 # perform JIT compilation if available # and query more expensive than this; # -1 disables #jit_inline_above_cost = 500000 # inline small functions if query is # more expensive than this; -1 disables #jit_optimize_above_cost = 500000 # use expensive JIT optimizations if # query is more expensive than this; # -1 disables # - Genetic Query Optimizer - #geqo = on #geqo_threshold = 12 #geqo_effort = 5 # range 1-10 #geqo_pool_size = 0 # selects default based on effort #geqo_generations = 0 # selects default based on effort #geqo_selection_bias = 2.0 # range 1.5-2.0 #geqo_seed = 0.0 # range 0.0-1.0 # - Other Planner Options - #default_statistics_target = 100 # range 1-10000 #constraint_exclusion = partition # on, off, or partition #cursor_tuple_fraction = 0.1 # range 0.0-1.0 #from_collapse_limit = 8 #jit = on # allow JIT compilation #join_collapse_limit = 8 # 1 disables collapsing of explicit # JOIN clauses #plan_cache_mode = auto # auto, force_generic_plan or # force_custom_plan #recursive_worktable_factor = 10.0 # range 0.001-1000000 #------------------------------------------------------------------------------ # REPORTING AND LOGGING #------------------------------------------------------------------------------ # - Where to Log - #log_destination = 'stderr' # Valid values are combinations of # From the Bulk Operations chapter, and CSV using the File FDW log_destination = 'stderr,csvlog' # Valid values are combinations of # stderr, csvlog, jsonlog, syslog, and # eventlog, depending on platform. # csvlog and jsonlog require # logging_collector to be on. # This is used when logging to stderr: logging_collector = on # Enable capturing of stderr, jsonlog, # and csvlog into log files. Required # to be on for csvlogs and jsonlogs. # (change requires restart) # These are only used if logging_collector is on: #log_directory = 'log' # directory where log files are written, # can be absolute or relative to PGDATA #log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern, # can include strftime() escapes #log_file_mode = 0600 # creation mode for log files, # begin with 0 to use octal notation #log_rotation_age = 1d # Automatic rotation of logfiles will # happen after that time. 0 disables. #log_rotation_size = 10MB # Automatic rotation of logfiles will # happen after that much log output. # 0 disables. #log_truncate_on_rotation = off # If on, an existing log file with the # same name as the new log file will be # truncated rather than appended to. # But such truncation only occurs on # time-driven rotation, not on restarts # or size-driven rotation. Default is # off, meaning append to existing files # in all cases. # These are relevant when logging to syslog: #syslog_facility = 'LOCAL0' #syslog_ident = 'postgres' #syslog_sequence_numbers = on #syslog_split_messages = on # This is only relevant when logging to eventlog (Windows): # (change requires restart) #event_source = 'PostgreSQL' # - When to Log - #log_min_messages = warning # values in order of decreasing detail: # debug5 # debug4 # debug3 # debug2 # debug1 # info # notice # warning # error # log # fatal # panic #log_min_error_statement = error # values in order of decreasing detail: # debug5 # debug4 # debug3 # debug2 # debug1 # info # notice # warning # error # log # fatal # panic (effectively off) log_min_duration_statement = 1000 # -1 is disabled, 0 logs all statements # and their durations, > 0 logs only # statements running at least this number # of milliseconds #log_min_duration_sample = -1 # -1 is disabled, 0 logs a sample of statements # and their durations, > 0 logs only a sample of # statements running at least this number # of milliseconds; # sample fraction is determined by log_statement_sample_rate #log_statement_sample_rate = 1.0 # fraction of logged statements exceeding # log_min_duration_sample to be logged; # 1.0 logs all such statements, 0.0 never logs #log_transaction_sample_rate = 0.0 # fraction of transactions whose statements # are logged regardless of their duration; 1.0 logs all # statements from all transactions, 0.0 never logs #log_startup_progress_interval = 10s # Time between progress updates for # long-running startup operations. # 0 disables the feature, > 0 indicates # the interval in milliseconds. # - What to Log - #debug_print_parse = off #debug_print_rewritten = off #debug_print_plan = off #debug_pretty_print = on #log_autovacuum_min_duration = 10min # log autovacuum activity; # -1 disables, 0 logs all actions and # their durations, > 0 logs only # actions running at least this number # of milliseconds. #log_checkpoints = on #log_connections = off #log_disconnections = off #log_duration = off #log_error_verbosity = default # terse, default, or verbose messages #log_hostname = off #log_line_prefix = '%m [%p] ' # special values: # %a = application name # %u = user name # %d = database name # %r = remote host and port # %h = remote host # %b = backend type # %p = process ID # %P = process ID of parallel group leader # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) # %Q = query ID (0 if none or not computed) # %i = command tag # %e = SQL state # %c = session ID # %l = session line number # %s = session start timestamp # %v = virtual transaction ID # %x = transaction ID (0 if none) # %q = stop here in non-session # processes # %% = '%' # e.g. '<%u%%%d> ' #log_lock_waits = off # log lock waits >= deadlock_timeout #log_recovery_conflict_waits = off # log standby recovery conflict waits # >= deadlock_timeout #log_parameter_max_length = -1 # when logging statements, limit logged # bind-parameter values to N bytes; # -1 means print in full, 0 disables #log_parameter_max_length_on_error = 0 # when logging an error, limit logged # bind-parameter values to N bytes; # -1 means print in full, 0 disables #log_statement = 'none' # none, ddl, mod, all #log_replication_commands = off #log_temp_files = -1 # log temporary files equal or larger # than the specified size in kilobytes; # -1 disables, 0 logs all temp files log_timezone = 'America/Chicago' #------------------------------------------------------------------------------ # PROCESS TITLE #------------------------------------------------------------------------------ #cluster_name = '' # added to process titles if nonempty # (change requires restart) #update_process_title = on #------------------------------------------------------------------------------ # STATISTICS #------------------------------------------------------------------------------ # - Cumulative Query and Index Statistics - #track_activities = on #track_activity_query_size = 1024 # (change requires restart) #track_counts = on #track_io_timing = off #track_wal_io_timing = off #track_functions = none # none, pl, all #stats_fetch_consistency = cache # - Monitoring - #compute_query_id = auto #log_statement_stats = off #log_parser_stats = off #log_planner_stats = off #log_executor_stats = off #------------------------------------------------------------------------------ # AUTOVACUUM #------------------------------------------------------------------------------ #autovacuum = on # Enable autovacuum subprocess? 'on' # requires track_counts to also be on. #autovacuum_max_workers = 3 # max number of autovacuum subprocesses # (change requires restart) #autovacuum_naptime = 1min # time between autovacuum runs #autovacuum_vacuum_threshold = 50 # min number of row updates before # vacuum #autovacuum_vacuum_insert_threshold = 1000 # min number of row inserts # before vacuum; -1 disables insert # vacuums #autovacuum_analyze_threshold = 50 # min number of row updates before # analyze #autovacuum_vacuum_scale_factor = 0.2 # fraction of table size before vacuum #autovacuum_vacuum_insert_scale_factor = 0.2 # fraction of inserts over table # size before insert vacuum #autovacuum_analyze_scale_factor = 0.1 # fraction of table size before analyze #autovacuum_freeze_max_age = 200000000 # maximum XID age before forced vacuum # (change requires restart) #autovacuum_multixact_freeze_max_age = 400000000 # maximum multixact age # before forced vacuum # (change requires restart) #autovacuum_vacuum_cost_delay = 2ms # default vacuum cost delay for # autovacuum, in milliseconds; # -1 means use vacuum_cost_delay #autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for # autovacuum, -1 means use # vacuum_cost_limit #------------------------------------------------------------------------------ # CLIENT CONNECTION DEFAULTS #------------------------------------------------------------------------------ # - Statement Behavior - #client_min_messages = notice # values in order of decreasing detail: # debug5 # debug4 # debug3 # debug2 # debug1 # log # notice # warning # error #search_path = '"$user", public' # schema names #row_security = on #default_table_access_method = 'heap' #default_tablespace = '' # a tablespace name, '' uses the default #default_toast_compression = 'pglz' # 'pglz' or 'lz4' #temp_tablespaces = '' # a list of tablespace names, '' uses # only default tablespace #check_function_bodies = on #default_transaction_isolation = 'read committed' #default_transaction_read_only = off #default_transaction_deferrable = off #session_replication_role = 'origin' #statement_timeout = 0 # in milliseconds, 0 is disabled #lock_timeout = 0 # in milliseconds, 0 is disabled #idle_in_transaction_session_timeout = 0 # in milliseconds, 0 is disabled #idle_session_timeout = 0 # in milliseconds, 0 is disabled #vacuum_freeze_table_age = 150000000 #vacuum_freeze_min_age = 50000000 #vacuum_failsafe_age = 1600000000 #vacuum_multixact_freeze_table_age = 150000000 #vacuum_multixact_freeze_min_age = 5000000 #vacuum_multixact_failsafe_age = 1600000000 #bytea_output = 'hex' # hex, escape #xmlbinary = 'base64' #xmloption = 'content' #gin_pending_list_limit = 4MB # - Locale and Formatting - datestyle = 'iso, mdy' #intervalstyle = 'postgres' timezone = 'America/Chicago' #timezone_abbreviations = 'Default' # Select the set of available time zone # abbreviations. Currently, there are # Default # Australia (historical usage) # India # You can create your own file in # share/timezonesets/. #extra_float_digits = 1 # min -15, max 3; any value >0 actually # selects precise output mode #client_encoding = sql_ascii # actually, defaults to database # encoding # These settings are initialized by initdb, but they can be changed. lc_messages = 'C' # locale for system error message # strings lc_monetary = 'C' # locale for monetary formatting lc_numeric = 'C' # locale for number formatting lc_time = 'C' # locale for time formatting # default configuration for text search default_text_search_config = 'pg_catalog.english' # - Shared Library Preloading - #local_preload_libraries = '' #session_preload_libraries = '' #shared_preload_libraries = '' # (change requires restart) # # Customized shared_preload_libraries for Rideshare # These extensions needed to be compiled for macOS and available # export PGDATA="$(psql $DATABASE_URL -c 'SHOW data_directory' --tuples-only | sed 's/^[ \t]*//')" # echo $PGDATA # vim $PGDATA/postgresql.conf shared_preload_libraries = 'pg_stat_statements,auto_explain,pg_cron,pg_hint_plan' # (change requires restart) #jit_provider = 'llvmjit' # JIT library to use # - Other Defaults - #dynamic_library_path = '$libdir' #gin_fuzzy_search_limit = 0 #------------------------------------------------------------------------------ # LOCK MANAGEMENT #------------------------------------------------------------------------------ #deadlock_timeout = 1s #max_locks_per_transaction = 64 # min 10 # (change requires restart) #max_pred_locks_per_transaction = 64 # min 10 # (change requires restart) #max_pred_locks_per_relation = -2 # negative values mean # (max_pred_locks_per_transaction # / -max_pred_locks_per_relation) - 1 #max_pred_locks_per_page = 2 # min 0 #------------------------------------------------------------------------------ # VERSION AND PLATFORM COMPATIBILITY #------------------------------------------------------------------------------ # - Previous PostgreSQL Versions - #array_nulls = on #backslash_quote = safe_encoding # on, off, or safe_encoding #escape_string_warning = on #lo_compat_privileges = off #quote_all_identifiers = off #standard_conforming_strings = on #synchronize_seqscans = on # - Other Platforms and Clients - #transform_null_equals = off #------------------------------------------------------------------------------ # ERROR HANDLING #------------------------------------------------------------------------------ #exit_on_error = off # terminate session on any error? #restart_after_crash = on # reinitialize after backend crash? #data_sync_retry = off # retry or panic on failure to fsync # data? # (change requires restart) #recovery_init_sync_method = fsync # fsync, syncfs (Linux 5.8+) #------------------------------------------------------------------------------ # CONFIG FILE INCLUDES #------------------------------------------------------------------------------ # These options allow settings to be loaded from files other than the # default postgresql.conf. Note that these are directives, not variable # assignments, so they can usefully be given more than once. #include_dir = '...' # include files ending in '.conf' from # a directory, e.g., 'conf.d' #include_if_exists = '...' # include file only if it exists #include = '...' # include file #------------------------------------------------------------------------------ # CUSTOMIZED OPTIONS #------------------------------------------------------------------------------ # Add settings for extensions here # # For PostgreSQL 14+, compute the query ID (to show) compute_query_id = on # Log statement duration log_duration = on # Log customization #log_statement = 'all' #log_line_prefix = '%t [%p]: [%l-1] user=%u,db=%d,app=%a,client=%h,query_id=%Q ' log_line_prefix = 'pid=%p query_id=%Q: ' # log slow queries log_min_duration_statement = 1000 # auto_explain auto_explain.log_min_duration = 1000 # pg_cron: # - Use the "postgres", when running pg_cron for multiple databases # - For Rideshare, work locally, use rideshare_development # cron.database_name = 'postgres' cron.database_name = 'rideshare_development' # For pg_cron # Make sure above is applied, and PG has restarted # psql -U postgres -d rideshare_development # create extension pg_cron; # GRANT USAGE ON SCHEMA cron TO owner; ================================================ FILE: postgresql/userlist.sample.txt ================================================ "owner" "HSnDDgFtyW9fyFI" ================================================ FILE: public/404.html ================================================ The page you were looking for doesn't exist (404)

The page you were looking for doesn't exist.

You may have mistyped the address or the page may have moved.

If you are the application owner check the logs for more information.

================================================ FILE: public/422.html ================================================ The change you wanted was rejected (422)

The change you wanted was rejected.

Maybe you tried to change something you didn't have access to.

If you are the application owner check the logs for more information.

================================================ FILE: public/500.html ================================================ We're sorry, but something went wrong (500)

We're sorry, but something went wrong.

If you are the application owner check the logs for more information.

================================================ FILE: public/robots.txt ================================================ # See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file ================================================ FILE: test/application_system_test_case.rb ================================================ require 'test_helper' class ApplicationSystemTestCase < ActionDispatch::SystemTestCase driven_by :selenium, using: :chrome, screen_size: [1400, 1400] end ================================================ FILE: test/controllers/.keep ================================================ ================================================ FILE: test/controllers/api/trip_requests_controller_test.rb ================================================ require 'test_helper' class Api::TripRequestsControllerTest < ActionDispatch::IntegrationTest test 'CREATE a trip request works' do rider = riders(:jane) trip_request = { rider_id: rider.id, start_address: 'Boston, MA', end_address: 'New York, NY' } post api_trip_requests_url, params: { trip_request: trip_request } assert_response 201 end test 'SHOW status for trip_request' do trip_request = trip_requests(:big_trip) get api_trip_request_url(trip_request) assert_response 200 assert response.parsed_body['trip_request_id'].present? assert response.parsed_body['trip_id'].present? end end ================================================ FILE: test/controllers/api/trips_controller_test.rb ================================================ require 'test_helper' class Api::TripsControllerTest < ActionDispatch::IntegrationTest test 'GET to index works' do get api_trips_url assert_response 200 assert_equal 3, response.parsed_body.size end test 'searching by start location works' do get api_trips_url, params: { start_location: 'New York' } assert_response 200 assert_equal 2, response.parsed_body.size end test 'searching by driver name works' do get api_trips_url, params: { driver_name: 'Jack' } assert_response 200 assert_equal 1, response.parsed_body.size end test 'searching by start location and rider name works' do get api_trips_url, params: { start_location: 'JFK', rider_name: 'Jessica' } assert_response 200 assert_equal 1, response.parsed_body.size end test 'show a single trip' do get api_trip_url(trip) assert_response 200 assert_equal trip.id, response.parsed_body['id'] end ### API: /my ### test 'get my trips' do get my_api_trips_url, headers: { 'Authorization' => auth_token }, params: { rider_id: trip.rider.id } assert_response 200 assert json = JSON.parse(response.body) assert first_trip = json['data'][0] assert_equal 'Jane D.', first_trip['attributes']['rider_name'] assert_equal 'Meg W.', first_trip['attributes']['driver_name'] end test 'get my trips sparse fieldset all fields' do get my_api_trips_url, headers: { 'Authorization' => auth_token }, params: { rider_id: trip.rider.id, 'fields[trips]' => 'rider_name,driver_name' } assert_response 200 assert json = JSON.parse(response.body) assert first_trip = json['data'][0] assert_equal 'Jane D.', first_trip['attributes']['rider_name'] assert_equal 'Meg W.', first_trip['attributes']['driver_name'] end test 'get my trips sparse fieldset subset of fields' do get my_api_trips_url, headers: { 'Authorization' => auth_token }, params: { rider_id: trip.rider.id, 'fields[trips]' => 'rider_name' } assert_response 200 assert json = JSON.parse(response.body) assert first_trip = json['data'][0] assert_equal 'Jane D.', first_trip['attributes']['rider_name'] assert_nil first_trip['attributes']['driver_name'] end test 'get my trips no auth token' do get my_api_trips_url, params: { rider_id: trip.rider.id } assert_response 401 end test 'get trip details' do get details_api_trip_url(id: trip.id) assert_response 200 assert json = JSON.parse(response.body) assert json.has_key?('data') assert_equal 'trip', json['data']['type'] end test 'get trip details with driver fields as compound document' do get details_api_trip_url(id: trip.id), params: { include: 'driver' } assert_response 200 assert json = JSON.parse(response.body) assert json.has_key?('data') assert_equal 'trip', json['data']['type'] assert json.has_key?('included') assert driver_details = json['included'][0]['attributes'] assert driver_details.has_key?('display_name') assert driver_details.has_key?('average_rating') end test 'get trip details with driver fields as compound document with sparse fieldset on driver' do get details_api_trip_url(id: trip.id), params: { include: 'driver', 'fields[driver]' => 'average_rating' } assert_response 200 assert json = JSON.parse(response.body) assert json.has_key?('data') assert_equal 'trip', json['data']['type'] assert json.has_key?('included') assert driver_details = json['included'][0]['attributes'] assert driver_details.has_key?('average_rating') assert_not driver_details.has_key?('display_name'), 'did not expect display_name for driver to be included' end private def trip @trip ||= trips(:completed_trip) end def auth_token JsonWebToken.encode(user_id: trip.rider.id) end end ================================================ FILE: test/controllers/authentication_controller_test.rb ================================================ require 'test_helper' class AuthenticationControllerTest < ActionDispatch::IntegrationTest test 'POST to login with correct user credentials' do post auth_login_url, params: { email: rider.email, password: 'abcd1234' } assert_response :ok assert jwt_payload = JSON.parse(response.body) assert jwt_payload.has_key?('token') assert jwt_payload.has_key?('exp') assert jwt_payload.has_key?('username') assert_equal rider.display_name, jwt_payload['username'] end test 'POST to login with INVALID credentials' do post auth_login_url, params: { email: rider.email, password: 'abcd123' } assert_response :unauthorized end private def rider # Rider has the hashed password for "abcd1234" # Stored in the field `password_digest` @rider ||= riders(:jane) end end ================================================ FILE: test/fixtures/.keep ================================================ ================================================ FILE: test/fixtures/drivers.yml ================================================ jack: first_name: Jack last_name: White email: jack@email.com type: Driver password_digest: $3a$12$D0/yLVQ67zujhbLZUBkd3eD2w4oNaTg1MK2o2w3f4IOXiZ6az/X0O drivers_license_number: P800000224322 meg: first_name: Meg last_name: White email: meg@email.com type: Driver password_digest: $4a$12$D0/yLVQ67zujhbLZUBkd3eD2w4oNaTg1MK2o2w3f4IOXiZ6az/X0O drivers_license_number: P800000224323 ================================================ FILE: test/fixtures/files/.keep ================================================ ================================================ FILE: test/fixtures/locations.yml ================================================ nyc: address: New York, NY position: "(40.7143528,-74.0059731)" state: NY comedy_cellar: address: "117 MacDougal St, New York, NY 10012" position: "(40.7303492,-74.0003215)" state: NY bos: address: Boston, MA position: "(42.361145,-71.057083)" state: MA jfk: address: JFK Airport position: "(40.6413111,-73.7781391)" state: NY ================================================ FILE: test/fixtures/riders.yml ================================================ jane: first_name: Jane last_name: Doe email: jane@email.com type: Rider password_digest: $2a$12$D0/yLVQ67zujhbLZUBkd3eD2w4oNaTg1MK2o2w3f4IOXiZ6az/X0O jessica: first_name: Jessica last_name: Cruz email: jessica@email.com type: Rider ================================================ FILE: test/fixtures/trip_requests.yml ================================================ big_trip: rider: jane start_location: nyc end_location: bos airport_trip: rider: jessica start_location: jfk end_location: nyc girls_night_out: rider: jessica start_location: nyc end_location: comedy_cellar ================================================ FILE: test/fixtures/trips.yml ================================================ incomplete_trip: trip_request: big_trip driver: jack completed_at: rating: completed_trip: trip_request: big_trip driver: meg completed_at: <%= 1.day.from_now.to_fs(:db) %> rating: created_at: <%= 1.day.ago.to_fs(:db) %> rated_trip: trip_request: airport_trip driver: meg completed_at: <%= 1.day.ago.to_fs(:db) %> rating: 5 created_at: <%= 1.week.ago.to_fs(:db) %> ================================================ FILE: test/fixtures/vehicle_reservations.yml ================================================ party_bus: vehicle: party_bus trip_request: girls_night_out starts_at: <%= Time.zone.local(2022, 07, 28, 19, 00, 00) %> ends_at: <%= Time.zone.local(2022, 07, 28, 23, 00, 00) %> canceled: false # should be supplied by database as DEFAULT ================================================ FILE: test/fixtures/vehicles.yml ================================================ party_bus: name: Party Bus ================================================ FILE: test/helpers/.keep ================================================ ================================================ FILE: test/mailers/.keep ================================================ ================================================ FILE: test/models/.keep ================================================ ================================================ FILE: test/models/driver_test.rb ================================================ require 'test_helper' class DriverTest < ActiveSupport::TestCase test 'valid driver' do assert driver = Driver.new(email: 'email@email.com') assert_not driver.valid? assert !driver.errors[:first_name].include?('be blank') end test "driver's license number format is validated" do assert driver = drivers(:meg) driver.drivers_license_number = '123' assert_not driver.valid? assert_equal ["is not a valid driver's license number"], driver.errors[:drivers_license_number] end test "driver's license number format must pass validation" do assert driver = drivers(:meg) driver.drivers_license_number = 'P800000224325' assert driver.valid?, driver.errors.full_messages end end ================================================ FILE: test/models/location_test.rb ================================================ require 'test_helper' class LocationTest < ActiveSupport::TestCase test 'valid location' do assert location = Location.new assert_not location.valid? assert !location.errors[:address].include?('be blank') end end ================================================ FILE: test/models/rider_test.rb ================================================ require 'test_helper' class RiderTest < ActiveSupport::TestCase test 'valid rider' do assert rider = Rider.new(email: 'email@email.com') assert_not rider.valid? assert !rider.errors[:first_name].include?('be blank') end end ================================================ FILE: test/models/trip_request_test.rb ================================================ require 'test_helper' class TripRequestTest < ActiveSupport::TestCase test 'trip request works' do trip_request = trip_requests(:airport_trip) assert trip_request.trip.present? assert trip_request.start_location.present? assert trip_request.end_location.present? assert trip_request.rider.present? end end ================================================ FILE: test/models/trip_test.rb ================================================ require 'test_helper' class TripTest < ActiveSupport::TestCase setup do @trip = trips(:completed_trip) end test 'rating values with valid values' do @trip.rating = 1 assert @trip.valid? @trip.rating = 5 assert @trip.valid? end test 'rating values with invalid values' do @trip.rating = 0 assert_not @trip.valid? assert_equal ['must be greater than or equal to 1'], @trip.errors[:rating] @trip.rating = 6 assert_not @trip.valid? assert_equal ['must be less than or equal to 5'], @trip.errors[:rating] @trip.rating = 2.5 assert_not @trip.valid? assert_equal ['must be an integer'], @trip.errors[:rating] end test 'rating requires a completed trip' do @incomplete_trip = trips(:incomplete_trip) @incomplete_trip.rating = 5 assert_not @incomplete_trip.valid? assert_equal ['must be completed before a rating can be added'], @incomplete_trip.errors[:rating] end end ================================================ FILE: test/models/user_test.rb ================================================ require 'test_helper' class UserTest < ActiveSupport::TestCase test 'user works' do driver = drivers(:jack) driver.email = '@email.com' assert_not driver.valid? assert_equal ['is not an email'], driver.errors[:email] end end ================================================ FILE: test/models/vehicle_reservation_test.rb ================================================ require 'test_helper' class VehicleReservationTest < ActiveSupport::TestCase test 'validity' do party_bus = VehicleReservation.new assert_not party_bus.valid? assert !party_bus.errors[:vehicle_id].include?('be blank') assert !party_bus.errors[:starts_at].include?('be blank') assert !party_bus.errors[:ends_at].include?('be blank') end end ================================================ FILE: test/models/vehicle_test.rb ================================================ require 'test_helper' class VehicleTest < ActiveSupport::TestCase test 'validity' do party_bus = Vehicle.new assert_not party_bus.valid? assert !party_bus.errors[:name].include?('be blank') end test 'a vehicle is in a draft state by default' do vehicle = vehicles(:party_bus) assert vehicle.status_draft? end end ================================================ FILE: test/services/book_reservation_test.rb ================================================ require 'test_helper' class BookReservationTest < ActiveSupport::TestCase test 'can book reservation' do jane = riders(:jane) nyc = locations(:nyc) comedy_cellar = locations(:comedy_cellar) party_bus = vehicles(:party_bus) reservation = BookReservation.new( vehicle_id: party_bus.id, rider_id: jane.id, start_location_id: nyc.id, end_location_id: comedy_cellar.id, starts_at: Time.zone.local(2022, 0o7, 29, 20, 0o0, 0o0), ends_at: Time.zone.local(2022, 0o7, 29, 23, 0o0, 0o0) ) assert_difference -> { ::VehicleReservation.count }, +1 do reservation.reserve! end end test 'can NOT book overlapping reservation' do existing_reservation = vehicle_reservations(:party_bus) violation_msg = 'PG::ExclusionViolation: ERROR: " + "conflicting key value violates exclusion constraint "non_overlapping_vehicle_registration"' assert_no_difference -> { ::VehicleReservation.count } do assert_raises(ActiveRecord::StatementInvalid, violation_msg) do new_reservation = BookReservation.new( vehicle_id: existing_reservation.vehicle_id, rider_id: existing_reservation.trip_request.rider.id, start_location_id: existing_reservation.trip_request.start_location.id, end_location_id: existing_reservation.trip_request.end_location.id, starts_at: (existing_reservation.starts_at + 1.hour).to_s, ends_at: (existing_reservation.starts_at + 2.hours).to_s ) new_reservation.reserve! end end end end ================================================ FILE: test/services/trip_creator_test.rb ================================================ require 'test_helper' class TripCreatorTest < ActiveSupport::TestCase test 'can create trip' do drivers(:jack) # at least one exists trip_request = trip_requests(:big_trip) trip_creator = TripCreator.new( trip_request_id: trip_request.id ) assert_difference -> { Trip.count }, +1 do trip_creator.create_trip! end end end ================================================ FILE: test/services/trip_search_test.rb ================================================ require 'test_helper' class TripSearchTest < ActiveSupport::TestCase test 'trip search no params works' do trip_search = TripSearch.new({}) assert trip_search.start_location assert trip_search.driver_name assert trip_search.rider_name end test 'trip search start location params' do trip_search = TripSearch.new({ start_location: 'JFK' }) assert trip_search.start_location.count >= 1 end test 'trip search driver name' do trip_search = TripSearch.new({ driver_name: 'Meg' }) assert trip_search.driver_name.count >= 1 end test 'trip search rider name' do trip_search = TripSearch.new({ rider_name: 'Jane' }) assert trip_search.rider_name.count >= 1 end end ================================================ FILE: test/system/.keep ================================================ ================================================ FILE: test/test_helper.rb ================================================ ENV['RAILS_ENV'] ||= 'test' require_relative '../config/environment' require 'rails/test_help' class ActiveSupport::TestCase # Run tests in parallel with specified workers parallelize(workers: :number_of_processors) # Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order. fixtures :all # Add more helper methods to be used by all tests here... # Geocoder.configure(lookup: :test, ip_lookup: :test) Geocoder::Lookup::Test.add_stub( 'New York, NY', [ { 'coordinates' => [40.7143528, -74.0059731], 'address' => 'New York, NY, USA', 'state' => 'New York', 'state_code' => 'NY', 'country' => 'United States', 'country_code' => 'US' } ] ) Geocoder::Lookup::Test.add_stub( 'Boston, MA', [ { 'coordinates' => [42.361145, -71.057083], 'address' => 'Boston, MA, USA', 'state' => 'Boston', 'state_code' => 'MA', 'country' => 'United States', 'country_code' => 'US' } ] ) end