Repository: rightscale/right_aws Branch: master Commit: 5e45a72260f0 Files: 82 Total size: 978.3 KB Directory structure: gitextract_uj4pe4k8/ ├── .gitignore ├── Dockerfile ├── Gemfile ├── History.txt ├── LICENSE ├── Manifest.txt ├── README.txt ├── Rakefile ├── docker-compose.yml ├── lib/ │ ├── acf/ │ │ ├── right_acf_interface.rb │ │ ├── right_acf_invalidations.rb │ │ ├── right_acf_origin_access_identities.rb │ │ └── right_acf_streaming_interface.rb │ ├── acw/ │ │ └── right_acw_interface.rb │ ├── as/ │ │ └── right_as_interface.rb │ ├── awsbase/ │ │ ├── benchmark_fix.rb │ │ ├── right_awsbase.rb │ │ ├── support.rb │ │ └── version.rb │ ├── ec2/ │ │ ├── right_ec2.rb │ │ ├── right_ec2_ebs.rb │ │ ├── right_ec2_images.rb │ │ ├── right_ec2_instances.rb │ │ ├── right_ec2_monitoring.rb │ │ ├── right_ec2_placement_groups.rb │ │ ├── right_ec2_reserved_instances.rb │ │ ├── right_ec2_security_groups.rb │ │ ├── right_ec2_spot_instances.rb │ │ ├── right_ec2_tags.rb │ │ ├── right_ec2_vpc.rb │ │ ├── right_ec2_vpc2.rb │ │ └── right_ec2_windows_mobility.rb │ ├── elb/ │ │ └── right_elb_interface.rb │ ├── emr/ │ │ └── right_emr_interface.rb │ ├── iam/ │ │ ├── right_iam_access_keys.rb │ │ ├── right_iam_groups.rb │ │ ├── right_iam_interface.rb │ │ ├── right_iam_mfa_devices.rb │ │ └── right_iam_users.rb │ ├── rds/ │ │ └── right_rds_interface.rb │ ├── right_aws.rb │ ├── route_53/ │ │ └── right_route_53_interface.rb │ ├── s3/ │ │ ├── right_s3.rb │ │ └── right_s3_interface.rb │ ├── sdb/ │ │ ├── active_sdb.rb │ │ └── right_sdb_interface.rb │ ├── sns/ │ │ └── right_sns_interface.rb │ └── sqs/ │ ├── right_sqs.rb │ ├── right_sqs_gen2.rb │ ├── right_sqs_gen2_interface.rb │ └── right_sqs_interface.rb ├── right_aws.gemspec └── test/ ├── README.mdown ├── acf/ │ ├── test_helper.rb │ └── test_right_acf.rb ├── awsbase/ │ ├── test_helper.rb │ └── test_right_awsbase.rb ├── ec2/ │ ├── test_helper.rb │ └── test_right_ec2.rb ├── elb/ │ ├── test_helper.rb │ └── test_right_elb.rb ├── http_connection.rb ├── rds/ │ ├── test_helper.rb │ └── test_right_rds.rb ├── route_53/ │ ├── fixtures/ │ │ ├── a_record.xml │ │ └── alias_record.xml │ ├── test_helper.rb │ └── test_right_route_53.rb ├── s3/ │ ├── test_helper.rb │ ├── test_right_s3.rb │ └── test_right_s3_stubbed.rb ├── sdb/ │ ├── test_active_sdb.rb │ ├── test_batch_put_attributes.rb │ ├── test_helper.rb │ └── test_right_sdb.rb ├── sns/ │ ├── test_helper.rb │ └── test_right_sns.rb ├── sqs/ │ ├── test_helper.rb │ ├── test_right_sqs.rb │ └── test_right_sqs_gen2.rb ├── test_credentials.rb └── ts_right_aws.rb ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ pkg/ .rvmrc ================================================ FILE: Dockerfile ================================================ FROM ruby:2.1.10 RUN apt-get update -qq && apt-get install -y \ build-essential \ libxml2 \ libxml2-dev \ && rm -rf /var/lib/apt/lists/* WORKDIR /right_aws COPY Gemfile Gemfile.lock /right_aws/ RUN bundle install ================================================ FILE: Gemfile ================================================ source "http://rubygems.org" gem "right_http_connection", ">= 1.5.0", git: "https://github.com/flexera-public/right_http_connection" gem 'libxml-ruby', "2.8.0" group :development do gem 'rake', '10.4.2' end ================================================ FILE: History.txt ================================================ == 1.1.0 2007-08-10 Initial release. == 1.2.0 2007-09-12 * r1718, todd, 2007-09-12 15:34:37 * # 458, Extensive documentation review, rework, and expansion. Also added coverage analysis to the test suite using RCov. * r1690, todd, 2007-09-07 15:23:11 * # 447, Add support.rb to manifest * r1688, todd, 2007-09-07 13:57:39 * # 447, Use Active Support if available, but don't require it. Load our own extensions if it's not present. This keeps us from overloading ActiveSupport if a user's already using it. * r1687, todd, 2007-09-07 11:36:43 * # 447, Removed dependency on activesupport * r1676, konstantin, 2007-09-06 01:27:09 * paid AMIs, small fix * r1667, konstantin, 2007-09-05 12:58:10 * # 427, paid AMI support for ec2_instances * r1658, konstantin, 2007-09-05 01:02:25 * params improvements for ec2.describe_xxx, now ones can use a String as well as an Array. * r1653, konstantin, 2007-09-04 12:31:19 * libxml and paid AMI support added * r1581, tve, 2007-08-24 16:21:45 * Improved RightAws documentation == 1.3.0 2007-09-26 * r1754, todd, 2007-09-19 13:48:34 * # 487, # 488, Consolidate a lot of code that was repeated in three places. Fix error handling path when using streaming GET interfaces with S3. Also add a stub for RightHttpConnection which allows more control over the unit tests. Expand the unit tests and coverage tests. * r1755, todd, 2007-09-19 14:29:19 * # 487, RDoc fixes after code consolidation * r1866, konstantin, 2007-10-05 06:17:36 * # 220, close connection on HTTP 5xx/4xx errors == 1.4.0 2007-10-10 * r1868, konstantin, 2007-10-05 06:38:37 * # 220, inst_type branch merge into 1.3.0 release * r1869, konstantin, 2007-10-05 07:32:34 * right_http_connection 1.1.1 requirements fixed * r1879, konstantin, 2007-10-07 02:11:24 * # 524, blocks added to ec2_describe_xxx to support aws_cache * r1924, konstantin, 2007-10-12 11:35:06 * # 536, user_data bug fix * r1929, tve, 2007-10-15 00:00:11 * Fix libxml/rexml selection bug * r1938, konstantin, 2007-10-16 10:53:56 * instance type support is set to default == 1.4.3 2007-10-25 * r1983, konstantin, 2007-10-25 22:33:00 +0400 * Fixed ActiveSupport requirement bug (thanks to Toby) * Fixed HttpConnection logging to stdout bug (thanks to Toby) == 1.4.4 * r1999, tve, 2007-11-01 00:07:00 -0700 * Fixed escaping issue affecting key names in S3 gem * Fixed duplicate marker in S3 incremental bucket listing * r2001, konstantin, 2007-11-01 12:03:13 +0300 * Fixed multiple permissions assignment on Grantee#grant/revoke * Fixed new grantee permissions set ingnore (Grantee#apply) * S3::Grantee#exists? method added * r2109, konstantin, 2007-11-12 21:49:36 +0300 * RightAwsBaseInterface: caching implemented. (The Ec2 functions are being cached: describe_images, describe_instances, describe_security_groups and describe_key_pairs) == 1.4.5 - 1.4.6 * r 2619, konstantin, 01-17-08 16:18:36 +0300 * S3 Location constraints support added. * Fixed bug with trailing '/' in the bucket name for 'EU' located buckets * Added: S3Interface#bucket_location, S3::Bucket#location == 1.4.7 * r 2622, konstantin, 01-18-08 13:52:20 +0300 * Virtual domains doc added * S3 Query API fixed to support virtual domains. == 1.4.8 * r 2650, konstantin, 01-24-08 11:12:00 +0300 * net_fix.rb moved to right_http_connection == 1.5.0 * r 2688, konstantin, 02-30-08 15:42:00 +0300 * SDB support added. * RightAws::S3::bucket and RightAws::S3::Bucket.create methods behaviour changed: param +create+ is set to +false+ by default. == 1.6.0 * r2780, todd, 2008-02-11 11:41:07 -0800 (Mon, 11 Feb 2008), 4 lines * Some doc updates & tweaks: we now support ultra-large PUTs to S3, small SDB cleanups, add SDB to the list of interfaces in RightAws, update some copyrights, warn about loading attachment_fu AFTER right_aws (big no-no). * r2784, todd, 2008-02-11 13:46:47 -0800 (Mon, 11 Feb 2008), 2 lines * One final clarification: you may get a Net::HTTP bad monkey patch exception * r2880, todd, 2008-02-25 18:06:22 -0800 (Mon, 25 Feb 2008), 2 lines * Add SQS 'Gen 2' interface implementation * r2913, todd, 2008-02-29 16:57:07 -0800 (Fri, 29 Feb 2008), 3 lines * SqsGen2 (object interface), unit tests for both interfaces, documentation updates. * r2922, todd, 2008-03-03 15:26:42 -0800 (Mon, 03 Mar 2008), 2 lines * couple of documentation tweaks in prep for 1.6.0 == 1.6.1 * r2963, todd, 2008-03-06 19:10:23 -0800 (Thu, 06 Mar 2008), 3 lines * (#950) Many minor fixes in incrementally_list_bucket to prevent a death loop in certain rare conditions == 1.7.0 * r3051, konstantin, 2008-03-14 21:26:12 +0300 (Fri, 14 Mar 2008), 1 line * #897, ActiveSdb alpha release == 1.7.1 Do not autoload right_sdb with the rest of the modules; it requires uuidtools. We want the user to explicly request ActiveSdb, at least as long as it is alpha/beta. Fix escaping problem in SqsGen2Interface: POST bodies did not properly escape the '&' character == 1.7.2 Release Notes: RightAws includes some new features, including: - Support in RightAws::S3 and RightAws::S3Interface for S3 key copy, move, and rename - Support for signature version 0 request authentication to EC2, SQS, and SDB - Enhanced S3 object meta-header read and update - Interoperability with clouds running Eucalyptus (http://eucalyptus.cs.ucsb.edu) [ Contributed by the Eucalyptus group ] - Support for c1.medium and c1.xlarge instance types Bug fixes include: - Corrected the failure, under certain conditions, of retries of streaming PUTs to S3. We now reset the seek pointer of the streaming IO object to its initial position. - Removal of an accidental dependency on ActiveSupport in RightAws::S3Interface.get_link(). - Monkey-patch of the Ruby File class on Windows platforms to correct a problem in lstat. The lstat bug was causing failure of very large file uploads on Windows [ Contributed by Benjamin Allfree ] - Fixed parsing of the ETag field for S3 objects == 1.7.3 Release Notes: - Removed the 1.7.2 monkey-patch of the Ruby File class on Windows. This patch broke Rails 2.0. The patch is now included in the README for anyone to use at their own risk. == 1.8.0 Release Notes: This release adds major new features to RightAws to support Amazon's new Elastic Block Store (EBS). Via the RightAws::Ec2 module, users can create and delete EBS volumes, attach and detach them from instances, snapshot volumes, and list the available volumes and snapshots. Bug fixes include correction of RightAws::S3 copy's failure to url-encode the source key. == 1.8.1 Release Notes: RightScale::SdbInterface & ::ActiveSdb have several enhancements, including: - RightAws::SdbInterface#last_query_expression added for debug puposes - RightAws::ActiveSdb::Base#query :order and :auto_load options added to support query result sorting and attributes auto loading - RightAws::ActiveSdb::Base#find_all_by_ and find_by_ helpers improved to support :order, :auto_load, :limit and :next_token options - RightAws::SdbInterface#delete_attributes bug fixed - SdbInterface allows specification of a string value to use for representing Ruby nil in SDB. - Sdb tests fixed and improved The ::S3 interface now has support for S3's server access logging. Amazon considers server access logging to be a beta or provisional feature. === 1.9.0 Release Notes: - RightAws::Ec2 now supports Windows instances. Added: - Ec2::get_initial_password - Ec2::bundle_instance - Ec2::describe_bundle_tasks - Ec::cancel_bundle_task - Full Amazon CloudFront support added with RightAws::AcfInterface - Bug fixes to S3Interface::store_object_and_verify and S3Interface::retrieve_object_and_verify (thanks to numerous user reports) - Updates to caching for Ec2::describe_images_by methods - Ec2 now has Ec2::last_request_id === 1.10.0 Release Notes: - AwsBase: signature v2 support added - Ec2: describe_availability_zones improved to support regions - CloudFront: docs fixes - SDB: added: SQL-like query, select and query_with_attributes support - SDB: fixed no method error when searching for id that doesn't exist === 1.11.0 Release Notes: - Full Amazon RDS instances support added with RightAws::RdsInterface - Boot from EBS support added - VPC support added - Latest EC2 API 2009-10-31 support added - Some of bugs fixed === 2.0.0 Release Notes: - Added: - Ruby 1.9 support - Ec2: - SpotInstances support - m2.xlarge instances - GetPasswordData API call (see get_password_data_v2) - SecurityGroups support for Eucalyptus clouds - EBS: - :delete_on_termination field for volumes - SimpleDB: - BatchPutAttributes support - ActiveSDB: - Dynamic attribute accessors - "Columns" support - Simple Type Casting support - ELB: - API '2009-11-25' support (stickiness policies) - ACF: - API '2010-03-01' support (origin access policy and streaming distributions) - Bunch of small issues were fixed - Time objects were replaced by Strings (as Amazon returns them) to make the gem more consistent: - :last_modified_time in: RightAws::AcfInterface#incrementally_list_distributions, RightAws::AcfInterface#create_distribution_by_config, RightAws::AcfInterface#get_distribution, RightAws::AcfInterface#get_distribution_config - :timestamp in: RightAws::AcwInterface#get_metric_statistics - :aws_created_at in: RightAws::Ec2#create_volume, RightAws::Ec2#describe_volumes - :aws_attached_at in: RightAws::Ec2#attach_volume, RightAws::Ec2#detach_volume, RightAws::Ec2#describe_volumes - :aws_started_at in: RightAws::Ec2#describe_snapshots, RightAws::Ec2#create_snapshot, RightAws::Ec2#try_create_snapshot - :created_time in: RightAws::ElbInterface#describe_load_balancers === 2.1.0 Release Notes: - Added: - Route 53: API '2010-10-01' - ACF: API '2010-11-01' - EC2: - API '2010-08-31' - Port based group permissions support - HPC Support - Tags Suport - ClientToken support added on instance launch - RDS: API "2010-07-28" - ELB: API '2010-07-01' (SSL support) - IAM: API '2010-05-08' (AWS Identity and Access Management interface) - 301 Redirect support added - Removed: - ActiveSupport dependency - SDB: uuid gem dependency - this gem requires right_http_connection 0bc3343232133bdb38c237d8285525d74495d3f5 or later - "Raise On Timeout On Action" feature added to avoid duplicate resources creation if a timeout error occures and a retry is performed === 3.0.0 Release Notes: - Fixed/Added: - ClientToken (launch_instances, run_instances) is not used for Eucalyptus clouds - VPC2, stage1. Next methods were updated: - associate_address, modify_security_group, create_security_group, create_vpc, delete_security_group, describe_addresses, describe_images, describe_instance_attribute, describe_regions, describe_reserved_instances_offerings, describe_security_groups, describe_snapshots, describe_spot_instance_requests, describe_spot_price_history, describe_vpcs, disassociate_address, modify_image_attribute, modify_snapshot_attribute, release_address, request_spot_instances, stop_instances - EC2: ClientToken (launch_instances, run_instances) is not used for Eucalyptus clouds - RDS: - RDS: API 2011-04-01 - Make :instance_class param more consistent: :db_instance_class --> :instance_class - Issue 53: regression in latest master version of right_rds_interface - Issue 73: Can't get list of instances with RdsInterface - EBS: Issue 54: regression in right_ec2_ebs.rb - Add the port number with server name in the v2 signature string if it is not the RFC standard number (author: unakatsuo). - EMR (Elastic Map Reduce) support - SNS (Simple Notification Service) support - SDB: ConsistentRead support - bunch of micro bugs === 3.0.1 Release Notes: - Fixed: - SignatureDoesNotMatch on file download via get_link() - S3#bucket should not fail for non admin creds - couple doc typos === 3.0.2 Release Notes: - Fixed: - S3 Content-Type not set in Ruby 1.9.2 - error in rds_interface describe_db_snapshots in Ruby 1.9.2 === 3.0.4 Release Notes: - Fixed: - #125 - fixes redirect bug in file PUT requests (Cary) - some other minor fixes === 3.0.5 Release Notes: - Added: API '2012-06-15' support for CreateVolume and DescribeVolumes API calls (to support IOPS) - Fixed: - Single-threaded multipart upload support (https://github.com/rightscale/right_aws/pull/116) - S3 multi object delete (https://github.com/rightscale/right_aws/pull/106) - Support for "ami_version" added in emr interface (https://github.com/rightscale/right_aws/pull/129) - S3: Added block references to several methods (https://github.com/rightscale/right_aws/pull/130) - Some other minor changes === 3.1.0 Release Notes: - Added: - EC2: - hs1.8xlarge, cr1.8xlarge instance types - API version '2012-10-01' support for ReservedInstances and ReservedInstancesOfferings - API version '2012-10-15' for some of VPC calls (including new DescribeAccountAttributes) - Removed: UUID dependency - Fixed: - EC2: - describe_reserved_instances_offerings was fixed to support pagination - typo in create_vpc - RDS: instances types list - Some other minor bugs ================================================ FILE: LICENSE ================================================ Copyright (c) 2009-2012 RightScale, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Manifest.txt ================================================ History.txt Manifest.txt README.txt Rakefile lib/awsbase/support.rb lib/awsbase/benchmark_fix.rb lib/awsbase/right_awsbase.rb lib/ec2/right_ec2.rb lib/ec2/right_ec2_images.rb lib/ec2/right_ec2_instances.rb lib/ec2/right_ec2_security_groups.rb lib/ec2/right_ec2_spot_instances.rb lib/ec2/right_ec2_ebs.rb lib/ec2/right_ec2_reserved_instances.rb lib/ec2/right_ec2_vpc.rb lib/ec2/right_ec2_vpc2.rb lib/ec2/right_ec2_monitoring.rb lib/ec2/right_ec2_placement_groups.rb lib/ec2/right_ec2_windows_mobility.rb lib/ec2/right_ec2_tags.rb lib/right_aws.rb lib/s3/right_s3.rb lib/acw/right_acw_interface.rb lib/elb/right_elb_interface.rb lib/as/right_as_interface.rb lib/s3/right_s3_interface.rb lib/sdb/active_sdb.rb lib/sdb/right_sdb_interface.rb lib/sqs/right_sqs.rb lib/sqs/right_sqs_gen2.rb lib/sqs/right_sqs_gen2_interface.rb lib/sqs/right_sqs_interface.rb lib/acf/right_acf_interface.rb lib/acf/right_acf_streaming_interface.rb lib/acf/right_acf_origin_access_identities.rb lib/acf/right_acf_invalidations.rb lib/rds/right_rds_interface.rb lib/iam/right_iam_interface.rb lib/iam/right_iam_groups.rb lib/iam/right_iam_users.rb lib/iam/right_iam_access_keys.rb lib/iam/right_iam_mfa_devices.rb lib/route_53/right_route_53_interface.rb test/ec2/test_helper.rb test/ec2/test_right_ec2.rb test/http_connection.rb test/s3/test_helper.rb test/s3/test_right_s3.rb test/s3/test_right_s3_stubbed.rb test/sdb/test_active_sdb.rb test/sdb/test_helper.rb test/sdb/test_right_sdb.rb test/sqs/test_helper.rb test/sqs/test_right_sqs.rb test/sqs/test_right_sqs_gen2.rb test/test_credentials.rb test/ts_right_aws.rb test/acf/test_helper.rb test/acf/test_right_acf.rb test/rds/test_helper.rb test/rds/test_right_rds.rb ================================================ FILE: README.txt ================================================ = RightScale Amazon Web Services Ruby Gems Published by RightScale, Inc. under the MIT License. For information about RightScale, see http://www.rightscale.com == THE GEM IS NO LONGER MAINTAINED We recommend you use https://github.com/rightscale/right_aws_api instead. == DESCRIPTION: The RightScale AWS gems have been designed to provide a robust, fast, and secure interface to Amazon EC2, EBS, S3, SQS, SDB, and CloudFront. These gems have been used in production by RightScale since late 2006 and are being maintained to track enhancements made by Amazon. The RightScale AWS gems comprise: - RightAws::Ec2 -- interface to Amazon EC2 (Elastic Compute Cloud), VPC (Virtual Private Cloud) and the associated EBS (Elastic Block Store) - RightAws::S3 and RightAws::S3Interface -- interface to Amazon S3 (Simple Storage Service) - RightAws::Sqs and RightAws::SqsInterface -- interface to first-generation Amazon SQS (Simple Queue Service) - RightAws::SqsGen2 and RightAws::SqsGen2Interface -- interface to second-generation Amazon SQS (Simple Queue Service) - RightAws::SdbInterface and RightAws::ActiveSdb -- interface to Amazon SDB (SimpleDB) - RightAws::AcfInterface -- interface to Amazon CloudFront, a content distribution service - RightAws::AsInterface -- interface to Amazon Auto Scaling - RightAws::AcwInterface -- interface to Amazon Cloud Watch - RightAws::ElbInterface -- interface to Amazon Elastic Load Balancer - RightAws::RdsInterface -- interface to Amazon RDS instances == FEATURES: - Full programmmatic access to EC2, EBS, S3, SQS, SDB, CloudFront, AS, ACW, ELB and RDS. - Complete error handling: all operations check for errors and report complete error information by raising an AwsError. - Persistent HTTP connections with robust network-level retry layer using RightHttpConnection). This includes socket timeouts and retries. - Robust HTTP-level retry layer. Certain (user-adjustable) HTTP errors returned by Amazon's services are classified as temporary errors. These errors are automaticallly retried using exponentially increasing intervals. The number of retries is user-configurable. - Fast REXML-based parsing of responses (as fast as a pure Ruby solution allows). - Uses libxml (if available) for faster response parsing. - Support for large S3 list operations. Buckets and key subfolders containing many (> 1000) keys are listed in entirety. Operations based on list (like bucket clear) work on arbitrary numbers of keys. - Support for streaming GETs from S3, and streaming PUTs to S3 if the data source is a file. - Support for single-threaded usage, multithreaded usage, as well as usage with multiple AWS accounts. - Support for both first- and second-generation SQS (API versions 2007-05-01 and 2008-01-01). These versions of SQS are not compatible. - Support for signature versions 0 and 1 on SQS, SDB, and EC2. - Interoperability with any cloud running Eucalyptus (http://eucalyptus.cs.ucsb.edu) - Test suite (requires AWS account to do "live" testing). == THREADING: All RightScale AWS interfaces offer two threading options: 1. Use a single persistent HTTP connection per process. 2. Use a persistent HTTP connection per Ruby thread. Either way, it doesn't matter how many (for example) RightAws::S3 objects you create, they all use the same per-program or per-thread connection. The purpose of sharing the connection is to keep a single persistent HTTP connection open to avoid paying connection overhead on every request. However, if you have multiple concurrent threads, you may want or need an HTTP connection per thread to enable concurrent requests to AWS. The way this plays out in practice is: 1. If you have a non-multithreaded Ruby program, use the non-multithreaded setting. 2. If you have a multi-threaded Ruby program, use the multithreaded setting to enable concurrent requests to S3 (or SQS, or SDB, or EC2). 3. For running under Mongrel/Rails, use the non-multithreaded setting even though mongrel is multithreaded. This is because only one Rails handler is invoked at time (i.e. it acts like a single-threaded program) Note that due to limitations in the I/O of the Ruby interpreter you may not get the degree of parallelism you may expect with the multi-threaded setting. == GETTING STARTED: * For EC2 read RightAws::Ec2 and consult the Amazon EC2 API documentation at http://developer.amazonwebservices.com/connect/kbcategory.jspa?categoryID=87 * For S3 read RightAws::S3 and consult the Amazon S3 API documentation at http://developer.amazonwebservices.com/connect/kbcategory.jspa?categoryID=48 * For first generation SQS read RightAws::Sqs and consult the Amazon SQS API documentation at http://developer.amazonwebservices.com/connect/kbcategory.jspa?categoryID=31 * For second generation SQS read RightAws::SqsGen2, RightAws::SqsGen2Interface, and consult the Amazon SQS API documentation at http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1214&categoryID=31 Amazon's Migration Guide for moving from first to second generation SQS is avalable at: http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1148 * For SDB read RightAws::SdbInterface, RightAws::ActiveSdb, and consult the Amazon SDB API documentation at http://developer.amazonwebservices.com/connect/kbcategory.jspa?categoryID=141 * For CloudFront (ACF) read RightAws::AcfInterface and consult the Amazon CloudFront API documentation at http://developer.amazonwebservices.com/connect/kbcategory.jspa?categoryID=213 == KNOWN ISSUES: - 7/08: A user has reported that uploads of large files on Windows may be broken on some Win platforms due to a buggy File.lstat.size. Use the following monkey-patch at your own risk, as it has been proven to break Rails 2.0 on Windows: require 'win32/file' class File def lstat self.stat end end - Attempting to use the Gibberish plugin (used by the Beast forum app) will break right_aws as well as lots of other code. Gibberish changes the semantics of core Ruby (specifically, the String class) and thus presents a reliability problem for most Ruby programs. - 2/11/08: If you use RightAws in conjunction with attachment_fu, the right_aws gem must be included (using the require statement) AFTER attachment_fu. If right_aws is loaded before attachment_fu, you'll encounter errors similar to: s3.amazonaws.com temporarily unavailable: (wrong number of arguments (5 for 4)) or 'incompatible Net::HTTP monkey-patch' This is due to a conflict between the right_http_connection gem and another gem required by attachment_fu. - 8/07: Amazon has changed the semantics of the SQS service. A new queue may not be created within 60 seconds of the destruction of any older queue with the same name. Certain methods of RightAws::Sqs and RightAws::SqsInterface will fail with the message: "AWS.SimpleQueueService.QueueDeletedRecently: You must wait 60 seconds after deleting a queue before you can create another with the same name." == REQUIREMENTS: RightAws requires REXML and the right_http_connection gem. If libxml and its Ruby bindings (distributed in the libxml-ruby gem) are present, RightAws can be configured to use them: RightAws::RightAWSParser.xml_lib = 'libxml' Any error with the libxml installation will result in RightAws failing-safe to REXML parsing. == INSTALL: sudo gem install right_aws == LICENSE: Copyright (c) 2007-2013 RightScale, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Rakefile ================================================ # -*- ruby -*- require 'rubygems' require "rake/testtask" require 'rubygems/package_task' require 'rake/clean' $: << File.dirname(__FILE__) testglobs = ["test/ts_right_aws.rb"] begin require 'bundler' rescue LoadError => e STDERR.puts("Bundler is not available, some rake tasks will not be defined: #{e.message}") else Bundler::GemHelper.install_tasks :name => 'right_aws' end # == Gem == # gemtask = Gem::PackageTask.new(Gem::Specification.load("right_aws.gemspec")) do |package| package.package_dir = ENV['PACKAGE_DIR'] || 'pkg' package.need_zip = true package.need_tar = true end directory gemtask.package_dir CLEAN.include(gemtask.package_dir) desc "Test just the SQS interface" task :testsqs do require 'test/test_credentials' require 'test/http_connection' TestCredentials.get_credentials require 'test/sqs/test_right_sqs.rb' end desc "Test just the second generation SQS interface" task :testsqs2 do require 'test/test_credentials' require 'test/http_connection' TestCredentials.get_credentials require 'test/sqs/test_right_sqs_gen2.rb' end desc "Test just the S3 interface" task :tests3 do require 'test/test_credentials' require 'test/http_connection' TestCredentials.get_credentials require 'test/s3/test_right_s3.rb' end desc "Test just the S3 interface using local stubs" task :tests3local do require 'test/test_credentials' require 'test/http_connection' TestCredentials.get_credentials require 'test/s3/test_right_s3_stubbed.rb' end desc "Test just the EC2 interface" task :testec2 do require 'test/test_credentials' TestCredentials.get_credentials require 'test/ec2/test_right_ec2.rb' end desc "Test just the SDB interface" task :testsdb do require 'test/test_credentials' TestCredentials.get_credentials require 'test/sdb/test_right_sdb.rb' end desc "Test active SDB interface" task :testactivesdb do require 'test/test_credentials' TestCredentials.get_credentials require 'test/sdb/test_active_sdb.rb' end desc "Test CloudFront interface" task :testacf do require 'test/test_credentials' TestCredentials.get_credentials require 'test/acf/test_right_acf.rb' end desc "Test RDS interface" task :testrds do require 'test/test_credentials' TestCredentials.get_credentials require 'test/rds/test_right_rds.rb' end desc "Test just the SNS interface" task :testsns do require 'test/test_credentials' TestCredentials.get_credentials require 'test/sns/test_right_sns.rb' end desc "Test Route 53 interface" task :testroute53 do require 'test/test_credentials' TestCredentials.get_credentials require 'test/route_53/test_right_route_53' end desc "Test ELB interface" task :testelb do require 'test/test_credentials' TestCredentials.get_credentials require 'test/elb/test_right_elb' end # vim: syntax=Ruby ================================================ FILE: docker-compose.yml ================================================ version: "3.8" services: app: build: . command: bash -c "bundle exec rake -T" volumes: - .:/right_aws ================================================ FILE: lib/acf/right_acf_interface.rb ================================================ # # Copyright (c) 2008 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws # = RightAws::AcfInterface -- RightScale Amazon's CloudFront interface # The AcfInterface class provides a complete interface to Amazon's # CloudFront service. # # For explanations of the semantics of each call, please refer to # Amazon's documentation at # http://developer.amazonwebservices.com/connect/kbcategory.jspa?categoryID=211 # # Example: # # acf = RightAws::AcfInterface.new('1E3GDYEOGFJPIT7XXXXXX','hgTHt68JY07JKUY08ftHYtERkjgtfERn57XXXXXX') # # list = acf.list_distributions #=> # [{:status => "Deployed", # :domain_name => "d74zzrxmpmygb.6hops.net", # :aws_id => "E4U91HCJHGXVC", # :s3_origin => {:dns_name=>"bucket-for-konstantin-00.s3.amazonaws.com"}, # :cnames => ["x1.my-awesome-site.net", "x1.my-awesome-site.net"] # :comment => "My comments", # :last_modified_time => "2008-09-10T17:00:04.000Z" }, ..., {...} ] # # distibution = list.first # # info = acf.get_distribution(distibution[:aws_id]) #=> # {:last_modified_time=>"2010-05-19T18:54:38.242Z", # :status=>"Deployed", # :domain_name=>"dpzl38cuix402.cloudfront.net", # :caller_reference=>"201005181943052207677116", # :e_tag=>"EJSXFGM5JL8ER", # :s3_origin=> # {:dns_name=>"bucket-for-konstantin-eu.s3.amazonaws.com", # :origin_access_identity=> # "origin-access-identity/cloudfront/E3JPJZ80ZBX24G"}, # :aws_id=>"E5P8HQ3ZAZIXD", # :enabled=>false} # # config = acf.get_distribution_config(distibution[:aws_id]) #=> # {:enabled => true, # :caller_reference => "200809102100536497863003", # :e_tag => "E39OHHU1ON65SI", # :cnames => ["web1.my-awesome-site.net", "web2.my-awesome-site.net"] # :comment => "Woo-Hoo!", # :s3_origin => {:dns_name => "my-bucket.s3.amazonaws.com"}} # # config[:comment] = 'Olah-lah!' # config[:enabled] = false # config[:cnames] << "web3.my-awesome-site.net" # # acf.set_distribution_config(distibution[:aws_id], config) #=> true # class AcfInterface < RightAwsBase include RightAwsBaseInterface API_VERSION = "2010-11-01" DEFAULT_HOST = 'cloudfront.amazonaws.com' DEFAULT_PORT = 443 DEFAULT_PROTOCOL = 'https' DEFAULT_PATH = '/' @@bench = AwsBenchmarkingBlock.new def self.bench_xml @@bench.xml end def self.bench_service @@bench.service end # Create a new handle to a CloudFront account. All handles share the same per process or per thread # HTTP connection to CloudFront. Each handle is for a specific account. The params have the # following options: # * :endpoint_url a fully qualified url to Amazon API endpoint (this overwrites: :server, :port, :service, :protocol). Example: 'https://cloudfront.amazonaws.com' # * :server: CloudFront service host, default: DEFAULT_HOST # * :port: CloudFront service port, default: DEFAULT_PORT # * :protocol: 'http' or 'https', default: DEFAULT_PROTOCOL # * :logger: for log messages, default: RAILS_DEFAULT_LOGGER else STDOUT # # acf = RightAws::AcfInterface.new('1E3GDYEOGFJPIT7XXXXXX','hgTHt68JY07JKUY08ftHYtERkjgtfERn57XXXXXX', # {:logger => Logger.new('/tmp/x.log')}) #=> # # def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={}) init({ :name => 'ACF', :default_host => ENV['ACF_URL'] ? URI.parse(ENV['ACF_URL']).host : DEFAULT_HOST, :default_port => ENV['ACF_URL'] ? URI.parse(ENV['ACF_URL']).port : DEFAULT_PORT, :default_service => ENV['ACF_URL'] ? URI.parse(ENV['ACF_URL']).path : DEFAULT_PATH, :default_protocol => ENV['ACF_URL'] ? URI.parse(ENV['ACF_URL']).scheme : DEFAULT_PROTOCOL, :default_api_version => ENV['ACF_API_VERSION'] || API_VERSION }, aws_access_key_id || ENV['AWS_ACCESS_KEY_ID'], aws_secret_access_key || ENV['AWS_SECRET_ACCESS_KEY'], params) end #----------------------------------------------------------------- # Requests #----------------------------------------------------------------- # Generates request hash for REST API. def generate_request(method, path, params={}, body=nil, headers={}) # :nodoc: # Params params.delete_if{ |key, val| val.right_blank? } unless params.right_blank? path += "?" + params.to_a.collect{ |key,val| "#{AwsUtils::amz_escape(key)}=#{AwsUtils::amz_escape(val.to_s)}" }.join("&") end # Headers headers = AwsUtils::fix_headers(headers) headers['content-type'] ||= 'text/xml' if body headers['date'] = Time.now.httpdate # Auth signature = AwsUtils::sign(@aws_secret_access_key, headers['date']) headers['Authorization'] = "AWS #{@aws_access_key_id}:#{signature}" # Request path = "#{@params[:service]}#{@params[:api_version]}/#{path}" request = "Net::HTTP::#{method.capitalize}".right_constantize.new(path) request.body = body if body # Set request headers headers.each { |key, value| request[key.to_s] = value } # prepare output hash { :request => request, :server => @params[:server], :port => @params[:port], :protocol => @params[:protocol] } end # Sends request to Amazon and parses the response. # Raises AwsError if any banana happened. def request_info(request, parser, &block) # :nodoc: request_info_impl(:acf_connection, @@bench, request, parser, &block) end #----------------------------------------------------------------- # Helpers: #----------------------------------------------------------------- def generate_call_reference # :nodoc: result = Time.now.strftime('%Y%m%d%H%M%S') 10.times{ result << rand(10).to_s } result end def merge_headers(hash) # :nodoc: hash[:location] = @last_response['Location'] if @last_response['Location'] hash[:e_tag] = @last_response['ETag'] if @last_response['ETag'] hash end def distribution_config_to_xml(config, xml_wrapper='DistributionConfig') # :nodoc: cnames = logging = trusted_signers = s3_origin = custom_origin = default_root_object = '' # CNAMES unless config[:cnames].right_blank? Array(config[:cnames]).each { |cname| cnames += " #{cname}\n" } end # Logging unless config[:logging].right_blank? logging = " \n" + " #{config[:logging][:bucket]}\n" + " #{config[:logging][:prefix]}\n" + " \n" end unless config[:required_protocols].right_blank? required_protocols = " \n" + " #{config[:required_protocols]}\n" + " \n" else required_protocols = "" end # Default Root Object unless config[:default_root_object].right_blank? default_root_object = " #{config[:default_root_object]}\n" unless config[:default_root_object].right_blank? end # Trusted Signers unless config[:trusted_signers].right_blank? trusted_signers = " \n" Array(config[:trusted_signers]).each do |trusted_signer| trusted_signers += if trusted_signer.to_s[/self/i] " \n" else " #{trusted_signer}\n" end end trusted_signers += " \n" end # S3Origin unless config[:s3_origin].right_blank? origin_access_identity = '' # Origin Access Identity unless config[:s3_origin][:origin_access_identity].right_blank? origin_access_identity = config[:s3_origin][:origin_access_identity] unless origin_access_identity[%r{^origin-access-identity}] origin_access_identity = "origin-access-identity/cloudfront/#{origin_access_identity}" end origin_access_identity = " #{origin_access_identity}\n" end s3_origin = " \n" + " #{config[:s3_origin][:dns_name]}\n" + "#{origin_access_identity}" + " \n" end # Custom Origin unless config[:custom_origin].right_blank? http_port = https_port = origin_protocol_policy = '' http_port = " #{config[:custom_origin][:http_port]}\n" unless config[:custom_origin][:http_port].right_blank? https_port = " #{config[:custom_origin][:https_port]}" unless config[:custom_origin][:https_port].right_blank? origin_protocol_policy = " #{config[:custom_origin][:origin_protocol_policy]}\n" unless config[:custom_origin][:origin_protocol_policy].right_blank? custom_origin = " \n" + " #{config[:custom_origin][:dns_name]}\n" + "#{http_port}" + "#{https_port}" + "#{origin_protocol_policy}" + " \n" end # XML "\n" + "<#{xml_wrapper} xmlns=\"http://#{@params[:server]}/doc/#{API_VERSION}/\">\n" + " #{config[:caller_reference]}\n" + " #{AwsUtils::xml_escape(config[:comment].to_s)}\n" + " #{config[:enabled]}\n" + s3_origin + custom_origin + default_root_object + cnames + logging + required_protocols + trusted_signers + "" end #----------------------------------------------------------------- # API Calls: #----------------------------------------------------------------- # List all distributions. # Returns an array of distributions or RightAws::AwsError exception. # # acf.list_distributions #=> # [{:status=>"Deployed", # :domain_name=>"dgmde.6os.net", # :comment=>"ONE LINE OF COMMENT", # :last_modified_time=>"2009-06-16T16:10:02.210Z", # :s3_origin=>{:dns_name=>"example.s3.amazonaws.com"}, # :aws_id=>"12Q05OOMFN7SYL", # :enabled=>true}, ... ] # def list_distributions result = [] incrementally_list_distributions do |response| result += response[:distributions] true end result end # Incrementally list distributions. # # Optional params: +:marker+ and +:max_items+. # # # get first distribution # incrementally_list_distributions(:max_items => 1) #=> # {:distributions=> # [{:status=>"Deployed", # :aws_id=>"E2Q0AOOMFNPSYL", # :s3_origin=>{:dns_name=>"example.s3.amazonaws.com"}, # :domain_name=>"d1s5gmdtmafnre.6hops.net", # :comment=>"ONE LINE OF COMMENT", # :last_modified_time=>"2008-10-22T19:31:23.000Z", # :enabled=>true, # :cnames=>[]}], # :is_truncated=>true, # :max_items=>1, # :marker=>"", # :next_marker=>"E2Q0AOOMFNPSYL"} # # # get max 100 distributions (the list will be restricted by a default MaxItems value ==100 ) # incrementally_list_distributions # # # list distributions by 10 # incrementally_list_distributions(:max_items => 10) do |response| # puts response.inspect # a list of 10 distributions # true # return false if the listing should be broken otherwise use true # end # def incrementally_list_distributions(params={}, &block) opts = {} opts['MaxItems'] = params[:max_items] if params[:max_items] opts['Marker'] = params[:marker] if params[:marker] last_response = nil loop do link = generate_request('GET', 'distribution', opts) last_response = request_info(link, AcfDistributionListParser.new(:logger => @logger)) opts['Marker'] = last_response[:next_marker] break unless block && block.call(last_response) && !last_response[:next_marker].right_blank? end last_response end # Create a new distribution. # Returns the just created distribution or RightAws::AwsError exception. # # # S3 Origin # # config = { :comment => "kd: delete me please", # :s3_origin => { :dns_name => "devs-us-east.s3.amazonaws.com", # :origin_access_identity => "origin-access-identity/cloudfront/E3JPJZ80ZBX24G"}, # :enabled => true, # :logging => { :prefix => "kd/log/", # :bucket => "devs-us-west.s3.amazonaws.com"}} # acf.create_distribution(config) #=> # { :status=>"InProgress", # :enabled=>true, # :caller_reference=>"201012071910051044304704", # :logging=>{:prefix=>"kd/log/", :bucket=>"devs-us-west.s3.amazonaws.com"}, # :e_tag=>"ESCTG5WJCFWJK", # :location=> "https://cloudfront.amazonaws.com/2010-11-01/distribution/E3KUBANZ7N1B2", # :comment=>"kd: delete me please", # :domain_name=>"d3stykk6upgs20.cloudfront.net", # :aws_id=>"E3KUBANZ7N1B2", # :s3_origin=> # {:origin_access_identity=>"origin-access-identity/cloudfront/E3JPJZ80ZBX24G", # :dns_name=>"devs-us-east.s3.amazonaws.com"}, # :last_modified_time=>"2010-12-07T16:10:07.087Z", # :in_progress_invalidation_batches=>0} # # # Custom Origin # # custom_config = { :comment => "kd: delete me please", # :custom_origin => { :dns_name => "custom_origin.my-site.com", # :http_port => 80, # :https_port => 443, # :origin_protocol_policy => 'match-viewer' }, # :enabled => true, # :logging => { :prefix => "kd/log/", # :bucket => "my-bucket.s3.amazonaws.com"}} #=> # { :last_modified_time=>"2010-12-08T14:23:43.522Z", # :status=>"InProgress", # :custom_origin=> # {:http_port=>"80", # :https_port=>"443", # :origin_protocol_policy=>"match-viewer", # :dns_name=>"custom_origin.my-site.com"}, # :enabled=>true, # :caller_reference=>"201012081723428499167245", # :in_progress_invalidation_batches=>0, # :e_tag=>"E1ZCJ8N5E52KO6", # :location=> # "https://cloudfront.amazonaws.com/2010-11-01/distribution/EK0AJ4RMNIF2P", # :logging=>{:prefix=>"kd/log/", :bucket=>"my-bucket.s3.amazonaws.com"}, # :domain_name=>"do36k7s2wxklg.cloudfront.net", # :comment=>"kd: delete me please", # :aws_id=>"EK0AJ4RMNIF2P"} # def create_distribution(config) config[:caller_reference] ||= generate_call_reference link = generate_request('POST', 'distribution', {}, distribution_config_to_xml(config)) merge_headers(request_info(link, AcfDistributionListParser.new(:logger => @logger))[:distributions].first) end alias_method :create_distribution_by_config, :create_distribution # Get a distribution's information. # Returns a distribution's information or RightAws::AwsError exception. # # acf.get_distribution('E2REJM3VUN5RSI') #=> # {:last_modified_time=>"2010-05-19T18:54:38.242Z", # :status=>"Deployed", # :domain_name=>"dpzl38cuix402.cloudfront.net", # :caller_reference=>"201005181943052207677116", # :e_tag=>"EJSXFGM5JL8ER", # :s3_origin=> # {:dns_name=>"bucket-for-konstantin-eu.s3.amazonaws.com", # :origin_access_identity=> # "origin-access-identity/cloudfront/E3JPJZ80ZBX24G"}, # :aws_id=>"E5P8HQ3ZAZIXD", # :enabled=>false} # # acf.get_distribution('E2FNSBHNVVF11E') #=> # {:e_tag=>"E1Q2DJEPTQOLJD", # :status=>"InProgress", # :last_modified_time=>"2010-04-17T17:24:25.000Z", # :cnames=>["web1.my-awesome-site.net", "web2.my-awesome-site.net"], # :aws_id=>"E2FNSBHNVVF11E", # :logging=>{:prefix=>"xlog/", :bucket=>"my-bucket.s3.amazonaws.com"}, # :enabled=>true, # :active_trusted_signers=> # [{:aws_account_number=>"120288270000", # :key_pair_ids=>["APKAJTD5OHNDX0000000", "APKAIK74BJWCL0000000"]}, # {:aws_account_number=>"self"}, # {:aws_account_number=>"648772220000"}], # :caller_reference=>"201004171154450740700072", # :domain_name=>"d1f6lpevremt5m.cloudfront.net", # :s3_origin=> # {:dns_name=>"bucket-for-konstantin-eu.s3.amazonaws.com", # :origin_access_identity=> # "origin-access-identity/cloudfront/E3JPJZ80ZBX24G"}, # :trusted_signers=>["self", "648772220000", "120288270000"]} # def get_distribution(aws_id) link = generate_request('GET', "distribution/#{aws_id}") merge_headers(request_info(link, AcfDistributionListParser.new(:logger => @logger))[:distributions].first) end # Get a distribution's configuration. # Returns a distribution's configuration or RightAws::AwsError exception. # # acf.get_distribution_config('E2REJM3VUN5RSI') #=> # {:caller_reference=>"201005181943052207677116", # :e_tag=>"EJSXFGM5JL8ER", # :s3_origin=> # {:dns_name=>"bucket-for-konstantin-eu.s3.amazonaws.com", # :origin_access_identity=> # "origin-access-identity/cloudfront/E3JPJZ80ZBX24G"}, # :enabled=>false} # # acf.get_distribution_config('E2FNSBHNVVF11E') #=> # {:e_tag=>"E1Q2DJEPTQOLJD", # :logging=>{:prefix=>"xlog/", :bucket=>"my-bucket.s3.amazonaws.com"}, # :enabled=>true, # :caller_reference=>"201004171154450740700072", # :trusted_signers=>["self", "648772220000", "120288270000"], # :s3_origin=> # {:dns_name=>"bucket-for-konstantin-eu.s3.amazonaws.com", # :origin_access_identity=> # "origin-access-identity/cloudfront/E3JPJZ80ZBX24G"}} # def get_distribution_config(aws_id) link = generate_request('GET', "distribution/#{aws_id}/config") merge_headers(request_info(link, AcfDistributionListParser.new(:logger => @logger))[:distributions].first) end # Set a distribution's configuration # Returns +true+ on success or RightAws::AwsError exception. # # config = acf.get_distribution_config('E2REJM3VUN5RSI') #=> # {:enabled => true, # :caller_reference => "200809102100536497863003", # :e_tag => "E39OHHU1ON65SI", # :cnames => ["web1.my-awesome-site.net", "web2.my-awesome-site.net"] # :comment => "Woo-Hoo!", # :s3_origin => { :dns_name => "my-bucket.s3.amazonaws.com"}} # # config[:comment] = 'Olah-lah!' # config[:enabled] = false # config[:s3_origin][:origin_access_identity] = "origin-access-identity/cloudfront/E3JPJZ80ZBX24G" # # or just # # config[:s3_origin][:origin_access_identity] = "E3JPJZ80ZBX24G" # config[:trusted_signers] = ['self', '648772220000', '120288270000'] # config[:logging] = { :bucket => 'my-bucket.s3.amazonaws.com', :prefix => 'xlog/' } # # acf.set_distribution_config('E2REJM3VUN5RSI', config) #=> true # def set_distribution_config(aws_id, config) link = generate_request('PUT', "distribution/#{aws_id}/config", {}, distribution_config_to_xml(config), 'If-Match' => config[:e_tag]) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Delete a distribution. The enabled distribution cannot be deleted. # Returns +true+ on success or RightAws::AwsError exception. # # acf.delete_distribution('E2REJM3VUN5RSI', 'E39OHHU1ON65SI') #=> true # def delete_distribution(aws_id, e_tag) link = generate_request('DELETE', "distribution/#{aws_id}", {}, nil, 'If-Match' => e_tag) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end #----------------------------------------------------------------- # PARSERS: #----------------------------------------------------------------- class AcfDistributionListParser < RightAWSParser # :nodoc: def reset @result = { :distributions => [] } end def tagstart(name, attributes) case full_tag_name when %r{/Signer$} @active_signer = {} when %r{(Streaming)?DistributionSummary$}, %r{^(Streaming)?Distribution$}, %r{^(Streaming)?DistributionConfig$} @distribution = { } when %r{/S3Origin$} then @distribution[:s3_origin] = {} when %r{/CustomOrigin$} then @distribution[:custom_origin] = {} end end def tagend(name) case name when 'Marker' then @result[:marker] = @text when 'NextMarker' then @result[:next_marker] = @text when 'MaxItems' then @result[:max_items] = @text.to_i when 'IsTruncated' then @result[:is_truncated] = (@text == 'true') when 'Id' then @distribution[:aws_id] = @text when 'Status' then @distribution[:status] = @text when 'LastModifiedTime' then @distribution[:last_modified_time] = @text when 'DomainName' then @distribution[:domain_name] = @text when 'Comment' then @distribution[:comment] = AwsUtils::xml_unescape(@text) when 'CallerReference' then @distribution[:caller_reference] = @text when 'CNAME' then (@distribution[:cnames] ||= []) << @text when 'Enabled' then @distribution[:enabled] = (@text == 'true') when 'Bucket' then (@distribution[:logging] ||= {})[:bucket] = @text when 'Prefix' then (@distribution[:logging] ||= {})[:prefix] = @text when 'Protocol' then (@distribution[:required_protocols] ||= {})[:protocol] = @text when 'InProgressInvalidationBatches' then @distribution[:in_progress_invalidation_batches] = @text.to_i when 'DefaultRootObject' then @distribution[:default_root_object] = @text else case full_tag_name when %r{/S3Origin/DNSName$} then @distribution[:s3_origin][:dns_name] = @text when %r{/S3Origin/OriginAccessIdentity$} then @distribution[:s3_origin][:origin_access_identity] = @text when %r{/CustomOrigin/DNSName$} then @distribution[:custom_origin][:dns_name] = @text when %r{/CustomOrigin/HTTPPort} then @distribution[:custom_origin][:http_port] = @text when %r{/CustomOrigin/HTTPSPort$} then @distribution[:custom_origin][:https_port] = @text when %r{/CustomOrigin/OriginProtocolPolicy$} then @distribution[:custom_origin][:origin_protocol_policy] = @text when %r{/TrustedSigners/Self$} then (@distribution[:trusted_signers] ||= []) << 'self' when %r{/TrustedSigners/AwsAccountNumber$} then (@distribution[:trusted_signers] ||= []) << @text when %r{/Signer/Self$} then @active_signer[:aws_account_number] = 'self' when %r{/Signer/AwsAccountNumber$} then @active_signer[:aws_account_number] = @text when %r{/Signer/KeyPairId$} then (@active_signer[:key_pair_ids] ||= []) << @text when %r{/Signer$} then (@distribution[:active_trusted_signers] ||= []) << @active_signer when %r{(Streaming)?DistributionSummary$}, %r{^(Streaming)?Distribution$}, %r{^(Streaming)?DistributionConfig$} @result[:distributions] << @distribution end end end end end end ================================================ FILE: lib/acf/right_acf_invalidations.rb ================================================ # # Copyright (c) 2010 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws class AcfInterface # List Invalidations # # acf.list_invalidations('E3LTBMK4EAQS7D') #=> # [{:status=>"InProgress", :aws_id=>"I3AW9PPQS0CBKV"}, # {:status=>"InProgress", :aws_id=>"I1HV23N5KD3XH9"}] # def list_invalidations(distribution_aws_id) result = [] incrementally_list_invalidations(distribution_aws_id) do |response| result += response[:invalidations] true end result end # Incrementally list Invalidations. # Optional params: +:marker+ and +:max_items+. # def incrementally_list_invalidations(distribution_aws_id, params={}, &block) opts = {} opts['MaxItems'] = params[:max_items] if params[:max_items] opts['Marker'] = params[:marker] if params[:marker] last_response = nil loop do link = generate_request('GET', "distribution/#{distribution_aws_id}/invalidation", opts) last_response = request_info(link, AcfInvalidationsListParser.new(:logger => @logger)) opts['Marker'] = last_response[:next_marker] break unless block && block.call(last_response) && !last_response[:next_marker].right_blank? end last_response end #----------------------------------------------------------------- # Origin Access Identity #----------------------------------------------------------------- # Create a new Invalidation batch. # # acf.create_invalidation('E3LTBMK4EAQS7D', :path => ['/boot.jpg', '/kd/boot.public.1.jpg']) #=> # {:status=>"InProgress", # :create_time=>"2010-12-08T14:03:38.449Z", # :location=> "https://cloudfront.amazonaws.com/2010-11-01/distribution/E3LTBMK4EAQS7D/invalidation/I3AW9PPQS0CBKV", # :aws_id=>"I3AW9PPQS0CBKV", # :invalidation_batch=> # {:caller_reference=>"201012081703372555972012", # :path=>["/boot.jpg", "/kd/boot.public.1.jpg"]}} # def create_invalidation(distribution_aws_id, invalidation_batch) invalidation_batch[:caller_reference] ||= generate_call_reference link = generate_request('POST', "/distribution/#{distribution_aws_id}/invalidation", {}, invalidation_batch_to_xml(invalidation_batch)) merge_headers(request_info(link, AcfInvalidationsListParser.new(:logger => @logger))[:invalidations].first) end # Get Invalidation # # acf.get_invalidation('E3LTBMK4EAQS7D', 'I3AW9PPQS0CBKV') #=> # {:create_time=>"2010-12-08T14:03:38.449Z", # :status=>"InProgress", # :aws_id=>"I3AW9PPQS0CBKV", # :invalidation_batch=> # {:caller_reference=>"201012081703372555972012", # :path=>["/boot.jpg", "/kd/boot.public.1.jpg"]}} # def get_invalidation(distribution_aws_id, aws_id) link = generate_request('GET', "distribution/#{distribution_aws_id}/invalidation/#{aws_id}") merge_headers(request_info(link, AcfInvalidationsListParser.new(:logger => @logger))[:invalidations].first) end #----------------------------------------------------------------- # Batch #----------------------------------------------------------------- def invalidation_batch_to_xml(invalidation_batch) # :nodoc: paths = '' Array(invalidation_batch[:path]).each do |path| paths << " #{AwsUtils::xml_escape(path)}\n" end "\n" + "\n" + " #{invalidation_batch[:caller_reference]}\n" + paths + "" end #----------------------------------------------------------------- # PARSERS: #----------------------------------------------------------------- class AcfInvalidationsListParser < RightAWSParser # :nodoc: def reset @result = { :invalidations => [] } end def tagstart(name, attributes) case name when %r{(InvalidationSummary|Invalidation)$} then @item = {} when %r{InvalidationBatch} then @item[:invalidation_batch] = {} end end def tagend(name) case name when 'Marker' then @result[:marker] = @text when 'NextMarker' then @result[:next_marker] = @text when 'MaxItems' then @result[:max_items] = @text.to_i when 'IsTruncated' then @result[:is_truncated] = (@text == 'true') when 'Id' then @item[:aws_id] = @text when 'Status' then @item[:status] = @text when 'CreateTime' then @item[:create_time] = @text when 'Path' then (@item[:invalidation_batch][:path] ||= []) << @text when 'CallerReference' then @item[:invalidation_batch][:caller_reference] = @text when %r{(InvalidationSummary|Invalidation)$} @item[:invalidation_batch][:path].sort! if @item[:invalidation_batch] && !@item[:invalidation_batch][:path].right_blank? @result[:invalidations] << @item end end end end end ================================================ FILE: lib/acf/right_acf_origin_access_identities.rb ================================================ # # Copyright (c) 2010 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws class AcfInterface # List Origin Access Identities. # # acf.list_origin_access_identities #=> # [{:comment=>"kd: TEST", # :s3_canonical_user_id=> # "c7ca36f6c5d384e60aeca02032ac748bae3c458c5322a2e279382935f1f71b16d9ac251f7f71f1ea91c37d3c214645b8", # :aws_id=>"E3TL4XWF5KTGH"}, # {:comment=>"kd: TEST-2", # :s3_canonical_user_id=> # "9af7058b1d197c2c03fdcc3ddad07012a7822f5fc4a8156025409ffac646bdae4dc714820482c92e6988e5703c8d9954", # :aws_id=>"E3HJ7V8C3324VF"}, # {:comment=>"MyTestAccessIdentity", # :s3_canonical_user_id=> # "de4361b33dbaf499d3d77159bfa1571d3451eaec25a2b16553de5e534da8089bb8c31a4898d73d1a658155d0e48872a7", # :aws_id=>"E3JPJZ80ZBX24G"}] # def list_origin_access_identities result = [] incrementally_list_origin_access_identities do |response| result += response[:origin_access_identities] true end result end # Incrementally list Origin Access Identities. # Optional params: +:marker+ and +:max_items+. # # acf.incrementally_list_origin_access_identities(:max_items => 2) #=> # {:origin_access_identities=> # [{:comment=>"kd: TEST", # :s3_canonical_user_id=> # "c7ca36f6c5d384e60aeca02032ac748bae3c458c5322a2e279382935f1f71b16d9ac251f7f71f1ea91c37d3c214645b8", # :aws_id=>"E3TL4XWF5KTGH"}, # {:comment=>"kd: TEST-2", # :s3_canonical_user_id=> # "9af7058b1d197c2c03fdcc3ddad07012a7822f5fc4a8156025409ffac646bdae4dc714820482c92e6988e5703c8d9954", # :aws_id=>"E3HJ7V8C3324VF"}], # :is_truncated=>true, # :max_items=>2, # :marker=>"", # :next_marker=>"E3HJ7V8C3324VF"} # # # get max 100 origin access identities (the list will be restricted by a default MaxItems value ==100 ) # incrementally_list_origin_access_identities # # # list origin access identities by 10 # acf.incrementally_list_origin_access_identities(:max_items => 10) do |response| # puts response.inspect # a list of 10 distributions # true # return false if the listing should be broken otherwise use true # end # def incrementally_list_origin_access_identities(params={}, &block) opts = {} opts['MaxItems'] = params[:max_items] if params[:max_items] opts['Marker'] = params[:marker] if params[:marker] last_response = nil loop do link = generate_request('GET', 'origin-access-identity/cloudfront', opts) last_response = request_info(link, AcfOriginAccesIdentitiesListParser.new(:logger => @logger)) opts['Marker'] = last_response[:next_marker] break unless block && block.call(last_response) && !last_response[:next_marker].right_blank? end last_response end #----------------------------------------------------------------- # Origin Access Identity #----------------------------------------------------------------- # Create a new CloudFront Origin Access Identity. # # acf.create_origin_access_identity('MyTestAccessIdentity') #=> # {:e_tag=>"E2QOKZEXCUWHJX", # :comment=>"MyTestAccessIdentity", # :location=> # "https://cloudfront.amazonaws.com/origin-access-identity/cloudfront/E3JPJZ80ZBX24G", # :caller_reference=>"201004161657467493031273", # :s3_canonical_user_id=> # "de4361b33dbaf499d3d77159bfa1571d3451eaec25a2b16553de5e534da8089bb8c31a4898d73d1a658155d0e48872a7", # :aws_id=>"E3JPJZ80ZBX24G"} # def create_origin_access_identity(comment='', caller_reference=nil) config = { :comment => comment, :caller_reference => caller_reference } create_origin_access_identity_by_config(config) end def create_origin_access_identity_by_config(config) config[:caller_reference] ||= generate_call_reference link = generate_request('POST', 'origin-access-identity/cloudfront', {}, origin_access_identity_config_to_xml(config)) merge_headers(request_info(link, AcfOriginAccesIdentitiesListParser.new(:logger => @logger))[:origin_access_identities].first) end # Get Origin Access Identity # # acf.get_origin_access_identity('E3HJ7V8C3324VF') #=> # {:comment=>"kd: TEST-2", # :caller_reference=>"201004161655035372351604", # :aws_id=>"E3HJ7V8C3324VF", # :s3_canonical_user_id=> # "9af7058b1d197c2c03fdcc3ddad07012a7822f5fc4a8156025409ffac646bdae4dc714820482c92e6988e5703c8d9954", # :e_tag=>"E309Q4IM450498"} # def get_origin_access_identity(aws_id) link = generate_request('GET', "origin-access-identity/cloudfront/#{aws_id}") merge_headers(request_info(link, AcfOriginAccesIdentitiesListParser.new(:logger => @logger))[:origin_access_identities].first) end # Get Origin Access Identity # # acf.get_origin_access_identity('E3HJ7V8C3324VF') #=> # {:comment=>"kd: TEST-2", # :caller_reference=>"201004161655035372351604", # :aws_id=>"E3HJ7V8C3324VF", # :s3_canonical_user_id=> # "9af7058b1d197c2c03fdcc3ddad07012a7822f5fc4a8156025409ffac646bdae4dc714820482c92e6988e5703c8d9954", # :e_tag=>"E309Q4IM450498"} # # acf.delete_origin_access_identity("E3HJ7V8C3324VF","E309Q4IM450498") #=> true # def delete_origin_access_identity(aws_id, e_tag) link = generate_request('DELETE', "origin-access-identity/cloudfront/#{aws_id}", {}, nil, 'If-Match' => e_tag) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end #----------------------------------------------------------------- # Config #----------------------------------------------------------------- def origin_access_identity_config_to_xml(config) # :nodoc: "\n" + "\n" + " #{config[:caller_reference]}\n" + " #{AwsUtils::xml_escape(config[:comment].to_s)}\n" + "" end # Get Origin Access Identity config # # acf.get_origin_access_identity_config("E3JPJZ80ZBX24G") #=> # {:comment=>"MyTestAccessIdentity", # :caller_reference=>"201004161657467493031273", # :e_tag=>"E2QOKZEXCUWHJX"} # def get_origin_access_identity_config(aws_id) link = generate_request('GET', "origin-access-identity/cloudfront/#{aws_id}/config") merge_headers(request_info(link, AcfOriginAccesIdentitiesListParser.new(:logger => @logger))[:origin_access_identities].first) end # Set Origin Access Identity config # # # acf.set_origin_access_identity_config("E2QOKZEXCUWHJX", # :comment => "MyBestOriginAccessConfig", # :caller_reference => '01234567890', # :e_tag=>"E2QOKZEXCUWHJX") #=> true # # P.S. This guy is not tested yet: http://developer.amazonwebservices.com/connect/thread.jspa?threadID=45256 def set_origin_access_identity_config(aws_id, config) link = generate_request('PUT', "origin-access-identity/cloudfront/#{aws_id}/config", {}, origin_access_identity_config_to_xml(config), 'If-Match' => config[:e_tag]) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end #----------------------------------------------------------------- # PARSERS: #----------------------------------------------------------------- class AcfOriginAccesIdentitiesListParser < RightAWSParser # :nodoc: def reset @result = { :origin_access_identities => [] } end def tagstart(name, attributes) case full_tag_name when %r{CloudFrontOriginAccessIdentitySummary$}, %r{^CloudFrontOriginAccessIdentity$}, %r{^CloudFrontOriginAccessIdentityConfig$} @item = {} end end def tagend(name) case name when 'Marker' then @result[:marker] = @text when 'NextMarker' then @result[:next_marker] = @text when 'MaxItems' then @result[:max_items] = @text.to_i when 'IsTruncated' then @result[:is_truncated] = (@text == 'true') when 'Id' then @item[:aws_id] = @text when 'S3CanonicalUserId' then @item[:s3_canonical_user_id] = @text when 'CallerReference' then @item[:caller_reference] = @text when 'Comment' then @item[:comment] = AwsUtils::xml_unescape(@text) end case full_tag_name when %r{CloudFrontOriginAccessIdentitySummary$}, %r{^CloudFrontOriginAccessIdentity$}, %r{^CloudFrontOriginAccessIdentityConfig$} @result[:origin_access_identities] << @item end end end end end ================================================ FILE: lib/acf/right_acf_streaming_interface.rb ================================================ # # Copyright (c) 2008 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws class AcfInterface def streaming_distribution_config_to_xml(config) # :nodoc: distribution_config_to_xml(config, 'StreamingDistributionConfig') end #----------------------------------------------------------------- # API Calls: #----------------------------------------------------------------- # List all streaming distributions. # Returns an array of distributions or RightAws::AwsError exception. # # acf.list_streaming_distributions #=> # [{:status=>"Deployed", # :aws_id=>"E3CWE2Z9USOS6B", # :enabled=>true, # :domain_name=>"s2jz1ourvss1fj.cloudfront.net", # :s3_origin=> {:dns_name=>"bucket-for-konstantin-00.s3.amazonaws.com"}, # :last_modified_time=>"2010-04-19T08:53:32.574Z", # :comment=>"Woo-Hoo!", # :cnames=>["stream.web.my-awesome-site.net"]}, # ... # {:status=>"Deployed", # :aws_id=>"E3NPQZY4LKAYQ8", # :enabled=>true, # :domain_name=>"sw9nrsq9pudk3.cloudfront.net", # :s3_origin=> {:dns_name=>"bucket-for-konstantin-00.s3.amazonaws.com"}, # :last_modified_time=>"2010-04-19T08:59:09.600Z", # :comment=>"Woo-Hoo!", # :cnames=>["stream-6.web.my-awesome-site.net"]}] # def list_streaming_distributions result = [] incrementally_list_streaming_distributions do |response| result += response[:distributions] true end result end # Incrementally list streaming distributions. # # Optional params: +:marker+ and +:max_items+. # # # get first streaming distribution # incrementally_list_distributions(:max_items => 1) #=> # {:marker=>"", # :next_marker=>"E3CWE2Z9USOS6B", # :distributions=> # [{:status=>"Deployed", # :cnames=>["stream.web.my-awesome-site.net"], # :aws_id=>"E3CWE2Z9USOS6B", # :enabled=>true, # :last_modified_time=>"2010-04-19T08:53:32.574Z", # :domain_name=>"s2jz1ourvss1fj.cloudfront.net", # :s3_origin=> {:dns_name=>"bucket-for-konstantin-00.s3.amazonaws.com"}, # :comment=>"Woo-Hoo!"}], # :max_items=>1, # :is_truncated=>true} # # # get max 100 streaming distributions (the list will be restricted by a default MaxItems value ==100 ) # incrementally_list_streaming_distributions # # # list streaming distributions by 10 # incrementally_list_streaming_distributions(:max_items => 10) do |response| # puts response.inspect # a list of 10 distributions # true # return false if the listing should be broken otherwise use true # end # def incrementally_list_streaming_distributions(params={}, &block) opts = {} opts['MaxItems'] = params[:max_items] if params[:max_items] opts['Marker'] = params[:marker] if params[:marker] last_response = nil loop do link = generate_request('GET', 'streaming-distribution', opts) last_response = request_info(link, AcfDistributionListParser.new(:logger => @logger)) opts['Marker'] = last_response[:next_marker] break unless block && block.call(last_response) && !last_response[:next_marker].right_blank? end last_response end # Create a new streaming distribution. # Returns the just created distribution or RightAws::AwsError exception. # # acf.create_streaming_distribution('bucket-for-konstantin-00.s3.amazonaws.com', 'Woo-Hoo!', true, # ['stream-1.web.my-awesome-site.net']) #=> # {:status=>"InProgress", # :caller_reference=>"201004191254412191173215", # :cnames=>["stream-1.web.my-awesome-site.net"], # :aws_id=>"E1M5LERJLU636F", # :e_tag=>"E2588L5QL4BLXH", # :enabled=>true, # :domain_name=>"s1di8imd85wgld.cloudfront.net", # :s3_origin=> {:dns_name=>"bucket-for-konstantin-00.s3.amazonaws.com"}, # :last_modified_time=>Mon Apr 19 08:54:42 UTC 2010, # :location=> # "https://cloudfront.amazonaws.com/streaming-distribution/E1M5LERJLU636F", # :comment=>"Woo-Hoo!"} # def create_streaming_distribution(config) config[:caller_reference] ||= generate_call_reference link = generate_request('POST', 'streaming-distribution', {}, streaming_distribution_config_to_xml(config)) merge_headers(request_info(link, AcfDistributionListParser.new(:logger => @logger))[:distributions].first) end alias_method :create_streaming_distribution_by_config, :create_streaming_distribution # Get a streaming distribution's information. # Returns a distribution's information or RightAws::AwsError exception. # # acf.get_streaming_distribution('E3CWE2Z9USOS6B') #=> # {:status=>"Deployed", # :e_tag=>"EXTZ2SXAQT39K", # :cnames=>["stream.web.my-awesome-site.net"], # :aws_id=>"E3CWE2Z9USOS6B", # :enabled=>true, # :domain_name=>"s2jz1ourvss1fj.cloudfront.net", # :s3_origin=> {:dns_name=>"bucket-for-konstantin-00.s3.amazonaws.com"}, # :last_modified_time=>"2010-04-19T08:53:32.574Z", # :comment=>"Woo-Hoo!", # :caller_reference=>"201004191253311625537161"} # # acf.get_streaming_distribution('E1M5LERJLU636F') #=> # {:trusted_signers=>["self", "648772220000", "120288270000"], # :status=>"InProgress", # :e_tag=>"E2K6XD13RCJQ6E", # :cnames=>["stream-1.web.my-awesome-site.net"], # :active_trusted_signers=> # [{:key_pair_ids=>["APKAIK74BJWCLXZUMEJA"], # :aws_account_number=>"120288270000"}, # {:aws_account_number=>"self"}, # {:aws_account_number=>"648772220000"}], # :aws_id=>"E1M5LERJLU636F", # :enabled=>false, # :domain_name=>"s1di8imd85wgld.cloudfront.net", # :s3_origin=> { # :dns_name=>"bucket-for-konstantin-00.s3.amazonaws.com", # :origin_access_identity=>"origin-access-identity/cloudfront/E3JPJZ80ZBX24G"}, # :last_modified_time=>"2010-04-19T09:14:07.160Z", # :comment=>"Olah-lah!", # :caller_reference=>"201004191254412191173215"} # def get_streaming_distribution(aws_id) link = generate_request('GET', "streaming-distribution/#{aws_id}") merge_headers(request_info(link, AcfDistributionListParser.new(:logger => @logger))[:distributions].first) end # Get a streaming distribution's configuration. # Returns a distribution's configuration or RightAws::AwsError exception. # # acf.get_streaming_distribution_config('E1M5LERJLU636F') #=> # {:trusted_signers=>["self", "648772220000", "120288270000"], # :e_tag=>"E2K6XD13RCJQ6E", # :cnames=>["stream-1.web.my-awesome-site.net"], # :enabled=>false, # :s3_origin=> { # :dns_name=>"bucket-for-konstantin-00.s3.amazonaws.com", # :origin_access_identity=>"origin-access-identity/cloudfront/E3JPJZ80ZBX24G",}, # :comment=>"Olah-lah!", # :caller_reference=>"201004191254412191173215"} # def get_streaming_distribution_config(aws_id) link = generate_request('GET', "streaming-distribution/#{aws_id}/config") merge_headers(request_info(link, AcfDistributionListParser.new(:logger => @logger))[:distributions].first) end # Set a streaming distribution's configuration # Returns +true+ on success or RightAws::AwsError exception. # # acf.get_streaming_distribution_config('E1M5LERJLU636F') #=> # {:e_tag=>"E2588L5QL4BLXH", # :cnames=>["stream-1.web.my-awesome-site.net"], # :enabled=>true, # :s3_origin=> {:dns_name=>"bucket-for-konstantin-00.s3.amazonaws.com"}, # :comment=>"Woo-Hoo!", # :caller_reference=>"201004191254412191173215"} # # config[:comment] = 'Olah-lah!' # config[:enabled] = false # config[:s3_origin][:origin_access_identity] = "origin-access-identity/cloudfront/E3JPJZ80ZBX24G" # config[:trusted_signers] = ['self', '648772220000', '120288270000'] # # acf.set_distribution_config('E2REJM3VUN5RSI', config) #=> true # def set_streaming_distribution_config(aws_id, config) link = generate_request('PUT', "streaming-distribution/#{aws_id}/config", {}, streaming_distribution_config_to_xml(config), 'If-Match' => config[:e_tag]) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Delete a streaming distribution. The enabled distribution cannot be deleted. # Returns +true+ on success or RightAws::AwsError exception. # # acf.delete_streaming_distribution('E1M5LERJLU636F', 'E2588L5QL4BLXH') #=> true # def delete_streaming_distribution(aws_id, e_tag) link = generate_request('DELETE', "streaming-distribution/#{aws_id}", {}, nil, 'If-Match' => e_tag) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end end end ================================================ FILE: lib/acw/right_acw_interface.rb ================================================ # # Copyright (c) 2007-2009 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws # = RightAWS::AcwInterface -- RightScale Amazon Cloud Watch interface # The RightAws::AcwInterface class provides a complete interface to Amazon Cloud Watch service. # # For explanations of the semantics of each call, please refer to Amazon's documentation at # http://docs.amazonwebservices.com/AmazonCloudWatch/latest/DeveloperGuide/ # class AcwInterface < RightAwsBase include RightAwsBaseInterface # Amazon ACW API version being used API_VERSION = "2009-05-15" DEFAULT_HOST = "monitoring.amazonaws.com" DEFAULT_PATH = '/' DEFAULT_PROTOCOL = 'https' DEFAULT_PORT = 443 @@bench = AwsBenchmarkingBlock.new def self.bench_xml @@bench.xml end def self.bench_service @@bench.service end # Create a new handle to an ACW account. All handles share the same per process or per thread # HTTP connection to Amazon ACW. Each handle is for a specific account. The params have the # following options: # * :endpoint_url a fully qualified url to Amazon API endpoint (this overwrites: :server, :port, :service, :protocol). Example: 'https://monitoring.amazonaws.com/' # * :server: ACW service host, default: DEFAULT_HOST # * :port: ACW service port, default: DEFAULT_PORT # * :protocol: 'http' or 'https', default: DEFAULT_PROTOCOL # * :logger: for log messages, default: RAILS_DEFAULT_LOGGER else STDOUT # * :signature_version: The signature version : '0','1' or '2'(default) # * :cache: true/false(default): list_metrics # def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={}) init({ :name => 'ACW', :default_host => ENV['ACW_URL'] ? URI.parse(ENV['ACW_URL']).host : DEFAULT_HOST, :default_port => ENV['ACW_URL'] ? URI.parse(ENV['ACW_URL']).port : DEFAULT_PORT, :default_service => ENV['ACW_URL'] ? URI.parse(ENV['ACW_URL']).path : DEFAULT_PATH, :default_protocol => ENV['ACW_URL'] ? URI.parse(ENV['ACW_URL']).scheme : DEFAULT_PROTOCOL, :default_api_version => ENV['ACW_API_VERSION'] || API_VERSION }, aws_access_key_id || ENV['AWS_ACCESS_KEY_ID'] , aws_secret_access_key|| ENV['AWS_SECRET_ACCESS_KEY'], params) end def generate_request(action, params={}) #:nodoc: generate_request_impl(:get, action, params ) end # Sends request to Amazon and parses the response # Raises AwsError if any banana happened def request_info(request, parser) #:nodoc: request_info_impl(:ams_connection, @@bench, request, parser) end #----------------------------------------------------------------- # MetricStatistics #----------------------------------------------------------------- # Get time-series data for one or more statistics of given a Metric # Returns a hash of stat data. # # Options are: # # :period - x*60 seconds interval (where x > 0) # :statistics - Average, Minimum. Maximum, Sum, Samples # :start_time - The timestamp of the first datapoint to return, inclusive. # :end_time - The timestamp to use for determining the last datapoint to return. This is the last datapoint to fetch, exclusive. # :namespace - The namespace corresponding to the service of interest. For example, AWS/EC2 represents Amazon EC2. # :unit - Seconds, Percent, Bytes, Bits, Count, Bytes/Second, Bits/Second, Count/Second, and None # :custom_unit - The user-defined CustomUnit applied to a Measure. Please see the key term Unit. # # :dimentions # Dimensions for EC2 Metrics: # * ImageId - shows the requested metric for all instances running this EC2 Amazon Machine Image(AMI) # * AvailabilityZone - shows the requested metric for all instances running in that EC2 Availability Zone # * CapacityGroupName - shows the requested metric for all instances in the specified capacity group - this dimension is # only available for EC2 metrics when the instances are in an Amazon Automatic Scaling Service # Capacity Group # * InstanceId - shows the requested metric for only the identified instance # * InstanceType - shows the requested metric for all instances running with that instance type # * Service (required) - the name of the service that reported the monitoring data - for EC2 metrics, use "EC2" # * Namespace (required) - in private beta, the available metrics are all reported by AWS services, so set this to "AWS" # Dimensions for Load Balancing Metrics: # * AccessPointName - shows the requested metric for the specified AccessPoint name # * AvailabilityZone - shows the requested metric for all instances running in that EC2 Availability Zone # * Service (required) - the name of the service that reported the monitoring data - for LoadBalancing metrics, use "LBS" # * Namespace (required) - in private beta, the available metrics are all reported by AWS services, so set this to "AWS" # # :measure_name # EC2 Metrics: # * CPUUtilization the percentage of allocated EC2 Compute Units that are currently in use on the instance. Units are Percent. # * NetworkIn - the number of bytes received on all network interfaces by the instance. Units are Bytes. # * NetworkOut - the number of bytes sent out on all network interfaces by the instance. Units are Bytes. # * DiskReadOps - completed read operations from all disks available to the instance in one minute. Units are Count/Second. # * DiskWriteOps - completed writes operations to all disks available to the instance in one minute. Units are Count/Second. # * DiskReadBytes - bytes read from all disks available to the instance in one minute. Units are Bytes/Second. # * DiskWriteBytes - bytes written to all disks available to the instance in one minute. Units are Bytes/Second. # Load Balancing Metrics: # * Latency - time taken between a request and the corresponding response as seen by the load balancer. Units are in # seconds, and the available statistics include minimum, maximum, average and count. # * RequestCount - number of requests processed by the AccessPoint over the valid period. Units are count per second, and # the available statistics include minimum, maximum and sum. A valid period can be anything equal to or # multiple of sixty (60) seconds. # * HealthyHostCount - number of healthy EndPoints for the valid Period. A valid period can be anything equal to or a multiple # of sixty (60) seconds. Units are the count of EndPoints. The meaningful statistic for HealthyHostCount # is the average for an AccessPoint within an Availability Zone. Both Load Balancing dimensions, # AccessPointName and AvailabilityZone, should be specified when retreiving HealthyHostCount. # * UnHealthyHostCount - number of unhealthy EndPoints for the valid Period. A valid period can be anything equal to or a multiple # of sixty (60) seconds. Units are the count of EndPoints. The meaningful statistic for UnHealthyHostCount # is the average for an AccessPoint within Availability Amazon Monitoring Service Developer Guide Load # Balancing Metrics Version PRIVATE BETA 2009-01-22 19 Zone. Both Load Balancing dimensions, AccessPointName # and AvailabilityZone, should be specified when retreiving UnHealthyHostCount. # def get_metric_statistics(options={}) # Period (60 sec by default) period = (options[:period] && options[:period].to_i) || 60 # Statistics ('Average' by default) statistics = Array(options[:statistics]).flatten statistics = statistics.right_blank? ? ['Average'] : statistics.map{|statistic| statistic.to_s.capitalize } # Times (5.min.ago up to now by default) start_time = options[:start_time] || (Time.now.utc - 5*60) start_time = start_time.utc.strftime("%Y-%m-%dT%H:%M:%S+00:00") if start_time.is_a?(Time) end_time = options[:end_time] || Time.now.utc end_time = end_time.utc.strftime("%Y-%m-%dT%H:%M:%S+00:00") if end_time.is_a?(Time) # Measure name measure_name = options[:measure_name] || 'CPUUtilization' # Dimentions (a hash, empty by default) dimentions = options[:dimentions] || {} # request_hash = { 'Period' => period, 'StartTime' => start_time, 'EndTime' => end_time, 'MeasureName' => measure_name } request_hash['Unit'] = options[:unit] if options[:unit] request_hash['CustomUnit'] = options[:custom_unit] if options[:custom_unit] request_hash['Namespace'] = options[:namespace] if options[:namespace] request_hash.merge!(amazonize_list('Statistics.member', statistics)) # dimentions dim = [] dimentions.each do |key, values| Array(values).each { |value| dim << [key, value] } end request_hash.merge!(amazonize_list(['Dimensions.member.?.Name', 'Dimensions.member.?.Value'], dim)) # link = generate_request("GetMetricStatistics", request_hash) request_info(link, GetMetricStatisticsParser.new(:logger => @logger)) end # This call returns a list of the valid metrics for which there is recorded data available to a you. # # acw.list_metrics #=> # [ { :namespace => "AWS/ELB", # :measure_name => "HealthyHostCount", # :dimentions => { "LoadBalancerName"=>"test-kd1" } }, # { :namespace => "AWS/ELB", # :measure_name => "UnHealthyHostCount", # :dimentions => { "LoadBalancerName"=>"test-kd1" } } ] def list_metrics link = generate_request("ListMetrics") request_cache_or_info :list_metrics, link, ListMetricsParser, @@bench, true end #----------------------------------------------------------------- # PARSERS: MetricStatistics #----------------------------------------------------------------- class GetMetricStatisticsParser < RightAWSParser #:nodoc: def tagstart(name, attributes) @item = {} if name == 'member' end def tagend(name) case name when 'Timestamp' then @item[:timestamp] = @text when 'Unit' then @item[:unit] = @text when 'CustomUnit' then @item[:custom_unit] = @text when 'Samples' then @item[:samples] = @text.to_f when 'Average' then @item[:average] = @text.to_f when 'Minimum' then @item[:minimum] = @text.to_f when 'Maximum' then @item[:maximum] = @text.to_f when 'Sum' then @item[:sum] = @text.to_f when 'member' then @result[:datapoints] << @item when 'Label' then @result[:label] = @text end end def reset @result = { :datapoints => [] } end end class ListMetricsParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case name when 'member' case @xmlpath when @p then @item = { :dimentions => {} } end end end def tagend(name) case name when 'MeasureName' then @item[:measure_name] = @text when 'Namespace' then @item[:namespace] = @text when 'Name' then @dname = @text when 'Value' then @dvalue = @text when 'member' case @xmlpath when "#@p/member/Dimensions" then @item[:dimentions][@dname] = @dvalue when @p then @result << @item end end end def reset @p = 'ListMetricsResponse/ListMetricsResult/Metrics' @result = [] end end end end ================================================ FILE: lib/as/right_as_interface.rb ================================================ # # Copyright (c) 2007-2009 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws # = RightAWS::AsInterface -- RightScale Amazon Auto Scaling interface # The RightAws::AsInterface class provides a complete interface to Amazon Auto Scaling service. # # For explanations of the semantics of each call, please refer to Amazon's documentation at # http://docs.amazonwebservices.com/AutoScaling/latest/DeveloperGuide/ # # Create an interface handle: # # as = RightAws::AsInterface.new(aws_access_key_id, aws_security_access_key) # # Create a launch configuration: # # as.create_launch_configuration('CentOS.5.1-c', 'ami-08f41161', 'm1.small', # :key_name => 'kd-moo-test', # :security_groups => ['default'], # :user_data => "Woohoo: CentOS.5.1-c" ) # # Create an AutoScaling group: # # as.create_auto_scaling_group('CentOS.5.1-c-array', 'CentOS.5.1-c', 'us-east-1c', # :min_size => 2, # :max_size => 5) # # Create a new trigger: # # as.create_or_update_scaling_trigger('kd.tr.1', 'CentOS.5.1-c-array', # :measure_name => 'CPUUtilization', # :statistic => :average, # :dimensions => { # 'AutoScalingGroupName' => 'CentOS.5.1-c-array', # 'Namespace' => 'AWS', # 'Service' => 'EC2' }, # :period => 60, # :lower_threshold => 5, # :lower_breach_scale_increment => -1, # :upper_threshold => 60, # :upper_breach_scale_increment => 1, # :breach_duration => 300 ) # # Describe scaling activity: # # as.incrementally_describe_scaling_activities('CentOS.5.1-c-array') #=> List of activities # # Describe the Auto Scaling group status: # # as.describe_auto_scaling_groups('CentOS.5.1-c-array') #=> Current group status # class AsInterface < RightAwsBase include RightAwsBaseInterface # Amazon AS API version being used API_VERSION = '2009-05-15' DEFAULT_HOST = 'autoscaling.amazonaws.com' DEFAULT_PATH = '/' DEFAULT_PROTOCOL = 'https' DEFAULT_PORT = 443 @@bench = AwsBenchmarkingBlock.new def self.bench_xml @@bench.xml end def self.bench_service @@bench.service end # Create a new handle to an CSLS account. All handles share the same per process or per thread # HTTP connection to Amazon CSLS. Each handle is for a specific account. The params have the # following options: # * :endpoint_url a fully qualified url to Amazon API endpoint (this overwrites: :server, :port, :service, :protocol). Example: 'https://autoscaling.amazonaws.com/' # * :server: AS service host, default: DEFAULT_HOST # * :port: AS service port, default: DEFAULT_PORT # * :protocol: 'http' or 'https', default: DEFAULT_PROTOCOL # * :logger: for log messages, default: RAILS_DEFAULT_LOGGER else STDOUT # * :signature_version: The signature version : '0','1' or '2'(default) # * :cache: true/false(default): describe_auto_scaling_groups # def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={}) init({ :name => 'AS', :default_host => ENV['AS_URL'] ? URI.parse(ENV['AS_URL']).host : DEFAULT_HOST, :default_port => ENV['AS_URL'] ? URI.parse(ENV['AS_URL']).port : DEFAULT_PORT, :default_service => ENV['AS_URL'] ? URI.parse(ENV['AS_URL']).path : DEFAULT_PATH, :default_protocol => ENV['AS_URL'] ? URI.parse(ENV['AS_URL']).scheme : DEFAULT_PROTOCOL, :default_api_version => ENV['AS_API_VERSION'] || API_VERSION }, aws_access_key_id || ENV['AWS_ACCESS_KEY_ID'] , aws_secret_access_key|| ENV['AWS_SECRET_ACCESS_KEY'], params) end def generate_request(action, params={}) #:nodoc: generate_request_impl(:get, action, params ) end # Sends request to Amazon and parses the response # Raises AwsError if any banana happened def request_info(request, parser) #:nodoc: request_info_impl(:aass_connection, @@bench, request, parser) end #----------------------------------------------------------------- # Auto Scaling Groups #----------------------------------------------------------------- # Describe auto scaling groups. # Returns a full description of the AutoScalingGroups from the given list. # This includes all EC2 instances that are members of the group. If a list # of names is not provided, then the full details of all AutoScalingGroups # is returned. This style conforms to the EC2 DescribeInstances API behavior. # def describe_auto_scaling_groups(*auto_scaling_group_names) auto_scaling_group_names = auto_scaling_group_names.flatten.compact request_hash = amazonize_list('AutoScalingGroupNames.member', auto_scaling_group_names) link = generate_request("DescribeAutoScalingGroups", request_hash) request_cache_or_info(:describe_auto_scaling_groups, link, DescribeAutoScalingGroupsParser, @@bench, auto_scaling_group_names.right_blank?) end # Creates a new auto scaling group with the specified name. # Returns +true+ or raises an exception. # # Options: +:min_size+, +:max_size+, +:cooldown+, +:load_balancer_names+ # # as.create_auto_scaling_group('CentOS.5.1-c-array', 'CentOS.5.1-c', 'us-east-1c', # :min_size => 2, # :max_size => 5) #=> true # # Amazon's notice: Constraints: Restricted to one Availability Zone def create_auto_scaling_group(auto_scaling_group_name, launch_configuration_name, availability_zones, options={}) options[:min_size] ||= 1 options[:max_size] ||= 20 options[:cooldown] ||= 0 request_hash = amazonize_list('AvailabilityZones.member', availability_zones) request_hash.merge!( amazonize_list('LoadBalancerNames.member', options[:load_balancer_names]) ) request_hash.merge!( 'AutoScalingGroupName' => auto_scaling_group_name, 'LaunchConfigurationName' => launch_configuration_name, 'MinSize' => options[:min_size], 'MaxSize' => options[:max_size], 'Cooldown' => options[:cooldown] ) link = generate_request("CreateAutoScalingGroup", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Deletes all configuration for this auto scaling group and also deletes the group. # Returns +true+ or raises an exception. # def delete_auto_scaling_group(auto_scaling_group_name) link = generate_request('DeleteAutoScalingGroup', 'AutoScalingGroupName' => auto_scaling_group_name) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Adjusts the desired size of the Capacity Group by using scaling actions, as necessary. When # adjusting the size of the group downward, it is not possible to define which EC2 instances will be # terminated. This also applies to any auto-scaling decisions that might result in the termination of # instances. # # Returns +true+ or raises an exception. # # as.set_desired_capacity('CentOS.5.1-c',3) #=> 3 # def set_desired_capacity(auto_scaling_group_name, desired_capacity) link = generate_request('SetDesiredCapacity', 'AutoScalingGroupName' => auto_scaling_group_name, 'DesiredCapacity' => desired_capacity ) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Updates the configuration for the given AutoScalingGroup. If MaxSize is lower than the current size, # then there will be an implicit call to SetDesiredCapacity to set the group to the new MaxSize. The # same is true for MinSize there will also be an implicit call to SetDesiredCapacity. All optional # parameters are left unchanged if not passed in the request. # # The new settings are registered upon the completion of this call. Any launch configuration settings # will take effect on any triggers after this call returns. However, triggers that are currently in # progress can not be affected. See key term Trigger. # # Returns +true+ or raises an exception. # # Options: +:launch_configuration_name+, +:min_size+, +:max_size+, +:cooldown+, +:availability_zones+. # (Amazon's notice: +:availability_zones+ is reserved for future use.) # # as.update_auto_scaling_group('CentOS.5.1-c', :min_size => 1, :max_size => 4) #=> true # def update_auto_scaling_group(auto_scaling_group_name, options={}) request_hash = amazonize_list('AvailabilityZones.member', options[:availability_zones]) request_hash['AutoScalingGroupName'] = auto_scaling_group_name request_hash['LaunchConfigurationName'] = options[:launch_configuration_name] if options[:launch_configuration_name] request_hash['MinSize'] = options[:min_size] if options[:min_size] request_hash['MaxSize'] = options[:max_size] if options[:max_size] request_hash['Cooldown'] = options[:cooldown] if options[:cooldown] link = generate_request("UpdateAutoScalingGroup", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end #----------------------------------------------------------------- # Scaling Activities #----------------------------------------------------------------- # Describe all Scaling Activities. # # describe_scaling_activities('CentOS.5.1-c-array') #=> # [{:cause=> # "At 2009-05-28 10:11:35Z trigger kd.tr.1 breached high threshold value for # CPUUtilization, 10.0, adjusting the desired capacity from 1 to 2. At 2009-05-28 10:11:35Z # a breaching trigger explicitly set group desired capacity changing the desired capacity # from 1 to 2. At 2009-05-28 10:11:40Z an instance was started in response to a difference # between desired and actual capacity, increasing the capacity from 1 to 2.", # :activity_id=>"067c9abb-f8a7-4cf8-8f3c-dc6f280457c4", # :progress=>0, # :description=>"Launching a new EC2 instance", # :status_code=>"InProgress", # :start_time=>Thu May 28 10:11:40 UTC 2009}, # {:end_time=>Thu May 28 09:35:23 UTC 2009, # :cause=> # "At 2009-05-28 09:31:21Z a user request created an AutoScalingGroup changing the desired # capacity from 0 to 1. At 2009-05-28 09:32:35Z an instance was started in response to a # difference between desired and actual capacity, increasing the capacity from 0 to 1.", # :activity_id=>"90d506ba-1b75-4d29-8739-0a75b1ba8030", # :progress=>100, # :description=>"Launching a new EC2 instance", # :status_code=>"Successful", # :start_time=>Thu May 28 09:32:35 UTC 2009}]} # def describe_scaling_activities(auto_scaling_group_name, *activity_ids) result = [] incrementally_describe_scaling_activities(auto_scaling_group_name, *activity_ids) do |response| result += response[:scaling_activities] true end result end # Incrementally describe Scaling Activities. # Returns the scaling activities specified for the given group. If the input list is empty, all the # activities from the past six weeks will be returned. Activities will be sorted by completion time. # Activities that have no completion time will be considered as using the most recent possible time. # # Optional params: +:max_records+, +:next_token+. # # # get max 100 first activities # as.incrementally_describe_scaling_activities('CentOS.5.1-c-array') #=> # {:scaling_activities=> # [{:cause=> # "At 2009-05-28 10:11:35Z trigger kd.tr.1 breached high threshold value for # CPUUtilization, 10.0, adjusting the desired capacity from 1 to 2. At 2009-05-28 10:11:35Z # a breaching trigger explicitly set group desired capacity changing the desired capacity # from 1 to 2. At 2009-05-28 10:11:40Z an instance was started in response to a difference # between desired and actual capacity, increasing the capacity from 1 to 2.", # :activity_id=>"067c9abb-f8a7-4cf8-8f3c-dc6f280457c4", # :progress=>0, # :description=>"Launching a new EC2 instance", # :status_code=>"InProgress", # :start_time=>Thu May 28 10:11:40 UTC 2009}, # {:end_time=>Thu May 28 09:35:23 UTC 2009, # :cause=> # "At 2009-05-28 09:31:21Z a user request created an AutoScalingGroup changing the desired # capacity from 0 to 1. At 2009-05-28 09:32:35Z an instance was started in response to a # difference between desired and actual capacity, increasing the capacity from 0 to 1.", # :activity_id=>"90d506ba-1b75-4d29-8739-0a75b1ba8030", # :progress=>100, # :description=>"Launching a new EC2 instance", # :status_code=>"Successful", # :start_time=>Thu May 28 09:32:35 UTC 2009}]} # # # list by 5 records # incrementally_describe_scaling_activities('CentOS.5.1-c-array', :max_records => 5) do |response| # puts response.inspect # true # end # def incrementally_describe_scaling_activities(auto_scaling_group_name, *activity_ids, &block) activity_ids = activity_ids.flatten.compact params = activity_ids.last.kind_of?(Hash) ? activity_ids.pop : {} request_hash = amazonize_list('ActivityIds.member', activity_ids) request_hash['AutoScalingGroupName'] = auto_scaling_group_name request_hash['MaxRecords'] = params[:max_records] if params[:max_records] request_hash['NextToken'] = params[:next_token] if params[:next_token] last_response = nil loop do link = generate_request("DescribeScalingActivities", request_hash) last_response = request_info( link, DescribeScalingActivitiesParser.new(:logger => @logger)) request_hash['NextToken'] = last_response[:next_token] break unless block && block.call(last_response) && !last_response[:next_token].right_blank? end last_response end #----------------------------------------------------------------- # Instance and Instance Workflow Operations #----------------------------------------------------------------- # This call will terminate the specified Instance. Optionally, the desired group size can be adjusted. # If set to true, the default, the AutoScalingGroup size will decrease by one. If the AutoScalingGroup # is associated with a LoadBalancer, the system will deregister the instance before terminating it. # This call simply registers a termination request. The termination of the instance can not happen # immediately. # # Returns the activity to terminate the instance. # def terminate_instance_in_auto_scaling_group(instance_id, should_decrement_desired_capacity=true) request_hash = { 'InstanceId' => instance_id } request_hash['ShouldDecrementDesiredCapacity'] = should_decrement_desired_capacity link = generate_request('TerminateInstanceInAutoScalingGroup', request_hash ) request_info(link, DescribeScalingActivitiesParser.new(:logger => @logger))[:scaling_activities].first end #----------------------------------------------------------------- # Launch Configuration Operations #----------------------------------------------------------------- # Creates a new Launch Configuration. Please note that the launch configuration name used must # be unique, within the scope of your Amazon Web Services AWS account, and the maximum limit of # launch configurations must not yet have been met, or else the call will fail. # # Once created, the new launch configuration is available for immediate use. # # Options: +:security_groups+, +:block_device_mappings+, +:key_name+, # +:user_data+, +:kernel_id+, +:ramdisk_id+ # # as.create_launch_configuration('kd: CentOS.5.1-c.1', 'ami-08f41161', 'c1.medium', # :key_name => 'tim', # :security_groups => ['default'], # :user_data => "Woohoo: CentOS.5.1-c", # :block_device_mappings => [ { :device_name => '/dev/sdk', # :ebs_snapshot_id => 'snap-145cbc7d', # :ebs_delete_on_termination => true, # :ebs_volume_size => 3, # :virtual_name => 'ephemeral2' # } ] # ) #=> true # def create_launch_configuration(launch_configuration_name, image_id, instance_type, options={}) request_hash = { 'LaunchConfigurationName' => launch_configuration_name, 'ImageId' => image_id, 'InstanceType' => instance_type } request_hash.merge!(amazonize_list('SecurityGroups.member', options[:security_groups])) unless options[:security_groups].right_blank? request_hash.merge!(amazonize_block_device_mappings(options[:block_device_mappings], 'BlockDeviceMappings.member')) request_hash['KeyName'] = options[:key_name] if options[:key_name] request_hash['UserData'] = Base64.encode64(options[:user_data]).delete("\n") unless options[:user_data].right_blank? if options[:user_data] request_hash['KernelId'] = options[:kernel_id] if options[:kernel_id] request_hash['RamdiskId'] = options[:ramdisk_id] if options[:ramdisk_id] link = generate_request("CreateLaunchConfiguration", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Describe all Launch Configurations. # Returns an array of configurations. # # as.describe_launch_configurations #=> # [{:security_groups=>["default"], # :ramdisk_id=>"", # :user_data=>"V29vaG9vOiBDZW50T1MuNS4xLWM=", # :instance_type=>"c1.medium", # :block_device_mappings=> # [{:virtual_name=>"ephemeral2", :device_name=>"/dev/sdk"}], # :launch_configuration_name=>"kd: CentOS.5.1-c.1", # :created_time=>"2010-03-29T10:00:32.742Z", # :image_id=>"ami-08f41161", # :key_name=>"tim", # :kernel_id=>""}, ...] # def describe_launch_configurations(*launch_configuration_names) result = [] incrementally_describe_launch_configurations(*launch_configuration_names) do |response| result += response[:launch_configurations] true end result end # Incrementally describe Launch Configurations. # Returns a full description of the launch configurations given the specified names. If no names # are specified, then the full details of all launch configurations are returned. # # Optional params: +:max_records+, +:next_token+. # # # get max 100 first configurations # as.incrementally_describe_launch_configurations #=> # {:launch_configurations=> # [{:created_time=>Thu May 28 09:31:20 UTC 2009, # :kernel_id=>"", # :launch_configuration_name=>"CentOS.5.1-c", # :ramdisk_id=>"", # :security_groups=>["default"], # :key_name=>"kd-moo-test", # :user_data=>"Woohoo: CentOS.5.1-c-array", # :image_id=>"ami-08f41161", # :block_device_mappings=>[], # :instance_type=>"m1.small"}, ... ]} # # # list by 5 records # incrementally_describe_launch_configurations(:max_records => 5) do |response| # puts response.inspect # true # end # def incrementally_describe_launch_configurations(*launch_configuration_names, &block) launch_configuration_names = launch_configuration_names.flatten.compact params = launch_configuration_names.last.kind_of?(Hash) ? launch_configuration_names.pop : {} request_hash = amazonize_list('LaunchConfigurationNames.member', launch_configuration_names) request_hash['MaxRecords'] = params[:max_records] if params[:max_records] request_hash['NextToken'] = params[:next_token] if params[:next_token] last_response = nil loop do link = generate_request("DescribeLaunchConfigurations", request_hash) last_response = request_info( link, DescribeLaunchConfigurationsParser.new(:logger => @logger) ) request_hash['NextToken'] = last_response[:next_token] break unless block && block.call(last_response) && !last_response[:next_token].right_blank? end last_response end # Delete launch configuration. # Returns +true+ or an exception. # # as.delete_launch_configuration('CentOS.5.1') #=> true # def delete_launch_configuration(launch_configuration_name) link = generate_request('DeleteLaunchConfiguration', 'LaunchConfigurationName' => launch_configuration_name) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end #----------------------------------------------------------------- # Trigger Operations #----------------------------------------------------------------- # Create or update specified trigger. # This call sets the parameters that governs when and how to scale an AutoScalingGroup. # If the Trigger, within the scope of the caller's AWS account, specified already exists, # it will be updated. If a trigger with a different name already exists, this call will fail. # # Returns +true+ or an exception. # # Options: +:measure_name+, +:statistic+, +:period+, +:lower_threshold+, +:lower_breach_scale_increment+, # +:upper_threshold+, +:upper_breach_scale_increment+, +:dimensions+, +:breach_duration+, +:unit+, +:custom_unit+ # # as.create_or_update_scaling_trigger('kd.tr.1', 'CentOS.5.1-c-array', # :measure_name => 'CPUUtilization', # :statistic => :average, # :dimensions => { # 'AutoScalingGroupName' => 'CentOS.5.1-c-array', # 'Namespace' => 'AWS', # 'Service' => 'EC2' }, # :period => 60, # :lower_threshold => 5, # :lower_breach_scale_increment => -1, # :upper_threshold => 60, # :upper_breach_scale_increment => 1, # :breach_duration => 300 ) #=> true # def create_or_update_scaling_trigger(trigger_name, auto_scaling_group_name, options={}) request_hash = { 'TriggerName' => trigger_name, 'AutoScalingGroupName' => auto_scaling_group_name, 'MeasureName' => options[:measure_name], 'Statistic' => options[:statistic].to_s.capitalize, 'Period' => options[:period], 'LowerThreshold' => options[:lower_threshold], 'LowerBreachScaleIncrement' => options[:lower_breach_scale_increment], 'UpperThreshold' => options[:upper_threshold], 'UpperBreachScaleIncrement' => options[:upper_breach_scale_increment], 'BreachDuration' => options[:breach_duration] } request_hash['Unit'] = options[:unit] if options[:unit] request_hash['CustomUnit'] = options[:custom_unit] if options[:custom_unit] dimensions = [] (options[:dimensions] || {}).each do |key, values| Array(values).each { |value| dimensions << [key, value] } end request_hash.merge!(amazonize_list(['Dimensions.member.?.Name', 'Dimensions.member.?.Value'], dimensions)) link = generate_request("CreateOrUpdateScalingTrigger", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Describe triggers. # Returns a full description of the trigger in the specified Auto Scaling Group. # # as.describe_triggers('CentOS.5.1-c-array') #=> # [{:status=>"HighBreaching", # :breach_duration=>300, # :measure_name=>"CPUUtilization", # :trigger_name=>"kd.tr.1", # :period=>60, # :lower_threshold=>0.0, # :lower_breach_scale_increment=>-1, # :dimensions=> # {"Namespace"=>"AWS", # "AutoScalingGroupName"=>"CentOS.5.1-c-array", # "Service"=>"EC2"}, # :statistic=>"Average", # :upper_threshold=>10.0, # :created_time=>Thu May 28 09:48:46 UTC 2009, # :auto_scaling_group_name=>"CentOS.5.1-c-array", # :upper_breach_scale_increment=>1}] # def describe_triggers(auto_scaling_group_name) link = generate_request("DescribeTriggers", 'AutoScalingGroupName' => auto_scaling_group_name) request_info(link, DescribeTriggersParser.new(:logger => @logger)) end # Delete specified trigger. # Returns +true+ or an exception. # # as.delete_trigger('kd.tr.1', 'CentOS.5.1-c-array') #=> true # def delete_trigger(trigger_name, auto_scaling_group_name) link = generate_request('DeleteTrigger', 'TriggerName' => trigger_name, 'AutoScalingGroupName' => auto_scaling_group_name) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end #----------------------------------------------------------------- # PARSERS: Scaling Activity #----------------------------------------------------------------- class DescribeScalingActivitiesParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case name when 'member', 'Activity' then @item = {} end end def tagend(name) case name when 'ActivityId' then @item[:activity_id] = @text when 'StartTime' then @item[:start_time] = @text when 'EndTime' then @item[:end_time] = @text when 'Progress' then @item[:progress] = @text.to_i when 'StatusCode' then @item[:status_code] = @text when 'Cause' then @item[:cause] = @text when 'Description' then @item[:description] = @text when 'member', 'Activity' then @result[:scaling_activities] << @item when 'NextToken' then @result[:next_token] = @text end end def reset @result = { :scaling_activities => []} end end #----------------------------------------------------------------- # PARSERS: Auto Scaling Groups #----------------------------------------------------------------- class DescribeAutoScalingGroupsParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case name when 'member' case @xmlpath when @p then @item = { :instances => [ ], :availability_zones => [], :load_balancer_names => [] } when "#@p/member/Instances" then @instance = { } end end end def tagend(name) case name when 'CreatedTime' then @item[:created_time] = @text when 'MinSize' then @item[:min_size] = @text.to_i when 'MaxSize' then @item[:max_size] = @text.to_i when 'DesiredCapacity' then @item[:desired_capacity] = @text.to_i when 'Cooldown' then @item[:cooldown] = @text.to_i when 'LaunchConfigurationName' then @item[:launch_configuration_name] = @text when 'AutoScalingGroupName' then @item[:auto_scaling_group_name] = @text when 'InstanceId' then @instance[:instance_id] = @text when 'LifecycleState' then @instance[:lifecycle_state] = @text when 'AvailabilityZone' then @instance[:availability_zone] = @text when 'member' case @xmlpath when @p then @item[:availability_zones].sort! @result << @item when "#@p/member/AvailabilityZones" then @item[:availability_zones] << @text when "#@p/member/LoadBalancerNames" then @item[:load_balancer_names] << @text when "#@p/member/Instances" then @item[:instances] << @instance end end end def reset @p = 'DescribeAutoScalingGroupsResponse/DescribeAutoScalingGroupsResult/AutoScalingGroups' @result = [] end end #----------------------------------------------------------------- # PARSERS: Launch Configurations #----------------------------------------------------------------- class DescribeLaunchConfigurationsParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{/LaunchConfigurations/member$} @item = { :block_device_mappings => [], :security_groups => [] } when %r{/BlockDeviceMappings/member$} @block_device_mapping = {} end end def tagend(name) case name when 'CreatedTime' then @item[:created_time] = @text when 'InstanceType' then @item[:instance_type] = @text when 'KeyName' then @item[:key_name] = @text when 'ImageId' then @item[:image_id] = @text when 'KernelId' then @item[:kernel_id] = @text when 'RamdiskId' then @item[:ramdisk_id] = @text when 'LaunchConfigurationName' then @item[:launch_configuration_name] = @text when 'UserData' then @item[:user_data] = @text when 'NextToken' then @result[:next_token] = @text else case full_tag_name when %r{/BlockDeviceMappings/member} # no trailing $ case name when 'DeviceName' then @block_device_mapping[:device_name] = @text when 'VirtualName' then @block_device_mapping[:virtual_name] = @text when 'member' then @item[:block_device_mappings] << @block_device_mapping end when %r{member/SecurityGroups/member$} @item[:security_groups] << @text when %r{/LaunchConfigurations/member$} @item[:security_groups].sort! @result[:launch_configurations] << @item end end end def reset @result = { :launch_configurations => []} end end #----------------------------------------------------------------- # PARSERS: Triggers #----------------------------------------------------------------- class DescribeTriggersParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case name when 'member' case @xmlpath when 'DescribeTriggersResponse/DescribeTriggersResult/Triggers' @item = { :dimensions => {} } when 'DescribeTriggersResponse/DescribeTriggersResult/Triggers/member/Dimensions' @dimension = {} end end end def tagend(name) case name when 'AutoScalingGroupName' then @item[:auto_scaling_group_name] = @text when 'MeasureName' then @item[:measure_name] = @text when 'CreatedTime' then @item[:created_time] = @text when 'BreachDuration' then @item[:breach_duration] = @text.to_i when 'UpperBreachScaleIncrement' then @item[:upper_breach_scale_increment] = @text.to_i when 'UpperThreshold' then @item[:upper_threshold] = @text.to_f when 'LowerThreshold' then @item[:lower_threshold] = @text.to_f when 'LowerBreachScaleIncrement' then @item[:lower_breach_scale_increment] = @text.to_i when 'Period' then @item[:period] = @text.to_i when 'Status' then @item[:status] = @text when 'TriggerName' then @item[:trigger_name] = @text when 'Statistic' then @item[:statistic] = @text when 'Unit' then @item[:unit] = @text when 'Name' then @dimension[:name] = @text when 'Value' then @dimension[:value] = @text when 'member' case @xmlpath when "#@p/member/Dimensions" then @item[:dimensions][@dimension[:name]] = @dimension[:value] when @p then @result << @item end end end def reset @p = 'DescribeTriggersResponse/DescribeTriggersResult/Triggers' @result = [] end end end end ================================================ FILE: lib/awsbase/benchmark_fix.rb ================================================ # # Copyright (c) 2007-2008 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # # # A hack because there's a bug in add! in Benchmark::Tms module Benchmark #:nodoc: class Tms #:nodoc: def add!(&blk) t = Benchmark::measure(&blk) @utime = utime + t.utime @stime = stime + t.stime @cutime = cutime + t.cutime @cstime = cstime + t.cstime @real = real + t.real self end end end ================================================ FILE: lib/awsbase/right_awsbase.rb ================================================ # # Copyright (c) 2007-2008 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # # Test module RightAws require 'digest/md5' class AwsUtils #:nodoc: @@digest1 = OpenSSL::Digest.new("sha1") @@digest256 = nil if OpenSSL::OPENSSL_VERSION_NUMBER > 0x00908000 @@digest256 = OpenSSL::Digest.new("sha256") rescue nil # Some installation may not support sha256 end def self.utc_iso8601(time) if time.is_a?(Fixnum) then time = Time::at(time) elsif time.is_a?(String) then time = Time::parse(time) end time.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z") end def self.sign(aws_secret_access_key, auth_string) Base64.encode64(OpenSSL::HMAC.digest(@@digest1, aws_secret_access_key, auth_string)).strip end # Calculates 'Content-MD5' header value for some content def self.content_md5(content) Base64.encode64(Digest::MD5::new.update(content).digest).strip end # Escape a string accordingly Amazon rulles # http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html def self.amz_escape(param) param = param.flatten.join('') if param.is_a?(Array) # ruby 1.9.x Array#to_s fix param.to_s.gsub(/([^a-zA-Z0-9._~-]+)/n) do '%' + $1.unpack('H2' * $1.size).join('%').upcase end end def self.xml_escape(text) # :nodoc: REXML::Text::normalize(text) end def self.xml_unescape(text) # :nodoc: REXML::Text::unnormalize(text) end # Set a timestamp and a signature version def self.fix_service_params(service_hash, signature) service_hash["Timestamp"] ||= utc_iso8601(Time.now) unless service_hash["Expires"] service_hash["SignatureVersion"] = signature service_hash end def self.fix_headers(headers) result = {} headers.each do |header, value| next if !header.is_a?(String) || value.nil? header = header.downcase result[header] = value if result[header].right_blank? end result end # Signature Version 0 # A deprecated guy (should work till septemper 2009) def self.sign_request_v0(aws_secret_access_key, service_hash) fix_service_params(service_hash, '0') string_to_sign = "#{service_hash['Action']}#{service_hash['Timestamp'] || service_hash['Expires']}" service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign) service_hash.to_a.collect{|key,val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&") end # Signature Version 1 # Another deprecated guy (should work till septemper 2009) def self.sign_request_v1(aws_secret_access_key, service_hash) fix_service_params(service_hash, '1') string_to_sign = service_hash.sort{|a,b| (a[0].to_s.downcase)<=>(b[0].to_s.downcase)}.to_s service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign) service_hash.to_a.collect{|key,val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&") end # Signature Version 2 # EC2, SQS and SDB requests must be signed by this guy. # See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html # http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1928 def self.sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, uri) fix_service_params(service_hash, '2') # select a signing method (make an old openssl working with sha1) # make 'HmacSHA256' to be a default one service_hash['SignatureMethod'] = 'HmacSHA256' unless ['HmacSHA256', 'HmacSHA1'].include?(service_hash['SignatureMethod']) service_hash['SignatureMethod'] = 'HmacSHA1' unless @@digest256 # select a digest digest = (service_hash['SignatureMethod'] == 'HmacSHA256' ? @@digest256 : @@digest1) # form string to sign canonical_string = service_hash.keys.sort.map do |key| "#{amz_escape(key)}=#{amz_escape(service_hash[key])}" end.join('&') string_to_sign = "#{http_verb.to_s.upcase}\n#{host.downcase}\n#{uri}\n#{canonical_string}" # sign the string signature = amz_escape(Base64.encode64(OpenSSL::HMAC.digest(digest, aws_secret_access_key, string_to_sign)).strip) "#{canonical_string}&Signature=#{signature}" end # From Amazon's SQS Dev Guide, a brief description of how to escape: # "URL encode the computed signature and other query parameters as specified in # RFC1738, section 2.2. In addition, because the + character is interpreted as a blank space # by Sun Java classes that perform URL decoding, make sure to encode the + character # although it is not required by RFC1738." # Avoid using CGI::escape to escape URIs. # CGI::escape will escape characters in the protocol, host, and port # sections of the URI. Only target chars in the query # string should be escaped. def self.URLencode(raw) e = URI.escape(raw) e.gsub(/\+/, "%2b") end def self.allow_only(allowed_keys, params) bogus_args = [] params.keys.each {|p| bogus_args.push(p) unless allowed_keys.include?(p) } raise AwsError.new("The following arguments were given but are not legal for the function call #{caller_method}: #{bogus_args.inspect}") if bogus_args.length > 0 end def self.mandatory_arguments(required_args, params) rargs = required_args.dup params.keys.each {|p| rargs.delete(p)} raise AwsError.new("The following mandatory arguments were not provided to #{caller_method}: #{rargs.inspect}") if rargs.length > 0 end def self.caller_method caller[1]=~/`(.*?)'/ $1 end def self.split_items_and_params(array) items = Array(array).flatten.compact params = items.last.kind_of?(Hash) ? items.pop : {} [items, params] end # Generates a token in format of: # 1. "1dd8d4e4-db6b-11df-b31d-0025b37efad0 (if UUID gem is loaded) # 2. "1287483761-855215-zSv2z-bWGj2-31M5t-ags9m" (if UUID gem is not loaded) TOKEN_GENERATOR_CHARSET = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a def self.generate_unique_token time = Time.now token = "%d-%06d" % [time.to_i, time.usec] 4.times do token << "-" 5.times { token << TOKEN_GENERATOR_CHARSET[rand(TOKEN_GENERATOR_CHARSET.size)] } end token end end class AwsBenchmarkingBlock #:nodoc: attr_accessor :xml, :service def initialize # Benchmark::Tms instance for service (Ec2, S3, or SQS) access benchmarking. @service = Benchmark::Tms.new() # Benchmark::Tms instance for XML parsing benchmarking. @xml = Benchmark::Tms.new() end end class AwsNoChange < RuntimeError end class RightAwsBase # Amazon HTTP Error handling # Text, if found in an error message returned by AWS, indicates that this may be a transient # error. Transient errors are automatically retried with exponential back-off. AMAZON_PROBLEMS = [ 'internal service error', 'is currently unavailable', 'no response from', 'Please try again', 'InternalError', 'Internal Server Error', 'ServiceUnavailable', #from SQS docs 'Unavailable', 'This application is not currently available', 'InsufficientInstanceCapacity' ] @@amazon_problems = AMAZON_PROBLEMS # Returns a list of Amazon service responses which are known to be transient problems. # We have to re-request if we get any of them, because the problem will probably disappear. # By default this method returns the same value as the AMAZON_PROBLEMS const. def self.amazon_problems @@amazon_problems end # Sets the list of Amazon side problems. Use in conjunction with the # getter to append problems. def self.amazon_problems=(problems_list) @@amazon_problems = problems_list end # Raise an exception if a timeout occures while an API call is in progress. # This helps to avoid a duplicate resources creation when Amazon hangs for some time and # RightHttpConnection is forced to use retries to get a response from it. # # If an API call action is in the list then no attempts to retry are performed. # RAISE_ON_TIMEOUT_ON_ACTIONS = %w{ AllocateAddress CreateSnapshot CreateVolume PurchaseReservedInstancesOffering RequestSpotInstances RunInstances } @@raise_on_timeout_on_actions = RAISE_ON_TIMEOUT_ON_ACTIONS.dup def self.raise_on_timeout_on_actions @@raise_on_timeout_on_actions end def self.raise_on_timeout_on_actions=(actions_list) @@raise_on_timeout_on_actions = actions_list end end module RightAwsBaseInterface DEFAULT_SIGNATURE_VERSION = '2' @@caching = false def self.caching @@caching end def self.caching=(caching) @@caching = caching end # Current aws_access_key_id attr_reader :aws_access_key_id # Current aws_secret_access_key attr_reader :aws_secret_access_key # Last HTTP request object attr_reader :last_request # Last HTTP response object attr_reader :last_response # Last AWS errors list (used by AWSErrorHandler) attr_accessor :last_errors # Last AWS request id (used by AWSErrorHandler) attr_accessor :last_request_id # Logger object attr_accessor :logger # Initial params hash attr_accessor :params # RightHttpConnection instance attr_reader :connection # Cache attr_reader :cache # Signature version (all services except s3) attr_reader :signature_version def init(service_info, aws_access_key_id, aws_secret_access_key, params={}) #:nodoc: @params = params # If one defines EC2_URL he may forget to use a single slash as an "empty service" path. # Amazon does not like this therefore add this bad boy if he is missing... service_info[:default_service] = '/' if service_info[:default_service].right_blank? raise AwsError.new("AWS access keys are required to operate on #{service_info[:name]}") \ if aws_access_key_id.right_blank? || aws_secret_access_key.right_blank? @aws_access_key_id = aws_access_key_id @aws_secret_access_key = aws_secret_access_key # if the endpoint was explicitly defined - then use it if @params[:endpoint_url] uri = URI.parse(@params[:endpoint_url]) @params[:server] = uri.host @params[:port] = uri.port @params[:service] = uri.path @params[:protocol] = uri.scheme # make sure the 'service' path is not empty @params[:service] = service_info[:default_service] if @params[:service].right_blank? @params[:region] = nil default_port = uri.default_port else @params[:server] ||= service_info[:default_host] @params[:server] = "#{@params[:region]}.#{@params[:server]}" if @params[:region] @params[:port] ||= service_info[:default_port] @params[:service] ||= service_info[:default_service] @params[:protocol] ||= service_info[:default_protocol] default_port = @params[:protocol] == 'https' ? 443 : 80 end # build a host name to sign @params[:host_to_sign] = @params[:server].dup @params[:host_to_sign] << ":#{@params[:port]}" unless default_port == @params[:port].to_i # a set of options to be passed to RightHttpConnection object @params[:connection_options] = {} unless @params[:connection_options].is_a?(Hash) @with_connection_options = {} @params[:connections] ||= :shared # || :dedicated @params[:max_connections] ||= 10 @params[:connection_lifetime] ||= 20*60 @params[:api_version] ||= service_info[:default_api_version] @logger = @params[:logger] @logger = ::Rails.logger if !@logger && defined?(::Rails) && ::Rails.respond_to?(:logger) @logger = RAILS_DEFAULT_LOGGER if !@logger && defined?(RAILS_DEFAULT_LOGGER) @logger = Logger.new(STDOUT) if !@logger @logger.info "New #{self.class.name} using #{@params[:connections]} connections mode" @error_handler = nil @cache = {} @signature_version = (params[:signature_version] || DEFAULT_SIGNATURE_VERSION).to_s end def signed_service_params(aws_secret_access_key, service_hash, http_verb=nil, host=nil, service=nil ) case signature_version.to_s when '0' then AwsUtils::sign_request_v0(aws_secret_access_key, service_hash) when '1' then AwsUtils::sign_request_v1(aws_secret_access_key, service_hash) when '2' then AwsUtils::sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, service) else raise AwsError.new("Unknown signature version (#{signature_version.to_s}) requested") end end # Returns +true+ if the describe_xxx responses are being cached def caching? @params.key?(:cache) ? @params[:cache] : @@caching end # Check if the aws function response hits the cache or not. # If the cache hits: # - raises an +AwsNoChange+ exception if +do_raise+ == +:raise+. # - returnes parsed response from the cache if it exists or +true+ otherwise. # If the cache miss or the caching is off then returns +false+. def cache_hits?(function, response, do_raise=:raise) result = false if caching? function = function.to_sym # get rid of requestId (this bad boy was added for API 2008-08-08+ and it is uniq for every response) # feb 04, 2009 (load balancer uses 'RequestId' hence use 'i' modifier to hit it also) response = response.sub(%r{.+?}i, '') # this should work for both ruby 1.8.x and 1.9.x response_md5 = Digest::MD5::new.update(response).to_s # check for changes unless @cache[function] && @cache[function][:response_md5] == response_md5 # well, the response is new, reset cache data update_cache(function, {:response_md5 => response_md5, :timestamp => Time.now, :hits => 0, :parsed => nil}) else # aha, cache hits, update the data and throw an exception if needed @cache[function][:hits] += 1 if do_raise == :raise raise(AwsNoChange, "Cache hit: #{function} response has not changed since "+ "#{@cache[function][:timestamp].strftime('%Y-%m-%d %H:%M:%S')}, "+ "hits: #{@cache[function][:hits]}.") else result = @cache[function][:parsed] || true end end end result end def update_cache(function, hash) (@cache[function.to_sym] ||= {}).merge!(hash) if caching? end def on_exception(options={:raise=>true, :log=>true}) # :nodoc: raise if $!.is_a?(AwsNoChange) AwsError::on_aws_exception(self, options) end #---------------------------- # HTTP Connections handling #---------------------------- def get_server_url(request) # :nodoc: "#{request[:protocol]}://#{request[:server]}:#{request[:port]}" end def get_connections_storage(aws_service) # :nodoc: case @params[:connections].to_s when 'dedicated' then @connections_storage ||= {} else Thread.current[aws_service] ||= {} end end def destroy_connection(request, reason) # :nodoc: connections = get_connections_storage(request[:aws_service]) server_url = get_server_url(request) if connections[server_url] connections[server_url][:connection].finish(reason) connections.delete(server_url) end end # Expire the connection if it has expired. def get_connection(request) # :nodoc: server_url = get_server_url(request) connection_storage = get_connections_storage(request[:aws_service]) life_time_scratch = Time.now-@params[:connection_lifetime] # Delete out-of-dated connections connections_in_list = 0 connection_storage.to_a.sort{|conn1, conn2| conn2[1][:last_used_at] <=> conn1[1][:last_used_at]}.each do |serv_url, conn_opts| if @params[:max_connections] <= connections_in_list conn_opts[:connection].finish('out-of-limit') connection_storage.delete(server_url) elsif conn_opts[:last_used_at] < life_time_scratch conn_opts[:connection].finish('out-of-date') connection_storage.delete(server_url) else connections_in_list += 1 end end connection = (connection_storage[server_url] ||= {}) connection[:last_used_at] = Time.now connection[:connection] ||= Rightscale::HttpConnection.new(:exception => RightAws::AwsError, :logger => @logger) end #---------------------------- # HTTP Requests handling #---------------------------- # ACF, AMS, EC2, LBS and SDB uses this guy # SQS and S3 use their own methods def generate_request_impl(verb, action, options={}, custom_options={}) #:nodoc: # Form a valid http verb: 'GET' or 'POST' (all the other are not supported now) http_verb = verb.to_s.upcase # remove empty keys from request options options.delete_if { |key, value| value.nil? } # prepare service data service_hash = {"Action" => action, "AWSAccessKeyId" => @aws_access_key_id, "Version" => custom_options[:api_version] || @params[:api_version] } service_hash.merge!(options) service_hash["SecurityToken"] = @params[:token] if @params[:token] # Sign request options service_params = signed_service_params(@aws_secret_access_key, service_hash, http_verb, @params[:host_to_sign], @params[:service]) # Use POST if the length of the query string is too large # see http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/MakingRESTRequests.html if http_verb != 'POST' && service_params.size > 2000 http_verb = 'POST' if signature_version == '2' service_params = signed_service_params(@aws_secret_access_key, service_hash, http_verb, @params[:host_to_sign], @params[:service]) end end # create a request case http_verb when 'GET' request = Net::HTTP::Get.new("#{@params[:service]}?#{service_params}") when 'POST' request = Net::HTTP::Post.new(@params[:service]) request.body = service_params request['Content-Type'] = 'application/x-www-form-urlencoded; charset=utf-8' else raise "Unsupported HTTP verb #{verb.inspect}!" end # prepare output hash request_hash = { :request => request, :server => @params[:server], :port => @params[:port], :protocol => @params[:protocol] } request_hash.merge!(@params[:connection_options]) request_hash.merge!(@with_connection_options) # If an action is marked as "non-retryable" and there was no :raise_on_timeout option set # explicitly then do set that option if Array(RightAwsBase::raise_on_timeout_on_actions).include?(action) && !request_hash.has_key?(:raise_on_timeout) request_hash.merge!(:raise_on_timeout => true) end request_hash end # All services uses this guy. def request_info_impl(aws_service, benchblock, request, parser, &block) #:nodoc: request[:aws_service] = aws_service @connection = get_connection(request) @last_request = request[:request] @last_response = nil response = nil blockexception = nil if(block != nil) # TRB 9/17/07 Careful - because we are passing in blocks, we get a situation where # an exception may get thrown in the block body (which is high-level # code either here or in the application) but gets caught in the # low-level code of HttpConnection. The solution is not to let any # exception escape the block that we pass to HttpConnection::request. # Exceptions can originate from code directly in the block, or from user # code called in the other block which is passed to response.read_body. benchblock.service.add! do begin responsehdr = @connection.request(request) do |response| ######### begin @last_response = response if response.is_a?(Net::HTTPSuccess) @error_handler = nil response.read_body(&block) else @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler check_result = @error_handler.check(request) if check_result @error_handler = nil return check_result end raise AwsError.new(@last_errors, @last_response.code, @last_request_id) end rescue Exception => e blockexception = e end end rescue Exception => e # Kill a connection if we run into a low level connection error destroy_connection(request, "error: #{e.message}") raise e end ######### #OK, now we are out of the block passed to the lower level if(blockexception) raise blockexception end benchblock.xml.add! do parser.parse(responsehdr) end return parser.result end else benchblock.service.add! do begin response = @connection.request(request) rescue Exception => e # Kill a connection if we run into a low level connection error destroy_connection(request, "error: #{e.message}") raise e end end # check response for errors... @last_response = response if response.is_a?(Net::HTTPSuccess) @error_handler = nil benchblock.xml.add! { parser.parse(response) } return parser.result else @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler check_result = @error_handler.check(request) if check_result @error_handler = nil return check_result end raise AwsError.new(@last_errors, @last_response.code, @last_request_id) end end rescue @error_handler = nil raise end def request_cache_or_info(method, link, parser_class, benchblock, use_cache=true, &block) #:nodoc: # We do not want to break the logic of parsing hence will use a dummy parser to process all the standard # steps (errors checking etc). The dummy parser does nothig - just returns back the params it received. # If the caching is enabled and hit then throw AwsNoChange. # P.S. caching works for the whole images list only! (when the list param is blank) # check cache response, params = request_info(link, RightDummyParser.new) cache_hits?(method.to_sym, response.body) if use_cache parser = parser_class.new(:logger => @logger) benchblock.xml.add!{ parser.parse(response, params) } result = block ? block.call(parser) : parser.result # update parsed data update_cache(method.to_sym, :parsed => result) if use_cache result end # Returns Amazons request ID for the latest request def last_request_id @last_response && @last_response.body.to_s[%r{(.+?)}i] && $1 end # Incrementally lists something. def incrementally_list_items(action, parser_class, params={}, &block) # :nodoc: params = params.dup params['MaxItems'] = params.delete(:max_items) if params[:max_items] params['Marker'] = params.delete(:marker) if params[:marker] last_response = nil loop do last_response = request_info( generate_request(action, params), parser_class.new(:logger => @logger)) params['Marker'] = last_response[:marker] break unless block && block.call(last_response) && !last_response[:marker].right_blank? end last_response end # Format array of items into Amazons handy hash ('?' is a place holder): # Options: # :default => "something" : Set a value to "something" when it is nil # :default => :skip_nils : Skip nil values # # amazonize_list('Item', ['a', 'b', 'c']) => # { 'Item.1' => 'a', 'Item.2' => 'b', 'Item.3' => 'c' } # # amazonize_list('Item.?.instance', ['a', 'c']) #=> # { 'Item.1.instance' => 'a', 'Item.2.instance' => 'c' } # # amazonize_list(['Item.?.Name', 'Item.?.Value'], {'A' => 'a', 'B' => 'b'}) #=> # { 'Item.1.Name' => 'A', 'Item.1.Value' => 'a', # 'Item.2.Name' => 'B', 'Item.2.Value' => 'b' } # # amazonize_list(['Item.?.Name', 'Item.?.Value'], [['A','a'], ['B','b']]) #=> # { 'Item.1.Name' => 'A', 'Item.1.Value' => 'a', # 'Item.2.Name' => 'B', 'Item.2.Value' => 'b' } # # amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], {'A' => ['aa','ab'], 'B' => ['ba','bb']}) #=> # amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], [['A',['aa','ab']], ['B',['ba','bb']]]) #=> # {"Filter.1.Key"=>"A", # "Filter.1.Value.1"=>"aa", # "Filter.1.Value.2"=>"ab", # "Filter.2.Key"=>"B", # "Filter.2.Value.1"=>"ba", # "Filter.2.Value.2"=>"bb"} def amazonize_list(masks, list, options={}) #:nodoc: groups = {} list_idx = options[:index] || 1 Array(list).each do |list_item| Array(masks).each_with_index do |mask, mask_idx| key = mask[/\?/] ? mask.dup : mask.dup + '.?' key.sub!('?', list_idx.to_s) value = Array(list_item)[mask_idx] if value.is_a?(Array) groups.merge!(amazonize_list(key, value, options)) else if value.nil? next if options[:default] == :skip_nils value = options[:default] end # Hack to avoid having unhandled '?' in keys : do replace them all with '1': # bad: ec2.amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], { a: => :b }) => {"Filter.1.Key"=>:a, "Filter.1.Value.?"=>1} # good: ec2.amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], { a: => :b }) => {"Filter.1.Key"=>:a, "Filter.1.Value.1"=>1} key.gsub!('?', '1') groups[key] = value end end list_idx += 1 end groups end BLOCK_DEVICE_KEY_MAPPING = { # :nodoc: :device_name => 'DeviceName', :virtual_name => 'VirtualName', :no_device => 'NoDevice', :ebs_snapshot_id => 'Ebs.SnapshotId', :ebs_volume_size => 'Ebs.VolumeSize', :ebs_delete_on_termination => 'Ebs.DeleteOnTermination' } def amazonize_block_device_mappings(block_device_mappings, key = 'BlockDeviceMapping') # :nodoc: result = {} unless block_device_mappings.right_blank? block_device_mappings = [block_device_mappings] unless block_device_mappings.is_a?(Array) block_device_mappings.each_with_index do |b, idx| BLOCK_DEVICE_KEY_MAPPING.each do |local_name, remote_name| value = b[local_name] case local_name when :no_device then value = value ? '' : nil # allow to pass :no_device as boolean end result["#{key}.#{idx+1}.#{remote_name}"] = value unless value.nil? end end end result end # Build API request keys set. # # Options is a hash, expectations is a set of keys [and rules] how to represent options. # Mappings is an Array (may include hashes) or a Hash. # # Example: # # options = { :valid_from => Time.now - 10, # :instance_count => 3, # :image_id => 'ami-08f41161', # :spot_price => 0.059, # :instance_type => 'c1.medium', # :instance_count => 1, # :key_name => 'tim', # :availability_zone => 'us-east-1a', # :monitoring_enabled => true, # :launch_group => 'lg1', # :availability_zone_group => 'azg1', # :groups => ['a', 'b', 'c'], # :group_ids => 'sg-1', # :user_data => 'konstantin', # :block_device_mappings => [ { :device_name => '/dev/sdk', # :ebs_snapshot_id => 'snap-145cbc7d', # :ebs_delete_on_termination => true, # :ebs_volume_size => 3, # :virtual_name => 'ephemeral2' }]} # mappings = { :spot_price, # :availability_zone_group, # :launch_group, # :type, # :instance_count, # :image_id => 'LaunchSpecification.ImageId', # :instance_type => 'LaunchSpecification.InstanceType', # :key_name => 'LaunchSpecification.KeyName', # :addressing_type => 'LaunchSpecification.AddressingType', # :kernel_id => 'LaunchSpecification.KernelId', # :ramdisk_id => 'LaunchSpecification.RamdiskId', # :subnet_id => 'LaunchSpecification.SubnetId', # :availability_zone => 'LaunchSpecification.Placement.AvailabilityZone', # :monitoring_enabled => 'LaunchSpecification.Monitoring.Enabled', # :valid_from => { :value => Proc.new { !options[:valid_from].right_blank? && AwsUtils::utc_iso8601(options[:valid_from]) }}, # :valid_until => { :value => Proc.new { !options[:valid_until].right_blank? && AwsUtils::utc_iso8601(options[:valid_until]) }}, # :user_data => { :name => 'LaunchSpecification.UserData', # :value => Proc.new { !options[:user_data].right_blank? && Base64.encode64(options[:user_data]).delete("\n") }}, # :groups => { :amazonize_list => 'LaunchSpecification.SecurityGroup'}, # :group_ids => { :amazonize_list => 'LaunchSpecification.SecurityGroupId'}, # :block_device_mappings => { :amazonize_bdm => 'LaunchSpecification.BlockDeviceMapping'}) # # map_api_keys_and_values( options, mappings) #=> # {"LaunchSpecification.BlockDeviceMapping.1.Ebs.DeleteOnTermination" => true, # "LaunchSpecification.BlockDeviceMapping.1.VirtualName" => "ephemeral2", # "LaunchSpecification.BlockDeviceMapping.1.Ebs.VolumeSize" => 3, # "LaunchSpecification.BlockDeviceMapping.1.Ebs.SnapshotId" => "snap-145cbc7d", # "LaunchSpecification.BlockDeviceMapping.1.DeviceName" => "/dev/sdk", # "LaunchSpecification.SecurityGroupId.1" => "sg-1", # "LaunchSpecification.InstanceType" => "c1.medium", # "LaunchSpecification.KeyName" => "tim", # "LaunchSpecification.ImageId" => "ami-08f41161", # "LaunchSpecification.SecurityGroup.1" => "a", # "LaunchSpecification.SecurityGroup.2" => "b", # "LaunchSpecification.SecurityGroup.3" => "c", # "LaunchSpecification.Placement.AvailabilityZone" => "us-east-1a", # "LaunchSpecification.Monitoring.Enabled" => true, # "LaunchGroup" => "lg1", # "InstanceCount" => 1, # "SpotPrice" => 0.059, # "AvailabilityZoneGroup" => "azg1", # "ValidFrom" => "2011-06-30T08:06:30.000Z", # "LaunchSpecification.UserData" => "a29uc3RhbnRpbg=="} # def map_api_keys_and_values(options, *mappings) # :nodoc: result = {} vars = {} # Fix inputs and make them all to be hashes mappings.flatten.each do |mapping| unless mapping.is_a?(Hash) # mapping is just a :key_name mapping = { mapping => { :name => mapping.to_s.right_camelize, :value => options[mapping] }} else mapping.each do |local_key, api_opts| unless api_opts.is_a?(Hash) # mapping is a { :key_name => 'ApiKeyName' } mapping[local_key] = { :name => api_opts.to_s, :value => options[local_key]} else # mapping is a { :key_name => { :name => 'ApiKeyName', :value => 'Value', ... etc} } api_opts[:name] = local_key.to_s.right_camelize if (api_opts.keys & [:name, :amazonize_list, :amazonize_bdm]).right_blank? api_opts[:value] = options[local_key] unless api_opts.has_key?(:value) end end end vars.merge! mapping end # Build API keys set # vars now is a Hash: # { :key1 => { :name => 'ApiKey1', :value => 'BlahBlah'}, # :key2 => { :amazonize_list => 'ApiKey2.?', :value => [1, ...] }, # :key3 => { :amazonize_bdm => 'BDM', :value => [{..}, ...] }, ... } # vars.each do |local_key, api_opts| if api_opts[:amazonize_list] result.merge!(amazonize_list( api_opts[:amazonize_list], api_opts[:value] )) unless api_opts[:value].right_blank? elsif api_opts[:amazonize_bdm] result.merge!(amazonize_block_device_mappings( api_opts[:value], api_opts[:amazonize_bdm] )) unless api_opts[:value].right_blank? else api_key = api_opts[:name] value = api_opts[:value] value = value.call if value.is_a?(Proc) next if value.right_blank? result[api_key] = value end end # result end # Transform a hash of parameters into a hash suitable for sending # to Amazon using a key mapping. # # amazonize_hash_with_key_mapping('Group.Filter', # {:some_param => 'SomeParam'}, # {:some_param => 'value'}) #=> {'Group.Filter.SomeParam' => 'value'} # def amazonize_hash_with_key_mapping(key, mapping, hash, options={}) result = {} unless hash.right_blank? mapping.each do |local_name, remote_name| value = hash[local_name] next if value.nil? result["#{key}.#{remote_name}"] = value end end result end # Transform a list of hashes of parameters into a hash suitable for sending # to Amazon using a key mapping. # # amazonize_list_with_key_mapping('Group.Filter', # [{:some_param => 'SomeParam'}, {:some_param => 'SomeParam'}], # {:some_param => 'value'}) #=> # {'Group.Filter.1.SomeParam' => 'value', # 'Group.Filter.2.SomeParam' => 'value'} # def amazonize_list_with_key_mapping(key, mapping, list, options={}) result = {} unless list.right_blank? list.each_with_index do |item, index| mapping.each do |local_name, remote_name| value = item[local_name] next if value.nil? result["#{key}.#{index+1}.#{remote_name}"] = value end end end end # Execute a block of code with custom set of settings for right_http_connection. # Accepts next options (see Rightscale::HttpConnection for explanation): # :raise_on_timeout # :http_connection_retry_count # :http_connection_open_timeout # :http_connection_read_timeout # :http_connection_retry_delay # :user_agent # :exception # # Example #1: # # # Try to create a snapshot but stop with exception if timeout is received # # to avoid having a duplicate API calls that create duplicate snapshots. # ec2 = Rightscale::Ec2::new(aws_access_key_id, aws_secret_access_key) # ec2.with_connection_options(:raise_on_timeout => true) do # ec2.create_snapshot('vol-898a6fe0', 'KD: WooHoo!!') # end # # Example #2: # # # Opposite case when the setting is global: # @ec2 = Rightscale::Ec2::new(aws_access_key_id, aws_secret_access_key, # :connection_options => { :raise_on_timeout => true }) # # Create an SSHKey but do tries on timeout # ec2.with_connection_options(:raise_on_timeout => false) do # new_key = ec2.create_key_pair('my_test_key') # end # # Example #3: # # # Global settings (HttpConnection level): # Rightscale::HttpConnection::params[:http_connection_open_timeout] = 5 # Rightscale::HttpConnection::params[:http_connection_read_timeout] = 250 # Rightscale::HttpConnection::params[:http_connection_retry_count] = 2 # # # Local setings (RightAws level) # ec2 = Rightscale::Ec2::new(AWS_ID, AWS_KEY, # :region => 'us-east-1', # :connection_options => { # :http_connection_read_timeout => 2, # :http_connection_retry_count => 5, # :user_agent => 'Mozilla 4.0' # }) # # # Custom settings (API call level) # ec2.with_connection_options(:raise_on_timeout => true, # :http_connection_read_timeout => 10, # :user_agent => '') do # pp ec2.describe_images # end # def with_connection_options(options, &block) @with_connection_options = options block.call self ensure @with_connection_options = {} end end # Exception class to signal any Amazon errors. All errors occuring during calls to Amazon's # web services raise this type of error. # Attribute inherited by RuntimeError: # message - the text of the error, generally as returned by AWS in its XML response. class AwsError < RuntimeError # either an array of errors where each item is itself an array of [code, message]), # or an error string if the error was raised manually, as in AwsError.new('err_text') attr_reader :errors # Request id (if exists) attr_reader :request_id # Response HTTP error code attr_reader :http_code def initialize(errors=nil, http_code=nil, request_id=nil) @errors = errors @request_id = request_id @http_code = http_code super(@errors.is_a?(Array) ? @errors.map{|code, msg| "#{code}: #{msg}"}.join("; ") : @errors.to_s) end # Does any of the error messages include the regexp +pattern+? # Used to determine whether to retry request. def include?(pattern) if @errors.is_a?(Array) @errors.each{ |code, msg| return true if code =~ pattern } else return true if @errors_str =~ pattern end false end # Generic handler for AwsErrors. +aws+ is the RightAws::S3, RightAws::EC2, or RightAws::SQS # object that caused the exception (it must provide last_request and last_response). Supported # boolean options are: # * :log print a message into the log using aws.logger to access the Logger # * :puts do a "puts" of the error # * :raise re-raise the error after logging def self.on_aws_exception(aws, options={:raise=>true, :log=>true}) # Only log & notify if not user error if !options[:raise] || system_error?($!) error_text = "#{$!.inspect}\n#{$@}.join('\n')}" puts error_text if options[:puts] # Log the error if options[:log] request = aws.last_request ? aws.last_request.path : '-none-' response = aws.last_response ? "#{aws.last_response.code} -- #{aws.last_response.message} -- #{aws.last_response.body}" : '-none-' aws.logger.error error_text aws.logger.error "Request was: #{request}" aws.logger.error "Response was: #{response}" end end raise if options[:raise] # re-raise an exception return nil end # True if e is an AWS system error, i.e. something that is for sure not the caller's fault. # Used to force logging. def self.system_error?(e) !e.is_a?(self) || e.message =~ /InternalError|InsufficientInstanceCapacity|Unavailable/ end end class AWSErrorHandler # 0-100 (%) DEFAULT_CLOSE_ON_4XX_PROBABILITY = 10 @@reiteration_start_delay = 0.2 def self.reiteration_start_delay @@reiteration_start_delay end def self.reiteration_start_delay=(reiteration_start_delay) @@reiteration_start_delay = reiteration_start_delay end @@reiteration_time = 5 def self.reiteration_time @@reiteration_time end def self.reiteration_time=(reiteration_time) @@reiteration_time = reiteration_time end @@close_on_error = true def self.close_on_error @@close_on_error end def self.close_on_error=(close_on_error) @@close_on_error = close_on_error end @@close_on_4xx_probability = DEFAULT_CLOSE_ON_4XX_PROBABILITY def self.close_on_4xx_probability @@close_on_4xx_probability end def self.close_on_4xx_probability=(close_on_4xx_probability) @@close_on_4xx_probability = close_on_4xx_probability end # params: # :reiteration_time # :errors_list # :close_on_error = true | false # :close_on_4xx_probability = 1-100 def initialize(aws, parser, params={}) #:nodoc: @aws = aws # Link to RightEc2 | RightSqs | RightS3 instance @parser = parser # parser to parse Amazon response @started_at = Time.now @stop_at = @started_at + (params[:reiteration_time] || @@reiteration_time) @errors_list = params[:errors_list] || [] @reiteration_delay = @@reiteration_start_delay @retries = 0 # close current HTTP(S) connection on 5xx, errors from list and 4xx errors @close_on_error = params[:close_on_error].nil? ? @@close_on_error : params[:close_on_error] @close_on_4xx_probability = params[:close_on_4xx_probability] || @@close_on_4xx_probability end # Returns false if def check(request) #:nodoc: result = false error_found = false redirect_detected= false error_match = nil last_errors_text = '' response = @aws.last_response # log error request_text_data = "#{request[:protocol]}://#{request[:server]}:#{request[:port]}#{request[:request].path}" # is this a redirect? # yes! if response.is_a?(Net::HTTPRedirection) redirect_detected = true else # no, it's an error ... @aws.logger.warn("##### #{@aws.class.name} returned an error: #{response.code} #{response.message}\n#{response.body} #####") @aws.logger.warn("##### #{@aws.class.name} request: #{request_text_data} ####") end # Extract error/redirection message from the response body # Amazon claims that a redirection must have a body but somethimes it is nil.... if response.body && response.body[/^(<\?xml|(.*?)<\/Endpoint>/] && $1 location = "#{request[:protocol]}://#{new_endpoint}:#{request[:port]}#{request[:request].path}" end # ... log information and ... @aws.logger.info("##### #{@aws.class.name} redirect requested: #{response.code} #{response.message} #####") @aws.logger.info(" Old location: #{request_text_data}") @aws.logger.info(" New location: #{location}") @aws.logger.info(" Request Verb: #{request[:request].class.name}") # ... fix the connection data request[:server] = URI.parse(location).host request[:protocol] = URI.parse(location).scheme request[:port] = URI.parse(location).port else # Not a redirect but an error: try to find the error in our list @errors_list.each do |error_to_find| if last_errors_text[/#{error_to_find}/i] error_found = true error_match = error_to_find @aws.logger.warn("##### Retry is needed, error pattern match: #{error_to_find} #####") break end end end # check the time has gone from the first error come if redirect_detected || error_found # Close the connection to the server and recreate a new one. # It may have a chance that one server is a semi-down and reconnection # will help us to connect to the other server if !redirect_detected && @close_on_error @aws.destroy_connection(request, "#{self.class.name}: error match to pattern '#{error_match}'") end if (Time.now < @stop_at) @retries += 1 unless redirect_detected @aws.logger.warn("##### Retry ##{@retries} is being performed. Sleeping for #{@reiteration_delay} sec. Whole time: #{Time.now-@started_at} sec ####") sleep @reiteration_delay @reiteration_delay *= 2 else @aws.logger.info("##### Retry ##{@retries} is being performed due to a redirect. ####") end # Always make sure that the fp is set to point to the beginning(?) # of the File/IO. TODO: it assumes that offset is 0, which is bad. if(request[:request].body_stream && request[:request].body_stream.respond_to?(:pos)) begin request[:request].body_stream.pos = 0 rescue Exception => e @logger.warn("Retry may fail due to unable to reset the file pointer" + " -- #{self.class.name} : #{e.inspect}") end end result = @aws.request_info(request, @parser) else @aws.logger.warn("##### Ooops, time is over... ####") end # aha, this is unhandled error: elsif @close_on_error # On 5xx(Server errors), 403(RequestTimeTooSkewed) and 408(Request Timeout) a conection has to be closed if @aws.last_response.code.to_s[/^(5\d\d|403|408)$/] @aws.destroy_connection(request, "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}'") # Is this a 4xx error ? elsif @aws.last_response.code.to_s[/^4\d\d$/] && @close_on_4xx_probability > rand(100) @aws.destroy_connection(request, "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}', " + "probability: #{@close_on_4xx_probability}%") end end result end end #----------------------------------------------------------------- class RightSaxParserCallbackTemplate #:nodoc: def initialize(right_aws_parser) @right_aws_parser = right_aws_parser end def on_characters(chars) @right_aws_parser.text(chars) end def on_start_document; end def on_comment(msg); end def on_processing_instruction(target, data); end def on_cdata_block(cdata); end def on_end_document; end end class RightSaxParserCallback < RightSaxParserCallbackTemplate def self.include_callback include XML::SaxParser::Callbacks end def on_start_element(name, attr_hash) @right_aws_parser.tag_start(name, attr_hash) end def on_end_element(name) @right_aws_parser.tag_end(name) end end class RightSaxParserCallbackNs < RightSaxParserCallbackTemplate def on_start_element_ns(name, attr_hash, prefix, uri, namespaces) @right_aws_parser.tag_start(name, attr_hash) end def on_end_element_ns(name, prefix, uri) @right_aws_parser.tag_end(name) end end class RightAWSParser #:nodoc: # default parsing library DEFAULT_XML_LIBRARY = 'rexml' # a list of supported parsers @@supported_xml_libs = [DEFAULT_XML_LIBRARY, 'libxml'] @@xml_lib = DEFAULT_XML_LIBRARY # xml library name: 'rexml' | 'libxml' def self.xml_lib @@xml_lib end def self.xml_lib=(new_lib_name) @@xml_lib = new_lib_name end attr_accessor :result attr_reader :xmlpath attr_accessor :xml_lib attr_reader :full_tag_name attr_reader :tag def initialize(params={}) @xmlpath = '' @full_tag_name = '' @result = false @text = '' @tag = '' @xml_lib = params[:xml_lib] || @@xml_lib @logger = params[:logger] reset end def tag_start(name, attributes) @text = '' @tag = name @full_tag_name += @full_tag_name.empty? ? name : "/#{name}" tagstart(name, attributes) @xmlpath = @full_tag_name end def tag_end(name) @xmlpath = @full_tag_name[/^(.*?)\/?#{name}$/] && $1 tagend(name) @full_tag_name = @xmlpath end def text(text) @text += text tagtext(text) end # Parser method. # Params: # xml_text - xml message text(String) or Net:HTTPxxx instance (response) # params[:xml_lib] - library name: 'rexml' | 'libxml' def parse(xml_text, params={}) # Get response body xml_text = xml_text.body unless xml_text.is_a?(String) @xml_lib = params[:xml_lib] || @xml_lib # check that we had no problems with this library otherwise use default @xml_lib = DEFAULT_XML_LIBRARY unless @@supported_xml_libs.include?(@xml_lib) # load xml library if @xml_lib=='libxml' && !defined?(XML::SaxParser) begin require 'xml/libxml' # Setup SaxParserCallback if XML::Parser::VERSION >= '0.5.1' && XML::Parser::VERSION < '0.9.7' RightSaxParserCallback.include_callback end rescue LoadError => e @@supported_xml_libs.delete(@xml_lib) @xml_lib = DEFAULT_XML_LIBRARY if @logger @logger.error e.inspect @logger.error e.backtrace @logger.info "Can not load 'libxml' library. '#{DEFAULT_XML_LIBRARY}' is used for parsing." end end end # Parse the xml text case @xml_lib when 'libxml' if XML::Parser::VERSION >= '0.9.9' # avoid warning on every usage xml = XML::SaxParser.string(xml_text) else xml = XML::SaxParser.new xml.string = xml_text end # check libxml-ruby version if XML::Parser::VERSION >= '0.9.7' xml.callbacks = RightSaxParserCallbackNs.new(self) elsif XML::Parser::VERSION >= '0.5.1' xml.callbacks = RightSaxParserCallback.new(self) else xml.on_start_element{|name, attr_hash| self.tag_start(name, attr_hash)} xml.on_characters{ |text| self.text(text)} xml.on_end_element{ |name| self.tag_end(name)} end xml.parse else REXML::Document.parse_stream(xml_text, self) end end # Parser must have a lots of methods # (see /usr/lib/ruby/1.8/rexml/parsers/streamparser.rb) # We dont need most of them in RightAWSParser and method_missing helps us # to skip their definition def method_missing(method, *params) # if the method is one of known - just skip it ... return if [:comment, :attlistdecl, :notationdecl, :elementdecl, :entitydecl, :cdata, :xmldecl, :attlistdecl, :instruction, :doctype].include?(method) # ... else - call super to raise an exception super(method, params) end # the functions to be overriden by children (if nessesery) def reset ; end def tagstart(name, attributes); end def tagend(name) ; end def tagtext(text) ; end end #----------------------------------------------------------------- # PARSERS: Errors #----------------------------------------------------------------- # # TemporaryRedirect # Please re-send this request to the specified temporary endpoint. Continue to use the original request endpoint for future requests. # FD8D5026D1C5ABA3 # bucket-for-k.s3-external-3.amazonaws.com # ItJy8xPFPli1fq/JR3DzQd3iDvFCRqi1LTRmunEdM1Uf6ZtW2r2kfGPWhRE1vtaU # bucket-for-k # class RightErrorResponseParser < RightAWSParser #:nodoc: attr_accessor :errors # array of hashes: error/message attr_accessor :requestID # attr_accessor :endpoint, :host_id, :bucket def tagend(name) case name when 'RequestID' ; @requestID = @text when 'Code' ; @code = @text when 'Message' ; @message = @text # when 'Endpoint' ; @endpoint = @text # when 'HostId' ; @host_id = @text # when 'Bucket' ; @bucket = @text when 'Error' ; @errors << [ @code, @message ] end end def reset @errors = [] end end # Dummy parser - does nothing # Returns the original params back class RightDummyParser # :nodoc: attr_accessor :result def parse(response, params={}) @result = [response, params] end end class RightHttp2xxParser < RightAWSParser # :nodoc: def parse(response) @result = response.is_a?(Net::HTTPSuccess) end end class RightBoolResponseParser < RightAWSParser #:nodoc: def tagend(name) @result = (@text=='true') if name == 'return' end end end ================================================ FILE: lib/awsbase/support.rb ================================================ # These are ActiveSupport-;like extensions to do a few handy things in the gems # Derived from ActiveSupport, so the AS copyright notice applies: # # # # Copyright (c) 2005 David Heinemeier Hansson # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #++ # # class String #:nodoc: def right_underscore self.gsub(/[A-Z]/){|match| "#{$`=='' ? '' : '_'}#{match.downcase}" } end end ================================================ FILE: lib/awsbase/version.rb ================================================ module RightAws #:nodoc: module VERSION #:nodoc: MAJOR = 3 unless defined?(MAJOR) MINOR = 1 unless defined?(MINOR) TINY = 0 unless defined?(TINY) STRING = [MAJOR, MINOR, TINY].join('.') unless defined?(STRING) end end ================================================ FILE: lib/ec2/right_ec2.rb ================================================ # # Copyright (c) 2007-2009 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws # = RightAWS::EC2 -- RightScale Amazon EC2 interface # The RightAws::EC2 class provides a complete interface to Amazon's # Elastic Compute Cloud service, as well as the associated EBS (Elastic Block # Store). # For explanations of the semantics # of each call, please refer to Amazon's documentation at # http://developer.amazonwebservices.com/connect/kbcategory.jspa?categoryID=87 # # Examples: # # Create an EC2 interface handle: # # @ec2 = RightAws::Ec2.new(aws_access_key_id, # aws_secret_access_key) # Create a new SSH key pair: # @key = 'right_ec2_awesome_test_key' # new_key = @ec2.create_key_pair(@key) # keys = @ec2.describe_key_pairs # # Create a security group: # @group = 'right_ec2_awesome_test_security_group' # @ec2.create_security_group(@group,'My awesome test group') # group = @ec2.describe_security_groups([@group])[0] # # Configure a security group: # @ec2.authorize_security_group_named_ingress(@group, account_number, 'default') # @ec2.authorize_security_group_IP_ingress(@group, 80,80,'udp','192.168.1.0/8') # # Describe the available images: # images = @ec2.describe_images # # Launch an instance: # ec2.run_instances('ami-9a9e7bf3', 1, 1, ['default'], @key, 'SomeImportantUserData', 'public') # # # Describe running instances: # @ec2.describe_instances # # Error handling: all operations raise an RightAws::AwsError in case # of problems. Note that transient errors are automatically retried. class Ec2 < RightAwsBase include RightAwsBaseInterface # Amazon EC2 API version being used API_VERSION = "2011-02-28" DEFAULT_HOST = "ec2.amazonaws.com" DEFAULT_PATH = '/' DEFAULT_PROTOCOL = 'https' DEFAULT_PORT = 443 # Default addressing type (public=NAT, direct=no-NAT) used when launching instances. DEFAULT_ADDRESSING_TYPE = 'public' DNS_ADDRESSING_SET = ['public','direct'] # Amazon EC2 Instance Types : http://www.amazon.com/b?ie=UTF8&node=370375011 # Default EC2 instance type (platform) DEFAULT_INSTANCE_TYPE = 'm1.small' INSTANCE_TYPES = [ 't1.micro' , 'm1.small' , 'm1.medium' , 'm1.large' , 'm1.xlarge' , 'c1.medium' , 'c1.xlarge' , 'm2.xlarge' , 'm2.2xlarge', 'm2.4xlarge', 'm3.xlarge' , 'm3.2xlarge', 'cc1.4xlarge', 'cg1.4xlarge', 'cc2.8xlarge', 'hi1.4xlarge', 'hs1.8xlarge', 'cr1.8xlarge' ] @@bench = AwsBenchmarkingBlock.new def self.bench_xml @@bench.xml end def self.bench_ec2 @@bench.service end # Current API version (sometimes we have to check it outside the GEM). @@api = ENV['EC2_API_VERSION'] || API_VERSION def self.api @@api end # Create a new handle to an EC2 account. All handles share the same per process or per thread # HTTP connection to Amazon EC2. Each handle is for a specific account. The params have the # following options: # * :endpoint_url a fully qualified url to Amazon API endpoint (this overwrites: :server, :port, :service, :protocol and :region). Example: 'https://eu-west-1.ec2.amazonaws.com/' # * :server: EC2 service host, default: DEFAULT_HOST # * :region: EC2 region (North America by default) # * :port: EC2 service port, default: DEFAULT_PORT # * :protocol: 'http' or 'https', default: DEFAULT_PROTOCOL # * :logger: for log messages, default: RAILS_DEFAULT_LOGGER else STDOUT # * :signature_version: The signature version : '0','1' or '2'(default) # * :cache: true/false: caching for: ec2_describe_images, describe_instances, # * :token: Option SecurityToken for temporary credentials # describe_images_by_owner, describe_images_by_executable_by, describe_availability_zones, # describe_security_groups, describe_key_pairs, describe_addresses, # describe_volumes, describe_snapshots methods, default: false. # def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={}) init({ :name => 'EC2', :default_host => ENV['EC2_URL'] ? URI.parse(ENV['EC2_URL']).host : DEFAULT_HOST, :default_port => ENV['EC2_URL'] ? URI.parse(ENV['EC2_URL']).port : DEFAULT_PORT, :default_service => ENV['EC2_URL'] ? URI.parse(ENV['EC2_URL']).path : DEFAULT_PATH, :default_protocol => ENV['EC2_URL'] ? URI.parse(ENV['EC2_URL']).scheme : DEFAULT_PROTOCOL, :default_api_version => @@api }, aws_access_key_id || ENV['AWS_ACCESS_KEY_ID'] , aws_secret_access_key|| ENV['AWS_SECRET_ACCESS_KEY'], params) # Eucalyptus supports some yummy features but Amazon does not #if @params[:eucalyptus] # @params[:port_based_group_ingress] = true unless @params.has_key?(:port_based_group_ingress) #end end def generate_request(action, params={}, custom_options={}) #:nodoc: generate_request_impl(:get, action, params, custom_options) end # Sends request to Amazon and parses the response # Raises AwsError if any banana happened def request_info(request, parser) #:nodoc: request_info_impl(:ec2_connection, @@bench, request, parser) end def describe_resources_with_list_and_options(remote_function_name, remote_item_name, parser_class, list_and_options, &block) # :nodoc: # 'RemoteFunctionName' -> :remote_funtion_name cache_name = remote_function_name.right_underscore.to_sym list, options = AwsUtils::split_items_and_params(list_and_options) custom_options = {} # Resource IDs to fetch request_hash = amazonize_list(remote_item_name, list) # Other custom options options.each do |key, values| next if values.right_blank? case key when :options custom_options = values when :filters then request_hash.merge!(amazonize_list(['Filter.?.Name', 'Filter.?.Value.?'], values)) else request_hash.merge!(amazonize_list(key.to_s.right_camelize, values)) end end cache_for = (list.right_blank? && options.right_blank?) ? cache_name : nil link = generate_request(remote_function_name, request_hash, custom_options) request_cache_or_info(cache_for, link, parser_class, @@bench, cache_for, &block) rescue Exception on_exception end # Incrementally lists given API call. # # All params are the same as for describe_resources_with_list_and_options call. # # Block is called on every chunk of resources received. If you need to stop the loop # just make the block to return nil or false. # # The API call should support 'NextToken' parameter and the response should return :next_token # as well. # # Returns the last response from the cloud. # def incrementally_list_items(remote_function_name, remote_item_name, parser_class, list_and_options, &block) # :nodoc: last_response = nil loop do last_response = describe_resources_with_list_and_options(remote_function_name, remote_item_name, parser_class, list_and_options) break unless block && block.call(last_response) && !last_response[:next_token].right_blank? list, options = AwsUtils::split_items_and_params(list_and_options) options[:next_token] = last_response[:next_token] list_and_options = list + [options] end last_response end def merge_new_options_into_list_and_options(list_and_options, new_options) list, options = AwsUtils::split_items_and_params(list_and_options) list << options.merge(new_options) end #----------------------------------------------------------------- # Keys #----------------------------------------------------------------- # Retrieve a list of SSH keys. # # Accepts a list of ssh keys and/or a set of filters as the last parameter. # # Filters: fingerprint, key-name # # Returns an array of keys or an exception. Each key is represented as a two-element hash. # # ec2.describe_key_pairs #=> # [{:aws_fingerprint=> "01:02:03:f4:25:e6:97:e8:9b:02:1a:26:32:4e:58:6b:7a:8c:9f:03", :aws_key_name=>"key-1"}, # {:aws_fingerprint=> "1e:29:30:47:58:6d:7b:8c:9f:08:11:20:3c:44:52:69:74:80:97:08", :aws_key_name=>"key-2"}, # ..., {...} ] # # ec2.describe_key_pairs(:filters => {'fingerprint' => ["53:0b:73:c9:c8:18:98:6e:bc:98:9e:51:97:04:74:4b:07:f9:00:00", # "9f:57:a5:bb:4b:e8:a7:f8:3c:fe:d6:db:41:f5:7e:97:b5:b2:00:00"]}) # # P.S. filters: http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference-query-DescribeKeyPairs.html # def describe_key_pairs(*list_and_options) describe_resources_with_list_and_options('DescribeKeyPairs', 'KeyName', QEc2DescribeKeyPairParser, list_and_options) end # Import new SSH key. Returns a hash of the key's data or an exception. # # ec2.import_key_pair('my_awesome_key', 'C:\keys\myfavoritekeypair_public.ppk') #=> # {:aws_key_name => "my_awesome_key", # :aws_fingerprint => "01:02:03:f4:25:e6:97:e8:9b:02:1a:26:32:4e:58:6b:7a:8c:9f:03"} # def import_key_pair(name, public_key_material) link = generate_request("ImportKeyPair", 'KeyName' => name.to_s, 'PublicKeyMaterial' => Base64.encode64(public_key_material.to_s)) request_info(link, QEc2ImportKeyPairParser.new(:logger => @logger)) rescue Exception on_exception end # Create new SSH key. Returns a hash of the key's data or an exception. # # ec2.create_key_pair('my_awesome_key') #=> # {:aws_key_name => "my_awesome_key", # :aws_fingerprint => "01:02:03:f4:25:e6:97:e8:9b:02:1a:26:32:4e:58:6b:7a:8c:9f:03", # :aws_material => "-----BEGIN RSA PRIVATE KEY-----\nMIIEpQIBAAK...Q8MDrCbuQ=\n-----END RSA PRIVATE KEY-----"} # def create_key_pair(name) link = generate_request("CreateKeyPair", 'KeyName' => name.to_s) request_info(link, QEc2CreateKeyPairParser.new(:logger => @logger)) rescue Exception on_exception end # Delete a key pair. Returns +true+ or an exception. # # ec2.delete_key_pair('my_awesome_key') #=> true # def delete_key_pair(name) link = generate_request("DeleteKeyPair", 'KeyName' => name.to_s) request_info(link, RightBoolResponseParser.new(:logger => @logger)) rescue Exception on_exception end #----------------------------------------------------------------- # Elastic IPs #----------------------------------------------------------------- # Acquire a new elastic IP address for use with your account. # Options: :domain. # Returns allocated IP address or or an exception. # # ec2.allocate_address #=> # { :public_ip => "50.19.214.224", # :domain => "standard"} # # ec2.allocate_address(:domain => 'vpc') #=> # { :allocation_id => "eipalloc-c6abfeaf", # :domain => "vpc", # :public_ip => "184.72.112.39"} # def allocate_address(options={}) request_hash = {} request_hash['Domain'] = options[:domain] unless options[:domain].right_blank? link = generate_request("AllocateAddress", request_hash) request_info(link, QEc2AllocateAddressParser.new(:logger => @logger)) rescue Exception on_exception end # Associate an elastic IP address with an instance. # Options: :public_ip, :allocation_id. # Returns a hash of data or an exception. # # ec2.associate_address('i-d630cbbf', :public_ip => '75.101.154.140') #=> # { :return => true } # # ec2.associate_address(inst, :allocation_id => "eipalloc-c6abfeaf") #=> # { :return => true, # :association_id => 'eipassoc-fc5ca095'} # def associate_address(instance_id, options={}) request_hash = { "InstanceId" => instance_id.to_s } request_hash['PublicIp'] = options[:public_ip] unless options[:public_ip].right_blank? request_hash['AllocationId'] = options[:allocation_id] unless options[:allocation_id].right_blank? link = generate_request("AssociateAddress", request_hash) request_info(link, QEc2AssociateAddressParser.new(:logger => @logger)) rescue Exception on_exception end # List elastic IPs by public addresses. # # Accepts a list of addresses and/or a set of filters as the last parameter. # # Filters: instance-id, public-ip # # Returns an array of 2 keys (:instance_id and :public_ip) hashes: # # ec2.describe_addresses #=> [{:instance_id=>"i-75ebd41b", :domain=>"standard", :public_ip=>"50.17.211.96"}, # :domain=>"vpc", :public_ip=>"184.72.112.39", :allocation_id=>"eipalloc-c6abfeaf"}] # # ec2.describe_addresses('75.101.154.140') #=> [{:instance_id=>"i-d630cbbf", :public_ip=>"75.101.154.140", :domain=>"standard"}] # # ec2.describe_addresses(:filters => { 'public-ip' => "75.101.154.140" }) # # P.S. http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-DescribeAddresses.html # def describe_addresses(*list_and_options) describe_resources_with_list_and_options('DescribeAddresses', 'PublicIp', QEc2DescribeAddressesParser, list_and_options) end # List elastic IPs by allocation ids. # # Accepts a list of allocations and/or a set of filters as the last parameter. # # describe_addresses_by_allocation_ids("eipalloc-c6abfeaf") #=> # [{:domain=>"vpc", # :public_ip=>"184.72.112.39", # :allocation_id=>"eipalloc-c6abfeaf"}] # # P.S. http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-DescribeAddresses.html # def describe_addresses_by_allocation_ids(*list_and_options) describe_resources_with_list_and_options('DescribeAddresses', 'AllocationId', QEc2DescribeAddressesParser, list_and_options) end # Disassociate the specified elastic IP address from the instance to which it is assigned. # Options: :public_ip, :association_id. # Returns +true+ or an exception. # # ec2.disassociate_address(:public_ip => '75.101.154.140') #=> true # def disassociate_address(options = {}) request_hash = {} request_hash['PublicIp'] = options[:public_ip] unless options[:public_ip].right_blank? request_hash['AssociationId'] = options[:association_id] unless options[:association_id].right_blank? link = generate_request("DisassociateAddress", request_hash) request_info(link, RightBoolResponseParser.new(:logger => @logger)) rescue Exception on_exception end # Release an elastic IP address associated with your account. # Options: :public_ip, :allocation_id. # Returns +true+ or an exception. # # ec2.release_address(:public_ip => '75.101.154.140') #=> true # def release_address(options = {}) request_hash = {} request_hash['PublicIp'] = options[:public_ip] unless options[:public_ip].right_blank? request_hash['AllocationId'] = options[:allocation_id] unless options[:allocation_id].right_blank? link = generate_request("ReleaseAddress", request_hash) request_info(link, RightBoolResponseParser.new(:logger => @logger)) rescue Exception on_exception end #----------------------------------------------------------------- # Availability zones #----------------------------------------------------------------- # Describes availability zones that are currently available to the account and their states. # # Accepts a list of availability zones and/or a set of filters as the last parameter. # # Filters: message, region-name, state, zone-name # # Returns an array of 2 keys (:zone_name and :zone_state) hashes: # # ec2.describe_availability_zones #=> [{:region_name=>"us-east-1", # :zone_name=>"us-east-1a", # :zone_state=>"available"}, ... ] # # ec2.describe_availability_zones('us-east-1c') #=> [{:region_name=>"us-east-1", # :zone_state=>"available", # :zone_name=>"us-east-1c"}] # # P.S. filters: http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-DescribeAvailabilityZones.html # def describe_availability_zones(*list_and_options) describe_resources_with_list_and_options('DescribeAvailabilityZones', 'ZoneName', QEc2DescribeAvailabilityZonesParser, list_and_options) end #----------------------------------------------------------------- # Regions #----------------------------------------------------------------- # Describe regions. # # Accepts a list of regions and/or a set of filters as the last parameter. # # Filters: endpoint, region-name # # ec2.describe_regions #=> # [{:region_endpoint=>"ec2.eu-west-1.amazonaws.com", :region_name=>"eu-west-1"}, # {:region_endpoint=>"ec2.us-east-1.amazonaws.com", :region_name=>"us-east-1"}, # {:region_endpoint=>"ec2.ap-northeast-1.amazonaws.com", :region_name=>"ap-northeast-1"}, # {:region_endpoint=>"ec2.us-west-1.amazonaws.com", :region_name=>"us-west-1"}, # {:region_endpoint=>"ec2.ap-southeast-1.amazonaws.com", :region_name=>"ap-southeast-1"}] # # P.S. filters: http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference-query-DescribeRegions.html # def describe_regions(*list_and_options) describe_resources_with_list_and_options('DescribeRegions', 'RegionName', QEc2DescribeRegionsParser, list_and_options) end #----------------------------------------------------------------- # Accounts #----------------------------------------------------------------- # Describe the specified attribute of your AWS account. # # ec2.describe_account_attributes(:attribute_name => ['default-vpc','supported-platforms']) #=> # {"default-vpc" => "vpc-8c3b00e7", # "supported-platforms" => "VPC"} # # ec2.describe_account_attributes #=> # {"vpc-max-security-groups-per-interface" => "5", # "max-instances" => "150", # "supported-platforms" => ["EC2", "VPC"], # "default-vpc" => "none"} # def describe_account_attributes(*list_and_options) list_and_options = merge_new_options_into_list_and_options(list_and_options, :options => {:api_version => VPC_API_VERSION}) describe_resources_with_list_and_options('DescribeAccountAttributes', 'accountAttributeValues', QEc2DescribeAccountAttributesParser, list_and_options) end #----------------------------------------------------------------- # PARSERS: Key Pair #----------------------------------------------------------------- class QEc2DescribeKeyPairParser < RightAWSParser #:nodoc: def tagstart(name, attributes) @item = {} if name == 'item' end def tagend(name) case name when 'keyName' then @item[:aws_key_name] = @text when 'keyFingerprint' then @item[:aws_fingerprint] = @text when 'item' then @result << @item end end def reset @result = []; end end class QEc2CreateKeyPairParser < RightAWSParser #:nodoc: def tagstart(name, attributes) @result = {} if name == 'CreateKeyPairResponse' end def tagend(name) case name when 'keyName' then @result[:aws_key_name] = @text when 'keyFingerprint' then @result[:aws_fingerprint] = @text when 'keyMaterial' then @result[:aws_material] = @text end end end class QEc2ImportKeyPairParser < RightAWSParser #:nodoc: def tagstart(name, attributes) @result = {} if name == 'ImportKeyPairResponse' end def tagend(name) case name when 'keyName' then @result[:aws_key_name] = @text when 'keyFingerprint' then @result[:aws_fingerprint] = @text end end end #----------------------------------------------------------------- # PARSERS: Elastic IPs #----------------------------------------------------------------- class QEc2AllocateAddressParser < RightAWSParser #:nodoc: def tagend(name) case name when 'publicIp' then @result[:public_ip] = @text when 'allocationId' then @result[:allocation_id] = @text when 'domain' then @result[:domain] = @text end end def reset @result = {} end end class QEc2AssociateAddressParser < RightAWSParser #:nodoc: def tagend(name) case name when 'return' then @result[:return] = @text == 'true' when 'associationId' then @result[:association_id] = @text end end def reset @result = {} end end class QEc2DescribeAddressesParser < RightAWSParser #:nodoc: def tagstart(name, attributes) @item = {} if name == 'item' end def tagend(name) case name when 'instanceId' then (@item[:instance_id] = @text unless @text.right_blank?) when 'publicIp' then @item[:public_ip] = @text when 'allocationId' then @item[:allocation_id] = @text when 'associationId' then @item[:association_id] = @text when 'domain' then @item[:domain] = @text when 'item' then @result << @item end end def reset @result = [] end end #----------------------------------------------------------------- # PARSERS: AvailabilityZones #----------------------------------------------------------------- class QEc2DescribeAvailabilityZonesParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{/availabilityZoneInfo/item$} then @item = {} end end def tagend(name) case name when 'regionName' then @item[:region_name] = @text when 'zoneName' then @item[:zone_name] = @text when 'zoneState' then @item[:zone_state] = @text else case full_tag_name when %r{/messageSet/item/message$} then (@item[:messages] ||= []) << @text when %r{/availabilityZoneInfo/item$} then @result << @item end end end def reset @result = [] end end #----------------------------------------------------------------- # PARSERS: Regions #----------------------------------------------------------------- class QEc2DescribeRegionsParser < RightAWSParser #:nodoc: def tagstart(name, attributes) @item = {} if name == 'item' end def tagend(name) case name when 'regionName' then @item[:region_name] = @text when 'regionEndpoint' then @item[:region_endpoint] = @text when 'item' then @result << @item end end def reset @result = [] end end #----------------------------------------------------------------- # PARSERS: Account #----------------------------------------------------------------- class QEc2DescribeAccountAttributesParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case name when 'attributeValueSet' then @values = [] end end def tagend(name) case name when 'attributeName' then @name = @text when 'attributeValue' then @values << @text when 'attributeValueSet' then @result[@name] = (@values.size == 1) ? @values.first : @values end end def reset @result = {} end end end end ================================================ FILE: lib/ec2/right_ec2_ebs.rb ================================================ # # Copyright (c) 2009 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws class Ec2 #----------------------------------------------------------------- # EBS: Volumes #----------------------------------------------------------------- VOLUME_API_VERSION = (API_VERSION > '2012-06-15') ? API_VERSION : '2012-06-15' VOLUME_TYPES = ['standard', 'io1'] # Describe EBS volumes. # # Accepts a list of volumes and/or a set of filters as the last parameter. # # Filters: attachement.attach-time, attachment.delete-on-termination, attachement.device, attachment.instance-id, # attachment.status, availability-zone, create-time, size, snapshot-id, status, tag-key, tag-value, tag:key, volume-id # # ec2.describe_volumes #=> # [{:aws_size => 94, # :aws_device => "/dev/sdc", # :aws_attachment_status => "attached", # :zone => "merlot", # :snapshot_id => nil, # :aws_attached_at => "2008-06-18T08:19:28.000Z", # :aws_status => "in-use", # :aws_id => "vol-60957009", # :aws_created_at => "2008-06-18T08:19:20.000Z", # :aws_instance_id => "i-c014c0a9"}, # {:aws_id => "vol-71de8b1f", # :aws_size => 5, # :snapshot_id => nil, # :zone => "us-east-1a", # :aws_status => "available", # :aws_created_at => "2012-06-21T18:47:34.000Z", # :volume_type => "io1", # :iop => "5"},# # {:aws_size => 1, # :zone => "merlot", # :snapshot_id => nil, # :aws_status => "available", # :aws_id => "vol-58957031", # :aws_created_at => Wed Jun 18 08:19:21 UTC 2008,}, ... ] # # ec2.describe_volumes(:filters => { 'availability-zone' => 'us-east-1a', 'size' => '10' }) # # P.S. filters: http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-DescribeVolumes.html # def describe_volumes(*list_and_options) list_and_options = merge_new_options_into_list_and_options(list_and_options, :options => {:api_version => VOLUME_API_VERSION}) describe_resources_with_list_and_options('DescribeVolumes', 'VolumeId', QEc2DescribeVolumesParser, list_and_options) end # Create new EBS volume based on previously created snapshot. # +Size+ in Gigabytes. # # ec2.create_volume('snap-000000', 10, zone) #=> # {:snapshot_id => "snap-e21df98b", # :aws_status => "creating", # :aws_id => "vol-fc9f7a95", # :zone => "merlot", # :aws_created_at => "2008-06-24T18:13:32.000Z", # :aws_size => 94} # # ec2.create_volume(nil, 5, 'us-east-1a', :iops => '5', :volume_type => 'io1') #=> # {:aws_id=>"vol-71de8b1f", # :aws_size=>5, # :snapshot_id=>nil, # :zone=>"us-east-1a", # :aws_status=>"creating", # :aws_created_at=>"2012-06-21T18:47:34.000Z", # :volume_type=>"io1", # :iops=>"5"} # def create_volume(snapshot_id, size, zone, options={}) hash = { "Size" => size.to_s, "AvailabilityZone" => zone.to_s } # Get rig of empty snapshot: e8s guys do not like it hash["SnapshotId"] = snapshot_id.to_s unless snapshot_id.right_blank? # Add IOPS support (default behavior) but skip it when an old API version call is requested options[:options] ||= {} options[:options][:api_version] ||= VOLUME_API_VERSION if options[:options][:api_version] >= VOLUME_API_VERSION hash["VolumeType"] = options[:volume_type] unless options[:volume_type].right_blank? hash["Iops"] = options[:iops] unless options[:iops].right_blank? end link = generate_request("CreateVolume", hash, options[:options]) request_info(link, QEc2CreateVolumeParser.new(:logger => @logger)) rescue Exception on_exception end # Delete the specified EBS volume. # This does not deletes any snapshots created from this volume. # # ec2.delete_volume('vol-b48a6fdd') #=> true # def delete_volume(volume_id) link = generate_request("DeleteVolume", "VolumeId" => volume_id.to_s) request_info(link, RightBoolResponseParser.new(:logger => @logger)) rescue Exception on_exception end # Attach the specified EBS volume to a specified instance, exposing the # volume using the specified device name. # # ec2.attach_volume('vol-898a6fe0', 'i-7c905415', '/dev/sdh') #=> # { :aws_instance_id => "i-7c905415", # :aws_device => "/dev/sdh", # :aws_status => "attaching", # :aws_attached_at => "2008-03-28T14:14:39.000Z", # :aws_id => "vol-898a6fe0" } # def attach_volume(volume_id, instance_id, device) link = generate_request("AttachVolume", "VolumeId" => volume_id.to_s, "InstanceId" => instance_id.to_s, "Device" => device.to_s) request_info(link, QEc2AttachAndDetachVolumeParser.new(:logger => @logger)) rescue Exception on_exception end # Detach the specified EBS volume from the instance to which it is attached. # # ec2.detach_volume('vol-898a6fe0') #=> # { :aws_instance_id => "i-7c905415", # :aws_device => "/dev/sdh", # :aws_status => "detaching", # :aws_attached_at => "2008-03-28T14:38:34.000Z", # :aws_id => "vol-898a6fe0"} # def detach_volume(volume_id, instance_id=nil, device=nil, force=nil) hash = { "VolumeId" => volume_id.to_s } hash["InstanceId"] = instance_id.to_s unless instance_id.right_blank? hash["Device"] = device.to_s unless device.right_blank? hash["Force"] = 'true' if force # link = generate_request("DetachVolume", hash) request_info(link, QEc2AttachAndDetachVolumeParser.new(:logger => @logger)) rescue Exception on_exception end #----------------------------------------------------------------- # EBS: Snapshots #----------------------------------------------------------------- # Describe EBS snapshots. # # Accepts a list of snapshots and/or options: :restorable_by, :owner and :filters # # Options: :restorable_by => Array or String, :owner => Array or String, :filters => Hash # # Filters: description, owner-alias, owner-id, progress, snapshot-id, start-time, status, tag-key, # tag-value, tag:key, volume-id, volume-size # # ec2.describe_snapshots #=> # [{:aws_volume_size=>2, # :tags=>{}, # :aws_id=>"snap-d010f6b9", # :owner_alias=>"amazon", # :aws_progress=>"100%", # :aws_status=>"completed", # :aws_description=> # "Windows 2003 R2 Installation Media [Deprecated] - Enterprise Edition 64-bit", # :aws_owner=>"711940113766", # :aws_volume_id=>"vol-351efb5c", # :aws_started_at=>"2008-10-20T18:23:59.000Z"}, # {:aws_volume_size=>2, # :tags=>{}, # :aws_id=>"snap-a310f6ca", # :owner_alias=>"amazon", # :aws_progress=>"100%", # :aws_status=>"completed", # :aws_description=>"Windows 2003 R2 Installation Media 64-bit", # :aws_owner=>"711940113766", # :aws_volume_id=>"vol-001efb69", # :aws_started_at=>"2008-10-20T18:25:53.000Z"}, ... ] # # ec2.describe_snapshots("snap-e676e28a", "snap-e176e281") # # ec2.describe_snapshots(:restorable_by => ['123456781234'], # :owner => ['self', 'amazon'], # :filters => {'tag:MyTag' => 'MyValue'}) # # P.S. filters: http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-DescribeSnapshots.html # def describe_snapshots(*list_and_options) describe_resources_with_list_and_options('DescribeSnapshots', 'SnapshotId', QEc2DescribeSnapshotsParser, list_and_options) end def describe_snapshots_by_restorable_by(*list_and_options) describe_resources_with_list_and_options('DescribeSnapshots', 'RestorableBy', QEc2DescribeSnapshotsParser, list_and_options) end # Create a snapshot of specified volume. # # ec2.create_snapshot('vol-898a6fe0', 'KD: WooHoo!!') #=> # {:aws_volume_id=>"vol-e429db8d", # :aws_started_at=>"2009-10-01T09:23:38.000Z", # :aws_description=>"KD: WooHoo!!", # :aws_owner=>"648770000000", # :aws_progress=>"", # :aws_status=>"pending", # :aws_volume_size=>1, # :aws_id=>"snap-3df54854"} # def create_snapshot(volume_id, description='') link = generate_request("CreateSnapshot", "VolumeId" => volume_id.to_s, "Description" => description) request_info(link, QEc2DescribeSnapshotsParser.new(:logger => @logger)).first rescue Exception on_exception end # Create a snapshot of specified volume, but with the normal retry algorithms disabled. # This method will return immediately upon error. The user can specify connect and read timeouts (in s) # for the connection to AWS. If the user does not specify timeouts, try_create_snapshot uses the default values # in Rightscale::HttpConnection. # # ec2.try_create_snapshot('vol-898a6fe0', 'KD: WooHoo!!') #=> # {:aws_volume_id=>"vol-e429db8d", # :aws_started_at=>"2009-10-01T09:23:38.000Z", # :aws_description=>"KD: WooHoo!!", # :aws_owner=>"648770000000", # :aws_progress=>"", # :aws_status=>"pending", # :aws_volume_size=>1, # :aws_id=>"snap-3df54854"} # def try_create_snapshot(volume_id, connect_timeout = nil, read_timeout = nil, description='') # For safety in the ensure block...we don't want to restore values # if we never read them in the first place orig_reiteration_time = nil orig_http_params = nil orig_reiteration_time = RightAws::AWSErrorHandler::reiteration_time RightAws::AWSErrorHandler::reiteration_time = 0 orig_http_params = Rightscale::HttpConnection::params() new_http_params = orig_http_params.dup new_http_params[:http_connection_retry_count] = 0 new_http_params[:http_connection_open_timeout] = connect_timeout if !connect_timeout.nil? new_http_params[:http_connection_read_timeout] = read_timeout if !read_timeout.nil? Rightscale::HttpConnection::params = new_http_params link = generate_request("CreateSnapshot", "VolumeId" => volume_id.to_s, "Description" => description) request_info(link, QEc2DescribeSnapshotsParser.new(:logger => @logger)).first rescue Exception on_exception ensure RightAws::AWSErrorHandler::reiteration_time = orig_reiteration_time if orig_reiteration_time Rightscale::HttpConnection::params = orig_http_params if orig_http_params end # Describe snapshot attribute. # # ec2.describe_snapshot_attribute('snap-36fe435f') #=> # {:create_volume_permission=> # {:users=>["826690000000", "826690000001"], :groups=>['all']}} # def describe_snapshot_attribute(snapshot_id, attribute='createVolumePermission') link = generate_request("DescribeSnapshotAttribute", 'SnapshotId'=> snapshot_id, 'Attribute' => attribute) request_info(link, QEc2DescribeSnapshotAttributeParser.new(:logger => @logger)) rescue Exception on_exception end # Reset permission settings for the specified snapshot. # # ec2.reset_snapshot_attribute('snap-cecd29a7') #=> true # def reset_snapshot_attribute(snapshot_id, attribute='createVolumePermission') link = generate_request("ResetSnapshotAttribute", 'SnapshotId' => snapshot_id, 'Attribute' => attribute) request_info(link, RightBoolResponseParser.new(:logger => @logger)) rescue Exception on_exception end # Modify snapshot attribute. # # Attribute can take only 'createVolumePermission' value. # Value is a Hash {:add_user_ids, :add_groups, :remove_user_ids, :remove_groups }. # def modify_snapshot_attribute(snapshot_id, attribute, value) params = { 'SnapshotId' => snapshot_id } case attribute.to_s when 'createVolumePermission' params.update(amazonize_list('CreateVolumePermission.Add.?.UserId', value[:add_user_ids])) params.update(amazonize_list('CreateVolumePermission.Add.?.Group', value[:add_groups])) params.update(amazonize_list('CreateVolumePermission.Remove.?.UserId', value[:remove_user_ids])) params.update(amazonize_list('CreateVolumePermission.Remove.?.Group', value[:remove_groups])) end link = generate_request("ModifySnapshotAttribute", params) request_info(link, RightBoolResponseParser.new(:logger => @logger)) rescue Exception on_exception end # Grant create volume permission for a list of users. # # ec2.modify_snapshot_attribute_create_volume_permission_add_users('snap-36fe435f', '000000000000', '000000000001') #=> true # def modify_snapshot_attribute_create_volume_permission_add_users(snapshot_id, *user_ids) modify_snapshot_attribute(snapshot_id, 'createVolumePermission', :add_user_ids => user_ids.flatten ) end # Revoke create volume permission for a list of users. # # ec2.modify_snapshot_attribute_create_volume_permission_remove_users('snap-36fe435f', '000000000000', '000000000001') #=> true # def modify_snapshot_attribute_create_volume_permission_remove_users(snapshot_id, *user_ids) modify_snapshot_attribute(snapshot_id, 'createVolumePermission', :remove_user_ids => user_ids.flatten ) end # Grant create volume permission for user groups (currently only 'all' is supported). # # ec2.modify_snapshot_attribute_create_volume_permission_add_groups('snap-36fe435f') #=> true # def modify_snapshot_attribute_create_volume_permission_add_groups(snapshot_id, *groups) groups.flatten! groups = ['all'] if groups.right_blank? modify_snapshot_attribute(snapshot_id, 'createVolumePermission', :add_groups => groups ) end # Remove create volume permission for user groups (currently only 'all' is supported). # # ec2.modify_snapshot_attribute_create_volume_permission_remove_groups('snap-36fe435f') #=> true # def modify_snapshot_attribute_create_volume_permission_remove_groups(snapshot_id, *groups) groups.flatten! groups = ['all'] if groups.right_blank? modify_snapshot_attribute(snapshot_id, 'createVolumePermission', :remove_groups => groups ) end # Delete the specified snapshot. # # ec2.delete_snapshot('snap-55a5403c') #=> true # def delete_snapshot(snapshot_id) link = generate_request("DeleteSnapshot", "SnapshotId" => snapshot_id.to_s) request_info(link, RightBoolResponseParser.new(:logger => @logger)) rescue Exception on_exception end #----------------------------------------------------------------- # PARSERS: EBS - Volumes #----------------------------------------------------------------- class QEc2CreateVolumeParser < RightAWSParser #:nodoc: def tagend(name) case name when 'volumeId' then @result[:aws_id] = @text when 'status' then @result[:aws_status] = @text when 'createTime' then @result[:aws_created_at] = @text when 'size' then @result[:aws_size] = @text.to_i ### when 'snapshotId' then @result[:snapshot_id] = @text.right_blank? ? nil : @text ### when 'availabilityZone' then @result[:zone] = @text ### when 'volumeType' then @result[:volume_type] = @text when 'iops' then @result[:iops] = @text end end def reset @result = {} end end class QEc2AttachAndDetachVolumeParser < RightAWSParser #:nodoc: def tagend(name) case name when 'volumeId' then @result[:aws_id] = @text when 'instanceId' then @result[:aws_instance_id] = @text when 'device' then @result[:aws_device] = @text when 'status' then @result[:aws_attachment_status] = @text when 'attachTime' then @result[:aws_attached_at] = @text end end def reset @result = {} end end class QEc2DescribeVolumesParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{volumeSet/item$} then @item = { :tags => {} } when %r{/tagSet/item$} then @aws_tag = {} end end def tagend(name) case name when 'size' then @item[:aws_size] = @text.to_i when 'createTime' then @item[:aws_created_at] = @text when 'instanceId' then @item[:aws_instance_id] = @text when 'device' then @item[:aws_device] = @text when 'attachTime' then @item[:aws_attached_at] = @text when 'snapshotId' then @item[:snapshot_id] = @text.right_blank? ? nil : @text when 'availabilityZone' then @item[:zone] = @text when 'deleteOnTermination' then @item[:delete_on_termination] = (@text == 'true') when 'volumeType' then @item[:volume_type] = @text when 'iops' then @item[:iops] = @text else case full_tag_name when %r{/volumeSet/item/volumeId$} then @item[:aws_id] = @text when %r{/volumeSet/item/status$} then @item[:aws_status] = @text when %r{/attachmentSet/item/status$} then @item[:aws_attachment_status] = @text when %r{/tagSet/item/key$} then @aws_tag[:key] = @text when %r{/tagSet/item/value$} then @aws_tag[:value] = @text when %r{/tagSet/item$} then @item[:tags][@aws_tag[:key]] = @aws_tag[:value] when %r{/volumeSet/item$} then @result << @item end end end def reset @result = [] end end #----------------------------------------------------------------- # PARSERS: EBS - Snapshots #----------------------------------------------------------------- class QEc2DescribeSnapshotsParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{CreateSnapshotResponse$}, %r{/snapshotSet/item$} then @item = { :tags => {} } when %r{/tagSet/item$} then @aws_tag = {} end end def tagend(name) case name when 'volumeId' then @item[:aws_volume_id] = @text when 'snapshotId' then @item[:aws_id] = @text when 'status' then @item[:aws_status] = @text when 'startTime' then @item[:aws_started_at] = @text when 'progress' then @item[:aws_progress] = @text when 'description' then @item[:aws_description] = @text when 'ownerId' then @item[:aws_owner] = @text when 'volumeSize' then @item[:aws_volume_size] = @text.to_i when 'ownerAlias' then @item[:owner_alias] = @text else case full_tag_name when %r{/tagSet/item/key$} then @aws_tag[:key] = @text when %r{/tagSet/item/value$} then @aws_tag[:value] = @text when %r{/tagSet/item$} then @item[:tags][@aws_tag[:key]] = @aws_tag[:value] when %r{CreateSnapshotResponse$}, %r{/snapshotSet/item$} then @result << @item end end end def reset @result = [] end end class QEc2DescribeSnapshotAttributeParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case name when 'createVolumePermission' then @result[:create_volume_permission] = { :groups => [], :users => [] } end end def tagend(name) case full_tag_name when "#{@create_volume_permission}/group" then @result[:create_volume_permission][:groups] << @text when "#{@create_volume_permission}/userId" then @result[:create_volume_permission][:users] << @text end end def reset @create_volume_permission = "DescribeSnapshotAttributeResponse/createVolumePermission/item" @result = {} end end end end ================================================ FILE: lib/ec2/right_ec2_images.rb ================================================ # # Copyright (c) 2009 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws class Ec2 #----------------------------------------------------------------- # Images #----------------------------------------------------------------- # Describe images helper # params: # { 'ImageId' => ['id1', ..., 'idN'], # 'Owner' => ['self', ..., 'userN'], # 'ExecutableBy' => ['self', 'all', ..., 'userN'] # } def ec2_describe_images(params={}, options={}, cache_for=nil) #:nodoc: request_hash = {} params.each { |list_by, list| request_hash.merge! amazonize_list(list_by, Array(list)) } request_hash.merge!(amazonize_list(['Filter.?.Name', 'Filter.?.Value.?'], options[:filters])) unless options[:filters].right_blank? link = generate_request("DescribeImages", request_hash) request_cache_or_info cache_for, link, QEc2DescribeImagesParser, @@bench, cache_for rescue Exception on_exception end # Retrieve a list of images. # # Accepts a list of images and/or a set of filters as the last parameter. # # Filters: architecture, block-device-mapping.delete-on-termination block-device-mapping.device-name, # block-device-mapping.snapshot-id, block-device-mapping.volume-size, description, image-id, image-type, # is-public, kernel-id, manifest-location, name, owner-alias, owner-id, platform, product-code, # ramdisk-id, root-device-name, root-device-type, state, state-reason-code, state-reason-message, # tag-key, tag-value, tag:key, virtualization-type # # ec2.describe_images #=> # [{:description=>"EBS backed Fedora core 8 i386", # :aws_architecture=>"i386", # :aws_id=>"ami-c2a3f5d4", # :aws_image_type=>"machine", # :root_device_name=>"/dev/sda1", # :image_class=>"elastic", # :aws_owner=>"937766719418", # :aws_location=>"937766719418/EBS backed FC8 i386", # :aws_state=>"available", # :block_device_mappings=> # [{:ebs_snapshot_id=>"snap-829a20eb", # :ebs_delete_on_termination=>true, # :device_name=>"/dev/sda1"}], # :name=>"EBS backed FC8 i386", # :aws_is_public=>true}, ... ] # # ec2.describe_images(:filters => { 'image-type' => 'kernel', 'state' => 'available', 'tag:MyTag' => 'MyValue'}) # # ec2.describe_images("ari-fda54b94", "ami-2ee80247", "aki-00896a69", # :filters => { 'image-type' => 'kernel', 'state' => 'available' }) #=> # [{:root_device_type=>"instance-store", # :aws_id=>"aki-00896a69", # :aws_image_type=>"kernel", # :aws_location=> # "karmic-kernel-zul/ubuntu-kernel-2.6.31-300-ec2-i386-20091002-test-04.manifest.xml", # :virtualization_type=>"paravirtual", # :aws_state=>"available", # :aws_owner=>"099720109477", # :tags=>{}, # :aws_is_public=>true, # :aws_architecture=>"i386"}] # # P.S. filters: http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-DescribeImages.html # def describe_images(*list_and_options) list, options = AwsUtils::split_items_and_params(list_and_options) cache_for = (list.right_blank? && options[:filters].right_blank?) ? :describe_images : nil ec2_describe_images( {'ImageId'=>list}, options, cache_for) end # Retrieve a list of images by image owner. # # Accepts a list of images and/or a set of filters as the last parameter. # # Filters: architecture, block-device-mapping.delete-on-termination block-device-mapping.device-name, # block-device-mapping.snapshot-id, block-device-mapping.volume-size, description, image-id, image-type, # is-public, kernel-id, manifest-location, name, owner-alias, owner-id, platform, product-code, # ramdisk-id, root-device-name, root-device-type, state, state-reason-code, state-reason-message, # tag-key, tag-value, tag:key, virtualization-type # # ec2.describe_images_by_owner('522821470517') # ec2.describe_images_by_owner('self', :filters => { 'block-device-mapping.delete-on-termination' => 'false' }) # # P.S. filters: http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-DescribeImages.html # def describe_images_by_owner(*list_and_options) list, options = AwsUtils::split_items_and_params(list_and_options) list = ['self'] if list.right_blank? cache_for = (list==['self'] && options[:filters].right_blank?) ? :describe_images_by_owner : nil ec2_describe_images( {'Owner'=>list}, options, cache_for) end # Retrieve a list of images by image executable by. # # Accepts a list of images and/or a set of filters as the last parameter. # # Filters: architecture, block-device-mapping.delete-on-termination block-device-mapping.device-name, # block-device-mapping.snapshot-id, block-device-mapping.volume-size, description, image-id, image-type, # is-public, kernel-id, manifest-location, name, owner-alias, owner-id, platform, product-code, # ramdisk-id, root-device-name, root-device-type, state, state-reason-code, state-reason-message, # tag-key, tag-value, tag:key, virtualization-type # # ec2.describe_images_by_executable_by('522821470517') # ec2.describe_images_by_executable_by('self') # ec2.describe_images_by_executable_by('all', :filters => { 'architecture' => 'i386' }) # # P.S. filters: http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-DescribeImages.html # def describe_images_by_executable_by(*list_and_options) list, options = AwsUtils::split_items_and_params(list_and_options) list = ['self'] if list.right_blank? cache_for = (list==['self'] && options[:filters].right_blank?) ? :describe_images_by_executable_by : nil ec2_describe_images( {'ExecutableBy'=>list}, options, cache_for) end # Register new image at Amazon. # Options: :image_location, :name, :description, :architecture, :kernel_id, :ramdisk_id, # :root_device_name, :block_device_mappings, :virtualizationt_type(hvm|paravirtual) # # Returns new image id. # # # Register S3 image # ec2.register_image('bucket_for_k_dzreyev/image_bundles/kd__CentOS_1_10_2009_10_21_13_30_43_MSD/image.manifest.xml') #=> 'ami-e444444d' # # # or # image_reg_params = { :image_location => 'bucket_for_k_dzreyev/image_bundles/kd__CentOS_1_10_2009_10_21_13_30_43_MSD/image.manifest.xml', # :name => 'my-test-one-1', # :description => 'My first test image' } # ec2.register_image(image_reg_params) #=> "ami-bca1f7aa" # # # Register EBS image # image_reg_params = { :name => 'my-test-image', # :description => 'My first test image', # :root_device_name => "/dev/sda1", # :block_device_mappings => [ { :ebs_snapshot_id=>"snap-7360871a", # :ebs_delete_on_termination=>true, # :device_name=>"/dev/sda1"}, # { :virtual_name => 'ephemeral0',} # :device_name=>"/dev/sdb"} ] # ec2.register_image(image_reg_params) #=> "ami-b2a1f7a4" # def register_image(options) case when options.is_a?(String) options = { :image_location => options } when !options.is_a?(Hash) raise "Unsupported options type" end params = {} params['ImageLocation'] = options[:image_location] if options[:image_location] params['Name'] = options[:name] if options[:name] params['Description'] = options[:description] if options[:description] params['Architecture'] = options[:architecture] if options[:architecture] params['KernelId'] = options[:kernel_id] if options[:kernel_id] params['RamdiskId'] = options[:ramdisk_id] if options[:ramdisk_id] params['RootDeviceName'] = options[:root_device_name] if options[:root_device_name] params['VirtualizationType'] = options[:virtualization_type] if options[:virtualization_type] # params['SnapshotId'] = options[:snapshot_id] if options[:snapshot_id] params.merge!(amazonize_block_device_mappings(options[:block_device_mappings])) link = generate_request("RegisterImage", params) request_info(link, QEc2RegisterImageParser.new(:logger => @logger)) rescue Exception on_exception end # Deregister image at Amazon. Returns +true+ or an exception. # # ec2.deregister_image('ami-e444444d') #=> true # def deregister_image(image_id) link = generate_request("DeregisterImage", 'ImageId' => image_id.to_s) request_info(link, RightBoolResponseParser.new(:logger => @logger)) rescue Exception on_exception end # Describe image attributes. # # Returns: String (or nil) for 'description', 'kernel', 'ramdisk'; Hash for 'launchPermission'; Array for 'productCodes', 'blockDeviceMapping' # # ec2.describe_image_attribute('ami-00000000', 'description') #=> 'My cool Image' # ec2.describe_image_attribute('ami-00000000', 'launchPermission') #=> {:user_ids=>["443739700000", "115864000000", "309179000000", "857501300000"]} # ec2.describe_image_attribute('ami-00000000', 'productCodes') #=> ["8ED10000"] # ec2.describe_image_attribute('ami-00000000', 'kernel') #=> "aki-9b00e5f2" # ec2.describe_image_attribute('ami-00000000', 'ramdisk') #=> nil # ec2.describe_image_attribute('ami-00000000', 'blockDeviceMapping') #=> [{:device_name=>"sda2", :virtual_name=>"ephemeral0"}, # {:device_name=>"sda1", :virtual_name=>"ami"}, # {:device_name=>"/dev/sda1", :virtual_name=>"root"}, # {:device_name=>"sda3", :virtual_name=>"swap"}] # def describe_image_attribute(image_id, attribute='launchPermission') link = generate_request("DescribeImageAttribute", 'ImageId' => image_id, 'Attribute' => attribute) request_info(link, QEc2DescribeImageAttributeParser.new(:logger => @logger)) rescue Exception on_exception end # Reset image attribute. Currently, only 'launchPermission' is supported. Returns +true+ or an exception. # # ec2.reset_image_attribute('ami-e444444d') #=> true # def reset_image_attribute(image_id, attribute='launchPermission') link = generate_request("ResetImageAttribute", 'ImageId' => image_id, 'Attribute' => attribute) request_info(link, RightBoolResponseParser.new(:logger => @logger)) rescue Exception on_exception end # Modify an image's attributes. It is recommended that you use # modify_image_launch_perm_add_users, modify_image_launch_perm_remove_users, etc. # instead of modify_image_attribute because the signature of # modify_image_attribute may change with EC2 service changes. # # Attribute can take next values: 'launchPermission', 'productCode', 'description'. # Value is a String for'description'. is a String or an Array for 'productCode' and # is a Hash {:add_user_ids, :add_groups, :remove_user_ids, :remove_groups } for 'launchPermission'. # def modify_image_attribute(image_id, attribute, value) params = { 'ImageId' => image_id } case attribute.to_s when 'launchPermission' params.update(amazonize_list('LaunchPermission.Add.?.UserId', value[:add_user_ids])) params.update(amazonize_list('LaunchPermission.Add.?.Group', value[:add_groups])) params.update(amazonize_list('LaunchPermission.Remove.?.UserId', value[:remove_user_ids])) params.update(amazonize_list('LaunchPermission.Remove.?.Group', value[:remove_groups])) when 'productCode' params.update(amazonize_list('ProductCode', value)) when 'description' params['Description.Value'] = value end link = generate_request("ModifyImageAttribute", params) request_info(link, RightBoolResponseParser.new(:logger => @logger)) rescue Exception on_exception end # Grant image launch permissions to users. # Parameter +user_id+ is a list of user AWS account ids. # Returns +true+ or an exception. # # ec2.modify_image_launch_perm_add_users('ami-e444444d',['000000000777','000000000778']) #=> true def modify_image_launch_perm_add_users(image_id, *user_ids) modify_image_attribute(image_id, 'launchPermission', :add_user_ids => user_ids.flatten) end # Revokes image launch permissions for users. +user_id+ is a list of users AWS accounts ids. Returns +true+ or an exception. # # ec2.modify_image_launch_perm_remove_users('ami-e444444d',['000000000777','000000000778']) #=> true # def modify_image_launch_perm_remove_users(image_id, *user_ids) modify_image_attribute(image_id, 'launchPermission', :remove_user_ids => user_ids.flatten) end # Add image launch permissions for users groups (currently only 'all' is supported, which gives public launch permissions). # Returns +true+ or an exception. # # ec2.modify_image_launch_perm_add_groups('ami-e444444d') #=> true # def modify_image_launch_perm_add_groups(image_id, *groups) modify_image_attribute(image_id, 'launchPermission', :add_groups => groups.flatten) end # Remove image launch permissions for users groups (currently only 'all' is supported, which gives public launch permissions). # # ec2.modify_image_launch_perm_remove_groups('ami-e444444d') #=> true # def modify_image_launch_perm_remove_groups(image_id, *groups) modify_image_attribute(image_id, 'launchPermission', :remove_groups => groups.flatten) end # Add product code to image # # ec2.modify_image_product_code('ami-e444444d','0ABCDEF') #=> true # def modify_image_product_code(image_id, product_codes=[]) modify_image_attribute(image_id, 'productCodes',product_codes) end # Modify image description # # ec2.modify_image_product_code('ami-e444444d','My cool image') #=> true # def modify_image_description(image_id, description) modify_image_attribute(image_id, 'description', description) end # Create a new image. # Options: :name, :description, :no_reboot(bool) # # ec2.create_image(instance, :description => 'KD: test#1', # :no_reboot => true, # :name => 'kd-1' ) #=> "ami-84a1f792" # def create_image(instance_aws_id, options={}) params = { 'InstanceId' => instance_aws_id } params['Name'] = options[:name] unless options[:name].right_blank? params['Description'] = options[:description] unless options[:description].right_blank? params['NoReboot'] = options[:no_reboot].to_s unless options[:no_reboot].nil? link = generate_request("CreateImage", params) request_info(link, QEc2RegisterImageParser.new(:logger => @logger)) end #----------------------------------------------------------------- # PARSERS: Images #----------------------------------------------------------------- class QEc2DescribeImagesParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{/imagesSet/item$} @item = { :tags => {} } when %r{/blockDeviceMapping/item$} @item[:block_device_mappings] ||= [] @block_device_mapping = {} when %r{/tagSet/item$} @aws_tag = {} end end def tagend(name) case name when 'imageId' then @item[:aws_id] = @text when 'imageLocation' then @item[:aws_location] = @text when 'imageState' then @item[:aws_state] = @text when 'imageOwnerId' then @item[:aws_owner] = @text when 'isPublic' then @item[:aws_is_public] = @text == 'true' ? true : false when 'productCode' then (@item[:aws_product_codes] ||= []) << @text when 'architecture' then @item[:aws_architecture] = @text when 'imageType' then @item[:aws_image_type] = @text when 'kernelId' then @item[:aws_kernel_id] = @text when 'ramdiskId' then @item[:aws_ramdisk_id] = @text when 'platform' then @item[:aws_platform] = @text when 'imageOwnerAlias' then @item[:image_owner_alias] = @text when 'name' then @item[:name] = @text when 'description' then @item[:description] = @text when 'rootDeviceType' then @item[:root_device_type] = @text when 'rootDeviceName' then @item[:root_device_name] = @text when 'imageClass' then @item[:image_class] = @text when 'virtualizationType' then @item[:virtualization_type] = @text when 'hypervisor' then @item [:hypervisor] = @text else case full_tag_name when %r{/stateReason/code$} then @item[:state_reason_code] = @text.to_i when %r{/stateReason/message$} then @item[:state_reason_message] = @text when %r{/blockDeviceMapping/item} # no trailing $ case name when 'deviceName' then @block_device_mapping[:device_name] = @text when 'virtualName' then @block_device_mapping[:virtual_name] = @text when 'volumeSize' then @block_device_mapping[:ebs_volume_size] = @text.to_i when 'snapshotId' then @block_device_mapping[:ebs_snapshot_id] = @text when 'deleteOnTermination' then @block_device_mapping[:ebs_delete_on_termination] = @text == 'true' ? true : false when 'item' then @item[:block_device_mappings] << @block_device_mapping end when %r{/tagSet/item/key$} then @aws_tag[:key] = @text when %r{/tagSet/item/value$} then @aws_tag[:value] = @text when %r{/tagSet/item$} then @item[:tags][@aws_tag[:key]] = @aws_tag[:value] when %r{/imagesSet/item$} then @result << @item end end end def reset @result = [] end end class QEc2RegisterImageParser < RightAWSParser #:nodoc: def tagend(name) @result = @text if name == 'imageId' end end #----------------------------------------------------------------- # PARSERS: Image Attribute #----------------------------------------------------------------- class QEc2DescribeImageAttributeParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{launchPermission$} then @result = {} when %r{productCodes$} then @result = [] when %r{blockDeviceMapping$} then @result = [] when %r{blockDeviceMapping/item$} then @block_device_mapping = {} end end def tagend(name) case full_tag_name when %r{/kernel/value$} then @result = @text when %r{/ramdisk/value$} then @result = @text when %r{/description/value$} then @result = @text when %r{/productCode$} then @result << @text when %r{launchPermission/item/group$} then (@result[:groups] ||=[]) << @text when %r{launchPermission/item/userId$} then (@result[:user_ids]||=[]) << @text when %r{/blockDeviceMapping/item} # no trailing $ case name when 'deviceName' then @block_device_mapping[:device_name] = @text when 'virtualName' then @block_device_mapping[:virtual_name] = @text when 'noDevice' then @block_device_mapping[:no_device] = @text when 'snapshotId' then @block_device_mapping[:ebs_snapshot_id] = @text when 'volumeSize' then @block_device_mapping[:ebs_volume_size] = @text when 'deleteOnTermination' then @block_device_mapping[:ebs_delete_on_termination] = @text == 'true' ? true : false when 'item' then @result << @block_device_mapping end end end def reset @result = nil end end end end ================================================ FILE: lib/ec2/right_ec2_instances.rb ================================================ # # Copyright (c) 2009 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws class Ec2 INSTANCE_API_VERSION = (API_VERSION > '2012-07-20') ? API_VERSION : '2012-07-20' #----------------------------------------------------------------- # Instances #----------------------------------------------------------------- def get_desc_instances(instances) # :nodoc: result = [] instances.each do |reservation| reservation[:instances_set].each do |instance| # Parse and remove timestamp from the reason string. The timestamp is of # the request, not when EC2 took action, thus confusing & useless... instance[:aws_reason] = instance[:aws_reason].sub(/\(\d[^)]*GMT\) */, '') instance[:aws_owner] = reservation[:aws_owner] instance[:aws_reservation_id] = reservation[:aws_reservation_id] # Security Groups instance[:groups] = instance[:groups].right_blank? ? reservation[:aws_groups] : instance[:groups] result << instance end end result rescue Exception on_exception end # Retrieve information about EC2 instances. # # Accepts a list of instances and/or a set of filters as the last parameter. # # Filters: architecture, availability-zone, block-device-mapping.attach-time, block-device-mapping.delete-on-termination, # block-device-mapping.device-name, block-device-mapping.status, block-device-mapping.volume-id, client-token, dns-name, # group-id, image-id, instance-id, instance-lifecycle, instance-state-code, instance-state-name, instance-type, ip-address, # kernel-id, key-name, launch-index, launch-time, monitoring-state, owner-id, placement-group-name, platform, # private-dns-name, private-ip-address, product-code, ramdisk-id, reason, requester-id, reservation-id, root-device-name, # root-device-type, spot-instance-request-id, state-reason-code, state-reason-message, subnet-id, tag-key, tag-value, # tag:key, virtualization-type, vpc-id, # # # ec2.describe_instances #=> # [{:source_dest_check=>true, # :subnet_id=>"subnet-da6cf9b3", # :aws_kernel_id=>"aki-3932d150", # :ami_launch_index=>"0", # :tags=>{}, # :aws_reservation_id=>"r-7cd25c11", # :aws_owner=>"826693181925", # :state_reason_code=>"Client.UserInitiatedShutdown", # :aws_instance_id=>"i-2d898e41", # :hypervisor=>"xen", # :root_device_name=>"/dev/sda1", # :aws_ramdisk_id=>"ari-c515f6ac", # :aws_instance_type=>"m1.large", # :groups=>[{:group_name=>"2009-07-15-default", :group_id=>"sg-90c5d6fc"}], # :block_device_mappings=> # [{:device_name=>"/dev/sda1", # :ebs_status=>"attached", # :ebs_attach_time=>"2011-03-04T18:51:58.000Z", # :ebs_delete_on_termination=>true, # :ebs_volume_id=>"vol-38f2bd50"}], # :state_reason_message=> # "Client.UserInitiatedShutdown: User initiated shutdown", # :aws_image_id=>"ami-a3638cca", # :virtualization_type=>"paravirtual", # :aws_launch_time=>"2011-03-04T18:13:59.000Z", # :private_dns_name=>"", # :aws_product_codes=>[], # :aws_availability_zone=>"us-east-1a", # :aws_state_code=>80, # :architecture=>"x86_64", # :dns_name=>"", # :client_token=>"1299262447-684266-NNgyH-ouPTI-MzG6h-5AIRk", # :root_device_type=>"ebs", # :vpc_id=>"vpc-e16cf988", # :monitoring_state=>"disabled", # :ssh_key_name=>"default", # :private_ip_address=>"192.168.0.52", # :aws_reason=>"User initiated ", # :aws_state=>"stopped"}, ...] # # ec2.describe_instances("i-8ce84ae6", "i-8ce84ae8", "i-8ce84ae0") # ec2.describe_instances(:filters => { 'availability-zone' => 'us-east-1a', 'instance-type' => 'c1.medium' }) # # P.S. filters: http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-DescribeInstances.html # def describe_instances(*list_and_options) list_and_options = merge_new_options_into_list_and_options(list_and_options, :options => {:api_version => INSTANCE_API_VERSION}) describe_resources_with_list_and_options('DescribeInstances', 'InstanceId', QEc2DescribeInstancesParser, list_and_options) do |parser| get_desc_instances(parser.result) end end # Return the product code attached to instance or +nil+ otherwise. # # ec2.confirm_product_instance('ami-e444444d','12345678') #=> nil # ec2.confirm_product_instance('ami-e444444d','00001111') #=> "000000000888" # def confirm_product_instance(instance, product_code) link = generate_request("ConfirmProductInstance", { 'ProductCode' => product_code, 'InstanceId' => instance }) request_info(link, QEc2ConfirmProductInstanceParser.new(:logger => @logger)) end # Launch new EC2 instances. Returns a list of launched instances or an exception. # # ec2.run_instances('ami-e444444d',1,1,['2009-07-15-default'],'my_awesome_key', 'Woohoo!!!', 'public') #=> # [{:aws_image_id => "ami-e444444d", # :aws_reason => "", # :aws_state_code => "0", # :aws_owner => "000000000888", # :aws_instance_id => "i-123f1234", # :aws_reservation_id => "r-aabbccdd", # :aws_state => "pending", # :dns_name => "", # :ssh_key_name => "my_awesome_key", # :groups => [{:group_name=>"2009-07-15-default", :group_id=>"sg-90c5d6fc"}], # :private_dns_name => "", # :aws_instance_type => "m1.small", # :aws_launch_time => "2008-1-1T00:00:00.000Z" # :aws_ramdisk_id => "ari-8605e0ef" # :aws_kernel_id => "aki-9905e0f0", # :ami_launch_index => "0", # :aws_availability_zone => "us-east-1b" # }] # def run_instances(image_id, min_count, max_count, group_names, key_name, user_data='', addressing_type = nil, instance_type = nil, kernel_id = nil, ramdisk_id = nil, availability_zone = nil, monitoring_enabled = nil, subnet_id = nil, disable_api_termination = nil, instance_initiated_shutdown_behavior = nil, block_device_mappings = nil, placement_group_name = nil, client_token = nil) launch_instances(image_id, { :min_count => min_count, :max_count => max_count, :user_data => user_data, :group_names => group_names, :key_name => key_name, :instance_type => instance_type, :kernel_id => kernel_id, :ramdisk_id => ramdisk_id, :availability_zone => availability_zone, :monitoring_enabled => monitoring_enabled, :subnet_id => subnet_id, :disable_api_termination => disable_api_termination, :instance_initiated_shutdown_behavior => instance_initiated_shutdown_behavior, :block_device_mappings => block_device_mappings, :placement_group_name => placement_group_name, :client_token => client_token }) end # Launch new EC2 instances. # # Options: :image_id, :min_count, max_count, :key_name, :kernel_id, :ramdisk_id, # :availability_zone, :monitoring_enabled, :subnet_id, :disable_api_termination, :instance_initiated_shutdown_behavior, # :block_device_mappings, :placement_group_name, :license_pool, :group_ids, :group_names, :private_ip_address, # :ebs_optimized # # Returns a list of launched instances or an exception. # # ec2.launch_instances( "ami-78779511", # :min_count => 1, # :group_names => ["default", "eugeg223123123"], # :user_data => 'Ohoho!', # :availability_zone => "us-east-1a", # :disable_api_termination => false, # :instance_initiated_shutdown_behavior => 'terminate', # :block_device_mappings => [ {:ebs_snapshot_id=>"snap-e40fd188", # :ebs_delete_on_termination=>true, # :device_name => "/dev/sdk", # :virtual_name => "mystorage"} ] ) #=> # [{:hypervisor=>"xen", # :private_dns_name=>"", # :client_token=>"1309532374-551037-gcsBj-gEypk-piG06-ODfQm", # :monitoring_state=>"disabled", # :aws_availability_zone=>"us-east-1a", # :root_device_name=>"/dev/sda1", # :state_reason_code=>"pending", # :dns_name=>"", # :tags=>{}, # :aws_reason=>"", # :virtualization_type=>"paravirtual", # :state_reason_message=>"pending", # :aws_reservation_id=>"r-6fada703", # :aws_ramdisk_id=>"ari-a51cf9cc", # :ami_launch_index=>"0", # :groups=> # [{:group_id=>"sg-a0b85dc9", :group_name=>"default"}, # {:group_id=>"sg-70733019", :group_name=>"eugeg223123123"}], # :aws_owner=>"826693181925", # :aws_instance_type=>"m1.small", # :aws_state=>"pending", # :root_device_type=>"ebs", # :aws_image_id=>"ami-78779511", # :aws_kernel_id=>"aki-a71cf9ce", # :aws_launch_time=>"2011-07-01T14:59:35.000Z", # :aws_state_code=>0, # :aws_instance_id=>"i-4f202621", # :aws_product_codes=>[]}] # def launch_instances(image_id, options={}) options[:user_data] = options[:user_data].to_s params = map_api_keys_and_values( options, :key_name, :kernel_id, :ramdisk_id, :subnet_id, :instance_initiated_shutdown_behavior, :private_ip_address, :additional_info, :license_pool, :ebs_optimized, :image_id => { :value => image_id }, :min_count => { :value => options[:min_count] || 1 }, :max_count => { :value => options[:max_count] || options[:min_count] || 1 }, :placement_tenancy => 'Placement.Tenancy', :placement_group_name => 'Placement.GroupName', :availability_zone => 'Placement.AvailabilityZone', :group_names => { :amazonize_list => 'SecurityGroup' }, :group_ids => { :amazonize_list => 'SecurityGroupId' }, :block_device_mappings => { :amazonize_bdm => 'BlockDeviceMapping' }, :instance_type => { :value => options[:instance_type] || DEFAULT_INSTANCE_TYPE }, :disable_api_termination => { :value => Proc.new{ !options[:disable_api_termination].nil? && options[:disable_api_termination].to_s }}, :client_token => { :value => !@params[:eucalyptus] && (options[:client_token] || AwsUtils::generate_unique_token)}, :user_data => { :value => Proc.new { !options[:user_data].empty? && Base64.encode64(options[:user_data]).delete("\n") }}, :monitoring_enabled => { :name => 'Monitoring.Enabled', :value => Proc.new{ options[:monitoring_enabled] && options[:monitoring_enabled].to_s }}) # Log debug information @logger.info("Launching instance of image #{image_id}. Options: #{params.inspect}") # Add IOPS support (default behavior) but skip it when an old API version call is requested options[:options] ||= {} options[:options][:api_version] ||= INSTANCE_API_VERSION params.delete("EbsOptimized") if options[:options][:api_version] < INSTANCE_API_VERSION # link = generate_request("RunInstances", params, options[:options]) instances = request_info(link, QEc2DescribeInstancesParser.new(:logger => @logger)) get_desc_instances(instances) rescue Exception on_exception end # Start instances. # # ec2.start_instances("i-36e84a5e") #=> # [{:aws_prev_state_name=>"stopped", # :aws_instance_id=>"i-36e84a5e", # :aws_current_state_code=>16, # :aws_current_state_name=>"running", # :aws_prev_state_code=>80}] # def start_instances(*instance_aws_ids) instance_aws_ids = instance_aws_ids.flatten link = generate_request("StartInstances", amazonize_list('InstanceId', instance_aws_ids)) request_info(link, QEc2TerminateInstancesParser.new(:logger => @logger)) end # Stop instances. # # Options: :force => true|false # # ec2.stop_instances("i-36e84a5e") #=> # [{:aws_prev_state_code=>16, # :aws_prev_state_name=>"running", # :aws_instance_id=>"i-36e84a5e", # :aws_current_state_code=>64, # :aws_current_state_name=>"stopping"}] # def stop_instances(*instance_aws_ids_and_options) list, options = AwsUtils::split_items_and_params(instance_aws_ids_and_options) request_hash = {} request_hash['Force'] = true if options[:force] request_hash.merge!(amazonize_list('InstanceId', list)) link = generate_request("StopInstances", request_hash) request_info(link, QEc2TerminateInstancesParser.new(:logger => @logger)) end # Terminates EC2 instances. Returns a list of termination params or an exception. # # ec2.terminate_instances(['i-cceb49a4']) #=> # [{:aws_instance_id=>"i-cceb49a4", # :aws_current_state_code=>32, # :aws_current_state_name=>"shutting-down", # :aws_prev_state_code=>16, # :aws_prev_state_name=>"running"}] # def terminate_instances(*instance_aws_ids) instance_aws_ids = instance_aws_ids.flatten link = generate_request("TerminateInstances", amazonize_list('InstanceId', instance_aws_ids)) request_info(link, QEc2TerminateInstancesParser.new(:logger => @logger)) rescue Exception on_exception end # Retreive EC2 instance OS logs. Returns a hash of data or an exception. # # ec2.get_console_output('i-f222222d') => # {:aws_instance_id => 'i-f222222d', # :aws_timestamp => "2007-05-23T14:36:07.000-07:00", # :timestamp => Wed May 23 21:36:07 UTC 2007, # Time instance # :aws_output => "Linux version 2.6.16-xenU (builder@patchbat.amazonsa) (gcc version 4.0.1 20050727 ..." def get_console_output(instance_id) link = generate_request("GetConsoleOutput", { 'InstanceId.1' => instance_id }) request_info(link, QEc2GetConsoleOutputParser.new(:logger => @logger)) rescue Exception on_exception end # Reboot an EC2 instance. Returns +true+ or an exception. # # ec2.reboot_instances(['i-f222222d','i-f222222e']) #=> true # def reboot_instances(*instances) instances = instances.flatten link = generate_request("RebootInstances", amazonize_list('InstanceId', instances)) request_info(link, RightBoolResponseParser.new(:logger => @logger)) rescue Exception on_exception end # Describe instance attribute. # # Attributes: 'instanceType', 'kernel', 'ramdisk', 'userData', 'rootDeviceName', 'disableApiTermination', # 'instanceInitiatedShutdownBehavior', 'sourceDestCheck', 'blockDeviceMapping', 'groupSet' # # ec2.describe_instance_attribute(instance, "blockDeviceMapping") #=> # [{:ebs_delete_on_termination=>true, # :ebs_volume_id=>"vol-683dc401", # :device_name=>"/dev/sda1"}] # # ec2.describe_instance_attribute(instance, "instanceType") #=> "m1.small" # # ec2.describe_instance_attribute(instance, "instanceInitiatedShutdownBehavior") #=> "stop" # def describe_instance_attribute(instance_id, attribute) link = generate_request('DescribeInstanceAttribute', 'InstanceId' => instance_id, 'Attribute' => attribute) value = request_info(link, QEc2DescribeInstanceAttributeParser.new(:logger => @logger)) value = Base64.decode64(value) if attribute == "userData" && !value.right_blank? value rescue Exception on_exception end # Describe instance attribute. # # Attributes: 'kernel', 'ramdisk', 'sourceDestCheck' # # ec2.reset_instance_attribute(instance, 'kernel') #=> true # def reset_instance_attribute(instance_id, attribute) link = generate_request('ResetInstanceAttribute', 'InstanceId' => instance_id, 'Attribute' => attribute ) request_info(link, RightBoolResponseParser.new(:logger => @logger)) rescue Exception on_exception end # Modify instance attribute. # # Attributes: 'InstanceType', 'Kernel', 'Ramdisk', 'UserData', 'DisableApiTermination', # 'InstanceInitiatedShutdownBehavior', 'SourceDestCheck', 'GroupId' # # ec2.modify_instance_attribute(instance, 'instanceInitiatedShutdownBehavior", "stop") #=> true # def modify_instance_attribute(instance_id, attribute, value) request_hash = {'InstanceId' => instance_id} attribute = attribute.to_s.right_underscore.right_camelize case attribute when 'UserData' then request_hash["#{attribute}.Value"] = Base64.encode64(value).delete("\n") when 'GroupId' then request_hash.merge!(amazonize_list('GroupId', value)) else request_hash["#{attribute}.Value"] = value end link = generate_request('ModifyInstanceAttribute', request_hash, :api_version => INSTANCE_API_VERSION) request_info(link, RightBoolResponseParser.new(:logger => @logger)) rescue Exception on_exception end #----------------------------------------------------------------- # Instances: Windows addons #----------------------------------------------------------------- # Get initial Windows Server setup password from an instance console output. # # my_awesome_key = ec2.create_key_pair('my_awesome_key') #=> # {:aws_key_name => "my_awesome_key", # :aws_fingerprint => "01:02:03:f4:25:e6:97:e8:9b:02:1a:26:32:4e:58:6b:7a:8c:9f:03", # :aws_material => "-----BEGIN RSA PRIVATE KEY-----\nMIIEpQIBAAK...Q8MDrCbuQ=\n-----END RSA PRIVATE KEY-----"} # # my_awesome_instance = ec2.run_instances('ami-a000000a',1,1,['my_awesome_group'],'my_awesome_key', 'WindowsInstance!!!') #=> # [{:aws_image_id => "ami-a000000a", # :aws_instance_id => "i-12345678", # ... # :aws_availability_zone => "us-east-1b" # }] # # # wait until instance enters 'operational' state and get it's initial password # # puts ec2.get_initial_password(my_awesome_instance[:aws_instance_id], my_awesome_key[:aws_material]) #=> "MhjWcgZuY6" # def get_initial_password(instance_id, private_key) console_output = get_console_output(instance_id) crypted_password = console_output[:aws_output][%r{(.+)}m] && $1 unless crypted_password raise AwsError.new("Initial password was not found in console output for #{instance_id}") else OpenSSL::PKey::RSA.new(private_key).private_decrypt(Base64.decode64(crypted_password)) end rescue Exception on_exception end # Get Initial windows instance password using Amazon API call GetPasswordData. # # puts ec2.get_initial_password_v2(my_awesome_instance[:aws_instance_id], my_awesome_key[:aws_material]) #=> "MhjWcgZuY6" # # P.S. To say the truth there is absolutely no any speedup if to compare to the old get_initial_password method... ;( # def get_initial_password_v2(instance_id, private_key) link = generate_request('GetPasswordData', 'InstanceId' => instance_id ) response = request_info(link, QEc2GetPasswordDataParser.new(:logger => @logger)) if response[:password_data].right_blank? raise AwsError.new("Initial password is not yet created for #{instance_id}") else OpenSSL::PKey::RSA.new(private_key).private_decrypt(Base64.decode64(response[:password_data])) end rescue Exception on_exception end # Bundle a Windows image. # Internally, it queues the bundling task and shuts down the instance. # It then takes a snapshot of the Windows volume bundles it, and uploads it to # S3. After bundling completes, Rightaws::Ec2#register_image may be used to # register the new Windows AMI for subsequent launches. # # ec2.bundle_instance('i-e3e24e8a', 'my-awesome-bucket', 'my-win-image-1') #=> # [{:aws_update_time => "2008-10-16T13:58:25.000Z", # :s3_bucket => "kd-win-1", # :s3_prefix => "win2pr", # :aws_state => "pending", # :aws_id => "bun-26a7424f", # :aws_instance_id => "i-878a25ee", # :aws_start_time => "2008-10-16T13:58:02.000Z"}] # def bundle_instance(instance_id, s3_bucket, s3_prefix, s3_owner_aws_access_key_id=nil, s3_owner_aws_secret_access_key=nil, s3_expires = S3Interface::DEFAULT_EXPIRES_AFTER, s3_upload_policy='ec2-bundle-read') # S3 access and signatures s3_owner_aws_access_key_id ||= @aws_access_key_id s3_owner_aws_secret_access_key ||= @aws_secret_access_key s3_expires = Time.now.utc + s3_expires if s3_expires.is_a?(Fixnum) && (s3_expires < S3Interface::ONE_YEAR_IN_SECONDS) # policy policy = { 'expiration' => AwsUtils::utc_iso8601(s3_expires), 'conditions' => [ { 'bucket' => s3_bucket }, { 'acl' => s3_upload_policy }, [ 'starts-with', '$key', s3_prefix ] ] }.to_json policy64 = Base64.encode64(policy).gsub("\n","") signed_policy64 = AwsUtils.sign(s3_owner_aws_secret_access_key, policy64) # fill request params params = { 'InstanceId' => instance_id, 'Storage.S3.AWSAccessKeyId' => s3_owner_aws_access_key_id, 'Storage.S3.UploadPolicy' => policy64, 'Storage.S3.UploadPolicySignature' => signed_policy64, 'Storage.S3.Bucket' => s3_bucket, 'Storage.S3.Prefix' => s3_prefix, } link = generate_request("BundleInstance", params) request_info(link, QEc2BundleInstanceParser.new) rescue Exception on_exception end # Describe the status of the Windows AMI bundlings. # # Accepts a list of tasks and/or a set of filters as the last parameter. # # Filters" bundle-id, error-code, error-message, instance-id, progress, s3-aws-access-key-id, s3-bucket, s3-prefix, # start-time, state, update-time # # ec2.describe_bundle_tasks('bun-4fa74226') #=> # [{:s3_bucket => "my-awesome-bucket" # :aws_id => "bun-0fa70206", # :s3_prefix => "win1pr", # :aws_start_time => "2008-10-14T16:27:57.000Z", # :aws_update_time => "2008-10-14T16:37:10.000Z", # :aws_error_code => "Client.S3Error", # :aws_error_message => # "AccessDenied(403)- Invalid according to Policy: Policy Condition failed: [\"eq\", \"$acl\", \"aws-exec-read\"]", # :aws_state => "failed", # :aws_instance_id => "i-e3e24e8a"}] # # ec2.describe_bundle_tasks(:filters => { 'state' => 'pending' }) # # P.S. filters: http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference-query-DescribeBundleTasks.html # def describe_bundle_tasks(*list_and_options) describe_resources_with_list_and_options('DescribeBundleTasks', 'BundleId', QEc2DescribeBundleTasksParser, list_and_options) end # Cancel an in‐progress or pending bundle task by id. # # ec2.cancel_bundle_task('bun-73a7421a') #=> # [{:s3_bucket => "my-awesome-bucket" # :aws_id => "bun-0fa70206", # :s3_prefix => "win02", # :aws_start_time => "2008-10-14T13:00:29.000Z", # :aws_error_message => "User has requested bundling operation cancellation", # :aws_state => "failed", # :aws_update_time => "2008-10-14T13:01:31.000Z", # :aws_error_code => "Client.Cancelled", # :aws_instance_id => "i-e3e24e8a"} # def cancel_bundle_task(bundle_id) link = generate_request("CancelBundleTask", { 'BundleId' => bundle_id }) request_info(link, QEc2BundleInstanceParser.new) rescue Exception on_exception end #----------------------------------------------------------------- # PARSERS: Instances #----------------------------------------------------------------- class QEc2DescribeInstancesParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{(RunInstancesResponse|DescribeInstancesResponse/reservationSet/item)$} @reservation = { :aws_groups => [], :instances_set => [] } when %r{(/groupSet/item|instancesSet/item/placement)$} @group = {} when %r{instancesSet/item$} # the optional params (sometimes are missing and we dont want them to be nil) @item = { :aws_product_codes => [], :groups => [], :tags => {} } when %r{blockDeviceMapping/item$} @item[:block_device_mappings] ||= [] @block_device_mapping = {} when %r{/tagSet/item$} @aws_tag = {} end end def tagend(name) case name when 'reservationId' then @reservation[:aws_reservation_id] = @text when 'ownerId' then @reservation[:aws_owner] = @text when 'instanceId' then @item[:aws_instance_id] = @text when 'imageId' then @item[:aws_image_id] = @text when 'privateDnsName' then @item[:private_dns_name] = @text when 'dnsName' then @item[:dns_name] = @text when 'reason' then @item[:aws_reason] = @text when 'keyName' then @item[:ssh_key_name] = @text when 'amiLaunchIndex' then @item[:ami_launch_index] = @text when 'productCode' then @item[:aws_product_codes] << @text when 'instanceType' then @item[:aws_instance_type] = @text when 'launchTime' then @item[:aws_launch_time] = @text when 'availabilityZone' then @item[:aws_availability_zone] = @text when 'kernelId' then @item[:aws_kernel_id] = @text when 'ramdiskId' then @item[:aws_ramdisk_id] = @text when 'platform' then @item[:aws_platform] = @text when 'subnetId' then @item[:subnet_id] = @text when 'vpcId' then @item[:vpc_id] = @text when 'privateIpAddress' then @item[:private_ip_address] = @text when 'ipAddress' then @item[:ip_address] = @text when 'architecture' then @item[:architecture] = @text when 'rootDeviceType' then @item[:root_device_type] = @text when 'rootDeviceName' then @item[:root_device_name] = @text when 'instanceClass' then @item[:instance_class] = @text when 'instanceLifecycle' then @item[:instance_lifecycle] = @text when 'spotInstanceRequestId' then @item[:spot_instance_request_id] = @text when 'requesterId' then @item[:requester_id] = @text when 'virtualizationType' then @item[:virtualization_type] = @text when 'clientToken' then @item[:client_token] = @text when 'sourceDestCheck' then @item[:source_dest_check] = @text == 'true' when 'tenancy' then @item[:placement_tenancy] = @text when 'hypervisor' then @item[:hypervisor] = @text when 'ebsOptimized' then @item[:ebs_optimized] = @text == 'true' else case full_tag_name # EC2 Groups when %r{(RunInstancesResponse|/reservationSet/item)/groupSet/item/groupId$} then @group[:group_id] = @text when %r{(RunInstancesResponse|/reservationSet/item)/groupSet/item/groupName$} then @group[:group_name] = @text when %r{(RunInstancesResponse|/reservationSet/item)/groupSet/item$} then @reservation[:aws_groups] << @group # VPC Groups # KD: It seems that these groups are always present when the groups above present for non VPC instances only when %r{/instancesSet/item/groupSet/item/groupId$} then @group[:group_id] = @text when %r{/instancesSet/item/groupSet/item/groupName$} then @group[:group_name] = @text when %r{/instancesSet/item/groupSet/item$} then @item[:groups] << @group # Placement Group Name when %r{/placement/groupName$} then @group[:placement_group_name]= @text # Codes when %r{/stateReason/code$} then @item[:state_reason_code] = @text when %r{/stateReason/message$} then @item[:state_reason_message] = @text when %r{/instanceState/code$} then @item[:aws_state_code] = @text.to_i when %r{/instanceState/name$} then @item[:aws_state] = @text when %r{/monitoring/state$} then @item[:monitoring_state] = @text when %r{/license/pool$} then @item[:license_pool] = @text when %r{/blockDeviceMapping/item} # no trailing $ case name when 'deviceName' then @block_device_mapping[:device_name] = @text when 'virtualName' then @block_device_mapping[:virtual_name] = @text when 'volumeId' then @block_device_mapping[:ebs_volume_id] = @text when 'status' then @block_device_mapping[:ebs_status] = @text when 'attachTime' then @block_device_mapping[:ebs_attach_time] = @text when 'deleteOnTermination' then @block_device_mapping[:ebs_delete_on_termination] = @text == 'true' when 'item' then @item[:block_device_mappings] << @block_device_mapping end when %r{/instancesSet/item$} then @reservation[:instances_set] << @item when %r{/tagSet/item/key$} then @aws_tag[:key] = @text when %r{/tagSet/item/value$} then @aws_tag[:value] = @text when %r{/tagSet/item$} then @item[:tags][@aws_tag[:key]] = @aws_tag[:value] when %r{(RunInstancesResponse|DescribeInstancesResponse/reservationSet/item)$} then @result << @reservation end end end def reset @result = [] end end class QEc2ConfirmProductInstanceParser < RightAWSParser #:nodoc: def tagend(name) @result = @text if name == 'ownerId' end end class QEc2TerminateInstancesParser < RightAWSParser #:nodoc: def tagstart(name, attributes) @instance = {} if name == 'item' end def tagend(name) case full_tag_name when %r{/instanceId$} then @instance[:aws_instance_id] = @text when %r{/currentState/code$} then @instance[:aws_current_state_code] = @text.to_i when %r{/currentState/name$} then @instance[:aws_current_state_name] = @text when %r{/previousState/code$} then @instance[:aws_prev_state_code] = @text.to_i when %r{/previousState/name$} then @instance[:aws_prev_state_name] = @text when %r{/item$} then @result << @instance end end def reset @result = [] end end class QEc2DescribeInstanceAttributeParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{groupSet$} then @result = [] when %r{groupSet/item$} then @group = {} when %r{blockDeviceMapping$} then @result = [] when %r{blockDeviceMapping/item$} then @block_device_mapping = {} end end def tagend(name) case full_tag_name when %r{/instanceType/value$} then @result = @text when %r{/kernel/value$} then @result = @text when %r{/ramdisk/value$} then @result = @text when %r{/userData/value$} then @result = @text when %r{/rootDeviceName/value$} then @result = @text when %r{/disableApiTermination/value} then @result = @text == 'true' when %r{/instanceInitiatedShutdownBehavior/value$} then @result = @text when %r{/sourceDestCheck/value$} then @result = @text == 'true' when %r{/groupSet/item} # no trailing $ case name when 'groupId' then @group[:group_id] = @text when 'groupName' then @group[:group_name] = @text when 'item' then @result << @group end when %r{/blockDeviceMapping/item} # no trailing $ case name when 'deviceName' then @block_device_mapping[:device_name] = @text when 'virtualName' then @block_device_mapping[:virtual_name] = @text when 'noDevice' then @block_device_mapping[:no_device] = @text when 'volumeId' then @block_device_mapping[:ebs_volume_id] = @text when 'status' then @block_device_mapping[:ebs_status] = @text when 'attachTime' then @block_device_mapping[:ebs_attach_time] = @text when 'deleteOnTermination' then @block_device_mapping[:ebs_delete_on_termination] = @text == 'true' when 'item' then @result << @block_device_mapping end end end def reset @result = nil end end #----------------------------------------------------------------- # PARSERS: Console #----------------------------------------------------------------- class QEc2GetConsoleOutputParser < RightAWSParser #:nodoc: def tagend(name) case name when 'instanceId' then @result[:aws_instance_id] = @text when 'timestamp' then @result[:aws_timestamp] = @text @result[:timestamp] = (Time.parse(@text)).utc when 'output' then @result[:aws_output] = Base64.decode64(@text) end end def reset @result = {} end end #----------------------------------------------------------------- # Instances: Windows related part #----------------------------------------------------------------- class QEc2DescribeBundleTasksParser < RightAWSParser #:nodoc: def tagstart(name, attributes) @bundle = {} if name == 'item' end def tagend(name) case name # when 'requestId' then @bundle[:request_id] = @text when 'instanceId' then @bundle[:aws_instance_id] = @text when 'bundleId' then @bundle[:aws_id] = @text when 'bucket' then @bundle[:s3_bucket] = @text when 'prefix' then @bundle[:s3_prefix] = @text when 'startTime' then @bundle[:aws_start_time] = @text when 'updateTime' then @bundle[:aws_update_time] = @text when 'state' then @bundle[:aws_state] = @text when 'progress' then @bundle[:aws_progress] = @text when 'code' then @bundle[:aws_error_code] = @text when 'message' then @bundle[:aws_error_message] = @text when 'item' then @result << @bundle end end def reset @result = [] end end class QEc2BundleInstanceParser < RightAWSParser #:nodoc: def tagend(name) case name # when 'requestId' then @result[:request_id] = @text when 'instanceId' then @result[:aws_instance_id] = @text when 'bundleId' then @result[:aws_id] = @text when 'bucket' then @result[:s3_bucket] = @text when 'prefix' then @result[:s3_prefix] = @text when 'startTime' then @result[:aws_start_time] = @text when 'updateTime' then @result[:aws_update_time] = @text when 'state' then @result[:aws_state] = @text when 'progress' then @result[:aws_progress] = @text when 'code' then @result[:aws_error_code] = @text when 'message' then @result[:aws_error_message] = @text end end def reset @result = {} end end class QEc2GetPasswordDataParser < RightAWSParser #:nodoc: def tagend(name) case name when 'instanceId' then @result[:aws_instance_id] = @text when 'timestamp' then @result[:timestamp] = @text when 'passwordData' then @result[:password_data] = @text end end def reset @result = {} end end end end ================================================ FILE: lib/ec2/right_ec2_monitoring.rb ================================================ # # Copyright (c) 2009 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws class Ec2 # Enables monitoring for a running instances. For more information, refer to the Amazon CloudWatch Developer Guide. # # ec2.monitor_instances('i-8437ddec') #=> # {:instance_id=>"i-8437ddec", :monitoring_state=>"pending"} # def monitor_instances(*list) link = generate_request("MonitorInstances", amazonize_list('InstanceId', list.flatten) ) request_info(link, QEc2MonitorInstancesParser.new(:logger => @logger)).first rescue Exception on_exception end # Disables monitoring for a running instances. For more information, refer to the Amazon CloudWatch Developer Guide. # # ec2.unmonitor_instances('i-8437ddec') #=> # {:instance_id=>"i-8437ddec", :monitoring_state=>"disabling"} # def unmonitor_instances(*list) link = generate_request("UnmonitorInstances", amazonize_list('InstanceId', list.flatten) ) request_info(link, QEc2MonitorInstancesParser.new(:logger => @logger)).first rescue Exception on_exception end class QEc2MonitorInstancesParser < RightAWSParser #:nodoc: def tagstart(name, attributes) @item = {} if name == 'item' end def tagend(name) case name when 'instanceId' then @item[:instance_id] = @text when 'state' then @item[:monitoring_state] = @text when 'item' then @result << @item end end def reset @result = [] end end end end ================================================ FILE: lib/ec2/right_ec2_placement_groups.rb ================================================ # # Copyright (c) 2010 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws class Ec2 #----------------------------------------------------------------- # Placement Groups #----------------------------------------------------------------- # Describe placement groups. # # Accepts a list of placement groups and/or a set of filters as the last parameter. # # Filters: group-name, state, strategy # # If you don’t specify a particular placement group, the response includes # information about all of them. The information includes the group name, the strategy, # and the group state (e.g., pending, available, etc.). # # ec2.describe_placement_groups #=> # [{:state=>"available", :strategy=>"cluster", :group_name=>"kd_first"}, # {:state=>"available", :strategy=>"cluster", :group_name=>"kd_second"}] # # ec2.describe_placement_groups('kd_second') #=> # [{:strategy=>"cluster", :group_name=>"kd_second", :state=>"available"}] # # P.S. filters: http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference_query_DescribePlacementGroups.html # def describe_placement_groups(*list_and_options) describe_resources_with_list_and_options('DescribePlacementGroups', 'GroupName', QEc2DescribePlacementGroupsParser, list_and_options) end # Create placement group creates a placement group (i.e. logical cluster group) # into which you can then launch instances. You must provide a name for the group # that is unique within the scope of your account. You must also provide a strategy # value. Currently the only value accepted is cluster. # # ec2.create_placement_group('kd_second') #=> true # def create_placement_group(placement_group_name, strategy = 'cluster') link = generate_request('CreatePlacementGroup', 'GroupName' => placement_group_name.to_s, 'Strategy' => strategy.to_s) request_info(link, RightBoolResponseParser.new(:logger => @logger)) rescue Exception on_exception end # Delete placement group deletes a placement group that you own. The group must not # contain any instances. # # ec2.delete_placement_group('kd_second') #=> true # def delete_placement_group(placement_group_name) link = generate_request('DeletePlacementGroup', 'GroupName' => placement_group_name.to_s) request_info(link, RightBoolResponseParser.new(:logger => @logger)) rescue Exception on_exception end #----------------------------------------------------------------- # PARSERS: Placement Groups #----------------------------------------------------------------- class QEc2DescribePlacementGroupsParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case name when 'item' then @item = {} end end def tagend(name) case name when 'groupName' then @item[:group_name] = @text when 'strategy' then @item[:strategy] = @text when 'state' then @item[:state] = @text when 'item' then @result << @item end end def reset @result = [] end end end end ================================================ FILE: lib/ec2/right_ec2_reserved_instances.rb ================================================ # # Copyright (c) 2009 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws class Ec2 RESERVED_INSTANCE_API_VERSION = (API_VERSION > '2012-10-01') ? API_VERSION : '2012-10-01' #----------------------------------------------------------------- # Reserved instances #----------------------------------------------------------------- # Retrieve reserved instances list. # # Accepts a list of reserved instances and/or a set of filters as the last parameter. # # Filters: availability-zone, duration, fixed-price, instance-type, product-description, # reserved-instances-id, start, state, tag-key, tag-value, tag:key, usage-price # # ec2.describe_reserved_instances #=> # [{:tags=>{}, # :aws_id=>"4357912c-0000-0000-0000-15ca71a8e66d", # :aws_instance_type=>"m1.small", # :aws_availability_zone=>"us-east-1c", # :aws_start=>"2010-03-18T20:39:39.569Z", # :aws_duration=>94608000, # :aws_fixed_price=>350.0, # :aws_usage_price=>0.03, # :aws_instance_count=>1, # :aws_product_description=>"Linux/UNIX", # :aws_state=>"active", # :instance_tenancy=>"default", # :currency_code=>"USD", # :offering_type=>"Medium Utilization"}] # # ec2.describe_reserved_instances(:filters => {'availability-zone' => 'us-east-1a'}) # # P.S. filters: http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-DescribeReservedInstances.html # def describe_reserved_instances(*list_and_options) list_and_options = merge_new_options_into_list_and_options(list_and_options, :options => {:api_version => RESERVED_INSTANCE_API_VERSION}) describe_resources_with_list_and_options('DescribeReservedInstances', 'ReservedInstancesId', QEc2DescribeReservedInstancesParser, list_and_options) end # Retrieve reserved instances offerings. # # Accepts a list of reserved instances offerings and/or a set of filters as the last parameter. # # Filters: availability-zone, duration, fixed-price, instance-type, product-description, reserved-instances-offering-id, usage-price # # ec2.describe_reserved_instances_offerings #=> # [{:recurring_charges=>[{:frequency=>"Hourly", :amount=>"0.095"}], # :pricing_details_set=>[], # :aws_id=>"438012d3-4031-43ff-9241-2964d1bf71d8", # :aws_instance_type=>"c1.medium", # :aws_availability_zone=>"us-east-1e", # :aws_duration=>94608000, # :aws_fixed_price=>775.0, # :aws_usage_price=>0.0, # :aws_product_description=>"Red Hat Enterprise Linux", # :instance_tenancy=>"default", # :currency_code=>"USD", # :offering_type=>"Heavy Utilization", # :marketplace=>false}, # { :recurring_charges=>[{:frequency=>"Hourly", :amount=>"0.095"}], # :pricing_details_set=>[], # :aws_id=>"649fd0c8-6cb4-47bf-83db-7a844016afa7", # :aws_instance_type=>"c1.medium", # :aws_availability_zone=>"us-east-1e", # :aws_duration=>94608000, # :aws_fixed_price=>775.0, # :aws_usage_price=>0.0, # :aws_product_description=>"Red Hat Enterprise Linux (Amazon VPC)", # :instance_tenancy=>"default", # :currency_code=>"USD", # :offering_type=>"Heavy Utilization", # :marketplace=>false}, ... ] # # ec2.describe_reserved_instances_offerings(:filters => {'availability-zone' => 'us-east-1c'}) # # # Get all ReservedInstancesOfferings (list by 50 items) # result = ec2.describe_reserved_instances_offerings(:max_results => 50) do |response| # puts response[:items].count # true # end # # # Get first 400 ReservedInstancesOfferings. # # P.S. it stops making calls one the block below returns false. # max_count_to_get = 400 # counter = 0 # result = ec2.describe_reserved_instances_offerings do |response| # counter += response[:items].count # max_count_to_get <= counter # end # # P.S. filters: http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-DescribeReservedInstancesOfferings.html # def describe_reserved_instances_offerings(*list_and_options, &block) result = [] list_and_options = merge_new_options_into_list_and_options(list_and_options, :options => {:api_version => RESERVED_INSTANCE_API_VERSION}) incrementally_list_items('DescribeReservedInstancesOfferings', 'ReservedInstancesOfferingId', QEc2DescribeReservedInstancesOfferingsParser, list_and_options) do |response| result += response[:items] block ? block.call(response) : true end result end # Purchase a Reserved Instance. # Returns ReservedInstancesId value. # # ec2.purchase_reserved_instances_offering('e5a2ff3b-f6eb-4b4e-83f8-b879d7060257', 3) # => '4b2293b4-5813-4cc8-9ce3-1957fc1dcfc8' # def purchase_reserved_instances_offering(reserved_instances_offering_id, instance_count=1, options={}) options[:options] ||= {} options[:options][:api_version] ||= RESERVED_INSTANCE_API_VERSION api_params = { 'ReservedInstancesOfferingId' => reserved_instances_offering_id, 'InstanceCount' => instance_count } link = generate_request("PurchaseReservedInstancesOffering", api_params, options) request_info(link, QEc2PurchaseReservedInstancesOfferingParser.new) rescue Exception on_exception end #----------------------------------------------------------------- # PARSERS: ReservedInstances #----------------------------------------------------------------- class QEc2DescribeReservedInstancesParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{/recurringCharges/item$} then @recurring_charge = {} when %r{/tagSet/item$} then @aws_tag = {} when %r{/reservedInstancesSet/item$} then @item = { :tags=> {} } end end def tagend(name) case name when 'reservedInstancesId' then @item[:aws_id] = @text when 'instanceType' then @item[:aws_instance_type] = @text when 'availabilityZone' then @item[:aws_availability_zone] = @text when 'duration' then @item[:aws_duration] = @text.to_i when 'usagePrice' then @item[:aws_usage_price] = @text.to_f when 'fixedPrice' then @item[:aws_fixed_price] = @text.to_f when 'instanceCount' then @item[:aws_instance_count] = @text.to_i when 'productDescription' then @item[:aws_product_description] = @text when 'state' then @item[:aws_state] = @text when 'start' then @item[:aws_start] = @text when 'instanceTenancy' then @item[:instance_tenancy] = @text when 'currencyCode' then @item[:currency_code] = @text when 'offeringType' then @item[:offering_type] = @text else case full_tag_name when %r{/tagSet/item/key$} then @aws_tag[:key] = @text when %r{/tagSet/item/value$} then @aws_tag[:value] = @text when %r{/tagSet/item$} then @item[:tags][@aws_tag[:key]] = @aws_tag[:value] when %r{/recurringCharges/item/frequency$} then @recurring_charge[:frequency] = @text when %r{/recurringCharges/item/amount$} then @recurring_charge[:amount] = @text when %r{/recurringCharges/item$} then (@item[:recurring_charges] ||= []) << @recurring_charge when %r{/reservedInstancesSet/item$} then @result << @item end end end def reset @result = [] end end class QEc2DescribeReservedInstancesOfferingsParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{/pricingDetailsSet/item$} then @pricing_details = {} when %r{/recurringCharges/item$} then @recurring_charge = {} when %r{/reservedInstancesOfferingsSet/item$} then @item = {} end end def tagend(name) case name when 'nextToken' then @result[:next_token] = @text when 'reservedInstancesOfferingId' then @item[:aws_id] = @text when 'instanceType' then @item[:aws_instance_type] = @text when 'availabilityZone' then @item[:aws_availability_zone] = @text when 'duration' then @item[:aws_duration] = @text.to_i when 'usagePrice' then @item[:aws_usage_price] = @text.to_f when 'fixedPrice' then @item[:aws_fixed_price] = @text.to_f when 'instanceTenancy' then @item[:instance_tenancy] = @text when 'currencyCode' then @item[:currency_code] = @text when 'productDescription' then @item[:aws_product_description] = @text when 'offeringType' then @item[:offering_type] = @text when 'marketplace' then @item[:marketplace] = (@text == 'true') else case full_tag_name when %r{/recurringCharges/item/frequency$} then @recurring_charge[:frequency] = @text when %r{/recurringCharges/item/amount$} then @recurring_charge[:amount] = @text when %r{/recurringCharges/item$} then (@item[:recurring_charges] ||= []) << @recurring_charge when %r{/pricingDetailsSet/item/price$} then @pricing_details[:price] = @text when %r{/pricingDetailsSet/item/count$} then @pricing_details[:count] = @text when %r{/pricingDetailsSet/item$} then (@item[:pricing_details_set] ||= []) << @pricing_details when %r{/reservedInstancesOfferingsSet/item$} then @result[:items] << @item end end end def reset @result = { :items => [] } end end class QEc2PurchaseReservedInstancesOfferingParser < RightAWSParser #:nodoc: def tagend(name) if name == 'reservedInstancesId' @result = @text end end def reset @result = '' end end end end ================================================ FILE: lib/ec2/right_ec2_security_groups.rb ================================================ # # Copyright (c) 2010 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws class Ec2 #----------------------------------------------------------------- # Security groups #----------------------------------------------------------------- # Retrieve Security Groups information. # Options: By default this methods expects security group ids but if you wanna pass their names then :describe_by => :group_name option must be set. # # Accepts a list of security groups and/or a set of filters as the last parameter. # # Filters: description, group-name, ip-permission.cidr, ip-permission.from-port, ip-permission.group-name, # ip-permission.protocol, ip-permission.to-port, ip-permission.user-id, owner-id # # # Amazon cloud: # ec2 = Rightscale::Ec2.new(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) # ec2.describe_security_groups #=> # [{:aws_perms=> # [{:protocol=>"-1", :cidr_ips=>"0.0.0.0/0", :direction=>:egress}, # {:protocol=>"tcp", # :cidr_ips=>"127.0.0.2/32", # :direction=>:egress, # :from_port=>"1111", # :to_port=>"1111"}, # {:protocol=>"tcp", # :cidr_ips=>"127.0.0.1/32", # :direction=>:egress, # :from_port=>"1111", # :to_port=>"1111"}], # :aws_group_name=>"kd-vpc-egress-test-1", # :vpc_id=>"vpc-e16cf988", # :aws_description=>"vpc test", # :aws_owner=>"826693181925", # :group_id=>"sg-b72032db"}] # # # Describe by group ids # ec2.describe_security_groups("sg-a0b85dc9", "sg-00b05d39", "sg-a1b86dc8") # # # Describe by group names # ec2.describe_security_groups("default", "default1", "kd", :describe_by => :group_name) # # # Eucalyptus cloud: # ec2 = Rightscale::Ec2.new(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, :eucalyptus => true) # ec2.describe_security_groups #=> # [{:aws_perms=> # [{:to_port=>"65535", # :group_name=>"default", # :protocol=>"tcp", # :owner=>"048291609141", # :from_port=>"1"}, # {:to_port=>"65535", # :group_name=>"default", # :protocol=>"udp", # :owner=>"048291609141", # :from_port=>"1"}, # {:to_port=>"-1", # :group_name=>"default", # :protocol=>"icmp", # :owner=>"048291609141", # :from_port=>"-1"}, # {:to_port=>"22", # :protocol=>"tcp", # :from_port=>"22", # :cidr_ip=>"0.0.0.0/0"}, # {:to_port=>"9997", # :protocol=>"tcp", # :from_port=>"9997", # :cidr_ip=>"0.0.0.0/0"}], # :aws_group_name=>"photo_us", # :aws_description=>"default group", # :aws_owner=>"826693181925"}] # # ec2.describe_security_groups(:filters => {'ip-permission.from-port' => '22'}) # # P.S. filters: http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference-query-DescribeSecurityGroups.html # def describe_security_groups(*list_and_options) list, options = AwsUtils::split_items_and_params(list_and_options) describe_by = options.delete(:describe_by) == :group_name ? 'GroupName' : 'GroupId' describe_resources_with_list_and_options('DescribeSecurityGroups', describe_by, QEc2DescribeSecurityGroupsParser, list_and_options) do |parser| result = [] parser.result.each do |item| result_item = { :aws_owner => item[:owner_id], :aws_group_name => item[:group_name], :aws_description => item[:group_description] } result_item[:group_id] = item[:group_id] unless item[:group_id].right_blank? result_item[:vpc_id] = item[:vpc_id] unless item[:vpc_id].right_blank? aws_perms = [] item[:ip_permissions].each do |permission| result_perm = {} result_perm[:from_port] = permission[:from_port] unless permission[:from_port].right_blank? result_perm[:to_port] = permission[:to_port] unless permission[:to_port].right_blank? result_perm[:protocol] = permission[:ip_protocol] result_perm[:direction] = permission[:direction] # IP permissions Array(permission[:ip_ranges]).each do |ip_range| perm = result_perm.dup # Mhhh... For Eucalyptus we somehow get used to use ":cidr_ip" instead of ":cidr_ips"... if @params[:eucalyptus] then perm[:cidr_ip] = ip_range else perm[:cidr_ips] = ip_range end aws_perms << perm end # Group permissions Array(permission[:groups]).each do |group| perm = result_perm.dup perm[:group_name] = group[:group_name] unless group[:group_name].right_blank? perm[:group_id] = group[:group_id] unless group[:group_id].right_blank? perm[:owner] = group[:user_id] unless group[:user_id].right_blank? aws_perms << perm end end result_item[:aws_perms] = aws_perms.uniq result << result_item end result end end def describe_security_groups_by_name(*list) describe_security_groups(list, :describe_by => :group_name) end # Create new Security Group. Returns +true+ or an exception. # Options: :vpc_id # # ec2.create_security_group('default-1',"Default allowing SSH, HTTP, and HTTPS ingress") #=> # { :group_id=>"sg-f0227599", :return=>true } # # ec2.create_security_group('default-2',"my VPC group", :vpc_id => 'vpc-e16c0000') #=> # { :group_id=>"sg-76d1c31a", :return=>true } # def create_security_group(name, description = nil, options = {}) options = options.dup options[:group_name] = name options[:group_description] = description.right_blank? ? '-' : description # EC2 rejects an empty description... link = generate_request("CreateSecurityGroup", map_api_keys_and_values(options, :group_name, :group_description, :vpc_id)) request_info(link, QEc2CreateSecurityGroupsParser.new(:logger => @logger)) rescue Exception on_exception end # Remove Security Group. Returns +true+ or an exception. # Options: :group_name, :group_id # # # Delete security group by group_id: # ec2.delete_security_group('sg-90054ef9') #=> true # ec2.delete_security_group(:group_id => 'sg-90054ef9') #=> true # # # Delete security group by name (EC2 only): # ec2.delete_security_group(:group_name => 'my-group']) #=> true # def delete_security_group(group_id_or_options={}) options = group_id_or_options.is_a?(Hash) ? group_id_or_options : { :group_id => group_id_or_options } link = generate_request("DeleteSecurityGroup", map_api_keys_and_values(options, :group_name, :group_id)) request_info(link, RightBoolResponseParser.new(:logger => @logger)) rescue Exception on_exception end def grant_security_group_ingress(group_id, permissions) modify_security_group(:grant, :ingress, group_id, permissions) end def revoke_security_group_ingress(group_id, permissions) modify_security_group(:revoke, :ingress, group_id, permissions) end def grant_security_group_egress(group_id, permissions) modify_security_group(:grant, :egress, group_id, permissions) end def revoke_security_group_egress(group_id, permissions) modify_security_group(:revoke, :egress, group_id, permissions) end # Modify AWS security group permissions. # # Options: # action - :authorize (or :grant) | :revoke (or :remove) # direction - :ingress | :egress # group_name - security group name # permissions - a combination of options below: # # Ports: # :from_port => from port # :to_port => to port # :port => set both :from_port and to_port with the same value # # Protocol # :protocol => :tcp | :udp | :icmp | -1 # # or (ingress) # :groups => { UserId1 => GroupId1, UserName2 => GroupId2 } # :groups => [ [ UserId1, GroupId1 ], [ UserName2 => GroupId2 ] ] # # or (egress) # :groups => [ GroupId1, GroupId2 ] # # CidrIp(s) # :cidr_ip => '0.0.0.0/0' # :cidr_ips => ['1.1.1.1/1', '2.2.2.2/2'] # # # CidrIP based permissions: # # ec2.modify_security_group(:authorize, :ingress, 'sg-75d1c319', # :cidr_ip => "127.0.0.0/31", # :port => 811, # :protocol => 'tcp' ) #=> true # # ec2.modify_security_group(:revoke, :ingress, 'sg-75d1c319', # :cidr_ips => ["127.0.0.1/32", "127.0.0.2/32"], # :port => 812, # :protocol => 'tcp' ) #=> true # # # Group based permissions: # # ec2.modify_security_group(:authorize, :ingress, 'sg-75d1c319', # :groups => { "586789340000" => "sg-75d1c300", # "635201710000" => "sg-75d1c301" }, # :port => 801, # :protocol => 'tcp' ) #=> true # # ec2.modify_security_group(:revoke, :ingress, 'sg-75d1c319', # :groups => [[ "586789340000", "sg-75d1c300" ], # [ "586789340000", "sg-75d1c302" ]], # :port => 809, # :protocol => 'tcp' ) #=> true # # # +Permissions+ can be an array of permission hashes: # # ec2.modify_security_group(:authorize, :ingress, 'sg-75d1c319', # [{ :groups => { "586789340000" => "sg-75d1c300", # "635201710000" => "sg-75d1c301" }, # :port => 803, # :protocol => 'tcp'}, # { :cidr_ips => ["127.0.0.1/32", "127.0.0.2/32"], # :port => 812, # :protocol => 'tcp' }]) #=> true # def modify_security_group(action, direction, group_id, permissions) hash = {} raise "Unknown action #{action.inspect}!" unless [:authorize, :grant, :revoke, :remove].include?(action) raise "Unknown direction #{direction.inspect}!" unless [:ingress, :egress].include?(direction) # Remote action remote_action = case action when :authorize, :grant then direction == :ingress ? "AuthorizeSecurityGroupIngress" : "AuthorizeSecurityGroupEgress" when :revoke, :remove then direction == :ingress ? "RevokeSecurityGroupIngress" : "RevokeSecurityGroupEgress" end # Group Name hash["GroupId"] = group_id # Permissions permissions = [permissions] unless permissions.is_a?(Array) permissions.each_with_index do |permission, idx| pid = idx+1 # Protocol hash["IpPermissions.#{pid}.IpProtocol"] = permission[:protocol] # Port unless permission[:port].right_blank? hash["IpPermissions.#{pid}.FromPort"] = permission[:port] hash["IpPermissions.#{pid}.ToPort"] = permission[:port] else hash["IpPermissions.#{pid}.FromPort"] = permission[:from_port] hash["IpPermissions.#{pid}.ToPort"] = permission[:to_port] end # Groups case direction when :ingress # :groups => {UserId1 => GroupId1, ... UserIdN => GroupIdN} # or (this allows using same UserId multiple times ) # :groups => [[UserId1, GroupId1], ... [UserIdN, GroupIdN]] # or even (unset user is == current account user) # :groups => [GroupId1, GroupId2, ... GroupIdN] # :groups => [[UserId1, GroupId1], GroupId2, ... GroupIdN, ... [UserIdM, GroupIdM]] # index = 1 unless permission[:group_names].right_blank? owner_and_groups = [] groups_only = [] Array(permission[:group_names]).each do |item| if item.is_a?(Array) && item.size == 2 owner_and_groups << item else groups_only << item end end hash.merge!(amazonize_list( ["IpPermissions.#{pid}.Groups.?.UserId", "IpPermissions.#{pid}.Groups.?.GroupName"], owner_and_groups, :index => index )) index += owner_and_groups.size groups_only = groups_only.flatten hash.merge!(amazonize_list( "IpPermissions.#{pid}.Groups.?.GroupName", groups_only, :index => index )) index += groups_only.size end unless permission[:groups].right_blank? owner_and_groups = [] groups_only = [] Array(permission[:groups]).each do |item| if item.is_a?(Array) && item.size == 2 owner_and_groups << item else groups_only << item end end hash.merge!(amazonize_list( ["IpPermissions.#{pid}.Groups.?.UserId", "IpPermissions.#{pid}.Groups.?.GroupId"], owner_and_groups, :index => index )) index += owner_and_groups.size groups_only = groups_only.flatten hash.merge!(amazonize_list( "IpPermissions.#{pid}.Groups.?.GroupId", groups_only, :index => index )) end when :egress # :groups => [GroupId1, ... GroupIdN] hash.merge!(amazonize_list( "IpPermissions.#{pid}.Groups.?.GroupId", permission[:groups] )) end # CidrIp(s) cidr_ips = permission[:cidr_ips] unless permission[:cidr_ips].right_blank? cidr_ips ||= permission[:cidr_ip] unless permission[:cidr_ip].right_blank? hash.merge!(amazonize_list("IpPermissions.1.IpRanges.?.CidrIp", cidr_ips)) end # link = generate_request(remote_action, hash) request_info(link, RightBoolResponseParser.new(:logger => @logger)) rescue Exception on_exception end #----------------------------------------------------------------- # Eucalyptus #----------------------------------------------------------------- # Edit AWS/Eucaliptus security group permissions. # # Options: # action - :authorize (or :grant) | :revoke (or :remove) # group_name - security group name # permissions - a combination of options below: # :source_group_owner => UserId # :source_group => GroupName # :from_port => from port # :to_port => to port # :port => set both :from_port and to_port with the same value # :protocol => :tcp | :udp | :icmp # :cidr_ip => '0.0.0.0/0' # # ec2.edit_security_group( :grant, # 'kd-sg-test', # :source_group => "sketchy", # :source_group_owner => "600000000006", # :protocol => 'tcp', # :port => '80', # :cidr_ip => '127.0.0.1/32') #=> true # # P.S. This method is deprecated for AWS and but still good for Eucaliptus clouds. # Use +modify_security_group_ingress+ method for AWS clouds. # def edit_security_group(action, group_name, params) hash = {} case action when :authorize, :grant then action = "AuthorizeSecurityGroupIngress" when :revoke, :remove then action = "RevokeSecurityGroupIngress" else raise "Unknown action #{action.inspect}!" end hash['GroupName'] = group_name hash['SourceSecurityGroupName'] = params[:source_group] unless params[:source_group].right_blank? hash['IpProtocol'] = params[:protocol] unless params[:protocol].right_blank? unless params[:source_group_owner].right_blank? # Do remove dashes only if the source owner is in format of "7011-0219-8268" source_group_owner = params[:source_group_owner].to_s source_group_owner.gsub!(/-/,'') if source_group_owner[/^\d{4}-\d{4}-\d{4}$/] hash['SourceSecurityGroupOwnerId'] = source_group_owner end unless params[:port].right_blank? hash['FromPort'] = params[:port] hash['ToPort'] = params[:port] end hash['FromPort'] = params[:from_port] unless params[:from_port].right_blank? hash['ToPort'] = params[:to_port] unless params[:to_port].right_blank? hash['CidrIp'] = params[:cidr_ip] unless params[:cidr_ip].right_blank? # link = generate_request(action, hash) request_info(link, RightBoolResponseParser.new(:logger => @logger)) rescue Exception on_exception end # Authorize named ingress for security group. Allows instances that are member of someone # else's security group to open connections to instances in my group. # # ec2.authorize_security_group_named_ingress('my_awesome_group', '7011-0219-8268', 'their_group_name') #=> true # def authorize_security_group_named_ingress(name, owner, group) edit_security_group( :authorize, name, :source_group_owner => owner, :source_group => group) end # Revoke named ingress for security group. # # ec2.revoke_security_group_named_ingress('my_awesome_group', aws_user_id, 'another_group_name') #=> true # def revoke_security_group_named_ingress(name, owner, group) edit_security_group( :revoke, name, :source_group_owner => owner, :source_group => group) end # Add permission to a security group. Returns +true+ or an exception. +protocol+ is one of :'tcp'|'udp'|'icmp'. # # ec2.authorize_security_group_IP_ingress('my_awesome_group', 80, 82, 'udp', '192.168.1.0/8') #=> true # ec2.authorize_security_group_IP_ingress('my_awesome_group', -1, -1, 'icmp') #=> true # def authorize_security_group_IP_ingress(name, from_port, to_port, protocol='tcp', cidr_ip='0.0.0.0/0') edit_security_group( :authorize, name, :from_port => from_port, :to_port => to_port, :protocol => protocol, :cidr_ip => cidr_ip ) end # Remove permission from a security group. Returns +true+ or an exception. +protocol+ is one of :'tcp'|'udp'|'icmp' ('tcp' is default). # # ec2.revoke_security_group_IP_ingress('my_awesome_group', 80, 82, 'udp', '192.168.1.0/8') #=> true # def revoke_security_group_IP_ingress(name, from_port, to_port, protocol='tcp', cidr_ip='0.0.0.0/0') edit_security_group( :revoke, name, :from_port => from_port, :to_port => to_port, :protocol => protocol, :cidr_ip => cidr_ip ) end #----------------------------------------------------------------- # PARSERS: Security Groups #----------------------------------------------------------------- class QEc2CreateSecurityGroupsParser < RightAWSParser #:nodoc: def tagend(name) case name when 'groupId' then @result[:group_id] = @text when 'return' then @result[:return] = @text == 'true' end end def reset @result = {} end end class QEc2DescribeSecurityGroupsParser < RightAWSParser #:nodoc: def tagstart(name, attributes) if name == 'item' case full_tag_name when %r{securityGroupInfo/item$} then @item = { :ip_permissions => [] } when %r{ipPermissions/item$} then @ip_perm = { :groups => [], :ip_ranges => [], :direction => :ingress } when %r{ipPermissionsEgress/item$} then @ip_perm = { :groups => [], :ip_ranges => [], :direction => :egress } when %r{ipPermissions(Egress)?/item/groups/item$} then @group = {} end end end def tagend(name) case name when 'ownerId' then @item[:owner_id] = @text when 'groupDescription' then @item[:group_description] = @text when 'vpcId' then @item[:vpc_id] = @text else case full_tag_name when %r{securityGroupInfo/item/groupName$} then @item[:group_name] = @text when %r{securityGroupInfo/item/groupId$} then @item[:group_id] = @text # ipPermission[Egress] when %r{ipPermissions(Egress)?/item/ipProtocol$} then @ip_perm[:ip_protocol] = @text when %r{ipPermissions(Egress)?/item/fromPort$} then @ip_perm[:from_port] = @text when %r{ipPermissions(Egress)?/item/toPort$} then @ip_perm[:to_port] = @text when %r{ipPermissions(Egress)?/item/ipRanges/item/cidrIp$} then @ip_perm[:ip_ranges] << @text # ipPermissions[Egress]/Groups when %r{ipPermissions(Egress)?/item/groups/item/groupName$} then @group[:group_name] = @text when %r{ipPermissions(Egress)?/item/groups/item/groupId$} then @group[:group_id] = @text when %r{ipPermissions(Egress)?/item/groups/item/userId$} then @group[:user_id] = @text # Sets when %r{ipPermissions(Egress)?/item/groups/item$} then @ip_perm[:groups] << @group when %r{ipPermissions/item$} then @item[:ip_permissions] << @ip_perm when %r{ipPermissionsEgress/item$} then @item[:ip_permissions] << @ip_perm when %r{securityGroupInfo/item$} then @result << @item end end end def reset @result = [] end end end end ================================================ FILE: lib/ec2/right_ec2_spot_instances.rb ================================================ # # Copyright (c) 2009 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws class Ec2 #----------------------------------------------------------------- # Spot Instances #----------------------------------------------------------------- # Describe Spot Price history. # # Options: :start_time, :end_time, instance_types, product_description # # Filters: instance-type, product-description, spot-price, timestamp # # ec2.describe_spot_price_history #=> # [{:spot_price=>0.054, # :timestamp=>"2009-12-07T12:12:58.000Z", # :product_description=>"Windows", # :instance_type=>"m1.small"}, # {:spot_price=>0.06, # :timestamp=>"2009-12-07T12:18:32.000Z", # :product_description=>"Linux/UNIX", # :instance_type=>"c1.medium"}, # {:spot_price=>0.198, # :timestamp=>"2009-12-07T12:58:00.000Z", # :product_description=>"Windows", # :instance_type=>"m1.large"}, # {:spot_price=>0.028, # :timestamp=>"2009-12-07T13:48:50.000Z", # :product_description=>"Linux/UNIX", # :instance_type=>"m1.small"}, ... ] # # ec2.describe_spot_price_history(:start_time => 1.day.ago, # :end_time => 10.minutes.ago, # :instance_types => ["c1.medium", "m1.small"], # :product_description => "Linux/UNIX" ) #=> # [{:product_description=>"Linux/UNIX", # :timestamp=>"2010-02-04T05:44:36.000Z", # :spot_price=>0.031, # :instance_type=>"m1.small"}, # {:product_description=>"Linux/UNIX", # :timestamp=>"2010-02-04T17:56:25.000Z", # :spot_price=>0.058, # :instance_type=>"c1.medium"}, ... ] # # ec2.describe_spot_price_history(:filters => {'spot-price' => '0.2' }) # # ec2.describe_spot_price_history(:instance_types => ["c1.medium"], :filters => {'spot-price' => '0.2' }) # # # P.S. filters: http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-DescribeSpotPriceHistory.html # def describe_spot_price_history(options={}) options = options.dup request_hash = {} request_hash.merge!(amazonize_list(['Filter.?.Name', 'Filter.?.Value.?'], options[:filters])) unless options[:filters].right_blank? request_hash['StartTime'] = AwsUtils::utc_iso8601(options[:start_time]) unless options[:start_time].right_blank? request_hash['EndTime'] = AwsUtils::utc_iso8601(options[:end_time]) unless options[:end_time].right_blank? request_hash['ProductDescription'] = options[:product_description] unless options[:product_description].right_blank? request_hash.merge!(amazonize_list('InstanceType', Array(options[:instance_types]))) unless options[:instance_types].right_blank? link = generate_request("DescribeSpotPriceHistory", request_hash, :api_version => '2011-05-15') request_info(link, QEc2DescribeSpotPriceHistoryParser.new) rescue Exception on_exception end # Describe Spot Instance requests. # # Accepts a list of requests and/or a set of filters as the last parameter. # # Filters: availability-zone-group, create-time, fault-code, fault-message, instance-id, launch-group, # launch.block-device-mapping.delete-on-termination, launch.block-device-mapping.device-name, # launch.block-device-mapping.snapshot-id, launch.group-id, launch.image-id, launch.instance-type, # launch.kernel-id, launch.key-name, launch.monitoring-enabled, launch.ramdisk-id, product-description, # spot-instance-request-id, spot-price, state, tag-key, tag-value, tag:key, type, valid-from, valid-until # # ec2.describe_spot_instance_requests #=> # [{:product_description=>"Linux/UNIX", # :type=>"one-time", # :availability_zone=>"us-east-1b", # :monitoring_enabled=>false, # :tags=>{}, # :image_id=>"ami-08f41161", # :groups=>[{:group_id=>"sg-a0b85dc9", :group_name=>"default"}], # :spot_price=>0.01, # :create_time=>"2010-03-24T10:41:28.000Z", # :instance_type=>"c1.medium", # :state=>"open", # :spot_instance_request_id=>"sir-9652a604", # :key_name=>"rightscale_test"}, # {:product_description=>"Linux/UNIX", # :type=>"one-time", # :availability_zone=>"us-east-1b", # :monitoring_enabled=>false, # :tags=>{}, # :image_id=>"ami-08f41161", # :groups=>[{:group_id=>"sg-a0b85dc9", :group_name=>"default"}], # :spot_price=>0.01, # :create_time=>"2010-03-24T11:40:27.000Z", # :instance_type=>"c1.medium", # :state=>"open", # :spot_instance_request_id=>"sir-fa912802", # :key_name=>"rightscale_test"}, ... ] # # ec2.describe_spot_instance_requests(:filters => {'type'=>"one-time", 'state'=>"open"}) # # P.S. filters: http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-DescribeSpotInstanceRequests.html # def describe_spot_instance_requests(*list_and_options) describe_resources_with_list_and_options('DescribeSpotInstanceRequests', 'SpotInstanceRequestId', QEc2DescribeSpotInstanceParser, list_and_options) end # Create a Spot Instance request. # # Mandatory params: :image_id, :spot_price, :instance_type # Optional params: :valid_from, :valid_until, :instance_count, :type, :launch_group, # :availability_zone_group, :key_name, :user_data, :addressing_type, :kernel_id, # :ramdisk_id, :subnet_id, :availability_zone, :monitoring_enabled, :groups, # :block_device_mappings # # ec2.request_spot_instances( # :image_id => 'ami-08f41161', # :spot_price => 0.01, # :key_name => 'tim', # :instance_count => 2, # :group_ids => ["sg-a0b85dc9"], # :instance_type => 'c1.medium') #=> # # [{:product_description=>"Linux/UNIX", # :type=>"one-time", # :spot_instance_requestId=>"sir-7a893003", # :monitoring_enabled=>false, # :image_id=>"ami-08f41161", # :state=>"open", # :spot_price=>0.01, # :groups=>[{:group_id=>"sg-a0b85dc9", :group_name=>"default"}], # :key_name=>"tim", # :create_time=>"2010-03-10T10:33:09.000Z", # :instance_type=>"c1.medium"}, # {:product_description=>"Linux/UNIX", # :type=>"one-time", # :spot_instance_requestId=>"sir-13dc9a03", # :monitoring_enabled=>false, # :image_id=>"ami-08f41161", # :state=>"open", # :spot_price=>0.01, # :groups=>[{:group_id=>"sg-a0b85dc9", :group_name=>"default"}], # :key_name=>"tim", # :create_time=>"2010-03-10T10:33:09.000Z", # :instance_type=>"c1.medium"}] # # ec2.request_spot_instances( # :image_id => 'ami-08f41161', # :spot_price => 0.01, # :instance_type => 'm1.small', # :valid_from => 10.minutes.since, # :valid_until => 1.hour.since, # :instance_count => 1, # :key_name => 'tim', # :group_names => ['default'], # :availability_zone => 'us-east-1a', # :monitoring_enabled => true, # :launch_group => 'lg1', # :availability_zone_group => 'azg1', # :block_device_mappings => [ { :device_name => '/dev/sdk', # :ebs_snapshot_id => 'snap-145cbc7d', # :ebs_delete_on_termination => true, # :ebs_volume_size => 3, # :virtual_name => 'ephemeral2' # } ] ) #=> # [{:type=>"one-time", # :image_id=>"ami-08f41161", # :availability_zone_group=>"azg1", # :key_name=>"default", # :spot_instance_request_id=>"sir-66c79a12", # :block_device_mappings=> # [{:ebs_volume_size=>3, # :virtual_name=>"ephemeral2", # :device_name=>"/dev/sdk", # :ebs_snapshot_id=>"snap-145cbc7d", # :ebs_delete_on_termination=>true}], # :spot_price=>0.01, # :product_description=>"Linux/UNIX", # :state=>"open", # :instance_type=>"m1.small", # :availability_zone=>"us-east-1a", # :groups=>[{:group_id=>"sg-a0b85dc9", :group_name=>"default"}], # :valid_from=>"2011-07-01T14:26:33.000Z", # :tags=>{}, # :monitoring_enabled=>true, # :valid_until=>"2011-07-01T14:28:03.000Z", # :create_time=>"2011-07-01T14:26:24.000Z", # :launch_group=>"lg1"}] # def request_spot_instances(options) options[:user_data] = options[:user_data].to_s request_hash = map_api_keys_and_values( options, :spot_price, :availability_zone_group, :launch_group, :type, :instance_count, :image_id => 'LaunchSpecification.ImageId', :instance_type => 'LaunchSpecification.InstanceType', :key_name => 'LaunchSpecification.KeyName', :addressing_type => 'LaunchSpecification.AddressingType', :kernel_id => 'LaunchSpecification.KernelId', :ramdisk_id => 'LaunchSpecification.RamdiskId', :subnet_id => 'LaunchSpecification.SubnetId', :availability_zone => 'LaunchSpecification.Placement.AvailabilityZone', :monitoring_enabled => 'LaunchSpecification.Monitoring.Enabled', :valid_from => { :value => Proc.new { !options[:valid_from].right_blank? && AwsUtils::utc_iso8601(options[:valid_from]) }}, :valid_until => { :value => Proc.new { !options[:valid_until].right_blank? && AwsUtils::utc_iso8601(options[:valid_until]) }}, :user_data => { :name => 'LaunchSpecification.UserData', :value => Proc.new { !options[:user_data].empty? && Base64.encode64(options[:user_data]).delete("\n") }}, :group_names => { :amazonize_list => 'LaunchSpecification.SecurityGroup'}, :group_ids => { :amazonize_list => 'LaunchSpecification.SecurityGroupId'}, :block_device_mappings => { :amazonize_bdm => 'LaunchSpecification.BlockDeviceMapping'}) link = generate_request("RequestSpotInstances", request_hash) request_info(link, QEc2DescribeSpotInstanceParser.new(:logger => @logger)) end # Cancel one or more Spot Instance requests. # # ec2.cancel_spot_instance_requests('sir-60662c03',"sir-d3c96e04", "sir-4fa8d804","sir-6992ce04") #=> # [{:state=>"cancelled", :spot_instance_request_id=>"sir-60662c03"}, # {:state=>"cancelled", :spot_instance_request_id=>"sir-6992ce04"}, # {:state=>"cancelled", :spot_instance_request_id=>"sir-4fa8d804"}, # {:state=>"cancelled", :spot_instance_request_id=>"sir-d3c96e04"}] # def cancel_spot_instance_requests(*spot_instance_request_ids) link = generate_request("CancelSpotInstanceRequests", amazonize_list('SpotInstanceRequestId', spot_instance_request_ids.flatten)) request_info(link, QEc2CancelSpotInstanceParser.new(:logger => @logger)) end # Create the data feed for Spot Instances # (Enables to view Spot Instance usage logs) # # ec2.create_spot_datafeed_subscription('bucket-for-konstantin-eu', 'splogs/') #=> # { :owner_id=>"826693181925", # :bucket=>"bucket-for-konstantin-eu", # :prefix=>"splogs/", # :state=>"Active"} # def create_spot_datafeed_subscription(bucket, prefix=nil) request_hash = { 'Bucket' => bucket } request_hash['Prefix'] = prefix unless prefix.right_blank? link = generate_request("CreateSpotDatafeedSubscription", request_hash) request_info(link, QEc2DescribeSpotDatafeedSubscriptionParser.new(:logger => @logger)) end # Describe the data feed for Spot Instances. # # ec2.describe_spot_datafeed_subscription #=> # { :owner_id=>"826693181925", # :bucket=>"bucket-for-konstantin-eu", # :prefix=>"splogs/", # :state=>"Active"} # def describe_spot_datafeed_subscription link = generate_request("DescribeSpotDatafeedSubscription") request_info(link, QEc2DescribeSpotDatafeedSubscriptionParser.new(:logger => @logger)) end # Delete the data feed for Spot Instances. # # ec2.delete_spot_datafeed_subscription #=> true # def delete_spot_datafeed_subscription() link = generate_request("DeleteSpotDatafeedSubscription") request_info(link, RightBoolResponseParser.new(:logger => @logger)) end #----------------------------------------------------------------- # PARSERS: Spot Instances #----------------------------------------------------------------- class QEc2DescribeSpotPriceHistoryParser < RightAWSParser #:nodoc: def tagstart(name, attributes) @item = {} if name == 'item' end def tagend(name) case name when 'instanceType' then @item[:instance_type] = @text when 'productDescription' then @item[:product_description] = @text when 'spotPrice' then @item[:spot_price] = @text.to_f when 'timestamp' then @item[:timestamp] = @text when 'availabilityZone' then @item[:availability_zone] = @text when 'item' then @result << @item end end def reset @result = [] end end class QEc2DescribeSpotInstanceParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{spotInstanceRequestSet/item$} @item = { :tags => {} } when %r{groupSet$} @item[:groups] = [] when %r{groupSet/item$} @group = {} when %r{/blockDeviceMapping/item$} @item[:block_device_mappings] ||= [] @block_device_mapping = {} when %r{/tagSet/item$} @aws_tag = {} end end def tagend(name) case name when 'spotInstanceRequestId' then @item[:spot_instance_request_id]= @text when 'spotPrice' then @item[:spot_price] = @text.to_f when 'type' then @item[:type] = @text when 'state' then @item[:state] = @text when 'code' then @item[:fault_code] = @text when 'message' then @item[:fault_message] = @text when 'validFrom' then @item[:valid_from] = @text when 'validUntil' then @item[:valid_until] = @text when 'launchGroup' then @item[:launch_group] = @text when 'availabilityZoneGroup' then @item[:availability_zone_group] = @text when 'imageId' then @item[:image_id] = @text when 'keyName' then @item[:key_name] = @text when 'userData' then @item[:userData] = @text when 'data' then @item[:data] = @text when 'addressingType' then @item[:addressing_type] = @text when 'instanceType' then @item[:instance_type] = @text when 'availabilityZone' then @item[:availability_zone] = @text when 'kernelId' then @item[:kernel_id] = @text when 'ramdiskId' then @item[:ramdisk_id] = @text when 'subnetId' then @item[:subnet_id] = @text when 'instanceId' then @item[:instance_id] = @text when 'createTime' then @item[:create_time] = @text when 'productDescription' then @item[:product_description] = @text else case full_tag_name when %r{/groupSet/item} # no trailing $ case name when 'groupId' then @group[:group_id] = @text when 'groupName' then @group[:group_name] = @text when 'item' then @item[:groups] << @group end when %r{monitoring/enabled$} @item[:monitoring_enabled] = @text == 'true' when %r{/blockDeviceMapping/item} # no trailing $ case name when 'deviceName' then @block_device_mapping[:device_name] = @text when 'virtualName' then @block_device_mapping[:virtual_name] = @text when 'volumeSize' then @block_device_mapping[:ebs_volume_size] = @text.to_i when 'snapshotId' then @block_device_mapping[:ebs_snapshot_id] = @text when 'deleteOnTermination' then @block_device_mapping[:ebs_delete_on_termination] = @text == 'true' ? true : false when 'item' then @item[:block_device_mappings] << @block_device_mapping end when %r{/tagSet/item/key$} then @aws_tag[:key] = @text when %r{/tagSet/item/value$} then @aws_tag[:value] = @text when %r{/tagSet/item$} then @item[:tags][@aws_tag[:key]] = @aws_tag[:value] when %r{spotInstanceRequestSet/item$} then @result << @item end end end def reset @result = [] end end class QEc2CancelSpotInstanceParser < RightAWSParser #:nodoc: def tagstart(name, attributes) @item = {} if name == 'item' end def tagend(name) case name when 'spotInstanceRequestId' then @item[:spot_instance_request_id] = @text when 'state' then @item[:state] = @text when 'item' then @result << @item end end def reset @result = [] end end class QEc2DescribeSpotDatafeedSubscriptionParser < RightAWSParser #:nodoc: def tagend(name) case name when 'ownerId' then @result[:owner_id] = @text when 'bucket' then @result[:bucket] = @text when 'prefix' then @result[:prefix] = @text when 'state' then @result[:state] = @text when 'code' then @result[:fault_code] = @text when 'message' then @result[:fault_message] = @text end end def reset @result = {} end end end end ================================================ FILE: lib/ec2/right_ec2_tags.rb ================================================ # # Copyright (c) 2007-2010 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws class Ec2 #----------------------------------------------------------------- # Tags #----------------------------------------------------------------- # Describe tags. # # Accepts set of filters. # # Filters: key, resource-id, resource-type, value # # ec2.describe_tags #=> [{:resource_id => "i-12345678", # :value => "foo", # :resource_type => "instance", # :key => "myKey"}] # # ec2.describe_tags(:filters => { 'resource-id' => "i-12345678"}) # # P.S. filters: http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference_query_DescribeTags.html def describe_tags(options={}) request_hash = {} request_hash.merge!(amazonize_list(['Filter.?.Name', 'Filter.?.Value.?'], options[:filters])) unless options[:filters].right_blank? cache_for = (options[:filters].right_blank?) ? :describe_tags : nil link = generate_request("DescribeTags", request_hash) request_cache_or_info cache_for, link, QEc2DescribeTagsParser, @@bench, cache_for rescue Exception on_exception end # Create tags. # Options: # :default => 'something' : a default value for keys without (or with nil) values. # # Add a single tag with no value to a resource: # ec2.create_tags("i-12345678", "myKey") => true # # Add multiple tags with no values (actually Amazon sets the values to '') # ec2.create_tags("i-12345678", ["myKey1", "myKey2", "myKey3"]) => true # # Add multiple tags with 'true' # ec2.create_tags("i-12345678", ["myKey1", "myKey2", "myKey3"], :default => true ) => true # # Add multiple keys and values to a resource: # ec2.create_tags("i-12345678", {"myKey1" => "foo", "myKey2" => "bar", "myKeyWithoutVal" => nil }) #=> true # # Add a key and value to multiple resources: # ec2.create_tags(["i-12345678","i-86fb3eec","i-86fb3eed"], {"myKey" => "foo"}) #=> true # def create_tags(resources, tags, options={}) default = options[:default].nil? ? '' : options[:default] params = amazonize_list("ResourceId", resources) params.merge! amazonize_list(['Tag.?.Key', 'Tag.?.Value'], tags, :default => default) link = generate_request("CreateTags", params) request_info(link, RightBoolResponseParser.new(:logger => @logger)) rescue Exception on_exception end # Delete tags. # Options: # :default => 'something' : a default value for keys without (or with nil) values. # # Delete a tag from a resource regardless of value: # ec2.delete_tags("i-12345678", "myKey") => true # # Delete multiple tags (regardless of their values) # ec2.delete_tags("i-12345678", ["myKey1", "myKey2", "myKey3"]) => true # # Add multiple tags with value 'true' # ec2.delete_tags("i-12345678", ["myKey1", "myKey2", "myKey3"], :default => true) => true # # Delete multiple keys and values to a resource: # ec2.delete_tags("i-12345678", [{"myKey1" => "foo", "myKey2" => "bar","myKeyForAnyVal" => nil }]) #=> true # # Delete a key and value on multiple resources: # ec2.delete_tags(["i-12345678", "i-a1234567", "i-b1234567"], {"myKey" => "foo"}) #=> true # def delete_tags(resources, tags, options={}) default = options[:default].nil? ? :skip_nils : options[:default] params = amazonize_list("ResourceId", resources) params.merge! amazonize_list(['Tag.?.Key', 'Tag.?.Value'], tags, :default => default) link = generate_request("DeleteTags", params) request_info(link, RightBoolResponseParser.new(:logger => @logger)) rescue Exception on_exception end #----------------------------------------------------------------- # PARSERS: Tags #----------------------------------------------------------------- class QEc2DescribeTagsParser < RightAWSParser #:nodoc: def tagstart(name, attributes) @resource_tag = {} if name == 'item' end def tagend(name) case name when 'resourceId' then @resource_tag[:resource_id] = @text when 'resourceType' then @resource_tag[:resource_type] = @text when 'key' then @resource_tag[:key] = @text when 'value' then @resource_tag[:value] = @text when 'item' then @result << @resource_tag end end def reset @result = [] end end end end ================================================ FILE: lib/ec2/right_ec2_vpc.rb ================================================ # # Copyright (c) 2009 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws class Ec2 VPC_API_VERSION = (API_VERSION > '2012-10-15') ? API_VERSION : '2012-10-15' public #----------------- # VPC #----------------- # Describe VPCs. # # Accepts a list of vpcs and/or a set of filters as the last parameter. # # Filters: cidr, dchp-options-id, state, tag-key, tag-value, tag:key, vpc-id # # ec2.describe_vpcs #=> # [{:instance_tenancy=>"default", # :vpc_id=>"vpc-e16cf988", # :tags=>{}, # :dhcp_options_id=>"default", # :cidr_block=>"192.168.0.0/24", # :state=>"available"}] # # ec2.describe_vpcs("vpc-890ce2e0") # # ec2.describe_vpcs( :filters => {'tag:MyTag' => 'MyValue'} ) # # ec2.describe_vpcs( :filters => {'cidr' => "192.168.1.0/24"} ) # # P.S. filters: http://docs.amazonwebservices.com/AmazonVPC/latest/APIReference/index.html?ApiReference-query-DescribeVpcs.html # def describe_vpcs(*list_and_options) list_and_options = merge_new_options_into_list_and_options(list_and_options, :options => {:api_version => VPC_API_VERSION}) describe_resources_with_list_and_options('DescribeVpcs', 'VpcId', QEc2DescribeVpcsParser, list_and_options) end # Create VPC. # # ec2.create_vpc('10.0.0.0/23') #=> # {:vpc_id=>"vpc-890ce2e0", # :dhcp_options_id=>"default", # :cidr_block=>"10.0.0.0/23", # :state=>"pending"} # def create_vpc(cidr_block, options = {}) request_hash = {'CidrBlock' => cidr_block} request_hash['InstanceTenancy'] = options[:instance_tenancy] unless options[:instance_tenancy].right_blank? link = generate_request("CreateVpc", request_hash ) request_info(link, QEc2DescribeVpcsParser.new(:logger => @logger)).first rescue Exception on_exception end # Delete VPC. # # ec2.delete_vpc("vpc-890ce2e0") #=> true # def delete_vpc(vpc_id) link = generate_request("DeleteVpc", 'VpcId' => vpc_id ) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue Exception on_exception end #----------------- # Subnets #----------------- # Describe Subnet. # # Accepts a list of subnets and/or a set of filters as the last parameter. # # Filters: availability-zone, available-ip-address-count, cidr, state, subnet-id, tag-key, tag-value, tag:key, vpc-id # # ec2.describe_subnets #=> # [{:available_ip_address_count=>"251", # :vpc_id=>"vpc-890ce2e0", # :availability_zone=>"us-east-1a", # :subnet_id=>"subnet-770de31e", # :cidr_block=>"10.0.1.0/24", # :state=>"available"}] # # ec2.describe_subnets(:filters => {'cidr' => "192.168.1.128/25"}) # # P.S. filters: http://docs.amazonwebservices.com/AmazonVPC/latest/APIReference/index.html?ApiReference-query-DescribeSubnets.html # def describe_subnets(*list_and_options) list_and_options = merge_new_options_into_list_and_options(list_and_options, :options => {:api_version => VPC_API_VERSION}) describe_resources_with_list_and_options('DescribeSubnets', 'SubnetId', QEc2DescribeSubnetsParser, list_and_options) end # Create Subnet. # # ec2.create_subnet("vpc-890ce2e0",'10.0.1.0/24') #=> # {:available_ip_address_count=>"251", # :vpc_id=>"vpc-890ce2e0", # :availability_zone=>"us-east-1a", # :subnet_id=>"subnet-770de31e", # :cidr_block=>"10.0.1.0/24", # :state=>"pending"} # def create_subnet(vpc_id, cidr_block, availability_zone = nil) request_hash = { 'VpcId' => vpc_id, 'CidrBlock' => cidr_block } request_hash['AvailabilityZone'] = availability_zone unless availability_zone.right_blank? link = generate_request("CreateSubnet", request_hash) request_info(link, QEc2DescribeSubnetsParser.new(:logger => @logger)).first rescue Exception on_exception end # Delete Subnet. # # ec2.delete_subnet("subnet-770de31e") #=> true # def delete_subnet(subnet_id) link = generate_request("DeleteSubnet", 'SubnetId' => subnet_id ) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue Exception on_exception end #----------------- # DHCP Options #----------------- # Describe DHCP options. # # Accepts a list of DHCP options and/or a set of filters as the last parameter. # # Filters: dchp-options-id, key, value, tag-key, tag-value, tag:key # # ec2.describe_dhcp_options #=> # [{:dhcp_options_id=>"dopt-cb0de3a2", # :dhcp_configuration_set=> # {"netbios-node-type"=>["1"], "domain-name"=>["my.awesomesite.ru"]}}] # # ec2.describe_dhcp_options(:filters => {'tag:MyTag' => 'MyValue'}) # # P.S. filters: http://docs.amazonwebservices.com/AmazonVPC/latest/APIReference/index.html?ApiReference-query-DescribeDhcpOptions.html # def describe_dhcp_options(*list_and_options) describe_resources_with_list_and_options('DescribeDhcpOptions', 'DhcpOptionsId', QEc2DescribeDhcpOptionsParser, list_and_options) end # Create DHCP options. # # ec2.create_dhcp_options('domain-name' => 'my.awesomesite.ru', # 'netbios-node-type' => 1) #=> # {:dhcp_options_id=>"dopt-cb0de3a2", # :dhcp_configuration_set=> # {"netbios-node-type"=>["1"], "domain-name"=>["my.awesomesite.ru"]}} # def create_dhcp_options(dhcp_configuration) dhcp_configuration.each{ |key, values| dhcp_configuration[key] = Array(values) } request_hash = amazonize_list(['DhcpConfiguration.?.Key','DhcpConfiguration.?.Value.?'], dhcp_configuration) link = generate_request("CreateDhcpOptions", request_hash) request_info(link, QEc2DescribeDhcpOptionsParser.new(:logger => @logger)).first rescue Exception on_exception end # Associate DHCP options # # ec2.associate_dhcp_options("dopt-cb0de3a2", "vpc-890ce2e0" ) #=> true # ec2.describe_vpcs #=> # [{:vpc_id=>"vpc-890ce2e0", # :dhcp_options_id=>"dopt-cb0de3a2", # :cidr_block=>"10.0.0.0/23", # :state=>"available"}] # def associate_dhcp_options(dhcp_options_id, vpc_id) link = generate_request("AssociateDhcpOptions", 'DhcpOptionsId' => dhcp_options_id, 'VpcId' => vpc_id) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue Exception on_exception end # Delete DHCP Options. # # ec2.delete_dhcp_options("dopt-cb0de3a2") #=> true # def delete_dhcp_options(dhcp_options_id) link = generate_request("DeleteDhcpOptions", 'DhcpOptionsId' => dhcp_options_id ) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue Exception on_exception end #----------------- # Customer Gateways #----------------- # Describe customer gateways. # # Accepts a list of gateways and/or a set of filters as the last parameter. # # Filters: bgp-asn, customer-gateway-id, state, type, tag-key, tag-value, tag:key # # ec2.describe_customer_gateways #=> # [{:type=>"ipsec.1", # :ip_address=>"12.1.2.3", # :bgp_asn=>"65534", # :state=>"available", # :customer_gateway_id=>"cgw-d5a643bc"}] # # ec2.describe_customer_gateways(:filters => {'tag:MyTag' => 'MyValue'}) # # P.S. filters: http://docs.amazonwebservices.com/AmazonVPC/latest/APIReference/index.html?ApiReference-query-DescribeCustomerGateways.html # def describe_customer_gateways(*list_and_options) describe_resources_with_list_and_options('DescribeCustomerGateways', 'CustomerGatewayId', QEc2DescribeCustomerGatewaysParser, list_and_options) end # Create customer gateway. # # ec2.create_customer_gateway('ipsec.1', '12.1.2.3', 65534) #=> # {:type=>"ipsec.1", # :bgp_asn=>"65534", # :ip_address=>"12.1.2.3", # :state=>"pending", # :customer_gateway_id=>"cgw-d5a643bc"} # def create_customer_gateway(type, ip_address, bgp_asn) link = generate_request("CreateCustomerGateway", 'Type' => type, 'IpAddress' => ip_address, 'BgpAsn' => bgp_asn ) request_info(link, QEc2DescribeCustomerGatewaysParser.new(:logger => @logger)).first rescue Exception on_exception end # Delete customer gateway. # # ec2.delete_customer_gateway("cgw-d5a643bc") #=> true # def delete_customer_gateway(customer_gateway_id) link = generate_request("DeleteCustomerGateway", 'CustomerGatewayId' => customer_gateway_id ) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue Exception on_exception end #----------------- # VPN Gateways #----------------- # Describe VPN gateways. # # Accepts a list of VPN gateways and/or a set of filters as the last parameter. # # Filters: attachment.state, attachment.vpc-id, availability-zone, state, tag-key, tag-value, tag:key, type, vpn-gateway-id # # ec2.describe_vpn_gateways #=> # [{:type=>"ipsec.1", # :availability_zone=>"us-east-1a", # :attachments=>[{:vpc_id=>"vpc-890ce2e0", :state=>"attached"}], # :vpn_gateway_id=>"vgw-dfa144b6"}] # # ec2.describe_vpn_gateways(:filters => {'tag:MyTag' => 'MyValue'}) # # P.S. filters: http://docs.amazonwebservices.com/AmazonVPC/latest/APIReference/index.html?ApiReference-query-DescribeVpnGateways.html # def describe_vpn_gateways(*list_and_options) describe_resources_with_list_and_options('DescribeVpnGateways', 'VpnGatewayId', QEc2DescribeVpnGatewaysParser, list_and_options) end # Create VPN gateway. # # ec2.create_vpn_gateway('ipsec.1') #=> # {:type=>"ipsec.1", # :availability_zone=>"us-east-1a", # :attachments=>[nil], # :vpn_gateway_id=>"vgw-dfa144b6"} # def create_vpn_gateway(type, availability_zone=nil) request_hash = { 'Type' => type } request_hash['AvailabilityZone'] = availability_zone unless availability_zone.right_blank? link = generate_request("CreateVpnGateway", request_hash ) request_info(link, QEc2DescribeVpnGatewaysParser.new(:logger => @logger)).first rescue Exception on_exception end # Attach VPN gateway. # # ec2.attach_vpn_gateway('vgw-dfa144b6','vpc-890ce2e0') #=> # {:vpc_id=>"vpc-890ce2e0", :state=>"attaching"} # def attach_vpn_gateway(vpn_gateway_id, vpc_id) link = generate_request("AttachVpnGateway", 'VpnGatewayId' => vpn_gateway_id, 'VpcId' => vpc_id ) request_info(link, QEc2AttachVpnGatewayParser.new(:logger => @logger)) rescue Exception on_exception end # Detach VPN gateway. # # ec2.detach_vpn_gateway('vgw-dfa144b6','vpc-890ce2e0') #=> true # def detach_vpn_gateway(vpn_gateway_id, vpc_id) link = generate_request("DetachVpnGateway", 'VpnGatewayId' => vpn_gateway_id, 'VpcId' => vpc_id ) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue Exception on_exception end # Delete vpn gateway. # # ec2.delete_vpn_gateway("vgw-dfa144b6") #=> true # def delete_vpn_gateway(vpn_gateway_id) link = generate_request("DeleteVpnGateway", 'VpnGatewayId' => vpn_gateway_id ) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue Exception on_exception end #----------------- # VPN Connections #----------------- # Describe VPN connections. # # Accepts a list of VPN gateways and/or a set of filters as the last parameter. # # Filters: customer-gateway-configuration, customer-gateway-id, state, tag-key, tag-value, tag:key, # type, vpn-connection-id, vpn-gateway-id # # ec2.describe_vpn_connections #=> # [{:type=>"ipsec.1", # :vpn_connection_id=>"vpn-a9a643c0", # :customer_gateway_configuration=> # "\n\n...\n", # :state=>"available", # :vpn_gateway_id=>"vgw-dfa144b6", # :customer_gateway_id=>"cgw-81a643e8"}] # # ec2.describe_vpn_gateways(:filters => {'tag:MyTag' => 'MyValue'}) # # P.S. filters: http://docs.amazonwebservices.com/AmazonVPC/latest/APIReference/index.html?ApiReference-query-DescribeVpnConnections.html # def describe_vpn_connections(*list_and_options) describe_resources_with_list_and_options('DescribeVpnConnections', 'VpnConnectionId', QEc2DescribeVpnConnectionsParser, list_and_options) end # Create VPN connection. # # ec2.create_vpn_connection('ipsec.1', 'cgw-81a643e8' ,'vgw-dfa144b6') # {:customer_gateway_id=>"cgw-81a643e8", # :vpn_connection_id=>"vpn-a9a643c0", # :customer_gateway_configuration=> # "\n\n...\n", # :state=>"pending", # :vpn_gateway_id=>"vgw-dfa144b6"} # def create_vpn_connection(type, customer_gateway_id, vpn_gateway_id) link = generate_request("CreateVpnConnection", 'Type' => type, 'CustomerGatewayId' => customer_gateway_id, 'VpnGatewayId' => vpn_gateway_id ) request_info(link, QEc2DescribeVpnConnectionsParser.new(:logger => @logger)).first rescue Exception on_exception end # Delete VPN connection. # # ec2.delete_vpn_connection("vpn-a9a643c0") #=> true # def delete_vpn_connection(vpn_connection_id) link = generate_request("DeleteVpnConnection", 'VpnConnectionId' => vpn_connection_id ) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue Exception on_exception end #----------------- # Parsers #----------------- class QEc2DescribeVpcsParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{/(vpcSet/item|vpc)$} then @item = { :tags => {} } when %r{/tagSet/item$} then @aws_tag = {} end end def tagend(name) case name when 'vpcId' then @item[:vpc_id] = @text when 'state' then @item[:state] = @text when 'dhcpOptionsId' then @item[:dhcp_options_id] = @text when 'cidrBlock' then @item[:cidr_block] = @text when 'instanceTenancy' then @item[:instance_tenancy] = @text when 'isDefault' then @item[:is_default] = @text == 'true' else case full_tag_name when %r{/tagSet/item/key$} then @aws_tag[:key] = @text when %r{/tagSet/item/value$} then @aws_tag[:value] = @text when %r{/tagSet/item$} then @item[:tags][@aws_tag[:key]] = @aws_tag[:value] when %r{(vpcSet/item|vpc)$} then @result << @item end end end def reset @result = [] end end class QEc2DescribeSubnetsParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{/(subnetSet/item|subnet)$} then @item = { :tags => {} } when %r{/tagSet/item$} then @aws_tag = {} end end def tagend(name) case name when 'subnetId' then @item[:subnet_id] = @text when 'state' then @item[:state] = @text when 'vpcId' then @item[:vpc_id] = @text when 'cidrBlock' then @item[:cidr_block] = @text when 'availabilityZone' then @item[:availability_zone] = @text when 'availableIpAddressCount' then @item[:available_ip_address_count] = @text when 'defaultForAz' then @item[:default_for_az] = @text == 'true' when 'mapPublicIpOnLaunch' then @item[:map_public_ip_on_launch] = @text == 'true' else case full_tag_name when %r{/tagSet/item/key$} then @aws_tag[:key] = @text when %r{/tagSet/item/value$} then @aws_tag[:value] = @text when %r{/tagSet/item$} then @item[:tags][@aws_tag[:key]] = @aws_tag[:value] when %r{/(subnetSet/item|subnet)$} then @result << @item end end end def reset @result = [] end end class QEc2DescribeDhcpOptionsParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{/(dhcpOptionsSet/item|dhcpOptions)$} then @item = { :tags => {}, :dhcp_configuration_set => {} } when %r{/tagSet/item$} then @aws_tag = {} end end def tagend(name) case full_tag_name when %r{/tagSet/item/key$} then @aws_tag[:key] = @text when %r{/tagSet/item/value$} then @aws_tag[:value] = @text when %r{/tagSet/item$} then @item[:tags][@aws_tag[:key]] = @aws_tag[:value] when %r{/dhcpOptionsId$} then @item[:dhcp_options_id] = @text when %r{/dhcpConfigurationSet/item/key$} then @conf_item_key = @text when %r{/dhcpConfigurationSet/item/valueSet/item/value$} then (@item[:dhcp_configuration_set][@conf_item_key] ||= []) << @text when %r{/(dhcpOptionsSet/item|dhcpOptions)$} then @result << @item end end def reset @result = [] end end class QEc2DescribeCustomerGatewaysParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{/(customerGatewaySet/item|customerGateway)$} then @item = { :tags => {} } when %r{/tagSet/item$} then @aws_tag = {} end end def tagend(name) case name when 'customerGatewayId' then @item[:customer_gateway_id] = @text when 'state' then @item[:state] = @text when 'type' then @item[:type] = @text when 'ipAddress' then @item[:ip_address] = @text when 'bgpAsn' then @item[:bgp_asn] = @text else case full_tag_name when %r{/tagSet/item/key$} then @aws_tag[:key] = @text when %r{/tagSet/item/value$} then @aws_tag[:value] = @text when %r{/tagSet/item$} then @item[:tags][@aws_tag[:key]] = @aws_tag[:value] when %r{/(customerGatewaySet/item|customerGateway)$} then @result << @item end end end def reset @result = [] end end class QEc2DescribeVpnGatewaysParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{/(vpnGatewaySet/item|vpnGateway)$} then @item = { :tags => {}, :attachments => [] } when %r{/attachments/item$} then @attachment = {} when %r{/tagSet/item$} then @aws_tag = {} end end def tagend(name) case name when 'vpnGatewayId' then @item[:vpn_gateway_id] = @text when 'availabilityZone' then @item[:availability_zone] = @text when 'type' then @item[:type] = @text when 'vpcId' then @attachment[:vpc_id] = @text else case full_tag_name when %r{/vpnGatewaySet/item/state$} then @item[:state] = @text when %r{/attachments/item/state$} then @attachment[:state] = @text when %r{/attachments/item$} then @item[:attachments] << @attachment unless @attachment.right_blank? when %r{/tagSet/item/key$} then @aws_tag[:key] = @text when %r{/tagSet/item/value$} then @aws_tag[:value] = @text when %r{/tagSet/item$} then @item[:tags][@aws_tag[:key]] = @aws_tag[:value] when %r{/(vpnGatewaySet/item|vpnGateway)$} then @result << @item end end end def reset @result = [] end end class QEc2AttachVpnGatewayParser < RightAWSParser #:nodoc: def tagend(name) case name when 'vpcId' then @result[:vpc_id] = @text when 'state' then @result[:state] = @text end end def reset @result = {} end end class QEc2DescribeVpnConnectionsParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{/(vpnConnectionSet/item|vpnConnection)$} then @item = { :tags => {} } when %r{/tagSet/item$} then @aws_tag = {} end end def tagend(name) case name when 'vpnConnectionId' then @item[:vpn_connection_id] = @text when 'state' then @item[:state] = @text when 'type' then @item[:type] = @text when 'vpnGatewayId' then @item[:vpn_gateway_id] = @text when 'customerGatewayId' then @item[:customer_gateway_id] = @text when 'customerGatewayConfiguration' then @item[:customer_gateway_configuration] = @text else case full_tag_name when %r{/tagSet/item/key$} then @aws_tag[:key] = @text when %r{/tagSet/item/value$} then @aws_tag[:value] = @text when %r{/tagSet/item$} then @item[:tags][@aws_tag[:key]] = @aws_tag[:value] when %r{/(vpnConnectionSet/item|vpnConnection)$} then @result << @item end end end def reset @result = [] end end end end ================================================ FILE: lib/ec2/right_ec2_vpc2.rb ================================================ # # Copyright (c) 2011 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws class Ec2 # VPC v2: Limits # # http://docs.amazonwebservices.com/AmazonVPC/latest/UserGuide/index.html?VPC_Appendix_Limits.html #---------------------- # InternetGateways #---------------------- # Create internet gateway # # ec2.create_internet_gateway #=> # { :internet_gateway_id=>"igw-6585c10c", :tags=>{}} # # P.S. http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-CreateInternetGateway.html # def create_internet_gateway link = generate_request("CreateInternetGateway") request_info(link, QEc2DescribeInternetGatewaysParser.new(:logger => @logger)).first rescue Exception on_exception end # Describe internet gateways. # # ec2.describe_internet_gateways #=> # [{:state=>"available", # :internet_gateway_id=>"igw-6585c10c", # :vpc_id=>"vpc-df80a6b6", # :tags=>{}}, # {:internet_gateway_id=>"igw-883503e1", # :tags=>{}}] # # P.S. http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-CreateInternetGateway.html # def describe_internet_gateways(*list_and_options) describe_resources_with_list_and_options('DescribeInternetGateways', 'InternetGatewayId', QEc2DescribeInternetGatewaysParser, list_and_options) rescue Exception on_exception end # Delete internet gateway. # # ec2.delete_internet_gateway("igw-6585c10c") #=> true # # P.S. http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference-query-DeleteInternetGateway.html # def delete_internet_gateway(internet_gateway_id) link = generate_request("DeleteInternetGateway", 'InternetGatewayId' => internet_gateway_id ) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue Exception on_exception end # Attaches an Internet gateway to a VPC, enabling connectivity between the Internet and the VPC. # # ec2.attach_internet_gateway("igw-6585c10c", "vpc-df80a6b6") #=> true # # P.S. http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference-query-AttachInternetGateway.html # def attach_internet_gateway(internet_gateway_id, vpc_id) request_hash = { 'InternetGatewayId' => internet_gateway_id, 'VpcId' => vpc_id } link = generate_request("AttachInternetGateway", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue Exception on_exception end # Detaches an Internet gateway from a VPC, disabling connectivity between the Internet and the VPC. # The VPC must not contain any running instances with Elastic IP addresses. # # ec2.detach_internet_gateway("igw-6585c10c", "vpc-df80a6b6") #=> true # # P.S. http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference-query-DetachInternetGateway.html # def detach_internet_gateway(internet_gateway_id, vpc_id) request_hash = { 'InternetGatewayId' => internet_gateway_id, 'VpcId' => vpc_id } link = generate_request("DetachInternetGateway", request_hash) request_info(link, RightHttp2xxParser::new(:logger => @logger)) rescue Exception on_exception end #---------------------- # RouteTables #---------------------- # Describe route tables. # # # List all tables # ec2.describe_route_tables #=> # [{:route_table_id=>"rtb-be3006d7", # :route_set=> # [{:state=>"active", # :destination_cidr_block=>"10.0.3.0/24", # :gateway_id=>"local"}], # :vpc_id=>"vpc-df80a6b6", # :association_set=>[], # :tags=>{}}, # {:route_table_id=>"rtb-e36cf98a", # :route_set=> # [{:state=>"active", # :destination_cidr_block=>"192.168.0.0/24", # :gateway_id=>"local"}], # :vpc_id=>"vpc-e16cf988", # :association_set=> # [{:route_table_id=>"rtb-e36cf98a", # :main=>true, # :route_table_association_id=>"rtbassoc-e26cf98b"}], # :tags=>{}}, ... ] # # # Filter tables by VpcId # ec2.describe_route_tables(:filters => {'vpc-id' => "vpc-df80a6b6"}) # # # Custom route table # ec2.describe_route_tables("rtb-be3006d7") #=> # [{:vpc_id=>"vpc-df80a6b6", # :route_set=> # [{:state=>"active", # :destination_cidr_block=>"0.0.0.1/32", # :gateway_id=>"igw-6585c10c"}, # {:state=>"active", # :destination_cidr_block=>"10.0.3.0/24", # :gateway_id=>"local"}], # :route_table_id=>"rtb-be3006d7", # :tags=>{}, # :association_set=> # [{:route_table_association_id=>"rtbassoc-a02610c9", # :subnet_id=>"subnet-b95f76d0", # :route_table_id=>"rtb-be3006d7"}]}] # # P.S. http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-DescribeRouteTables.html # def describe_route_tables(*list_and_options) describe_resources_with_list_and_options('DescribeRouteTables', 'RouteTableId', QEc2DescribeRouteTablesParser, list_and_options) rescue Exception on_exception end # Creates a new route table within a VPC. After you create a new route table, you can add routes and associate the table with a subne # # ec2.create_route_table("vpc-df80a6b6") #=> # {:route_table_id=>"rtb-4331072a", # :route_set=> # [{:state=>"active", # :destination_cidr_block=>"10.0.3.0/24", # :gateway_id=>"local"}], # :vpc_id=>"vpc-df80a6b6", # :association_set=>[], # :tags=>{}} # # P.S. http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-CreateRouteTable.html # def create_route_table(vpc_id) link = generate_request("CreateRouteTable", 'VpcId' => vpc_id ) request_info(link, QEc2DescribeRouteTablesParser::new(:logger => @logger)).first rescue Exception on_exception end # Deletes a route table from a VPC. # The route table must not be associated with a subnet. You can't delete the main route table. # # ec2.delete_route_table("rtb-4331072a") #=> true # # P.S. http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-DeleteRouteTable.html def delete_route_table(route_table_id) link = generate_request("DeleteRouteTable", 'RouteTableId' => route_table_id ) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue Exception on_exception end # Associates a subnet with a route table. The subnet and route table must be in the same VPC. # This association causes traffic originating from the subnet to be routed according to the routes in # the route table. The action returns an association ID, which you need if you want to disassociate # the route table from the subnet later. A route table can be associated with multiple subnets. # # ec2.associate_route_table("rtb-be3006d7", "subnet-b95f76d0") #=> true # # P.S. http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-AssociateRouteTable.html # def associate_route_table(route_table_id, subnet_id) request_hash = { 'RouteTableId' => route_table_id, 'SubnetId' => subnet_id } link = generate_request("AssociateRouteTable", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue Exception on_exception end # Disassociates a subnet from a route table. # # ec2.disassociate_route_table(route_table_association_id) #=> true # # P.S. http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference-query-DisassociateRouteTable.html # def disassociate_route_table(route_table_association_id) link = generate_request("DisassociateRouteTable", 'AssociationId' => route_table_association_id ) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue Exception on_exception end # Changes the route table associated with a given subnet in a VPC. After you execute this action, the subnet # uses the routes in the new route table it's associated with. # You can also use this action to change which table is the main route table in the VPC. You just specify # the main route table's association ID and the route table that you want to be the new main route table. # # ec2.replace_route_table_association("rtb-be3006d7", "rtbassoc-a02610c9") #=> true # # P.S. http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference-query-ReplaceRouteTableAssociation.html # def replace_route_table_association(route_table_id, route_table_association_id) request_hash = { 'RouteTableId' => route_table_id, 'AssociationId' => route_table_association_id } link = generate_request("ReplaceRouteTableAssociation", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue Exception on_exception end #--------------------- # Routes #--------------------- # Creates a new route in a route table within a VPC. The route's target can be either a gateway attached to # the VPC or a NAT instance in the VPC. # Options: :gateway_id, :instance_id # # ec2.create_route("rtb-be3006d7", "0.0.0.1/32", :gateway_id => 'igw-6585c10c') #=> true # # P.S. http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference-query-CreateRoute.html # def create_route(route_table_id, destination_cidr_block, options = {}) request_hash = { 'RouteTableId' => route_table_id, 'DestinationCidrBlock' => destination_cidr_block } request_hash['GatewayId'] = options[:gateway_id] unless options[:gateway_id].right_blank? request_hash['InstanceId'] = options[:instance_id] unless options[:instance_id].right_blank? link = generate_request("CreateRoute", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue Exception on_exception end # Deletes a route from a route table in a VPC. # # ec2.delete_route("rtb-be3006d7", "0.0.0.1/32") #=> true # # P.S. http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference-query-DeleteRoute.html # def delete_route(route_table_id, destination_cidr_block) link = generate_request("DeleteRoute", 'RouteTableId' => route_table_id, 'DestinationCidrBlock' => destination_cidr_block ) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue Exception on_exception end # Replaces an existing route within a route table in a VPC # Options: :gateway_id, :instance_id # # ec2.replace_route("rtb-be3006d7", "0.0.0.2/32", :gateway_id => 'igw-6585c10c') #=> true # # P.S. http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference-query-ReplaceRoute.html # def replace_route(route_table_id, destination_cidr_block, options = {}) request_hash = { 'RouteTableId' => route_table_id, 'DestinationCidrBlock' => destination_cidr_block } request_hash['GatewayId'] = options[:gateway_id] unless options[:gateway_id].right_blank? request_hash['InstanceId'] = options[:instance_id] unless options[:instance_id].right_blank? link = generate_request("ReplaceRoute", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue Exception on_exception end #--------------------- # InternetGateways #--------------------- class QEc2DescribeInternetGatewaysParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{/(internetGatewaySet/item|internetGateway)$} then @item = { :tags => {} } when %r{/tagSet/item$} then @aws_tag = {} end end def tagend(name) case full_tag_name # item when %r{/(internetGatewaySet/item|internetGateway)$} then @result << @item when %r{/internetGatewayId$} then @item[:internet_gateway_id] = @text when %r{/vpcId$} then @item[:vpc_id] = @text when %r{/state$} then @item[:state] = @text # tags when %r{/tagSet/item/key$} then @aws_tag[:key] = @text when %r{/tagSet/item/value$} then @aws_tag[:value] = @text when %r{/tagSet/item$} then @item[:tags][@aws_tag[:key]] = @aws_tag[:value] end end def reset @result = [] end end #--------------------- # Routes #--------------------- class QEc2DescribeRouteTablesParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{/(routeTableSet/item|routeTable)$} @item = { :route_set => [], :association_set => [], :tags => {}} when %r{/routeSet/item$} then @route_set = {} when %r{/associationSet/item$} then @association_set = {} when %r{/tagSet/item$} then @aws_tag = {} end end def tagend(name) case full_tag_name # item when %r{/(routeTableSet/item|routeTable)/routeTableId$} then @item[:route_table_id] = @text when %r{/(routeTableSet/item|routeTable)/vpcId$} then @item[:vpc_id] = @text when %r{/(routeTableSet/item|routeTable)$} then @result << @item # route set when %r{/routeSet/item/destinationCidrBlock$} then @route_set[:destination_cidr_block] = @text when %r{/routeSet/item/gatewayId$} then @route_set[:gateway_id] = @text when %r{/routeSet/item/instanceId$} then @route_set[:instance_id] = @text when %r{/routeSet/item/state$} then @route_set[:state] = @text when %r{/routeSet/item$} then @item[:route_set] << @route_set # association set when %r{/associationSet/item/routeTableId$} then @association_set[:route_table_id] = @text when %r{/associationSet/item/routeTableAssociationId$} then @association_set[:route_table_association_id] = @text when %r{/associationSet/item/subnetId$} then @association_set[:subnet_id] = @text when %r{/associationSet/item/main} then @association_set[:main] = @text == 'true' when %r{/associationSet/item$} then @item[:association_set] << @association_set # tags when %r{/tagSet/item/key$} then @aws_tag[:key] = @text when %r{/tagSet/item/value$} then @aws_tag[:value] = @text when %r{/tagSet/item$} then @item[:tags][@aws_tag[:key]] = @aws_tag[:value] end end def reset @result = [] end end end end ================================================ FILE: lib/ec2/right_ec2_windows_mobility.rb ================================================ # # Copyright (c) 2010 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws class Ec2 def describe_licenses(*license_ids) link = generate_request("DescribeLicenses", amazonize_list('LicenseId', license_ids)) request_info(link, QEc2DescribeLicensesParser.new(:logger => @logger)) end def activate_license(license_id, capacity) link = generate_request("ActivateLicense", 'LicenseId' => license_id, 'Capacity' => capacity) request_info(link, RightBoolResponseParser.new(:logger => @logger)) end # def get_license_capacity(license_id) # link = generate_request("GetLicenseCapacity", 'LicenseId' => license_id) # request_info(link, RightBoolResponseParser.new(:logger => @logger)) # end def deactivate_license(license_id, capacity) link = generate_request("DeactivateLicense", 'LicenseId' => license_id, 'Capacity' => capacity) request_info(link, RightBoolResponseParser.new(:logger => @logger)) end #----------------------------------------------------------------- # PARSERS: Images #----------------------------------------------------------------- class QEc2DescribeLicensesParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{/licenseSet/item$} then @item = { :capacities => [] } when %r{/capacitySet/item$} then @capacity_item = {} end end def tagend(name) case name when 'licenseId' then @item[:license_id] = @text when 'type' then @item[:type] = @text when 'pool' then @item[:pool] = @text when 'capacity' then @capacity_item[:capacity] = @text.to_i when 'instanceCapacity' then @capacity_item[:instance_capacity] = @text.to_i when 'state' then @capacity_item[:state] = @text when 'earliestAllowedDeactivationTime' then @capacity_item[:earliest_allowed_deactivation_time] = @text else case full_tag_name when %r{/capacitySet/item$} then @item[:capacities] << @capacity_item when %r{/licenseSet/item$} then @result << @item end end end def reset @result = [] end end end end ================================================ FILE: lib/elb/right_elb_interface.rb ================================================ # # Copyright (c) 2007-2009 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws # = RightAWS::ElbInterface -- RightScale Amazon Elastic Load Balancer interface # The RightAws::ElbInterface class provides a complete interface to Amazon's # Elastic Load Balancer service. # # For explanations of the semantics of each call, please refer to Amazon's documentation at # http://docs.amazonwebservices.com/ElasticLoadBalancing/latest/DeveloperGuide/ # # Create an interface handle: # # elb = RightAws::ElbInterface.new(aws_access_key_id, aws_security_access_key) # # Create an new load balancer: # # elb.create_load_balancer( 'test-kd1', # ['us-east-1a', 'us-east-1b'], # [ { :protocol => :http, :load_balancer_port => 80, :instance_port => 80 }, # { :protocol => :tcp, :load_balancer_port => 443, :instance_port => 443 } ]) # # Configure its health checking: # # elb.configure_health_check( 'test-kd1', # { :healthy_threshold => 9, # :unhealthy_threshold => 3, # :target => "TCP:433", # :timeout => 6, # :interval => 31} # # Register instances with the balancer: # # elb.register_instances_with_load_balancer('test-kd1', 'i-8b8bcbe2', 'i-bf8bcbd6') #=> ["i-8b8bcbe2", "i-bf8bcbd6"] # # Add new availability zones: # # elb.enable_availability_zones_for_load_balancer("test-kd1", "us-east-1c") # class ElbInterface < RightAwsBase include RightAwsBaseInterface # Amazon ELB API version being used API_VERSION = "2011-04-05" DEFAULT_HOST = "elasticloadbalancing.amazonaws.com" DEFAULT_PATH = '/' DEFAULT_PROTOCOL = 'https' DEFAULT_PORT = 443 LISTENER_PROTOCOLS = [ 'HTTP', 'HTTPS', 'TCP', 'SSL' ] @@bench = AwsBenchmarkingBlock.new def self.bench_xml @@bench.xml end def self.bench_service @@bench.service end # Create a new handle to an ELB account. All handles share the same per process or per thread # HTTP connection to Amazon ELB. Each handle is for a specific account. The params have the # following options: # * :endpoint_url a fully qualified url to Amazon API endpoint (this overwrites: :server, :port, :service, :protocol). Example: 'https://elasticloadbalancing.amazonaws.com' # * :server: ELB service host, default: DEFAULT_HOST # * :port: ELB service port, default: DEFAULT_PORT # * :protocol: 'http' or 'https', default: DEFAULT_PROTOCOL # * :logger: for log messages, default: RAILS_DEFAULT_LOGGER else STDOUT # * :signature_version: The signature version : '0','1' or '2'(default) # * :cache: true/false(default): caching works for: describe_load_balancers # def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={}) init({ :name => 'ELB', :default_host => ENV['ELB_URL'] ? URI.parse(ENV['ELB_URL']).host : DEFAULT_HOST, :default_port => ENV['ELB_URL'] ? URI.parse(ENV['ELB_URL']).port : DEFAULT_PORT, :default_service => ENV['ELB_URL'] ? URI.parse(ENV['ELB_URL']).path : DEFAULT_PATH, :default_protocol => ENV['ELB_URL'] ? URI.parse(ENV['ELB_URL']).scheme : DEFAULT_PROTOCOL, :default_api_version => ENV['ELB_API_VERSION'] || API_VERSION }, aws_access_key_id || ENV['AWS_ACCESS_KEY_ID'] , aws_secret_access_key|| ENV['AWS_SECRET_ACCESS_KEY'], params) end def generate_request(action, params={}) #:nodoc: generate_request_impl(:get, action, params ) end # Sends request to Amazon and parses the response # Raises AwsError if any banana happened def request_info(request, parser) #:nodoc: request_info_impl(:lbs_connection, @@bench, request, parser) end #----------------------------------------------------------------- # Load Balancers #----------------------------------------------------------------- # Describe load balancers. # Returns an array of load balancers. # # elb.describe_load_balancers #=> # [ { :health_check => # { :healthy_threshold => 10, # :unhealthy_threshold => 2, # :target => "TCP:80", # :timeout => 5, # :interval => 30}, # :load_balancer_name => "test-kd1", # :availability_zones => ["us-east-1a", "us-east-1b"], # :listeners => # [ { :protocol => "HTTP", :load_balancer_port => "80", :instance_port => "80" }, # { :protocol => "TCP", :load_balancer_port => "443", :instance_port => "443" } ], # :created_time => "2009-05-27T11:59:11.000Z", # :dns_name => "test-kd1-1519253964.us-east-1.elb.amazonaws.com", # :instances => [] } ] # # elb.describe_load_balancers("test-kd1") #=> # [{:load_balancer_name=>"test-kd1", # :instances=>["i-9fc056f4", "i-b3debfd8"], # :health_check=> # {:interval=>30, # :healthy_threshold=>10, # :target=>"TCP:80", # :unhealthy_threshold=>2, # :timeout=>5}, # :dns_name=>"test-kd1-869291821.us-east-1.elb.amazonaws.com", # :listeners=> # [{:load_balancer_port=>"80", # :policy_names=>["my-policy-1"], # :instance_port=>"80", # :protocol=>"HTTP"}, # {:load_balancer_port=>"8080", # :policy_names=>["my-policy-lb-1"], # :instance_port=>"8080", # :protocol=>"HTTP"}, # {:load_balancer_port=>"443", # :policy_names=>[], # :instance_port=>"443", # :protocol=>"TCP"}], # :created_time=>"2010-04-15T12:04:49.000Z", # :availability_zones=>["us-east-1a", "us-east-1b"], # :app_cookie_stickiness_policies=> # [{:policy_name=>"my-policy-1", :cookie_name=>"my-cookie-1"}], # :lb_cookie_stickiness_policies=> # [{:cookie_expiration_period=>60, :policy_name=>"my-policy-lb-1"}]}] # def describe_load_balancers(*load_balancers) load_balancers = load_balancers.flatten.compact request_hash = amazonize_list("LoadBalancerNames.member", load_balancers) link = generate_request("DescribeLoadBalancers", request_hash) request_cache_or_info(:describe_load_balancers, link, DescribeLoadBalancersParser, @@bench, load_balancers.right_blank?) end # Create new load balancer. # Returns a new load balancer DNS name. # # Listener options: :protocol, :load_balancer_port, :instance_port and :ssl_certificate_id # Protocols: :tcp, :http, :https or :ssl # # elb.create_load_balancer( 'test-kd1', # ['us-east-1a', 'us-east-1b'], # [ { :protocol => :http, :load_balancer_port => 80, :instance_port => 80 }, # { :protocol => :https, :load_balancer_port => 443, :instance_port => 443, # :ssl_certificate_id => 'arn:aws:iam::123456789012:user/division_abc/subdivision_xyz/Bob' } ]) # #=> "test-kd1-1519253964.us-east-1.elb.amazonaws.com" # def create_load_balancer(load_balancer_name, availability_zones=[], listeners=[]) request_hash = { 'LoadBalancerName' => load_balancer_name } # merge zones request_hash.merge!( amazonize_list("AvailabilityZones.member", availability_zones) ) # merge listeners if listeners.right_blank? listeners = { :protocol => :http, :load_balancer_port => 80, :instance_port => 80 } end request_hash = merge_listeners_into_request_hash(request_hash, listeners) link = generate_request("CreateLoadBalancer", request_hash) request_info(link, CreateLoadBalancerParser.new(:logger => @logger)) end # Delete load balancer. # Returns +true+ on success. # # elb.delete_load_balancer('test-kd1') #=> true # # Amazon: Because this API has been designed to be idempotent, even if the LoadBalancer does not exist or # has been deleted, DeleteLoadBalancer still returns a success. # def delete_load_balancer(load_balancer_name) link = generate_request("DeleteLoadBalancer", 'LoadBalancerName' => load_balancer_name) request_info(link, DeleteLoadBalancerParser.new(:logger => @logger)) end # Creates one or more new listeners on a LoadBalancer for the specified port. If a listener with the given # port does not already exist, it will be created; otherwise, the properties of the new listener must match # the the properties of the existing listener. # # Listener options: :protocol, :load_balancer_port, :instance_port and :ssl_certificate_id # Protocols: :tcp, :http, :https or :ssl # # elb.create_load_balancer_listeners( 'test-kd1', # [ { :protocol => :http, :load_balancer_port => 80, :instance_port => 80 }, # { :protocol => :https, :load_balancer_port => 443, :instance_port => 443, # :ssl_certificate_id => 'arn:aws:iam::123456789012:user/division_abc/subdivision_xyz/Bob' } ]) #=> true # def create_load_balancer_listeners(load_balancer_name, listeners) request_hash = { 'LoadBalancerName' => load_balancer_name } request_hash = merge_listeners_into_request_hash(request_hash, listeners) link = generate_request("CreateLoadBalancerListeners", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Removes listeners from the load balancer for the specified port number. # # elb.delete_load_balancer_listeners( 'kd_test', 80, 443) #=> true # def delete_load_balancer_listeners(load_balancer_name, *load_balancer_ports) load_balancer_ports.flatten! request_hash = { 'LoadBalancerName' => load_balancer_name } request_hash.merge!( amazonize_list("LoadBalancerPorts.member", load_balancer_ports ) ) link = generate_request("DeleteLoadBalancerListeners", request_hash ) request_info(link, DeleteLoadBalancerParser.new(:logger => @logger)) end # Add one or more zones to a load balancer. # Returns a list of updated availability zones for the load balancer. # # elb.enable_availability_zones_for_load_balancer("test-kd1", "us-east-1c") #=> ["us-east-1a", "us-east-1c"] # def enable_availability_zones_for_load_balancer(load_balancer_name, *availability_zones) availability_zones.flatten! request_hash = amazonize_list("AvailabilityZones.member", availability_zones) request_hash.merge!( 'LoadBalancerName' => load_balancer_name ) link = generate_request("EnableAvailabilityZonesForLoadBalancer", request_hash) request_info(link, AvailabilityZonesForLoadBalancerParser.new(:logger => @logger)) end # Remove one or more zones from a load balancer. # Returns a list of updated availability zones for the load balancer. # # elb.disable_availability_zones_for_load_balancer("test-kd1", "us-east-1c") #=> ["us-east-1a"] # def disable_availability_zones_for_load_balancer(load_balancer_name, *availability_zones) availability_zones.flatten! request_hash = amazonize_list("AvailabilityZones.member", availability_zones) request_hash.merge!( 'LoadBalancerName' => load_balancer_name ) link = generate_request("DisableAvailabilityZonesForLoadBalancer", request_hash) request_info(link, AvailabilityZonesForLoadBalancerParser.new(:logger => @logger)) end # Define an application healthcheck for the instances. # Returns an updated health check configuration for the load balancer. # # hc = elb.configure_health_check( 'test-kd1', # { :healthy_threshold => 9, # :unhealthy_threshold => 3, # :target => "TCP:433", # :timeout => 6, # :interval => 31} # pp hc #=> { :target=>"TCP:433", :timeout=>6, :interval=>31, :healthy_threshold=>9, :unhealthy_threshold=>3 } # def configure_health_check(load_balancer_name, health_check) request_hash = { 'LoadBalancerName' => load_balancer_name } health_check.each{ |key, value| request_hash["HealthCheck.#{key.to_s.right_camelize}"] = value } link = generate_request("ConfigureHealthCheck", request_hash) request_info(link, HealthCheckParser.new(:logger => @logger)) end #----------------------------------------------------------------- # Instances #----------------------------------------------------------------- # Describe the current state of the instances of the specified load balancer. # Returns a list of the instances. # # elb.describe_instance_health('test-kd1', 'i-8b8bcbe2', 'i-bf8bcbd6') #=> # [ { :description => "Instance registration is still in progress", # :reason_code => "ELB", # :instance_id => "i-8b8bcbe2", # :state => "OutOfService" }, # { :description => "Instance has failed at least the UnhealthyThreshold number of health checks consecutively.", # :reason_code => "Instance", # :instance_id => "i-bf8bcbd6", # :state => "OutOfService" } ] # def describe_instance_health(load_balancer_name, *instances) instances.flatten! request_hash = amazonize_list("Instances.member.?.InstanceId", instances) request_hash.merge!( 'LoadBalancerName' => load_balancer_name ) link = generate_request("DescribeInstanceHealth", request_hash) request_info(link, DescribeInstanceHealthParser.new(:logger => @logger)) end # Add new instance(s) to the load balancer. # Returns an updated list of instances for the load balancer. # # elb.register_instances_with_load_balancer('test-kd1', 'i-8b8bcbe2', 'i-bf8bcbd6') #=> ["i-8b8bcbe2", "i-bf8bcbd6"] # def register_instances_with_load_balancer(load_balancer_name, *instances) instances.flatten! request_hash = amazonize_list("Instances.member.?.InstanceId", instances) request_hash.merge!( 'LoadBalancerName' => load_balancer_name ) link = generate_request("RegisterInstancesWithLoadBalancer", request_hash) request_info(link, InstancesWithLoadBalancerParser.new(:logger => @logger)) end # Remove instance(s) from the load balancer. # Returns an updated list of instances for the load balancer. # # elb.deregister_instances_with_load_balancer('test-kd1', 'i-8b8bcbe2') #=> ["i-bf8bcbd6"] # def deregister_instances_with_load_balancer(load_balancer_name, *instances) instances.flatten! request_hash = amazonize_list("Instances.member.?.InstanceId", instances) request_hash.merge!( 'LoadBalancerName' => load_balancer_name ) link = generate_request("DeregisterInstancesFromLoadBalancer", request_hash) request_info(link, InstancesWithLoadBalancerParser.new(:logger => @logger)) end #----------------------------------------------------------------- # Cookies #----------------------------------------------------------------- # Generates a stickiness policy with sticky session lifetimes that follow # that of an application-generated cookie. # This policy can only be associated with HTTP listeners. # # elb.create_app_cookie_stickiness_policy('my-load-balancer', 'MyLoadBalancerPolicy', 'MyCookie') #=> true # def create_app_cookie_stickiness_policy(load_balancer_name, policy_name, cookie_name) request_hash = { 'LoadBalancerName' => load_balancer_name, 'PolicyName' => policy_name, 'CookieName' => cookie_name } link = generate_request("CreateAppCookieStickinessPolicy", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Generates a stickiness policy with sticky session lifetimes controlled by the # lifetime of the browser (user-agent) or a specified expiration period. # This policy can only be associated only with HTTP listeners. # # elb.create_lb_cookie_stickiness_policy('my-load-balancer', 'MyLoadBalancerPolicy', 60) #=> true # def create_lb_cookie_stickiness_policy(load_balancer_name, policy_name, cookie_expiration_period) request_hash = { 'LoadBalancerName' => load_balancer_name, 'PolicyName' => policy_name, 'CookieExpirationPeriod' => cookie_expiration_period } link = generate_request("CreateLBCookieStickinessPolicy", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Associates, updates, or disables a policy with a listener on the load balancer. # Only zero(0) or one(1) policy can be associated with a listener. # # elb.set_load_balancer_policies_of_listener('my-load-balancer', 80, 'MyLoadBalancerPolicy') #=> true # def set_load_balancer_policies_of_listener(load_balancer_name, load_balancer_port, *policy_names) policy_names.flatten! request_hash = { 'LoadBalancerName' => load_balancer_name, 'LoadBalancerPort' => load_balancer_port } if policy_names.right_blank? request_hash['PolicyNames'] = '' else request_hash.merge!(amazonize_list('PolicyNames.member', policy_names)) end link = generate_request("SetLoadBalancerPoliciesOfListener", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Deletes a policy from the load balancer. The specified policy must not be enabled for any listeners. # # elb.delete_load_balancer_policy('my-load-balancer', 'MyLoadBalancerPolicy') #=> true # def delete_load_balancer_policy(load_balancer_name, policy_name) request_hash = { 'LoadBalancerName' => load_balancer_name, 'PolicyName' => policy_name } link = generate_request("DeleteLoadBalancerPolicy", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end def set_load_balancer_listener_ssl_certificate(load_balancer_name, load_balancer_port, ssl_sertificate_id) request_hash = { 'LoadBalancerName' => load_balancer_name, 'LoadBalancerPort' => load_balancer_port, 'SSLCertificateId' => ssl_sertificate_id } link = generate_request("SetLoadBalancerListenerSSLCertificate", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end #----------------------------------------------------------------- # Helpers #----------------------------------------------------------------- def merge_listeners_into_request_hash(request_hash, listeners) # :nodoc: listeners = [listeners] unless listeners.is_a?(Array) request_hash.merge(amazonize_list( ['Listeners.member.?.Protocol', 'Listeners.member.?.LoadBalancerPort', 'Listeners.member.?.InstancePort', 'Listeners.member.?.SSLCertificateId'], listeners.map{ |i| [ (i[:protocol] || 'HTTP').to_s.upcase, i[:load_balancer_port] || 80, i[:instance_port] || 80, i[:ssl_certificate_id]] }, :default => :skip_nils ) ) end #----------------------------------------------------------------- # PARSERS: Load Balancers #----------------------------------------------------------------- class DescribeLoadBalancersParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{LoadBalancerDescriptions/member$} @item = { :availability_zones => [], :health_check => {}, :listeners => [], :instances => [], :app_cookie_stickiness_policies => [], :lb_cookie_stickiness_policies => []} when %r{ListenerDescriptions/member$} then @listener = {:policy_names => []} when %r{AppCookieStickinessPolicies/member$} then @app_cookie_stickiness_policy = {} when %r{LBCookieStickinessPolicies/member$} then @lb_cookie_stickiness_policy = {} end end def tagend(name) case name when 'LoadBalancerName' then @item[:load_balancer_name] = @text when 'DNSName' then @item[:dns_name] = @text when 'CreatedTime' then @item[:created_time] = @text when 'Interval' then @item[:health_check][:interval] = @text.to_i when 'Target' then @item[:health_check][:target] = @text when 'HealthyThreshold' then @item[:health_check][:healthy_threshold] = @text.to_i when 'Timeout' then @item[:health_check][:timeout] = @text.to_i when 'UnhealthyThreshold' then @item[:health_check][:unhealthy_threshold] = @text.to_i when 'Protocol' then @listener[:protocol] = @text when 'LoadBalancerPort' then @listener[:load_balancer_port] = @text when 'InstancePort' then @listener[:instance_port] = @text when 'SSLCertificateId' then @listener[:ssl_certificate_id] = @text when 'CanonicalHostedZoneName' then @item[:canonical_hosted_zone_name] = @text when 'CanonicalHostedZoneNameID' then @item[:canonical_hosted_zone_name_id] = @text end case full_tag_name when %r{AvailabilityZones/member$} then @item[:availability_zones] << @text when %r{Instances/member/InstanceId$} then @item[:instances] << @text when %r{ListenerDescriptions/member$} then @item[:listeners] << @listener when %r{ListenerDescriptions/member/PolicyNames/member$} then @listener[:policy_names] << @text when %r{AppCookieStickinessPolicies/member} case name when 'PolicyName' then @app_cookie_stickiness_policy[:policy_name] = @text when 'CookieName' then @app_cookie_stickiness_policy[:cookie_name] = @text when 'member' then @item[:app_cookie_stickiness_policies] << @app_cookie_stickiness_policy end when %r{LBCookieStickinessPolicies/member} case name when 'PolicyName' then @lb_cookie_stickiness_policy[:policy_name] = @text when 'CookieExpirationPeriod' then @lb_cookie_stickiness_policy[:cookie_expiration_period] = @text.to_i when 'member' then @item[:lb_cookie_stickiness_policies] << @lb_cookie_stickiness_policy end when %r{LoadBalancerDescriptions/member$} @item[:availability_zones].sort! @item[:instances].sort! @result << @item end end def reset @result = [] end end class CreateLoadBalancerParser < RightAWSParser #:nodoc: def tagend(name) @result = @text if name == 'DNSName' end end class DeleteLoadBalancerParser < RightAWSParser #:nodoc: def tagend(name) @result = true if name == 'DeleteLoadBalancerResult' end end class AvailabilityZonesForLoadBalancerParser < RightAWSParser #:nodoc: def tagend(name) case name when 'member' @result << @text when 'AvailabilityZones' @result.sort! end end def reset @result = [] end end class HealthCheckParser < RightAWSParser #:nodoc: def tagend(name) case name when 'Interval' then @result[:interval] = @text.to_i when 'Target' then @result[:target] = @text when 'HealthyThreshold' then @result[:healthy_threshold] = @text.to_i when 'Timeout' then @result[:timeout] = @text.to_i when 'UnhealthyThreshold' then @result[:unhealthy_threshold] = @text.to_i end end def reset @result = {} end end #----------------------------------------------------------------- # PARSERS: Instances #----------------------------------------------------------------- class DescribeInstanceHealthParser < RightAWSParser #:nodoc: def tagstart(name, attributes) @item = {} if name == 'member' end def tagend(name) case name when 'Description' then @item[:description] = @text when 'State' then @item[:state] = @text when 'InstanceId' then @item[:instance_id] = @text when 'ReasonCode' then @item[:reason_code] = @text when 'member' then @result << @item end end def reset @result = [] end end class InstancesWithLoadBalancerParser < RightAWSParser #:nodoc: def tagend(name) case name when 'InstanceId' @result << @text when 'Instances' @result.sort! end end def reset @result = [] end end end end ================================================ FILE: lib/emr/right_emr_interface.rb ================================================ # # Copyright (c) 2011 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws # = RightAWS::EmrInterface -- RightScale Amazon Elastic Map Reduce interface # # The RightAws::EmrInterface class provides a complete interface to Amazon # Elastic Map Reduce service. # # For explanations of the semantics of each call, please refer to Amazon's # documentation at # http://aws.amazon.com/documentation/elasticmapreduce/ # # Create an interface handle: # # emr = RightAws::EmrInterface.new(aws_access_key_id, aws_secret_access_key) # # Create a job flow: # # emr.run_job_flow( # :name => 'job flow 1', # :master_instance_type => 'm1.large', # :slave_instance_type => 'm1.large', # :instance_count => 5, # :log_uri => 's3n://bucket/path/to/logs', # :steps => [{ # :name => 'step 1', # :jar => 's3n://bucket/path/to/code.jar', # :main_class => 'com.foobar.emr.Step1', # :args => ['arg', 'arg'], # }]) #=> "j-9K18HM82Q0AE7" # # Describe a job flow: # # emr.describe_job_flows('j-9K18HM82Q0AE7') #=> {...} # # Terminate a job flow: # # emr.terminate_job_flows('j-9K18HM82Q0AE7') #=> true # class EmrInterface < RightAwsBase include RightAwsBaseInterface # Amazon EMR API version being used API_VERSION = '2009-03-31' DEFAULT_HOST = 'elasticmapreduce.amazonaws.com' DEFAULT_PATH = '/' DEFAULT_PROTOCOL = 'https' DEFAULT_PORT = 443 @@bench = AwsBenchmarkingBlock.new def self.bench_xml @@bench.xml end def self.bench_service @@bench.service end # Create a new handle to a EMR service. # # All handles share the same per process or per thread HTTP connection # to EMR. Each handle is for a specific account. The params have # the following options: # # * :endpoint_url a fully qualified url to Amazon API endpoint # (this overwrites: :server, :port, :service, :protocol). Example: # 'https://elasticmapreduce.amazonaws.com' # * :server: EMR service host, default: DEFAULT_HOST # * :port: EMR service port, default: DEFAULT_PORT # * :protocol: 'http' or 'https', default: DEFAULT_PROTOCOL # * :logger: for log messages, default: RAILS_DEFAULT_LOGGER else STDOUT # # emr = RightAws::EmrInterface.new('xxxxxxxxxxxxxxxxxxxxx','xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', # {:logger => Logger.new('/tmp/x.log')}) #=> # # def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={}) init({ :name => 'EMR', :default_host => ENV['EMR_URL'] ? URI.parse(ENV['EMR_URL']).host : DEFAULT_HOST, :default_port => ENV['EMR_URL'] ? URI.parse(ENV['EMR_URL']).port : DEFAULT_PORT, :default_service => ENV['EMR_URL'] ? URI.parse(ENV['EMR_URL']).path : DEFAULT_PATH, :default_protocol => ENV['EMR_URL'] ? URI.parse(ENV['EMR_URL']).scheme : DEFAULT_PROTOCOL, :default_api_version => ENV['EMR_API_VERSION'] || API_VERSION }, aws_access_key_id || ENV['AWS_ACCESS_KEY_ID'] , aws_secret_access_key|| ENV['AWS_SECRET_ACCESS_KEY'], params) end def generate_request(action, params={}) #:nodoc: generate_request_impl(:get, action, params ) end # Sends request to Amazon and parses the response # Raises AwsError if any banana happened def request_info(request, parser) #:nodoc: request_info_impl(:emr_connection, @@bench, request, parser) end #----------------------------------------------------------------- # Job Flows #----------------------------------------------------------------- EMR_INSTANCES_KEY_MAPPING = { # :nodoc: :additional_info => 'AdditionalInfo', :log_uri => 'LogUri', :name => 'Name', :ami_version => 'AmiVersion', # JobFlowInstancesConfig :ec2_key_name => 'Instances.Ec2KeyName', :hadoop_version => 'Instances.HadoopVersion', :instance_count => 'Instances.InstanceCount', :keep_job_flow_alive_when_no_steps => 'Instances.KeepJobFlowAliveWhenNoSteps', :master_instance_type => 'Instances.MasterInstanceType', :slave_instance_type => 'Instances.SlaveInstanceType', :termination_protected => 'Instances.TerminationProtected', # PlacementType :availability_zone => 'Instances.Placement.AvailabilityZone', } BOOTSTRAP_ACTION_KEY_MAPPING = { # :nodoc: :name => 'Name', # ScriptBootstrapActionConfig :args => 'ScriptBootstrapAction.Args', :path => 'ScriptBootstrapAction.Path', } INSTANCE_GROUP_KEY_MAPPING = { # :nodoc: :bid_price => 'BidPrice', :instance_count => 'InstanceCount', :instance_role => 'InstanceRole', :instance_type => 'InstanceType', :market => 'Market', :name => 'Name', } STEP_CONFIG_KEY_MAPPING = { # :nodoc: :action_on_failure => 'ActionOnFailure', :name => 'Name', # HadoopJarStepConfig :args => 'HadoopJarStep.Args', :jar => 'HadoopJarStep.Jar', :main_class => 'HadoopJarStep.MainClass', :properties => 'HadoopJarStep.Properties', } KEY_VALUE_KEY_MAPPINGS = { :key => 'Key', :value => 'Value', } # Creates and starts running a new job flow. # # The job flow will run the steps specified and terminate (unless # keep alive option is set). # # A maximum of 256 steps are allowed in a job flow. # # At least the name, instance types, instance count and one step # must be specified. # # # simple usage: # emr.run_job_flow( # :name => 'job flow 1', # :master_instance_type => 'm1.large', # :slave_instance_type => 'm1.large', # :instance_count => 5, # :log_uri => 's3n://bucket/path/to/logs', # :steps => [{ # :name => 'step 1', # :jar => 's3n://bucket/path/to/code.jar', # :main_class => 'com.foobar.emr.Step1', # :args => ['arg', 'arg'], # }]) #=> "j-9K18HM82Q0AE7" # # # advanced usage: # emr.run_job_flow( # :name => 'job flow 1', # :ec2_key_name => 'gsg-keypair', # :hadoop_version => '0.20', # :instance_groups => [{ # :bid_price => '0.1', # :instance_count => '1', # :instance_role => 'MASTER', # :instance_type => 'm1.small', # :market => 'SPOT', # :name => 'master group', # }, { # :bid_price => '0.1', # :instance_count => '2', # :instance_role => 'CORE', # :instance_type => 'm1.small', # :market => 'SPOT', # :name => 'core group', # }, { # :bid_price => '0.1', # :instance_count => '2', # :instance_role => 'TASK', # :instance_type => 'm1.small', # :market => 'SPOT', # :name => 'task group', # }], # :keep_job_flow_alive_when_no_steps => true, # :availability_zone => 'us-east-1a', # :termination_protected => true, # :log_uri => 's3n://bucket/path/to/logs', # :steps => [{ # :name => 'step 1', # :jar => 's3n://bucket/path/to/code.jar', # :main_class => 'com.foobar.emr.Step1', # :args => ['arg', 'arg'], # :properties => { # 'property' => 'value', # }, # :action_on_failure => 'TERMINATE_JOB_FLOW', # }], # :additional_info => '', # :bootstrap_actions => [{ # :name => 'bootstrap action 1', # :path => 's3n://bucket/path/to/bootstrap', # :args => ['hello', 'world'], # }], # ) #=> "j-9K18HM82Q0AE7" # def run_job_flow(options={}) request_hash = amazonize_run_job_flow(options) request_hash.update(amazonize_bootstrap_actions(options[:bootstrap_actions])) request_hash.update(amazonize_instance_groups(options[:instance_groups])) request_hash.update(amazonize_steps(options[:steps])) link = generate_request("RunJobFlow", request_hash) request_info(link, RunJobFlowParser.new(:logger => @logger)) rescue on_exception end # Returns a list of job flows that match all of supplied parameters. # # Without parameters, returns job flows started in the last two weeks # or running job flows started in the last two months. # # Regardless of parameters, only jobs started in the last two months # are returned. # # # default list: # emr.describe_job_flows #=> [ # {:keep_job_flow_alive_when_no_steps=>false, # :log_uri=>"s3n://bucket/path/to/logs", # :master_instance_type=>"m1.small", # :availability_zone=>"us-east-1d", # :last_state_change_reason=>"Steps completed", # :termination_protected=>false, # :master_instance_id=>"i-1fe51278", # :instance_count=>1, # :ready_date_time=>"2011-08-31T18:58:58Z", # :bootstrap_actions=>[], # :master_public_dns_name=>"ec2-184-78-29-127.compute-1.amazonaws.com", # :instance_groups=> # [{:instance_request_count=>1, # :last_state_change_reason=>"Job flow terminated", # :instance_role=>"MASTER", # :ready_date_time=>"2011-08-31T18:58:56Z", # :instance_running_count=>0, # :start_date_time=>"2011-08-31T18:58:19Z", # :market=>"ON_DEMAND", # :creation_date_time=>"2011-08-31T18:55:36Z", # :name=>"master", # :instance_group_id=>"ig-1D91GQR7A9H2K", # :state=>"ENDED", # :instance_type=>"m1.small", # :end_date_time=>"2011-08-31T19:01:09Z"}], # :start_date_time=>"2011-08-31T18:58:58Z", # :steps=> # [{:jar=>"s3n://bucket/path/to/code.jar", # :main_class=>"com.foobar.emr.Step1", # :start_date_time=>"2011-08-31T18:58:58Z", # :properties=>{}, # :args=>[], # :creation_date_time=>"2011-08-31T18:55:36Z", # :action_on_failure=>"TERMINATE_JOB_FLOW", # :name=>"step 1", # :state=>"COMPLETED", # :end_date_time=>"2011-08-31T19:00:34Z"}], # :normalized_instance_hours=>1, # :ami_version=>"1.0", # :creation_date_time=>"2011-08-31T18:55:36Z", # :name=>"jobflow 1", # :hadoop_version=>"0.18", # :job_flow_id=>"j-9K18HM82Q0AE7", # :state=>"COMPLETED", # :end_date_time=>"2011-08-31T19:01:09Z"}] # # # describe specific job flows: # emr.describe_job_flows('j-9K18HM82Q0AE7', 'j-2QE0KHA1LP4GS') #=> [...] # # # specify parameters: # emr.describe_job_flows( # :created_after => Time.now - 86400, # :created_before => Time.now - 3600, # :job_flow_ids => ['j-9K18HM82Q0AE7', 'j-2QE0KHA1LP4GS'], # :job_flow_states => ['RUNNING'] # ) #=> [...] # # # combined job flow list and parameters syntax: # emr.describe_job_flows('j-9K18HM82Q0AE7', 'j-2QE0KHA1LP4GS', # :job_flow_states => ['RUNNING'] # ) #=> [...] # def describe_job_flows(*job_flow_ids_and_options) job_flow_ids, options = AwsUtils::split_items_and_params(job_flow_ids_and_options) # merge job flow ids passed in as arguments and in options unless job_flow_ids.empty? # do not modify passed in options options = options.dup if job_flow_ids_in_options = options[:job_flow_ids] # allow the same ids to be passed in either location; # remove duplicates options[:job_flow_ids] = (job_flow_ids_in_options + job_flow_ids).uniq else options[:job_flow_ids] = job_flow_ids end end request_hash = {} unless (job_flow_ids = options[:job_flow_ids]).right_blank? request_hash.update(amazonize_list("JobFlowIds.member", job_flow_ids)) end unless (job_flow_states = options[:job_flow_states]).right_blank? request_hash = amazonize_list("JobFlowStates.member", job_flow_states) end request_hash['CreatedAfter'] = AwsUtils::utc_iso8601(options[:created_after]) unless options[:created_after].right_blank? request_hash['CreatedBefore'] = AwsUtils::utc_iso8601(options[:created_before]) unless options[:created_before].right_blank? link = generate_request("DescribeJobFlows", request_hash) request_cache_or_info(:describe_job_flows, link, DescribeJobFlowsParser, @@bench, nil) rescue on_exception end # Terminates specified job flows. # # emr.terminate_job_flows('j-9K18HM82Q0AE7') #=> true # def terminate_job_flows(*job_flow_ids) link = generate_request("TerminateJobFlows", amazonize_list('JobFlowIds.member', job_flow_ids)) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue on_exception end # Locks a job flow so the EC2 instances in the cluster cannot be # terminated by user intervention, an API call, or in the event of a # job flow error. Cluster will still terminate upon successful completion # of the job flow. # # emr.set_termination_protection( # 'j-9K18HM82Q0AE7', 'j-2QE0KHA1LP4GS', :termination_protected => true # ) #=> true # # Protection can be enabled using the shortcut syntax: # # emr.set_termination_protection('j-9K18HM82Q0AE7') #=> true # def set_termination_protection(*job_flow_ids_and_options) job_flow_ids, options = AwsUtils::split_items_and_params(job_flow_ids_and_options) request_hash = amazonize_list('JobFlowIds.member', job_flow_ids) request_hash['TerminationProtected'] = case value = options[:termination_protected] when true 'true' when false 'false' when nil # if :termination_protected => nil was given, then unprotect; # if no :termination_protected option was given, protect if options.has_key?(:termination_protected) 'false' else 'true' end else # pass value through value end link = generate_request("SetTerminationProtection", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue on_exception end #----------------------------------------------------------------- # Steps #----------------------------------------------------------------- # Adds steps to a running job flow. # # A maximum of 256 steps are allowed in a job flow. Steps can only be # added to job flows that are starting, bootstrapping, running or waiting. # # Step configuration options are the same as the ones accepted by # run_job_flow. # # emr.add_job_flow_steps('j-2QE0KHA1LP4GS', { # :name => 'step 1', # :jar => 's3n://bucket/path/to/code.jar', # :main_class => 'com.foobar.emr.Step1', # :args => ['arg', 'arg'], # :properties => { # 'property' => 'value', # }, # :action_on_failure => 'TERMINATE_JOB_FLOW', # }) #=> true # def add_job_flow_steps(job_flow_id, *steps) request_hash = amazonize_steps(steps) request_hash['JobFlowId'] = job_flow_id link = generate_request("AddJobFlowSteps", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue on_exception end #----------------------------------------------------------------- # Instance Groups #----------------------------------------------------------------- # Adds instance groups to a running job flow. # # Instance group configuration options are the same as the ones accepted # by run_job_flow. # # Only task instance groups may be added at runtime. # Instance groups cannot be added to job flows that have only a master # instance (i.e. 1 instance in total). # # emr.add_instance_groups('j-2QE0KHA1LP4GS', { # :bid_price => '0.1', # :instance_count => '2', # :instance_role => 'TASK', # :instance_type => 'm1.small', # :market => 'SPOT', # :name => 'core group', # }) #=> true # def add_instance_groups(job_flow_id, *instance_groups) request_hash = amazonize_instance_groups(instance_groups, 'InstanceGroups') request_hash['JobFlowId'] = job_flow_id link = generate_request("AddInstanceGroups", request_hash) request_info(link, AddInstanceGroupsParser.new(:logger => @logger)) rescue on_exception end MODIFY_INSTANCE_GROUP_KEY_MAPPINGS = { :instance_group_id => 'InstanceGroupId', :instance_count => 'InstanceCount', } # Modifies instance groups. # # The only modifiable parameter is instance count. # # An instance group may only be modified when the job flow is running # or waiting. Additionally, hadoop 0.20 is required to resize job flows. # # # general syntax # emr.modify_instance_groups( # {:instance_group_id => 'ig-P2OPM2L9ZQ4P', :instance_count => 5}, # {:instance_group_id => 'ig-J82ML0M94A7E', :instance_count => 1} # ) #=> true # # # shortcut syntax # emr.modify_instance_groups('ig-P2OPM2L9ZQ4P', 5) #=> true # # Shortcut syntax supports modifying only one instance group at a time. # def modify_instance_groups(*args) unless args.first.is_a?(Hash) if args.length != 2 raise ArgumentError, "Must be given two arguments if arguments are not hashes" end args = [{:instance_group_id => args.first, :instance_count => args.last}] end request_hash = amazonize_list_with_key_mapping('InstanceGroups.member', MODIFY_INSTANCE_GROUP_KEY_MAPPINGS, args) link = generate_request("ModifyInstanceGroups", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) rescue on_exception end private def amazonize_run_job_flow(options) # :nodoc: result = {} unless options.right_blank? EMR_INSTANCES_KEY_MAPPING.each do |local_name, remote_name| value = options[local_name] result[remote_name] = value unless value.nil? end end result end def amazonize_bootstrap_actions(bootstrap_actions, key = 'BootstrapActions.member') # :nodoc: result = {} unless bootstrap_actions.right_blank? bootstrap_actions.each_with_index do |item, index| BOOTSTRAP_ACTION_KEY_MAPPING.each do |local_name, remote_name| value = item[local_name] case local_name when :args result.update(amazonize_list("#{key}.#{index+1}.#{remote_name}.member", value)) else next if value.nil? result["#{key}.#{index+1}.#{remote_name}"] = value end end end end result end def amazonize_instance_groups(instance_groups, key = 'Instances.InstanceGroups') # :nodoc: result = {} unless instance_groups.right_blank? instance_groups.each_with_index do |item, index| INSTANCE_GROUP_KEY_MAPPING.each do |local_name, remote_name| value = item[local_name] case local_name when :instance_groups result.update(amazonize_list_with_key_mapping("#{key}.member.#{index+1}.#{remote_name}", INSTANCE_GROUP_KEY_MAPPING, value)) else next if value.nil? result["#{key}.member.#{index+1}.#{remote_name}"] = value end end end end result end def amazonize_steps(steps, key = 'Steps.member') # :nodoc: result = {} unless steps.right_blank? steps.each_with_index do |item, index| STEP_CONFIG_KEY_MAPPING.each do |local_name, remote_name| value = item[local_name] case local_name when :args result.update(amazonize_list("#{key}.#{index+1}.#{remote_name}.member", value)) when :properties next if value.right_blank? list = value.inject([]) do |l, (k, v)| l << {:key => k, :value => v} end result.update(amazonize_list_with_key_mapping("#{key}.#{index+1}.#{remote_name}.member", KEY_VALUE_KEY_MAPPINGS, list)) else next if value.nil? result["#{key}.#{index+1}.#{remote_name}"] = value end end end end result end #----------------------------------------------------------------- # PARSERS: Run Job Flow #----------------------------------------------------------------- class RunJobFlowParser < RightAWSParser #:nodoc: def tagend(name) case name when 'JobFlowId' then @result = @text end end def reset @result = nil end end #----------------------------------------------------------------- # PARSERS: Describe Job Flows #----------------------------------------------------------------- class DescribeJobFlowsParser < RightAWSParser #:nodoc: def tagstart(name, attributes) case full_tag_name when %r{/JobFlows/member$} @item = { :instance_groups => [], :steps => [], :bootstrap_actions => [] } when %r{/BootstrapActionConfig$} @bootstrap_action = {} when %r{/InstanceGroups/member$} @instance_group = {} when %r{/Steps/member$} @step = { :args => [], :properties => {} } end end def tagend(name) case full_tag_name when %r{/BootstrapActionConfig} # no trailing $ case name when 'Name' @bootstrap_action[:name] = @text when 'ScriptBootstrapAction' @bootstrap_action[:script_bootstrap_action] = @text when 'BootstrapActionConfig' @step[:bootstrap_actions] << @bootstrap_action end when %r{/InstanceGroups/member} # no trailing $ case name when 'BidPrice' then @instance_group[:bid_price] = @text when 'CreationDateTime' then @instance_group[:creation_date_time] = @text when 'EndDateTime' then @instance_group[:end_date_time] = @text when 'InstanceGroupId' then @instance_group[:instance_group_id] = @text when 'InstanceRequestCount' then @instance_group[:instance_request_count] = @text.to_i when 'InstanceRole' then @instance_group[:instance_role] = @text when 'InstanceRunningCount' then @instance_group[:instance_running_count] = @text.to_i when 'InstanceType' then @instance_group[:instance_type] = @text when 'LastStateChangeReason' then @instance_group[:last_state_change_reason] = @text when 'Market' then @instance_group[:market] = @text when 'Name' then @instance_group[:name] = @text when 'ReadyDateTime' then @instance_group[:ready_date_time] = @text when 'StartDateTime' then @instance_group[:start_date_time] = @text when 'State' then @instance_group[:state] = @text when 'member' then @item[:instance_groups] << @instance_group end when %r{/Steps/member/StepConfig/HadoopJarStep/Args/member} @step[:args] << @text when %r{/Steps/member/StepConfig/HadoopJarStep/Properties$} @step[:properties][@key] = @value when %r{/Steps/member/StepConfig/HadoopJarStep/Properties} case name when 'Key' @key = @text when 'Value' @value = @text end when %r{/Steps/member$} @item[:steps] << @step when %r{/Steps/member} # no trailing $ case name # ExecutionStatusDetail when 'CreationDateTime' then @step[:creation_date_time] = @text when 'EndDateTime' then @step[:end_date_time] = @text when 'LastStateChangeReason' then @step[:last_state_change_reason] = @text when 'StartDateTime' then @step[:start_date_time] = @text when 'State' then @step[:state] = @text # StepConfig when 'ActionOnFailure' then @step[:action_on_failure] = @text when 'Name' then @step[:name] = @text # HadoopJarStepConfig when 'Jar' then @step[:jar] = @text when 'MainClass' then @step[:main_class] = @text end when %r{/JobFlows/member$} @result << @item else case name when 'AmiVersion' then @item[:ami_version] = @text when 'JobFlowId' then @item[:job_flow_id] = @text when 'LogUri' then @item[:log_uri] = @text when 'Name' then @item[:name] = @text # JobFlowExecutionStatusDetail when 'CreationDateTime' then @item[:creation_date_time] = @text when 'EndDateTime' then @item[:end_date_time] = @text when 'LastStateChangeReason' then @item[:last_state_change_reason] = @text when 'ReadyDateTime' then @item[:ready_date_time] = @text when 'StartDateTime' then @item[:start_date_time] = @text when 'State' then @item[:state] = @text # JobFlowInstancesDetail when 'Ec2KeyName' then @item[:ec2_key_name] = @text when 'HadoopVersion' then @item[:hadoop_version] = @text when 'InstanceCount' then @item[:instance_count] = @text.to_i when 'KeepJobFlowAliveWhenNoSteps' then @item[:keep_job_flow_alive_when_no_steps] = case @text when 'true' then true when 'false' then false else @text end when 'MasterInstanceId' then @item[:master_instance_id] = @text when 'MasterInstanceType' then @item[:master_instance_type] = @text when 'MasterPublicDnsName' then @item[:master_public_dns_name] = @text when 'NormalizedInstanceHours' then @item[:normalized_instance_hours] = @text.to_i # Placement when 'AvailabilityZone' then @item[:availability_zone] = @text when 'SlaveInstanceType' then @item[:slave_instance_type] = @text when 'TerminationProtected' then @item[:termination_protected] = case @text when 'true' then true when 'false' then false else @text end end end end def reset @result = [] end end #----------------------------------------------------------------- # PARSERS: Add Instance Groups #----------------------------------------------------------------- class AddInstanceGroupsParser < RightAWSParser #:nodoc: def tagend(name) case name when 'InstanceGroupIds' then @result << @text.strip end end def reset @result = [] end end end end ================================================ FILE: lib/iam/right_iam_access_keys.rb ================================================ module RightAws class IamInterface < RightAwsBase #----------------------------------------------------------------- # Access Keys #----------------------------------------------------------------- # Returns information about the Access Key IDs associated with the specified User. # # Options: :user_name, :max_items, :marker # # iam.list_access_keys #=> # [{:create_date=>"2007-01-09T06:16:30Z", # :status=>"Active", # :access_key_id=>"00000000000000000000"}] # def list_access_keys(options={}, &block) incrementally_list_iam_resources('ListAccessKeys', options, &block) end # Creates a new AWS Secret Access Key and corresponding AWS Access Key ID for the specified User. # # Options: :user_name # # iam.create_access_key(:user_name => 'kd1') #=> # {:access_key_id=>"AK0000000000000000ZQ", # :status=>"Active", # :secret_access_key=>"QXN0000000000000000000000000000000000Ioj", # :create_date=>"2010-10-29T07:16:32.210Z", # :user_name=>"kd1"} # def create_access_key(options={}) request_hash = {} request_hash['UserName'] = options[:user_name] unless options[:user_name].right_blank? link = generate_request("CreateAccessKey", request_hash) request_info(link, CreateAccessKeyParser.new(:logger => @logger)) end # Deletes the access key associated with the specified User. # # Options: :user_name # # iam.delete_access_key('AK00000000000000006A', :user_name => 'kd1') #=> true # def delete_access_key(access_key_id, options={}) request_hash = { 'AccessKeyId' => access_key_id } request_hash['UserName'] = options[:user_name] unless options[:user_name].right_blank? link = generate_request("DeleteAccessKey", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end #----------------------------------------------------------------- # PARSERS #----------------------------------------------------------------- class ListAccessKeysParser < BasicIamListParser #:nodoc: def reset @expected_tags = %w{ AccessKeyId CreateDate Status UserName } end end class CreateAccessKeyParser < BasicIamParser #:nodoc: def reset @expected_tags = %w{ AccessKeyId CreateDate SecretAccessKey Status UserName } end end end end ================================================ FILE: lib/iam/right_iam_groups.rb ================================================ module RightAws class IamInterface < RightAwsBase #----------------------------------------------------------------- # Groups #----------------------------------------------------------------- # Lists the groups that have the specified path prefix. # # Options: :path_prefix, :max_items, :marker # # iam.list_groups #=> # [{:group_id=>"AGP000000000000000UTY", # :arn=>"arn:aws:iam::640000000037:group/kd_test", # :path=>"/", # :group_name=>"kd_test"}] # def list_groups(options={}, &block) incrementally_list_iam_resources('ListGroups', options, &block) end # Creates a new group. # # iam.create_group('kd_group') #=> # {:group_id=>"AGP000000000000000UTY", # :arn=>"arn:aws:iam::640000000037:group/kd_test", # :path=>"/", # :group_name=>"kd_test"} # # iam.create_group('kd_test_3', '/kd/') #=> # {:group_id=>"AGP000000000000000G6Q", # :arn=>"arn:aws:iam::640000000037:group/kd/kd_test_3", # :path=>"/kd/", # :group_name=>"kd_test_3"} # def create_group(group_name, path=nil) request_hash = { 'GroupName' => group_name } request_hash['Path'] = path unless path.right_blank? link = generate_request("CreateGroup", request_hash) request_info(link, CreateGroupParser.new(:logger => @logger)) end # Updates the name and/or the path of the specified group # # Options: :new_group_name, :new_path # # iam.update_group('kd_test', :new_group_name => 'kd_test_1', :new_path => '/kd1/') #=> true # def update_group(group_name, options={}) request_hash = { 'GroupName' => group_name} request_hash['NewGroupName'] = options[:new_group_name] unless options[:new_group_name].right_blank? request_hash['NewPath'] = options[:new_path] unless options[:new_path].right_blank? link = generate_request("UpdateGroup", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Returns a list of Users that are in the specified group. # # Options: :max_items, :marker # # iam.get_group('kd_test') #=> # {:arn=>"arn:aws:iam::640000000037:group/kd1/kd_test_1", # :users=> # [{:arn=>"arn:aws:iam::640000000037:user/kd", # :path=>"/", # :user_name=>"kd", # :user_id=>"AID000000000000000WZ2"}], # :group_name=>"kd_test_1", # :group_id=>"AGP000000000000000UTY", # :path=>"/kd1/"} # def get_group(group_name, options={}, &block) options[:group_name] = group_name incrementally_list_iam_resources('GetGroup', options, :items => :users, :except => [:marker, :is_truncated], &block) end # Deletes the specified group. The group must not contain any Users or have any attached policies. # # iam.delete_group('kd_test_3') #=> true # def delete_group(group_name) request_hash = { 'GroupName' => group_name } link = generate_request("DeleteGroup", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end #----------------------------------------------------------------- # Group Policies #----------------------------------------------------------------- # Lists the names of the policies associated with the specified group. # # Options: :max_items, :marker # # iam.list_group_policies('kd_test') #=> ["kd_policy_1"] # def list_group_policies(group_name, options={}, &block) options[:group_name] = group_name incrementally_list_iam_resources('ListGroupPolicies', options, :parser => BasicIamListParser, &block) end # Adds (or updates) a policy document associated with the specified group. # # iam.put_group_policy('kd_test', 'kd_policy_1', %Q({"Statement":[{"Effect":"Allow","Action":"*","Resource":"*"}]})) #=> true # def put_group_policy(group_name, policy_name, policy_document) request_hash = { 'GroupName' => group_name, 'PolicyDocument' => policy_document, 'PolicyName' => policy_name } link = generate_request_impl(:post, "PutGroupPolicy", request_hash) result = request_info(link, RightHttp2xxParser.new(:logger => @logger)) result[:policy_document] = URI::decode(result[:policy_document]) result end # Retrieves the specified policy document for the specified group. # # iam.get_group_policy('kd_test', 'kd_policy_1') #=> # {:policy_name=>"kd_policy_1", # :policy_document=>"{\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"*\",\"Resource\":\"*\"}]}", # :group_name=>"kd_test"} # def get_group_policy(group_name, policy_name) request_hash = { 'GroupName' => group_name, 'PolicyName' => policy_name } link = generate_request("GetGroupPolicy", request_hash) request_info(link, GetGroupPolicyParser.new(:logger => @logger)) end # Deletes the specified policy that is associated with the specified group # # iam.delete_group_policy('kd_test', 'kd_policy_1') #=> true # def delete_group_policy(group_name, policy_name) request_hash = { 'GroupName' => group_name, 'PolicyName' => policy_name } link = generate_request("DeleteGroupPolicy", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end #----------------------------------------------------------------- # PARSERS: #----------------------------------------------------------------- class ListGroupsParser < BasicIamListParser #:nodoc: def reset @expected_tags = %w{ Arn GroupId GroupName Path } end end class CreateGroupParser < BasicIamParser #:nodoc: def reset @expected_tags = %w{ Arn GroupId GroupName Path } end end class GetGroupParser < RightAWSParser #:nodoc: def tagstart(name, attributes) @item = {} if name == 'member' end def tagend(name) case name when 'Marker' then @result[:marker] = @text when 'IsTruncated' then @result[:is_truncated] = @text == 'true' when 'GroupName' then @result[:group_name] = @text when 'GroupId' then @result[:group_id] = @text when 'UserName' then @item[:user_name] = @text when 'UserId' then @item[:user_id] = @text when 'member' then @result[:users] << @item else case full_tag_name when %r{/Group/Path$} then @result[:path] = @text when %r{/Group/Arn$} then @result[:arn] = @text when %r{/member/Path$} then @item[:path] = @text when %r{/member/Arn$} then @item[:arn] = @text end end end def reset @result = { :users => [] } end end class GetGroupPolicyParser < BasicIamParser #:nodoc: def reset @expected_tags = %w{ GroupName PolicyDocument PolicyName } end end end end ================================================ FILE: lib/iam/right_iam_interface.rb ================================================ # # Copyright (c) 2007-2010 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws # = RightAWS::Iam -- RightScale AWS Identity and Access Management (IAM) interface # # The RightAws::Iam class provides a complete interface to Amazon's Identity and # Access Management service. # # For explanations of the semantics of each call, please refer to Amazon's documentation at # http://aws.amazon.com/documentation/iam/ # # Examples: # # Create an EC2 interface handle: # # iam = RightAws::IamInterface.new(aws_access_key_id, aws_secret_access_key) # iam.list_access_keys # iam.list_users # iam.list_groups # class IamInterface < RightAwsBase include RightAwsBaseInterface API_VERSION = "2010-05-08" DEFAULT_HOST = "iam.amazonaws.com" DEFAULT_PATH = '/' DEFAULT_PROTOCOL = 'https' DEFAULT_PORT = 443 @@bench = AwsBenchmarkingBlock.new def self.bench_xml @@bench.xml end def self.bench_service @@bench.service end # Create a new handle to an IAM account. All handles share the same per process or per thread # HTTP connection to Amazon IAM. Each handle is for a specific account. The params have the # following options: # * :endpoint_url a fully qualified url to Amazon API endpoint (this overwrites: :server, :port, :service, :protocol). # * :server: IAM service host, default: DEFAULT_HOST # * :port: IAM service port, default: DEFAULT_PORT # * :protocol: 'http' or 'https', default: DEFAULT_PROTOCOL # * :logger: for log messages, default: RAILS_DEFAULT_LOGGER else STDOUT # * :signature_version: The signature version : '0','1' or '2'(default) # * :cache: true/false(default): caching works for: describe_load_balancers # def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={}) init({ :name => 'IAM', :default_host => ENV['IAM_URL'] ? URI.parse(ENV['IAM_URL']).host : DEFAULT_HOST, :default_port => ENV['IAM_URL'] ? URI.parse(ENV['IAM_URL']).port : DEFAULT_PORT, :default_service => ENV['IAM_URL'] ? URI.parse(ENV['IAM_URL']).path : DEFAULT_PATH, :default_protocol => ENV['IAM_URL'] ? URI.parse(ENV['IAM_URL']).scheme : DEFAULT_PROTOCOL, :default_api_version => ENV['IAM_API_VERSION'] || API_VERSION }, aws_access_key_id || ENV['AWS_ACCESS_KEY_ID'] , aws_secret_access_key|| ENV['AWS_SECRET_ACCESS_KEY'], params) end def generate_request(action, params={}) #:nodoc: generate_request_impl(:get, action, params ) end # Sends request to Amazon and parses the response # Raises AwsError if any banana happened def request_info(request, parser) #:nodoc: request_info_impl(:iam_connection, @@bench, request, parser) end # Options: :parser, :except, :items # def incrementally_list_iam_resources(api_function, params={}, options={}, &block) #:nodoc: items = options[:items] || :items result = { items => [] } parser = options[:parser] || "RightAws::IamInterface::#{api_function}Parser".right_constantize request_hash = {} params.each { |key,value| request_hash[key.to_s.right_camelize] = value unless value.right_blank? } incrementally_list_items(api_function, parser, request_hash) do |response| if result[items].right_blank? result = response else result[items] += response[items] end block ? block.call(response) : true end if options[:except] Array(options[:except]).each{ |key| result.delete(key)} result else result[items] end end #----------------------------------------------------------------- # Server Certificates #----------------------------------------------------------------- # Lists the server certificates that have the specified path prefix. If none exist, the action returns an empty list. # # Options: :path_prefix, :max_items, :marker # # iam.list_server_certificates #=> # {:server_certificate_id=>"ASCDJN5K5HRGS1N2UJWWU", # :server_certificate_name=>"KdCert1", # :upload_date=>"2010-12-09T13:21:07.226Z", # :path=>"/kdcert/", # :arn=>"arn:aws:iam::600000000007:server-certificate/kdcert/KdCert1"} # def list_server_certificates(options={}, &block) incrementally_list_iam_resources('ListServerCertificates', options, &block) end # Uploads a server certificate entity for the AWS Account. The server certificate # entity includes a public key certificate, a private key, and an optional certificate # chain, which should all be PEM-encoded. # # Options: :certificate_chain, :path # # certificate_body =<<-EOB # -----BEGIN CERTIFICATE----- # MIICdzCCAeCgAwIBAgIGANc+Ha2wMA0GCSqGSIb3DQEBBQUAMFMxCzAJBgNVBAYT # AlVTMRMwEQYDVQQKEwpBbWF6b24uY29tMQwwCgYDVQQLEwNBV1MxITAfBgNVBAMT # GEFXUyBMaW1pdGVkLUFzc3VyYW5jZSBDQTAeFw0wOTAyMDQxNzE5MjdaFw0xMDAy # AEaHzTpmEXAMPLE= # EOB # # private_key =<'/kdcert/') #=> # {:server_certificate_id=>"ASCDJN5K5HRGS1N2UJWWU", # :server_certificate_name=>"KdCert1", # :upload_date=>"2010-12-09T13:21:07.226Z", # :path=>"/kdcert/", # :arn=>"arn:aws:iam::600000000007:server-certificate/kdcert/KdCert1"} # def upload_server_certificate(server_certificate_name, certificate_body, private_key, options={}) request_hash = { 'CertificateBody' => certificate_body, 'PrivateKey' => private_key, 'ServerCertificateName' => server_certificate_name } request_hash['CertificateChain'] = options[:certificate_chain] unless options[:certificate_chain].right_blank? request_hash['Path'] = options[:path] unless options[:path].right_blank? link = generate_request_impl(:post, "UploadServerCertificate", request_hash) request_info(link, GetServerCertificateParser.new(:logger => @logger)) end # Updates the name and/or the path of the specified server certificate. # # Options: :new_server_certificate_name, :new_path # # iam.update_server_certificate('ProdServerCert', :new_server_certificate_name => 'OldServerCert') #=> true # def update_server_certificate(server_certificate_name, options={}) request_hash = { 'ServerCertificateName' => server_certificate_name} request_hash['NewServerCertificateName'] = options[:new_server_certificate_name] unless options[:new_server_certificate_name].right_blank? request_hash['NewPath'] = options[:new_path] unless options[:new_path].right_blank? link = generate_request("UpdateServerCertificate", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Retrieves information about the specified server certificate. # # iam.get_server_certificate('KdCert1') # {:certificate_body=> # "-----BEGIN CERTIFICATE-----\nMIICATC...TiU5TibMpD1g==\n-----END CERTIFICATE-----", # :server_certificate_id=>"ASCDJN5K5HRGS1N2UJWWU", # :server_certificate_name=>"KdCert1", # :upload_date=>"2010-12-09T13:21:07Z", # :path=>"/kdcert/", # :certificate_chain=>"", # :arn=>"arn:aws:iam::600000000007:server-certificate/kdcert/KdCert1"} # def get_server_certificate(server_certificate_name) request_hash = { 'ServerCertificateName' => server_certificate_name} link = generate_request("GetServerCertificate", request_hash) request_info(link, GetServerCertificateParser.new(:logger => @logger)) end # Deletes the specified server certificate # # iam.delete_server_certificate('ProdServerCert') #=> true # def delete_server_certificate(server_certificate_name) request_hash = { 'ServerCertificateName' => server_certificate_name } link = generate_request("DeleteServerCertificate", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end #----------------------------------------------------------------- # Signing Certificates #----------------------------------------------------------------- # Returns information about the signing certificates associated with the specified User. # # Options: :user_name, :max_items, :marker # # iam.list_signing_certificates #=> # [{:upload_date => "2007-08-11T06:48:35Z", # :status => "Active", # :certificate_id => "00000000000000000000000000000000", # :certificate_body => "-----BEGIN CERTIFICATE-----\nMIICd...PPHQ=\n-----END CERTIFICATE-----\n"}] # def list_signing_certificates(options={}, &block) incrementally_list_iam_resources('ListSigningCertificates', options, &block) end # Uploads an X.509 signing certificate and associates it with the specified User. # # Options: :user_name # # certificate_body =<<-EOB # -----BEGIN CERTIFICATE----- # MIICdzCCAeCgAwIBAgIGANc+Ha2wMA0GCSqGSIb3DQEBBQUAMFMxCzAJBgNVBAYT # AlVTMRMwEQYDVQQKEwpBbWF6b24uY29tMQwwCgYDVQQLEwNBV1MxITAfBgNVBAMT # GEFXUyBMaW1pdGVkLUFzc3VyYW5jZSBDQTAeFw0wOTAyMDQxNzE5MjdaFw0xMDAy # AEaHzTpmEXAMPLE= # EOB # # iam.upload_signing_certificate(certificate_body, :user_name => 'kd1') #=> # {:user_name => "kd1", # :certificate_id => "OBG00000000000000000000000000DHY", # :status => "Active", # :certificate_body => "-----BEGIN CERTIFICATE-----\nMII...5GS\n-----END CERTIFICATE-----\n", # :upload_date => "2010-10-29T10:02:05.929Z"} # def upload_signing_certificate(certificate_body, options={}) request_hash = { 'CertificateBody' => certificate_body } request_hash['UserName'] = options[:user_name] unless options[:user_name].right_blank? link = generate_request_impl(:post, "UploadSigningCertificate", request_hash) request_info(link, GetSigningCertificateParser.new(:logger => @logger)) end # Deletes the specified signing certificate associated with the specified User. # # Options: :user_name # # pp iam.delete_signing_certificate('OB0000000000000000000000000000HY', :user_name => 'kd1') # def delete_signing_certificate(certificate_id, options={}) request_hash = { 'CertificateId' => certificate_id } request_hash['UserName'] = options[:user_name] unless options[:user_name].right_blank? link = generate_request("DeleteSigningCertificate", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end #----------------------------------------------------------------- # PARSERS: #----------------------------------------------------------------- class BasicIamParser < RightAWSParser #:nodoc: def tagstart(name, attributes) @result ||= {} end def tagend(name) if Array(@expected_tags).include?(name) @result[name.right_underscore.to_sym] = @text end end end class BasicIamListParser < RightAWSParser #:nodoc: def tagstart(name, attributes) @result ||= { :items => [] } @item = {} if name == (@items_splitter || 'member') end def tagend(name) case name when 'Marker' then @result[:marker] = @text when 'IsTruncated' then @result[:is_truncated] = @text == 'true' when (@items_splitter || 'member') @result[:items] << (@item.right_blank? ? @text : @item) else if Array(@expected_tags).include?(name) @item[name.right_underscore.to_sym] = @text end end end end #----------------------------------------------------------------- # Server Certificates #----------------------------------------------------------------- class GetServerCertificateParser < BasicIamParser #:nodoc: def reset @expected_tags = %w{ Arn Path ServerCertificateId ServerCertificateName UploadDate CertificateBody CertificateChain } end end class ListServerCertificatesParser < BasicIamListParser #:nodoc: def reset @expected_tags = %w{ Arn Path ServerCertificateId ServerCertificateName UploadDate } end end #----------------------------------------------------------------- # Signing Certificates #----------------------------------------------------------------- class ListSigningCertificatesParser < BasicIamListParser #:nodoc: def reset @expected_tags = %w{ CertificateBody CertificateId Status UploadDate UserName } end end class GetSigningCertificateParser < BasicIamParser #:nodoc: def reset @expected_tags = %w{ CertificateBody CertificateId Status UploadDate UserName } end end end end ================================================ FILE: lib/iam/right_iam_mfa_devices.rb ================================================ module RightAws class IamInterface < RightAwsBase #----------------------------------------------------------------- # MFADevices #----------------------------------------------------------------- # Lists the MFA devices associated with the specified User name. # # Options: :user_name, :max_items, :marker # def list_mfa_devices(options={}, &block) incrementally_list_iam_resources('ListMFADevices', options, &block) end # Enables the specified MFA device and associates it with the specified User name. # Once enabled, the MFA device is required for every subsequent login by the User name associated with the device. # # iam.enable_mfa_device('kd1', 'x12345', '12345', '67890') #=> true # def enable_mfa_device(user_name, serial_number, auth_code1, auth_code2) request_hash = { 'UserName' => user_name, 'SerialNumber' => serial_number, 'AuthenticationCode1' => auth_code1, 'AuthenticationCode2' => auth_code2 } link = generate_request("EnableMFADevice", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Synchronizes the specified MFA device with AWS servers. # # iam.resync_mfa_device('kd1', 'x12345', '12345', '67890') #=> true # def resync_mfa_device(user_name, serial_number, auth_code1, auth_code2) request_hash = { 'UserName' => user_name, 'SerialNumber' => serial_number, 'AuthenticationCode1' => auth_code1, 'AuthenticationCode2' => auth_code2 } link = generate_request("ResyncMFADevice", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Deactivates the specified MFA device and removes it from association with the User name for which it was originally enabled. # # deactivate_mfa_device('kd1', 'dev1234567890') #=> true # def deactivate_mfa_device(user_name, serial_number) request_hash = { 'UserName' => user_name, 'SerialNumber' => serial_number } link = generate_request("DeactivateMFADevice", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end #----------------------------------------------------------------- # PARSERS #----------------------------------------------------------------- class ListMFADevicesParser < BasicIamListParser #:nodoc: def reset @expected_tags = %w{ SerialNumber UserName } end end end end ================================================ FILE: lib/iam/right_iam_users.rb ================================================ module RightAws class IamInterface < RightAwsBase #----------------------------------------------------------------- # Users #----------------------------------------------------------------- # Lists the Users that have the specified path prefix. # # Options: :path_prefix, :max_items, :marker # # iam.list_users #=> # [{:user_name=>"kd", # :user_id=>"AI000000000000000006A", # :arn=>"arn:aws:iam::640000000037:user/kd", # :path=>"/"}] # def list_users(options={}, &block) incrementally_list_iam_resources('ListUsers', options, &block) end # Creates a new User for your AWS Account. # # Options: :path # # iam.create_user('kd') #=> # {:user_name=>"kd", # :user_id=>"AI000000000000000006A", # :arn=>"arn:aws:iam::640000000037:user/kd", # :path=>"/"} # def create_user(user_name, options={}) request_hash = { 'UserName' => user_name } request_hash['Path'] = options[:path] unless options[:path] link = generate_request("CreateUser", request_hash) request_info(link, GetUserParser.new(:logger => @logger)) end # Updates the name and/or the path of the specified User. # # iam.update_user('kd1', :new_user_name => 'kd1', :new_path => '/kd1/') #=> true # def update_user(user_name, options={}) request_hash = { 'UserName' => user_name} request_hash['NewUserName'] = options[:new_user_name] unless options[:new_user_name].right_blank? request_hash['NewPath'] = options[:new_path] unless options[:new_path].right_blank? link = generate_request("UpdateUser", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Retrieves information about the specified User, including the User's path, GUID, and ARN. # # iam.get_user('kd') #=> # {:user_name=>"kd", # :user_id=>"AI000000000000000006A", # :arn=>"arn:aws:iam::640000000037:user/kd", # :path=>"/"} # def get_user(user_name) request_hash = { 'UserName' => user_name } link = generate_request("GetUser", request_hash) request_info(link, GetUserParser.new(:logger => @logger)) end # Deletes the specified User. The User must not belong to any groups, have any keys or signing certificates, or have any attached policies. # # iam.delete_user('kd') #=> true # def delete_user(user_name) request_hash = { 'UserName' => user_name } link = generate_request("DeleteUser", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end #----------------------------------------------------------------- # User Policies #----------------------------------------------------------------- # Lists the names of the policies associated with the specified User. # # Options: :max_items, :marker # # iam.list_user_policies('kd') #=> ["kd_user_policy_1"] # def list_user_policies(user_name, options={}, &block) options[:user_name] = user_name incrementally_list_iam_resources('ListUserPolicies', options, :parser => BasicIamListParser, &block) end # Adds (or updates) a policy document associated with the specified User # # iam.put_user_policy('kd', 'kd_user_policy_1', %Q({"Statement":[{"Effect":"Allow","Action":"*","Resource":"*"}]})) #=> true # def put_user_policy(user_name, policy_name, policy_document) request_hash = { 'UserName' => user_name, 'PolicyDocument' => policy_document, 'PolicyName' => policy_name } link = generate_request_impl(:post, "PutUserPolicy", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Retrieves the specified policy document for the specified User. # # iam.get_user_policy('kd','kd_user_policy_1') #=> # {:user_name=>"kd", # :policy_name=>"kd_user_policy_1", # :policy_document=>"{\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"*\",\"Resource\":\"*\"}]}"} # def get_user_policy(user_name, policy_name) request_hash = { 'UserName' => user_name, 'PolicyName' => policy_name } link = generate_request("GetUserPolicy", request_hash) result = request_info(link, GetUserPolicyParser.new(:logger => @logger)) result[:policy_document] = URI::decode(result[:policy_document]) result end # Deletes the specified policy associated with the specified User. # # iam.delete_user_policy('kd','kd_user_policy_1') #=> true # def delete_user_policy(user_name, policy_name) request_hash = { 'UserName' => user_name, 'PolicyName' => policy_name } link = generate_request("DeleteUserPolicy", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end #----------------------------------------------------------------- # User Groups #----------------------------------------------------------------- # Lists the names of the policies associated with the specified group. If there are none, # the action returns an empty list. # # Options: :max_items, :marker # # iam.list_groups_for_user('kd') #=> # [{:group_name=>"kd_test_1", # :group_id=>"AGP000000000000000UTY", # :arn=>"arn:aws:iam::640000000037:group/kd1/kd_test_1", # :path=>"/kd1/"}] # def list_groups_for_user(user_name, options={}, &block) options[:user_name] = user_name incrementally_list_iam_resources('ListGroupsForUser', options, :parser => ListGroupsParser, &block) end # Adds the specified User to the specified group. # # iam.add_user_to_group('kd', 'kd_test_1') #=> true # def add_user_to_group(user_name, group_name) request_hash = { 'UserName' => user_name, 'GroupName' => group_name } link = generate_request("AddUserToGroup", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Removes the specified User from the specified group. # # iam.remove_user_from_group('kd', 'kd_test_1') #=> true # def remove_user_from_group(user_name, group_name) request_hash = { 'UserName' => user_name, 'GroupName' => group_name } link = generate_request("RemoveUserFromGroup", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end #----------------------------------------------------------------- # User Login Profiles #----------------------------------------------------------------- # Creates a login profile for the specified User, giving the User the ability to access # AWS services such as the AWS Management Console. # # iam.create_login_profile('kd','q1w2e3r4t5') #=> { :user_name => 'kd' } # def create_login_profile(user_name, password) request_hash = { 'UserName' => user_name, 'Password' => password} link = generate_request("CreateLoginProfile", request_hash) request_info(link, GetLoginProfileParser.new(:logger => @logger)) end # Updates the login profile for the specified User. Use this API to change the User's password. # # update_login_profile('kd', '00000000') #=> true # def update_login_profile(user_name, options={}) request_hash = { 'UserName' => user_name} request_hash['Password'] = options[:password] unless options[:passwrod].right_blank? link = generate_request("UpdateLoginProfile", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Retrieves the login profile for the specified User # # iam.create_login_profile('kd','q1w2e3r4t5') #=> { :user_name => 'kd' } # def get_login_profile(user_name) request_hash = { 'UserName' => user_name } link = generate_request("GetLoginProfile", request_hash) request_info(link, GetLoginProfileParser.new(:logger => @logger)) end # Deletes the login profile for the specified User, which terminates the User's ability to access # AWS services through the IAM login page. # # iam.delete_login_profile('kd') #=> true # def delete_login_profile(user_name) request_hash = { 'UserName' => user_name } link = generate_request("DeleteLoginProfile", request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end #----------------------------------------------------------------- # PARSERS #----------------------------------------------------------------- class ListUsersParser < BasicIamListParser #:nodoc: def reset @expected_tags = %w{ Arn Path UserId UserName } end end class GetUserParser < BasicIamParser #:nodoc: def reset @expected_tags = %w{ Arn Path UserId UserName } end end class GetUserPolicyParser < BasicIamParser #:nodoc: def reset @expected_tags = %w{ PolicyDocument PolicyName UserName } end end class GetLoginProfileParser < BasicIamParser #:nodoc: def reset @expected_tags = %w{ UserName } end end end end ================================================ FILE: lib/rds/right_rds_interface.rb ================================================ # # Copyright (c) 2009 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws class RdsInterface < RightAwsBase include RightAwsBaseInterface API_VERSION = "2012-09-17" DEFAULT_HOST = 'rds.amazonaws.com' DEFAULT_PORT = 443 DEFAULT_PROTOCOL = 'https' DEFAULT_PATH = '/' DEFAULT_INSTANCE_CLASS = 'db.m1.small' INSTANCE_CLASSES = [ 'db.t1.micro', 'db.m1.small', 'db.m1.medium', 'db.m1.large', 'db.m1.xlarge', 'db.m2.xlarge', 'db.m2.2xlarge', 'db.m2.4xlarge'] LICENSE_MODELS = ['bring-your-own-license', 'license-included', 'general-public-license'] @@bench = AwsBenchmarkingBlock.new def self.bench_xml @@bench.xml end def self.bench_service @@bench.service end # Create a new handle to a RDS account. All handles share the same per process or per thread # HTTP connection to RDS. Each handle is for a specific account. The params have the # following options: # * :endpoint_url a fully qualified url to Amazon API endpoint (this overwrites: :server, :port, :service, :protocol). Example: 'https://rds.amazonaws.com' # * :server: RDS service host, default: DEFAULT_HOST # * :port: RDS service port, default: DEFAULT_PORT # * :protocol: 'http' or 'https', default: DEFAULT_PROTOCOL # * :logger: for log messages, default: RAILS_DEFAULT_LOGGER else STDOUT # # rds = RightAws::RdsInterface.new('xxxxxxxxxxxxxxxxxxxxx','xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', # {:logger => Logger.new('/tmp/x.log')}) #=> # # def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={}) init({ :name => 'RDS', :default_host => ENV['RDS_URL'] ? URI.parse(ENV['RDS_URL']).host : DEFAULT_HOST, :default_port => ENV['RDS_URL'] ? URI.parse(ENV['RDS_URL']).port : DEFAULT_PORT, :default_service => ENV['RDS_URL'] ? URI.parse(ENV['RDS_URL']).path : DEFAULT_PATH, :default_protocol => ENV['RDS_URL'] ? URI.parse(ENV['RDS_URL']).scheme : DEFAULT_PROTOCOL, :default_api_version => ENV['RDS_API_VERSION'] || API_VERSION }, aws_access_key_id || ENV['AWS_ACCESS_KEY_ID'], aws_secret_access_key || ENV['AWS_SECRET_ACCESS_KEY'], params) end #----------------------------------------------------------------- # Requests #----------------------------------------------------------------- # Generates request hash for REST API. def generate_request(action, params={}) #:nodoc: generate_request_impl(:get, action, params ) end # Sends request to Amazon and parses the response. # Raises AwsError if any banana happened. def request_info(request, parser, &block) # :nodoc: request_info_impl(:rds_connection, @@bench, request, parser, &block) end # Incrementally lists something. def incrementally_list_items(action, parser_class, params={}, &block) # :nodoc: params = params.dup params['MaxRecords'] = params.delete(:max_records) if params[:max_records] params['Marker'] = params.delete(:marker) if params[:marker] last_response = nil loop do link = generate_request(action, params) last_response = request_info( link, parser_class.new(:logger => @logger)) params['Marker'] = last_response[:marker] break unless block && block.call(last_response) && !last_response[:marker].right_blank? end last_response end #----------------------------------------------------------------- # API Calls: #----------------------------------------------------------------- # -------------------------------------------- # DB Instances # -------------------------------------------- # List DB instances. # # Optional params: +:aws_id+, +:max_records+, +:marker+ # # # Get a list of DB instances. The response is an +Array+ of instances. # rds.describe_db_instances #=> # [{:instance_class=>"db.m1.small", # :status=>"creating", # :backup_retention_period=>1, # :read_replica_db_instance_identifiers=>["kd-delete-me-01-replica-01"], # :master_username=>"username", # :preferred_maintenance_window=>"sun:05:00-sun:09:00", # :db_parameter_group=>{:status=>"in-sync", :name=>"default.mysql5.1"}, # :multi_az=>true, # :engine=>"mysql", # :auto_minor_version_upgrade=>false, # :allocated_storage=>25, # :availability_zone=>"us-east-1d", # :aws_id=>"kd-delete-me-01", # :preferred_backup_window=>"03:00-05:00", # :engine_version=>"5.1.50", # :pending_modified_values=>{:master_user_password=>"****"}, # :db_security_groups=>[{:status=>"active", :name=>"default"}]}] # # # Retrieve a custom DB instance. # # The response is an +Array+ with a single instance record. # rds.describe_db_instances("kd-test-n3") # # # Incrementally a list DB instances. Every response part is a +Hash+. # rds.describe_db_instances(:max_records => 30) do |x| # puts x.inspect #=> # {:db_instances=> # [{:instance_class=>"db.m1.small", # :status=>"creating", # :backup_retention_period=>1, # :read_replica_db_instance_identifiers=>["kd-delete-me-01-replica-01"], # :master_username=>"username", # :preferred_maintenance_window=>"sun:05:00-sun:09:00", # :db_parameter_group=>{:status=>"in-sync", :name=>"default.mysql5.1"}, # :multi_az=>true, # :engine=>"mysql", # :auto_minor_version_upgrade=>false, # :allocated_storage=>25, # :availability_zone=>"us-east-1d", # :aws_id=>"kd-delete-me-01", # :preferred_backup_window=>"03:00-05:00", # :engine_version=>"5.1.50", # :pending_modified_values=>{:master_user_password=>"****"}, # :db_security_groups=>[{:status=>"active", :name=>"default"}]}]} # true # end # def describe_db_instances(*params, &block) item, params = AwsUtils::split_items_and_params(params) params = params.dup params['DBInstanceIdentifier'] = item.first unless item.right_blank? result = [] incrementally_list_items('DescribeDBInstances', DescribeDbInstancesParser, params) do |response| result += response[:db_instances] block ? block.call(response) : true end result end # Create a new RDS instance of the type and size specified by you. The default storage engine for RDS Instances is InnoDB. # # Mandatory arguments: +aws_id+, +master_username+, +master_user_password+ # Optional params: +:allocated_storage+ (25 by def), +:instance_class+, +:engine+ ('MySQL' by def), # +:endpoint_port+, +:db_name+, +:db_security_groups+, +:db_parameter_group+, +:availability_zone+, +:preferred_maintenance_window+ # +:backup_retention_period+, +:preferred_backup_window+, +:multi_az+, +:engine_version+, +:auto_minor_version_upgrade+, # +:license_model+, +:iops+, +:db_subnet_group_name+, +:character_set_name+, +:option_group_name+ # # rds.create_db_instance('kd-delete-me-01', 'username', 'password', # :instance_class => 'db.m1.small', # :multi_az => true, # :auto_minor_version_upgrade => false ) #=> # {:instance_class=>"db.m1.small", # :multi_az=>true, # :status=>"creating", # :backup_retention_period=>1, # :read_replica_db_instance_identifiers=>[], # :master_username=>"username", # :preferred_maintenance_window=>"sun:05:00-sun:09:00", # :auto_minor_version_upgrade=>false, # :db_parameter_group=>{:status=>"in-sync", :name=>"default.mysql5.1"}, # :engine=>"mysql", # :allocated_storage=>25, # :aws_id=>"kd-delete-me-01", # :preferred_backup_window=>"03:00-05:00", # :engine_version=>"5.1.50", # :pending_modified_values=>{:master_user_password=>"****"}, # :db_security_groups=>[{:status=>"active", :name=>"default"}]} # def create_db_instance(aws_id, master_username, master_user_password, params={}) request_hash = {} # Mandatory request_hash['DBInstanceIdentifier'] = aws_id request_hash['MasterUsername'] = master_username request_hash['MasterUserPassword'] = master_user_password # Mandatory with default values request_hash['DBInstanceClass'] = params[:instance_class].right_blank? ? DEFAULT_INSTANCE_CLASS : params[:instance_class].to_s request_hash['AllocatedStorage'] = params[:allocated_storage].right_blank? ? 25 : params[:allocated_storage] request_hash['Engine'] = params[:engine].right_blank? ? 'mysql' : params[:engine] # Optional request_hash['Port'] = params[:endpoint_port] unless params[:endpoint_port].right_blank? request_hash['DBName'] = params[:db_name] unless params[:db_name].right_blank? request_hash['AvailabilityZone'] = params[:availability_zone] unless params[:availability_zone].right_blank? request_hash['MultiAZ'] = params[:multi_az].to_s unless params[:multi_az].nil? request_hash['PreferredMaintenanceWindow'] = params[:preferred_maintenance_window] unless params[:preferred_maintenance_window].right_blank? request_hash['BackupRetentionPeriod'] = params[:backup_retention_period] unless params[:backup_retention_period].right_blank? request_hash['PreferredBackupWindow'] = params[:preferred_backup_window] unless params[:preferred_backup_window].right_blank? request_hash['DBParameterGroupName'] = params[:db_parameter_group] unless params[:db_parameter_group].right_blank? request_hash['EngineVersion'] = params[:engine_version] unless params[:engine_version].right_blank? request_hash['AutoMinorVersionUpgrade'] = params[:auto_minor_version_upgrade].to_s unless params[:auto_minor_version_upgrade].nil? request_hash['LicenseModel'] = params[:license_model] unless params[:license_model].right_blank? request_hash['CharacterSetName'] = params[:character_set_name] unless params[:character_set_name].right_blank? request_hash['DBSubnetGroupName'] = params[:db_subnet_group_name] unless params[:db_subnet_group_name].right_blank? request_hash['Iops'] = params[:iops] unless params[:iops].right_blank? request_hash['OptionGroupName'] = params[:option_group_name] unless params[:option_group_name].right_blank? request_hash.merge!(amazonize_list('DBSecurityGroups.member', params[:db_security_groups])) link = generate_request('CreateDBInstance', request_hash) request_info(link, DescribeDbInstancesParser.new(:logger => @logger))[:db_instances].first end # Modify a DB instance. # # Mandatory arguments: +aws_id+. # Optional params: +:master_user_password+, +:instance_class+, +:db_security_groups+, # +:db_parameter_group+, +:preferred_maintenance_window+, +:allocated_storage+, +:apply_immediately+, # +:backup_retention_period+, +:preferred_backup_window+, +:multi_az+, +:engine_version+, # +:auto_minor_version_upgrade+, +:allow_major_version_upgrade+, +:iops+, +::option_group_name+ # # rds.modify_db_instance('kd-delete-me-01', # :master_user_password => 'newpassword', # :instance_class => 'db.m1.large', # :multi_az => false, # :allocated_storage => 30, # :allow_major_version_upgrade => true, # :auto_minor_version_upgrade => true, # :preferred_maintenance_window => 'sun:06:00-sun:10:00', # :preferred_backup_window => '02:00-04:00', # :apply_immediately => true, # :backup_retention_period => 2) #=> # {:engine_version=>"5.1.50", # :aws_id=>"kd-delete-me-01", # :multi_az=>true, # :status=>"available", # :read_replica_db_instance_identifiers=>[], # :availability_zone=>"us-east-1d", # :auto_minor_version_upgrade=>true, # :master_username=>"username", # :preferred_maintenance_window=>"sun:06:00-sun:10:00", # :db_parameter_group=>{:status=>"in-sync", :name=>"default.mysql5.1"}, # :create_time=>"2010-11-17T10:21:59.720Z", # :preferred_backup_window=>"02:00-04:00", # :engine=>"mysql", # :db_security_groups=>[{:status=>"active", :name=>"default"}], # :endpoint_address=>"kd-delete-me-01.chxspydgchoo.us-east-1.rds.amazonaws.com", # :instance_class=>"db.m1.small", # :latest_restorable_time=>"2010-11-17T10:27:17.089Z", # :backup_retention_period=>2, # :pending_modified_values=> # {:multi_az=>false, :master_user_password=>"****", :allocated_storage=>30, :instance_class=>"db.m1.large"}, # :allocated_storage=>25} # def modify_db_instance(aws_id, params={}) request_hash = {} # Mandatory request_hash['DBInstanceIdentifier'] = aws_id # Optional request_hash['MasterUserPassword'] = params[:master_user_password] unless params[:master_user_password].right_blank? request_hash['DBInstanceClass'] = params[:instance_class].to_s.capitalize unless params[:instance_class].right_blank? request_hash['PreferredMaintenanceWindow'] = params[:preferred_maintenance_window] unless params[:preferred_maintenance_window].right_blank? request_hash['BackupRetentionPeriod'] = params[:backup_retention_period] unless params[:backup_retention_period].right_blank? request_hash['PreferredBackupWindow'] = params[:preferred_backup_window] unless params[:preferred_backup_window].right_blank? request_hash['AllocatedStorage'] = params[:allocated_storage] unless params[:allocated_storage].right_blank? request_hash['MultiAZ'] = params[:multi_az].to_s unless params[:multi_az].nil? request_hash['EngineVersion'] = params[:engine_version] unless params[:engine_version].right_blank? request_hash['AutoMinorVersionUpgrade'] = params[:auto_minor_version_upgrade].to_s unless params[:auto_minor_version_upgrade].nil? request_hash['AllowMajorVersionUpgrade'] = params[:allow_major_version_upgrade].to_s unless params[:allow_major_version_upgrade].nil? request_hash['ApplyImmediately'] = params[:apply_immediately].to_s unless params[:apply_immediately].right_blank? request_hash.merge!(amazonize_list('DBSecurityGroups.member', params[:db_security_groups])) request_hash['DBParameterGroupName'] = params[:db_parameter_group] unless params[:db_parameter_group].right_blank? request_hash['Iops'] = params[:iops] unless params[:iops].right_blank? request_hash['OptionGroupName'] = params[:option_group_name] unless params[:option_group_name].right_blank? link = generate_request('ModifyDBInstance', request_hash) request_info(link, DescribeDbInstancesParser.new(:logger => @logger))[:db_instances].first end # Reboot Db instance. # # Options: +:force_failover+ # # rds.reboot_db_instance('kd-my-awesome-db') #=> # {:status=>"rebooting", # :pending_modified_values=>{}, # :allocated_storage=>42, # :master_username=>"kd", # :db_security_groups=>[], # :instance_class=>"Medium", # :availability_zone=>"us-east-1a", # :aws_id=>"kd-my-awesome-db", # :create_time=>"2009-08-28T08:34:21.858Z", # :engine=>"MySQL5.1", # :preferred_maintenance_window=>"Sun:05:00-Sun:09:00"} # # P.S. http://docs.amazonwebservices.com/AmazonRDS/latest/APIReference/API_RebootDBInstance.html # def reboot_db_instance(aws_id, params={}) params = params.dup params['DBInstanceIdentifier'] = aws_id params['ForceFailover'] = !!params[:force_failover] if params.has_key?(:force_failover) link = generate_request('RebootDBInstance', params) request_info(link, DescribeDbInstancesParser.new(:logger => @logger))[:db_instances].first end # Delete a DB instance # # Mandatory arguments: aws_id # Optional params: :skip_final_snapshot ('false' by def), # :snapshot_aws_id ('{instance_aws_id}-final-snapshot-YYYYMMDDHHMMSS') # # rds.delete_db_instance('my-awesome-db-g2') #=> true # def delete_db_instance(aws_id, params={}) request_hash = {} request_hash['DBInstanceIdentifier'] = aws_id request_hash['SkipFinalSnapshot'] = params.has_key?(:skip_final_snapshot) ? params[:skip_final_snapshot].to_s : 'false' if request_hash['SkipFinalSnapshot'] == 'false' && params[:snapshot_aws_id].right_blank? params = params.dup params[:snapshot_aws_id] = "#{aws_id}-final-snapshot-#{Time.now.utc.strftime('%Y%m%d%H%M%S')}" end request_hash['FinalDBSnapshotIdentifier'] = params[:snapshot_aws_id] unless params[:snapshot_aws_id].right_blank? link = generate_request('DeleteDBInstance', request_hash) request_info(link, DescribeDbInstancesParser.new(:logger => @logger))[:db_instances].first end # -------------------------------------------- # DB SecurityGroups # -------------------------------------------- # # rds.describe_db_security_groups #=> # [{:owner_id=>"82...25", # :description=>"Default", # :ec2_security_groups=>[], # :ip_ranges=>[], # :name=>"Default"}, # {:owner_id=>"82...25", # :description=>"kd", # :ec2_security_groups=>[], # :ip_ranges=>[], # :name=>"kd2"}, # {:owner_id=>"82...25", # :description=>"kd", # :ec2_security_groups=> # [{:status=>"Authorized", :owner_id=>"82...23", :name=>"default"}, # {:status=>"Authorized", :owner_id=>"82...24", :name=>"default1"}, # {:status=>"Authorized", :owner_id=>"82...25", :name=>"default"}, # {:status=>"Authorized", :owner_id=>"82...26", :name=>"default"}, # {:status=>"Authorized", :owner_id=>"82...26", :name=>"default1"}, # {:status=>"Authorized", :owner_id=>"82...29", :name=>"default22"}], # :ip_ranges=> # [{:status=>"Authorized", :cidrip=>"127.0.0.1/8"}, # {:status=>"Authorized", :cidrip=>"128.0.0.1/8"}, # {:status=>"Authorized", :cidrip=>"129.0.0.1/8"}, # {:status=>"Authorized", :cidrip=>"130.0.0.1/8"}, # {:status=>"Authorized", :cidrip=>"131.0.0.1/8"}], # :name=>"kd3"}] # # # get a custom group # rds.describe_db_security_groups('kd3') # def describe_db_security_groups(*db_security_group_name, &block) items, params = AwsUtils::split_items_and_params(db_security_group_name) params['DBSecurityGroupName'] = items.first unless items.right_blank? result = [] incrementally_list_items('DescribeDBSecurityGroups', DescribeDbSecurityGroupsParser, params) do |response| result += response[:db_security_groups] block ? block.call(response) : true end result end # Create a database security group so that ingress to an RDS Instance can be controlled. # A new security group cannot have the same name as an existing group. # # Options: :vpc_id # # ds.create_db_security_group('kd3', 'kd') #=> # {:ec2_security_groups=>[], # :description=>"kd", # :ip_ranges=>[], # :name=>"kd3", # :owner_id=>"82...25"} # def create_db_security_group(db_security_group_name, db_security_group_description, params={}) request_hash = { 'DBSecurityGroupName' => db_security_group_name, 'DBSecurityGroupDescription' => db_security_group_description } request_hash['EC2VpcId'] = params[:vpc_id] unless params[:vpc_id].right_blank? link = generate_request('CreateDBSecurityGroup', request_hash) request_info(link, DescribeDbSecurityGroupsParser.new(:logger => @logger))[:db_security_groups].first end def modify_db_security_group_ingress(action, db_security_group_name, params={}) # :nodoc: request_hash = { 'DBSecurityGroupName' => db_security_group_name} request_hash['CIDRIP'] = params[:cidrip] unless params[:cidrip].right_blank? request_hash['EC2SecurityGroupName'] = params[:ec2_security_group_name] unless params[:ec2_security_group_name].right_blank? request_hash['EC2SecurityGroupOwnerId'] = params[:ec2_security_group_owner] unless params[:ec2_security_group_owner].right_blank? request_hash['EC2SecurityGroupId'] = params[:ec2_security_group_id] unless params[:ec2_security_group_id].right_blank? link = generate_request(action, request_hash) request_info(link, DescribeDbSecurityGroupsParser.new(:logger => @logger))[:db_security_groups].first end # Authorize an ingress. Params: +:cidrip+ or (+:ec2_security_group_name+ and +:ec2_security_group_owner+) # # rds.authorize_db_security_group_ingress('kd3', :cidrip => '131.0.0.1/8') # {:owner_id=>"82...25", # :ec2_security_groups=>[], # :description=>"kd", # :ip_ranges=> # [{:status=>"Authorized", :cidrip=>"127.0.0.1/8"}, # {:status=>"Authorized", :cidrip=>"128.0.0.1/8"}, # {:status=>"Authorized", :cidrip=>"129.0.0.1/8"}, # {:status=>"Authorized", :cidrip=>"130.0.0.1/8"}, # {:status=>"Authorizing", :cidrip=>"131.0.0.1/8"}], # :name=>"kd3"} # # rds.authorize_db_security_group_ingress('kd3',:ec2_security_group_owner => '82...27', # :ec2_security_group_name => 'default') #=> # {:owner_id=>"82...25", # :ec2_security_groups=> # [{:status=>"Authorized", :owner_id=>"82...25", :name=>"g1"}, # {:status=>"Authorized", :owner_id=>"82...26", :name=>"g2"}, # {:status=>"Authorizing", :owner_id=>"82...27", :name=>"default"}], # :ip_ranges=> # [{:status=>"Authorized", :cidrip=>"127.0.0.1/8"}, # {:status=>"Authorized", :cidrip=>"128.0.0.1/8"}, # {:status=>"Authorized", :cidrip=>"129.0.0.1/8"}, # {:status=>"Authorized", :cidrip=>"130.0.0.1/8"}, # {:status=>"Authorized", :cidrip=>"131.0.0.1/8"}], # :name=>"kd3"} # def authorize_db_security_group_ingress(db_security_group_name, params={}) modify_db_security_group_ingress('AuthorizeDBSecurityGroupIngress', db_security_group_name, params) end # Revoke an ingress. # Optional params: +:cidrip+ or (+:ec2_security_group_name+ and +:ec2_security_group_owner+) # # rds.revoke_db_security_group_ingress('kd3', :ec2_security_group_owner => '82...25', # :ec2_security_group_name => 'default') #=> # {:owner_id=>"82...25", # :ec2_security_groups=> # [{:status=>"Revoking", :owner_id=>"826693181925", :name=>"default"}], # :name=>"kd3", # :description=>"kd", # :ip_ranges=> # [{:status=>"Authorized", :cidrip=>"127.0.0.1/8"}, # {:status=>"Authorized", :cidrip=>"128.0.0.1/8"}, # {:status=>"Authorized", :cidrip=>"129.0.0.1/8"}, # {:status=>"Authorized", :cidrip=>"130.0.0.1/8"}, # {:status=>"Authorized", :cidrip=>"131.0.0.1/8"}]} # def revoke_db_security_group_ingress(db_security_group_name, params={}) modify_db_security_group_ingress('RevokeDBSecurityGroupIngress', db_security_group_name, params) end # Delete a database security group. Database security group must not be associated with any # RDS Instances. # # rds.delete_db_security_group('kd3') #=> true # def delete_db_security_group(db_security_group_name) link = generate_request('DeleteDBSecurityGroup', 'DBSecurityGroupName' => db_security_group_name) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # -------------------------------------------- # DB ParameterGroups # -------------------------------------------- # Describe DBParameterGroups. # # rds.describe_db_parameter_groups #=> # [{:engine=>"MySQL5.1", # :description=>"Default parameter group for MySQL5.1", # :name=>"default.MySQL5.1"}] # # # List parameter groups by 20 # rds.describe_db_parameter_groups(:max_records=>20) do |response| # puts response.inspect # true # end # def describe_db_parameter_groups(*db_parameter_group_name, &block) items, params = AwsUtils::split_items_and_params(db_parameter_group_name) params['DBParameterGroupName'] = items.first unless items.right_blank? result = [] incrementally_list_items('DescribeDBParameterGroups', DescribeDbParameterGroupsParser, params) do |response| result += response[:db_parameter_groups] block ? block.call(response) : true end result end # Creates a database parameter group so that configuration of an RDS Instance can be controlled. # # rds.create_db_parameter_group('my-new-group-1','My new group') #=> {} # # TODO: this call returns an empty hash, but should be a parameter group data - ask Amazon guys. # def create_db_parameter_group(db_parameter_group_name, db_parameter_group_description, db_parameter_group_family='mysql5.1', params={}) params['DBParameterGroupName'] = db_parameter_group_name params['Description'] = db_parameter_group_description params['DBParameterGroupFamily'] = db_parameter_group_family link = generate_request('CreateDBParameterGroup', params ) request_info(link, DescribeDbParameterGroupsParser.new(:logger => @logger))[:db_parameter_groups].first end # Modify DBParameterGroup paramaters. Up to 20 params can be midified at once. # # rds.modify_db_parameter_group('kd1', 'max_allowed_packet' => 2048) #=> true # # rds.modify_db_parameter_group('kd1', 'max_allowed_packet' => {:value => 2048, :method => 'immediate') #=> true # def modify_db_parameter_group(db_parameter_group_name, params={}) # :nodoc: request_hash = { 'DBParameterGroupName' => db_parameter_group_name} parameters = [] params.each do |key, value| method = 'pending-reboot' if value.is_a?(Hash) method = value[:method] unless value[:method].right_blank? value = value[:value] end parameters << [key, value, method] end request_hash.merge!( amazonize_list(['Parameters.member.?.ParameterName', 'Parameters.member.?.ParameterValue', 'Parameters.member.?.ApplyMethod'], parameters )) link = generate_request('ModifyDBParameterGroup', request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Delete DBParameter Group. # # rds.delete_db_parameter_group('kd1') #=> true # def delete_db_parameter_group(db_parameter_group_name) link = generate_request('DeleteDBParameterGroup', 'DBParameterGroupName' => db_parameter_group_name) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Modify the parameters of a DBParameterGroup to the engine/system default value. # # # Reset all parameters # rds.reset_db_parameter_group('kd2', :all ) #=> true # # # Reset custom parameters # rds.reset_db_parameter_group('kd2', 'max_allowed_packet', 'auto_increment_increment' ) #=> true # rds.reset_db_parameter_group('kd2', 'max_allowed_packet', 'auto_increment_increment' => 'immediate' ) #=> true # def reset_db_parameter_group(db_parameter_group_name, *params) params = params.flatten request_hash = { 'DBParameterGroupName' => db_parameter_group_name } if params.first.to_s == 'all' request_hash['ResetAllParameters'] = true else tmp = [] params.each{ |item| tmp |= item.to_a } params = [] tmp.each do |key, method| method = 'pending-reboot' unless method params << [key, method] end request_hash.merge!( amazonize_list(['Parameters.member.?.ParameterName', 'Parameters.member.?.ApplyMethod'], params )) end link = generate_request('ResetDBParameterGroup', request_hash) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # Get the detailed parameters list for a particular DBParameterGroup. # # rds.describe_db_parameters('kd1') #=> # [{:is_modifiable=>true, # :apply_type=>"static", # :source=>"engine-default", # :allowed_values=>"ON,OFF", # :description=>"Controls whether user-defined functions that have only an xxx symbol for the main function can be loaded", # :name=>"allow-suspicious-udfs", # :data_type=>"boolean"}, # {:is_modifiable=>true, # :apply_type=>"dynamic", # :source=>"engine-default", # :allowed_values=>"1-65535", # :description=>"Intended for use with master-to-master replication, and can be used to control the operation of AUTO_INCREMENT columns", # :name=>"auto_increment_increment", # :data_type=>"integer"}, ... ] # # # List parameters by 20 # rds.describe_db_parameters('kd1', :max_records=>20) do |response| # puts response.inspect # true # end # def describe_db_parameters(*db_parameter_group_name, &block) item, params = AwsUtils::split_items_and_params(db_parameter_group_name) params['DBParameterGroupName'] = item result = [] incrementally_list_items('DescribeDBParameters', DescribeDbParametersParser, params) do |response| result += response[:parameters] block ? block.call(response) : true end result end # Describe a default parameters for the parameter group family. # # rds.describe_engine_default_parameters('MySQL5.1') #=> # [{:is_modifiable=>true, # :apply_type=>"static", # :source=>"engine-default", # :allowed_values=>"ON,OFF", # :description=>"Controls whether user-defined functions that have only an xxx symbol for the main function can be loaded", # :name=>"allow-suspicious-udfs", # :data_type=>"boolean"}, # {:is_modifiable=>true, # :apply_type=>"dynamic", # :source=>"engine-default", # :allowed_values=>"1-65535", # :description=>"Intended for use with master-to-master replication, and can be used to control the operation of AUTO_INCREMENT columns", # :name=>"auto_increment_increment", # :data_type=>"integer"}, ... ] # def describe_engine_default_parameters(*db_parameter_group_family, &block) db_parameter_group_family = ['MySQL5.1'] if db_parameter_group_family.right_blank? item, params = AwsUtils::split_items_and_params(db_parameter_group_family) params['DBParameterGroupFamily'] = item if item result = [] incrementally_list_items('DescribeEngineDefaultParameters', DescribeDbParametersParser, params) do |response| result += response[:parameters] block ? block.call(response) : true end result end # Describe a list of orderable DB Instance options for the specified engine. # Optionals: +:instance_class+, +:engine_version+ , +:license_model+, +:vpc+ # # rds.describe_orderable_db_instance_options('oracle-ee', :engine_version => '11.2.0.2.v2') #=> # [{:read_replica_capable=>false, # :instance_class=>"db.m1.large", # :availability_zones=>["us-east-1a", "us-east-1b", "us-east-1d"], # :engine=>"oracle-ee", # :license_model=>"bring-your-own-license", # :engine_version=>"11.2.0.2.v2", # :multi_az_capable=>"false"}, ... ] # def describe_orderable_db_instance_options(engine, params={}, &block) request_hash = { 'Engine' => engine } request_hash['DBInstanceClass'] = params[:instance_class] unless params[:instance_class].right_blank? request_hash['EngineVersion'] = params[:engine_version] unless params[:engine_version].right_blank? request_hash['LicenseModel'] = params[:license_model] unless params[:license_model].right_blank? request_hash['Vpc'] = !!params[:vpc] if params.has_key?(:vpc) result = [] incrementally_list_items('DescribeOrderableDBInstanceOptions', DescribeOrderableDBInstanceOptionsParser, request_hash) do |response| result += response[:items] block ? block.call(response) : true end result end # -------------------------------------------- # DB Snapshots # -------------------------------------------- # Get DBSecurityGroup details for a particular customer or for a particular DBSecurityGroup if a name is specified. # Optional params: +:instance_aws_id+ # # # all snapshots # rds.describe_db_snapshots #=> # [{:status=>"Available", # :instance_aws_id=>"kd-test-n1", # :allocated_storage=>25, # :availability_zone=>"us-east-1b", # :aws_id=>"kd-test-n1-final-snapshot-at-20090630131215", # :engine=>"MySQL5.1", # :endpoint_port=>3306, # :instance_create_time=>"2009-06-30T12:48:15.590Z", # :master_username=>"payless", # :snapshot_time=>"2009-06-30T13:16:48.496Z"}, ...] # # # all snapshots for a custom instance # rds.describe_db_snapshots(:instance_aws_id => 'kd-test-n3') #=> # [{:status=>"Available", # :instance_aws_id=>"kd-test-n3", # :allocated_storage=>25, # :availability_zone=>"us-east-1a", # :aws_id=>"kd-test-n3-final-snapshot-20090713074916", # :engine=>"MySQL5.1", # :endpoint_port=>3306, # :instance_create_time=>"2009-06-30T12:51:32.540Z", # :master_username=>"payless", # :snapshot_time=>"2009-07-13T07:52:35.542Z"}] # # # a snapshot by id # rds.describe_db_snapshots('my-awesome-db-final-snapshot-20090713075554') #=> # [{:status=>"Available", # :allocated_storage=>25, # :engine=>"MySQL5.1", # :instance_aws_id=>"my-awesome-db", # :availability_zone=>"us-east-1a", # :instance_create_time=>"2009-07-13T07:53:08.912Z", # :endpoint_port=>3306, # :master_username=>"medium", # :aws_id=>"my-awesome-db-final-snapshot-20090713075554", # :snapshot_time=>"2009-07-13T07:59:17.537Z"}] # def describe_db_snapshots(params={}, &block) item, params = AwsUtils::split_items_and_params(params) params['DBSnapshotIdentifier'] = item if item params['DBInstanceIdentifier'] = params.delete(:instance_aws_id) unless params[:instance_aws_id].right_blank? result = [] incrementally_list_items('DescribeDBSnapshots', DescribeDbSnapshotsParser, params) do |response| result += response[:db_snapshots] block ? block.call(response) : true end result end # Create a DBSnapshot. The source DBInstance must be in Available state # # rds.create_db_snapshot('remove-me-tomorrow-2', 'my-awesome-db-g7' ) #=> # {:status=>"PendingCreation", # :allocated_storage=>50, # :availability_zone=>"us-east-1b", # :engine=>"MySQL5.1", # :aws_id=>"remove-me-tomorrow-2", # :instance_create_time=>"2009-07-13T09:35:39.243Z", # :endpoint_port=>3306, # :instance_aws_id=>"my-awesome-db-g7", # :db_master_username=>"username"} # def create_db_snapshot(aws_id, instance_aws_id) link = generate_request('CreateDBSnapshot', 'DBSnapshotIdentifier' => aws_id, 'DBInstanceIdentifier' => instance_aws_id) request_info(link, DescribeDbSnapshotsParser.new(:logger => @logger))[:db_snapshots].first end # Create a new RDS instance from a DBSnapshot. The source DBSnapshot must be # in the "Available" state. The new RDS instance is created with the Default security group. # # Optional params: +:instance_class+, +:endpoint_port+, +:availability_zone+, +:multi_az+, # +:auto_minor_version_upgrade+, +:license_model+, +:db_name+, +:engine+, +:db_subnet_group_name+, # +:iops+, +:option_group_name+ # # rds.restore_db_instance_from_db_snapshot('ahahahaha-final-snapshot-20090828081159', 'q1') #=> # {:status=>"creating", # :pending_modified_values=>{}, # :allocated_storage=>42, # :db_security_groups=>[], # :master_username=>"kd", # :availability_zone=>"us-east-1a", # :aws_id=>"q1", # :create_time=>"2009-08-29T18:07:01.510Z", # :instance_class=>"Medium", # :preferred_maintenance_window=>"Sun:05:00-Sun:09:00", # :engine=>"MySQL", # :engine_version=>"5.1.49"} # def restore_db_instance_from_db_snapshot(snapshot_aws_id, instance_aws_id, params={}) request_hash = { 'DBSnapshotIdentifier' => snapshot_aws_id, 'DBInstanceIdentifier' => instance_aws_id } request_hash['DBInstanceClass'] = params[:instance_class] unless params[:instance_class].right_blank? request_hash['Port'] = params[:endpoint_port] unless params[:endpoint_port].right_blank? request_hash['AvailabilityZone'] = params[:availability_zone] unless params[:availability_zone].right_blank? request_hash['MultiAZ'] = params[:multi_az] unless params[:multi_az].nil? request_hash['AutoMinorVersionUpgrade'] = params[:auto_minor_version_upgrade] unless params[:auto_minor_version_upgrade].nil? request_hash['LicenseModel'] = params[:license_model] unless params[:license_model].right_blank? request_hash['DBName'] = params[:db_name] unless params[:db_name].right_blank? request_hash['Engine'] = params[:engine] unless params[:enginel].right_blank? request_hash['DBSubnetGroupName'] = params[:db_subnet_group_name] unless params[:db_subnet_group_name].right_blank? request_hash['Iops'] = params[:iops] unless params[:iops].right_blank? request_hash['OptionGroupName'] = params[:option_group_name] unless params[:option_group_name].right_blank? link = generate_request('RestoreDBInstanceFromDBSnapshot', request_hash) request_info(link, DescribeDbInstancesParser.new(:logger => @logger))[:db_instances].first end # Create a new RDS instance from a point-in-time system snapshot. The target # database is created from the source database restore point with the same configuration as # the original source database, except that the new RDS instance is created with the default # security group. # # Optional params: +:instance_class+, +:endpoint_port+, +:availability_zone+, +:multi_az+, +:restore_time+, # +:auto_minor_version_upgrade+, +:use_latest_restorable_time+, +:license_model+, +:db_name+, +:engine+ # +:db_subnet_group_name+, +:iops+, +:option_group_name+ # def restore_db_instance_to_point_in_time(instance_aws_id, new_instance_aws_id, params={}) request_hash = { 'SourceDBInstanceIdentifier' => instance_aws_id, 'TargetDBInstanceIdentifier' => new_instance_aws_id} request_hash['UseLatestRestorableTime'] = params[:use_latest_restorable_time].to_s unless params[:use_latest_restorable_time].nil? request_hash['RestoreTime'] = params[:restore_time] unless params[:restore_time].right_blank? request_hash['DBInstanceClass'] = params[:instance_class] unless params[:instance_class].right_blank? request_hash['MultiAZ'] = params[:multi_az] unless params[:multi_az].nil? request_hash['Port'] = params[:endpoint_port] unless params[:endpoint_port].right_blank? request_hash['AvailabilityZone'] = params[:availability_zone] unless params[:availability_zone].right_blank? request_hash['AutoMinorVersionUpgrade'] = params[:auto_minor_version_upgrade] unless params[:auto_minor_version_upgrade].nil? request_hash['LicenseModel'] = params[:license_model] unless params[:license_model].right_blank? request_hash['DBName'] = params[:db_name] unless params[:db_name].right_blank? request_hash['Engine'] = params[:engine] unless params[:enginel].right_blank? request_hash['DBSubnetGroupName'] = params[:db_subnet_group_name] unless params[:db_subnet_group_name].right_blank? request_hash['Iops'] = params[:iops] unless params[:iops].right_blank? request_hash['OptionGroupName'] = params[:option_group_name] unless params[:option_group_name].right_blank? link = generate_request('RestoreDBInstanceToPointInTime', request_hash) request_info(link, DescribeDbInstancesParser.new(:logger => @logger))[:db_instances].first end # Delete a DBSnapshot. The DBSnapshot must be in the Available state to be deleted. # # rds.delete_db_snapshot('remove-me-tomorrow-1') #=> # {:status=>"Deleted", # :allocated_storage=>50, # :instance_create_time=>"2009-07-13T09:27:01.053Z", # :availability_zone=>"us-east-1a", # :db_master_username=>"username", # :aws_id=>"remove-me-tomorrow-1", # :snapshot_time=>"2009-07-13T10:59:30.227Z", # :endpoint_port=>3306, # :instance_aws_id=>"my-awesome-db-g5", # :engine=>"MySQL5.1"} # def delete_db_snapshot(aws_id) link = generate_request('DeleteDBSnapshot', 'DBSnapshotIdentifier' => aws_id) request_info(link, DescribeDbSnapshotsParser.new(:logger => @logger))[:db_snapshots].first end # -------------------------------------------- # DB Events # -------------------------------------------- # Get events related to RDS instances and DBSecurityGroups for the past 14 days. # Optional params: +:duration+, +:start_time+, +:end_time+, +:aws_id+, # +:source_type+('db-instance', 'db-security-group', 'db-snapshot', 'db-parameter-group') # # # get all enevts # rds.describe_events #=> # [{:aws_id=>"my-awesome-db-g4", # :source_type=>"DBInstance", # :message=>"Started user snapshot for database instance:my-awesome-db-g4", # :date=>"2009-07-13T10:54:13.661Z"}, # {:aws_id=>"my-awesome-db-g5", # :source_type=>"DBInstance", # :message=>"Started user snapshot for database instance:my-awesome-db-g5", # :date=>"2009-07-13T10:55:13.674Z"}, # {:aws_id=>"my-awesome-db-g7", # :source_type=>"DBInstance", # :message=>"Started user snapshot for database instance:my-awesome-db-g7", # :date=>"2009-07-13T10:56:34.226Z"}] # # # get all events since yesterday # rds.describe_events(:start_date => 1.day.ago) # # # get last 60 min events # rds.describe_events(:duration => 60) # def describe_events(params={}, &block) params = params.dup params['SourceIdentifier'] = params.delete(:aws_id) unless params[:aws_id].right_blank? params['SourceType'] = params.delete(:source_type) unless params[:source_type].right_blank? params['Duration'] = params.delete(:duration) unless params[:duration].right_blank? params['StartDate'] = fix_date(params.delete(:start_date)) unless params[:start_date].right_blank? params['EndDate'] = fix_date(params.delete(:end_date)) unless params[:end_date].right_blank? result = [] incrementally_list_items('DescribeEvents', DescribeEventsParser, params) do |response| result += response[:events] block ? block.call(response) : true end result end def fix_date(date) # :nodoc: date = Time.at(date) if date.is_a?(Fixnum) date = date.utc.strftime('%Y-%m-%dT%H:%M:%SZ') if date.is_a?(Time) date end # -------------------------------------------- # DB Engine Versions # -------------------------------------------- # Get a list of the available DB engines. # Optional params: +:db_parameter_group_family+, +:default_only+, +:engine+, +:engine_version+. +:engine_version+ # # rds.describe_db_engine_versions #=> # [{:db_parameter_group_family=>"mysql5.1", # :engine=>"mysql", # :db_engine_description=>"MySQL Community Edition", # :db_engine_version_description=>"Mysql 5.1.45", # :engine_version=>"5.1.45"}, # {:db_parameter_group_family=>"oracle-se1-11.2", # :engine=>"oracle-se1", # :db_engine_description=>"Oracle Database Standard Edition One", # :db_engine_version_description=> # "Oracle Standard Edition One - DB Engine Version 11.2.0.2.v2", # :engine_version=>"11.2.0.2.v2"}] # # rds.describe_db_engine_versions(:list_supported_character_sets => true) #=> # [{:db_parameter_group_family=>"mysql5.1", # :engine=>"mysql", # :db_engine_description=>"MySQL Community Edition", # :engine_version=>"5.1.45", # :db_engine_version_description=>"MySQL 5.1.45"}, # {:db_parameter_group_family=>"oracle-ee-11.2", # :engine=>"oracle-ee", # :supported_character_sets=> # [{:name=>"AL32UTF8", # :description=>"Unicode 5.0 UTF-8 Universal character set"}, # {:name=>"JA16EUC", :description=>"EUC 24-bit Japanese"}, # {:name=>"JA16EUCTILDE", # :description=> # "The same as JA16EUC except for the way that the wave dash and the tilde are mapped to and from Unicode."}, # {:name=>"JA16SJIS", :description=>"Shift-JIS 16-bit Japanese"}, # {:name=>"WE8ISO8859P9", # :description=>"ISO 8859-9 West European and Turkish"}, # {:name=>"US7ASCII", :description=>"ASCII 7-bit American"}], # :db_engine_description=>"Oracle Database Enterprise Edition", # :default_character_set=> # {:name=>"AL32UTF8", # :description=>"Unicode 5.0 UTF-8 Universal character set"}, # :engine_version=>"11.2.0.2.v3", # :db_engine_version_description=>"Oracle 11.2.0.2.v3"}, ... ] # # P.S. http://docs.amazonwebservices.com/AmazonRDS/latest/APIReference/API_DescribeDBEngineVersions.html # def describe_db_engine_versions(params={}, &block) params = params.dup params['DBParameterGroupFamily'] = params.delete(:db_parameter_group_family) unless params[:db_parameter_group_family].right_blank? params['DefaultOnly'] = params.delete(:default_only).to_s unless params[:default_only].nil? params['Engine'] = params.delete(:engine) unless params[:engine].right_blank? params['EngineVersion'] = params.delete(:engine_version) unless params[:engine_version].right_blank? params['ListSupportedCharacterSets'] = !!params.delete(:list_supported_character_sets) if params.has_key?(:list_supported_character_sets) result = [] incrementally_list_items('DescribeDBEngineVersions', DescribeDBEngineVersionsParser, params) do |response| result += response[:db_engine_versions] block ? block.call(response) : true end result end # -------------------------------------------- # DB Replicas # -------------------------------------------- # Create a DB Instance that acts as a Read Replica of a source DB Instance. # # Optional params: +:endpoint_port+, +:availability_zone+, +:instance_class+, +:auto_minor_version_upgrade+, # +:iops+, +:option_group_name+ # # rds.create_db_instance_read_replica('kd-delete-me-01-replica-01', 'kd-delete-me-01', # :instance_class => 'db.m1.small', # :endpoint_port => '11000', # :auto_minor_version_upgrade => false ) #=> # {:auto_minor_version_upgrade=>false, # :read_replica_source_db_instance_identifier=>"kd-delete-me-01", # :status=>"creating", # :backup_retention_period=>0, # :allocated_storage=>30, # :read_replica_db_instance_identifiers=>[], # :engine_version=>"5.1.50", # :aws_id=>"kd-delete-me-01-replica-01", # :multi_az=>false, # :preferred_maintenance_window=>"sun:06:00-sun:10:00", # :master_username=>"username", # :preferred_backup_window=>"02:00-04:00", # :db_parameter_group=>{:status=>"in-sync", :name=>"default.mysql5.1"}, # :engine=>"mysql", # :db_security_groups=>[{:status=>"active", :name=>"default"}], # :instance_class=>"db.m1.small", # :pending_modified_values=>{}} # def create_db_instance_read_replica(aws_id, source_db_instance_identifier, params={}) request_hash = { 'DBInstanceIdentifier' => aws_id, 'SourceDBInstanceIdentifier' => source_db_instance_identifier} request_hash['Port'] = params[:endpoint_port] unless params[:endpoint_port].right_blank? request_hash['AvailabilityZone'] = params[:availability_zone] unless params[:availability_zone].right_blank? request_hash['DBInstanceClass'] = params[:instance_class] unless params[:instance_class].right_blank? request_hash['AutoMinorVersionUpgrade'] = params[:auto_minor_version_upgrade].to_s unless params[:auto_minor_version_upgrade].nil? request_hash['Iops'] = params[:iops] unless params[:iops].right_blank? request_hash['OptionGroupName'] = params[:option_group_name] unless params[:option_group_name].right_blank? link = generate_request('CreateDBInstanceReadReplica', request_hash) request_info(link, DescribeDbInstancesParser.new(:logger => @logger))[:db_instances].first end #--------------------------------------------- # Reserved Instances #--------------------------------------------- # Lists available reserved DB Instance offerings. # Options: :aws_id, :instance_class, :duration, :product_description, :multi_az # # rds.describe_reserved_db_instances_offerings #=> # [{:recurring_charges=>[], # :offering_type=>"Medium Utilization", # :duration=>94608000, # :currency_code=>"USD", # :fixed_price=>82.0, # :product_description=>"oracle-se(byol)", # :usage_price=>0.01, # :aws_id=>"248e7b75-01ff-4f1d-8fad-918b76337c13", # :multi_az=>false, # :instance_class=>"db.t1.micro"}, # {:recurring_charges=>[{:frequency=>"Hourly", :amount=>"0.24"}], # :offering_type=>"Heavy Utilization", # :duration=>31536000, # :currency_code=>"USD", # :fixed_price=>1040.0, # :product_description=>"sqlserver-web(li)", # :usage_price=>0.0, # :aws_id=>"248e7b75-05eb-46df-a7b8-4117b5001667", # :multi_az=>false, # :instance_class=>"db.m2.xlarge"}, ... ] # # rds.describe_reserved_db_instances_offerings(:aws_id => "248e7b75-49a7-4cd7-9a9b-354f4906a9b1") #=> # [{:duration=>94608000, # :multi_az=>true, # :fixed_price=>700.0, # :usage_price=>0.092, # :currency_code=>"USD", # :aws_id=>"248e7b75-49a7-4cd7-9a9b-354f4906a9b1", # :instance_class=>"db.m1.small", # :product_description=>"mysql"}] # # rds.describe_reserved_db_instances_offerings(:instance_class => "db.m1.small") # rds.describe_reserved_db_instances_offerings(:duration => 31536000) # rds.describe_reserved_db_instances_offerings(:product_description => 'mysql') # rds.describe_reserved_db_instances_offerings(:multi_az => true) # def describe_reserved_db_instances_offerings(params={}, &block) params = params.dup params['ReservedDBInstancesOfferingId'] = params.delete(:aws_id) unless params[:aws_id].right_blank? params['DBInstanceClass'] = params.delete(:instance_class) unless params[:instance_class].right_blank? params['Duration'] = params.delete(:duration) unless params[:duration].right_blank? params['ProductDescription'] = params.delete(:product_description) unless params[:product_description].right_blank? params['MultiAZ'] = params.delete(:multi_az).to_s unless params[:multi_az].nil? result = [] incrementally_list_items('DescribeReservedDBInstancesOfferings', DescribeReservedDBInstancesOfferingsParser, params) do |response| result += response[:reserved_db_instances_offerings] block ? block.call(response) : true end result end # Returns information about reserved DB Instances for this account, or about # a specified reserved DB Instance. # Options: :aws_id, :offering_aws_id, :instance_class, :duration, :product_description, :multi_az # # rds.describe_reserved_db_instances # rds.describe_reserved_db_instances(:aws_id => "myreservedinstance") # rds.describe_reserved_db_instances(:offering_aws_id => "248e7b75-49a7-4cd7-9a9b-354f4906a9b1") # rds.describe_reserved_db_instances(:instance_class => "db.m1.small") # rds.describe_reserved_db_instances(:duration => 31536000) # rds.describe_reserved_db_instances(:product_description => 'mysql') # rds.describe_reserved_db_instances_offerings(:multi_az => true) # def describe_reserved_db_instances(params={}, &block) params = params.dup params['ReservedDBInstancesId'] = params.delete(:aws_id) unless params[:aws_id].right_blank? params['ReservedDBInstancesOfferingId'] = params.delete(:offering_aws_id) unless params[:offering_aws_id].right_blank? params['DBInstanceClass'] = params.delete(:instance_class) unless params[:instance_class].right_blank? params['Duration'] = params.delete(:duration) unless params[:duration].right_blank? params['ProductDescription'] = params.delete(:product_description) unless params[:product_description].right_blank? params['MultiAZ'] = params.delete(:multi_az).to_s unless params[:multi_az].nil? result = [] incrementally_list_items('DescribeReservedDBInstances', DescribeReservedDBInstancesParser, params) do |response| result += response[:reserved_db_instances] block ? block.call(response) : true end result end # Purchases a reserved DB Instance offering. # Options: :aws_id, :count def purchase_reserved_db_instances_offering(offering_aws_id, params={}) request_hash = { 'ReservedDBInstancesOfferingId' => offering_aws_id } request_hash['ReservedDBInstanceId'] = params[:aws_id] unless params[:aws_id].right_blank? request_hash['DBInstanceCount'] = params[:count] unless params[:count].right_blank? link = generate_request('PurchaseReservedDBInstancesOffering', request_hash) request_info(link, DescribeReservedDBInstancesParser.new(:logger => @logger))[:reserved_db_instances].first end #--------------------------------------------- # Subnet Groups #--------------------------------------------- # Lists available DB Subnet Groups. # Options: :name, :max_records, :marker # # rds.describe_db_subnet_groups #=> # [{:subnets=> # [{:availability_zone=> # {:name=>"us-east-1d", :provisioned_iops_capable=>false}, # :status=>"Active", # :subnet_id=>"subnet-5259d03a"}, # {:availability_zone=> # {:name=>"us-east-1a", :provisioned_iops_capable=>false}, # :status=>"Active", # :subnet_id=>"subnet-eb518f83"}], # :vpc_id=>"vpc-10518f78", # :status=>"Complete", # :description=>"delete me please", # :name=>"kd-subnet-group"}, # {:subnets=> # [{:availability_zone=> # {:name=>"us-east-1a", :provisioned_iops_capable=>false}, # :status=>"Active", # :subnet_id=>"subnet-eb518f83"}, # {:availability_zone=> # {:name=>"us-east-1d", :provisioned_iops_capable=>false}, # :status=>"Active", # :subnet_id=>"subnet-5259d03a"}], # :vpc_id=>"vpc-10518f78", # :status=>"Complete", # :description=>"delete me please", # :name=>"kd-subnet-group-1"}] # # P.S. http://docs.amazonwebservices.com/AmazonRDS/latest/APIReference/API_DescribeDBSubnetGroups.html # def describe_db_subnet_groups(params={}, &block) params = params.dup params['DBSubnetGroupName'] = params.delete(:name) unless params[:name].right_blank? result = [] incrementally_list_items('DescribeDBSubnetGroups', DescribeDBSubnetGroupsParser, params) do |response| result += response[:subnet_groups] block ? block.call(response) : true end result end def get_db_subnet_group(group_name, &block) describe_db_subnet_groups(:name => group_name, &block) end # Create a new DB Subnet Group. # # rds.create_db_subnet_group('kd-subnet-group-1', ['subnet-5259d03a', 'subnet-eb518f83'], 'delete me please') #=> # {:subnets=> # [{:availability_zone=> # {:name=>"us-east-1a", :provisioned_iops_capable=>false}, # :status=>"Active", # :subnet_id=>"subnet-eb518f83"}, # {:availability_zone=> # {:name=>"us-east-1d", :provisioned_iops_capable=>false}, # :status=>"Active", # :subnet_id=>"subnet-5259d03a"}], # :vpc_id=>"vpc-10518f78", # :status=>"Complete", # :description=>"delete me please", # :name=>"kd-subnet-group-1"} # # P.S. http://docs.amazonwebservices.com/AmazonRDS/latest/APIReference/API_CreateDBSubnetGroup.html # def create_db_subnet_group(subnet_group_name, subnets, subnet_group_description = '-') request_hash = { 'DBSubnetGroupName' => subnet_group_name, 'DBSubnetGroupDescription' => subnet_group_description } request_hash.merge!(amazonize_list('SubnetIds.member', subnets)) link = generate_request('CreateDBSubnetGroup', request_hash) request_info(link, DescribeDBSubnetGroupsParser.new(:logger => @logger))[:subnet_groups].first end # Modify an existing DB Subnet Group. # # rds.modify_db_subnet_group('kd-subnet-group', ['subnet-5259d03a', 'subnet-eb518f83'], 'hahaha!') #=> # {:subnets=> # [{:availability_zone=> # {:name=>"us-east-1d", :provisioned_iops_capable=>false}, # :status=>"Active", # :subnet_id=>"subnet-5259d03a"}, # {:availability_zone=> # {:name=>"us-east-1a", :provisioned_iops_capable=>false}, # :status=>"Active", # :subnet_id=>"subnet-eb518f83"}], # :vpc_id=>"vpc-10518f78", # :status=>"Complete", # :description=>"hahaha!", # :name=>"kd-subnet-group"} # # P.S. http://docs.amazonwebservices.com/AmazonRDS/latest/APIReference/API_ModifyDBSubnetGroup.html # def modify_db_subnet_group(subnet_group_name, subnets, subnet_group_description = '') request_hash = { 'DBSubnetGroupName' => subnet_group_name} request_hash['DBSubnetGroupDescription'] = subnet_group_description unless subnet_group_description.right_blank? request_hash.merge!(amazonize_list('SubnetIds.member', subnets)) link = generate_request('ModifyDBSubnetGroup', request_hash) request_info(link, DescribeDBSubnetGroupsParser.new(:logger => @logger))[:subnet_groups].first end # Delete a DB Subnet Group. # # rds.delete_db_subnet_group("kd-subnet-group-1") #=> true # # P.S. http://docs.amazonwebservices.com/AmazonRDS/latest/APIReference/API_DeleteDBSubnetGroup.html # def delete_db_subnet_group(name) link = generate_request('DeleteDBSubnetGroup', 'DBSubnetGroupName' => name) request_info(link, RightHttp2xxParser.new(:logger => @logger)) end # -------------------------------------------- # Parsers # -------------------------------------------- # -------------------------------------------- # DB Instances # -------------------------------------------- class DescribeDbInstancesParser < RightAWSParser # :nodoc: def reset @result = { :db_instances => [] } end def tagstart(name, attributes) case name when 'DBInstance' then @item = { :db_security_groups => [], :pending_modified_values => {}, :read_replica_db_instance_identifiers => [], :option_group_membership => {} } when 'DBSecurityGroup' then @db_security_group = {} when 'DBSubnetGroup' then @item[:db_subnet_group] = {} when 'Subnet' then @subnet = { :availability_zone => {} } when 'DBParameterGroup', 'DBParameterGroupStatus' then @db_parameter_group = {} end end def tagend(name) case name when 'Marker' then @result[:marker] = @text when 'MaxRecords' then @result[:max_records] = @text.to_i when 'DBInstanceIdentifier' then @item[:aws_id] = @text when 'InstanceCreateTime' then @item[:create_time] = @text when 'Engine' then @item[:engine] = @text when 'DBInstanceStatus' then @item[:status] = @text when 'Address' then @item[:endpoint_address] = @text when 'Port' then @item[:endpoint_port] = @text.to_i when 'MasterUsername' then @item[:master_username] = @text when 'AvailabilityZone' then @item[:availability_zone] = @text when 'LatestRestorableTime' then @item[:latest_restorable_time] = @text when 'LicenseModel' then @item[:license_model] = @text when 'DBName' then @item[:db_name] = @text when 'Iops' then @item[:iops] = @text when 'CharacterSetName' then @item[:character_set_name] = @text when 'ReadReplicaSourceDBInstanceIdentifier' then @item[:read_replica_source_db_instance_identifier] = @text when 'ReadReplicaDBInstanceIdentifier' then @item[:read_replica_db_instance_identifiers] << @text when 'DBParameterGroupName' then @db_parameter_group[:name] = @text when 'ParameterApplyStatus' then @db_parameter_group[:status] = @text when 'DBSecurityGroup' then @item[:db_security_groups] << @db_security_group when 'DBParameterGroup', 'DBParameterGroupStatus' then @item[:db_parameter_group] = @db_parameter_group when 'DBInstance' then @result[:db_instances] << @item else case full_tag_name when %r{DBInstance/DBInstanceClass$} then @item[:instance_class] = @text when %r{DBInstance/AllocatedStorage$} then @item[:allocated_storage] = @text.to_i when %r{DBInstance/MultiAZ$} then @item[:multi_az] = (@text == 'true') when %r{DBInstance/BackupRetentionPeriod$} then @item[:backup_retention_period] = @text.to_i when %r{DBInstance/PreferredMaintenanceWindow$} then @item[:preferred_maintenance_window] = @text when %r{DBInstance/PreferredBackupWindow$} then @item[:preferred_backup_window] = @text when %r{DBInstance/EngineVersion$} then @item[:engine_version] = @text when %r{DBInstance/AutoMinorVersionUpgrade$} then @item[:auto_minor_version_upgrade] = (@text == 'true') when %r{DBInstance/AllowMajorVersionUpgrade$} then @item[:allow_major_version_upgrade] = (@text == 'true') when %r{PendingModifiedValues/DBInstanceClass$} then @item[:pending_modified_values][:instance_class] = @text when %r{PendingModifiedValues/AllocatedStorage$} then @item[:pending_modified_values][:allocated_storage] = @text.to_i when %r{PendingModifiedValues/MasterUserPassword$} then @item[:pending_modified_values][:master_user_password] = @text when %r{PendingModifiedValues/MultiAZ$} then @item[:pending_modified_values][:multi_az] = (@text == 'true') when %r{PendingModifiedValues/BackupRetentionPeriod$} then @item[:pending_modified_values][:backup_retention_period] = @text.to_i when %r{PendingModifiedValues/PreferredMaintenanceWindow$} then @item[:pending_modified_values][:preferred_maintenance_window] = @text when %r{PendingModifiedValues/PreferredBackupWindow$} then @item[:pending_modified_values][:preferred_backup_window] = @text when %r{PendingModifiedValues/EngineVersion$} then @item[:pending_modified_values][:engine_version] = @text when %r{PendingModifiedValues/AutoMinorVersionUpgrade$} then @item[:pending_modified_values][:auto_minor_version_upgrade] = (@text == 'true') when %r{PendingModifiedValues/AllowMajorVersionUpgrade$} then @item[:pending_modified_values][:allow_major_version_upgrade] = (@text == 'true') when %r{OptionGroupMembership/Status$} then @item[:option_group_membership][:status] = @text when %r{OptionGroupMembership/OptionGroupName$} then @item[:option_group_membership][:name] = @text when %r{DBSecurityGroup/Status$} then @db_security_group[:status] = @text when %r{DBSecurityGroup/DBSecurityGroupName$} then @db_security_group[:name] = @text when %r{DBSubnetGroup/DBSubnetGroupDescription$} then @item[:db_subnet_group][:description] = @text when %r{DBSubnetGroup/DBSubnetGroupName$} then @item[:db_subnet_group][:name] = @text when %r{DBSubnetGroup/SubnetGroupStatus$} then @item[:db_subnet_group][:status] = @text when %r{Subnet/SubnetIdentifier$} then @subnet[:subnet_id] = @text when %r{Subnet/SubnetStatus$} then @subnet[:status] = @text when %r{Subnet/AvailabilityZone/Name$} then @subnet[:availability_zone][:name] = @text when %r{Subnet/AvailabilityZone/ProvisionedIopsCapable$} then @subnet[:availability_zone][:provisioned_iops_capable] = @text == 'true' when %r{DBSubnetGroup/Subnet$} then (@item[:db_subnet_group][:subnets] ||= []) << @subnet when %r{DBSubnetGroup/VpcId$} then @item[:db_subnet_group][:vpc_id] = @text end end end end class DescribeOrderableDBInstanceOptionsParser < RightAWSParser # :nodoc: def reset @result = { :items => [] } end def tagstart(name, attributes) case name when 'OrderableDBInstanceOption' then @item = { :availability_zones => [] } end end def tagend(name) case name when 'Marker' then @result[:marker] = @text when 'MaxRecords' then @result[:max_records] = @text.to_i when 'DBInstanceClass' then @item[:instance_class] = @text when 'Engine' then @item[:engine] = @text when 'EngineVersion' then @item[:engine_version] = @text when 'LicenseModel' then @item[:license_model] = @text when 'MultiAZCapable' then @item[:multi_az_capable] = @text when 'ReadReplicaCapable' then @item[:read_replica_capable] = @text == 'true' when 'Vpc' then @item[:vpc] = @text == 'true' when 'Name' then @item[:availability_zones] << @text when 'OrderableDBInstanceOption' then @result[:items] << @item end end end # -------------------------------------------- # DB Security Groups # -------------------------------------------- class DescribeDbSecurityGroupsParser < RightAWSParser # :nodoc: def reset @result = { :db_security_groups => [] } end def tagstart(name, attributes) case name when 'DBSecurityGroup' then @item = { :ec2_security_groups => [], :ip_ranges => [] } when 'IPRange' then @ip_range = {} when 'EC2SecurityGroup' then @ec2_security_group = {} end end def tagend(name) case name when 'Marker' then @result[:marker] = @text when 'MaxRecords' then @result[:max_records] = @text.to_i when 'DBSecurityGroupDescription' then @item[:description ] = @text when 'OwnerId' then @item[:owner_id] = @text when 'DBSecurityGroupName' then @item[:name] = @text when 'EC2SecurityGroupId' then @ec2_security_group[:group_id] = @text when 'EC2SecurityGroupName' then @ec2_security_group[:name] = @text when 'EC2SecurityGroupOwnerId' then @ec2_security_group[:owner_id] = @text when 'CIDRIP' then @ip_range[:cidrip] = @text when 'IPRange' then @item[:ip_ranges] << @ip_range when 'EC2SecurityGroup' then @item[:ec2_security_groups] << @ec2_security_group when 'VpcId' then @item[:vpc_id] = @text when 'DBSecurityGroup' # Sort the ip_ranges and ec2_security_groups @item[:ip_ranges].sort!{ |i1,i2| "#{i1[:cidrip]}" <=> "#{i2[:cidrip]}" } @item[:ec2_security_groups].sort!{ |i1,i2| "#{i1[:owner_id]}#{i1[:name]}" <=> "#{i2[:owner_id]}#{i2[:name]}" } @result[:db_security_groups] << @item else case full_tag_name when %r{IPRange/Status$} then @ip_range[:status] = @text when %r{EC2SecurityGroup/Status$} then @ec2_security_group[:status] = @text end end end end # -------------------------------------------- # DB Security Groups # -------------------------------------------- class DescribeDbParameterGroupsParser < RightAWSParser # :nodoc: def reset @result = { :db_parameter_groups => [] } end def tagstart(name, attributes) case name when 'DBParameterGroup', 'ModifyDBParameterGroupResult' then @item = { } end end def tagend(name) case name when 'Marker' then @result[:marker] = @text when 'MaxRecords' then @result[:max_records] = @text.to_i when 'DBParameterGroupName' then @item[:name] = @text when 'Description' then @item[:description] = @text when 'DBParameterGroupFamily' then @item[:db_parameter_group_family] = @text when 'DBParameterGroup', 'ModifyDBParameterGroupResult' then @result[:db_parameter_groups] << @item end end end class DescribeDbParametersParser < RightAWSParser # :nodoc: def reset @result = { :parameters => [] } end def tagstart(name, attributes) case name when 'Parameter' then @item = {} end end def tagend(name) case name when 'Marker' then @result[:marker] = @text when 'MaxRecords' then @result[:max_records] = @text.to_i when 'DBParameterGroupName' then @result[:group_name] = @text # DescribeDbParametersResponse when 'DBParameterGroupFamily' then @result[:db_parameter_group_family] = @text # DescribeDBEngineDefaultParametersResponse when 'DataType' then @item[:data_type] = @text when 'Source' then @item[:source] = @text when 'Description' then @item[:description] = @text when 'IsModifiable' then @item[:is_modifiable] = (@text == 'true') when 'ApplyType' then @item[:apply_type] = @text when 'ApplyMethod' then @item[:apply_method] = @text when 'MinimumEngineVersion' then @item[:minimum_engine_version] = @text when 'AllowedValues' then @item[:allowed_values] = @text when 'ParameterName' then @item[:name] = @text when 'ParameterValue' then @item[:value] = @text when 'Parameter' then @result[:parameters] << @item end end end # -------------------------------------------- # DB Snapshots # -------------------------------------------- class DescribeDbSnapshotsParser < RightAWSParser # :nodoc: def reset @result = { :db_snapshots => [] } end def tagstart(name, attributes) case name when 'DBSnapshot' then @item = {} end end def tagend(name) case name when 'Marker' then @result[:marker] = @text when 'MaxRecords' then @result[:max_records] = @text.to_i # ? when 'Engine' then @item[:engine] = @text when 'EngineVersion' then @item[:engine_version] = @text when 'InstanceCreateTime' then @item[:instance_create_time] = @text when 'Port' then @item[:endpoint_port] = @text.to_i when 'Status' then @item[:status] = @text when 'AvailabilityZone' then @item[:availability_zone] = @text when 'MasterUsername' then @item[:master_username] = @text when 'AllocatedStorage' then @item[:allocated_storage] = @text.to_i when 'SnapshotCreateTime' then @item[:create_time] = @text when 'DBInstanceIdentifier' then @item[:instance_aws_id] = @text when 'DBSnapshotIdentifier' then @item[:aws_id] = @text when 'LicenseModel' then @item[:license_model] = @text when 'Iops' then @item[:iops] = @text when 'SnapshotType' then @item[:snapshot_type] = @text when 'VpcId' then @item[:vpc_id] = @text when 'DBSnapshot' then @result[:db_snapshots] << @item end end end # -------------------------------------------- # DB Events # -------------------------------------------- class DescribeEventsParser < RightAWSParser # :nodoc: def reset @result = { :events => [] } end def tagstart(name, attributes) case name when 'Event' then @item = {} end end def tagend(name) case name when 'Marker' then @result[:marker] = @text when 'MaxRecords' then @result[:max_records] = @text.to_i # ? when 'Date' then @item[:date] = @text when 'SourceIdentifier' then @item[:aws_id] = @text when 'SourceType' then @item[:source_type] = @text when 'Message' then @item[:message] = @text when 'Event' then @result[:events] << @item end end end # -------------------------------------------- # DB Engine Versions # -------------------------------------------- class DescribeDBEngineVersionsParser < RightAWSParser # :nodoc: def reset @result = { :db_engine_versions => [] } end def tagstart(name, attributes) case name when 'DBEngineVersion' then @item = {} else case full_tag_name when %r{DefaultCharacterSet$} then @item[:default_character_set] = {} when %r{SupportedCharacterSets/CharacterSet$} then @set = {} end end end def tagend(name) case name when 'Marker' then @result[:marker] = @text when 'MaxRecords' then @result[:max_records] = @text.to_i when 'DBParameterGroupFamily' then @item[:db_parameter_group_family] = @text when 'Engine' then @item[:engine] = @text when 'EngineVersion' then @item[:engine_version] = @text when 'DBEngineDescription' then @item[:db_engine_description] = @text when 'DBEngineVersionDescription' then @item[:db_engine_version_description] = @text when 'DBEngineVersion' then @result[:db_engine_versions] << @item else case full_tag_name when %r{DefaultCharacterSet/CharacterSetDescription$} then @item[:default_character_set][:description] = @text when %r{DefaultCharacterSet/CharacterSetName$} then @item[:default_character_set][:name] = @text when %r{SupportedCharacterSets/CharacterSet/CharacterSetDescription$} then @set[:description] = @text when %r{SupportedCharacterSets/CharacterSet/CharacterSetName$} then @set[:name] = @text when %r{SupportedCharacterSets/CharacterSet$} then (@item[:supported_character_sets] ||= []) << @set end end end end # -------------------------------------------- # DB Reserved Instances # -------------------------------------------- class DescribeReservedDBInstancesOfferingsParser < RightAWSParser # :nodoc: def reset @result = { :reserved_db_instances_offerings => [] } end def tagstart(name, attributes) case name when 'ReservedDBInstancesOffering' then @item = { :recurring_charges => [] } when 'RecurringCharge' then @recurring_charge = {} end end def tagend(name) case name when 'Marker' then @result[:marker] = @text when 'MaxRecords' then @result[:max_records] = @text.to_i when 'CurrencyCode' then @item[:currency_code] = @text when 'DBInstanceClass' then @item[:instance_class] = @text when 'Duration' then @item[:duration] = @text.to_i when 'FixedPrice' then @item[:fixed_price] = @text.to_f when 'UsagePrice' then @item[:usage_price] = @text.to_f when 'MultiAZ' then @item[:multi_az] = (@text == 'true') when 'ProductDescription' then @item[:product_description] = @text when 'OfferingType' then @item[:offering_type] = @text when 'ReservedDBInstancesOfferingId' then @item[:aws_id] = @text when 'RecurringCharge' then @item[:recurring_charges] << @recurring_charge when 'ReservedDBInstancesOffering' then @result[:reserved_db_instances_offerings] << @item else case full_tag_name when %r{RecurringCharge/RecurringChargeAmount$} then @recurring_charge[:amount] = @text when %r{RecurringCharge/RecurringChargeFrequency$} then @recurring_charge[:frequency] = @text end end end end class DescribeReservedDBInstancesParser < RightAWSParser # :nodoc: def reset @result = { :reserved_db_instances => [] } end def tagstart(name, attributes) case name when 'ReservedDBInstance' then @item = { :recurring_charges => [] } when 'RecurringCharge' then @recurring_charge = {} end end def tagend(name) case name when 'Marker' then @result[:marker] = @text when 'MaxRecords' then @result[:max_records] = @text.to_i when 'DBInstanceClass' then @item[:instance_class] = @text when 'CurrencyCode' then @item[:currency_code] = @text when 'Duration' then @item[:duration] = @text.to_i when 'FixedPrice' then @item[:fixed_price] = @text.to_f when 'UsagePrice' then @item[:usage_price] = @text.to_f when 'MultiAZ' then @item[:multi_az] = (@text == 'true') when 'ProductDescription' then @item[:product_description] = @text when 'ReservedDBInstancesOfferingId' then @item[:offering_aws_id] = @text when 'ReservedDBInstanceId' then @item[:aws_id] = @text when 'State' then @item[:state] = @text when 'DBInstanceCount' then @item[:instance_count] = @text.to_i when 'StartTime' then @item[:start_time] = @text when 'OfferingType' then @item[:offering_type] = @text when 'RecurringCharge' then @item[:recurring_charges] << @recurring_charge when 'ReservedDBInstance' then @result[:reserved_db_instances] << @item else case full_tag_name when %r{RecurringCharge/RecurringChargeAmount$} then @recurring_charge[:amount] = @text when %r{RecurringCharge/RecurringChargeFrequency$} then @recurring_charge[:frequency] = @text end end end end # -------------------------------------------- # DB Subnet Groups # -------------------------------------------- class DescribeDBSubnetGroupsParser < RightAWSParser # :nodoc: def reset @result = { :subnet_groups => [] } end def tagstart(name, attributes) case name when 'DBSubnetGroup' then @item = { :subnets => [] } when 'Subnet' then @subnet = { :availability_zone => {}} end end def tagend(name) case name when 'Marker' then @result[:marker] = @text when 'MaxRecords' then @result[:max_records] = @text.to_i when 'DBSubnetGroupName' then @item[:name] = @text when 'DBSubnetGroupDescription' then @item[:description] = @text when 'SubnetGroupStatus' then @item[:status] = @text when 'Subnet' then @item[:subnets] << @subnet when 'VpcId' then @item[:vpc_id] = @text when 'DBSubnetGroup' then @result[:subnet_groups] << @item else case full_tag_name when %r{Subnet/SubnetIdentifier$} then @subnet[:subnet_id] = @text when %r{Subnet/SubnetStatus$} then @subnet[:status] = @text when %r{AvailabilityZone/Name$} then @subnet[:availability_zone][:name] = @text when %r{AvailabilityZone/ProvisionedIopsCapable$} then @subnet[:availability_zone][:provisioned_iops_capable] = @text == 'true' end end end end end end ================================================ FILE: lib/right_aws.rb ================================================ # # Copyright (c) 2007-2011 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # require 'benchmark' require 'net/https' require 'uri' require 'time' require "cgi" require "base64" require "rexml/document" require "openssl" require "digest/sha1" require 'rubygems' require 'right_http_connection' $:.unshift(File.dirname(__FILE__)) require 'awsbase/version' require 'awsbase/support' require 'awsbase/benchmark_fix' require 'awsbase/right_awsbase' require 'ec2/right_ec2' require 'ec2/right_ec2_images' require 'ec2/right_ec2_instances' require 'ec2/right_ec2_security_groups' require 'ec2/right_ec2_spot_instances' require 'ec2/right_ec2_ebs' require 'ec2/right_ec2_reserved_instances' require 'ec2/right_ec2_vpc' require 'ec2/right_ec2_vpc2' require 'ec2/right_ec2_monitoring' require 'ec2/right_ec2_placement_groups' require 'ec2/right_ec2_windows_mobility' require 'ec2/right_ec2_tags' require 'elb/right_elb_interface' require 'emr/right_emr_interface' require 'acw/right_acw_interface' require 'as/right_as_interface' require 's3/right_s3_interface' require 's3/right_s3' require 'sqs/right_sqs_interface' require 'sqs/right_sqs' require 'sqs/right_sqs_gen2_interface' require 'sqs/right_sqs_gen2' require 'sdb/right_sdb_interface' require 'acf/right_acf_interface' require 'acf/right_acf_streaming_interface' require 'acf/right_acf_origin_access_identities' require 'acf/right_acf_invalidations' require 'rds/right_rds_interface' require 'iam/right_iam_interface' require 'iam/right_iam_groups' require 'iam/right_iam_users' require 'iam/right_iam_access_keys' require 'iam/right_iam_mfa_devices' require 'route_53/right_route_53_interface' require 'sns/right_sns_interface' #- # We also want everything available in the Rightscale namespace for backward # compatibility reasons. module Rightscale #:nodoc: include RightAws extend RightAws end ================================================ FILE: lib/route_53/right_route_53_interface.rb ================================================ # Copyright (c) 2007-2011 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws # = RightAws::Route53Interface -- Amazon Route 53 web service interface. # # The RightAws::Route53Interface class provides a complete interface to Amazon Route 53: a web # service that enables you to manage your DNS service. # # For explanations of the semantics of each call, please refer to Amazon's documentation at # http://aws.amazon.com/documentation/route53/ # # Examples: # # # Create Route53 handler # r53 = RightAws::Route53Interface.new(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) # # #------------------------ # # Create Hosted Zone # #------------------------ # # hosted_zone_config = { # :name => 'my-awesome-site.com.', # :config => { # :comment => 'My test site!' # } # } # r53.create_hosted_zone(hosted_zone_config) #=> # {:name_servers=> # ["ns-1115.awsdns-11.org", # "ns-696.awsdns-23.net", # "ns-1963.awsdns-53.co.uk", # "ns-362.awsdns-45.com"], # :aws_id=>"/hostedzone/Z1K6NCF0EB26FB", # :caller_reference=>"1295424990-710392-gqMuw-KcY8F-LFlrB-SQhp9", # :config=>{:comment=>"My test site!"}, # :change_info=> # {:status=>"PENDING", # :aws_id=>"/change/C23QGMT8XTCAJY", # :submitted_at=>"2011-01-19T08:16:31.046Z"}, # :name=>"my-awesome-site.com."} # # # List Hosted Zones # r53.list_hosted_zones #=> [] # [{:aws_id=>"/hostedzone/Z1K6NCF0EB26FB", # :caller_reference=>"1295424990-710392-gqMuw-KcY8F-LFlrB-SQhp9", # :config=>{:comment=>"My test site!"}, # :name=>"my-awesome-site.com."}] # # #-------------------------------- # # Manage DNS Records and Changes # #-------------------------------- # # # Create DNS Records # resource_record_sets = [ { :name => 'www1.my-awesome-site.com.', # :type => 'NS', # :ttl => 600, # :resource_records => 'www.mysite.com' }, # { :name => 'www2.my-awesome-site.com.', # :type => 'A', # :ttl => 600, # :resource_records => ['10.0.0.1'] } ] # r53.create_resource_record_sets("/hostedzone/Z1K6NCF0EB26FB", resource_record_sets, 'my first set of records') #=> # { :status=>"PENDING", # :aws_id=>"/change/C2C6IGNRTKA0AY", # :submitted_at=>"2011-01-19T08:29:26.160Z" } # # # Delete DNS records # r53.delete_resource_record_sets("/hostedzone/Z1K6NCF0EB26FB", resource_record_sets, 'I dont need them any more') #=> # { :status=>"PENDING", # :aws_id=>"/change/C1CYJ10EZBFLO7", # :submitted_at=>"2011-01-19T08:26:41.220Z" } # # # Create or delete DNS records (:action key must be provided): # resource_record_sets = [ { :action => :create, # :name => 'www1.my-awesome-site.com.', # :type => 'NS', # :ttl => 600, # :resource_records => 'www.mysite.com' }, # { :action => :delete, # :name => 'www2.my-awesome-site.com.', # :type => 'A', # :ttl => 600, # :resource_records => ['10.0.0.1'] } ] # r53.change_resource_record_sets("/hostedzone/Z1K6NCF0EB26FB", resource_record_sets, 'do change records') # { :status=>"PENDING", # :aws_id=>"/change/C2PWXVECN794LK", # :submitted_at=>"2011-01-19T08:31:33.301Z" } # # # List DNS Records # r53.list_resource_record_sets("/hostedzone/Z1K6NCF0EB26FB") #=> # [{:type=>"NS", # :ttl=>172800, # :resource_records=> # ["ns-1115.awsdns-11.org.", # "ns-696.awsdns-23.net.", # "ns-1963.awsdns-53.co.uk.", # "ns-362.awsdns-45.com."], # :name=>"my-awesome-site.com."}, # {:type=>"SOA", # :ttl=>900, # :resource_records=> # ["ns-1115.awsdns-11.org. awsdns-hostmaster.amazon.com. 1 7200 900 1209600 86400"], # :name=>"my-awesome-site.com."}, # {:type=>"NS", # :ttl=>600, # :resource_records=>["www.mysite.com"], # :name=>"www1.my-awesome-site.com."}] # # # Get Change info # r53.get_change("/change/C2C6IGNRTKA0AY") # {:status=>"INSYNC", # :aws_id=>"/change/C2C6IGNRTKA0AY", # :submitted_at=>"2011-01-19T08:29:26.160Z"} # # #------------------------ # # Delete Hosted Zone # #------------------------ # # # Get a list of DNS records I have created (the first 2 records were added by Amazon and cannot be deleted) # resource_record_sets = r53.list_resource_record_sets("/hostedzone/Z1K6NCF0EB26FB") # resource_record_sets.shift # resource_record_sets.shift # # # Delete them all # delete_resource_record_sets("/hostedzone/Z1K6NCF0EB26FB", resource_record_sets, 'kill all records I have created') #=> # { :status=>"PENDING", # :aws_id=>"/change/C6NCO8Z50MHXV", # :submitted_at=>"2011-01-19T08:46:37.307Z" } # # # Delete Hosted Zone # r53.delete_hosted_zone("/hostedzone/Z1K6NCF0EB26FB") #=> # { :status=>"PENDING", # :aws_id=>"/change/C3OJ31D4V5P2LU", # :submitted_at=>"2011-01-19T08:46:37.530Z" } # class Route53Interface < RightAwsBase include RightAwsBaseInterface API_VERSION = "2011-05-05" DEFAULT_HOST = "route53.amazonaws.com" DEFAULT_PATH = '/' DEFAULT_PROTOCOL = 'https' DEFAULT_PORT = 443 @@bench = AwsBenchmarkingBlock.new def self.bench_xml @@bench.xml end def self.bench_service @@bench.service end # Create a new handle to an Route53 account. All handles share the same per process or per thread # HTTP connection to Amazon Route53. Each handle is for a specific account. The params have the # following options: # * :endpoint_url a fully qualified url to Amazon API endpoint (this overwrites: :server, :port, :service, :protocol). # * :server: Route53 service host, default: DEFAULT_HOST # * :port: Route53 service port, default: DEFAULT_PORT # * :protocol: 'http' or 'https', default: DEFAULT_PROTOCOL # * :logger: for log messages, default: RAILS_DEFAULT_LOGGER else STDOUT # * :signature_version: The signature version : '0','1' or '2'(default) # def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={}) init({ :name => 'ROUTE_53', :default_host => ENV['ROUTE_53_URL'] ? URI.parse(ENV['ROUTE_53_URL']).host : DEFAULT_HOST, :default_port => ENV['ROUTE_53_URL'] ? URI.parse(ENV['ROUTE_53_URL']).port : DEFAULT_PORT, :default_service => ENV['ROUTE_53_URL'] ? URI.parse(ENV['ROUTE_53_URL']).path : DEFAULT_PATH, :default_protocol => ENV['ROUTE_53_URL'] ? URI.parse(ENV['ROUTE_53_URL']).scheme : DEFAULT_PROTOCOL, :default_api_version => ENV['ROUTE_53_API_VERSION'] || API_VERSION }, aws_access_key_id || ENV['AWS_ACCESS_KEY_ID'] , aws_secret_access_key|| ENV['AWS_SECRET_ACCESS_KEY'], params) end #----------------------------------------------------------------- # Requests #----------------------------------------------------------------- # Generates request hash for REST API. def generate_request(method, path, params={}, body=nil, headers={}) # :nodoc: # Params params.delete_if{ |key, val| val.right_blank? } unless params.right_blank? path += "?" + params.to_a.collect{ |key,val| "#{AwsUtils::amz_escape(key)}=#{AwsUtils::amz_escape(val.to_s)}" }.join("&") end # Headers headers = AwsUtils::fix_headers(headers) headers['content-type'] ||= 'text/xml' if body headers['date'] = Time.now.httpdate # Auth signature = AwsUtils::sign(@aws_secret_access_key, headers['date']) headers['X-Amzn-Authorization'] = "AWS3-HTTPS AWSAccessKeyId=#{@aws_access_key_id},Algorithm=HmacSHA1,Signature=#{signature}" # Request path = "#{@params[:service]}#{@params[:api_version]}/#{path}" request = "Net::HTTP::#{method.capitalize}".right_constantize.new(path) request.body = body if body # Set request headers headers.each { |key, value| request[key.to_s] = value } # prepare output hash { :request => request, :server => @params[:server], :port => @params[:port], :protocol => @params[:protocol] } end # Sends request to Amazon and parses the response. # Raises AwsError if any banana happened. def request_info(request, parser, &block) # :nodoc: request_info_impl(:acf_connection, @@bench, request, parser, &block) end def incrementally_list_hosted_zones(path, parser, params={}, &block) # :nodoc: opts = {} opts['MaxItems'] = params[:max_items] if params[:max_items] opts['Marker'] = params[:marker] if params[:marker] last_response = nil loop do link = generate_request('GET', path, opts) last_response = request_info(link, parser.new(:logger => @logger)) opts['Marker'] = last_response[:next_marker] break unless block && block.call(last_response) && !last_response[:next_marker].right_blank? end last_response end def incrementally_list_resource_records(path, parser, params={}, &block) # :nodoc: opts = {} opts[:maxitems] = params.delete(:max_items) if params[:max_items] last_response = nil loop do link = generate_request('GET', path, opts) last_response = request_info(link, parser.new(:logger => @logger)) opts[:maxitems] = last_response[:max_items] opts[:name] = last_response[:next_record_name] opts[:type] = last_response[:next_record_type] break unless block && block.call(last_response) && last_response[:is_truncated] end last_response end def expand_hosted_zone_id(aws_id) # :nodoc: aws_id[%r{^/hostedzone/}] ? aws_id : "/hostedzone/#{aws_id}" end def expand_change_id(aws_id) # :nodoc: aws_id[%r{^/change/}] ? aws_id : "/change/#{aws_id}" end def hosted_zone_config_to_xml(config) # :nodoc: config[:caller_reference] ||= AwsUtils::generate_call_reference hosted_zone_config = '' unless config[:config].right_blank? hosted_zone_config = " \n" + " #{AwsUtils::xml_escape config[:config][:comment]}\n" + " \n" end # XML "\n" + "\n" + " #{config[:name]}\n" + " #{config[:caller_reference]}\n" + hosted_zone_config + "\n" end def resource_record_sets_to_xml(resource_record_changes, comment) # :nodoc: # Comment xml_comment = comment.right_blank? ? '' : " #{AwsUtils::xml_escape(comment)}\n" # Changes xml_changes = '' resource_record_changes.each do |change| xml_resource_records = Array(change[:resource_records]).map{|record| " #{AwsUtils::xml_escape(record)}\n" }.join('') xml_changes += " \n" + " #{AwsUtils::xml_escape(change[:action].to_s.upcase)}\n" + " \n" + " #{AwsUtils::xml_escape(change[:name])}\n" + " #{AwsUtils::xml_escape(change[:type].to_s.upcase)}\n" if change[:alias_target] alias_target = change[:alias_target] xml_changes += " \n" + " #{AwsUtils::xml_escape(alias_target[:hosted_zone_id].to_s)}\n" + " #{AwsUtils::xml_escape(alias_target[:dns_name].to_s)}\n" + " \n" else xml_changes += " #{AwsUtils::xml_escape(change[:ttl].to_s)}\n" + " \n" + xml_resource_records + " \n" end xml_changes += " \n" + " \n" end # XML "\n" + "\n" + " \n" + xml_comment + " \n" + xml_changes + " \n" + " \n" + "\n" end #----------------------------------------------------------------- # Hosted Zones #----------------------------------------------------------------- # List your hosted zones. # # r53.list_hosted_zones #=> # [{:config=>{:comment=>"KD1, description"}, # :aws_id=>"/hostedzone/Z2P714ENJN23PN", # :caller_reference=>"1295424990-710392-gqMuw-KcY8F-LFlrB-SQhp9", # :name=>"patch-island.com."}, # {:config=>{:comment=>"My awesome site!"}, # :aws_id=>"/hostedzone/ZWEC7PPVACGQ4", # :caller_reference=>"1295422234-657482-hfkeo-JFKid-Ldfle-Sdrty", # :name=>"mysite.patch-island.com."}, ...] # # PS: http://docs.amazonwebservices.com/Route53/latest/APIReference/API_ListHostedZones.html # def list_hosted_zones result = [] incrementally_list_hosted_zones('hostedzone', ListHostedZonesParser) do |response| result += response[:items] true end result end # Create new hosted zone # # config = { # :name => 'mysite.patch-island.com.', # :config => { # :comment => 'My awesome site!' # } # } # r53.create_hosted_zone(config) #=> # {:config=>{:comment=>"My awesome site!"}, # :change_info=> # {:status=>"PENDING", # :aws_id=>"/change/C2NOTVGL7IOFFF", # :submitted_at=>"2011-01-18T15:34:18.086Z"}, # :aws_id=>"/hostedzone/ZWEC7PPVACGQ4", # :caller_reference=>"1295365357-227168-NfZ4P-VGHWi-Yq0p7-nuN6q", # :name_servers=> # ["ns-794.awsdns-35.net", # "ns-459.awsdns-57.com", # "ns-1537.awsdns-00.co.uk", # "ns-1165.awsdns-17.org"], # :name=>"mysite.patch-island.com."} # # PS: http://docs.amazonwebservices.com/Route53/latest/APIReference/index.html?API_CreateHostedZone.html # def create_hosted_zone(config) config[:caller_reference] ||= AwsUtils::generate_unique_token link = generate_request('POST', 'hostedzone', {}, hosted_zone_config_to_xml(config)) request_info(link, GetHostedZoneParser.new(:logger => @logger)) end # Get your hosted zone. # # r53.get_hosted_zone("ZWEC7PPVACGQ4") #=> # {:config=>{:comment=>"My awesome site!"}, # :aws_id=>"/hostedzone/ZWEC7PPVACGQ4", # :caller_reference=>"1295422234-657482-hfkeo-JFKid-Ldfle-Sdrty", # :name_servers=> # ["ns-794.awsdns-35.net", # "ns-459.awsdns-57.com", # "ns-1537.awsdns-00.co.uk", # "ns-1165.awsdns-17.org"], # :name=>"mysite.patch-island.com."} # # PS: http://docs.amazonwebservices.com/Route53/latest/APIReference/API_GetHostedZone.html # def get_hosted_zone(hosted_zone_aws_id) link = generate_request('GET', expand_hosted_zone_id(hosted_zone_aws_id)) request_info(link, GetHostedZoneParser.new(:logger => @logger)) end # Delete hosted zone. # # r53.delete_hosted_zone("/hostedzone/Z2P714ENJN23PN") #=> # {:status=>"PENDING", # :submitted_at=>"2011-01-18T15:45:45.060Z", # :aws_id=>"/change/C1PN1SDWZKPTAC"} # # PS: http://docs.amazonwebservices.com/Route53/latest/APIReference/API_DeleteHostedZone.html # def delete_hosted_zone(hosted_zone_aws_id) link = generate_request('DELETE', expand_hosted_zone_id(hosted_zone_aws_id)) request_info(link, GetChangeParser.new(:logger => @logger)) end #----------------------------------------------------------------- # Resource Records Set #----------------------------------------------------------------- # List your resource record sets. # Options: :type, :name, :max_items # # r53.list_resource_record_sets("/hostedzone/ZWEC7PPVACGQ4") #=> # [{:type=>"NS", # :ttl=>172800, # :name=>"mysite.patch-island.com.", # :resource_records=> # ["ns-459.awsdns-57.com.", # "ns-794.awsdns-35.net.", # "ns-1165.awsdns-17.org.", # "ns-1537.awsdns-00.co.uk."]}, # {:type=>"SOA", # :ttl=>900, # :name=>"mysite.patch-island.com.", # :resource_records=> # ["ns-794.awsdns-35.net. awsdns-hostmaster.amazon.com. 1 7200 900 1209600 86400"]}, # {:type=>"NS", # :ttl=>600, # :resource_records=>["xxx.mysite.com"], # :name=>"m1.mysite.patch-island.com."}] # # PS: http://docs.amazonwebservices.com/Route53/latest/APIReference/API_ListResourceRecordSets.html # def list_resource_record_sets(hosted_zone_aws_id, options={}) options = options.dup result = [] incrementally_list_resource_records("#{expand_hosted_zone_id(hosted_zone_aws_id)}/rrset", ListResourceRecordSetsParser, options) do |response| result += response[:items] true end result end # Create or delete DNS records. # # resource_record_sets = [{ :action => :create, # :name => 'm3.mysite.patch-island.com', # :type => 'NS', # :ttl => 600, # :resource_records => 'xxx.mysite.com' }, # { :action => :delete, # :name => 'm2.mysite.patch-island.com', # :type => 'A', # :ttl => 600, # :resource_records => ['10.0.0.1'] }] # r53.change_resource_record_sets("/hostedzone/Z1K6NCF0EB26FB", resource_record_sets, 'KD: Comment#1') #=> # {:status=>"PENDING", # :submitted_at=>"2011-01-18T20:21:56.828Z", # :aws_id=>"/change/C394PNLM1B2P08"} # # PS: resource_record_sets must have an :action key set (== :create or :delete) # PPS: http://docs.amazonwebservices.com/Route53/latest/APIReference/API_ChangeResourceRecordSets.html # def change_resource_record_sets(hosted_zone_aws_id, resource_record_sets, comment = '') link = generate_request('POST', "#{expand_hosted_zone_id(hosted_zone_aws_id)}/rrset", {}, resource_record_sets_to_xml(resource_record_sets, comment)) request_info(link, GetChangeParser.new(:logger => @logger)) end # Create DNS records. # # resource_record_sets = [{ :name => 'm3.mysite.patch-island.com', # :type => 'NS', # :ttl => 600, # :resource_records => 'xxx.mysite.com' }, # { :name => 'm2.mysite.patch-island.com', # :type => 'A', # :ttl => 600, # :resource_records => ['10.0.0.1'] }] # r53.create_resource_record_sets("/hostedzone/Z1K6NCF0EB26FB", resource_record_sets, 'KD: Comment#1') #=> # {:status=>"PENDING", # :submitted_at=>"2011-01-18T20:21:56.828Z", # :aws_id=>"/change/C394PNLM1B2P08"} # # PS: http://docs.amazonwebservices.com/Route53/latest/APIReference/API_ChangeResourceRecordSets.html # def create_resource_record_sets(hosted_zone_aws_id, resource_record_sets, comment = '') resource_record_sets.each{|rrs| rrs[:action] = :create} change_resource_record_sets(hosted_zone_aws_id, resource_record_sets, comment) end # Delete DNS records. # # resource_record_sets = [{ :name => 'm3.mysite.patch-island.com', # :type => 'NS', # :ttl => 600, # :resource_records => 'xxx.mysite.com' }, # { :name => 'm2.mysite.patch-island.com', # :type => 'A', # :ttl => 600, # :resource_records => ['10.0.0.1'] }] # r53.create_resource_record_sets("/hostedzone/Z1K6NCF0EB26FB", resource_record_sets, 'KD: Comment#1') #=> # {:status=>"PENDING", # :submitted_at=>"2011-01-18T20:21:56.828Z", # :aws_id=>"/change/C394PNLM1B2P08"} # # PS: http://docs.amazonwebservices.com/Route53/latest/APIReference/API_ChangeResourceRecordSets.html # def delete_resource_record_sets(hosted_zone_aws_id, resource_record_sets, comment = '') resource_record_sets.each{|rrs| rrs[:action] = :delete} change_resource_record_sets(hosted_zone_aws_id, resource_record_sets, comment) end # Get the current state of a change request. # # r53.get_change("/change/C1PN1SDWZKPTAC") #=> # {:status=>"INSYNC", # :aws_id=>"/change/C1PN1SDWZKPTAC", # :submitted_at=>"2011-01-18T15:45:45.060Z"} # # PS: http://docs.amazonwebservices.com/Route53/latest/APIReference/API_GetChange.html # def get_change(change_aws_id) link = generate_request('GET', expand_change_id(change_aws_id)) request_info(link, GetChangeParser.new(:logger => @logger)) end #----------------------------------------------------------------- # Hosted Zones #----------------------------------------------------------------- class ListHostedZonesParser < RightAWSParser # :nodoc: def reset @result = { :items => [] } end def tagstart(name, attributes) case name when 'HostedZone' then @item = { :config => {} } end end def tagend(name) case name when 'IsTruncated' then @result[:is_truncated] = @text == 'true' when 'NextMarker' then @result[:next_marker] = @text when 'MaxItems' then @result[:max_items] = @text.to_i when 'Id' then @item[:aws_id] = @text when 'Name' then @item[:name] = @text when 'CallerReference' then @item[:caller_reference] = @text when 'HostedZone' then @result[:items] << @item else case full_tag_name when %r{/Config/Comment$} then @item[:config][:comment] = @text end end end end class GetHostedZoneParser < RightAWSParser # :nodoc: def reset @result = {} end def tagend(name) case full_tag_name when %r{/HostedZone/Id} then @result[:aws_id] = @text when %r{/HostedZone/Name} then @result[:name] = @text when %r{/HostedZone/CallerReference} then @result[:caller_reference] = @text when %r{/Config/Comment$} then (@result[:config] ||= {})[:comment] = AwsUtils::xml_unescape(@text) when %r{/ChangeInfo/Id$} then (@result[:change_info] ||= {})[:aws_id] = @text when %r{/ChangeInfo/Status$} then (@result[:change_info] ||= {})[:status] = @text when %r{/ChangeInfo/SubmittedAt$} then (@result[:change_info] ||= {})[:submitted_at] = @text when %r{/DelegationSet/NameServers/NameServer$} then (@result[:name_servers] ||= []) << @text end end end #----------------------------------------------------------------- # Resource Records Set #----------------------------------------------------------------- class ListResourceRecordSetsParser < RightAWSParser # :nodoc: def reset @result = { :items => [] } end def tagstart(name, attributes) case name when 'ResourceRecordSet' then @item = {} end end def tagend(name) case name when 'IsTruncated' then @result[:is_truncated] = @text == 'true' when 'NextRecordName' then @result[:next_record_name] = @text when 'NextRecordType' then @result[:next_record_type] = @text when 'MaxItems' then @result[:max_items] = @text.to_i when 'Type' then @item[:type] = @text when 'Name' then @item[:name] = @text when 'TTL' then @item[:ttl] = @text.to_i when 'ResourceRecordSet' then @result[:items] << @item else case full_tag_name when %r{/ResourceRecord/Value} then (@item[:resource_records] ||= []) << @text when %r{/AliasTarget/DNSName} then (@item[:alias_target] ||= {})[:dns_name] = @text when %r{/AliasTarget/HostedZoneId} then (@item[:alias_target] ||= {})[:hosted_zone_id] = @text end end end end class GetChangeParser < RightAWSParser # :nodoc: def reset @result = { } end def tagend(name) case name when 'Id' then @result[:aws_id] = @text when 'Status' then @result[:status] = @text when 'SubmittedAt' then @result[:submitted_at] = @text end end end end end ================================================ FILE: lib/s3/right_s3.rb ================================================ # # Copyright (c) 2007-2008 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws # = RightAws::S3 -- RightScale's Amazon S3 interface # The RightAws::S3 class provides a complete interface to Amazon's Simple # Storage Service. # For explanations of the semantics # of each call, please refer to Amazon's documentation at # http://developer.amazonwebservices.com/connect/kbcategory.jspa?categoryID=48 # # See examples below for the bucket and buckets methods. # # Error handling: all operations raise an RightAws::AwsError in case # of problems. Note that transient errors are automatically retried. # # It is a good way to use domain naming style getting a name for the buckets. # See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingBucket.html # about the naming convention for the buckets. This case they can be accessed using a virtual domains. # # Let assume you have 3 buckets: 'awesome-bucket', 'awesome_bucket' and 'AWEsomE-bucket'. # The first ones objects can be accessed as: http:// awesome-bucket.s3.amazonaws.com/key/object # # But the rest have to be accessed as: # http:// s3.amazonaws.com/awesome_bucket/key/object and http:// s3.amazonaws.com/AWEsomE-bucket/key/object # # See: http://docs.amazonwebservices.com/AmazonS3/2006-03-01/VirtualHosting.html for better explanation. # class S3 attr_reader :interface # Create a new handle to an S3 account. All handles share the same per process or per thread # HTTP connection to Amazon S3. Each handle is for a specific account. # The +params+ are passed through as-is to RightAws::S3Interface.new # # Params is a hash: # # {:server => 's3.amazonaws.com' # Amazon service host: 's3.amazonaws.com'(default) # :port => 443 # Amazon service port: 80 or 443(default) # :protocol => 'https' # Amazon service protocol: 'http' or 'https'(default) # :logger => Logger Object} # Logger instance: logs to STDOUT if omitted } def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={}) @interface = S3Interface.new(aws_access_key_id, aws_secret_access_key, params) end # Retrieve a list of buckets. # Returns an array of RightAws::S3::Bucket instances. # # Create handle to S3 account # s3 = RightAws::S3.new(aws_access_key_id, aws_secret_access_key) # my_buckets_names = s3.buckets.map{|b| b.name} # puts "Buckets on S3: #{my_bucket_names.join(', ')}" def buckets @interface.list_all_my_buckets.map! do |entry| owner = Owner.new(entry[:owner_id], entry[:owner_display_name]) Bucket.new(self, entry[:name], entry[:creation_date], owner) end end # Retrieve an individual bucket. # If the bucket does not exist and +create+ is set, a new bucket # is created on S3. Launching this method with +create+=+true+ may # affect on the bucket's ACL if the bucket already exists. # Returns a RightAws::S3::Bucket instance or +nil+ if the bucket does not exist # and +create+ is not set. # # s3 = RightAws::S3.new(aws_access_key_id, aws_secret_access_key) # bucket1 = s3.bucket('my_awesome_bucket_1') # bucket1.keys #=> exception here if the bucket does not exists # ... # bucket2 = s3.bucket('my_awesome_bucket_2', true) # bucket2.keys #=> list of keys # # create a bucket at the European location with public read access # bucket3 = s3.bucket('my-awesome-bucket-3', true, 'public-read', :location => :eu) # # see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTAccessPolicy.html # (section: Canned Access Policies) # def bucket(name, create=false, perms=nil, headers={}) result = nil if create headers['x-amz-acl'] = perms if perms @interface.create_bucket(name, headers) end begin buckets.each do |bucket| if bucket.name == name result = bucket break end end rescue RightAws::AwsError => e # With non root creds one can use bucket(s) but can't list them raise e unless e.message['AccessDenied'] result = Bucket::new(self, name) end result end class Bucket attr_reader :s3, :name, :owner, :creation_date # Create a Bucket instance. # If the bucket does not exist and +create+ is set, a new bucket # is created on S3. Launching this method with +create+=+true+ may # affect on the bucket's ACL if the bucket already exists. # Returns Bucket instance or +nil+ if the bucket does not exist # and +create+ is not set. # # s3 = RightAws::S3.new(aws_access_key_id, aws_secret_access_key) # ... # bucket1 = RightAws::S3::Bucket.create(s3, 'my_awesome_bucket_1') # bucket1.keys #=> exception here if the bucket does not exists # ... # bucket2 = RightAws::S3::Bucket.create(s3, 'my_awesome_bucket_2', true) # bucket2.keys #=> list of keys # # create a bucket at the European location with public read access # bucket3 = RightAws::S3::Bucket.create(s3,'my-awesome-bucket-3', true, 'public-read', :location => :eu) # # see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTAccessPolicy.html # (section: Canned Access Policies) # def self.create(s3, name, create=false, perms=nil, headers={}) s3.bucket(name, create, perms, headers) end # Create a bucket instance. In normal use this method should # not be called directly. # Use RightAws::S3::Bucket.create or RightAws::S3.bucket instead. def initialize(s3, name, creation_date=nil, owner=nil) @s3 = s3 @name = name @owner = owner @creation_date = creation_date if @creation_date && !@creation_date.is_a?(Time) @creation_date = Time.parse(@creation_date) end end # Return bucket name as a String. # # bucket = RightAws::S3.bucket('my_awesome_bucket') # puts bucket #=> 'my_awesome_bucket' # def to_s @name.to_s end alias_method :full_name, :to_s # Return a public link to bucket. # # bucket.public_link #=> 'https://s3.amazonaws.com:443/my_awesome_bucket' # def public_link params = @s3.interface.params "#{params[:protocol]}://#{params[:server]}:#{params[:port]}/#{full_name}" end # Returns the bucket location def location @location ||= @s3.interface.bucket_location(@name) end # Retrieves the logging configuration for a bucket. # Returns a hash of {:enabled, :targetbucket, :targetprefix} # # bucket.logging_info() # => {:enabled=>true, :targetbucket=>"mylogbucket", :targetprefix=>"loggylogs/"} def logging_info @s3.interface.get_logging_parse(:bucket => @name) end # Enables S3 server access logging on a bucket. The target bucket must have been properly configured to receive server # access logs. # Params: # :targetbucket - either the target bucket object or the name of the target bucket # :targetprefix - the prefix under which all logs should be stored # # bucket.enable_logging(:targetbucket=>"mylogbucket", :targetprefix=>"loggylogs/") # => true def enable_logging(params) AwsUtils.mandatory_arguments([:targetbucket, :targetprefix], params) AwsUtils.allow_only([:targetbucket, :targetprefix], params) xmldoc = "#{params[:targetbucket]}#{params[:targetprefix]}" @s3.interface.put_logging(:bucket => @name, :xmldoc => xmldoc) end # Disables S3 server access logging on a bucket. Takes no arguments. def disable_logging xmldoc = "" @s3.interface.put_logging(:bucket => @name, :xmldoc => xmldoc) end # Retrieve a group of keys from Amazon. # +options+ is a hash: { 'prefix'=>'', 'marker'=>'', 'max-keys'=>5, 'delimiter'=>'' }). # Retrieves meta-headers information if +head+ it +true+. # Returns an array of Key instances. # # bucket.keys #=> # returns all keys from bucket # bucket.keys('prefix' => 'logs') #=> # returns all keys that starts with 'logs' # def keys(options={}, head=false) keys_and_service(options, head)[0] end # Same as +keys+ method but return an array of [keys, service_data]. # where +service_data+ is a hash with additional output information. # # keys, service = bucket.keys_and_service({'max-keys'=> 2, 'prefix' => 'logs'}) # p keys #=> # 2 keys array # p service #=> {"max-keys"=>"2", "prefix"=>"logs", "name"=>"my_awesome_bucket", "marker"=>"", "is_truncated"=>true, :common_prefixes=>[]} # def keys_and_service(options={}, head=false) opt = {}; options.each{ |key, value| opt[key.to_s] = value } service = {} keys = [] @s3.interface.incrementally_list_bucket(@name, opt) do |_service| service = _service service[:contents].each do |entry| owner = Owner.new(entry[:owner_id], entry[:owner_display_name]) key = Key.new(self, entry[:key], nil, {}, {}, entry[:last_modified], entry[:e_tag], entry[:size], entry[:storage_class], owner) key.head if head keys << key end end service.delete(:contents) [keys, service] end # Retrieve key information from Amazon. # The +key_name+ is a +String+ or Key instance. # Retrieves meta-header information if +head+ is +true+. # Returns new Key instance. # # key = bucket.key('logs/today/1.log', true) #=> # # # is the same as: # key = RightAws::S3::Key.create(bucket, 'logs/today/1.log') # key.head # def key(key_name, head=false, &blck) raise 'Key name can not be empty.' if key_name.right_blank? key_instance = nil # if this key exists - find it .... keys({'prefix'=>key_name}, head).each do |key| blck.call if block_given? if key.name == key_name.to_s key_instance = key break end end # .... else this key is unknown unless key_instance key_instance = Key.create(self, key_name.to_s) end key_instance end # Store object data. # The +key+ is a +String+ or Key instance. # Returns +true+. # # bucket.put('logs/today/1.log', 'Olala!') #=> true # def put(key, data=nil, meta_headers={}, perms=nil, headers={}, &blck) key = Key.create(self, key.to_s, data, meta_headers) unless key.is_a?(Key) key.put(data, perms, headers, &blck) end # Retrieve data object from Amazon. # The +key+ is a +String+ or Key. # Returns String instance. # # data = bucket.get('logs/today/1.log') #=> # puts data #=> 'sasfasfasdf' # def get(key, headers={}) key = Key.create(self, key.to_s) unless key.is_a?(Key) key.get(headers) end # Rename object. Returns RightAws::S3::Key instance. # # new_key = bucket.rename_key('logs/today/1.log','logs/today/2.log') #=> # # puts key.name #=> 'logs/today/2.log' # key.exists? #=> true # def rename_key(old_key_or_name, new_name) old_key_or_name = Key.create(self, old_key_or_name.to_s) unless old_key_or_name.is_a?(Key) old_key_or_name.rename(new_name) old_key_or_name end # Create an object copy. Returns a destination RightAws::S3::Key instance. # # new_key = bucket.copy_key('logs/today/1.log','logs/today/2.log') #=> # # puts key.name #=> 'logs/today/2.log' # key.exists? #=> true # def copy_key(old_key_or_name, new_key_or_name) old_key_or_name = Key.create(self, old_key_or_name.to_s) unless old_key_or_name.is_a?(Key) old_key_or_name.copy(new_key_or_name) end # Move an object to other location. Returns a destination RightAws::S3::Key instance. # # new_key = bucket.copy_key('logs/today/1.log','logs/today/2.log') #=> # # puts key.name #=> 'logs/today/2.log' # key.exists? #=> true # def move_key(old_key_or_name, new_key_or_name) old_key_or_name = Key.create(self, old_key_or_name.to_s) unless old_key_or_name.is_a?(Key) old_key_or_name.move(new_key_or_name) end # Remove all keys from a bucket. # Returns +true+. # # bucket.clear #=> true # def clear @s3.interface.clear_bucket(@name) end # Delete all keys where the 'folder_key' can be interpreted # as a 'folder' name. # Returns an array of string keys that have been deleted. # # bucket.keys.map{|key| key.name}.join(', ') #=> 'test, test/2/34, test/3, test1, test1/logs' # bucket.delete_folder('test') #=> ['test','test/2/34','test/3'] # def delete_folder(folder, separator='/') @s3.interface.delete_folder(@name, folder, separator) end # Delete a bucket. Bucket must be empty. # If +force+ is set, clears and deletes the bucket. # Returns +true+. # # bucket.delete(:force => true) #=> true # def delete(options={}) force = options.is_a?(Hash) && options[:force]==true force ? @s3.interface.force_delete_bucket(@name) : @s3.interface.delete_bucket(@name) end # Return a list of grantees. # def grantees Grantee::grantees(self) end end class Key attr_reader :bucket, :name, :last_modified, :e_tag, :size, :storage_class, :owner attr_accessor :headers, :meta_headers attr_writer :data # Separate Amazon meta headers from other headers def self.split_meta(headers) #:nodoc: hash = headers.dup meta = {} hash.each do |key, value| if key[/^#{S3Interface::AMAZON_METADATA_PREFIX}/] meta[key.gsub(S3Interface::AMAZON_METADATA_PREFIX,'')] = value hash.delete(key) end end [hash, meta] end def self.add_meta_prefix(meta_headers, prefix=S3Interface::AMAZON_METADATA_PREFIX) meta = {} meta_headers.each do |meta_header, value| if meta_header[/#{prefix}/] meta[meta_header] = value else meta["#{S3Interface::AMAZON_METADATA_PREFIX}#{meta_header}"] = value end end meta end # Create a new Key instance, but do not create the actual key. # The +name+ is a +String+. # Returns a new Key instance. # # key = RightAws::S3::Key.create(bucket, 'logs/today/1.log') #=> # # key.exists? #=> true | false # key.put('Woohoo!') #=> true # key.exists? #=> true # def self.create(bucket, name, data=nil, meta_headers={}) new(bucket, name, data, {}, meta_headers) end # Create a new Key instance, but do not create the actual key. # In normal use this method should not be called directly. # Use RightAws::S3::Key.create or bucket.key() instead. # def initialize(bucket, name, data=nil, headers={}, meta_headers={}, last_modified=nil, e_tag=nil, size=nil, storage_class=nil, owner=nil) raise 'Bucket must be a Bucket instance.' unless bucket.is_a?(Bucket) @bucket = bucket @name = name @data = data @e_tag = e_tag @size = size.to_i @storage_class = storage_class @owner = owner @last_modified = last_modified if @last_modified && !@last_modified.is_a?(Time) @last_modified = Time.parse(@last_modified) end @headers, @meta_headers = self.class.split_meta(headers) @meta_headers.merge!(meta_headers) end # Return key name as a String. # # key = RightAws::S3::Key.create(bucket, 'logs/today/1.log') #=> # # puts key #=> 'logs/today/1.log' # def to_s @name.to_s end # Return the full S3 path to this key (bucket/key). # # key.full_name #=> 'my_awesome_bucket/cool_key' # def full_name(separator='/') "#{@bucket.to_s}#{separator}#{@name}" end # Return a public link to a key. # # key.public_link #=> 'https://s3.amazonaws.com:443/my_awesome_bucket/cool_key' # def public_link params = @bucket.s3.interface.params "#{params[:protocol]}://#{params[:server]}:#{params[:port]}/#{full_name('/')}" end # Return Key data. Retrieve this data from Amazon if it is the first time call. # TODO TRB 6/19/07 What does the above mean? Clarify. # def data get if !@data and exists? @data end # Getter for the 'content-type' metadata def content_type @headers['content-type'] if @headers end # Helper to get and URI-decode a header metadata. # Metadata have to be HTTP encoded (rfc2616) as we use the Amazon S3 REST api # see http://docs.amazonwebservices.com/AmazonS3/latest/index.html?UsingMetadata.html def decoded_meta_headers(key = nil) if key # Get one metadata value by its key URI.decode(@meta_headers[key.to_s]) else # Get a hash of all metadata with a decoded value @decoded_meta_headers ||= begin metadata = {} @meta_headers.each do |key, value| metadata[key.to_sym] = URI.decode(value) end metadata end end end # Retrieve object data and attributes from Amazon. # Returns a +String+. # def get(headers={}) response = @bucket.s3.interface.get(@bucket.name, @name, headers) @data = response[:object] @headers, @meta_headers = self.class.split_meta(response[:headers]) refresh(false) @data end # Store object data on S3. # Parameter +data+ is a +String+ or S3Object instance. # Returns +true+. # # key = RightAws::S3::Key.create(bucket, 'logs/today/1.log') # key.data = 'Qwerty' # key.put #=> true # ... # key.put('Olala!') #=> true # def put(data=nil, perms=nil, headers={}, &blck) headers['x-amz-acl'] = perms if perms @data = data || @data meta = self.class.add_meta_prefix(@meta_headers) @bucket.s3.interface.put(@bucket.name, @name, @data, meta.merge(headers), &blck) end # Store object data on S3 using the Multipart Upload API. This is useful if you do not know the file size # upfront (for example reading from pipe or socket) or if you are transmitting data over an unreliable network. # # Parameter +data+ is an object which responds to :read or an object which can be converted to a String prior to upload. # Parameter +part_size+ determines the size of each part sent (must be > 5MB per Amazon's API requirements) # # If data is a stream the caller is responsible for calling close() on the stream after this methods returns # # Returns +true+. # # upload_data = StringIO.new('My sample data') # key = RightAws::S3::Key.create(bucket, 'logs/today/1.log') # key.data = upload_data # key.put_multipart(:part_size => 5*1024*1024) #=> true # def put_multipart(data=nil, perms=nil, headers={}, part_size=nil) headers['x-amz-acl'] = perms if perms @data = data || @data meta = self.class.add_meta_prefix(@meta_headers) @bucket.s3.interface.store_object_multipart({:bucket => @bucket.name, :key => @name, :data => @data, :headers => meta.merge(headers), :part_size => part_size}) end # Rename an object. Returns new object name. # # key = RightAws::S3::Key.create(bucket, 'logs/today/1.log') #=> # # key.rename('logs/today/2.log') #=> 'logs/today/2.log' # puts key.name #=> 'logs/today/2.log' # key.exists? #=> true # def rename(new_name) @bucket.s3.interface.rename(@bucket.name, @name, new_name) @name = new_name end # Create an object copy. Returns a destination RightAws::S3::Key instance. # # # Key instance as destination # key1 = RightAws::S3::Key.create(bucket, 'logs/today/1.log') #=> # # key2 = RightAws::S3::Key.create(bucket, 'logs/today/2.log') #=> # # key1.put('Olala!') #=> true # key1.copy(key2) #=> # # key1.exists? #=> true # key2.exists? #=> true # puts key2.data #=> 'Olala!' # # # String as destination # key = RightAws::S3::Key.create(bucket, 'logs/today/777.log') #=> # # key.put('Olala!') #=> true # new_key = key.copy('logs/today/888.log') #=> # # key.exists? #=> true # new_key.exists? #=> true # def copy(new_key_or_name) new_key_or_name = Key.create(@bucket, new_key_or_name.to_s) unless new_key_or_name.is_a?(Key) @bucket.s3.interface.copy(@bucket.name, @name, new_key_or_name.bucket.name, new_key_or_name.name) new_key_or_name end # Move an object to other location. Returns a destination RightAws::S3::Key instance. # # # Key instance as destination # key1 = RightAws::S3::Key.create(bucket, 'logs/today/1.log') #=> # # key2 = RightAws::S3::Key.create(bucket, 'logs/today/2.log') #=> # # key1.put('Olala!') #=> true # key1.move(key2) #=> # # key1.exists? #=> false # key2.exists? #=> true # puts key2.data #=> 'Olala!' # # # String as destination # key = RightAws::S3::Key.create(bucket, 'logs/today/777.log') #=> # # key.put('Olala!') #=> true # new_key = key.move('logs/today/888.log') #=> # # key.exists? #=> false # new_key.exists? #=> true # def move(new_key_or_name) new_key_or_name = Key.create(@bucket, new_key_or_name.to_s) unless new_key_or_name.is_a?(Key) @bucket.s3.interface.move(@bucket.name, @name, new_key_or_name.bucket.name, new_key_or_name.name) new_key_or_name end # Retrieve key info from bucket and update attributes. # Refresh meta-headers (by calling +head+ method) if +head+ is set. # Returns +true+ if the key exists in bucket and +false+ otherwise. # # key = RightAws::S3::Key.create(bucket, 'logs/today/1.log') # key.e_tag #=> nil # key.meta_headers #=> {} # key.refresh #=> true # key.e_tag #=> '12345678901234567890bf11094484b6' # key.meta_headers #=> {"family"=>"qwerty", "name"=>"asdfg"} # def refresh(head=true) new_key = @bucket.key(self) @last_modified = new_key.last_modified @e_tag = new_key.e_tag @size = new_key.size @storage_class = new_key.storage_class @owner = new_key.owner if @last_modified self.head true else @headers = @meta_headers = {} false end end # Updates headers and meta-headers from S3. # Returns +true+. # # key.meta_headers #=> {"family"=>"qwerty"} # key.head #=> true # key.meta_headers #=> {"family"=>"qwerty", "name"=>"asdfg"} # def head @headers, @meta_headers = self.class.split_meta(@bucket.s3.interface.head(@bucket, @name)) true end # Reload meta-headers only. Returns meta-headers hash. # # key.reload_meta #=> {"family"=>"qwerty", "name"=>"asdfg"} # def reload_meta @meta_headers = self.class.split_meta(@bucket.s3.interface.head(@bucket, @name)).last end # Replace meta-headers by new hash at S3. Returns new meta-headers hash. # # key.reload_meta #=> {"family"=>"qwerty", "name"=>"asdfg"} # key.save_meta #=> {"family"=>"oops", "race" => "troll"} # key.reload_meta #=> {"family"=>"oops", "race" => "troll"} # def save_meta(meta_headers) meta = self.class.add_meta_prefix(meta_headers) @bucket.s3.interface.copy(@bucket.name, @name, @bucket.name, @name, :replace, meta) @meta_headers = self.class.split_meta(meta)[1] end # Check for existence of the key in the given bucket. # Returns +true+ or +false+. # # key = RightAws::S3::Key.create(bucket,'logs/today/1.log') # key.exists? #=> false # key.put('Woohoo!') #=> true # key.exists? #=> true # def exists? @bucket.key(self).last_modified ? true : false end # Remove key from bucket. # Returns +true+. # # key.delete #=> true # def delete raise 'Key name must be specified.' if @name.right_blank? @bucket.s3.interface.delete(@bucket, @name) end # Return a list of grantees. # def grantees Grantee::grantees(self) end end class Owner attr_reader :id, :name def initialize(id, name) @id = id @name = name end # Return Owner name as a +String+. def to_s @name end end # There are 2 ways to set permissions for a bucket or key (called a +thing+ below): # # 1 . Use +perms+ param to set 'Canned Access Policies' when calling the bucket.create, # bucket.put and key.put methods. # The +perms+ param can take these values: 'private', 'public-read', 'public-read-write' and # 'authenticated-read'. # (see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTAccessPolicy.html). # # bucket = s3.bucket('bucket_for_kd_test_13', true, 'public-read') # key.put('Woohoo!','public-read-write' ) # # 2 . Use Grantee instances (the permission is a +String+ or an +Array+ of: 'READ', 'WRITE', # 'READ_ACP', 'WRITE_ACP', 'FULL_CONTROL'): # # bucket = s3.bucket('my_awesome_bucket', true) # grantee1 = RightAws::S3::Grantee.new(bucket, 'a123b...223c', FULL_CONTROL, :apply) # grantee2 = RightAws::S3::Grantee.new(bucket, 'xy3v3...5fhp', [READ, WRITE], :apply) # # There is only one way to get and to remove permission (via Grantee instances): # # grantees = bucket.grantees # a list of Grantees that have any access for this bucket # grantee1 = RightAws::S3::Grantee.new(bucket, 'a123b...223c') # grantee1.perms #=> returns a list of perms for this grantee to that bucket # ... # grantee1.drop # remove all perms for this grantee # grantee2.revoke('WRITE') # revoke write access only # class Grantee # A bucket or a key the grantee has an access to. attr_reader :thing # Grantee Amazon id. attr_reader :id # Grantee display name. attr_reader :name # Array of permissions. attr_accessor :perms # Retrieve Owner information and a list of Grantee instances that have # a access to this thing (bucket or key). # # bucket = s3.bucket('my_awesome_bucket', true, 'public-read') # ... # RightAws::S3::Grantee.owner_and_grantees(bucket) #=> [owner, grantees] # def self.owner_and_grantees(thing) if thing.is_a?(Bucket) bucket, key = thing, '' else bucket, key = thing.bucket, thing end hash = bucket.s3.interface.get_acl_parse(bucket.to_s, key.to_s) owner = Owner.new(hash[:owner][:id], hash[:owner][:display_name]) grantees = [] hash[:grantees].each do |id, params| grantees << new(thing, id, params[:permissions], nil, params[:display_name]) end [owner, grantees] end # Retrieves a list of Grantees instances that have an access to this thing(bucket or key). # # bucket = s3.bucket('my_awesome_bucket', true, 'public-read') # ... # RightAws::S3::Grantee.grantees(bucket) #=> grantees # def self.grantees(thing) owner_and_grantees(thing)[1] end def self.put_acl(thing, owner, grantees) #:nodoc: if thing.is_a?(Bucket) bucket, key = thing, '' else bucket, key = thing.bucket, thing end body = "" + "" + "#{owner.id}" + "#{owner.name}" + "" + "" + grantees.map{|grantee| grantee.to_xml}.join + "" + "" bucket.s3.interface.put_acl(bucket.to_s, key.to_s, body) end # Create a new Grantee instance. # Grantee +id+ must exist on S3. If +action+ == :refresh, then retrieve # permissions from S3 and update @perms. If +action+ == :apply, then apply # perms to +thing+ at S3. If +action+ == :apply_and_refresh then it performs. # both the actions. This is used for the new grantees that had no perms to # this thing before. The default action is :refresh. # # bucket = s3.bucket('my_awesome_bucket', true, 'public-read') # grantee1 = RightAws::S3::Grantee.new(bucket, 'a123b...223c', FULL_CONTROL) # ... # grantee2 = RightAws::S3::Grantee.new(bucket, 'abcde...asdf', [FULL_CONTROL, READ], :apply) # grantee3 = RightAws::S3::Grantee.new(bucket, 'aaaaa...aaaa', 'READ', :apply_and_refresh) # def initialize(thing, id, perms=[], action=:refresh, name=nil) @thing = thing @id = id @name = name @perms = Array(perms) case action when :apply then apply when :refresh then refresh when :apply_and_refresh then apply; refresh end end # Return +true+ if the grantee has any permissions to the thing. def exists? self.class.grantees(@thing).each do |grantee| return true if @id == grantee.id end false end # Return Grantee type (+String+): "Group", "AmazonCustomerByEmail" or "CanonicalUser". def type case @id when /^http:/ then "Group" when /@/ then "AmazonCustomerByEmail" else "CanonicalUser" end end # Return a name or an id. def to_s @name || @id end # Add permissions for grantee. # Permissions: 'READ', 'WRITE', 'READ_ACP', 'WRITE_ACP', 'FULL_CONTROL'. # See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingPermissions.html . # Returns +true+. # # grantee.grant('FULL_CONTROL') #=> true # grantee.grant('FULL_CONTROL','WRITE','READ') #=> true # grantee.grant(['WRITE_ACP','READ','READ_ACP']) #=> true # def grant(*permissions) permissions.flatten! old_perms = @perms.dup @perms += permissions @perms.uniq! return true if @perms == old_perms apply end # Revoke permissions for grantee. # Permissions: 'READ', 'WRITE', 'READ_ACP', 'WRITE_ACP', 'FULL_CONTROL' # See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingPermissions.html . # Default value is 'FULL_CONTROL'. # Returns +true+. # # grantee.revoke('READ') #=> true # grantee.revoke('FULL_CONTROL','WRITE') #=> true # grantee.revoke(['READ_ACP','WRITE_ACP']) #=> true # def revoke(*permissions) permissions.flatten! old_perms = @perms.dup @perms -= permissions @perms.uniq! return true if @perms == old_perms apply end # Revoke all permissions for this grantee. # Returns +true+. # # grantee.drop #=> true # def drop @perms = [] apply end # Refresh grantee perms for its +thing+. # Returns +true+ if the grantee has perms for this +thing+ or # +false+ otherwise, and updates @perms value as a side-effect. # # grantee.grant('FULL_CONTROL') #=> true # grantee.refresh #=> true # grantee.drop #=> true # grantee.refresh #=> false # def refresh @perms = [] self.class.grantees(@thing).each do |grantee| if @id == grantee.id @name = grantee.name @perms = grantee.perms return true end end false end # Apply current grantee @perms to +thing+. This method is called internally by the +grant+ # and +revoke+ methods. In normal use this method should not # be called directly. # # grantee.perms = ['FULL_CONTROL'] # grantee.apply #=> true # def apply @perms.uniq! owner, grantees = self.class.owner_and_grantees(@thing) # walk through all the grantees and replace the data for the current one and ... grantees.map! { |grantee| grantee.id == @id ? self : grantee } # ... if this grantee is not known - add this bad boy to a list grantees << self unless grantees.include?(self) # set permissions self.class.put_acl(@thing, owner, grantees) end def to_xml # :nodoc: id_str = case @id when /^http/ then "#{@id}" when /@/ then "#{@id}" else "#{@id}" end grants = '' @perms.each do |perm| grants << "" + "#{id_str}" + "#{perm}" + "" end grants end end end # RightAws::S3Generator and RightAws::S3Generator::Bucket methods: # # s3g = RightAws::S3Generator.new('1...2', 'nx...Y6') #=> # # # # List all buckets(method 'GET'): # buckets_list = s3g.buckets #=> 'https://s3.amazonaws.com:443/?Signature=Y...D&Expires=1180941864&AWSAccessKeyId=1...2' # # Create bucket link (method 'PUT'): # bucket = s3g.bucket('my_awesome_bucket') #=> # # link_to_create = bucket.create_link(1.hour) #=> https://s3.amazonaws.com:443/my_awesome_bucket?Signature=4...D&Expires=1180942132&AWSAccessKeyId=1...2 # # ... or: # bucket = RightAws::S3Generator::Bucket.create(s3g, 'my_awesome_bucket') #=> # # link_to_create = bucket.create_link(1.hour) #=> https://s3.amazonaws.com:443/my_awesome_bucket?Signature=4...D&Expires=1180942132&AWSAccessKeyId=1...2 # # ... or: # bucket = RightAws::S3Generator::Bucket.new(s3g, 'my_awesome_bucket') #=> # # link_to_create = bucket.create_link(1.hour) #=> https://s3.amazonaws.com:443/my_awesome_bucket?Signature=4...D&Expires=1180942132&AWSAccessKeyId=1...2 # # List bucket(method 'GET'): # bucket.keys(1.day) #=> https://s3.amazonaws.com:443/my_awesome_bucket?Signature=i...D&Expires=1180942620&AWSAccessKeyId=1...2 # # Create/put key (method 'PUT'): # bucket.put('my_cool_key') #=> https://s3.amazonaws.com:443/my_awesome_bucket/my_cool_key?Signature=q...D&Expires=1180943094&AWSAccessKeyId=1...2 # # Get key data (method 'GET'): # bucket.get('logs/today/1.log', 1.hour) #=> https://s3.amazonaws.com:443/my_awesome_bucket/my_cool_key?Signature=h...M%3D&Expires=1180820032&AWSAccessKeyId=1...2 # # Delete bucket (method 'DELETE'): # bucket.delete(2.hour) #=> https://s3.amazonaws.com:443/my_awesome_bucket/logs%2Ftoday%2F1.log?Signature=4...D&Expires=1180820032&AWSAccessKeyId=1...2 # # RightAws::S3Generator::Key methods: # # # Create Key instance: # key = RightAws::S3Generator::Key.new(bicket, 'my_cool_key') #=> # # # Put key data (method 'PUT'): # key.put #=> https://s3.amazonaws.com:443/my_awesome_bucket/my_cool_key?Signature=2...D&Expires=1180943302&AWSAccessKeyId=1...2 # # Get key data (method 'GET'): # key.get #=> https://s3.amazonaws.com:443/my_awesome_bucket/my_cool_key?Signature=a...D&Expires=1180820032&AWSAccessKeyId=1...2 # # Head key (method 'HEAD'): # key.head #=> https://s3.amazonaws.com:443/my_awesome_bucket/my_cool_key?Signature=b...D&Expires=1180820032&AWSAccessKeyId=1...2 # # Delete key (method 'DELETE'): # key.delete #=> https://s3.amazonaws.com:443/my_awesome_bucket/my_cool_key?Signature=x...D&Expires=1180820032&AWSAccessKeyId=1...2 # class S3Generator attr_reader :interface def initialize(aws_access_key_id, aws_secret_access_key, params={}) @interface = S3Interface.new(aws_access_key_id, aws_secret_access_key, params) end # Generate link to list all buckets # # s3.buckets(1.hour) # def buckets(expires=nil, headers={}) @interface.list_all_my_buckets_link(expires, headers) end # Create new S3LinkBucket instance and generate link to create it at S3. # # bucket= s3.bucket('my_owesome_bucket') # def bucket(name, expires=nil, headers={}) Bucket.create(self, name.to_s) end class Bucket attr_reader :s3, :name def to_s @name end alias_method :full_name, :to_s # Return a public link to bucket. # # bucket.public_link #=> 'https://s3.amazonaws.com:443/my_awesome_bucket' # def public_link params = @s3.interface.params "#{params[:protocol]}://#{params[:server]}:#{params[:port]}/#{full_name}" end # Create new S3LinkBucket instance and generate creation link for it. def self.create(s3, name, expires=nil, headers={}) new(s3, name.to_s) end # Create new S3LinkBucket instance. def initialize(s3, name) @s3, @name = s3, name.to_s end # Return a link to create this bucket. # def create_link(expires=nil, headers={}) @s3.interface.create_bucket_link(@name, expires, headers) end # Generate link to list keys. # # bucket.keys # bucket.keys('prefix'=>'logs') # def keys(options=nil, expires=nil, headers={}) @s3.interface.list_bucket_link(@name, options, expires, headers) end # Return a S3Generator::Key instance. # # bucket.key('my_cool_key').get #=> https://s3.amazonaws.com:443/my_awesome_bucket/my_cool_key?Signature=B...D&Expires=1180820032&AWSAccessKeyId=1...2 # bucket.key('my_cool_key').delete #=> https://s3.amazonaws.com:443/my_awesome_bucket/my_cool_key?Signature=B...D&Expires=1180820098&AWSAccessKeyId=1...2 # def key(name) Key.new(self, name) end # Generates link to PUT key data. # # puts bucket.put('logs/today/1.log', 2.hour) # def put(key, meta_headers={}, expires=nil, headers={}) meta = RightAws::S3::Key.add_meta_prefix(meta_headers) @s3.interface.put_link(@name, key.to_s, nil, expires, meta.merge(headers)) end # Generate link to GET key data. # # bucket.get('logs/today/1.log', 1.hour) # def get(key, expires=nil, headers={}, response_params={}) @s3.interface.get_link(@name, key.to_s, expires, headers, response_params) end # Generate link to delete bucket. # # bucket.delete(2.hour) # def delete(expires=nil, headers={}) @s3.interface.delete_bucket_link(@name, expires, headers) end end class Key attr_reader :bucket, :name def to_s @name end # Return a full S# name (bucket/key). # # key.full_name #=> 'my_awesome_bucket/cool_key' # def full_name(separator='/') "#{@bucket.to_s}#{separator}#{@name}" end # Return a public link to key. # # key.public_link #=> 'https://s3.amazonaws.com:443/my_awesome_bucket/cool_key' # def public_link params = @bucket.s3.interface.params "#{params[:protocol]}://#{params[:server]}:#{params[:port]}/#{full_name('/')}" end def initialize(bucket, name, meta_headers={}) @bucket = bucket @name = name.to_s @meta_headers = meta_headers raise 'Key name can not be empty.' if @name.right_blank? end # Generate link to PUT key data. # # puts bucket.put('logs/today/1.log', '123', 2.hour) #=> https://s3.amazonaws.com:443/my_awesome_bucket/logs%2Ftoday%2F1.log?Signature=B...D&Expires=1180820032&AWSAccessKeyId=1...2 # def put(expires=nil, headers={}) @bucket.put(@name.to_s, @meta_headers, expires, headers) end # Generate link to GET key data. # # bucket.get('logs/today/1.log', 1.hour) #=> https://s3.amazonaws.com:443/my_awesome_bucket/logs%2Ftoday%2F1.log?Signature=h...M%3D&Expires=1180820032&AWSAccessKeyId=1...2 # def get(expires=nil, headers={}, response_params={}) @bucket.s3.interface.get_link(@bucket.to_s, @name, expires, headers, response_params) end # Generate link to delete key. # # bucket.delete(2.hour) #=> https://s3.amazonaws.com:443/my_awesome_bucket/logs%2Ftoday%2F1.log?Signature=4...D&Expires=1180820032&AWSAccessKeyId=1...2 # def delete(expires=nil, headers={}) @bucket.s3.interface.delete_link(@bucket.to_s, @name, expires, headers) end # Generate link to head key. # # bucket.head(2.hour) #=> https://s3.amazonaws.com:443/my_awesome_bucket/logs%2Ftoday%2F1.log?Signature=4...D&Expires=1180820032&AWSAccessKeyId=1...2 # def head(expires=nil, headers={}) @bucket.s3.interface.head_link(@bucket.to_s, @name, expires, headers) end end end end ================================================ FILE: lib/s3/right_s3_interface.rb ================================================ # # Copyright (c) 2007-2008 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws class S3Interface < RightAwsBase USE_100_CONTINUE_PUT_SIZE = 1_000_000 MINIMUM_PART_SIZE = 5 * 1024 * 1024 DEFAULT_RETRY_COUNT = 5 include RightAwsBaseInterface DEFAULT_HOST = 's3.amazonaws.com' DEFAULT_PORT = 443 DEFAULT_PROTOCOL = 'https' DEFAULT_SERVICE = '/' REQUEST_TTL = 30 DEFAULT_EXPIRES_AFTER = 1 * 24 * 60 * 60 # One day's worth of seconds ONE_YEAR_IN_SECONDS = 365 * 24 * 60 * 60 AMAZON_HEADER_PREFIX = 'x-amz-' AMAZON_METADATA_PREFIX = 'x-amz-meta-' S3_REQUEST_PARAMETERS = [ 'acl', 'location', 'logging', # this one is beta, no support for now 'partNumber', 'response-content-type', 'response-content-language', 'response-expires', 'response-cache-control', 'response-content-disposition', 'response-content-encoding', 'torrent', 'uploadId', 'uploads', 'delete'].sort MULTI_OBJECT_DELETE_MAX_KEYS = 1000 @@bench = AwsBenchmarkingBlock.new def self.bench_xml @@bench.xml end def self.bench_s3 @@bench.service end # Params supported: # :no_subdomains => true # do not use bucket as a part of domain name but as a part of path @@params = {} def self.params @@params end # get custom option def param(name) # - check explicitly defined param (@params) # - otherwise check implicitly defined one (@@params) @params.has_key?(name) ? @params[name] : @@params[name] end # Creates new RightS3 instance. # # s3 = RightAws::S3Interface.new('1E3GDYEOGFJPIT7XXXXXX','hgTHt68JY07JKUY08ftHYtERkjgtfERn57XXXXXX', {:logger => Logger.new('/tmp/x.log')}) #=> # # # Params is a hash: # # {:server => 's3.amazonaws.com' # Amazon service host: 's3.amazonaws.com'(default) # :port => 443 # Amazon service port: 80 or 443(default) # :protocol => 'https' # Amazon service protocol: 'http' or 'https'(default) # :logger => Logger Object # Logger instance: logs to STDOUT if omitted # :no_subdomains => true} # Force placing bucket name into path instead of domain name # def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={}) init({ :name => 'S3', :default_host => ENV['S3_URL'] ? URI.parse(ENV['S3_URL']).host : DEFAULT_HOST, :default_port => ENV['S3_URL'] ? URI.parse(ENV['S3_URL']).port : DEFAULT_PORT, :default_service => ENV['S3_URL'] ? URI.parse(ENV['S3_URL']).path : DEFAULT_SERVICE, :default_protocol => ENV['S3_URL'] ? URI.parse(ENV['S3_URL']).scheme : DEFAULT_PROTOCOL }, aws_access_key_id || ENV['AWS_ACCESS_KEY_ID'], aws_secret_access_key || ENV['AWS_SECRET_ACCESS_KEY'], params) end #----------------------------------------------------------------- # Requests #----------------------------------------------------------------- # Produces canonical string for signing. def canonical_string(method, path, headers={}, expires=nil) # :nodoc: s3_headers = {} headers.each do |key, value| key = key.downcase value = case when value.is_a?(Array) then value.join('') else value.to_s end s3_headers[key] = value.strip if key[/^#{AMAZON_HEADER_PREFIX}|^content-md5$|^content-type$|^date$/o] end s3_headers['content-type'] ||= '' s3_headers['content-md5'] ||= '' s3_headers['date'] = '' if s3_headers.has_key? 'x-amz-date' s3_headers['date'] = expires if expires # prepare output string out_string = "#{method}\n" s3_headers.sort { |a, b| a[0] <=> b[0] }.each do |key, value| out_string << (key[/^#{AMAZON_HEADER_PREFIX}/o] ? "#{key}:#{value}\n" : "#{value}\n") end # ignore everything after the question mark by default... out_string << path.gsub(/\?.*$/, '') # ... unless there is a parameter that we care about. S3_REQUEST_PARAMETERS.each do |parameter| if path[/[&?]#{parameter}(=[^&]*)?($|&)/] if $1 value = CGI::unescape($1) else value = '' end out_string << (out_string[/[?]/] ? "&#{parameter}#{value}" : "?#{parameter}#{value}") end end out_string end # http://docs.amazonwebservices.com/AmazonS3/2006-03-01/index.html?BucketRestrictions.html def is_dns_bucket?(bucket_name) bucket_name = bucket_name.to_s return nil unless (3..63) === bucket_name.size bucket_name.split('.').each do |component| return nil unless component[/^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/] end true end def fetch_request_params(headers) #:nodoc: # default server to use server = @params[:server] service = @params[:service].to_s service.chop! if service[%r{/$}] # remove trailing '/' from service # extract bucket name and check it's dns compartibility headers[:url].to_s[%r{^([a-z0-9._-]*)(/[^?]*)?(\?.+)?}i] bucket_name, key_path, params_list = $1, $2, $3 key_path = key_path.gsub( '%2F', '/' ) if key_path # select request model if !param(:no_subdomains) && is_dns_bucket?(bucket_name) # fix a path server = "#{bucket_name}.#{server}" key_path ||= '/' path = "#{service}#{key_path}#{params_list}" else path = "#{service}/#{bucket_name}#{key_path}#{params_list}" end path_to_sign = "#{service}/#{bucket_name}#{key_path}#{params_list}" # path_to_sign = "/#{bucket_name}#{key_path}#{params_list}" [ server, path, path_to_sign ] end # Generates request hash for REST API. # Assumes that headers[:url] is URL encoded (use CGI::escape) def generate_rest_request(method, headers) # :nodoc: # calculate request data server, path, path_to_sign = fetch_request_params(headers) data = headers[:data] # make sure headers are downcased strings headers = AwsUtils::fix_headers(headers) # headers['content-type'] ||= '' headers['date'] = Time.now.httpdate # create request request = "Net::HTTP::#{method.capitalize}".right_constantize.new(path) request.body = data if data # set request headers and meta headers headers.each { |key, value| request[key.to_s] = value } #generate auth strings auth_string = canonical_string(request.method, path_to_sign, request.to_hash) signature = AwsUtils::sign(@aws_secret_access_key, auth_string) # set other headers request['Authorization'] = "AWS #{@aws_access_key_id}:#{signature}" # prepare output hash { :request => request, :server => server, :port => @params[:port], :protocol => @params[:protocol] } end # Sends request to Amazon and parses the response. # Raises AwsError if any banana happened. def request_info(request, parser, &block) # :nodoc: request_info_impl(:s3_connection, @@bench, request, parser, &block) end # Returns an array of customer's buckets. Each item is a +hash+. # # s3.list_all_my_buckets #=> # [{:owner_id => "00000000009314cc309ffe736daa2b264357476c7fea6efb2c3347ac3ab2792a", # :owner_display_name => "root", # :name => "bucket_name", # :creation_date => "2007-04-19T18:47:43.000Z"}, ..., {...}] # def list_all_my_buckets(headers={}) req_hash = generate_rest_request('GET', headers.merge(:url=>'')) request_info(req_hash, S3ListAllMyBucketsParser.new(:logger => @logger)) rescue on_exception end # Creates new bucket. Returns +true+ or an exception. # # # create a bucket at American server # s3.create_bucket('my-awesome-bucket-us') #=> true # # create a bucket at European server # s3.create_bucket('my-awesome-bucket-eu', :location => :eu) #=> true # def create_bucket(bucket, headers={}) data = nil location = case headers[:location].to_s when 'us','US' then '' when 'eu' then 'EU' else headers[:location].to_s end unless location.right_blank? data = "#{location}" end req_hash = generate_rest_request('PUT', headers.merge(:url=>bucket, :data => data)) request_info(req_hash, RightHttp2xxParser.new) rescue Exception => e # if the bucket exists AWS returns an error for the location constraint interface. Drop it e.is_a?(RightAws::AwsError) && e.message.include?('BucketAlreadyOwnedByYou') ? true : on_exception end # Retrieve bucket location # # s3.create_bucket('my-awesome-bucket-us') #=> true # puts s3.bucket_location('my-awesome-bucket-us') #=> '' (Amazon's default value assumed) # # s3.create_bucket('my-awesome-bucket-eu', :location => :eu) #=> true # puts s3.bucket_location('my-awesome-bucket-eu') #=> 'EU' # def bucket_location(bucket, headers={}) req_hash = generate_rest_request('GET', headers.merge(:url=>"#{bucket}?location")) request_info(req_hash, S3BucketLocationParser.new) rescue on_exception end # Retrieves the logging configuration for a bucket. # Returns a hash of {:enabled, :targetbucket, :targetprefix} # # s3.interface.get_logging_parse(:bucket => "asset_bucket") # => {:enabled=>true, :targetbucket=>"mylogbucket", :targetprefix=>"loggylogs/"} # # def get_logging_parse(params) AwsUtils.mandatory_arguments([:bucket], params) AwsUtils.allow_only([:bucket, :headers], params) params[:headers] = {} unless params[:headers] req_hash = generate_rest_request('GET', params[:headers].merge(:url=>"#{params[:bucket]}?logging")) request_info(req_hash, S3LoggingParser.new) rescue on_exception end # Sets logging configuration for a bucket from the XML configuration document. # params: # :bucket # :xmldoc def put_logging(params) AwsUtils.mandatory_arguments([:bucket,:xmldoc], params) AwsUtils.allow_only([:bucket,:xmldoc, :headers], params) params[:headers] = {} unless params[:headers] req_hash = generate_rest_request('PUT', params[:headers].merge(:url=>"#{params[:bucket]}?logging", :data => params[:xmldoc])) request_info(req_hash, RightHttp2xxParser.new) rescue on_exception end # Deletes new bucket. Bucket must be empty! Returns +true+ or an exception. # # s3.delete_bucket('my_awesome_bucket') #=> true # # See also: force_delete_bucket method # def delete_bucket(bucket, headers={}) req_hash = generate_rest_request('DELETE', headers.merge(:url=>bucket)) request_info(req_hash, RightHttp2xxParser.new) rescue on_exception end # Returns an array of bucket's keys. Each array item (key data) is a +hash+. # # s3.list_bucket('my_awesome_bucket', { 'prefix'=>'t', 'marker'=>'', 'max-keys'=>5, delimiter=>'' }) #=> # [{:key => "test1", # :last_modified => "2007-05-18T07:00:59.000Z", # :owner_id => "00000000009314cc309ffe736daa2b264357476c7fea6efb2c3347ac3ab2792a", # :owner_display_name => "root", # :e_tag => "000000000059075b964b07152d234b70", # :storage_class => "STANDARD", # :size => 3, # :service=> {'is_truncated' => false, # 'prefix' => "t", # 'marker' => "", # 'name' => "my_awesome_bucket", # 'max-keys' => "5"}, ..., {...}] # def list_bucket(bucket, options={}, headers={}) bucket += '?'+options.map{|k, v| "#{k.to_s}=#{CGI::escape v.to_s}"}.join('&') unless options.right_blank? req_hash = generate_rest_request('GET', headers.merge(:url=>bucket)) request_info(req_hash, S3ListBucketParser.new(:logger => @logger)) rescue on_exception end # Incrementally list the contents of a bucket. Yields the following hash to a block: # s3.incrementally_list_bucket('my_awesome_bucket', { 'prefix'=>'t', 'marker'=>'', 'max-keys'=>5, delimiter=>'' }) yields # { # :name => 'bucketname', # :prefix => 'subfolder/', # :marker => 'fileN.jpg', # :max_keys => 234, # :delimiter => '/', # :is_truncated => true, # :next_marker => 'fileX.jpg', # :contents => [ # { :key => "file1", # :last_modified => "2007-05-18T07:00:59.000Z", # :e_tag => "000000000059075b964b07152d234b70", # :size => 3, # :storage_class => "STANDARD", # :owner_id => "00000000009314cc309ffe736daa2b264357476c7fea6efb2c3347ac3ab2792a", # :owner_display_name => "root" # }, { :key, ...}, ... {:key, ...} # ] # :common_prefixes => [ # "prefix1", # "prefix2", # ..., # "prefixN" # ] # } def incrementally_list_bucket(bucket, options={}, headers={}, &block) internal_options = options.right_symbolize_keys begin internal_bucket = bucket.dup internal_bucket += '?'+internal_options.map{|k, v| "#{k.to_s}=#{CGI::escape v.to_s}"}.join('&') unless internal_options.right_blank? req_hash = generate_rest_request('GET', headers.merge(:url=>internal_bucket)) response = request_info(req_hash, S3ImprovedListBucketParser.new(:logger => @logger)) there_are_more_keys = response[:is_truncated] if(there_are_more_keys) internal_options[:marker] = decide_marker(response) total_results = response[:contents].length + response[:common_prefixes].length internal_options[:'max-keys'] ? (internal_options[:'max-keys'] -= total_results) : nil end yield response end while there_are_more_keys && under_max_keys(internal_options) true rescue on_exception end private def decide_marker(response) return response[:next_marker].dup if response[:next_marker] last_key = response[:contents].last[:key] last_prefix = response[:common_prefixes].last if(!last_key) return nil if(!last_prefix) last_prefix.dup elsif(!last_prefix) last_key.dup else last_key > last_prefix ? last_key.dup : last_prefix.dup end end def under_max_keys(internal_options) internal_options[:'max-keys'] ? internal_options[:'max-keys'] > 0 : true end public # Saves object to Amazon. Returns +true+ or an exception. # Any header starting with AMAZON_METADATA_PREFIX is considered # user metadata. It will be stored with the object and returned # when you retrieve the object. The total size of the HTTP # request, not including the body, must be less than 4 KB. # # s3.put('my_awesome_bucket', 'log/current/1.log', 'Ola-la!', 'x-amz-meta-family'=>'Woho556!') #=> true # # This method is capable of 'streaming' uploads; that is, it can upload # data from a file or other IO object without first reading all the data # into memory. This is most useful for large PUTs - it is difficult to read # a 2 GB file entirely into memory before sending it to S3. # To stream an upload, pass an object that responds to 'read' (like the read # method of IO) and to either 'lstat' or 'size'. For files, this means # streaming is enabled by simply making the call: # # s3.put(bucket_name, 'S3keyname.forthisfile', File.open('localfilename.dat')) # # If the IO object you wish to stream from responds to the read method but # doesn't implement lstat or size, you can extend the object dynamically # to implement these methods, or define your own class which defines these # methods. Be sure that your class returns 'nil' from read() after having # read 'size' bytes. Otherwise S3 will drop the socket after # 'Content-Length' bytes have been uploaded, and HttpConnection will # interpret this as an error. # # This method now supports very large PUTs, where very large # is > 2 GB. # # For Win32 users: Files and IO objects should be opened in binary mode. If # a text mode IO object is passed to PUT, it will be converted to binary # mode. # def put(bucket, key, data=nil, headers={}, &blck) # On Windows, if someone opens a file in text mode, we must reset it so # to binary mode for streaming to work properly if(data.respond_to?(:binmode)) data.binmode end if (data.respond_to?(:lstat) && data.lstat.size >= USE_100_CONTINUE_PUT_SIZE) || (data.respond_to?(:size) && data.size >= USE_100_CONTINUE_PUT_SIZE) headers['expect'] = '100-continue' end req_hash = generate_rest_request('PUT', headers.merge(:url=>"#{bucket}/#{CGI::escape key}", :data=>data)) request_info(req_hash, RightHttp2xxParser.new, &blck) rescue on_exception end # New experimental API for uploading objects, introduced in RightAws 1.8.1. # store_object is similar in function to the older function put, but returns the full response metadata. It also allows for optional verification # of object md5 checksums on upload. Parameters are passed as hash entries and are checked for completeness as well as for spurious arguments. # The hash of the response headers contains useful information like the Amazon request ID and the object ETag (MD5 checksum). # # If the optional :md5 argument is provided, store_object verifies that the given md5 matches the md5 returned by S3. The :verified_md5 field in the response hash is # set true or false depending on the outcome of this check. If no :md5 argument is given, :verified_md5 will be false in the response. # # The optional argument of :headers allows the caller to specify arbitrary request header values. # # s3.store_object(:bucket => "foobucket", :key => "foo", :md5 => "a507841b1bc8115094b00bbe8c1b2954", :data => "polemonium" ) # => {"x-amz-id-2"=>"SVsnS2nfDaR+ixyJUlRKM8GndRyEMS16+oZRieamuL61pPxPaTuWrWtlYaEhYrI/", # "etag"=>"\"a507841b1bc8115094b00bbe8c1b2954\"", # "date"=>"Mon, 29 Sep 2008 18:57:46 GMT", # :verified_md5=>true, # "x-amz-request-id"=>"63916465939995BA", # "server"=>"AmazonS3", # "content-length"=>"0"} # # s3.store_object(:bucket => "foobucket", :key => "foo", :data => "polemonium" ) # => {"x-amz-id-2"=>"MAt9PLjgLX9UYJ5tV2fI/5dBZdpFjlzRVpWgBDpvZpl+V+gJFcBMW2L+LBstYpbR", # "etag"=>"\"a507841b1bc8115094b00bbe8c1b2954\"", # "date"=>"Mon, 29 Sep 2008 18:58:56 GMT", # :verified_md5=>false, # "x-amz-request-id"=>"3B25A996BC2CDD3B", # "server"=>"AmazonS3", # "content-length"=>"0"} def store_object(params) AwsUtils.allow_only([:bucket, :key, :data, :headers, :md5], params) AwsUtils.mandatory_arguments([:bucket, :key, :data], params) params[:headers] = {} unless params[:headers] params[:data].binmode if(params[:data].respond_to?(:binmode)) # On Windows, if someone opens a file in text mode, we must reset it to binary mode for streaming to work properly if (params[:data].respond_to?(:lstat) && params[:data].lstat.size >= USE_100_CONTINUE_PUT_SIZE) || (params[:data].respond_to?(:size) && params[:data].size >= USE_100_CONTINUE_PUT_SIZE) params[:headers]['expect'] = '100-continue' end req_hash = generate_rest_request('PUT', params[:headers].merge(:url=>"#{params[:bucket]}/#{CGI::escape params[:key]}", :data=>params[:data])) resp = request_info(req_hash, S3HttpResponseHeadParser.new) if(params[:md5]) resp[:verified_md5] = (resp['etag'].gsub(/\"/, '') == params[:md5]) ? true : false else resp[:verified_md5] = false end resp rescue on_exception end # Identical in function to store_object, but requires verification that the returned ETag is identical to the checksum passed in by the user as the 'md5' argument. # If the check passes, returns the response metadata with the "verified_md5" field set true. Raises an exception if the checksums conflict. # This call is implemented as a wrapper around store_object and the user may gain different semantics by creating a custom wrapper. # # s3.store_object_and_verify(:bucket => "foobucket", :key => "foo", :md5 => "a507841b1bc8115094b00bbe8c1b2954", :data => "polemonium" ) # => {"x-amz-id-2"=>"IZN3XsH4FlBU0+XYkFTfHwaiF1tNzrm6dIW2EM/cthKvl71nldfVC0oVQyydzWpb", # "etag"=>"\"a507841b1bc8115094b00bbe8c1b2954\"", # "date"=>"Mon, 29 Sep 2008 18:38:32 GMT", # :verified_md5=>true, # "x-amz-request-id"=>"E8D7EA4FE00F5DF7", # "server"=>"AmazonS3", # "content-length"=>"0"} # # s3.store_object_and_verify(:bucket => "foobucket", :key => "foo", :md5 => "a507841b1bc8115094b00bbe8c1b2953", :data => "polemonium" ) # RightAws::AwsError: Uploaded object failed MD5 checksum verification: {"x-amz-id-2"=>"HTxVtd2bf7UHHDn+WzEH43MkEjFZ26xuYvUzbstkV6nrWvECRWQWFSx91z/bl03n", # "etag"=>"\"a507841b1bc8115094b00bbe8c1b2954\"", # "date"=>"Mon, 29 Sep 2008 18:38:41 GMT", # :verified_md5=>false, # "x-amz-request-id"=>"0D7ADE09F42606F2", # "server"=>"AmazonS3", # "content-length"=>"0"} def store_object_and_verify(params) AwsUtils.mandatory_arguments([:md5], params) r = store_object(params) r[:verified_md5] ? (return r) : (raise AwsError.new("Uploaded object failed MD5 checksum verification: #{r.inspect}")) end # New experimental API for uploading objects using the multipart upload API. # store_object_multipart is similar in function to the store_object method, but breaks the input into parts and transmits each # part separately. The multipart upload API has the benefit of being be able to retransmit a part in isolation without needing to # restart the entire upload. This makes it ideal for uploading large files over unreliable networks. It also does not # require the file size to be known before starting the upload, making it useful for stream data as it is created (say via reading a pipe or socket). # The hash of the response headers contains useful information like the location (the URI for the newly created object), bucket, key, and etag). # # The optional argument of :headers allows the caller to specify arbitrary request header values. # # s3.store_object_multipart(:bucket => "foobucket", :key => "foo", :data => "polemonium" ) # => {:location=>"https://s3.amazonaws.com/right_s3_awesome_test_bucket_000B1_officedrop/test%2Flarge_multipart_file", # :e_tag=>"\"72b81ac08aed4d4d1055c11f56c2a258-1\"", # :key=>"test/large_multipart_file", # :bucket=>"right_s3_awesome_test_bucket_000B1_officedrop"} # # f = File.new("some_file", "r") # s3.store_object_multipart(:bucket => "foobucket", :key => "foo", :data => f ) # => {:location=>"https://s3.amazonaws.com/right_s3_awesome_test_bucket_000B1_officedrop/test%2Flarge_multipart_file", # :e_tag=>"\"72b81ac08aed4d4d1055c11f56c2a258-1\"", # :key=>"test/large_multipart_file", # :bucket=>"right_s3_awesome_test_bucket_000B1_officedrop"} def store_object_multipart(params) AwsUtils.allow_only([:bucket, :key, :data, :headers, :part_size, :retry_count], params) AwsUtils.mandatory_arguments([:bucket, :key, :data], params) params[:headers] = {} unless params[:headers] params[:data].binmode if(params[:data].respond_to?(:binmode)) # On Windows, if someone opens a file in text mode, we must reset it to binary mode for streaming to work properly # detect whether we are using straight read or converting to string first unless(params[:data].respond_to?(:read)) params[:data] = StringIO.new(params[:data].to_s) end # make sure part size is > 5 MB minimum params[:part_size] ||= MINIMUM_PART_SIZE if params[:part_size] < MINIMUM_PART_SIZE raise AwsError.new("Part size for a multipart upload must be greater than or equal to #{5 * 1024 * 1024} bytes. #{params[:part_size]} bytes was provided.") end # make sure retry_count is positive params[:retry_count] ||= DEFAULT_RETRY_COUNT if params[:retry_count] < 0 raise AwsError.new("Retry count must be positive. #{params[:retry_count]} bytes was provided.") end # Set 100-continue for large part sizes if (params[:part_size] >= USE_100_CONTINUE_PUT_SIZE) params[:headers]['expect'] = '100-continue' end # initiate upload initiate_hash = generate_rest_request('POST', params[:headers].merge(:url=>"#{params[:bucket]}/#{CGI::escape params[:key]}?uploads")) initiate_resp = request_info(initiate_hash, S3MultipartUploadInitiateResponseParser.new) upload_id = initiate_resp[:upload_id] # split into parts and upload each one, re-trying if necessary # upload occurs serially at this time. part_etags = [] part_data = "" index = 1 until params[:data].eof? part_data = params[:data].read(params[:part_size]) unless part_data.size == 0 retry_attempts = 1 while true begin send_part_hash = generate_rest_request('PUT', params[:headers].merge({ :url=>"#{params[:bucket]}/#{CGI::escape params[:key]}?partNumber=#{index}&uploadId=#{upload_id}", :data=>part_data } )) send_part_resp = request_info(send_part_hash, S3HttpResponseHeadParser.new) part_etags << {:part_num => index, :etag => send_part_resp['etag']} index += 1 break # successful, can move to next part rescue AwsError => e if retry_attempts >= params[:retry_count] raise e else #Hit an error attempting to transmit part, retry until retry_attemts have been exhausted retry_attempts += 1 end end end end end # assemble complete upload message complete_body = "" part_etags.each do |part_hash| complete_body << "#{part_hash[:part_num]}#{part_hash[:etag]}" end complete_body << "" complete_req_hash = generate_rest_request('POST', {:url=>"#{params[:bucket]}/#{CGI::escape params[:key]}?uploadId=#{upload_id}", :data => complete_body}) return request_info(complete_req_hash, S3CompleteMultipartParser.new) rescue on_exception end class S3MultipartUploadInitiateResponseParser < RightAWSParser def reset @result = {} end def headers_to_string(headers) result = {} headers.each do |key, value| value = value.first if value.is_a?(Array) && value.size<2 result[key] = value end result end def tagend(name) case name when 'UploadId' then @result[:upload_id] = @text end end end class S3CompleteMultipartParser < RightAWSParser # :nodoc: def reset @result = {} end def tagend(name) case name when 'Location' then @result[:location] = @text when 'Bucket' then @result[:bucket] = @text when 'Key' then @result[:key] = @text when 'ETag' then @result[:e_tag] = @text end end end # Retrieves object data from Amazon. Returns a +hash+ or an exception. # # s3.get('my_awesome_bucket', 'log/curent/1.log') #=> # # {:object => "Ola-la!", # :headers => {"last-modified" => "Wed, 23 May 2007 09:08:04 GMT", # "content-type" => "", # "etag" => "\"000000000096f4ee74bc4596443ef2a4\"", # "date" => "Wed, 23 May 2007 09:08:03 GMT", # "x-amz-id-2" => "ZZZZZZZZZZZZZZZZZZZZ1HJXZoehfrS4QxcxTdNGldR7w/FVqblP50fU8cuIMLiu", # "x-amz-meta-family" => "Woho556!", # "x-amz-request-id" => "0000000C246D770C", # "server" => "AmazonS3", # "content-length" => "7"}} # # If a block is provided, yields incrementally to the block as # the response is read. For large responses, this function is ideal as # the response can be 'streamed'. The hash containing header fields is # still returned. # Example: # foo = File.new('./chunder.txt', File::CREAT|File::RDWR) # rhdr = s3.get('aws-test', 'Cent5V1_7_1.img.part.00') do |chunk| # foo.write(chunk) # end # foo.close # def get(bucket, key, headers={}, &block) req_hash = generate_rest_request('GET', headers.merge(:url=>"#{bucket}/#{CGI::escape key}")) request_info(req_hash, S3HttpResponseBodyParser.new, &block) rescue on_exception end # New experimental API for retrieving objects, introduced in RightAws 1.8.1. # retrieve_object is similar in function to the older function get. It allows for optional verification # of object md5 checksums on retrieval. Parameters are passed as hash entries and are checked for completeness as well as for spurious arguments. # # If the optional :md5 argument is provided, retrieve_object verifies that the given md5 matches the md5 returned by S3. The :verified_md5 field in the response hash is # set true or false depending on the outcome of this check. If no :md5 argument is given, :verified_md5 will be false in the response. # # The optional argument of :headers allows the caller to specify arbitrary request header values. # Mandatory arguments: # :bucket - the bucket in which the object is stored # :key - the object address (or path) within the bucket # Optional arguments: # :headers - hash of additional HTTP headers to include with the request # :md5 - MD5 checksum against which to verify the retrieved object # # s3.retrieve_object(:bucket => "foobucket", :key => "foo") # => {:verified_md5=>false, # :headers=>{"last-modified"=>"Mon, 29 Sep 2008 18:58:56 GMT", # "x-amz-id-2"=>"2Aj3TDz6HP5109qly//18uHZ2a1TNHGLns9hyAtq2ved7wmzEXDOPGRHOYEa3Qnp", # "content-type"=>"", # "etag"=>"\"a507841b1bc8115094b00bbe8c1b2954\"", # "date"=>"Tue, 30 Sep 2008 00:52:44 GMT", # "x-amz-request-id"=>"EE4855DE27A2688C", # "server"=>"AmazonS3", # "content-length"=>"10"}, # :object=>"polemonium"} # # s3.retrieve_object(:bucket => "foobucket", :key => "foo", :md5=>'a507841b1bc8115094b00bbe8c1b2954') # => {:verified_md5=>true, # :headers=>{"last-modified"=>"Mon, 29 Sep 2008 18:58:56 GMT", # "x-amz-id-2"=>"mLWQcI+VuKVIdpTaPXEo84g0cz+vzmRLbj79TS8eFPfw19cGFOPxuLy4uGYVCvdH", # "content-type"=>"", "etag"=>"\"a507841b1bc8115094b00bbe8c1b2954\"", # "date"=>"Tue, 30 Sep 2008 00:53:08 GMT", # "x-amz-request-id"=>"6E7F317356580599", # "server"=>"AmazonS3", # "content-length"=>"10"}, # :object=>"polemonium"} # If a block is provided, yields incrementally to the block as # the response is read. For large responses, this function is ideal as # the response can be 'streamed'. The hash containing header fields is # still returned. def retrieve_object(params, &block) AwsUtils.mandatory_arguments([:bucket, :key], params) AwsUtils.allow_only([:bucket, :key, :headers, :md5], params) params[:headers] = {} unless params[:headers] req_hash = generate_rest_request('GET', params[:headers].merge(:url=>"#{params[:bucket]}/#{CGI::escape params[:key]}")) resp = request_info(req_hash, S3HttpResponseBodyParser.new, &block) resp[:verified_md5] = false if(params[:md5] && (resp[:headers]['etag'].gsub(/\"/,'') == params[:md5])) resp[:verified_md5] = true end resp rescue on_exception end # Identical in function to retrieve_object, but requires verification that the returned ETag is identical to the checksum passed in by the user as the 'md5' argument. # If the check passes, returns the response metadata with the "verified_md5" field set true. Raises an exception if the checksums conflict. # This call is implemented as a wrapper around retrieve_object and the user may gain different semantics by creating a custom wrapper. def retrieve_object_and_verify(params, &block) AwsUtils.mandatory_arguments([:md5], params) resp = retrieve_object(params, &block) return resp if resp[:verified_md5] raise AwsError.new("Retrieved object failed MD5 checksum verification: #{resp.inspect}") end # Retrieves object metadata. Returns a +hash+ of http_response_headers. # # s3.head('my_awesome_bucket', 'log/curent/1.log') #=> # {"last-modified" => "Wed, 23 May 2007 09:08:04 GMT", # "content-type" => "", # "etag" => "\"000000000096f4ee74bc4596443ef2a4\"", # "date" => "Wed, 23 May 2007 09:08:03 GMT", # "x-amz-id-2" => "ZZZZZZZZZZZZZZZZZZZZ1HJXZoehfrS4QxcxTdNGldR7w/FVqblP50fU8cuIMLiu", # "x-amz-meta-family" => "Woho556!", # "x-amz-request-id" => "0000000C246D770C", # "server" => "AmazonS3", # "content-length" => "7"} # def head(bucket, key, headers={}) req_hash = generate_rest_request('HEAD', headers.merge(:url=>"#{bucket}/#{CGI::escape key}")) request_info(req_hash, S3HttpResponseHeadParser.new) rescue on_exception end # Deletes key. Returns +true+ or an exception. # # s3.delete('my_awesome_bucket', 'log/curent/1.log') #=> true # def delete(bucket, key='', headers={}) req_hash = generate_rest_request('DELETE', headers.merge(:url=>"#{bucket}/#{CGI::escape key}")) request_info(req_hash, RightHttp2xxParser.new) rescue on_exception end # Deletes multiple keys. Returns an array with errors, if any. # # s3.delete_multiple('my_awesome_bucket', ['key1', 'key2', ...) # #=> [ { :key => 'key2', :code => 'AccessDenied', :message => "Access Denied" } ] # def delete_multiple(bucket, keys=[], headers={}) errors = [] keys = Array.new(keys) while keys.length > 0 data = "\n" data += "\ntrue\n" keys.take(MULTI_OBJECT_DELETE_MAX_KEYS).each do |key| data += "#{AwsUtils::xml_escape(key)}\n" end data += "" req_hash = generate_rest_request('POST', headers.merge( :url => "#{bucket}?delete", :data => data, 'content-md5' => AwsUtils::content_md5(data) )) errors += request_info(req_hash, S3DeleteMultipleParser.new) keys = keys.drop(MULTI_OBJECT_DELETE_MAX_KEYS) end errors rescue on_exception end # Copy an object. # directive: :copy - copy meta-headers from source (default value) # :replace - replace meta-headers by passed ones # # # copy a key with meta-headers # s3.copy('b1', 'key1', 'b1', 'key1_copy') #=> {:e_tag=>"\"e8b...8d\"", :last_modified=>"2008-05-11T10:25:22.000Z"} # # # copy a key, overwrite meta-headers # s3.copy('b1', 'key2', 'b1', 'key2_copy', :replace, 'x-amz-meta-family'=>'Woho555!') #=> {:e_tag=>"\"e8b...8d\"", :last_modified=>"2008-05-11T10:26:22.000Z"} # # see: http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingCopyingObjects.html # http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTObjectCOPY.html # def copy(src_bucket, src_key, dest_bucket, dest_key=nil, directive=:copy, headers={}) dest_key ||= src_key headers['x-amz-metadata-directive'] = directive.to_s.upcase headers['x-amz-copy-source'] = "#{src_bucket}/#{CGI::escape src_key}" req_hash = generate_rest_request('PUT', headers.merge(:url=>"#{dest_bucket}/#{CGI::escape dest_key}")) request_info(req_hash, S3CopyParser.new) rescue on_exception end # Move an object. # directive: :copy - copy meta-headers from source (default value) # :replace - replace meta-headers by passed ones # # # move bucket1/key1 to bucket1/key2 # s3.move('bucket1', 'key1', 'bucket1', 'key2') #=> {:e_tag=>"\"e8b...8d\"", :last_modified=>"2008-05-11T10:27:22.000Z"} # # # move bucket1/key1 to bucket2/key2 with new meta-headers assignment # s3.copy('bucket1', 'key1', 'bucket2', 'key2', :replace, 'x-amz-meta-family'=>'Woho555!') #=> {:e_tag=>"\"e8b...8d\"", :last_modified=>"2008-05-11T10:28:22.000Z"} # def move(src_bucket, src_key, dest_bucket, dest_key=nil, directive=:copy, headers={}) copy_result = copy(src_bucket, src_key, dest_bucket, dest_key, directive, headers) # delete an original key if it differs from a destination one delete(src_bucket, src_key) unless src_bucket == dest_bucket && src_key == dest_key copy_result end # Rename an object. # # # rename bucket1/key1 to bucket1/key2 # s3.rename('bucket1', 'key1', 'key2') #=> {:e_tag=>"\"e8b...8d\"", :last_modified=>"2008-05-11T10:29:22.000Z"} # def rename(src_bucket, src_key, dest_key, headers={}) move(src_bucket, src_key, src_bucket, dest_key, :copy, headers) end # Retieves the ACL (access control policy) for a bucket or object. Returns a hash of headers and xml doc with ACL data. See: http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTAccessPolicy.html. # # s3.get_acl('my_awesome_bucket', 'log/curent/1.log') #=> # {:headers => {"x-amz-id-2"=>"B3BdDMDUz+phFF2mGBH04E46ZD4Qb9HF5PoPHqDRWBv+NVGeA3TOQ3BkVvPBjgxX", # "content-type"=>"application/xml;charset=ISO-8859-1", # "date"=>"Wed, 23 May 2007 09:40:16 GMT", # "x-amz-request-id"=>"B183FA7AB5FBB4DD", # "server"=>"AmazonS3", # "transfer-encoding"=>"chunked"}, # :object => "\n # 16144ab2929314cc309ffe736daa2b264357476c7fea6efb2c3347ac3ab2792aroot # # 16144ab2929314cc309ffe736daa2b264357476c7fea6efb2c3347ac3ab2792aroot # FULL_CONTROL" } # def get_acl(bucket, key='', headers={}) key = key.right_blank? ? '' : "/#{CGI::escape key}" req_hash = generate_rest_request('GET', headers.merge(:url=>"#{bucket}#{key}?acl")) request_info(req_hash, S3HttpResponseBodyParser.new) rescue on_exception end # Retieves the ACL (access control policy) for a bucket or object. # Returns a hash of {:owner, :grantees} # # s3.get_acl_parse('my_awesome_bucket', 'log/curent/1.log') #=> # # { :grantees=> # { "16...2a"=> # { :display_name=>"root", # :permissions=>["FULL_CONTROL"], # :attributes=> # { "xsi:type"=>"CanonicalUser", # "xmlns:xsi"=>"http://www.w3.org/2001/XMLSchema-instance"}}, # "http://acs.amazonaws.com/groups/global/AllUsers"=> # { :display_name=>"AllUsers", # :permissions=>["READ"], # :attributes=> # { "xsi:type"=>"Group", # "xmlns:xsi"=>"http://www.w3.org/2001/XMLSchema-instance"}}}, # :owner=> # { :id=>"16..2a", # :display_name=>"root"}} # def get_acl_parse(bucket, key='', headers={}) key = key.right_blank? ? '' : "/#{CGI::escape key}" req_hash = generate_rest_request('GET', headers.merge(:url=>"#{bucket}#{key}?acl")) acl = request_info(req_hash, S3AclParser.new(:logger => @logger)) result = {} result[:owner] = acl[:owner] result[:grantees] = {} acl[:grantees].each do |grantee| key = grantee[:id] || grantee[:uri] if result[:grantees].key?(key) result[:grantees][key][:permissions] << grantee[:permissions] else result[:grantees][key] = { :display_name => grantee[:display_name] || grantee[:uri].to_s[/[^\/]*$/], :permissions => Array(grantee[:permissions]), :attributes => grantee[:attributes] } end end result rescue on_exception end # Sets the ACL on a bucket or object. def put_acl(bucket, key, acl_xml_doc, headers={}) key = key.right_blank? ? '' : "/#{CGI::escape key}" req_hash = generate_rest_request('PUT', headers.merge(:url=>"#{bucket}#{key}?acl", :data=>acl_xml_doc)) request_info(req_hash, S3HttpResponseBodyParser.new) rescue on_exception end # Retieves the ACL (access control policy) for a bucket. Returns a hash of headers and xml doc with ACL data. def get_bucket_acl(bucket, headers={}) return get_acl(bucket, '', headers) rescue on_exception end # Sets the ACL on a bucket only. def put_bucket_acl(bucket, acl_xml_doc, headers={}) return put_acl(bucket, '', acl_xml_doc, headers) rescue on_exception end # Removes all keys from bucket. Returns +true+ or an exception. # # s3.clear_bucket('my_awesome_bucket') #=> true # def clear_bucket(bucket) incrementally_list_bucket(bucket) do |results| results[:contents].each { |key| delete(bucket, key[:key]) } end true rescue on_exception end # Deletes all keys in bucket then deletes bucket. Returns +true+ or an exception. # # s3.force_delete_bucket('my_awesome_bucket') # def force_delete_bucket(bucket) clear_bucket(bucket) delete_bucket(bucket) rescue on_exception end # Deletes all keys where the 'folder_key' may be assumed as 'folder' name. Returns an array of string keys that have been deleted. # # s3.list_bucket('my_awesome_bucket').map{|key_data| key_data[:key]} #=> ['test','test/2/34','test/3','test1','test1/logs'] # s3.delete_folder('my_awesome_bucket','test') #=> ['test','test/2/34','test/3'] # def delete_folder(bucket, folder_key, separator='/') folder_key.chomp!(separator) allkeys = [] incrementally_list_bucket(bucket, { 'prefix' => folder_key }) do |results| keys = results[:contents].map{ |s3_key| s3_key[:key][/^#{folder_key}($|#{separator}.*)/] ? s3_key[:key] : nil}.compact keys.each{ |key| delete(bucket, key) } allkeys << keys end allkeys rescue on_exception end # Retrieves object data only (headers are omitted). Returns +string+ or an exception. # # s3.get('my_awesome_bucket', 'log/curent/1.log') #=> 'Ola-la!' # def get_object(bucket, key, headers={}) get(bucket, key, headers)[:object] rescue on_exception end #----------------------------------------------------------------- # Query API: Links #----------------------------------------------------------------- def s3_link_escape(text) #CGI::escape(text.to_s).gsub(/[+]/, '%20') AwsUtils::amz_escape(text.to_s) end # Generates link for QUERY API def generate_link(method, headers={}, expires=nil) #:nodoc: # calculate request data server, path, path_to_sign = fetch_request_params(headers) # expiration time expires ||= DEFAULT_EXPIRES_AFTER expires = Time.now.utc + expires if expires.is_a?(Fixnum) && (expires < ONE_YEAR_IN_SECONDS) expires = expires.to_i # make sure headers are downcased strings headers = AwsUtils::fix_headers(headers) #generate auth strings auth_string = canonical_string(method, path_to_sign, headers, expires) signature = CGI::escape(AwsUtils::sign( @aws_secret_access_key, auth_string)) # path building addon = "Signature=#{signature}&Expires=#{expires}&AWSAccessKeyId=#{@aws_access_key_id}" path += path[/\?/] ? "&#{addon}" : "?#{addon}" "#{@params[:protocol]}://#{server}:#{@params[:port]}#{path}" rescue on_exception end # Generates link for 'ListAllMyBuckets'. # # s3.list_all_my_buckets_link #=> url string # def list_all_my_buckets_link(expires=nil, headers={}) generate_link('GET', headers.merge(:url=>''), expires) rescue on_exception end # Generates link for 'CreateBucket'. # # s3.create_bucket_link('my_awesome_bucket') #=> url string # def create_bucket_link(bucket, expires=nil, headers={}) generate_link('PUT', headers.merge(:url=>bucket), expires) rescue on_exception end # Generates link for 'DeleteBucket'. # # s3.delete_bucket_link('my_awesome_bucket') #=> url string # def delete_bucket_link(bucket, expires=nil, headers={}) generate_link('DELETE', headers.merge(:url=>bucket), expires) rescue on_exception end # Generates link for 'ListBucket'. # # s3.list_bucket_link('my_awesome_bucket') #=> url string # def list_bucket_link(bucket, options=nil, expires=nil, headers={}) bucket += '?' + options.map{|k, v| "#{k.to_s}=#{s3_link_escape(v)}"}.join('&') unless options.right_blank? generate_link('GET', headers.merge(:url=>bucket), expires) rescue on_exception end # Generates link for 'PutObject'. # # s3.put_link('my_awesome_bucket',key, object) #=> url string # def put_link(bucket, key, data=nil, expires=nil, headers={}) generate_link('PUT', headers.merge(:url=>"#{bucket}/#{s3_link_escape(key)}", :data=>data), expires) rescue on_exception end # Generates link for 'GetObject'. # # if a bucket comply with virtual hosting naming then retuns a link with the # bucket as a part of host name: # # s3.get_link('my-awesome-bucket',key) #=> https://my-awesome-bucket.s3.amazonaws.com:443/asia%2Fcustomers?Signature=nh7... # # otherwise returns an old style link (the bucket is a part of path): # # s3.get_link('my_awesome_bucket',key) #=> https://s3.amazonaws.com:443/my_awesome_bucket/asia%2Fcustomers?Signature=QAO... # # see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/VirtualHosting.html # # To specify +response+-* parameters, define them in the response_params hash: # # s3.get_link('my_awesome_bucket',key,nil,{},{ "response-content-disposition" => "attachment; filename=caf�.png", "response-content-type" => "image/png"}) # # #=> https://s3.amazonaws.com:443/my_awesome_bucket/asia%2Fcustomers?response-content-disposition=attachment%3B%20filename%3Dcaf%25C3%25A9.png&response-content-type=image%2Fpng&Signature=wio... # def get_link(bucket, key, expires=nil, headers={}, response_params={}) if response_params.size > 0 response_params = '?' + response_params.map { |k, v| "#{k}=#{s3_link_escape(v)}" }.join('&') else response_params = '' end generate_link('GET', headers.merge(:url=>"#{bucket}/#{s3_link_escape(key)}#{response_params}"), expires) rescue on_exception end # Generates link for 'HeadObject'. # # s3.head_link('my_awesome_bucket',key) #=> url string # def head_link(bucket, key, expires=nil, headers={}) generate_link('HEAD', headers.merge(:url=>"#{bucket}/#{s3_link_escape(key)}"), expires) rescue on_exception end # Generates link for 'DeleteObject'. # # s3.delete_link('my_awesome_bucket',key) #=> url string # def delete_link(bucket, key, expires=nil, headers={}) generate_link('DELETE', headers.merge(:url=>"#{bucket}/#{s3_link_escape(key)}"), expires) rescue on_exception end # Generates link for 'GetACL'. # # s3.get_acl_link('my_awesome_bucket',key) #=> url string # def get_acl_link(bucket, key='', headers={}) return generate_link('GET', headers.merge(:url=>"#{bucket}/#{s3_link_escape(key)}?acl")) rescue on_exception end # Generates link for 'PutACL'. # # s3.put_acl_link('my_awesome_bucket',key) #=> url string # def put_acl_link(bucket, key='', headers={}) return generate_link('PUT', headers.merge(:url=>"#{bucket}/#{s3_link_escape(key)}?acl")) rescue on_exception end # Generates link for 'GetBucketACL'. # # s3.get_acl_link('my_awesome_bucket',key) #=> url string # def get_bucket_acl_link(bucket, headers={}) return get_acl_link(bucket, '', headers) rescue on_exception end # Generates link for 'PutBucketACL'. # # s3.put_acl_link('my_awesome_bucket',key) #=> url string # def put_bucket_acl_link(bucket, acl_xml_doc, headers={}) return put_acl_link(bucket, '', acl_xml_doc, headers) rescue on_exception end class S3DeleteMultipleParser < RightAWSParser # :nodoc: def reset @result = [] end def tagstart(name, attributes) @error = {} if name == 'Error' end def tagend(name) case name when 'Key' then @error[:key] = @text when 'Code' then @error[:code] = @text when 'Message' then @error[:message] = @text when 'Error' then @result << @error end end end #----------------------------------------------------------------- # PARSERS: #----------------------------------------------------------------- class S3ListAllMyBucketsParser < RightAWSParser # :nodoc: def reset @result = [] @owner = {} end def tagstart(name, attributes) @current_bucket = {} if name == 'Bucket' end def tagend(name) case name when 'ID' then @owner[:owner_id] = @text when 'DisplayName' then @owner[:owner_display_name] = @text when 'Name' then @current_bucket[:name] = @text when 'CreationDate'then @current_bucket[:creation_date] = @text when 'Bucket' then @result << @current_bucket.merge(@owner) end end end class S3ListBucketParser < RightAWSParser # :nodoc: def reset @result = [] @service = {} @current_key = {} end def tagstart(name, attributes) @current_key = {} if name == 'Contents' end def tagend(name) case name # service info when 'Name' then @service['name'] = @text when 'Prefix' then @service['prefix'] = @text when 'Marker' then @service['marker'] = @text when 'MaxKeys' then @service['max-keys'] = @text when 'Delimiter' then @service['delimiter'] = @text when 'IsTruncated' then @service['is_truncated'] = (@text =~ /false/ ? false : true) # key data when 'Key' then @current_key[:key] = @text when 'LastModified'then @current_key[:last_modified] = @text when 'ETag' then @current_key[:e_tag] = @text when 'Size' then @current_key[:size] = @text.to_i when 'StorageClass'then @current_key[:storage_class] = @text when 'ID' then @current_key[:owner_id] = @text when 'DisplayName' then @current_key[:owner_display_name] = @text when 'Contents' @current_key[:service] = @service @result << @current_key end end end class S3ImprovedListBucketParser < RightAWSParser # :nodoc: def reset @result = {} @result[:contents] = [] @result[:common_prefixes] = [] @contents = [] @current_key = {} @common_prefixes = [] @in_common_prefixes = false end def tagstart(name, attributes) @current_key = {} if name == 'Contents' @in_common_prefixes = true if name == 'CommonPrefixes' end def tagend(name) case name # service info when 'Name' then @result[:name] = @text # Amazon uses the same tag for the search prefix and for the entries # in common prefix...so use our simple flag to see which element # we are parsing when 'Prefix' then @in_common_prefixes ? @common_prefixes << @text : @result[:prefix] = @text when 'Marker' then @result[:marker] = @text when 'MaxKeys' then @result[:max_keys] = @text when 'Delimiter' then @result[:delimiter] = @text when 'IsTruncated' then @result[:is_truncated] = (@text =~ /false/ ? false : true) when 'NextMarker' then @result[:next_marker] = @text # key data when 'Key' then @current_key[:key] = @text when 'LastModified'then @current_key[:last_modified] = @text when 'ETag' then @current_key[:e_tag] = @text when 'Size' then @current_key[:size] = @text.to_i when 'StorageClass'then @current_key[:storage_class] = @text when 'ID' then @current_key[:owner_id] = @text when 'DisplayName' then @current_key[:owner_display_name] = @text when 'Contents' then @result[:contents] << @current_key # Common Prefix stuff when 'CommonPrefixes' @result[:common_prefixes] = @common_prefixes @in_common_prefixes = false end end end class S3BucketLocationParser < RightAWSParser # :nodoc: def reset @result = '' end def tagend(name) @result = @text if name == 'LocationConstraint' end end class S3AclParser < RightAWSParser # :nodoc: def reset @result = {:grantees=>[], :owner=>{}} @current_grantee = {} end def tagstart(name, attributes) @current_grantee = { :attributes => attributes } if name=='Grantee' end def tagend(name) case name # service info when 'ID' if @xmlpath == 'AccessControlPolicy/Owner' @result[:owner][:id] = @text else @current_grantee[:id] = @text end when 'DisplayName' if @xmlpath == 'AccessControlPolicy/Owner' @result[:owner][:display_name] = @text else @current_grantee[:display_name] = @text end when 'URI' @current_grantee[:uri] = @text when 'Permission' @current_grantee[:permissions] = @text when 'Grant' @result[:grantees] << @current_grantee end end end class S3LoggingParser < RightAWSParser # :nodoc: def reset @result = {:enabled => false, :targetbucket => '', :targetprefix => ''} @current_grantee = {} end def tagend(name) case name # service info when 'TargetBucket' if @xmlpath == 'BucketLoggingStatus/LoggingEnabled' @result[:targetbucket] = @text @result[:enabled] = true end when 'TargetPrefix' if @xmlpath == 'BucketLoggingStatus/LoggingEnabled' @result[:targetprefix] = @text @result[:enabled] = true end end end end class S3CopyParser < RightAWSParser # :nodoc: def reset @result = {} end def tagend(name) case name when 'LastModified' then @result[:last_modified] = @text when 'ETag' then @result[:e_tag] = @text end end end #----------------------------------------------------------------- # PARSERS: Non XML #----------------------------------------------------------------- class S3HttpResponseParser # :nodoc: attr_reader :result def parse(response) @result = response end def headers_to_string(headers) result = {} headers.each do |key, value| value = value.first if value.is_a?(Array) && value.size<2 result[key] = value end result end end class S3HttpResponseBodyParser < S3HttpResponseParser # :nodoc: def parse(response) @result = { :object => response.body, :headers => headers_to_string(response.to_hash) } end end class S3HttpResponseHeadParser < S3HttpResponseParser # :nodoc: def parse(response) @result = headers_to_string(response.to_hash) end end end end ================================================ FILE: lib/sdb/active_sdb.rb ================================================ # # Copyright (c) 2008 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws # = RightAws::ActiveSdb -- RightScale SDB interface (alpha release) # The RightAws::ActiveSdb class provides a complete interface to Amazon's Simple # Database Service. # # ActiveSdb is in alpha and does not load by default with the rest of RightAws. You must use an additional require statement to load the ActiveSdb class. For example: # # require 'right_aws' # require 'sdb/active_sdb' # # Simple ActiveSdb usage example: # # class Client < RightAws::ActiveSdb::Base # end # # # connect to SDB # RightAws::ActiveSdb.establish_connection # # # create domain # Client.create_domain # # # create initial DB # Client.create 'name' => 'Bush', 'country' => 'USA', 'gender' => 'male', 'expiration' => '2009', 'post' => 'president' # Client.create 'name' => 'Putin', 'country' => 'Russia', 'gender' => 'male', 'expiration' => '2008', 'post' => 'president' # Client.create 'name' => 'Medvedev', 'country' => 'Russia', 'gender' => 'male', 'expiration' => '2012', 'post' => 'president' # Client.create 'name' => 'Mary', 'country' => 'USA', 'gender' => 'female', 'hobby' => ['patchwork', 'bundle jumping'] # Client.create 'name' => 'Mary', 'country' => 'Russia', 'gender' => 'female', 'hobby' => ['flowers', 'cats', 'cooking'] # sandy_id = Client.create('name' => 'Sandy', 'country' => 'Russia', 'gender' => 'female', 'hobby' => ['flowers', 'cats', 'cooking']).id # # # find all Bushes in USA # Client.find(:all, :conditions => ["['name'=?] intersection ['country'=?]",'Bush','USA']).each do |client| # client.reload # puts client.attributes.inspect # end # # # find all Maries through the world # Client.find_all_by_name_and_gender('Mary','female').each do |mary| # mary.reload # puts "#{mary[:name]}, gender: #{mary[:gender]}, hobbies: #{mary[:hobby].join(',')}" # end # # # find new russian president # medvedev = Client.find_by_post_and_country_and_expiration('president','Russia','2012') # if medvedev # medvedev.reload # medvedev.save_attributes('age' => '42', 'hobby' => 'Gazprom') # end # # # retire old president # Client.find_by_name('Putin').delete # # # Sandy disappointed in 'cooking' and decided to hide her 'gender' and 'country' () # sandy = Client.find(sandy_id) # sandy.reload # sandy.delete_values('hobby' => ['cooking'] ) # sandy.delete_attributes('country', 'gender') # # # remove domain # Client.delete_domain # # # Dynamic attribute accessors # # class KdClient < RightAws::ActiveSdb::Base # end # # client = KdClient.select(:all, :order => 'expiration').first # pp client.attributes #=> # {"name"=>["Putin"], # "post"=>["president"], # "country"=>["Russia"], # "expiration"=>["2008"], # "id"=>"376d2e00-75b0-11dd-9557-001bfc466dd7", # "gender"=>["male"]} # # pp client.name #=> ["Putin"] # pp client.country #=> ["Russia"] # pp client.post #=> ["president"] # # # Columns and simple typecasting # # class Person < RightAws::ActiveSdb::Base # columns do # name # email # score :Integer # is_active :Boolean # registered_at :DateTime # created_at :DateTime, :default => lambda{ Time.now } # end # end # Person::create( :name => 'Yetta E. Andrews', :email => 'nulla.facilisis@metus.com', :score => 100, :is_active => true, :registered_at => Time.local(2000, 1, 1) ) # # person = Person.find_by_email 'nulla.facilisis@metus.com' # person.reload # # pp person.attributes #=> # {"name"=>["Yetta E. Andrews"], # "created_at"=>["2010-04-02T20:51:58+0400"], # "id"=>"0ee24946-3e60-11df-9d4c-0025b37efad0", # "registered_at"=>["2000-01-01T00:00:00+0300"], # "is_active"=>["T"], # "score"=>["100"], # "email"=>["nulla.facilisis@metus.com"]} # pp person.name #=> "Yetta E. Andrews" # pp person.name.class #=> String # pp person.registered_at.to_s #=> "2000-01-01T00:00:00+03:00" # pp person.registered_at.class #=> DateTime # pp person.is_active #=> true # pp person.is_active.class #=> TrueClass # pp person.score #=> 100 # pp person.score.class #=> Fixnum # pp person.created_at.to_s #=> "2010-04-02T20:51:58+04:00" # class ActiveSdb module ActiveSdbConnect def connection @connection || raise(ActiveSdbError.new('Connection to SDB is not established')) end # Create a new handle to an Sdb account. All handles share the same per process or per thread # HTTP connection to Amazon Sdb. Each handle is for a specific account. # The +params+ are passed through as-is to RightAws::SdbInterface.new # Params: # { :server => 'sdb.amazonaws.com' # Amazon service host: 'sdb.amazonaws.com'(default) # :port => 443 # Amazon service port: 80 or 443(default) # :protocol => 'https' # Amazon service protocol: 'http' or 'https'(default) # :signature_version => '0' # The signature version : '0' or '1'(default) # :logger => Logger Object # Logger instance: logs to STDOUT if omitted # :nil_representation => 'mynil'} # interpret Ruby nil as this string value; i.e. use this string in SDB to represent Ruby nils (default is the string 'nil') def establish_connection(aws_access_key_id=nil, aws_secret_access_key=nil, params={}) @connection = RightAws::SdbInterface.new(aws_access_key_id, aws_secret_access_key, params) end end class ActiveSdbError < RuntimeError ; end class << self include ActiveSdbConnect # Retreive a list of domains. # # put RightAws::ActiveSdb.domains #=> ['co-workers','family','friends','clients'] # def domains connection.list_domains[:domains] end # Create new domain. # Raises no errors if the domain already exists. # # RightAws::ActiveSdb.create_domain('alpha') #=> {:request_id=>"6fc652a0-0000-41d5-91f4-3ed390a3d3b2", :box_usage=>"0.0055590278"} # def create_domain(domain_name) connection.create_domain(domain_name) end # Remove domain from SDB. # Raises no errors if the domain does not exist. # # RightAws::ActiveSdb.create_domain('alpha') #=> {:request_id=>"6fc652a0-0000-41c6-91f4-3ed390a3d3b2", :box_usage=>"0.0055590001"} # def delete_domain(domain_name) connection.delete_domain(domain_name) end end class Base class << self include ActiveSdbConnect # next_token value returned by last find: is useful to continue finding attr_accessor :next_token # Returns a RightAws::SdbInterface object # # class A < RightAws::ActiveSdb::Base # end # # class B < RightAws::ActiveSdb::Base # end # # class C < RightAws::ActiveSdb::Base # end # # RightAws::ActiveSdb.establish_connection 'key_id_1', 'secret_key_1' # # C.establish_connection 'key_id_2', 'secret_key_2' # # # A and B uses the default connection, C - uses its own # puts A.connection #=> # # puts B.connection #=> # # puts C.connection #=> # # def connection @connection || ActiveSdb::connection end @domain = nil # Current domain name. # # # if 'ActiveSupport' is not loaded then class name converted to downcase # class Client < RightAws::ActiveSdb::Base # end # puts Client.domain #=> 'client' # # # if 'ActiveSupport' is loaded then class name being tableized # require 'activesupport' # class Client < RightAws::ActiveSdb::Base # end # puts Client.domain #=> 'clients' # # # Explicit domain name definition # class Client < RightAws::ActiveSdb::Base # set_domain_name :foreign_clients # end # puts Client.domain #=> 'foreign_clients' # def domain unless @domain if defined? ActiveSupport::CoreExtensions::String::Inflections @domain = name.tableize else @domain = name.downcase end end @domain end # Change the default domain name to user defined. # # class Client < RightAws::ActiveSdb::Base # set_domain_name :foreign_clients # end # def set_domain_name(domain) @domain = domain.to_s end # Create domain at SDB. # Raises no errors if the domain already exists. # # class Client < RightAws::ActiveSdb::Base # end # Client.create_domain #=> {:request_id=>"6fc652a0-0000-41d5-91f4-3ed390a3d3b2", :box_usage=>"0.0055590278"} # def create_domain connection.create_domain(domain) end # Remove domain from SDB. # Raises no errors if the domain does not exist. # # class Client < RightAws::ActiveSdb::Base # end # Client.delete_domain #=> {:request_id=>"e14d90d3-0000-4898-9995-0de28cdda270", :box_usage=>"0.0055590278"} # def delete_domain connection.delete_domain(domain) end def columns(&block) @columns ||= ColumnSet.new @columns.instance_eval(&block) if block @columns end def column?(col_name) columns.include?(col_name) end def type_of(col_name) columns.type_of(col_name) end def serialize(attribute, value) s = serialization_for_type(type_of(attribute)) s ? s.serialize(value) : value.to_s end def deserialize(attribute, value) s = serialization_for_type(type_of(attribute)) s ? s.deserialize(value) : value end # Perform a find request. # # Single record: # # Client.find(:first) # Client.find(:first, :conditions=> [ "['name'=?] intersection ['wife'=?]", "Jon", "Sandy"]) # # Bunch of records: # # Client.find(:all) # Client.find(:all, :limit => 10) # Client.find(:all, :conditions=> [ "['name'=?] intersection ['girlfriend'=?]", "Jon", "Judy"]) # Client.find(:all, :conditions=> [ "['name'=?]", "Sandy"], :limit => 3) # # Records by ids: # # Client.find('1') # Client.find('1234987b4583475347523948') # Client.find('1','2','3','4', :conditions=> [ "['toys'=?]", "beer"]) # # Find helpers: RightAws::ActiveSdb::Base.find_by_... and RightAws::ActiveSdb::Base.find_all_by_... # # Client.find_by_name('Matias Rust') # Client.find_by_name_and_city('Putin','Moscow') # Client.find_by_name_and_city_and_post('Medvedev','Moscow','president') # # Client.find_all_by_author('G.Bush jr') # Client.find_all_by_age_and_gender_and_ethnicity('34','male','russian') # Client.find_all_by_gender_and_country('male', 'Russia', :auto_load => true, :order => 'name desc') # # Returned records have to be +reloaded+ to access their attributes. # # item = Client.find_by_name('Cat') #=> #"2937601a-e45d-11dc-a75f-001bfc466dd7"}, @new_record=false> # item.reload #=> #"2937601a-e45d-11dc-a75f-001bfc466dd7", "name"=>["Cat"], "toys"=>["Jons socks", "clew", "mice"]}, @new_record=false> # # Continue listing: # # initial listing # Client.find(:all, :limit => 10) # # continue listing # begin # Client.find(:all, :limit => 10, :next_token => Client.next_token) # end while Client.next_token # # Sort oder: # Client.find(:all, :order => 'gender') # Client.find(:all, :order => 'name desc') # # Attributes auto load (be carefull - this may take lot of time for a huge bunch of records): # Client.find(:first) #=> #"2937601a-e45d-11dc-a75f-001bfc466dd7"}, @new_record=false> # Client.find(:first, :auto_load => true) #=> #"2937601a-e45d-11dc-a75f-001bfc466dd7", "name"=>["Cat"], "toys"=>["Jons socks", "clew", "mice"]}, @new_record=false> # # see http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?UsingQuery.html # def find(*args) options = args.last.is_a?(Hash) ? args.pop : {} case args.first when :all then find_every options when :first then find_initial options else find_from_ids args, options end end # Perform a SQL-like select request. # # Single record: # # Client.select(:first) # Client.select(:first, :conditions=> [ "name=? AND wife=?", "Jon", "Sandy"]) # Client.select(:first, :conditions=> { :name=>"Jon", :wife=>"Sandy" }, :select => :girfriends) # # Bunch of records: # # Client.select(:all) # Client.select(:all, :limit => 10) # Client.select(:all, :conditions=> [ "name=? AND 'girlfriend'=?", "Jon", "Judy"]) # Client.select(:all, :conditions=> { :name=>"Sandy" }, :limit => 3) # # Records by ids: # # Client.select('1') # Client.select('1234987b4583475347523948') # Client.select('1','2','3','4', :conditions=> ["toys=?", "beer"]) # # Find helpers: RightAws::ActiveSdb::Base.select_by_... and RightAws::ActiveSdb::Base.select_all_by_... # # Client.select_by_name('Matias Rust') # Client.select_by_name_and_city('Putin','Moscow') # Client.select_by_name_and_city_and_post('Medvedev','Moscow','president') # # Client.select_all_by_author('G.Bush jr') # Client.select_all_by_age_and_gender_and_ethnicity('34','male','russian') # Client.select_all_by_gender_and_country('male', 'Russia', :order => 'name') # # Continue listing: # # # initial listing # Client.select(:all, :limit => 10) # # continue listing # begin # Client.select(:all, :limit => 10, :next_token => Client.next_token) # end while Client.next_token # # Sort oder: # If :order=>'attribute' option is specified then result response (ordered by 'attribute') will contain only items where attribute is defined (is not null). # # Client.select(:all) # returns all records # Client.select(:all, :order => 'gender') # returns all records ordered by gender where gender attribute exists # Client.select(:all, :order => 'name desc') # returns all records ordered by name in desc order where name attribute exists # # see http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?UsingSelect.html # def select(*args) options = args.last.is_a?(Hash) ? args.pop : {} case args.first when :all then sql_select(options) when :first then sql_select(options.merge(:limit => 1)).first else select_from_ids args, options end end def generate_id # :nodoc: AwsUtils::generate_unique_token end protected # Select def select_from_ids(args, options) # :nodoc: cond = [] # detect amount of records requested bunch_of_records_requested = args.size > 1 || args.first.is_a?(Array) # flatten ids args = Array(args).flatten args.each { |id| cond << "id=#{self.connection.escape(id)}" } ids_cond = "(#{cond.join(' OR ')})" # user defined :conditions to string (if it was defined) options[:conditions] = build_conditions(options[:conditions]) # join ids condition and user defined conditions options[:conditions] = options[:conditions].right_blank? ? ids_cond : "(#{options[:conditions]}) AND #{ids_cond}" result = sql_select(options) # if one record was requested then return it unless bunch_of_records_requested record = result.first # railse if nothing was found raise ActiveSdbError.new("Couldn't find #{name} with ID #{args}") unless record record else # if a bunch of records was requested then return check that we found all of them # and return as an array unless args.size == result.size id_list = args.map{|i| "'#{i}'"}.join(',') raise ActiveSdbError.new("Couldn't find all #{name} with IDs (#{id_list}) (found #{result.size} results, but was looking for #{args.size})") else result end end end def sql_select(options) # :nodoc: @next_token = options[:next_token] select_expression = build_select(options) # request items query_result = self.connection.select(select_expression, @next_token) @next_token = query_result[:next_token] items = query_result[:items].map do |hash| id, attributes = hash.shift new_item = self.new( attributes.merge({ 'id' => id })) new_item.mark_as_old new_item end items end # select_by helpers def select_all_by_(format_str, args, options) # :nodoc: fields = format_str.to_s.sub(/^select_(all_)?by_/,'').split('_and_') conditions = fields.map { |field| "#{field}=?" }.join(' AND ') options[:conditions] = [conditions, *args] select(:all, options) end def select_by_(format_str, args, options) # :nodoc: options[:limit] = 1 select_all_by_(format_str, args, options).first end # Query # Returns an array of query attributes. # Query_expression must be a well formated SDB query string: # query_attributes("['title' starts-with 'O\\'Reily'] intersection ['year' = '2007']") #=> ["title", "year"] def query_attributes(query_expression) # :nodoc: attrs = [] array = query_expression.scan(/['"](.*?[^\\])['"]/).flatten until array.empty? do attrs << array.shift # skip it's value array.shift # end attrs end # Returns an array of [attribute_name, 'asc'|'desc'] def sort_options(sort_string) sort_string[/['"]?(\w+)['"]? *(asc|desc)?/i] [$1, ($2 || 'asc')] end # Perform a query request. # # Options # :query_expression nil | string | array # :max_number_of_items nil | integer # :next_token nil | string # :sort_option nil | string "name desc|asc" # def query(options) # :nodoc: @next_token = options[:next_token] query_expression = build_conditions(options[:query_expression]) # add sort_options to the query_expression if options[:sort_option] sort_by, sort_order = sort_options(options[:sort_option]) sort_query_expression = "['#{sort_by}' starts-with '']" sort_by_expression = " sort '#{sort_by}' #{sort_order}" # make query_expression to be a string (it may be null) query_expression = query_expression.to_s # quote from Amazon: # The sort attribute must be present in at least one of the predicates of the query expression. if query_expression.right_blank? query_expression = sort_query_expression elsif !query_attributes(query_expression).include?(sort_by) query_expression += " intersection #{sort_query_expression}" end query_expression += sort_by_expression end # request items query_result = self.connection.query(domain, query_expression, options[:max_number_of_items], @next_token) @next_token = query_result[:next_token] items = query_result[:items].map do |name| new_item = self.new('id' => name) new_item.mark_as_old reload_if_exists(record) if options[:auto_load] new_item end items end # reload a record unless it is nil def reload_if_exists(record) # :nodoc: record && record.reload end def reload_all_records(*list) # :nodoc: list.flatten.each { |record| reload_if_exists(record) } end def find_every(options) # :nodoc: records = query( :query_expression => options[:conditions], :max_number_of_items => options[:limit], :next_token => options[:next_token], :sort_option => options[:sort] || options[:order] ) options[:auto_load] ? reload_all_records(records) : records end def find_initial(options) # :nodoc: options[:limit] = 1 record = find_every(options).first options[:auto_load] ? reload_all_records(record).first : record end def find_from_ids(args, options) # :nodoc: cond = [] # detect amount of records requested bunch_of_records_requested = args.size > 1 || args.first.is_a?(Array) # flatten ids args = Array(args).flatten args.each { |id| cond << "'id'=#{self.connection.escape(id)}" } ids_cond = "[#{cond.join(' OR ')}]" # user defined :conditions to string (if it was defined) options[:conditions] = build_conditions(options[:conditions]) # join ids condition and user defined conditions options[:conditions] = options[:conditions].right_blank? ? ids_cond : "#{options[:conditions]} intersection #{ids_cond}" result = find_every(options) # if one record was requested then return it unless bunch_of_records_requested record = result.first # railse if nothing was found raise ActiveSdbError.new("Couldn't find #{name} with ID #{args}") unless record options[:auto_load] ? reload_all_records(record).first : record else # if a bunch of records was requested then return check that we found all of them # and return as an array unless args.size == result.size id_list = args.map{|i| "'#{i}'"}.join(',') raise ActiveSdbError.new("Couldn't find all #{name} with IDs (#{id_list}) (found #{result.size} results, but was looking for #{args.size})") else options[:auto_load] ? reload_all_records(result) : result end end end # find_by helpers def find_all_by_(format_str, args, options) # :nodoc: fields = format_str.to_s.sub(/^find_(all_)?by_/,'').split('_and_') conditions = fields.map { |field| "['#{field}'=?]" }.join(' intersection ') options[:conditions] = [conditions, *args] find(:all, options) end def find_by_(format_str, args, options) # :nodoc: options[:limit] = 1 find_all_by_(format_str, args, options).first end # Misc def method_missing(method, *args) # :nodoc: if method.to_s[/^(find_all_by_|find_by_|select_all_by_|select_by_)/] options = args.last.is_a?(Hash) ? args.pop : {} __send__($1, method, args, options) else super(method, *args) end end def build_select(options) # :nodoc: select = options[:select] || '*' from = options[:from] || domain conditions = options[:conditions] ? " WHERE #{build_conditions(options[:conditions])}" : '' order = options[:order] ? " ORDER BY #{options[:order]}" : '' limit = options[:limit] ? " LIMIT #{options[:limit]}" : '' # mix sort by argument (it must present in response) unless order.right_blank? sort_by, sort_order = sort_options(options[:order]) conditions << (conditions.right_blank? ? " WHERE " : " AND ") << "(#{sort_by} IS NOT NULL)" end "SELECT #{select} FROM #{from}#{conditions}#{order}#{limit}" end def build_conditions(conditions) # :nodoc: case when conditions.is_a?(Array) then connection.query_expression_from_array(conditions) when conditions.is_a?(Hash) then connection.query_expression_from_hash(conditions) else conditions end end def serialization_for_type(type) @serializations ||= {} unless @serializations.has_key? type @serializations[type] = ::RightAws::ActiveSdb.const_get("#{type}Serialization") rescue false end @serializations[type] end end public # instance attributes attr_accessor :attributes # item name attr_accessor :id # Create new Item instance. # +attrs+ is a hash: { attribute1 => values1, ..., attributeN => valuesN }. # # item = Client.new('name' => 'Jon', 'toys' => ['girls', 'beer', 'pub']) # puts item.inspect #=> #["Jon"], "toys"=>["girls", "beer", "pub"]}> # item.save #=> {"name"=>["Jon"], "id"=>"c03edb7e-e45c-11dc-bede-001bfc466dd7", "toys"=>["girls", "beer", "pub"]} # puts item.inspect #=> #["Jon"], "id"=>"c03edb7e-e45c-11dc-bede-001bfc466dd7", "toys"=>["girls", "beer", "pub"]}> # def initialize(attrs={}) @attributes = uniq_values(attrs) @new_record = true end # Create and save new Item instance. # +Attributes+ is a hash: { attribute1 => values1, ..., attributeN => valuesN }. # # item = Client.create('name' => 'Cat', 'toys' => ['Jons socks', 'mice', 'clew']) # puts item.inspect #=> #["Cat"], "id"=>"2937601a-e45d-11dc-a75f-001bfc466dd7", "toys"=>["Jons socks", "mice", "clew"]}> # def self.create(attributes={}) item = self.new(attributes) item.save item end # Returns an item id. Same as: item['id'] or item.attributes['id'] def id @attributes['id'] end # Sets an item id. def id=(id) @attributes['id'] = id.to_s end # Returns a hash of all the attributes. # # puts item.attributes.inspect #=> {"name"=>["Cat"], "id"=>"2937601a-e45d-11dc-a75f-001bfc466dd7", "toys"=>["Jons socks", "clew", "mice"]} # def attributes @attributes.dup end # Allows one to set all the attributes at once by passing in a hash with keys matching the attribute names. # if '+id+' attribute is not set in new attributes has then it being derived from old attributes. # # puts item.attributes.inspect #=> {"name"=>["Cat"], "id"=>"2937601a-e45d-11dc-a75f-001bfc466dd7", "toys"=>["Jons socks", "clew", "mice"]} # # set new attributes ('id' is missed) # item.attributes = { 'name'=>'Dog', 'toys'=>['bones','cats'] } # puts item.attributes.inspect #=> {"name"=>["Dog"], "id"=>"2937601a-e45d-11dc-a75f-001bfc466dd7", "toys"=>["bones", "cats"]} # # set new attributes ('id' is set) # item.attributes = { 'id' => 'blah-blah', 'name'=>'Birds', 'toys'=>['seeds','dogs tail'] } # puts item.attributes.inspect #=> {"name"=>["Birds"], "id"=>"blah-blah", "toys"=>["seeds", "dogs tail"]} # def attributes=(attrs) old_id = @attributes['id'] @attributes = uniq_values(attrs) @attributes['id'] = old_id if @attributes['id'].right_blank? && !old_id.right_blank? self.attributes end def columns self.class.columns end def connection self.class.connection end # Item domain name. def domain self.class.domain end # Returns the values of the attribute identified by +attribute+. # # puts item['Cat'].inspect #=> ["Jons socks", "clew", "mice"] # def [](attribute) raw = @attributes[attribute.to_s] self.class.column?(attribute) && raw ? self.class.deserialize(attribute, raw.first) : raw end # Updates the attribute identified by +attribute+ with the specified +values+. # # puts item['Cat'].inspect #=> ["Jons socks", "clew", "mice"] # item['Cat'] = ["Whiskas", "chicken"] # puts item['Cat'].inspect #=> ["Whiskas", "chicken"] # def []=(attribute, values) attribute = attribute.to_s @attributes[attribute] = case when attribute == 'id' values.to_s when self.class.column?(attribute) self.class.serialize(attribute, values) else Array(values).uniq end end # Reload attributes from SDB. Replaces in-memory attributes. # # item = Client.find_by_name('Cat') #=> #"2937601a-e45d-11dc-a75f-001bfc466dd7"}, @new_record=false> # item.reload #=> #"2937601a-e45d-11dc-a75f-001bfc466dd7", "name"=>["Cat"], "toys"=>["Jons socks", "clew", "mice"]}, @new_record=false> # def reload raise_on_id_absence old_id = id attrs = connection.get_attributes(domain, id)[:attributes] @attributes = {} unless attrs.right_blank? attrs.each { |attribute, values| @attributes[attribute] = values } @attributes['id'] = old_id end mark_as_old @attributes end # Reload a set of attributes from SDB. Adds the loaded list to in-memory data. # +attrs_list+ is an array or comma separated list of attributes names. # Returns a hash of loaded attributes. # # This is not the best method to get a bunch of attributes because # a web service call is being performed for every attribute. # # item = Client.find_by_name('Cat') # item.reload_attributes('toys', 'name') #=> {"name"=>["Cat"], "toys"=>["Jons socks", "clew", "mice"]} # def reload_attributes(*attrs_list) raise_on_id_absence attrs_list = attrs_list.flatten.map{ |attribute| attribute.to_s } attrs_list.delete('id') result = {} attrs_list.flatten.uniq.each do |attribute| attribute = attribute.to_s values = connection.get_attributes(domain, id, attribute)[:attributes][attribute] unless values.right_blank? @attributes[attribute] = result[attribute] = values else @attributes.delete(attribute) end end mark_as_old result end # Stores in-memory attributes to SDB. # Adds the attributes values to already stored at SDB. # Returns a hash of stored attributes. # # sandy = Client.new(:name => 'Sandy') #=> #["Sandy"]}, @new_record=true> # sandy['toys'] = 'boys' # sandy.put # sandy['toys'] = 'patchwork' # sandy.put # sandy['toys'] = 'kids' # sandy.put # puts sandy.attributes.inspect #=> {"name"=>["Sandy"], "id"=>"b2832ce2-e461-11dc-b13c-001bfc466dd7", "toys"=>["kids"]} # sandy.reload #=> {"name"=>["Sandy"], "id"=>"b2832ce2-e461-11dc-b13c-001bfc466dd7", "toys"=>["boys", "kids", "patchwork"]} # # compare to +save+ method def put @attributes = uniq_values(@attributes) prepare_for_update attrs = @attributes.dup attrs.delete('id') connection.put_attributes(domain, id, attrs) unless attrs.right_blank? connection.put_attributes(domain, id, { 'id' => id }, :replace) mark_as_old @attributes end # Stores specified attributes. # +attrs+ is a hash: { attribute1 => values1, ..., attributeN => valuesN }. # Returns a hash of saved attributes. # # see to +put+ method def put_attributes(attrs) attrs = uniq_values(attrs) prepare_for_update # if 'id' is present in attrs hash: # replace internal 'id' attribute and remove it from the attributes to be sent @attributes['id'] = attrs['id'] unless attrs['id'].right_blank? attrs.delete('id') # add new values to all attributes from list connection.put_attributes(domain, id, attrs) unless attrs.right_blank? connection.put_attributes(domain, id, { 'id' => id }, :replace) attrs.each do |attribute, values| @attributes[attribute] ||= [] @attributes[attribute] += values @attributes[attribute].uniq! end mark_as_old attributes end # Store in-memory attributes to SDB. # Replaces the attributes values already stored at SDB by in-memory data. # Returns a hash of stored attributes. # # sandy = Client.new(:name => 'Sandy') #=> #["Sandy"]}, @new_record=true> # sandy['toys'] = 'boys' # sandy.put # sandy['toys'] = 'patchwork' # sandy.put # sandy['toys'] = 'kids' # sandy.put # puts sandy.attributes.inspect #=> {"name"=>["Sandy"], "id"=>"b2832ce2-e461-11dc-b13c-001bfc466dd7", "toys"=>["kids"]} # sandy.reload #=> {"name"=>["Sandy"], "id"=>"b2832ce2-e461-11dc-b13c-001bfc466dd7", "toys"=>["kids"]} # # compare to +put+ method def save @attributes = uniq_values(@attributes) prepare_for_update connection.put_attributes(domain, id, @attributes, :replace) mark_as_old @attributes end # Replaces the attributes at SDB by the given values. # +Attrs+ is a hash: { attribute1 => values1, ..., attributeN => valuesN }. # The other in-memory attributes are not being saved. # Returns a hash of stored attributes. # # see +save+ method def save_attributes(attrs) prepare_for_update attrs = uniq_values(attrs) # if 'id' is present in attrs hash then replace internal 'id' attribute unless attrs['id'].right_blank? @attributes['id'] = attrs['id'] else attrs['id'] = id end connection.put_attributes(domain, id, attrs, :replace) unless attrs.right_blank? attrs.each { |attribute, values| attrs[attribute] = values } mark_as_old attrs end # Remove specified values from corresponding attributes. # +attrs+ is a hash: { attribute1 => values1, ..., attributeN => valuesN }. # # sandy = Client.find_by_name 'Sandy' # sandy.reload # puts sandy.inspect #=> #["Sandy"], "id"=>"b2832ce2-e461-11dc-b13c-001bfc466dd7", "toys"=>["boys", "kids", "patchwork"]}> # puts sandy.delete_values('toys' => 'patchwork') #=> { 'toys' => ['patchwork'] } # puts sandy.inspect #=> #["Sandy"], "id"=>"b2832ce2-e461-11dc-b13c-001bfc466dd7", "toys"=>["boys", "kids"]}> # def delete_values(attrs) raise_on_id_absence attrs = uniq_values(attrs) attrs.delete('id') unless attrs.right_blank? connection.delete_attributes(domain, id, attrs) attrs.each do |attribute, values| # remove the values from the attribute if @attributes[attribute] @attributes[attribute] -= values else # if the attribute is unknown remove it from a resulting list of fixed attributes attrs.delete(attribute) end end end attrs end # Removes specified attributes from the item. # +attrs_list+ is an array or comma separated list of attributes names. # Returns the list of deleted attributes. # # sandy = Client.find_by_name 'Sandy' # sandy.reload # puts sandy.inspect #=> #["Sandy"], "id"=>"b2832ce2-e461-11dc-b13c-001bfc466dd7", "toys"=>["boys", "kids", "patchwork"}> # puts sandy.delete_attributes('toys') #=> ['toys'] # puts sandy.inspect #=> #["Sandy"], "id"=>"b2832ce2-e461-11dc-b13c-001bfc466dd7"}> # def delete_attributes(*attrs_list) raise_on_id_absence attrs_list = attrs_list.flatten.map{ |attribute| attribute.to_s } attrs_list.delete('id') unless attrs_list.right_blank? connection.delete_attributes(domain, id, attrs_list) attrs_list.each { |attribute| @attributes.delete(attribute) } end attrs_list end # Delete the Item entirely from SDB. # # sandy = Client.find_by_name 'Sandy' # sandy.reload # sandy.inspect #=> #["Sandy"], "id"=>"b2832ce2-e461-11dc-b13c-001bfc466dd7", "toys"=>["boys", "kids", "patchwork"}> # puts sandy.delete # sandy.reload # puts sandy.inspect #=> # # def delete raise_on_id_absence connection.delete_attributes(domain, id) end # Item ID def to_s @id end # Returns true if this object hasn‘t been saved yet. def new_record? @new_record end def mark_as_old # :nodoc: @new_record = false end # support accessing attribute values via method call def method_missing(method_sym, *args) method_name = method_sym.to_s setter = method_name[-1,1] == '=' method_name.chop! if setter if @attributes.has_key?(method_name) || self.class.column?(method_name) setter ? self[method_name] = args.first : self[method_name] else super end end private def raise_on_id_absence raise ActiveSdbError.new('Unknown record id') unless id end def prepare_for_update @attributes['id'] = self.class.generate_id if @attributes['id'].right_blank? columns.all.each do |col_name| self[col_name] ||= columns.default(col_name) end end def uniq_values(attributes=nil) # :nodoc: attrs = {} attributes.each do |attribute, values| attribute = attribute.to_s attrs[attribute] = case when attribute == 'id' values.to_s when self.class.column?(attribute) values.is_a?(String) ? values : self.class.serialize(attribute, values) else Array(values).uniq end attrs.delete(attribute) if values.right_blank? end attrs end end class ColumnSet attr_accessor :columns def initialize @columns = {} end def all @columns.keys end def column(col_name) @columns[col_name.to_s] end alias_method :include?, :column def type_of(col_name) column(col_name) && column(col_name)[:type] end def default(col_name) return nil unless include?(col_name) default = column(col_name)[:default] default.respond_to?(:call) ? default.call : default end def method_missing(method_sym, *args) data_type = args.shift || :String options = args.shift || {} @columns[method_sym.to_s] = options.merge( :type => data_type ) end end class DateTimeSerialization class << self def serialize(date) date.strftime('%Y-%m-%dT%H:%M:%S%z') end def deserialize(string) r = DateTime.parse(string) rescue nil end end end class BooleanSerialization class << self def serialize(boolean) boolean ? 'T' : 'F' end def deserialize(string) string == 'T' end end end class IntegerSerialization class << self def serialize(int) int.to_s end def deserialize(string) string.to_i end end end end end ================================================ FILE: lib/sdb/right_sdb_interface.rb ================================================ # # Copyright (c) 2008 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # require "right_aws" module RightAws class SdbInterface < RightAwsBase include RightAwsBaseInterface DEFAULT_HOST = 'sdb.amazonaws.com' DEFAULT_PORT = 443 DEFAULT_PROTOCOL = 'https' DEFAULT_PATH = '/' API_VERSION = '2009-04-15' DEFAULT_NIL_REPRESENTATION = 'nil' @@bench = AwsBenchmarkingBlock.new def self.bench_xml; @@bench.xml; end def self.bench_sdb; @@bench.service; end attr_reader :last_query_expression # Creates new RightSdb instance. # # Params: # { :server => 'sdb.amazonaws.com' # Amazon service host: 'sdb.amazonaws.com'(default) # :port => 443 # Amazon service port: 80 or 443(default) # :protocol => 'https' # Amazon service protocol: 'http' or 'https'(default) # :signature_version => '0' # The signature version : '0','1 or '2'(default) # :logger => Logger Object # Logger instance: logs to STDOUT if omitted # :nil_representation => 'mynil'} # interpret Ruby nil as this string value; i.e. use this string in SDB to represent Ruby nils (default is the string 'nil') # # Example: # # sdb = RightAws::SdbInterface.new('1E3GDYEOGFJPIT7XXXXXX','hgTHt68JY07JKUY08ftHYtERkjgtfERn57XXXXXX', {:logger => Logger.new('/tmp/x.log')}) #=> # # # see: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/ # def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={}) @nil_rep = params[:nil_representation] ? params[:nil_representation] : DEFAULT_NIL_REPRESENTATION params.delete(:nil_representation) init({ :name => 'SDB', :default_host => ENV['SDB_URL'] ? URI.parse(ENV['SDB_URL']).host : DEFAULT_HOST, :default_port => ENV['SDB_URL'] ? URI.parse(ENV['SDB_URL']).port : DEFAULT_PORT, :default_service => ENV['SDB_URL'] ? URI.parse(ENV['SDB_URL']).path : DEFAULT_PATH, :default_protocol => ENV['SDB_URL'] ? URI.parse(ENV['SDB_URL']).scheme : DEFAULT_PROTOCOL, :default_api_version => ENV['SDB_API_VERSION'] || API_VERSION }, aws_access_key_id || ENV['AWS_ACCESS_KEY_ID'], aws_secret_access_key || ENV['AWS_SECRET_ACCESS_KEY'], params) end #----------------------------------------------------------------- # Requests #----------------------------------------------------------------- def generate_request(action, params={}) #:nodoc: generate_request_impl(:get, action, params ) end # Sends request to Amazon and parses the response # Raises AwsError if any banana happened def request_info(request, parser) #:nodoc: request_info_impl(:sdb_connection, @@bench, request, parser) end # Prepare attributes for putting. # (used by put_attributes) def pack_attributes(items_or_attributes, replace = false, batch = false) #:nodoc: if batch index = 0 items_or_attributes.inject({}){|result, (item_name, attributes)| item_prefix = "Item.#{index}." result["#{item_prefix}ItemName"] = item_name.to_s result.merge!( pack_single_item_attributes(attributes, replace, item_prefix)) index += 1 result } else pack_single_item_attributes(items_or_attributes, replace) end end def pack_single_item_attributes(attributes, replace, prefix = "") result = {} if attributes idx = 0 skip_values = attributes.is_a?(Array) attributes.each do |attribute, values| # set replacement attribute result["#{prefix}Attribute.#{idx}.Replace"] = 'true' if replace # pack Name/Value unless values.nil? # Array(values) does not work here: # - Array('') => [] but we wanna get here [''] [values].flatten.each do |value| result["#{prefix}Attribute.#{idx}.Name"] = attribute result["#{prefix}Attribute.#{idx}.Value"] = ruby_to_sdb(value) unless skip_values idx += 1 end else result["#{prefix}Attribute.#{idx}.Name"] = attribute result["#{prefix}Attribute.#{idx}.Value"] = ruby_to_sdb(nil) unless skip_values idx += 1 end end end result end # Use this helper to manually escape the fields in the query expressions. # To escape the single quotes and backslashes and to wrap the string into the single quotes. # # see: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/SDB_API.html # def escape(value) %Q{'#{value.to_s.gsub(/(['\\])/){ "\\#{$1}" }}'} if value end # Convert a Ruby language value to a SDB value by replacing Ruby nil with the user's chosen string representation of nil. # Non-nil values are unaffected by this filter. def ruby_to_sdb(value) value.nil? ? @nil_rep : value end # Convert a SDB value to a Ruby language value by replacing the user's chosen string representation of nil with Ruby nil. # Values are unaffected by this filter unless they match the nil representation exactly. def sdb_to_ruby(value) value.eql?(@nil_rep) ? nil : value end # Convert select and query_with_attributes responses to a Ruby language values by replacing the user's chosen string representation of nil with Ruby nil. # (This method affects on a passed response value) def select_response_to_ruby(response) #:nodoc: response[:items].each_with_index do |item, idx| item.each do |key, attributes| attributes.each do |name, values| values.collect! { |value| sdb_to_ruby(value) } end end end response end # Create query expression from an array. # (similar to ActiveRecord::Base#find using :conditions => ['query', param1, .., paramN]) # def query_expression_from_array(params) #:nodoc: return '' if params.right_blank? query = params.shift.to_s query.gsub(/(\\)?(\?)/) do if $1 # if escaped '\?' is found - replace it by '?' without backslash "?" else # well, if no backslash precedes '?' then replace it by next param from the list escape(params.shift) end end end def query_expression_from_hash(hash) return '' if hash.right_blank? expression = [] hash.each do |key, value| expression << "#{key}=#{escape(value)}" end expression.join(' AND ') end # Retrieve a list of SDB domains from Amazon. # # Returns a hash: # { :domains => [domain1, ..., domainN], # :next_token => string || nil, # :box_usage => string, # :request_id => string } # # Example: # # sdb = RightAws::SdbInterface.new # sdb.list_domains #=> { :box_usage => "0.0000071759", # :request_id => "976709f9-0111-2345-92cb-9ce90acd0982", # :domains => ["toys", "dolls"]} # # If a block is given, this method yields to it. If the block returns true, list_domains will continue looping the request. If the block returns false, # list_domains will end. # # sdb.list_domains(10) do |result| # list by 10 domains per iteration # puts result.inspect # true # end # # see: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/SDB_API_ListDomains.html # def list_domains(max_number_of_domains = nil, next_token = nil ) request_params = { 'MaxNumberOfDomains' => max_number_of_domains, 'NextToken' => next_token } link = generate_request("ListDomains", request_params) result = request_info(link, QSdbListDomainParser.new) # return result if no block given return result unless block_given? # loop if block if given begin # the block must return true if it wanna continue break unless yield(result) && result[:next_token] # make new request request_params['NextToken'] = result[:next_token] link = generate_request("ListDomains", request_params) result = request_info(link, QSdbListDomainParser.new) end while true rescue Exception on_exception end # Create new SDB domain at Amazon. # # Returns a hash: { :box_usage, :request_id } on success or an exception on error. # (Amazon raises no errors if the domain already exists). # # Example: # # sdb = RightAws::SdbInterface.new # sdb.create_domain('toys') # => { :box_usage => "0.0000071759", # :request_id => "976709f9-0111-2345-92cb-9ce90acd0982" } # # see: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/SDB_API_CreateDomain.html def create_domain(domain_name) link = generate_request("CreateDomain", 'DomainName' => domain_name) request_info(link, QSdbSimpleParser.new) rescue Exception on_exception end # Query Metadata for Domain # # Returns a hash on success or an exception on error. # # example: # sdb = RightAWS:::SdbInterface.new # sdb.domain_metadata('toys') # => {:attribute_values_size_bytes=>"2754", # :item_count=>"25", # :item_names_size_bytes=>"900", # :timestamp=>"1291890409", # :attribute_name_count=>"7", # :box_usage=>"0.0000071759", # :attribute_names_size_bytes=>"48", # :attribute_value_count=>"154", # :request_id=>"79bbfe8f-f0c9-59a2-0963-16d5fc6c3c52"} # see http://docs.amazonwebservices.com/AmazonSimpleDB/latest/DeveloperGuide/index.html?SDB_API_DomainMetadata.html def domain_metadata(domain) link = generate_request("DomainMetadata","DomainName"=>domain) request_info(link,QSdbGenericParser.new) end # Delete SDB domain at Amazon. # # Returns a hash: { :box_usage, :request_id } on success or an exception on error. # (Amazon raises no errors if the domain does not exist). # # Example: # # sdb = RightAws::SdbInterface.new # sdb.delete_domain('toys') # => { :box_usage => "0.0000071759", # :request_id => "976709f9-0111-2345-92cb-9ce90acd0982" } # # see: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/SDB_API_DeleteDomain.html # def delete_domain(domain_name) link = generate_request("DeleteDomain", 'DomainName' => domain_name) request_info(link, QSdbSimpleParser.new) rescue Exception on_exception end # Add/Replace item attributes. # # Params: # domain_name = DomainName # item_name = ItemName # attributes = { # 'nameA' => [valueA1,..., valueAN], # ... # 'nameZ' => [valueZ1,..., valueZN] # } # replace = :replace | any other value to skip replacement # # Returns a hash: { :box_usage, :request_id } on success or an exception on error. # (Amazon raises no errors if the attribute was not overridden, as when the :replace param is unset). # # Example: # # sdb = RightAws::SdbInterface.new # sdb.create_domain 'family' # # attributes = {} # # create attributes for Jon and Silvia # attributes['Jon'] = %w{ car beer } # attributes['Silvia'] = %w{ beetle rolling_pin kids } # sdb.put_attributes 'family', 'toys', attributes #=> ok # # now: Jon=>[car, beer], Silvia=>[beetle, rolling_pin, kids] # # # add attributes to Jon # attributes.delete('Silvia') # attributes['Jon'] = %w{ girls pub } # sdb.put_attributes 'family', 'toys', attributes #=> ok # # now: Jon=>[car, beer, girls, pub], Silvia=>[beetle, rolling_pin, kids] # # # replace attributes for Jon and add to a cat (the cat had no attributes before) # attributes['Jon'] = %w{ vacuum_cleaner hammer spade } # attributes['cat'] = %w{ mouse clew Jons_socks } # sdb.put_attributes 'family', 'toys', attributes, :replace #=> ok # # now: Jon=>[vacuum_cleaner, hammer, spade], Silvia=>[beetle, rolling_pin, kids], cat=>[mouse, clew, Jons_socks] # # see: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/SDB_API_PutAttributes.html # def put_attributes(domain_name, item_name, attributes, replace = false) params = { 'DomainName' => domain_name, 'ItemName' => item_name }.merge(pack_attributes(attributes, replace)) link = generate_request("PutAttributes", params) request_info( link, QSdbSimpleParser.new ) rescue Exception on_exception end # Add/Replace attributes for multiple items at a time. # # Params: # domain_name = DomainName # items = { # 'Item1' => { # 'nameA' => [valueA1, valueA2,..., valueAN], # ... # 'nameB' => [valueB1, valueB2,..., valueBN] # }, # 'Item2' => { # 'nameC' => [valueC1, valueC2,..., valueCN], # ... # 'nameD' => [valueD1, valueD2,..., valueDN] # } # } # replace = :replace | any other value to skip replacement # # Usage of batch_put_attributes is similar to put_attributes except that # instead of supplying an item_name and a hash of attributes, you supply a # hash of item names to attributes. # # See: http://docs.amazonwebservices.com/AmazonSimpleDB/latest/DeveloperGuide/index.html?SDB_API_BatchPutAttributes.html def batch_put_attributes(domain_name, items, replace = false) params = { 'DomainName' => domain_name }.merge(pack_attributes(items, replace, true)) link = generate_request("BatchPutAttributes", params) request_info( link, QSdbSimpleParser.new) rescue Exception on_exception end # Retrieve SDB item's attribute(s). # # Returns a hash: # { :box_usage => string, # :request_id => string, # :attributes => { 'nameA' => [valueA1,..., valueAN], # ... , # 'nameZ' => [valueZ1,..., valueZN] } } # # Example: # # request all attributes # sdb.get_attributes('family', 'toys') # => { :attributes => {"cat" => ["clew", "Jons_socks", "mouse"] }, # "Silvia" => ["beetle", "rolling_pin", "kids"], # "Jon" => ["vacuum_cleaner", "hammer", "spade"]}, # :box_usage => "0.0000093222", # :request_id => "81273d21-000-1111-b3f9-512d91d29ac8" } # # # request cat's attributes only # sdb.get_attributes('family', 'toys', 'cat') # => { :attributes => {"cat" => ["clew", "Jons_socks", "mouse"] }, # :box_usage => "0.0000093222", # :request_id => "81273d21-001-1111-b3f9-512d91d29ac8" } # # # request all attributes using a consistent read # # see: http://docs.amazonwebservices.com/AmazonSimpleDB/latest/DeveloperGuide/index.html?ConsistencySummary.html # sdb.get_attributes('family', 'toys', nil, true) # => { :attributes => {"cat" => ["clew", "Jons_socks", "mouse"] }, # "Silvia" => ["beetle", "rolling_pin", "kids"], # "Jon" => ["vacuum_cleaner", "hammer", "spade"]}, # :box_usage => "0.0000093222", # :request_id => "81273d21-000-1111-b3f9-512d91d29ac8" } # # see: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/SDB_API_GetAttributes.html # def get_attributes(domain_name, item_name, attribute_name=nil, consistent_read=nil) link = generate_request("GetAttributes", 'DomainName' => domain_name, 'ItemName' => item_name, 'AttributeName' => attribute_name, 'ConsistentRead' => consistent_read ) res = request_info(link, QSdbGetAttributesParser.new) res[:attributes].each_value do |values| values.collect! { |e| sdb_to_ruby(e) } end res rescue Exception on_exception end # Delete value, attribute or item. # # Example: # # delete 'vodka' and 'girls' from 'Jon' and 'mice' from 'cat'. # sdb.delete_attributes 'family', 'toys', { 'Jon' => ['vodka', 'girls'], 'cat' => ['mice'] } # # # delete the all the values from attributes (i.e. delete the attributes) # sdb.delete_attributes 'family', 'toys', { 'Jon' => [], 'cat' => [] } # # or # sdb.delete_attributes 'family', 'toys', [ 'Jon', 'cat' ] # # # delete all the attributes from item 'toys' (i.e. delete the item) # sdb.delete_attributes 'family', 'toys' # # see http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/SDB_API_DeleteAttributes.html # def delete_attributes(domain_name, item_name, attributes = nil) params = { 'DomainName' => domain_name, 'ItemName' => item_name }.merge(pack_attributes(attributes)) link = generate_request("DeleteAttributes", params) request_info( link, QSdbSimpleParser.new ) rescue Exception on_exception end # QUERY: # Perform a query on SDB. # # Returns a hash: # { :box_usage => string, # :request_id => string, # :next_token => string, # :items => [ItemName1,..., ItemNameN] } # # Example: # # query = "['cat' = 'clew']" # sdb.query('family', query) #=> hash of data # sdb.query('family', query, 10) #=> hash of data with max of 10 items # # If a block is given, query will iteratively yield results to it as long as the block continues to return true. # # # List 10 items per iteration. Don't # # forget to escape single quotes and backslashes and wrap all the items in single quotes. # query = "['cat'='clew'] union ['dog'='Jon\\'s boot']" # sdb.query('family', query, 10) do |result| # puts result.inspect # true # end # # # Same query using automatic escaping...to use the auto escape, pass the query and its params as an array: # query = [ "['cat'=?] union ['dog'=?]", "clew", "Jon's boot" ] # sdb.query('family', query) # # query = [ "['cat'=?] union ['dog'=?] sort 'cat' desc", "clew", "Jon's boot" ] # sdb.query('family', query) # # see: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/SDB_API_Query.html # http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?SortingData.html # def query(domain_name, query_expression = nil, max_number_of_items = nil, next_token = nil) query_expression = query_expression_from_array(query_expression) if query_expression.is_a?(Array) @last_query_expression = query_expression # request_params = { 'DomainName' => domain_name, 'QueryExpression' => query_expression, 'MaxNumberOfItems' => max_number_of_items, 'NextToken' => next_token } link = generate_request("Query", request_params) result = request_info( link, QSdbQueryParser.new ) # return result if no block given return result unless block_given? # loop if block if given begin # the block must return true if it wanna continue break unless yield(result) && result[:next_token] # make new request request_params['NextToken'] = result[:next_token] link = generate_request("Query", request_params) result = request_info( link, QSdbQueryParser.new ) end while true rescue Exception on_exception end # Perform a query and fetch specified attributes. # If attributes are not specified then fetches the whole list of attributes. # # # Returns a hash: # { :box_usage => string, # :request_id => string, # :next_token => string, # :items => [ { ItemName1 => { attribute1 => value1, ... attributeM => valueM } }, # { ItemName2 => {...}}, ... ] # # Example: # # sdb.query_with_attributes(domain, ['hobby', 'country'], "['gender'='female'] intersection ['name' starts-with ''] sort 'name'") #=> # { :request_id => "06057228-70d0-4487-89fb-fd9c028580d3", # :items => # [ { "035f1ba8-dbd8-11dd-80bd-001bfc466dd7"=> # { "hobby" => ["cooking", "flowers", "cats"], # "country" => ["Russia"]}}, # { "0327614a-dbd8-11dd-80bd-001bfc466dd7"=> # { "hobby" => ["patchwork", "bundle jumping"], # "country" => ["USA"]}}, ... ], # :box_usage=>"0.0000504786"} # # sdb.query_with_attributes(domain, [], "['gender'='female'] intersection ['name' starts-with ''] sort 'name'") #=> # { :request_id => "75bb19db-a529-4f69-b86f-5e3800f79a45", # :items => # [ { "035f1ba8-dbd8-11dd-80bd-001bfc466dd7"=> # { "hobby" => ["cooking", "flowers", "cats"], # "name" => ["Mary"], # "country" => ["Russia"], # "gender" => ["female"], # "id" => ["035f1ba8-dbd8-11dd-80bd-001bfc466dd7"]}}, # { "0327614a-dbd8-11dd-80bd-001bfc466dd7"=> # { "hobby" => ["patchwork", "bundle jumping"], # "name" => ["Mary"], # "country" => ["USA"], # "gender" => ["female"], # "id" => ["0327614a-dbd8-11dd-80bd-001bfc466dd7"]}}, ... ], # :box_usage=>"0.0000506668"} # # see: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?SDB_API_QueryWithAttributes.html # def query_with_attributes(domain_name, attributes=[], query_expression = nil, max_number_of_items = nil, next_token = nil) attributes = Array(attributes) query_expression = query_expression_from_array(query_expression) if query_expression.is_a?(Array) @last_query_expression = query_expression # request_params = { 'DomainName' => domain_name, 'QueryExpression' => query_expression, 'MaxNumberOfItems' => max_number_of_items, 'NextToken' => next_token } attributes.each_with_index do |attribute, idx| request_params["AttributeName.#{idx+1}"] = attribute end link = generate_request("QueryWithAttributes", request_params) result = select_response_to_ruby(request_info( link, QSdbQueryWithAttributesParser.new )) # return result if no block given return result unless block_given? # loop if block if given begin # the block must return true if it wanna continue break unless yield(result) && result[:next_token] # make new request request_params['NextToken'] = result[:next_token] link = generate_request("QueryWithAttributes", request_params) result = select_response_to_ruby(request_info( link, QSdbQueryWithAttributesParser.new )) end while true rescue Exception on_exception end # Perform SQL-like select and fetch attributes. # Attribute values must be quoted with a single or double quote. If a quote appears within the attribute value, it must be escaped with the same quote symbol as shown in the following example. # (Use array to pass select_expression params to avoid manual escaping). # # sdb.select(["select * from my_domain where gender=?", 'female']) #=> # {:request_id =>"8241b843-0fb9-4d66-9100-effae12249ec", # :items => # [ { "035f1ba8-dbd8-11dd-80bd-001bfc466dd7"=> # {"hobby" => ["cooking", "flowers", "cats"], # "name" => ["Mary"], # "country" => ["Russia"], # "gender" => ["female"], # "id" => ["035f1ba8-dbd8-11dd-80bd-001bfc466dd7"]}}, # { "0327614a-dbd8-11dd-80bd-001bfc466dd7"=> # {"hobby" => ["patchwork", "bundle jumping"], # "name" => ["Mary"], # "country" => ["USA"], # "gender" => ["female"], # "id" => ["0327614a-dbd8-11dd-80bd-001bfc466dd7"]}}, ... ] # :box_usage =>"0.0000506197"} # # sdb.select('select country, name from my_domain') #=> # {:request_id=>"b1600198-c317-413f-a8dc-4e7f864a940a", # :items=> # [ { "035f1ba8-dbd8-11dd-80bd-001bfc466dd7"=> {"name"=>["Mary"], "country"=>["Russia"]} }, # { "376d2e00-75b0-11dd-9557-001bfc466dd7"=> {"name"=>["Putin"], "country"=>["Russia"]} }, # { "0327614a-dbd8-11dd-80bd-001bfc466dd7"=> {"name"=>["Mary"], "country"=>["USA"]} }, # { "372ebbd4-75b0-11dd-9557-001bfc466dd7"=> {"name"=>["Bush"], "country"=>["USA"]} }, # { "37a4e552-75b0-11dd-9557-001bfc466dd7"=> {"name"=>["Medvedev"], "country"=>["Russia"]} }, # { "38278dfe-75b0-11dd-9557-001bfc466dd7"=> {"name"=>["Mary"], "country"=>["Russia"]} }, # { "37df6c36-75b0-11dd-9557-001bfc466dd7"=> {"name"=>["Mary"], "country"=>["USA"]} } ], # :box_usage=>"0.0000777663"} # # see: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?SDB_API_Select.html # http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?UsingSelect.html # http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?SDBLimits.html # def select(select_expression, next_token = nil) select_expression = query_expression_from_array(select_expression) if select_expression.is_a?(Array) @last_query_expression = select_expression # request_params = { 'SelectExpression' => select_expression, 'NextToken' => next_token } link = generate_request("Select", request_params) result = select_response_to_ruby(request_info( link, QSdbSelectParser.new )) return result unless block_given? # loop if block if given begin # the block must return true if it wanna continue break unless yield(result) && result[:next_token] # make new request request_params['NextToken'] = result[:next_token] link = generate_request("Select", request_params) result = select_response_to_ruby(request_info( link, QSdbSelectParser.new )) end while true rescue Exception on_exception end #----------------------------------------------------------------- # PARSERS: #----------------------------------------------------------------- class QSdbGenericParser < RightAWSParser #:nodoc: def reset @result = {} end def tagend(name) case full_tag_name when %r{/(DomainMetadataResult|ResponseMetadata)/} @result[name.right_underscore.to_sym] = @text end end end class QSdbListDomainParser < RightAWSParser #:nodoc: def reset @result = { :domains => [] } end def tagend(name) case name when 'NextToken' then @result[:next_token] = @text when 'DomainName' then @result[:domains] << @text when 'BoxUsage' then @result[:box_usage] = @text when 'RequestId' then @result[:request_id] = @text end end end class QSdbSimpleParser < RightAWSParser #:nodoc: def reset @result = {} end def tagend(name) case name when 'BoxUsage' then @result[:box_usage] = @text when 'RequestId' then @result[:request_id] = @text end end end class QSdbGetAttributesParser < RightAWSParser #:nodoc: def reset @last_attribute_name = nil @result = { :attributes => {} } end def tagend(name) case name when 'Name' then @last_attribute_name = @text when 'Value' then (@result[:attributes][@last_attribute_name] ||= []) << @text when 'BoxUsage' then @result[:box_usage] = @text when 'RequestId' then @result[:request_id] = @text end end end class QSdbQueryParser < RightAWSParser #:nodoc: def reset @result = { :items => [] } end def tagend(name) case name when 'ItemName' then @result[:items] << @text when 'BoxUsage' then @result[:box_usage] = @text when 'RequestId' then @result[:request_id] = @text when 'NextToken' then @result[:next_token] = @text end end end class QSdbQueryWithAttributesParser < RightAWSParser #:nodoc: def reset @result = { :items => [] } end def tagend(name) case name when 'Name' case @xmlpath when 'QueryWithAttributesResponse/QueryWithAttributesResult/Item' @item = @text @result[:items] << { @item => {} } when 'QueryWithAttributesResponse/QueryWithAttributesResult/Item/Attribute' @attribute = @text @result[:items].last[@item][@attribute] ||= [] end when 'RequestId' then @result[:request_id] = @text when 'BoxUsage' then @result[:box_usage] = @text when 'NextToken' then @result[:next_token] = @text when 'Value' then @result[:items].last[@item][@attribute] << @text end end end class QSdbSelectParser < RightAWSParser #:nodoc: def reset @result = { :items => [] } end def tagend(name) case name when 'Name' case @xmlpath when 'SelectResponse/SelectResult/Item' @item = @text @result[:items] << { @item => {} } when 'SelectResponse/SelectResult/Item/Attribute' @attribute = @text @result[:items].last[@item][@attribute] ||= [] end when 'RequestId' then @result[:request_id] = @text when 'BoxUsage' then @result[:box_usage] = @text when 'NextToken' then @result[:next_token] = @text when 'Value' then @result[:items].last[@item][@attribute] << @text end end end end end ================================================ FILE: lib/sns/right_sns_interface.rb ================================================ # # Copyright (c) 2007-2008 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws class SnsInterface < RightAwsBase include RightAwsBaseInterface DEFAULT_HOST = 'sns.us-east-1.amazonaws.com' DEFAULT_PORT = 443 DEFAULT_PROTOCOL = 'https' DEFAULT_SERVICE = '/' REQUEST_TTL = 30 # Apparently boilerplate stuff @@bench = AwsBenchmarkingBlock.new def self.bench_xml @@bench.xml end def self.bench_service @@bench.service end def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={}) if params[:region] server = "sns.#{params[:region]}.amazonaws.com" params.delete(:region) else server = DEFAULT_HOST end init({ :name => 'SNS', :default_host => ENV['SNS_URL'] ? URI.parse(ENV['SNS_URL']).host : server, :default_port => ENV['SNS_URL'] ? URI.parse(ENV['SNS_URL']).port : DEFAULT_PORT, :default_service => ENV['SNS_URL'] ? URI.parse(ENV['SNS_URL']).path : DEFAULT_SERVICE, :default_protocol => ENV['SNS_URL'] ? URI.parse(ENV['SNS_URL']).scheme : DEFAULT_PROTOCOL}, aws_access_key_id || ENV['AWS_ACCESS_KEY_ID'], aws_secret_access_key || ENV['AWS_SECRET_ACCESS_KEY'], params) end # TODO: RJG - Seems like generate_request and generate_rest_request could be in a sub class? # Generates a request hash for the sns API def generate_request(action, params={}) # :nodoc: # Sometimes we need to use queue uri (delete queue etc) # In that case we will use Symbol key: 'param[:queue_url]' service = params[:sns_url] ? URI(params[:sns_url]).path : '/' # remove unset(=optional) and symbolyc keys params.each{ |key, value| params.delete(key) if (value.nil? || key.is_a?(Symbol)) } # prepare output hash service_hash = { "Action" => action, "Expires" => (Time.now + REQUEST_TTL).utc.strftime("%Y-%m-%dT%H:%M:%SZ"), "AWSAccessKeyId" => @aws_access_key_id } #"Version" => API_VERSION } service_hash.update(params) service_params = signed_service_params(@aws_secret_access_key, service_hash, :get, @params[:server], service) request = Net::HTTP::Get.new("#{AwsUtils::URLencode(service)}?#{service_params}") # prepare output hash { :request => request, :server => @params[:server], :port => @params[:port], :protocol => @params[:protocol] } end # Generates a request hash for the REST API def generate_rest_request(method, param) # :nodoc: sns_uri = param[:sns_url] ? URI(param[:sns_url]).path : '/' message = param[:message] # extract message body if nesessary # remove unset(=optional) and symbolyc keys param.each{ |key, value| param.delete(key) if (value.nil? || key.is_a?(Symbol)) } # created request param_to_str = param.to_a.collect{|key,val| key.to_s + "=" + CGI::escape(val.to_s) }.join("&") param_to_str = "?#{param_to_str}" unless param_to_str.right_blank? request = "Net::HTTP::#{method.capitalize}".right_constantize.new("#{sns_uri}#{param_to_str}") request.body = message if message # set main headers request['content-md5'] = '' request['Content-Type'] = 'text/plain' request['Date'] = Time.now.httpdate # generate authorization string auth_string = "#{method.upcase}\n#{request['content-md5']}\n#{request['Content-Type']}\n#{request['Date']}\n#{CGI::unescape(sns_uri)}" signature = AwsUtils::sign(@aws_secret_access_key, auth_string) # set other headers request['Authorization'] = "AWS #{@aws_access_key_id}:#{signature}" #request['AWS-Version'] = API_VERSION # prepare output hash { :request => request, :server => @params[:server], :port => @params[:port], :protocol => @params[:protocol] } end # Sends request to Amazon and parses the response # Raises AwsError if any banana happened def request_info(request, parser) # :nodoc: request_info_impl(:sns_connection, @@bench, request, parser) end def create_topic(topic_name) req_hash = generate_request('CreateTopic', 'Name' => topic_name) request_info(req_hash, SnsCreateTopicParser.new) end def list_topics() req_hash = generate_request('ListTopics') request_info(req_hash, SnsListTopicsParser.new) end def delete_topic(topic_arn) req_hash = generate_request('DeleteTopic', 'TopicArn' => topic_arn) request_info(req_hash, RightHttp2xxParser.new) end def subscribe(topic_arn, protocol, endpoint) req_hash = generate_request('Subscribe', 'TopicArn' => topic_arn, 'Protocol' => protocol, 'Endpoint' => endpoint) request_info(req_hash, SnsSubscribeParser.new) end def unsubscribe(subscription_arn) req_hash = generate_request('Unsubscribe', 'SubscriptionArn' => subscription_arn) request_info(req_hash, RightHttp2xxParser.new) end def publish(topic_arn, message, subject) req_hash = generate_request('Publish', 'TopicArn' => topic_arn, 'Message' => message, 'Subject' => subject) request_info(req_hash, SnsPublishParser.new) end def set_topic_attribute(topic_arn, attribute_name, attribute_value) if attribute_name != 'Policy' && attribute_name != 'DisplayName' raise(ArgumentError, "The only values accepted for the attribute_name parameter are (Policy, DisplayName)") end req_hash = generate_request('SetTopicAttributes', 'TopicArn' => topic_arn, 'AttributeName' => attribute_name, 'AttributeValue' => attribute_value) request_info(req_hash, RightHttp2xxParser.new) end def get_topic_attributes(topic_arn) req_hash = generate_request('GetTopicAttributes', 'TopicArn' => topic_arn) request_info(req_hash, SnsGetTopicAttributesParser.new) end # Calls either the ListSubscriptions or ListSubscriptionsByTopic depending on whether or not the topic_arn parameter is provided. def list_subscriptions(topic_arn = nil) req_hash = topic_arn ? generate_request('ListSubscriptionsByTopic', 'TopicArn' => topic_arn) : generate_request('ListSubscriptions') request_info(req_hash, SnsListSubscriptionsParser.new) end def confirm_subscription(topic_arn, token, authenticate_on_unsubscribe=false) req_hash = generate_request('ConfirmSubscription', 'AuthenticateOnUnsubscribe' => authenticate_on_unsubscribe.to_s, 'Token' => token, 'TopicArn' => topic_arn) request_info(req_hash, SnsConfirmSubscriptionParser.new) end def add_permission(topic_arn, label, acct_action_hash_ary) n_hash = { 'TopicArn' => topic_arn, 'Label' => label } acct_action_hash_ary.each_with_index do |hash_val, idx| n_hash["AWSAccountId.member.#{idx+1}"] = hash_val[:aws_account_id] n_hash["ActionName.member.#{idx+1}"] = hash_val[:action] end req_hash = generate_request('AddPermission', n_hash) request_info(req_hash, RightHttp2xxParser.new) end def remove_permission(topic_arn, label) req_hash = generate_request('RemovePermission', 'TopicArn' => topic_arn, 'Label' => label) request_info(req_hash, RightHttp2xxParser.new) end class SnsCreateTopicParser < RightAWSParser # :nodoc: def reset @result = '' @request_id = '' end def tagend(name) case name when 'RequestId' then @result_id = @text when 'TopicArn' then @result = @text end end end class SnsListTopicsParser < RightAWSParser # :nodoc: def reset @result = [] @request_id = '' end def tagstart(name, attributes) @current_key = {} if name == 'member' end def tagend(name) case name when 'RequestId' then @result_id = @text when 'TopicArn' then @current_key[:arn] = @text when 'member' then @result << @current_key end end end class SnsSubscribeParser < RightAWSParser # :nodoc: def reset @result = '' end def tagend(name) case name when 'SubscriptionArn' then @result = @text end end end class SnsPublishParser < RightAWSParser # :nodoc: def reset @result = '' end def tagend(name) case name when 'MessageId' then @result = @text end end end class SnsGetTopicAttributesParser < RightAWSParser # :nodoc: def reset @result = {} end def tagend(name) case name when 'key' then @current_attr = @text when 'value' then @result[@current_attr] = @text end end end class SnsListSubscriptionsParser < RightAWSParser # :nodoc: def reset @result = [] end def tagstart(name, attributes) @current_key = {} if name == 'member' end def tagend(name) case name when 'TopicArn' then @current_key[:topic_arn] = @text when 'Protocol' then @current_key[:protocol] = @text when 'SubscriptionArn' then @current_key[:subscription_arn] = @text when 'Owner' then @current_key[:owner] = @text when 'Endpoint' then @current_key[:endpoint] = @text when 'member' then @result << @current_key end end end class SnsConfirmSubscriptionParser < RightAWSParser # :nodoc: def reset @result = '' end def tagend(name) case name when 'SubscriptionArn' then @result = @text end end end end end ================================================ FILE: lib/sqs/right_sqs.rb ================================================ # # Copyright (c) 2007-2008 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws # # = RightAws::Sqs -- RightScale's Amazon SQS interface # The RightAws::Sqs class provides a complete interface to Amazon's Simple # Queue Service. # For explanations of the semantics # of each call, please refer to Amazon's documentation at # http://developer.amazonwebservices.com/connect/kbcategory.jspa?categoryID=31 # # Error handling: all operations raise an RightAws::AwsError in case # of problems. Note that transient errors are automatically retried. # # sqs = RightAws::Sqs.new(aws_access_key_id, aws_secret_access_key) # queue1 = sqs.queue('my_awesome_queue') # ... # queue2 = RightAws::Sqs::Queue.create(sqs, 'my_cool_queue', true) # puts queue2.size # ... # message1 = queue2.receive # message1.visibility = 0 # puts message1 # ... # queue2.clear(true) # queue2.send_message('Ola-la!') # message2 = queue2.pop # ... # grantee1 = RightAws::Sqs::Grantee.create(queue2,'one_cool_guy@email.address') # grantee1.grant('FULLCONTROL') # grantee1.drop # ... # grantee2 = queue.grantees('another_cool_guy@email.address') # grantee2.revoke('SENDMESSAGE') # # Params is a hash: # # {:server => 'queue.amazonaws.com' # Amazon service host: 'queue.amazonaws.com' (default) # :port => 443 # Amazon service port: 80 or 443 (default) # :signature_version => '0' # The signature version : '0' or '1'(default) # :logger => Logger Object} # Logger instance: logs to STDOUT if omitted } # class Sqs attr_reader :interface def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={}) @interface = SqsInterface.new(aws_access_key_id, aws_secret_access_key, params) end # Retrieves a list of queues. # Returns an +array+ of +Queue+ instances. # # RightAws::Sqs.queues #=> array of queues # def queues(prefix=nil) @interface.list_queues(prefix).map do |url| Queue.new(self, url) end end # Returns Queue instance by queue name. # If the queue does not exist at Amazon SQS and +create+ is true, the method creates it. # # RightAws::Sqs.queue('my_awesome_queue') #=> # # def queue(queue_name, create=true, visibility=nil) url = @interface.queue_url_by_name(queue_name) url = (create ? @interface.create_queue(queue_name, visibility) : nil) unless url url ? Queue.new(self, url) : nil end class Queue attr_reader :name, :url, :sqs # Returns Queue instance by queue name. # If the queue does not exist at Amazon SQS and +create+ is true, the method creates it. # # RightAws::Sqs::Queue.create(sqs, 'my_awesome_queue') #=> # # def self.create(sqs, url_or_name, create=true, visibility=nil) sqs.queue(url_or_name, create, visibility) end # Creates new Queue instance. # Does not create a queue at Amazon. # # queue = RightAws::Sqs::Queue.new(sqs, 'my_awesome_queue') # def initialize(sqs, url_or_name) @sqs = sqs @url = @sqs.interface.queue_url_by_name(url_or_name) @name = @sqs.interface.queue_name_by_url(@url) end # Retrieves queue size. # # queue.size #=> 1 # def size @sqs.interface.get_queue_length(@url) end # Clears queue. # Deletes only the visible messages unless +force+ is +true+. # # queue.clear(true) #=> true # # P.S. when force==true the queue deletes then creates again. This is # the quickest method to clear a big queue or a queue with 'locked' messages. All queue # attributes are restored. But there is no way to restore grantees' permissions to # this queue. If you have no grantees except 'root' then you have no problems. # Otherwise, it's better to use queue.clear(false). # # PS This function is no longer supported. Amazon has changed the SQS semantics to require at least 60 seconds between # queue deletion and creation. Hence this method will fail with an exception. # def clear(force=false) ## if force ## @sqs.interface.force_clear_queue(@url) ## else @sqs.interface.clear_queue(@url) ## end end # Deletes queue. # Queue must be empty or +force+ must be set to +true+. # Returns +true+. # # queue.delete(true) #=> true # def delete(force=false) @sqs.interface.delete_queue(@url, force) end # Sends new message to queue. # Returns new Message instance that has been sent to queue. def send_message(message) message = message.to_s msg = Message.new(self, @sqs.interface.send_message(@url, message), message) msg.sent_at = Time.now msg end alias_method :push, :send_message # Retrieves several messages from queue. # Returns an array of Message instances. # # queue.receive_messages(2,10) #=> array of messages # def receive_messages(number_of_messages=1, visibility=nil) list = @sqs.interface.receive_messages(@url, number_of_messages, visibility) list.map! do |entry| msg = Message.new(self, entry[:id], entry[:body], visibility) msg.received_at = Time.now msg end end # Retrieves first accessible message from queue. # Returns Message instance or +nil+ it the queue is empty. # # queue.receive #=> # # def receive(visibility=nil) list = receive_messages(1, visibility) list.empty? ? nil : list[0] end # Peeks message body. # # queue.peek #=> # # def peek(message_id) entry = @sqs.interface.peek_message(@url, message_id) msg = Message.new(self, entry[:id], entry[:body]) msg.received_at = Time.now msg end # Pops (and deletes) first accessible message from queue. # Returns Message instance or +nil+ it the queue is empty. # # queue.pop #=> # # def pop msg = receive msg.delete if msg msg end # Retrieves +VisibilityTimeout+ value for the queue. # Returns new timeout value. # # queue.visibility #=> 30 # def visibility @sqs.interface.get_visibility_timeout(@url) end # Sets new +VisibilityTimeout+ for the queue. # Returns new timeout value. # # queue.visibility #=> 30 # queue.visibility = 33 # queue.visibility #=> 33 # def visibility=(visibility_timeout) @sqs.interface.set_visibility_timeout(@url, visibility_timeout) visibility_timeout end # Sets new queue attribute value. # Not all attributes may be changed: +ApproximateNumberOfMessages+ (for example) is a read only attribute. # Returns a value to be assigned to attribute. # # queue.set_attribute('VisibilityTimeout', '100') #=> '100' # queue.get_attribute('VisibilityTimeout') #=> '100' # def set_attribute(attribute, value) @sqs.interface.set_queue_attributes(@url, attribute, value) value end # Retrieves queue attributes. # At this moment Amazon supports +VisibilityTimeout+ and +ApproximateNumberOfMessages+ only. # If the name of attribute is set, returns its value. Otherwise, returns a hash of attributes. # # queue.get_attribute('VisibilityTimeout') #=> '100' # def get_attribute(attribute='All') attributes = @sqs.interface.get_queue_attributes(@url, attribute) attribute=='All' ? attributes : attributes[attribute] end # Retrieves a list of grantees. # Returns an +array+ of Grantee instances if the +grantee_email_address+ is unset. # Otherwise returns a Grantee instance that points to +grantee_email_address+ or +nil+. # # grantees = queue.grantees #=> [#, ...] # ... # grantee = queue.grantees('cool_guy@email.address') #=> nil | # # def grantees(grantee_email_address=nil, permission = nil) hash = @sqs.interface.list_grants(@url, grantee_email_address, permission) grantees = [] hash.each do |key, value| grantees << Grantee.new(self, grantee_email_address, key, value[:name], value[:perms]) end if grantee_email_address grantees.right_blank? ? nil : grantees.shift else grantees end end end class Message attr_reader :queue, :id, :body, :visibility attr_accessor :sent_at, :received_at def initialize(queue, id=nil, body=nil, visibility=nil) @queue = queue @id = id @body = body @visibility = visibility @sent_at = nil @received_at = nil end # Returns +Message+ instance body. def to_s @body end # Changes +VisibilityTimeout+ for current message. # Returns new +VisibilityTimeout+ value. def visibility=(visibility_timeout) @queue.sqs.interface.change_message_visibility(@queue.url, @id, visibility_timeout) @visibility = visibility_timeout end # Removes message from queue. # Returns +true+. def delete @queue.sqs.interface.delete_message(@queue.url, @id) end end class Grantee attr_accessor :queue, :id, :name, :perms, :email # Creates new Grantee instance. # To create new grantee for queue use: # # grantee = Grantee.new(queue, grantee@email.address) # grantee.grant('FULLCONTROL') # def initialize(queue, email=nil, id=nil, name=nil, perms=[]) @queue = queue @id = id @name = name @perms = perms @email = email retrieve unless id end # Retrieves security information for grantee identified by email. # Returns +nil+ if the named user has no privileges on this queue, or # +true+ if perms updated successfully. def retrieve # :nodoc: @id = nil @name = nil @perms = [] hash = @queue.sqs.interface.list_grants(@queue.url, @email) return nil if hash.empty? grantee = hash.shift @id = grantee[0] @name = grantee[1][:name] @perms = grantee[1][:perms] true end # Adds permissions for grantee. # Permission: 'FULLCONTROL' | 'RECEIVEMESSAGE' | 'SENDMESSAGE'. # The caller must have set the email instance variable. def grant(permission=nil) raise "You can't grant permission without defining a grantee email address!" unless @email @queue.sqs.interface.add_grant(@queue.url, @email, permission) retrieve end # Revokes permissions for grantee. # Permission: 'FULLCONTROL' | 'RECEIVEMESSAGE' | 'SENDMESSAGE'. # Default value is 'FULLCONTROL'. # User must have +@email+ or +@id+ set. # Returns +true+. def revoke(permission='FULLCONTROL') @queue.sqs.interface.remove_grant(@queue.url, @email || @id, permission) unless @email # if email is unknown - just remove permission from local perms list... @perms.delete(permission) else # ... else retrieve updated information from Amazon retrieve end true end # Revokes all permissions for this grantee. # Returns +true+ def drop @perms.each do |permission| @queue.sqs.interface.remove_grant(@queue.url, @email || @id, permission) end retrieve true end end end end ================================================ FILE: lib/sqs/right_sqs_gen2.rb ================================================ # # Copyright (c) 2008 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws # # RightAws::SqsGen2 -- RightScale's Amazon SQS interface, API version # 2008-01-01 and later. # The RightAws::SqsGen2 class provides a complete interface to the second generation of Amazon's Simple # Queue Service. # For explanations of the semantics # of each call, please refer to Amazon's documentation at # http://developer.amazonwebservices.com/connect/kbcategory.jspa?categoryID=31 # # # RightAws::SqsGen2 is built atop RightAws::SqsGen2Interface, a lower-level # procedural API that may be appropriate for certain programs. # # Error handling: all operations raise an RightAws::AwsError in case # of problems. Note that transient errors are automatically retried. # # sqs = RightAws::SqsGen2.new(aws_access_key_id, aws_secret_access_key) # queue1 = sqs.queue('my_awesome_queue') # ... # queue2 = RightAws::SqsGen2::Queue.create(sqs, 'my_cool_queue', true) # puts queue2.size # ... # message1 = queue2.receive # message1.visibility = 0 # puts message1 # ... # queue2.clear(true) # queue2.send_message('Ola-la!') # message2 = queue2.pop # ... # # NB: Second-generation SQS has eliminated the entire access grant mechanism present in Gen 1. # # Params is a hash: # # {:server => 'queue.amazonaws.com' # Amazon service host: 'queue.amazonaws.com' (default) # :port => 443 # Amazon service port: 80 or 443 (default) # :signature_version => '0' # The signature version : '0' or '1'(default) # :logger => Logger Object} # Logger instance: logs to STDOUT if omitted } class SqsGen2 attr_reader :interface def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={}) @interface = SqsGen2Interface.new(aws_access_key_id, aws_secret_access_key, params) end # Retrieves a list of queues. # Returns an +array+ of +Queue+ instances. # # RightAws::Sqs.queues #=> array of queues # def queues(prefix=nil) @interface.list_queues(prefix).map do |url| Queue.new(self, url) end end # Returns Queue instance by queue name. # If the queue does not exist at Amazon SQS and +create+ is true, the method creates it. # # RightAws::SqsGen2.queue('my_awesome_queue') #=> # # def queue(queue_name, create=true, visibility=nil) url = @interface.queue_url_by_name(queue_name) url = (create ? @interface.create_queue(queue_name, visibility) : nil) unless url url ? Queue.new(self, url) : nil end class Queue attr_reader :name, :url, :sqs # Returns Queue instance by queue name. # If the queue does not exist at Amazon SQS and +create+ is true, the method creates it. # # RightAws::SqsGen2::Queue.create(sqs, 'my_awesome_queue') #=> # # def self.create(sqs, url_or_name, create=true, visibility=nil) sqs.queue(url_or_name, create, visibility) end # Creates new Queue instance. # Does not create a queue at Amazon. # # queue = RightAws::SqsGen2::Queue.new(sqs, 'my_awesome_queue') # def initialize(sqs, url_or_name) @sqs = sqs @url = @sqs.interface.queue_url_by_name(url_or_name) @name = @sqs.interface.queue_name_by_url(@url) end # Retrieves queue size. # # queue.size #=> 1 # def size @sqs.interface.get_queue_length(@url) end # Clears queue, deleting only the visible messages. Any message within its visibility # timeout will not be deleted, and will re-appear in the queue in the # future when the timeout expires. # # To delete all messages in a queue and eliminate the chance of any # messages re-appearing in the future, it's best to delete the queue and # re-create it as a new queue. Note that doing this will take at least 60 # s since SQS does not allow re-creation of a queue within this interval. # # queue.clear() #=> true # def clear() @sqs.interface.clear_queue(@url) end # Deletes queue. Any messages in the queue will be permanently lost. # Returns +true+. # # NB: Use with caution; severe data loss is possible! # # queue.delete(true) #=> true # def delete(force=false) @sqs.interface.delete_queue(@url) end # Sends new message to queue. # Returns new Message instance that has been sent to queue. def send_message(message) message = message.to_s res = @sqs.interface.send_message(@url, message) msg = Message.new(self, res['MessageId'], nil, message) msg.send_checksum = res['MD5OfMessageBody'] msg.sent_at = Time.now msg end alias_method :push, :send_message # Retrieves several messages from queue. # Returns an array of Message instances. # # queue.receive_messages(2,10) #=> array of messages # def receive_messages(number_of_messages=1, visibility=nil, attributes=nil) list = @sqs.interface.receive_message(@url, number_of_messages, visibility, attributes) list.map! do |entry| msg = Message.new(self, entry['MessageId'], entry['ReceiptHandle'], entry['Body'], visibility, entry['Attributes']) msg.received_at = Time.now msg.receive_checksum = entry['MD5OfBody'] msg end end # Retrieves first accessible message from queue. # Returns Message instance or +nil+ it the queue is empty. # # queue.receive #=> # # def receive(visibility=nil, attributes=nil) list = receive_messages(1, visibility, attributes) list.empty? ? nil : list[0] end # Pops (and deletes) first accessible message from queue. # Returns Message instance or +nil+ if the queue is empty. # # queue.pop #=> # # # # pop a message with custom attributes # m = queue.pop(['SenderId', 'SentTimestamp']) #=> # # m.attributes #=> {"SentTimestamp"=>"1240991906937", "SenderId"=>"800000000005"} # def pop(attributes=nil) list = @sqs.interface.pop_messages(@url, 1, attributes) return nil if list.empty? entry = list[0] msg = Message.new(self, entry['MessageId'], entry['ReceiptHandle'], entry['Body'], visibility, entry['Attributes']) msg.received_at = Time.now msg.receive_checksum = entry['MD5OfBody'] msg end # Retrieves +VisibilityTimeout+ value for the queue. # Returns new timeout value. # # queue.visibility #=> 30 # def visibility @sqs.interface.get_queue_attributes(@url, 'VisibilityTimeout')['VisibilityTimeout'] end # Sets new +VisibilityTimeout+ for the queue. # Returns new timeout value. # # queue.visibility #=> 30 # queue.visibility = 33 # queue.visibility #=> 33 # def visibility=(visibility_timeout) @sqs.interface.set_queue_attributes(@url, 'VisibilityTimeout', visibility_timeout) visibility_timeout end # Sets new queue attribute value. # Not all attributes may be changed: +ApproximateNumberOfMessages+ (for example) is a read only attribute. # Returns a value to be assigned to attribute. # Currently, 'VisibilityTimeout' is the only settable queue attribute. # Attempting to set non-existent attributes generates an indignant # exception. # # queue.set_attribute('VisibilityTimeout', '100') #=> '100' # queue.get_attribute('VisibilityTimeout') #=> '100' # def set_attribute(attribute, value) @sqs.interface.set_queue_attributes(@url, attribute, value) value end # Retrieves queue attributes. # If the name of attribute is set, returns its value. Otherwise, returns a hash of attributes. # # queue.get_attribute('VisibilityTimeout') #=> {"VisibilityTimeout"=>"45"} # # P.S. This guy is deprecated. Use +get_attributes+ instead. def get_attribute(attribute='All') attributes = get_attributes(attribute) attribute=='All' ? attributes : attributes[attribute] end # Retrieves queue attributes. # # queue.get_attributes #=> # {"ApproximateNumberOfMessages" => "0", # "LastModifiedTimestamp" => "1240946032", # "CreatedTimestamp" => "1240816887", # "VisibilityTimeout" => "30", # "Policy" => "{"Version":"2008-10-17","Id":...}"} # # queue.get_attributes("LastModifiedTimestamp", "VisibilityTimeout") #=> # {"LastModifiedTimestamp" => "1240946032", # "VisibilityTimeout" => "30"} # def get_attributes(*attributes) @sqs.interface.get_queue_attributes(@url, attributes) end # Add permission to the queue. # # queue.add_permissions('testLabel',['125074342641', '125074342642'], # ['SendMessage','SendMessage','ReceiveMessage']) #=> true # def add_permissions(label, grantees, actions) @sqs.interface.add_permissions(@url, label, grantees, actions) end # Revoke any permissions in the queue policy that matches the +label+ parameter. # # sqs.remove_permissions('testLabel') # => true # def remove_permissions(label) @sqs.interface.remove_permissions(@url, label) end # Get current permissions set. The set is JSON packed. # # sqs.get_permissions #=> # '{"Version":"2008-10-17","Id":"/826693181925/kd-test-gen-2_5/SQSDefaultPolicy", # "Statement":[{"Sid":"kd-perm-04","Effect":"Allow","Principal":{"AWS":"100000000001", # "AWS":"100000000001","AWS":"100000000002"},"Action":["SQS:SendMessage","SQS:DeleteMessage", # "SQS:ReceiveMessage"],"Resource":"/826693181925/kd-test-gen-2_5"},{"Sid":"kd-perm-03", # "Effect":"Allow","Principal":{"AWS":"648772224137"},"Action":"SQS:SendMessage", # "Resource":"/826693181925/kd-test-gen-2_5"}]}' # def get_permissions get_attributes('Policy')['Policy'] end end class Message attr_reader :queue, :id, :body, :visibility, :receipt_handle, :attributes attr_accessor :sent_at, :received_at, :send_checksum, :receive_checksum def initialize(queue, id=nil, rh = nil, body=nil, visibility=nil, attributes=nil) @queue = queue @id = id @receipt_handle = rh @body = body @visibility = visibility @attributes = attributes @sent_at = nil @received_at = nil @send_checksum = nil @receive_checksum = nil end # Returns +Message+ instance body. def to_s @body end # Set message visibility timeout. def visibility=(visibility_timeout) @queue.sqs.interface.change_message_visibility(@queue.url, @receipt_handle, visibility_timeout) @visibility = visibility_timeout end # Removes message from queue. # Returns +true+. def delete @queue.sqs.interface.delete_message(@queue.url, @receipt_handle) if @receipt_handle end end end end ================================================ FILE: lib/sqs/right_sqs_gen2_interface.rb ================================================ # # Copyright (c) 2007-2008 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws # # Right::Aws::SqsGen2Interface - RightScale's low-level Amazon SQS interface # for API version 2008-01-01 and later. # For explanations of the semantics # of each call, please refer to Amazon's documentation at # http://developer.amazonwebservices.com/connect/kbcategory.jspa?categoryID=31 # # This class provides a procedural interface to SQS. Conceptually it is # mostly a pass-through interface to SQS and its API is very similar to the # bare SQS API. For a somewhat higher-level and object-oriented interface, see # RightAws::SqsGen2. class SqsGen2Interface < RightAwsBase include RightAwsBaseInterface API_VERSION = "2009-02-01" DEFAULT_HOST = "queue.amazonaws.com" DEFAULT_PORT = 443 DEFAULT_PROTOCOL = 'https' REQUEST_TTL = 30 DEFAULT_VISIBILITY_TIMEOUT = 30 @@bench = AwsBenchmarkingBlock.new def self.bench_xml @@bench.xml end def self.bench_sqs @@bench.service end @@api = API_VERSION def self.api @@api end # Creates a new SqsInterface instance. This instance is limited to # operations on SQS objects created with Amazon's 2008-01-01 API version. This # interface will not work on objects created with prior API versions. See # Amazon's article "Migrating to Amazon SQS API version 2008-01-01" at: # http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1148 # # sqs = RightAws::SqsGen2Interface.new('1E3GDYEOGFJPIT75KDT40','hgTHt68JY07JKUY08ftHYtERkjgtfERn57DFE379', {:logger => Logger.new('/tmp/x.log')}) # # Params is a hash: # # {:server => 'queue.amazonaws.com' # Amazon service host: 'queue.amazonaws.com' (default) # :port => 443 # Amazon service port: 80 or 443 (default) # :signature_version => '0' # The signature version : '0', '1' or '2'(default) # :logger => Logger Object} # Logger instance: logs to STDOUT if omitted } # def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={}) init({ :name => 'SQS', :default_host => ENV['SQS_URL'] ? URI.parse(ENV['SQS_URL']).host : DEFAULT_HOST, :default_port => ENV['SQS_URL'] ? URI.parse(ENV['SQS_URL']).port : DEFAULT_PORT, :default_protocol => ENV['SQS_URL'] ? URI.parse(ENV['SQS_URL']).scheme : DEFAULT_PROTOCOL }, aws_access_key_id || ENV['AWS_ACCESS_KEY_ID'], aws_secret_access_key || ENV['AWS_SECRET_ACCESS_KEY'], params) end #----------------------------------------------------------------- # Requests #----------------------------------------------------------------- # Generates a request hash for the query API def generate_request(action, param={}) # :nodoc: # For operation requests on a queue, the queue URI will be a parameter, # so we first extract it from the call parameters. Next we remove any # parameters with no value or with symbolic keys. We add the header # fields required in all requests, and then the headers passed in as # params. We sort the header fields alphabetically and then generate the # signature before URL escaping the resulting query and sending it. service = param[:queue_url] ? URI(param[:queue_url]).path : '/' param.each{ |key, value| param.delete(key) if (value.nil? || key.is_a?(Symbol)) } service_hash = { "Action" => action, "Expires" => (Time.now + REQUEST_TTL).utc.strftime("%Y-%m-%dT%H:%M:%SZ"), "AWSAccessKeyId" => @aws_access_key_id, "Version" => API_VERSION } service_hash.update(param) service_params = signed_service_params(@aws_secret_access_key, service_hash, :get, @params[:server], service) request = Net::HTTP::Get.new("#{AwsUtils.URLencode(service)}?#{service_params}") # prepare output hash { :request => request, :server => @params[:server], :port => @params[:port], :protocol => @params[:protocol] } end def generate_post_request(action, param={}) # :nodoc: service = param[:queue_url] ? URI(param[:queue_url]).path : '/' message = param[:message] # extract message body if nesessary param.each{ |key, value| param.delete(key) if (value.nil? || key.is_a?(Symbol)) } service_hash = { "Action" => action, "Expires" => (Time.now + REQUEST_TTL).utc.strftime("%Y-%m-%dT%H:%M:%SZ"), "AWSAccessKeyId" => @aws_access_key_id, "MessageBody" => message, "Version" => API_VERSION } service_hash.update(param) # service_params = signed_service_params(@aws_secret_access_key, service_hash, :post, @params[:server], service) request = Net::HTTP::Post.new(AwsUtils::URLencode(service)) request['Content-Type'] = 'application/x-www-form-urlencoded; charset=utf-8' request.body = service_params # prepare output hash { :request => request, :server => @params[:server], :port => @params[:port], :protocol => @params[:protocol] } end # Sends request to Amazon and parses the response # Raises AwsError if any banana happened def request_info(request, parser) # :nodoc: request_info_impl(:sqs_connection, @@bench, request, parser) end # Creates a new queue, returning its URI. # # sqs.create_queue('my_awesome_queue') #=> 'https://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue' # def create_queue(queue_name, default_visibility_timeout=nil) req_hash = generate_request('CreateQueue', 'QueueName' => queue_name, 'DefaultVisibilityTimeout' => default_visibility_timeout || DEFAULT_VISIBILITY_TIMEOUT ) request_info(req_hash, SqsCreateQueueParser.new(:logger => @logger)) rescue on_exception end # Lists all queues owned by this user that have names beginning with +queue_name_prefix+. # If +queue_name_prefix+ is omitted then retrieves a list of all queues. # Queue creation is an eventual operation and created queues may not show up in immediately subsequent list_queues calls. # # sqs.create_queue('my_awesome_queue') # sqs.create_queue('my_awesome_queue_2') # sqs.list_queues('my_awesome') #=> ['https://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue','https://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue_2'] # def list_queues(queue_name_prefix=nil) req_hash = generate_request('ListQueues', 'QueueNamePrefix' => queue_name_prefix) request_info(req_hash, SqsListQueuesParser.new(:logger => @logger)) rescue on_exception end # Deletes queue. Any messages in the queue are permanently lost. # Returns +true+ or an exception. # Queue deletion can take up to 60 s to propagate through SQS. Thus, after a deletion, subsequent list_queues calls # may still show the deleted queue. It is not unusual within the 60 s window to see the deleted queue absent from # one list_queues call but present in the subsequent one. Deletion is eventual. # # sqs.delete_queue('https://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue_2') #=> true # def delete_queue(queue_url) req_hash = generate_request('DeleteQueue', :queue_url => queue_url) request_info(req_hash, SqsStatusParser.new(:logger => @logger)) rescue on_exception end # Retrieves the queue attribute(s). Returns a hash of attribute(s) or an exception. # # sqs.get_queue_attributes('https://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue') #=> # {"ApproximateNumberOfMessages" => "0", # "LastModifiedTimestamp" => "1240946032", # "CreatedTimestamp" => "1240816887", # "VisibilityTimeout" => "30", # "Policy" => "{"Version":"2008-10-17","Id":...}"} # # queue.get_queue_attributes('https://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue', "LastModifiedTimestamp", "VisibilityTimeout") #=> # {"LastModifiedTimestamp" => "1240946032", # "VisibilityTimeout" => "30"} # # http://docs.amazonwebservices.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/index.html?Query_QueryGetQueueAttributes.html def get_queue_attributes(queue_url, *attributes) attributes.flatten! attributes << 'All' if attributes.right_blank? params = amazonize_list('AttributeName', attributes) params.merge!(:queue_url => queue_url) req_hash = generate_request('GetQueueAttributes', params) request_info(req_hash, SqsGetQueueAttributesParser.new(:logger => @logger)) rescue on_exception end # Sets queue attribute. Returns +true+ or an exception. # # sqs.set_queue_attributes('https://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue', "VisibilityTimeout", 10) #=> true # # From the SQS Dev Guide: # "When you change a queue's attributes, the change can take up to 60 seconds to propagate # throughout the SQS system." # # NB: Attribute values may not be immediately available to other queries # for some time after an update. See the SQS documentation for # semantics, but in general propagation can take up to 60 s. # # see http://docs.amazonwebservices.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/index.html?Query_QuerySetQueueAttributes.html def set_queue_attributes(queue_url, attribute, value) req_hash = generate_request('SetQueueAttributes', 'Attribute.Name' => attribute, 'Attribute.Value' => value, :queue_url => queue_url) request_info(req_hash, SqsStatusParser.new(:logger => @logger)) rescue on_exception end # Add permissions to a queue. # # sqs.add_permissions('https://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue', # 'testLabel', ['125074342641','125074342642'], # ['SendMessage','SendMessage','ReceiveMessage']) #=> true # # +permissions+ is a hash of: AccountId => ActionName # (valid ActionNames: * | SendMessage | ReceiveMessage | DeleteMessage | ChangeMessageVisibility | GetQueueAttributes ) # # see http://docs.amazonwebservices.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/index.html?Query_QueryAddPermission.html # http://docs.amazonwebservices.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/index.html?acp-overview.html def add_permissions(queue_url, label, grantees, actions) params = amazonize_list('AWSAccountId', Array(grantees)) params.merge!(amazonize_list('ActionName', Array(actions))) params.merge!('Label' => label, :queue_url => queue_url ) req_hash = generate_request('AddPermission', params) request_info(req_hash, SqsStatusParser.new(:logger => @logger)) rescue on_exception end # Revoke any permissions in the queue policy that matches the +label+ parameter. # # sqs.remove_permissions('https://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue', # 'testLabel') # => true # # see http://docs.amazonwebservices.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/index.html?Query_QueryRemovePermission.html def remove_permissions(queue_url, label) req_hash = generate_request('RemovePermission', 'Label' => label, :queue_url => queue_url ) request_info(req_hash, SqsStatusParser.new(:logger => @logger)) rescue on_exception end # Retrieves a list of messages from queue. Returns an array of hashes in format: {:id=>'message_id', :body=>'message_body'} # # sqs.receive_message('https://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue',10, 5) #=> # [{"ReceiptHandle"=>"Euvo62...kw==", "MD5OfBody"=>"16af2171b5b83cfa35ce254966ba81e3", # "Body"=>"Goodbyte World!", "MessageId"=>"MUM4WlAyR...pYOTA="}, ..., {}] # # Normally this call returns fewer messages than the maximum specified, # even if they are available. # def receive_message(queue_url, max_number_of_messages=1, visibility_timeout=nil, attributes=nil) return [] if max_number_of_messages == 0 params = {} params.merge!(amazonize_list('AttributeName', Array(attributes))) unless attributes.right_blank? params.merge!('MaxNumberOfMessages' => max_number_of_messages, 'VisibilityTimeout' => visibility_timeout, :queue_url => queue_url ) req_hash = generate_post_request('ReceiveMessage', params) request_info(req_hash, SqsReceiveMessageParser.new(:logger => @logger)) rescue on_exception end # Change the visibility timeout of a specified message in a queue. # # sqs.change_message_visibility('https://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue', 'Euvo62...kw==', 33) #=> true # # see http://docs.amazonwebservices.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/index.html?Query_QueryChangeMessageVisibility.html def change_message_visibility(queue_url, receipt_handle, visibility_timeout) req_hash = generate_request('ChangeMessageVisibility', 'ReceiptHandle' => receipt_handle, 'VisibilityTimeout' => visibility_timeout, :queue_url => queue_url ) request_info(req_hash, SqsStatusParser.new(:logger => @logger)) rescue on_exception end # Sends a new message to a queue. Message size is limited to 8 KB. # If successful, this call returns a hash containing key/value pairs for # "MessageId" and "MD5OfMessageBody": # # sqs.send_message('https://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue', 'message_1') #=> # {"MessageId"=>"MEs4M0JKNlRCRTBBSENaMjROTk58QVFRNzNEREhDVFlFOVJDQ1JKNjF8UTdBRllCUlJUMjhKMUI1WDJSWDE=", # "MD5OfMessageBody"=>"16af2171b5b83cfa35ce254966ba81e3"} # # On failure, send_message raises an exception. # # def send_message(queue_url, message) req_hash = generate_post_request('SendMessage', :message => message, :queue_url => queue_url) request_info(req_hash, SqsSendMessagesParser.new(:logger => @logger)) rescue on_exception end # Same as send_message alias_method :push_message, :send_message # Deletes message from queue. Returns +true+ or an exception. Amazon # returns +true+ on deletion of non-existent messages. You must use the # receipt handle for a message to delete it, not the message ID. # # From the SQS Developer Guide: # "It is possible you will receive a message even after you have deleted it. This might happen # on rare occasions if one of the servers storing a copy of the message is unavailable when # you request to delete the message. The copy remains on the server and might be returned to # you again on a subsequent receive request. You should create your system to be # idempotent so that receiving a particular message more than once is not a problem. " # # sqs.delete_message('https://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue', 'Euvo62/1nlIet...ao03hd9Sa0w==') #=> true # def delete_message(queue_url, receipt_handle) req_hash = generate_request('DeleteMessage', 'ReceiptHandle' => receipt_handle, :queue_url => queue_url) request_info(req_hash, SqsStatusParser.new(:logger => @logger)) rescue on_exception end # Given the queue's short name, this call returns the queue URL or +nil+ if queue is not found # sqs.queue_url_by_name('my_awesome_queue') #=> 'https://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue' # def queue_url_by_name(queue_name) return queue_name if queue_name.include?('/') queue_urls = list_queues(queue_name) queue_urls.each do |queue_url| return queue_url if queue_name_by_url(queue_url) == queue_name end nil rescue on_exception end # Returns short queue name by url. # # RightSqs.queue_name_by_url('https://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue') #=> 'my_awesome_queue' # def self.queue_name_by_url(queue_url) queue_url[/[^\/]*$/] rescue on_exception end # Returns short queue name by url. # # sqs.queue_name_by_url('https://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue') #=> 'my_awesome_queue' # def queue_name_by_url(queue_url) self.class.queue_name_by_url(queue_url) rescue on_exception end # Returns approximate number of messages in queue. # # sqs.get_queue_length('https://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue') #=> 3 # def get_queue_length(queue_url) attrs = get_queue_attributes(queue_url) attrs['ApproximateNumberOfMessages'].to_i + attrs['ApproximateNumberOfMessagesNotVisible'].to_i rescue on_exception end # Removes all visible messages from queue. Return +true+ or an exception. # # sqs.clear_queue('https://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue') #=> true # def clear_queue(queue_url) while (pop_messages(queue_url, 10).length > 0) ; end # delete all messages in queue true rescue on_exception end # Pops (retrieves and deletes) up to 'number_of_messages' from queue. Returns an array of retrieved messages in format: [{:id=>'message_id', :body=>'message_body'}]. # # sqs.pop_messages('https://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue', 3) #=> # [{"ReceiptHandle"=>"Euvo62/...+Zw==", "MD5OfBody"=>"16af2...81e3", "Body"=>"Goodbyte World!", # "MessageId"=>"MEZI...JSWDE="}, {...}, ... , {...} ] # def pop_messages(queue_url, number_of_messages=1, attributes=nil) messages = receive_message(queue_url, number_of_messages, nil, attributes) messages.each do |message| delete_message(queue_url, message['ReceiptHandle']) end messages rescue on_exception end # Pops (retrieves and deletes) first accessible message from queue. Returns the message in format {:id=>'message_id', :body=>'message_body'} or +nil+. # # sqs.pop_message('https://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue') #=> # {:id=>"12345678904GEZX9746N|0N9ED344VK5Z3SV1DTM0|1RVYH4X3TJ0987654321", :body=>"message_1"} # def pop_message(queue_url, attributes=nil) messages = pop_messages(queue_url, 1, attributes) messages.right_blank? ? nil : messages[0] rescue on_exception end #----------------------------------------------------------------- # PARSERS: Status Response Parser #----------------------------------------------------------------- class SqsStatusParser < RightAWSParser # :nodoc: def tagend(name) if name == 'ResponseMetadata' @result = true end end end #----------------------------------------------------------------- # PARSERS: Queue #----------------------------------------------------------------- class SqsCreateQueueParser < RightAWSParser # :nodoc: def tagend(name) @result = @text if name == 'QueueUrl' end end class SqsListQueuesParser < RightAWSParser # :nodoc: def reset @result = [] end def tagend(name) @result << @text if name == 'QueueUrl' end end class SqsGetQueueAttributesParser < RightAWSParser # :nodoc: def reset @result = {} end def tagend(name) case name when 'Name' then @current_attribute = @text when 'Value' then @result[@current_attribute] = @text end end end #----------------------------------------------------------------- # PARSERS: Messages #----------------------------------------------------------------- class SqsReceiveMessageParser < RightAWSParser # :nodoc: def reset @result = [] end def tagstart(name, attributes) case name when 'Message' then @current_message = { } when 'Attribute' then @current_message['Attributes'] ||= {} @current_attribute_name = '' @current_attribute_value = '' end end def tagend(name) case name when 'MessageId' then @current_message['MessageId'] = @text when 'ReceiptHandle' then @current_message['ReceiptHandle'] = @text when 'MD5OfBody' then @current_message['MD5OfBody'] = @text when 'Name' then @current_attribute_name = @text when 'Value' then @current_attribute_value = @text when 'Attribute' then @current_message['Attributes'][@current_attribute_name] = @current_attribute_value when 'Body' then @current_message['Body'] = @text; @result << @current_message end end end class SqsSendMessagesParser < RightAWSParser # :nodoc: def reset @result = {} end def tagend(name) case name when 'MessageId' then @result['MessageId'] = @text when 'MD5OfMessageBody' then @result['MD5OfMessageBody'] = @text end end end end end ================================================ FILE: lib/sqs/right_sqs_interface.rb ================================================ # # Copyright (c) 2007-2008 RightScale Inc # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # module RightAws class SqsInterface < RightAwsBase include RightAwsBaseInterface API_VERSION = "2007-05-01" DEFAULT_HOST = "queue.amazonaws.com" DEFAULT_PORT = 443 DEFAULT_PROTOCOL = 'https' REQUEST_TTL = 30 DEFAULT_VISIBILITY_TIMEOUT = 30 @@bench = AwsBenchmarkingBlock.new def self.bench_xml @@bench.xml end def self.bench_sqs @@bench.service end @@api = API_VERSION def self.api @@api end # Creates a new SqsInterface instance. # # sqs = RightAws::SqsInterface.new('1E3GDYEOGFJPIT75KDT40','hgTHt68JY07JKUY08ftHYtERkjgtfERn57DFE379', {:logger => Logger.new('/tmp/x.log')}) # # Params is a hash: # # {:server => 'queue.amazonaws.com' # Amazon service host: 'queue.amazonaws.com'(default) # :port => 443 # Amazon service port: 80 or 443(default) # :signature_version => '0' # The signature version : '0', '1' or '2'(default) # :logger => Logger Object} # Logger instance: logs to STDOUT if omitted } # def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={}) init({ :name => 'SQS', :default_host => ENV['SQS_URL'] ? URI.parse(ENV['SQS_URL']).host : DEFAULT_HOST, :default_port => ENV['SQS_URL'] ? URI.parse(ENV['SQS_URL']).port : DEFAULT_PORT, :default_protocol => ENV['SQS_URL'] ? URI.parse(ENV['SQS_URL']).scheme : DEFAULT_PROTOCOL }, aws_access_key_id || ENV['AWS_ACCESS_KEY_ID'], aws_secret_access_key || ENV['AWS_SECRET_ACCESS_KEY'], params) end #----------------------------------------------------------------- # Requests #----------------------------------------------------------------- # Generates a request hash for the query API def generate_request(action, params={}) # :nodoc: # Sometimes we need to use queue uri (delete queue etc) # In that case we will use Symbol key: 'param[:queue_url]' service = params[:queue_url] ? URI(params[:queue_url]).path : '/' # remove unset(=optional) and symbolyc keys params.each{ |key, value| params.delete(key) if (value.nil? || key.is_a?(Symbol)) } # prepare output hash service_hash = { "Action" => action, "Expires" => (Time.now + REQUEST_TTL).utc.strftime("%Y-%m-%dT%H:%M:%SZ"), "AWSAccessKeyId" => @aws_access_key_id, "Version" => API_VERSION } service_hash.update(params) service_params = signed_service_params(@aws_secret_access_key, service_hash, :get, @params[:server], service) request = Net::HTTP::Get.new("#{AwsUtils::URLencode(service)}?#{service_params}") # prepare output hash { :request => request, :server => @params[:server], :port => @params[:port], :protocol => @params[:protocol] } end # Generates a request hash for the REST API def generate_rest_request(method, param) # :nodoc: queue_uri = param[:queue_url] ? URI(param[:queue_url]).path : '/' message = param[:message] # extract message body if nesessary # remove unset(=optional) and symbolyc keys param.each{ |key, value| param.delete(key) if (value.nil? || key.is_a?(Symbol)) } # created request param_to_str = param.to_a.collect{|key,val| key.to_s + "=" + CGI::escape(val.to_s) }.join("&") param_to_str = "?#{param_to_str}" unless param_to_str.right_blank? request = "Net::HTTP::#{method.capitalize}".right_constantize.new("#{queue_uri}#{param_to_str}") request.body = message if message # set main headers request['content-md5'] = '' request['Content-Type'] = 'text/plain' request['Date'] = Time.now.httpdate # generate authorization string auth_string = "#{method.upcase}\n#{request['content-md5']}\n#{request['Content-Type']}\n#{request['Date']}\n#{CGI::unescape(queue_uri)}" signature = AwsUtils::sign(@aws_secret_access_key, auth_string) # set other headers request['Authorization'] = "AWS #{@aws_access_key_id}:#{signature}" request['AWS-Version'] = API_VERSION # prepare output hash { :request => request, :server => @params[:server], :port => @params[:port], :protocol => @params[:protocol] } end # Sends request to Amazon and parses the response # Raises AwsError if any banana happened def request_info(request, parser) # :nodoc: request_info_impl(:sqs_connection, @@bench, request, parser) end # Creates new queue. Returns new queue link. # # sqs.create_queue('my_awesome_queue') #=> 'http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue' # # PS Some queue based requests may not become available until a couple of minutes after queue creation # (permission grant and removal for example) # def create_queue(queue_name, default_visibility_timeout=nil) req_hash = generate_request('CreateQueue', 'QueueName' => queue_name, 'DefaultVisibilityTimeout' => default_visibility_timeout || DEFAULT_VISIBILITY_TIMEOUT ) request_info(req_hash, SqsCreateQueueParser.new(:logger => @logger)) end # Lists all queues owned by this user that have names beginning with +queue_name_prefix+. If +queue_name_prefix+ is omitted then retrieves a list of all queues. # # sqs.create_queue('my_awesome_queue') # sqs.create_queue('my_awesome_queue_2') # sqs.list_queues('my_awesome') #=> ['http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue','http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue_2'] # def list_queues(queue_name_prefix=nil) req_hash = generate_request('ListQueues', 'QueueNamePrefix' => queue_name_prefix) request_info(req_hash, SqsListQueuesParser.new(:logger => @logger)) rescue on_exception end # Deletes queue (queue must be empty or +force_deletion+ must be set to true). Queue is identified by url. Returns +true+ or an exception. # # sqs.delete_queue('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue_2') #=> true # def delete_queue(queue_url, force_deletion = false) req_hash = generate_request('DeleteQueue', 'ForceDeletion' => force_deletion.to_s, :queue_url => queue_url) request_info(req_hash, SqsStatusParser.new(:logger => @logger)) rescue on_exception end # Retrieves the queue attribute(s). Returns a hash of attribute(s) or an exception. # # sqs.get_queue_attributes('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue') #=> {"ApproximateNumberOfMessages"=>"0", "VisibilityTimeout"=>"30"} # def get_queue_attributes(queue_url, attribute='All') req_hash = generate_request('GetQueueAttributes', 'Attribute' => attribute, :queue_url => queue_url) request_info(req_hash, SqsGetQueueAttributesParser.new(:logger => @logger)) rescue on_exception end # Sets queue attribute. Returns +true+ or an exception. # # sqs.set_queue_attributes('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue', "VisibilityTimeout", 10) #=> true # # P.S. Amazon returns success even if the attribute does not exist. Also, attribute values may not be immediately available to other queries # for some time after an update (see the SQS documentation for # semantics). def set_queue_attributes(queue_url, attribute, value) req_hash = generate_request('SetQueueAttributes', 'Attribute' => attribute, 'Value' => value, :queue_url => queue_url) request_info(req_hash, SqsStatusParser.new(:logger => @logger)) rescue on_exception end # Sets visibility timeout. Returns +true+ or an exception. # # sqs.set_visibility_timeout('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue', 15) #=> true # # See also: +set_queue_attributes+ # def set_visibility_timeout(queue_url, visibility_timeout=nil) req_hash = generate_request('SetVisibilityTimeout', 'VisibilityTimeout' => visibility_timeout || DEFAULT_VISIBILITY_TIMEOUT, :queue_url => queue_url ) request_info(req_hash, SqsStatusParser.new(:logger => @logger)) rescue on_exception end # Retrieves visibility timeout. # # sqs.get_visibility_timeout('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue') #=> 15 # # See also: +get_queue_attributes+ # def get_visibility_timeout(queue_url) req_hash = generate_request('GetVisibilityTimeout', :queue_url => queue_url ) request_info(req_hash, SqsGetVisibilityTimeoutParser.new(:logger => @logger)) rescue on_exception end # Adds grants for user (identified by email he registered at Amazon). Returns +true+ or an exception. Permission = 'FULLCONTROL' | 'RECEIVEMESSAGE' | 'SENDMESSAGE'. # # sqs.add_grant('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue', 'my_awesome_friend@gmail.com', 'FULLCONTROL') #=> true # def add_grant(queue_url, grantee_email_address, permission = nil) req_hash = generate_request('AddGrant', 'Grantee.EmailAddress' => grantee_email_address, 'Permission' => permission, :queue_url => queue_url) request_info(req_hash, SqsStatusParser.new(:logger => @logger)) rescue on_exception end # Retrieves hash of +grantee_id+ => +perms+ for this queue: # # sqs.list_grants('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue') #=> # {"000000000000000000000001111111111117476c7fea6efb2c3347ac3ab2792a"=>{:name=>"root", :perms=>["FULLCONTROL"]}, # "00000000000000000000000111111111111e5828344600fc9e4a784a09e97041"=>{:name=>"myawesomefriend", :perms=>["FULLCONTROL"]} # def list_grants(queue_url, grantee_email_address=nil, permission = nil) req_hash = generate_request('ListGrants', 'Grantee.EmailAddress' => grantee_email_address, 'Permission' => permission, :queue_url => queue_url) response = request_info(req_hash, SqsListGrantsParser.new(:logger => @logger)) # One user may have up to 3 permission records for every queue. # We will join these records to one. result = {} response.each do |perm| id = perm[:id] # create hash for new user if unexisit result[id] = {:perms=>[]} unless result[id] # fill current grantee params result[id][:perms] << perm[:permission] result[id][:name] = perm[:name] end result end # Revokes permission from user. Returns +true+ or an exception. # # sqs.remove_grant('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue', 'my_awesome_friend@gmail.com', 'FULLCONTROL') #=> true # def remove_grant(queue_url, grantee_email_address_or_id, permission = nil) grantee_key = grantee_email_address_or_id.include?('@') ? 'Grantee.EmailAddress' : 'Grantee.ID' req_hash = generate_request('RemoveGrant', grantee_key => grantee_email_address_or_id, 'Permission' => permission, :queue_url => queue_url) request_info(req_hash, SqsStatusParser.new(:logger => @logger)) rescue on_exception end # Retrieves a list of messages from queue. Returns an array of hashes in format: {:id=>'message_id', body=>'message_body'} # # sqs.receive_messages('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue',10, 5) #=> # [{:id=>"12345678904GEZX9746N|0N9ED344VK5Z3SV1DTM0|1RVYH4X3TJ0987654321", :body=>"message_1"}, ..., {}] # # P.S. Usually returns fewer messages than requested even if they are available. # def receive_messages(queue_url, number_of_messages=1, visibility_timeout=nil) return [] if number_of_messages == 0 req_hash = generate_rest_request('GET', 'NumberOfMessages' => number_of_messages, 'VisibilityTimeout' => visibility_timeout, :queue_url => "#{queue_url}/front" ) request_info(req_hash, SqsReceiveMessagesParser.new(:logger => @logger)) rescue on_exception end # Peeks message from queue by message id. Returns message in format of {:id=>'message_id', :body=>'message_body'} or +nil+. # # sqs.peek_message('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue', '1234567890...0987654321') #=> # {:id=>"12345678904GEZX9746N|0N9ED344VK5Z3SV1DTM0|1RVYH4X3TJ0987654321", :body=>"message_1"} # def peek_message(queue_url, message_id) req_hash = generate_rest_request('GET', :queue_url => "#{queue_url}/#{CGI::escape message_id}" ) messages = request_info(req_hash, SqsReceiveMessagesParser.new(:logger => @logger)) messages.right_blank? ? nil : messages[0] rescue on_exception end # Sends new message to queue.Returns 'message_id' or raises an exception. # # sqs.send_message('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue', 'message_1') #=> "1234567890...0987654321" # def send_message(queue_url, message) req_hash = generate_rest_request('PUT', :message => message, :queue_url => "#{queue_url}/back") request_info(req_hash, SqsSendMessagesParser.new(:logger => @logger)) rescue on_exception end # Deletes message from queue. Returns +true+ or an exception. Amazon # returns +true+ on deletion of non-existent messages. # # sqs.delete_message('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue', '12345678904...0987654321') #=> true # def delete_message(queue_url, message_id) req_hash = generate_request('DeleteMessage', 'MessageId' => message_id, :queue_url => queue_url) request_info(req_hash, SqsStatusParser.new(:logger => @logger)) rescue on_exception end # Changes message visibility timeout. Returns +true+ or an exception. # # sqs.change_message_visibility('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue', '1234567890...0987654321', 10) #=> true # def change_message_visibility(queue_url, message_id, visibility_timeout=0) req_hash = generate_request('ChangeMessageVisibility', 'MessageId' => message_id, 'VisibilityTimeout' => visibility_timeout.to_s, :queue_url => queue_url) request_info(req_hash, SqsStatusParser.new(:logger => @logger)) rescue on_exception end # Returns queue url by queue short name or +nil+ if queue is not found # # sqs.queue_url_by_name('my_awesome_queue') #=> 'http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue' # def queue_url_by_name(queue_name) return queue_name if queue_name.include?('/') queue_urls = list_queues(queue_name) queue_urls.each do |queue_url| return queue_url if queue_name_by_url(queue_url) == queue_name end nil rescue on_exception end # Returns short queue name by url. # # RightSqs.queue_name_by_url('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue') #=> 'my_awesome_queue' # def self.queue_name_by_url(queue_url) queue_url[/[^\/]*$/] rescue on_exception end # Returns short queue name by url. # # sqs.queue_name_by_url('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue') #=> 'my_awesome_queue' # def queue_name_by_url(queue_url) self.class.queue_name_by_url(queue_url) rescue on_exception end # Returns approximate number of messages in queue. # # sqs.get_queue_length('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue') #=> 3 # def get_queue_length(queue_url) get_queue_attributes(queue_url)['ApproximateNumberOfMessages'].to_i rescue on_exception end # Removes all visible messages from queue. Return +true+ or an exception. # # sqs.clear_queue('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue') #=> true # def clear_queue(queue_url) while (m = pop_message(queue_url)) ; end # delete all messages in queue true rescue on_exception end # Deletes queue then re-creates it (restores attributes also). The fastest method to clear big queue or queue with invisible messages. Return +true+ or an exception. # # sqs.force_clear_queue('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue') #=> true # # PS This function is no longer supported. Amazon has changed the SQS semantics to require at least 60 seconds between # queue deletion and creation. Hence this method will fail with an exception. # def force_clear_queue(queue_url) queue_name = queue_name_by_url(queue_url) queue_attributes = get_queue_attributes(queue_url) force_delete_queue(queue_url) create_queue(queue_name) # hmmm... The next line is a trick. Amazon do not want change attributes immediately after queue creation # So we do 'empty' get_queue_attributes. Probably they need some time to allow attributes change. get_queue_attributes(queue_url) queue_attributes.each{ |attribute, value| set_queue_attributes(queue_url, attribute, value) } true rescue on_exception end # Deletes queue even if it has messages. Return +true+ or an exception. # # force_delete_queue('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue') #=> true # # P.S. same as delete_queue('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue', true) def force_delete_queue(queue_url) delete_queue(queue_url, true) rescue on_exception end # Reads first accessible message from queue. Returns message as a hash: {:id=>'message_id', :body=>'message_body'} or +nil+. # # sqs.receive_message('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue', 10) #=> # {:id=>"12345678904GEZX9746N|0N9ED344VK5Z3SV1DTM0|1RVYH4X3TJ0987654321", :body=>"message_1"} # def receive_message(queue_url, visibility_timeout=nil) result = receive_messages(queue_url, 1, visibility_timeout) result.right_blank? ? nil : result[0] rescue on_exception end # Same as send_message alias_method :push_message, :send_message # Pops (retrieves and deletes) up to 'number_of_messages' from queue. Returns an array of retrieved messages in format: [{:id=>'message_id', :body=>'message_body'}]. # # sqs.pop_messages('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue', 3) #=> # [{:id=>"12345678904GEZX9746N|0N9ED344VK5Z3SV1DTM0|1RVYH4X3TJ0987654321", :body=>"message_1"}, ..., {}] # def pop_messages(queue_url, number_of_messages=1) messages = receive_messages(queue_url, number_of_messages) messages.each do |message| delete_message(queue_url, message[:id]) end messages rescue on_exception end # Pops (retrieves and deletes) first accessible message from queue. Returns the message in format {:id=>'message_id', :body=>'message_body'} or +nil+. # # sqs.pop_message('http://queue.amazonaws.com/ZZ7XXXYYYBINS/my_awesome_queue') #=> # {:id=>"12345678904GEZX9746N|0N9ED344VK5Z3SV1DTM0|1RVYH4X3TJ0987654321", :body=>"message_1"} # def pop_message(queue_url) messages = pop_messages(queue_url) messages.right_blank? ? nil : messages[0] rescue on_exception end #----------------------------------------------------------------- # PARSERS: Status Response Parser #----------------------------------------------------------------- class SqsStatusParser < RightAWSParser # :nodoc: def tagend(name) if name == 'StatusCode' @result = @text=='Success' ? true : false end end end #----------------------------------------------------------------- # PARSERS: Queue #----------------------------------------------------------------- class SqsCreateQueueParser < RightAWSParser # :nodoc: def tagend(name) @result = @text if name == 'QueueUrl' end end class SqsListQueuesParser < RightAWSParser # :nodoc: def reset @result = [] end def tagend(name) @result << @text if name == 'QueueUrl' end end class SqsGetQueueAttributesParser < RightAWSParser # :nodoc: def reset @result = {} end def tagend(name) case name when 'Attribute' ; @current_attribute = @text when 'Value' ; @result[@current_attribute] = @text # when 'StatusCode'; @result['status_code'] = @text # when 'RequestId' ; @result['request_id'] = @text end end end #----------------------------------------------------------------- # PARSERS: Timeouts #----------------------------------------------------------------- class SqsGetVisibilityTimeoutParser < RightAWSParser # :nodoc: def tagend(name) @result = @text.to_i if name == 'VisibilityTimeout' end end #----------------------------------------------------------------- # PARSERS: Permissions #----------------------------------------------------------------- class SqsListGrantsParser < RightAWSParser # :nodoc: def reset @result = [] end def tagstart(name, attributes) @current_perms = {} if name == 'GrantList' end def tagend(name) case name when 'ID' ; @current_perms[:id] = @text when 'DisplayName'; @current_perms[:name] = @text when 'Permission' ; @current_perms[:permission] = @text when 'GrantList' ; @result << @current_perms end end end #----------------------------------------------------------------- # PARSERS: Messages #----------------------------------------------------------------- class SqsReceiveMessagesParser < RightAWSParser # :nodoc: def reset @result = [] end def tagstart(name, attributes) @current_message = {} if name == 'Message' end def tagend(name) case name when 'MessageId' ; @current_message[:id] = @text when 'MessageBody'; @current_message[:body] = @text when 'Message' ; @result << @current_message end end end class SqsSendMessagesParser < RightAWSParser # :nodoc: def tagend(name) @result = @text if name == 'MessageId' end end end end ================================================ FILE: right_aws.gemspec ================================================ #-- -*- mode: ruby; encoding: utf-8 -*- # Copyright: Copyright (c) 2010 RightScale, Inc. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # 'Software'), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #++ require 'rubygems' require File.expand_path(File.join(File.dirname(__FILE__), "lib", "awsbase", "version")) Gem::Specification.new do |spec| spec.name = 'right_aws' spec.rubyforge_project = 'rightaws' spec.version = RightAws::VERSION::STRING spec.authors = ['RightScale, Inc.'] spec.email = 'support@rightscale.com' spec.summary = 'Interface classes for the Amazon EC2, SQS, and S3 Web Services' spec.rdoc_options = ['--main', 'README.txt', '--title', ''] spec.extra_rdoc_files = ['README.txt'] spec.required_ruby_version = '>= 1.8.7' spec.require_path = 'lib' spec.add_dependency('right_http_connection', '>= 1.2.5') #spec.requirements << "uuidtools >= 1.0.7 if you want to use ActiveSdb" spec.requirements << "libxml-ruby >= 0.5.2.0 is encouraged" spec.add_development_dependency('rake') spec.summary = 'The RightScale AWS gems have been designed to provide a robust, fast, and secure interface to Amazon EC2, EBS, S3, SQS, SDB, and CloudFront.' spec.description = <<-EOF == DESCRIPTION: The RightScale AWS gems have been designed to provide a robust, fast, and secure interface to Amazon EC2, EBS, S3, SQS, SDB, and CloudFront. These gems have been used in production by RightScale since late 2006 and are being maintained to track enhancements made by Amazon. The RightScale AWS gems comprise: - RightAws::Ec2 -- interface to Amazon EC2 (Elastic Compute Cloud) and the associated EBS (Elastic Block Store) - RightAws::S3 and RightAws::S3Interface -- interface to Amazon S3 (Simple Storage Service) - RightAws::Sqs and RightAws::SqsInterface -- interface to first-generation Amazon SQS (Simple Queue Service) (API version 2007-05-01) - RightAws::SqsGen2 and RightAws::SqsGen2Interface -- interface to second-generation Amazon SQS (Simple Queue Service) (API version 2008-01-01) - RightAws::SdbInterface and RightAws::ActiveSdb -- interface to Amazon SDB (SimpleDB) - RightAws::AcfInterface -- interface to Amazon CloudFront, a content distribution service == FEATURES: - Full programmmatic access to EC2, EBS, S3, SQS, SDB, and CloudFront. - Complete error handling: all operations check for errors and report complete error information by raising an AwsError. - Persistent HTTP connections with robust network-level retry layer using RightHttpConnection). This includes socket timeouts and retries. - Robust HTTP-level retry layer. Certain (user-adjustable) HTTP errors returned by Amazon's services are classified as temporary errors. These errors are automaticallly retried using exponentially increasing intervals. The number of retries is user-configurable. - Fast REXML-based parsing of responses (as fast as a pure Ruby solution allows). - Uses libxml (if available) for faster response parsing. - Support for large S3 list operations. Buckets and key subfolders containing many (> 1000) keys are listed in entirety. Operations based on list (like bucket clear) work on arbitrary numbers of keys. - Support for streaming GETs from S3, and streaming PUTs to S3 if the data source is a file. - Support for single-threaded usage, multithreaded usage, as well as usage with multiple AWS accounts. - Support for both first- and second-generation SQS (API versions 2007-05-01 and 2008-01-01). These versions of SQS are not compatible. - Support for signature versions 0 and 1 on SQS, SDB, and EC2. - Interoperability with any cloud running Eucalyptus (http://eucalyptus.cs.ucsb.edu) - Test suite (requires AWS account to do "live" testing). EOF candidates = Dir.glob('{lib,test}/**/*') + ['History.txt', 'Manifest.txt', 'README.txt', 'Rakefile', 'right_aws.gemspec'] spec.files = candidates.sort spec.test_files = Dir.glob('test/**/*') end ================================================ FILE: test/README.mdown ================================================ # Notes and tips for developers ## Setting up credentials for testing Before you can run any tests, you need to set up credentials (API key and secret) that will be used when talking to AWS. The credentials are loaded in `test/test_credentials.rb` and are expected to be found in `~/.rightscale/testcredentials.rb` and look like this: TestCredentials.aws_access_key_id= 'AAAAAAAAAAAAAAAAAAAA' TestCredentials.aws_secret_access_key= 'asdfasdfsadf' TestCredentials.account_number= '???' If you prefer to store your secret key in the OS X keychain, you can do this: def secret_access_key_from_keychain (key_id) dump = `security -q find-generic-password -a "#{key_id}" -g 2>&1` dump[/password: "(.*)"/, 1] end TestCredentials.aws_access_key_id= 'AAAAAAAAAAAAAAAAAAAA' TestCredentials.aws_secret_access_key= secret_access_key_from_keychain(TestCredentials.aws_access_key_id) TestCredentials.account_number= '???' ## Running tests There is no test suite that runs all tests. Each module is tested separately. E.g., to run the Load Balancer tests, run `rake testelb`. Run `rake -T` for a full list. Some tests need to launch services on AWS to have something to test. This means two things: 1. Running all the tests will cost you money. 2. You will need to shut down some services separately once you are done, or things will keep running and cost you money. As an example, the ELB and Route 53 tests need a load balancer for testing. Starting a load balancer for every test would make every test case cost as much as running the LB for one hour, so it makes more sense to leave it running until it's no longer needed. The ELB tests contain instructions for shutting down the load balancer. ================================================ FILE: test/acf/test_helper.rb ================================================ require 'test/unit' require File.dirname(__FILE__) + '/../../lib/right_aws' ================================================ FILE: test/acf/test_right_acf.rb ================================================ require File.dirname(__FILE__) + '/test_helper.rb' class TestAcf < Test::Unit::TestCase RIGHT_OBJECT_TEXT = 'Right test message' STDOUT.sync = true def setup @acf= Rightscale::AcfInterface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key) @s3 = Rightscale::S3.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key) @bucket_name = "right-acf-awesome-test-bucket-xxx1" @bucket_domain = "#{@bucket_name}.s3.amazonaws.com" end def test_01_list_distributions_part1 distributions = nil assert_nothing_raised(Rightscale::AwsError) do distributions = @acf.list_distributions end assert distributions.is_a?(Array) end def test_02_try_to_create_for_bad_bucket # a bucket does not exist assert_raise(Rightscale::AwsError) do @acf.create_distribution("right-cloudfront-awesome-test-bucket-not-exist", "Mustn't to be born", true) end # a bucket is not a domain naming complied guy bucket_name = 'right_cloudfront_awesome_test_bucket_BAD_XXX' @s3.bucket(bucket_name, :create) assert_raise(Rightscale::AwsError) do @acf.create_distribution(bucket_name, "Mustn't to be born", true) end end def test_02_x_delete_bad_bucket bucket_name = 'right_cloudfront_awesome_test_bucket_BAD_XXX' @s3.bucket(bucket_name, false).delete end def test_03_create comment = 'WooHoo!!!' # create a test bucket @s3.bucket(@bucket_name, :create) # create a distribution distribution = @acf.create_distribution(@bucket_domain, comment, true) assert_equal comment, distribution[:comment] assert distribution[:cnames].size == 0 assert distribution[:enabled] end def test_04_list_distributions_part2 distributions = @acf.list_distributions assert distributions.size > 0 end def get_test_distribution @acf.list_distributions.select{ |d| d[:origin] == @bucket_domain }.first end def test_05_get_distribution old = get_test_distribution assert_nothing_raised do @acf.get_distribution(old[:aws_id]) end end def test_06_get_and_set_config config = nil old = get_test_distribution assert_nothing_raised do config = @acf.get_distribution_config(old[:aws_id]) end # change a config config[:enabled] = false config[:cnames] << 'xxx1.myawesomesite.com' config[:cnames] << 'xxx2.myawesomesite.com' # set config set_config_result = nil assert_nothing_raised do set_config_result = @acf.set_distribution_config(old[:aws_id], config) end assert set_config_result # reget the config and check new_config = nil assert_nothing_raised do new_config = @acf.get_distribution_config(old[:aws_id]) end assert !new_config[:enabled] assert_equal new_config[:cnames].sort, ['xxx1.myawesomesite.com', 'xxx2.myawesomesite.com'] assert_not_equal config[:e_tag], new_config[:e_tag] # try to update the old config again (must fail because ETAG has changed) assert_raise(Rightscale::AwsError) do @acf.set_distribution_config(old[:aws_id], config) end end def test_08_delete_distribution # we need ETAG so use get_distribution distribution = @acf.get_distribution(get_test_distribution[:aws_id]) # try to delete a distribution # should fail because if distribution[:status] == 'InProgress' # should fail because the distribution is not deployed yet assert_raise(Rightscale::AwsError) do @acf.delete_distribution(distribution[:aws_id], distribution[:e_tag]) end # wait for a deployed state print "waiting up to 5 min while the distribution is being deployed: " 100.times do print '.' distribution = @acf.get_distribution(distribution[:aws_id]) if distribution[:status] == 'Deployed' print ' done' break end sleep 3 end puts end # only disabled and deployed distribution can be deleted assert_equal 'Deployed', distribution[:status] assert !distribution[:enabled] # delete the distribution assert_nothing_raised do @acf.delete_distribution(distribution[:aws_id], distribution[:e_tag]) end end def test_09_drop_bucket assert @s3.bucket(@bucket_name).delete end end ================================================ FILE: test/awsbase/test_helper.rb ================================================ require 'test/unit' require File.dirname(__FILE__) + '/../../lib/right_aws' ================================================ FILE: test/awsbase/test_right_awsbase.rb ================================================ require File.dirname(__FILE__) + '/test_helper.rb' class TestAwsbase < Test::Unit::TestCase def setup end def test_01_create_describe_key_pairs end end ================================================ FILE: test/ec2/test_helper.rb ================================================ require 'test/unit' require File.dirname(__FILE__) + '/../../lib/right_aws' ================================================ FILE: test/ec2/test_right_ec2.rb ================================================ require File.dirname(__FILE__) + '/test_helper.rb' class TestEc2 < Test::Unit::TestCase # Some of RightEc2 instance methods concerning instance launching and image registration # are not tested here due to their potentially risk. def setup @ec2 = Rightscale::Ec2.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key) @key = 'right_ec2_awesome_test_key' @group = 'right_ec2_awesome_test_security_group' end def test_01_create_describe_key_pairs new_key = @ec2.create_key_pair(@key) assert new_key[:aws_material][/BEGIN RSA PRIVATE KEY/], "New key material is absent" keys = @ec2.describe_key_pairs assert keys.map{|key| key[:aws_key_name] }.include?(@key), "#{@key} must exist" end def test_02_create_security_group assert @ec2.create_security_group(@group,'My awesone test group'), 'Create_security_group fail' group = @ec2.describe_security_groups([@group])[0] assert_equal @group, group[:aws_group_name], 'Group must be created but does not exist' end def test_03_perms_add assert @ec2.authorize_security_group_named_ingress(@group, TestCredentials.account_number, 'default') assert @ec2.authorize_security_group_IP_ingress(@group, 80,80,'udp','192.168.1.0/8') end def test_04_check_new_perms_exist assert_equal 2, @ec2.describe_security_groups([@group])[0][:aws_perms].size end def test_05_perms_remove assert @ec2.revoke_security_group_IP_ingress(@group, 80,80,'udp','192.168.1.0/8') assert @ec2.revoke_security_group_named_ingress(@group, TestCredentials.account_number, 'default') end def test_06_describe_images images = @ec2.describe_images assert images.size>0, 'Amazon must have at least some public images' # unknown image assert_raise(Rightscale::AwsError){ @ec2.describe_images(['ami-ABCDEFGH'])} end def test_07_describe_instanses assert @ec2.describe_instances # unknown image assert_raise(Rightscale::AwsError){ @ec2.describe_instances(['i-ABCDEFGH'])} end def test_08_delete_security_group assert @ec2.delete_security_group(@group), 'Delete_security_group fail' end def test_09_delete_key_pair assert @ec2.delete_key_pair(@key), 'Delete_key_pair fail' ## Hmmm... Amazon does not through the exception any more. It now just returns a 'true' if the key does not exist any more... ## # key must be deleted already ## assert_raise(Rightscale::AwsError) { @ec2.delete_key_pair(@key) } end def test_10_signature_version_0 ec2 = Rightscale::Ec2.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key, :signature_version => '0') images = ec2.describe_images assert images.size>0, 'Amazon must have at least some public images' # check that the request has correct signature version assert ec2.last_request.path.include?('SignatureVersion=0') end def test_11_regions regions = nil assert_nothing_raised do regions = @ec2.describe_regions end # check we got more that 0 regions assert regions.size > 0 # check an access to regions regions.each do |region| regional_ec2 = Rightscale::Ec2.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key, :region => region) # do we have a correct endpoint server? assert_equal "#{region}.ec2.amazonaws.com", regional_ec2.params[:server] # get a list of images from every region images = nil assert_nothing_raised do images = regional_ec2.describe_regions end # every region must have images assert images.size > 0 end end def test_12_endpoint_url ec2 = Rightscale::Ec2.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key, :endpoint_url => 'a://b.c:0/d/', :region => 'z') # :endpoint_url has a priority hence :region should be ommitted assert_equal 'a', ec2.params[:protocol] assert_equal 'b.c', ec2.params[:server] assert_equal '/d/', ec2.params[:service] assert_equal 0, ec2.params[:port] assert_nil ec2.params[:region] end end ================================================ FILE: test/elb/test_helper.rb ================================================ require 'test/unit' require File.dirname(__FILE__) + '/../../lib/right_aws' ================================================ FILE: test/elb/test_right_elb.rb ================================================ require File.dirname(__FILE__) + '/test_helper.rb' class TestElb < Test::Unit::TestCase STDOUT.sync = true BALANCER_NAME = 'right-aws-test-lb' def setup @elb = Rightscale::ElbInterface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key, :logger => Logger.new('/dev/null')) unless @elb.describe_load_balancers.detect { |lb| lb[:load_balancer_name] == BALANCER_NAME } @elb.create_load_balancer(BALANCER_NAME, %w[us-east-1b], []) end @lb = @elb.describe_load_balancers.detect { |lb| lb[:load_balancer_name] == BALANCER_NAME } end # At the end of the day when you want to shut down the test balancer: # * Uncomment this method. # * Comment out all test except one. # * Run this test file. # # def teardown # @elb.delete_load_balancer BALANCER_NAME # end def test_00_describe_load_balancers items = @elb.describe_load_balancers assert items.is_a?(Array) end def test_description assert_match /^#{BALANCER_NAME}-\d+\.us-east-1\.elb\.amazonaws\.com$/, @lb[:dns_name] end def test_description_has_canonical_hosted_zone_name assert_match /^#{BALANCER_NAME}-\d+\.us-east-1\.elb\.amazonaws\.com$/, @lb[:canonical_hosted_zone_name] end def test_description_has_canonical_hosted_zone_name_id assert_match /^[A-Z0-9]+$/, @lb[:canonical_hosted_zone_name_id] end end ================================================ FILE: test/http_connection.rb ================================================ =begin Copyright (c) 2007 RightScale, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. =end # Stub extension/redefinition of RightHttpConnection for testing purposes. require 'net/http' require 'rubygems' require 'right_http_connection' module Net class HTTPResponse alias_method :real_body, :body def setmsg(msg) @mymsg = msg end def body # defined?() helps us to get rid of a bunch of 'warnings' (defined?(@mymsg) && @mymsg) ? @mymsg : real_body end end end module Rightscale class HttpConnection @@response_stack = [] alias_method :real_request, :request def request(request_params, &block) if(@@response_stack.length == 0) return real_request(request_params, &block) end if(block) # Do something special else next_response = HttpConnection::pop() classname = Net::HTTPResponse::CODE_TO_OBJ["#{next_response[:code]}"] response = classname.new("1.1", next_response[:code], next_response[:msg]) if(next_response[:msg]) response.setmsg(next_response[:msg]) end response end end def self.reset @@response_stack = [] end def self.push(code, msg=nil) response = {:code => code, :msg => msg} @@response_stack << response end def self.pop @@response_stack.pop end def self.length @@response_stack.length end end end ================================================ FILE: test/rds/test_helper.rb ================================================ require 'test/unit' require File.dirname(__FILE__) + '/../../lib/right_aws' ================================================ FILE: test/rds/test_right_rds.rb ================================================ require File.dirname(__FILE__) + '/test_helper.rb' class TestRds < Test::Unit::TestCase STDOUT.sync = true TEST_SG_NAME = 'RightRdsSGTest0123456789' def setup @rds = Rightscale::RdsInterface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key, :logger => Logger.new('/dev/null')) end def test_00_instances assert_nothing_raised do items = @rds.describe_db_instances assert items.is_a?(Array) end # assert_nothing_raised do @rds.describe_db_instances do |response| assert response.is_a?(Hash) assert response[:db_instances].is_a?(Array) end end end def test_10_security_groups assert_nothing_raised do items = @rds.describe_db_security_groups assert items.is_a?(Array) end # assert_nothing_raised do @rds.describe_db_security_groups do |response| assert response.is_a?(Hash) assert response[:db_security_groups].is_a?(Array) end end end def test_11_remove_security_group @rds.delete_db_security_group rescue nil end def test_12_create_security_group sg = nil assert_nothing_raised do sg = @rds.create_db_security_group(TEST_SG_NAME, 'sg-description') end assert sg.is_a?(Hash) end def test_13_authorize assert_nothing_raised do sg = @rds.authorize_db_security_group_ingress(TEST_SG_NAME, :cidrip => '131.0.0.1/8') assert sg.is_a?(Hash) assert_equal 1, sg[:ip_ranges].size end assert_nothing_raised do sg = @rds.authorize_db_security_group_ingress(TEST_SG_NAME, :ec2_security_group_owner => '826693181925', :ec2_security_group_name => 'default' ) assert sg.is_a?(Hash) assert_equal 1, sg[:ec2_security_groups].size end sleep 30 end def test_14_revoke assert_nothing_raised do sg = @rds.revoke_db_security_group_ingress(TEST_SG_NAME, :cidrip => '131.0.0.1/8') assert sg.is_a?(Hash) end assert_nothing_raised do sg = @rds.revoke_db_security_group_ingress(TEST_SG_NAME, :ec2_security_group_owner => '826693181925', :ec2_security_group_name => 'default' ) assert sg.is_a?(Hash) end sleep 30 # sg = @rds.describe_db_security_groups(TEST_SG_NAME).first assert_equal 0, sg[:ip_ranges].size assert_equal 0, sg[:ec2_security_groups].size end def test_15_delete_security_group assert_nothing_raised do @rds.delete_db_security_group(TEST_SG_NAME) end end def test_20_db_snapshots assert_nothing_raised do items = @rds.describe_db_snapshots assert items.is_a?(Array) end # assert_nothing_raised do @rds.describe_db_snapshots do |response| assert response.is_a?(Hash) assert response[:db_snapshots].is_a?(Array) end end end def test_30_events assert_nothing_raised do items = @rds.describe_events assert items.is_a?(Array) end # assert_nothing_raised do @rds.describe_events do |response| assert response.is_a?(Hash) assert response[:events].is_a?(Array) end end end end ================================================ FILE: test/route_53/fixtures/a_record.xml ================================================ CREATE host.right-aws.example.com. A 600 10.0.0.1 ================================================ FILE: test/route_53/fixtures/alias_record.xml ================================================ CREATE right-aws.example.com. A Z1234567890123 example-load-balancer-1111111111.us-east-1.elb.amazonaws.com. ================================================ FILE: test/route_53/test_helper.rb ================================================ require 'test/unit' require File.dirname(__FILE__) + '/../../lib/right_aws' ================================================ FILE: test/route_53/test_right_route_53.rb ================================================ require File.dirname(__FILE__) + '/test_helper.rb' class TestRoute53 < Test::Unit::TestCase STDOUT.sync = true BALANCER_NAME = 'right-aws-test-lb' def setup @r53 = Rightscale::Route53Interface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key, :logger => Logger.new('/dev/null')) @zone_config = {:name => "right-aws.example.com.", :config => {:comment => 'a comment'}} @elb = Rightscale::ElbInterface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key, :logger => Logger.new('/dev/null')) end def teardown @r53.list_hosted_zones.each do |zone| next unless zone[:name] == @zone_config[:name] zone_id = zone[:aws_id] puts zone_id records = @r53.list_resource_record_sets(zone_id) # The NS and SOA records are provided by AWS and must not be deleted records.reject! { |rr| %w[NS SOA].include? rr[:type] } if records.any? response = @r53.delete_resource_record_sets(zone_id, records) puts response.inspect end puts @r53.delete_hosted_zone(zone_id).inspect end # Uncomment to shut down at the end of the day. # @elb.delete_load_balancer BALANCER_NAME end def test_00_list_hosted_zones items = @r53.list_hosted_zones assert items.is_a?(Array) end def test_create_and_delete_zone response = @r53.create_hosted_zone(@zone_config) assert_equal response[:name], @zone_config[:name] assert response[:aws_id].is_a?(String) assert response[:name_servers].is_a?(Array) response2 = @r53.delete_hosted_zone(response[:aws_id]) assert_equal response2[:status], 'PENDING' end def test_add_and_remove_A_record zone = @r53.create_hosted_zone(@zone_config) zone_id = zone[:aws_id] # add a_record = { :name => 'host.right-aws.example.com.', :type => 'A', :ttl => 600, :resource_records => ['10.0.0.1'] } response = @r53.create_resource_record_sets(zone_id, [a_record.dup]) # .dup since it will get :action => :create assert_equal response[:status], 'PENDING' # It should be there now records = @r53.list_resource_record_sets(zone_id) assert records.is_a?(Array) assert records.detect { |rr| rr == a_record }, "Could not find '#{a_record.inspect}' in '#{records.inspect}'" # remove response = @r53.delete_resource_record_sets(zone_id, [a_record.dup]) assert_equal response[:status], 'PENDING' # It should not be there anymore records = @r53.list_resource_record_sets(zone_id) assert records.is_a?(Array) assert ! records.detect { |rr| rr == a_record }, "Record '#{a_record.inspect}' is still in '#{records.inspect}'" @r53.delete_hosted_zone(zone_id) end def test_add_and_remove_Alias_record lb = find_or_create_load_balancer zone = @r53.create_hosted_zone(@zone_config) zone_id = zone[:aws_id] # add alias_target = { :hosted_zone_id => lb[:canonical_hosted_zone_name_id], :dns_name => lb[:dns_name] } alias_record = { :name => 'right-aws.example.com', :type => 'A', :alias_target => alias_target } response = @r53.create_resource_record_sets(zone_id, [alias_record.dup]) # .dup since it will get :action => :create assert_equal response[:status], 'PENDING' # It should be there now records = @r53.list_resource_record_sets(zone_id) assert records.is_a?(Array) record = records.detect { |rr| rr[:alias_target] } assert record, "Could not find '#{alias_record.inspect}' in '#{records.inspect}'" # AWS adds final dots to names assert_equal "#{alias_record[:name]}.", record[:name] assert_equal "#{alias_target[:dns_name]}.", record[:alias_target][:dns_name] # remove response = @r53.delete_resource_record_sets(zone_id, [alias_record.dup]) assert_equal response[:status], 'PENDING' # It should not be there anymore records = @r53.list_resource_record_sets(zone_id) assert records.is_a?(Array) record = records.detect { |rr| rr[:alias_target] } assert ! record, "Record '#{alias_record.inspect}' is still in '#{records.inspect}'" @r53.delete_hosted_zone(zone_id) end def find_or_create_load_balancer unless @elb.describe_load_balancers.detect { |lb| lb[:load_balancer_name] == BALANCER_NAME } @elb.create_load_balancer(BALANCER_NAME, %w[us-east-1b], []) puts "WARNING: Started load balancer. Remember to shut it down (see teardown)." puts "NOTE: Tests might not pass during first few seconds after load balancer is created." end @elb.describe_load_balancers.detect { |lb| lb[:load_balancer_name] == BALANCER_NAME } end def test_rr_sets_to_xml a_record = { :name => 'host.right-aws.example.com.', :type => 'A', :ttl => 600, :resource_records => ['10.0.0.1'], :action => :create } expected = load_fixture('a_record.xml') assert_equal expected, @r53.resource_record_sets_to_xml([a_record], '') # Note final full stop alias_target = { :hosted_zone_id => 'Z1234567890123', :dns_name => 'example-load-balancer-1111111111.us-east-1.elb.amazonaws.com.' } alias_record = { :name => 'right-aws.example.com.', :type => 'A', :alias_target => alias_target, :action => :create } expected = load_fixture('alias_record.xml') assert_same_lines expected, @r53.resource_record_sets_to_xml([alias_record], '') end def load_fixture (name) File.read(File.join(File.dirname(__FILE__), 'fixtures', name)) end def assert_same_lines (expected, actual) expected = expected.split "\n" actual = actual.split "\n" previous = [] while e = expected.shift and a = actual.shift assert_equal e, a, "After:\n#{previous.join("\n")}" previous << e end end end ================================================ FILE: test/s3/test_helper.rb ================================================ require 'test/unit' require File.dirname(__FILE__) + '/../../lib/right_aws' ================================================ FILE: test/s3/test_right_s3.rb ================================================ require File.dirname(__FILE__) + '/test_helper.rb' class TestS3 < Test::Unit::TestCase RIGHT_OBJECT_TEXT = 'Right test message' INTERFACE = Rightscale::S3Interface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key) CONNECTION = Rightscale::S3.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key) def setup @s3 = INTERFACE @bucket = 'right_s3_awesome_test_bucket_000B1_officedrop' @bucket2 = 'right_s3_awesome_test_bucket_000B2_officedrop' @key1 = 'test/woohoo1/' @key2 = 'test1/key/woohoo2' @key3 = 'test2/A%B@C_D&E?F+G=H"I' @key4 = 'test/large_multipart_file_string' @key5 = 'test/large_multipart_file_stream' @key1_copy = 'test/woohoo1_2' @key1_new_name = 'test/woohoo1_3' @key2_new_name = 'test1/key/woohoo2_new' @s = CONNECTION end #--------------------------- # Rightscale::S3Interface #--------------------------- def test_01_create_bucket assert @s3.create_bucket(@bucket), 'Create_bucket fail' end def test_02_list_all_my_buckets assert @s3.list_all_my_buckets.map{|bucket| bucket[:name]}.include?(@bucket), "#{@bucket} must exist in bucket list" end def test_03_list_empty_bucket assert_equal 0, @s3.list_bucket(@bucket).size, "#{@bucket} isn't empty, arrgh!" end def test_04_put assert @s3.put(@bucket, @key1, RIGHT_OBJECT_TEXT, 'x-amz-meta-family'=>'Woohoo1!'), 'Put bucket fail' assert @s3.put(@bucket, @key2, RIGHT_OBJECT_TEXT, 'x-amz-meta-family'=>'Woohoo2!'), 'Put bucket fail' assert @s3.put(@bucket, @key3, RIGHT_OBJECT_TEXT, 'x-amz-meta-family'=>'Woohoo3!'), 'Put bucket fail' end def test_04_put_multipart_string test_text = "" for i in 1..100000 test_text << "Testing test text #{i}\n" end assert @s3.store_object_multipart({:bucket => @bucket, :key => @key4, :data => StringIO.new(test_text)}), 'Put bucket multipart fail' end def test_04b_store_object_multipart_stream rd, wr = IO.pipe producer = Thread.new(wr) do |out| for i in 1..100000 out.write("Testing stream text #{i}\n") end out.close end assert @s3.store_object_multipart({:bucket => @bucket, :key => @key5, :data => rd , :part_size => (20*1024*1024)}), 'Put bucket multipart fail' rd.close end def test_05_get_and_get_object assert_raise(Rightscale::AwsError) { @s3.get(@bucket, 'undefined/key') } data1 = @s3.get(@bucket, @key1) assert_equal RIGHT_OBJECT_TEXT, data1[:object], "Object text must be equal to '#{RIGHT_OBJECT_TEXT}'" assert_equal RIGHT_OBJECT_TEXT, @s3.get_object(@bucket, @key1), "Get_object text must return '#{RIGHT_OBJECT_TEXT}'" assert_equal 'Woohoo1!', data1[:headers]['x-amz-meta-family'], "x-amz-meta-family header must be equal to 'Woohoo1!'" assert_equal RIGHT_OBJECT_TEXT, @s3.get_object(@bucket, @key3), "Get_object text must return '#{RIGHT_OBJECT_TEXT}'" end def test_06_head assert_equal 'Woohoo1!', @s3.head(@bucket,@key1)['x-amz-meta-family'], "x-amz-meta-family header must be equal to 'Woohoo1!'" end def test_07_streaming_get resp = String.new assert_raise(Rightscale::AwsError) do @s3.get(@bucket, 'undefined/key') do |chunk| resp += chunk end end resp = String.new data1 = @s3.get(@bucket, @key1) do |chunk| resp += chunk end assert_equal RIGHT_OBJECT_TEXT, resp, "Object text must be equal to '#{RIGHT_OBJECT_TEXT}'" assert_equal @s3.get_object(@bucket, @key1), resp, "Streaming iface must return same as non-streaming" assert_equal 'Woohoo1!', data1[:headers]['x-amz-meta-family'], "x-amz-meta-family header must be equal to 'Woohoo1!'" end def test_08_keys keys = @s3.list_bucket(@bucket).map{|b| b[:key]} assert_equal keys.size, 5, "There should be 5 keys" assert(keys.include?(@key1)) assert(keys.include?(@key2)) assert(keys.include?(@key3)) assert(keys.include?(@key4)) assert(keys.include?(@key5)) end def test_09_copy_key #--- test COPY # copy a key assert @s3.copy(@bucket, @key1, @bucket, @key1_copy) # check it was copied well assert_equal RIGHT_OBJECT_TEXT, @s3.get_object(@bucket, @key1_copy), "copied object must have the same data" # check meta-headers were copied headers = @s3.head(@bucket, @key1_copy) assert_equal 'Woohoo1!', headers['x-amz-meta-family'], "x-amz-meta-family header must be equal to 'Woohoo1!'" #--- test REPLACE assert @s3.copy(@bucket, @key1, @bucket, @key1_copy, :replace, 'x-amz-meta-family' => 'oooops!') # check it was copied well assert_equal RIGHT_OBJECT_TEXT, @s3.get_object(@bucket, @key1_copy), "copied object must have the same data" # check meta-headers were overwrittenn headers = @s3.head(@bucket, @key1_copy) assert_equal 'oooops!', headers['x-amz-meta-family'], "x-amz-meta-family header must be equal to 'oooops!'" end def test_10_move_key # move a key assert @s3.move(@bucket, @key1, @bucket, @key1_new_name) # check it's data was moved correctly assert_equal RIGHT_OBJECT_TEXT, @s3.get_object(@bucket, @key1_new_name), "moved object must have the same data" # check meta-headers were moved headers = @s3.head(@bucket, @key1_new_name) assert_equal 'Woohoo1!', headers['x-amz-meta-family'], "x-amz-meta-family header must be equal to 'Woohoo1!'" # check the original key is not exists any more keys = @s3.list_bucket(@bucket).map{|b| b[:key]} assert(!keys.include?(@key1)) end def test_11_rename_key # rename a key assert @s3.rename(@bucket, @key2, @key2_new_name) # check the new key data assert_equal RIGHT_OBJECT_TEXT, @s3.get_object(@bucket, @key2_new_name), "moved object must have the same data" # check meta-headers headers = @s3.head(@bucket, @key2_new_name) assert_equal 'Woohoo2!', headers['x-amz-meta-family'], "x-amz-meta-family header must be equal to 'Woohoo2!'" # check the original key is not exists any more keys = @s3.list_bucket(@bucket).map{|b| b[:key]} assert(!keys.include?(@key2)) end def test_12_retrieve_object assert_raise(Rightscale::AwsError) { @s3.retrieve_object(:bucket => @bucket, :key => 'undefined/key') } data1 = @s3.retrieve_object(:bucket => @bucket, :key => @key1_new_name) assert_equal RIGHT_OBJECT_TEXT, data1[:object], "Object text must be equal to '#{RIGHT_OBJECT_TEXT}'" assert_equal 'Woohoo1!', data1[:headers]['x-amz-meta-family'], "x-amz-meta-family header must be equal to 'Woohoo1!'" end def test_13_delete_folder assert_equal 1, @s3.delete_folder(@bucket, 'test').size, "Only one key(#{@key1}) must be deleted!" end def test_14_delete_bucket assert_raise(Rightscale::AwsError) { @s3.delete_bucket(@bucket) } assert @s3.clear_bucket(@bucket), 'Clear_bucket fail' assert_equal 0, @s3.list_bucket(@bucket).size, 'Bucket must be empty' assert @s3.delete_bucket(@bucket) assert !@s3.list_all_my_buckets.map{|bucket| bucket[:name]}.include?(@bucket), "#{@bucket} must not exist" end #--------------------------- # Rightscale::S3 classes #--------------------------- def test_20_s3 # create bucket bucket = @s.bucket(@bucket, true) assert bucket # check that the bucket exists assert @s.buckets.map{|b| b.name}.include?(@bucket) # delete bucket assert bucket.clear assert bucket.delete end def test_21_bucket_create_put_get_key bucket = Rightscale::S3::Bucket.create(@s, @bucket, true) # check that the bucket exists assert @s.buckets.map{|b| b.name}.include?(@bucket) assert bucket.keys.empty? # put data assert bucket.put(@key3, RIGHT_OBJECT_TEXT, {'family'=>'123456'}) # get data and compare assert_equal RIGHT_OBJECT_TEXT, bucket.get(@key3) # get key object key = bucket.key(@key3, true) assert_equal Rightscale::S3::Key, key.class assert key.exists? assert_equal '123456', key.meta_headers['family'] end def test_22_keys bucket = Rightscale::S3::Bucket.create(@s, @bucket, false) # create first key key3 = Rightscale::S3::Key.create(bucket, @key3) key3.refresh assert key3.exists? assert_equal '123456', key3.meta_headers['family'] # create second key key2 = Rightscale::S3::Key.create(bucket, @key2) assert !key2.refresh assert !key2.exists? assert_raise(Rightscale::AwsError) { key2.head } # store key key2.meta_headers = {'family'=>'111222333'} assert key2.put(RIGHT_OBJECT_TEXT) # make sure that the key exists assert key2.refresh assert key2.exists? assert key2.head # get its data assert_equal RIGHT_OBJECT_TEXT, key2.get # drop key assert key2.delete assert !key2.exists? end def test_23_rename_key bucket = Rightscale::S3::Bucket.create(@s, @bucket, false) # -- 1 -- (key based rename) # create a key key = bucket.key('test/copy/1') key.put(RIGHT_OBJECT_TEXT) original_key = key.clone assert key.exists?, "'test/copy/1' should exist" # rename it key.rename('test/copy/2') assert_equal 'test/copy/2', key.name assert key.exists?, "'test/copy/2' should exist" # the original key should not exist assert !original_key.exists?, "'test/copy/1' should not exist" # -- 2 -- (bucket based rename) bucket.rename_key('test/copy/2', 'test/copy/3') assert bucket.key('test/copy/3').exists?, "'test/copy/3' should exist" assert !bucket.key('test/copy/2').exists?, "'test/copy/2' should not exist" end def test_24_copy_key bucket = Rightscale::S3::Bucket.create(@s, @bucket, false) # -- 1 -- (key based copy) # create a key key = bucket.key('test/copy/10') key.put(RIGHT_OBJECT_TEXT) # make copy new_key = key.copy('test/copy/11') # make sure both the keys exist and have a correct data assert key.exists?, "'test/copy/10' should exist" assert new_key.exists?, "'test/copy/11' should exist" assert_equal RIGHT_OBJECT_TEXT, key.get assert_equal RIGHT_OBJECT_TEXT, new_key.get # -- 2 -- (bucket based copy) bucket.copy_key('test/copy/11', 'test/copy/12') assert bucket.key('test/copy/11').exists?, "'test/copy/11' should exist" assert bucket.key('test/copy/12').exists?, "'test/copy/12' should exist" assert_equal RIGHT_OBJECT_TEXT, bucket.key('test/copy/11').get assert_equal RIGHT_OBJECT_TEXT, bucket.key('test/copy/12').get end def test_25_move_key bucket = Rightscale::S3::Bucket.create(@s, @bucket, false) # -- 1 -- (key based copy) # create a key key = bucket.key('test/copy/20') key.put(RIGHT_OBJECT_TEXT) # move new_key = key.move('test/copy/21') # make sure both the keys exist and have a correct data assert !key.exists?, "'test/copy/20' should not exist" assert new_key.exists?, "'test/copy/21' should exist" assert_equal RIGHT_OBJECT_TEXT, new_key.get # -- 2 -- (bucket based copy) bucket.copy_key('test/copy/21', 'test/copy/22') assert bucket.key('test/copy/21').exists?, "'test/copy/21' should not exist" assert bucket.key('test/copy/22').exists?, "'test/copy/22' should exist" assert_equal RIGHT_OBJECT_TEXT, bucket.key('test/copy/22').get end def test_26_save_meta bucket = Rightscale::S3::Bucket.create(@s, @bucket, false) # create a key key = bucket.key('test/copy/30') key.put(RIGHT_OBJECT_TEXT) assert key.meta_headers.right_blank? # store some meta keys meta = {'family' => 'oops','race' => 'troll'} assert_equal meta, key.save_meta(meta) # reload meta assert_equal meta, key.reload_meta end def test_27_clear_delete bucket = Rightscale::S3::Bucket.create(@s, @bucket, false) # add another key bucket.put(@key2, RIGHT_OBJECT_TEXT) # delete 'folder' assert_equal 1, bucket.delete_folder(@key1).size # delete assert_raise(Rightscale::AwsError) { bucket.delete } bucket.delete(:force => true) end # Grantees def test_30_create_bucket bucket = @s.bucket(@bucket, true, 'public-read') assert bucket end def test_31_list_grantees bucket = Rightscale::S3::Bucket.create(@s, @bucket, false) # get grantees list grantees = bucket.grantees # check that the grantees count equal to 2 (root, AllUsers) assert_equal 2, grantees.size end def test_32_grant_revoke_drop bucket = Rightscale::S3::Bucket.create(@s, @bucket, false) # Take 'AllUsers' grantee grantee = Rightscale::S3::Grantee.new(bucket,'http://acs.amazonaws.com/groups/global/AllUsers') # Check exists? assert grantee.exists? # Add grant as String assert grantee.grant('WRITE') # Add grants as Array assert grantee.grant(['READ_ACP', 'WRITE_ACP']) # Check perms count assert_equal 4, grantee.perms.size # revoke 'WRITE_ACP' assert grantee.revoke('WRITE_ACP') # Check manual perm removal method grantee.perms -= ['READ_ACP'] grantee.apply assert_equal 2, grantee.perms.size # Check grantee removal if it has no permissions assert grantee.perms = [] assert grantee.apply assert !grantee.exists? # Check multiple perms assignment assert grantee.grant('FULL_CONTROL', 'READ', 'WRITE') assert_equal ['FULL_CONTROL','READ','WRITE'].sort, grantee.perms.sort # Check multiple perms removal assert grantee.revoke('FULL_CONTROL', 'WRITE') assert_equal ['READ'], grantee.perms # check 'Drop' method assert grantee.drop assert !grantee.exists? assert_equal 1, bucket.grantees.size # Delete bucket bucket.delete(:force => true) end def test_33_key_grantees # Create bucket bucket = @s.bucket(@bucket, true) # Create key key = bucket.key(@key1) assert key.put(RIGHT_OBJECT_TEXT, 'public-read') # Get grantees list (must be == 2) grantees = key.grantees assert grantees assert_equal 2, grantees.size # Take one of grantees and give him 'Write' perms grantee = grantees[0] assert grantee.grant('WRITE') # Drop grantee assert grantee.drop # Drop bucket bucket.delete(:force => true) end def test_34_bucket_create_put_with_perms bucket = Rightscale::S3::Bucket.create(@s, @bucket, true) # check that the bucket exists assert @s.buckets.map{|b| b.name}.include?(@bucket) assert bucket.keys.empty? # put data (with canned ACL) assert bucket.put(@key1, RIGHT_OBJECT_TEXT, {'family'=>'123456'}, "public-read") # get data and compare assert_equal RIGHT_OBJECT_TEXT, bucket.get(@key1) # get key object key = bucket.key(@key1, true) assert_equal Rightscale::S3::Key, key.class assert key.exists? assert_equal '123456', key.meta_headers['family'] end def test_35_key_put_with_perms bucket = Rightscale::S3::Bucket.create(@s, @bucket, false) # create first key key1 = Rightscale::S3::Key.create(bucket, @key1) key1.refresh assert key1.exists? assert key1.put(RIGHT_OBJECT_TEXT, "public-read") # get its data assert_equal RIGHT_OBJECT_TEXT, key1.get # drop key assert key1.delete assert !key1.exists? end def test_36_set_amazon_problems original_problems = RightAws::S3Interface.amazon_problems assert(original_problems.length > 0) RightAws::S3Interface.amazon_problems= original_problems << "A New Problem" new_problems = RightAws::S3Interface.amazon_problems assert_equal(new_problems, original_problems) RightAws::S3Interface.amazon_problems= nil assert_nil(RightAws::S3Interface.amazon_problems) end def test_37_access_logging bucket = Rightscale::S3::Bucket.create(@s, @bucket, false) targetbucket = Rightscale::S3::Bucket.create(@s, @bucket2, true) # Take 'AllUsers' grantee grantee = Rightscale::S3::Grantee.new(targetbucket,'http://acs.amazonaws.com/groups/s3/LogDelivery') assert grantee.grant(['READ_ACP', 'WRITE']) assert bucket.enable_logging(:targetbucket => targetbucket, :targetprefix => "loggylogs/") sleep 10 assert_equal({:enabled => true, :targetbucket => @bucket2, :targetprefix => "loggylogs/"}, bucket.logging_info) assert bucket.disable_logging # check 'Drop' method assert grantee.drop end def test_40_delete_buckets Rightscale::S3::Bucket.create(@s, @bucket, false).delete(:force => true) Rightscale::S3::Bucket.create(@s, @bucket2, false).delete(:force => true) end def test_41_add_cache_control_response_parameter @cache_control = 'max-age=3600' perform_request( 'response-cache-control' => @cache_control ) do |response| assert_equal response['Cache-Control'], @cache_control end end def test_42_add_cache_control_and_content_type_and_content_disposition @cache_control = 'max-age=3600' @content_type = 'text/plain' @content_disposition = 'attachment; filename=sample.txt' perform_request( 'response-cache-control' => @cache_control, 'response-content-type' => @content_type, 'response-content-disposition' => @content_disposition ) do |response| assert_equal response['Cache-Control'], @cache_control assert_equal response['Content-Type'], @content_type assert_equal response['Content-Disposition'], @content_disposition end end def test_43_add_expires_and_content_type_and_content_disposition @expires = 'Sun, 26 Jun 2011 02:58:26 GMT' @content_type = 'text/plain' @content_disposition = 'attachment; filename=sample.txt' perform_request( 'response-expires' => @expires, 'response-content-type' => @content_type, 'response-content-disposition' => @content_disposition ) do |response| assert_equal response['Expires'], @expires assert_equal response['Content-Type'], @content_type assert_equal response['Content-Disposition'], @content_disposition end end def test_44_delete_multiple bucket = RightAws::S3::Bucket.create(@s, @bucket, true) key1 = Rightscale::S3::Key.create(bucket, @key1) key2 = Rightscale::S3::Key.create(bucket, @key2) key3 = Rightscale::S3::Key.create(bucket, @key3) assert @s3.put(@bucket, @key1, RIGHT_OBJECT_TEXT), 'Put bucket fail' assert @s3.put(@bucket, @key2, RIGHT_OBJECT_TEXT), 'Put bucket fail' assert @s3.put(@bucket, @key3, RIGHT_OBJECT_TEXT), 'Put bucket fail' key1.refresh key2.refresh key3.refresh assert key1.exists? assert key2.exists? assert key3.exists? result = @s3.delete_multiple(@bucket, [@key1, @key2, @key3]) assert result.empty? key1.refresh key2.refresh key3.refresh assert !key1.exists? assert !key2.exists? assert !key3.exists? end def test_45_delete_multiple_more_than_1000_objects n = 1200 keys = (1..n).map { |i| "key-#{i}"} keys.each do |key| assert @s3.put(@bucket, key, RIGHT_OBJECT_TEXT), 'Put bucket fail' end result = @s3.delete_multiple(@bucket, keys) assert result.empty? keys_after = @s3.list_bucket(@bucket).map { |obj| obj[:key] } keys.each do |key| assert !keys_after.include?(key) end end private def request( uri ) url = URI.parse( uri ) http = Net::HTTP.new(url.host, 80) # http.use_ssl = true # http.verify_mode = OpenSSL::SSL::VERIFY_PEER http.request(Net::HTTP::Get.new( url.request_uri )) end def perform_request( headers, &block ) @s3.create_bucket( @bucket ) @s3.put( @bucket, @key2, RIGHT_OBJECT_TEXT ) response = request( @s3.get_link( @bucket, @key2, nil, {}, headers ) ) block.call( response ) @s3.force_delete_bucket(@bucket) end end ================================================ FILE: test/s3/test_right_s3_stubbed.rb ================================================ require File.dirname(__FILE__) + '/test_helper.rb' class TestS3Stubbed < Test::Unit::TestCase RIGHT_OBJECT_TEXT = 'Right test message' def setup @s3 = Rightscale::S3Interface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key) @bucket = 'right_s3_awesome_test_bucket' @key1 = 'test/woohoo1' @key2 = 'test1/key/woohoo2' @s = Rightscale::S3.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key) Rightscale::HttpConnection.reset end # Non-remote tests: these use the stub version of Rightscale::HTTPConnection def test_101_create_bucket Rightscale::HttpConnection.push(409, 'The named bucket you tried to create already exists') Rightscale::HttpConnection.push(500, 'We encountered an internal error. Please try again.') Rightscale::HttpConnection.push(500, 'We encountered an internal error. Please try again.') assert_raise RightAws::AwsError do @s3.create_bucket(@bucket) end end def test_102_list_all_my_buckets_failure Rightscale::HttpConnection.push(401, 'Unauthorized') assert_raise RightAws::AwsError do @s3.list_all_my_buckets end end def test_103_list_empty_bucket Rightscale::HttpConnection.push(403, 'Access Denied') assert_raise RightAws::AwsError do @s3.list_bucket(@bucket) end end def test_104_put Rightscale::HttpConnection.push(400, 'Your proposed upload exceeds the maximum allowed object size.') Rightscale::HttpConnection.push(400, 'The Content-MD5 you specified was an invalid.') Rightscale::HttpConnection.push(409, 'Please try again') assert_raise RightAws::AwsError do assert @s3.put(@bucket, @key1, RIGHT_OBJECT_TEXT, 'x-amz-meta-family'=>'Woohoo1!'), 'Put bucket fail' end assert_raise RightAws::AwsError do assert @s3.put(@bucket, @key2, RIGHT_OBJECT_TEXT, 'x-amz-meta-family'=>'Woohoo2!'), 'Put bucket fail' end end def test_105_get_and_get_object Rightscale::HttpConnection.push(404, 'not found') assert_raise(Rightscale::AwsError) { @s3.get(@bucket, 'undefined/key') } end def test_106_head Rightscale::HttpConnection.push(404, 'Good Luck!') assert_raise RightAws::AwsError do @s3.head(@bucket,@key1) end end def test_109_delete_bucket Rightscale::HttpConnection.push(403, 'Good Luck!') assert_raise(Rightscale::AwsError) { @s3.delete_bucket(@bucket) } end def test_115_copy_key Rightscale::HttpConnection.push(500, 'not found') #--- test COPY # copy a key assert_raise RightAws::AwsError do @s3.copy(@bucket, @key1, @bucket, @key1_copy) end end def test_116_move_key # move a key Rightscale::HttpConnection.push(413, 'not found') assert_raise RightAws::AwsError do @s3.move(@bucket, @key1, @bucket, @key1_new_name) end end def test_117_rename_key # rename a key Rightscale::HttpConnection.push(500, 'not found') assert_raise RightAws::AwsError do @s3.rename(@bucket, @key2, @key2_new_name) end end end ================================================ FILE: test/sdb/test_active_sdb.rb ================================================ require File.dirname(__FILE__) + '/test_helper.rb' class TestSdb < Test::Unit::TestCase DOMAIN_PREFIX = 'right_sdb_awesome_test' CLIENT_DOMAIN = "#{DOMAIN_PREFIX}_client" PERSON_DOMAIN = "#{DOMAIN_PREFIX}_person" class Client < RightAws::ActiveSdb::Base set_domain_name CLIENT_DOMAIN end class Person < RightAws::ActiveSdb::Base set_domain_name PERSON_DOMAIN columns do name email score :Integer is_active :Boolean registered_at :DateTime created_at :DateTime, :default => lambda{ Time.now } end end def setup STDOUT.sync = true @clients = [ { 'name' => 'Bush', 'country' => 'USA', 'gender' => 'male', 'expiration' => '2009', 'post' => 'president' }, { 'name' => 'Putin', 'country' => 'Russia', 'gender' => 'male', 'expiration' => '2008', 'post' => 'president' }, { 'name' => 'Medvedev', 'country' => 'Russia', 'gender' => 'male', 'expiration' => '2012', 'post' => 'president' }, { 'name' => 'Mary', 'country' => 'USA', 'gender' => 'female', 'hobby' => ['patchwork', 'bundle jumping'] }, { 'name' => 'Sandy', 'country' => 'Russia', 'gender' => 'female', 'hobby' => ['flowers', 'cats', 'cooking'] }, { 'name' => 'Mary', 'country' => 'Russia', 'gender' => 'female', 'hobby' => ['flowers', 'cats', 'cooking'] } ] @people = [ { :name => 'Yetta E. Andrews', :email => 'nulla.facilisis@metus.com', :score => 100, :is_active => true, :registered_at => Time.local(2000, 1, 1) }, { :name => 'Sybill O. Olson', :email => 'nisi.Aenean.eget@urna.com', :score => 87, :is_active => true, :registered_at => Time.local(2008, 7, 6) }, { :name => 'Isabelle K. Flynn', :email => 'velit@amet.com', :score => 98, :is_active => false, :registered_at => Time.local(2003, 5, 20) }, { :name => 'Juliet H. Witt', :email => 'egestas@pretiumaliquet.ca', :score => 72, :is_active => true, :registered_at => Time.local(2007, 2, 28) }, { :name => 'Lucy N. Christensen', :email => 'lacus.v12@stu.edu', :score => 94, :is_active => false, :registered_at => Time.local(2005, 10, 26) } ] RightAws::ActiveSdb.establish_connection(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key) end SDB_DELAY = 3 def wait(delay, msg='') print " waiting #{delay} seconds: #{msg}" while delay>0 do delay -= 1 print '.' sleep 1 end puts end #--------------------------- # Rightscale::SdbInterface #--------------------------- def test_00_delete_domain assert RightAws::ActiveSdb.delete_domain(CLIENT_DOMAIN) assert RightAws::ActiveSdb.delete_domain(PERSON_DOMAIN) wait SDB_DELAY, 'test 00: after domain deletion' end def test_01_create_domain # check that domain does not exist assert !RightAws::ActiveSdb.domains.include?(CLIENT_DOMAIN) # create domain assert Client.create_domain assert Person.create_domain wait SDB_DELAY, 'test 01: after domain creation' # check that we have received new domain from Amazin assert RightAws::ActiveSdb.domains.include?(CLIENT_DOMAIN) end def test_02_create_items # check that DB is empty clients = Client.find(:all) assert clients.right_blank? # put some clients there @clients.each do |client| Client.create client end wait SDB_DELAY, 'test 02: after clients creation' # check that DB has all the clients we just putted clients = Client.find(:all) assert_equal @clients.size, clients.size end def test_03_create_and_save_new_item # get the db old_clients = Client.find(:all) # create new client new_client = Client.new('country' => 'unknown', 'dummy' => 'yes') wait SDB_DELAY, 'test 03: after in-memory client creation' # get the db and ensure we created the client in-memory only assert_equal old_clients.size, Client.find(:all).size # put the client to DB new_client.save wait SDB_DELAY, 'test 03: after in-memory client saving' # get all db again and compare to original list assert_equal old_clients.size+1, Client.find(:all).size end def test_04_find_all # retrieve all the DB, make sure all are in place clients = Client.find(:all) ids = clients.map{|client| client.id }[0..1] assert_equal @clients.size + 1, clients.size # retrieve all presidents (must find: Bush, Putin, Medvedev) assert_equal 3, Client.find(:all, :conditions => ["[?=?]",'post','president']).size # retrieve all russian presidents (must find: Putin, Medvedev) assert_equal 2, Client.find(:all, :conditions => ["['post'=?] intersection ['country'=?]",'president', 'Russia']).size # retrieve all russian presidents and all women (must find: Putin, Medvedev, 2 Maries and Sandy) assert_equal 5, Client.find(:all, :conditions => ["['post'=?] intersection ['country'=?] union ['gender'=?]",'president', 'Russia','female']).size # find all rissian presidents Bushes assert_equal 0, Client.find(:all, :conditions => ["['post'=?] intersection ['country'=?] intersection ['name'=?]",'president', 'Russia','Bush']).size # --- find by ids # must find 1 rec (by rec id) and return it assert_equal ids.first, Client.find(ids.first).id # must find 1 rec (by one item array) and return an array assert_equal ids.first, Client.find([ids.first]).first.id # must find 2 recs (by a list of comma separated ids) and return an array assert_equal ids.size, Client.find(*ids).size # must find 2 recs (by an array of ids) and return an array assert_equal ids.size, Client.find(ids).size ids << 'dummy_id' # must raise an error when getting unexistent record assert_raise(RightAws::ActiveSdb::ActiveSdbError) do Client.find(ids) end # find one record by unknown id assert_raise(RightAws::ActiveSdb::ActiveSdbError) do Client.find('dummy_id') end end def test_05_find_first # find any record assert Client.find(:first) # find any president assert Client.find(:first, :conditions => ["[?=?]",'post','president']) # find any rissian president assert Client.find(:first, :conditions => ["['post'=?] intersection ['country'=?]",'president','Russia']) # find any unexistent record assert_nil Client.find(:first, :conditions => ["['post'=?] intersection ['country'=?]",'president','Rwanda']) end def test_06_find_all_by_helpers # find all Bushes assert_equal 1, Client.find_all_by_name('Bush').size # find all russian presidents assert_equal 2, Client.find_all_by_post_and_country('president','Russia').size # find all women in USA that love flowers assert_equal 2, Client.find_all_by_gender_and_country_and_hobby('female','Russia','flowers').size # order and auto_load: clients = Client.find_all_by_post('president', :order => 'name', :auto_load => true) assert_equal [['Bush'], ['Medvedev'], ['Putin']], clients.map{|c| c['name']} clients = Client.find_all_by_post('president', :order => 'name desc', :auto_load => true) assert_equal [['Putin'], ['Medvedev'], ['Bush']], clients.map{|c| c['name']} end def test_07_find_by_helpers # find mr Bush assert Client.find_by_name('Bush') # find any russian president assert Client.find_by_post_and_country('president','Russia') # find Mary in Russia that loves flowers # order and auto_load: assert_equal ['Bush'], Client.find_by_post('president', :order => 'name', :auto_load => true)['name'] assert_equal ['Putin'], Client.find_by_post('president', :order => 'name desc', :auto_load => true)['name'] end def test_08_reload putin = Client.find_by_name('Putin') # attributes must be empty until reload (except 'id' field) assert_nil putin['name'] assert_nil putin['country'] assert_nil putin['gender'] assert_nil putin['expiration'] assert_nil putin['post'] # reloaded attributes must have 5 items + id putin.reload assert_equal 6, putin.attributes.size # check all attributes assert_equal ['Putin'], putin['name'] assert_equal ['Russia'], putin['country'] assert_equal ['male'], putin['gender'] assert_equal ['2008'], putin['expiration'] assert_equal ['president'], putin['post'] end def test_09_select # select all records assert_equal 7, Client.select(:all).size # LIMIT # 1 record assert Client.select(:first).is_a?(Client) # select 2 recs assert_equal 2, Client.select(:all, :limit => 2).size # ORDER # select all recs ordered by 'expration' (must find only recs where 'expration' attribute presents) result = Client.select(:all, :order => 'expiration') assert_equal 3, result.size assert_equal ['2008', '2009', '2012'], result.map{ |c| c['expiration'] }.flatten # desc order result = Client.select(:all, :order => 'expiration desc') assert_equal ['2012', '2009', '2008'], result.map{ |c| c['expiration'] }.flatten # CONDITIONS result = Client.select(:all, :conditions => ["expiration >= ?", 2009], :order => 'name') assert_equal ['Bush', 'Medvedev'], result.map{ |c| c['name'] }.flatten result = Client.select(:all, :conditions => "hobby='flowers' AND gender='female'", :order => 'name') assert_equal ['Mary', 'Sandy'], result.map{ |c| c['name'] }.flatten # SELECT result = Client.select(:all, :select => 'hobby', :conditions => "gender IS NOT NULL", :order => 'name') hobbies = result.map{|c| c['hobby']} # must return all recs assert_equal 6, result.size # but anly 3 of them have this field set assert_equal 3, hobbies.compact.size end def test_10_select_by assert_equal 2, Client.select_all_by_hobby('flowers').size assert_equal 2, Client.select_all_by_hobby_and_country('flowers', 'Russia').size assert_equal ['Putin'], Client.select_by_post_and_expiration('president','2008')['name'] end def test_11_save_and_put putin = Client.find_by_name('Putin') putin.reload putin['hobby'] = 'ski' # SAVE method (replace values) putin.save wait SDB_DELAY, 'test 09: after saving' # check that DB was updated with 'ski' new_putin = Client.find_by_name('Putin') new_putin.reload assert ['ski'], new_putin['hobby'] # replace hobby putin['hobby'] = 'dogs' putin.save wait SDB_DELAY, 'test 09: after saving' # check that 'ski' in DB was replaced by 'dogs' new_putin = Client.find_by_name('Putin') new_putin.reload assert ['dogs'], new_putin['hobby'] # PUT method (add values) putin['hobby'] = 'ski' putin.put wait SDB_DELAY, 'test 09: after putting' # check that 'ski' was added to 'dogs' new_putin = Client.find_by_name('Putin') new_putin.reload assert ['dogs', 'ski'], new_putin['hobby'].sort end def test_12_save_and_put_attributes putin = Client.find_by_name('Putin') putin.reload # SAVE method (replace values) putin.save_attributes('language' => 'russian') wait SDB_DELAY, 'test 10: after save_attributes' # check that DB was updated with 'ski' new_putin = Client.find_by_name('Putin') new_putin.reload assert ['russian'], new_putin['language'] # replace 'russian' by 'german' putin.save_attributes('language' => 'german') wait SDB_DELAY, 'test 10: after save_attributes' # check that 'russian' in DB was replaced by 'german' new_putin = Client.find_by_name('Putin') new_putin.reload assert ['german'], new_putin['language'] # PUT method (add values) putin.put_attributes('language' => ['russian', 'english']) wait SDB_DELAY, 'test 10: after put_attributes' # now Putin must know all the languages new_putin = Client.find_by_name('Putin') new_putin.reload assert ['english', 'german', 'russian'], new_putin['language'].sort end def test_13_delete putin = Client.find_by_name('Putin') putin.reload # --- delete_values # remove an unknown attribute # should return an empty hash assert_equal( {}, putin.delete_values('undefined_attribute' => 'ohoho')) # remove 2 languages lang_hash = {'language' => ['english', 'german']} assert_equal lang_hash, putin.delete_values(lang_hash) wait SDB_DELAY, 'test 11: after put_attributes' # now Putin must know only russian lang new_putin = Client.find_by_name('Putin') new_putin.reload assert ['russian'], new_putin['language'].sort # --- delete_attributes putin.delete_attributes('language', 'hobby') wait SDB_DELAY, 'test 11: after delete_attributes' # trash hoddy and langs new_putin = Client.find_by_name('Putin') new_putin.reload assert_nil new_putin['language'] assert_nil new_putin['hobby'] # --- delete item putin.delete wait SDB_DELAY, 'test 11: after delete item' assert_nil Client.find_by_name('Putin') end def test_14_dynamic_attribute_accessors bush = Client.find_by_name('Bush') bush.reload assert_nothing_raised { assert_equal ['male'], bush.gender bush.name = 'George' assert_equal ['George'], bush.name } assert_raise(NoMethodError) { bush.flarble } end def test_15_column_emulation @people.each do |person| Person.create person end wait SDB_DELAY, 'test 15: after people creation' person = Person.find_by_email 'nulla.facilisis@metus.com' person.reload assert_equal 'Yetta E. Andrews', person.name assert_equal DateTime, person.registered_at.class assert person['registered_at'].is_a?(DateTime) assert person[:registered_at].is_a?(DateTime) assert ! person[:created_at].nil? assert_equal DateTime, person.created_at.class assert person['created_at'].is_a?(DateTime) assert person[:created_at].is_a?(DateTime) assert person.is_active assert_equal 100, person.score end def test_999_delete_domain assert Client.delete_domain assert Person.delete_domain wait SDB_DELAY, 'test 999: after delete domain' assert_raise(Rightscale::AwsError) do Client.find :all end end end ================================================ FILE: test/sdb/test_batch_put_attributes.rb ================================================ # -*- coding: utf-8 -*- require File.dirname(__FILE__) + '/test_helper.rb' class TestSdb < Test::Unit::TestCase def setup STDOUT.sync = true @domain = 'right_sdb_awesome_test_domain' @attributes = { 'a' => { 'foo' => '123' }, 'b' => { 'bar' => '456' } } # Interface instance @sdb = Rightscale::SdbInterface.new @sdb.delete_domain(@domain) wait(SDB_DELAY, "after removing domain") @sdb.create_domain(@domain) wait(SDB_DELAY, "after recreating domain") end SDB_DELAY = 2 def wait(delay, msg='') print "waiting #{delay} seconds #{msg}" while delay>0 do delay -= 1 print '.' sleep 1 end puts end def test_batch_put_attributes @sdb.batch_put_attributes(@domain, @attributes) wait(SDB_DELAY, "after putting attributes") a = @sdb.get_attributes(@domain, 'a')[:attributes] b = @sdb.get_attributes(@domain, 'b')[:attributes] assert_equal( {'foo' => ['123']}, a) assert_equal( {'bar' => ['456']}, b) # Replace = false @sdb.batch_put_attributes(@domain, { 'a' => {'foo' => ['789']}}) wait(SDB_DELAY, "after putting attributes") a = @sdb.get_attributes(@domain, 'a')[:attributes] assert_equal ['123', '789'], a['foo'].sort # Replace = true @sdb.batch_put_attributes(@domain, {'b' => {'bar' => ['789']}}, true) wait(SDB_DELAY, "after putting attributes") b = @sdb.get_attributes(@domain, 'b')[:attributes] assert_equal ['789'], b['bar'].sort end end ================================================ FILE: test/sdb/test_helper.rb ================================================ require 'test/unit' require File.dirname(__FILE__) + '/../../lib/right_aws' require 'sdb/active_sdb' ================================================ FILE: test/sdb/test_right_sdb.rb ================================================ require File.dirname(__FILE__) + '/test_helper.rb' class TestSdb < Test::Unit::TestCase def setup STDOUT.sync = true @domain = 'right_sdb_awesome_test_domain' @item = 'toys' @attr = { 'Jon' => %w{beer car} } # Interface instance @sdb = Rightscale::SdbInterface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key) end SDB_DELAY = 7 def wait(delay, msg='') print "waiting #{delay} seconds #{msg}" while delay>0 do delay -= 1 print '.' sleep 1 end puts end #--------------------------- # Rightscale::SdbInterface #--------------------------- def test_00_delete_domain # delete the domain to reset all the things assert @sdb.delete_domain(@domain), 'delete_domain fail' wait SDB_DELAY, 'after domain deletion' end def test_01_create_domain # check that domain does not exist assert !@sdb.list_domains[:domains].include?(@domain) # create domain assert @sdb.create_domain(@domain), 'create_domain fail' wait SDB_DELAY, 'after domain creation' # check that we have received new domain from Amazin assert @sdb.list_domains[:domains].include?(@domain) end def test_02_put_attributes # put attributes assert @sdb.put_attributes(@domain, @item, @attr) wait SDB_DELAY, 'after putting attributes' end def test_03_get_attributes # get attributes values = Array(@sdb.get_attributes(@domain, @item)[:attributes]['Jon']).sort # compare to original list assert_equal values, @attr['Jon'].sort end def test_04_add_attributes # add new attribute new_value = 'girls' @sdb.put_attributes @domain, @item, {'Jon' => new_value} wait SDB_DELAY, 'after putting attributes' # get attributes ('girls' must be added to already existent attributes) values = Array(@sdb.get_attributes(@domain, @item)[:attributes]['Jon']).sort assert_equal values, (@attr['Jon'] << new_value).sort end def test_05_replace_attributes # replace attributes @sdb.put_attributes @domain, @item, {'Jon' => 'pub'}, :replace wait SDB_DELAY, 'after replacing attributes' # get attributes (all must be removed except of 'pub') values = @sdb.get_attributes(@domain, @item)[:attributes]['Jon'] assert_equal values, ['pub'] end def test_06_delete_attribute # add value 'girls' and 'vodka' to 'Jon' @sdb.put_attributes @domain, @item, {'Jon' => ['girls','vodka']} wait SDB_DELAY, 'after adding attributes' # get attributes ('girls' and 'vodka' must be added 'pub') values = Array(@sdb.get_attributes(@domain, @item)[:attributes]['Jon']).sort assert_equal values, ['girls', 'pub', 'vodka'] # delete a single value 'girls' from attribute 'Jon' @sdb.delete_attributes @domain, @item, 'Jon' => ['girls'] wait SDB_DELAY, 'after the deletion of attribute' # get attributes ('girls' must be removed) values = @sdb.get_attributes(@domain, @item)[:attributes]['Jon'] assert_equal values, ['pub', 'vodka'] # delete all values from attribute 'Jon' @sdb.delete_attributes @domain, @item, ['Jon'] wait SDB_DELAY, 'after the deletion of attributes' # get attributes (values must be empty) values = @sdb.get_attributes(@domain, @item)[:attributes]['Jon'] assert_equal values, nil end def test_07_delete_item @sdb.put_attributes @domain, @item, {'Volodya' => ['girls','vodka']} wait SDB_DELAY, 'after adding attributes' # get attributes ('girls' and 'vodka' must be there) values = Array(@sdb.get_attributes(@domain, @item)[:attributes]['Volodya']).sort assert_equal values, ['girls', 'vodka'] # delete an item @sdb.delete_attributes @domain, @item wait SDB_DELAY, 'after deleting attributes' # get attributes (values must be empty) values = @sdb.get_attributes(@domain, @item)[:attributes]['Volodya'] assert_equal values, nil end def test_08_query # add some values for query @sdb.put_attributes @domain, @item, {'Jon' => ['girls','vodka']} wait SDB_DELAY, 'after adding attributes' items = @sdb.query(@domain, ['[?=?]', 'Jon','vodka'])[:items] assert_equal items.size, 1 assert_equal items.first, @item end def test_09_signature_version_0 sdb = Rightscale::SdbInterface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key, :signature_version => '0') item = 'toys' # TODO: need to change the below test. I think Juergen's intention was to include some umlauts in the values # put attributes # mhhh... Not sure how to translate this: hölzchehn klötzchen grÃŒnspan buße... Lets assume this is: attributes = { 'Jurgen' => %w{kitten puppy chickabiddy piglet} } assert sdb.put_attributes(@domain, item, attributes) wait SDB_DELAY, 'after putting attributes' # get attributes values = Array(sdb.get_attributes(@domain, item)[:attributes]['Jurgen']).sort # compare to original list assert_equal values, attributes['Jurgen'].sort # check that the request has correct signature version assert sdb.last_request.path.include?('SignatureVersion=0') end def test_10_signature_version_1 sdb = Rightscale::SdbInterface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key, :signature_version => '1') domains = nil assert_nothing_thrown "Failed to use signature V1" do domains = sdb.list_domains end assert domains end def test_11_signature_version_1 sdb = Rightscale::SdbInterface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key, :signature_version => '2') domains = nil assert_nothing_thrown "Failed to use signature V2" do domains = sdb.list_domains end assert domains end def test_12_array_of_attrs item = 'multiples' assert_nothing_thrown "Failed to put multiple attrs" do @sdb.put_attributes(@domain, item, {:one=>1, :two=>2, :three=>3}) end end def test_13_zero_len_attrs item = 'zeroes' assert_nothing_thrown "Failed to put zero-length attributes" do @sdb.put_attributes(@domain, item, {:one=>"", :two=>"", :three=>""}) end end def test_14_nil_attrs item = 'nils' res = nil assert_nothing_thrown do @sdb.put_attributes(@domain, item, {:one=>nil, :two=>nil, :three=>'chunder'}) end wait SDB_DELAY, 'after putting attributes' assert_nothing_thrown do res = @sdb.get_attributes(@domain, item) end assert_nil(res[:attributes]['one'][0]) assert_nil(res[:attributes]['two'][0]) assert_not_nil(res[:attributes]['three'][0]) end def test_15_url_escape item = 'urlescapes' content = {:a=>"one & two & three", :b=>"one ? two / three"} @sdb.put_attributes(@domain, item, content) wait SDB_DELAY, 'after putting attributes' res = @sdb.get_attributes(@domain, item) assert_equal(content[:a], res[:attributes]['a'][0]) assert_equal(content[:b], res[:attributes]['b'][0]) end def test_16_put_attrs_by_post item = 'reqgirth' i = 0 sa = "" while(i < 64) do sa += "aaaaaaaa" i += 1 end @sdb.put_attributes(@domain, item, {:a => sa, :b => sa, :c => sa, :d => sa, :e => sa}) wait SDB_DELAY, 'after putting attributes' end def test_20_query_with_atributes response = @sdb.query_with_attributes(@domain) # convers response to a hash representation items = {}; response[:items].each{ |item| items.merge!(item) } # check we have receied all 5 items each full of attributes assert_equal 6, items.keys.size assert items['toys'].size > 0 assert items['nils'].size > 0 assert items['urlescapes'].size > 0 assert items['multiples'].size > 0 assert items['reqgirth'].size > 0 assert items['zeroes'].size > 0 # fetch only Jon's attributes from all items response = @sdb.query_with_attributes(@domain,['Jon']) items = {}; response[:items].each{ |item| items.merge!(item) } # check we have receied all 5 items # check we have receied all 5 items, but only 'toys' has attributes puts items.inspect assert_equal 2, items['toys']['Jon'].size assert_equal 0, items['nils'].size assert_equal 0, items['urlescapes'].size assert_equal 0, items['multiples'].size assert_equal 0, items['reqgirth'].size assert_equal 0, items['zeroes'].size # kust Jurgen's attriburs response = @sdb.query_with_attributes(@domain,['Jurgen'], "['Jurgen'='piglet']") items = {}; response[:items].each{ |item| items.merge!(item) } # check we have receied an only item assert_equal 1, items.keys.size assert_equal ["chickabiddy", "kitten", "piglet", "puppy"], items['toys']['Jurgen'].sort end # Keep this test last, because it deletes the domain... def test_21_delete_domain assert @sdb.delete_domain(@domain), 'delete_domain fail' wait SDB_DELAY, 'after domain deletion' # check that domain does not exist assert !@sdb.list_domains[:domains].include?(@domain) end end ================================================ FILE: test/sns/test_helper.rb ================================================ require 'test/unit' require File.dirname(__FILE__) + '/../../lib/right_aws' ================================================ FILE: test/sns/test_right_sns.rb ================================================ require File.dirname(__FILE__) + '/test_helper.rb' class TestSns < Test::Unit::TestCase # You can change these things to whatever you like @@subscriber_email = 'foo@bar.baz' @@topic_name = 'RightTestTopic' @@topic_display_name = 'right_aws test notification topic' @@queue_name = "sns_subscribe_queue_#{Time.now.utc.to_i}" # These are placeholders for values that get set, and consumed during the course of testing. @@topic_arn = '' @@subscription_arn = '' @@queue_url = '' @@queue_arn = '' @@policy_template = <<-EOF { "Id": "Policy1300753700208", "Statement": [ { "Sid": "Stmt1300753696680", "Action": [ "SNS:Publish", "SNS:RemovePermission", "SNS:SetTopicAttributes", "SNS:DeleteTopic", "SNS:ListSubscriptionsByTopic", "SNS:GetTopicAttributes", "SNS:Receive", "SNS:AddPermission", "SNS:Subscribe" ], "Effect": "Allow", "Resource": "@@topic_arn@@", "Principal": { "AWS": [ "*" ] } } ] } EOF def policy_text @@policy_template.gsub('@@topic_arn@@', @@topic_arn) end def setup @sns = Rightscale::SnsInterface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key) @sqs = Rightscale::SqsGen2Interface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key) end def test_01_create_topic response = @sns.create_topic(@@topic_name) assert_not_nil(response) @@topic_arn = response end def test_02_set_topic_attributes response = @sns.set_topic_attribute(@@topic_arn, 'Policy', policy_text()) assert_not_nil(response) response = @sns.set_topic_attribute(@@topic_arn, 'DisplayName', @@topic_display_name) assert_not_nil(response) assert_raise ArgumentError do @sns.set_topic_attribute(@@topic_arn, 'Foo', 'val') end end def test_03_get_topic_attributes response = @sns.get_topic_attributes(@@topic_arn) assert_not_nil(response) assert(response['DisplayName'] == @@topic_display_name) assert(response['Policy'] =~ /Policy1300753700208/) end def test_04_list_topics sleep(1) assert(@sns.list_topics.collect{|topic| topic[:arn] }.include?(@@topic_arn)) end def test_05_subscribe_email response = @sns.subscribe(@@topic_arn, 'email', @@subscriber_email) assert_not_nil(response) @@subscription_arn = response end def test_06_list_subscriptions sleep(1) response = @sns.list_subscriptions() assert_not_nil(response) assert(response.count == 1) assert(response[0][:endpoint] == @@subscriber_email) assert(response[0][:protocol] == 'email') assert(response[0][:subscription_arn] == 'PendingConfirmation') assert(response[0][:topic_arn] == @@topic_arn) end def test_07_list_subscriptions_by_topic response = @sns.list_subscriptions(@@topic_arn) assert_not_nil(response) assert(response.count == 1) assert(response[0][:endpoint] == @@subscriber_email) assert(response[0][:protocol] == 'email') assert(response[0][:subscription_arn] == 'PendingConfirmation') assert(response[0][:topic_arn] == @@topic_arn) end def test_08_unsubscribe @@queue_url = @sqs.create_queue(@@queue_name) @@queue_arn = "arn:aws:sqs:us-east-1:#{TestCredentials.account_number.gsub('-','')}:#{@@queue_name}" sub_response = @sns.subscribe(@@topic_arn, 'sqs', @@queue_arn) assert_not_nil(sub_response) unsub_response = @sns.unsubscribe(sub_response) @sqs.delete_queue(@@queue_url) end def test_09_publish response = @sns.publish(@@topic_arn, 'Message to publish', 'Message Subject') assert_not_nil(response) end def test_10_add_and_remove_permission acct_num = TestCredentials.account_number.gsub('-','') add_response = @sns.add_permission(@@topic_arn, 'PermissionLbl', [ {:aws_account_id => acct_num, :action => "GetTopicAttributes"}, {:aws_account_id => acct_num, :action => "Publish"} ]) assert_not_nil(add_response) remove_response = @sns.remove_permission(@@topic_arn, 'PermissionLbl') assert_not_nil(remove_response) end # TODO: Cannot easily test confirming subscription because it's only valid for http(s) and email subscriptions. # Since we don't want to setup an email box or HTTP server to recive the token, we can't really simulate this # def test_10_confirm_subscription # response = @sns.confirm_subscription(@@topic_arn, 'SomeToken') # assert_not_null(response) # end def test_30_delete_topic response = @sns.delete_topic(@@topic_arn) assert_not_nil(response) sleep(1) assert(!@sns.list_topics.collect{|topic| topic[:arn] }.include?(@@topic_arn)) end end ================================================ FILE: test/sqs/test_helper.rb ================================================ require 'test/unit' require File.dirname(__FILE__) + '/../../lib/right_aws' ================================================ FILE: test/sqs/test_right_sqs.rb ================================================ require File.dirname(__FILE__) + '/test_helper.rb' class TestSqs < Test::Unit::TestCase GRANTEE_EMAIL_ADDRESS = 'madhur@amazon.com' RIGHT_MESSAGE_TEXT = 'Right test message' def setup @sqs = Rightscale::SqsInterface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key) @queue_name = 'right_sqs_test_awesome_queue' # for classes @s = Rightscale::Sqs.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key) end # Wait for the queue to appears in the queues list. # Amazon needs some time to after the queue creation to place # it to the accessible queues list. If we dont want to get # the additional faults then wait a bit... def wait_for_queue_url(queue_name) queue_url = nil until queue_url queue_url = @sqs.queue_url_by_name(queue_name) unless queue_url print '-' STDOUT.flush sleep 1 end end queue_url end def assert_eventually_equal(value, timeout=30, failmsg="", &block) start_time = Time.now.to_i tries = 0 while(yield != value) do tries += 1 print '-' STDOUT.flush s = Time.now.to_i - start_time flunk("Timeout: #{failmsg}: did not equal \"#{value}\" after #{tries} tries in #{s}s.") if s > timeout sleep(1) setup if (tries % 10) == 0 end end #--------------------------- # Rightscale::SqsInterface #--------------------------- def test_01_create_queue queue_url = @sqs.create_queue @queue_name assert queue_url[/http.*#{@queue_name}/], 'New queue creation fail' end def test_02_list_queues wait_for_queue_url(@queue_name) queues = @sqs.list_queues('right_') assert queues.size>0, 'Must more that 0 queues in list' end def test_03_set_and_get_queue_attributes queue_url = @sqs.queue_url_by_name(@queue_name) assert queue_url[/http.*#{@queue_name}/], "#{@queue_name} must exist!" assert @sqs.set_queue_attributes(queue_url, 'VisibilityTimeout', 111), 'Set_queue_attributes fail' sleep 20 # Amazon needs some time to change attribute assert_equal '111', @sqs.get_queue_attributes(queue_url)['VisibilityTimeout'], 'New VisibilityTimeout must be equal to 111' end def test_04_set_and_get_visibility_timeout queue_url = @sqs.queue_url_by_name(@queue_name) assert @sqs.set_visibility_timeout(queue_url, 222), 'Set_visibility_timeout fail' sleep 20 # Amazon needs some time to change attribute #assert_equal 222, @sqs.get_visibility_timeout(queue_url), 'Get_visibility_timeout must return to 222' assert_eventually_equal(222, 60, 'Get_visibility_timeout must return to 222') do @sqs.get_visibility_timeout(queue_url) end end def test_05_add_test_remove_grant queue_url = @sqs.queue_url_by_name(@queue_name) assert @sqs.add_grant(queue_url, GRANTEE_EMAIL_ADDRESS, 'FULLCONTROL'), 'Add grant fail' grants_list = @sqs.list_grants(queue_url, GRANTEE_EMAIL_ADDRESS) assert grants_list.size>0, 'List_grants must return at least 1 record for user #{GRANTEE_EMAIL_ADDRESS}' assert @sqs.remove_grant(queue_url, GRANTEE_EMAIL_ADDRESS, 'FULLCONTROL'), 'Remove_grant fail' end def test_06_send_message queue_url = @sqs.queue_url_by_name(@queue_name) # send 5 messages for the tests below assert @sqs.send_message(queue_url, RIGHT_MESSAGE_TEXT) assert @sqs.send_message(queue_url, RIGHT_MESSAGE_TEXT) assert @sqs.send_message(queue_url, RIGHT_MESSAGE_TEXT) assert @sqs.send_message(queue_url, RIGHT_MESSAGE_TEXT) assert @sqs.send_message(queue_url, RIGHT_MESSAGE_TEXT) end def test_07_get_queue_length queue_url = @sqs.queue_url_by_name(@queue_name) assert_equal 5, @sqs.get_queue_length(queue_url), 'Queue must have 5 messages' end def test_08_receive_message queue_url = @sqs.queue_url_by_name(@queue_name) r_message = @sqs.receive_message(queue_url, 1) assert_equal RIGHT_MESSAGE_TEXT, r_message[:body], 'Receive message get wron message text' p_message = @sqs.peek_message(queue_url, r_message[:id]) assert_equal r_message[:body], p_message[:body], 'Received and Peeked messages must be equal' assert @sqs.change_message_visibility(queue_url, r_message[:id], 0), 'Change_message_visibility fail' end def test_09_delete_message queue_url = @sqs.queue_url_by_name(@queue_name) message = @sqs.receive_message(queue_url) assert @sqs.delete_message(queue_url, message[:id]), 'Delete_message fail' assert @sqs.pop_message(queue_url), 'Pop_message fail' end def test_10_clear_and_delete_queue queue_url = @sqs.queue_url_by_name(@queue_name) assert_raise(Rightscale::AwsError) { @sqs.delete_queue(queue_url) } ## oops, force_clear_queue does not work any more - amazon expects for 60 secs timeout between ## queue deletion and recreation... ## assert @sqs.force_clear_queue(queue_url), 'Force_clear_queue fail' assert @sqs.clear_queue(queue_url), 'Clear_queue fail' assert @sqs.delete_queue(queue_url), 'Delete_queue fail' end #--------------------------- # Rightscale::Sqs classes #--------------------------- def test_20_sqs_create_delete_queue assert @s, 'Rightscale::Sqs must exist' # get queues list queues_size = @s.queues.size # create new queue queue = @s.queue("#{@queue_name}_20", true) # check that it is created assert queue.is_a?(Rightscale::Sqs::Queue) wait_for_queue_url(@queue_name) # check that amount of queues has increased assert_eventually_equal(queues_size + 1, 60, "The number of queues did not increase by one") do @s.queues.size end # delete queue assert queue.delete end def test_21_queue_create # create new queue queue = Rightscale::Sqs::Queue.create(@s, "#{@queue_name}_21", true) # check that it is created assert queue.is_a?(Rightscale::Sqs::Queue) wait_for_queue_url(@queue_name) end def test_22_queue_attributes queue = Rightscale::Sqs::Queue.create(@s, "#{@queue_name}_21", false) # get a list of attrinutes attributes = queue.get_attribute assert attributes.is_a?(Hash) && attributes.size>0 # get attribute value and increase it by 10 v = (queue.get_attribute('VisibilityTimeout').to_i + 10).to_s # set attribute assert queue.set_attribute('VisibilityTimeout', v) # wait a bit sleep 20 # check that attribute has changed assert_equal v, queue.get_attribute('VisibilityTimeout') # get queue visibility timeout assert v.to_i, queue.visibility # change it queue.visibility += 10 # make sure that it is changed assert v.to_i + 10, queue.visibility end def test_23_grantees queue = Rightscale::Sqs::Queue.create(@s, "#{@queue_name}_21", false) # get a list of grantees grantees = queue.grantees # well, queue must exist at least some seconds before we could add grantees to it.... # otherwise we get "Queue does not exists" message. Hence we use the queue # has been created at previous step. # # create new grantee grantee = Rightscale::Sqs::Grantee.new(queue, GRANTEE_EMAIL_ADDRESS) assert grantee.perms.empty? # grant perms assert grantee.grant('FULLCONTROL') assert grantee.grant('RECEIVEMESSAGE') assert_equal 2, grantee.perms.size # make sure that amount of grantees has increased assert grantees.size < queue.grantees.size # revoke perms assert grantee.revoke('RECEIVEMESSAGE') assert_equal 1, grantee.perms.size # remove grantee assert grantee.drop # Don't test this - just for cleanup purposes queue.delete end def test_24_send_size queue_url = @sqs.queue_url_by_name("#{@queue_name}_24") @sqs.delete_queue(queue_url) queue = Rightscale::Sqs::Queue.create(@s, "#{@queue_name}_24", true) # send 5 messages assert queue.push('a1') assert queue.push('a2') assert queue.push('a3') assert queue.push('a4') assert queue.push('a5') # check queue size assert_equal 5, queue.size # send one more assert queue.push('a6') # check queue size again assert_equal 6, queue.size end def test_25_message_receive_pop_peek_delete queue = Rightscale::Sqs::Queue.create(@s, "#{@queue_name}_24", false) # get queue size size = queue.size # get first message m1 = queue.receive(10) assert m1.is_a?(Rightscale::Sqs::Message) # pop second message m2 = queue.pop assert m2.is_a?(Rightscale::Sqs::Message) # make sure that queue size has decreased assert_equal size-1, queue.size # peek message 1 m1p = queue.peek(m1.id) assert m1p.is_a?(Rightscale::Sqs::Message) assert_equal m1.id, m1p.id assert_equal m1.body, m1p.body # change message visibility assert m1.visibility = 30 # delete messsage assert m1.delete # make sure that queue size has decreased again assert_equal size-2, queue.size end def test_26 queue = Rightscale::Sqs::Queue.create(@s, "#{@queue_name}_24", false) # lock message queue.receive(100) # clear queue assert queue.clear # queue size is greater than zero assert queue.size>0 queue.push('123456') assert_raise(Rightscale::AwsError) { queue.delete } assert queue.delete(true) end def test_27_set_amazon_problems original_problems = Rightscale::SqsInterface.amazon_problems assert(original_problems.length > 0) Rightscale::SqsInterface.amazon_problems= original_problems << "A New Problem" new_problems = Rightscale::SqsInterface.amazon_problems assert_equal(new_problems, original_problems) Rightscale::SqsInterface.amazon_problems= nil assert_nil(Rightscale::SqsInterface.amazon_problems) end def test_29_signature_version_0 sqs = Rightscale::SqsInterface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key, :signature_version => '0') assert_nothing_raised do sqs.list_queues end # check that the request has correct signature version assert sqs.last_request.path.include?('SignatureVersion=0') end end ================================================ FILE: test/sqs/test_right_sqs_gen2.rb ================================================ require File.dirname(__FILE__) + '/test_helper.rb' class TestSqsGen2 < Test::Unit::TestCase GRANTEE_EMAIL_ADDRESS = 'fester@example.com' RIGHT_MESSAGE_TEXT = 'Right test message' def setup $stdout.sync = true @grantee_aws_id = '100000000001' @sqs = Rightscale::SqsGen2Interface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key) @queue_name = 'right_sqs_test_gen2_queue' @queue2_name = @queue_name + '_2' @queue3_name = @queue_name + '_3' @queue4_name = @queue_name + '_4' # for classes @s = Rightscale::SqsGen2.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key) end # Wait for the queue to appear in the queues list. # Amazon needs some time to after the queue creation to place # it to the accessible queues list. If we dont want to get # the additional faults then wait a bit... def wait_for_queue_url(queue_name) queue_url = nil do_sleep(180) do queue_url = @sqs.queue_url_by_name(queue_name) end sleep 30 queue_url end def do_sleep(delay, &block) puts "sleeping #{block ? 'up to ' : ''}#{delay} seconds:" wake_up_at = Time.now+delay while Time.now < wake_up_at do sleep 1 print '.' break if block && block.call end puts end #--------------------------- # Rightscale::SqsInterface #--------------------------- def test_01_create_queue queue_url = @sqs.create_queue @queue_name assert queue_url[/http.*#{@queue_name}/], 'New queue creation failed' end def test_02_list_queues wait_for_queue_url(@queue_name) queues = @sqs.list_queues('right_') assert queues.size>0, 'Must more that 0 queues in list' end def test_03_set_and_get_queue_attributes queue_url = @sqs.queue_url_by_name(@queue_name) assert queue_url[/https.*#{@queue_name}/], "#{@queue_name} must exist!" assert @sqs.set_queue_attributes(queue_url, 'VisibilityTimeout', 111), 'Set_queue_attributes fail' do_sleep 60 # Amazon needs some time to change attribute assert_equal '111', @sqs.get_queue_attributes(queue_url)['VisibilityTimeout'], 'New VisibilityTimeout must be equal to 111' end def test_04_get_queue_attributes_forms queue_url = @sqs.queue_url_by_name(@queue_name) all = nil assert_nothing_raised do all = @sqs.get_queue_attributes(queue_url, 'All') end assert_nothing_raised do assert all, @sqs.get_queue_attributes(queue_url) end assert_nothing_raised do attributes = @sqs.get_queue_attributes(queue_url, 'ApproximateNumberOfMessages', 'VisibilityTimeout') assert_equal 2, attributes.size end assert_nothing_raised do attributes = @sqs.get_queue_attributes(queue_url, ['ApproximateNumberOfMessages', 'VisibilityTimeout']) assert_equal 2, attributes.size end end def test_05_add_permissions queue_url = @sqs.queue_url_by_name(@queue_name) assert @sqs.add_permissions(queue_url, 'test01', @grantee_aws_id, 'SendMessage') assert @sqs.add_permissions(queue_url, 'test02', @grantee_aws_id, ['DeleteMessage','ReceiveMessage']) do_sleep 60 end def test_06_test_permissions queue_url = @sqs.queue_url_by_name(@queue_name) permissions = @sqs.get_queue_attributes(queue_url, 'Policy') assert !permissions.right_blank? end def test_07_revoke_permissions queue_url = @sqs.queue_url_by_name(@queue_name) assert @sqs.remove_permissions(queue_url, 'test01') assert @sqs.remove_permissions(queue_url, 'test02') end def test_14_send_message queue_url = @sqs.queue_url_by_name(@queue_name) # send 5 messages for the tests below assert @sqs.send_message(queue_url, RIGHT_MESSAGE_TEXT) assert @sqs.send_message(queue_url, RIGHT_MESSAGE_TEXT) assert @sqs.send_message(queue_url, RIGHT_MESSAGE_TEXT) assert @sqs.send_message(queue_url, RIGHT_MESSAGE_TEXT) assert @sqs.send_message(queue_url, RIGHT_MESSAGE_TEXT) do_sleep 60 end def test_15_get_queue_length queue_url = @sqs.queue_url_by_name(@queue_name) assert_equal 5, @sqs.get_queue_length(queue_url), 'Queue must have 5 messages' end def test_16_receive_message queue_url = @sqs.queue_url_by_name(@queue_name) r_message = @sqs.receive_message(queue_url, 1)[0] assert r_message, "Receive returned no message(s), but this is not necessarily incorrect" assert_equal RIGHT_MESSAGE_TEXT, r_message['Body'], 'Receive message got wrong message text' end def test_17_delete_message queue_url = @sqs.queue_url_by_name(@queue_name) message = @sqs.receive_message(queue_url)[0] assert @sqs.delete_message(queue_url, message['ReceiptHandle']), 'Delete_message fail' assert @sqs.pop_message(queue_url), 'Pop_message fail' end def test_18_clear_and_delete_queue queue_url = @sqs.queue_url_by_name(@queue_name) assert @sqs.delete_queue(queue_url) end #--------------------------- # Rightscale::Sqs classes #--------------------------- def test_20_sqs_create_queue assert @s, 'Rightscale::SqsGen2 must exist' # get queues list queues_size = @s.queues.size # create new queue queue = @s.queue(@queue2_name, true) # check that it is created assert queue.is_a?(Rightscale::SqsGen2::Queue) wait_for_queue_url(queue.name) # check that amount of queues has increased assert_equal queues_size + 1, @s.queues.size do_sleep 10 end def test_21_sqs_delete_queue queue = @s.queue(@queue2_name, false) assert queue.delete end def test_22_queue_create # create new queue queue = Rightscale::SqsGen2::Queue.create(@s, @queue3_name, true) # check that it is created assert queue.is_a?(Rightscale::SqsGen2::Queue) wait_for_queue_url("#{@queue_name}_21") do_sleep 10 end def test_23_queue_attributes queue = Rightscale::SqsGen2::Queue.create(@s, @queue3_name, false) # get a list of attrinutes attributes = queue.get_attribute assert attributes.is_a?(Hash) && attributes.size>0 # get attribute value and increase it by 10 v = (queue.get_attribute('VisibilityTimeout').to_i + 10).to_s # set attribute assert queue.set_attribute('VisibilityTimeout', v) # wait a bit do_sleep 60 # check that attribute has changed assert_equal v, queue.get_attribute('VisibilityTimeout') # get queue visibility timeout assert_equal v, queue.visibility # change it queue.visibility = queue.visibility.to_i + 10 # make sure that it is changed assert v.to_i + 10, queue.visibility end def test_24 queue = Rightscale::SqsGen2::Queue.create(@s, @queue3_name, false) assert queue.delete end def test_25_send_size queue = Rightscale::SqsGen2::Queue.create(@s, @queue4_name, true) # send 5 messages assert queue.push('a1') assert queue.push('a2') assert queue.push('a3') assert queue.push('a4') assert queue.push('a5') # do_sleep(300){ queue.size == 5 } # check queue size assert_equal 5, queue.size # send one more assert queue.push('a6') # do_sleep(300){ queue.size == 6 } # check queue size again assert_equal 6, queue.size end def test_26_message_receive_pop_delete queue = Rightscale::SqsGen2::Queue.create(@s, @queue4_name, false) # get queue size size = queue.size # get first message m1 = queue.receive(10) assert m1.is_a?(Rightscale::SqsGen2::Message) # pop second message m2 = queue.pop assert m2.is_a?(Rightscale::SqsGen2::Message) # do_sleep 30 # make sure that queue size has decreased assert_equal size-1, queue.size # delete messsage assert m1.delete # do_sleep 15 # make sure that queue size has decreased again assert_equal size-2, queue.size end def test_27 queue = Rightscale::SqsGen2::Queue.create(@s, @queue4_name, false) # lock message queue.receive(100) # clear queue assert queue.clear # queue size is greater than zero assert queue.size>0 # delete queue assert queue.delete end def test_28_set_amazon_problems original_problems = Rightscale::SqsGen2Interface.amazon_problems assert(original_problems.length > 0) Rightscale::SqsGen2Interface.amazon_problems= original_problems << "A New Problem" new_problems = Rightscale::SqsGen2Interface.amazon_problems assert_equal(new_problems, original_problems) Rightscale::SqsGen2Interface.amazon_problems= nil assert_nil(Rightscale::SqsGen2Interface.amazon_problems) end end ================================================ FILE: test/test_credentials.rb ================================================ class TestCredentials @@aws_access_key_id = nil @@aws_secret_access_key = nil @@account_number = nil def self.aws_access_key_id @@aws_access_key_id end def self.aws_access_key_id=(newval) @@aws_access_key_id = newval end def self.account_number @@account_number end def self.account_number=(newval) @@account_number = newval end def self.aws_secret_access_key @@aws_secret_access_key end def self.aws_secret_access_key=(newval) @@aws_secret_access_key = newval end def self.get_credentials Dir.chdir do begin Dir.chdir('./.rightscale') do require 'testcredentials' end rescue Exception => e puts "Couldn't chdir to ~/.rightscale: #{e.message}" end end end end ================================================ FILE: test/ts_right_aws.rb ================================================ require 'test/unit' $: << File.dirname(__FILE__) require 'test_credentials' TestCredentials.get_credentials require 'http_connection' require 'awsbase/test_right_awsbase.rb' require 'ec2/test_right_ec2.rb' require 's3/test_right_s3.rb' require 's3/test_right_s3_stubbed.rb' require 'sqs/test_right_sqs.rb' require 'sqs/test_right_sqs_gen2.rb' require 'sdb/test_right_sdb.rb' require 'acf/test_right_acf.rb' require 'sns/test_right_sns.rb'